aarch64: initial pauth support

1. Support PAuth dynamic relocs:

   - R_AARCH64_AUTH_ABS64;
   - R_AARCH64_AUTH_RELATIVE (including relr);
   - R_AARCH64_JUMP_SLOT (sign slot contents if DT_AARCH64_PAC_PLT
     dynamic tag is present);
   - R_AARCH64_AUTH_GLOB_DAT;
   - R_AARCH64_AUTH_TLSDESC.

2. Support signed function pointers in init/fini arrays (with optional
   address discrimination enabled).

3. Check PAuth core info compatibility for DSOs in the process.

TODO:

1. Support function pointer type discrimination. This should not be
   enabled under normal conditions though, and pauthtest ABI in LLVM
   intentionally does not include that.

2. Do not store unsigned LR in memory in raw assembly code: see
   #3.

3. Support non-null `__ehdr_start`. See in-code comment in `__dls2`
   function in src/dynlink.c for details.

4. Enhance test coverage: current proof-of-concept is good enough to run
   llvm-test-suite, but we lack test coverage of some parts of musl.

Co-authored-by: Evgeny Leviant <[email protected]>
Co-authored-by: Anatoly Trosinenko <[email protected]>

Author: 3 people

arch/aarch64/reloc.h

@@ -18,7 +18,39 @@
 #define REL_DTPMOD      R_AARCH64_TLS_DTPMOD64
 #define REL_DTPOFF      R_AARCH64_TLS_DTPREL64
 #define REL_TPOFF       R_AARCH64_TLS_TPREL64
+
+#ifndef __has_feature
+#define __has_feature(x) 0
+#endif
+
+#if __has_feature(ptrauth_elf_got)
+#define R_AARCH64_AUTH_TLSDESC 0x413
+#define REL_TLSDESC     R_AARCH64_AUTH_TLSDESC
+#else
 #define REL_TLSDESC     R_AARCH64_TLSDESC
+#endif
 
 #define CRTJMP(pc,sp) __asm__ __volatile__( \
 	"mov sp,%1 ; br %0" : : "r"(pc), "r"(sp) : "memory" )
+
+#if __has_feature(ptrauth_intrinsics)
+
+#include <stdint.h>
+
+#define TARGET_RELOCATE(dso, type, reladdr, sym, addend, is_phase_2, dyn, error_sym) \
+	do_target_reloc(dso, type, reladdr, sym, addend, is_phase_2, dyn, error_sym)
+#define DO_TARGET_RELR(dso, dyn) do_pauth_relr(dso, dyn)
+
+int do_target_reloc(int type, uint64_t* reladdr, uint64_t base, uint64_t symval,
+                    uint64_t addend, int is_phase_2, uint64_t* dyn, uint64_t error_sym);
+
+void do_pauth_relr(uint64_t base, uint64_t* dyn);
+
+#define GETFUNCSYM(fp, sym, got) do { \
+	hidden void sym(); \
+	*(fp) = sym; } while(0)
+
+#define FPTR_CAST(fty, p) \
+	((fty)__builtin_ptrauth_sign_unauthenticated((void*)(p), 0, 0))
+
+#endif

crt/aarch64/crti.s

@@ -2,12 +2,14 @@
 .global _init
 .type _init,%function
 _init:
+	paciasp
 	stp x29,x30,[sp,-16]!
 	mov x29,sp
 
 .section .fini
 .global _fini
 .type _fini,%function
 _fini:
+	paciasp
 	stp x29,x30,[sp,-16]!
 	mov x29,sp

crt/aarch64/crtn.s

@@ -1,7 +1,9 @@
 .section .init
 	ldp x29,x30,[sp],#16
+	autiasp
 	ret
 
 .section .fini
 	ldp x29,x30,[sp],#16
+	autiasp
 	ret

ldso/dynlink.c

@@ -1,5 +1,6 @@
 #define _GNU_SOURCE
 #define SYSCALL_NO_TLS 1
+#include <inttypes.h>
 #include <stdlib.h>
 #include <stdarg.h>
 #include <stddef.h>
@@ -19,6 +20,9 @@
 #include <dlfcn.h>
 #include <semaphore.h>
 #include <sys/membarrier.h>
+#if __has_feature(ptrauth_intrinsics)
+#include <ptrauth.h>
+#endif
 #include "pthread_impl.h"
 #include "fork_impl.h"
 #include "dynlink.h"
@@ -45,6 +49,18 @@ static void (*error)(const char *, ...) = error_noop;
 #define container_of(p,t,m) ((t*)((char *)(p)-offsetof(t,m)))
 #define countof(a) ((sizeof (a))/(sizeof (a)[0]))
 
+#ifndef TARGET_RELOCATE
+#define TARGET_RELOCATE(...) 0
+#endif
+
+#ifndef DO_TARGET_RELR
+#define DO_TARGET_RELR(...)
+#endif
+
+#ifndef FPTR_CAST
+#define FPTR_CAST(fty, p) ((fty)(p))
+#endif
+
 struct debug {
 	int ver;
 	void *head;
@@ -111,6 +127,11 @@ struct dso {
 		size_t *got;
 	} *funcdescs;
 	size_t *got;
+#ifdef __aarch64__
+	/* PAuth core info as defined in PAUTHABIELF64:
+	 * https://github.com/ARM-software/abi-aa/blob/2025Q1/pauthabielf64/pauthabielf64.rst#core-information */
+	size_t* pauth;
+#endif
 	char buf[];
 };
 
@@ -471,6 +492,9 @@ static void do_relocs(struct dso *dso, size_t *rel, size_t rel_size, size_t stri
 		case REL_GOT:
 		case REL_PLT:
 			*reloc_addr = sym_val + addend;
+			/* If AArch64 PAC is enabled and DT_AARCH64_PAC_PLT is present, sign the contents of R_AARCH64_JUMP_SLOT.
+			 * Otherwise, do nothing. */
+			TARGET_RELOCATE(type, reloc_addr, (size_t)base, sym_val, addend, head == &ldso, dso->dynv, (uint64_t)error);
 			break;
 		case REL_USYMBOLIC:
 			memcpy(reloc_addr, &(size_t){sym_val + addend}, sizeof(size_t));
@@ -518,6 +542,15 @@ static void do_relocs(struct dso *dso, size_t *rel, size_t rel_size, size_t stri
 #endif
 		case REL_TLSDESC:
 			if (stride<3) addend = reloc_addr[!TLSDESC_BACKWARDS];
+#ifdef __aarch64__
+			/* TODO: Submit implementation of undefined weak TLS symbols support to
+			 * mainline musl when it's implemented for other architectures.
+			 * The patch is work-in-progress. */
+			if (sym && sym->st_info>>4 == STB_WEAK && sym->st_shndx == SHN_UNDEF) {
+				reloc_addr[0] = (size_t)__tlsdesc_undef_weak;
+				reloc_addr[1] = 0;
+			} else
+#endif
 			if (def.dso->tls_id > static_tls_cnt) {
 				struct td_index *new = malloc(sizeof *new);
 				if (!new) {
@@ -533,7 +566,11 @@ static void do_relocs(struct dso *dso, size_t *rel, size_t rel_size, size_t stri
 				reloc_addr[0] = (size_t)__tlsdesc_dynamic;
 				reloc_addr[1] = (size_t)new;
 			} else {
+#if __has_feature(ptrauth_intrinsics) && !__has_feature(ptrauth_elf_got)
+				reloc_addr[0] = (size_t)ptrauth_strip(&__tlsdesc_static, 0);
+#else
 				reloc_addr[0] = (size_t)__tlsdesc_static;
+#endif
 #ifdef TLS_ABOVE_TP
 				reloc_addr[1] = tls_val + def.dso->tls.offset
 					+ TPOFF_K + addend;
@@ -549,8 +586,18 @@ static void do_relocs(struct dso *dso, size_t *rel, size_t rel_size, size_t stri
 				reloc_addr[0] = reloc_addr[1];
 				reloc_addr[1] = tmp;
 			}
+#if __has_feature(ptrauth_elf_got)
+			/* FIXME: actually, signing scheme is written in-place in relocation slot, and we should read and use that.
+			 * However, the scheme is known (IA key + addr div for function and DA key + addr div for data).
+			 * So, we just hard-code that. See also:
+			 * https://github.com/ARM-software/abi-aa/blob/2025Q1/pauthabielf64/pauthabielf64.rst#default-signing-schema */
+			reloc_addr[0] = (size_t)(ptrauth_auth_and_resign((void*)(reloc_addr[0]), 0, 0, 0, (size_t)(reloc_addr)));
+			reloc_addr[1] = (size_t)(ptrauth_sign_unauthenticated((void*)(reloc_addr[1]), 2, (size_t)(reloc_addr) + 8));
+#endif
 			break;
 		default:
+			if (TARGET_RELOCATE(type, reloc_addr, (size_t)base, sym_val, addend, head == &ldso, dso->dynv, (uint64_t)error))
+				break;
 			error("Error relocating %s: unsupported relocation type %d",
 				dso->name, type);
 			if (runtime) longjmp(*rtld_fail, 1);
@@ -684,6 +731,84 @@ static void unmap_library(struct dso *dso)
 	}
 }
 
+#ifdef __aarch64__
+
+/* See https://github.com/ARM-software/abi-aa/blob/2025Q1/pauthabielf64/pauthabielf64.rst#elf-marking */
+#define GNU_PROPERTY_AARCH64_FEATURE_PAUTH 0xc0000001
+
+static uint32_t align8(uint32_t val) {
+	if (val % 8 == 0)
+		return val;
+	return val + 8 - (val % 8);
+}
+
+static void get_pauth_core_info(struct dso *dso) {
+	for (int i = 0; i < dso->phnum; ++i) {
+		Phdr *ph = &dso->phdr[i];
+
+		/* Minimal GNU property section containing PAuth core info has 40 bytes size. */
+		if (!(ph->p_type == PT_NOTE && ph->p_memsz >= 40))
+			continue;
+
+		uint32_t *note = laddr(dso, ph->p_vaddr);
+		uint32_t *note_arr_end = (uint32_t*)((uintptr_t)note + ph->p_memsz);
+
+		for (; note != note_arr_end;
+		     /* We can hardcode 8-byte alignment since this code runs only on AArch64. */
+		     note = (uint32_t*)((uintptr_t)note + 4 + 4 + align8(4 + note[0]) + align8(note[1]))) {
+			/* Note segment is ill-formed: last note information entry exceeds the right segment boundary. */
+			if (note > note_arr_end) return;
+
+			if (!(note[0] == 4 && note[2] == NT_GNU_PROPERTY_TYPE_0 && strncmp((char*)&note[3], "GNU", 4) == 0))
+				continue;
+
+			uint32_t *prop = &note[4];
+			uint32_t *prop_arr_end = (uint32_t*)((uintptr_t)prop + note[1]);
+			for (; prop != prop_arr_end;
+			     /* We can hardcode 8-byte alignment since this code runs only on AArch64. */
+			     prop = (uint32_t*)((uintptr_t)prop + 4 + 4 + align8(prop[1]))) {
+				/* GNU property array is ill-formed: its last element end exceeds the right array boundary. */
+				if (prop > prop_arr_end) return;
+
+				if (prop[0] != GNU_PROPERTY_AARCH64_FEATURE_PAUTH) continue;
+
+				/* PAuth GNU property must have exactly 16 bytes length:
+				 * 8 bytes for platform and 8 bytes for version value. */
+				if (prop[1] != 16) return;
+
+				/* We do not expect multiple PAuth GNU properties. */
+				if (dso->pauth != 0) return;
+
+				dso->pauth = (size_t*)&prop[2];
+			}
+		}
+	}
+}
+
+static void print_pauth_core_info(size_t *pauth, const char *name) {
+	if (pauth == 0) {
+		dprintf(2, "%s: no PAuth core info\n", name);
+		return;
+	}
+	dprintf(2, "%s: (platform: 0x%" PRIx64 "; version: 0x%" PRIx64 ")\n", name, pauth[0], pauth[1]);
+}
+
+static int check_pauth_core_info_compatibility(size_t *pauth1, const char *name1, size_t *pauth2, const char *name2) {
+	if (pauth1 == pauth2)
+		return 1;
+
+	if (pauth1 == 0 || pauth2 == 0 || pauth1[0] != pauth2[0] || pauth1[1] != pauth2[1]) {
+		dprintf(2, "incompatible PAuth core info between %s and %s\n", name1, name2);
+		print_pauth_core_info(pauth1, name1);
+		print_pauth_core_info(pauth2, name2);
+		return 0;
+	}
+
+	return 1;
+}
+
+#endif
+
 static void *map_library(int fd, struct dso *dso)
 {
 	Ehdr buf[(896+sizeof(Ehdr))/sizeof(Ehdr)];
@@ -860,6 +985,9 @@ static void *map_library(int fd, struct dso *dso)
 	dso->base = base;
 	dso->dynv = laddr(dso, dyn);
 	if (dso->tls.size) dso->tls.image = laddr(dso, tls_image);
+#ifdef __aarch64__
+	get_pauth_core_info(dso);
+#endif
 	free(allocated_buf);
 	return map;
 noexec:
@@ -1184,6 +1312,11 @@ static struct dso *load_library(const char *name, struct dso *needed_by)
 	close(fd);
 	if (!map) return 0;
 
+#ifdef __aarch64__
+	if (!check_pauth_core_info_compatibility(head->pauth, head->name, temp_dso.pauth, name))
+		return 0;
+#endif
+
 	/* Avoid the danger of getting two versions of libc mapped into the
 	 * same process when an absolute pathname was used. The symbols
 	 * checked are chosen to catch both musl and glibc, and to avoid
@@ -1421,6 +1554,9 @@ static void reloc_all(struct dso *p)
 		do_relocs(p, laddr(p, dyn[DT_RELA]), dyn[DT_RELASZ], 3);
 		if (!DL_FDPIC)
 			do_relr_relocs(p, laddr(p, dyn[DT_RELR]), dyn[DT_RELRSZ]);
+		if (p != &ldso) {
+			DO_TARGET_RELR((uint64_t)p->base, p->dynv);
+		}
 
 		if (head != &ldso && p->relro_start != p->relro_end) {
 			long ret = __syscall(SYS_mprotect, laddr(p, p->relro_start),
@@ -1464,6 +1600,9 @@ static void kernel_mapped_dso(struct dso *p)
 	p->map = p->base + min_addr;
 	p->map_len = max_addr - min_addr;
 	p->kernel_mapped = 1;
+#ifdef __aarch64__
+	get_pauth_core_info(p);
+#endif
 }
 
 void __libc_exit_fini()
@@ -1487,7 +1626,20 @@ void __libc_exit_fini()
 		if (dyn[0] & (1<<DT_FINI_ARRAY)) {
 			size_t n = dyn[DT_FINI_ARRAYSZ]/sizeof(size_t);
 			size_t *fn = (size_t *)laddr(p, dyn[DT_FINI_ARRAY])+n;
-			while (n--) ((void (*)(void))*--fn)();
+#if __has_feature(ptrauth_init_fini)
+			while (n--) {
+				ptrauth_auth_function(
+					(void (*)(void))*--fn,
+					ptrauth_key_asia,
+#if __has_feature(ptrauth_init_fini_address_discrimination)
+					ptrauth_blend_discriminator(fn, __ptrauth_init_fini_discriminator))();
+#else
+					__ptrauth_init_fini_discriminator)();
+#endif
+			}
+#else
+			while (n--) FPTR_CAST(void (*)(void), (void*)*(--fn))();
+#endif
 		}
 #ifndef NO_LEGACY_INITFINI
 		if ((dyn[0] & (1<<DT_FINI)) && dyn[DT_FINI])
@@ -1605,7 +1757,21 @@ static void do_init_fini(struct dso **queue)
 		if (dyn[0] & (1<<DT_INIT_ARRAY)) {
 			size_t n = dyn[DT_INIT_ARRAYSZ]/sizeof(size_t);
 			size_t *fn = laddr(p, dyn[DT_INIT_ARRAY]);
-			while (n--) ((void (*)(void))*fn++)();
+#if __has_feature(ptrauth_init_fini)
+			while (n--) {
+				ptrauth_auth_function(
+					(void (*)(void))*fn,
+					ptrauth_key_asia,
+#if __has_feature(ptrauth_init_fini_address_discrimination)
+					ptrauth_blend_discriminator(fn, __ptrauth_init_fini_discriminator))();
+#else
+					__ptrauth_init_fini_discriminator)();
+#endif
+				++fn;
+			}
+#else
+			while (n--) FPTR_CAST(void (*)(void), (void*)*fn++)();
+#endif
 		}
 
 		pthread_mutex_lock(&init_fini_lock);
@@ -1727,7 +1893,16 @@ hidden void __dls2(unsigned char *base, size_t *sp)
 	} else {
 		ldso.base = base;
 	}
+#if __has_feature(ptrauth_elf_got)
+	/* TODO: support non-null __ehdr_start. Since at this point relocations
+	 * are not resolved yet, the contents of the GOT slot contains signing schema
+	 * so the value is treated as non-null even when it is actually null.
+	 * For undefined weak symbols with non-null values, implicit authentication
+	 * sequence is emitted on access attempt, and this authentication obviously fails. */
+	Ehdr *ehdr = (void *)ldso.base;
+#else
 	Ehdr *ehdr = __ehdr_start ? (void *)__ehdr_start : (void *)ldso.base;
+#endif
 	ldso.name = ldso.shortname = "libc.so";
 	ldso.phnum = ehdr->e_phnum;
 	ldso.phdr = laddr(&ldso, ehdr->e_phoff);
@@ -1764,7 +1939,7 @@ hidden void __dls2(unsigned char *base, size_t *sp)
 	 * load across the above relocation processing. */
 	struct symdef dls2b_def = find_sym(&ldso, "__dls2b", 0);
 	if (DL_FDPIC) ((stage3_func)&ldso.funcdescs[dls2b_def.sym-ldso.syms])(sp, auxv);
-	else ((stage3_func)laddr(&ldso, dls2b_def.sym->st_value))(sp, auxv);
+	else FPTR_CAST(stage3_func, laddr(&ldso, dls2b_def.sym->st_value))(sp, auxv);
 }
 
 /* Stage 2b sets up a valid thread pointer, which requires relocations
@@ -1788,7 +1963,7 @@ void __dls2b(size_t *sp, size_t *auxv)
 
 	struct symdef dls3_def = find_sym(&ldso, "__dls3", 0);
 	if (DL_FDPIC) ((stage3_func)&ldso.funcdescs[dls3_def.sym-ldso.syms])(sp, auxv);
-	else ((stage3_func)laddr(&ldso, dls3_def.sym->st_value))(sp, auxv);
+	else FPTR_CAST(stage3_func, laddr(&ldso, dls3_def.sym->st_value))(sp, auxv);
 }
 
 /* Stage 3 of the dynamic linker is called with the dynamic linker/libc

src/internal/dynlink.h

@@ -111,7 +111,7 @@ hidden void __dl_seterr(const char *, ...);
 hidden int __dl_invalid_handle(void *);
 hidden void __dl_vseterr(const char *, va_list);
 
-hidden ptrdiff_t __tlsdesc_static(), __tlsdesc_dynamic();
+hidden ptrdiff_t __tlsdesc_static(), __tlsdesc_dynamic(), __tlsdesc_undef_weak();
 
 hidden extern int __malloc_replaced;
 hidden extern int __aligned_alloc_replaced;