mm/arm64: support large pfn mappings

jira LE-3557
Rebuild_History Non-Buildable kernel-5.14.0-570.26.1.el9_6
commit-author Peter Xu <[email protected]>
commit 3e509c9
Empty-Commit: Cherry-Pick Conflicts during history rebuild.
Will be included in final tarball splat. Ref for failed cherry-pick at:
ciq/ciq_backports/kernel-5.14.0-570.26.1.el9_6/3e509c9b.failed

Support huge pfnmaps by using bit 56 (PTE_SPECIAL) for "special" on
pmds/puds.  Provide the pmd/pud helpers to set/get special bit.

There's one more thing missing for arm64 which is the pxx_pgprot() for
pmd/pud.  Add them too, which is mostly the same as the pte version by
dropping the pfn field.  These helpers are essential to be used in the new
follow_pfnmap*() API to report valid pgprot_t results.

Note that arm64 doesn't yet support huge PUD yet, but it's still
straightforward to provide the pud helpers that we need altogether.  Only
PMD helpers will make an immediate benefit until arm64 will support huge
PUDs first in general (e.g.  in THPs).

Link: https://lkml.kernel.org/r/[email protected]
	Signed-off-by: Peter Xu <[email protected]>
	Cc: Catalin Marinas <[email protected]>
	Cc: Will Deacon <[email protected]>
	Cc: Alexander Gordeev <[email protected]>
	Cc: Alex Williamson <[email protected]>
	Cc: Aneesh Kumar K.V <[email protected]>
	Cc: Borislav Petkov <[email protected]>
	Cc: Christian Borntraeger <[email protected]>
	Cc: Dave Hansen <[email protected]>
	Cc: David Hildenbrand <[email protected]>
	Cc: Gavin Shan <[email protected]>
	Cc: Gerald Schaefer <[email protected]>
	Cc: Heiko Carstens <[email protected]>
	Cc: Ingo Molnar <[email protected]>
	Cc: Jason Gunthorpe <[email protected]>
	Cc: Matthew Wilcox <[email protected]>
	Cc: Niklas Schnelle <[email protected]>
	Cc: Paolo Bonzini <[email protected]>
	Cc: Ryan Roberts <[email protected]>
	Cc: Sean Christopherson <[email protected]>
	Cc: Sven Schnelle <[email protected]>
	Cc: Thomas Gleixner <[email protected]>
	Cc: Vasily Gorbik <[email protected]>
	Cc: Zi Yan <[email protected]>
	Signed-off-by: Andrew Morton <[email protected]>
(cherry picked from commit 3e509c9)
	Signed-off-by: Jonathan Maple <[email protected]>

# Conflicts:
#	arch/arm64/Kconfig

Author: PlaidCat

ciq/ciq_backports/kernel-5.14.0-570.26.1.el9_6/3e509c9b.failed

@@ -0,0 +1,118 @@
+mm/arm64: support large pfn mappings
+
+jira LE-3557
+Rebuild_History Non-Buildable kernel-5.14.0-570.26.1.el9_6
+commit-author Peter Xu <[email protected]>
+commit 3e509c9b03f9abc7804c80bed266a6cc4286a5a8
+Empty-Commit: Cherry-Pick Conflicts during history rebuild.
+Will be included in final tarball splat. Ref for failed cherry-pick at:
+ciq/ciq_backports/kernel-5.14.0-570.26.1.el9_6/3e509c9b.failed
+
+Support huge pfnmaps by using bit 56 (PTE_SPECIAL) for "special" on
+pmds/puds.  Provide the pmd/pud helpers to set/get special bit.
+
+There's one more thing missing for arm64 which is the pxx_pgprot() for
+pmd/pud.  Add them too, which is mostly the same as the pte version by
+dropping the pfn field.  These helpers are essential to be used in the new
+follow_pfnmap*() API to report valid pgprot_t results.
+
+Note that arm64 doesn't yet support huge PUD yet, but it's still
+straightforward to provide the pud helpers that we need altogether.  Only
+PMD helpers will make an immediate benefit until arm64 will support huge
+PUDs first in general (e.g.  in THPs).
+
+Link: https://lkml.kernel.org/r/[email protected]
+	Signed-off-by: Peter Xu <[email protected]>
+	Cc: Catalin Marinas <[email protected]>
+	Cc: Will Deacon <[email protected]>
+	Cc: Alexander Gordeev <[email protected]>
+	Cc: Alex Williamson <[email protected]>
+	Cc: Aneesh Kumar K.V <[email protected]>
+	Cc: Borislav Petkov <[email protected]>
+	Cc: Christian Borntraeger <[email protected]>
+	Cc: Dave Hansen <[email protected]>
+	Cc: David Hildenbrand <[email protected]>
+	Cc: Gavin Shan <[email protected]>
+	Cc: Gerald Schaefer <[email protected]>
+	Cc: Heiko Carstens <[email protected]>
+	Cc: Ingo Molnar <[email protected]>
+	Cc: Jason Gunthorpe <[email protected]>
+	Cc: Matthew Wilcox <[email protected]>
+	Cc: Niklas Schnelle <[email protected]>
+	Cc: Paolo Bonzini <[email protected]>
+	Cc: Ryan Roberts <[email protected]>
+	Cc: Sean Christopherson <[email protected]>
+	Cc: Sven Schnelle <[email protected]>
+	Cc: Thomas Gleixner <[email protected]>
+	Cc: Vasily Gorbik <[email protected]>
+	Cc: Zi Yan <[email protected]>
+	Signed-off-by: Andrew Morton <[email protected]>
+(cherry picked from commit 3e509c9b03f9abc7804c80bed266a6cc4286a5a8)
+	Signed-off-by: Jonathan Maple <[email protected]>
+
+# Conflicts:
+#	arch/arm64/Kconfig
+diff --cc arch/arm64/Kconfig
+index ca1f39e49631,6607ed8fdbb4..000000000000
+--- a/arch/arm64/Kconfig
++++ b/arch/arm64/Kconfig
+@@@ -97,7 -99,7 +97,11 @@@ config ARM6
+  	select ARCH_SUPPORTS_NUMA_BALANCING
+  	select ARCH_SUPPORTS_PAGE_TABLE_CHECK
+  	select ARCH_SUPPORTS_PER_VMA_LOCK
+++<<<<<<< HEAD
+ +	select ARCH_SUPPORTS_RT
+++=======
++ 	select ARCH_SUPPORTS_HUGE_PFNMAP if TRANSPARENT_HUGEPAGE
+++>>>>>>> 3e509c9b03f9 (mm/arm64: support large pfn mappings)
+  	select ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH
+  	select ARCH_WANT_COMPAT_IPC_PARSE_VERSION if COMPAT
+  	select ARCH_WANT_DEFAULT_BPF_JIT
+* Unmerged path arch/arm64/Kconfig
+diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
+index e3ea0ef9673d..7fa291d3f90a 100644
+--- a/arch/arm64/include/asm/pgtable.h
++++ b/arch/arm64/include/asm/pgtable.h
+@@ -528,6 +528,14 @@ static inline pmd_t pmd_mkdevmap(pmd_t pmd)
+ 	return pte_pmd(set_pte_bit(pmd_pte(pmd), __pgprot(PTE_DEVMAP)));
+ }
+ 
++#ifdef CONFIG_ARCH_SUPPORTS_PMD_PFNMAP
++#define pmd_special(pte)	(!!((pmd_val(pte) & PTE_SPECIAL)))
++static inline pmd_t pmd_mkspecial(pmd_t pmd)
++{
++	return set_pmd_bit(pmd, __pgprot(PTE_SPECIAL));
++}
++#endif
++
+ #define __pmd_to_phys(pmd)	__pte_to_phys(pmd_pte(pmd))
+ #define __phys_to_pmd_val(phys)	__phys_to_pte_val(phys)
+ #define pmd_pfn(pmd)		((__pmd_to_phys(pmd) & PMD_MASK) >> PAGE_SHIFT)
+@@ -545,6 +553,27 @@ static inline pmd_t pmd_mkdevmap(pmd_t pmd)
+ #define pud_pfn(pud)		((__pud_to_phys(pud) & PUD_MASK) >> PAGE_SHIFT)
+ #define pfn_pud(pfn,prot)	__pud(__phys_to_pud_val((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot))
+ 
++#ifdef CONFIG_ARCH_SUPPORTS_PUD_PFNMAP
++#define pud_special(pte)	pte_special(pud_pte(pud))
++#define pud_mkspecial(pte)	pte_pud(pte_mkspecial(pud_pte(pud)))
++#endif
++
++#define pmd_pgprot pmd_pgprot
++static inline pgprot_t pmd_pgprot(pmd_t pmd)
++{
++	unsigned long pfn = pmd_pfn(pmd);
++
++	return __pgprot(pmd_val(pfn_pmd(pfn, __pgprot(0))) ^ pmd_val(pmd));
++}
++
++#define pud_pgprot pud_pgprot
++static inline pgprot_t pud_pgprot(pud_t pud)
++{
++	unsigned long pfn = pud_pfn(pud);
++
++	return __pgprot(pud_val(pfn_pud(pfn, __pgprot(0))) ^ pud_val(pud));
++}
++
+ static inline void __set_pte_at(struct mm_struct *mm,
+ 				unsigned long __always_unused addr,
+ 				pte_t *ptep, pte_t pte, unsigned int nr)