mm/x86: support large pfn mappings

jira LE-3557
Rebuild_History Non-Buildable kernel-5.14.0-570.26.1.el9_6
commit-author Peter Xu <[email protected]>
commit 7518202
Empty-Commit: Cherry-Pick Conflicts during history rebuild.
Will be included in final tarball splat. Ref for failed cherry-pick at:
ciq/ciq_backports/kernel-5.14.0-570.26.1.el9_6/75182022.failed

Helpers to install and detect special pmd/pud entries.  In short, bit 9 on
x86 is not used for pmd/pud, so we can directly define them the same as
the pte level.  One note is that it's also used in _PAGE_BIT_CPA_TEST but
that is only used in the debug test, and shouldn't conflict in this case.

One note is that pxx_set|clear_flags() for pmd/pud will need to be moved
upper so that they can be referenced by the new special bit helpers.
There's no change in the code that was moved.

Link: https://lkml.kernel.org/r/[email protected]
	Signed-off-by: Peter Xu <[email protected]>
	Cc: Thomas Gleixner <[email protected]>
	Cc: Ingo Molnar <[email protected]>
	Cc: Borislav Petkov <[email protected]>
	Cc: Dave Hansen <[email protected]>
	Cc: Alexander Gordeev <[email protected]>
	Cc: Alex Williamson <[email protected]>
	Cc: Aneesh Kumar K.V <[email protected]>
	Cc: Catalin Marinas <[email protected]>
	Cc: Christian Borntraeger <[email protected]>
	Cc: David Hildenbrand <[email protected]>
	Cc: Gavin Shan <[email protected]>
	Cc: Gerald Schaefer <[email protected]>
	Cc: Heiko Carstens <[email protected]>
	Cc: Jason Gunthorpe <[email protected]>
	Cc: Matthew Wilcox <[email protected]>
	Cc: Niklas Schnelle <[email protected]>
	Cc: Paolo Bonzini <[email protected]>
	Cc: Ryan Roberts <[email protected]>
	Cc: Sean Christopherson <[email protected]>
	Cc: Sven Schnelle <[email protected]>
	Cc: Vasily Gorbik <[email protected]>
	Cc: Will Deacon <[email protected]>
	Cc: Zi Yan <[email protected]>
	Signed-off-by: Andrew Morton <[email protected]>
(cherry picked from commit 7518202)
	Signed-off-by: Jonathan Maple <[email protected]>

# Conflicts:
#	arch/x86/Kconfig

Author: PlaidCat

ciq/ciq_backports/kernel-5.14.0-570.26.1.el9_6/75182022.failed

@@ -0,0 +1,179 @@
+mm/x86: support large pfn mappings
+
+jira LE-3557
+Rebuild_History Non-Buildable kernel-5.14.0-570.26.1.el9_6
+commit-author Peter Xu <[email protected]>
+commit 75182022a0439788415b2dd1db3086e07aa506f7
+Empty-Commit: Cherry-Pick Conflicts during history rebuild.
+Will be included in final tarball splat. Ref for failed cherry-pick at:
+ciq/ciq_backports/kernel-5.14.0-570.26.1.el9_6/75182022.failed
+
+Helpers to install and detect special pmd/pud entries.  In short, bit 9 on
+x86 is not used for pmd/pud, so we can directly define them the same as
+the pte level.  One note is that it's also used in _PAGE_BIT_CPA_TEST but
+that is only used in the debug test, and shouldn't conflict in this case.
+
+One note is that pxx_set|clear_flags() for pmd/pud will need to be moved
+upper so that they can be referenced by the new special bit helpers. 
+There's no change in the code that was moved.
+
+Link: https://lkml.kernel.org/r/[email protected]
+	Signed-off-by: Peter Xu <[email protected]>
+	Cc: Thomas Gleixner <[email protected]>
+	Cc: Ingo Molnar <[email protected]>
+	Cc: Borislav Petkov <[email protected]>
+	Cc: Dave Hansen <[email protected]>
+	Cc: Alexander Gordeev <[email protected]>
+	Cc: Alex Williamson <[email protected]>
+	Cc: Aneesh Kumar K.V <[email protected]>
+	Cc: Catalin Marinas <[email protected]>
+	Cc: Christian Borntraeger <[email protected]>
+	Cc: David Hildenbrand <[email protected]>
+	Cc: Gavin Shan <[email protected]>
+	Cc: Gerald Schaefer <[email protected]>
+	Cc: Heiko Carstens <[email protected]>
+	Cc: Jason Gunthorpe <[email protected]>
+	Cc: Matthew Wilcox <[email protected]>
+	Cc: Niklas Schnelle <[email protected]>
+	Cc: Paolo Bonzini <[email protected]>
+	Cc: Ryan Roberts <[email protected]>
+	Cc: Sean Christopherson <[email protected]>
+	Cc: Sven Schnelle <[email protected]>
+	Cc: Vasily Gorbik <[email protected]>
+	Cc: Will Deacon <[email protected]>
+	Cc: Zi Yan <[email protected]>
+	Signed-off-by: Andrew Morton <[email protected]>
+(cherry picked from commit 75182022a0439788415b2dd1db3086e07aa506f7)
+	Signed-off-by: Jonathan Maple <[email protected]>
+
+# Conflicts:
+#	arch/x86/Kconfig
+diff --cc arch/x86/Kconfig
+index a5a59118efe4,d4dbe9717e96..000000000000
+--- a/arch/x86/Kconfig
++++ b/arch/x86/Kconfig
+@@@ -28,7 -28,7 +28,11 @@@ config X86_6
+  	select ARCH_HAS_GIGANTIC_PAGE
+  	select ARCH_SUPPORTS_INT128 if CC_HAS_INT128
+  	select ARCH_SUPPORTS_PER_VMA_LOCK
+++<<<<<<< HEAD
+ +	select ARCH_SUPPORTS_RT
+++=======
++ 	select ARCH_SUPPORTS_HUGE_PFNMAP if TRANSPARENT_HUGEPAGE
+++>>>>>>> 75182022a043 (mm/x86: support large pfn mappings)
+  	select HAVE_ARCH_SOFT_DIRTY
+  	select MODULES_USE_ELF_RELA
+  	select NEED_DMA_MAP_STATE
+* Unmerged path arch/x86/Kconfig
+diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h
+index 8149afec43a4..c5bc120fade8 100644
+--- a/arch/x86/include/asm/pgtable.h
++++ b/arch/x86/include/asm/pgtable.h
+@@ -121,6 +121,34 @@ extern pmdval_t early_pmd_flags;
+ #define arch_end_context_switch(prev)	do {} while(0)
+ #endif	/* CONFIG_PARAVIRT_XXL */
+ 
++static inline pmd_t pmd_set_flags(pmd_t pmd, pmdval_t set)
++{
++	pmdval_t v = native_pmd_val(pmd);
++
++	return native_make_pmd(v | set);
++}
++
++static inline pmd_t pmd_clear_flags(pmd_t pmd, pmdval_t clear)
++{
++	pmdval_t v = native_pmd_val(pmd);
++
++	return native_make_pmd(v & ~clear);
++}
++
++static inline pud_t pud_set_flags(pud_t pud, pudval_t set)
++{
++	pudval_t v = native_pud_val(pud);
++
++	return native_make_pud(v | set);
++}
++
++static inline pud_t pud_clear_flags(pud_t pud, pudval_t clear)
++{
++	pudval_t v = native_pud_val(pud);
++
++	return native_make_pud(v & ~clear);
++}
++
+ /*
+  * The following only work if pte_present() is true.
+  * Undefined behaviour if not..
+@@ -310,6 +338,30 @@ static inline int pud_devmap(pud_t pud)
+ }
+ #endif
+ 
++#ifdef CONFIG_ARCH_SUPPORTS_PMD_PFNMAP
++static inline bool pmd_special(pmd_t pmd)
++{
++	return pmd_flags(pmd) & _PAGE_SPECIAL;
++}
++
++static inline pmd_t pmd_mkspecial(pmd_t pmd)
++{
++	return pmd_set_flags(pmd, _PAGE_SPECIAL);
++}
++#endif	/* CONFIG_ARCH_SUPPORTS_PMD_PFNMAP */
++
++#ifdef CONFIG_ARCH_SUPPORTS_PUD_PFNMAP
++static inline bool pud_special(pud_t pud)
++{
++	return pud_flags(pud) & _PAGE_SPECIAL;
++}
++
++static inline pud_t pud_mkspecial(pud_t pud)
++{
++	return pud_set_flags(pud, _PAGE_SPECIAL);
++}
++#endif	/* CONFIG_ARCH_SUPPORTS_PUD_PFNMAP */
++
+ static inline int pgd_devmap(pgd_t pgd)
+ {
+ 	return 0;
+@@ -480,20 +532,6 @@ static inline pte_t pte_mkdevmap(pte_t pte)
+ 	return pte_set_flags(pte, _PAGE_SPECIAL|_PAGE_DEVMAP);
+ }
+ 
+-static inline pmd_t pmd_set_flags(pmd_t pmd, pmdval_t set)
+-{
+-	pmdval_t v = native_pmd_val(pmd);
+-
+-	return native_make_pmd(v | set);
+-}
+-
+-static inline pmd_t pmd_clear_flags(pmd_t pmd, pmdval_t clear)
+-{
+-	pmdval_t v = native_pmd_val(pmd);
+-
+-	return native_make_pmd(v & ~clear);
+-}
+-
+ /* See comments above mksaveddirty_shift() */
+ static inline pmd_t pmd_mksaveddirty(pmd_t pmd)
+ {
+@@ -588,20 +626,6 @@ static inline pmd_t pmd_mkwrite_novma(pmd_t pmd)
+ pmd_t pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma);
+ #define pmd_mkwrite pmd_mkwrite
+ 
+-static inline pud_t pud_set_flags(pud_t pud, pudval_t set)
+-{
+-	pudval_t v = native_pud_val(pud);
+-
+-	return native_make_pud(v | set);
+-}
+-
+-static inline pud_t pud_clear_flags(pud_t pud, pudval_t clear)
+-{
+-	pudval_t v = native_pud_val(pud);
+-
+-	return native_make_pud(v & ~clear);
+-}
+-
+ /* See comments above mksaveddirty_shift() */
+ static inline pud_t pud_mksaveddirty(pud_t pud)
+ {