mm: new follow_pfnmap API

jira LE-3557
Rebuild_History Non-Buildable kernel-5.14.0-570.26.1.el9_6
commit-author Peter Xu <[email protected]>
commit 6da8e96
Empty-Commit: Cherry-Pick Conflicts during history rebuild.
Will be included in final tarball splat. Ref for failed cherry-pick at:
ciq/ciq_backports/kernel-5.14.0-570.26.1.el9_6/6da8e963.failed

Introduce a pair of APIs to follow pfn mappings to get entry information.
It's very similar to what follow_pte() does before, but different in that
it recognizes huge pfn mappings.

Link: https://lkml.kernel.org/r/[email protected]
	Signed-off-by: Peter Xu <[email protected]>
	Cc: Alexander Gordeev <[email protected]>
	Cc: Alex Williamson <[email protected]>
	Cc: Aneesh Kumar K.V <[email protected]>
	Cc: Borislav Petkov <[email protected]>
	Cc: Catalin Marinas <[email protected]>
	Cc: Christian Borntraeger <[email protected]>
	Cc: Dave Hansen <[email protected]>
	Cc: David Hildenbrand <[email protected]>
	Cc: Gavin Shan <[email protected]>
	Cc: Gerald Schaefer <[email protected]>
	Cc: Heiko Carstens <[email protected]>
	Cc: Ingo Molnar <[email protected]>
	Cc: Jason Gunthorpe <[email protected]>
	Cc: Matthew Wilcox <[email protected]>
	Cc: Niklas Schnelle <[email protected]>
	Cc: Paolo Bonzini <[email protected]>
	Cc: Ryan Roberts <[email protected]>
	Cc: Sean Christopherson <[email protected]>
	Cc: Sven Schnelle <[email protected]>
	Cc: Thomas Gleixner <[email protected]>
	Cc: Vasily Gorbik <[email protected]>
	Cc: Will Deacon <[email protected]>
	Cc: Zi Yan <[email protected]>
	Signed-off-by: Andrew Morton <[email protected]>
(cherry picked from commit 6da8e96)
	Signed-off-by: Jonathan Maple <[email protected]>

# Conflicts:
#	mm/memory.c

Author: PlaidCat

ciq/ciq_backports/kernel-5.14.0-570.26.1.el9_6/6da8e963.failed

@@ -0,0 +1,316 @@
+mm: new follow_pfnmap API
+
+jira LE-3557
+Rebuild_History Non-Buildable kernel-5.14.0-570.26.1.el9_6
+commit-author Peter Xu <[email protected]>
+commit 6da8e9634bb7e3fdad9ae0e4db873a05036c4343
+Empty-Commit: Cherry-Pick Conflicts during history rebuild.
+Will be included in final tarball splat. Ref for failed cherry-pick at:
+ciq/ciq_backports/kernel-5.14.0-570.26.1.el9_6/6da8e963.failed
+
+Introduce a pair of APIs to follow pfn mappings to get entry information. 
+It's very similar to what follow_pte() does before, but different in that
+it recognizes huge pfn mappings.
+
+Link: https://lkml.kernel.org/r/[email protected]
+	Signed-off-by: Peter Xu <[email protected]>
+	Cc: Alexander Gordeev <[email protected]>
+	Cc: Alex Williamson <[email protected]>
+	Cc: Aneesh Kumar K.V <[email protected]>
+	Cc: Borislav Petkov <[email protected]>
+	Cc: Catalin Marinas <[email protected]>
+	Cc: Christian Borntraeger <[email protected]>
+	Cc: Dave Hansen <[email protected]>
+	Cc: David Hildenbrand <[email protected]>
+	Cc: Gavin Shan <[email protected]>
+	Cc: Gerald Schaefer <[email protected]>
+	Cc: Heiko Carstens <[email protected]>
+	Cc: Ingo Molnar <[email protected]>
+	Cc: Jason Gunthorpe <[email protected]>
+	Cc: Matthew Wilcox <[email protected]>
+	Cc: Niklas Schnelle <[email protected]>
+	Cc: Paolo Bonzini <[email protected]>
+	Cc: Ryan Roberts <[email protected]>
+	Cc: Sean Christopherson <[email protected]>
+	Cc: Sven Schnelle <[email protected]>
+	Cc: Thomas Gleixner <[email protected]>
+	Cc: Vasily Gorbik <[email protected]>
+	Cc: Will Deacon <[email protected]>
+	Cc: Zi Yan <[email protected]>
+	Signed-off-by: Andrew Morton <[email protected]>
+(cherry picked from commit 6da8e9634bb7e3fdad9ae0e4db873a05036c4343)
+	Signed-off-by: Jonathan Maple <[email protected]>
+
+# Conflicts:
+#	mm/memory.c
+diff --cc mm/memory.c
+index e2794e3b8919,3878bf69bc14..000000000000
+--- a/mm/memory.c
++++ b/mm/memory.c
+@@@ -5666,71 -6172,157 +5666,223 @@@ out
+  }
+  EXPORT_SYMBOL_GPL(follow_pte);
+  
+++<<<<<<< HEAD
+ +/**
+ + * follow_pfn - look up PFN at a user virtual address
+ + * @vma: memory mapping
+ + * @address: user virtual address
+ + * @pfn: location to store found PFN
+ + *
+ + * Only IO mappings and raw PFN mappings are allowed.
+ + *
+ + * This function does not allow the caller to read the permissions
+ + * of the PTE.  Do not use it.
+ + *
+ + * Return: zero and the pfn at @pfn on success, -ve otherwise.
+ + */
+ +int follow_pfn(struct vm_area_struct *vma, unsigned long address,
+ +	unsigned long *pfn)
+ +{
+ +	int ret = -EINVAL;
+ +	spinlock_t *ptl;
+ +	pte_t *ptep;
+ +
+ +	if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
+ +		return ret;
+ +
+ +	ret = follow_pte(vma->vm_mm, address, &ptep, &ptl);
+ +	if (ret)
+ +		return ret;
+ +	*pfn = pte_pfn(ptep_get(ptep));
+ +	pte_unmap_unlock(ptep, ptl);
+ +	return 0;
+ +}
+ +EXPORT_SYMBOL(follow_pfn);
+++=======
++ static inline void pfnmap_args_setup(struct follow_pfnmap_args *args,
++ 				     spinlock_t *lock, pte_t *ptep,
++ 				     pgprot_t pgprot, unsigned long pfn_base,
++ 				     unsigned long addr_mask, bool writable,
++ 				     bool special)
++ {
++ 	args->lock = lock;
++ 	args->ptep = ptep;
++ 	args->pfn = pfn_base + ((args->address & ~addr_mask) >> PAGE_SHIFT);
++ 	args->pgprot = pgprot;
++ 	args->writable = writable;
++ 	args->special = special;
++ }
++ 
++ static inline void pfnmap_lockdep_assert(struct vm_area_struct *vma)
++ {
++ #ifdef CONFIG_LOCKDEP
++ 	struct address_space *mapping = vma->vm_file->f_mapping;
++ 
++ 	if (mapping)
++ 		lockdep_assert(lockdep_is_held(&vma->vm_file->f_mapping->i_mmap_rwsem) ||
++ 			       lockdep_is_held(&vma->vm_mm->mmap_lock));
++ 	else
++ 		lockdep_assert(lockdep_is_held(&vma->vm_mm->mmap_lock));
++ #endif
++ }
++ 
++ /**
++  * follow_pfnmap_start() - Look up a pfn mapping at a user virtual address
++  * @args: Pointer to struct @follow_pfnmap_args
++  *
++  * The caller needs to setup args->vma and args->address to point to the
++  * virtual address as the target of such lookup.  On a successful return,
++  * the results will be put into other output fields.
++  *
++  * After the caller finished using the fields, the caller must invoke
++  * another follow_pfnmap_end() to proper releases the locks and resources
++  * of such look up request.
++  *
++  * During the start() and end() calls, the results in @args will be valid
++  * as proper locks will be held.  After the end() is called, all the fields
++  * in @follow_pfnmap_args will be invalid to be further accessed.  Further
++  * use of such information after end() may require proper synchronizations
++  * by the caller with page table updates, otherwise it can create a
++  * security bug.
++  *
++  * If the PTE maps a refcounted page, callers are responsible to protect
++  * against invalidation with MMU notifiers; otherwise access to the PFN at
++  * a later point in time can trigger use-after-free.
++  *
++  * Only IO mappings and raw PFN mappings are allowed.  The mmap semaphore
++  * should be taken for read, and the mmap semaphore cannot be released
++  * before the end() is invoked.
++  *
++  * This function must not be used to modify PTE content.
++  *
++  * Return: zero on success, negative otherwise.
++  */
++ int follow_pfnmap_start(struct follow_pfnmap_args *args)
++ {
++ 	struct vm_area_struct *vma = args->vma;
++ 	unsigned long address = args->address;
++ 	struct mm_struct *mm = vma->vm_mm;
++ 	spinlock_t *lock;
++ 	pgd_t *pgdp;
++ 	p4d_t *p4dp, p4d;
++ 	pud_t *pudp, pud;
++ 	pmd_t *pmdp, pmd;
++ 	pte_t *ptep, pte;
++ 
++ 	pfnmap_lockdep_assert(vma);
++ 
++ 	if (unlikely(address < vma->vm_start || address >= vma->vm_end))
++ 		goto out;
++ 
++ 	if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
++ 		goto out;
++ retry:
++ 	pgdp = pgd_offset(mm, address);
++ 	if (pgd_none(*pgdp) || unlikely(pgd_bad(*pgdp)))
++ 		goto out;
++ 
++ 	p4dp = p4d_offset(pgdp, address);
++ 	p4d = READ_ONCE(*p4dp);
++ 	if (p4d_none(p4d) || unlikely(p4d_bad(p4d)))
++ 		goto out;
++ 
++ 	pudp = pud_offset(p4dp, address);
++ 	pud = READ_ONCE(*pudp);
++ 	if (pud_none(pud))
++ 		goto out;
++ 	if (pud_leaf(pud)) {
++ 		lock = pud_lock(mm, pudp);
++ 		if (!unlikely(pud_leaf(pud))) {
++ 			spin_unlock(lock);
++ 			goto retry;
++ 		}
++ 		pfnmap_args_setup(args, lock, NULL, pud_pgprot(pud),
++ 				  pud_pfn(pud), PUD_MASK, pud_write(pud),
++ 				  pud_special(pud));
++ 		return 0;
++ 	}
++ 
++ 	pmdp = pmd_offset(pudp, address);
++ 	pmd = pmdp_get_lockless(pmdp);
++ 	if (pmd_leaf(pmd)) {
++ 		lock = pmd_lock(mm, pmdp);
++ 		if (!unlikely(pmd_leaf(pmd))) {
++ 			spin_unlock(lock);
++ 			goto retry;
++ 		}
++ 		pfnmap_args_setup(args, lock, NULL, pmd_pgprot(pmd),
++ 				  pmd_pfn(pmd), PMD_MASK, pmd_write(pmd),
++ 				  pmd_special(pmd));
++ 		return 0;
++ 	}
++ 
++ 	ptep = pte_offset_map_lock(mm, pmdp, address, &lock);
++ 	if (!ptep)
++ 		goto out;
++ 	pte = ptep_get(ptep);
++ 	if (!pte_present(pte))
++ 		goto unlock;
++ 	pfnmap_args_setup(args, lock, ptep, pte_pgprot(pte),
++ 			  pte_pfn(pte), PAGE_MASK, pte_write(pte),
++ 			  pte_special(pte));
++ 	return 0;
++ unlock:
++ 	pte_unmap_unlock(ptep, lock);
++ out:
++ 	return -EINVAL;
++ }
++ EXPORT_SYMBOL_GPL(follow_pfnmap_start);
++ 
++ /**
++  * follow_pfnmap_end(): End a follow_pfnmap_start() process
++  * @args: Pointer to struct @follow_pfnmap_args
++  *
++  * Must be used in pair of follow_pfnmap_start().  See the start() function
++  * above for more information.
++  */
++ void follow_pfnmap_end(struct follow_pfnmap_args *args)
++ {
++ 	if (args->lock)
++ 		spin_unlock(args->lock);
++ 	if (args->ptep)
++ 		pte_unmap(args->ptep);
++ }
++ EXPORT_SYMBOL_GPL(follow_pfnmap_end);
+++>>>>>>> 6da8e9634bb7 (mm: new follow_pfnmap API)
+  
+  #ifdef CONFIG_HAVE_IOREMAP_PROT
+ +int follow_phys(struct vm_area_struct *vma,
+ +		unsigned long address, unsigned int flags,
+ +		unsigned long *prot, resource_size_t *phys)
+ +{
+ +	int ret = -EINVAL;
+ +	pte_t *ptep, pte;
+ +	spinlock_t *ptl;
+ +
+ +	if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
+ +		goto out;
+ +
+ +	if (follow_pte(vma->vm_mm, address, &ptep, &ptl))
+ +		goto out;
+ +	pte = ptep_get(ptep);
+ +
+ +	/* Never return PFNs of anon folios in COW mappings. */
+ +	if (vm_normal_folio(vma, address, pte))
+ +		goto unlock;
+ +
+ +	if ((flags & FOLL_WRITE) && !pte_write(pte))
+ +		goto unlock;
+ +
+ +	*prot = pgprot_val(pte_pgprot(pte));
+ +	*phys = (resource_size_t)pte_pfn(pte) << PAGE_SHIFT;
+ +
+ +	ret = 0;
+ +unlock:
+ +	pte_unmap_unlock(ptep, ptl);
+ +out:
+ +	return ret;
+ +}
+ +
+  /**
+   * generic_access_phys - generic implementation for iomem mmap access
+   * @vma: the vma to access
+diff --git a/include/linux/mm.h b/include/linux/mm.h
+index 196c481ec160..51f28b4e78fc 100644
+--- a/include/linux/mm.h
++++ b/include/linux/mm.h
+@@ -2436,6 +2436,37 @@ int follow_phys(struct vm_area_struct *vma, unsigned long address,
+ int generic_access_phys(struct vm_area_struct *vma, unsigned long addr,
+ 			void *buf, int len, int write);
+ 
++struct follow_pfnmap_args {
++	/**
++	 * Inputs:
++	 * @vma: Pointer to @vm_area_struct struct
++	 * @address: the virtual address to walk
++	 */
++	struct vm_area_struct *vma;
++	unsigned long address;
++	/**
++	 * Internals:
++	 *
++	 * The caller shouldn't touch any of these.
++	 */
++	spinlock_t *lock;
++	pte_t *ptep;
++	/**
++	 * Outputs:
++	 *
++	 * @pfn: the PFN of the address
++	 * @pgprot: the pgprot_t of the mapping
++	 * @writable: whether the mapping is writable
++	 * @special: whether the mapping is a special mapping (real PFN maps)
++	 */
++	unsigned long pfn;
++	pgprot_t pgprot;
++	bool writable;
++	bool special;
++};
++int follow_pfnmap_start(struct follow_pfnmap_args *args);
++void follow_pfnmap_end(struct follow_pfnmap_args *args);
++
+ extern void truncate_pagecache(struct inode *inode, loff_t new);
+ extern void truncate_setsize(struct inode *inode, loff_t newsize);
+ void pagecache_isize_extended(struct inode *inode, loff_t from, loff_t to);
+* Unmerged path mm/memory.c