mm: Provide address mask in struct follow_pfnmap_args

jira LE-3557
Rebuild_History Non-Buildable kernel-5.14.0-570.26.1.el9_6
commit-author Alex Williamson <[email protected]>
commit 62fb8ad
Empty-Commit: Cherry-Pick Conflicts during history rebuild.
Will be included in final tarball splat. Ref for failed cherry-pick at:
ciq/ciq_backports/kernel-5.14.0-570.26.1.el9_6/62fb8adc.failed

follow_pfnmap_start() walks the page table for a given address and
fills out the struct follow_pfnmap_args in pfnmap_args_setup().
The address mask of the page table level is already provided to this
latter function for calculating the pfn.  This address mask can also
be useful for the caller to determine the extent of the contiguous
mapping.

For example, vfio-pci now supports huge_fault for pfnmaps and is able
to insert pud and pmd mappings.  When we DMA map these pfnmaps, ex.
PCI MMIO BARs, we iterate follow_pfnmap_start() to get each pfn to test
for a contiguous pfn range.  Providing the mapping address mask allows
us to skip the extent of the mapping level.  Assuming a 1GB pud level
and 4KB page size, iterations are reduced by a factor of 256K.  In wall
clock time, mapping a 32GB PCI BAR is reduced from ~1s to <1ms.

	Cc: Andrew Morton <[email protected]>
	Cc: David Hildenbrand <[email protected]>
	Cc: [email protected]
	Reviewed-by: Peter Xu <[email protected]>
	Reviewed-by: Mitchell Augustin <[email protected]>
	Tested-by: Mitchell Augustin <[email protected]>
	Reviewed-by: Jason Gunthorpe <[email protected]>
	Acked-by: David Hildenbrand <[email protected]>
Link: https://lore.kernel.org/r/[email protected]
	Signed-off-by: Alex Williamson <[email protected]>
(cherry picked from commit 62fb8ad)
	Signed-off-by: Jonathan Maple <[email protected]>

# Conflicts:
#	include/linux/mm.h
#	mm/memory.c

Author: PlaidCat

ciq/ciq_backports/kernel-5.14.0-570.26.1.el9_6/62fb8adc.failed

@@ -0,0 +1,268 @@
+mm: Provide address mask in struct follow_pfnmap_args
+
+jira LE-3557
+Rebuild_History Non-Buildable kernel-5.14.0-570.26.1.el9_6
+commit-author Alex Williamson <[email protected]>
+commit 62fb8adc43afad5fa1c9cadc6f3a8e9fb72af194
+Empty-Commit: Cherry-Pick Conflicts during history rebuild.
+Will be included in final tarball splat. Ref for failed cherry-pick at:
+ciq/ciq_backports/kernel-5.14.0-570.26.1.el9_6/62fb8adc.failed
+
+follow_pfnmap_start() walks the page table for a given address and
+fills out the struct follow_pfnmap_args in pfnmap_args_setup().
+The address mask of the page table level is already provided to this
+latter function for calculating the pfn.  This address mask can also
+be useful for the caller to determine the extent of the contiguous
+mapping.
+
+For example, vfio-pci now supports huge_fault for pfnmaps and is able
+to insert pud and pmd mappings.  When we DMA map these pfnmaps, ex.
+PCI MMIO BARs, we iterate follow_pfnmap_start() to get each pfn to test
+for a contiguous pfn range.  Providing the mapping address mask allows
+us to skip the extent of the mapping level.  Assuming a 1GB pud level
+and 4KB page size, iterations are reduced by a factor of 256K.  In wall
+clock time, mapping a 32GB PCI BAR is reduced from ~1s to <1ms.
+
+	Cc: Andrew Morton <[email protected]>
+	Cc: David Hildenbrand <[email protected]>
+	Cc: [email protected]
+	Reviewed-by: Peter Xu <[email protected]>
+	Reviewed-by: Mitchell Augustin <[email protected]>
+	Tested-by: Mitchell Augustin <[email protected]>
+	Reviewed-by: Jason Gunthorpe <[email protected]>
+	Acked-by: David Hildenbrand <[email protected]>
+Link: https://lore.kernel.org/r/[email protected]
+	Signed-off-by: Alex Williamson <[email protected]>
+(cherry picked from commit 62fb8adc43afad5fa1c9cadc6f3a8e9fb72af194)
+	Signed-off-by: Jonathan Maple <[email protected]>
+
+# Conflicts:
+#	include/linux/mm.h
+#	mm/memory.c
+diff --cc include/linux/mm.h
+index 196c481ec160,92b30dba7e38..000000000000
+--- a/include/linux/mm.h
++++ b/include/linux/mm.h
+@@@ -2436,6 -2398,39 +2436,42 @@@ int follow_phys(struct vm_area_struct *
+  int generic_access_phys(struct vm_area_struct *vma, unsigned long addr,
+  			void *buf, int len, int write);
+  
+++<<<<<<< HEAD
+++=======
++ struct follow_pfnmap_args {
++ 	/**
++ 	 * Inputs:
++ 	 * @vma: Pointer to @vm_area_struct struct
++ 	 * @address: the virtual address to walk
++ 	 */
++ 	struct vm_area_struct *vma;
++ 	unsigned long address;
++ 	/**
++ 	 * Internals:
++ 	 *
++ 	 * The caller shouldn't touch any of these.
++ 	 */
++ 	spinlock_t *lock;
++ 	pte_t *ptep;
++ 	/**
++ 	 * Outputs:
++ 	 *
++ 	 * @pfn: the PFN of the address
++ 	 * @addr_mask: address mask covering pfn
++ 	 * @pgprot: the pgprot_t of the mapping
++ 	 * @writable: whether the mapping is writable
++ 	 * @special: whether the mapping is a special mapping (real PFN maps)
++ 	 */
++ 	unsigned long pfn;
++ 	unsigned long addr_mask;
++ 	pgprot_t pgprot;
++ 	bool writable;
++ 	bool special;
++ };
++ int follow_pfnmap_start(struct follow_pfnmap_args *args);
++ void follow_pfnmap_end(struct follow_pfnmap_args *args);
++ 
+++>>>>>>> 62fb8adc43af (mm: Provide address mask in struct follow_pfnmap_args)
+  extern void truncate_pagecache(struct inode *inode, loff_t new);
+  extern void truncate_setsize(struct inode *inode, loff_t newsize);
+  void pagecache_isize_extended(struct inode *inode, loff_t from, loff_t to);
+diff --cc mm/memory.c
+index e2794e3b8919,68aa0f11633e..000000000000
+--- a/mm/memory.c
++++ b/mm/memory.c
+@@@ -5607,60 -6479,137 +5607,92 @@@ int __pmd_alloc(struct mm_struct *mm, p
+  }
+  #endif /* __PAGETABLE_PMD_FOLDED */
+  
+++<<<<<<< HEAD
+++=======
++ static inline void pfnmap_args_setup(struct follow_pfnmap_args *args,
++ 				     spinlock_t *lock, pte_t *ptep,
++ 				     pgprot_t pgprot, unsigned long pfn_base,
++ 				     unsigned long addr_mask, bool writable,
++ 				     bool special)
++ {
++ 	args->lock = lock;
++ 	args->ptep = ptep;
++ 	args->pfn = pfn_base + ((args->address & ~addr_mask) >> PAGE_SHIFT);
++ 	args->addr_mask = addr_mask;
++ 	args->pgprot = pgprot;
++ 	args->writable = writable;
++ 	args->special = special;
++ }
++ 
++ static inline void pfnmap_lockdep_assert(struct vm_area_struct *vma)
++ {
++ #ifdef CONFIG_LOCKDEP
++ 	struct file *file = vma->vm_file;
++ 	struct address_space *mapping = file ? file->f_mapping : NULL;
++ 
++ 	if (mapping)
++ 		lockdep_assert(lockdep_is_held(&mapping->i_mmap_rwsem) ||
++ 			       lockdep_is_held(&vma->vm_mm->mmap_lock));
++ 	else
++ 		lockdep_assert(lockdep_is_held(&vma->vm_mm->mmap_lock));
++ #endif
++ }
++ 
+++>>>>>>> 62fb8adc43af (mm: Provide address mask in struct follow_pfnmap_args)
+  /**
+ - * follow_pfnmap_start() - Look up a pfn mapping at a user virtual address
+ - * @args: Pointer to struct @follow_pfnmap_args
+ - *
+ - * The caller needs to setup args->vma and args->address to point to the
+ - * virtual address as the target of such lookup.  On a successful return,
+ - * the results will be put into other output fields.
+ + * follow_pte - look up PTE at a user virtual address
+ + * @mm: the mm_struct of the target address space
+ + * @address: user virtual address
+ + * @ptepp: location to store found PTE
+ + * @ptlp: location to store the lock for the PTE
+   *
+ - * After the caller finished using the fields, the caller must invoke
+ - * another follow_pfnmap_end() to proper releases the locks and resources
+ - * of such look up request.
+ - *
+ - * During the start() and end() calls, the results in @args will be valid
+ - * as proper locks will be held.  After the end() is called, all the fields
+ - * in @follow_pfnmap_args will be invalid to be further accessed.  Further
+ - * use of such information after end() may require proper synchronizations
+ - * by the caller with page table updates, otherwise it can create a
+ - * security bug.
+ - *
+ - * If the PTE maps a refcounted page, callers are responsible to protect
+ - * against invalidation with MMU notifiers; otherwise access to the PFN at
+ - * a later point in time can trigger use-after-free.
+ + * On a successful return, the pointer to the PTE is stored in @ptepp;
+ + * the corresponding lock is taken and its location is stored in @ptlp.
+ + * The contents of the PTE are only stable until @ptlp is released;
+ + * any further use, if any, must be protected against invalidation
+ + * with MMU notifiers.
+   *
+   * Only IO mappings and raw PFN mappings are allowed.  The mmap semaphore
+ - * should be taken for read, and the mmap semaphore cannot be released
+ - * before the end() is invoked.
+ + * should be taken for read.
+   *
+ - * This function must not be used to modify PTE content.
+ + * KVM uses this function.  While it is arguably less bad than ``follow_pfn``,
+ + * it is not a good general-purpose API.
+   *
+ - * Return: zero on success, negative otherwise.
+ + * Return: zero on success, -ve otherwise.
+   */
+ -int follow_pfnmap_start(struct follow_pfnmap_args *args)
+ +int follow_pte(struct mm_struct *mm, unsigned long address,
+ +	       pte_t **ptepp, spinlock_t **ptlp)
+  {
+ -	struct vm_area_struct *vma = args->vma;
+ -	unsigned long address = args->address;
+ -	struct mm_struct *mm = vma->vm_mm;
+ -	spinlock_t *lock;
+ -	pgd_t *pgdp;
+ -	p4d_t *p4dp, p4d;
+ -	pud_t *pudp, pud;
+ -	pmd_t *pmdp, pmd;
+ -	pte_t *ptep, pte;
+ -
+ -	pfnmap_lockdep_assert(vma);
+ -
+ -	if (unlikely(address < vma->vm_start || address >= vma->vm_end))
+ -		goto out;
+ +	pgd_t *pgd;
+ +	p4d_t *p4d;
+ +	pud_t *pud;
+ +	pmd_t *pmd;
+ +	pte_t *ptep;
+  
+ -	if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
+ -		goto out;
+ -retry:
+ -	pgdp = pgd_offset(mm, address);
+ -	if (pgd_none(*pgdp) || unlikely(pgd_bad(*pgdp)))
+ +	pgd = pgd_offset(mm, address);
+ +	if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd)))
+  		goto out;
+  
+ -	p4dp = p4d_offset(pgdp, address);
+ -	p4d = READ_ONCE(*p4dp);
+ -	if (p4d_none(p4d) || unlikely(p4d_bad(p4d)))
+ +	p4d = p4d_offset(pgd, address);
+ +	if (p4d_none(*p4d) || unlikely(p4d_bad(*p4d)))
+  		goto out;
+  
+ -	pudp = pud_offset(p4dp, address);
+ -	pud = READ_ONCE(*pudp);
+ -	if (pud_none(pud))
+ +	pud = pud_offset(p4d, address);
+ +	if (pud_none(*pud) || unlikely(pud_bad(*pud)))
+  		goto out;
+ -	if (pud_leaf(pud)) {
+ -		lock = pud_lock(mm, pudp);
+ -		if (!unlikely(pud_leaf(pud))) {
+ -			spin_unlock(lock);
+ -			goto retry;
+ -		}
+ -		pfnmap_args_setup(args, lock, NULL, pud_pgprot(pud),
+ -				  pud_pfn(pud), PUD_MASK, pud_write(pud),
+ -				  pud_special(pud));
+ -		return 0;
+ -	}
+  
+ -	pmdp = pmd_offset(pudp, address);
+ -	pmd = pmdp_get_lockless(pmdp);
+ -	if (pmd_leaf(pmd)) {
+ -		lock = pmd_lock(mm, pmdp);
+ -		if (!unlikely(pmd_leaf(pmd))) {
+ -			spin_unlock(lock);
+ -			goto retry;
+ -		}
+ -		pfnmap_args_setup(args, lock, NULL, pmd_pgprot(pmd),
+ -				  pmd_pfn(pmd), PMD_MASK, pmd_write(pmd),
+ -				  pmd_special(pmd));
+ -		return 0;
+ -	}
+ +	pmd = pmd_offset(pud, address);
+ +	VM_BUG_ON(pmd_trans_huge(*pmd));
+  
+ -	ptep = pte_offset_map_lock(mm, pmdp, address, &lock);
+ +	ptep = pte_offset_map_lock(mm, pmd, address, ptlp);
+  	if (!ptep)
+  		goto out;
+ -	pte = ptep_get(ptep);
+ -	if (!pte_present(pte))
+ +	if (!pte_present(ptep_get(ptep)))
+  		goto unlock;
+ -	pfnmap_args_setup(args, lock, ptep, pte_pgprot(pte),
+ -			  pte_pfn(pte), PAGE_MASK, pte_write(pte),
+ -			  pte_special(pte));
+ +	*ptepp = ptep;
+  	return 0;
+  unlock:
+ -	pte_unmap_unlock(ptep, lock);
+ +	pte_unmap_unlock(ptep, *ptlp);
+  out:
+  	return -EINVAL;
+  }
+* Unmerged path include/linux/mm.h
+* Unmerged path mm/memory.c