mirror of
git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
synced 2025-08-04 00:06:36 +00:00
6bbd42e2df
The invalidate_range() is going to become an architecture specific mmu notifier used to keep the TLB of secondary MMUs such as an IOMMU in sync with the CPU page tables. Currently it is called from separate code paths to the main CPU TLB invalidations. This can lead to a secondary TLB not getting invalidated when required and makes it hard to reason about when exactly the secondary TLB is invalidated. To fix this move the notifier call to the architecture specific TLB maintenance functions for architectures that have secondary MMUs requiring explicit software invalidations. This fixes a SMMU bug on ARM64. On ARM64 PTE permission upgrades require a TLB invalidation. This invalidation is done by the architecture specific ptep_set_access_flags() which calls flush_tlb_page() if required. However this doesn't call the notifier resulting in infinite faults being generated by devices using the SMMU if it has previously cached a read-only PTE in it's TLB. Moving the invalidations into the TLB invalidation functions ensures all invalidations happen at the same time as the CPU invalidation. The architecture specific flush_tlb_all() routines do not call the notifier as none of the IOMMUs require this. Link: https://lkml.kernel.org/r/0287ae32d91393a582897d6c4db6f7456b1001f2.1690292440.git-series.apopple@nvidia.com Signed-off-by: Alistair Popple <apopple@nvidia.com> Suggested-by: Jason Gunthorpe <jgg@ziepe.ca> Tested-by: SeongJae Park <sj@kernel.org> Acked-by: Catalin Marinas <catalin.marinas@arm.com> Reviewed-by: Jason Gunthorpe <jgg@nvidia.com> Tested-by: Luis Chamberlain <mcgrof@kernel.org> Cc: Andrew Donnellan <ajd@linux.ibm.com> Cc: Chaitanya Kumar Borah <chaitanya.kumar.borah@intel.com> Cc: Frederic Barrat <fbarrat@linux.ibm.com> Cc: John Hubbard <jhubbard@nvidia.com> Cc: Kevin Tian <kevin.tian@intel.com> Cc: Michael Ellerman <mpe@ellerman.id.au> Cc: Nicholas Piggin <npiggin@gmail.com> Cc: Nicolin Chen <nicolinc@nvidia.com> Cc: Robin Murphy <robin.murphy@arm.com> Cc: Sean Christopherson <seanjc@google.com> Cc: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com> Cc: Will Deacon <will@kernel.org> Cc: Zhi Wang <zhi.wang.linux@gmail.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
62 lines
1.8 KiB
C
62 lines
1.8 KiB
C
// SPDX-License-Identifier: GPL-2.0
|
|
#include <linux/mm.h>
|
|
#include <linux/hugetlb.h>
|
|
#include <linux/security.h>
|
|
#include <asm/cacheflush.h>
|
|
#include <asm/machdep.h>
|
|
#include <asm/mman.h>
|
|
#include <asm/tlb.h>
|
|
|
|
void radix__flush_hugetlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
|
|
{
|
|
int psize;
|
|
struct hstate *hstate = hstate_file(vma->vm_file);
|
|
|
|
psize = hstate_get_psize(hstate);
|
|
radix__flush_tlb_page_psize(vma->vm_mm, vmaddr, psize);
|
|
}
|
|
|
|
void radix__local_flush_hugetlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
|
|
{
|
|
int psize;
|
|
struct hstate *hstate = hstate_file(vma->vm_file);
|
|
|
|
psize = hstate_get_psize(hstate);
|
|
radix__local_flush_tlb_page_psize(vma->vm_mm, vmaddr, psize);
|
|
}
|
|
|
|
void radix__flush_hugetlb_tlb_range(struct vm_area_struct *vma, unsigned long start,
|
|
unsigned long end)
|
|
{
|
|
int psize;
|
|
struct hstate *hstate = hstate_file(vma->vm_file);
|
|
|
|
psize = hstate_get_psize(hstate);
|
|
/*
|
|
* Flush PWC even if we get PUD_SIZE hugetlb invalidate to keep this simpler.
|
|
*/
|
|
if (end - start >= PUD_SIZE)
|
|
radix__flush_tlb_pwc_range_psize(vma->vm_mm, start, end, psize);
|
|
else
|
|
radix__flush_tlb_range_psize(vma->vm_mm, start, end, psize);
|
|
mmu_notifier_invalidate_range(vma->vm_mm, start, end);
|
|
}
|
|
|
|
void radix__huge_ptep_modify_prot_commit(struct vm_area_struct *vma,
|
|
unsigned long addr, pte_t *ptep,
|
|
pte_t old_pte, pte_t pte)
|
|
{
|
|
struct mm_struct *mm = vma->vm_mm;
|
|
|
|
/*
|
|
* POWER9 NMMU must flush the TLB after clearing the PTE before
|
|
* installing a PTE with more relaxed access permissions, see
|
|
* radix__ptep_set_access_flags.
|
|
*/
|
|
if (!cpu_has_feature(CPU_FTR_ARCH_31) &&
|
|
is_pte_rw_upgrade(pte_val(old_pte), pte_val(pte)) &&
|
|
atomic_read(&mm->context.copros) > 0)
|
|
radix__flush_hugetlb_page(vma, addr);
|
|
|
|
set_huge_pte_at(vma->vm_mm, addr, ptep, pte);
|
|
}
|