mirror of
git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
synced 2025-09-18 22:14:16 +00:00
fa48e6f780
The tlb maintainence functions: __cpu_flush_user_tlb_range and __cpu_flush_kern_tlb_range do not take into consideration the page granule when looping through the address range, and repeatedly flush tlb entries for the same page when operating with 64K pages. This patch re-works the logic s.t. we instead advance the loop by 1 << (PAGE_SHIFT - 12), so avoid repeating ourselves. Also the routines have been converted from assembler to static inline functions to aid with legibility and potential compiler optimisations. The isb() has been removed from flush_tlb_kernel_range(.) as it is only needed when changing the execute permission of a mapping. If one needs to set an area of the kernel as execute/non-execute an isb() must be inserted after the call to flush_tlb_kernel_range. Cc: Laura Abbott <lauraa@codeaurora.org> Signed-off-by: Steve Capper <steve.capper@linaro.org> Acked-by: Will Deacon <will.deacon@arm.com> Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
144 lines
3.7 KiB
C
144 lines
3.7 KiB
C
/*
|
|
* Based on arch/arm/include/asm/tlbflush.h
|
|
*
|
|
* Copyright (C) 1999-2003 Russell King
|
|
* Copyright (C) 2012 ARM Ltd.
|
|
*
|
|
* This program is free software; you can redistribute it and/or modify
|
|
* it under the terms of the GNU General Public License version 2 as
|
|
* published by the Free Software Foundation.
|
|
*
|
|
* This program is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU General Public License
|
|
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
*/
|
|
#ifndef __ASM_TLBFLUSH_H
|
|
#define __ASM_TLBFLUSH_H
|
|
|
|
#ifndef __ASSEMBLY__
|
|
|
|
#include <linux/sched.h>
|
|
#include <asm/cputype.h>
|
|
|
|
extern void __cpu_flush_user_tlb_range(unsigned long, unsigned long, struct vm_area_struct *);
|
|
extern void __cpu_flush_kern_tlb_range(unsigned long, unsigned long);
|
|
|
|
extern struct cpu_tlb_fns cpu_tlb;
|
|
|
|
/*
|
|
* TLB Management
|
|
* ==============
|
|
*
|
|
* The arch/arm64/mm/tlb.S files implement these methods.
|
|
*
|
|
* The TLB specific code is expected to perform whatever tests it needs
|
|
* to determine if it should invalidate the TLB for each call. Start
|
|
* addresses are inclusive and end addresses are exclusive; it is safe to
|
|
* round these addresses down.
|
|
*
|
|
* flush_tlb_all()
|
|
*
|
|
* Invalidate the entire TLB.
|
|
*
|
|
* flush_tlb_mm(mm)
|
|
*
|
|
* Invalidate all TLB entries in a particular address space.
|
|
* - mm - mm_struct describing address space
|
|
*
|
|
* flush_tlb_range(mm,start,end)
|
|
*
|
|
* Invalidate a range of TLB entries in the specified address
|
|
* space.
|
|
* - mm - mm_struct describing address space
|
|
* - start - start address (may not be aligned)
|
|
* - end - end address (exclusive, may not be aligned)
|
|
*
|
|
* flush_tlb_page(vaddr,vma)
|
|
*
|
|
* Invalidate the specified page in the specified address range.
|
|
* - vaddr - virtual address (may not be aligned)
|
|
* - vma - vma_struct describing address range
|
|
*
|
|
* flush_kern_tlb_page(kaddr)
|
|
*
|
|
* Invalidate the TLB entry for the specified page. The address
|
|
* will be in the kernels virtual memory space. Current uses
|
|
* only require the D-TLB to be invalidated.
|
|
* - kaddr - Kernel virtual memory address
|
|
*/
|
|
static inline void flush_tlb_all(void)
|
|
{
|
|
dsb();
|
|
asm("tlbi vmalle1is");
|
|
dsb();
|
|
isb();
|
|
}
|
|
|
|
static inline void flush_tlb_mm(struct mm_struct *mm)
|
|
{
|
|
unsigned long asid = (unsigned long)ASID(mm) << 48;
|
|
|
|
dsb();
|
|
asm("tlbi aside1is, %0" : : "r" (asid));
|
|
dsb();
|
|
}
|
|
|
|
static inline void flush_tlb_page(struct vm_area_struct *vma,
|
|
unsigned long uaddr)
|
|
{
|
|
unsigned long addr = uaddr >> 12 |
|
|
((unsigned long)ASID(vma->vm_mm) << 48);
|
|
|
|
dsb();
|
|
asm("tlbi vae1is, %0" : : "r" (addr));
|
|
dsb();
|
|
}
|
|
|
|
static inline void flush_tlb_range(struct vm_area_struct *vma,
|
|
unsigned long start, unsigned long end)
|
|
{
|
|
unsigned long asid = (unsigned long)ASID(vma->vm_mm) << 48;
|
|
unsigned long addr;
|
|
start = asid | (start >> 12);
|
|
end = asid | (end >> 12);
|
|
|
|
dsb(ishst);
|
|
for (addr = start; addr < end; addr += 1 << (PAGE_SHIFT - 12))
|
|
asm("tlbi vae1is, %0" : : "r"(addr));
|
|
dsb(ish);
|
|
}
|
|
|
|
static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end)
|
|
{
|
|
unsigned long addr;
|
|
start >>= 12;
|
|
end >>= 12;
|
|
|
|
dsb(ishst);
|
|
for (addr = start; addr < end; addr += 1 << (PAGE_SHIFT - 12))
|
|
asm("tlbi vaae1is, %0" : : "r"(addr));
|
|
dsb(ish);
|
|
}
|
|
|
|
/*
|
|
* On AArch64, the cache coherency is handled via the set_pte_at() function.
|
|
*/
|
|
static inline void update_mmu_cache(struct vm_area_struct *vma,
|
|
unsigned long addr, pte_t *ptep)
|
|
{
|
|
/*
|
|
* set_pte() does not have a DSB, so make sure that the page table
|
|
* write is visible.
|
|
*/
|
|
dsb();
|
|
}
|
|
|
|
#define update_mmu_cache_pmd(vma, address, pmd) do { } while (0)
|
|
|
|
#endif
|
|
|
|
#endif
|