2019-05-29 07:18:00 -07:00
|
|
|
/* SPDX-License-Identifier: GPL-2.0-only */
|
2017-07-10 18:06:09 -07:00
|
|
|
/*
|
|
|
|
|
* Copyright (C) 2009 Chen Liqin <liqin.chen@sunplusct.com>
|
|
|
|
|
* Copyright (C) 2012 Regents of the University of California
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
#ifndef _ASM_RISCV_PGALLOC_H
|
|
|
|
|
#define _ASM_RISCV_PGALLOC_H
|
|
|
|
|
|
|
|
|
|
#include <linux/mm.h>
|
riscv: Use IPIs for remote cache/TLB flushes by default
An IPI backend is always required in an SMP configuration, but an SBI
implementation is not. For example, SBI will be unavailable when the
kernel runs in M mode. For this reason, consider IPI delivery of cache
and TLB flushes to be the base case, and any other implementation (such
as the SBI remote fence extension) to be an optimization.
Generally, if IPIs can be delivered without firmware assistance, they
are assumed to be faster than SBI calls due to the SBI context switch
overhead. However, when SBI is used as the IPI backend, then the context
switch cost must be paid anyway, and performing the cache/TLB flush
directly in the SBI implementation is more efficient than injecting an
interrupt to S-mode. This is the only existing scenario where
riscv_ipi_set_virq_range() is called with use_for_rfence set to false.
sbi_ipi_init() already checks riscv_ipi_have_virq_range(), so it only
calls riscv_ipi_set_virq_range() when no other IPI device is available.
This allows moving the static key and dropping the use_for_rfence
parameter. This decouples the static key from the irqchip driver probe
order.
Furthermore, the static branch only makes sense when CONFIG_RISCV_SBI is
enabled. Optherwise, IPIs must be used. Add a fallback definition of
riscv_use_sbi_for_rfence() which handles this case and removes the need
to check CONFIG_RISCV_SBI elsewhere, such as in cacheflush.c.
Reviewed-by: Anup Patel <anup@brainfault.org>
Signed-off-by: Samuel Holland <samuel.holland@sifive.com>
Reviewed-by: Alexandre Ghiti <alexghiti@rivosinc.com>
Link: https://lore.kernel.org/r/20240327045035.368512-4-samuel.holland@sifive.com
Signed-off-by: Palmer Dabbelt <palmer@rivosinc.com>
2024-03-26 21:49:44 -07:00
|
|
|
#include <asm/sbi.h>
|
2017-07-10 18:06:09 -07:00
|
|
|
#include <asm/tlb.h>
|
|
|
|
|
|
2019-10-28 13:10:41 +01:00
|
|
|
#ifdef CONFIG_MMU
|
2021-12-06 11:46:51 +01:00
|
|
|
#define __HAVE_ARCH_PUD_FREE
|
2020-08-06 23:22:39 -07:00
|
|
|
#include <asm-generic/pgalloc.h>
|
2019-07-11 20:58:31 -07:00
|
|
|
|
2017-07-10 18:06:09 -07:00
|
|
|
static inline void pmd_populate_kernel(struct mm_struct *mm,
|
|
|
|
|
pmd_t *pmd, pte_t *pte)
|
|
|
|
|
{
|
|
|
|
|
unsigned long pfn = virt_to_pfn(pte);
|
|
|
|
|
|
|
|
|
|
set_pmd(pmd, __pmd((pfn << _PAGE_PFN_SHIFT) | _PAGE_TABLE));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline void pmd_populate(struct mm_struct *mm,
|
|
|
|
|
pmd_t *pmd, pgtable_t pte)
|
|
|
|
|
{
|
|
|
|
|
unsigned long pfn = virt_to_pfn(page_address(pte));
|
|
|
|
|
|
|
|
|
|
set_pmd(pmd, __pmd((pfn << _PAGE_PFN_SHIFT) | _PAGE_TABLE));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#ifndef __PAGETABLE_PMD_FOLDED
|
|
|
|
|
static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
|
|
|
|
|
{
|
|
|
|
|
unsigned long pfn = virt_to_pfn(pmd);
|
|
|
|
|
|
|
|
|
|
set_pud(pud, __pud((pfn << _PAGE_PFN_SHIFT) | _PAGE_TABLE));
|
|
|
|
|
}
|
2021-12-06 11:46:51 +01:00
|
|
|
|
|
|
|
|
static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4d, pud_t *pud)
|
|
|
|
|
{
|
|
|
|
|
if (pgtable_l4_enabled) {
|
|
|
|
|
unsigned long pfn = virt_to_pfn(pud);
|
|
|
|
|
|
|
|
|
|
set_p4d(p4d, __p4d((pfn << _PAGE_PFN_SHIFT) | _PAGE_TABLE));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline void p4d_populate_safe(struct mm_struct *mm, p4d_t *p4d,
|
|
|
|
|
pud_t *pud)
|
|
|
|
|
{
|
|
|
|
|
if (pgtable_l4_enabled) {
|
|
|
|
|
unsigned long pfn = virt_to_pfn(pud);
|
|
|
|
|
|
|
|
|
|
set_p4d_safe(p4d,
|
|
|
|
|
__p4d((pfn << _PAGE_PFN_SHIFT) | _PAGE_TABLE));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2022-01-27 10:48:41 +08:00
|
|
|
static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4d)
|
|
|
|
|
{
|
|
|
|
|
if (pgtable_l5_enabled) {
|
|
|
|
|
unsigned long pfn = virt_to_pfn(p4d);
|
|
|
|
|
|
|
|
|
|
set_pgd(pgd, __pgd((pfn << _PAGE_PFN_SHIFT) | _PAGE_TABLE));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline void pgd_populate_safe(struct mm_struct *mm, pgd_t *pgd,
|
|
|
|
|
p4d_t *p4d)
|
|
|
|
|
{
|
|
|
|
|
if (pgtable_l5_enabled) {
|
|
|
|
|
unsigned long pfn = virt_to_pfn(p4d);
|
|
|
|
|
|
|
|
|
|
set_pgd_safe(pgd,
|
|
|
|
|
__pgd((pfn << _PAGE_PFN_SHIFT) | _PAGE_TABLE));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2021-12-06 11:46:51 +01:00
|
|
|
#define pud_free pud_free
|
|
|
|
|
static inline void pud_free(struct mm_struct *mm, pud_t *pud)
|
|
|
|
|
{
|
|
|
|
|
if (pgtable_l4_enabled)
|
|
|
|
|
__pud_free(mm, pud);
|
|
|
|
|
}
|
|
|
|
|
|
2023-12-20 01:50:44 +08:00
|
|
|
static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pud,
|
|
|
|
|
unsigned long addr)
|
|
|
|
|
{
|
2025-01-08 14:57:26 +08:00
|
|
|
if (pgtable_l4_enabled)
|
riscv: pgtable: unconditionally use tlb_remove_ptdesc()
To support fast gup, the commit 69be3fb111e7 ("riscv: enable
MMU_GATHER_RCU_TABLE_FREE for SMP && MMU") did the following:
1) use tlb_remove_page_ptdesc() for those platforms which use IPI to
perform TLB shootdown
2) use tlb_remove_ptdesc() for those platforms which use SBI to perform
TLB shootdown
The tlb_remove_page_ptdesc() is the wrapper of the tlb_remove_page(). By
design, the tlb_remove_page() should be used to remove a normal page from
a page table entry, and should not be used for page table pages.
The tlb_remove_ptdesc() is the wrapper of the tlb_remove_table(), which is
designed specifically for freeing page table pages. If the
CONFIG_MMU_GATHER_TABLE_FREE is enabled, the tlb_remove_table() will use
semi RCU to free page table pages, that is:
- batch table freeing: asynchronous free by RCU
- single table freeing: IPI + synchronous free
If the CONFIG_MMU_GATHER_TABLE_FREE is disabled, the tlb_remove_table()
will fall back to pagetable_dtor() + tlb_remove_page().
For case 1), since we need to perform TLB shootdown before freeing the
page table page, the local_irq_save() in fast gup can block the freeing
and protect the fast gup page walker. Therefore we can ensure safety by
just using tlb_remove_page_ptdesc(). In addition, we can also the
tlb_remove_ptdesc()/tlb_remove_table() to achieve it, and it doesn't
matter whether CONFIG_MMU_GATHER_RCU_TABLE_FREE is selected. And in
theory, the performance of freeing pages asynchronously via RCU will not
be lower than synchronous free.
For case 2), since local_irq_save() only disable S-privilege IPI irq but
not M-privilege's, which is used by the SBI implementation to perform TLB
shootdown, so we must select CONFIG_MMU_GATHER_RCU_TABLE_FREE and use
tlb_remove_ptdesc() to ensure safety. The riscv selects this config for
SMP && MMU, the CONFIG_RISCV_SBI is dependent on MMU. Therefore, only the
UP system may have the situation where CONFIG_MMU_GATHER_RCU_TABLE_FREE is
disabled but CONFIG_RISCV_SBI is enabled. But there is no freeing vs fast
gup race in the UP system.
So, in summary, we can use tlb_remove_ptdesc() to support fast gup in all
cases, and this interface is specifically designed for page table pages.
So let's use it unconditionally.
Link: https://lkml.kernel.org/r/9025595e895515515c95e48db54b29afa489c41d.1740454179.git.zhengqi.arch@bytedance.com
Signed-off-by: Qi Zheng <zhengqi.arch@bytedance.com>
Suggested-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Alexandre Ghiti <alexghiti@rivosinc.com>
Cc: "Aneesh Kumar K.V" <aneesh.kumar@kernel.org>
Cc: Arnd Bergmann <arnd@arndb.de>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: David Hildenbrand <david@redhat.com>
Cc: Hugh Dickens <hughd@google.com>
Cc: Jann Horn <jannh@google.com>
Cc: Kevin Brodsky <kevin.brodsky@arm.com>
Cc: Matthew Wilcow (Oracle) <willy@infradead.org>
Cc: "Mike Rapoport (IBM)" <rppt@kernel.org>
Cc: Muchun Song <muchun.song@linux.dev>
Cc: Nicholas Piggin <npiggin@gmail.com>
Cc: Rik van Riel <riel@surriel.com>
Cc: Vishal Moola (Oracle) <vishal.moola@gmail.com>
Cc: Will Deacon <will@kernel.org>
Cc: Yu Zhao <yuzhao@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
2025-02-25 11:45:54 +08:00
|
|
|
tlb_remove_ptdesc(tlb, virt_to_ptdesc(pud));
|
2023-12-20 01:50:44 +08:00
|
|
|
}
|
2022-01-27 10:48:41 +08:00
|
|
|
|
2023-12-20 01:50:44 +08:00
|
|
|
static inline void __p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d,
|
|
|
|
|
unsigned long addr)
|
|
|
|
|
{
|
2025-01-08 14:57:26 +08:00
|
|
|
if (pgtable_l5_enabled)
|
riscv: pgtable: unconditionally use tlb_remove_ptdesc()
To support fast gup, the commit 69be3fb111e7 ("riscv: enable
MMU_GATHER_RCU_TABLE_FREE for SMP && MMU") did the following:
1) use tlb_remove_page_ptdesc() for those platforms which use IPI to
perform TLB shootdown
2) use tlb_remove_ptdesc() for those platforms which use SBI to perform
TLB shootdown
The tlb_remove_page_ptdesc() is the wrapper of the tlb_remove_page(). By
design, the tlb_remove_page() should be used to remove a normal page from
a page table entry, and should not be used for page table pages.
The tlb_remove_ptdesc() is the wrapper of the tlb_remove_table(), which is
designed specifically for freeing page table pages. If the
CONFIG_MMU_GATHER_TABLE_FREE is enabled, the tlb_remove_table() will use
semi RCU to free page table pages, that is:
- batch table freeing: asynchronous free by RCU
- single table freeing: IPI + synchronous free
If the CONFIG_MMU_GATHER_TABLE_FREE is disabled, the tlb_remove_table()
will fall back to pagetable_dtor() + tlb_remove_page().
For case 1), since we need to perform TLB shootdown before freeing the
page table page, the local_irq_save() in fast gup can block the freeing
and protect the fast gup page walker. Therefore we can ensure safety by
just using tlb_remove_page_ptdesc(). In addition, we can also the
tlb_remove_ptdesc()/tlb_remove_table() to achieve it, and it doesn't
matter whether CONFIG_MMU_GATHER_RCU_TABLE_FREE is selected. And in
theory, the performance of freeing pages asynchronously via RCU will not
be lower than synchronous free.
For case 2), since local_irq_save() only disable S-privilege IPI irq but
not M-privilege's, which is used by the SBI implementation to perform TLB
shootdown, so we must select CONFIG_MMU_GATHER_RCU_TABLE_FREE and use
tlb_remove_ptdesc() to ensure safety. The riscv selects this config for
SMP && MMU, the CONFIG_RISCV_SBI is dependent on MMU. Therefore, only the
UP system may have the situation where CONFIG_MMU_GATHER_RCU_TABLE_FREE is
disabled but CONFIG_RISCV_SBI is enabled. But there is no freeing vs fast
gup race in the UP system.
So, in summary, we can use tlb_remove_ptdesc() to support fast gup in all
cases, and this interface is specifically designed for page table pages.
So let's use it unconditionally.
Link: https://lkml.kernel.org/r/9025595e895515515c95e48db54b29afa489c41d.1740454179.git.zhengqi.arch@bytedance.com
Signed-off-by: Qi Zheng <zhengqi.arch@bytedance.com>
Suggested-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Alexandre Ghiti <alexghiti@rivosinc.com>
Cc: "Aneesh Kumar K.V" <aneesh.kumar@kernel.org>
Cc: Arnd Bergmann <arnd@arndb.de>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: David Hildenbrand <david@redhat.com>
Cc: Hugh Dickens <hughd@google.com>
Cc: Jann Horn <jannh@google.com>
Cc: Kevin Brodsky <kevin.brodsky@arm.com>
Cc: Matthew Wilcow (Oracle) <willy@infradead.org>
Cc: "Mike Rapoport (IBM)" <rppt@kernel.org>
Cc: Muchun Song <muchun.song@linux.dev>
Cc: Nicholas Piggin <npiggin@gmail.com>
Cc: Rik van Riel <riel@surriel.com>
Cc: Vishal Moola (Oracle) <vishal.moola@gmail.com>
Cc: Will Deacon <will@kernel.org>
Cc: Yu Zhao <yuzhao@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
2025-02-25 11:45:54 +08:00
|
|
|
tlb_remove_ptdesc(tlb, virt_to_ptdesc(p4d));
|
2023-12-20 01:50:44 +08:00
|
|
|
}
|
2017-07-10 18:06:09 -07:00
|
|
|
#endif /* __PAGETABLE_PMD_FOLDED */
|
|
|
|
|
|
2022-11-21 14:33:03 +01:00
|
|
|
static inline void sync_kernel_mappings(pgd_t *pgd)
|
|
|
|
|
{
|
|
|
|
|
memcpy(pgd + USER_PTRS_PER_PGD,
|
|
|
|
|
init_mm.pgd + USER_PTRS_PER_PGD,
|
|
|
|
|
(PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t));
|
|
|
|
|
}
|
|
|
|
|
|
2017-07-10 18:06:09 -07:00
|
|
|
static inline pgd_t *pgd_alloc(struct mm_struct *mm)
|
|
|
|
|
{
|
|
|
|
|
pgd_t *pgd;
|
|
|
|
|
|
asm-generic: pgalloc: provide generic __pgd_{alloc,free}
We already have a generic implementation of alloc/free up to P4D level, as
well as pgd_free(). Let's finish the work and add a generic PGD-level
alloc helper as well.
Unlike at lower levels, almost all architectures need some specific magic
at PGD level (typically initialising PGD entries), so introducing a
generic pgd_alloc() isn't worth it. Instead we introduce two new helpers,
__pgd_alloc() and __pgd_free(), and make use of them in the arch-specific
pgd_alloc() and pgd_free() wherever possible. To accommodate as many arch
as possible, __pgd_alloc() takes a page allocation order.
Because pagetable_alloc() allocates zeroed pages, explicit zeroing in
pgd_alloc() becomes redundant and we can get rid of it. Some trivial
implementations of pgd_free() also become unnecessary once __pgd_alloc()
is used; remove them.
Another small improvement is consistent accounting of PGD pages by using
GFP_PGTABLE_{USER,KERNEL} as appropriate.
Not all PGD allocations can be handled by the generic helpers. In
particular, multiple architectures allocate PGDs from a kmem_cache, and
those PGDs may not be page-sized.
Link: https://lkml.kernel.org/r/20250103184415.2744423-6-kevin.brodsky@arm.com
Signed-off-by: Kevin Brodsky <kevin.brodsky@arm.com>
Acked-by: Dave Hansen <dave.hansen@linux.intel.com>
Acked-by: Qi Zheng <zhengqi.arch@bytedance.com>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Linus Walleij <linus.walleij@linaro.org>
Cc: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: Mike Rapoport (Microsoft) <rppt@kernel.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Ryan Roberts <ryan.roberts@arm.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Will Deacon <will@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
2025-01-03 18:44:14 +00:00
|
|
|
pgd = __pgd_alloc(mm, 0);
|
2017-07-10 18:06:09 -07:00
|
|
|
if (likely(pgd != NULL)) {
|
|
|
|
|
/* Copy kernel mappings */
|
2022-11-21 14:33:03 +01:00
|
|
|
sync_kernel_mappings(pgd);
|
2017-07-10 18:06:09 -07:00
|
|
|
}
|
|
|
|
|
return pgd;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#ifndef __PAGETABLE_PMD_FOLDED
|
|
|
|
|
|
2023-12-20 01:50:44 +08:00
|
|
|
static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd,
|
|
|
|
|
unsigned long addr)
|
|
|
|
|
{
|
riscv: pgtable: unconditionally use tlb_remove_ptdesc()
To support fast gup, the commit 69be3fb111e7 ("riscv: enable
MMU_GATHER_RCU_TABLE_FREE for SMP && MMU") did the following:
1) use tlb_remove_page_ptdesc() for those platforms which use IPI to
perform TLB shootdown
2) use tlb_remove_ptdesc() for those platforms which use SBI to perform
TLB shootdown
The tlb_remove_page_ptdesc() is the wrapper of the tlb_remove_page(). By
design, the tlb_remove_page() should be used to remove a normal page from
a page table entry, and should not be used for page table pages.
The tlb_remove_ptdesc() is the wrapper of the tlb_remove_table(), which is
designed specifically for freeing page table pages. If the
CONFIG_MMU_GATHER_TABLE_FREE is enabled, the tlb_remove_table() will use
semi RCU to free page table pages, that is:
- batch table freeing: asynchronous free by RCU
- single table freeing: IPI + synchronous free
If the CONFIG_MMU_GATHER_TABLE_FREE is disabled, the tlb_remove_table()
will fall back to pagetable_dtor() + tlb_remove_page().
For case 1), since we need to perform TLB shootdown before freeing the
page table page, the local_irq_save() in fast gup can block the freeing
and protect the fast gup page walker. Therefore we can ensure safety by
just using tlb_remove_page_ptdesc(). In addition, we can also the
tlb_remove_ptdesc()/tlb_remove_table() to achieve it, and it doesn't
matter whether CONFIG_MMU_GATHER_RCU_TABLE_FREE is selected. And in
theory, the performance of freeing pages asynchronously via RCU will not
be lower than synchronous free.
For case 2), since local_irq_save() only disable S-privilege IPI irq but
not M-privilege's, which is used by the SBI implementation to perform TLB
shootdown, so we must select CONFIG_MMU_GATHER_RCU_TABLE_FREE and use
tlb_remove_ptdesc() to ensure safety. The riscv selects this config for
SMP && MMU, the CONFIG_RISCV_SBI is dependent on MMU. Therefore, only the
UP system may have the situation where CONFIG_MMU_GATHER_RCU_TABLE_FREE is
disabled but CONFIG_RISCV_SBI is enabled. But there is no freeing vs fast
gup race in the UP system.
So, in summary, we can use tlb_remove_ptdesc() to support fast gup in all
cases, and this interface is specifically designed for page table pages.
So let's use it unconditionally.
Link: https://lkml.kernel.org/r/9025595e895515515c95e48db54b29afa489c41d.1740454179.git.zhengqi.arch@bytedance.com
Signed-off-by: Qi Zheng <zhengqi.arch@bytedance.com>
Suggested-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Alexandre Ghiti <alexghiti@rivosinc.com>
Cc: "Aneesh Kumar K.V" <aneesh.kumar@kernel.org>
Cc: Arnd Bergmann <arnd@arndb.de>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: David Hildenbrand <david@redhat.com>
Cc: Hugh Dickens <hughd@google.com>
Cc: Jann Horn <jannh@google.com>
Cc: Kevin Brodsky <kevin.brodsky@arm.com>
Cc: Matthew Wilcow (Oracle) <willy@infradead.org>
Cc: "Mike Rapoport (IBM)" <rppt@kernel.org>
Cc: Muchun Song <muchun.song@linux.dev>
Cc: Nicholas Piggin <npiggin@gmail.com>
Cc: Rik van Riel <riel@surriel.com>
Cc: Vishal Moola (Oracle) <vishal.moola@gmail.com>
Cc: Will Deacon <will@kernel.org>
Cc: Yu Zhao <yuzhao@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
2025-02-25 11:45:54 +08:00
|
|
|
tlb_remove_ptdesc(tlb, virt_to_ptdesc(pmd));
|
2023-12-20 01:50:44 +08:00
|
|
|
}
|
2017-07-10 18:06:09 -07:00
|
|
|
|
|
|
|
|
#endif /* __PAGETABLE_PMD_FOLDED */
|
|
|
|
|
|
2023-12-20 01:50:44 +08:00
|
|
|
static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte,
|
|
|
|
|
unsigned long addr)
|
|
|
|
|
{
|
riscv: pgtable: unconditionally use tlb_remove_ptdesc()
To support fast gup, the commit 69be3fb111e7 ("riscv: enable
MMU_GATHER_RCU_TABLE_FREE for SMP && MMU") did the following:
1) use tlb_remove_page_ptdesc() for those platforms which use IPI to
perform TLB shootdown
2) use tlb_remove_ptdesc() for those platforms which use SBI to perform
TLB shootdown
The tlb_remove_page_ptdesc() is the wrapper of the tlb_remove_page(). By
design, the tlb_remove_page() should be used to remove a normal page from
a page table entry, and should not be used for page table pages.
The tlb_remove_ptdesc() is the wrapper of the tlb_remove_table(), which is
designed specifically for freeing page table pages. If the
CONFIG_MMU_GATHER_TABLE_FREE is enabled, the tlb_remove_table() will use
semi RCU to free page table pages, that is:
- batch table freeing: asynchronous free by RCU
- single table freeing: IPI + synchronous free
If the CONFIG_MMU_GATHER_TABLE_FREE is disabled, the tlb_remove_table()
will fall back to pagetable_dtor() + tlb_remove_page().
For case 1), since we need to perform TLB shootdown before freeing the
page table page, the local_irq_save() in fast gup can block the freeing
and protect the fast gup page walker. Therefore we can ensure safety by
just using tlb_remove_page_ptdesc(). In addition, we can also the
tlb_remove_ptdesc()/tlb_remove_table() to achieve it, and it doesn't
matter whether CONFIG_MMU_GATHER_RCU_TABLE_FREE is selected. And in
theory, the performance of freeing pages asynchronously via RCU will not
be lower than synchronous free.
For case 2), since local_irq_save() only disable S-privilege IPI irq but
not M-privilege's, which is used by the SBI implementation to perform TLB
shootdown, so we must select CONFIG_MMU_GATHER_RCU_TABLE_FREE and use
tlb_remove_ptdesc() to ensure safety. The riscv selects this config for
SMP && MMU, the CONFIG_RISCV_SBI is dependent on MMU. Therefore, only the
UP system may have the situation where CONFIG_MMU_GATHER_RCU_TABLE_FREE is
disabled but CONFIG_RISCV_SBI is enabled. But there is no freeing vs fast
gup race in the UP system.
So, in summary, we can use tlb_remove_ptdesc() to support fast gup in all
cases, and this interface is specifically designed for page table pages.
So let's use it unconditionally.
Link: https://lkml.kernel.org/r/9025595e895515515c95e48db54b29afa489c41d.1740454179.git.zhengqi.arch@bytedance.com
Signed-off-by: Qi Zheng <zhengqi.arch@bytedance.com>
Suggested-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Alexandre Ghiti <alexghiti@rivosinc.com>
Cc: "Aneesh Kumar K.V" <aneesh.kumar@kernel.org>
Cc: Arnd Bergmann <arnd@arndb.de>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: David Hildenbrand <david@redhat.com>
Cc: Hugh Dickens <hughd@google.com>
Cc: Jann Horn <jannh@google.com>
Cc: Kevin Brodsky <kevin.brodsky@arm.com>
Cc: Matthew Wilcow (Oracle) <willy@infradead.org>
Cc: "Mike Rapoport (IBM)" <rppt@kernel.org>
Cc: Muchun Song <muchun.song@linux.dev>
Cc: Nicholas Piggin <npiggin@gmail.com>
Cc: Rik van Riel <riel@surriel.com>
Cc: Vishal Moola (Oracle) <vishal.moola@gmail.com>
Cc: Will Deacon <will@kernel.org>
Cc: Yu Zhao <yuzhao@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
2025-02-25 11:45:54 +08:00
|
|
|
tlb_remove_ptdesc(tlb, page_ptdesc(pte));
|
2023-12-20 01:50:44 +08:00
|
|
|
}
|
2019-10-28 13:10:41 +01:00
|
|
|
#endif /* CONFIG_MMU */
|
2017-07-10 18:06:09 -07:00
|
|
|
|
|
|
|
|
#endif /* _ASM_RISCV_PGALLOC_H */
|