Kernel/linux
1
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2025-01-08 14:23:19 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

riscv: add ASID-based tlbflushing methods

Browse Source 
Implement optimized version of the tlb flushing routines for systems
using ASIDs. These are behind the use_asid_allocator static branch to
not affect existing systems not using ASIDs.

Signed-off-by: Guo Ren <guoren@linux.alibaba.com>
[hch: rebased on top of previous cleanups, use the same algorithm as
      the non-ASID based code for local vs global flushes, keep functions
      as local as possible]
Signed-off-by: Christoph Hellwig <hch@lst.de>
Tested-by: Guo Ren <guoren@kernel.org>
Signed-off-by: Palmer Dabbelt <palmerdabbelt@google.com>
This commit is contained in:
Guo Ren 2021-06-06 17:20:50 +02:00 committed by Palmer Dabbelt
parent 70c7605c08
commit 3f1e782998
No known key found for this signature in database
GPG Key ID: 2E1319F35FBB1889
3 changed files with 44 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
arch/riscv/include/asm/mmu_context.h
View File

@ -33,6 +33,8 @@ static inline int init_new_context(struct task_struct *tsk,
return 0;
}
DECLARE_STATIC_KEY_FALSE(use_asid_allocator);
#include <asm-generic/mmu_context.h>
#endif /* _ASM_RISCV_MMU_CONTEXT_H */

2
arch/riscv/mm/context.c
View File

@ -18,7 +18,7 @@
#ifdef CONFIG_MMU
static DEFINE_STATIC_KEY_FALSE(use_asid_allocator);
DEFINE_STATIC_KEY_FALSE(use_asid_allocator);
static unsigned long asid_bits;
static unsigned long num_asids;

49
arch/riscv/mm/tlbflush.c
View File

@ -4,6 +4,24 @@
#include <linux/smp.h>
#include <linux/sched.h>
#include <asm/sbi.h>
#include <asm/mmu_context.h>
static inline void local_flush_tlb_all_asid(unsigned long asid)
{
__asm__ __volatile__ ("sfence.vma x0, %0"
:
: "r" (asid)
: "memory");
}
static inline void local_flush_tlb_page_asid(unsigned long addr,
unsigned long asid)
{
__asm__ __volatile__ ("sfence.vma %0, %1"
:
: "r" (addr), "r" (asid)
: "memory");
}
void flush_tlb_all(void)
{
@ -16,21 +34,36 @@ static void __sbi_tlb_flush_range(struct mm_struct *mm, unsigned long start,
struct cpumask *cmask = mm_cpumask(mm);
struct cpumask hmask;
unsigned int cpuid;
bool broadcast;
if (cpumask_empty(cmask))
return;
cpuid = get_cpu();
/* check if the tlbflush needs to be sent to other CPUs */
broadcast = cpumask_any_but(cmask, cpuid) < nr_cpu_ids;
if (static_branch_unlikely(&use_asid_allocator)) {
unsigned long asid = atomic_long_read(&mm->context.id);
if (cpumask_any_but(cmask, cpuid) >= nr_cpu_ids) {
/* local cpu is the only cpu present in cpumask */
if (size <= stride)
local_flush_tlb_page(start);
else
local_flush_tlb_all();
if (broadcast) {
riscv_cpuid_to_hartid_mask(cmask, &hmask);
sbi_remote_sfence_vma_asid(cpumask_bits(&hmask),
start, size, asid);
} else if (size <= stride) {
local_flush_tlb_page_asid(start, asid);
} else {
local_flush_tlb_all_asid(asid);
}
} else {
riscv_cpuid_to_hartid_mask(cmask, &hmask);
sbi_remote_sfence_vma(cpumask_bits(&hmask), start, size);
if (broadcast) {
riscv_cpuid_to_hartid_mask(cmask, &hmask);
sbi_remote_sfence_vma(cpumask_bits(&hmask),
start, size);
} else if (size <= stride) {
local_flush_tlb_page(start);
} else {
local_flush_tlb_all();
}
}
put_cpu();