Kernel/linux-stable
1
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git synced 2024-12-29 09:16:33 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

srcu: Allow inlining of __srcu_read_{,un}lock_lite()

Browse Source 
This commit moves __srcu_read_lock_lite() and __srcu_read_unlock_lite()
into include/linux/srcu.h and marks them "static inline" so that they
can be inlined into srcu_read_lock_lite() and srcu_read_unlock_lite(),
respectively.  They are not hand-inlined due to Tree SRCU and Tiny SRCU
having different implementations.

The earlier removal of smp_mb() combined with the inlining produce
significant single-percentage performance wins.

Link: https://lore.kernel.org/all/CAEf4BzYgiNmSb=ZKQ65tm6nJDi1UX2Gq26cdHSH1mPwXJYZj5g@mail.gmail.com/

Reported-by: Alexei Starovoitov <ast@kernel.org>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Cc: Alexei Starovoitov <ast@kernel.org>
Cc: Andrii Nakryiko <andrii@kernel.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Kent Overstreet <kent.overstreet@linux.dev>
Cc: <bpf@vger.kernel.org>
Reviewed-by: Neeraj Upadhyay <Neeraj.Upadhyay@amd.com>
Signed-off-by: Frederic Weisbecker <frederic@kernel.org>
This commit is contained in:
Paul E. McKenney 2024-10-15 09:11:06 -07:00 committed by Frederic Weisbecker
parent 6364dd8191
commit bb94b12e45
2 changed files with 39 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
include/linux/srcutree.h
View File

@ -209,4 +209,43 @@ void synchronize_srcu_expedited(struct srcu_struct *ssp);
void srcu_barrier(struct srcu_struct *ssp);
void srcu_torture_stats_print(struct srcu_struct *ssp, char *tt, char *tf);
/*
* Counts the new reader in the appropriate per-CPU element of the
* srcu_struct. Returns an index that must be passed to the matching
* srcu_read_unlock_lite().
*
* Note that this_cpu_inc() is an RCU read-side critical section either
* because it disables interrupts, because it is a single instruction,
* or because it is a read-modify-write atomic operation, depending on
* the whims of the architecture.
*/
static inline int __srcu_read_lock_lite(struct srcu_struct *ssp)
{
int idx;
RCU_LOCKDEP_WARN(!rcu_is_watching(), "RCU must be watching srcu_read_lock_lite().");
idx = READ_ONCE(ssp->srcu_idx) & 0x1;
this_cpu_inc(ssp->sda->srcu_lock_count[idx].counter); /* Y */
barrier(); /* Avoid leaking the critical section. */
return idx;
}
/*
* Removes the count for the old reader from the appropriate
* per-CPU element of the srcu_struct. Note that this may well be a
* different CPU than that which was incremented by the corresponding
* srcu_read_lock_lite(), but it must be within the same task.
*
* Note that this_cpu_inc() is an RCU read-side critical section either
* because it disables interrupts, because it is a single instruction,
* or because it is a read-modify-write atomic operation, depending on
* the whims of the architecture.
*/
static inline void __srcu_read_unlock_lite(struct srcu_struct *ssp, int idx)
{
barrier(); /* Avoid leaking the critical section. */
this_cpu_inc(ssp->sda->srcu_unlock_count[idx].counter); /* Z */
RCU_LOCKDEP_WARN(!rcu_is_watching(), "RCU must be watching srcu_read_unlock_lite().");
}
#endif

41
kernel/rcu/srcutree.c
View File

@ -763,47 +763,6 @@ void __srcu_read_unlock(struct srcu_struct *ssp, int idx)
}
EXPORT_SYMBOL_GPL(__srcu_read_unlock);
/*
* Counts the new reader in the appropriate per-CPU element of the
* srcu_struct. Returns an index that must be passed to the matching
* srcu_read_unlock_lite().
*
* Note that this_cpu_inc() is an RCU read-side critical section either
* because it disables interrupts, because it is a single instruction,
* or because it is a read-modify-write atomic operation, depending on
* the whims of the architecture.
*/
int __srcu_read_lock_lite(struct srcu_struct *ssp)
{
int idx;
RCU_LOCKDEP_WARN(!rcu_is_watching(), "RCU must be watching srcu_read_lock_lite().");
idx = READ_ONCE(ssp->srcu_idx) & 0x1;
this_cpu_inc(ssp->sda->srcu_lock_count[idx].counter); /* Y */
barrier(); /* Avoid leaking the critical section. */
return idx;
}
EXPORT_SYMBOL_GPL(__srcu_read_lock_lite);
/*
* Removes the count for the old reader from the appropriate
* per-CPU element of the srcu_struct. Note that this may well be a
* different CPU than that which was incremented by the corresponding
* srcu_read_lock_lite(), but it must be within the same task.
*
* Note that this_cpu_inc() is an RCU read-side critical section either
* because it disables interrupts, because it is a single instruction,
* or because it is a read-modify-write atomic operation, depending on
* the whims of the architecture.
*/
void __srcu_read_unlock_lite(struct srcu_struct *ssp, int idx)
{
barrier(); /* Avoid leaking the critical section. */
this_cpu_inc(ssp->sda->srcu_unlock_count[idx].counter); /* Z */
RCU_LOCKDEP_WARN(!rcu_is_watching(), "RCU must be watching srcu_read_unlock_lite().");
}
EXPORT_SYMBOL_GPL(__srcu_read_unlock_lite);
#ifdef CONFIG_NEED_SRCU_NMI_SAFE
/*