Kernel/linux
1
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2025-01-07 13:53:24 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

srcu: Make Tiny SRCU use full-sized grace-period counters

Browse Source 
This commit makes Tiny SRCU use full-sized grace-period counters to
further avoid counter-wrap issues when using polled grace-period APIs.

Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
This commit is contained in:
Paul E. McKenney 2022-08-02 15:32:47 -07:00
parent de3f2671ae
commit 5fe89191e4
2 changed files with 10 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
include/linux/srcutiny.h
View File

@ -15,10 +15,10 @@
struct srcu_struct { struct srcu_struct {
short srcu_lock_nesting[2]; /* srcu_read_lock() nesting depth. */ short srcu_lock_nesting[2]; /* srcu_read_lock() nesting depth. */
unsigned short srcu_idx; /* Current reader array element in bit 0x2. */
unsigned short srcu_idx_max; /* Furthest future srcu_idx request. */
u8 srcu_gp_running; /* GP workqueue running? */ u8 srcu_gp_running; /* GP workqueue running? */
u8 srcu_gp_waiting; /* GP waiting for readers? */ u8 srcu_gp_waiting; /* GP waiting for readers? */
unsigned long srcu_idx; /* Current reader array element in bit 0x2. */
unsigned long srcu_idx_max; /* Furthest future srcu_idx request. */
struct swait_queue_head srcu_wq; struct swait_queue_head srcu_wq;
/* Last srcu_read_unlock() wakes GP. */ /* Last srcu_read_unlock() wakes GP. */
struct rcu_head *srcu_cb_head; /* Pending callbacks: Head. */ struct rcu_head *srcu_cb_head; /* Pending callbacks: Head. */
@ -82,7 +82,7 @@ static inline void srcu_torture_stats_print(struct srcu_struct *ssp,
int idx; int idx;
idx = ((data_race(READ_ONCE(ssp->srcu_idx)) + 1) & 0x2) >> 1; idx = ((data_race(READ_ONCE(ssp->srcu_idx)) + 1) & 0x2) >> 1;
pr_alert("%s%s Tiny SRCU per-CPU(idx=%d): (%hd,%hd) gp: %hu->%hu\n", pr_alert("%s%s Tiny SRCU per-CPU(idx=%d): (%hd,%hd) gp: %lu->%lu\n",
tt, tf, idx, tt, tf, idx,
data_race(READ_ONCE(ssp->srcu_lock_nesting[!idx])), data_race(READ_ONCE(ssp->srcu_lock_nesting[!idx])),
data_race(READ_ONCE(ssp->srcu_lock_nesting[idx])), data_race(READ_ONCE(ssp->srcu_lock_nesting[idx])),

14
kernel/rcu/srcutiny.c
View File

@ -117,7 +117,7 @@ void srcu_drive_gp(struct work_struct *wp)
struct srcu_struct *ssp; struct srcu_struct *ssp;
ssp = container_of(wp, struct srcu_struct, srcu_work); ssp = container_of(wp, struct srcu_struct, srcu_work);
if (ssp->srcu_gp_running || USHORT_CMP_GE(ssp->srcu_idx, READ_ONCE(ssp->srcu_idx_max))) if (ssp->srcu_gp_running || ULONG_CMP_GE(ssp->srcu_idx, READ_ONCE(ssp->srcu_idx_max)))
return; /* Already running or nothing to do. */ return; /* Already running or nothing to do. */
/* Remove recently arrived callbacks and wait for readers. */ /* Remove recently arrived callbacks and wait for readers. */
@ -150,17 +150,17 @@ void srcu_drive_gp(struct work_struct *wp)
* straighten that out. * straighten that out.
*/ */
WRITE_ONCE(ssp->srcu_gp_running, false); WRITE_ONCE(ssp->srcu_gp_running, false);
if (USHORT_CMP_LT(ssp->srcu_idx, READ_ONCE(ssp->srcu_idx_max))) if (ULONG_CMP_LT(ssp->srcu_idx, READ_ONCE(ssp->srcu_idx_max)))
schedule_work(&ssp->srcu_work); schedule_work(&ssp->srcu_work);
} }
EXPORT_SYMBOL_GPL(srcu_drive_gp); EXPORT_SYMBOL_GPL(srcu_drive_gp);
static void srcu_gp_start_if_needed(struct srcu_struct *ssp) static void srcu_gp_start_if_needed(struct srcu_struct *ssp)
{ {
unsigned short cookie; unsigned long cookie;
cookie = get_state_synchronize_srcu(ssp); cookie = get_state_synchronize_srcu(ssp);
if (USHORT_CMP_GE(READ_ONCE(ssp->srcu_idx_max), cookie)) if (ULONG_CMP_GE(READ_ONCE(ssp->srcu_idx_max), cookie))
return; return;
WRITE_ONCE(ssp->srcu_idx_max, cookie); WRITE_ONCE(ssp->srcu_idx_max, cookie);
if (!READ_ONCE(ssp->srcu_gp_running)) { if (!READ_ONCE(ssp->srcu_gp_running)) {
@ -215,7 +215,7 @@ unsigned long get_state_synchronize_srcu(struct srcu_struct *ssp)
barrier(); barrier();
ret = (READ_ONCE(ssp->srcu_idx) + 3) & ~0x1; ret = (READ_ONCE(ssp->srcu_idx) + 3) & ~0x1;
barrier(); barrier();
return ret & USHRT_MAX; return ret;
} }
EXPORT_SYMBOL_GPL(get_state_synchronize_srcu); EXPORT_SYMBOL_GPL(get_state_synchronize_srcu);
@ -240,10 +240,10 @@ EXPORT_SYMBOL_GPL(start_poll_synchronize_srcu);
*/ */
bool poll_state_synchronize_srcu(struct srcu_struct *ssp, unsigned long cookie) bool poll_state_synchronize_srcu(struct srcu_struct *ssp, unsigned long cookie)
{ {
unsigned short cur_s = READ_ONCE(ssp->srcu_idx); unsigned long cur_s = READ_ONCE(ssp->srcu_idx);
barrier(); barrier();
return USHORT_CMP_GE(cur_s, cookie) || USHORT_CMP_LT(cur_s, cookie - 3); return ULONG_CMP_GE(cur_s, cookie) || ULONG_CMP_LT(cur_s, cookie - 3);
} }
EXPORT_SYMBOL_GPL(poll_state_synchronize_srcu); EXPORT_SYMBOL_GPL(poll_state_synchronize_srcu);