Kernel/linux-stable
1
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git synced 2025-01-04 04:06:26 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

srcu: Begin offloading srcu_struct fields to srcu_update

Browse Source 
The current srcu_struct structure is on the order of 200 bytes in size
(depending on architecture and .config), which is much better than the
old-style 26K bytes, but still all too inconvenient when one is trying
to achieve good cache locality on a fastpath involving SRCU readers.

However, only a few fields in srcu_struct are used by SRCU readers.
The remaining fields could be offloaded to a new srcu_update
structure, thus shrinking the srcu_struct structure down to a few
tens of bytes.  This commit begins this noble quest, a quest that is
complicated by open-coded initialization of the srcu_struct within the
srcu_notifier_head structure.  This complication is addressed by updating
the srcu_notifier_head structure's open coding, given that there does
not appear to be a straightforward way of abstracting that initialization.

This commit moves only the ->node pointer to srcu_update.  Later commits
will move additional fields.

[ paulmck: Fold in qiang1.zhang@intel.com's memory-leak fix. ]

Link: https://lore.kernel.org/all/20230320055751.4120251-1-qiang1.zhang@intel.com/
Suggested-by: Christoph Hellwig <hch@lst.de>
Cc: "Rafael J. Wysocki" <rafael.j.wysocki@intel.com>
Cc: "Michał Mirosław" <mirq-linux@rere.qmqm.pl>
Cc: Dmitry Osipenko <dmitry.osipenko@collabora.com>
Tested-by: Sachin Sant <sachinp@linux.ibm.com>
Tested-by: "Zhang, Qiang1" <qiang1.zhang@intel.com>
Acked-by: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
Tested-by: Joel Fernandes (Google) <joel@joelfernandes.org>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
This commit is contained in:
Paul E. McKenney 2023-03-16 17:58:51 -07:00
parent f4d01a2593
commit 95433f7263
5 changed files with 48 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
include/linux/notifier.h
View File

@ -73,6 +73,9 @@ struct raw_notifier_head {
struct srcu_notifier_head { struct srcu_notifier_head {
struct mutex mutex; struct mutex mutex;
#ifdef CONFIG_TREE_SRCU
struct srcu_usage srcuu;
#endif
struct srcu_struct srcu; struct srcu_struct srcu;
struct notifier_block __rcu *head; struct notifier_block __rcu *head;
}; };
@ -107,7 +110,7 @@ extern void srcu_init_notifier_head(struct srcu_notifier_head *nh);
{ \ { \
.mutex = __MUTEX_INITIALIZER(name.mutex), \ .mutex = __MUTEX_INITIALIZER(name.mutex), \
.head = NULL, \ .head = NULL, \
.srcu = __SRCU_STRUCT_INIT(name.srcu, pcpu), \ .srcu = __SRCU_STRUCT_INIT(name.srcu, name.srcuu, pcpu), \
} }
#define ATOMIC_NOTIFIER_HEAD(name) \ #define ATOMIC_NOTIFIER_HEAD(name) \

6
include/linux/srcutiny.h
View File

@ -31,7 +31,7 @@ struct srcu_struct {
void srcu_drive_gp(struct work_struct *wp); void srcu_drive_gp(struct work_struct *wp);
#define __SRCU_STRUCT_INIT(name, __ignored) \ #define __SRCU_STRUCT_INIT(name, __ignored, ___ignored) \
{ \ { \
.srcu_wq = __SWAIT_QUEUE_HEAD_INITIALIZER(name.srcu_wq), \ .srcu_wq = __SWAIT_QUEUE_HEAD_INITIALIZER(name.srcu_wq), \
.srcu_cb_tail = &name.srcu_cb_head, \ .srcu_cb_tail = &name.srcu_cb_head, \
@ -44,9 +44,9 @@ void srcu_drive_gp(struct work_struct *wp);
* Tree SRCU, which needs some per-CPU data. * Tree SRCU, which needs some per-CPU data.
*/ */
#define DEFINE_SRCU(name) \ #define DEFINE_SRCU(name) \
struct srcu_struct name = __SRCU_STRUCT_INIT(name, name) struct srcu_struct name = __SRCU_STRUCT_INIT(name, name, name)
#define DEFINE_STATIC_SRCU(name) \ #define DEFINE_STATIC_SRCU(name) \
static struct srcu_struct name = __SRCU_STRUCT_INIT(name, name) static struct srcu_struct name = __SRCU_STRUCT_INIT(name, name, name)
void synchronize_srcu(struct srcu_struct *ssp); void synchronize_srcu(struct srcu_struct *ssp);

27
include/linux/srcutree.h
View File

@ -57,11 +57,17 @@ struct srcu_node {
int grphi; /* Biggest CPU for node. */ int grphi; /* Biggest CPU for node. */
}; };
/*
* Per-SRCU-domain structure, update-side data linked from srcu_struct.
*/
struct srcu_usage {
struct srcu_node *node; /* Combining tree. */
};
/* /*
* Per-SRCU-domain structure, similar in function to rcu_state. * Per-SRCU-domain structure, similar in function to rcu_state.
*/ */
struct srcu_struct { struct srcu_struct {
struct srcu_node *node; /* Combining tree. */
struct srcu_node *level[RCU_NUM_LVLS + 1]; struct srcu_node *level[RCU_NUM_LVLS + 1];
/* First node at each level. */ /* First node at each level. */
int srcu_size_state; /* Small-to-big transition state. */ int srcu_size_state; /* Small-to-big transition state. */
@ -90,6 +96,7 @@ struct srcu_struct {
unsigned long reschedule_count; unsigned long reschedule_count;
struct delayed_work work; struct delayed_work work;
struct lockdep_map dep_map; struct lockdep_map dep_map;
struct srcu_usage *srcu_sup; /* Update-side data. */
}; };
/* Values for size state variable (->srcu_size_state). */ /* Values for size state variable (->srcu_size_state). */
@ -108,24 +115,24 @@ struct srcu_struct {
#define SRCU_STATE_SCAN1 1 #define SRCU_STATE_SCAN1 1
#define SRCU_STATE_SCAN2 2 #define SRCU_STATE_SCAN2 2
#define __SRCU_STRUCT_INIT_COMMON(name) \ #define __SRCU_STRUCT_INIT_COMMON(name, usage_name) \
.lock = __SPIN_LOCK_UNLOCKED(name.lock), \ .lock = __SPIN_LOCK_UNLOCKED(name.lock), \
.srcu_gp_seq_needed = -1UL, \ .srcu_gp_seq_needed = -1UL, \
.work = __DELAYED_WORK_INITIALIZER(name.work, NULL, 0), \ .work = __DELAYED_WORK_INITIALIZER(name.work, NULL, 0), \
.srcu_sup = &usage_name, \
__SRCU_DEP_MAP_INIT(name) __SRCU_DEP_MAP_INIT(name)
#define __SRCU_STRUCT_INIT_MODULE(name) \ #define __SRCU_STRUCT_INIT_MODULE(name, usage_name) \
{ \ { \
__SRCU_STRUCT_INIT_COMMON(name) \ __SRCU_STRUCT_INIT_COMMON(name, usage_name) \
} }
#define __SRCU_STRUCT_INIT(name, pcpu_name) \ #define __SRCU_STRUCT_INIT(name, usage_name, pcpu_name) \
{ \ { \
.sda = &pcpu_name, \ .sda = &pcpu_name, \
__SRCU_STRUCT_INIT_COMMON(name) \ __SRCU_STRUCT_INIT_COMMON(name, usage_name) \
} }
/* /*
* Define and initialize a srcu struct at build time. * Define and initialize a srcu struct at build time.
* Do -not- call init_srcu_struct() nor cleanup_srcu_struct() on it. * Do -not- call init_srcu_struct() nor cleanup_srcu_struct() on it.
@ -147,15 +154,17 @@ struct srcu_struct {
*/ */
#ifdef MODULE #ifdef MODULE
# define __DEFINE_SRCU(name, is_static) \ # define __DEFINE_SRCU(name, is_static) \
is_static struct srcu_struct name = __SRCU_STRUCT_INIT_MODULE(name); \ static struct srcu_usage name##_srcu_usage; \
is_static struct srcu_struct name = __SRCU_STRUCT_INIT_MODULE(name, name##_srcu_usage); \
extern struct srcu_struct * const __srcu_struct_##name; \ extern struct srcu_struct * const __srcu_struct_##name; \
struct srcu_struct * const __srcu_struct_##name \ struct srcu_struct * const __srcu_struct_##name \
__section("___srcu_struct_ptrs") = &name __section("___srcu_struct_ptrs") = &name
#else #else
# define __DEFINE_SRCU(name, is_static) \ # define __DEFINE_SRCU(name, is_static) \
static DEFINE_PER_CPU(struct srcu_data, name##_srcu_data); \ static DEFINE_PER_CPU(struct srcu_data, name##_srcu_data); \
static struct srcu_usage name##_srcu_usage; \
is_static struct srcu_struct name = \ is_static struct srcu_struct name = \
__SRCU_STRUCT_INIT(name, name##_srcu_data) __SRCU_STRUCT_INIT(name, name##_srcu_usage, name##_srcu_data)
#endif #endif
#define DEFINE_SRCU(name) __DEFINE_SRCU(name, /* not static */) #define DEFINE_SRCU(name) __DEFINE_SRCU(name, /* not static */)
#define DEFINE_STATIC_SRCU(name) __DEFINE_SRCU(name, static) #define DEFINE_STATIC_SRCU(name) __DEFINE_SRCU(name, static)

6
kernel/rcu/rcu.h
View File

@ -341,11 +341,13 @@ extern void rcu_init_geometry(void);
* specified state structure (for SRCU) or the only rcu_state structure * specified state structure (for SRCU) or the only rcu_state structure
* (for RCU). * (for RCU).
*/ */
#define srcu_for_each_node_breadth_first(sp, rnp) \ #define _rcu_for_each_node_breadth_first(sp, rnp) \
for ((rnp) = &(sp)->node[0]; \ for ((rnp) = &(sp)->node[0]; \
(rnp) < &(sp)->node[rcu_num_nodes]; (rnp)++) (rnp) < &(sp)->node[rcu_num_nodes]; (rnp)++)
#define rcu_for_each_node_breadth_first(rnp) \ #define rcu_for_each_node_breadth_first(rnp) \
srcu_for_each_node_breadth_first(&rcu_state, rnp) _rcu_for_each_node_breadth_first(&rcu_state, rnp)
#define srcu_for_each_node_breadth_first(ssp, rnp) \
_rcu_for_each_node_breadth_first(ssp->srcu_sup, rnp)
/* /*
* Scan the leaves of the rcu_node hierarchy for the rcu_state structure. * Scan the leaves of the rcu_node hierarchy for the rcu_state structure.

28
kernel/rcu/srcutree.c
View File

@ -173,12 +173,12 @@ static bool init_srcu_struct_nodes(struct srcu_struct *ssp, gfp_t gfp_flags)
/* Initialize geometry if it has not already been initialized. */ /* Initialize geometry if it has not already been initialized. */
rcu_init_geometry(); rcu_init_geometry();
ssp->node = kcalloc(rcu_num_nodes, sizeof(*ssp->node), gfp_flags); ssp->srcu_sup->node = kcalloc(rcu_num_nodes, sizeof(*ssp->srcu_sup->node), gfp_flags);
if (!ssp->node) if (!ssp->srcu_sup->node)
return false; return false;
/* Work out the overall tree geometry. */ /* Work out the overall tree geometry. */
ssp->level[0] = &ssp->node[0]; ssp->level[0] = &ssp->srcu_sup->node[0];
for (i = 1; i < rcu_num_lvls; i++) for (i = 1; i < rcu_num_lvls; i++)
ssp->level[i] = ssp->level[i - 1] + num_rcu_lvl[i - 1]; ssp->level[i] = ssp->level[i - 1] + num_rcu_lvl[i - 1];
rcu_init_levelspread(levelspread, num_rcu_lvl); rcu_init_levelspread(levelspread, num_rcu_lvl);
@ -195,7 +195,7 @@ static bool init_srcu_struct_nodes(struct srcu_struct *ssp, gfp_t gfp_flags)
snp->srcu_gp_seq_needed_exp = SRCU_SNP_INIT_SEQ; snp->srcu_gp_seq_needed_exp = SRCU_SNP_INIT_SEQ;
snp->grplo = -1; snp->grplo = -1;
snp->grphi = -1; snp->grphi = -1;
if (snp == &ssp->node[0]) { if (snp == &ssp->srcu_sup->node[0]) {
/* Root node, special case. */ /* Root node, special case. */
snp->srcu_parent = NULL; snp->srcu_parent = NULL;
continue; continue;
@ -236,8 +236,12 @@ static bool init_srcu_struct_nodes(struct srcu_struct *ssp, gfp_t gfp_flags)
*/ */
static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static) static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static)
{ {
if (!is_static)
ssp->srcu_sup = kzalloc(sizeof(*ssp->srcu_sup), GFP_KERNEL);
if (!ssp->srcu_sup)
return -ENOMEM;
ssp->srcu_size_state = SRCU_SIZE_SMALL; ssp->srcu_size_state = SRCU_SIZE_SMALL;
ssp->node = NULL; ssp->srcu_sup->node = NULL;
mutex_init(&ssp->srcu_cb_mutex); mutex_init(&ssp->srcu_cb_mutex);
mutex_init(&ssp->srcu_gp_mutex); mutex_init(&ssp->srcu_gp_mutex);
ssp->srcu_idx = 0; ssp->srcu_idx = 0;
@ -249,8 +253,11 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static)
ssp->sda_is_static = is_static; ssp->sda_is_static = is_static;
if (!is_static) if (!is_static)
ssp->sda = alloc_percpu(struct srcu_data); ssp->sda = alloc_percpu(struct srcu_data);
if (!ssp->sda) if (!ssp->sda) {
if (!is_static)
kfree(ssp->srcu_sup);
return -ENOMEM; return -ENOMEM;
}
init_srcu_struct_data(ssp); init_srcu_struct_data(ssp);
ssp->srcu_gp_seq_needed_exp = 0; ssp->srcu_gp_seq_needed_exp = 0;
ssp->srcu_last_gp_end = ktime_get_mono_fast_ns(); ssp->srcu_last_gp_end = ktime_get_mono_fast_ns();
@ -259,6 +266,7 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static)
if (!ssp->sda_is_static) { if (!ssp->sda_is_static) {
free_percpu(ssp->sda); free_percpu(ssp->sda);
ssp->sda = NULL; ssp->sda = NULL;
kfree(ssp->srcu_sup);
return -ENOMEM; return -ENOMEM;
} }
} else { } else {
@ -656,13 +664,15 @@ void cleanup_srcu_struct(struct srcu_struct *ssp)
rcu_seq_current(&ssp->srcu_gp_seq), ssp->srcu_gp_seq_needed); rcu_seq_current(&ssp->srcu_gp_seq), ssp->srcu_gp_seq_needed);
return; /* Caller forgot to stop doing call_srcu()? */ return; /* Caller forgot to stop doing call_srcu()? */
} }
kfree(ssp->srcu_sup->node);
ssp->srcu_sup->node = NULL;
ssp->srcu_size_state = SRCU_SIZE_SMALL;
if (!ssp->sda_is_static) { if (!ssp->sda_is_static) {
free_percpu(ssp->sda); free_percpu(ssp->sda);
ssp->sda = NULL; ssp->sda = NULL;
kfree(ssp->srcu_sup);
ssp->srcu_sup = NULL;
} }
kfree(ssp->node);
ssp->node = NULL;
ssp->srcu_size_state = SRCU_SIZE_SMALL;
} }
EXPORT_SYMBOL_GPL(cleanup_srcu_struct); EXPORT_SYMBOL_GPL(cleanup_srcu_struct);