Kernel/linux
1
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2025-01-10 07:10:27 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

six locks: Simplify dispatch

Browse Source 
Originally, we used inlining/flattening to cause the compiler to
generate different versions of lock/trylock/relock/unlock for each lock
type - read, intent, and write. This made the individual functions
smaller and let the compiler eliminate table lookups: however, as the
code has gotten more complicated these optimizations have gotten less
worthwhile, and all the tricky inlining and dispatching made the code
less readable.

Text size: 11015 bytes -> 7467 bytes, and benchmarks show no loss of
performance.

Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>
This commit is contained in:
Kent Overstreet 2023-05-20 21:44:30 -04:00
parent d2c86b77de
commit c4bd3491b1
2 changed files with 106 additions and 128 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

86
fs/bcachefs/six.c
View File

@ -245,9 +245,10 @@ unlock:
} }
} }
static inline void six_lock_wakeup(struct six_lock *lock, __always_inline
union six_lock_state state, static void six_lock_wakeup(struct six_lock *lock,
enum six_lock_type lock_type) union six_lock_state state,
enum six_lock_type lock_type)
{ {
if (lock_type == SIX_LOCK_write && state.read_lock) if (lock_type == SIX_LOCK_write && state.read_lock)
return; return;
@ -258,6 +259,7 @@ static inline void six_lock_wakeup(struct six_lock *lock,
__six_lock_wakeup(lock, lock_type); __six_lock_wakeup(lock, lock_type);
} }
__always_inline
static bool do_six_trylock_type(struct six_lock *lock, static bool do_six_trylock_type(struct six_lock *lock,
enum six_lock_type type, enum six_lock_type type,
bool try) bool try)
@ -271,9 +273,8 @@ static bool do_six_trylock_type(struct six_lock *lock,
return ret > 0; return ret > 0;
} }
__always_inline __flatten bool six_trylock_ip_type(struct six_lock *lock, enum six_lock_type type,
static bool __six_trylock_type(struct six_lock *lock, enum six_lock_type type, unsigned long ip)
unsigned long ip)
{ {
if (!do_six_trylock_type(lock, type, true)) if (!do_six_trylock_type(lock, type, true))
return false; return false;
@ -283,9 +284,8 @@ static bool __six_trylock_type(struct six_lock *lock, enum six_lock_type type,
return true; return true;
} }
__always_inline __flatten bool six_relock_ip_type(struct six_lock *lock, enum six_lock_type type,
static bool __six_relock_type(struct six_lock *lock, enum six_lock_type type, unsigned seq, unsigned long ip)
unsigned seq, unsigned long ip)
{ {
const struct six_lock_vals l[] = LOCK_VALS; const struct six_lock_vals l[] = LOCK_VALS;
union six_lock_state old; union six_lock_state old;
@ -335,6 +335,7 @@ static bool __six_relock_type(struct six_lock *lock, enum six_lock_type type,
six_acquire(&lock->dep_map, 1, type == SIX_LOCK_read, ip); six_acquire(&lock->dep_map, 1, type == SIX_LOCK_read, ip);
return true; return true;
} }
EXPORT_SYMBOL_GPL(six_relock_ip_type);
#ifdef CONFIG_SIX_LOCK_SPIN_ON_OWNER #ifdef CONFIG_SIX_LOCK_SPIN_ON_OWNER
@ -566,11 +567,10 @@ out:
return ret; return ret;
} }
__always_inline __flatten int six_lock_type_ip_waiter(struct six_lock *lock, enum six_lock_type type,
static int __six_lock_type_waiter(struct six_lock *lock, enum six_lock_type type, struct six_lock_waiter *wait,
struct six_lock_waiter *wait, six_lock_should_sleep_fn should_sleep_fn, void *p,
six_lock_should_sleep_fn should_sleep_fn, void *p, unsigned long ip)
unsigned long ip)
{ {
int ret; int ret;
@ -589,18 +589,9 @@ static int __six_lock_type_waiter(struct six_lock *lock, enum six_lock_type type
return ret; return ret;
} }
EXPORT_SYMBOL_GPL(six_lock_type_ip_waiter);
__always_inline __always_inline
static int __six_lock_type(struct six_lock *lock, enum six_lock_type type,
six_lock_should_sleep_fn should_sleep_fn, void *p,
unsigned long ip)
{
struct six_lock_waiter wait;
return __six_lock_type_waiter(lock, type, &wait, should_sleep_fn, p, ip);
}
__always_inline __flatten
static void do_six_unlock_type(struct six_lock *lock, enum six_lock_type type) static void do_six_unlock_type(struct six_lock *lock, enum six_lock_type type)
{ {
const struct six_lock_vals l[] = LOCK_VALS; const struct six_lock_vals l[] = LOCK_VALS;
@ -628,9 +619,7 @@ static void do_six_unlock_type(struct six_lock *lock, enum six_lock_type type)
six_lock_wakeup(lock, state, l[type].unlock_wakeup); six_lock_wakeup(lock, state, l[type].unlock_wakeup);
} }
__always_inline __flatten void six_unlock_ip_type(struct six_lock *lock, enum six_lock_type type, unsigned long ip)
static void __six_unlock_type(struct six_lock *lock, enum six_lock_type type,
unsigned long ip)
{ {
EBUG_ON(type == SIX_LOCK_write && EBUG_ON(type == SIX_LOCK_write &&
!(lock->state.v & __SIX_LOCK_HELD_intent)); !(lock->state.v & __SIX_LOCK_HELD_intent));
@ -649,48 +638,7 @@ static void __six_unlock_type(struct six_lock *lock, enum six_lock_type type,
do_six_unlock_type(lock, type); do_six_unlock_type(lock, type);
} }
EXPORT_SYMBOL_GPL(six_unlock_ip_type);
#define __SIX_LOCK(type) \
bool six_trylock_ip_##type(struct six_lock *lock, unsigned long ip) \
{ \
return __six_trylock_type(lock, SIX_LOCK_##type, ip); \
} \
EXPORT_SYMBOL_GPL(six_trylock_ip_##type); \
\
bool six_relock_ip_##type(struct six_lock *lock, u32 seq, unsigned long ip)\
{ \
return __six_relock_type(lock, SIX_LOCK_##type, seq, ip); \
} \
EXPORT_SYMBOL_GPL(six_relock_ip_##type); \
\
int six_lock_ip_##type(struct six_lock *lock, \
six_lock_should_sleep_fn should_sleep_fn, void *p, \
unsigned long ip) \
{ \
return __six_lock_type(lock, SIX_LOCK_##type, should_sleep_fn, p, ip);\
} \
EXPORT_SYMBOL_GPL(six_lock_ip_##type); \
\
int six_lock_ip_waiter_##type(struct six_lock *lock, \
struct six_lock_waiter *wait, \
six_lock_should_sleep_fn should_sleep_fn, void *p,\
unsigned long ip) \
{ \
return __six_lock_type_waiter(lock, SIX_LOCK_##type, wait, should_sleep_fn, p, ip);\
} \
EXPORT_SYMBOL_GPL(six_lock_ip_waiter_##type); \
\
void six_unlock_ip_##type(struct six_lock *lock, unsigned long ip) \
{ \
__six_unlock_type(lock, SIX_LOCK_##type, ip); \
} \
EXPORT_SYMBOL_GPL(six_unlock_ip_##type);
__SIX_LOCK(read)
__SIX_LOCK(intent)
__SIX_LOCK(write)
#undef __SIX_LOCK
/* Convert from intent to read: */ /* Convert from intent to read: */
void six_lock_downgrade(struct six_lock *lock) void six_lock_downgrade(struct six_lock *lock)

148
fs/bcachefs/six.h
View File

@ -148,37 +148,116 @@ do { \
__six_lock_init((lock), #lock, &__key, flags); \ __six_lock_init((lock), #lock, &__key, flags); \
} while (0) } while (0)
bool six_trylock_ip_type(struct six_lock *lock, enum six_lock_type type,
unsigned long ip);
static inline bool six_trylock_type(struct six_lock *lock, enum six_lock_type type)
{
return six_trylock_ip_type(lock, type, _THIS_IP_);
}
int six_lock_type_ip_waiter(struct six_lock *lock, enum six_lock_type type,
struct six_lock_waiter *wait,
six_lock_should_sleep_fn should_sleep_fn, void *p,
unsigned long ip);
static inline int six_lock_type_waiter(struct six_lock *lock, enum six_lock_type type,
struct six_lock_waiter *wait,
six_lock_should_sleep_fn should_sleep_fn, void *p)
{
return six_lock_type_ip_waiter(lock, type, wait, should_sleep_fn, p, _THIS_IP_);
}
static inline int six_lock_ip_type(struct six_lock *lock, enum six_lock_type type,
six_lock_should_sleep_fn should_sleep_fn, void *p,
unsigned long ip)
{
struct six_lock_waiter wait;
return six_lock_type_ip_waiter(lock, type, &wait, should_sleep_fn, p, ip);
}
static inline int six_lock_type(struct six_lock *lock, enum six_lock_type type,
six_lock_should_sleep_fn should_sleep_fn, void *p)
{
struct six_lock_waiter wait;
return six_lock_type_ip_waiter(lock, type, &wait, should_sleep_fn, p, _THIS_IP_);
}
bool six_relock_ip_type(struct six_lock *lock, enum six_lock_type type,
unsigned seq, unsigned long ip);
static inline bool six_relock_type(struct six_lock *lock, enum six_lock_type type,
unsigned seq)
{
return six_relock_ip_type(lock, type, seq, _THIS_IP_);
}
void six_unlock_ip_type(struct six_lock *lock, enum six_lock_type type, unsigned long ip);
static inline void six_unlock_type(struct six_lock *lock, enum six_lock_type type)
{
six_unlock_ip_type(lock, type, _THIS_IP_);
}
#define __SIX_LOCK(type) \ #define __SIX_LOCK(type) \
bool six_trylock_ip_##type(struct six_lock *, unsigned long); \ static inline bool six_trylock_ip_##type(struct six_lock *lock, unsigned long ip)\
bool six_relock_ip_##type(struct six_lock *, u32, unsigned long); \ { \
int six_lock_ip_##type(struct six_lock *, six_lock_should_sleep_fn, \ return six_trylock_ip_type(lock, SIX_LOCK_##type, ip); \
void *, unsigned long); \ } \
int six_lock_ip_waiter_##type(struct six_lock *, struct six_lock_waiter *,\
six_lock_should_sleep_fn, void *, unsigned long);\
void six_unlock_ip_##type(struct six_lock *, unsigned long); \
\ \
static inline bool six_trylock_##type(struct six_lock *lock) \ static inline bool six_trylock_##type(struct six_lock *lock) \
{ \ { \
return six_trylock_ip_##type(lock, _THIS_IP_); \ return six_trylock_ip_type(lock, SIX_LOCK_##type, _THIS_IP_); \
} \ } \
\
static inline int six_lock_ip_waiter_##type(struct six_lock *lock, \
struct six_lock_waiter *wait, \
six_lock_should_sleep_fn should_sleep_fn, void *p,\
unsigned long ip) \
{ \
return six_lock_type_ip_waiter(lock, SIX_LOCK_##type, wait, should_sleep_fn, p, ip);\
} \
\
static inline int six_lock_ip_##type(struct six_lock *lock, \
six_lock_should_sleep_fn should_sleep_fn, void *p, \
unsigned long ip) \
{ \
return six_lock_ip_type(lock, SIX_LOCK_##type, should_sleep_fn, p, ip);\
} \
\
static inline bool six_relock_ip_##type(struct six_lock *lock, u32 seq, unsigned long ip)\
{ \
return six_relock_ip_type(lock, SIX_LOCK_##type, seq, ip); \
} \
\
static inline bool six_relock_##type(struct six_lock *lock, u32 seq) \ static inline bool six_relock_##type(struct six_lock *lock, u32 seq) \
{ \ { \
return six_relock_ip_##type(lock, seq, _THIS_IP_); \ return six_relock_ip_type(lock, SIX_LOCK_##type, seq, _THIS_IP_);\
} \ } \
\
static inline int six_lock_##type(struct six_lock *lock, \ static inline int six_lock_##type(struct six_lock *lock, \
six_lock_should_sleep_fn fn, void *p)\ six_lock_should_sleep_fn fn, void *p)\
{ \ { \
return six_lock_ip_##type(lock, fn, p, _THIS_IP_); \ return six_lock_ip_##type(lock, fn, p, _THIS_IP_); \
} \ } \
\
static inline int six_lock_waiter_##type(struct six_lock *lock, \ static inline int six_lock_waiter_##type(struct six_lock *lock, \
struct six_lock_waiter *wait, \ struct six_lock_waiter *wait, \
six_lock_should_sleep_fn fn, void *p) \ six_lock_should_sleep_fn fn, void *p) \
{ \ { \
return six_lock_ip_waiter_##type(lock, wait, fn, p, _THIS_IP_); \ return six_lock_ip_waiter_##type(lock, wait, fn, p, _THIS_IP_); \
} \ } \
\
static inline void six_unlock_ip_##type(struct six_lock *lock, unsigned long ip) \
{ \
six_unlock_ip_type(lock, SIX_LOCK_##type, ip); \
} \
\
static inline void six_unlock_##type(struct six_lock *lock) \ static inline void six_unlock_##type(struct six_lock *lock) \
{ \ { \
return six_unlock_ip_##type(lock, _THIS_IP_); \ six_unlock_ip_type(lock, SIX_LOCK_##type, _THIS_IP_); \
} }
__SIX_LOCK(read) __SIX_LOCK(read)
@ -186,55 +265,6 @@ __SIX_LOCK(intent)
__SIX_LOCK(write) __SIX_LOCK(write)
#undef __SIX_LOCK #undef __SIX_LOCK
#define SIX_LOCK_DISPATCH(type, fn, ...) \
switch (type) { \
case SIX_LOCK_read: \
return fn##_read(__VA_ARGS__); \
case SIX_LOCK_intent: \
return fn##_intent(__VA_ARGS__); \
case SIX_LOCK_write: \
return fn##_write(__VA_ARGS__); \
default: \
BUG(); \
}
static inline bool six_trylock_type(struct six_lock *lock, enum six_lock_type type)
{
SIX_LOCK_DISPATCH(type, six_trylock, lock);
}
static inline bool six_relock_type(struct six_lock *lock, enum six_lock_type type,
unsigned seq)
{
SIX_LOCK_DISPATCH(type, six_relock, lock, seq);
}
static inline int six_lock_type(struct six_lock *lock, enum six_lock_type type,
six_lock_should_sleep_fn should_sleep_fn, void *p)
{
SIX_LOCK_DISPATCH(type, six_lock, lock, should_sleep_fn, p);
}
static inline int six_lock_type_ip_waiter(struct six_lock *lock, enum six_lock_type type,
struct six_lock_waiter *wait,
six_lock_should_sleep_fn should_sleep_fn, void *p,
unsigned long ip)
{
SIX_LOCK_DISPATCH(type, six_lock_ip_waiter, lock, wait, should_sleep_fn, p, ip);
}
static inline int six_lock_type_waiter(struct six_lock *lock, enum six_lock_type type,
struct six_lock_waiter *wait,
six_lock_should_sleep_fn should_sleep_fn, void *p)
{
SIX_LOCK_DISPATCH(type, six_lock_waiter, lock, wait, should_sleep_fn, p);
}
static inline void six_unlock_type(struct six_lock *lock, enum six_lock_type type)
{
SIX_LOCK_DISPATCH(type, six_unlock, lock);
}
void six_lock_downgrade(struct six_lock *); void six_lock_downgrade(struct six_lock *);
bool six_lock_tryupgrade(struct six_lock *); bool six_lock_tryupgrade(struct six_lock *);
bool six_trylock_convert(struct six_lock *, enum six_lock_type, bool six_trylock_convert(struct six_lock *, enum six_lock_type,