mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2025-01-09 14:43:16 +00:00
bcachefs: btree_locking.c
Start to centralize some of the locking code in a new file; more locking code will be moving here in the future. Signed-off-by: Kent Overstreet <kent.overstreet@gmail.com>
This commit is contained in:
Kent Overstreet
Kent Overstreet
parent
02afcb8c26
commit
cd5afabea1
@ -13,6 +13,7 @@ bcachefs-y := \
|
|||||||
btree_io.o \
|
btree_io.o \
|
||||||
btree_iter.o \
|
btree_iter.o \
|
||||||
btree_key_cache.o \
|
btree_key_cache.o \
|
||||||
|
btree_locking.o \
|
||||||
btree_update_interior.o \
|
btree_update_interior.o \
|
||||||
btree_update_leaf.o \
|
btree_update_leaf.o \
|
||||||
buckets.o \
|
buckets.o \
|
||||||
|
|||||||
@ -118,444 +118,6 @@ static inline bool btree_path_pos_in_node(struct btree_path *path,
|
|||||||
!btree_path_pos_after_node(path, b);
|
!btree_path_pos_after_node(path, b);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Btree node locking: */
|
|
||||||
|
|
||||||
void bch2_btree_node_unlock_write(struct btree_trans *trans,
|
|
||||||
struct btree_path *path, struct btree *b)
|
|
||||||
{
|
|
||||||
bch2_btree_node_unlock_write_inlined(trans, path, b);
|
|
||||||
}
|
|
||||||
|
|
||||||
struct six_lock_count bch2_btree_node_lock_counts(struct btree_trans *trans,
|
|
||||||
struct btree_path *skip,
|
|
||||||
struct btree *b,
|
|
||||||
unsigned level)
|
|
||||||
{
|
|
||||||
struct btree_path *path;
|
|
||||||
struct six_lock_count ret = { 0, 0 };
|
|
||||||
|
|
||||||
if (IS_ERR_OR_NULL(b))
|
|
||||||
return ret;
|
|
||||||
|
|
||||||
trans_for_each_path(trans, path)
|
|
||||||
if (path != skip && path->l[level].b == b) {
|
|
||||||
ret.read += btree_node_read_locked(path, level);
|
|
||||||
ret.intent += btree_node_intent_locked(path, level);
|
|
||||||
}
|
|
||||||
|
|
||||||
return ret;
|
|
||||||
}
|
|
||||||
|
|
||||||
static inline void six_lock_readers_add(struct six_lock *lock, int nr)
|
|
||||||
{
|
|
||||||
if (!lock->readers)
|
|
||||||
atomic64_add(__SIX_VAL(read_lock, nr), &lock->state.counter);
|
|
||||||
else
|
|
||||||
this_cpu_add(*lock->readers, nr);
|
|
||||||
}
|
|
||||||
|
|
||||||
void __bch2_btree_node_lock_write(struct btree_trans *trans, struct btree *b)
|
|
||||||
{
|
|
||||||
int readers = bch2_btree_node_lock_counts(trans, NULL, b, b->c.level).read;
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Must drop our read locks before calling six_lock_write() -
|
|
||||||
* six_unlock() won't do wakeups until the reader count
|
|
||||||
* goes to 0, and it's safe because we have the node intent
|
|
||||||
* locked:
|
|
||||||
*/
|
|
||||||
six_lock_readers_add(&b->c.lock, -readers);
|
|
||||||
six_lock_write(&b->c.lock, NULL, NULL);
|
|
||||||
six_lock_readers_add(&b->c.lock, readers);
|
|
||||||
}
|
|
||||||
|
|
||||||
bool __bch2_btree_node_relock(struct btree_trans *trans,
|
|
||||||
struct btree_path *path, unsigned level)
|
|
||||||
{
|
|
||||||
struct btree *b = btree_path_node(path, level);
|
|
||||||
int want = __btree_lock_want(path, level);
|
|
||||||
|
|
||||||
if (!is_btree_node(path, level))
|
|
||||||
goto fail;
|
|
||||||
|
|
||||||
if (race_fault())
|
|
||||||
goto fail;
|
|
||||||
|
|
||||||
if (six_relock_type(&b->c.lock, want, path->l[level].lock_seq) ||
|
|
||||||
(btree_node_lock_seq_matches(path, b, level) &&
|
|
||||||
btree_node_lock_increment(trans, b, level, want))) {
|
|
||||||
mark_btree_node_locked(trans, path, level, want);
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
fail:
|
|
||||||
if (b != ERR_PTR(-BCH_ERR_no_btree_node_cached) &&
|
|
||||||
b != ERR_PTR(-BCH_ERR_no_btree_node_init))
|
|
||||||
trace_btree_node_relock_fail(trans, _RET_IP_, path, level);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
bool bch2_btree_node_upgrade(struct btree_trans *trans,
|
|
||||||
struct btree_path *path, unsigned level)
|
|
||||||
{
|
|
||||||
struct btree *b = path->l[level].b;
|
|
||||||
|
|
||||||
if (!is_btree_node(path, level))
|
|
||||||
return false;
|
|
||||||
|
|
||||||
switch (btree_lock_want(path, level)) {
|
|
||||||
case BTREE_NODE_UNLOCKED:
|
|
||||||
BUG_ON(btree_node_locked(path, level));
|
|
||||||
return true;
|
|
||||||
case BTREE_NODE_READ_LOCKED:
|
|
||||||
BUG_ON(btree_node_intent_locked(path, level));
|
|
||||||
return bch2_btree_node_relock(trans, path, level);
|
|
||||||
case BTREE_NODE_INTENT_LOCKED:
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (btree_node_intent_locked(path, level))
|
|
||||||
return true;
|
|
||||||
|
|
||||||
if (race_fault())
|
|
||||||
return false;
|
|
||||||
|
|
||||||
if (btree_node_locked(path, level)
|
|
||||||
? six_lock_tryupgrade(&b->c.lock)
|
|
||||||
: six_relock_type(&b->c.lock, SIX_LOCK_intent, path->l[level].lock_seq))
|
|
||||||
goto success;
|
|
||||||
|
|
||||||
if (btree_node_lock_seq_matches(path, b, level) &&
|
|
||||||
btree_node_lock_increment(trans, b, level, BTREE_NODE_INTENT_LOCKED)) {
|
|
||||||
btree_node_unlock(trans, path, level);
|
|
||||||
goto success;
|
|
||||||
}
|
|
||||||
|
|
||||||
trace_btree_node_upgrade_fail(trans, _RET_IP_, path, level);
|
|
||||||
return false;
|
|
||||||
success:
|
|
||||||
mark_btree_node_intent_locked(trans, path, level);
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
static inline bool btree_path_get_locks(struct btree_trans *trans,
|
|
||||||
struct btree_path *path,
|
|
||||||
bool upgrade)
|
|
||||||
{
|
|
||||||
unsigned l = path->level;
|
|
||||||
int fail_idx = -1;
|
|
||||||
|
|
||||||
do {
|
|
||||||
if (!btree_path_node(path, l))
|
|
||||||
break;
|
|
||||||
|
|
||||||
if (!(upgrade
|
|
||||||
? bch2_btree_node_upgrade(trans, path, l)
|
|
||||||
: bch2_btree_node_relock(trans, path, l)))
|
|
||||||
fail_idx = l;
|
|
||||||
|
|
||||||
l++;
|
|
||||||
} while (l < path->locks_want);
|
|
||||||
|
|
||||||
/*
|
|
||||||
* When we fail to get a lock, we have to ensure that any child nodes
|
|
||||||
* can't be relocked so bch2_btree_path_traverse has to walk back up to
|
|
||||||
* the node that we failed to relock:
|
|
||||||
*/
|
|
||||||
if (fail_idx >= 0) {
|
|
||||||
__bch2_btree_path_unlock(trans, path);
|
|
||||||
btree_path_set_dirty(path, BTREE_ITER_NEED_TRAVERSE);
|
|
||||||
|
|
||||||
do {
|
|
||||||
path->l[fail_idx].b = upgrade
|
|
||||||
? ERR_PTR(-BCH_ERR_no_btree_node_upgrade)
|
|
||||||
: ERR_PTR(-BCH_ERR_no_btree_node_relock);
|
|
||||||
--fail_idx;
|
|
||||||
} while (fail_idx >= 0);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (path->uptodate == BTREE_ITER_NEED_RELOCK)
|
|
||||||
path->uptodate = BTREE_ITER_UPTODATE;
|
|
||||||
|
|
||||||
bch2_trans_verify_locks(trans);
|
|
||||||
|
|
||||||
return path->uptodate < BTREE_ITER_NEED_RELOCK;
|
|
||||||
}
|
|
||||||
|
|
||||||
static struct bpos btree_node_pos(struct btree_bkey_cached_common *_b,
|
|
||||||
bool cached)
|
|
||||||
{
|
|
||||||
return !cached
|
|
||||||
? container_of(_b, struct btree, c)->key.k.p
|
|
||||||
: container_of(_b, struct bkey_cached, c)->key.pos;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Slowpath: */
|
|
||||||
int __bch2_btree_node_lock(struct btree_trans *trans,
|
|
||||||
struct btree_path *path,
|
|
||||||
struct btree *b,
|
|
||||||
struct bpos pos, unsigned level,
|
|
||||||
enum six_lock_type type,
|
|
||||||
six_lock_should_sleep_fn should_sleep_fn, void *p,
|
|
||||||
unsigned long ip)
|
|
||||||
{
|
|
||||||
struct btree_path *linked;
|
|
||||||
unsigned reason;
|
|
||||||
|
|
||||||
/* Check if it's safe to block: */
|
|
||||||
trans_for_each_path(trans, linked) {
|
|
||||||
if (!linked->nodes_locked)
|
|
||||||
continue;
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Can't block taking an intent lock if we have _any_ nodes read
|
|
||||||
* locked:
|
|
||||||
*
|
|
||||||
* - Our read lock blocks another thread with an intent lock on
|
|
||||||
* the same node from getting a write lock, and thus from
|
|
||||||
* dropping its intent lock
|
|
||||||
*
|
|
||||||
* - And the other thread may have multiple nodes intent locked:
|
|
||||||
* both the node we want to intent lock, and the node we
|
|
||||||
* already have read locked - deadlock:
|
|
||||||
*/
|
|
||||||
if (type == SIX_LOCK_intent &&
|
|
||||||
linked->nodes_locked != linked->nodes_intent_locked) {
|
|
||||||
reason = 1;
|
|
||||||
goto deadlock;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (linked->btree_id != path->btree_id) {
|
|
||||||
if (linked->btree_id < path->btree_id)
|
|
||||||
continue;
|
|
||||||
|
|
||||||
reason = 3;
|
|
||||||
goto deadlock;
|
|
||||||
}
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Within the same btree, non-cached paths come before cached
|
|
||||||
* paths:
|
|
||||||
*/
|
|
||||||
if (linked->cached != path->cached) {
|
|
||||||
if (!linked->cached)
|
|
||||||
continue;
|
|
||||||
|
|
||||||
reason = 4;
|
|
||||||
goto deadlock;
|
|
||||||
}
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Interior nodes must be locked before their descendants: if
|
|
||||||
* another path has possible descendants locked of the node
|
|
||||||
* we're about to lock, it must have the ancestors locked too:
|
|
||||||
*/
|
|
||||||
if (level > __fls(linked->nodes_locked)) {
|
|
||||||
reason = 5;
|
|
||||||
goto deadlock;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Must lock btree nodes in key order: */
|
|
||||||
if (btree_node_locked(linked, level) &&
|
|
||||||
bpos_cmp(pos, btree_node_pos((void *) linked->l[level].b,
|
|
||||||
linked->cached)) <= 0) {
|
|
||||||
reason = 7;
|
|
||||||
goto deadlock;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return btree_node_lock_type(trans, path, b, pos, level,
|
|
||||||
type, should_sleep_fn, p);
|
|
||||||
deadlock:
|
|
||||||
trace_trans_restart_would_deadlock(trans, ip, reason, linked, path, &pos);
|
|
||||||
return btree_trans_restart(trans, BCH_ERR_transaction_restart_would_deadlock);
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Btree iterator locking: */
|
|
||||||
|
|
||||||
#ifdef CONFIG_BCACHEFS_DEBUG
|
|
||||||
|
|
||||||
static void bch2_btree_path_verify_locks(struct btree_path *path)
|
|
||||||
{
|
|
||||||
unsigned l;
|
|
||||||
|
|
||||||
if (!path->nodes_locked) {
|
|
||||||
BUG_ON(path->uptodate == BTREE_ITER_UPTODATE &&
|
|
||||||
btree_path_node(path, path->level));
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
for (l = 0; btree_path_node(path, l); l++)
|
|
||||||
BUG_ON(btree_lock_want(path, l) !=
|
|
||||||
btree_node_locked_type(path, l));
|
|
||||||
}
|
|
||||||
|
|
||||||
void bch2_trans_verify_locks(struct btree_trans *trans)
|
|
||||||
{
|
|
||||||
struct btree_path *path;
|
|
||||||
|
|
||||||
trans_for_each_path(trans, path)
|
|
||||||
bch2_btree_path_verify_locks(path);
|
|
||||||
}
|
|
||||||
#else
|
|
||||||
static inline void bch2_btree_path_verify_locks(struct btree_path *path) {}
|
|
||||||
#endif
|
|
||||||
|
|
||||||
/* Btree path locking: */
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Only for btree_cache.c - only relocks intent locks
|
|
||||||
*/
|
|
||||||
int bch2_btree_path_relock_intent(struct btree_trans *trans,
|
|
||||||
struct btree_path *path)
|
|
||||||
{
|
|
||||||
unsigned l;
|
|
||||||
|
|
||||||
for (l = path->level;
|
|
||||||
l < path->locks_want && btree_path_node(path, l);
|
|
||||||
l++) {
|
|
||||||
if (!bch2_btree_node_relock(trans, path, l)) {
|
|
||||||
__bch2_btree_path_unlock(trans, path);
|
|
||||||
btree_path_set_dirty(path, BTREE_ITER_NEED_TRAVERSE);
|
|
||||||
trace_trans_restart_relock_path_intent(trans, _RET_IP_, path);
|
|
||||||
return btree_trans_restart(trans, BCH_ERR_transaction_restart_relock_path_intent);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
__flatten
|
|
||||||
static bool bch2_btree_path_relock_norestart(struct btree_trans *trans,
|
|
||||||
struct btree_path *path, unsigned long trace_ip)
|
|
||||||
{
|
|
||||||
return btree_path_get_locks(trans, path, false);
|
|
||||||
}
|
|
||||||
|
|
||||||
static int __bch2_btree_path_relock(struct btree_trans *trans,
|
|
||||||
struct btree_path *path, unsigned long trace_ip)
|
|
||||||
{
|
|
||||||
if (!bch2_btree_path_relock_norestart(trans, path, trace_ip)) {
|
|
||||||
trace_trans_restart_relock_path(trans, trace_ip, path);
|
|
||||||
return btree_trans_restart(trans, BCH_ERR_transaction_restart_relock_path);
|
|
||||||
}
|
|
||||||
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
static inline int bch2_btree_path_relock(struct btree_trans *trans,
|
|
||||||
struct btree_path *path, unsigned long trace_ip)
|
|
||||||
{
|
|
||||||
return btree_node_locked(path, path->level)
|
|
||||||
? 0
|
|
||||||
: __bch2_btree_path_relock(trans, path, trace_ip);
|
|
||||||
}
|
|
||||||
|
|
||||||
bool __bch2_btree_path_upgrade(struct btree_trans *trans,
|
|
||||||
struct btree_path *path,
|
|
||||||
unsigned new_locks_want)
|
|
||||||
{
|
|
||||||
struct btree_path *linked;
|
|
||||||
|
|
||||||
EBUG_ON(path->locks_want >= new_locks_want);
|
|
||||||
|
|
||||||
path->locks_want = new_locks_want;
|
|
||||||
|
|
||||||
if (btree_path_get_locks(trans, path, true))
|
|
||||||
return true;
|
|
||||||
|
|
||||||
/*
|
|
||||||
* XXX: this is ugly - we'd prefer to not be mucking with other
|
|
||||||
* iterators in the btree_trans here.
|
|
||||||
*
|
|
||||||
* On failure to upgrade the iterator, setting iter->locks_want and
|
|
||||||
* calling get_locks() is sufficient to make bch2_btree_path_traverse()
|
|
||||||
* get the locks we want on transaction restart.
|
|
||||||
*
|
|
||||||
* But if this iterator was a clone, on transaction restart what we did
|
|
||||||
* to this iterator isn't going to be preserved.
|
|
||||||
*
|
|
||||||
* Possibly we could add an iterator field for the parent iterator when
|
|
||||||
* an iterator is a copy - for now, we'll just upgrade any other
|
|
||||||
* iterators with the same btree id.
|
|
||||||
*
|
|
||||||
* The code below used to be needed to ensure ancestor nodes get locked
|
|
||||||
* before interior nodes - now that's handled by
|
|
||||||
* bch2_btree_path_traverse_all().
|
|
||||||
*/
|
|
||||||
if (!path->cached && !trans->in_traverse_all)
|
|
||||||
trans_for_each_path(trans, linked)
|
|
||||||
if (linked != path &&
|
|
||||||
linked->cached == path->cached &&
|
|
||||||
linked->btree_id == path->btree_id &&
|
|
||||||
linked->locks_want < new_locks_want) {
|
|
||||||
linked->locks_want = new_locks_want;
|
|
||||||
btree_path_get_locks(trans, linked, true);
|
|
||||||
}
|
|
||||||
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
void __bch2_btree_path_downgrade(struct btree_trans *trans,
|
|
||||||
struct btree_path *path,
|
|
||||||
unsigned new_locks_want)
|
|
||||||
{
|
|
||||||
unsigned l;
|
|
||||||
|
|
||||||
EBUG_ON(path->locks_want < new_locks_want);
|
|
||||||
|
|
||||||
path->locks_want = new_locks_want;
|
|
||||||
|
|
||||||
while (path->nodes_locked &&
|
|
||||||
(l = __fls(path->nodes_locked)) >= path->locks_want) {
|
|
||||||
if (l > path->level) {
|
|
||||||
btree_node_unlock(trans, path, l);
|
|
||||||
} else {
|
|
||||||
if (btree_node_intent_locked(path, l)) {
|
|
||||||
six_lock_downgrade(&path->l[l].b->c.lock);
|
|
||||||
path->nodes_intent_locked ^= 1 << l;
|
|
||||||
}
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
bch2_btree_path_verify_locks(path);
|
|
||||||
}
|
|
||||||
|
|
||||||
void bch2_trans_downgrade(struct btree_trans *trans)
|
|
||||||
{
|
|
||||||
struct btree_path *path;
|
|
||||||
|
|
||||||
trans_for_each_path(trans, path)
|
|
||||||
bch2_btree_path_downgrade(trans, path);
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Btree transaction locking: */
|
|
||||||
|
|
||||||
int bch2_trans_relock(struct btree_trans *trans)
|
|
||||||
{
|
|
||||||
struct btree_path *path;
|
|
||||||
|
|
||||||
if (unlikely(trans->restarted))
|
|
||||||
return -BCH_ERR_transaction_restart_relock;
|
|
||||||
|
|
||||||
trans_for_each_path(trans, path)
|
|
||||||
if (path->should_be_locked &&
|
|
||||||
bch2_btree_path_relock(trans, path, _RET_IP_)) {
|
|
||||||
trace_trans_restart_relock(trans, _RET_IP_, path);
|
|
||||||
BUG_ON(!trans->restarted);
|
|
||||||
return -BCH_ERR_transaction_restart_relock;
|
|
||||||
}
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
void bch2_trans_unlock(struct btree_trans *trans)
|
|
||||||
{
|
|
||||||
struct btree_path *path;
|
|
||||||
|
|
||||||
trans_for_each_path(trans, path)
|
|
||||||
__bch2_btree_path_unlock(trans, path);
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Btree iterator: */
|
/* Btree iterator: */
|
||||||
|
|
||||||
#ifdef CONFIG_BCACHEFS_DEBUG
|
#ifdef CONFIG_BCACHEFS_DEBUG
|
||||||
@ -2036,10 +1598,8 @@ struct btree_path *bch2_path_get(struct btree_trans *trans,
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
locks_want = min(locks_want, BTREE_MAX_DEPTH);
|
locks_want = min(locks_want, BTREE_MAX_DEPTH);
|
||||||
if (locks_want > path->locks_want) {
|
if (locks_want > path->locks_want)
|
||||||
path->locks_want = locks_want;
|
bch2_btree_path_upgrade_noupgrade_sibs(trans, path, locks_want);
|
||||||
btree_path_get_locks(trans, path, true);
|
|
||||||
}
|
|
||||||
|
|
||||||
return path;
|
return path;
|
||||||
}
|
}
|
||||||
|
|||||||
@ -181,12 +181,10 @@ struct bkey_i *bch2_btree_journal_peek_slot(struct btree_trans *,
|
|||||||
|
|
||||||
#ifdef CONFIG_BCACHEFS_DEBUG
|
#ifdef CONFIG_BCACHEFS_DEBUG
|
||||||
void bch2_trans_verify_paths(struct btree_trans *);
|
void bch2_trans_verify_paths(struct btree_trans *);
|
||||||
void bch2_trans_verify_locks(struct btree_trans *);
|
|
||||||
void bch2_assert_pos_locked(struct btree_trans *, enum btree_id,
|
void bch2_assert_pos_locked(struct btree_trans *, enum btree_id,
|
||||||
struct bpos, bool);
|
struct bpos, bool);
|
||||||
#else
|
#else
|
||||||
static inline void bch2_trans_verify_paths(struct btree_trans *trans) {}
|
static inline void bch2_trans_verify_paths(struct btree_trans *trans) {}
|
||||||
static inline void bch2_trans_verify_locks(struct btree_trans *trans) {}
|
|
||||||
static inline void bch2_assert_pos_locked(struct btree_trans *trans, enum btree_id id,
|
static inline void bch2_assert_pos_locked(struct btree_trans *trans, enum btree_id id,
|
||||||
struct bpos pos, bool key_cache) {}
|
struct bpos pos, bool key_cache) {}
|
||||||
#endif
|
#endif
|
||||||
@ -231,20 +229,6 @@ static inline int btree_trans_restart(struct btree_trans *trans, int err)
|
|||||||
bool bch2_btree_node_upgrade(struct btree_trans *,
|
bool bch2_btree_node_upgrade(struct btree_trans *,
|
||||||
struct btree_path *, unsigned);
|
struct btree_path *, unsigned);
|
||||||
|
|
||||||
bool __bch2_btree_path_upgrade(struct btree_trans *,
|
|
||||||
struct btree_path *, unsigned);
|
|
||||||
|
|
||||||
static inline bool bch2_btree_path_upgrade(struct btree_trans *trans,
|
|
||||||
struct btree_path *path,
|
|
||||||
unsigned new_locks_want)
|
|
||||||
{
|
|
||||||
new_locks_want = min(new_locks_want, BTREE_MAX_DEPTH);
|
|
||||||
|
|
||||||
return path->locks_want < new_locks_want
|
|
||||||
? __bch2_btree_path_upgrade(trans, path, new_locks_want)
|
|
||||||
: path->uptodate == BTREE_ITER_UPTODATE;
|
|
||||||
}
|
|
||||||
|
|
||||||
void __bch2_btree_path_downgrade(struct btree_trans *, struct btree_path *, unsigned);
|
void __bch2_btree_path_downgrade(struct btree_trans *, struct btree_path *, unsigned);
|
||||||
|
|
||||||
static inline void bch2_btree_path_downgrade(struct btree_trans *trans,
|
static inline void bch2_btree_path_downgrade(struct btree_trans *trans,
|
||||||
|
|||||||
442
fs/bcachefs/btree_locking.c
Normal file
442
fs/bcachefs/btree_locking.c
Normal file
@ -0,0 +1,442 @@
|
|||||||
|
// SPDX-License-Identifier: GPL-2.0
|
||||||
|
|
||||||
|
#include "bcachefs.h"
|
||||||
|
#include "btree_locking.h"
|
||||||
|
#include "btree_types.h"
|
||||||
|
|
||||||
|
struct lock_class_key bch2_btree_node_lock_key;
|
||||||
|
|
||||||
|
/* Btree node locking: */
|
||||||
|
|
||||||
|
void bch2_btree_node_unlock_write(struct btree_trans *trans,
|
||||||
|
struct btree_path *path, struct btree *b)
|
||||||
|
{
|
||||||
|
bch2_btree_node_unlock_write_inlined(trans, path, b);
|
||||||
|
}
|
||||||
|
|
||||||
|
struct six_lock_count bch2_btree_node_lock_counts(struct btree_trans *trans,
|
||||||
|
struct btree_path *skip,
|
||||||
|
struct btree *b,
|
||||||
|
unsigned level)
|
||||||
|
{
|
||||||
|
struct btree_path *path;
|
||||||
|
struct six_lock_count ret = { 0, 0 };
|
||||||
|
|
||||||
|
if (IS_ERR_OR_NULL(b))
|
||||||
|
return ret;
|
||||||
|
|
||||||
|
trans_for_each_path(trans, path)
|
||||||
|
if (path != skip && path->l[level].b == b) {
|
||||||
|
ret.read += btree_node_read_locked(path, level);
|
||||||
|
ret.intent += btree_node_intent_locked(path, level);
|
||||||
|
}
|
||||||
|
|
||||||
|
return ret;
|
||||||
|
}
|
||||||
|
|
||||||
|
static inline void six_lock_readers_add(struct six_lock *lock, int nr)
|
||||||
|
{
|
||||||
|
if (!lock->readers)
|
||||||
|
atomic64_add(__SIX_VAL(read_lock, nr), &lock->state.counter);
|
||||||
|
else
|
||||||
|
this_cpu_add(*lock->readers, nr);
|
||||||
|
}
|
||||||
|
|
||||||
|
void __bch2_btree_node_lock_write(struct btree_trans *trans, struct btree *b)
|
||||||
|
{
|
||||||
|
int readers = bch2_btree_node_lock_counts(trans, NULL, b, b->c.level).read;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Must drop our read locks before calling six_lock_write() -
|
||||||
|
* six_unlock() won't do wakeups until the reader count
|
||||||
|
* goes to 0, and it's safe because we have the node intent
|
||||||
|
* locked:
|
||||||
|
*/
|
||||||
|
six_lock_readers_add(&b->c.lock, -readers);
|
||||||
|
six_lock_write(&b->c.lock, NULL, NULL);
|
||||||
|
six_lock_readers_add(&b->c.lock, readers);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool __bch2_btree_node_relock(struct btree_trans *trans,
|
||||||
|
struct btree_path *path, unsigned level)
|
||||||
|
{
|
||||||
|
struct btree *b = btree_path_node(path, level);
|
||||||
|
int want = __btree_lock_want(path, level);
|
||||||
|
|
||||||
|
if (!is_btree_node(path, level))
|
||||||
|
goto fail;
|
||||||
|
|
||||||
|
if (race_fault())
|
||||||
|
goto fail;
|
||||||
|
|
||||||
|
if (six_relock_type(&b->c.lock, want, path->l[level].lock_seq) ||
|
||||||
|
(btree_node_lock_seq_matches(path, b, level) &&
|
||||||
|
btree_node_lock_increment(trans, b, level, want))) {
|
||||||
|
mark_btree_node_locked(trans, path, level, want);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
fail:
|
||||||
|
if (b != ERR_PTR(-BCH_ERR_no_btree_node_cached) &&
|
||||||
|
b != ERR_PTR(-BCH_ERR_no_btree_node_init))
|
||||||
|
trace_btree_node_relock_fail(trans, _RET_IP_, path, level);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool bch2_btree_node_upgrade(struct btree_trans *trans,
|
||||||
|
struct btree_path *path, unsigned level)
|
||||||
|
{
|
||||||
|
struct btree *b = path->l[level].b;
|
||||||
|
|
||||||
|
if (!is_btree_node(path, level))
|
||||||
|
return false;
|
||||||
|
|
||||||
|
switch (btree_lock_want(path, level)) {
|
||||||
|
case BTREE_NODE_UNLOCKED:
|
||||||
|
BUG_ON(btree_node_locked(path, level));
|
||||||
|
return true;
|
||||||
|
case BTREE_NODE_READ_LOCKED:
|
||||||
|
BUG_ON(btree_node_intent_locked(path, level));
|
||||||
|
return bch2_btree_node_relock(trans, path, level);
|
||||||
|
case BTREE_NODE_INTENT_LOCKED:
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (btree_node_intent_locked(path, level))
|
||||||
|
return true;
|
||||||
|
|
||||||
|
if (race_fault())
|
||||||
|
return false;
|
||||||
|
|
||||||
|
if (btree_node_locked(path, level)
|
||||||
|
? six_lock_tryupgrade(&b->c.lock)
|
||||||
|
: six_relock_type(&b->c.lock, SIX_LOCK_intent, path->l[level].lock_seq))
|
||||||
|
goto success;
|
||||||
|
|
||||||
|
if (btree_node_lock_seq_matches(path, b, level) &&
|
||||||
|
btree_node_lock_increment(trans, b, level, BTREE_NODE_INTENT_LOCKED)) {
|
||||||
|
btree_node_unlock(trans, path, level);
|
||||||
|
goto success;
|
||||||
|
}
|
||||||
|
|
||||||
|
trace_btree_node_upgrade_fail(trans, _RET_IP_, path, level);
|
||||||
|
return false;
|
||||||
|
success:
|
||||||
|
mark_btree_node_intent_locked(trans, path, level);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
static inline bool btree_path_get_locks(struct btree_trans *trans,
|
||||||
|
struct btree_path *path,
|
||||||
|
bool upgrade)
|
||||||
|
{
|
||||||
|
unsigned l = path->level;
|
||||||
|
int fail_idx = -1;
|
||||||
|
|
||||||
|
do {
|
||||||
|
if (!btree_path_node(path, l))
|
||||||
|
break;
|
||||||
|
|
||||||
|
if (!(upgrade
|
||||||
|
? bch2_btree_node_upgrade(trans, path, l)
|
||||||
|
: bch2_btree_node_relock(trans, path, l)))
|
||||||
|
fail_idx = l;
|
||||||
|
|
||||||
|
l++;
|
||||||
|
} while (l < path->locks_want);
|
||||||
|
|
||||||
|
/*
|
||||||
|
* When we fail to get a lock, we have to ensure that any child nodes
|
||||||
|
* can't be relocked so bch2_btree_path_traverse has to walk back up to
|
||||||
|
* the node that we failed to relock:
|
||||||
|
*/
|
||||||
|
if (fail_idx >= 0) {
|
||||||
|
__bch2_btree_path_unlock(trans, path);
|
||||||
|
btree_path_set_dirty(path, BTREE_ITER_NEED_TRAVERSE);
|
||||||
|
|
||||||
|
do {
|
||||||
|
path->l[fail_idx].b = upgrade
|
||||||
|
? ERR_PTR(-BCH_ERR_no_btree_node_upgrade)
|
||||||
|
: ERR_PTR(-BCH_ERR_no_btree_node_relock);
|
||||||
|
--fail_idx;
|
||||||
|
} while (fail_idx >= 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (path->uptodate == BTREE_ITER_NEED_RELOCK)
|
||||||
|
path->uptodate = BTREE_ITER_UPTODATE;
|
||||||
|
|
||||||
|
bch2_trans_verify_locks(trans);
|
||||||
|
|
||||||
|
return path->uptodate < BTREE_ITER_NEED_RELOCK;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Slowpath: */
|
||||||
|
int __bch2_btree_node_lock(struct btree_trans *trans,
|
||||||
|
struct btree_path *path,
|
||||||
|
struct btree *b,
|
||||||
|
struct bpos pos, unsigned level,
|
||||||
|
enum six_lock_type type,
|
||||||
|
six_lock_should_sleep_fn should_sleep_fn, void *p,
|
||||||
|
unsigned long ip)
|
||||||
|
{
|
||||||
|
struct btree_path *linked;
|
||||||
|
unsigned reason;
|
||||||
|
|
||||||
|
/* Check if it's safe to block: */
|
||||||
|
trans_for_each_path(trans, linked) {
|
||||||
|
if (!linked->nodes_locked)
|
||||||
|
continue;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Can't block taking an intent lock if we have _any_ nodes read
|
||||||
|
* locked:
|
||||||
|
*
|
||||||
|
* - Our read lock blocks another thread with an intent lock on
|
||||||
|
* the same node from getting a write lock, and thus from
|
||||||
|
* dropping its intent lock
|
||||||
|
*
|
||||||
|
* - And the other thread may have multiple nodes intent locked:
|
||||||
|
* both the node we want to intent lock, and the node we
|
||||||
|
* already have read locked - deadlock:
|
||||||
|
*/
|
||||||
|
if (type == SIX_LOCK_intent &&
|
||||||
|
linked->nodes_locked != linked->nodes_intent_locked) {
|
||||||
|
reason = 1;
|
||||||
|
goto deadlock;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (linked->btree_id != path->btree_id) {
|
||||||
|
if (linked->btree_id < path->btree_id)
|
||||||
|
continue;
|
||||||
|
|
||||||
|
reason = 3;
|
||||||
|
goto deadlock;
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Within the same btree, non-cached paths come before cached
|
||||||
|
* paths:
|
||||||
|
*/
|
||||||
|
if (linked->cached != path->cached) {
|
||||||
|
if (!linked->cached)
|
||||||
|
continue;
|
||||||
|
|
||||||
|
reason = 4;
|
||||||
|
goto deadlock;
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Interior nodes must be locked before their descendants: if
|
||||||
|
* another path has possible descendants locked of the node
|
||||||
|
* we're about to lock, it must have the ancestors locked too:
|
||||||
|
*/
|
||||||
|
if (level > __fls(linked->nodes_locked)) {
|
||||||
|
reason = 5;
|
||||||
|
goto deadlock;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Must lock btree nodes in key order: */
|
||||||
|
if (btree_node_locked(linked, level) &&
|
||||||
|
bpos_cmp(pos, btree_node_pos((void *) linked->l[level].b,
|
||||||
|
linked->cached)) <= 0) {
|
||||||
|
reason = 7;
|
||||||
|
goto deadlock;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return btree_node_lock_type(trans, path, b, pos, level,
|
||||||
|
type, should_sleep_fn, p);
|
||||||
|
deadlock:
|
||||||
|
trace_trans_restart_would_deadlock(trans, ip, reason, linked, path, &pos);
|
||||||
|
return btree_trans_restart(trans, BCH_ERR_transaction_restart_would_deadlock);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Btree iterator locking: */
|
||||||
|
|
||||||
|
#ifdef CONFIG_BCACHEFS_DEBUG
|
||||||
|
|
||||||
|
void bch2_btree_path_verify_locks(struct btree_path *path)
|
||||||
|
{
|
||||||
|
unsigned l;
|
||||||
|
|
||||||
|
if (!path->nodes_locked) {
|
||||||
|
BUG_ON(path->uptodate == BTREE_ITER_UPTODATE &&
|
||||||
|
btree_path_node(path, path->level));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
for (l = 0; btree_path_node(path, l); l++)
|
||||||
|
BUG_ON(btree_lock_want(path, l) !=
|
||||||
|
btree_node_locked_type(path, l));
|
||||||
|
}
|
||||||
|
|
||||||
|
void bch2_trans_verify_locks(struct btree_trans *trans)
|
||||||
|
{
|
||||||
|
struct btree_path *path;
|
||||||
|
|
||||||
|
trans_for_each_path(trans, path)
|
||||||
|
bch2_btree_path_verify_locks(path);
|
||||||
|
}
|
||||||
|
|
||||||
|
#endif
|
||||||
|
|
||||||
|
/* Btree path locking: */
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Only for btree_cache.c - only relocks intent locks
|
||||||
|
*/
|
||||||
|
int bch2_btree_path_relock_intent(struct btree_trans *trans,
|
||||||
|
struct btree_path *path)
|
||||||
|
{
|
||||||
|
unsigned l;
|
||||||
|
|
||||||
|
for (l = path->level;
|
||||||
|
l < path->locks_want && btree_path_node(path, l);
|
||||||
|
l++) {
|
||||||
|
if (!bch2_btree_node_relock(trans, path, l)) {
|
||||||
|
__bch2_btree_path_unlock(trans, path);
|
||||||
|
btree_path_set_dirty(path, BTREE_ITER_NEED_TRAVERSE);
|
||||||
|
trace_trans_restart_relock_path_intent(trans, _RET_IP_, path);
|
||||||
|
return btree_trans_restart(trans, BCH_ERR_transaction_restart_relock_path_intent);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
__flatten
|
||||||
|
bool bch2_btree_path_relock_norestart(struct btree_trans *trans,
|
||||||
|
struct btree_path *path, unsigned long trace_ip)
|
||||||
|
{
|
||||||
|
return btree_path_get_locks(trans, path, false);
|
||||||
|
}
|
||||||
|
|
||||||
|
int __bch2_btree_path_relock(struct btree_trans *trans,
|
||||||
|
struct btree_path *path, unsigned long trace_ip)
|
||||||
|
{
|
||||||
|
if (!bch2_btree_path_relock_norestart(trans, path, trace_ip)) {
|
||||||
|
trace_trans_restart_relock_path(trans, trace_ip, path);
|
||||||
|
return btree_trans_restart(trans, BCH_ERR_transaction_restart_relock_path);
|
||||||
|
}
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
__flatten
|
||||||
|
bool bch2_btree_path_upgrade_norestart(struct btree_trans *trans,
|
||||||
|
struct btree_path *path, unsigned long trace_ip)
|
||||||
|
{
|
||||||
|
return btree_path_get_locks(trans, path, true);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool bch2_btree_path_upgrade_noupgrade_sibs(struct btree_trans *trans,
|
||||||
|
struct btree_path *path,
|
||||||
|
unsigned new_locks_want)
|
||||||
|
{
|
||||||
|
EBUG_ON(path->locks_want >= new_locks_want);
|
||||||
|
|
||||||
|
path->locks_want = new_locks_want;
|
||||||
|
|
||||||
|
return btree_path_get_locks(trans, path, true);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool __bch2_btree_path_upgrade(struct btree_trans *trans,
|
||||||
|
struct btree_path *path,
|
||||||
|
unsigned new_locks_want)
|
||||||
|
{
|
||||||
|
struct btree_path *linked;
|
||||||
|
|
||||||
|
if (bch2_btree_path_upgrade_noupgrade_sibs(trans, path, new_locks_want))
|
||||||
|
return true;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* XXX: this is ugly - we'd prefer to not be mucking with other
|
||||||
|
* iterators in the btree_trans here.
|
||||||
|
*
|
||||||
|
* On failure to upgrade the iterator, setting iter->locks_want and
|
||||||
|
* calling get_locks() is sufficient to make bch2_btree_path_traverse()
|
||||||
|
* get the locks we want on transaction restart.
|
||||||
|
*
|
||||||
|
* But if this iterator was a clone, on transaction restart what we did
|
||||||
|
* to this iterator isn't going to be preserved.
|
||||||
|
*
|
||||||
|
* Possibly we could add an iterator field for the parent iterator when
|
||||||
|
* an iterator is a copy - for now, we'll just upgrade any other
|
||||||
|
* iterators with the same btree id.
|
||||||
|
*
|
||||||
|
* The code below used to be needed to ensure ancestor nodes get locked
|
||||||
|
* before interior nodes - now that's handled by
|
||||||
|
* bch2_btree_path_traverse_all().
|
||||||
|
*/
|
||||||
|
if (!path->cached && !trans->in_traverse_all)
|
||||||
|
trans_for_each_path(trans, linked)
|
||||||
|
if (linked != path &&
|
||||||
|
linked->cached == path->cached &&
|
||||||
|
linked->btree_id == path->btree_id &&
|
||||||
|
linked->locks_want < new_locks_want) {
|
||||||
|
linked->locks_want = new_locks_want;
|
||||||
|
btree_path_get_locks(trans, linked, true);
|
||||||
|
}
|
||||||
|
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
void __bch2_btree_path_downgrade(struct btree_trans *trans,
|
||||||
|
struct btree_path *path,
|
||||||
|
unsigned new_locks_want)
|
||||||
|
{
|
||||||
|
unsigned l;
|
||||||
|
|
||||||
|
EBUG_ON(path->locks_want < new_locks_want);
|
||||||
|
|
||||||
|
path->locks_want = new_locks_want;
|
||||||
|
|
||||||
|
while (path->nodes_locked &&
|
||||||
|
(l = __fls(path->nodes_locked)) >= path->locks_want) {
|
||||||
|
if (l > path->level) {
|
||||||
|
btree_node_unlock(trans, path, l);
|
||||||
|
} else {
|
||||||
|
if (btree_node_intent_locked(path, l)) {
|
||||||
|
six_lock_downgrade(&path->l[l].b->c.lock);
|
||||||
|
path->nodes_intent_locked ^= 1 << l;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
bch2_btree_path_verify_locks(path);
|
||||||
|
}
|
||||||
|
|
||||||
|
void bch2_trans_downgrade(struct btree_trans *trans)
|
||||||
|
{
|
||||||
|
struct btree_path *path;
|
||||||
|
|
||||||
|
trans_for_each_path(trans, path)
|
||||||
|
bch2_btree_path_downgrade(trans, path);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Btree transaction locking: */
|
||||||
|
|
||||||
|
int bch2_trans_relock(struct btree_trans *trans)
|
||||||
|
{
|
||||||
|
struct btree_path *path;
|
||||||
|
|
||||||
|
if (unlikely(trans->restarted))
|
||||||
|
return -BCH_ERR_transaction_restart_relock;
|
||||||
|
|
||||||
|
trans_for_each_path(trans, path)
|
||||||
|
if (path->should_be_locked &&
|
||||||
|
bch2_btree_path_relock(trans, path, _RET_IP_)) {
|
||||||
|
trace_trans_restart_relock(trans, _RET_IP_, path);
|
||||||
|
BUG_ON(!trans->restarted);
|
||||||
|
return -BCH_ERR_transaction_restart_relock;
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
void bch2_trans_unlock(struct btree_trans *trans)
|
||||||
|
{
|
||||||
|
struct btree_path *path;
|
||||||
|
|
||||||
|
trans_for_each_path(trans, path)
|
||||||
|
__bch2_btree_path_unlock(trans, path);
|
||||||
|
}
|
||||||
@ -13,6 +13,8 @@
|
|||||||
#include "btree_iter.h"
|
#include "btree_iter.h"
|
||||||
#include "six.h"
|
#include "six.h"
|
||||||
|
|
||||||
|
extern struct lock_class_key bch2_btree_node_lock_key;
|
||||||
|
|
||||||
static inline bool is_btree_node(struct btree_path *path, unsigned l)
|
static inline bool is_btree_node(struct btree_path *path, unsigned l)
|
||||||
{
|
{
|
||||||
return l < BTREE_MAX_DEPTH && !IS_ERR_OR_NULL(path->l[l].b);
|
return l < BTREE_MAX_DEPTH && !IS_ERR_OR_NULL(path->l[l].b);
|
||||||
@ -300,6 +302,22 @@ static inline void bch2_btree_node_lock_write(struct btree_trans *trans,
|
|||||||
__bch2_btree_node_lock_write(trans, b);
|
__bch2_btree_node_lock_write(trans, b);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool bch2_btree_path_upgrade_noupgrade_sibs(struct btree_trans *,
|
||||||
|
struct btree_path *, unsigned);
|
||||||
|
bool __bch2_btree_path_upgrade(struct btree_trans *,
|
||||||
|
struct btree_path *, unsigned);
|
||||||
|
|
||||||
|
static inline bool bch2_btree_path_upgrade(struct btree_trans *trans,
|
||||||
|
struct btree_path *path,
|
||||||
|
unsigned new_locks_want)
|
||||||
|
{
|
||||||
|
new_locks_want = min(new_locks_want, BTREE_MAX_DEPTH);
|
||||||
|
|
||||||
|
return path->locks_want < new_locks_want
|
||||||
|
? __bch2_btree_path_upgrade(trans, path, new_locks_want)
|
||||||
|
: path->uptodate == BTREE_ITER_UPTODATE;
|
||||||
|
}
|
||||||
|
|
||||||
static inline void btree_path_set_should_be_locked(struct btree_path *path)
|
static inline void btree_path_set_should_be_locked(struct btree_path *path)
|
||||||
{
|
{
|
||||||
EBUG_ON(!btree_node_locked(path, path->level));
|
EBUG_ON(!btree_node_locked(path, path->level));
|
||||||
@ -326,4 +344,27 @@ static inline void btree_path_set_level_up(struct btree_trans *trans,
|
|||||||
struct six_lock_count bch2_btree_node_lock_counts(struct btree_trans *,
|
struct six_lock_count bch2_btree_node_lock_counts(struct btree_trans *,
|
||||||
struct btree_path *, struct btree *, unsigned);
|
struct btree_path *, struct btree *, unsigned);
|
||||||
|
|
||||||
|
bool bch2_btree_path_relock_norestart(struct btree_trans *,
|
||||||
|
struct btree_path *, unsigned long);
|
||||||
|
int __bch2_btree_path_relock(struct btree_trans *,
|
||||||
|
struct btree_path *, unsigned long);
|
||||||
|
|
||||||
|
static inline int bch2_btree_path_relock(struct btree_trans *trans,
|
||||||
|
struct btree_path *path, unsigned long trace_ip)
|
||||||
|
{
|
||||||
|
return btree_node_locked(path, path->level)
|
||||||
|
? 0
|
||||||
|
: __bch2_btree_path_relock(trans, path, trace_ip);
|
||||||
|
}
|
||||||
|
|
||||||
|
int bch2_btree_path_relock(struct btree_trans *, struct btree_path *, unsigned long);
|
||||||
|
|
||||||
|
#ifdef CONFIG_BCACHEFS_DEBUG
|
||||||
|
void bch2_btree_path_verify_locks(struct btree_path *);
|
||||||
|
void bch2_trans_verify_locks(struct btree_trans *);
|
||||||
|
#else
|
||||||
|
static inline void bch2_btree_path_verify_locks(struct btree_path *path) {}
|
||||||
|
static inline void bch2_trans_verify_locks(struct btree_trans *trans) {}
|
||||||
|
#endif
|
||||||
|
|
||||||
#endif /* _BCACHEFS_BTREE_LOCKING_H */
|
#endif /* _BCACHEFS_BTREE_LOCKING_H */
|
||||||
|
|||||||
@ -336,6 +336,14 @@ struct bkey_cached {
|
|||||||
struct bkey_i *k;
|
struct bkey_i *k;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
static inline struct bpos btree_node_pos(struct btree_bkey_cached_common *b,
|
||||||
|
bool cached)
|
||||||
|
{
|
||||||
|
return !cached
|
||||||
|
? container_of(b, struct btree, c)->key.k.p
|
||||||
|
: container_of(b, struct bkey_cached, c)->key.pos;
|
||||||
|
}
|
||||||
|
|
||||||
struct btree_insert_entry {
|
struct btree_insert_entry {
|
||||||
unsigned flags;
|
unsigned flags;
|
||||||
u8 bkey_type;
|
u8 bkey_type;
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user