Kernel/linux-stable
1
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git synced 2024-12-29 17:25:38 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

sbitmap: flush deferred clears for resize and shallow gets

Browse Source 
We're missing a deferred clear off the shallow get, which can cause
a hang. Additionally, when we resize the sbitmap, we should also
flush deferred clears for good measure.

Ensure we have full coverage on batch clears, even for paths where
we would not be doing deferred clear. This makes it less error
prone for future additions.

Reported-by: Bart Van Assche <bvanassche@acm.org>
Tested-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Jens Axboe 2018-12-11 18:39:41 -07:00
parent 2c4d5356e6
commit b2dbff1bb8
1 changed files with 51 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

94
lib/sbitmap.c
View File

@ -20,6 +20,47 @@
#include <linux/sbitmap.h>
#include <linux/seq_file.h>
/*
* See if we have deferred clears that we can batch move
*/
static inline bool sbitmap_deferred_clear(struct sbitmap *sb, int index)
{
unsigned long mask, val;
unsigned long __maybe_unused flags;
bool ret = false;
/* Silence bogus lockdep warning */
#if defined(CONFIG_LOCKDEP)
local_irq_save(flags);
#endif
spin_lock(&sb->map[index].swap_lock);
if (!sb->map[index].cleared)
goto out_unlock;
/*
* First get a stable cleared mask, setting the old mask to 0.
*/
do {
mask = sb->map[index].cleared;
} while (cmpxchg(&sb->map[index].cleared, mask, 0) != mask);
/*
* Now clear the masked bits in our free word
*/
do {
val = sb->map[index].word;
} while (cmpxchg(&sb->map[index].word, val, val & ~mask) != val);
ret = true;
out_unlock:
spin_unlock(&sb->map[index].swap_lock);
#if defined(CONFIG_LOCKDEP)
local_irq_restore(flags);
#endif
return ret;
}
int sbitmap_init_node(struct sbitmap *sb, unsigned int depth, int shift,
gfp_t flags, int node)
{
@ -70,6 +111,9 @@ void sbitmap_resize(struct sbitmap *sb, unsigned int depth)
unsigned int bits_per_word = 1U << sb->shift;
unsigned int i;
for (i = 0; i < sb->map_nr; i++)
sbitmap_deferred_clear(sb, i);
sb->depth = depth;
sb->map_nr = DIV_ROUND_UP(sb->depth, bits_per_word);
@ -112,47 +156,6 @@ static int __sbitmap_get_word(unsigned long *word, unsigned long depth,
return nr;
}
/*
* See if we have deferred clears that we can batch move
*/
static inline bool sbitmap_deferred_clear(struct sbitmap *sb, int index)
{
unsigned long mask, val;
unsigned long __maybe_unused flags;
bool ret = false;
/* Silence bogus lockdep warning */
#if defined(CONFIG_LOCKDEP)
local_irq_save(flags);
#endif
spin_lock(&sb->map[index].swap_lock);
if (!sb->map[index].cleared)
goto out_unlock;
/*
* First get a stable cleared mask, setting the old mask to 0.
*/
do {
mask = sb->map[index].cleared;
} while (cmpxchg(&sb->map[index].cleared, mask, 0) != mask);
/*
* Now clear the masked bits in our free word
*/
do {
val = sb->map[index].word;
} while (cmpxchg(&sb->map[index].word, val, val & ~mask) != val);
ret = true;
out_unlock:
spin_unlock(&sb->map[index].swap_lock);
#if defined(CONFIG_LOCKDEP)
local_irq_restore(flags);
#endif
return ret;
}
static int sbitmap_find_bit_in_index(struct sbitmap *sb, int index,
unsigned int alloc_hint, bool round_robin)
{
@ -215,6 +218,7 @@ int sbitmap_get_shallow(struct sbitmap *sb, unsigned int alloc_hint,
index = SB_NR_TO_INDEX(sb, alloc_hint);
for (i = 0; i < sb->map_nr; i++) {
again:
nr = __sbitmap_get_word(&sb->map[index].word,
min(sb->map[index].depth, shallow_depth),
SB_NR_TO_BIT(sb, alloc_hint), true);
@ -223,6 +227,9 @@ int sbitmap_get_shallow(struct sbitmap *sb, unsigned int alloc_hint,
break;
}
if (sbitmap_deferred_clear(sb, index))
goto again;
/* Jump to next index. */
index++;
alloc_hint = index << sb->shift;
@ -242,7 +249,7 @@ bool sbitmap_any_bit_set(const struct sbitmap *sb)
unsigned int i;
for (i = 0; i < sb->map_nr; i++) {
if (sb->map[i].word)
if (sb->map[i].word & ~sb->map[i].cleared)
return true;
}
return false;
@ -255,9 +262,10 @@ bool sbitmap_any_bit_clear(const struct sbitmap *sb)
for (i = 0; i < sb->map_nr; i++) {
const struct sbitmap_word *word = &sb->map[i];
unsigned long mask = word->word & ~word->cleared;
unsigned long ret;
ret = find_first_zero_bit(&word->word, word->depth);
ret = find_first_zero_bit(&mask, word->depth);
if (ret < word->depth)
return true;
}