Kernel/linux-next
1
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git synced 2025-01-08 15:04:45 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

btrfs: make buffered write to copy one page a time

Browse Source 
Currently the btrfs_buffered_write() is preparing multiple page a time,
allowing a better performance.

But the current trend is to support larger folio as an optimization,
instead of implementing own multi-page optimization.

This is inspired by generic_perform_write(), which is copying one folio
a time.

Such change will prepare us to migrate to implement the write_begin()
and write_end() callbacks, and make every involved function a little
easier.

Signed-off-by: Qu Wenruo <wqu@suse.com>
Signed-off-by: David Sterba <dsterba@suse.com>
This commit is contained in:
Qu Wenruo 2024-10-10 15:16:12 +10:30 committed by David Sterba
parent b1c5f6eda2
commit c87c299776
3 changed files with 95 additions and 164 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

239
fs/btrfs/file.c
View File

@ -37,22 +37,19 @@
#include "file.h" #include "file.h"
#include "super.h" #include "super.h"
/* simple helper to fault in pages and copy. This should go away /*
* and be replaced with calls into generic code. * Helper to fault in page and copy. This should go away and be replaced with
* calls into generic code.
*/ */
static noinline int btrfs_copy_from_user(loff_t pos, size_t write_bytes, static noinline int btrfs_copy_from_user(loff_t pos, size_t write_bytes,
struct page **prepared_pages, struct page *page, struct iov_iter *i)
struct iov_iter *i)
{ {
size_t copied = 0; size_t copied = 0;
size_t total_copied = 0; size_t total_copied = 0;
int pg = 0;
int offset = offset_in_page(pos); int offset = offset_in_page(pos);
while (write_bytes > 0) { while (write_bytes > 0) {
size_t count = min_t(size_t, size_t count = min_t(size_t, PAGE_SIZE - offset, write_bytes);
PAGE_SIZE - offset, write_bytes);
struct page *page = prepared_pages[pg];
/* /*
* Copy data from userspace to the current page * Copy data from userspace to the current page
*/ */
@ -63,7 +60,7 @@ static noinline int btrfs_copy_from_user(loff_t pos, size_t write_bytes,
/* /*
* if we get a partial write, we can end up with * if we get a partial write, we can end up with
* partially up to date pages. These add * partially up to date page. These add
* a lot of complexity, so make sure they don't * a lot of complexity, so make sure they don't
* happen by forcing this copy to be retried. * happen by forcing this copy to be retried.
* *
@ -82,10 +79,6 @@ static noinline int btrfs_copy_from_user(loff_t pos, size_t write_bytes,
write_bytes -= copied; write_bytes -= copied;
total_copied += copied; total_copied += copied;
offset += copied; offset += copied;
if (offset == PAGE_SIZE) {
pg++;
offset = 0;
}
} }
return total_copied; return total_copied;
} }
@ -93,27 +86,24 @@ static noinline int btrfs_copy_from_user(loff_t pos, size_t write_bytes,
/* /*
* unlocks pages after btrfs_file_write is done with them * unlocks pages after btrfs_file_write is done with them
*/ */
static void btrfs_drop_pages(struct btrfs_fs_info *fs_info, static void btrfs_drop_page(struct btrfs_fs_info *fs_info, struct page *page,
struct page **pages, size_t num_pages, u64 pos, u64 copied)
u64 pos, u64 copied)
{ {
size_t i;
u64 block_start = round_down(pos, fs_info->sectorsize); u64 block_start = round_down(pos, fs_info->sectorsize);
u64 block_len = round_up(pos + copied, fs_info->sectorsize) - block_start; u64 block_len = round_up(pos + copied, fs_info->sectorsize) - block_start;
ASSERT(block_len <= U32_MAX); ASSERT(block_len <= U32_MAX);
for (i = 0; i < num_pages; i++) { /*
/* page checked is some magic around finding pages that * Page checked is some magic around finding pages that have been
* have been modified without going through btrfs_set_page_dirty * modified without going through btrfs_set_page_dirty clear it here.
* clear it here. There should be no need to mark the pages * There should be no need to mark the pages accessed as
* accessed as prepare_pages should have marked them accessed * prepare_one_page() should have marked them accessed in
* in prepare_pages via find_or_create_page() * prepare_one_page() via find_or_create_page()
*/ */
btrfs_folio_clamp_clear_checked(fs_info, page_folio(pages[i]), btrfs_folio_clamp_clear_checked(fs_info, page_folio(page), block_start,
block_start, block_len); block_len);
unlock_page(pages[i]); unlock_page(page);
put_page(pages[i]); put_page(page);
}
} }
/* /*
@ -123,19 +113,16 @@ static void btrfs_drop_pages(struct btrfs_fs_info *fs_info,
* - Mark modified pages as Uptodate/Dirty and not needing COW fixup * - Mark modified pages as Uptodate/Dirty and not needing COW fixup
* - Update inode size for past EOF write * - Update inode size for past EOF write
*/ */
int btrfs_dirty_pages(struct btrfs_inode *inode, struct page **pages, int btrfs_dirty_page(struct btrfs_inode *inode, struct page *page, loff_t pos,
loff_t pos, size_t write_bytes, size_t write_bytes, struct extent_state **cached, bool noreserve)
struct extent_state **cached, bool noreserve)
{ {
struct btrfs_fs_info *fs_info = inode->root->fs_info; struct btrfs_fs_info *fs_info = inode->root->fs_info;
int ret = 0; int ret = 0;
int i;
const int num_pages = (round_up(pos + write_bytes, PAGE_SIZE) -
round_down(pos, PAGE_SIZE)) >> PAGE_SHIFT;
u64 num_bytes; u64 num_bytes;
u64 start_pos; u64 start_pos;
u64 end_of_last_block; u64 end_of_last_block;
u64 end_pos = pos + write_bytes; u64 end_pos = pos + write_bytes;
struct folio *folio = page_folio(page);
loff_t isize = i_size_read(&inode->vfs_inode); loff_t isize = i_size_read(&inode->vfs_inode);
unsigned int extra_bits = 0; unsigned int extra_bits = 0;
@ -149,6 +136,8 @@ int btrfs_dirty_pages(struct btrfs_inode *inode, struct page **pages,
num_bytes = round_up(write_bytes + pos - start_pos, num_bytes = round_up(write_bytes + pos - start_pos,
fs_info->sectorsize); fs_info->sectorsize);
ASSERT(num_bytes <= U32_MAX); ASSERT(num_bytes <= U32_MAX);
ASSERT(folio_pos(folio) <= pos &&
folio_pos(folio) + folio_size(folio) >= pos + write_bytes);
end_of_last_block = start_pos + num_bytes - 1; end_of_last_block = start_pos + num_bytes - 1;
@ -165,16 +154,9 @@ int btrfs_dirty_pages(struct btrfs_inode *inode, struct page **pages,
if (ret) if (ret)
return ret; return ret;
for (i = 0; i < num_pages; i++) { btrfs_folio_clamp_set_uptodate(fs_info, folio, start_pos, num_bytes);
struct page *p = pages[i]; btrfs_folio_clamp_clear_checked(fs_info, folio, start_pos, num_bytes);
btrfs_folio_clamp_set_dirty(fs_info, folio, start_pos, num_bytes);
btrfs_folio_clamp_set_uptodate(fs_info, page_folio(p),
start_pos, num_bytes);
btrfs_folio_clamp_clear_checked(fs_info, page_folio(p),
start_pos, num_bytes);
btrfs_folio_clamp_set_dirty(fs_info, page_folio(p),
start_pos, num_bytes);
}
/* /*
* we've only changed i_size in ram, and we haven't updated * we've only changed i_size in ram, and we haven't updated
@ -922,62 +904,47 @@ static gfp_t get_prepare_gfp_flags(struct inode *inode, bool nowait)
} }
/* /*
* this just gets pages into the page cache and locks them down. * this just gets page into the page cache and locks them down.
*/ */
static noinline int prepare_pages(struct inode *inode, struct page **pages, static noinline int prepare_one_page(struct inode *inode, struct page **page_ret,
size_t num_pages, loff_t pos, loff_t pos, size_t write_bytes,
size_t write_bytes, bool force_uptodate, bool force_uptodate, bool nowait)
bool nowait)
{ {
int i;
unsigned long index = pos >> PAGE_SHIFT; unsigned long index = pos >> PAGE_SHIFT;
gfp_t mask = get_prepare_gfp_flags(inode, nowait); gfp_t mask = get_prepare_gfp_flags(inode, nowait);
fgf_t fgp_flags = get_prepare_fgp_flags(nowait); fgf_t fgp_flags = get_prepare_fgp_flags(nowait);
struct page *page;
int ret = 0; int ret = 0;
int faili;
for (i = 0; i < num_pages; i++) {
again: again:
pages[i] = pagecache_get_page(inode->i_mapping, index + i, page = pagecache_get_page(inode->i_mapping, index, fgp_flags,
fgp_flags, mask | __GFP_WRITE); mask | __GFP_WRITE);
if (!pages[i]) { if (!page) {
faili = i - 1; if (nowait)
if (nowait) ret = -EAGAIN;
ret = -EAGAIN; else
else ret = -ENOMEM;
ret = -ENOMEM; return ret;
goto fail;
}
ret = set_page_extent_mapped(pages[i]);
if (ret < 0) {
faili = i;
goto fail;
}
ret = prepare_uptodate_page(inode, pages[i], pos, write_bytes,
force_uptodate);
if (ret) {
put_page(pages[i]);
if (!nowait && ret == -EAGAIN) {
ret = 0;
goto again;
}
faili = i - 1;
goto fail;
}
wait_on_page_writeback(pages[i]);
} }
ret = set_page_extent_mapped(page);
if (ret < 0) {
unlock_page(page);
put_page(page);
return ret;
}
ret = prepare_uptodate_page(inode, page, pos, write_bytes, force_uptodate);
if (ret) {
/* The page is already unlocked. */
put_page(page);
if (!nowait && ret == -EAGAIN) {
ret = 0;
goto again;
}
return ret;
}
wait_on_page_writeback(page);
*page_ret = page;
return 0; return 0;
fail:
while (faili >= 0) {
unlock_page(pages[faili]);
put_page(pages[faili]);
faili--;
}
return ret;
} }
/* /*
@ -988,19 +955,16 @@ static noinline int prepare_pages(struct inode *inode, struct page **pages,
* 1 - the extent is locked * 1 - the extent is locked
* 0 - the extent is not locked, and everything is OK * 0 - the extent is not locked, and everything is OK
* -EAGAIN - need re-prepare the pages * -EAGAIN - need re-prepare the pages
* the other < 0 number - Something wrong happens
*/ */
static noinline int static noinline int
lock_and_cleanup_extent_if_need(struct btrfs_inode *inode, struct page **pages, lock_and_cleanup_extent_if_need(struct btrfs_inode *inode, struct page *page,
size_t num_pages, loff_t pos, loff_t pos, size_t write_bytes,
size_t write_bytes,
u64 *lockstart, u64 *lockend, bool nowait, u64 *lockstart, u64 *lockend, bool nowait,
struct extent_state **cached_state) struct extent_state **cached_state)
{ {
struct btrfs_fs_info *fs_info = inode->root->fs_info; struct btrfs_fs_info *fs_info = inode->root->fs_info;
u64 start_pos; u64 start_pos;
u64 last_pos; u64 last_pos;
int i;
int ret = 0; int ret = 0;
start_pos = round_down(pos, fs_info->sectorsize); start_pos = round_down(pos, fs_info->sectorsize);
@ -1012,12 +976,8 @@ lock_and_cleanup_extent_if_need(struct btrfs_inode *inode, struct page **pages,
if (nowait) { if (nowait) {
if (!try_lock_extent(&inode->io_tree, start_pos, last_pos, if (!try_lock_extent(&inode->io_tree, start_pos, last_pos,
cached_state)) { cached_state)) {
for (i = 0; i < num_pages; i++) { unlock_page(page);
unlock_page(pages[i]); put_page(page);
put_page(pages[i]);
pages[i] = NULL;
}
return -EAGAIN; return -EAGAIN;
} }
} else { } else {
@ -1031,10 +991,8 @@ lock_and_cleanup_extent_if_need(struct btrfs_inode *inode, struct page **pages,
ordered->file_offset <= last_pos) { ordered->file_offset <= last_pos) {
unlock_extent(&inode->io_tree, start_pos, last_pos, unlock_extent(&inode->io_tree, start_pos, last_pos,
cached_state); cached_state);
for (i = 0; i < num_pages; i++) { unlock_page(page);
unlock_page(pages[i]); put_page(page);
put_page(pages[i]);
}
btrfs_start_ordered_extent(ordered); btrfs_start_ordered_extent(ordered);
btrfs_put_ordered_extent(ordered); btrfs_put_ordered_extent(ordered);
return -EAGAIN; return -EAGAIN;
@ -1048,11 +1006,10 @@ lock_and_cleanup_extent_if_need(struct btrfs_inode *inode, struct page **pages,
} }
/* /*
* We should be called after prepare_pages() which should have locked * We should be called after prepare_one_page() which should have locked
* all pages in the range. * all pages in the range.
*/ */
for (i = 0; i < num_pages; i++) WARN_ON(!PageLocked(page));
WARN_ON(!PageLocked(pages[i]));
return ret; return ret;
} }
@ -1196,20 +1153,17 @@ ssize_t btrfs_buffered_write(struct kiocb *iocb, struct iov_iter *i)
loff_t pos; loff_t pos;
struct inode *inode = file_inode(file); struct inode *inode = file_inode(file);
struct btrfs_fs_info *fs_info = inode_to_fs_info(inode); struct btrfs_fs_info *fs_info = inode_to_fs_info(inode);
struct page **pages = NULL;
struct extent_changeset *data_reserved = NULL; struct extent_changeset *data_reserved = NULL;
u64 release_bytes = 0; u64 release_bytes = 0;
u64 lockstart; u64 lockstart;
u64 lockend; u64 lockend;
size_t num_written = 0; size_t num_written = 0;
int nrptrs;
ssize_t ret; ssize_t ret;
bool only_release_metadata = false;
bool force_page_uptodate = false;
loff_t old_isize = i_size_read(inode); loff_t old_isize = i_size_read(inode);
unsigned int ilock_flags = 0; unsigned int ilock_flags = 0;
const bool nowait = (iocb->ki_flags & IOCB_NOWAIT); const bool nowait = (iocb->ki_flags & IOCB_NOWAIT);
unsigned int bdp_flags = (nowait ? BDP_ASYNC : 0); unsigned int bdp_flags = (nowait ? BDP_ASYNC : 0);
bool only_release_metadata = false;
if (nowait) if (nowait)
ilock_flags |= BTRFS_ILOCK_TRY; ilock_flags |= BTRFS_ILOCK_TRY;
@ -1227,32 +1181,21 @@ ssize_t btrfs_buffered_write(struct kiocb *iocb, struct iov_iter *i)
goto out; goto out;
pos = iocb->ki_pos; pos = iocb->ki_pos;
nrptrs = min(DIV_ROUND_UP(iov_iter_count(i), PAGE_SIZE),
PAGE_SIZE / (sizeof(struct page *)));
nrptrs = min(nrptrs, current->nr_dirtied_pause - current->nr_dirtied);
nrptrs = max(nrptrs, 8);
pages = kmalloc_array(nrptrs, sizeof(struct page *), GFP_KERNEL);
if (!pages) {
ret = -ENOMEM;
goto out;
}
while (iov_iter_count(i) > 0) { while (iov_iter_count(i) > 0) {
struct extent_state *cached_state = NULL; struct extent_state *cached_state = NULL;
size_t offset = offset_in_page(pos); size_t offset = offset_in_page(pos);
size_t sector_offset; size_t sector_offset;
size_t write_bytes = min(iov_iter_count(i), size_t write_bytes = min(iov_iter_count(i), PAGE_SIZE - offset);
nrptrs * (size_t)PAGE_SIZE -
offset);
size_t num_pages;
size_t reserve_bytes; size_t reserve_bytes;
size_t copied; size_t copied;
size_t dirty_sectors; size_t dirty_sectors;
size_t num_sectors; size_t num_sectors;
struct page *page = NULL;
int extents_locked; int extents_locked;
bool force_page_uptodate = false;
/* /*
* Fault pages before locking them in prepare_pages * Fault pages before locking them in prepare_one_page()
* to avoid recursive lock * to avoid recursive lock
*/ */
if (unlikely(fault_in_iov_iter_readable(i, write_bytes))) { if (unlikely(fault_in_iov_iter_readable(i, write_bytes))) {
@ -1291,8 +1234,6 @@ ssize_t btrfs_buffered_write(struct kiocb *iocb, struct iov_iter *i)
only_release_metadata = true; only_release_metadata = true;
} }
num_pages = DIV_ROUND_UP(write_bytes + offset, PAGE_SIZE);
WARN_ON(num_pages > nrptrs);
reserve_bytes = round_up(write_bytes + sector_offset, reserve_bytes = round_up(write_bytes + sector_offset,
fs_info->sectorsize); fs_info->sectorsize);
WARN_ON(reserve_bytes == 0); WARN_ON(reserve_bytes == 0);
@ -1320,23 +1261,17 @@ ssize_t btrfs_buffered_write(struct kiocb *iocb, struct iov_iter *i)
break; break;
} }
/* ret = prepare_one_page(inode, &page, pos, write_bytes,
* This is going to setup the pages array with the number of force_page_uptodate, false);
* pages we want, so we don't really need to worry about the
* contents of pages from loop to loop
*/
ret = prepare_pages(inode, pages, num_pages,
pos, write_bytes, force_page_uptodate, false);
if (ret) { if (ret) {
btrfs_delalloc_release_extents(BTRFS_I(inode), btrfs_delalloc_release_extents(BTRFS_I(inode),
reserve_bytes); reserve_bytes);
break; break;
} }
extents_locked = lock_and_cleanup_extent_if_need( extents_locked = lock_and_cleanup_extent_if_need(BTRFS_I(inode),
BTRFS_I(inode), pages, page, pos, write_bytes, &lockstart,
num_pages, pos, write_bytes, &lockstart, &lockend, nowait, &cached_state);
&lockend, nowait, &cached_state);
if (extents_locked < 0) { if (extents_locked < 0) {
if (!nowait && extents_locked == -EAGAIN) if (!nowait && extents_locked == -EAGAIN)
goto again; goto again;
@ -1347,20 +1282,13 @@ ssize_t btrfs_buffered_write(struct kiocb *iocb, struct iov_iter *i)
break; break;
} }
copied = btrfs_copy_from_user(pos, write_bytes, pages, i); copied = btrfs_copy_from_user(pos, write_bytes, page, i);
num_sectors = BTRFS_BYTES_TO_BLKS(fs_info, reserve_bytes); num_sectors = BTRFS_BYTES_TO_BLKS(fs_info, reserve_bytes);
dirty_sectors = round_up(copied + sector_offset, dirty_sectors = round_up(copied + sector_offset,
fs_info->sectorsize); fs_info->sectorsize);
dirty_sectors = BTRFS_BYTES_TO_BLKS(fs_info, dirty_sectors); dirty_sectors = BTRFS_BYTES_TO_BLKS(fs_info, dirty_sectors);
/*
* if we have trouble faulting in the pages, fall
* back to one page at a time
*/
if (copied < write_bytes)
nrptrs = 1;
if (copied == 0) { if (copied == 0) {
force_page_uptodate = true; force_page_uptodate = true;
dirty_sectors = 0; dirty_sectors = 0;
@ -1386,15 +1314,14 @@ ssize_t btrfs_buffered_write(struct kiocb *iocb, struct iov_iter *i)
release_bytes = round_up(copied + sector_offset, release_bytes = round_up(copied + sector_offset,
fs_info->sectorsize); fs_info->sectorsize);
ret = btrfs_dirty_pages(BTRFS_I(inode), pages, ret = btrfs_dirty_page(BTRFS_I(inode), page, pos, copied,
pos, copied, &cached_state, only_release_metadata);
&cached_state, only_release_metadata);
/* /*
* If we have not locked the extent range, because the range's * If we have not locked the extent range, because the range's
* start offset is >= i_size, we might still have a non-NULL * start offset is >= i_size, we might still have a non-NULL
* cached extent state, acquired while marking the extent range * cached extent state, acquired while marking the extent range
* as delalloc through btrfs_dirty_pages(). Therefore free any * as delalloc through btrfs_dirty_page(). Therefore free any
* possible cached extent state to avoid a memory leak. * possible cached extent state to avoid a memory leak.
*/ */
if (extents_locked) if (extents_locked)
@ -1405,7 +1332,7 @@ ssize_t btrfs_buffered_write(struct kiocb *iocb, struct iov_iter *i)
btrfs_delalloc_release_extents(BTRFS_I(inode), reserve_bytes); btrfs_delalloc_release_extents(BTRFS_I(inode), reserve_bytes);
if (ret) { if (ret) {
btrfs_drop_pages(fs_info, pages, num_pages, pos, copied); btrfs_drop_page(fs_info, page, pos, copied);
break; break;
} }
@ -1413,7 +1340,7 @@ ssize_t btrfs_buffered_write(struct kiocb *iocb, struct iov_iter *i)
if (only_release_metadata) if (only_release_metadata)
btrfs_check_nocow_unlock(BTRFS_I(inode)); btrfs_check_nocow_unlock(BTRFS_I(inode));
btrfs_drop_pages(fs_info, pages, num_pages, pos, copied); btrfs_drop_page(fs_info, page, pos, copied);
cond_resched(); cond_resched();
@ -1421,8 +1348,6 @@ ssize_t btrfs_buffered_write(struct kiocb *iocb, struct iov_iter *i)
num_written += copied; num_written += copied;
} }
kfree(pages);
if (release_bytes) { if (release_bytes) {
if (only_release_metadata) { if (only_release_metadata) {
btrfs_check_nocow_unlock(BTRFS_I(inode)); btrfs_check_nocow_unlock(BTRFS_I(inode));

5
fs/btrfs/file.h
View File

@ -34,9 +34,8 @@ int btrfs_mark_extent_written(struct btrfs_trans_handle *trans,
ssize_t btrfs_do_write_iter(struct kiocb *iocb, struct iov_iter *from, ssize_t btrfs_do_write_iter(struct kiocb *iocb, struct iov_iter *from,
const struct btrfs_ioctl_encoded_io_args *encoded); const struct btrfs_ioctl_encoded_io_args *encoded);
int btrfs_release_file(struct inode *inode, struct file *file); int btrfs_release_file(struct inode *inode, struct file *file);
int btrfs_dirty_pages(struct btrfs_inode *inode, struct page **pages, int btrfs_dirty_page(struct btrfs_inode *inode, struct page *page, loff_t pos,
loff_t pos, size_t write_bytes, size_t write_bytes, struct extent_state **cached, bool noreserve);
struct extent_state **cached, bool noreserve);
int btrfs_fdatawrite_range(struct btrfs_inode *inode, loff_t start, loff_t end); int btrfs_fdatawrite_range(struct btrfs_inode *inode, loff_t start, loff_t end);
int btrfs_check_nocow_lock(struct btrfs_inode *inode, loff_t pos, int btrfs_check_nocow_lock(struct btrfs_inode *inode, loff_t pos,
size_t *write_bytes, bool nowait); size_t *write_bytes, bool nowait);

15
fs/btrfs/free-space-cache.c
View File

@ -1388,6 +1388,7 @@ static int __btrfs_write_out_cache(struct inode *inode,
int bitmaps = 0; int bitmaps = 0;
int ret; int ret;
int must_iput = 0; int must_iput = 0;
int i_size;
if (!i_size_read(inode)) if (!i_size_read(inode))
return -EIO; return -EIO;
@ -1458,10 +1459,16 @@ static int __btrfs_write_out_cache(struct inode *inode,
io_ctl_zero_remaining_pages(io_ctl); io_ctl_zero_remaining_pages(io_ctl);
/* Everything is written out, now we dirty the pages in the file. */ /* Everything is written out, now we dirty the pages in the file. */
ret = btrfs_dirty_pages(BTRFS_I(inode), io_ctl->pages, 0, i_size_read(inode), i_size = i_size_read(inode);
&cached_state, false); for (int i = 0; i < round_up(i_size, PAGE_SIZE) / PAGE_SIZE; i++) {
if (ret) u64 dirty_start = i * PAGE_SIZE;
goto out_nospc; u64 dirty_len = min_t(u64, dirty_start + PAGE_SIZE, i_size) - dirty_start;
ret = btrfs_dirty_page(BTRFS_I(inode), io_ctl->pages[i],
dirty_start, dirty_len, &cached_state, false);
if (ret < 0)
goto out_nospc;
}
if (block_group && (block_group->flags & BTRFS_BLOCK_GROUP_DATA)) if (block_group && (block_group->flags & BTRFS_BLOCK_GROUP_DATA))
up_write(&block_group->data_rwsem); up_write(&block_group->data_rwsem);