diff --git a/fs/bcachefs/Kconfig b/fs/bcachefs/Kconfig index 49776ba0a031..df13a4f9a6e3 100644 --- a/fs/bcachefs/Kconfig +++ b/fs/bcachefs/Kconfig @@ -9,6 +9,8 @@ config BCACHEFS_FS select FS_POSIX_ACL select LZ4_COMPRESS select LZ4_DECOMPRESS + select LZ4HC_COMPRESS + select LZ4HC_DECOMPRESS select ZLIB_DEFLATE select ZLIB_INFLATE select ZSTD_COMPRESS diff --git a/fs/bcachefs/checksum.h b/fs/bcachefs/checksum.h index 409ad534d9f4..1ad1d5f03939 100644 --- a/fs/bcachefs/checksum.h +++ b/fs/bcachefs/checksum.h @@ -120,12 +120,6 @@ static inline enum bch_csum_type bch2_meta_checksum_type(struct bch_fs *c) return bch2_csum_opt_to_type(c->opts.metadata_checksum, false); } -static const unsigned bch2_compression_opt_to_type[] = { -#define x(t, n) [BCH_COMPRESSION_OPT_##t] = BCH_COMPRESSION_TYPE_##t, - BCH_COMPRESSION_OPTS() -#undef x -}; - static inline bool bch2_checksum_type_valid(const struct bch_fs *c, unsigned type) { diff --git a/fs/bcachefs/compress.c b/fs/bcachefs/compress.c index 48427a270840..560214c15da3 100644 --- a/fs/bcachefs/compress.c +++ b/fs/bcachefs/compress.c @@ -296,21 +296,32 @@ static int attempt_compress(struct bch_fs *c, void *workspace, void *dst, size_t dst_len, void *src, size_t src_len, - enum bch_compression_type compression_type) + struct bch_compression_opt compression) { + enum bch_compression_type compression_type = + __bch2_compression_opt_to_type[compression.type]; + switch (compression_type) { - case BCH_COMPRESSION_TYPE_lz4: { - int len = src_len; - int ret = LZ4_compress_destSize( - src, dst, - &len, dst_len, - workspace); + case BCH_COMPRESSION_TYPE_lz4: + if (compression.level < LZ4HC_MIN_CLEVEL) { + int len = src_len; + int ret = LZ4_compress_destSize( + src, dst, + &len, dst_len, + workspace); + if (len < src_len) + return -len; - if (len < src_len) - return -len; + return ret; + } else { + int ret = LZ4_compress_HC( + src, dst, + src_len, dst_len, + compression.level, + workspace); - return ret; - } + return ret ?: -1; + } case BCH_COMPRESSION_TYPE_gzip: { z_stream strm = { .next_in = src, @@ -320,7 +331,11 @@ static int attempt_compress(struct bch_fs *c, }; zlib_set_workspace(&strm, workspace); - zlib_deflateInit2(&strm, Z_DEFAULT_COMPRESSION, + zlib_deflateInit2(&strm, + compression.level + ? clamp_t(unsigned, compression.level, + Z_BEST_SPEED, Z_BEST_COMPRESSION) + : Z_DEFAULT_COMPRESSION, Z_DEFLATED, -MAX_WBITS, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY); @@ -333,8 +348,14 @@ static int attempt_compress(struct bch_fs *c, return strm.total_out; } case BCH_COMPRESSION_TYPE_zstd: { + /* + * rescale: + * zstd max compression level is 22, our max level is 15 + */ + unsigned level = min((compression.level * 3) / 2, zstd_max_clevel()); + ZSTD_parameters params = zstd_get_params(level, c->opts.encoded_extent_max); ZSTD_CCtx *ctx = zstd_init_cctx(workspace, - zstd_cctx_workspace_bound(&c->zstd_params.cParams)); + zstd_cctx_workspace_bound(¶ms.cParams)); /* * ZSTD requires that when we decompress we pass in the exact @@ -365,10 +386,12 @@ static int attempt_compress(struct bch_fs *c, static unsigned __bio_compress(struct bch_fs *c, struct bio *dst, size_t *dst_len, struct bio *src, size_t *src_len, - enum bch_compression_type compression_type) + struct bch_compression_opt compression) { struct bbuf src_data = { NULL }, dst_data = { NULL }; void *workspace; + enum bch_compression_type compression_type = + __bch2_compression_opt_to_type[compression.type]; unsigned pad; int ret = 0; @@ -400,7 +423,7 @@ static unsigned __bio_compress(struct bch_fs *c, ret = attempt_compress(c, workspace, dst_data.b, *dst_len, src_data.b, *src_len, - compression_type); + compression); if (ret > 0) { *dst_len = ret; ret = 0; @@ -447,22 +470,24 @@ static unsigned __bio_compress(struct bch_fs *c, BUG_ON(!*src_len || *src_len > src->bi_iter.bi_size); BUG_ON(*dst_len & (block_bytes(c) - 1)); BUG_ON(*src_len & (block_bytes(c) - 1)); + ret = compression_type; out: bio_unmap_or_unbounce(c, src_data); bio_unmap_or_unbounce(c, dst_data); - return compression_type; + return ret; err: - compression_type = BCH_COMPRESSION_TYPE_incompressible; + ret = BCH_COMPRESSION_TYPE_incompressible; goto out; } unsigned bch2_bio_compress(struct bch_fs *c, struct bio *dst, size_t *dst_len, struct bio *src, size_t *src_len, - unsigned compression_type) + unsigned compression_opt) { unsigned orig_dst = dst->bi_iter.bi_size; unsigned orig_src = src->bi_iter.bi_size; + unsigned compression_type; /* Don't consume more than BCH_ENCODED_EXTENT_MAX from @src: */ src->bi_iter.bi_size = min_t(unsigned, src->bi_iter.bi_size, @@ -470,11 +495,9 @@ unsigned bch2_bio_compress(struct bch_fs *c, /* Don't generate a bigger output than input: */ dst->bi_iter.bi_size = min(dst->bi_iter.bi_size, src->bi_iter.bi_size); - if (compression_type == BCH_COMPRESSION_TYPE_lz4_old) - compression_type = BCH_COMPRESSION_TYPE_lz4; - compression_type = - __bio_compress(c, dst, dst_len, src, src_len, compression_type); + __bio_compress(c, dst, dst_len, src, src_len, + bch2_compression_decode(compression_opt)); dst->bi_iter.bi_size = orig_dst; src->bi_iter.bi_size = orig_src; @@ -521,8 +544,10 @@ static int __bch2_check_set_has_compressed_data(struct bch_fs *c, u64 f) } int bch2_check_set_has_compressed_data(struct bch_fs *c, - unsigned compression_type) + unsigned compression_opt) { + unsigned compression_type = bch2_compression_decode(compression_opt).type; + BUG_ON(compression_type >= ARRAY_SIZE(bch2_compression_opt_to_feature)); return compression_type @@ -546,14 +571,16 @@ static int __bch2_fs_compress_init(struct bch_fs *c, u64 features) { size_t decompress_workspace_size = 0; bool decompress_workspace_needed; - ZSTD_parameters params = zstd_get_params(0, c->opts.encoded_extent_max); + ZSTD_parameters params = zstd_get_params(zstd_max_clevel(), + c->opts.encoded_extent_max); struct { - unsigned feature; - unsigned type; - size_t compress_workspace; - size_t decompress_workspace; + unsigned feature; + enum bch_compression_type type; + size_t compress_workspace; + size_t decompress_workspace; } compression_types[] = { - { BCH_FEATURE_lz4, BCH_COMPRESSION_TYPE_lz4, LZ4_MEM_COMPRESS, 0 }, + { BCH_FEATURE_lz4, BCH_COMPRESSION_TYPE_lz4, + max_t(size_t, LZ4_MEM_COMPRESS, LZ4HC_MEM_COMPRESS) }, { BCH_FEATURE_gzip, BCH_COMPRESSION_TYPE_gzip, zlib_deflate_workspacesize(MAX_WBITS, DEF_MEM_LEVEL), zlib_inflate_workspacesize(), }, @@ -612,16 +639,74 @@ static int __bch2_fs_compress_init(struct bch_fs *c, u64 features) return 0; } +static u64 compression_opt_to_feature(unsigned v) +{ + unsigned type = bch2_compression_decode(v).type; + return 1ULL << bch2_compression_opt_to_feature[type]; +} + int bch2_fs_compress_init(struct bch_fs *c) { u64 f = c->sb.features; - if (c->opts.compression) - f |= 1ULL << bch2_compression_opt_to_feature[c->opts.compression]; - - if (c->opts.background_compression) - f |= 1ULL << bch2_compression_opt_to_feature[c->opts.background_compression]; + f |= compression_opt_to_feature(c->opts.compression); + f |= compression_opt_to_feature(c->opts.background_compression); return __bch2_fs_compress_init(c, f); - +} + +int bch2_opt_compression_parse(struct bch_fs *c, const char *_val, u64 *res, + struct printbuf *err) +{ + char *val = kstrdup(_val, GFP_KERNEL); + char *p = val, *type_str, *level_str; + struct bch_compression_opt opt = { 0 }; + int ret; + + if (!val) + return -ENOMEM; + + type_str = strsep(&p, ":"); + level_str = p; + + ret = match_string(bch2_compression_opts, -1, type_str); + if (ret < 0 && err) + prt_str(err, "invalid compression type"); + if (ret < 0) + goto err; + + opt.type = ret; + + if (level_str) { + unsigned level; + + ret = kstrtouint(level_str, 10, &level); + if (!ret && !opt.type && level) + ret = -EINVAL; + if (!ret && level > 15) + ret = -EINVAL; + if (ret < 0 && err) + prt_str(err, "invalid compression level"); + if (ret < 0) + goto err; + + opt.level = level; + } + + *res = bch2_compression_encode(opt); +err: + kfree(val); + return ret; +} + +void bch2_opt_compression_to_text(struct printbuf *out, + struct bch_fs *c, + struct bch_sb *sb, + u64 v) +{ + struct bch_compression_opt opt = bch2_compression_decode(v); + + prt_str(out, bch2_compression_opts[opt.type]); + if (opt.level) + prt_printf(out, ":%u", opt.level); } diff --git a/fs/bcachefs/compress.h b/fs/bcachefs/compress.h index 4bab1f61b3b5..052ea303241f 100644 --- a/fs/bcachefs/compress.h +++ b/fs/bcachefs/compress.h @@ -4,6 +4,35 @@ #include "extents_types.h" +struct bch_compression_opt { + u8 type:4, + level:4; +}; + +static inline struct bch_compression_opt bch2_compression_decode(unsigned v) +{ + return (struct bch_compression_opt) { + .type = v & 15, + .level = v >> 4, + }; +} + +static inline unsigned bch2_compression_encode(struct bch_compression_opt opt) +{ + return opt.type|(opt.level << 4); +} + +static const unsigned __bch2_compression_opt_to_type[] = { +#define x(t, n) [BCH_COMPRESSION_OPT_##t] = BCH_COMPRESSION_TYPE_##t, + BCH_COMPRESSION_OPTS() +#undef x +}; + +static inline enum bch_compression_type bch2_compression_opt_to_type(unsigned v) +{ + return __bch2_compression_opt_to_type[bch2_compression_decode(v).type]; +} + int bch2_bio_uncompress_inplace(struct bch_fs *, struct bio *, struct bch_extent_crc_unpacked *); int bch2_bio_uncompress(struct bch_fs *, struct bio *, struct bio *, @@ -15,4 +44,12 @@ int bch2_check_set_has_compressed_data(struct bch_fs *, unsigned); void bch2_fs_compress_exit(struct bch_fs *); int bch2_fs_compress_init(struct bch_fs *); +int bch2_opt_compression_parse(struct bch_fs *, const char *, u64 *, struct printbuf *); +void bch2_opt_compression_to_text(struct printbuf *, struct bch_fs *, struct bch_sb *, u64); + +#define bch2_opt_compression (struct bch_opt_fn) { \ + .parse = bch2_opt_compression_parse, \ + .to_text = bch2_opt_compression_to_text, \ +} + #endif /* _BCACHEFS_COMPRESS_H */ diff --git a/fs/bcachefs/data_update.c b/fs/bcachefs/data_update.c index 3c918368b2ec..cfc624463700 100644 --- a/fs/bcachefs/data_update.c +++ b/fs/bcachefs/data_update.c @@ -455,9 +455,7 @@ int bch2_data_update_init(struct btree_trans *trans, BCH_WRITE_DATA_ENCODED| BCH_WRITE_MOVE| m->data_opts.write_flags; - m->op.compression_type = - bch2_compression_opt_to_type[io_opts.background_compression ?: - io_opts.compression]; + m->op.compression_opt = io_opts.background_compression ?: io_opts.compression; m->op.watermark = m->data_opts.btree_insert_flags & BCH_WATERMARK_MASK; bkey_for_each_ptr(ptrs, ptr) diff --git a/fs/bcachefs/io.c b/fs/bcachefs/io.c index 33762e4a0f05..8604df80a3e2 100644 --- a/fs/bcachefs/io.c +++ b/fs/bcachefs/io.c @@ -1078,7 +1078,7 @@ static enum prep_encoded_ret { /* Can we just write the entire extent as is? */ if (op->crc.uncompressed_size == op->crc.live_size && op->crc.compressed_size <= wp->sectors_free && - (op->crc.compression_type == op->compression_type || + (op->crc.compression_type == bch2_compression_opt_to_type(op->compression_opt) || op->incompressible)) { if (!crc_is_compressed(op->crc) && op->csum_type != op->crc.csum_type && @@ -1126,7 +1126,7 @@ static enum prep_encoded_ret { /* * If we want to compress the data, it has to be decrypted: */ - if ((op->compression_type || + if ((op->compression_opt || bch2_csum_type_is_encryption(op->crc.csum_type) != bch2_csum_type_is_encryption(op->csum_type)) && bch2_write_decrypt(op)) @@ -1173,7 +1173,7 @@ static int bch2_write_extent(struct bch_write_op *op, struct write_point *wp, } if (ec_buf || - op->compression_type || + op->compression_opt || (op->csum_type && !(op->flags & BCH_WRITE_PAGES_STABLE)) || (bch2_csum_type_is_encryption(op->csum_type) && @@ -1196,16 +1196,16 @@ static int bch2_write_extent(struct bch_write_op *op, struct write_point *wp, dst->bi_iter.bi_size < c->opts.encoded_extent_max) break; - BUG_ON(op->compression_type && + BUG_ON(op->compression_opt && (op->flags & BCH_WRITE_DATA_ENCODED) && bch2_csum_type_is_encryption(op->crc.csum_type)); - BUG_ON(op->compression_type && !bounce); + BUG_ON(op->compression_opt && !bounce); crc.compression_type = op->incompressible ? BCH_COMPRESSION_TYPE_incompressible - : op->compression_type + : op->compression_opt ? bch2_bio_compress(c, dst, &dst_len, src, &src_len, - op->compression_type) + op->compression_opt) : 0; if (!crc_is_compressed(crc)) { dst_len = min(dst->bi_iter.bi_size, src->bi_iter.bi_size); diff --git a/fs/bcachefs/io.h b/fs/bcachefs/io.h index 7a243a5f3f89..1476380d5fbf 100644 --- a/fs/bcachefs/io.h +++ b/fs/bcachefs/io.h @@ -86,7 +86,7 @@ static inline void bch2_write_op_init(struct bch_write_op *op, struct bch_fs *c, op->written = 0; op->error = 0; op->csum_type = bch2_data_checksum_type(c, opts); - op->compression_type = bch2_compression_opt_to_type[opts.compression]; + op->compression_opt = opts.compression; op->nr_replicas = 0; op->nr_replicas_required = c->opts.data_replicas_required; op->watermark = BCH_WATERMARK_normal; diff --git a/fs/bcachefs/io_types.h b/fs/bcachefs/io_types.h index 0fbdfbf90ad8..737f16d78c48 100644 --- a/fs/bcachefs/io_types.h +++ b/fs/bcachefs/io_types.h @@ -115,8 +115,8 @@ struct bch_write_op { u16 flags; s16 error; /* dio write path expects it to hold -ERESTARTSYS... */ + unsigned compression_opt:8; unsigned csum_type:4; - unsigned compression_type:4; unsigned nr_replicas:4; unsigned nr_replicas_required:4; unsigned watermark:3; diff --git a/fs/bcachefs/opts.h b/fs/bcachefs/opts.h index 92e2e5e759d9..8a9db110d64f 100644 --- a/fs/bcachefs/opts.h +++ b/fs/bcachefs/opts.h @@ -174,12 +174,12 @@ enum fsck_err_opts { NULL, NULL) \ x(compression, u8, \ OPT_FS|OPT_INODE|OPT_FORMAT|OPT_MOUNT|OPT_RUNTIME, \ - OPT_STR(bch2_compression_opts), \ + OPT_FN(bch2_opt_compression), \ BCH_SB_COMPRESSION_TYPE, BCH_COMPRESSION_OPT_none, \ NULL, NULL) \ x(background_compression, u8, \ OPT_FS|OPT_INODE|OPT_FORMAT|OPT_MOUNT|OPT_RUNTIME, \ - OPT_STR(bch2_compression_opts), \ + OPT_FN(bch2_opt_compression), \ BCH_SB_BACKGROUND_COMPRESSION_TYPE,BCH_COMPRESSION_OPT_none, \ NULL, NULL) \ x(str_hash, u8, \ diff --git a/fs/bcachefs/rebalance.c b/fs/bcachefs/rebalance.c index 989f37a3b46a..c3d577236ce2 100644 --- a/fs/bcachefs/rebalance.c +++ b/fs/bcachefs/rebalance.c @@ -5,6 +5,7 @@ #include "btree_iter.h" #include "buckets.h" #include "clock.h" +#include "compress.h" #include "disk_groups.h" #include "errcode.h" #include "extents.h" @@ -45,7 +46,7 @@ static bool rebalance_pred(struct bch_fs *c, void *arg, bkey_for_each_ptr_decode(k.k, ptrs, p, entry) { if (!p.ptr.cached && p.crc.compression_type != - bch2_compression_opt_to_type[io_opts->background_compression]) + bch2_compression_opt_to_type(io_opts->background_compression)) data_opts->rewrite_ptrs |= 1U << i; i++; }