mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2025-01-16 10:17:32 +00:00
0701db7439
Rather than allocate an io_rsrc_node for an empty/sparse buffer entry, add a const entry that can be used for that. This just needs checking for writing the tag, and the put check needs to check for that sparse node rather than NULL for validity. This avoids allocating rsrc nodes for sparse buffer entries. Signed-off-by: Jens Axboe <axboe@kernel.dk>
152 lines
3.6 KiB
C
152 lines
3.6 KiB
C
// SPDX-License-Identifier: GPL-2.0
|
|
#include <linux/kernel.h>
|
|
#include <linux/errno.h>
|
|
#include <linux/fs.h>
|
|
#include <linux/file.h>
|
|
#include <linux/mm.h>
|
|
#include <linux/slab.h>
|
|
#include <linux/namei.h>
|
|
#include <linux/io_uring.h>
|
|
#include <linux/splice.h>
|
|
|
|
#include <uapi/linux/io_uring.h>
|
|
|
|
#include "io_uring.h"
|
|
#include "splice.h"
|
|
|
|
struct io_splice {
|
|
struct file *file_out;
|
|
loff_t off_out;
|
|
loff_t off_in;
|
|
u64 len;
|
|
int splice_fd_in;
|
|
unsigned int flags;
|
|
struct io_rsrc_node *rsrc_node;
|
|
};
|
|
|
|
static int __io_splice_prep(struct io_kiocb *req,
|
|
const struct io_uring_sqe *sqe)
|
|
{
|
|
struct io_splice *sp = io_kiocb_to_cmd(req, struct io_splice);
|
|
unsigned int valid_flags = SPLICE_F_FD_IN_FIXED | SPLICE_F_ALL;
|
|
|
|
sp->len = READ_ONCE(sqe->len);
|
|
sp->flags = READ_ONCE(sqe->splice_flags);
|
|
if (unlikely(sp->flags & ~valid_flags))
|
|
return -EINVAL;
|
|
sp->splice_fd_in = READ_ONCE(sqe->splice_fd_in);
|
|
sp->rsrc_node = rsrc_empty_node;
|
|
req->flags |= REQ_F_FORCE_ASYNC;
|
|
return 0;
|
|
}
|
|
|
|
int io_tee_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
|
|
{
|
|
if (READ_ONCE(sqe->splice_off_in) || READ_ONCE(sqe->off))
|
|
return -EINVAL;
|
|
return __io_splice_prep(req, sqe);
|
|
}
|
|
|
|
void io_splice_cleanup(struct io_kiocb *req)
|
|
{
|
|
struct io_splice *sp = io_kiocb_to_cmd(req, struct io_splice);
|
|
|
|
io_put_rsrc_node(sp->rsrc_node);
|
|
}
|
|
|
|
static struct file *io_splice_get_file(struct io_kiocb *req,
|
|
unsigned int issue_flags)
|
|
{
|
|
struct io_splice *sp = io_kiocb_to_cmd(req, struct io_splice);
|
|
struct io_ring_ctx *ctx = req->ctx;
|
|
struct io_rsrc_node *node;
|
|
struct file *file = NULL;
|
|
|
|
if (!(sp->flags & SPLICE_F_FD_IN_FIXED))
|
|
return io_file_get_normal(req, sp->splice_fd_in);
|
|
|
|
io_ring_submit_lock(ctx, issue_flags);
|
|
if (unlikely(sp->splice_fd_in >= ctx->nr_user_files))
|
|
goto out;
|
|
sp->splice_fd_in = array_index_nospec(sp->splice_fd_in, ctx->nr_user_files);
|
|
node = ctx->file_table.nodes[sp->splice_fd_in];
|
|
if (node) {
|
|
node->refs++;
|
|
sp->rsrc_node = node;
|
|
file = io_slot_file(node);
|
|
req->flags |= REQ_F_NEED_CLEANUP;
|
|
}
|
|
out:
|
|
io_ring_submit_unlock(ctx, issue_flags);
|
|
return file;
|
|
}
|
|
|
|
int io_tee(struct io_kiocb *req, unsigned int issue_flags)
|
|
{
|
|
struct io_splice *sp = io_kiocb_to_cmd(req, struct io_splice);
|
|
struct file *out = sp->file_out;
|
|
unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED;
|
|
struct file *in;
|
|
ssize_t ret = 0;
|
|
|
|
WARN_ON_ONCE(issue_flags & IO_URING_F_NONBLOCK);
|
|
|
|
in = io_splice_get_file(req, issue_flags);
|
|
if (!in) {
|
|
ret = -EBADF;
|
|
goto done;
|
|
}
|
|
|
|
if (sp->len)
|
|
ret = do_tee(in, out, sp->len, flags);
|
|
|
|
if (!(sp->flags & SPLICE_F_FD_IN_FIXED))
|
|
fput(in);
|
|
done:
|
|
if (ret != sp->len)
|
|
req_set_fail(req);
|
|
io_req_set_res(req, ret, 0);
|
|
return IOU_OK;
|
|
}
|
|
|
|
int io_splice_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
|
|
{
|
|
struct io_splice *sp = io_kiocb_to_cmd(req, struct io_splice);
|
|
|
|
sp->off_in = READ_ONCE(sqe->splice_off_in);
|
|
sp->off_out = READ_ONCE(sqe->off);
|
|
return __io_splice_prep(req, sqe);
|
|
}
|
|
|
|
int io_splice(struct io_kiocb *req, unsigned int issue_flags)
|
|
{
|
|
struct io_splice *sp = io_kiocb_to_cmd(req, struct io_splice);
|
|
struct file *out = sp->file_out;
|
|
unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED;
|
|
loff_t *poff_in, *poff_out;
|
|
struct file *in;
|
|
ssize_t ret = 0;
|
|
|
|
WARN_ON_ONCE(issue_flags & IO_URING_F_NONBLOCK);
|
|
|
|
in = io_splice_get_file(req, issue_flags);
|
|
if (!in) {
|
|
ret = -EBADF;
|
|
goto done;
|
|
}
|
|
|
|
poff_in = (sp->off_in == -1) ? NULL : &sp->off_in;
|
|
poff_out = (sp->off_out == -1) ? NULL : &sp->off_out;
|
|
|
|
if (sp->len)
|
|
ret = do_splice(in, poff_in, out, poff_out, sp->len, flags);
|
|
|
|
if (!(sp->flags & SPLICE_F_FD_IN_FIXED))
|
|
fput(in);
|
|
done:
|
|
if (ret != sp->len)
|
|
req_set_fail(req);
|
|
io_req_set_res(req, ret, 0);
|
|
return IOU_OK;
|
|
}
|