mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git
synced 2025-01-01 18:52:02 +00:00
3597f2786b
For files, there's nr_user_files/file_table/file_data, and buffers have nr_user_bufs/user_bufs/buf_data. There's no reason why file_table and file_data can't be the same thing, and ditto for the buffer side. That gets rid of more io_ring_ctx state that's in two spots rather than just being in one spot, as it should be. Put all the registered file data in one locations, and ditto on the buffer front. This also avoids having both io_rsrc_data->nodes being an allocated array, and ->user_bufs[] or ->file_table.nodes. There's no reason to have this information duplicated. Keep it in one spot, io_rsrc_data, along with how many resources are available. Signed-off-by: Jens Axboe <axboe@kernel.dk>
152 lines
3.7 KiB
C
152 lines
3.7 KiB
C
// SPDX-License-Identifier: GPL-2.0
|
|
#include <linux/kernel.h>
|
|
#include <linux/errno.h>
|
|
#include <linux/fs.h>
|
|
#include <linux/file.h>
|
|
#include <linux/mm.h>
|
|
#include <linux/slab.h>
|
|
#include <linux/namei.h>
|
|
#include <linux/io_uring.h>
|
|
#include <linux/splice.h>
|
|
|
|
#include <uapi/linux/io_uring.h>
|
|
|
|
#include "io_uring.h"
|
|
#include "splice.h"
|
|
|
|
struct io_splice {
|
|
struct file *file_out;
|
|
loff_t off_out;
|
|
loff_t off_in;
|
|
u64 len;
|
|
int splice_fd_in;
|
|
unsigned int flags;
|
|
struct io_rsrc_node *rsrc_node;
|
|
};
|
|
|
|
static int __io_splice_prep(struct io_kiocb *req,
|
|
const struct io_uring_sqe *sqe)
|
|
{
|
|
struct io_splice *sp = io_kiocb_to_cmd(req, struct io_splice);
|
|
unsigned int valid_flags = SPLICE_F_FD_IN_FIXED | SPLICE_F_ALL;
|
|
|
|
sp->len = READ_ONCE(sqe->len);
|
|
sp->flags = READ_ONCE(sqe->splice_flags);
|
|
if (unlikely(sp->flags & ~valid_flags))
|
|
return -EINVAL;
|
|
sp->splice_fd_in = READ_ONCE(sqe->splice_fd_in);
|
|
sp->rsrc_node = rsrc_empty_node;
|
|
req->flags |= REQ_F_FORCE_ASYNC;
|
|
return 0;
|
|
}
|
|
|
|
int io_tee_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
|
|
{
|
|
if (READ_ONCE(sqe->splice_off_in) || READ_ONCE(sqe->off))
|
|
return -EINVAL;
|
|
return __io_splice_prep(req, sqe);
|
|
}
|
|
|
|
void io_splice_cleanup(struct io_kiocb *req)
|
|
{
|
|
struct io_splice *sp = io_kiocb_to_cmd(req, struct io_splice);
|
|
|
|
io_put_rsrc_node(sp->rsrc_node);
|
|
}
|
|
|
|
static struct file *io_splice_get_file(struct io_kiocb *req,
|
|
unsigned int issue_flags)
|
|
{
|
|
struct io_splice *sp = io_kiocb_to_cmd(req, struct io_splice);
|
|
struct io_ring_ctx *ctx = req->ctx;
|
|
struct io_rsrc_node *node;
|
|
struct file *file = NULL;
|
|
|
|
if (!(sp->flags & SPLICE_F_FD_IN_FIXED))
|
|
return io_file_get_normal(req, sp->splice_fd_in);
|
|
|
|
io_ring_submit_lock(ctx, issue_flags);
|
|
if (unlikely(sp->splice_fd_in >= ctx->file_table.data.nr))
|
|
goto out;
|
|
sp->splice_fd_in = array_index_nospec(sp->splice_fd_in, ctx->file_table.data.nr);
|
|
node = ctx->file_table.data.nodes[sp->splice_fd_in];
|
|
if (node) {
|
|
node->refs++;
|
|
sp->rsrc_node = node;
|
|
file = io_slot_file(node);
|
|
req->flags |= REQ_F_NEED_CLEANUP;
|
|
}
|
|
out:
|
|
io_ring_submit_unlock(ctx, issue_flags);
|
|
return file;
|
|
}
|
|
|
|
int io_tee(struct io_kiocb *req, unsigned int issue_flags)
|
|
{
|
|
struct io_splice *sp = io_kiocb_to_cmd(req, struct io_splice);
|
|
struct file *out = sp->file_out;
|
|
unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED;
|
|
struct file *in;
|
|
ssize_t ret = 0;
|
|
|
|
WARN_ON_ONCE(issue_flags & IO_URING_F_NONBLOCK);
|
|
|
|
in = io_splice_get_file(req, issue_flags);
|
|
if (!in) {
|
|
ret = -EBADF;
|
|
goto done;
|
|
}
|
|
|
|
if (sp->len)
|
|
ret = do_tee(in, out, sp->len, flags);
|
|
|
|
if (!(sp->flags & SPLICE_F_FD_IN_FIXED))
|
|
fput(in);
|
|
done:
|
|
if (ret != sp->len)
|
|
req_set_fail(req);
|
|
io_req_set_res(req, ret, 0);
|
|
return IOU_OK;
|
|
}
|
|
|
|
int io_splice_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
|
|
{
|
|
struct io_splice *sp = io_kiocb_to_cmd(req, struct io_splice);
|
|
|
|
sp->off_in = READ_ONCE(sqe->splice_off_in);
|
|
sp->off_out = READ_ONCE(sqe->off);
|
|
return __io_splice_prep(req, sqe);
|
|
}
|
|
|
|
int io_splice(struct io_kiocb *req, unsigned int issue_flags)
|
|
{
|
|
struct io_splice *sp = io_kiocb_to_cmd(req, struct io_splice);
|
|
struct file *out = sp->file_out;
|
|
unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED;
|
|
loff_t *poff_in, *poff_out;
|
|
struct file *in;
|
|
ssize_t ret = 0;
|
|
|
|
WARN_ON_ONCE(issue_flags & IO_URING_F_NONBLOCK);
|
|
|
|
in = io_splice_get_file(req, issue_flags);
|
|
if (!in) {
|
|
ret = -EBADF;
|
|
goto done;
|
|
}
|
|
|
|
poff_in = (sp->off_in == -1) ? NULL : &sp->off_in;
|
|
poff_out = (sp->off_out == -1) ? NULL : &sp->off_out;
|
|
|
|
if (sp->len)
|
|
ret = do_splice(in, poff_in, out, poff_out, sp->len, flags);
|
|
|
|
if (!(sp->flags & SPLICE_F_FD_IN_FIXED))
|
|
fput(in);
|
|
done:
|
|
if (ret != sp->len)
|
|
req_set_fail(req);
|
|
io_req_set_res(req, ret, 0);
|
|
return IOU_OK;
|
|
}
|