From 51e08eaf954de51b991889a5f5b5b5edcc712c6c Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Fri, 12 Jun 2026 11:48:37 -0700 Subject: io_uring/rsrc: rename io_buffer_register_bvec()/io_buffer_unregister_bvec() Currently, io_buffer_register_bvec() takes in a request. In preparation for supporting kernel-populated buffers in fuse io-uring (which will need to register bvecs directly, not through a struct request), rename this to io_buffer_register_request(). A subsequent patch will commandeer the "io_buffer_register_bvec()" function name to support registering bvecs directly. Rename io_buffer_unregister_bvec() to a more generic name, io_buffer_unregister(), as both io_buffer_register_request() and io_buffer_register_bvec() callers will use it for unregistration. Signed-off-by: Joanne Koong Reviewed-by: Caleb Sander Mateos Link: https://patch.msgid.link/20260612184840.4058966-2-joannelkoong@gmail.com Signed-off-by: Jens Axboe Signed-off-by: Miklos Szeredi --- io_uring/rsrc.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'io_uring') diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c index 8d0f2ee24e0c..40807994a8f4 100644 --- a/io_uring/rsrc.c +++ b/io_uring/rsrc.c @@ -1015,9 +1015,9 @@ int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg, return ret; } -int io_buffer_register_bvec(struct io_uring_cmd *cmd, struct request *rq, - void (*release)(void *), unsigned int index, - unsigned int issue_flags) +int io_buffer_register_request(struct io_uring_cmd *cmd, struct request *rq, + void (*release)(void *), unsigned int index, + unsigned int issue_flags) { struct io_ring_ctx *ctx = cmd_to_io_kiocb(cmd)->ctx; struct io_rsrc_data *data = &ctx->buf_table; @@ -1076,10 +1076,10 @@ unlock: io_ring_submit_unlock(ctx, issue_flags); return ret; } -EXPORT_SYMBOL_GPL(io_buffer_register_bvec); +EXPORT_SYMBOL_GPL(io_buffer_register_request); -int io_buffer_unregister_bvec(struct io_uring_cmd *cmd, unsigned int index, - unsigned int issue_flags) +int io_buffer_unregister(struct io_uring_cmd *cmd, unsigned int index, + unsigned int issue_flags) { struct io_ring_ctx *ctx = cmd_to_io_kiocb(cmd)->ctx; struct io_rsrc_data *data = &ctx->buf_table; @@ -1109,7 +1109,7 @@ unlock: io_ring_submit_unlock(ctx, issue_flags); return ret; } -EXPORT_SYMBOL_GPL(io_buffer_unregister_bvec); +EXPORT_SYMBOL_GPL(io_buffer_unregister); static int validate_fixed_range(u64 buf_addr, size_t len, const struct io_mapped_ubuf *imu) -- cgit From fbc32d5f44d3e2134443909e816c5f9c657e81ad Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Fri, 12 Jun 2026 11:48:38 -0700 Subject: io_uring/rsrc: split io_buffer_register_request() logic Split the main initialization logic in io_buffer_register_request() into a helper function. This is a preparatory patch for supporting kernel-populated buffers in fuse io-uring, which will be reusing this logic. Signed-off-by: Joanne Koong Reviewed-by: Caleb Sander Mateos Link: https://patch.msgid.link/20260612184840.4058966-3-joannelkoong@gmail.com Signed-off-by: Jens Axboe Signed-off-by: Miklos Szeredi --- io_uring/rsrc.c | 84 ++++++++++++++++++++++++++++++++++----------------------- 1 file changed, 51 insertions(+), 33 deletions(-) (limited to 'io_uring') diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c index 40807994a8f4..5d50b967645b 100644 --- a/io_uring/rsrc.c +++ b/io_uring/rsrc.c @@ -1015,63 +1015,81 @@ int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg, return ret; } -int io_buffer_register_request(struct io_uring_cmd *cmd, struct request *rq, - void (*release)(void *), unsigned int index, - unsigned int issue_flags) +static struct io_mapped_ubuf *io_kernel_buffer_init(struct io_ring_ctx *ctx, + unsigned int nr_bvecs, + unsigned int total_bytes, + u8 dir, + void (*release)(void *), + void *priv, + unsigned int index) { - struct io_ring_ctx *ctx = cmd_to_io_kiocb(cmd)->ctx; struct io_rsrc_data *data = &ctx->buf_table; - struct req_iterator rq_iter; struct io_mapped_ubuf *imu; struct io_rsrc_node *node; - struct bio_vec bv; - unsigned int nr_bvecs = 0; - int ret = 0; - io_ring_submit_lock(ctx, issue_flags); - if (index >= data->nr) { - ret = -EINVAL; - goto unlock; - } + if (index >= data->nr) + return ERR_PTR(-EINVAL); index = array_index_nospec(index, data->nr); - if (data->nodes[index]) { - ret = -EBUSY; - goto unlock; - } + if (data->nodes[index]) + return ERR_PTR(-EBUSY); node = io_rsrc_node_alloc(ctx, IORING_RSRC_BUFFER); - if (!node) { - ret = -ENOMEM; - goto unlock; - } + if (!node) + return ERR_PTR(-ENOMEM); - /* - * blk_rq_nr_phys_segments() may overestimate the number of bvecs - * but avoids needing to iterate over the bvecs - */ - imu = io_alloc_imu(ctx, blk_rq_nr_phys_segments(rq)); + imu = io_alloc_imu(ctx, nr_bvecs); if (!imu) { io_cache_free(&ctx->node_cache, node); - ret = -ENOMEM; - goto unlock; + return ERR_PTR(-ENOMEM); } imu->ubuf = 0; - imu->len = blk_rq_bytes(rq); + imu->len = total_bytes; imu->folio_shift = PAGE_SHIFT; + imu->nr_bvecs = nr_bvecs; refcount_set(&imu->refs, 1); imu->release = release; - imu->priv = rq; + imu->priv = priv; + imu->dir = dir; imu->flags = IO_REGBUF_F_KBUF; - imu->dir = 1 << rq_data_dir(rq); + node->buf = imu; + data->nodes[index] = node; + + return imu; +} + +int io_buffer_register_request(struct io_uring_cmd *cmd, struct request *rq, + void (*release)(void *), unsigned int index, + unsigned int issue_flags) +{ + struct io_ring_ctx *ctx = cmd_to_io_kiocb(cmd)->ctx; + struct req_iterator rq_iter; + struct io_mapped_ubuf *imu; + struct bio_vec bv; + /* + * blk_rq_nr_phys_segments() may overestimate the number of bvecs + * but avoids needing to iterate over the bvecs + */ + unsigned int nr_bvecs = blk_rq_nr_phys_segments(rq); + unsigned int total_bytes = blk_rq_bytes(rq); + int ret = 0; + + io_ring_submit_lock(ctx, issue_flags); + + imu = io_kernel_buffer_init(ctx, nr_bvecs, total_bytes, + 1 << rq_data_dir(rq), release, rq, index); + if (IS_ERR(imu)) { + ret = PTR_ERR(imu); + goto unlock; + } + + nr_bvecs = 0; rq_for_each_bvec(bv, rq, rq_iter) imu->bvec[nr_bvecs++] = bv; imu->nr_bvecs = nr_bvecs; - node->buf = imu; - data->nodes[index] = node; unlock: io_ring_submit_unlock(ctx, issue_flags); return ret; -- cgit From bd62a2cfff9f37d85927050f082ffc3a423e6b53 Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Fri, 12 Jun 2026 11:48:39 -0700 Subject: io_uring/rsrc: add io_buffer_register_bvec() Add io_buffer_register_bvec() for registering a bvec array. This is a preparatory patch for fuse-over-io-uring zero-copy. Signed-off-by: Joanne Koong Reviewed-by: Caleb Sander Mateos Link: https://patch.msgid.link/20260612184840.4058966-4-joannelkoong@gmail.com Signed-off-by: Jens Axboe Signed-off-by: Miklos Szeredi --- include/linux/io_uring/cmd.h | 13 +++++++++++++ io_uring/rsrc.c | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) (limited to 'io_uring') diff --git a/include/linux/io_uring/cmd.h b/include/linux/io_uring/cmd.h index bbf57da1e4c8..42801f0b6456 100644 --- a/include/linux/io_uring/cmd.h +++ b/include/linux/io_uring/cmd.h @@ -94,6 +94,10 @@ bool io_uring_mshot_cmd_post_cqe(struct io_uring_cmd *ioucmd, int io_buffer_register_request(struct io_uring_cmd *cmd, struct request *rq, void (*release)(void *), unsigned int index, unsigned int issue_flags); +int io_buffer_register_bvec(struct io_uring_cmd *cmd, const struct bio_vec *bvs, + unsigned int nr_bvecs, void (*release)(void *), + void *priv, u8 dir, unsigned int index, + unsigned int issue_flags); int io_buffer_unregister(struct io_uring_cmd *cmd, unsigned int index, unsigned int issue_flags); #else @@ -146,6 +150,15 @@ static inline int io_buffer_register_request(struct io_uring_cmd *cmd, { return -EOPNOTSUPP; } +static inline int io_buffer_register_bvec(struct io_uring_cmd *cmd, + const struct bio_vec *bvs, + unsigned int nr_bvecs, + void (*release)(void *), void *priv, + u8 dir, unsigned int index, + unsigned int issue_flags) +{ + return -EOPNOTSUPP; +} static inline int io_buffer_unregister(struct io_uring_cmd *cmd, unsigned int index, unsigned int issue_flags) diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c index 5d50b967645b..819c5087d8d3 100644 --- a/io_uring/rsrc.c +++ b/io_uring/rsrc.c @@ -1096,6 +1096,41 @@ unlock: } EXPORT_SYMBOL_GPL(io_buffer_register_request); +/* + * bvs is copied internally. caller may free it on return. + */ +int io_buffer_register_bvec(struct io_uring_cmd *cmd, const struct bio_vec *bvs, + unsigned int nr_bvecs, void (*release)(void *), + void *priv, u8 dir, unsigned int index, + unsigned int issue_flags) +{ + struct io_ring_ctx *ctx = cmd_to_io_kiocb(cmd)->ctx; + struct io_mapped_ubuf *imu; + struct bio_vec *bvec; + unsigned int i, total_bytes = 0; + int ret = 0; + + for (i = 0; i < nr_bvecs; i++) + total_bytes += bvs[i].bv_len; + + io_ring_submit_lock(ctx, issue_flags); + imu = io_kernel_buffer_init(ctx, nr_bvecs, total_bytes, dir, release, + priv, index); + if (IS_ERR(imu)) { + ret = PTR_ERR(imu); + goto unlock; + } + + bvec = imu->bvec; + for (i = 0; i < nr_bvecs; i++) + bvec[i] = bvs[i]; + +unlock: + io_ring_submit_unlock(ctx, issue_flags); + return ret; +} +EXPORT_SYMBOL_GPL(io_buffer_register_bvec); + int io_buffer_unregister(struct io_uring_cmd *cmd, unsigned int index, unsigned int issue_flags) { -- cgit From 95961b72c57b29a96c14f86d16f1d32787f2e009 Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Fri, 12 Jun 2026 11:48:40 -0700 Subject: io_uring/rsrc: rename and export IO_IMU_DEST / IO_IMU_SOURCE Rename IO_IMU_DEST and IO_IMU_SOURCE to IO_BUF_DEST and IO_BUF_SOURCE and export it so subsystems may use it. This is needed by the io_buffer_register_bvec() path for callers who may need the buffer to be both readable and writable. Signed-off-by: Joanne Koong Link: https://patch.msgid.link/20260612184840.4058966-5-joannelkoong@gmail.com Signed-off-by: Jens Axboe Signed-off-by: Miklos Szeredi --- include/linux/io_uring_types.h | 5 +++++ io_uring/io_uring.c | 2 +- io_uring/rsrc.c | 2 +- io_uring/rsrc.h | 5 ----- 4 files changed, 7 insertions(+), 7 deletions(-) (limited to 'io_uring') diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index 87151a5b62c1..db42a548c7a5 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -44,6 +44,11 @@ enum io_uring_cmd_flags { IO_URING_F_COMPAT = (1 << 12), }; +enum { + IO_BUF_DEST = 1 << ITER_DEST, + IO_BUF_SOURCE = 1 << ITER_SOURCE, +}; + struct iou_loop_params; struct io_wq_work_node { diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 1ea2fca34a36..ba685b6052ed 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -3245,7 +3245,7 @@ static int __init io_uring_init(void) io_uring_optable_init(); /* imu->dir is u8 */ - BUILD_BUG_ON((IO_IMU_DEST | IO_IMU_SOURCE) > U8_MAX); + BUILD_BUG_ON((IO_BUF_DEST | IO_BUF_SOURCE) > U8_MAX); /* * Allow user copy in the per-command field, which starts after the diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c index 819c5087d8d3..f3f01e0c8102 100644 --- a/io_uring/rsrc.c +++ b/io_uring/rsrc.c @@ -912,7 +912,7 @@ static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx, imu->release = io_release_ubuf; imu->priv = imu; imu->flags = 0; - imu->dir = IO_IMU_DEST | IO_IMU_SOURCE; + imu->dir = IO_BUF_DEST | IO_BUF_SOURCE; if (coalesced) imu->folio_shift = data.folio_shift; refcount_set(&imu->refs, 1); diff --git a/io_uring/rsrc.h b/io_uring/rsrc.h index 98ae8ef51009..e503b02aa61a 100644 --- a/io_uring/rsrc.h +++ b/io_uring/rsrc.h @@ -23,11 +23,6 @@ struct io_rsrc_node { }; }; -enum { - IO_IMU_DEST = 1 << ITER_DEST, - IO_IMU_SOURCE = 1 << ITER_SOURCE, -}; - enum { IO_REGBUF_F_KBUF = 1, }; -- cgit