blob: 14297add848508026e3d8450d77e3b170a861e9f [file] [log] [blame]
Jens Axboe2b188cc2019-01-07 10:46:33 -07001// SPDX-License-Identifier: GPL-2.0
2/*
3 * Shared application/kernel submission and completion ring pairs, for
4 * supporting fast/efficient IO.
5 *
6 * A note on the read/write ordering memory barriers that are matched between
Stefan Bühler1e84b972019-04-24 23:54:16 +02007 * the application and kernel side.
8 *
9 * After the application reads the CQ ring tail, it must use an
10 * appropriate smp_rmb() to pair with the smp_wmb() the kernel uses
11 * before writing the tail (using smp_load_acquire to read the tail will
12 * do). It also needs a smp_mb() before updating CQ head (ordering the
13 * entry load(s) with the head store), pairing with an implicit barrier
Pavel Begunkovd068b502021-05-16 22:58:11 +010014 * through a control-dependency in io_get_cqe (smp_store_release to
Stefan Bühler1e84b972019-04-24 23:54:16 +020015 * store head will do). Failure to do so could lead to reading invalid
16 * CQ entries.
17 *
18 * Likewise, the application must use an appropriate smp_wmb() before
19 * writing the SQ tail (ordering SQ entry stores with the tail store),
20 * which pairs with smp_load_acquire in io_get_sqring (smp_store_release
21 * to store the tail will do). And it needs a barrier ordering the SQ
22 * head load before writing new SQ entries (smp_load_acquire to read
23 * head will do).
24 *
25 * When using the SQ poll thread (IORING_SETUP_SQPOLL), the application
26 * needs to check the SQ flags for IORING_SQ_NEED_WAKEUP *after*
27 * updating the SQ tail; a full memory barrier smp_mb() is needed
28 * between.
Jens Axboe2b188cc2019-01-07 10:46:33 -070029 *
30 * Also see the examples in the liburing library:
31 *
32 * git://git.kernel.dk/liburing
33 *
34 * io_uring also uses READ/WRITE_ONCE() for _any_ store or load that happens
35 * from data shared between the kernel and application. This is done both
36 * for ordering purposes, but also to ensure that once a value is loaded from
37 * data that the application could potentially modify, it remains stable.
38 *
39 * Copyright (C) 2018-2019 Jens Axboe
Christoph Hellwigc992fe22019-01-11 09:43:02 -070040 * Copyright (c) 2018-2019 Christoph Hellwig
Jens Axboe2b188cc2019-01-07 10:46:33 -070041 */
42#include <linux/kernel.h>
43#include <linux/init.h>
44#include <linux/errno.h>
45#include <linux/syscalls.h>
46#include <linux/compat.h>
Jens Axboe52de1fe2020-02-27 10:15:42 -070047#include <net/compat.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070048#include <linux/refcount.h>
49#include <linux/uio.h>
Pavel Begunkov6b47ee62020-01-18 20:22:41 +030050#include <linux/bits.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070051
52#include <linux/sched/signal.h>
53#include <linux/fs.h>
54#include <linux/file.h>
55#include <linux/fdtable.h>
56#include <linux/mm.h>
57#include <linux/mman.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070058#include <linux/percpu.h>
59#include <linux/slab.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070060#include <linux/blkdev.h>
Jens Axboeedafcce2019-01-09 09:16:05 -070061#include <linux/bvec.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070062#include <linux/net.h>
63#include <net/sock.h>
64#include <net/af_unix.h>
Jens Axboe6b063142019-01-10 22:13:58 -070065#include <net/scm.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070066#include <linux/anon_inodes.h>
67#include <linux/sched/mm.h>
68#include <linux/uaccess.h>
69#include <linux/nospec.h>
Jens Axboeedafcce2019-01-09 09:16:05 -070070#include <linux/sizes.h>
71#include <linux/hugetlb.h>
Jens Axboeaa4c3962019-11-29 10:14:00 -070072#include <linux/highmem.h>
Jens Axboe15b71ab2019-12-11 11:20:36 -070073#include <linux/namei.h>
74#include <linux/fsnotify.h>
Jens Axboe4840e412019-12-25 22:03:45 -070075#include <linux/fadvise.h>
Jens Axboe3e4827b2020-01-08 15:18:09 -070076#include <linux/eventpoll.h>
Pavel Begunkov7d67af22020-02-24 11:32:45 +030077#include <linux/splice.h>
Jens Axboeb41e9852020-02-17 09:52:41 -070078#include <linux/task_work.h>
Jens Axboebcf5a062020-05-22 09:24:42 -060079#include <linux/pagemap.h>
Jens Axboe0f212202020-09-13 13:09:39 -060080#include <linux/io_uring.h>
Nadav Amitef98eb02021-08-07 17:13:41 -070081#include <linux/tracehook.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070082
Dmitrii Dolgovc826bd72019-10-15 19:02:01 +020083#define CREATE_TRACE_POINTS
84#include <trace/events/io_uring.h>
85
Jens Axboe2b188cc2019-01-07 10:46:33 -070086#include <uapi/linux/io_uring.h>
87
Jens Axboef435c662022-05-23 17:05:03 -060088#include "../fs/internal.h"
Jens Axboe561fb042019-10-24 07:25:42 -060089#include "io-wq.h"
Jens Axboe2b188cc2019-01-07 10:46:33 -070090
Daniel Xu5277dea2019-09-14 14:23:45 -070091#define IORING_MAX_ENTRIES 32768
Jens Axboe33a107f2019-10-04 12:10:03 -060092#define IORING_MAX_CQ_ENTRIES (2 * IORING_MAX_ENTRIES)
Olivier Langlois4ce8ad92021-06-23 11:50:18 -070093#define IORING_SQPOLL_CAP_ENTRIES_VALUE 8
Jens Axboe65e19f52019-10-26 07:20:21 -060094
wangyangbo187f08c2021-08-19 13:56:57 +080095/* only define max */
Pavel Begunkov042b0d82021-08-09 13:04:01 +010096#define IORING_MAX_FIXED_FILES (1U << 15)
Stefano Garzarella21b55db2020-08-27 16:58:30 +020097#define IORING_MAX_RESTRICTIONS (IORING_RESTRICTION_LAST + \
98 IORING_REGISTER_LAST + IORING_OP_LAST)
Jens Axboe2b188cc2019-01-07 10:46:33 -070099
wangyangbo187f08c2021-08-19 13:56:57 +0800100#define IO_RSRC_TAG_TABLE_SHIFT (PAGE_SHIFT - 3)
Pavel Begunkov2d091d62021-06-14 02:36:21 +0100101#define IO_RSRC_TAG_TABLE_MAX (1U << IO_RSRC_TAG_TABLE_SHIFT)
102#define IO_RSRC_TAG_TABLE_MASK (IO_RSRC_TAG_TABLE_MAX - 1)
103
Pavel Begunkov489809e2021-05-14 12:06:44 +0100104#define IORING_MAX_REG_BUFFERS (1U << 14)
105
Pavel Begunkovb16fed662021-02-18 18:29:40 +0000106#define SQE_VALID_FLAGS (IOSQE_FIXED_FILE|IOSQE_IO_DRAIN|IOSQE_IO_LINK| \
107 IOSQE_IO_HARDLINK | IOSQE_ASYNC | \
108 IOSQE_BUFFER_SELECT)
Pavel Begunkovc8543572021-06-17 18:14:04 +0100109#define IO_REQ_CLEAN_FLAGS (REQ_F_BUFFER_SELECTED | REQ_F_NEED_CLEANUP | \
110 REQ_F_POLLED | REQ_F_INFLIGHT | REQ_F_CREDS)
Pavel Begunkovb16fed662021-02-18 18:29:40 +0000111
Pavel Begunkov09899b12021-06-14 02:36:22 +0100112#define IO_TCTX_REFS_CACHE_NR (1U << 10)
113
Jens Axboe2b188cc2019-01-07 10:46:33 -0700114struct io_uring {
115 u32 head ____cacheline_aligned_in_smp;
116 u32 tail ____cacheline_aligned_in_smp;
117};
118
Stefan Bühler1e84b972019-04-24 23:54:16 +0200119/*
Hristo Venev75b28af2019-08-26 17:23:46 +0000120 * This data is shared with the application through the mmap at offsets
121 * IORING_OFF_SQ_RING and IORING_OFF_CQ_RING.
Stefan Bühler1e84b972019-04-24 23:54:16 +0200122 *
123 * The offsets to the member fields are published through struct
124 * io_sqring_offsets when calling io_uring_setup.
125 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000126struct io_rings {
Stefan Bühler1e84b972019-04-24 23:54:16 +0200127 /*
128 * Head and tail offsets into the ring; the offsets need to be
129 * masked to get valid indices.
130 *
Hristo Venev75b28af2019-08-26 17:23:46 +0000131 * The kernel controls head of the sq ring and the tail of the cq ring,
132 * and the application controls tail of the sq ring and the head of the
133 * cq ring.
Stefan Bühler1e84b972019-04-24 23:54:16 +0200134 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000135 struct io_uring sq, cq;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200136 /*
Hristo Venev75b28af2019-08-26 17:23:46 +0000137 * Bitmasks to apply to head and tail offsets (constant, equals
Stefan Bühler1e84b972019-04-24 23:54:16 +0200138 * ring_entries - 1)
139 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000140 u32 sq_ring_mask, cq_ring_mask;
141 /* Ring sizes (constant, power of 2) */
142 u32 sq_ring_entries, cq_ring_entries;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200143 /*
144 * Number of invalid entries dropped by the kernel due to
145 * invalid index stored in array
146 *
147 * Written by the kernel, shouldn't be modified by the
148 * application (i.e. get number of "new events" by comparing to
149 * cached value).
150 *
151 * After a new SQ head value was read by the application this
152 * counter includes all submissions that were dropped reaching
153 * the new SQ head (and possibly more).
154 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000155 u32 sq_dropped;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200156 /*
Stefano Garzarella0d9b5b32020-05-15 18:38:04 +0200157 * Runtime SQ flags
Stefan Bühler1e84b972019-04-24 23:54:16 +0200158 *
159 * Written by the kernel, shouldn't be modified by the
160 * application.
161 *
162 * The application needs a full memory barrier before checking
163 * for IORING_SQ_NEED_WAKEUP after updating the sq tail.
164 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000165 u32 sq_flags;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200166 /*
Stefano Garzarella0d9b5b32020-05-15 18:38:04 +0200167 * Runtime CQ flags
168 *
169 * Written by the application, shouldn't be modified by the
170 * kernel.
171 */
Pavel Begunkovfe7e3252021-06-24 15:09:57 +0100172 u32 cq_flags;
Stefano Garzarella0d9b5b32020-05-15 18:38:04 +0200173 /*
Stefan Bühler1e84b972019-04-24 23:54:16 +0200174 * Number of completion events lost because the queue was full;
175 * this should be avoided by the application by making sure
LimingWu0b4295b2019-12-05 20:18:18 +0800176 * there are not more requests pending than there is space in
Stefan Bühler1e84b972019-04-24 23:54:16 +0200177 * the completion queue.
178 *
179 * Written by the kernel, shouldn't be modified by the
180 * application (i.e. get number of "new events" by comparing to
181 * cached value).
182 *
183 * As completion events come in out of order this counter is not
184 * ordered with any other data.
185 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000186 u32 cq_overflow;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200187 /*
188 * Ring buffer of completion events.
189 *
190 * The kernel writes completion events fresh every time they are
191 * produced, so the application is allowed to modify pending
192 * entries.
193 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000194 struct io_uring_cqe cqes[] ____cacheline_aligned_in_smp;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700195};
196
Pavel Begunkov45d189c2021-02-10 00:03:07 +0000197enum io_uring_cmd_flags {
198 IO_URING_F_NONBLOCK = 1,
Pavel Begunkov889fca72021-02-10 00:03:09 +0000199 IO_URING_F_COMPLETE_DEFER = 2,
Pavel Begunkov45d189c2021-02-10 00:03:07 +0000200};
201
Jens Axboeedafcce2019-01-09 09:16:05 -0700202struct io_mapped_ubuf {
203 u64 ubuf;
Pavel Begunkov4751f532021-04-01 15:43:55 +0100204 u64 ubuf_end;
Jens Axboeedafcce2019-01-09 09:16:05 -0700205 unsigned int nr_bvecs;
Jens Axboede293932020-09-17 16:19:16 -0600206 unsigned long acct_pages;
Pavel Begunkov41edf1a2021-04-25 14:32:23 +0100207 struct bio_vec bvec[];
Jens Axboeedafcce2019-01-09 09:16:05 -0700208};
209
Bijan Mottahedeh50238532021-01-15 17:37:45 +0000210struct io_ring_ctx;
211
Pavel Begunkov6c2450a2021-02-23 12:40:22 +0000212struct io_overflow_cqe {
213 struct io_uring_cqe cqe;
214 struct list_head list;
215};
216
Pavel Begunkova04b0ac2021-04-01 15:44:04 +0100217struct io_fixed_file {
218 /* file * with additional FFS_* flags */
219 unsigned long file_ptr;
220};
221
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +0000222struct io_rsrc_put {
223 struct list_head list;
Pavel Begunkovb60c8dc2021-04-25 14:32:18 +0100224 u64 tag;
Bijan Mottahedeh50238532021-01-15 17:37:45 +0000225 union {
226 void *rsrc;
227 struct file *file;
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +0100228 struct io_mapped_ubuf *buf;
Bijan Mottahedeh50238532021-01-15 17:37:45 +0000229 };
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +0000230};
231
Pavel Begunkovaeca2412021-04-11 01:46:37 +0100232struct io_file_table {
Pavel Begunkov042b0d82021-08-09 13:04:01 +0100233 struct io_fixed_file *files;
Jens Axboe31b51512019-01-18 22:56:34 -0700234};
235
Pavel Begunkovb895c9a2021-04-01 15:43:40 +0100236struct io_rsrc_node {
Xiaoguang Wang05589552020-03-31 14:05:18 +0800237 struct percpu_ref refs;
238 struct list_head node;
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +0000239 struct list_head rsrc_list;
Pavel Begunkovb895c9a2021-04-01 15:43:40 +0100240 struct io_rsrc_data *rsrc_data;
Jens Axboe4a38aed22020-05-14 17:21:15 -0600241 struct llist_node llist;
Pavel Begunkove2978222020-11-18 14:56:26 +0000242 bool done;
Xiaoguang Wang05589552020-03-31 14:05:18 +0800243};
244
Pavel Begunkov40ae0ff2021-04-01 15:43:44 +0100245typedef void (rsrc_put_fn)(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc);
246
Pavel Begunkovb895c9a2021-04-01 15:43:40 +0100247struct io_rsrc_data {
Jens Axboe05f3fb32019-12-09 11:22:50 -0700248 struct io_ring_ctx *ctx;
249
Pavel Begunkov2d091d62021-06-14 02:36:21 +0100250 u64 **tags;
251 unsigned int nr;
Pavel Begunkov40ae0ff2021-04-01 15:43:44 +0100252 rsrc_put_fn *do_put;
Pavel Begunkov3e942492021-04-11 01:46:34 +0100253 atomic_t refs;
Jens Axboe05f3fb32019-12-09 11:22:50 -0700254 struct completion done;
Hao Xu8bad28d2021-02-19 17:19:36 +0800255 bool quiesce;
Jens Axboe05f3fb32019-12-09 11:22:50 -0700256};
257
Jens Axboe5a2e7452020-02-23 16:23:11 -0700258struct io_buffer {
259 struct list_head list;
260 __u64 addr;
Thadeu Lima de Souza Cascardod1f82802021-05-05 09:47:06 -0300261 __u32 len;
Jens Axboe5a2e7452020-02-23 16:23:11 -0700262 __u16 bid;
263};
264
Stefano Garzarella21b55db2020-08-27 16:58:30 +0200265struct io_restriction {
266 DECLARE_BITMAP(register_op, IORING_REGISTER_LAST);
267 DECLARE_BITMAP(sqe_op, IORING_OP_LAST);
268 u8 sqe_flags_allowed;
269 u8 sqe_flags_required;
Stefano Garzarella7e84e1c2020-08-27 16:58:31 +0200270 bool registered;
Stefano Garzarella21b55db2020-08-27 16:58:30 +0200271};
272
Jens Axboe37d1e2e2021-02-17 21:03:43 -0700273enum {
274 IO_SQ_THREAD_SHOULD_STOP = 0,
275 IO_SQ_THREAD_SHOULD_PARK,
276};
277
Jens Axboe534ca6d2020-09-02 13:52:19 -0600278struct io_sq_data {
279 refcount_t refs;
Pavel Begunkov9e138a42021-03-14 20:57:12 +0000280 atomic_t park_pending;
Pavel Begunkov09a6f4e2021-03-14 20:57:10 +0000281 struct mutex lock;
Jens Axboe69fb2132020-09-14 11:16:23 -0600282
283 /* ctx's that are using this sqd */
284 struct list_head ctx_list;
Jens Axboe69fb2132020-09-14 11:16:23 -0600285
Jens Axboe534ca6d2020-09-02 13:52:19 -0600286 struct task_struct *thread;
287 struct wait_queue_head wait;
Xiaoguang Wang08369242020-11-03 14:15:59 +0800288
289 unsigned sq_thread_idle;
Jens Axboe37d1e2e2021-02-17 21:03:43 -0700290 int sq_cpu;
291 pid_t task_pid;
Jens Axboe5c2469e2021-03-11 10:17:56 -0700292 pid_t task_tgid;
Jens Axboe37d1e2e2021-02-17 21:03:43 -0700293
294 unsigned long state;
Jens Axboe37d1e2e2021-02-17 21:03:43 -0700295 struct completion exited;
Jens Axboe534ca6d2020-09-02 13:52:19 -0600296};
297
Pavel Begunkov6dd0be12021-02-10 00:03:13 +0000298#define IO_COMPL_BATCH 32
Pavel Begunkov6ff119a2021-02-10 00:03:18 +0000299#define IO_REQ_CACHE_SIZE 32
Pavel Begunkovbf019da2021-02-10 00:03:17 +0000300#define IO_REQ_ALLOC_BATCH 8
Pavel Begunkov258b29a2021-02-10 00:03:10 +0000301
Pavel Begunkova1ab7b32021-02-18 18:29:42 +0000302struct io_submit_link {
303 struct io_kiocb *head;
304 struct io_kiocb *last;
305};
306
Pavel Begunkov258b29a2021-02-10 00:03:10 +0000307struct io_submit_state {
308 struct blk_plug plug;
Pavel Begunkova1ab7b32021-02-18 18:29:42 +0000309 struct io_submit_link link;
Pavel Begunkov258b29a2021-02-10 00:03:10 +0000310
311 /*
312 * io_kiocb alloc cache
313 */
Pavel Begunkovbf019da2021-02-10 00:03:17 +0000314 void *reqs[IO_REQ_CACHE_SIZE];
Pavel Begunkov258b29a2021-02-10 00:03:10 +0000315 unsigned int free_reqs;
316
317 bool plug_started;
318
319 /*
320 * Batch completion logic
321 */
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +0100322 struct io_kiocb *compl_reqs[IO_COMPL_BATCH];
323 unsigned int compl_nr;
324 /* inline/task_work completion list, under ->uring_lock */
325 struct list_head free_list;
Pavel Begunkov258b29a2021-02-10 00:03:10 +0000326
Pavel Begunkov258b29a2021-02-10 00:03:10 +0000327 unsigned int ios_left;
328};
329
Jens Axboe2b188cc2019-01-07 10:46:33 -0700330struct io_ring_ctx {
Pavel Begunkovb52ecf82021-06-14 23:37:21 +0100331 /* const or read-mostly hot data */
Jens Axboe2b188cc2019-01-07 10:46:33 -0700332 struct {
333 struct percpu_ref refs;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700334
Pavel Begunkovb52ecf82021-06-14 23:37:21 +0100335 struct io_rings *rings;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700336 unsigned int flags;
Randy Dunlape1d85332020-02-05 20:57:10 -0800337 unsigned int compat: 1;
Randy Dunlape1d85332020-02-05 20:57:10 -0800338 unsigned int drain_next: 1;
339 unsigned int eventfd_async: 1;
Stefano Garzarella21b55db2020-08-27 16:58:30 +0200340 unsigned int restricted: 1;
Pavel Begunkovf18ee4c2021-06-14 23:37:25 +0100341 unsigned int off_timeout_used: 1;
Pavel Begunkov10c66902021-06-15 16:47:56 +0100342 unsigned int drain_active: 1;
Pavel Begunkovb52ecf82021-06-14 23:37:21 +0100343 } ____cacheline_aligned_in_smp;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700344
Pavel Begunkov7f1129d2021-06-14 23:37:22 +0100345 /* submission data */
Pavel Begunkovb52ecf82021-06-14 23:37:21 +0100346 struct {
Pavel Begunkov0499e582021-06-14 23:37:29 +0100347 struct mutex uring_lock;
348
Hristo Venev75b28af2019-08-26 17:23:46 +0000349 /*
350 * Ring buffer of indices into array of io_uring_sqe, which is
351 * mmapped by the application using the IORING_OFF_SQES offset.
352 *
353 * This indirection could e.g. be used to assign fixed
354 * io_uring_sqe entries to operations and only submit them to
355 * the queue when needed.
356 *
357 * The kernel modifies neither the indices array nor the entries
358 * array.
359 */
360 u32 *sq_array;
Pavel Begunkovc7af47c2021-06-14 23:37:20 +0100361 struct io_uring_sqe *sq_sqes;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700362 unsigned cached_sq_head;
363 unsigned sq_entries;
Jens Axboede0617e2019-04-06 21:51:27 -0600364 struct list_head defer_list;
Pavel Begunkov7f1129d2021-06-14 23:37:22 +0100365
366 /*
367 * Fixed resources fast path, should be accessed only under
368 * uring_lock, and updated through io_uring_register(2)
369 */
370 struct io_rsrc_node *rsrc_node;
371 struct io_file_table file_table;
372 unsigned nr_user_files;
373 unsigned nr_user_bufs;
374 struct io_mapped_ubuf **user_bufs;
375
376 struct io_submit_state submit_state;
Jens Axboe5262f562019-09-17 12:26:57 -0600377 struct list_head timeout_list;
Pavel Begunkovef9dd632021-08-28 19:54:38 -0600378 struct list_head ltimeout_list;
Jens Axboe1d7bb1d2019-11-06 11:31:17 -0700379 struct list_head cq_overflow_list;
Pavel Begunkov7f1129d2021-06-14 23:37:22 +0100380 struct xarray io_buffers;
381 struct xarray personalities;
382 u32 pers_next;
383 unsigned sq_thread_idle;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700384 } ____cacheline_aligned_in_smp;
385
Pavel Begunkovd0acdee2021-05-16 22:58:12 +0100386 /* IRQ completion list, under ->completion_lock */
387 struct list_head locked_free_list;
388 unsigned int locked_free_nr;
Jens Axboe3c1a2ea2021-02-11 10:48:03 -0700389
Stefan Metzmacher7c30f36a2021-03-07 11:54:28 +0100390 const struct cred *sq_creds; /* cred used for __io_sq_thread() */
Jens Axboe534ca6d2020-09-02 13:52:19 -0600391 struct io_sq_data *sq_data; /* if using sq thread polling */
392
Jens Axboe90554202020-09-03 12:12:41 -0600393 struct wait_queue_head sqo_sq_wait;
Jens Axboe69fb2132020-09-14 11:16:23 -0600394 struct list_head sqd_list;
Hristo Venev75b28af2019-08-26 17:23:46 +0000395
Pavel Begunkov5ed7a372021-06-14 23:37:27 +0100396 unsigned long check_cq_overflow;
397
Jens Axboe206aefd2019-11-07 18:27:42 -0700398 struct {
399 unsigned cached_cq_tail;
400 unsigned cq_entries;
Jens Axboe206aefd2019-11-07 18:27:42 -0700401 struct eventfd_ctx *cq_ev_fd;
Pavel Begunkov0499e582021-06-14 23:37:29 +0100402 struct wait_queue_head poll_wait;
403 struct wait_queue_head cq_wait;
404 unsigned cq_extra;
405 atomic_t cq_timeouts;
Pavel Begunkov0499e582021-06-14 23:37:29 +0100406 unsigned cq_last_tm_flush;
Jens Axboe206aefd2019-11-07 18:27:42 -0700407 } ____cacheline_aligned_in_smp;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700408
409 struct {
Jens Axboe2b188cc2019-01-07 10:46:33 -0700410 spinlock_t completion_lock;
Jens Axboee94f1412019-12-19 12:06:02 -0700411
Jens Axboe89850fc2021-08-10 15:11:51 -0600412 spinlock_t timeout_lock;
413
Jens Axboedef596e2019-01-09 08:59:42 -0700414 /*
Pavel Begunkov540e32a2020-07-13 23:37:09 +0300415 * ->iopoll_list is protected by the ctx->uring_lock for
Jens Axboedef596e2019-01-09 08:59:42 -0700416 * io_uring instances that don't use IORING_SETUP_SQPOLL.
417 * For SQPOLL, only the single threaded io_sq_thread() will
418 * manipulate the list, hence no extra locking is needed there.
419 */
Pavel Begunkov540e32a2020-07-13 23:37:09 +0300420 struct list_head iopoll_list;
Jens Axboe78076bb2019-12-04 19:56:40 -0700421 struct hlist_head *cancel_hash;
422 unsigned cancel_hash_bits;
Hao Xu915b3dd2021-06-28 05:37:30 +0800423 bool poll_multi_queue;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700424 } ____cacheline_aligned_in_smp;
Jens Axboe85faa7b2020-04-09 18:14:00 -0600425
Stefano Garzarella21b55db2020-08-27 16:58:30 +0200426 struct io_restriction restrictions;
Jens Axboe3c1a2ea2021-02-11 10:48:03 -0700427
Pavel Begunkovb13a8912021-05-16 22:58:07 +0100428 /* slow path rsrc auxilary data, used by update/register */
429 struct {
430 struct io_rsrc_node *rsrc_backup_node;
431 struct io_mapped_ubuf *dummy_ubuf;
432 struct io_rsrc_data *file_data;
433 struct io_rsrc_data *buf_data;
434
435 struct delayed_work rsrc_put_work;
436 struct llist_head rsrc_put_llist;
437 struct list_head rsrc_ref_list;
438 spinlock_t rsrc_ref_lock;
439 };
440
Jens Axboe3c1a2ea2021-02-11 10:48:03 -0700441 /* Keep this last, we don't need it for the fast path */
Pavel Begunkovb986af72021-05-16 22:58:06 +0100442 struct {
443 #if defined(CONFIG_UNIX)
444 struct socket *ring_sock;
445 #endif
446 /* hashed buffered write serialization */
447 struct io_wq_hash *hash_map;
448
449 /* Only used for accounting purposes */
450 struct user_struct *user;
451 struct mm_struct *mm_account;
452
453 /* ctx exit and cancelation */
Pavel Begunkov9011bf92021-06-30 21:54:03 +0100454 struct llist_head fallback_llist;
455 struct delayed_work fallback_work;
Pavel Begunkovb986af72021-05-16 22:58:06 +0100456 struct work_struct exit_work;
457 struct list_head tctx_list;
458 struct completion ref_comp;
Pavel Begunkove139a1e2021-10-19 23:43:46 +0100459 u32 iowq_limits[2];
460 bool iowq_limits_set;
Pavel Begunkovb986af72021-05-16 22:58:06 +0100461 };
Jens Axboe2b188cc2019-01-07 10:46:33 -0700462};
463
Stefan Metzmacher53e043b2021-03-15 12:56:56 +0100464struct io_uring_task {
465 /* submission side */
Pavel Begunkov09899b12021-06-14 02:36:22 +0100466 int cached_refs;
Stefan Metzmacher53e043b2021-03-15 12:56:56 +0100467 struct xarray xa;
468 struct wait_queue_head wait;
Stefan Metzmacheree53fb22021-03-15 12:56:57 +0100469 const struct io_ring_ctx *last;
470 struct io_wq *io_wq;
Stefan Metzmacher53e043b2021-03-15 12:56:56 +0100471 struct percpu_counter inflight;
Pavel Begunkovb303fe22021-04-11 01:46:26 +0100472 atomic_t inflight_tracked;
Stefan Metzmacher53e043b2021-03-15 12:56:56 +0100473 atomic_t in_idle;
Stefan Metzmacher53e043b2021-03-15 12:56:56 +0100474
475 spinlock_t task_lock;
476 struct io_wq_work_list task_list;
Stefan Metzmacher53e043b2021-03-15 12:56:56 +0100477 struct callback_head task_work;
Pavel Begunkov6294f362021-08-10 17:53:55 +0100478 bool task_running;
Stefan Metzmacher53e043b2021-03-15 12:56:56 +0100479};
480
Jens Axboe09bb8392019-03-13 12:39:28 -0600481/*
482 * First field must be the file pointer in all the
483 * iocb unions! See also 'struct kiocb' in <linux/fs.h>
484 */
Jens Axboe221c5eb2019-01-17 09:41:58 -0700485struct io_poll_iocb {
486 struct file *file;
Pavel Begunkov018043b2020-10-27 23:17:18 +0000487 struct wait_queue_head *head;
Jens Axboe221c5eb2019-01-17 09:41:58 -0700488 __poll_t events;
Jens Axboe392edb42019-12-09 17:52:20 -0700489 struct wait_queue_entry wait;
Jens Axboe221c5eb2019-01-17 09:41:58 -0700490};
491
Pavel Begunkov9d805892021-04-13 02:58:40 +0100492struct io_poll_update {
Pavel Begunkov018043b2020-10-27 23:17:18 +0000493 struct file *file;
Pavel Begunkov9d805892021-04-13 02:58:40 +0100494 u64 old_user_data;
495 u64 new_user_data;
496 __poll_t events;
Jens Axboeb69de282021-03-17 08:37:41 -0600497 bool update_events;
498 bool update_user_data;
Pavel Begunkov018043b2020-10-27 23:17:18 +0000499};
500
Jens Axboeb5dba592019-12-11 14:02:38 -0700501struct io_close {
502 struct file *file;
Jens Axboeb5dba592019-12-11 14:02:38 -0700503 int fd;
Pavel Begunkov7df778b2021-09-24 20:04:29 +0100504 u32 file_slot;
Jens Axboeb5dba592019-12-11 14:02:38 -0700505};
506
Jens Axboead8a48a2019-11-15 08:49:11 -0700507struct io_timeout_data {
508 struct io_kiocb *req;
509 struct hrtimer timer;
510 struct timespec64 ts;
511 enum hrtimer_mode mode;
Jens Axboe50c1df22021-08-27 17:11:06 -0600512 u32 flags;
Jens Axboead8a48a2019-11-15 08:49:11 -0700513};
514
Jens Axboe8ed8d3c2019-12-16 11:55:28 -0700515struct io_accept {
516 struct file *file;
517 struct sockaddr __user *addr;
518 int __user *addr_len;
519 int flags;
Pavel Begunkovaaa4db12021-08-25 12:25:47 +0100520 u32 file_slot;
Jens Axboe09952e32020-03-19 20:16:56 -0600521 unsigned long nofile;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -0700522};
523
524struct io_sync {
525 struct file *file;
526 loff_t len;
527 loff_t off;
528 int flags;
Jens Axboed63d1b52019-12-10 10:38:56 -0700529 int mode;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -0700530};
531
Jens Axboefbf23842019-12-17 18:45:56 -0700532struct io_cancel {
533 struct file *file;
534 u64 addr;
535};
536
Jens Axboeb29472e2019-12-17 18:50:29 -0700537struct io_timeout {
538 struct file *file;
Pavel Begunkovbfe68a22020-05-30 14:54:18 +0300539 u32 off;
540 u32 target_seq;
Pavel Begunkov135fcde2020-07-13 23:37:12 +0300541 struct list_head list;
Pavel Begunkov90cd7e42020-10-27 23:25:36 +0000542 /* head of the link, used by linked timeouts only */
543 struct io_kiocb *head;
Jens Axboe89b263f2021-08-10 15:14:18 -0600544 /* for linked completions */
545 struct io_kiocb *prev;
Jens Axboeb29472e2019-12-17 18:50:29 -0700546};
547
Pavel Begunkov0bdf7a22020-10-10 18:34:10 +0100548struct io_timeout_rem {
549 struct file *file;
550 u64 addr;
Pavel Begunkov9c8e11b2020-11-30 19:11:16 +0000551
552 /* timeout update */
553 struct timespec64 ts;
554 u32 flags;
Pavel Begunkovf1042b62021-08-28 19:54:39 -0600555 bool ltimeout;
Pavel Begunkov0bdf7a22020-10-10 18:34:10 +0100556};
557
Jens Axboe9adbd452019-12-20 08:45:55 -0700558struct io_rw {
559 /* NOTE: kiocb has the file as the first member, so don't do it here */
560 struct kiocb kiocb;
561 u64 addr;
562 u64 len;
563};
564
Jens Axboe3fbb51c2019-12-20 08:51:52 -0700565struct io_connect {
566 struct file *file;
567 struct sockaddr __user *addr;
568 int addr_len;
569};
570
Jens Axboee47293f2019-12-20 08:58:21 -0700571struct io_sr_msg {
572 struct file *file;
Jens Axboefddafac2020-01-04 20:19:44 -0700573 union {
Pavel Begunkov4af34172021-04-11 01:46:30 +0100574 struct compat_msghdr __user *umsg_compat;
575 struct user_msghdr __user *umsg;
576 void __user *buf;
Jens Axboefddafac2020-01-04 20:19:44 -0700577 };
Jens Axboee47293f2019-12-20 08:58:21 -0700578 int msg_flags;
Jens Axboebcda7ba2020-02-23 16:42:51 -0700579 int bgid;
Jens Axboefddafac2020-01-04 20:19:44 -0700580 size_t len;
Jens Axboe9b7b0f22023-01-21 10:21:22 -0700581 size_t done_io;
Jens Axboebcda7ba2020-02-23 16:42:51 -0700582 struct io_buffer *kbuf;
Jens Axboee47293f2019-12-20 08:58:21 -0700583};
584
Jens Axboe15b71ab2019-12-11 11:20:36 -0700585struct io_open {
586 struct file *file;
587 int dfd;
Pavel Begunkovb9445592021-08-25 12:25:45 +0100588 u32 file_slot;
Jens Axboe15b71ab2019-12-11 11:20:36 -0700589 struct filename *filename;
Jens Axboec12cedf2020-01-08 17:41:21 -0700590 struct open_how how;
Jens Axboe4022e7a2020-03-19 19:23:18 -0600591 unsigned long nofile;
Jens Axboe15b71ab2019-12-11 11:20:36 -0700592};
593
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +0000594struct io_rsrc_update {
Jens Axboe05f3fb32019-12-09 11:22:50 -0700595 struct file *file;
596 u64 arg;
597 u32 nr_args;
598 u32 offset;
599};
600
Jens Axboe4840e412019-12-25 22:03:45 -0700601struct io_fadvise {
602 struct file *file;
603 u64 offset;
604 u32 len;
605 u32 advice;
606};
607
Jens Axboec1ca7572019-12-25 22:18:28 -0700608struct io_madvise {
609 struct file *file;
610 u64 addr;
611 u32 len;
612 u32 advice;
613};
614
Jens Axboe3e4827b2020-01-08 15:18:09 -0700615struct io_epoll {
616 struct file *file;
617 int epfd;
618 int op;
619 int fd;
620 struct epoll_event event;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700621};
622
Pavel Begunkov7d67af22020-02-24 11:32:45 +0300623struct io_splice {
624 struct file *file_out;
Pavel Begunkov7d67af22020-02-24 11:32:45 +0300625 loff_t off_out;
626 loff_t off_in;
627 u64 len;
Jens Axboeae6cba32022-03-29 10:59:20 -0600628 int splice_fd_in;
Pavel Begunkov7d67af22020-02-24 11:32:45 +0300629 unsigned int flags;
630};
631
Jens Axboeddf0322d2020-02-23 16:41:33 -0700632struct io_provide_buf {
633 struct file *file;
634 __u64 addr;
Pavel Begunkov38134ad2021-04-15 13:07:39 +0100635 __u32 len;
Jens Axboeddf0322d2020-02-23 16:41:33 -0700636 __u32 bgid;
637 __u16 nbufs;
638 __u16 bid;
639};
640
Bijan Mottahedeh1d9e1282020-05-22 21:31:16 -0700641struct io_statx {
642 struct file *file;
643 int dfd;
644 unsigned int mask;
645 unsigned int flags;
Bijan Mottahedehe62753e2020-05-22 21:31:18 -0700646 const char __user *filename;
Bijan Mottahedeh1d9e1282020-05-22 21:31:16 -0700647 struct statx __user *buffer;
648};
649
Jens Axboe36f4fa62020-09-05 11:14:22 -0600650struct io_shutdown {
651 struct file *file;
652 int how;
653};
654
Jens Axboe80a261f2020-09-28 14:23:58 -0600655struct io_rename {
656 struct file *file;
657 int old_dfd;
658 int new_dfd;
659 struct filename *oldpath;
660 struct filename *newpath;
661 int flags;
662};
663
Jens Axboe14a11432020-09-28 14:27:37 -0600664struct io_unlink {
665 struct file *file;
666 int dfd;
667 int flags;
668 struct filename *filename;
669};
670
Dmitry Kadasheve34a02d2021-07-08 13:34:45 +0700671struct io_mkdir {
672 struct file *file;
673 int dfd;
674 umode_t mode;
675 struct filename *filename;
676};
677
Dmitry Kadashev7a8721f2021-07-08 13:34:46 +0700678struct io_symlink {
679 struct file *file;
680 int new_dfd;
681 struct filename *oldpath;
682 struct filename *newpath;
683};
684
Dmitry Kadashevcf30da92021-07-08 13:34:47 +0700685struct io_hardlink {
686 struct file *file;
687 int old_dfd;
688 int new_dfd;
689 struct filename *oldpath;
690 struct filename *newpath;
691 int flags;
692};
693
Pavel Begunkov3ca405e2020-07-13 23:37:08 +0300694struct io_completion {
695 struct file *file;
Pavel Begunkov8c3f9cd2021-02-28 22:35:15 +0000696 u32 cflags;
Pavel Begunkov3ca405e2020-07-13 23:37:08 +0300697};
698
Jens Axboef499a022019-12-02 16:28:46 -0700699struct io_async_connect {
700 struct sockaddr_storage address;
701};
702
Jens Axboe03b12302019-12-02 18:50:25 -0700703struct io_async_msghdr {
704 struct iovec fast_iov[UIO_FASTIOV];
Pavel Begunkov257e84a2021-02-05 00:58:00 +0000705 /* points to an allocated iov, if NULL we use fast_iov instead */
706 struct iovec *free_iov;
Jens Axboe03b12302019-12-02 18:50:25 -0700707 struct sockaddr __user *uaddr;
708 struct msghdr msg;
Jens Axboeb5379162020-02-09 11:29:15 -0700709 struct sockaddr_storage addr;
Jens Axboe03b12302019-12-02 18:50:25 -0700710};
711
Jens Axboef67676d2019-12-02 11:03:47 -0700712struct io_async_rw {
713 struct iovec fast_iov[UIO_FASTIOV];
Jens Axboeff6165b2020-08-13 09:47:43 -0600714 const struct iovec *free_iovec;
715 struct iov_iter iter;
Jens Axboecd658692021-09-10 11:19:14 -0600716 struct iov_iter_state iter_state;
Jens Axboe227c0c92020-08-13 11:51:40 -0600717 size_t bytes_done;
Jens Axboebcf5a062020-05-22 09:24:42 -0600718 struct wait_page_queue wpq;
Jens Axboef67676d2019-12-02 11:03:47 -0700719};
720
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300721enum {
722 REQ_F_FIXED_FILE_BIT = IOSQE_FIXED_FILE_BIT,
723 REQ_F_IO_DRAIN_BIT = IOSQE_IO_DRAIN_BIT,
724 REQ_F_LINK_BIT = IOSQE_IO_LINK_BIT,
725 REQ_F_HARDLINK_BIT = IOSQE_IO_HARDLINK_BIT,
726 REQ_F_FORCE_ASYNC_BIT = IOSQE_ASYNC_BIT,
Jens Axboebcda7ba2020-02-23 16:42:51 -0700727 REQ_F_BUFFER_SELECT_BIT = IOSQE_BUFFER_SELECT_BIT,
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300728
Pavel Begunkovdddca222021-04-27 16:13:52 +0100729 /* first byte is taken by user flags, shift it to not overlap */
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +0100730 REQ_F_FAIL_BIT = 8,
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300731 REQ_F_INFLIGHT_BIT,
732 REQ_F_CUR_POS_BIT,
733 REQ_F_NOWAIT_BIT,
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300734 REQ_F_LINK_TIMEOUT_BIT,
Pavel Begunkov99bc4c32020-02-07 22:04:45 +0300735 REQ_F_NEED_CLEANUP_BIT,
Jens Axboed7718a92020-02-14 22:23:12 -0700736 REQ_F_POLLED_BIT,
Jens Axboebcda7ba2020-02-23 16:42:51 -0700737 REQ_F_BUFFER_SELECTED_BIT,
Pavel Begunkove342c802021-01-19 13:32:47 +0000738 REQ_F_COMPLETE_INLINE_BIT,
Jens Axboe230d50d2021-04-01 20:41:15 -0600739 REQ_F_REISSUE_BIT,
Pavel Begunkovb8e64b52021-06-17 18:14:02 +0100740 REQ_F_CREDS_BIT,
Pavel Begunkov20e60a32021-08-11 19:28:30 +0100741 REQ_F_REFCOUNT_BIT,
Pavel Begunkov4d13d1a2021-08-15 10:40:24 +0100742 REQ_F_ARM_LTIMEOUT_BIT,
Jens Axboe390b8812022-03-23 09:30:05 -0600743 REQ_F_PARTIAL_IO_BIT,
Jens Axboe7b29f922021-03-12 08:30:14 -0700744 /* keep async read/write and isreg together and in order */
Pavel Begunkovb191e2d2021-08-09 13:04:03 +0100745 REQ_F_NOWAIT_READ_BIT,
746 REQ_F_NOWAIT_WRITE_BIT,
Jens Axboe7b29f922021-03-12 08:30:14 -0700747 REQ_F_ISREG_BIT,
Jens Axboe84557872020-03-03 15:28:17 -0700748
749 /* not a real bit, just to check we're not overflowing the space */
750 __REQ_F_LAST_BIT,
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300751};
752
753enum {
754 /* ctx owns file */
755 REQ_F_FIXED_FILE = BIT(REQ_F_FIXED_FILE_BIT),
756 /* drain existing IO first */
757 REQ_F_IO_DRAIN = BIT(REQ_F_IO_DRAIN_BIT),
758 /* linked sqes */
759 REQ_F_LINK = BIT(REQ_F_LINK_BIT),
760 /* doesn't sever on completion < 0 */
761 REQ_F_HARDLINK = BIT(REQ_F_HARDLINK_BIT),
762 /* IOSQE_ASYNC */
763 REQ_F_FORCE_ASYNC = BIT(REQ_F_FORCE_ASYNC_BIT),
Jens Axboebcda7ba2020-02-23 16:42:51 -0700764 /* IOSQE_BUFFER_SELECT */
765 REQ_F_BUFFER_SELECT = BIT(REQ_F_BUFFER_SELECT_BIT),
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300766
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300767 /* fail rest of links */
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +0100768 REQ_F_FAIL = BIT(REQ_F_FAIL_BIT),
Pavel Begunkovb05a1bc2021-03-04 13:59:24 +0000769 /* on inflight list, should be cancelled and waited on exit reliably */
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300770 REQ_F_INFLIGHT = BIT(REQ_F_INFLIGHT_BIT),
771 /* read/write uses file position */
772 REQ_F_CUR_POS = BIT(REQ_F_CUR_POS_BIT),
773 /* must not punt to workers */
774 REQ_F_NOWAIT = BIT(REQ_F_NOWAIT_BIT),
Pavel Begunkov900fad42020-10-19 16:39:16 +0100775 /* has or had linked timeout */
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300776 REQ_F_LINK_TIMEOUT = BIT(REQ_F_LINK_TIMEOUT_BIT),
Pavel Begunkov99bc4c32020-02-07 22:04:45 +0300777 /* needs cleanup */
778 REQ_F_NEED_CLEANUP = BIT(REQ_F_NEED_CLEANUP_BIT),
Jens Axboed7718a92020-02-14 22:23:12 -0700779 /* already went through poll handler */
780 REQ_F_POLLED = BIT(REQ_F_POLLED_BIT),
Jens Axboebcda7ba2020-02-23 16:42:51 -0700781 /* buffer already selected */
782 REQ_F_BUFFER_SELECTED = BIT(REQ_F_BUFFER_SELECTED_BIT),
Pavel Begunkove342c802021-01-19 13:32:47 +0000783 /* completion is deferred through io_comp_state */
784 REQ_F_COMPLETE_INLINE = BIT(REQ_F_COMPLETE_INLINE_BIT),
Jens Axboe230d50d2021-04-01 20:41:15 -0600785 /* caller should reissue async */
786 REQ_F_REISSUE = BIT(REQ_F_REISSUE_BIT),
Jens Axboe7b29f922021-03-12 08:30:14 -0700787 /* supports async reads */
Pavel Begunkovb191e2d2021-08-09 13:04:03 +0100788 REQ_F_NOWAIT_READ = BIT(REQ_F_NOWAIT_READ_BIT),
Jens Axboe7b29f922021-03-12 08:30:14 -0700789 /* supports async writes */
Pavel Begunkovb191e2d2021-08-09 13:04:03 +0100790 REQ_F_NOWAIT_WRITE = BIT(REQ_F_NOWAIT_WRITE_BIT),
Jens Axboe7b29f922021-03-12 08:30:14 -0700791 /* regular file */
792 REQ_F_ISREG = BIT(REQ_F_ISREG_BIT),
Pavel Begunkovb8e64b52021-06-17 18:14:02 +0100793 /* has creds assigned */
794 REQ_F_CREDS = BIT(REQ_F_CREDS_BIT),
Pavel Begunkov20e60a32021-08-11 19:28:30 +0100795 /* skip refcounting if not set */
796 REQ_F_REFCOUNT = BIT(REQ_F_REFCOUNT_BIT),
Pavel Begunkov4d13d1a2021-08-15 10:40:24 +0100797 /* there is a linked timeout that has to be armed */
798 REQ_F_ARM_LTIMEOUT = BIT(REQ_F_ARM_LTIMEOUT_BIT),
Jens Axboe390b8812022-03-23 09:30:05 -0600799 /* request has already done partial IO */
800 REQ_F_PARTIAL_IO = BIT(REQ_F_PARTIAL_IO_BIT),
Jens Axboed7718a92020-02-14 22:23:12 -0700801};
802
803struct async_poll {
804 struct io_poll_iocb poll;
Jens Axboe807abcb2020-07-17 17:09:27 -0600805 struct io_poll_iocb *double_poll;
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300806};
807
Pavel Begunkovf237c302021-08-18 12:42:46 +0100808typedef void (*io_req_tw_func_t)(struct io_kiocb *req, bool *locked);
Pavel Begunkov5b0a6ac2021-06-30 21:54:04 +0100809
Jens Axboe7cbf1722021-02-10 00:03:20 +0000810struct io_task_work {
Pavel Begunkov5b0a6ac2021-06-30 21:54:04 +0100811 union {
812 struct io_wq_work_node node;
813 struct llist_node fallback_node;
814 };
815 io_req_tw_func_t func;
Jens Axboe7cbf1722021-02-10 00:03:20 +0000816};
817
Pavel Begunkov992da012021-06-10 16:37:37 +0100818enum {
819 IORING_RSRC_FILE = 0,
820 IORING_RSRC_BUFFER = 1,
821};
822
Jens Axboe09bb8392019-03-13 12:39:28 -0600823/*
824 * NOTE! Each of the iocb union members has the file pointer
825 * as the first entry in their struct definition. So you can
826 * access the file pointer through any of the sub-structs,
827 * or directly as just 'ki_filp' in this struct.
828 */
Jens Axboe2b188cc2019-01-07 10:46:33 -0700829struct io_kiocb {
Jens Axboe221c5eb2019-01-17 09:41:58 -0700830 union {
Jens Axboe09bb8392019-03-13 12:39:28 -0600831 struct file *file;
Jens Axboe9adbd452019-12-20 08:45:55 -0700832 struct io_rw rw;
Jens Axboe221c5eb2019-01-17 09:41:58 -0700833 struct io_poll_iocb poll;
Pavel Begunkov9d805892021-04-13 02:58:40 +0100834 struct io_poll_update poll_update;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -0700835 struct io_accept accept;
836 struct io_sync sync;
Jens Axboefbf23842019-12-17 18:45:56 -0700837 struct io_cancel cancel;
Jens Axboeb29472e2019-12-17 18:50:29 -0700838 struct io_timeout timeout;
Pavel Begunkov0bdf7a22020-10-10 18:34:10 +0100839 struct io_timeout_rem timeout_rem;
Jens Axboe3fbb51c2019-12-20 08:51:52 -0700840 struct io_connect connect;
Jens Axboee47293f2019-12-20 08:58:21 -0700841 struct io_sr_msg sr_msg;
Jens Axboe15b71ab2019-12-11 11:20:36 -0700842 struct io_open open;
Jens Axboeb5dba592019-12-11 14:02:38 -0700843 struct io_close close;
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +0000844 struct io_rsrc_update rsrc_update;
Jens Axboe4840e412019-12-25 22:03:45 -0700845 struct io_fadvise fadvise;
Jens Axboec1ca7572019-12-25 22:18:28 -0700846 struct io_madvise madvise;
Jens Axboe3e4827b2020-01-08 15:18:09 -0700847 struct io_epoll epoll;
Pavel Begunkov7d67af22020-02-24 11:32:45 +0300848 struct io_splice splice;
Jens Axboeddf0322d2020-02-23 16:41:33 -0700849 struct io_provide_buf pbuf;
Bijan Mottahedeh1d9e1282020-05-22 21:31:16 -0700850 struct io_statx statx;
Jens Axboe36f4fa62020-09-05 11:14:22 -0600851 struct io_shutdown shutdown;
Jens Axboe80a261f2020-09-28 14:23:58 -0600852 struct io_rename rename;
Jens Axboe14a11432020-09-28 14:27:37 -0600853 struct io_unlink unlink;
Dmitry Kadasheve34a02d2021-07-08 13:34:45 +0700854 struct io_mkdir mkdir;
Dmitry Kadashev7a8721f2021-07-08 13:34:46 +0700855 struct io_symlink symlink;
Dmitry Kadashevcf30da92021-07-08 13:34:47 +0700856 struct io_hardlink hardlink;
Pavel Begunkov3ca405e2020-07-13 23:37:08 +0300857 /* use only after cleaning per-op data, see io_clean_op() */
858 struct io_completion compl;
Jens Axboe221c5eb2019-01-17 09:41:58 -0700859 };
Jens Axboe2b188cc2019-01-07 10:46:33 -0700860
Jens Axboee8c2bc12020-08-15 18:44:09 -0700861 /* opcode allocated if it needs to store data for async defer */
862 void *async_data;
Jens Axboed625c6e2019-12-17 19:53:05 -0700863 u8 opcode;
Xiaoguang Wang65a65432020-06-11 23:39:36 +0800864 /* polled IO has completed */
865 u8 iopoll_completed;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700866
Bijan Mottahedeh4f4eeba2020-05-19 14:52:49 -0700867 u16 buf_index;
Pavel Begunkov9cf7c102020-07-13 23:37:15 +0300868 u32 result;
Bijan Mottahedeh4f4eeba2020-05-19 14:52:49 -0700869
Pavel Begunkov010e8e62020-07-30 18:43:45 +0300870 struct io_ring_ctx *ctx;
871 unsigned int flags;
Jens Axboeabc54d62021-02-24 13:32:30 -0700872 atomic_t refs;
Pavel Begunkov010e8e62020-07-30 18:43:45 +0300873 struct task_struct *task;
874 u64 user_data;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700875
Pavel Begunkovf2f87372020-10-27 23:25:37 +0000876 struct io_kiocb *link;
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +0000877 struct percpu_ref *fixed_rsrc_refs;
Jens Axboed7718a92020-02-14 22:23:12 -0700878
Pavel Begunkovb303fe22021-04-11 01:46:26 +0100879 /* used with ctx->iopoll_list with reads/writes */
Pavel Begunkov010e8e62020-07-30 18:43:45 +0300880 struct list_head inflight_entry;
Pavel Begunkov5b0a6ac2021-06-30 21:54:04 +0100881 struct io_task_work io_task_work;
Pavel Begunkov010e8e62020-07-30 18:43:45 +0300882 /* for polled requests, i.e. IORING_OP_POLL_ADD and async armed poll */
883 struct hlist_node hash_node;
884 struct async_poll *apoll;
885 struct io_wq_work work;
Pavel Begunkovfe7e3252021-06-24 15:09:57 +0100886 const struct cred *creds;
Pavel Begunkovc10d1f92021-06-17 18:14:01 +0100887
Pavel Begunkoveae071c2021-04-25 14:32:24 +0100888 /* store used ubuf, so we can prevent reloading */
889 struct io_mapped_ubuf *imu;
Pavel Begunkovf770fba2022-08-29 14:30:18 +0100890 /* stores selected buf, valid IFF REQ_F_BUFFER_SELECTED is set */
891 struct io_buffer *kbuf;
892 atomic_t poll_refs;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700893};
894
Pavel Begunkov13bf43f2021-03-06 11:02:12 +0000895struct io_tctx_node {
896 struct list_head ctx_node;
897 struct task_struct *task;
Pavel Begunkov13bf43f2021-03-06 11:02:12 +0000898 struct io_ring_ctx *ctx;
899};
900
Pavel Begunkov27dc8332020-07-13 23:37:14 +0300901struct io_defer_entry {
902 struct list_head list;
903 struct io_kiocb *req;
Pavel Begunkov9cf7c102020-07-13 23:37:15 +0300904 u32 seq;
Pavel Begunkov27dc8332020-07-13 23:37:14 +0300905};
906
Jens Axboed3656342019-12-18 09:50:26 -0700907struct io_op_def {
Jens Axboed3656342019-12-18 09:50:26 -0700908 /* needs req->file assigned */
909 unsigned needs_file : 1;
Jens Axboed3656342019-12-18 09:50:26 -0700910 /* hash wq insertion if file is a regular file */
911 unsigned hash_reg_file : 1;
912 /* unbound wq insertion if file is a non-regular file */
913 unsigned unbound_nonreg_file : 1;
Jens Axboe66f4af92020-01-16 15:36:52 -0700914 /* opcode is not supported by this kernel */
915 unsigned not_supported : 1;
Jens Axboe8a727582020-02-20 09:59:44 -0700916 /* set if opcode supports polled "wait" */
917 unsigned pollin : 1;
918 unsigned pollout : 1;
Jens Axboebcda7ba2020-02-23 16:42:51 -0700919 /* op supports buffer selection */
920 unsigned buffer_select : 1;
Pavel Begunkov26f05052021-02-28 22:35:18 +0000921 /* do prep async if is going to be punted */
922 unsigned needs_async_setup : 1;
Jens Axboe27926b62020-10-28 09:33:23 -0600923 /* should block plug */
924 unsigned plug : 1;
Jens Axboee8c2bc12020-08-15 18:44:09 -0700925 /* size of async data needed, if any */
926 unsigned short async_size;
Jens Axboed3656342019-12-18 09:50:26 -0700927};
928
Jens Axboe09186822020-10-13 15:01:40 -0600929static const struct io_op_def io_op_defs[] = {
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300930 [IORING_OP_NOP] = {},
931 [IORING_OP_READV] = {
Jens Axboed3656342019-12-18 09:50:26 -0700932 .needs_file = 1,
933 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -0700934 .pollin = 1,
Jens Axboe4d954c22020-02-27 07:31:19 -0700935 .buffer_select = 1,
Pavel Begunkov26f05052021-02-28 22:35:18 +0000936 .needs_async_setup = 1,
Jens Axboe27926b62020-10-28 09:33:23 -0600937 .plug = 1,
Jens Axboee8c2bc12020-08-15 18:44:09 -0700938 .async_size = sizeof(struct io_async_rw),
Jens Axboed3656342019-12-18 09:50:26 -0700939 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300940 [IORING_OP_WRITEV] = {
Jens Axboed3656342019-12-18 09:50:26 -0700941 .needs_file = 1,
942 .hash_reg_file = 1,
943 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -0700944 .pollout = 1,
Pavel Begunkov26f05052021-02-28 22:35:18 +0000945 .needs_async_setup = 1,
Jens Axboe27926b62020-10-28 09:33:23 -0600946 .plug = 1,
Jens Axboee8c2bc12020-08-15 18:44:09 -0700947 .async_size = sizeof(struct io_async_rw),
Jens Axboed3656342019-12-18 09:50:26 -0700948 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300949 [IORING_OP_FSYNC] = {
Jens Axboed3656342019-12-18 09:50:26 -0700950 .needs_file = 1,
951 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300952 [IORING_OP_READ_FIXED] = {
Jens Axboed3656342019-12-18 09:50:26 -0700953 .needs_file = 1,
954 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -0700955 .pollin = 1,
Jens Axboe27926b62020-10-28 09:33:23 -0600956 .plug = 1,
Jens Axboee8c2bc12020-08-15 18:44:09 -0700957 .async_size = sizeof(struct io_async_rw),
Jens Axboed3656342019-12-18 09:50:26 -0700958 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300959 [IORING_OP_WRITE_FIXED] = {
Jens Axboed3656342019-12-18 09:50:26 -0700960 .needs_file = 1,
961 .hash_reg_file = 1,
962 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -0700963 .pollout = 1,
Jens Axboe27926b62020-10-28 09:33:23 -0600964 .plug = 1,
Jens Axboee8c2bc12020-08-15 18:44:09 -0700965 .async_size = sizeof(struct io_async_rw),
Jens Axboed3656342019-12-18 09:50:26 -0700966 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300967 [IORING_OP_POLL_ADD] = {
Jens Axboed3656342019-12-18 09:50:26 -0700968 .needs_file = 1,
969 .unbound_nonreg_file = 1,
970 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300971 [IORING_OP_POLL_REMOVE] = {},
972 [IORING_OP_SYNC_FILE_RANGE] = {
Jens Axboed3656342019-12-18 09:50:26 -0700973 .needs_file = 1,
974 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300975 [IORING_OP_SENDMSG] = {
Jens Axboed3656342019-12-18 09:50:26 -0700976 .needs_file = 1,
977 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -0700978 .pollout = 1,
Pavel Begunkov26f05052021-02-28 22:35:18 +0000979 .needs_async_setup = 1,
Jens Axboee8c2bc12020-08-15 18:44:09 -0700980 .async_size = sizeof(struct io_async_msghdr),
Jens Axboed3656342019-12-18 09:50:26 -0700981 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300982 [IORING_OP_RECVMSG] = {
Jens Axboed3656342019-12-18 09:50:26 -0700983 .needs_file = 1,
984 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -0700985 .pollin = 1,
Jens Axboe52de1fe2020-02-27 10:15:42 -0700986 .buffer_select = 1,
Pavel Begunkov26f05052021-02-28 22:35:18 +0000987 .needs_async_setup = 1,
Jens Axboee8c2bc12020-08-15 18:44:09 -0700988 .async_size = sizeof(struct io_async_msghdr),
Jens Axboed3656342019-12-18 09:50:26 -0700989 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300990 [IORING_OP_TIMEOUT] = {
Jens Axboee8c2bc12020-08-15 18:44:09 -0700991 .async_size = sizeof(struct io_timeout_data),
Jens Axboed3656342019-12-18 09:50:26 -0700992 },
Pavel Begunkov9c8e11b2020-11-30 19:11:16 +0000993 [IORING_OP_TIMEOUT_REMOVE] = {
994 /* used by timeout updates' prep() */
Pavel Begunkov9c8e11b2020-11-30 19:11:16 +0000995 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300996 [IORING_OP_ACCEPT] = {
Jens Axboed3656342019-12-18 09:50:26 -0700997 .needs_file = 1,
998 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -0700999 .pollin = 1,
Jens Axboed3656342019-12-18 09:50:26 -07001000 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +03001001 [IORING_OP_ASYNC_CANCEL] = {},
1002 [IORING_OP_LINK_TIMEOUT] = {
Jens Axboee8c2bc12020-08-15 18:44:09 -07001003 .async_size = sizeof(struct io_timeout_data),
Jens Axboed3656342019-12-18 09:50:26 -07001004 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +03001005 [IORING_OP_CONNECT] = {
Jens Axboed3656342019-12-18 09:50:26 -07001006 .needs_file = 1,
1007 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -07001008 .pollout = 1,
Pavel Begunkov26f05052021-02-28 22:35:18 +00001009 .needs_async_setup = 1,
Jens Axboee8c2bc12020-08-15 18:44:09 -07001010 .async_size = sizeof(struct io_async_connect),
Jens Axboed3656342019-12-18 09:50:26 -07001011 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +03001012 [IORING_OP_FALLOCATE] = {
Jens Axboed3656342019-12-18 09:50:26 -07001013 .needs_file = 1,
1014 },
Jens Axboe44526be2021-02-15 13:32:18 -07001015 [IORING_OP_OPENAT] = {},
1016 [IORING_OP_CLOSE] = {},
1017 [IORING_OP_FILES_UPDATE] = {},
1018 [IORING_OP_STATX] = {},
Pavel Begunkov0463b6c2020-01-18 21:35:38 +03001019 [IORING_OP_READ] = {
Jens Axboe3a6820f2019-12-22 15:19:35 -07001020 .needs_file = 1,
1021 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -07001022 .pollin = 1,
Jens Axboebcda7ba2020-02-23 16:42:51 -07001023 .buffer_select = 1,
Jens Axboe27926b62020-10-28 09:33:23 -06001024 .plug = 1,
Jens Axboee8c2bc12020-08-15 18:44:09 -07001025 .async_size = sizeof(struct io_async_rw),
Jens Axboe3a6820f2019-12-22 15:19:35 -07001026 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +03001027 [IORING_OP_WRITE] = {
Jens Axboe3a6820f2019-12-22 15:19:35 -07001028 .needs_file = 1,
Jens Axboe7b3188e2021-08-30 19:37:41 -06001029 .hash_reg_file = 1,
Jens Axboe3a6820f2019-12-22 15:19:35 -07001030 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -07001031 .pollout = 1,
Jens Axboe27926b62020-10-28 09:33:23 -06001032 .plug = 1,
Jens Axboee8c2bc12020-08-15 18:44:09 -07001033 .async_size = sizeof(struct io_async_rw),
Jens Axboe3a6820f2019-12-22 15:19:35 -07001034 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +03001035 [IORING_OP_FADVISE] = {
Jens Axboe4840e412019-12-25 22:03:45 -07001036 .needs_file = 1,
1037 },
Jens Axboe44526be2021-02-15 13:32:18 -07001038 [IORING_OP_MADVISE] = {},
Pavel Begunkov0463b6c2020-01-18 21:35:38 +03001039 [IORING_OP_SEND] = {
Jens Axboefddafac2020-01-04 20:19:44 -07001040 .needs_file = 1,
1041 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -07001042 .pollout = 1,
Jens Axboefddafac2020-01-04 20:19:44 -07001043 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +03001044 [IORING_OP_RECV] = {
Jens Axboefddafac2020-01-04 20:19:44 -07001045 .needs_file = 1,
1046 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -07001047 .pollin = 1,
Jens Axboebcda7ba2020-02-23 16:42:51 -07001048 .buffer_select = 1,
Jens Axboefddafac2020-01-04 20:19:44 -07001049 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +03001050 [IORING_OP_OPENAT2] = {
Jens Axboecebdb982020-01-08 17:59:24 -07001051 },
Jens Axboe3e4827b2020-01-08 15:18:09 -07001052 [IORING_OP_EPOLL_CTL] = {
1053 .unbound_nonreg_file = 1,
Jens Axboe3e4827b2020-01-08 15:18:09 -07001054 },
Pavel Begunkov7d67af22020-02-24 11:32:45 +03001055 [IORING_OP_SPLICE] = {
1056 .needs_file = 1,
1057 .hash_reg_file = 1,
1058 .unbound_nonreg_file = 1,
Jens Axboeddf0322d2020-02-23 16:41:33 -07001059 },
1060 [IORING_OP_PROVIDE_BUFFERS] = {},
Jens Axboe067524e2020-03-02 16:32:28 -07001061 [IORING_OP_REMOVE_BUFFERS] = {},
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03001062 [IORING_OP_TEE] = {
1063 .needs_file = 1,
1064 .hash_reg_file = 1,
1065 .unbound_nonreg_file = 1,
1066 },
Jens Axboe36f4fa62020-09-05 11:14:22 -06001067 [IORING_OP_SHUTDOWN] = {
1068 .needs_file = 1,
1069 },
Jens Axboe44526be2021-02-15 13:32:18 -07001070 [IORING_OP_RENAMEAT] = {},
1071 [IORING_OP_UNLINKAT] = {},
Dmitry Kadasheve34a02d2021-07-08 13:34:45 +07001072 [IORING_OP_MKDIRAT] = {},
Dmitry Kadashev7a8721f2021-07-08 13:34:46 +07001073 [IORING_OP_SYMLINKAT] = {},
Dmitry Kadashevcf30da92021-07-08 13:34:47 +07001074 [IORING_OP_LINKAT] = {},
Jens Axboed3656342019-12-18 09:50:26 -07001075};
1076
Pavel Begunkov0756a862021-08-15 10:40:25 +01001077/* requests with any of those set should undergo io_disarm_next() */
1078#define IO_DISARM_MASK (REQ_F_ARM_LTIMEOUT | REQ_F_LINK_TIMEOUT | REQ_F_FAIL)
1079
Pavel Begunkov7a612352021-03-09 00:37:59 +00001080static bool io_disarm_next(struct io_kiocb *req);
Pavel Begunkoveef51da2021-06-14 02:36:15 +01001081static void io_uring_del_tctx_node(unsigned long index);
Pavel Begunkov9936c7c2021-02-04 13:51:56 +00001082static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
1083 struct task_struct *task,
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01001084 bool cancel_all);
Pavel Begunkov78cc6872021-06-14 02:36:23 +01001085static void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd);
Pavel Begunkov1ffc5422020-12-30 21:34:15 +00001086
Pavel Begunkovb850d6d2022-08-29 14:30:13 +01001087static void io_fill_cqe_req(struct io_kiocb *req, s32 res, u32 cflags);
1088
Jackie Liuec9c02a2019-11-08 23:50:36 +08001089static void io_put_req(struct io_kiocb *req);
Pavel Begunkov91c2f692021-08-11 19:28:28 +01001090static void io_put_req_deferred(struct io_kiocb *req);
Jens Axboec7dae4b2021-02-09 19:53:37 -07001091static void io_dismantle_req(struct io_kiocb *req);
Jens Axboe94ae5e72019-11-14 19:39:52 -07001092static void io_queue_linked_timeout(struct io_kiocb *req);
Pavel Begunkovfdecb662021-04-25 14:32:20 +01001093static int __io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned type,
Pavel Begunkovc3bdad02021-04-25 14:32:22 +01001094 struct io_uring_rsrc_update2 *up,
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +01001095 unsigned nr_args);
Pavel Begunkov68fb8972021-03-19 17:22:41 +00001096static void io_clean_op(struct io_kiocb *req);
Pavel Begunkovac177052021-08-09 13:04:02 +01001097static struct file *io_file_get(struct io_ring_ctx *ctx,
Pavel Begunkov8371adf2020-10-10 18:34:08 +01001098 struct io_kiocb *req, int fd, bool fixed);
Pavel Begunkovc5eef2b2021-02-10 00:03:22 +00001099static void __io_queue_sqe(struct io_kiocb *req);
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00001100static void io_rsrc_put_work(struct work_struct *work);
Jens Axboede0617e2019-04-06 21:51:27 -06001101
Pavel Begunkov907d1df2021-01-26 23:35:10 +00001102static void io_req_task_queue(struct io_kiocb *req);
Pavel Begunkov2a2758f2021-06-17 18:14:00 +01001103static void io_submit_flush_completions(struct io_ring_ctx *ctx);
Pavel Begunkov179ae0d2021-02-28 22:35:20 +00001104static int io_req_prep_async(struct io_kiocb *req);
Jens Axboe9a56a232019-01-09 09:06:50 -07001105
Pavel Begunkovb9445592021-08-25 12:25:45 +01001106static int io_install_fixed_file(struct io_kiocb *req, struct file *file,
1107 unsigned int issue_flags, u32 slot_index);
Pavel Begunkov7df778b2021-09-24 20:04:29 +01001108static int io_close_fixed(struct io_kiocb *req, unsigned int issue_flags);
1109
Pavel Begunkovf1042b62021-08-28 19:54:39 -06001110static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer);
Pavel Begunkovb9445592021-08-25 12:25:45 +01001111
Jens Axboe2b188cc2019-01-07 10:46:33 -07001112static struct kmem_cache *req_cachep;
1113
Jens Axboe09186822020-10-13 15:01:40 -06001114static const struct file_operations io_uring_fops;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001115
1116struct sock *io_uring_get_socket(struct file *file)
1117{
1118#if defined(CONFIG_UNIX)
1119 if (file->f_op == &io_uring_fops) {
1120 struct io_ring_ctx *ctx = file->private_data;
1121
1122 return ctx->ring_sock->sk;
1123 }
1124#endif
1125 return NULL;
1126}
1127EXPORT_SYMBOL(io_uring_get_socket);
1128
Pavel Begunkovf237c302021-08-18 12:42:46 +01001129static inline void io_tw_lock(struct io_ring_ctx *ctx, bool *locked)
1130{
1131 if (!*locked) {
1132 mutex_lock(&ctx->uring_lock);
1133 *locked = true;
1134 }
1135}
1136
Pavel Begunkovf2f87372020-10-27 23:25:37 +00001137#define io_for_each_link(pos, head) \
1138 for (pos = (head); pos; pos = pos->link)
1139
Pavel Begunkov21c843d2021-08-11 19:28:27 +01001140/*
1141 * Shamelessly stolen from the mm implementation of page reference checking,
1142 * see commit f958d7b528b1 for details.
1143 */
1144#define req_ref_zero_or_close_to_overflow(req) \
1145 ((unsigned int) atomic_read(&(req->refs)) + 127u <= 127u)
1146
1147static inline bool req_ref_inc_not_zero(struct io_kiocb *req)
1148{
Pavel Begunkov20e60a32021-08-11 19:28:30 +01001149 WARN_ON_ONCE(!(req->flags & REQ_F_REFCOUNT));
Pavel Begunkov21c843d2021-08-11 19:28:27 +01001150 return atomic_inc_not_zero(&req->refs);
1151}
1152
Pavel Begunkov21c843d2021-08-11 19:28:27 +01001153static inline bool req_ref_put_and_test(struct io_kiocb *req)
1154{
Pavel Begunkov20e60a32021-08-11 19:28:30 +01001155 if (likely(!(req->flags & REQ_F_REFCOUNT)))
1156 return true;
1157
Pavel Begunkov21c843d2021-08-11 19:28:27 +01001158 WARN_ON_ONCE(req_ref_zero_or_close_to_overflow(req));
1159 return atomic_dec_and_test(&req->refs);
1160}
1161
Pavel Begunkov21c843d2021-08-11 19:28:27 +01001162static inline void req_ref_get(struct io_kiocb *req)
1163{
Pavel Begunkov20e60a32021-08-11 19:28:30 +01001164 WARN_ON_ONCE(!(req->flags & REQ_F_REFCOUNT));
Pavel Begunkov21c843d2021-08-11 19:28:27 +01001165 WARN_ON_ONCE(req_ref_zero_or_close_to_overflow(req));
1166 atomic_inc(&req->refs);
1167}
1168
Pavel Begunkov48dcd382021-08-15 10:40:18 +01001169static inline void __io_req_set_refcount(struct io_kiocb *req, int nr)
Pavel Begunkov20e60a32021-08-11 19:28:30 +01001170{
1171 if (!(req->flags & REQ_F_REFCOUNT)) {
1172 req->flags |= REQ_F_REFCOUNT;
Pavel Begunkov48dcd382021-08-15 10:40:18 +01001173 atomic_set(&req->refs, nr);
Pavel Begunkov20e60a32021-08-11 19:28:30 +01001174 }
1175}
1176
Pavel Begunkov48dcd382021-08-15 10:40:18 +01001177static inline void io_req_set_refcount(struct io_kiocb *req)
1178{
1179 __io_req_set_refcount(req, 1);
1180}
1181
Pavel Begunkovb895c9a2021-04-01 15:43:40 +01001182static inline void io_req_set_rsrc_node(struct io_kiocb *req)
Jens Axboec40f6372020-06-25 15:39:59 -06001183{
Pavel Begunkov36f72fe2020-11-18 19:57:26 +00001184 struct io_ring_ctx *ctx = req->ctx;
1185
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00001186 if (!req->fixed_rsrc_refs) {
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01001187 req->fixed_rsrc_refs = &ctx->rsrc_node->refs;
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00001188 percpu_ref_get(req->fixed_rsrc_refs);
Pavel Begunkov36f72fe2020-11-18 19:57:26 +00001189 }
1190}
1191
Pavel Begunkovf70865d2021-04-11 01:46:40 +01001192static void io_refs_resurrect(struct percpu_ref *ref, struct completion *compl)
1193{
1194 bool got = percpu_ref_tryget(ref);
1195
1196 /* already at zero, wait for ->release() */
1197 if (!got)
1198 wait_for_completion(compl);
1199 percpu_ref_resurrect(ref);
1200 if (got)
1201 percpu_ref_put(ref);
1202}
1203
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01001204static bool io_match_task(struct io_kiocb *head, struct task_struct *task,
1205 bool cancel_all)
Pavel Begunkov1c939a52021-11-26 14:38:15 +00001206 __must_hold(&req->ctx->timeout_lock)
Pavel Begunkov08d23632020-11-06 13:00:22 +00001207{
1208 struct io_kiocb *req;
1209
Pavel Begunkov68207682021-03-22 01:58:25 +00001210 if (task && head->task != task)
Pavel Begunkov08d23632020-11-06 13:00:22 +00001211 return false;
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01001212 if (cancel_all)
Pavel Begunkov08d23632020-11-06 13:00:22 +00001213 return true;
1214
1215 io_for_each_link(req, head) {
Pavel Begunkovb05a1bc2021-03-04 13:59:24 +00001216 if (req->flags & REQ_F_INFLIGHT)
Jens Axboe02a13672021-01-23 15:49:31 -07001217 return true;
Pavel Begunkov08d23632020-11-06 13:00:22 +00001218 }
1219 return false;
1220}
1221
Pavel Begunkov1c939a52021-11-26 14:38:15 +00001222static bool io_match_linked(struct io_kiocb *head)
1223{
1224 struct io_kiocb *req;
1225
1226 io_for_each_link(req, head) {
1227 if (req->flags & REQ_F_INFLIGHT)
1228 return true;
1229 }
1230 return false;
1231}
1232
1233/*
1234 * As io_match_task() but protected against racing with linked timeouts.
1235 * User must not hold timeout_lock.
1236 */
1237static bool io_match_task_safe(struct io_kiocb *head, struct task_struct *task,
1238 bool cancel_all)
1239{
1240 bool matched;
1241
1242 if (task && head->task != task)
1243 return false;
1244 if (cancel_all)
1245 return true;
1246
1247 if (head->flags & REQ_F_LINK_TIMEOUT) {
1248 struct io_ring_ctx *ctx = head->ctx;
1249
1250 /* protect against races with linked timeouts */
1251 spin_lock_irq(&ctx->timeout_lock);
1252 matched = io_match_linked(head);
1253 spin_unlock_irq(&ctx->timeout_lock);
1254 } else {
1255 matched = io_match_linked(head);
1256 }
1257 return matched;
1258}
1259
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01001260static inline void req_set_fail(struct io_kiocb *req)
Jens Axboec40f6372020-06-25 15:39:59 -06001261{
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01001262 req->flags |= REQ_F_FAIL;
Jens Axboec40f6372020-06-25 15:39:59 -06001263}
Jens Axboe4a38aed22020-05-14 17:21:15 -06001264
Hao Xua8295b92021-08-27 17:46:09 +08001265static inline void req_fail_link_node(struct io_kiocb *req, int res)
1266{
1267 req_set_fail(req);
1268 req->result = res;
1269}
1270
Jens Axboe2b188cc2019-01-07 10:46:33 -07001271static void io_ring_ctx_ref_free(struct percpu_ref *ref)
1272{
1273 struct io_ring_ctx *ctx = container_of(ref, struct io_ring_ctx, refs);
1274
Jens Axboe0f158b42020-05-14 17:18:39 -06001275 complete(&ctx->ref_comp);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001276}
1277
Pavel Begunkov8eb7e2d2020-06-29 13:13:02 +03001278static inline bool io_is_timeout_noseq(struct io_kiocb *req)
1279{
1280 return !req->timeout.off;
1281}
1282
Pavel Begunkovf56165e2021-08-09 20:18:07 +01001283static void io_fallback_req_func(struct work_struct *work)
1284{
1285 struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx,
1286 fallback_work.work);
1287 struct llist_node *node = llist_del_all(&ctx->fallback_llist);
1288 struct io_kiocb *req, *tmp;
Pavel Begunkovf237c302021-08-18 12:42:46 +01001289 bool locked = false;
Pavel Begunkovf56165e2021-08-09 20:18:07 +01001290
1291 percpu_ref_get(&ctx->refs);
1292 llist_for_each_entry_safe(req, tmp, node, io_task_work.fallback_node)
Pavel Begunkovf237c302021-08-18 12:42:46 +01001293 req->io_task_work.func(req, &locked);
Pavel Begunkov5636c002021-08-18 12:42:45 +01001294
Pavel Begunkovf237c302021-08-18 12:42:46 +01001295 if (locked) {
1296 if (ctx->submit_state.compl_nr)
1297 io_submit_flush_completions(ctx);
1298 mutex_unlock(&ctx->uring_lock);
1299 }
Pavel Begunkovf56165e2021-08-09 20:18:07 +01001300 percpu_ref_put(&ctx->refs);
Pavel Begunkovf237c302021-08-18 12:42:46 +01001301
Pavel Begunkovf56165e2021-08-09 20:18:07 +01001302}
1303
Jens Axboe2b188cc2019-01-07 10:46:33 -07001304static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
1305{
1306 struct io_ring_ctx *ctx;
Jens Axboe78076bb2019-12-04 19:56:40 -07001307 int hash_bits;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001308
1309 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
1310 if (!ctx)
1311 return NULL;
1312
Jens Axboe78076bb2019-12-04 19:56:40 -07001313 /*
1314 * Use 5 bits less than the max cq entries, that should give us around
1315 * 32 entries per hash list if totally full and uniformly spread.
1316 */
1317 hash_bits = ilog2(p->cq_entries);
1318 hash_bits -= 5;
1319 if (hash_bits <= 0)
1320 hash_bits = 1;
1321 ctx->cancel_hash_bits = hash_bits;
1322 ctx->cancel_hash = kmalloc((1U << hash_bits) * sizeof(struct hlist_head),
1323 GFP_KERNEL);
1324 if (!ctx->cancel_hash)
1325 goto err;
1326 __hash_init(ctx->cancel_hash, 1U << hash_bits);
1327
Pavel Begunkov62248432021-04-28 13:11:29 +01001328 ctx->dummy_ubuf = kzalloc(sizeof(*ctx->dummy_ubuf), GFP_KERNEL);
1329 if (!ctx->dummy_ubuf)
1330 goto err;
1331 /* set invalid range, so io_import_fixed() fails meeting it */
1332 ctx->dummy_ubuf->ubuf = -1UL;
1333
Roman Gushchin21482892019-05-07 10:01:48 -07001334 if (percpu_ref_init(&ctx->refs, io_ring_ctx_ref_free,
Jens Axboe206aefd2019-11-07 18:27:42 -07001335 PERCPU_REF_ALLOW_REINIT, GFP_KERNEL))
1336 goto err;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001337
1338 ctx->flags = p->flags;
Jens Axboe90554202020-09-03 12:12:41 -06001339 init_waitqueue_head(&ctx->sqo_sq_wait);
Jens Axboe69fb2132020-09-14 11:16:23 -06001340 INIT_LIST_HEAD(&ctx->sqd_list);
Pavel Begunkov311997b2021-06-14 23:37:28 +01001341 init_waitqueue_head(&ctx->poll_wait);
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001342 INIT_LIST_HEAD(&ctx->cq_overflow_list);
Jens Axboe0f158b42020-05-14 17:18:39 -06001343 init_completion(&ctx->ref_comp);
Jens Axboe9e15c3a2021-03-13 12:29:43 -07001344 xa_init_flags(&ctx->io_buffers, XA_FLAGS_ALLOC1);
Matthew Wilcox (Oracle)61cf9372021-03-08 14:16:16 +00001345 xa_init_flags(&ctx->personalities, XA_FLAGS_ALLOC1);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001346 mutex_init(&ctx->uring_lock);
Pavel Begunkov311997b2021-06-14 23:37:28 +01001347 init_waitqueue_head(&ctx->cq_wait);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001348 spin_lock_init(&ctx->completion_lock);
Jens Axboe89850fc2021-08-10 15:11:51 -06001349 spin_lock_init(&ctx->timeout_lock);
Pavel Begunkov540e32a2020-07-13 23:37:09 +03001350 INIT_LIST_HEAD(&ctx->iopoll_list);
Jens Axboede0617e2019-04-06 21:51:27 -06001351 INIT_LIST_HEAD(&ctx->defer_list);
Jens Axboe5262f562019-09-17 12:26:57 -06001352 INIT_LIST_HEAD(&ctx->timeout_list);
Pavel Begunkovef9dd632021-08-28 19:54:38 -06001353 INIT_LIST_HEAD(&ctx->ltimeout_list);
Bijan Mottahedehd67d2262021-01-15 17:37:46 +00001354 spin_lock_init(&ctx->rsrc_ref_lock);
1355 INIT_LIST_HEAD(&ctx->rsrc_ref_list);
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00001356 INIT_DELAYED_WORK(&ctx->rsrc_put_work, io_rsrc_put_work);
1357 init_llist_head(&ctx->rsrc_put_llist);
Pavel Begunkov13bf43f2021-03-06 11:02:12 +00001358 INIT_LIST_HEAD(&ctx->tctx_list);
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01001359 INIT_LIST_HEAD(&ctx->submit_state.free_list);
Pavel Begunkovd0acdee2021-05-16 22:58:12 +01001360 INIT_LIST_HEAD(&ctx->locked_free_list);
Pavel Begunkov9011bf92021-06-30 21:54:03 +01001361 INIT_DELAYED_WORK(&ctx->fallback_work, io_fallback_req_func);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001362 return ctx;
Jens Axboe206aefd2019-11-07 18:27:42 -07001363err:
Pavel Begunkov62248432021-04-28 13:11:29 +01001364 kfree(ctx->dummy_ubuf);
Jens Axboe78076bb2019-12-04 19:56:40 -07001365 kfree(ctx->cancel_hash);
Jens Axboe206aefd2019-11-07 18:27:42 -07001366 kfree(ctx);
1367 return NULL;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001368}
1369
Pavel Begunkov8f6ed492021-05-16 22:58:10 +01001370static void io_account_cq_overflow(struct io_ring_ctx *ctx)
1371{
1372 struct io_rings *r = ctx->rings;
1373
1374 WRITE_ONCE(r->cq_overflow, READ_ONCE(r->cq_overflow) + 1);
1375 ctx->cq_extra--;
1376}
1377
Pavel Begunkov9cf7c102020-07-13 23:37:15 +03001378static bool req_need_defer(struct io_kiocb *req, u32 seq)
Jens Axboede0617e2019-04-06 21:51:27 -06001379{
Jens Axboe2bc99302020-07-09 09:43:27 -06001380 if (unlikely(req->flags & REQ_F_IO_DRAIN)) {
1381 struct io_ring_ctx *ctx = req->ctx;
Jackie Liua197f662019-11-08 08:09:12 -07001382
Pavel Begunkov8f6ed492021-05-16 22:58:10 +01001383 return seq + READ_ONCE(ctx->cq_extra) != ctx->cached_cq_tail;
Jens Axboe2bc99302020-07-09 09:43:27 -06001384 }
Jens Axboe7adf4ea2019-10-10 21:42:58 -06001385
Bob Liu9d858b22019-11-13 18:06:25 +08001386 return false;
Jens Axboe7adf4ea2019-10-10 21:42:58 -06001387}
1388
Pavel Begunkovc97d8a02021-08-09 13:04:04 +01001389#define FFS_ASYNC_READ 0x1UL
1390#define FFS_ASYNC_WRITE 0x2UL
1391#ifdef CONFIG_64BIT
1392#define FFS_ISREG 0x4UL
1393#else
1394#define FFS_ISREG 0x0UL
1395#endif
1396#define FFS_MASK ~(FFS_ASYNC_READ|FFS_ASYNC_WRITE|FFS_ISREG)
1397
1398static inline bool io_req_ffs_set(struct io_kiocb *req)
1399{
1400 return IS_ENABLED(CONFIG_64BIT) && (req->flags & REQ_F_FIXED_FILE);
1401}
1402
Pavel Begunkovce3d5aa2021-02-01 18:59:55 +00001403static void io_req_track_inflight(struct io_kiocb *req)
1404{
Pavel Begunkovce3d5aa2021-02-01 18:59:55 +00001405 if (!(req->flags & REQ_F_INFLIGHT)) {
Pavel Begunkovce3d5aa2021-02-01 18:59:55 +00001406 req->flags |= REQ_F_INFLIGHT;
Jens Axboe3746d622022-06-23 11:06:43 -06001407 atomic_inc(&req->task->io_uring->inflight_tracked);
Pavel Begunkovce3d5aa2021-02-01 18:59:55 +00001408 }
1409}
1410
Pavel Begunkovfd08e532021-08-11 19:28:31 +01001411static struct io_kiocb *__io_prep_linked_timeout(struct io_kiocb *req)
1412{
Pavel Begunkov906c6ca2021-08-15 10:40:26 +01001413 if (WARN_ON_ONCE(!req->link))
1414 return NULL;
1415
Pavel Begunkov4d13d1a2021-08-15 10:40:24 +01001416 req->flags &= ~REQ_F_ARM_LTIMEOUT;
1417 req->flags |= REQ_F_LINK_TIMEOUT;
Pavel Begunkovfd08e532021-08-11 19:28:31 +01001418
1419 /* linked timeouts should have two refs once prep'ed */
Pavel Begunkov48dcd382021-08-15 10:40:18 +01001420 io_req_set_refcount(req);
Pavel Begunkov4d13d1a2021-08-15 10:40:24 +01001421 __io_req_set_refcount(req->link, 2);
1422 return req->link;
Pavel Begunkovfd08e532021-08-11 19:28:31 +01001423}
1424
1425static inline struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req)
1426{
Pavel Begunkov4d13d1a2021-08-15 10:40:24 +01001427 if (likely(!(req->flags & REQ_F_ARM_LTIMEOUT)))
Pavel Begunkovfd08e532021-08-11 19:28:31 +01001428 return NULL;
1429 return __io_prep_linked_timeout(req);
1430}
1431
Pavel Begunkovcbdcb432020-06-29 19:18:43 +03001432static void io_prep_async_work(struct io_kiocb *req)
Jens Axboe561fb042019-10-24 07:25:42 -06001433{
Jens Axboed3656342019-12-18 09:50:26 -07001434 const struct io_op_def *def = &io_op_defs[req->opcode];
Pavel Begunkov23329512020-10-10 18:34:06 +01001435 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe54a91f32019-09-10 09:15:04 -06001436
Pavel Begunkovb8e64b52021-06-17 18:14:02 +01001437 if (!(req->flags & REQ_F_CREDS)) {
1438 req->flags |= REQ_F_CREDS;
Pavel Begunkovc10d1f92021-06-17 18:14:01 +01001439 req->creds = get_current_cred();
Pavel Begunkovb8e64b52021-06-17 18:14:02 +01001440 }
Jens Axboe003e8dc2021-03-06 09:22:27 -07001441
Pavel Begunkove1d675d2021-03-22 01:58:29 +00001442 req->work.list.next = NULL;
1443 req->work.flags = 0;
Pavel Begunkovfeaadc42020-10-22 16:47:16 +01001444 if (req->flags & REQ_F_FORCE_ASYNC)
1445 req->work.flags |= IO_WQ_WORK_CONCURRENT;
1446
Jens Axboed3656342019-12-18 09:50:26 -07001447 if (req->flags & REQ_F_ISREG) {
Pavel Begunkov23329512020-10-10 18:34:06 +01001448 if (def->hash_reg_file || (ctx->flags & IORING_SETUP_IOPOLL))
Pavel Begunkov8766dd52020-03-14 00:31:04 +03001449 io_wq_hash_work(&req->work, file_inode(req->file));
Jens Axboe4b982bd2021-04-01 08:38:34 -06001450 } else if (!req->file || !S_ISBLK(file_inode(req->file)->i_mode)) {
Jens Axboed3656342019-12-18 09:50:26 -07001451 if (def->unbound_nonreg_file)
Jens Axboe3529d8c2019-12-19 18:24:38 -07001452 req->work.flags |= IO_WQ_WORK_UNBOUND;
Jens Axboe54a91f32019-09-10 09:15:04 -06001453 }
Jens Axboe561fb042019-10-24 07:25:42 -06001454}
1455
Pavel Begunkovcbdcb432020-06-29 19:18:43 +03001456static void io_prep_async_link(struct io_kiocb *req)
1457{
1458 struct io_kiocb *cur;
1459
Pavel Begunkov44eff402021-07-26 14:14:31 +01001460 if (req->flags & REQ_F_LINK_TIMEOUT) {
1461 struct io_ring_ctx *ctx = req->ctx;
1462
Pavel Begunkov09eb40f2021-11-23 01:45:35 +00001463 spin_lock_irq(&ctx->timeout_lock);
Pavel Begunkov44eff402021-07-26 14:14:31 +01001464 io_for_each_link(cur, req)
1465 io_prep_async_work(cur);
Pavel Begunkov09eb40f2021-11-23 01:45:35 +00001466 spin_unlock_irq(&ctx->timeout_lock);
Pavel Begunkov44eff402021-07-26 14:14:31 +01001467 } else {
1468 io_for_each_link(cur, req)
1469 io_prep_async_work(cur);
1470 }
Pavel Begunkovcbdcb432020-06-29 19:18:43 +03001471}
1472
Pavel Begunkovf237c302021-08-18 12:42:46 +01001473static void io_queue_async_work(struct io_kiocb *req, bool *locked)
Jens Axboe561fb042019-10-24 07:25:42 -06001474{
Jackie Liua197f662019-11-08 08:09:12 -07001475 struct io_ring_ctx *ctx = req->ctx;
Pavel Begunkovcbdcb432020-06-29 19:18:43 +03001476 struct io_kiocb *link = io_prep_linked_timeout(req);
Jens Axboe5aa75ed2021-02-16 12:56:50 -07001477 struct io_uring_task *tctx = req->task->io_uring;
Jens Axboe561fb042019-10-24 07:25:42 -06001478
Pavel Begunkovf237c302021-08-18 12:42:46 +01001479 /* must not take the lock, NULL it as a precaution */
1480 locked = NULL;
1481
Jens Axboe3bfe6102021-02-16 14:15:30 -07001482 BUG_ON(!tctx);
1483 BUG_ON(!tctx->io_wq);
Jens Axboe561fb042019-10-24 07:25:42 -06001484
Pavel Begunkovcbdcb432020-06-29 19:18:43 +03001485 /* init ->work of the whole link before punting */
1486 io_prep_async_link(req);
Jens Axboe991468d2021-07-23 11:53:54 -06001487
1488 /*
1489 * Not expected to happen, but if we do have a bug where this _can_
1490 * happen, catch it here and ensure the request is marked as
1491 * canceled. That will make io-wq go through the usual work cancel
1492 * procedure rather than attempt to run this request (or create a new
1493 * worker for it).
1494 */
1495 if (WARN_ON_ONCE(!same_thread_group(req->task, current)))
1496 req->work.flags |= IO_WQ_WORK_CANCEL;
1497
Pavel Begunkovd07f1e8a2021-03-22 01:45:58 +00001498 trace_io_uring_queue_async_work(ctx, io_wq_is_hashed(&req->work), req,
1499 &req->work, req->flags);
Pavel Begunkovebf93662021-03-01 18:20:47 +00001500 io_wq_enqueue(tctx->io_wq, &req->work);
Jens Axboe7271ef32020-08-10 09:55:22 -06001501 if (link)
1502 io_queue_linked_timeout(link);
Pavel Begunkovcbdcb432020-06-29 19:18:43 +03001503}
1504
Pavel Begunkov1ee41602021-03-25 18:32:42 +00001505static void io_kill_timeout(struct io_kiocb *req, int status)
Pavel Begunkov8c855882021-04-13 02:58:41 +01001506 __must_hold(&req->ctx->completion_lock)
Jens Axboe89850fc2021-08-10 15:11:51 -06001507 __must_hold(&req->ctx->timeout_lock)
Jens Axboe5262f562019-09-17 12:26:57 -06001508{
Jens Axboee8c2bc12020-08-15 18:44:09 -07001509 struct io_timeout_data *io = req->async_data;
Jens Axboe5262f562019-09-17 12:26:57 -06001510
Pavel Begunkovfd9c7bc2021-04-13 02:58:42 +01001511 if (hrtimer_try_to_cancel(&io->timer) != -1) {
Pavel Begunkov2ae2eb92021-09-09 13:56:27 +01001512 if (status)
1513 req_set_fail(req);
Pavel Begunkov01cec8c2020-07-30 18:43:50 +03001514 atomic_set(&req->ctx->cq_timeouts,
1515 atomic_read(&req->ctx->cq_timeouts) + 1);
Pavel Begunkov135fcde2020-07-13 23:37:12 +03001516 list_del_init(&req->timeout.list);
Pavel Begunkovb850d6d2022-08-29 14:30:13 +01001517 io_fill_cqe_req(req, status, 0);
Pavel Begunkov91c2f692021-08-11 19:28:28 +01001518 io_put_req_deferred(req);
Jens Axboe5262f562019-09-17 12:26:57 -06001519 }
1520}
1521
Pavel Begunkov441b8a72021-06-14 23:37:31 +01001522static void io_queue_deferred(struct io_ring_ctx *ctx)
Pavel Begunkov04518942020-05-26 20:34:05 +03001523{
Pavel Begunkov441b8a72021-06-14 23:37:31 +01001524 while (!list_empty(&ctx->defer_list)) {
Pavel Begunkov27dc8332020-07-13 23:37:14 +03001525 struct io_defer_entry *de = list_first_entry(&ctx->defer_list,
1526 struct io_defer_entry, list);
Pavel Begunkov04518942020-05-26 20:34:05 +03001527
Pavel Begunkov9cf7c102020-07-13 23:37:15 +03001528 if (req_need_defer(de->req, de->seq))
Pavel Begunkov04518942020-05-26 20:34:05 +03001529 break;
Pavel Begunkov27dc8332020-07-13 23:37:14 +03001530 list_del_init(&de->list);
Pavel Begunkov907d1df2021-01-26 23:35:10 +00001531 io_req_task_queue(de->req);
Pavel Begunkov27dc8332020-07-13 23:37:14 +03001532 kfree(de);
Pavel Begunkov441b8a72021-06-14 23:37:31 +01001533 }
Pavel Begunkov04518942020-05-26 20:34:05 +03001534}
1535
Pavel Begunkov360428f2020-05-30 14:54:17 +03001536static void io_flush_timeouts(struct io_ring_ctx *ctx)
Jens Axboe89850fc2021-08-10 15:11:51 -06001537 __must_hold(&ctx->completion_lock)
Pavel Begunkov360428f2020-05-30 14:54:17 +03001538{
Pavel Begunkov441b8a72021-06-14 23:37:31 +01001539 u32 seq = ctx->cached_cq_tail - atomic_read(&ctx->cq_timeouts);
Jens Axboeba7261a2022-04-08 11:08:58 -06001540 struct io_kiocb *req, *tmp;
Marcelo Diop-Gonzalezf0105052021-01-15 11:54:40 -05001541
Jens Axboe79ebeae2021-08-10 15:18:27 -06001542 spin_lock_irq(&ctx->timeout_lock);
Jens Axboeba7261a2022-04-08 11:08:58 -06001543 list_for_each_entry_safe(req, tmp, &ctx->timeout_list, timeout.list) {
Marcelo Diop-Gonzalezf0105052021-01-15 11:54:40 -05001544 u32 events_needed, events_got;
Pavel Begunkov360428f2020-05-30 14:54:17 +03001545
Pavel Begunkov8eb7e2d2020-06-29 13:13:02 +03001546 if (io_is_timeout_noseq(req))
Pavel Begunkov360428f2020-05-30 14:54:17 +03001547 break;
Marcelo Diop-Gonzalezf0105052021-01-15 11:54:40 -05001548
1549 /*
1550 * Since seq can easily wrap around over time, subtract
1551 * the last seq at which timeouts were flushed before comparing.
1552 * Assuming not more than 2^31-1 events have happened since,
1553 * these subtractions won't have wrapped, so we can check if
1554 * target is in [last_seq, current_seq] by comparing the two.
1555 */
1556 events_needed = req->timeout.target_seq - ctx->cq_last_tm_flush;
1557 events_got = seq - ctx->cq_last_tm_flush;
1558 if (events_got < events_needed)
Pavel Begunkov360428f2020-05-30 14:54:17 +03001559 break;
Pavel Begunkovbfe68a22020-05-30 14:54:18 +03001560
Pavel Begunkov1ee41602021-03-25 18:32:42 +00001561 io_kill_timeout(req, 0);
Pavel Begunkovf18ee4c2021-06-14 23:37:25 +01001562 }
Marcelo Diop-Gonzalezf0105052021-01-15 11:54:40 -05001563 ctx->cq_last_tm_flush = seq;
Jens Axboe79ebeae2021-08-10 15:18:27 -06001564 spin_unlock_irq(&ctx->timeout_lock);
Pavel Begunkov360428f2020-05-30 14:54:17 +03001565}
1566
Pavel Begunkov2335f6f2021-06-15 16:47:58 +01001567static void __io_commit_cqring_flush(struct io_ring_ctx *ctx)
Jens Axboede0617e2019-04-06 21:51:27 -06001568{
Pavel Begunkov2335f6f2021-06-15 16:47:58 +01001569 if (ctx->off_timeout_used)
1570 io_flush_timeouts(ctx);
1571 if (ctx->drain_active)
1572 io_queue_deferred(ctx);
1573}
1574
1575static inline void io_commit_cqring(struct io_ring_ctx *ctx)
1576{
1577 if (unlikely(ctx->off_timeout_used || ctx->drain_active))
1578 __io_commit_cqring_flush(ctx);
Pavel Begunkovec30e042021-01-19 13:32:38 +00001579 /* order cqe stores with ring update */
1580 smp_store_release(&ctx->rings->cq.tail, ctx->cached_cq_tail);
Jens Axboede0617e2019-04-06 21:51:27 -06001581}
1582
Jens Axboe90554202020-09-03 12:12:41 -06001583static inline bool io_sqring_full(struct io_ring_ctx *ctx)
1584{
1585 struct io_rings *r = ctx->rings;
1586
Pavel Begunkova566c552021-05-16 22:58:08 +01001587 return READ_ONCE(r->sq.tail) - ctx->cached_sq_head == ctx->sq_entries;
Jens Axboe90554202020-09-03 12:12:41 -06001588}
1589
Pavel Begunkov888aae22021-01-19 13:32:39 +00001590static inline unsigned int __io_cqring_events(struct io_ring_ctx *ctx)
1591{
1592 return ctx->cached_cq_tail - READ_ONCE(ctx->rings->cq.head);
1593}
1594
Pavel Begunkovd068b502021-05-16 22:58:11 +01001595static inline struct io_uring_cqe *io_get_cqe(struct io_ring_ctx *ctx)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001596{
Hristo Venev75b28af2019-08-26 17:23:46 +00001597 struct io_rings *rings = ctx->rings;
Pavel Begunkovea5ab3b2021-05-16 22:58:09 +01001598 unsigned tail, mask = ctx->cq_entries - 1;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001599
Stefan Bühler115e12e2019-04-24 23:54:18 +02001600 /*
1601 * writes to the cq entry need to come after reading head; the
1602 * control dependency is enough as we're using WRITE_ONCE to
1603 * fill the cq entry
1604 */
Pavel Begunkova566c552021-05-16 22:58:08 +01001605 if (__io_cqring_events(ctx) == ctx->cq_entries)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001606 return NULL;
1607
Pavel Begunkov888aae22021-01-19 13:32:39 +00001608 tail = ctx->cached_cq_tail++;
Pavel Begunkovea5ab3b2021-05-16 22:58:09 +01001609 return &rings->cqes[tail & mask];
Jens Axboe2b188cc2019-01-07 10:46:33 -07001610}
1611
Jens Axboef2842ab2020-01-08 11:04:00 -07001612static inline bool io_should_trigger_evfd(struct io_ring_ctx *ctx)
1613{
Pavel Begunkov44c769d2021-04-11 01:46:31 +01001614 if (likely(!ctx->cq_ev_fd))
Jens Axboef0b493e2020-02-01 21:30:11 -07001615 return false;
Stefano Garzarella7e55a192020-05-15 18:38:05 +02001616 if (READ_ONCE(ctx->rings->cq_flags) & IORING_CQ_EVENTFD_DISABLED)
1617 return false;
Pavel Begunkov44c769d2021-04-11 01:46:31 +01001618 return !ctx->eventfd_async || io_wq_current_is_worker();
Jens Axboef2842ab2020-01-08 11:04:00 -07001619}
1620
Jens Axboe2c5d7632021-08-21 07:21:19 -06001621/*
1622 * This should only get called when at least one event has been posted.
1623 * Some applications rely on the eventfd notification count only changing
1624 * IFF a new CQE has been added to the CQ ring. There's no depedency on
1625 * 1:1 relationship between how many times this function is called (and
1626 * hence the eventfd count) and number of CQEs posted to the CQ ring.
1627 */
Jens Axboeb41e9852020-02-17 09:52:41 -07001628static void io_cqring_ev_posted(struct io_ring_ctx *ctx)
Jens Axboe8c838782019-03-12 15:48:16 -06001629{
Jens Axboe5fd46172021-08-06 14:04:31 -06001630 /*
1631 * wake_up_all() may seem excessive, but io_wake_function() and
1632 * io_should_wake() handle the termination of the loop and only
1633 * wake as many waiters as we need to.
1634 */
1635 if (wq_has_sleeper(&ctx->cq_wait))
Jens Axboeccf06b52022-12-23 07:04:49 -07001636 __wake_up(&ctx->cq_wait, TASK_NORMAL, 0,
1637 poll_to_key(EPOLL_URING_WAKE | EPOLLIN));
Jens Axboe534ca6d2020-09-02 13:52:19 -06001638 if (ctx->sq_data && waitqueue_active(&ctx->sq_data->wait))
1639 wake_up(&ctx->sq_data->wait);
Jens Axboeb41e9852020-02-17 09:52:41 -07001640 if (io_should_trigger_evfd(ctx))
Jens Axboeccf06b52022-12-23 07:04:49 -07001641 eventfd_signal_mask(ctx->cq_ev_fd, 1, EPOLL_URING_WAKE);
Pavel Begunkov3f008382021-10-01 10:39:33 +01001642 if (waitqueue_active(&ctx->poll_wait))
Jens Axboeccf06b52022-12-23 07:04:49 -07001643 __wake_up(&ctx->poll_wait, TASK_INTERRUPTIBLE, 0,
1644 poll_to_key(EPOLL_URING_WAKE | EPOLLIN));
Jens Axboe8c838782019-03-12 15:48:16 -06001645}
1646
Pavel Begunkov80c18e42021-01-07 03:15:41 +00001647static void io_cqring_ev_posted_iopoll(struct io_ring_ctx *ctx)
1648{
Pavel Begunkovc57a91fb2021-09-08 20:49:17 +01001649 /* see waitqueue_active() comment */
1650 smp_mb();
1651
Pavel Begunkov80c18e42021-01-07 03:15:41 +00001652 if (ctx->flags & IORING_SETUP_SQPOLL) {
Pavel Begunkovc57a91fb2021-09-08 20:49:17 +01001653 if (waitqueue_active(&ctx->cq_wait))
Jens Axboeccf06b52022-12-23 07:04:49 -07001654 __wake_up(&ctx->cq_wait, TASK_NORMAL, 0,
1655 poll_to_key(EPOLL_URING_WAKE | EPOLLIN));
Pavel Begunkov80c18e42021-01-07 03:15:41 +00001656 }
1657 if (io_should_trigger_evfd(ctx))
Jens Axboeccf06b52022-12-23 07:04:49 -07001658 eventfd_signal_mask(ctx->cq_ev_fd, 1, EPOLL_URING_WAKE);
Pavel Begunkov3f008382021-10-01 10:39:33 +01001659 if (waitqueue_active(&ctx->poll_wait))
Jens Axboeccf06b52022-12-23 07:04:49 -07001660 __wake_up(&ctx->poll_wait, TASK_INTERRUPTIBLE, 0,
1661 poll_to_key(EPOLL_URING_WAKE | EPOLLIN));
Pavel Begunkov80c18e42021-01-07 03:15:41 +00001662}
1663
Jens Axboec4a2ed72019-11-21 21:01:26 -07001664/* Returns true if there are no backlogged entries after the flush */
Pavel Begunkov6c2450a2021-02-23 12:40:22 +00001665static bool __io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001666{
Jens Axboeb18032b2021-01-24 16:58:56 -07001667 bool all_flushed, posted;
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001668
Pavel Begunkova566c552021-05-16 22:58:08 +01001669 if (!force && __io_cqring_events(ctx) == ctx->cq_entries)
Pavel Begunkove23de152020-12-17 00:24:37 +00001670 return false;
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001671
Jens Axboeb18032b2021-01-24 16:58:56 -07001672 posted = false;
Jens Axboe79ebeae2021-08-10 15:18:27 -06001673 spin_lock(&ctx->completion_lock);
Pavel Begunkov6c2450a2021-02-23 12:40:22 +00001674 while (!list_empty(&ctx->cq_overflow_list)) {
Pavel Begunkovd068b502021-05-16 22:58:11 +01001675 struct io_uring_cqe *cqe = io_get_cqe(ctx);
Pavel Begunkov6c2450a2021-02-23 12:40:22 +00001676 struct io_overflow_cqe *ocqe;
Jens Axboee6c8aa92020-09-28 13:10:13 -06001677
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001678 if (!cqe && !force)
1679 break;
Pavel Begunkov6c2450a2021-02-23 12:40:22 +00001680 ocqe = list_first_entry(&ctx->cq_overflow_list,
1681 struct io_overflow_cqe, list);
1682 if (cqe)
1683 memcpy(cqe, &ocqe->cqe, sizeof(*cqe));
1684 else
Pavel Begunkov8f6ed492021-05-16 22:58:10 +01001685 io_account_cq_overflow(ctx);
1686
Jens Axboeb18032b2021-01-24 16:58:56 -07001687 posted = true;
Pavel Begunkov6c2450a2021-02-23 12:40:22 +00001688 list_del(&ocqe->list);
1689 kfree(ocqe);
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001690 }
1691
Pavel Begunkov09e88402020-12-17 00:24:38 +00001692 all_flushed = list_empty(&ctx->cq_overflow_list);
1693 if (all_flushed) {
Pavel Begunkov5ed7a372021-06-14 23:37:27 +01001694 clear_bit(0, &ctx->check_cq_overflow);
Nadav Amit20c0b382021-08-07 17:13:42 -07001695 WRITE_ONCE(ctx->rings->sq_flags,
1696 ctx->rings->sq_flags & ~IORING_SQ_CQ_OVERFLOW);
Pavel Begunkov09e88402020-12-17 00:24:38 +00001697 }
Pavel Begunkov46930142020-07-30 18:43:49 +03001698
Jens Axboeb18032b2021-01-24 16:58:56 -07001699 if (posted)
1700 io_commit_cqring(ctx);
Jens Axboe79ebeae2021-08-10 15:18:27 -06001701 spin_unlock(&ctx->completion_lock);
Jens Axboeb18032b2021-01-24 16:58:56 -07001702 if (posted)
1703 io_cqring_ev_posted(ctx);
Pavel Begunkov09e88402020-12-17 00:24:38 +00001704 return all_flushed;
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001705}
1706
Pavel Begunkov90f67362021-08-09 20:18:12 +01001707static bool io_cqring_overflow_flush(struct io_ring_ctx *ctx)
Pavel Begunkov6c503152021-01-04 20:36:36 +00001708{
Jens Axboeca0a2652021-03-04 17:15:48 -07001709 bool ret = true;
1710
Pavel Begunkov5ed7a372021-06-14 23:37:27 +01001711 if (test_bit(0, &ctx->check_cq_overflow)) {
Pavel Begunkov6c503152021-01-04 20:36:36 +00001712 /* iopoll syncs against uring_lock, not completion_lock */
1713 if (ctx->flags & IORING_SETUP_IOPOLL)
1714 mutex_lock(&ctx->uring_lock);
Pavel Begunkov90f67362021-08-09 20:18:12 +01001715 ret = __io_cqring_overflow_flush(ctx, false);
Pavel Begunkov6c503152021-01-04 20:36:36 +00001716 if (ctx->flags & IORING_SETUP_IOPOLL)
1717 mutex_unlock(&ctx->uring_lock);
1718 }
Jens Axboeca0a2652021-03-04 17:15:48 -07001719
1720 return ret;
Pavel Begunkov6c503152021-01-04 20:36:36 +00001721}
1722
Pavel Begunkov6a290a12021-08-09 13:04:13 +01001723/* must to be called somewhat shortly after putting a request */
1724static inline void io_put_task(struct task_struct *task, int nr)
1725{
1726 struct io_uring_task *tctx = task->io_uring;
1727
Pavel Begunkove98e49b2021-08-18 17:01:43 +01001728 if (likely(task == current)) {
1729 tctx->cached_refs += nr;
1730 } else {
1731 percpu_counter_sub(&tctx->inflight, nr);
1732 if (unlikely(atomic_read(&tctx->in_idle)))
1733 wake_up(&tctx->wait);
1734 put_task_struct_many(task, nr);
1735 }
Pavel Begunkov6a290a12021-08-09 13:04:13 +01001736}
1737
Pavel Begunkov9a108672021-08-27 11:55:01 +01001738static void io_task_refs_refill(struct io_uring_task *tctx)
1739{
1740 unsigned int refill = -tctx->cached_refs + IO_TCTX_REFS_CACHE_NR;
1741
1742 percpu_counter_add(&tctx->inflight, refill);
1743 refcount_add(refill, &current->usage);
1744 tctx->cached_refs += refill;
1745}
1746
1747static inline void io_get_task_refs(int nr)
1748{
1749 struct io_uring_task *tctx = current->io_uring;
1750
1751 tctx->cached_refs -= nr;
1752 if (unlikely(tctx->cached_refs < 0))
1753 io_task_refs_refill(tctx);
1754}
1755
Pavel Begunkovb168b1a2022-01-09 00:53:22 +00001756static __cold void io_uring_drop_tctx_refs(struct task_struct *task)
1757{
1758 struct io_uring_task *tctx = task->io_uring;
1759 unsigned int refs = tctx->cached_refs;
1760
1761 if (refs) {
1762 tctx->cached_refs = 0;
1763 percpu_counter_sub(&tctx->inflight, refs);
1764 put_task_struct_many(task, refs);
1765 }
1766}
1767
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01001768static bool io_cqring_event_overflow(struct io_ring_ctx *ctx, u64 user_data,
Pavel Begunkov5c0ea4c2022-08-29 14:30:12 +01001769 s32 res, u32 cflags)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001770{
Pavel Begunkovcce4b8b2021-04-13 02:58:44 +01001771 struct io_overflow_cqe *ocqe;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001772
Pavel Begunkovcce4b8b2021-04-13 02:58:44 +01001773 ocqe = kmalloc(sizeof(*ocqe), GFP_ATOMIC | __GFP_ACCOUNT);
1774 if (!ocqe) {
1775 /*
1776 * If we're in ring overflow flush mode, or in task cancel mode,
1777 * or cannot allocate an overflow entry, then we need to drop it
1778 * on the floor.
1779 */
Pavel Begunkov8f6ed492021-05-16 22:58:10 +01001780 io_account_cq_overflow(ctx);
Pavel Begunkovcce4b8b2021-04-13 02:58:44 +01001781 return false;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001782 }
Pavel Begunkovcce4b8b2021-04-13 02:58:44 +01001783 if (list_empty(&ctx->cq_overflow_list)) {
Pavel Begunkov5ed7a372021-06-14 23:37:27 +01001784 set_bit(0, &ctx->check_cq_overflow);
Nadav Amit20c0b382021-08-07 17:13:42 -07001785 WRITE_ONCE(ctx->rings->sq_flags,
1786 ctx->rings->sq_flags | IORING_SQ_CQ_OVERFLOW);
1787
Pavel Begunkovcce4b8b2021-04-13 02:58:44 +01001788 }
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01001789 ocqe->cqe.user_data = user_data;
Pavel Begunkovcce4b8b2021-04-13 02:58:44 +01001790 ocqe->cqe.res = res;
1791 ocqe->cqe.flags = cflags;
1792 list_add_tail(&ocqe->list, &ctx->cq_overflow_list);
1793 return true;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001794}
1795
Pavel Begunkovb850d6d2022-08-29 14:30:13 +01001796static inline bool __io_fill_cqe(struct io_ring_ctx *ctx, u64 user_data,
1797 s32 res, u32 cflags)
Pavel Begunkov8d133262021-04-11 01:46:33 +01001798{
Jens Axboe2b188cc2019-01-07 10:46:33 -07001799 struct io_uring_cqe *cqe;
1800
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01001801 trace_io_uring_complete(ctx, user_data, res, cflags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001802
1803 /*
1804 * If we can't get a cq entry, userspace overflowed the
1805 * submission (by quite a lot). Increment the overflow count in
1806 * the ring.
1807 */
Pavel Begunkovd068b502021-05-16 22:58:11 +01001808 cqe = io_get_cqe(ctx);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001809 if (likely(cqe)) {
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01001810 WRITE_ONCE(cqe->user_data, user_data);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001811 WRITE_ONCE(cqe->res, res);
1812 WRITE_ONCE(cqe->flags, cflags);
Pavel Begunkov8d133262021-04-11 01:46:33 +01001813 return true;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001814 }
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01001815 return io_cqring_event_overflow(ctx, user_data, res, cflags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001816}
1817
Pavel Begunkovb850d6d2022-08-29 14:30:13 +01001818static noinline void io_fill_cqe_req(struct io_kiocb *req, s32 res, u32 cflags)
Jens Axboebcda7ba2020-02-23 16:42:51 -07001819{
Pavel Begunkovb850d6d2022-08-29 14:30:13 +01001820 __io_fill_cqe(req->ctx, req->user_data, res, cflags);
1821}
1822
1823static noinline bool io_fill_cqe_aux(struct io_ring_ctx *ctx, u64 user_data,
1824 s32 res, u32 cflags)
1825{
1826 ctx->cq_extra++;
1827 return __io_fill_cqe(ctx, user_data, res, cflags);
Jens Axboebcda7ba2020-02-23 16:42:51 -07001828}
1829
Pavel Begunkov5c0ea4c2022-08-29 14:30:12 +01001830static void io_req_complete_post(struct io_kiocb *req, s32 res,
1831 u32 cflags)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001832{
Jens Axboe78e19bb2019-11-06 15:21:34 -07001833 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001834
Jens Axboe79ebeae2021-08-10 15:18:27 -06001835 spin_lock(&ctx->completion_lock);
Pavel Begunkovb850d6d2022-08-29 14:30:13 +01001836 __io_fill_cqe(ctx, req->user_data, res, cflags);
Jens Axboec7dae4b2021-02-09 19:53:37 -07001837 /*
1838 * If we're the last reference to this request, add to our locked
1839 * free_list cache.
1840 */
Jens Axboede9b4cc2021-02-24 13:28:27 -07001841 if (req_ref_put_and_test(req)) {
Pavel Begunkov7a612352021-03-09 00:37:59 +00001842 if (req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) {
Pavel Begunkov0756a862021-08-15 10:40:25 +01001843 if (req->flags & IO_DISARM_MASK)
Pavel Begunkov7a612352021-03-09 00:37:59 +00001844 io_disarm_next(req);
1845 if (req->link) {
1846 io_req_task_queue(req->link);
1847 req->link = NULL;
1848 }
1849 }
Jens Axboec7dae4b2021-02-09 19:53:37 -07001850 io_dismantle_req(req);
1851 io_put_task(req->task, 1);
Pavel Begunkovbb943b82021-08-09 20:18:10 +01001852 list_add(&req->inflight_entry, &ctx->locked_free_list);
Pavel Begunkovd0acdee2021-05-16 22:58:12 +01001853 ctx->locked_free_nr++;
Pavel Begunkov180f8292021-03-14 20:57:09 +00001854 } else {
1855 if (!percpu_ref_tryget(&ctx->refs))
1856 req = NULL;
1857 }
Pavel Begunkov7a612352021-03-09 00:37:59 +00001858 io_commit_cqring(ctx);
Jens Axboe79ebeae2021-08-10 15:18:27 -06001859 spin_unlock(&ctx->completion_lock);
Pavel Begunkov7a612352021-03-09 00:37:59 +00001860
Pavel Begunkov180f8292021-03-14 20:57:09 +00001861 if (req) {
1862 io_cqring_ev_posted(ctx);
Jens Axboec7dae4b2021-02-09 19:53:37 -07001863 percpu_ref_put(&ctx->refs);
Pavel Begunkov180f8292021-03-14 20:57:09 +00001864 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07001865}
1866
Jens Axboe4e3d9ff2021-04-15 17:44:34 -06001867static inline bool io_req_needs_clean(struct io_kiocb *req)
1868{
Pavel Begunkovc8543572021-06-17 18:14:04 +01001869 return req->flags & IO_REQ_CLEAN_FLAGS;
Jens Axboe4e3d9ff2021-04-15 17:44:34 -06001870}
1871
Pavel Begunkov5c0ea4c2022-08-29 14:30:12 +01001872static inline void io_req_complete_state(struct io_kiocb *req, s32 res,
1873 u32 cflags)
Jens Axboebcda7ba2020-02-23 16:42:51 -07001874{
Jens Axboe4e3d9ff2021-04-15 17:44:34 -06001875 if (io_req_needs_clean(req))
Pavel Begunkov68fb8972021-03-19 17:22:41 +00001876 io_clean_op(req);
Pavel Begunkova38d68d2021-01-19 13:32:45 +00001877 req->result = res;
1878 req->compl.cflags = cflags;
Pavel Begunkove342c802021-01-19 13:32:47 +00001879 req->flags |= REQ_F_COMPLETE_INLINE;
Jens Axboee1e16092020-06-22 09:17:17 -06001880}
Jens Axboe2b188cc2019-01-07 10:46:33 -07001881
Pavel Begunkov889fca72021-02-10 00:03:09 +00001882static inline void __io_req_complete(struct io_kiocb *req, unsigned issue_flags,
Pavel Begunkov5c0ea4c2022-08-29 14:30:12 +01001883 s32 res, u32 cflags)
Pavel Begunkova38d68d2021-01-19 13:32:45 +00001884{
Pavel Begunkov889fca72021-02-10 00:03:09 +00001885 if (issue_flags & IO_URING_F_COMPLETE_DEFER)
1886 io_req_complete_state(req, res, cflags);
Pavel Begunkova38d68d2021-01-19 13:32:45 +00001887 else
Jens Axboec7dae4b2021-02-09 19:53:37 -07001888 io_req_complete_post(req, res, cflags);
Pavel Begunkova38d68d2021-01-19 13:32:45 +00001889}
Jens Axboebcda7ba2020-02-23 16:42:51 -07001890
Pavel Begunkov5c0ea4c2022-08-29 14:30:12 +01001891static inline void io_req_complete(struct io_kiocb *req, s32 res)
Jens Axboee1e16092020-06-22 09:17:17 -06001892{
Pavel Begunkov889fca72021-02-10 00:03:09 +00001893 __io_req_complete(req, 0, res, 0);
Jens Axboebcda7ba2020-02-23 16:42:51 -07001894}
1895
Pavel Begunkov5c0ea4c2022-08-29 14:30:12 +01001896static void io_req_complete_failed(struct io_kiocb *req, s32 res)
Pavel Begunkovf41db2732021-02-28 22:35:12 +00001897{
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01001898 req_set_fail(req);
Pavel Begunkovf41db2732021-02-28 22:35:12 +00001899 io_req_complete_post(req, res, 0);
1900}
1901
Pavel Begunkovc6d3d9c2021-08-31 14:13:10 +01001902static void io_req_complete_fail_submit(struct io_kiocb *req)
1903{
1904 /*
1905 * We don't submit, fail them all, for that replace hardlinks with
1906 * normal links. Extra REQ_F_LINK is tolerated.
1907 */
1908 req->flags &= ~REQ_F_HARDLINK;
1909 req->flags |= REQ_F_LINK;
1910 io_req_complete_failed(req, req->result);
1911}
1912
Pavel Begunkov864ea922021-08-09 13:04:08 +01001913/*
1914 * Don't initialise the fields below on every allocation, but do that in
1915 * advance and keep them valid across allocations.
1916 */
1917static void io_preinit_req(struct io_kiocb *req, struct io_ring_ctx *ctx)
1918{
1919 req->ctx = ctx;
1920 req->link = NULL;
1921 req->async_data = NULL;
1922 /* not necessary, but safer to zero */
1923 req->result = 0;
1924}
1925
Pavel Begunkovdac7a092021-03-19 17:22:39 +00001926static void io_flush_cached_locked_reqs(struct io_ring_ctx *ctx,
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01001927 struct io_submit_state *state)
Pavel Begunkovdac7a092021-03-19 17:22:39 +00001928{
Jens Axboe79ebeae2021-08-10 15:18:27 -06001929 spin_lock(&ctx->completion_lock);
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01001930 list_splice_init(&ctx->locked_free_list, &state->free_list);
Pavel Begunkovd0acdee2021-05-16 22:58:12 +01001931 ctx->locked_free_nr = 0;
Jens Axboe79ebeae2021-08-10 15:18:27 -06001932 spin_unlock(&ctx->completion_lock);
Pavel Begunkovdac7a092021-03-19 17:22:39 +00001933}
1934
Pavel Begunkovdd78f492021-03-19 17:22:35 +00001935/* Returns true IFF there are requests in the cache */
Jens Axboec7dae4b2021-02-09 19:53:37 -07001936static bool io_flush_cached_reqs(struct io_ring_ctx *ctx)
Pavel Begunkove5d1bc02021-02-10 00:03:23 +00001937{
Jens Axboec7dae4b2021-02-09 19:53:37 -07001938 struct io_submit_state *state = &ctx->submit_state;
Pavel Begunkovdd78f492021-03-19 17:22:35 +00001939 int nr;
Pavel Begunkove5d1bc02021-02-10 00:03:23 +00001940
Jens Axboec7dae4b2021-02-09 19:53:37 -07001941 /*
1942 * If we have more than a batch's worth of requests in our IRQ side
1943 * locked cache, grab the lock and move them over to our submission
1944 * side cache.
1945 */
Pavel Begunkovd0acdee2021-05-16 22:58:12 +01001946 if (READ_ONCE(ctx->locked_free_nr) > IO_COMPL_BATCH)
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01001947 io_flush_cached_locked_reqs(ctx, state);
Jens Axboec7dae4b2021-02-09 19:53:37 -07001948
Pavel Begunkovdd78f492021-03-19 17:22:35 +00001949 nr = state->free_reqs;
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01001950 while (!list_empty(&state->free_list)) {
1951 struct io_kiocb *req = list_first_entry(&state->free_list,
Pavel Begunkovbb943b82021-08-09 20:18:10 +01001952 struct io_kiocb, inflight_entry);
Pavel Begunkovdd78f492021-03-19 17:22:35 +00001953
Pavel Begunkovbb943b82021-08-09 20:18:10 +01001954 list_del(&req->inflight_entry);
Pavel Begunkovdd78f492021-03-19 17:22:35 +00001955 state->reqs[nr++] = req;
1956 if (nr == ARRAY_SIZE(state->reqs))
Pavel Begunkove5d1bc02021-02-10 00:03:23 +00001957 break;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001958 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07001959
Pavel Begunkovdd78f492021-03-19 17:22:35 +00001960 state->free_reqs = nr;
1961 return nr != 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001962}
1963
Pavel Begunkov5d5901a2021-08-11 19:28:29 +01001964/*
1965 * A request might get retired back into the request caches even before opcode
1966 * handlers and io_issue_sqe() are done with it, e.g. inline completion path.
1967 * Because of that, io_alloc_req() should be called only under ->uring_lock
1968 * and with extra caution to not get a request that is still worked on.
1969 */
Pavel Begunkov258b29a2021-02-10 00:03:10 +00001970static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx)
Pavel Begunkov5d5901a2021-08-11 19:28:29 +01001971 __must_hold(&ctx->uring_lock)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001972{
Pavel Begunkov258b29a2021-02-10 00:03:10 +00001973 struct io_submit_state *state = &ctx->submit_state;
Pavel Begunkov864ea922021-08-09 13:04:08 +01001974 gfp_t gfp = GFP_KERNEL | __GFP_NOWARN;
1975 int ret, i;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001976
Pavel Begunkovfe7e3252021-06-24 15:09:57 +01001977 BUILD_BUG_ON(ARRAY_SIZE(state->reqs) < IO_REQ_ALLOC_BATCH);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001978
Pavel Begunkov864ea922021-08-09 13:04:08 +01001979 if (likely(state->free_reqs || io_flush_cached_reqs(ctx)))
1980 goto got_req;
Jens Axboe2579f912019-01-09 09:10:43 -07001981
Pavel Begunkov864ea922021-08-09 13:04:08 +01001982 ret = kmem_cache_alloc_bulk(req_cachep, gfp, IO_REQ_ALLOC_BATCH,
1983 state->reqs);
Pavel Begunkove5d1bc02021-02-10 00:03:23 +00001984
Pavel Begunkov864ea922021-08-09 13:04:08 +01001985 /*
1986 * Bulk alloc is all-or-nothing. If we fail to get a batch,
1987 * retry single alloc to be on the safe side.
1988 */
1989 if (unlikely(ret <= 0)) {
1990 state->reqs[0] = kmem_cache_alloc(req_cachep, gfp);
1991 if (!state->reqs[0])
1992 return NULL;
1993 ret = 1;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001994 }
Pavel Begunkov864ea922021-08-09 13:04:08 +01001995
1996 for (i = 0; i < ret; i++)
1997 io_preinit_req(state->reqs[i], ctx);
1998 state->free_reqs = ret;
Pavel Begunkove5d1bc02021-02-10 00:03:23 +00001999got_req:
Pavel Begunkov291b2822020-09-30 22:57:01 +03002000 state->free_reqs--;
2001 return state->reqs[state->free_reqs];
Jens Axboe2b188cc2019-01-07 10:46:33 -07002002}
2003
Pavel Begunkove1d767f2021-03-19 17:22:43 +00002004static inline void io_put_file(struct file *file)
Pavel Begunkov8da11c12020-02-24 11:32:44 +03002005{
Pavel Begunkove1d767f2021-03-19 17:22:43 +00002006 if (file)
Pavel Begunkov8da11c12020-02-24 11:32:44 +03002007 fput(file);
2008}
2009
Pavel Begunkov4edf20f2020-10-13 09:43:59 +01002010static void io_dismantle_req(struct io_kiocb *req)
Jens Axboe2b188cc2019-01-07 10:46:33 -07002011{
Pavel Begunkov094bae42021-03-19 17:22:42 +00002012 unsigned int flags = req->flags;
Pavel Begunkov929a3af2020-02-19 00:19:09 +03002013
Pavel Begunkov3a0a6902021-04-20 12:03:31 +01002014 if (io_req_needs_clean(req))
2015 io_clean_op(req);
Pavel Begunkove1d767f2021-03-19 17:22:43 +00002016 if (!(flags & REQ_F_FIXED_FILE))
2017 io_put_file(req->file);
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00002018 if (req->fixed_rsrc_refs)
2019 percpu_ref_put(req->fixed_rsrc_refs);
Pavel Begunkov99ebe4e2021-06-26 21:40:49 +01002020 if (req->async_data) {
Pavel Begunkov094bae42021-03-19 17:22:42 +00002021 kfree(req->async_data);
Pavel Begunkov99ebe4e2021-06-26 21:40:49 +01002022 req->async_data = NULL;
2023 }
Pavel Begunkove6543a82020-06-28 12:52:30 +03002024}
Pavel Begunkov2b85edf2019-12-28 14:13:03 +03002025
Pavel Begunkov216578e2020-10-13 09:44:00 +01002026static void __io_free_req(struct io_kiocb *req)
Pavel Begunkove6543a82020-06-28 12:52:30 +03002027{
Jens Axboe51a4cc12020-08-10 10:55:56 -06002028 struct io_ring_ctx *ctx = req->ctx;
Pavel Begunkovecfc5172020-06-29 13:13:03 +03002029
Pavel Begunkov216578e2020-10-13 09:44:00 +01002030 io_dismantle_req(req);
Pavel Begunkov7c660732021-01-25 11:42:21 +00002031 io_put_task(req->task, 1);
Pavel Begunkove6543a82020-06-28 12:52:30 +03002032
Jens Axboe79ebeae2021-08-10 15:18:27 -06002033 spin_lock(&ctx->completion_lock);
Pavel Begunkovbb943b82021-08-09 20:18:10 +01002034 list_add(&req->inflight_entry, &ctx->locked_free_list);
Pavel Begunkovc34b0252021-08-09 20:18:08 +01002035 ctx->locked_free_nr++;
Jens Axboe79ebeae2021-08-10 15:18:27 -06002036 spin_unlock(&ctx->completion_lock);
Pavel Begunkovc34b0252021-08-09 20:18:08 +01002037
Pavel Begunkovecfc5172020-06-29 13:13:03 +03002038 percpu_ref_put(&ctx->refs);
Jens Axboee65ef562019-03-12 10:16:44 -06002039}
2040
Pavel Begunkovf2f87372020-10-27 23:25:37 +00002041static inline void io_remove_next_linked(struct io_kiocb *req)
2042{
2043 struct io_kiocb *nxt = req->link;
2044
2045 req->link = nxt->link;
2046 nxt->link = NULL;
2047}
2048
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00002049static bool io_kill_linked_timeout(struct io_kiocb *req)
2050 __must_hold(&req->ctx->completion_lock)
Jens Axboe89b263f2021-08-10 15:14:18 -06002051 __must_hold(&req->ctx->timeout_lock)
Jens Axboe9e645e112019-05-10 16:07:28 -06002052{
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00002053 struct io_kiocb *link = req->link;
Pavel Begunkovf2f87372020-10-27 23:25:37 +00002054
Pavel Begunkovb97e7362021-08-15 10:40:23 +01002055 if (link && link->opcode == IORING_OP_LINK_TIMEOUT) {
Pavel Begunkovc9abd7a2020-10-22 16:43:11 +01002056 struct io_timeout_data *io = link->async_data;
Pavel Begunkov7c86ffe2020-06-29 13:12:59 +03002057
Pavel Begunkovf2f87372020-10-27 23:25:37 +00002058 io_remove_next_linked(req);
Pavel Begunkov90cd7e42020-10-27 23:25:36 +00002059 link->timeout.head = NULL;
Pavel Begunkovfd9c7bc2021-04-13 02:58:42 +01002060 if (hrtimer_try_to_cancel(&io->timer) != -1) {
Pavel Begunkovef9dd632021-08-28 19:54:38 -06002061 list_del(&link->timeout.list);
Pavel Begunkovb850d6d2022-08-29 14:30:13 +01002062 io_fill_cqe_req(link, -ECANCELED, 0);
Pavel Begunkov91c2f692021-08-11 19:28:28 +01002063 io_put_req_deferred(link);
Pavel Begunkovd4729fb2021-03-22 01:58:24 +00002064 return true;
Pavel Begunkovc9abd7a2020-10-22 16:43:11 +01002065 }
2066 }
Pavel Begunkovd4729fb2021-03-22 01:58:24 +00002067 return false;
Pavel Begunkov7c86ffe2020-06-29 13:12:59 +03002068}
2069
Pavel Begunkovd148ca42020-10-18 10:17:39 +01002070static void io_fail_links(struct io_kiocb *req)
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00002071 __must_hold(&req->ctx->completion_lock)
Jens Axboe9e645e112019-05-10 16:07:28 -06002072{
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00002073 struct io_kiocb *nxt, *link = req->link;
Jens Axboe9e645e112019-05-10 16:07:28 -06002074
Pavel Begunkovf2f87372020-10-27 23:25:37 +00002075 req->link = NULL;
Pavel Begunkovf2f87372020-10-27 23:25:37 +00002076 while (link) {
Hao Xua8295b92021-08-27 17:46:09 +08002077 long res = -ECANCELED;
2078
2079 if (link->flags & REQ_F_FAIL)
2080 res = link->result;
2081
Pavel Begunkovf2f87372020-10-27 23:25:37 +00002082 nxt = link->link;
2083 link->link = NULL;
2084
Dmitrii Dolgovc826bd72019-10-15 19:02:01 +02002085 trace_io_uring_fail_link(req, link);
Pavel Begunkovb850d6d2022-08-29 14:30:13 +01002086 io_fill_cqe_req(link, res, 0);
Pavel Begunkov91c2f692021-08-11 19:28:28 +01002087 io_put_req_deferred(link);
Pavel Begunkovf2f87372020-10-27 23:25:37 +00002088 link = nxt;
Jens Axboe9e645e112019-05-10 16:07:28 -06002089 }
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00002090}
Pavel Begunkov7c86ffe2020-06-29 13:12:59 +03002091
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00002092static bool io_disarm_next(struct io_kiocb *req)
2093 __must_hold(&req->ctx->completion_lock)
2094{
2095 bool posted = false;
2096
Pavel Begunkov0756a862021-08-15 10:40:25 +01002097 if (req->flags & REQ_F_ARM_LTIMEOUT) {
2098 struct io_kiocb *link = req->link;
2099
Pavel Begunkov906c6ca2021-08-15 10:40:26 +01002100 req->flags &= ~REQ_F_ARM_LTIMEOUT;
Pavel Begunkov0756a862021-08-15 10:40:25 +01002101 if (link && link->opcode == IORING_OP_LINK_TIMEOUT) {
2102 io_remove_next_linked(req);
Pavel Begunkovb850d6d2022-08-29 14:30:13 +01002103 io_fill_cqe_req(link, -ECANCELED, 0);
Pavel Begunkov0756a862021-08-15 10:40:25 +01002104 io_put_req_deferred(link);
2105 posted = true;
2106 }
2107 } else if (req->flags & REQ_F_LINK_TIMEOUT) {
Jens Axboe89b263f2021-08-10 15:14:18 -06002108 struct io_ring_ctx *ctx = req->ctx;
2109
2110 spin_lock_irq(&ctx->timeout_lock);
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00002111 posted = io_kill_linked_timeout(req);
Jens Axboe89b263f2021-08-10 15:14:18 -06002112 spin_unlock_irq(&ctx->timeout_lock);
2113 }
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01002114 if (unlikely((req->flags & REQ_F_FAIL) &&
Pavel Begunkove4335ed2021-04-11 01:46:39 +01002115 !(req->flags & REQ_F_HARDLINK))) {
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00002116 posted |= (req->link != NULL);
2117 io_fail_links(req);
2118 }
2119 return posted;
Jens Axboe9e645e112019-05-10 16:07:28 -06002120}
2121
Pavel Begunkov3fa5e0f2020-06-30 15:20:43 +03002122static struct io_kiocb *__io_req_find_next(struct io_kiocb *req)
Jens Axboe9e645e112019-05-10 16:07:28 -06002123{
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00002124 struct io_kiocb *nxt;
Jens Axboe2665abf2019-11-05 12:40:47 -07002125
Jens Axboe9e645e112019-05-10 16:07:28 -06002126 /*
2127 * If LINK is set, we have dependent requests in this chain. If we
2128 * didn't fail this request, queue the first one up, moving any other
2129 * dependencies to the next request. In case of failure, fail the rest
2130 * of the chain.
2131 */
Pavel Begunkov0756a862021-08-15 10:40:25 +01002132 if (req->flags & IO_DISARM_MASK) {
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00002133 struct io_ring_ctx *ctx = req->ctx;
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00002134 bool posted;
Pavel Begunkovf2f87372020-10-27 23:25:37 +00002135
Jens Axboe79ebeae2021-08-10 15:18:27 -06002136 spin_lock(&ctx->completion_lock);
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00002137 posted = io_disarm_next(req);
2138 if (posted)
2139 io_commit_cqring(req->ctx);
Jens Axboe79ebeae2021-08-10 15:18:27 -06002140 spin_unlock(&ctx->completion_lock);
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00002141 if (posted)
2142 io_cqring_ev_posted(ctx);
Pavel Begunkovf2f87372020-10-27 23:25:37 +00002143 }
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00002144 nxt = req->link;
2145 req->link = NULL;
2146 return nxt;
Jens Axboe4d7dd462019-11-20 13:03:52 -07002147}
Jens Axboe2665abf2019-11-05 12:40:47 -07002148
Pavel Begunkovf2f87372020-10-27 23:25:37 +00002149static inline struct io_kiocb *io_req_find_next(struct io_kiocb *req)
Pavel Begunkov3fa5e0f2020-06-30 15:20:43 +03002150{
Pavel Begunkovcdbff982021-02-12 18:41:16 +00002151 if (likely(!(req->flags & (REQ_F_LINK|REQ_F_HARDLINK))))
Pavel Begunkov3fa5e0f2020-06-30 15:20:43 +03002152 return NULL;
2153 return __io_req_find_next(req);
2154}
2155
Pavel Begunkovf237c302021-08-18 12:42:46 +01002156static void ctx_flush_and_put(struct io_ring_ctx *ctx, bool *locked)
Pavel Begunkov2c323952021-02-28 22:04:53 +00002157{
2158 if (!ctx)
2159 return;
Pavel Begunkovf237c302021-08-18 12:42:46 +01002160 if (*locked) {
Hao Xu99c8bc52021-08-21 06:19:54 +08002161 if (ctx->submit_state.compl_nr)
2162 io_submit_flush_completions(ctx);
Pavel Begunkov2c323952021-02-28 22:04:53 +00002163 mutex_unlock(&ctx->uring_lock);
Pavel Begunkovf237c302021-08-18 12:42:46 +01002164 *locked = false;
Pavel Begunkov2c323952021-02-28 22:04:53 +00002165 }
2166 percpu_ref_put(&ctx->refs);
2167}
2168
Jens Axboe7cbf1722021-02-10 00:03:20 +00002169static void tctx_task_work(struct callback_head *cb)
2170{
Pavel Begunkovf237c302021-08-18 12:42:46 +01002171 bool locked = false;
Pavel Begunkovebd0df22021-06-17 18:14:07 +01002172 struct io_ring_ctx *ctx = NULL;
Pavel Begunkov3f184072021-06-17 18:14:06 +01002173 struct io_uring_task *tctx = container_of(cb, struct io_uring_task,
2174 task_work);
Jens Axboe7cbf1722021-02-10 00:03:20 +00002175
Pavel Begunkov16f72072021-06-17 18:14:09 +01002176 while (1) {
Pavel Begunkov3f184072021-06-17 18:14:06 +01002177 struct io_wq_work_node *node;
2178
Pavel Begunkov8d4ad412021-09-02 00:38:23 +01002179 if (!tctx->task_list.first && locked && ctx->submit_state.compl_nr)
2180 io_submit_flush_completions(ctx);
2181
Pavel Begunkov3f184072021-06-17 18:14:06 +01002182 spin_lock_irq(&tctx->task_lock);
Pavel Begunkovc6538be2021-06-17 18:14:08 +01002183 node = tctx->task_list.first;
Pavel Begunkov3f184072021-06-17 18:14:06 +01002184 INIT_WQ_LIST(&tctx->task_list);
Pavel Begunkov6294f362021-08-10 17:53:55 +01002185 if (!node)
2186 tctx->task_running = false;
Pavel Begunkov3f184072021-06-17 18:14:06 +01002187 spin_unlock_irq(&tctx->task_lock);
Pavel Begunkov6294f362021-08-10 17:53:55 +01002188 if (!node)
2189 break;
Pavel Begunkov3f184072021-06-17 18:14:06 +01002190
Pavel Begunkov6294f362021-08-10 17:53:55 +01002191 do {
Pavel Begunkov3f184072021-06-17 18:14:06 +01002192 struct io_wq_work_node *next = node->next;
2193 struct io_kiocb *req = container_of(node, struct io_kiocb,
2194 io_task_work.node);
2195
2196 if (req->ctx != ctx) {
Pavel Begunkovf237c302021-08-18 12:42:46 +01002197 ctx_flush_and_put(ctx, &locked);
Pavel Begunkov3f184072021-06-17 18:14:06 +01002198 ctx = req->ctx;
Pavel Begunkov126180b2021-08-18 12:42:47 +01002199 /* if not contended, grab and improve batching */
2200 locked = mutex_trylock(&ctx->uring_lock);
Pavel Begunkov3f184072021-06-17 18:14:06 +01002201 percpu_ref_get(&ctx->refs);
2202 }
Pavel Begunkovf237c302021-08-18 12:42:46 +01002203 req->io_task_work.func(req, &locked);
Pavel Begunkov3f184072021-06-17 18:14:06 +01002204 node = next;
Pavel Begunkov6294f362021-08-10 17:53:55 +01002205 } while (node);
2206
Jens Axboe7cbf1722021-02-10 00:03:20 +00002207 cond_resched();
Pavel Begunkov3f184072021-06-17 18:14:06 +01002208 }
Pavel Begunkovebd0df22021-06-17 18:14:07 +01002209
Pavel Begunkovf237c302021-08-18 12:42:46 +01002210 ctx_flush_and_put(ctx, &locked);
Pavel Begunkovb168b1a2022-01-09 00:53:22 +00002211
2212 /* relaxed read is enough as only the task itself sets ->in_idle */
2213 if (unlikely(atomic_read(&tctx->in_idle)))
2214 io_uring_drop_tctx_refs(current);
Jens Axboe7cbf1722021-02-10 00:03:20 +00002215}
2216
Pavel Begunkove09ee512021-07-01 13:26:05 +01002217static void io_req_task_work_add(struct io_kiocb *req)
Jens Axboe7cbf1722021-02-10 00:03:20 +00002218{
Pavel Begunkovc15b79d2021-03-19 17:22:44 +00002219 struct task_struct *tsk = req->task;
Jens Axboe7cbf1722021-02-10 00:03:20 +00002220 struct io_uring_task *tctx = tsk->io_uring;
Pavel Begunkovc15b79d2021-03-19 17:22:44 +00002221 enum task_work_notify_mode notify;
Pavel Begunkove09ee512021-07-01 13:26:05 +01002222 struct io_wq_work_node *node;
Jens Axboe0b81e802021-02-16 10:33:53 -07002223 unsigned long flags;
Pavel Begunkov6294f362021-08-10 17:53:55 +01002224 bool running;
Jens Axboe7cbf1722021-02-10 00:03:20 +00002225
2226 WARN_ON_ONCE(!tctx);
2227
Jens Axboe0b81e802021-02-16 10:33:53 -07002228 spin_lock_irqsave(&tctx->task_lock, flags);
Jens Axboe7cbf1722021-02-10 00:03:20 +00002229 wq_list_add_tail(&req->io_task_work.node, &tctx->task_list);
Pavel Begunkov6294f362021-08-10 17:53:55 +01002230 running = tctx->task_running;
2231 if (!running)
2232 tctx->task_running = true;
Jens Axboe0b81e802021-02-16 10:33:53 -07002233 spin_unlock_irqrestore(&tctx->task_lock, flags);
Jens Axboe7cbf1722021-02-10 00:03:20 +00002234
2235 /* task_work already pending, we're done */
Pavel Begunkov6294f362021-08-10 17:53:55 +01002236 if (running)
Pavel Begunkove09ee512021-07-01 13:26:05 +01002237 return;
Jens Axboe7cbf1722021-02-10 00:03:20 +00002238
Pavel Begunkovc15b79d2021-03-19 17:22:44 +00002239 /*
2240 * SQPOLL kernel thread doesn't need notification, just a wakeup. For
2241 * all other cases, use TWA_SIGNAL unconditionally to ensure we're
2242 * processing task_work. There's no reliable way to tell if TWA_RESUME
2243 * will do the job.
2244 */
2245 notify = (req->ctx->flags & IORING_SETUP_SQPOLL) ? TWA_NONE : TWA_SIGNAL;
Pavel Begunkovc15b79d2021-03-19 17:22:44 +00002246 if (!task_work_add(tsk, &tctx->task_work, notify)) {
2247 wake_up_process(tsk);
Pavel Begunkove09ee512021-07-01 13:26:05 +01002248 return;
Pavel Begunkovc15b79d2021-03-19 17:22:44 +00002249 }
Pavel Begunkov2215bed2021-08-09 13:04:06 +01002250
Pavel Begunkove09ee512021-07-01 13:26:05 +01002251 spin_lock_irqsave(&tctx->task_lock, flags);
Pavel Begunkov6294f362021-08-10 17:53:55 +01002252 tctx->task_running = false;
Pavel Begunkove09ee512021-07-01 13:26:05 +01002253 node = tctx->task_list.first;
2254 INIT_WQ_LIST(&tctx->task_list);
2255 spin_unlock_irqrestore(&tctx->task_lock, flags);
Jens Axboe7cbf1722021-02-10 00:03:20 +00002256
Pavel Begunkove09ee512021-07-01 13:26:05 +01002257 while (node) {
2258 req = container_of(node, struct io_kiocb, io_task_work.node);
2259 node = node->next;
2260 if (llist_add(&req->io_task_work.fallback_node,
2261 &req->ctx->fallback_llist))
2262 schedule_delayed_work(&req->ctx->fallback_work, 1);
2263 }
Pavel Begunkoveab30c42021-01-19 13:32:42 +00002264}
2265
Pavel Begunkovf237c302021-08-18 12:42:46 +01002266static void io_req_task_cancel(struct io_kiocb *req, bool *locked)
Jens Axboec40f6372020-06-25 15:39:59 -06002267{
Jens Axboe87ceb6a2020-09-14 08:20:12 -06002268 struct io_ring_ctx *ctx = req->ctx;
Jens Axboec40f6372020-06-25 15:39:59 -06002269
Pavel Begunkovb18a1a42021-08-25 20:51:39 +01002270 /* not needed for normal modes, but SQPOLL depends on it */
Pavel Begunkovf237c302021-08-18 12:42:46 +01002271 io_tw_lock(ctx, locked);
Pavel Begunkov25935532021-03-19 17:22:40 +00002272 io_req_complete_failed(req, req->result);
Jens Axboec40f6372020-06-25 15:39:59 -06002273}
2274
Pavel Begunkovf237c302021-08-18 12:42:46 +01002275static void io_req_task_submit(struct io_kiocb *req, bool *locked)
Jens Axboec40f6372020-06-25 15:39:59 -06002276{
2277 struct io_ring_ctx *ctx = req->ctx;
2278
Pavel Begunkovf237c302021-08-18 12:42:46 +01002279 io_tw_lock(ctx, locked);
Jens Axboe316319e2021-08-19 09:41:42 -06002280 /* req->task == current here, checking PF_EXITING is safe */
Pavel Begunkovaf066f32021-08-09 13:04:19 +01002281 if (likely(!(req->task->flags & PF_EXITING)))
Pavel Begunkovc5eef2b2021-02-10 00:03:22 +00002282 __io_queue_sqe(req);
Pavel Begunkov81b6d052021-01-04 20:36:35 +00002283 else
Pavel Begunkov25935532021-03-19 17:22:40 +00002284 io_req_complete_failed(req, -EFAULT);
Jens Axboe9e645e112019-05-10 16:07:28 -06002285}
2286
Pavel Begunkova3df76982021-02-18 22:32:52 +00002287static void io_req_task_queue_fail(struct io_kiocb *req, int ret)
2288{
Pavel Begunkova3df76982021-02-18 22:32:52 +00002289 req->result = ret;
Pavel Begunkov5b0a6ac2021-06-30 21:54:04 +01002290 req->io_task_work.func = io_req_task_cancel;
Pavel Begunkove09ee512021-07-01 13:26:05 +01002291 io_req_task_work_add(req);
Pavel Begunkova3df76982021-02-18 22:32:52 +00002292}
2293
Pavel Begunkov2c4b8eb2021-02-28 22:35:10 +00002294static void io_req_task_queue(struct io_kiocb *req)
2295{
Pavel Begunkov5b0a6ac2021-06-30 21:54:04 +01002296 req->io_task_work.func = io_req_task_submit;
Pavel Begunkove09ee512021-07-01 13:26:05 +01002297 io_req_task_work_add(req);
Pavel Begunkov2c4b8eb2021-02-28 22:35:10 +00002298}
2299
Jens Axboe773af692021-07-27 10:25:55 -06002300static void io_req_task_queue_reissue(struct io_kiocb *req)
2301{
2302 req->io_task_work.func = io_queue_async_work;
2303 io_req_task_work_add(req);
2304}
2305
Pavel Begunkovf2f87372020-10-27 23:25:37 +00002306static inline void io_queue_next(struct io_kiocb *req)
Jackie Liuc69f8db2019-11-09 11:00:08 +08002307{
Pavel Begunkov9b5f7bd2020-06-29 13:13:00 +03002308 struct io_kiocb *nxt = io_req_find_next(req);
Pavel Begunkov944e58b2019-11-21 23:21:01 +03002309
Pavel Begunkov906a8c32020-06-27 14:04:55 +03002310 if (nxt)
2311 io_req_task_queue(nxt);
Jackie Liuc69f8db2019-11-09 11:00:08 +08002312}
2313
Jens Axboe9e645e112019-05-10 16:07:28 -06002314static void io_free_req(struct io_kiocb *req)
2315{
Pavel Begunkovc3524382020-06-28 12:52:32 +03002316 io_queue_next(req);
Jens Axboe9e645e112019-05-10 16:07:28 -06002317 __io_free_req(req);
Jens Axboee65ef562019-03-12 10:16:44 -06002318}
2319
Pavel Begunkovf237c302021-08-18 12:42:46 +01002320static void io_free_req_work(struct io_kiocb *req, bool *locked)
2321{
2322 io_free_req(req);
2323}
2324
Pavel Begunkov2d6500d2020-06-28 12:52:33 +03002325struct req_batch {
Pavel Begunkov5af1d132020-07-18 11:32:52 +03002326 struct task_struct *task;
2327 int task_refs;
Jens Axboe1b4c3512021-02-10 00:03:19 +00002328 int ctx_refs;
Pavel Begunkov2d6500d2020-06-28 12:52:33 +03002329};
2330
Pavel Begunkov5af1d132020-07-18 11:32:52 +03002331static inline void io_init_req_batch(struct req_batch *rb)
Pavel Begunkov7a743e22020-03-03 21:33:13 +03002332{
Pavel Begunkov5af1d132020-07-18 11:32:52 +03002333 rb->task_refs = 0;
Pavel Begunkov9ae72462021-02-10 00:03:16 +00002334 rb->ctx_refs = 0;
Pavel Begunkov5af1d132020-07-18 11:32:52 +03002335 rb->task = NULL;
2336}
Pavel Begunkov8766dd52020-03-14 00:31:04 +03002337
Pavel Begunkov2d6500d2020-06-28 12:52:33 +03002338static void io_req_free_batch_finish(struct io_ring_ctx *ctx,
2339 struct req_batch *rb)
2340{
Pavel Begunkov9ae72462021-02-10 00:03:16 +00002341 if (rb->ctx_refs)
2342 percpu_ref_put_many(&ctx->refs, rb->ctx_refs);
Pavel Begunkove98e49b2021-08-18 17:01:43 +01002343 if (rb->task)
Pavel Begunkove9dbe222021-08-09 13:04:20 +01002344 io_put_task(rb->task, rb->task_refs);
Pavel Begunkov2d6500d2020-06-28 12:52:33 +03002345}
2346
Pavel Begunkov6ff119a2021-02-10 00:03:18 +00002347static void io_req_free_batch(struct req_batch *rb, struct io_kiocb *req,
2348 struct io_submit_state *state)
Pavel Begunkov2d6500d2020-06-28 12:52:33 +03002349{
Pavel Begunkovf2f87372020-10-27 23:25:37 +00002350 io_queue_next(req);
Pavel Begunkov96670652021-03-19 17:22:32 +00002351 io_dismantle_req(req);
Pavel Begunkov2d6500d2020-06-28 12:52:33 +03002352
Jens Axboee3bc8e92020-09-24 08:45:57 -06002353 if (req->task != rb->task) {
Pavel Begunkov7c660732021-01-25 11:42:21 +00002354 if (rb->task)
2355 io_put_task(rb->task, rb->task_refs);
Jens Axboee3bc8e92020-09-24 08:45:57 -06002356 rb->task = req->task;
2357 rb->task_refs = 0;
Pavel Begunkov5af1d132020-07-18 11:32:52 +03002358 }
Jens Axboee3bc8e92020-09-24 08:45:57 -06002359 rb->task_refs++;
Pavel Begunkov9ae72462021-02-10 00:03:16 +00002360 rb->ctx_refs++;
Pavel Begunkov5af1d132020-07-18 11:32:52 +03002361
Pavel Begunkovbd759042021-02-12 03:23:50 +00002362 if (state->free_reqs != ARRAY_SIZE(state->reqs))
Pavel Begunkov6ff119a2021-02-10 00:03:18 +00002363 state->reqs[state->free_reqs++] = req;
Pavel Begunkovbd759042021-02-12 03:23:50 +00002364 else
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01002365 list_add(&req->inflight_entry, &state->free_list);
Pavel Begunkov7a743e22020-03-03 21:33:13 +03002366}
2367
Pavel Begunkov2a2758f2021-06-17 18:14:00 +01002368static void io_submit_flush_completions(struct io_ring_ctx *ctx)
Jens Axboea141dd82021-08-12 12:48:34 -06002369 __must_hold(&ctx->uring_lock)
Pavel Begunkov905c1722021-02-10 00:03:14 +00002370{
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01002371 struct io_submit_state *state = &ctx->submit_state;
2372 int i, nr = state->compl_nr;
Pavel Begunkov905c1722021-02-10 00:03:14 +00002373 struct req_batch rb;
2374
Jens Axboe79ebeae2021-08-10 15:18:27 -06002375 spin_lock(&ctx->completion_lock);
Pavel Begunkov905c1722021-02-10 00:03:14 +00002376 for (i = 0; i < nr; i++) {
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01002377 struct io_kiocb *req = state->compl_reqs[i];
Pavel Begunkov5182ed22021-06-26 21:40:48 +01002378
Pavel Begunkovb850d6d2022-08-29 14:30:13 +01002379 __io_fill_cqe(ctx, req->user_data, req->result,
2380 req->compl.cflags);
Pavel Begunkov905c1722021-02-10 00:03:14 +00002381 }
2382 io_commit_cqring(ctx);
Jens Axboe79ebeae2021-08-10 15:18:27 -06002383 spin_unlock(&ctx->completion_lock);
Pavel Begunkov905c1722021-02-10 00:03:14 +00002384 io_cqring_ev_posted(ctx);
Pavel Begunkov5182ed22021-06-26 21:40:48 +01002385
2386 io_init_req_batch(&rb);
Pavel Begunkov905c1722021-02-10 00:03:14 +00002387 for (i = 0; i < nr; i++) {
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01002388 struct io_kiocb *req = state->compl_reqs[i];
Pavel Begunkov905c1722021-02-10 00:03:14 +00002389
Pavel Begunkov91c2f692021-08-11 19:28:28 +01002390 if (req_ref_put_and_test(req))
Pavel Begunkov6ff119a2021-02-10 00:03:18 +00002391 io_req_free_batch(&rb, req, &ctx->submit_state);
Pavel Begunkov905c1722021-02-10 00:03:14 +00002392 }
2393
2394 io_req_free_batch_finish(ctx, &rb);
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01002395 state->compl_nr = 0;
Jens Axboee65ef562019-03-12 10:16:44 -06002396}
2397
Jens Axboeba816ad2019-09-28 11:36:45 -06002398/*
2399 * Drop reference to request, return next in chain (if there is one) if this
2400 * was the last reference to this request.
2401 */
Pavel Begunkov0d850352021-03-19 17:22:37 +00002402static inline struct io_kiocb *io_put_req_find_next(struct io_kiocb *req)
Jens Axboee65ef562019-03-12 10:16:44 -06002403{
Pavel Begunkov9b5f7bd2020-06-29 13:13:00 +03002404 struct io_kiocb *nxt = NULL;
2405
Jens Axboede9b4cc2021-02-24 13:28:27 -07002406 if (req_ref_put_and_test(req)) {
Pavel Begunkov9b5f7bd2020-06-29 13:13:00 +03002407 nxt = io_req_find_next(req);
Jens Axboe4d7dd462019-11-20 13:03:52 -07002408 __io_free_req(req);
Jens Axboe2a44f462020-02-25 13:25:41 -07002409 }
Pavel Begunkov9b5f7bd2020-06-29 13:13:00 +03002410 return nxt;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002411}
2412
Pavel Begunkov0d850352021-03-19 17:22:37 +00002413static inline void io_put_req(struct io_kiocb *req)
Jens Axboe2b188cc2019-01-07 10:46:33 -07002414{
Jens Axboede9b4cc2021-02-24 13:28:27 -07002415 if (req_ref_put_and_test(req))
Jens Axboedef596e2019-01-09 08:59:42 -07002416 io_free_req(req);
2417}
2418
Pavel Begunkov91c2f692021-08-11 19:28:28 +01002419static inline void io_put_req_deferred(struct io_kiocb *req)
Pavel Begunkov216578e2020-10-13 09:44:00 +01002420{
Pavel Begunkov91c2f692021-08-11 19:28:28 +01002421 if (req_ref_put_and_test(req)) {
Pavel Begunkovf237c302021-08-18 12:42:46 +01002422 req->io_task_work.func = io_free_req_work;
Pavel Begunkov543af3a2021-08-09 13:04:15 +01002423 io_req_task_work_add(req);
2424 }
Pavel Begunkov216578e2020-10-13 09:44:00 +01002425}
2426
Pavel Begunkov6c503152021-01-04 20:36:36 +00002427static unsigned io_cqring_events(struct io_ring_ctx *ctx)
Jens Axboea3a0e432019-08-20 11:03:11 -06002428{
2429 /* See comment at the top of this file */
2430 smp_rmb();
Pavel Begunkove23de152020-12-17 00:24:37 +00002431 return __io_cqring_events(ctx);
Jens Axboea3a0e432019-08-20 11:03:11 -06002432}
2433
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03002434static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx)
2435{
2436 struct io_rings *rings = ctx->rings;
2437
2438 /* make sure SQ entry isn't read before tail */
2439 return smp_load_acquire(&rings->sq.tail) - ctx->cached_sq_head;
2440}
2441
Pavel Begunkov8ff069b2020-07-16 23:28:04 +03002442static unsigned int io_put_kbuf(struct io_kiocb *req, struct io_buffer *kbuf)
Jens Axboee94f1412019-12-19 12:06:02 -07002443{
Pavel Begunkov8ff069b2020-07-16 23:28:04 +03002444 unsigned int cflags;
Jens Axboee94f1412019-12-19 12:06:02 -07002445
Jens Axboebcda7ba2020-02-23 16:42:51 -07002446 cflags = kbuf->bid << IORING_CQE_BUFFER_SHIFT;
2447 cflags |= IORING_CQE_F_BUFFER;
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03002448 req->flags &= ~REQ_F_BUFFER_SELECTED;
Jens Axboebcda7ba2020-02-23 16:42:51 -07002449 kfree(kbuf);
2450 return cflags;
2451}
2452
Pavel Begunkov8ff069b2020-07-16 23:28:04 +03002453static inline unsigned int io_put_rw_kbuf(struct io_kiocb *req)
2454{
2455 struct io_buffer *kbuf;
2456
Pavel Begunkovae421d92021-08-17 20:28:08 +01002457 if (likely(!(req->flags & REQ_F_BUFFER_SELECTED)))
2458 return 0;
Pavel Begunkov8ff069b2020-07-16 23:28:04 +03002459 kbuf = (struct io_buffer *) (unsigned long) req->rw.addr;
2460 return io_put_kbuf(req, kbuf);
2461}
2462
Jens Axboe4c6e2772020-07-01 11:29:10 -06002463static inline bool io_run_task_work(void)
2464{
Nadav Amitef98eb02021-08-07 17:13:41 -07002465 if (test_thread_flag(TIF_NOTIFY_SIGNAL) || current->task_works) {
Jens Axboe4c6e2772020-07-01 11:29:10 -06002466 __set_current_state(TASK_RUNNING);
Nadav Amitef98eb02021-08-07 17:13:41 -07002467 tracehook_notify_signal();
Jens Axboe4c6e2772020-07-01 11:29:10 -06002468 return true;
2469 }
2470
2471 return false;
2472}
2473
Jens Axboedef596e2019-01-09 08:59:42 -07002474/*
2475 * Find and free completed poll iocbs
2476 */
2477static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events,
Pavel Begunkova8576af2021-08-15 10:40:21 +01002478 struct list_head *done)
Jens Axboedef596e2019-01-09 08:59:42 -07002479{
Jens Axboe8237e042019-12-28 10:48:22 -07002480 struct req_batch rb;
Jens Axboedef596e2019-01-09 08:59:42 -07002481 struct io_kiocb *req;
Xiaoguang Wangbbde0172020-06-16 02:06:38 +08002482
2483 /* order with ->result store in io_complete_rw_iopoll() */
2484 smp_rmb();
Jens Axboedef596e2019-01-09 08:59:42 -07002485
Pavel Begunkov5af1d132020-07-18 11:32:52 +03002486 io_init_req_batch(&rb);
Jens Axboedef596e2019-01-09 08:59:42 -07002487 while (!list_empty(done)) {
Pavel Begunkoved4629d2023-01-14 09:14:03 -07002488 struct io_uring_cqe *cqe;
2489 unsigned cflags;
2490
Pavel Begunkovd21ffe72020-07-13 23:37:10 +03002491 req = list_first_entry(done, struct io_kiocb, inflight_entry);
Pavel Begunkovd21ffe72020-07-13 23:37:10 +03002492 list_del(&req->inflight_entry);
Pavel Begunkoved4629d2023-01-14 09:14:03 -07002493 cflags = io_put_rw_kbuf(req);
Jens Axboedef596e2019-01-09 08:59:42 -07002494 (*nr_events)++;
2495
Pavel Begunkoved4629d2023-01-14 09:14:03 -07002496 cqe = io_get_cqe(ctx);
2497 if (cqe) {
2498 WRITE_ONCE(cqe->user_data, req->user_data);
2499 WRITE_ONCE(cqe->res, req->result);
2500 WRITE_ONCE(cqe->flags, cflags);
2501 } else {
2502 spin_lock(&ctx->completion_lock);
2503 io_cqring_event_overflow(ctx, req->user_data,
2504 req->result, cflags);
2505 spin_unlock(&ctx->completion_lock);
2506 }
2507
Jens Axboede9b4cc2021-02-24 13:28:27 -07002508 if (req_ref_put_and_test(req))
Pavel Begunkov6ff119a2021-02-10 00:03:18 +00002509 io_req_free_batch(&rb, req, &ctx->submit_state);
Jens Axboedef596e2019-01-09 08:59:42 -07002510 }
Jens Axboedef596e2019-01-09 08:59:42 -07002511
Jens Axboe09bb8392019-03-13 12:39:28 -06002512 io_commit_cqring(ctx);
Pavel Begunkov80c18e42021-01-07 03:15:41 +00002513 io_cqring_ev_posted_iopoll(ctx);
Pavel Begunkov2d6500d2020-06-28 12:52:33 +03002514 io_req_free_batch_finish(ctx, &rb);
Bijan Mottahedeh581f9812020-04-03 13:51:33 -07002515}
2516
Jens Axboedef596e2019-01-09 08:59:42 -07002517static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events,
Pavel Begunkova8576af2021-08-15 10:40:21 +01002518 long min)
Jens Axboedef596e2019-01-09 08:59:42 -07002519{
2520 struct io_kiocb *req, *tmp;
2521 LIST_HEAD(done);
2522 bool spin;
Jens Axboedef596e2019-01-09 08:59:42 -07002523
2524 /*
2525 * Only spin for completions if we don't have multiple devices hanging
2526 * off our complete list, and we're under the requested amount.
2527 */
Hao Xu915b3dd2021-06-28 05:37:30 +08002528 spin = !ctx->poll_multi_queue && *nr_events < min;
Jens Axboedef596e2019-01-09 08:59:42 -07002529
Pavel Begunkovd21ffe72020-07-13 23:37:10 +03002530 list_for_each_entry_safe(req, tmp, &ctx->iopoll_list, inflight_entry) {
Jens Axboe9adbd452019-12-20 08:45:55 -07002531 struct kiocb *kiocb = &req->rw.kiocb;
Pavel Begunkova2416e12021-08-09 13:04:09 +01002532 int ret;
Jens Axboedef596e2019-01-09 08:59:42 -07002533
2534 /*
Bijan Mottahedeh581f9812020-04-03 13:51:33 -07002535 * Move completed and retryable entries to our local lists.
2536 * If we find a request that requires polling, break out
2537 * and complete those lists first, if we have entries there.
Jens Axboedef596e2019-01-09 08:59:42 -07002538 */
Xiaoguang Wang65a65432020-06-11 23:39:36 +08002539 if (READ_ONCE(req->iopoll_completed)) {
Pavel Begunkovd21ffe72020-07-13 23:37:10 +03002540 list_move_tail(&req->inflight_entry, &done);
Jens Axboedef596e2019-01-09 08:59:42 -07002541 continue;
2542 }
2543 if (!list_empty(&done))
2544 break;
2545
2546 ret = kiocb->ki_filp->f_op->iopoll(kiocb, spin);
Pavel Begunkova2416e12021-08-09 13:04:09 +01002547 if (unlikely(ret < 0))
2548 return ret;
2549 else if (ret)
2550 spin = false;
Jens Axboedef596e2019-01-09 08:59:42 -07002551
Pavel Begunkov3aadc232020-07-06 17:59:29 +03002552 /* iopoll may have completed current req */
2553 if (READ_ONCE(req->iopoll_completed))
Pavel Begunkovd21ffe72020-07-13 23:37:10 +03002554 list_move_tail(&req->inflight_entry, &done);
Jens Axboedef596e2019-01-09 08:59:42 -07002555 }
2556
2557 if (!list_empty(&done))
Pavel Begunkova8576af2021-08-15 10:40:21 +01002558 io_iopoll_complete(ctx, nr_events, &done);
Jens Axboedef596e2019-01-09 08:59:42 -07002559
Pavel Begunkova2416e12021-08-09 13:04:09 +01002560 return 0;
Jens Axboedef596e2019-01-09 08:59:42 -07002561}
2562
2563/*
Jens Axboedef596e2019-01-09 08:59:42 -07002564 * We can't just wait for polled events to come to us, we have to actively
2565 * find and complete them.
2566 */
Pavel Begunkovb2edc0a2020-07-07 16:36:22 +03002567static void io_iopoll_try_reap_events(struct io_ring_ctx *ctx)
Jens Axboedef596e2019-01-09 08:59:42 -07002568{
2569 if (!(ctx->flags & IORING_SETUP_IOPOLL))
2570 return;
2571
2572 mutex_lock(&ctx->uring_lock);
Pavel Begunkov540e32a2020-07-13 23:37:09 +03002573 while (!list_empty(&ctx->iopoll_list)) {
Jens Axboedef596e2019-01-09 08:59:42 -07002574 unsigned int nr_events = 0;
2575
Pavel Begunkova8576af2021-08-15 10:40:21 +01002576 io_do_iopoll(ctx, &nr_events, 0);
Jens Axboe08f54392019-08-21 22:19:11 -06002577
Pavel Begunkovb2edc0a2020-07-07 16:36:22 +03002578 /* let it sleep and repeat later if can't complete a request */
2579 if (nr_events == 0)
2580 break;
Jens Axboe08f54392019-08-21 22:19:11 -06002581 /*
2582 * Ensure we allow local-to-the-cpu processing to take place,
2583 * in this case we need to ensure that we reap all events.
Pavel Begunkov3fcee5a2020-07-06 17:59:31 +03002584 * Also let task_work, etc. to progress by releasing the mutex
Jens Axboe08f54392019-08-21 22:19:11 -06002585 */
Pavel Begunkov3fcee5a2020-07-06 17:59:31 +03002586 if (need_resched()) {
2587 mutex_unlock(&ctx->uring_lock);
2588 cond_resched();
2589 mutex_lock(&ctx->uring_lock);
2590 }
Jens Axboedef596e2019-01-09 08:59:42 -07002591 }
2592 mutex_unlock(&ctx->uring_lock);
2593}
2594
Pavel Begunkov7668b922020-07-07 16:36:21 +03002595static int io_iopoll_check(struct io_ring_ctx *ctx, long min)
Jens Axboedef596e2019-01-09 08:59:42 -07002596{
Pavel Begunkov7668b922020-07-07 16:36:21 +03002597 unsigned int nr_events = 0;
Pavel Begunkove9979b32021-04-13 02:58:45 +01002598 int ret = 0;
Jens Axboedef596e2019-01-09 08:59:42 -07002599
Xiaoguang Wangc7849be2020-02-22 14:46:05 +08002600 /*
2601 * We disallow the app entering submit/complete with polling, but we
2602 * still need to lock the ring to prevent racing with polled issue
2603 * that got punted to a workqueue.
2604 */
2605 mutex_lock(&ctx->uring_lock);
Pavel Begunkovf39c8a52021-04-13 02:58:46 +01002606 /*
2607 * Don't enter poll loop if we already have events pending.
2608 * If we do, we can potentially be spinning for commands that
2609 * already triggered a CQE (eg in error).
2610 */
Pavel Begunkov5ed7a372021-06-14 23:37:27 +01002611 if (test_bit(0, &ctx->check_cq_overflow))
Pavel Begunkovf39c8a52021-04-13 02:58:46 +01002612 __io_cqring_overflow_flush(ctx, false);
2613 if (io_cqring_events(ctx))
2614 goto out;
Jens Axboedef596e2019-01-09 08:59:42 -07002615 do {
Jens Axboe500f9fb2019-08-19 12:15:59 -06002616 /*
2617 * If a submit got punted to a workqueue, we can have the
2618 * application entering polling for a command before it gets
2619 * issued. That app will hold the uring_lock for the duration
2620 * of the poll right here, so we need to take a breather every
2621 * now and then to ensure that the issue has a chance to add
2622 * the poll to the issued list. Otherwise we can spin here
2623 * forever, while the workqueue is stuck trying to acquire the
2624 * very same mutex.
2625 */
Pavel Begunkove9979b32021-04-13 02:58:45 +01002626 if (list_empty(&ctx->iopoll_list)) {
Pavel Begunkov8f487ef2021-07-08 13:37:06 +01002627 u32 tail = ctx->cached_cq_tail;
2628
Jens Axboe500f9fb2019-08-19 12:15:59 -06002629 mutex_unlock(&ctx->uring_lock);
Jens Axboe4c6e2772020-07-01 11:29:10 -06002630 io_run_task_work();
Jens Axboe500f9fb2019-08-19 12:15:59 -06002631 mutex_lock(&ctx->uring_lock);
Pavel Begunkove9979b32021-04-13 02:58:45 +01002632
Pavel Begunkov8f487ef2021-07-08 13:37:06 +01002633 /* some requests don't go through iopoll_list */
2634 if (tail != ctx->cached_cq_tail ||
2635 list_empty(&ctx->iopoll_list))
Pavel Begunkove9979b32021-04-13 02:58:45 +01002636 break;
Jens Axboe500f9fb2019-08-19 12:15:59 -06002637 }
Pavel Begunkova8576af2021-08-15 10:40:21 +01002638 ret = io_do_iopoll(ctx, &nr_events, min);
Pavel Begunkovf39c8a52021-04-13 02:58:46 +01002639 } while (!ret && nr_events < min && !need_resched());
2640out:
Jens Axboe500f9fb2019-08-19 12:15:59 -06002641 mutex_unlock(&ctx->uring_lock);
Jens Axboedef596e2019-01-09 08:59:42 -07002642 return ret;
2643}
2644
Jens Axboe491381ce2019-10-17 09:20:46 -06002645static void kiocb_end_write(struct io_kiocb *req)
Jens Axboe2b188cc2019-01-07 10:46:33 -07002646{
Jens Axboe491381ce2019-10-17 09:20:46 -06002647 /*
2648 * Tell lockdep we inherited freeze protection from submission
2649 * thread.
2650 */
2651 if (req->flags & REQ_F_ISREG) {
Pavel Begunkov1c986792021-03-22 01:58:31 +00002652 struct super_block *sb = file_inode(req->file)->i_sb;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002653
Pavel Begunkov1c986792021-03-22 01:58:31 +00002654 __sb_writers_acquired(sb, SB_FREEZE_WRITE);
2655 sb_end_write(sb);
Jens Axboe2b188cc2019-01-07 10:46:33 -07002656 }
2657}
2658
Jens Axboeb63534c2020-06-04 11:28:00 -06002659#ifdef CONFIG_BLOCK
Pavel Begunkovdc2a6e92021-01-19 13:32:35 +00002660static bool io_resubmit_prep(struct io_kiocb *req)
Jens Axboeb63534c2020-06-04 11:28:00 -06002661{
Pavel Begunkovab454432021-03-22 01:58:33 +00002662 struct io_async_rw *rw = req->async_data;
Jens Axboeb63534c2020-06-04 11:28:00 -06002663
Pavel Begunkovab454432021-03-22 01:58:33 +00002664 if (!rw)
2665 return !io_req_prep_async(req);
Jens Axboecd658692021-09-10 11:19:14 -06002666 iov_iter_restore(&rw->iter, &rw->iter_state);
Pavel Begunkovab454432021-03-22 01:58:33 +00002667 return true;
Jens Axboeb63534c2020-06-04 11:28:00 -06002668}
Jens Axboeb63534c2020-06-04 11:28:00 -06002669
Jens Axboe3e6a0d32021-03-01 13:56:00 -07002670static bool io_rw_should_reissue(struct io_kiocb *req)
Jens Axboeb63534c2020-06-04 11:28:00 -06002671{
Jens Axboe355afae2020-09-02 09:30:31 -06002672 umode_t mode = file_inode(req->file)->i_mode;
Jens Axboe3e6a0d32021-03-01 13:56:00 -07002673 struct io_ring_ctx *ctx = req->ctx;
Jens Axboeb63534c2020-06-04 11:28:00 -06002674
Jens Axboe355afae2020-09-02 09:30:31 -06002675 if (!S_ISBLK(mode) && !S_ISREG(mode))
2676 return false;
Jens Axboe3e6a0d32021-03-01 13:56:00 -07002677 if ((req->flags & REQ_F_NOWAIT) || (io_wq_current_is_worker() &&
2678 !(ctx->flags & IORING_SETUP_IOPOLL)))
Jens Axboeb63534c2020-06-04 11:28:00 -06002679 return false;
Jens Axboe7c977a52021-02-23 19:17:35 -07002680 /*
2681 * If ref is dying, we might be running poll reap from the exit work.
2682 * Don't attempt to reissue from that path, just let it fail with
2683 * -EAGAIN.
2684 */
Jens Axboe3e6a0d32021-03-01 13:56:00 -07002685 if (percpu_ref_is_dying(&ctx->refs))
2686 return false;
Jens Axboeef046882021-07-27 10:50:31 -06002687 /*
2688 * Play it safe and assume not safe to re-import and reissue if we're
2689 * not in the original thread group (or in task context).
2690 */
2691 if (!same_thread_group(req->task, current) || !in_task())
2692 return false;
Jens Axboe3e6a0d32021-03-01 13:56:00 -07002693 return true;
2694}
Jens Axboee82ad482021-04-02 19:45:34 -06002695#else
Jens Axboea1ff1e32021-04-12 06:40:02 -06002696static bool io_resubmit_prep(struct io_kiocb *req)
2697{
2698 return false;
2699}
Jens Axboee82ad482021-04-02 19:45:34 -06002700static bool io_rw_should_reissue(struct io_kiocb *req)
2701{
2702 return false;
2703}
Jens Axboe3e6a0d32021-03-01 13:56:00 -07002704#endif
2705
Jens Axboe8ef12ef2021-08-10 15:15:25 -06002706static bool __io_complete_rw_common(struct io_kiocb *req, long res)
Jens Axboea1d7c392020-06-22 11:09:46 -06002707{
Jens Axboedf1ec532022-03-20 13:08:38 -06002708 if (req->rw.kiocb.ki_flags & IOCB_WRITE) {
Pavel Begunkovb65c1282021-03-22 01:45:59 +00002709 kiocb_end_write(req);
Jens Axboedf1ec532022-03-20 13:08:38 -06002710 fsnotify_modify(req->file);
2711 } else {
2712 fsnotify_access(req->file);
2713 }
Pavel Begunkov9532b992021-03-22 01:58:34 +00002714 if (res != req->result) {
2715 if ((res == -EAGAIN || res == -EOPNOTSUPP) &&
2716 io_rw_should_reissue(req)) {
2717 req->flags |= REQ_F_REISSUE;
Jens Axboe8ef12ef2021-08-10 15:15:25 -06002718 return true;
Pavel Begunkov9532b992021-03-22 01:58:34 +00002719 }
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01002720 req_set_fail(req);
Jens Axboe8ef12ef2021-08-10 15:15:25 -06002721 req->result = res;
Pavel Begunkov9532b992021-03-22 01:58:34 +00002722 }
Jens Axboe8ef12ef2021-08-10 15:15:25 -06002723 return false;
2724}
2725
Harshit Mogalapallie326ee02023-01-10 08:46:47 -08002726static inline int io_fixup_rw_res(struct io_kiocb *req, long res)
Pavel Begunkove8574572022-10-16 22:42:56 +01002727{
2728 struct io_async_rw *io = req->async_data;
2729
2730 /* add previously done IO, if any */
2731 if (io && io->bytes_done > 0) {
2732 if (res < 0)
2733 res = io->bytes_done;
2734 else
2735 res += io->bytes_done;
2736 }
2737 return res;
2738}
2739
Pavel Begunkovf237c302021-08-18 12:42:46 +01002740static void io_req_task_complete(struct io_kiocb *req, bool *locked)
Jens Axboe8ef12ef2021-08-10 15:15:25 -06002741{
Pavel Begunkov126180b2021-08-18 12:42:47 +01002742 unsigned int cflags = io_put_rw_kbuf(req);
Pavel Begunkov5c0ea4c2022-08-29 14:30:12 +01002743 int res = req->result;
Pavel Begunkov126180b2021-08-18 12:42:47 +01002744
2745 if (*locked) {
2746 struct io_ring_ctx *ctx = req->ctx;
2747 struct io_submit_state *state = &ctx->submit_state;
2748
2749 io_req_complete_state(req, res, cflags);
2750 state->compl_reqs[state->compl_nr++] = req;
2751 if (state->compl_nr == ARRAY_SIZE(state->compl_reqs))
2752 io_submit_flush_completions(ctx);
2753 } else {
2754 io_req_complete_post(req, res, cflags);
2755 }
Jens Axboe8ef12ef2021-08-10 15:15:25 -06002756}
2757
2758static void __io_complete_rw(struct io_kiocb *req, long res, long res2,
2759 unsigned int issue_flags)
2760{
2761 if (__io_complete_rw_common(req, res))
2762 return;
Pavel Begunkove8574572022-10-16 22:42:56 +01002763 __io_req_complete(req, issue_flags, io_fixup_rw_res(req, res), io_put_rw_kbuf(req));
Jens Axboeba816ad2019-09-28 11:36:45 -06002764}
2765
2766static void io_complete_rw(struct kiocb *kiocb, long res, long res2)
2767{
Jens Axboe9adbd452019-12-20 08:45:55 -07002768 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
Jens Axboeba816ad2019-09-28 11:36:45 -06002769
Jens Axboe8ef12ef2021-08-10 15:15:25 -06002770 if (__io_complete_rw_common(req, res))
2771 return;
Pavel Begunkove8574572022-10-16 22:42:56 +01002772 req->result = io_fixup_rw_res(req, res);
Jens Axboe8ef12ef2021-08-10 15:15:25 -06002773 req->io_task_work.func = io_req_task_complete;
2774 io_req_task_work_add(req);
Jens Axboe2b188cc2019-01-07 10:46:33 -07002775}
2776
Jens Axboedef596e2019-01-09 08:59:42 -07002777static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2)
2778{
Jens Axboe9adbd452019-12-20 08:45:55 -07002779 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
Jens Axboedef596e2019-01-09 08:59:42 -07002780
Jens Axboe491381ce2019-10-17 09:20:46 -06002781 if (kiocb->ki_flags & IOCB_WRITE)
2782 kiocb_end_write(req);
Pavel Begunkov9532b992021-03-22 01:58:34 +00002783 if (unlikely(res != req->result)) {
Pavel Begunkovb66ceaf2021-09-15 11:00:05 +01002784 if (res == -EAGAIN && io_rw_should_reissue(req)) {
2785 req->flags |= REQ_F_REISSUE;
2786 return;
Pavel Begunkov9532b992021-03-22 01:58:34 +00002787 }
Pavel Begunkov8c130822021-03-22 01:58:32 +00002788 }
Xiaoguang Wangbbde0172020-06-16 02:06:38 +08002789
2790 WRITE_ONCE(req->result, res);
Jens Axboeb9b0e0d2021-02-23 08:18:36 -07002791 /* order with io_iopoll_complete() checking ->result */
Pavel Begunkovcd664b02020-06-25 12:37:10 +03002792 smp_wmb();
2793 WRITE_ONCE(req->iopoll_completed, 1);
Jens Axboedef596e2019-01-09 08:59:42 -07002794}
2795
2796/*
2797 * After the iocb has been issued, it's safe to be found on the poll list.
2798 * Adding the kiocb to the list AFTER submission ensures that we don't
Pavel Begunkovf39c8a52021-04-13 02:58:46 +01002799 * find it from a io_do_iopoll() thread before the issuer is done
Jens Axboedef596e2019-01-09 08:59:42 -07002800 * accessing the kiocb cookie.
2801 */
Pavel Begunkovcb3d8972021-06-14 02:36:14 +01002802static void io_iopoll_req_issued(struct io_kiocb *req)
Jens Axboedef596e2019-01-09 08:59:42 -07002803{
2804 struct io_ring_ctx *ctx = req->ctx;
Pavel Begunkovcb3d8972021-06-14 02:36:14 +01002805 const bool in_async = io_wq_current_is_worker();
2806
2807 /* workqueue context doesn't hold uring_lock, grab it now */
2808 if (unlikely(in_async))
2809 mutex_lock(&ctx->uring_lock);
Jens Axboedef596e2019-01-09 08:59:42 -07002810
2811 /*
2812 * Track whether we have multiple files in our lists. This will impact
2813 * how we do polling eventually, not spinning if we're on potentially
2814 * different devices.
2815 */
Pavel Begunkov540e32a2020-07-13 23:37:09 +03002816 if (list_empty(&ctx->iopoll_list)) {
Hao Xu915b3dd2021-06-28 05:37:30 +08002817 ctx->poll_multi_queue = false;
2818 } else if (!ctx->poll_multi_queue) {
Jens Axboedef596e2019-01-09 08:59:42 -07002819 struct io_kiocb *list_req;
Hao Xu915b3dd2021-06-28 05:37:30 +08002820 unsigned int queue_num0, queue_num1;
Jens Axboedef596e2019-01-09 08:59:42 -07002821
Pavel Begunkov540e32a2020-07-13 23:37:09 +03002822 list_req = list_first_entry(&ctx->iopoll_list, struct io_kiocb,
Pavel Begunkovd21ffe72020-07-13 23:37:10 +03002823 inflight_entry);
Hao Xu915b3dd2021-06-28 05:37:30 +08002824
2825 if (list_req->file != req->file) {
2826 ctx->poll_multi_queue = true;
2827 } else {
2828 queue_num0 = blk_qc_t_to_queue_num(list_req->rw.kiocb.ki_cookie);
2829 queue_num1 = blk_qc_t_to_queue_num(req->rw.kiocb.ki_cookie);
2830 if (queue_num0 != queue_num1)
2831 ctx->poll_multi_queue = true;
2832 }
Jens Axboedef596e2019-01-09 08:59:42 -07002833 }
2834
2835 /*
2836 * For fast devices, IO may have already completed. If it has, add
2837 * it to the front so we find it first.
2838 */
Xiaoguang Wang65a65432020-06-11 23:39:36 +08002839 if (READ_ONCE(req->iopoll_completed))
Pavel Begunkovd21ffe72020-07-13 23:37:10 +03002840 list_add(&req->inflight_entry, &ctx->iopoll_list);
Jens Axboedef596e2019-01-09 08:59:42 -07002841 else
Pavel Begunkovd21ffe72020-07-13 23:37:10 +03002842 list_add_tail(&req->inflight_entry, &ctx->iopoll_list);
Xiaoguang Wangbdcd3ea2020-02-25 22:12:08 +08002843
Pavel Begunkovcb3d8972021-06-14 02:36:14 +01002844 if (unlikely(in_async)) {
2845 /*
2846 * If IORING_SETUP_SQPOLL is enabled, sqes are either handle
2847 * in sq thread task context or in io worker task context. If
2848 * current task context is sq thread, we don't need to check
2849 * whether should wake up sq thread.
2850 */
2851 if ((ctx->flags & IORING_SETUP_SQPOLL) &&
2852 wq_has_sleeper(&ctx->sq_data->wait))
2853 wake_up(&ctx->sq_data->wait);
2854
2855 mutex_unlock(&ctx->uring_lock);
2856 }
Jens Axboedef596e2019-01-09 08:59:42 -07002857}
2858
Jens Axboe4503b762020-06-01 10:00:27 -06002859static bool io_bdev_nowait(struct block_device *bdev)
2860{
Jeffle Xu9ba0d0c2020-10-19 16:59:42 +08002861 return !bdev || blk_queue_nowait(bdev_get_queue(bdev));
Jens Axboe4503b762020-06-01 10:00:27 -06002862}
2863
Jens Axboe2b188cc2019-01-07 10:46:33 -07002864/*
2865 * If we tracked the file through the SCM inflight mechanism, we could support
2866 * any file. For now, just ensure that anything potentially problematic is done
2867 * inline.
2868 */
Pavel Begunkovb191e2d2021-08-09 13:04:03 +01002869static bool __io_file_supports_nowait(struct file *file, int rw)
Jens Axboe2b188cc2019-01-07 10:46:33 -07002870{
2871 umode_t mode = file_inode(file)->i_mode;
2872
Jens Axboe4503b762020-06-01 10:00:27 -06002873 if (S_ISBLK(mode)) {
Christoph Hellwig4e7b5672020-11-23 13:38:40 +01002874 if (IS_ENABLED(CONFIG_BLOCK) &&
2875 io_bdev_nowait(I_BDEV(file->f_mapping->host)))
Jens Axboe4503b762020-06-01 10:00:27 -06002876 return true;
2877 return false;
2878 }
Pavel Begunkov976517f2021-06-09 12:07:25 +01002879 if (S_ISSOCK(mode))
Jens Axboe2b188cc2019-01-07 10:46:33 -07002880 return true;
Jens Axboe4503b762020-06-01 10:00:27 -06002881 if (S_ISREG(mode)) {
Christoph Hellwig4e7b5672020-11-23 13:38:40 +01002882 if (IS_ENABLED(CONFIG_BLOCK) &&
2883 io_bdev_nowait(file->f_inode->i_sb->s_bdev) &&
Jens Axboe4503b762020-06-01 10:00:27 -06002884 file->f_op != &io_uring_fops)
2885 return true;
2886 return false;
2887 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07002888
Jens Axboec5b85622020-06-09 19:23:05 -06002889 /* any ->read/write should understand O_NONBLOCK */
2890 if (file->f_flags & O_NONBLOCK)
2891 return true;
2892
Jens Axboeaf197f52020-04-28 13:15:06 -06002893 if (!(file->f_mode & FMODE_NOWAIT))
2894 return false;
2895
2896 if (rw == READ)
2897 return file->f_op->read_iter != NULL;
2898
2899 return file->f_op->write_iter != NULL;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002900}
2901
Pavel Begunkovb191e2d2021-08-09 13:04:03 +01002902static bool io_file_supports_nowait(struct io_kiocb *req, int rw)
Jens Axboe7b29f922021-03-12 08:30:14 -07002903{
Pavel Begunkovb191e2d2021-08-09 13:04:03 +01002904 if (rw == READ && (req->flags & REQ_F_NOWAIT_READ))
Jens Axboe7b29f922021-03-12 08:30:14 -07002905 return true;
Pavel Begunkovb191e2d2021-08-09 13:04:03 +01002906 else if (rw == WRITE && (req->flags & REQ_F_NOWAIT_WRITE))
Jens Axboe7b29f922021-03-12 08:30:14 -07002907 return true;
2908
Pavel Begunkovb191e2d2021-08-09 13:04:03 +01002909 return __io_file_supports_nowait(req->file, rw);
Jens Axboe7b29f922021-03-12 08:30:14 -07002910}
2911
Jens Axboe5d329e12021-09-14 11:08:37 -06002912static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,
2913 int rw)
Jens Axboe2b188cc2019-01-07 10:46:33 -07002914{
Jens Axboedef596e2019-01-09 08:59:42 -07002915 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe9adbd452019-12-20 08:45:55 -07002916 struct kiocb *kiocb = &req->rw.kiocb;
Pavel Begunkov75c668c2021-02-04 13:52:05 +00002917 struct file *file = req->file;
Jens Axboe09bb8392019-03-13 12:39:28 -06002918 unsigned ioprio;
2919 int ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002920
Pavel Begunkovc97d8a02021-08-09 13:04:04 +01002921 if (!io_req_ffs_set(req) && S_ISREG(file_inode(file)->i_mode))
Jens Axboe491381ce2019-10-17 09:20:46 -06002922 req->flags |= REQ_F_ISREG;
2923
Jens Axboe2b188cc2019-01-07 10:46:33 -07002924 kiocb->ki_pos = READ_ONCE(sqe->off);
Jens Axboe20fb0dc2021-12-22 20:26:56 -07002925 if (kiocb->ki_pos == -1) {
2926 if (!(file->f_mode & FMODE_STREAM)) {
2927 req->flags |= REQ_F_CUR_POS;
2928 kiocb->ki_pos = file->f_pos;
2929 } else {
2930 kiocb->ki_pos = 0;
2931 }
Jens Axboeba042912019-12-25 16:33:42 -07002932 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07002933 kiocb->ki_hint = ki_hint_validate(file_write_hint(kiocb->ki_filp));
Pavel Begunkov3e577dc2020-02-01 03:58:42 +03002934 kiocb->ki_flags = iocb_flags(kiocb->ki_filp);
2935 ret = kiocb_set_rw_flags(kiocb, READ_ONCE(sqe->rw_flags));
2936 if (unlikely(ret))
2937 return ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002938
Jens Axboe5d329e12021-09-14 11:08:37 -06002939 /*
2940 * If the file is marked O_NONBLOCK, still allow retry for it if it
2941 * supports async. Otherwise it's impossible to use O_NONBLOCK files
2942 * reliably. If not, or it IOCB_NOWAIT is set, don't retry.
2943 */
2944 if ((kiocb->ki_flags & IOCB_NOWAIT) ||
2945 ((file->f_flags & O_NONBLOCK) && !io_file_supports_nowait(req, rw)))
Pavel Begunkov75c668c2021-02-04 13:52:05 +00002946 req->flags |= REQ_F_NOWAIT;
2947
Jens Axboe2b188cc2019-01-07 10:46:33 -07002948 ioprio = READ_ONCE(sqe->ioprio);
2949 if (ioprio) {
2950 ret = ioprio_check_cap(ioprio);
2951 if (ret)
Jens Axboe09bb8392019-03-13 12:39:28 -06002952 return ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002953
2954 kiocb->ki_ioprio = ioprio;
2955 } else
2956 kiocb->ki_ioprio = get_current_ioprio();
2957
Jens Axboedef596e2019-01-09 08:59:42 -07002958 if (ctx->flags & IORING_SETUP_IOPOLL) {
Jens Axboedef596e2019-01-09 08:59:42 -07002959 if (!(kiocb->ki_flags & IOCB_DIRECT) ||
2960 !kiocb->ki_filp->f_op->iopoll)
Jens Axboe09bb8392019-03-13 12:39:28 -06002961 return -EOPNOTSUPP;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002962
Jens Axboe394918e2021-03-08 11:40:23 -07002963 kiocb->ki_flags |= IOCB_HIPRI | IOCB_ALLOC_CACHE;
Jens Axboedef596e2019-01-09 08:59:42 -07002964 kiocb->ki_complete = io_complete_rw_iopoll;
Xiaoguang Wang65a65432020-06-11 23:39:36 +08002965 req->iopoll_completed = 0;
Jens Axboedef596e2019-01-09 08:59:42 -07002966 } else {
Jens Axboe09bb8392019-03-13 12:39:28 -06002967 if (kiocb->ki_flags & IOCB_HIPRI)
2968 return -EINVAL;
Jens Axboedef596e2019-01-09 08:59:42 -07002969 kiocb->ki_complete = io_complete_rw;
2970 }
Jens Axboe9adbd452019-12-20 08:45:55 -07002971
Pavel Begunkovea512d52022-06-09 08:34:35 +01002972 /* used for fixed read/write too - just read unconditionally */
2973 req->buf_index = READ_ONCE(sqe->buf_index);
2974 req->imu = NULL;
2975
Pavel Begunkoveae071c2021-04-25 14:32:24 +01002976 if (req->opcode == IORING_OP_READ_FIXED ||
2977 req->opcode == IORING_OP_WRITE_FIXED) {
Pavel Begunkovea512d52022-06-09 08:34:35 +01002978 struct io_ring_ctx *ctx = req->ctx;
2979 u16 index;
2980
2981 if (unlikely(req->buf_index >= ctx->nr_user_bufs))
2982 return -EFAULT;
2983 index = array_index_nospec(req->buf_index, ctx->nr_user_bufs);
2984 req->imu = ctx->user_bufs[index];
Pavel Begunkoveae071c2021-04-25 14:32:24 +01002985 io_req_set_rsrc_node(req);
2986 }
2987
Jens Axboe3529d8c2019-12-19 18:24:38 -07002988 req->rw.addr = READ_ONCE(sqe->addr);
2989 req->rw.len = READ_ONCE(sqe->len);
Jens Axboe2b188cc2019-01-07 10:46:33 -07002990 return 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002991}
2992
2993static inline void io_rw_done(struct kiocb *kiocb, ssize_t ret)
2994{
2995 switch (ret) {
2996 case -EIOCBQUEUED:
2997 break;
2998 case -ERESTARTSYS:
2999 case -ERESTARTNOINTR:
3000 case -ERESTARTNOHAND:
3001 case -ERESTART_RESTARTBLOCK:
3002 /*
3003 * We can't just restart the syscall, since previously
3004 * submitted sqes may already be in progress. Just fail this
3005 * IO with EINTR.
3006 */
3007 ret = -EINTR;
Gustavo A. R. Silvadf561f662020-08-23 17:36:59 -05003008 fallthrough;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003009 default:
3010 kiocb->ki_complete(kiocb, ret, 0);
3011 }
3012}
3013
Jens Axboea1d7c392020-06-22 11:09:46 -06003014static void kiocb_done(struct kiocb *kiocb, ssize_t ret,
Pavel Begunkov889fca72021-02-10 00:03:09 +00003015 unsigned int issue_flags)
Jens Axboeba816ad2019-09-28 11:36:45 -06003016{
Jens Axboeba042912019-12-25 16:33:42 -07003017 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
Jens Axboe227c0c92020-08-13 11:51:40 -06003018
Jens Axboeba042912019-12-25 16:33:42 -07003019 if (req->flags & REQ_F_CUR_POS)
3020 req->file->f_pos = kiocb->ki_pos;
Pavel Begunkovb66ceaf2021-09-15 11:00:05 +01003021 if (ret >= 0 && (kiocb->ki_complete == io_complete_rw))
Pavel Begunkov889fca72021-02-10 00:03:09 +00003022 __io_complete_rw(req, ret, 0, issue_flags);
Jens Axboeba816ad2019-09-28 11:36:45 -06003023 else
3024 io_rw_done(kiocb, ret);
Pavel Begunkov97284632021-04-08 19:28:03 +01003025
Pavel Begunkovb66ceaf2021-09-15 11:00:05 +01003026 if (req->flags & REQ_F_REISSUE) {
Pavel Begunkov97284632021-04-08 19:28:03 +01003027 req->flags &= ~REQ_F_REISSUE;
Jens Axboea7be7c22021-04-15 11:31:14 -06003028 if (io_resubmit_prep(req)) {
Jens Axboe773af692021-07-27 10:25:55 -06003029 io_req_task_queue_reissue(req);
Pavel Begunkov8c130822021-03-22 01:58:32 +00003030 } else {
Pavel Begunkovb66ceaf2021-09-15 11:00:05 +01003031 unsigned int cflags = io_put_rw_kbuf(req);
3032 struct io_ring_ctx *ctx = req->ctx;
3033
Pavel Begunkove8574572022-10-16 22:42:56 +01003034 ret = io_fixup_rw_res(req, ret);
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01003035 req_set_fail(req);
Hao Xu14cfbb72021-10-14 22:04:00 +08003036 if (!(issue_flags & IO_URING_F_NONBLOCK)) {
Pavel Begunkovb66ceaf2021-09-15 11:00:05 +01003037 mutex_lock(&ctx->uring_lock);
3038 __io_req_complete(req, issue_flags, ret, cflags);
3039 mutex_unlock(&ctx->uring_lock);
3040 } else {
3041 __io_req_complete(req, issue_flags, ret, cflags);
3042 }
Pavel Begunkov97284632021-04-08 19:28:03 +01003043 }
3044 }
Jens Axboeba816ad2019-09-28 11:36:45 -06003045}
3046
Pavel Begunkoveae071c2021-04-25 14:32:24 +01003047static int __io_import_fixed(struct io_kiocb *req, int rw, struct iov_iter *iter,
3048 struct io_mapped_ubuf *imu)
Jens Axboeedafcce2019-01-09 09:16:05 -07003049{
Jens Axboe9adbd452019-12-20 08:45:55 -07003050 size_t len = req->rw.len;
Pavel Begunkov75769e32021-04-01 15:43:54 +01003051 u64 buf_end, buf_addr = req->rw.addr;
Jens Axboeedafcce2019-01-09 09:16:05 -07003052 size_t offset;
Jens Axboeedafcce2019-01-09 09:16:05 -07003053
Pavel Begunkov75769e32021-04-01 15:43:54 +01003054 if (unlikely(check_add_overflow(buf_addr, (u64)len, &buf_end)))
Jens Axboeedafcce2019-01-09 09:16:05 -07003055 return -EFAULT;
3056 /* not inside the mapped region */
Pavel Begunkov4751f532021-04-01 15:43:55 +01003057 if (unlikely(buf_addr < imu->ubuf || buf_end > imu->ubuf_end))
Jens Axboeedafcce2019-01-09 09:16:05 -07003058 return -EFAULT;
3059
3060 /*
3061 * May not be a start of buffer, set size appropriately
3062 * and advance us to the beginning.
3063 */
3064 offset = buf_addr - imu->ubuf;
3065 iov_iter_bvec(iter, rw, imu->bvec, imu->nr_bvecs, offset + len);
Jens Axboebd11b3a2019-07-20 08:37:31 -06003066
3067 if (offset) {
3068 /*
3069 * Don't use iov_iter_advance() here, as it's really slow for
3070 * using the latter parts of a big fixed buffer - it iterates
3071 * over each segment manually. We can cheat a bit here, because
3072 * we know that:
3073 *
3074 * 1) it's a BVEC iter, we set it up
3075 * 2) all bvecs are PAGE_SIZE in size, except potentially the
3076 * first and last bvec
3077 *
3078 * So just find our index, and adjust the iterator afterwards.
3079 * If the offset is within the first bvec (or the whole first
3080 * bvec, just use iov_iter_advance(). This makes it easier
3081 * since we can just skip the first segment, which may not
3082 * be PAGE_SIZE aligned.
3083 */
3084 const struct bio_vec *bvec = imu->bvec;
3085
3086 if (offset <= bvec->bv_len) {
3087 iov_iter_advance(iter, offset);
3088 } else {
3089 unsigned long seg_skip;
3090
3091 /* skip first vec */
3092 offset -= bvec->bv_len;
3093 seg_skip = 1 + (offset >> PAGE_SHIFT);
3094
3095 iter->bvec = bvec + seg_skip;
3096 iter->nr_segs -= seg_skip;
Aleix Roca Nonell99c79f62019-08-15 14:03:22 +02003097 iter->count -= bvec->bv_len + offset;
Jens Axboebd11b3a2019-07-20 08:37:31 -06003098 iter->iov_offset = offset & ~PAGE_MASK;
Jens Axboebd11b3a2019-07-20 08:37:31 -06003099 }
3100 }
3101
Pavel Begunkov847595d2021-02-04 13:52:06 +00003102 return 0;
Jens Axboeedafcce2019-01-09 09:16:05 -07003103}
3104
Pavel Begunkoveae071c2021-04-25 14:32:24 +01003105static int io_import_fixed(struct io_kiocb *req, int rw, struct iov_iter *iter)
3106{
Pavel Begunkovea512d52022-06-09 08:34:35 +01003107 if (WARN_ON_ONCE(!req->imu))
3108 return -EFAULT;
3109 return __io_import_fixed(req, rw, iter, req->imu);
Pavel Begunkoveae071c2021-04-25 14:32:24 +01003110}
3111
Jens Axboebcda7ba2020-02-23 16:42:51 -07003112static void io_ring_submit_unlock(struct io_ring_ctx *ctx, bool needs_lock)
3113{
3114 if (needs_lock)
3115 mutex_unlock(&ctx->uring_lock);
3116}
3117
3118static void io_ring_submit_lock(struct io_ring_ctx *ctx, bool needs_lock)
3119{
3120 /*
3121 * "Normal" inline submissions always hold the uring_lock, since we
3122 * grab it from the system call. Same is true for the SQPOLL offload.
3123 * The only exception is when we've detached the request and issue it
3124 * from an async worker thread, grab the lock for that case.
3125 */
3126 if (needs_lock)
3127 mutex_lock(&ctx->uring_lock);
3128}
3129
3130static struct io_buffer *io_buffer_select(struct io_kiocb *req, size_t *len,
3131 int bgid, struct io_buffer *kbuf,
3132 bool needs_lock)
3133{
3134 struct io_buffer *head;
3135
3136 if (req->flags & REQ_F_BUFFER_SELECTED)
3137 return kbuf;
3138
3139 io_ring_submit_lock(req->ctx, needs_lock);
3140
3141 lockdep_assert_held(&req->ctx->uring_lock);
3142
Jens Axboe9e15c3a2021-03-13 12:29:43 -07003143 head = xa_load(&req->ctx->io_buffers, bgid);
Jens Axboebcda7ba2020-02-23 16:42:51 -07003144 if (head) {
3145 if (!list_empty(&head->list)) {
3146 kbuf = list_last_entry(&head->list, struct io_buffer,
3147 list);
3148 list_del(&kbuf->list);
3149 } else {
3150 kbuf = head;
Jens Axboe9e15c3a2021-03-13 12:29:43 -07003151 xa_erase(&req->ctx->io_buffers, bgid);
Jens Axboebcda7ba2020-02-23 16:42:51 -07003152 }
3153 if (*len > kbuf->len)
3154 *len = kbuf->len;
3155 } else {
3156 kbuf = ERR_PTR(-ENOBUFS);
3157 }
3158
3159 io_ring_submit_unlock(req->ctx, needs_lock);
3160
3161 return kbuf;
3162}
3163
Jens Axboe4d954c22020-02-27 07:31:19 -07003164static void __user *io_rw_buffer_select(struct io_kiocb *req, size_t *len,
3165 bool needs_lock)
3166{
3167 struct io_buffer *kbuf;
Bijan Mottahedeh4f4eeba2020-05-19 14:52:49 -07003168 u16 bgid;
Jens Axboe4d954c22020-02-27 07:31:19 -07003169
3170 kbuf = (struct io_buffer *) (unsigned long) req->rw.addr;
Bijan Mottahedeh4f4eeba2020-05-19 14:52:49 -07003171 bgid = req->buf_index;
Jens Axboe4d954c22020-02-27 07:31:19 -07003172 kbuf = io_buffer_select(req, len, bgid, kbuf, needs_lock);
3173 if (IS_ERR(kbuf))
3174 return kbuf;
3175 req->rw.addr = (u64) (unsigned long) kbuf;
3176 req->flags |= REQ_F_BUFFER_SELECTED;
3177 return u64_to_user_ptr(kbuf->addr);
3178}
3179
3180#ifdef CONFIG_COMPAT
3181static ssize_t io_compat_import(struct io_kiocb *req, struct iovec *iov,
3182 bool needs_lock)
3183{
3184 struct compat_iovec __user *uiov;
3185 compat_ssize_t clen;
3186 void __user *buf;
3187 ssize_t len;
3188
3189 uiov = u64_to_user_ptr(req->rw.addr);
3190 if (!access_ok(uiov, sizeof(*uiov)))
3191 return -EFAULT;
3192 if (__get_user(clen, &uiov->iov_len))
3193 return -EFAULT;
3194 if (clen < 0)
3195 return -EINVAL;
3196
3197 len = clen;
3198 buf = io_rw_buffer_select(req, &len, needs_lock);
3199 if (IS_ERR(buf))
3200 return PTR_ERR(buf);
3201 iov[0].iov_base = buf;
3202 iov[0].iov_len = (compat_size_t) len;
3203 return 0;
3204}
3205#endif
3206
3207static ssize_t __io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov,
3208 bool needs_lock)
3209{
3210 struct iovec __user *uiov = u64_to_user_ptr(req->rw.addr);
3211 void __user *buf;
3212 ssize_t len;
3213
3214 if (copy_from_user(iov, uiov, sizeof(*uiov)))
3215 return -EFAULT;
3216
3217 len = iov[0].iov_len;
3218 if (len < 0)
3219 return -EINVAL;
3220 buf = io_rw_buffer_select(req, &len, needs_lock);
3221 if (IS_ERR(buf))
3222 return PTR_ERR(buf);
3223 iov[0].iov_base = buf;
3224 iov[0].iov_len = len;
3225 return 0;
3226}
3227
3228static ssize_t io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov,
3229 bool needs_lock)
3230{
Jens Axboedddb3e22020-06-04 11:27:01 -06003231 if (req->flags & REQ_F_BUFFER_SELECTED) {
3232 struct io_buffer *kbuf;
3233
3234 kbuf = (struct io_buffer *) (unsigned long) req->rw.addr;
3235 iov[0].iov_base = u64_to_user_ptr(kbuf->addr);
3236 iov[0].iov_len = kbuf->len;
Jens Axboe4d954c22020-02-27 07:31:19 -07003237 return 0;
Jens Axboedddb3e22020-06-04 11:27:01 -06003238 }
Pavel Begunkovdd201662020-12-19 03:15:43 +00003239 if (req->rw.len != 1)
Jens Axboe4d954c22020-02-27 07:31:19 -07003240 return -EINVAL;
3241
3242#ifdef CONFIG_COMPAT
3243 if (req->ctx->compat)
3244 return io_compat_import(req, iov, needs_lock);
3245#endif
3246
3247 return __io_iov_buffer_select(req, iov, needs_lock);
3248}
3249
Pavel Begunkov847595d2021-02-04 13:52:06 +00003250static int io_import_iovec(int rw, struct io_kiocb *req, struct iovec **iovec,
3251 struct iov_iter *iter, bool needs_lock)
Jens Axboe2b188cc2019-01-07 10:46:33 -07003252{
Jens Axboe9adbd452019-12-20 08:45:55 -07003253 void __user *buf = u64_to_user_ptr(req->rw.addr);
3254 size_t sqe_len = req->rw.len;
Pavel Begunkov847595d2021-02-04 13:52:06 +00003255 u8 opcode = req->opcode;
Jens Axboe4d954c22020-02-27 07:31:19 -07003256 ssize_t ret;
Jens Axboeedafcce2019-01-09 09:16:05 -07003257
Pavel Begunkov7d009162019-11-25 23:14:40 +03003258 if (opcode == IORING_OP_READ_FIXED || opcode == IORING_OP_WRITE_FIXED) {
Jens Axboeedafcce2019-01-09 09:16:05 -07003259 *iovec = NULL;
Jens Axboe9adbd452019-12-20 08:45:55 -07003260 return io_import_fixed(req, rw, iter);
Jens Axboeedafcce2019-01-09 09:16:05 -07003261 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07003262
Jens Axboebcda7ba2020-02-23 16:42:51 -07003263 /* buffer index only valid with fixed read/write, or buffer select */
Bijan Mottahedeh4f4eeba2020-05-19 14:52:49 -07003264 if (req->buf_index && !(req->flags & REQ_F_BUFFER_SELECT))
Jens Axboe9adbd452019-12-20 08:45:55 -07003265 return -EINVAL;
3266
Jens Axboe3a6820f2019-12-22 15:19:35 -07003267 if (opcode == IORING_OP_READ || opcode == IORING_OP_WRITE) {
Jens Axboebcda7ba2020-02-23 16:42:51 -07003268 if (req->flags & REQ_F_BUFFER_SELECT) {
Jens Axboe4d954c22020-02-27 07:31:19 -07003269 buf = io_rw_buffer_select(req, &sqe_len, needs_lock);
Pavel Begunkov867a23e2020-08-20 11:34:39 +03003270 if (IS_ERR(buf))
Jens Axboe4d954c22020-02-27 07:31:19 -07003271 return PTR_ERR(buf);
Jens Axboe3f9d6442020-03-11 12:27:04 -06003272 req->rw.len = sqe_len;
Jens Axboebcda7ba2020-02-23 16:42:51 -07003273 }
3274
Jens Axboe3a6820f2019-12-22 15:19:35 -07003275 ret = import_single_range(rw, buf, sqe_len, *iovec, iter);
3276 *iovec = NULL;
David Laight10fc72e2020-11-07 13:16:25 +00003277 return ret;
Jens Axboe3a6820f2019-12-22 15:19:35 -07003278 }
3279
Jens Axboe4d954c22020-02-27 07:31:19 -07003280 if (req->flags & REQ_F_BUFFER_SELECT) {
3281 ret = io_iov_buffer_select(req, *iovec, needs_lock);
Pavel Begunkov847595d2021-02-04 13:52:06 +00003282 if (!ret)
3283 iov_iter_init(iter, rw, *iovec, 1, (*iovec)->iov_len);
Jens Axboe4d954c22020-02-27 07:31:19 -07003284 *iovec = NULL;
3285 return ret;
3286 }
3287
Christoph Hellwig89cd35c2020-09-25 06:51:41 +02003288 return __import_iovec(rw, buf, sqe_len, UIO_FASTIOV, iovec, iter,
3289 req->ctx->compat);
Jens Axboe2b188cc2019-01-07 10:46:33 -07003290}
3291
Jens Axboe0fef9482020-08-26 10:36:20 -06003292static inline loff_t *io_kiocb_ppos(struct kiocb *kiocb)
3293{
Pavel Begunkov5b09e372020-09-30 22:57:15 +03003294 return (kiocb->ki_filp->f_mode & FMODE_STREAM) ? NULL : &kiocb->ki_pos;
Jens Axboe0fef9482020-08-26 10:36:20 -06003295}
3296
Jens Axboe32960612019-09-23 11:05:34 -06003297/*
3298 * For files that don't have ->read_iter() and ->write_iter(), handle them
3299 * by looping over ->read() or ->write() manually.
3300 */
Jens Axboe4017eb92020-10-22 14:14:12 -06003301static ssize_t loop_rw_iter(int rw, struct io_kiocb *req, struct iov_iter *iter)
Jens Axboe32960612019-09-23 11:05:34 -06003302{
Jens Axboe4017eb92020-10-22 14:14:12 -06003303 struct kiocb *kiocb = &req->rw.kiocb;
3304 struct file *file = req->file;
Jens Axboe32960612019-09-23 11:05:34 -06003305 ssize_t ret = 0;
Dylan Yudakenb7958ca2022-02-22 02:55:01 -08003306 loff_t *ppos;
Jens Axboe32960612019-09-23 11:05:34 -06003307
3308 /*
3309 * Don't support polled IO through this interface, and we can't
3310 * support non-blocking either. For the latter, this just causes
3311 * the kiocb to be handled from an async context.
3312 */
3313 if (kiocb->ki_flags & IOCB_HIPRI)
3314 return -EOPNOTSUPP;
3315 if (kiocb->ki_flags & IOCB_NOWAIT)
3316 return -EAGAIN;
3317
Dylan Yudakenb7958ca2022-02-22 02:55:01 -08003318 ppos = io_kiocb_ppos(kiocb);
3319
Jens Axboe32960612019-09-23 11:05:34 -06003320 while (iov_iter_count(iter)) {
Pavel Begunkov311ae9e2019-11-24 11:58:24 +03003321 struct iovec iovec;
Jens Axboe32960612019-09-23 11:05:34 -06003322 ssize_t nr;
3323
Pavel Begunkov311ae9e2019-11-24 11:58:24 +03003324 if (!iov_iter_is_bvec(iter)) {
3325 iovec = iov_iter_iovec(iter);
3326 } else {
Jens Axboe4017eb92020-10-22 14:14:12 -06003327 iovec.iov_base = u64_to_user_ptr(req->rw.addr);
3328 iovec.iov_len = req->rw.len;
Pavel Begunkov311ae9e2019-11-24 11:58:24 +03003329 }
3330
Jens Axboe32960612019-09-23 11:05:34 -06003331 if (rw == READ) {
3332 nr = file->f_op->read(file, iovec.iov_base,
Dylan Yudakenb7958ca2022-02-22 02:55:01 -08003333 iovec.iov_len, ppos);
Jens Axboe32960612019-09-23 11:05:34 -06003334 } else {
3335 nr = file->f_op->write(file, iovec.iov_base,
Dylan Yudakenb7958ca2022-02-22 02:55:01 -08003336 iovec.iov_len, ppos);
Jens Axboe32960612019-09-23 11:05:34 -06003337 }
3338
3339 if (nr < 0) {
3340 if (!ret)
3341 ret = nr;
3342 break;
3343 }
Jens Axboe109dda42022-03-18 11:28:13 -06003344 ret += nr;
Jens Axboe16c8d2d2021-09-12 06:45:07 -06003345 if (!iov_iter_is_bvec(iter)) {
3346 iov_iter_advance(iter, nr);
3347 } else {
Jens Axboe16c8d2d2021-09-12 06:45:07 -06003348 req->rw.addr += nr;
Jens Axboe109dda42022-03-18 11:28:13 -06003349 req->rw.len -= nr;
3350 if (!req->rw.len)
3351 break;
Jens Axboe16c8d2d2021-09-12 06:45:07 -06003352 }
Jens Axboe32960612019-09-23 11:05:34 -06003353 if (nr != iovec.iov_len)
3354 break;
Jens Axboe32960612019-09-23 11:05:34 -06003355 }
3356
3357 return ret;
3358}
3359
Jens Axboeff6165b2020-08-13 09:47:43 -06003360static void io_req_map_rw(struct io_kiocb *req, const struct iovec *iovec,
3361 const struct iovec *fast_iov, struct iov_iter *iter)
Jens Axboef67676d2019-12-02 11:03:47 -07003362{
Jens Axboee8c2bc12020-08-15 18:44:09 -07003363 struct io_async_rw *rw = req->async_data;
Pavel Begunkovb64e3442020-07-13 22:59:18 +03003364
Jens Axboeff6165b2020-08-13 09:47:43 -06003365 memcpy(&rw->iter, iter, sizeof(*iter));
Pavel Begunkovafb87652020-09-06 00:45:46 +03003366 rw->free_iovec = iovec;
Jens Axboe227c0c92020-08-13 11:51:40 -06003367 rw->bytes_done = 0;
Jens Axboeff6165b2020-08-13 09:47:43 -06003368 /* can only be fixed buffers, no need to do anything */
Pavel Begunkov9c3a2052020-11-23 23:20:27 +00003369 if (iov_iter_is_bvec(iter))
Jens Axboeff6165b2020-08-13 09:47:43 -06003370 return;
Pavel Begunkovb64e3442020-07-13 22:59:18 +03003371 if (!iovec) {
Jens Axboeff6165b2020-08-13 09:47:43 -06003372 unsigned iov_off = 0;
3373
3374 rw->iter.iov = rw->fast_iov;
3375 if (iter->iov != fast_iov) {
3376 iov_off = iter->iov - fast_iov;
3377 rw->iter.iov += iov_off;
3378 }
3379 if (rw->fast_iov != fast_iov)
3380 memcpy(rw->fast_iov + iov_off, fast_iov + iov_off,
Xiaoguang Wang45097da2020-04-08 22:29:58 +08003381 sizeof(struct iovec) * iter->nr_segs);
Pavel Begunkov99bc4c32020-02-07 22:04:45 +03003382 } else {
3383 req->flags |= REQ_F_NEED_CLEANUP;
Jens Axboef67676d2019-12-02 11:03:47 -07003384 }
3385}
3386
Pavel Begunkov6cb78682021-02-28 22:35:17 +00003387static inline int io_alloc_async_data(struct io_kiocb *req)
Xiaoguang Wang3d9932a2020-03-27 15:36:52 +08003388{
Jens Axboee8c2bc12020-08-15 18:44:09 -07003389 WARN_ON_ONCE(!io_op_defs[req->opcode].async_size);
3390 req->async_data = kmalloc(io_op_defs[req->opcode].async_size, GFP_KERNEL);
3391 return req->async_data == NULL;
Xiaoguang Wang3d9932a2020-03-27 15:36:52 +08003392}
3393
Jens Axboeff6165b2020-08-13 09:47:43 -06003394static int io_setup_async_rw(struct io_kiocb *req, const struct iovec *iovec,
3395 const struct iovec *fast_iov,
Jens Axboe227c0c92020-08-13 11:51:40 -06003396 struct iov_iter *iter, bool force)
Jens Axboeb7bb4f72019-12-15 22:13:43 -07003397{
Pavel Begunkov26f05052021-02-28 22:35:18 +00003398 if (!force && !io_op_defs[req->opcode].needs_async_setup)
Jens Axboe74566df2020-01-13 19:23:24 -07003399 return 0;
Jens Axboee8c2bc12020-08-15 18:44:09 -07003400 if (!req->async_data) {
Jens Axboecd658692021-09-10 11:19:14 -06003401 struct io_async_rw *iorw;
3402
Pavel Begunkov6cb78682021-02-28 22:35:17 +00003403 if (io_alloc_async_data(req)) {
Pavel Begunkov6bf985d2021-02-04 13:52:01 +00003404 kfree(iovec);
Jens Axboe5d204bc2020-01-31 12:06:52 -07003405 return -ENOMEM;
Pavel Begunkov6bf985d2021-02-04 13:52:01 +00003406 }
Jens Axboeb7bb4f72019-12-15 22:13:43 -07003407
Jens Axboeff6165b2020-08-13 09:47:43 -06003408 io_req_map_rw(req, iovec, fast_iov, iter);
Jens Axboecd658692021-09-10 11:19:14 -06003409 iorw = req->async_data;
3410 /* we've copied and mapped the iter, ensure state is saved */
3411 iov_iter_save_state(&iorw->iter, &iorw->iter_state);
Jens Axboe5d204bc2020-01-31 12:06:52 -07003412 }
Jens Axboeb7bb4f72019-12-15 22:13:43 -07003413 return 0;
Jens Axboef67676d2019-12-02 11:03:47 -07003414}
3415
Pavel Begunkov73debe62020-09-30 22:57:54 +03003416static inline int io_rw_prep_async(struct io_kiocb *req, int rw)
Pavel Begunkovc3e330a2020-07-13 22:59:19 +03003417{
Jens Axboee8c2bc12020-08-15 18:44:09 -07003418 struct io_async_rw *iorw = req->async_data;
Pavel Begunkovf4bff102020-09-06 00:45:45 +03003419 struct iovec *iov = iorw->fast_iov;
Pavel Begunkov847595d2021-02-04 13:52:06 +00003420 int ret;
Pavel Begunkovc3e330a2020-07-13 22:59:19 +03003421
Pavel Begunkov2846c482020-11-07 13:16:27 +00003422 ret = io_import_iovec(rw, req, &iov, &iorw->iter, false);
Pavel Begunkovc3e330a2020-07-13 22:59:19 +03003423 if (unlikely(ret < 0))
3424 return ret;
3425
Pavel Begunkovab0b1962020-09-06 00:45:47 +03003426 iorw->bytes_done = 0;
3427 iorw->free_iovec = iov;
3428 if (iov)
3429 req->flags |= REQ_F_NEED_CLEANUP;
Jens Axboecd658692021-09-10 11:19:14 -06003430 iov_iter_save_state(&iorw->iter, &iorw->iter_state);
Pavel Begunkovc3e330a2020-07-13 22:59:19 +03003431 return 0;
3432}
3433
Pavel Begunkov73debe62020-09-30 22:57:54 +03003434static int io_read_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboef67676d2019-12-02 11:03:47 -07003435{
Jens Axboe3529d8c2019-12-19 18:24:38 -07003436 if (unlikely(!(req->file->f_mode & FMODE_READ)))
3437 return -EBADF;
Jens Axboe5d329e12021-09-14 11:08:37 -06003438 return io_prep_rw(req, sqe, READ);
Jens Axboef67676d2019-12-02 11:03:47 -07003439}
3440
Jens Axboec1dd91d2020-08-03 16:43:59 -06003441/*
3442 * This is our waitqueue callback handler, registered through lock_page_async()
3443 * when we initially tried to do the IO with the iocb armed our waitqueue.
3444 * This gets called when the page is unlocked, and we generally expect that to
3445 * happen when the page IO is completed and the page is now uptodate. This will
3446 * queue a task_work based retry of the operation, attempting to copy the data
3447 * again. If the latter fails because the page was NOT uptodate, then we will
3448 * do a thread based blocking retry of the operation. That's the unexpected
3449 * slow path.
3450 */
Jens Axboebcf5a062020-05-22 09:24:42 -06003451static int io_async_buf_func(struct wait_queue_entry *wait, unsigned mode,
3452 int sync, void *arg)
3453{
3454 struct wait_page_queue *wpq;
3455 struct io_kiocb *req = wait->private;
Jens Axboebcf5a062020-05-22 09:24:42 -06003456 struct wait_page_key *key = arg;
Jens Axboebcf5a062020-05-22 09:24:42 -06003457
3458 wpq = container_of(wait, struct wait_page_queue, wait);
3459
Linus Torvaldscdc8fcb2020-08-03 13:01:22 -07003460 if (!wake_page_match(wpq, key))
3461 return 0;
3462
Hao Xuc8d317a2020-09-29 20:00:45 +08003463 req->rw.kiocb.ki_flags &= ~IOCB_WAITQ;
Jens Axboebcf5a062020-05-22 09:24:42 -06003464 list_del_init(&wait->entry);
Pavel Begunkov921b9052021-02-12 03:23:53 +00003465 io_req_task_queue(req);
Jens Axboebcf5a062020-05-22 09:24:42 -06003466 return 1;
3467}
3468
Jens Axboec1dd91d2020-08-03 16:43:59 -06003469/*
3470 * This controls whether a given IO request should be armed for async page
3471 * based retry. If we return false here, the request is handed to the async
3472 * worker threads for retry. If we're doing buffered reads on a regular file,
3473 * we prepare a private wait_page_queue entry and retry the operation. This
3474 * will either succeed because the page is now uptodate and unlocked, or it
3475 * will register a callback when the page is unlocked at IO completion. Through
3476 * that callback, io_uring uses task_work to setup a retry of the operation.
3477 * That retry will attempt the buffered read again. The retry will generally
3478 * succeed, or in rare cases where it fails, we then fall back to using the
3479 * async worker threads for a blocking retry.
3480 */
Jens Axboe227c0c92020-08-13 11:51:40 -06003481static bool io_rw_should_retry(struct io_kiocb *req)
Jens Axboebcf5a062020-05-22 09:24:42 -06003482{
Jens Axboee8c2bc12020-08-15 18:44:09 -07003483 struct io_async_rw *rw = req->async_data;
3484 struct wait_page_queue *wait = &rw->wpq;
Jens Axboebcf5a062020-05-22 09:24:42 -06003485 struct kiocb *kiocb = &req->rw.kiocb;
Jens Axboebcf5a062020-05-22 09:24:42 -06003486
3487 /* never retry for NOWAIT, we just complete with -EAGAIN */
3488 if (req->flags & REQ_F_NOWAIT)
3489 return false;
3490
Jens Axboe227c0c92020-08-13 11:51:40 -06003491 /* Only for buffered IO */
Jens Axboe3b2a4432020-08-16 10:58:43 -07003492 if (kiocb->ki_flags & (IOCB_DIRECT | IOCB_HIPRI))
Jens Axboebcf5a062020-05-22 09:24:42 -06003493 return false;
Jens Axboe3b2a4432020-08-16 10:58:43 -07003494
Jens Axboebcf5a062020-05-22 09:24:42 -06003495 /*
3496 * just use poll if we can, and don't attempt if the fs doesn't
3497 * support callback based unlocks
3498 */
3499 if (file_can_poll(req->file) || !(req->file->f_mode & FMODE_BUF_RASYNC))
3500 return false;
3501
Jens Axboe3b2a4432020-08-16 10:58:43 -07003502 wait->wait.func = io_async_buf_func;
3503 wait->wait.private = req;
3504 wait->wait.flags = 0;
3505 INIT_LIST_HEAD(&wait->wait.entry);
3506 kiocb->ki_flags |= IOCB_WAITQ;
Hao Xuc8d317a2020-09-29 20:00:45 +08003507 kiocb->ki_flags &= ~IOCB_NOWAIT;
Jens Axboe3b2a4432020-08-16 10:58:43 -07003508 kiocb->ki_waitq = wait;
Jens Axboe3b2a4432020-08-16 10:58:43 -07003509 return true;
Jens Axboebcf5a062020-05-22 09:24:42 -06003510}
3511
Pavel Begunkovaeab9502021-06-14 02:36:24 +01003512static inline int io_iter_do_read(struct io_kiocb *req, struct iov_iter *iter)
Jens Axboebcf5a062020-05-22 09:24:42 -06003513{
3514 if (req->file->f_op->read_iter)
3515 return call_read_iter(req->file, &req->rw.kiocb, iter);
Guoyu Huang2dd21112020-08-05 03:53:50 -07003516 else if (req->file->f_op->read)
Jens Axboe4017eb92020-10-22 14:14:12 -06003517 return loop_rw_iter(READ, req, iter);
Guoyu Huang2dd21112020-08-05 03:53:50 -07003518 else
3519 return -EINVAL;
Jens Axboebcf5a062020-05-22 09:24:42 -06003520}
3521
Ming Lei7db30432021-08-21 23:07:51 +08003522static bool need_read_all(struct io_kiocb *req)
3523{
3524 return req->flags & REQ_F_ISREG ||
3525 S_ISBLK(file_inode(req->file)->i_mode);
3526}
3527
Pavel Begunkov889fca72021-02-10 00:03:09 +00003528static int io_read(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe2b188cc2019-01-07 10:46:33 -07003529{
3530 struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
Jens Axboe9adbd452019-12-20 08:45:55 -07003531 struct kiocb *kiocb = &req->rw.kiocb;
Jens Axboeff6165b2020-08-13 09:47:43 -06003532 struct iov_iter __iter, *iter = &__iter;
Jens Axboee8c2bc12020-08-15 18:44:09 -07003533 struct io_async_rw *rw = req->async_data;
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003534 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
Jens Axboecd658692021-09-10 11:19:14 -06003535 struct iov_iter_state __state, *state;
3536 ssize_t ret, ret2;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003537
Pavel Begunkov2846c482020-11-07 13:16:27 +00003538 if (rw) {
Jens Axboee8c2bc12020-08-15 18:44:09 -07003539 iter = &rw->iter;
Jens Axboecd658692021-09-10 11:19:14 -06003540 state = &rw->iter_state;
3541 /*
3542 * We come here from an earlier attempt, restore our state to
3543 * match in case it doesn't. It's cheap enough that we don't
3544 * need to make this conditional.
3545 */
3546 iov_iter_restore(iter, state);
Pavel Begunkov2846c482020-11-07 13:16:27 +00003547 iovec = NULL;
3548 } else {
3549 ret = io_import_iovec(READ, req, &iovec, iter, !force_nonblock);
3550 if (ret < 0)
3551 return ret;
Jens Axboecd658692021-09-10 11:19:14 -06003552 state = &__state;
3553 iov_iter_save_state(iter, state);
Pavel Begunkov2846c482020-11-07 13:16:27 +00003554 }
Jens Axboecd658692021-09-10 11:19:14 -06003555 req->result = iov_iter_count(iter);
Jens Axboe2b188cc2019-01-07 10:46:33 -07003556
Jens Axboefd6c2e42019-12-18 12:19:41 -07003557 /* Ensure we clear previously set non-block flag */
3558 if (!force_nonblock)
Jens Axboe29de5f62020-02-20 09:56:08 -07003559 kiocb->ki_flags &= ~IOCB_NOWAIT;
Pavel Begunkova88fc402020-09-30 22:57:53 +03003560 else
3561 kiocb->ki_flags |= IOCB_NOWAIT;
3562
Pavel Begunkov24c74672020-06-21 13:09:51 +03003563 /* If the file doesn't support async, just async punt */
Pavel Begunkovb191e2d2021-08-09 13:04:03 +01003564 if (force_nonblock && !io_file_supports_nowait(req, READ)) {
Pavel Begunkov6713e7a2021-02-04 13:51:59 +00003565 ret = io_setup_async_rw(req, iovec, inline_vecs, iter, true);
Pavel Begunkov6bf985d2021-02-04 13:52:01 +00003566 return ret ?: -EAGAIN;
Pavel Begunkov6713e7a2021-02-04 13:51:59 +00003567 }
Jens Axboe9e645e112019-05-10 16:07:28 -06003568
Jens Axboecd658692021-09-10 11:19:14 -06003569 ret = rw_verify_area(READ, req->file, io_kiocb_ppos(kiocb), req->result);
Pavel Begunkov5ea5dd42021-02-04 13:52:03 +00003570 if (unlikely(ret)) {
3571 kfree(iovec);
3572 return ret;
3573 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07003574
Jens Axboe227c0c92020-08-13 11:51:40 -06003575 ret = io_iter_do_read(req, iter);
Jens Axboe32960612019-09-23 11:05:34 -06003576
Jens Axboe230d50d2021-04-01 20:41:15 -06003577 if (ret == -EAGAIN || (req->flags & REQ_F_REISSUE)) {
Pavel Begunkov6ad7f232021-04-08 01:54:39 +01003578 req->flags &= ~REQ_F_REISSUE;
Jens Axboeeefdf302020-08-27 16:40:19 -06003579 /* IOPOLL retry should happen for io-wq threads */
3580 if (!force_nonblock && !(req->ctx->flags & IORING_SETUP_IOPOLL))
Jens Axboef91daf52020-08-15 15:58:42 -07003581 goto done;
Pavel Begunkov75c668c2021-02-04 13:52:05 +00003582 /* no retry on NONBLOCK nor RWF_NOWAIT */
3583 if (req->flags & REQ_F_NOWAIT)
Jens Axboe355afae2020-09-02 09:30:31 -06003584 goto done;
Jens Axboef38c7e32020-09-25 15:23:43 -06003585 ret = 0;
Jens Axboe230d50d2021-04-01 20:41:15 -06003586 } else if (ret == -EIOCBQUEUED) {
3587 goto out_free;
Jens Axboecd658692021-09-10 11:19:14 -06003588 } else if (ret <= 0 || ret == req->result || !force_nonblock ||
Ming Lei7db30432021-08-21 23:07:51 +08003589 (req->flags & REQ_F_NOWAIT) || !need_read_all(req)) {
Pavel Begunkov7335e3b2021-02-04 13:52:02 +00003590 /* read all, failed, already did sync or don't want to retry */
Jens Axboe00d23d52020-08-25 12:59:22 -06003591 goto done;
Jens Axboe227c0c92020-08-13 11:51:40 -06003592 }
3593
Jens Axboecd658692021-09-10 11:19:14 -06003594 /*
3595 * Don't depend on the iter state matching what was consumed, or being
3596 * untouched in case of error. Restore it and we'll advance it
3597 * manually if we need to.
3598 */
3599 iov_iter_restore(iter, state);
3600
Jens Axboe227c0c92020-08-13 11:51:40 -06003601 ret2 = io_setup_async_rw(req, iovec, inline_vecs, iter, true);
Pavel Begunkov6bf985d2021-02-04 13:52:01 +00003602 if (ret2)
3603 return ret2;
3604
Pavel Begunkovfe1cdd52021-02-17 21:02:36 +00003605 iovec = NULL;
Jens Axboee8c2bc12020-08-15 18:44:09 -07003606 rw = req->async_data;
Jens Axboecd658692021-09-10 11:19:14 -06003607 /*
3608 * Now use our persistent iterator and state, if we aren't already.
3609 * We've restored and mapped the iter to match.
3610 */
3611 if (iter != &rw->iter) {
3612 iter = &rw->iter;
3613 state = &rw->iter_state;
3614 }
Jens Axboe227c0c92020-08-13 11:51:40 -06003615
Pavel Begunkovb23df912021-02-04 13:52:04 +00003616 do {
Jens Axboecd658692021-09-10 11:19:14 -06003617 /*
3618 * We end up here because of a partial read, either from
3619 * above or inside this loop. Advance the iter by the bytes
3620 * that were consumed.
3621 */
3622 iov_iter_advance(iter, ret);
3623 if (!iov_iter_count(iter))
3624 break;
Pavel Begunkovb23df912021-02-04 13:52:04 +00003625 rw->bytes_done += ret;
Jens Axboecd658692021-09-10 11:19:14 -06003626 iov_iter_save_state(iter, state);
3627
Pavel Begunkovb23df912021-02-04 13:52:04 +00003628 /* if we can retry, do so with the callbacks armed */
3629 if (!io_rw_should_retry(req)) {
3630 kiocb->ki_flags &= ~IOCB_WAITQ;
3631 return -EAGAIN;
3632 }
3633
Pavel Begunkov98aada62022-10-16 22:42:58 +01003634 req->result = iov_iter_count(iter);
Pavel Begunkovb23df912021-02-04 13:52:04 +00003635 /*
3636 * Now retry read with the IOCB_WAITQ parts set in the iocb. If
3637 * we get -EIOCBQUEUED, then we'll get a notification when the
3638 * desired page gets unlocked. We can also get a partial read
3639 * here, and if we do, then just retry at the new offset.
3640 */
3641 ret = io_iter_do_read(req, iter);
3642 if (ret == -EIOCBQUEUED)
3643 return 0;
Jens Axboe227c0c92020-08-13 11:51:40 -06003644 /* we got some bytes, but not all. retry. */
Jens Axboeb5b0ecb2021-03-04 21:02:58 -07003645 kiocb->ki_flags &= ~IOCB_WAITQ;
Jens Axboecd658692021-09-10 11:19:14 -06003646 iov_iter_restore(iter, state);
3647 } while (ret > 0);
Jens Axboe227c0c92020-08-13 11:51:40 -06003648done:
Pavel Begunkov889fca72021-02-10 00:03:09 +00003649 kiocb_done(kiocb, ret, issue_flags);
Pavel Begunkovfe1cdd52021-02-17 21:02:36 +00003650out_free:
3651 /* it's faster to check here then delegate to kfree */
3652 if (iovec)
3653 kfree(iovec);
Pavel Begunkov5ea5dd42021-02-04 13:52:03 +00003654 return 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003655}
3656
Pavel Begunkov73debe62020-09-30 22:57:54 +03003657static int io_write_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboef67676d2019-12-02 11:03:47 -07003658{
Jens Axboe3529d8c2019-12-19 18:24:38 -07003659 if (unlikely(!(req->file->f_mode & FMODE_WRITE)))
3660 return -EBADF;
Jens Axboe5d329e12021-09-14 11:08:37 -06003661 return io_prep_rw(req, sqe, WRITE);
Jens Axboef67676d2019-12-02 11:03:47 -07003662}
3663
Pavel Begunkov889fca72021-02-10 00:03:09 +00003664static int io_write(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe2b188cc2019-01-07 10:46:33 -07003665{
3666 struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
Jens Axboe9adbd452019-12-20 08:45:55 -07003667 struct kiocb *kiocb = &req->rw.kiocb;
Jens Axboeff6165b2020-08-13 09:47:43 -06003668 struct iov_iter __iter, *iter = &__iter;
Jens Axboee8c2bc12020-08-15 18:44:09 -07003669 struct io_async_rw *rw = req->async_data;
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003670 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
Jens Axboecd658692021-09-10 11:19:14 -06003671 struct iov_iter_state __state, *state;
3672 ssize_t ret, ret2;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003673
Pavel Begunkov2846c482020-11-07 13:16:27 +00003674 if (rw) {
Jens Axboee8c2bc12020-08-15 18:44:09 -07003675 iter = &rw->iter;
Jens Axboecd658692021-09-10 11:19:14 -06003676 state = &rw->iter_state;
3677 iov_iter_restore(iter, state);
Pavel Begunkov2846c482020-11-07 13:16:27 +00003678 iovec = NULL;
3679 } else {
3680 ret = io_import_iovec(WRITE, req, &iovec, iter, !force_nonblock);
3681 if (ret < 0)
3682 return ret;
Jens Axboecd658692021-09-10 11:19:14 -06003683 state = &__state;
3684 iov_iter_save_state(iter, state);
Pavel Begunkov2846c482020-11-07 13:16:27 +00003685 }
Jens Axboecd658692021-09-10 11:19:14 -06003686 req->result = iov_iter_count(iter);
Jens Axboe2b188cc2019-01-07 10:46:33 -07003687
Jens Axboefd6c2e42019-12-18 12:19:41 -07003688 /* Ensure we clear previously set non-block flag */
3689 if (!force_nonblock)
Pavel Begunkova88fc402020-09-30 22:57:53 +03003690 kiocb->ki_flags &= ~IOCB_NOWAIT;
3691 else
3692 kiocb->ki_flags |= IOCB_NOWAIT;
Jens Axboefd6c2e42019-12-18 12:19:41 -07003693
Pavel Begunkov24c74672020-06-21 13:09:51 +03003694 /* If the file doesn't support async, just async punt */
Pavel Begunkovb191e2d2021-08-09 13:04:03 +01003695 if (force_nonblock && !io_file_supports_nowait(req, WRITE))
Jens Axboef67676d2019-12-02 11:03:47 -07003696 goto copy_iov;
Jens Axboef67676d2019-12-02 11:03:47 -07003697
Jens Axboe10d59342019-12-09 20:16:22 -07003698 /* file path doesn't support NOWAIT for non-direct_IO */
3699 if (force_nonblock && !(kiocb->ki_flags & IOCB_DIRECT) &&
3700 (req->flags & REQ_F_ISREG))
Jens Axboef67676d2019-12-02 11:03:47 -07003701 goto copy_iov;
Jens Axboe9e645e112019-05-10 16:07:28 -06003702
Jens Axboecd658692021-09-10 11:19:14 -06003703 ret = rw_verify_area(WRITE, req->file, io_kiocb_ppos(kiocb), req->result);
Pavel Begunkovfa15baf2020-08-01 13:50:02 +03003704 if (unlikely(ret))
3705 goto out_free;
Roman Penyaev9bf79332019-03-25 20:09:24 +01003706
Pavel Begunkovfa15baf2020-08-01 13:50:02 +03003707 /*
3708 * Open-code file_start_write here to grab freeze protection,
3709 * which will be released by another thread in
3710 * io_complete_rw(). Fool lockdep by telling it the lock got
3711 * released so that it doesn't complain about the held lock when
3712 * we return to userspace.
3713 */
3714 if (req->flags & REQ_F_ISREG) {
Darrick J. Wong8a3c84b2020-11-10 16:50:21 -08003715 sb_start_write(file_inode(req->file)->i_sb);
Pavel Begunkovfa15baf2020-08-01 13:50:02 +03003716 __sb_writers_release(file_inode(req->file)->i_sb,
3717 SB_FREEZE_WRITE);
3718 }
3719 kiocb->ki_flags |= IOCB_WRITE;
Roman Penyaev9bf79332019-03-25 20:09:24 +01003720
Pavel Begunkovfa15baf2020-08-01 13:50:02 +03003721 if (req->file->f_op->write_iter)
Jens Axboeff6165b2020-08-13 09:47:43 -06003722 ret2 = call_write_iter(req->file, kiocb, iter);
Guoyu Huang2dd21112020-08-05 03:53:50 -07003723 else if (req->file->f_op->write)
Jens Axboe4017eb92020-10-22 14:14:12 -06003724 ret2 = loop_rw_iter(WRITE, req, iter);
Guoyu Huang2dd21112020-08-05 03:53:50 -07003725 else
3726 ret2 = -EINVAL;
Jens Axboe4ed734b2020-03-20 11:23:41 -06003727
Pavel Begunkov6ad7f232021-04-08 01:54:39 +01003728 if (req->flags & REQ_F_REISSUE) {
3729 req->flags &= ~REQ_F_REISSUE;
Jens Axboe230d50d2021-04-01 20:41:15 -06003730 ret2 = -EAGAIN;
Pavel Begunkov6ad7f232021-04-08 01:54:39 +01003731 }
Jens Axboe230d50d2021-04-01 20:41:15 -06003732
Pavel Begunkovfa15baf2020-08-01 13:50:02 +03003733 /*
3734 * Raw bdev writes will return -EOPNOTSUPP for IOCB_NOWAIT. Just
3735 * retry them without IOCB_NOWAIT.
3736 */
3737 if (ret2 == -EOPNOTSUPP && (kiocb->ki_flags & IOCB_NOWAIT))
3738 ret2 = -EAGAIN;
Pavel Begunkov75c668c2021-02-04 13:52:05 +00003739 /* no retry on NONBLOCK nor RWF_NOWAIT */
3740 if (ret2 == -EAGAIN && (req->flags & REQ_F_NOWAIT))
Jens Axboe355afae2020-09-02 09:30:31 -06003741 goto done;
Pavel Begunkovfa15baf2020-08-01 13:50:02 +03003742 if (!force_nonblock || ret2 != -EAGAIN) {
Jens Axboeeefdf302020-08-27 16:40:19 -06003743 /* IOPOLL retry should happen for io-wq threads */
3744 if ((req->ctx->flags & IORING_SETUP_IOPOLL) && ret2 == -EAGAIN)
3745 goto copy_iov;
Jens Axboe355afae2020-09-02 09:30:31 -06003746done:
Pavel Begunkov889fca72021-02-10 00:03:09 +00003747 kiocb_done(kiocb, ret2, issue_flags);
Pavel Begunkovfa15baf2020-08-01 13:50:02 +03003748 } else {
Jens Axboef67676d2019-12-02 11:03:47 -07003749copy_iov:
Jens Axboecd658692021-09-10 11:19:14 -06003750 iov_iter_restore(iter, state);
Jens Axboe227c0c92020-08-13 11:51:40 -06003751 ret = io_setup_async_rw(req, iovec, inline_vecs, iter, false);
Jens Axboe295219a2022-08-25 10:19:08 -06003752 if (!ret) {
3753 if (kiocb->ki_flags & IOCB_WRITE)
3754 kiocb_end_write(req);
3755 return -EAGAIN;
3756 }
3757 return ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003758 }
Jens Axboe31b51512019-01-18 22:56:34 -07003759out_free:
Pavel Begunkovf261c162020-08-20 11:34:10 +03003760 /* it's reportedly faster than delegating the null check to kfree() */
Pavel Begunkov252917c2020-07-13 22:59:20 +03003761 if (iovec)
Xiaoguang Wang6f2cc162020-06-18 15:01:56 +08003762 kfree(iovec);
Jens Axboe2b188cc2019-01-07 10:46:33 -07003763 return ret;
3764}
3765
Jens Axboe80a261f2020-09-28 14:23:58 -06003766static int io_renameat_prep(struct io_kiocb *req,
3767 const struct io_uring_sqe *sqe)
3768{
3769 struct io_rename *ren = &req->rename;
3770 const char __user *oldf, *newf;
3771
Jens Axboeed7eb252021-06-23 09:04:13 -06003772 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3773 return -EINVAL;
Pavel Begunkov26578cd2021-08-20 10:36:37 +01003774 if (sqe->ioprio || sqe->buf_index || sqe->splice_fd_in)
Jens Axboeed7eb252021-06-23 09:04:13 -06003775 return -EINVAL;
Jens Axboe80a261f2020-09-28 14:23:58 -06003776 if (unlikely(req->flags & REQ_F_FIXED_FILE))
3777 return -EBADF;
3778
3779 ren->old_dfd = READ_ONCE(sqe->fd);
3780 oldf = u64_to_user_ptr(READ_ONCE(sqe->addr));
3781 newf = u64_to_user_ptr(READ_ONCE(sqe->addr2));
3782 ren->new_dfd = READ_ONCE(sqe->len);
3783 ren->flags = READ_ONCE(sqe->rename_flags);
3784
3785 ren->oldpath = getname(oldf);
3786 if (IS_ERR(ren->oldpath))
3787 return PTR_ERR(ren->oldpath);
3788
3789 ren->newpath = getname(newf);
3790 if (IS_ERR(ren->newpath)) {
3791 putname(ren->oldpath);
3792 return PTR_ERR(ren->newpath);
3793 }
3794
3795 req->flags |= REQ_F_NEED_CLEANUP;
3796 return 0;
3797}
3798
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003799static int io_renameat(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe80a261f2020-09-28 14:23:58 -06003800{
3801 struct io_rename *ren = &req->rename;
3802 int ret;
3803
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003804 if (issue_flags & IO_URING_F_NONBLOCK)
Jens Axboe80a261f2020-09-28 14:23:58 -06003805 return -EAGAIN;
3806
3807 ret = do_renameat2(ren->old_dfd, ren->oldpath, ren->new_dfd,
3808 ren->newpath, ren->flags);
3809
3810 req->flags &= ~REQ_F_NEED_CLEANUP;
3811 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01003812 req_set_fail(req);
Jens Axboe80a261f2020-09-28 14:23:58 -06003813 io_req_complete(req, ret);
3814 return 0;
3815}
3816
Jens Axboe14a11432020-09-28 14:27:37 -06003817static int io_unlinkat_prep(struct io_kiocb *req,
3818 const struct io_uring_sqe *sqe)
3819{
3820 struct io_unlink *un = &req->unlink;
3821 const char __user *fname;
3822
Jens Axboe22634bc2021-06-23 09:07:45 -06003823 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3824 return -EINVAL;
Pavel Begunkov26578cd2021-08-20 10:36:37 +01003825 if (sqe->ioprio || sqe->off || sqe->len || sqe->buf_index ||
3826 sqe->splice_fd_in)
Jens Axboe22634bc2021-06-23 09:07:45 -06003827 return -EINVAL;
Jens Axboe14a11432020-09-28 14:27:37 -06003828 if (unlikely(req->flags & REQ_F_FIXED_FILE))
3829 return -EBADF;
3830
3831 un->dfd = READ_ONCE(sqe->fd);
3832
3833 un->flags = READ_ONCE(sqe->unlink_flags);
3834 if (un->flags & ~AT_REMOVEDIR)
3835 return -EINVAL;
3836
3837 fname = u64_to_user_ptr(READ_ONCE(sqe->addr));
3838 un->filename = getname(fname);
3839 if (IS_ERR(un->filename))
3840 return PTR_ERR(un->filename);
3841
3842 req->flags |= REQ_F_NEED_CLEANUP;
3843 return 0;
3844}
3845
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003846static int io_unlinkat(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe14a11432020-09-28 14:27:37 -06003847{
3848 struct io_unlink *un = &req->unlink;
3849 int ret;
3850
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003851 if (issue_flags & IO_URING_F_NONBLOCK)
Jens Axboe14a11432020-09-28 14:27:37 -06003852 return -EAGAIN;
3853
3854 if (un->flags & AT_REMOVEDIR)
3855 ret = do_rmdir(un->dfd, un->filename);
3856 else
3857 ret = do_unlinkat(un->dfd, un->filename);
3858
3859 req->flags &= ~REQ_F_NEED_CLEANUP;
3860 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01003861 req_set_fail(req);
Jens Axboe14a11432020-09-28 14:27:37 -06003862 io_req_complete(req, ret);
3863 return 0;
3864}
3865
Dmitry Kadasheve34a02d2021-07-08 13:34:45 +07003866static int io_mkdirat_prep(struct io_kiocb *req,
3867 const struct io_uring_sqe *sqe)
3868{
3869 struct io_mkdir *mkd = &req->mkdir;
3870 const char __user *fname;
3871
3872 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3873 return -EINVAL;
3874 if (sqe->ioprio || sqe->off || sqe->rw_flags || sqe->buf_index ||
3875 sqe->splice_fd_in)
3876 return -EINVAL;
3877 if (unlikely(req->flags & REQ_F_FIXED_FILE))
3878 return -EBADF;
3879
3880 mkd->dfd = READ_ONCE(sqe->fd);
3881 mkd->mode = READ_ONCE(sqe->len);
3882
3883 fname = u64_to_user_ptr(READ_ONCE(sqe->addr));
3884 mkd->filename = getname(fname);
3885 if (IS_ERR(mkd->filename))
3886 return PTR_ERR(mkd->filename);
3887
3888 req->flags |= REQ_F_NEED_CLEANUP;
3889 return 0;
3890}
3891
3892static int io_mkdirat(struct io_kiocb *req, int issue_flags)
3893{
3894 struct io_mkdir *mkd = &req->mkdir;
3895 int ret;
3896
3897 if (issue_flags & IO_URING_F_NONBLOCK)
3898 return -EAGAIN;
3899
3900 ret = do_mkdirat(mkd->dfd, mkd->filename, mkd->mode);
3901
3902 req->flags &= ~REQ_F_NEED_CLEANUP;
3903 if (ret < 0)
3904 req_set_fail(req);
3905 io_req_complete(req, ret);
3906 return 0;
3907}
3908
Dmitry Kadashev7a8721f2021-07-08 13:34:46 +07003909static int io_symlinkat_prep(struct io_kiocb *req,
3910 const struct io_uring_sqe *sqe)
3911{
3912 struct io_symlink *sl = &req->symlink;
3913 const char __user *oldpath, *newpath;
3914
3915 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3916 return -EINVAL;
3917 if (sqe->ioprio || sqe->len || sqe->rw_flags || sqe->buf_index ||
3918 sqe->splice_fd_in)
3919 return -EINVAL;
3920 if (unlikely(req->flags & REQ_F_FIXED_FILE))
3921 return -EBADF;
3922
3923 sl->new_dfd = READ_ONCE(sqe->fd);
3924 oldpath = u64_to_user_ptr(READ_ONCE(sqe->addr));
3925 newpath = u64_to_user_ptr(READ_ONCE(sqe->addr2));
3926
3927 sl->oldpath = getname(oldpath);
3928 if (IS_ERR(sl->oldpath))
3929 return PTR_ERR(sl->oldpath);
3930
3931 sl->newpath = getname(newpath);
3932 if (IS_ERR(sl->newpath)) {
3933 putname(sl->oldpath);
3934 return PTR_ERR(sl->newpath);
3935 }
3936
3937 req->flags |= REQ_F_NEED_CLEANUP;
3938 return 0;
3939}
3940
3941static int io_symlinkat(struct io_kiocb *req, int issue_flags)
3942{
3943 struct io_symlink *sl = &req->symlink;
3944 int ret;
3945
3946 if (issue_flags & IO_URING_F_NONBLOCK)
3947 return -EAGAIN;
3948
3949 ret = do_symlinkat(sl->oldpath, sl->new_dfd, sl->newpath);
3950
3951 req->flags &= ~REQ_F_NEED_CLEANUP;
3952 if (ret < 0)
3953 req_set_fail(req);
3954 io_req_complete(req, ret);
3955 return 0;
3956}
3957
Dmitry Kadashevcf30da92021-07-08 13:34:47 +07003958static int io_linkat_prep(struct io_kiocb *req,
3959 const struct io_uring_sqe *sqe)
3960{
3961 struct io_hardlink *lnk = &req->hardlink;
3962 const char __user *oldf, *newf;
3963
3964 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3965 return -EINVAL;
3966 if (sqe->ioprio || sqe->rw_flags || sqe->buf_index || sqe->splice_fd_in)
3967 return -EINVAL;
3968 if (unlikely(req->flags & REQ_F_FIXED_FILE))
3969 return -EBADF;
3970
3971 lnk->old_dfd = READ_ONCE(sqe->fd);
3972 lnk->new_dfd = READ_ONCE(sqe->len);
3973 oldf = u64_to_user_ptr(READ_ONCE(sqe->addr));
3974 newf = u64_to_user_ptr(READ_ONCE(sqe->addr2));
3975 lnk->flags = READ_ONCE(sqe->hardlink_flags);
3976
3977 lnk->oldpath = getname(oldf);
3978 if (IS_ERR(lnk->oldpath))
3979 return PTR_ERR(lnk->oldpath);
3980
3981 lnk->newpath = getname(newf);
3982 if (IS_ERR(lnk->newpath)) {
3983 putname(lnk->oldpath);
3984 return PTR_ERR(lnk->newpath);
3985 }
3986
3987 req->flags |= REQ_F_NEED_CLEANUP;
3988 return 0;
3989}
3990
3991static int io_linkat(struct io_kiocb *req, int issue_flags)
3992{
3993 struct io_hardlink *lnk = &req->hardlink;
3994 int ret;
3995
3996 if (issue_flags & IO_URING_F_NONBLOCK)
3997 return -EAGAIN;
3998
3999 ret = do_linkat(lnk->old_dfd, lnk->oldpath, lnk->new_dfd,
4000 lnk->newpath, lnk->flags);
4001
4002 req->flags &= ~REQ_F_NEED_CLEANUP;
4003 if (ret < 0)
4004 req_set_fail(req);
4005 io_req_complete(req, ret);
4006 return 0;
4007}
4008
Jens Axboe36f4fa62020-09-05 11:14:22 -06004009static int io_shutdown_prep(struct io_kiocb *req,
4010 const struct io_uring_sqe *sqe)
4011{
4012#if defined(CONFIG_NET)
4013 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4014 return -EINVAL;
Pavel Begunkov26578cd2021-08-20 10:36:37 +01004015 if (unlikely(sqe->ioprio || sqe->off || sqe->addr || sqe->rw_flags ||
4016 sqe->buf_index || sqe->splice_fd_in))
Jens Axboe36f4fa62020-09-05 11:14:22 -06004017 return -EINVAL;
4018
4019 req->shutdown.how = READ_ONCE(sqe->len);
4020 return 0;
4021#else
4022 return -EOPNOTSUPP;
4023#endif
4024}
4025
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004026static int io_shutdown(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe36f4fa62020-09-05 11:14:22 -06004027{
4028#if defined(CONFIG_NET)
4029 struct socket *sock;
4030 int ret;
4031
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004032 if (issue_flags & IO_URING_F_NONBLOCK)
Jens Axboe36f4fa62020-09-05 11:14:22 -06004033 return -EAGAIN;
4034
Linus Torvalds48aba792020-12-16 12:44:05 -08004035 sock = sock_from_file(req->file);
Jens Axboe36f4fa62020-09-05 11:14:22 -06004036 if (unlikely(!sock))
Linus Torvalds48aba792020-12-16 12:44:05 -08004037 return -ENOTSOCK;
Jens Axboe36f4fa62020-09-05 11:14:22 -06004038
4039 ret = __sys_shutdown_sock(sock, req->shutdown.how);
Jens Axboea1464682020-12-14 20:57:27 -07004040 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004041 req_set_fail(req);
Jens Axboe36f4fa62020-09-05 11:14:22 -06004042 io_req_complete(req, ret);
4043 return 0;
4044#else
4045 return -EOPNOTSUPP;
4046#endif
4047}
4048
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03004049static int __io_splice_prep(struct io_kiocb *req,
4050 const struct io_uring_sqe *sqe)
Pavel Begunkov7d67af22020-02-24 11:32:45 +03004051{
Pavel Begunkovfe7e3252021-06-24 15:09:57 +01004052 struct io_splice *sp = &req->splice;
Pavel Begunkov7d67af22020-02-24 11:32:45 +03004053 unsigned int valid_flags = SPLICE_F_FD_IN_FIXED | SPLICE_F_ALL;
Pavel Begunkov7d67af22020-02-24 11:32:45 +03004054
Pavel Begunkov3232dd02020-06-03 18:03:22 +03004055 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4056 return -EINVAL;
Pavel Begunkov7d67af22020-02-24 11:32:45 +03004057
Pavel Begunkov7d67af22020-02-24 11:32:45 +03004058 sp->len = READ_ONCE(sqe->len);
4059 sp->flags = READ_ONCE(sqe->splice_flags);
Pavel Begunkov7d67af22020-02-24 11:32:45 +03004060 if (unlikely(sp->flags & ~valid_flags))
4061 return -EINVAL;
Jens Axboeae6cba32022-03-29 10:59:20 -06004062 sp->splice_fd_in = READ_ONCE(sqe->splice_fd_in);
Pavel Begunkov7d67af22020-02-24 11:32:45 +03004063 return 0;
4064}
4065
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03004066static int io_tee_prep(struct io_kiocb *req,
4067 const struct io_uring_sqe *sqe)
4068{
4069 if (READ_ONCE(sqe->splice_off_in) || READ_ONCE(sqe->off))
4070 return -EINVAL;
4071 return __io_splice_prep(req, sqe);
4072}
4073
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004074static int io_tee(struct io_kiocb *req, unsigned int issue_flags)
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03004075{
4076 struct io_splice *sp = &req->splice;
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03004077 struct file *out = sp->file_out;
4078 unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED;
Jens Axboeae6cba32022-03-29 10:59:20 -06004079 struct file *in;
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03004080 long ret = 0;
4081
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004082 if (issue_flags & IO_URING_F_NONBLOCK)
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03004083 return -EAGAIN;
Jens Axboeae6cba32022-03-29 10:59:20 -06004084
4085 in = io_file_get(req->ctx, req, sp->splice_fd_in,
4086 (sp->flags & SPLICE_F_FD_IN_FIXED));
4087 if (!in) {
4088 ret = -EBADF;
4089 goto done;
4090 }
4091
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03004092 if (sp->len)
4093 ret = do_tee(in, out, sp->len, flags);
4094
Pavel Begunkove1d767f2021-03-19 17:22:43 +00004095 if (!(sp->flags & SPLICE_F_FD_IN_FIXED))
4096 io_put_file(in);
Jens Axboeae6cba32022-03-29 10:59:20 -06004097done:
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03004098 if (ret != sp->len)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004099 req_set_fail(req);
Jens Axboee1e16092020-06-22 09:17:17 -06004100 io_req_complete(req, ret);
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03004101 return 0;
4102}
4103
4104static int io_splice_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4105{
Pavel Begunkovfe7e3252021-06-24 15:09:57 +01004106 struct io_splice *sp = &req->splice;
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03004107
4108 sp->off_in = READ_ONCE(sqe->splice_off_in);
4109 sp->off_out = READ_ONCE(sqe->off);
4110 return __io_splice_prep(req, sqe);
4111}
4112
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004113static int io_splice(struct io_kiocb *req, unsigned int issue_flags)
Pavel Begunkov7d67af22020-02-24 11:32:45 +03004114{
4115 struct io_splice *sp = &req->splice;
Pavel Begunkov7d67af22020-02-24 11:32:45 +03004116 struct file *out = sp->file_out;
4117 unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED;
4118 loff_t *poff_in, *poff_out;
Jens Axboeae6cba32022-03-29 10:59:20 -06004119 struct file *in;
Pavel Begunkovc9687422020-05-04 23:00:54 +03004120 long ret = 0;
Pavel Begunkov7d67af22020-02-24 11:32:45 +03004121
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004122 if (issue_flags & IO_URING_F_NONBLOCK)
Pavel Begunkov2fb3e822020-05-01 17:09:38 +03004123 return -EAGAIN;
Pavel Begunkov7d67af22020-02-24 11:32:45 +03004124
Jens Axboeae6cba32022-03-29 10:59:20 -06004125 in = io_file_get(req->ctx, req, sp->splice_fd_in,
4126 (sp->flags & SPLICE_F_FD_IN_FIXED));
4127 if (!in) {
4128 ret = -EBADF;
4129 goto done;
4130 }
4131
Pavel Begunkov7d67af22020-02-24 11:32:45 +03004132 poff_in = (sp->off_in == -1) ? NULL : &sp->off_in;
4133 poff_out = (sp->off_out == -1) ? NULL : &sp->off_out;
Pavel Begunkovc9687422020-05-04 23:00:54 +03004134
Jens Axboe948a7742020-05-17 14:21:38 -06004135 if (sp->len)
Pavel Begunkovc9687422020-05-04 23:00:54 +03004136 ret = do_splice(in, poff_in, out, poff_out, sp->len, flags);
Pavel Begunkov7d67af22020-02-24 11:32:45 +03004137
Pavel Begunkove1d767f2021-03-19 17:22:43 +00004138 if (!(sp->flags & SPLICE_F_FD_IN_FIXED))
4139 io_put_file(in);
Jens Axboeae6cba32022-03-29 10:59:20 -06004140done:
Pavel Begunkov7d67af22020-02-24 11:32:45 +03004141 if (ret != sp->len)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004142 req_set_fail(req);
Jens Axboee1e16092020-06-22 09:17:17 -06004143 io_req_complete(req, ret);
Pavel Begunkov7d67af22020-02-24 11:32:45 +03004144 return 0;
4145}
4146
Jens Axboe2b188cc2019-01-07 10:46:33 -07004147/*
4148 * IORING_OP_NOP just posts a completion event, nothing else.
4149 */
Pavel Begunkov889fca72021-02-10 00:03:09 +00004150static int io_nop(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe2b188cc2019-01-07 10:46:33 -07004151{
4152 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe2b188cc2019-01-07 10:46:33 -07004153
Jens Axboedef596e2019-01-09 08:59:42 -07004154 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
4155 return -EINVAL;
4156
Pavel Begunkov889fca72021-02-10 00:03:09 +00004157 __io_req_complete(req, issue_flags, 0, 0);
Jens Axboe2b188cc2019-01-07 10:46:33 -07004158 return 0;
4159}
4160
Pavel Begunkov1155c762021-02-18 18:29:38 +00004161static int io_fsync_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Christoph Hellwigc992fe22019-01-11 09:43:02 -07004162{
Jens Axboe6b063142019-01-10 22:13:58 -07004163 struct io_ring_ctx *ctx = req->ctx;
Christoph Hellwigc992fe22019-01-11 09:43:02 -07004164
Jens Axboe6b063142019-01-10 22:13:58 -07004165 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
Jens Axboedef596e2019-01-09 08:59:42 -07004166 return -EINVAL;
Pavel Begunkov26578cd2021-08-20 10:36:37 +01004167 if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index ||
4168 sqe->splice_fd_in))
Christoph Hellwigc992fe22019-01-11 09:43:02 -07004169 return -EINVAL;
4170
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004171 req->sync.flags = READ_ONCE(sqe->fsync_flags);
4172 if (unlikely(req->sync.flags & ~IORING_FSYNC_DATASYNC))
4173 return -EINVAL;
4174
4175 req->sync.off = READ_ONCE(sqe->off);
4176 req->sync.len = READ_ONCE(sqe->len);
Christoph Hellwigc992fe22019-01-11 09:43:02 -07004177 return 0;
4178}
4179
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004180static int io_fsync(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe78912932020-01-14 22:09:06 -07004181{
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004182 loff_t end = req->sync.off + req->sync.len;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004183 int ret;
4184
Pavel Begunkovac45abc2020-06-08 21:08:18 +03004185 /* fsync always requires a blocking context */
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004186 if (issue_flags & IO_URING_F_NONBLOCK)
Pavel Begunkovac45abc2020-06-08 21:08:18 +03004187 return -EAGAIN;
4188
Jens Axboe9adbd452019-12-20 08:45:55 -07004189 ret = vfs_fsync_range(req->file, req->sync.off,
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004190 end > 0 ? end : LLONG_MAX,
4191 req->sync.flags & IORING_FSYNC_DATASYNC);
4192 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004193 req_set_fail(req);
Jens Axboee1e16092020-06-22 09:17:17 -06004194 io_req_complete(req, ret);
Christoph Hellwigc992fe22019-01-11 09:43:02 -07004195 return 0;
4196}
4197
Jens Axboed63d1b52019-12-10 10:38:56 -07004198static int io_fallocate_prep(struct io_kiocb *req,
4199 const struct io_uring_sqe *sqe)
4200{
Pavel Begunkov26578cd2021-08-20 10:36:37 +01004201 if (sqe->ioprio || sqe->buf_index || sqe->rw_flags ||
4202 sqe->splice_fd_in)
Jens Axboed63d1b52019-12-10 10:38:56 -07004203 return -EINVAL;
Pavel Begunkov3232dd02020-06-03 18:03:22 +03004204 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4205 return -EINVAL;
Jens Axboed63d1b52019-12-10 10:38:56 -07004206
4207 req->sync.off = READ_ONCE(sqe->off);
4208 req->sync.len = READ_ONCE(sqe->addr);
4209 req->sync.mode = READ_ONCE(sqe->len);
4210 return 0;
4211}
4212
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004213static int io_fallocate(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboed63d1b52019-12-10 10:38:56 -07004214{
Pavel Begunkovac45abc2020-06-08 21:08:18 +03004215 int ret;
Jens Axboed63d1b52019-12-10 10:38:56 -07004216
Pavel Begunkovac45abc2020-06-08 21:08:18 +03004217 /* fallocate always requiring blocking context */
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004218 if (issue_flags & IO_URING_F_NONBLOCK)
Pavel Begunkovac45abc2020-06-08 21:08:18 +03004219 return -EAGAIN;
Pavel Begunkovac45abc2020-06-08 21:08:18 +03004220 ret = vfs_fallocate(req->file, req->sync.mode, req->sync.off,
4221 req->sync.len);
Pavel Begunkovac45abc2020-06-08 21:08:18 +03004222 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004223 req_set_fail(req);
Jens Axboedf1ec532022-03-20 13:08:38 -06004224 else
4225 fsnotify_modify(req->file);
Jens Axboee1e16092020-06-22 09:17:17 -06004226 io_req_complete(req, ret);
Jens Axboed63d1b52019-12-10 10:38:56 -07004227 return 0;
4228}
4229
Pavel Begunkovec65fea2020-06-03 18:03:24 +03004230static int __io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboe15b71ab2019-12-11 11:20:36 -07004231{
Jens Axboef8748882020-01-08 17:47:02 -07004232 const char __user *fname;
Jens Axboe15b71ab2019-12-11 11:20:36 -07004233 int ret;
4234
Pavel Begunkovd3fddf62021-08-09 13:04:16 +01004235 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4236 return -EINVAL;
Pavel Begunkovb9445592021-08-25 12:25:45 +01004237 if (unlikely(sqe->ioprio || sqe->buf_index))
Jens Axboe15b71ab2019-12-11 11:20:36 -07004238 return -EINVAL;
Pavel Begunkovec65fea2020-06-03 18:03:24 +03004239 if (unlikely(req->flags & REQ_F_FIXED_FILE))
Jens Axboecf3040c2020-02-06 21:31:40 -07004240 return -EBADF;
Jens Axboe15b71ab2019-12-11 11:20:36 -07004241
Pavel Begunkovec65fea2020-06-03 18:03:24 +03004242 /* open.how should be already initialised */
4243 if (!(req->open.how.flags & O_PATH) && force_o_largefile())
Jens Axboe08a1d26eb2020-04-08 09:20:54 -06004244 req->open.how.flags |= O_LARGEFILE;
Jens Axboe15b71ab2019-12-11 11:20:36 -07004245
Pavel Begunkov25e72d12020-06-03 18:03:23 +03004246 req->open.dfd = READ_ONCE(sqe->fd);
4247 fname = u64_to_user_ptr(READ_ONCE(sqe->addr));
Jens Axboef8748882020-01-08 17:47:02 -07004248 req->open.filename = getname(fname);
Jens Axboe15b71ab2019-12-11 11:20:36 -07004249 if (IS_ERR(req->open.filename)) {
4250 ret = PTR_ERR(req->open.filename);
4251 req->open.filename = NULL;
4252 return ret;
4253 }
Pavel Begunkovb9445592021-08-25 12:25:45 +01004254
4255 req->open.file_slot = READ_ONCE(sqe->file_index);
4256 if (req->open.file_slot && (req->open.how.flags & O_CLOEXEC))
4257 return -EINVAL;
4258
Jens Axboe4022e7a2020-03-19 19:23:18 -06004259 req->open.nofile = rlimit(RLIMIT_NOFILE);
Pavel Begunkov8fef80b2020-02-07 23:59:53 +03004260 req->flags |= REQ_F_NEED_CLEANUP;
Jens Axboe15b71ab2019-12-11 11:20:36 -07004261 return 0;
4262}
4263
Pavel Begunkovec65fea2020-06-03 18:03:24 +03004264static int io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4265{
Pavel Begunkovd3fddf62021-08-09 13:04:16 +01004266 u64 mode = READ_ONCE(sqe->len);
4267 u64 flags = READ_ONCE(sqe->open_flags);
Pavel Begunkovec65fea2020-06-03 18:03:24 +03004268
Pavel Begunkovec65fea2020-06-03 18:03:24 +03004269 req->open.how = build_open_how(flags, mode);
4270 return __io_openat_prep(req, sqe);
4271}
4272
Jens Axboecebdb982020-01-08 17:59:24 -07004273static int io_openat2_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4274{
4275 struct open_how __user *how;
Jens Axboecebdb982020-01-08 17:59:24 -07004276 size_t len;
4277 int ret;
4278
Jens Axboecebdb982020-01-08 17:59:24 -07004279 how = u64_to_user_ptr(READ_ONCE(sqe->addr2));
4280 len = READ_ONCE(sqe->len);
Jens Axboecebdb982020-01-08 17:59:24 -07004281 if (len < OPEN_HOW_SIZE_VER0)
4282 return -EINVAL;
4283
4284 ret = copy_struct_from_user(&req->open.how, sizeof(req->open.how), how,
4285 len);
4286 if (ret)
4287 return ret;
4288
Pavel Begunkovec65fea2020-06-03 18:03:24 +03004289 return __io_openat_prep(req, sqe);
Jens Axboecebdb982020-01-08 17:59:24 -07004290}
4291
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004292static int io_openat2(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe15b71ab2019-12-11 11:20:36 -07004293{
4294 struct open_flags op;
Jens Axboe15b71ab2019-12-11 11:20:36 -07004295 struct file *file;
Pavel Begunkovb9445592021-08-25 12:25:45 +01004296 bool resolve_nonblock, nonblock_set;
4297 bool fixed = !!req->open.file_slot;
Jens Axboe15b71ab2019-12-11 11:20:36 -07004298 int ret;
4299
Jens Axboecebdb982020-01-08 17:59:24 -07004300 ret = build_open_flags(&req->open.how, &op);
Jens Axboe15b71ab2019-12-11 11:20:36 -07004301 if (ret)
4302 goto err;
Jens Axboe3a81fd02020-12-10 12:25:36 -07004303 nonblock_set = op.open_flag & O_NONBLOCK;
4304 resolve_nonblock = req->open.how.resolve & RESOLVE_CACHED;
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004305 if (issue_flags & IO_URING_F_NONBLOCK) {
Jens Axboe3a81fd02020-12-10 12:25:36 -07004306 /*
4307 * Don't bother trying for O_TRUNC, O_CREAT, or O_TMPFILE open,
4308 * it'll always -EAGAIN
4309 */
4310 if (req->open.how.flags & (O_TRUNC | O_CREAT | O_TMPFILE))
4311 return -EAGAIN;
4312 op.lookup_flags |= LOOKUP_CACHED;
4313 op.open_flag |= O_NONBLOCK;
4314 }
Jens Axboe15b71ab2019-12-11 11:20:36 -07004315
Pavel Begunkovb9445592021-08-25 12:25:45 +01004316 if (!fixed) {
4317 ret = __get_unused_fd_flags(req->open.how.flags, req->open.nofile);
4318 if (ret < 0)
4319 goto err;
4320 }
Jens Axboe15b71ab2019-12-11 11:20:36 -07004321
4322 file = do_filp_open(req->open.dfd, req->open.filename, &op);
Pavel Begunkov12dcb58a2021-06-24 15:10:00 +01004323 if (IS_ERR(file)) {
Jens Axboe3a81fd02020-12-10 12:25:36 -07004324 /*
Pavel Begunkov12dcb58a2021-06-24 15:10:00 +01004325 * We could hang on to this 'fd' on retrying, but seems like
4326 * marginal gain for something that is now known to be a slower
4327 * path. So just put it, and we'll get a new one when we retry.
Jens Axboe3a81fd02020-12-10 12:25:36 -07004328 */
Pavel Begunkovb9445592021-08-25 12:25:45 +01004329 if (!fixed)
4330 put_unused_fd(ret);
Pavel Begunkov12dcb58a2021-06-24 15:10:00 +01004331
4332 ret = PTR_ERR(file);
4333 /* only retry if RESOLVE_CACHED wasn't already set by application */
4334 if (ret == -EAGAIN &&
4335 (!resolve_nonblock && (issue_flags & IO_URING_F_NONBLOCK)))
4336 return -EAGAIN;
4337 goto err;
Jens Axboe3a81fd02020-12-10 12:25:36 -07004338 }
4339
Pavel Begunkov12dcb58a2021-06-24 15:10:00 +01004340 if ((issue_flags & IO_URING_F_NONBLOCK) && !nonblock_set)
4341 file->f_flags &= ~O_NONBLOCK;
4342 fsnotify_open(file);
Pavel Begunkovb9445592021-08-25 12:25:45 +01004343
4344 if (!fixed)
4345 fd_install(ret, file);
4346 else
4347 ret = io_install_fixed_file(req, file, issue_flags,
4348 req->open.file_slot - 1);
Jens Axboe15b71ab2019-12-11 11:20:36 -07004349err:
4350 putname(req->open.filename);
Pavel Begunkov8fef80b2020-02-07 23:59:53 +03004351 req->flags &= ~REQ_F_NEED_CLEANUP;
Jens Axboe15b71ab2019-12-11 11:20:36 -07004352 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004353 req_set_fail(req);
Pavel Begunkov0bdf3392021-04-11 01:46:29 +01004354 __io_req_complete(req, issue_flags, ret, 0);
Jens Axboe15b71ab2019-12-11 11:20:36 -07004355 return 0;
4356}
4357
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004358static int io_openat(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboecebdb982020-01-08 17:59:24 -07004359{
Pavel Begunkove45cff52021-02-28 22:35:14 +00004360 return io_openat2(req, issue_flags);
Jens Axboecebdb982020-01-08 17:59:24 -07004361}
4362
Jens Axboe067524e2020-03-02 16:32:28 -07004363static int io_remove_buffers_prep(struct io_kiocb *req,
4364 const struct io_uring_sqe *sqe)
4365{
4366 struct io_provide_buf *p = &req->pbuf;
4367 u64 tmp;
4368
Pavel Begunkov26578cd2021-08-20 10:36:37 +01004369 if (sqe->ioprio || sqe->rw_flags || sqe->addr || sqe->len || sqe->off ||
4370 sqe->splice_fd_in)
Jens Axboe067524e2020-03-02 16:32:28 -07004371 return -EINVAL;
4372
4373 tmp = READ_ONCE(sqe->fd);
4374 if (!tmp || tmp > USHRT_MAX)
4375 return -EINVAL;
4376
4377 memset(p, 0, sizeof(*p));
4378 p->nbufs = tmp;
4379 p->bgid = READ_ONCE(sqe->buf_group);
4380 return 0;
4381}
4382
4383static int __io_remove_buffers(struct io_ring_ctx *ctx, struct io_buffer *buf,
4384 int bgid, unsigned nbufs)
4385{
4386 unsigned i = 0;
4387
4388 /* shouldn't happen */
4389 if (!nbufs)
4390 return 0;
4391
4392 /* the head kbuf is the list itself */
4393 while (!list_empty(&buf->list)) {
4394 struct io_buffer *nxt;
4395
4396 nxt = list_first_entry(&buf->list, struct io_buffer, list);
4397 list_del(&nxt->list);
4398 kfree(nxt);
4399 if (++i == nbufs)
4400 return i;
Ye Bin2d447d32021-11-22 10:47:37 +08004401 cond_resched();
Jens Axboe067524e2020-03-02 16:32:28 -07004402 }
4403 i++;
4404 kfree(buf);
Jens Axboe9e15c3a2021-03-13 12:29:43 -07004405 xa_erase(&ctx->io_buffers, bgid);
Jens Axboe067524e2020-03-02 16:32:28 -07004406
4407 return i;
4408}
4409
Pavel Begunkov889fca72021-02-10 00:03:09 +00004410static int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe067524e2020-03-02 16:32:28 -07004411{
4412 struct io_provide_buf *p = &req->pbuf;
4413 struct io_ring_ctx *ctx = req->ctx;
4414 struct io_buffer *head;
4415 int ret = 0;
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004416 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
Jens Axboe067524e2020-03-02 16:32:28 -07004417
4418 io_ring_submit_lock(ctx, !force_nonblock);
4419
4420 lockdep_assert_held(&ctx->uring_lock);
4421
4422 ret = -ENOENT;
Jens Axboe9e15c3a2021-03-13 12:29:43 -07004423 head = xa_load(&ctx->io_buffers, p->bgid);
Jens Axboe067524e2020-03-02 16:32:28 -07004424 if (head)
4425 ret = __io_remove_buffers(ctx, head, p->bgid, p->nbufs);
Jens Axboe067524e2020-03-02 16:32:28 -07004426 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004427 req_set_fail(req);
Pavel Begunkov31bff9a2020-12-06 22:22:43 +00004428
Pavel Begunkov9fb8cb42021-02-28 22:35:13 +00004429 /* complete before unlock, IOPOLL may need the lock */
4430 __io_req_complete(req, issue_flags, ret, 0);
4431 io_ring_submit_unlock(ctx, !force_nonblock);
Jens Axboe067524e2020-03-02 16:32:28 -07004432 return 0;
4433}
4434
Jens Axboeddf0322d2020-02-23 16:41:33 -07004435static int io_provide_buffers_prep(struct io_kiocb *req,
4436 const struct io_uring_sqe *sqe)
4437{
Pavel Begunkov38134ad2021-04-15 13:07:39 +01004438 unsigned long size, tmp_check;
Jens Axboeddf0322d2020-02-23 16:41:33 -07004439 struct io_provide_buf *p = &req->pbuf;
4440 u64 tmp;
4441
Pavel Begunkov26578cd2021-08-20 10:36:37 +01004442 if (sqe->ioprio || sqe->rw_flags || sqe->splice_fd_in)
Jens Axboeddf0322d2020-02-23 16:41:33 -07004443 return -EINVAL;
4444
4445 tmp = READ_ONCE(sqe->fd);
4446 if (!tmp || tmp > USHRT_MAX)
4447 return -E2BIG;
4448 p->nbufs = tmp;
4449 p->addr = READ_ONCE(sqe->addr);
4450 p->len = READ_ONCE(sqe->len);
4451
Pavel Begunkov38134ad2021-04-15 13:07:39 +01004452 if (check_mul_overflow((unsigned long)p->len, (unsigned long)p->nbufs,
4453 &size))
4454 return -EOVERFLOW;
4455 if (check_add_overflow((unsigned long)p->addr, size, &tmp_check))
4456 return -EOVERFLOW;
4457
Pavel Begunkovd81269f2021-03-19 10:21:19 +00004458 size = (unsigned long)p->len * p->nbufs;
4459 if (!access_ok(u64_to_user_ptr(p->addr), size))
Jens Axboeddf0322d2020-02-23 16:41:33 -07004460 return -EFAULT;
4461
4462 p->bgid = READ_ONCE(sqe->buf_group);
4463 tmp = READ_ONCE(sqe->off);
4464 if (tmp > USHRT_MAX)
4465 return -E2BIG;
4466 p->bid = tmp;
4467 return 0;
4468}
4469
4470static int io_add_buffers(struct io_provide_buf *pbuf, struct io_buffer **head)
4471{
4472 struct io_buffer *buf;
4473 u64 addr = pbuf->addr;
4474 int i, bid = pbuf->bid;
4475
4476 for (i = 0; i < pbuf->nbufs; i++) {
Jens Axboe9990da92021-09-24 07:39:08 -06004477 buf = kmalloc(sizeof(*buf), GFP_KERNEL_ACCOUNT);
Jens Axboeddf0322d2020-02-23 16:41:33 -07004478 if (!buf)
4479 break;
4480
4481 buf->addr = addr;
Thadeu Lima de Souza Cascardod1f82802021-05-05 09:47:06 -03004482 buf->len = min_t(__u32, pbuf->len, MAX_RW_COUNT);
Jens Axboeddf0322d2020-02-23 16:41:33 -07004483 buf->bid = bid;
4484 addr += pbuf->len;
4485 bid++;
4486 if (!*head) {
4487 INIT_LIST_HEAD(&buf->list);
4488 *head = buf;
4489 } else {
4490 list_add_tail(&buf->list, &(*head)->list);
4491 }
Eric Dumazetc718ea42022-02-14 20:10:03 -08004492 cond_resched();
Jens Axboeddf0322d2020-02-23 16:41:33 -07004493 }
4494
4495 return i ? i : -ENOMEM;
4496}
4497
Pavel Begunkov889fca72021-02-10 00:03:09 +00004498static int io_provide_buffers(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboeddf0322d2020-02-23 16:41:33 -07004499{
4500 struct io_provide_buf *p = &req->pbuf;
4501 struct io_ring_ctx *ctx = req->ctx;
4502 struct io_buffer *head, *list;
4503 int ret = 0;
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004504 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
Jens Axboeddf0322d2020-02-23 16:41:33 -07004505
4506 io_ring_submit_lock(ctx, !force_nonblock);
4507
4508 lockdep_assert_held(&ctx->uring_lock);
4509
Jens Axboe9e15c3a2021-03-13 12:29:43 -07004510 list = head = xa_load(&ctx->io_buffers, p->bgid);
Jens Axboeddf0322d2020-02-23 16:41:33 -07004511
4512 ret = io_add_buffers(p, &head);
Jens Axboe9e15c3a2021-03-13 12:29:43 -07004513 if (ret >= 0 && !list) {
Pavel Begunkovfa304062022-08-04 15:13:46 +01004514 ret = xa_insert(&ctx->io_buffers, p->bgid, head,
4515 GFP_KERNEL_ACCOUNT);
Jens Axboe9e15c3a2021-03-13 12:29:43 -07004516 if (ret < 0)
Jens Axboe067524e2020-03-02 16:32:28 -07004517 __io_remove_buffers(ctx, head, p->bgid, -1U);
Jens Axboeddf0322d2020-02-23 16:41:33 -07004518 }
Jens Axboeddf0322d2020-02-23 16:41:33 -07004519 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004520 req_set_fail(req);
Pavel Begunkov9fb8cb42021-02-28 22:35:13 +00004521 /* complete before unlock, IOPOLL may need the lock */
4522 __io_req_complete(req, issue_flags, ret, 0);
4523 io_ring_submit_unlock(ctx, !force_nonblock);
Jens Axboeddf0322d2020-02-23 16:41:33 -07004524 return 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07004525}
4526
Jens Axboe3e4827b2020-01-08 15:18:09 -07004527static int io_epoll_ctl_prep(struct io_kiocb *req,
4528 const struct io_uring_sqe *sqe)
4529{
4530#if defined(CONFIG_EPOLL)
Pavel Begunkov26578cd2021-08-20 10:36:37 +01004531 if (sqe->ioprio || sqe->buf_index || sqe->splice_fd_in)
Jens Axboe3e4827b2020-01-08 15:18:09 -07004532 return -EINVAL;
Pavel Begunkov2d74d042021-05-14 12:05:46 +01004533 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
Pavel Begunkov3232dd02020-06-03 18:03:22 +03004534 return -EINVAL;
Jens Axboe3e4827b2020-01-08 15:18:09 -07004535
4536 req->epoll.epfd = READ_ONCE(sqe->fd);
4537 req->epoll.op = READ_ONCE(sqe->len);
4538 req->epoll.fd = READ_ONCE(sqe->off);
4539
4540 if (ep_op_has_event(req->epoll.op)) {
4541 struct epoll_event __user *ev;
4542
4543 ev = u64_to_user_ptr(READ_ONCE(sqe->addr));
4544 if (copy_from_user(&req->epoll.event, ev, sizeof(*ev)))
4545 return -EFAULT;
4546 }
4547
4548 return 0;
4549#else
4550 return -EOPNOTSUPP;
4551#endif
4552}
4553
Pavel Begunkov889fca72021-02-10 00:03:09 +00004554static int io_epoll_ctl(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe3e4827b2020-01-08 15:18:09 -07004555{
4556#if defined(CONFIG_EPOLL)
4557 struct io_epoll *ie = &req->epoll;
4558 int ret;
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004559 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
Jens Axboe3e4827b2020-01-08 15:18:09 -07004560
4561 ret = do_epoll_ctl(ie->epfd, ie->op, ie->fd, &ie->event, force_nonblock);
4562 if (force_nonblock && ret == -EAGAIN)
4563 return -EAGAIN;
4564
4565 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004566 req_set_fail(req);
Pavel Begunkov889fca72021-02-10 00:03:09 +00004567 __io_req_complete(req, issue_flags, ret, 0);
Jens Axboe3e4827b2020-01-08 15:18:09 -07004568 return 0;
4569#else
4570 return -EOPNOTSUPP;
4571#endif
4572}
4573
Jens Axboec1ca7572019-12-25 22:18:28 -07004574static int io_madvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4575{
4576#if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU)
Pavel Begunkov26578cd2021-08-20 10:36:37 +01004577 if (sqe->ioprio || sqe->buf_index || sqe->off || sqe->splice_fd_in)
Jens Axboec1ca7572019-12-25 22:18:28 -07004578 return -EINVAL;
Pavel Begunkov3232dd02020-06-03 18:03:22 +03004579 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4580 return -EINVAL;
Jens Axboec1ca7572019-12-25 22:18:28 -07004581
4582 req->madvise.addr = READ_ONCE(sqe->addr);
4583 req->madvise.len = READ_ONCE(sqe->len);
4584 req->madvise.advice = READ_ONCE(sqe->fadvise_advice);
4585 return 0;
4586#else
4587 return -EOPNOTSUPP;
4588#endif
4589}
4590
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004591static int io_madvise(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboec1ca7572019-12-25 22:18:28 -07004592{
4593#if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU)
4594 struct io_madvise *ma = &req->madvise;
4595 int ret;
4596
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004597 if (issue_flags & IO_URING_F_NONBLOCK)
Jens Axboec1ca7572019-12-25 22:18:28 -07004598 return -EAGAIN;
4599
Minchan Kim0726b012020-10-17 16:14:50 -07004600 ret = do_madvise(current->mm, ma->addr, ma->len, ma->advice);
Jens Axboec1ca7572019-12-25 22:18:28 -07004601 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004602 req_set_fail(req);
Jens Axboee1e16092020-06-22 09:17:17 -06004603 io_req_complete(req, ret);
Jens Axboec1ca7572019-12-25 22:18:28 -07004604 return 0;
4605#else
4606 return -EOPNOTSUPP;
4607#endif
4608}
4609
Jens Axboe4840e412019-12-25 22:03:45 -07004610static int io_fadvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4611{
Pavel Begunkov26578cd2021-08-20 10:36:37 +01004612 if (sqe->ioprio || sqe->buf_index || sqe->addr || sqe->splice_fd_in)
Jens Axboe4840e412019-12-25 22:03:45 -07004613 return -EINVAL;
Pavel Begunkov3232dd02020-06-03 18:03:22 +03004614 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4615 return -EINVAL;
Jens Axboe4840e412019-12-25 22:03:45 -07004616
4617 req->fadvise.offset = READ_ONCE(sqe->off);
4618 req->fadvise.len = READ_ONCE(sqe->len);
4619 req->fadvise.advice = READ_ONCE(sqe->fadvise_advice);
4620 return 0;
4621}
4622
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004623static int io_fadvise(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe4840e412019-12-25 22:03:45 -07004624{
4625 struct io_fadvise *fa = &req->fadvise;
4626 int ret;
4627
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004628 if (issue_flags & IO_URING_F_NONBLOCK) {
Jens Axboe3e694262020-02-01 09:22:49 -07004629 switch (fa->advice) {
4630 case POSIX_FADV_NORMAL:
4631 case POSIX_FADV_RANDOM:
4632 case POSIX_FADV_SEQUENTIAL:
4633 break;
4634 default:
4635 return -EAGAIN;
4636 }
4637 }
Jens Axboe4840e412019-12-25 22:03:45 -07004638
4639 ret = vfs_fadvise(req->file, fa->offset, fa->len, fa->advice);
4640 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004641 req_set_fail(req);
Pavel Begunkov0bdf3392021-04-11 01:46:29 +01004642 __io_req_complete(req, issue_flags, ret, 0);
Jens Axboe4840e412019-12-25 22:03:45 -07004643 return 0;
4644}
4645
Jens Axboeeddc7ef2019-12-13 21:18:10 -07004646static int io_statx_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4647{
Pavel Begunkov2d74d042021-05-14 12:05:46 +01004648 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
Pavel Begunkov3232dd02020-06-03 18:03:22 +03004649 return -EINVAL;
Pavel Begunkov26578cd2021-08-20 10:36:37 +01004650 if (sqe->ioprio || sqe->buf_index || sqe->splice_fd_in)
Jens Axboeeddc7ef2019-12-13 21:18:10 -07004651 return -EINVAL;
Pavel Begunkov9c280f92020-04-08 08:58:46 +03004652 if (req->flags & REQ_F_FIXED_FILE)
Jens Axboecf3040c2020-02-06 21:31:40 -07004653 return -EBADF;
Jens Axboeeddc7ef2019-12-13 21:18:10 -07004654
Bijan Mottahedeh1d9e1282020-05-22 21:31:16 -07004655 req->statx.dfd = READ_ONCE(sqe->fd);
4656 req->statx.mask = READ_ONCE(sqe->len);
Bijan Mottahedehe62753e2020-05-22 21:31:18 -07004657 req->statx.filename = u64_to_user_ptr(READ_ONCE(sqe->addr));
Bijan Mottahedeh1d9e1282020-05-22 21:31:16 -07004658 req->statx.buffer = u64_to_user_ptr(READ_ONCE(sqe->addr2));
4659 req->statx.flags = READ_ONCE(sqe->statx_flags);
Jens Axboeeddc7ef2019-12-13 21:18:10 -07004660
Jens Axboeeddc7ef2019-12-13 21:18:10 -07004661 return 0;
4662}
4663
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004664static int io_statx(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboeeddc7ef2019-12-13 21:18:10 -07004665{
Bijan Mottahedeh1d9e1282020-05-22 21:31:16 -07004666 struct io_statx *ctx = &req->statx;
Jens Axboeeddc7ef2019-12-13 21:18:10 -07004667 int ret;
4668
Pavel Begunkov59d70012021-03-22 01:58:30 +00004669 if (issue_flags & IO_URING_F_NONBLOCK)
Jens Axboeeddc7ef2019-12-13 21:18:10 -07004670 return -EAGAIN;
4671
Bijan Mottahedehe62753e2020-05-22 21:31:18 -07004672 ret = do_statx(ctx->dfd, ctx->filename, ctx->flags, ctx->mask,
4673 ctx->buffer);
Jens Axboeeddc7ef2019-12-13 21:18:10 -07004674
Jens Axboeeddc7ef2019-12-13 21:18:10 -07004675 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004676 req_set_fail(req);
Jens Axboee1e16092020-06-22 09:17:17 -06004677 io_req_complete(req, ret);
Jens Axboeeddc7ef2019-12-13 21:18:10 -07004678 return 0;
4679}
4680
Jens Axboeb5dba592019-12-11 14:02:38 -07004681static int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4682{
Jens Axboe14587a462020-09-05 11:36:08 -06004683 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
Pavel Begunkov3232dd02020-06-03 18:03:22 +03004684 return -EINVAL;
Jens Axboeb5dba592019-12-11 14:02:38 -07004685 if (sqe->ioprio || sqe->off || sqe->addr || sqe->len ||
Pavel Begunkov7df778b2021-09-24 20:04:29 +01004686 sqe->rw_flags || sqe->buf_index)
Jens Axboeb5dba592019-12-11 14:02:38 -07004687 return -EINVAL;
Pavel Begunkov9c280f92020-04-08 08:58:46 +03004688 if (req->flags & REQ_F_FIXED_FILE)
Jens Axboecf3040c2020-02-06 21:31:40 -07004689 return -EBADF;
Jens Axboeb5dba592019-12-11 14:02:38 -07004690
4691 req->close.fd = READ_ONCE(sqe->fd);
Pavel Begunkov7df778b2021-09-24 20:04:29 +01004692 req->close.file_slot = READ_ONCE(sqe->file_index);
4693 if (req->close.file_slot && req->close.fd)
4694 return -EINVAL;
4695
Jens Axboeb5dba592019-12-11 14:02:38 -07004696 return 0;
4697}
4698
Pavel Begunkov889fca72021-02-10 00:03:09 +00004699static int io_close(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboeb5dba592019-12-11 14:02:38 -07004700{
Jens Axboe9eac1902021-01-19 15:50:37 -07004701 struct files_struct *files = current->files;
Pavel Begunkov3af73b22020-06-08 21:08:17 +03004702 struct io_close *close = &req->close;
Jens Axboe9eac1902021-01-19 15:50:37 -07004703 struct fdtable *fdt;
Pavel Begunkova1fde922021-04-11 01:46:28 +01004704 struct file *file = NULL;
4705 int ret = -EBADF;
Jens Axboeb5dba592019-12-11 14:02:38 -07004706
Pavel Begunkov7df778b2021-09-24 20:04:29 +01004707 if (req->close.file_slot) {
4708 ret = io_close_fixed(req, issue_flags);
4709 goto err;
4710 }
4711
Jens Axboe9eac1902021-01-19 15:50:37 -07004712 spin_lock(&files->file_lock);
4713 fdt = files_fdtable(files);
4714 if (close->fd >= fdt->max_fds) {
4715 spin_unlock(&files->file_lock);
4716 goto err;
4717 }
4718 file = fdt->fd[close->fd];
Pavel Begunkova1fde922021-04-11 01:46:28 +01004719 if (!file || file->f_op == &io_uring_fops) {
Jens Axboe9eac1902021-01-19 15:50:37 -07004720 spin_unlock(&files->file_lock);
4721 file = NULL;
4722 goto err;
Pavel Begunkov3af73b22020-06-08 21:08:17 +03004723 }
Jens Axboeb5dba592019-12-11 14:02:38 -07004724
4725 /* if the file has a flush method, be safe and punt to async */
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004726 if (file->f_op->flush && (issue_flags & IO_URING_F_NONBLOCK)) {
Jens Axboe9eac1902021-01-19 15:50:37 -07004727 spin_unlock(&files->file_lock);
Pavel Begunkov0bf0eef2020-05-26 20:34:06 +03004728 return -EAGAIN;
Pavel Begunkova2100672020-03-02 23:45:16 +03004729 }
Jens Axboeb5dba592019-12-11 14:02:38 -07004730
Jens Axboe9eac1902021-01-19 15:50:37 -07004731 ret = __close_fd_get_file(close->fd, &file);
4732 spin_unlock(&files->file_lock);
4733 if (ret < 0) {
4734 if (ret == -ENOENT)
4735 ret = -EBADF;
4736 goto err;
4737 }
4738
Pavel Begunkov3af73b22020-06-08 21:08:17 +03004739 /* No ->flush() or already async, safely close from here */
Jens Axboe9eac1902021-01-19 15:50:37 -07004740 ret = filp_close(file, current->files);
4741err:
Pavel Begunkov3af73b22020-06-08 21:08:17 +03004742 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004743 req_set_fail(req);
Jens Axboe9eac1902021-01-19 15:50:37 -07004744 if (file)
4745 fput(file);
Pavel Begunkov889fca72021-02-10 00:03:09 +00004746 __io_req_complete(req, issue_flags, ret, 0);
Jens Axboe1a417f42020-01-31 17:16:48 -07004747 return 0;
Jens Axboeb5dba592019-12-11 14:02:38 -07004748}
4749
Pavel Begunkov1155c762021-02-18 18:29:38 +00004750static int io_sfr_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboe5d17b4a2019-04-09 14:56:44 -06004751{
4752 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe5d17b4a2019-04-09 14:56:44 -06004753
Jens Axboe5d17b4a2019-04-09 14:56:44 -06004754 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
4755 return -EINVAL;
Pavel Begunkov26578cd2021-08-20 10:36:37 +01004756 if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index ||
4757 sqe->splice_fd_in))
Jens Axboe5d17b4a2019-04-09 14:56:44 -06004758 return -EINVAL;
4759
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004760 req->sync.off = READ_ONCE(sqe->off);
4761 req->sync.len = READ_ONCE(sqe->len);
4762 req->sync.flags = READ_ONCE(sqe->sync_range_flags);
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004763 return 0;
4764}
4765
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004766static int io_sync_file_range(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004767{
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004768 int ret;
4769
Pavel Begunkovac45abc2020-06-08 21:08:18 +03004770 /* sync_file_range always requires a blocking context */
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004771 if (issue_flags & IO_URING_F_NONBLOCK)
Pavel Begunkovac45abc2020-06-08 21:08:18 +03004772 return -EAGAIN;
4773
Jens Axboe9adbd452019-12-20 08:45:55 -07004774 ret = sync_file_range(req->file, req->sync.off, req->sync.len,
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004775 req->sync.flags);
4776 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004777 req_set_fail(req);
Jens Axboee1e16092020-06-22 09:17:17 -06004778 io_req_complete(req, ret);
Jens Axboe5d17b4a2019-04-09 14:56:44 -06004779 return 0;
4780}
4781
YueHaibing469956e2020-03-04 15:53:52 +08004782#if defined(CONFIG_NET)
Jens Axboe3c1a3d02022-04-20 19:21:36 -06004783static bool io_net_retry(struct socket *sock, int flags)
4784{
4785 if (!(flags & MSG_WAITALL))
4786 return false;
4787 return sock->type == SOCK_STREAM || sock->type == SOCK_SEQPACKET;
4788}
4789
Pavel Begunkov02d27d82020-02-28 10:36:36 +03004790static int io_setup_async_msg(struct io_kiocb *req,
4791 struct io_async_msghdr *kmsg)
4792{
Jens Axboee8c2bc12020-08-15 18:44:09 -07004793 struct io_async_msghdr *async_msg = req->async_data;
4794
4795 if (async_msg)
Pavel Begunkov02d27d82020-02-28 10:36:36 +03004796 return -EAGAIN;
Jens Axboee8c2bc12020-08-15 18:44:09 -07004797 if (io_alloc_async_data(req)) {
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004798 kfree(kmsg->free_iov);
Pavel Begunkov02d27d82020-02-28 10:36:36 +03004799 return -ENOMEM;
4800 }
Jens Axboee8c2bc12020-08-15 18:44:09 -07004801 async_msg = req->async_data;
Pavel Begunkov02d27d82020-02-28 10:36:36 +03004802 req->flags |= REQ_F_NEED_CLEANUP;
Jens Axboee8c2bc12020-08-15 18:44:09 -07004803 memcpy(async_msg, kmsg, sizeof(*kmsg));
Pavel Begunkovf9dc33f2022-09-29 22:23:18 +01004804 if (async_msg->msg.msg_name)
4805 async_msg->msg.msg_name = &async_msg->addr;
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004806 /* if were using fast_iov, set it to the new one */
4807 if (!async_msg->free_iov)
4808 async_msg->msg.msg_iter.iov = async_msg->fast_iov;
4809
Pavel Begunkov02d27d82020-02-28 10:36:36 +03004810 return -EAGAIN;
4811}
4812
Pavel Begunkov2ae523e2020-07-12 20:41:06 +03004813static int io_sendmsg_copy_hdr(struct io_kiocb *req,
4814 struct io_async_msghdr *iomsg)
4815{
Pavel Begunkov2ae523e2020-07-12 20:41:06 +03004816 iomsg->msg.msg_name = &iomsg->addr;
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004817 iomsg->free_iov = iomsg->fast_iov;
Pavel Begunkov2ae523e2020-07-12 20:41:06 +03004818 return sendmsg_copy_msghdr(&iomsg->msg, req->sr_msg.umsg,
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004819 req->sr_msg.msg_flags, &iomsg->free_iov);
Pavel Begunkov2ae523e2020-07-12 20:41:06 +03004820}
4821
Pavel Begunkov93642ef2021-02-18 18:29:44 +00004822static int io_sendmsg_prep_async(struct io_kiocb *req)
4823{
4824 int ret;
4825
Pavel Begunkov93642ef2021-02-18 18:29:44 +00004826 ret = io_sendmsg_copy_hdr(req, req->async_data);
4827 if (!ret)
4828 req->flags |= REQ_F_NEED_CLEANUP;
4829 return ret;
4830}
4831
Jens Axboe3529d8c2019-12-19 18:24:38 -07004832static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboeaa1fa282019-04-19 13:38:09 -06004833{
Jens Axboee47293f2019-12-20 08:58:21 -07004834 struct io_sr_msg *sr = &req->sr_msg;
Jens Axboe03b12302019-12-02 18:50:25 -07004835
Pavel Begunkovd2b6f482020-06-03 18:03:25 +03004836 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4837 return -EINVAL;
Jens Axboe79c10cb2022-04-26 19:34:11 -06004838 if (unlikely(sqe->addr2 || sqe->file_index))
4839 return -EINVAL;
Jens Axboe50fefe52022-06-30 14:42:05 -06004840 if (unlikely(sqe->addr2 || sqe->file_index || sqe->ioprio))
4841 return -EINVAL;
Pavel Begunkovd2b6f482020-06-03 18:03:25 +03004842
Pavel Begunkov270a5942020-07-12 20:41:04 +03004843 sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
Jens Axboefddafac2020-01-04 20:19:44 -07004844 sr->len = READ_ONCE(sqe->len);
Pavel Begunkov04411802021-04-01 15:44:00 +01004845 sr->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL;
4846 if (sr->msg_flags & MSG_DONTWAIT)
4847 req->flags |= REQ_F_NOWAIT;
Jens Axboe3529d8c2019-12-19 18:24:38 -07004848
Jens Axboed8768362020-02-27 14:17:49 -07004849#ifdef CONFIG_COMPAT
4850 if (req->ctx->compat)
4851 sr->msg_flags |= MSG_CMSG_COMPAT;
4852#endif
Jens Axboe3c1a3d02022-04-20 19:21:36 -06004853 sr->done_io = 0;
Pavel Begunkov93642ef2021-02-18 18:29:44 +00004854 return 0;
Jens Axboe03b12302019-12-02 18:50:25 -07004855}
4856
Pavel Begunkov889fca72021-02-10 00:03:09 +00004857static int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe03b12302019-12-02 18:50:25 -07004858{
Pavel Begunkov6b754c82020-07-16 23:28:00 +03004859 struct io_async_msghdr iomsg, *kmsg;
Jens Axboe3c1a3d02022-04-20 19:21:36 -06004860 struct io_sr_msg *sr = &req->sr_msg;
Jens Axboe03b12302019-12-02 18:50:25 -07004861 struct socket *sock;
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004862 unsigned flags;
Stefan Metzmacher00312752021-03-20 20:33:36 +01004863 int min_ret = 0;
Jens Axboe03b12302019-12-02 18:50:25 -07004864 int ret;
4865
Florent Revestdba4a922020-12-04 12:36:04 +01004866 sock = sock_from_file(req->file);
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004867 if (unlikely(!sock))
Florent Revestdba4a922020-12-04 12:36:04 +01004868 return -ENOTSOCK;
Jens Axboe03b12302019-12-02 18:50:25 -07004869
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004870 kmsg = req->async_data;
4871 if (!kmsg) {
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004872 ret = io_sendmsg_copy_hdr(req, &iomsg);
Jens Axboefddafac2020-01-04 20:19:44 -07004873 if (ret)
4874 return ret;
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004875 kmsg = &iomsg;
Jens Axboefddafac2020-01-04 20:19:44 -07004876 }
4877
Pavel Begunkov04411802021-04-01 15:44:00 +01004878 flags = req->sr_msg.msg_flags;
4879 if (issue_flags & IO_URING_F_NONBLOCK)
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004880 flags |= MSG_DONTWAIT;
Stefan Metzmacher00312752021-03-20 20:33:36 +01004881 if (flags & MSG_WAITALL)
4882 min_ret = iov_iter_count(&kmsg->msg.msg_iter);
4883
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004884 ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags);
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004885
Pavel Begunkovcdc68e72021-11-23 00:07:47 +00004886 if (ret < min_ret) {
4887 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))
4888 return io_setup_async_msg(req, kmsg);
4889 if (ret == -ERESTARTSYS)
4890 ret = -EINTR;
Jens Axboe3c1a3d02022-04-20 19:21:36 -06004891 if (ret > 0 && io_net_retry(sock, flags)) {
4892 sr->done_io += ret;
4893 req->flags |= REQ_F_PARTIAL_IO;
4894 return io_setup_async_msg(req, kmsg);
4895 }
Pavel Begunkovcdc68e72021-11-23 00:07:47 +00004896 req_set_fail(req);
4897 }
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004898 /* fast path, check for non-NULL to avoid function call */
4899 if (kmsg->free_iov)
4900 kfree(kmsg->free_iov);
Jens Axboe03b12302019-12-02 18:50:25 -07004901 req->flags &= ~REQ_F_NEED_CLEANUP;
Jens Axboe3c1a3d02022-04-20 19:21:36 -06004902 if (ret >= 0)
4903 ret += sr->done_io;
4904 else if (sr->done_io)
4905 ret = sr->done_io;
Pavel Begunkov889fca72021-02-10 00:03:09 +00004906 __io_req_complete(req, issue_flags, ret, 0);
Jens Axboefddafac2020-01-04 20:19:44 -07004907 return 0;
Jens Axboefddafac2020-01-04 20:19:44 -07004908}
4909
Pavel Begunkov889fca72021-02-10 00:03:09 +00004910static int io_send(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe03b12302019-12-02 18:50:25 -07004911{
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004912 struct io_sr_msg *sr = &req->sr_msg;
4913 struct msghdr msg;
4914 struct iovec iov;
Jens Axboe03b12302019-12-02 18:50:25 -07004915 struct socket *sock;
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004916 unsigned flags;
Stefan Metzmacher00312752021-03-20 20:33:36 +01004917 int min_ret = 0;
Jens Axboe03b12302019-12-02 18:50:25 -07004918 int ret;
4919
Florent Revestdba4a922020-12-04 12:36:04 +01004920 sock = sock_from_file(req->file);
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004921 if (unlikely(!sock))
Florent Revestdba4a922020-12-04 12:36:04 +01004922 return -ENOTSOCK;
Jens Axboe03b12302019-12-02 18:50:25 -07004923
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004924 ret = import_single_range(WRITE, sr->buf, sr->len, &iov, &msg.msg_iter);
4925 if (unlikely(ret))
Zheng Bin14db8412020-09-09 20:12:37 +08004926 return ret;
Jens Axboe03b12302019-12-02 18:50:25 -07004927
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004928 msg.msg_name = NULL;
4929 msg.msg_control = NULL;
4930 msg.msg_controllen = 0;
4931 msg.msg_namelen = 0;
Jens Axboe03b12302019-12-02 18:50:25 -07004932
Pavel Begunkov04411802021-04-01 15:44:00 +01004933 flags = req->sr_msg.msg_flags;
4934 if (issue_flags & IO_URING_F_NONBLOCK)
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004935 flags |= MSG_DONTWAIT;
Stefan Metzmacher00312752021-03-20 20:33:36 +01004936 if (flags & MSG_WAITALL)
4937 min_ret = iov_iter_count(&msg.msg_iter);
4938
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004939 msg.msg_flags = flags;
4940 ret = sock_sendmsg(sock, &msg);
Pavel Begunkovcdc68e72021-11-23 00:07:47 +00004941 if (ret < min_ret) {
4942 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))
4943 return -EAGAIN;
4944 if (ret == -ERESTARTSYS)
4945 ret = -EINTR;
Jens Axboe3c1a3d02022-04-20 19:21:36 -06004946 if (ret > 0 && io_net_retry(sock, flags)) {
4947 sr->len -= ret;
4948 sr->buf += ret;
4949 sr->done_io += ret;
4950 req->flags |= REQ_F_PARTIAL_IO;
4951 return -EAGAIN;
4952 }
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004953 req_set_fail(req);
Pavel Begunkovcdc68e72021-11-23 00:07:47 +00004954 }
Jens Axboe3c1a3d02022-04-20 19:21:36 -06004955 if (ret >= 0)
4956 ret += sr->done_io;
4957 else if (sr->done_io)
4958 ret = sr->done_io;
Pavel Begunkov889fca72021-02-10 00:03:09 +00004959 __io_req_complete(req, issue_flags, ret, 0);
Jens Axboe03b12302019-12-02 18:50:25 -07004960 return 0;
Jens Axboe03b12302019-12-02 18:50:25 -07004961}
4962
Pavel Begunkov1400e692020-07-12 20:41:05 +03004963static int __io_recvmsg_copy_hdr(struct io_kiocb *req,
4964 struct io_async_msghdr *iomsg)
Jens Axboe52de1fe2020-02-27 10:15:42 -07004965{
4966 struct io_sr_msg *sr = &req->sr_msg;
4967 struct iovec __user *uiov;
4968 size_t iov_len;
4969 int ret;
4970
Pavel Begunkov1400e692020-07-12 20:41:05 +03004971 ret = __copy_msghdr_from_user(&iomsg->msg, sr->umsg,
4972 &iomsg->uaddr, &uiov, &iov_len);
Jens Axboe52de1fe2020-02-27 10:15:42 -07004973 if (ret)
4974 return ret;
4975
4976 if (req->flags & REQ_F_BUFFER_SELECT) {
4977 if (iov_len > 1)
4978 return -EINVAL;
Pavel Begunkov5476dfe2021-02-05 00:57:59 +00004979 if (copy_from_user(iomsg->fast_iov, uiov, sizeof(*uiov)))
Jens Axboe52de1fe2020-02-27 10:15:42 -07004980 return -EFAULT;
Pavel Begunkov5476dfe2021-02-05 00:57:59 +00004981 sr->len = iomsg->fast_iov[0].iov_len;
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004982 iomsg->free_iov = NULL;
Jens Axboe52de1fe2020-02-27 10:15:42 -07004983 } else {
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004984 iomsg->free_iov = iomsg->fast_iov;
Christoph Hellwig89cd35c2020-09-25 06:51:41 +02004985 ret = __import_iovec(READ, uiov, iov_len, UIO_FASTIOV,
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004986 &iomsg->free_iov, &iomsg->msg.msg_iter,
Christoph Hellwig89cd35c2020-09-25 06:51:41 +02004987 false);
Jens Axboe52de1fe2020-02-27 10:15:42 -07004988 if (ret > 0)
4989 ret = 0;
4990 }
4991
4992 return ret;
4993}
4994
4995#ifdef CONFIG_COMPAT
4996static int __io_compat_recvmsg_copy_hdr(struct io_kiocb *req,
Pavel Begunkov1400e692020-07-12 20:41:05 +03004997 struct io_async_msghdr *iomsg)
Jens Axboe52de1fe2020-02-27 10:15:42 -07004998{
Jens Axboe52de1fe2020-02-27 10:15:42 -07004999 struct io_sr_msg *sr = &req->sr_msg;
5000 struct compat_iovec __user *uiov;
5001 compat_uptr_t ptr;
5002 compat_size_t len;
5003 int ret;
5004
Pavel Begunkov4af34172021-04-11 01:46:30 +01005005 ret = __get_compat_msghdr(&iomsg->msg, sr->umsg_compat, &iomsg->uaddr,
5006 &ptr, &len);
Jens Axboe52de1fe2020-02-27 10:15:42 -07005007 if (ret)
5008 return ret;
5009
5010 uiov = compat_ptr(ptr);
5011 if (req->flags & REQ_F_BUFFER_SELECT) {
5012 compat_ssize_t clen;
5013
5014 if (len > 1)
5015 return -EINVAL;
5016 if (!access_ok(uiov, sizeof(*uiov)))
5017 return -EFAULT;
5018 if (__get_user(clen, &uiov->iov_len))
5019 return -EFAULT;
5020 if (clen < 0)
5021 return -EINVAL;
Pavel Begunkov2d280bc2020-11-29 18:33:32 +00005022 sr->len = clen;
Pavel Begunkov257e84a2021-02-05 00:58:00 +00005023 iomsg->free_iov = NULL;
Jens Axboe52de1fe2020-02-27 10:15:42 -07005024 } else {
Pavel Begunkov257e84a2021-02-05 00:58:00 +00005025 iomsg->free_iov = iomsg->fast_iov;
Christoph Hellwig89cd35c2020-09-25 06:51:41 +02005026 ret = __import_iovec(READ, (struct iovec __user *)uiov, len,
Pavel Begunkov257e84a2021-02-05 00:58:00 +00005027 UIO_FASTIOV, &iomsg->free_iov,
Christoph Hellwig89cd35c2020-09-25 06:51:41 +02005028 &iomsg->msg.msg_iter, true);
Jens Axboe52de1fe2020-02-27 10:15:42 -07005029 if (ret < 0)
5030 return ret;
5031 }
5032
5033 return 0;
5034}
Jens Axboe03b12302019-12-02 18:50:25 -07005035#endif
Jens Axboe52de1fe2020-02-27 10:15:42 -07005036
Pavel Begunkov1400e692020-07-12 20:41:05 +03005037static int io_recvmsg_copy_hdr(struct io_kiocb *req,
5038 struct io_async_msghdr *iomsg)
Jens Axboe52de1fe2020-02-27 10:15:42 -07005039{
Pavel Begunkov1400e692020-07-12 20:41:05 +03005040 iomsg->msg.msg_name = &iomsg->addr;
Jens Axboe52de1fe2020-02-27 10:15:42 -07005041
5042#ifdef CONFIG_COMPAT
5043 if (req->ctx->compat)
Pavel Begunkov1400e692020-07-12 20:41:05 +03005044 return __io_compat_recvmsg_copy_hdr(req, iomsg);
Jens Axboe52de1fe2020-02-27 10:15:42 -07005045#endif
5046
Pavel Begunkov1400e692020-07-12 20:41:05 +03005047 return __io_recvmsg_copy_hdr(req, iomsg);
Jens Axboe52de1fe2020-02-27 10:15:42 -07005048}
5049
Jens Axboebcda7ba2020-02-23 16:42:51 -07005050static struct io_buffer *io_recv_buffer_select(struct io_kiocb *req,
Pavel Begunkov7fbb1b52020-07-16 23:28:05 +03005051 bool needs_lock)
Jens Axboebcda7ba2020-02-23 16:42:51 -07005052{
5053 struct io_sr_msg *sr = &req->sr_msg;
5054 struct io_buffer *kbuf;
5055
Jens Axboebcda7ba2020-02-23 16:42:51 -07005056 kbuf = io_buffer_select(req, &sr->len, sr->bgid, sr->kbuf, needs_lock);
5057 if (IS_ERR(kbuf))
5058 return kbuf;
5059
5060 sr->kbuf = kbuf;
5061 req->flags |= REQ_F_BUFFER_SELECTED;
Jens Axboebcda7ba2020-02-23 16:42:51 -07005062 return kbuf;
Jens Axboe03b12302019-12-02 18:50:25 -07005063}
5064
Pavel Begunkov7fbb1b52020-07-16 23:28:05 +03005065static inline unsigned int io_put_recv_kbuf(struct io_kiocb *req)
5066{
5067 return io_put_kbuf(req, req->sr_msg.kbuf);
5068}
5069
Pavel Begunkov93642ef2021-02-18 18:29:44 +00005070static int io_recvmsg_prep_async(struct io_kiocb *req)
Jens Axboe03b12302019-12-02 18:50:25 -07005071{
Pavel Begunkov99bc4c32020-02-07 22:04:45 +03005072 int ret;
Jens Axboe06b76d42019-12-19 14:44:26 -07005073
Pavel Begunkov93642ef2021-02-18 18:29:44 +00005074 ret = io_recvmsg_copy_hdr(req, req->async_data);
5075 if (!ret)
5076 req->flags |= REQ_F_NEED_CLEANUP;
5077 return ret;
5078}
5079
5080static int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
5081{
5082 struct io_sr_msg *sr = &req->sr_msg;
5083
Pavel Begunkovd2b6f482020-06-03 18:03:25 +03005084 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
5085 return -EINVAL;
Jens Axboe37811e42022-04-26 19:34:57 -06005086 if (unlikely(sqe->addr2 || sqe->file_index))
5087 return -EINVAL;
Jens Axboe50fefe52022-06-30 14:42:05 -06005088 if (unlikely(sqe->addr2 || sqe->file_index || sqe->ioprio))
5089 return -EINVAL;
Pavel Begunkovd2b6f482020-06-03 18:03:25 +03005090
Pavel Begunkov270a5942020-07-12 20:41:04 +03005091 sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
Jens Axboe0b7b21e2020-01-31 08:34:59 -07005092 sr->len = READ_ONCE(sqe->len);
Jens Axboebcda7ba2020-02-23 16:42:51 -07005093 sr->bgid = READ_ONCE(sqe->buf_group);
Pavel Begunkov04411802021-04-01 15:44:00 +01005094 sr->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL;
5095 if (sr->msg_flags & MSG_DONTWAIT)
5096 req->flags |= REQ_F_NOWAIT;
Jens Axboe3529d8c2019-12-19 18:24:38 -07005097
Jens Axboed8768362020-02-27 14:17:49 -07005098#ifdef CONFIG_COMPAT
5099 if (req->ctx->compat)
5100 sr->msg_flags |= MSG_CMSG_COMPAT;
5101#endif
Jens Axboe9b7b0f22023-01-21 10:21:22 -07005102 sr->done_io = 0;
Pavel Begunkov93642ef2021-02-18 18:29:44 +00005103 return 0;
Jens Axboe03b12302019-12-02 18:50:25 -07005104}
5105
Pavel Begunkov889fca72021-02-10 00:03:09 +00005106static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe03b12302019-12-02 18:50:25 -07005107{
Pavel Begunkov6b754c82020-07-16 23:28:00 +03005108 struct io_async_msghdr iomsg, *kmsg;
Jens Axboe9b7b0f22023-01-21 10:21:22 -07005109 struct io_sr_msg *sr = &req->sr_msg;
Jens Axboe0fa03c62019-04-19 13:34:07 -06005110 struct socket *sock;
Pavel Begunkov7fbb1b52020-07-16 23:28:05 +03005111 struct io_buffer *kbuf;
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03005112 unsigned flags;
Stefan Metzmacher00312752021-03-20 20:33:36 +01005113 int min_ret = 0;
Jens Axboe52de1fe2020-02-27 10:15:42 -07005114 int ret, cflags = 0;
Pavel Begunkov45d189c2021-02-10 00:03:07 +00005115 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
Jens Axboe0fa03c62019-04-19 13:34:07 -06005116
Florent Revestdba4a922020-12-04 12:36:04 +01005117 sock = sock_from_file(req->file);
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03005118 if (unlikely(!sock))
Florent Revestdba4a922020-12-04 12:36:04 +01005119 return -ENOTSOCK;
Jens Axboe0fa03c62019-04-19 13:34:07 -06005120
Pavel Begunkov257e84a2021-02-05 00:58:00 +00005121 kmsg = req->async_data;
5122 if (!kmsg) {
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03005123 ret = io_recvmsg_copy_hdr(req, &iomsg);
5124 if (ret)
Pavel Begunkov681fda82020-07-15 22:20:45 +03005125 return ret;
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03005126 kmsg = &iomsg;
Jens Axboe0fa03c62019-04-19 13:34:07 -06005127 }
5128
Pavel Begunkovbc02ef32020-07-16 23:28:03 +03005129 if (req->flags & REQ_F_BUFFER_SELECT) {
Pavel Begunkov7fbb1b52020-07-16 23:28:05 +03005130 kbuf = io_recv_buffer_select(req, !force_nonblock);
Pavel Begunkovbc02ef32020-07-16 23:28:03 +03005131 if (IS_ERR(kbuf))
5132 return PTR_ERR(kbuf);
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03005133 kmsg->fast_iov[0].iov_base = u64_to_user_ptr(kbuf->addr);
Pavel Begunkov5476dfe2021-02-05 00:57:59 +00005134 kmsg->fast_iov[0].iov_len = req->sr_msg.len;
5135 iov_iter_init(&kmsg->msg.msg_iter, READ, kmsg->fast_iov,
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03005136 1, req->sr_msg.len);
5137 }
5138
Pavel Begunkov04411802021-04-01 15:44:00 +01005139 flags = req->sr_msg.msg_flags;
5140 if (force_nonblock)
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03005141 flags |= MSG_DONTWAIT;
Stefan Metzmacher00312752021-03-20 20:33:36 +01005142 if (flags & MSG_WAITALL)
5143 min_ret = iov_iter_count(&kmsg->msg.msg_iter);
5144
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03005145 ret = __sys_recvmsg_sock(sock, &kmsg->msg, req->sr_msg.umsg,
5146 kmsg->uaddr, flags);
Pavel Begunkovcdc68e72021-11-23 00:07:47 +00005147 if (ret < min_ret) {
5148 if (ret == -EAGAIN && force_nonblock)
5149 return io_setup_async_msg(req, kmsg);
5150 if (ret == -ERESTARTSYS)
5151 ret = -EINTR;
Jens Axboe9b7b0f22023-01-21 10:21:22 -07005152 if (ret > 0 && io_net_retry(sock, flags)) {
5153 sr->done_io += ret;
Jens Axboe390b8812022-03-23 09:30:05 -06005154 req->flags |= REQ_F_PARTIAL_IO;
Jens Axboe9b7b0f22023-01-21 10:21:22 -07005155 return io_setup_async_msg(req, kmsg);
5156 }
Pavel Begunkovcdc68e72021-11-23 00:07:47 +00005157 req_set_fail(req);
5158 } else if ((flags & MSG_WAITALL) && (kmsg->msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) {
5159 req_set_fail(req);
5160 }
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03005161
Pavel Begunkov7fbb1b52020-07-16 23:28:05 +03005162 if (req->flags & REQ_F_BUFFER_SELECTED)
5163 cflags = io_put_recv_kbuf(req);
Pavel Begunkov257e84a2021-02-05 00:58:00 +00005164 /* fast path, check for non-NULL to avoid function call */
5165 if (kmsg->free_iov)
5166 kfree(kmsg->free_iov);
Pavel Begunkov99bc4c32020-02-07 22:04:45 +03005167 req->flags &= ~REQ_F_NEED_CLEANUP;
Jens Axboe9b7b0f22023-01-21 10:21:22 -07005168 if (ret >= 0)
5169 ret += sr->done_io;
5170 else if (sr->done_io)
5171 ret = sr->done_io;
Pavel Begunkov889fca72021-02-10 00:03:09 +00005172 __io_req_complete(req, issue_flags, ret, cflags);
Jens Axboe0fa03c62019-04-19 13:34:07 -06005173 return 0;
Jens Axboe0fa03c62019-04-19 13:34:07 -06005174}
5175
Pavel Begunkov889fca72021-02-10 00:03:09 +00005176static int io_recv(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboefddafac2020-01-04 20:19:44 -07005177{
Pavel Begunkov6b754c82020-07-16 23:28:00 +03005178 struct io_buffer *kbuf;
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03005179 struct io_sr_msg *sr = &req->sr_msg;
5180 struct msghdr msg;
5181 void __user *buf = sr->buf;
Jens Axboefddafac2020-01-04 20:19:44 -07005182 struct socket *sock;
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03005183 struct iovec iov;
5184 unsigned flags;
Stefan Metzmacher00312752021-03-20 20:33:36 +01005185 int min_ret = 0;
Jens Axboebcda7ba2020-02-23 16:42:51 -07005186 int ret, cflags = 0;
Pavel Begunkov45d189c2021-02-10 00:03:07 +00005187 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
Jens Axboefddafac2020-01-04 20:19:44 -07005188
Florent Revestdba4a922020-12-04 12:36:04 +01005189 sock = sock_from_file(req->file);
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03005190 if (unlikely(!sock))
Florent Revestdba4a922020-12-04 12:36:04 +01005191 return -ENOTSOCK;
Jens Axboefddafac2020-01-04 20:19:44 -07005192
Pavel Begunkovbc02ef32020-07-16 23:28:03 +03005193 if (req->flags & REQ_F_BUFFER_SELECT) {
Pavel Begunkov7fbb1b52020-07-16 23:28:05 +03005194 kbuf = io_recv_buffer_select(req, !force_nonblock);
Jens Axboebcda7ba2020-02-23 16:42:51 -07005195 if (IS_ERR(kbuf))
5196 return PTR_ERR(kbuf);
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03005197 buf = u64_to_user_ptr(kbuf->addr);
Jens Axboefddafac2020-01-04 20:19:44 -07005198 }
5199
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03005200 ret = import_single_range(READ, buf, sr->len, &iov, &msg.msg_iter);
Pavel Begunkov14c32ee2020-07-16 23:28:01 +03005201 if (unlikely(ret))
5202 goto out_free;
Jens Axboefddafac2020-01-04 20:19:44 -07005203
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03005204 msg.msg_name = NULL;
5205 msg.msg_control = NULL;
5206 msg.msg_controllen = 0;
5207 msg.msg_namelen = 0;
5208 msg.msg_iocb = NULL;
5209 msg.msg_flags = 0;
5210
Pavel Begunkov04411802021-04-01 15:44:00 +01005211 flags = req->sr_msg.msg_flags;
5212 if (force_nonblock)
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03005213 flags |= MSG_DONTWAIT;
Stefan Metzmacher00312752021-03-20 20:33:36 +01005214 if (flags & MSG_WAITALL)
5215 min_ret = iov_iter_count(&msg.msg_iter);
5216
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03005217 ret = sock_recvmsg(sock, &msg, flags);
Pavel Begunkov14c32ee2020-07-16 23:28:01 +03005218out_free:
Pavel Begunkovcdc68e72021-11-23 00:07:47 +00005219 if (ret < min_ret) {
5220 if (ret == -EAGAIN && force_nonblock)
5221 return -EAGAIN;
5222 if (ret == -ERESTARTSYS)
5223 ret = -EINTR;
Jens Axboe9b7b0f22023-01-21 10:21:22 -07005224 if (ret > 0 && io_net_retry(sock, flags)) {
5225 sr->len -= ret;
5226 sr->buf += ret;
5227 sr->done_io += ret;
Jens Axboe390b8812022-03-23 09:30:05 -06005228 req->flags |= REQ_F_PARTIAL_IO;
Jens Axboe9b7b0f22023-01-21 10:21:22 -07005229 return -EAGAIN;
5230 }
Pavel Begunkovcdc68e72021-11-23 00:07:47 +00005231 req_set_fail(req);
5232 } else if ((flags & MSG_WAITALL) && (msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) {
5233 req_set_fail(req);
5234 }
Pavel Begunkov7fbb1b52020-07-16 23:28:05 +03005235 if (req->flags & REQ_F_BUFFER_SELECTED)
5236 cflags = io_put_recv_kbuf(req);
Jens Axboe9b7b0f22023-01-21 10:21:22 -07005237 if (ret >= 0)
5238 ret += sr->done_io;
5239 else if (sr->done_io)
5240 ret = sr->done_io;
Pavel Begunkov889fca72021-02-10 00:03:09 +00005241 __io_req_complete(req, issue_flags, ret, cflags);
Jens Axboefddafac2020-01-04 20:19:44 -07005242 return 0;
Jens Axboefddafac2020-01-04 20:19:44 -07005243}
5244
Jens Axboe3529d8c2019-12-19 18:24:38 -07005245static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboe17f2fe32019-10-17 14:42:58 -06005246{
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07005247 struct io_accept *accept = &req->accept;
5248
Jens Axboe14587a462020-09-05 11:36:08 -06005249 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
Jens Axboe17f2fe32019-10-17 14:42:58 -06005250 return -EINVAL;
Pavel Begunkovaaa4db12021-08-25 12:25:47 +01005251 if (sqe->ioprio || sqe->len || sqe->buf_index)
Jens Axboe17f2fe32019-10-17 14:42:58 -06005252 return -EINVAL;
5253
Jens Axboed55e5f52019-12-11 16:12:15 -07005254 accept->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));
5255 accept->addr_len = u64_to_user_ptr(READ_ONCE(sqe->addr2));
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07005256 accept->flags = READ_ONCE(sqe->accept_flags);
Jens Axboe09952e32020-03-19 20:16:56 -06005257 accept->nofile = rlimit(RLIMIT_NOFILE);
Pavel Begunkova7083ad2021-08-25 12:25:46 +01005258
Pavel Begunkovaaa4db12021-08-25 12:25:47 +01005259 accept->file_slot = READ_ONCE(sqe->file_index);
Jens Axboe13239762022-03-14 17:26:19 -06005260 if (accept->file_slot && (accept->flags & SOCK_CLOEXEC))
Pavel Begunkovaaa4db12021-08-25 12:25:47 +01005261 return -EINVAL;
Pavel Begunkova7083ad2021-08-25 12:25:46 +01005262 if (accept->flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK))
5263 return -EINVAL;
5264 if (SOCK_NONBLOCK != O_NONBLOCK && (accept->flags & SOCK_NONBLOCK))
5265 accept->flags = (accept->flags & ~SOCK_NONBLOCK) | O_NONBLOCK;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07005266 return 0;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07005267}
Jens Axboe17f2fe32019-10-17 14:42:58 -06005268
Pavel Begunkov889fca72021-02-10 00:03:09 +00005269static int io_accept(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07005270{
5271 struct io_accept *accept = &req->accept;
Pavel Begunkov45d189c2021-02-10 00:03:07 +00005272 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
Pavel Begunkovac45abc2020-06-08 21:08:18 +03005273 unsigned int file_flags = force_nonblock ? O_NONBLOCK : 0;
Pavel Begunkovaaa4db12021-08-25 12:25:47 +01005274 bool fixed = !!accept->file_slot;
Pavel Begunkova7083ad2021-08-25 12:25:46 +01005275 struct file *file;
5276 int ret, fd;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07005277
Pavel Begunkovaaa4db12021-08-25 12:25:47 +01005278 if (!fixed) {
5279 fd = __get_unused_fd_flags(accept->flags, accept->nofile);
5280 if (unlikely(fd < 0))
5281 return fd;
5282 }
Pavel Begunkova7083ad2021-08-25 12:25:46 +01005283 file = do_accept(req->file, file_flags, accept->addr, accept->addr_len,
5284 accept->flags);
5285 if (IS_ERR(file)) {
Pavel Begunkovaaa4db12021-08-25 12:25:47 +01005286 if (!fixed)
5287 put_unused_fd(fd);
Pavel Begunkova7083ad2021-08-25 12:25:46 +01005288 ret = PTR_ERR(file);
Dylan Yudaken30b90682023-01-21 09:13:12 -07005289 /* safe to retry */
5290 req->flags |= REQ_F_PARTIAL_IO;
Pavel Begunkova7083ad2021-08-25 12:25:46 +01005291 if (ret == -EAGAIN && force_nonblock)
5292 return -EAGAIN;
Pavel Begunkovac45abc2020-06-08 21:08:18 +03005293 if (ret == -ERESTARTSYS)
5294 ret = -EINTR;
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01005295 req_set_fail(req);
Pavel Begunkovaaa4db12021-08-25 12:25:47 +01005296 } else if (!fixed) {
Pavel Begunkova7083ad2021-08-25 12:25:46 +01005297 fd_install(fd, file);
5298 ret = fd;
Pavel Begunkovaaa4db12021-08-25 12:25:47 +01005299 } else {
5300 ret = io_install_fixed_file(req, file, issue_flags,
5301 accept->file_slot - 1);
Pavel Begunkovac45abc2020-06-08 21:08:18 +03005302 }
Pavel Begunkov889fca72021-02-10 00:03:09 +00005303 __io_req_complete(req, issue_flags, ret, 0);
Jens Axboe17f2fe32019-10-17 14:42:58 -06005304 return 0;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07005305}
5306
Pavel Begunkov93642ef2021-02-18 18:29:44 +00005307static int io_connect_prep_async(struct io_kiocb *req)
5308{
5309 struct io_async_connect *io = req->async_data;
5310 struct io_connect *conn = &req->connect;
5311
5312 return move_addr_to_kernel(conn->addr, conn->addr_len, &io->address);
5313}
5314
Jens Axboe3529d8c2019-12-19 18:24:38 -07005315static int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboef499a022019-12-02 16:28:46 -07005316{
Jens Axboe3529d8c2019-12-19 18:24:38 -07005317 struct io_connect *conn = &req->connect;
Jens Axboef499a022019-12-02 16:28:46 -07005318
Jens Axboe14587a462020-09-05 11:36:08 -06005319 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
Jens Axboe3fbb51c2019-12-20 08:51:52 -07005320 return -EINVAL;
Pavel Begunkov26578cd2021-08-20 10:36:37 +01005321 if (sqe->ioprio || sqe->len || sqe->buf_index || sqe->rw_flags ||
5322 sqe->splice_fd_in)
Jens Axboe3fbb51c2019-12-20 08:51:52 -07005323 return -EINVAL;
5324
Jens Axboe3529d8c2019-12-19 18:24:38 -07005325 conn->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));
5326 conn->addr_len = READ_ONCE(sqe->addr2);
Pavel Begunkov93642ef2021-02-18 18:29:44 +00005327 return 0;
Jens Axboef499a022019-12-02 16:28:46 -07005328}
5329
Pavel Begunkov889fca72021-02-10 00:03:09 +00005330static int io_connect(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboef8e85cf2019-11-23 14:24:24 -07005331{
Jens Axboee8c2bc12020-08-15 18:44:09 -07005332 struct io_async_connect __io, *io;
Jens Axboef8e85cf2019-11-23 14:24:24 -07005333 unsigned file_flags;
Jens Axboe3fbb51c2019-12-20 08:51:52 -07005334 int ret;
Pavel Begunkov45d189c2021-02-10 00:03:07 +00005335 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
Jens Axboef8e85cf2019-11-23 14:24:24 -07005336
Jens Axboee8c2bc12020-08-15 18:44:09 -07005337 if (req->async_data) {
5338 io = req->async_data;
Jens Axboef499a022019-12-02 16:28:46 -07005339 } else {
Jens Axboe3529d8c2019-12-19 18:24:38 -07005340 ret = move_addr_to_kernel(req->connect.addr,
5341 req->connect.addr_len,
Jens Axboee8c2bc12020-08-15 18:44:09 -07005342 &__io.address);
Jens Axboef499a022019-12-02 16:28:46 -07005343 if (ret)
5344 goto out;
5345 io = &__io;
5346 }
5347
Jens Axboe3fbb51c2019-12-20 08:51:52 -07005348 file_flags = force_nonblock ? O_NONBLOCK : 0;
5349
Jens Axboee8c2bc12020-08-15 18:44:09 -07005350 ret = __sys_connect_file(req->file, &io->address,
Jens Axboe3fbb51c2019-12-20 08:51:52 -07005351 req->connect.addr_len, file_flags);
Jens Axboe87f80d62019-12-03 11:23:54 -07005352 if ((ret == -EAGAIN || ret == -EINPROGRESS) && force_nonblock) {
Jens Axboee8c2bc12020-08-15 18:44:09 -07005353 if (req->async_data)
Jens Axboeb7bb4f72019-12-15 22:13:43 -07005354 return -EAGAIN;
Jens Axboee8c2bc12020-08-15 18:44:09 -07005355 if (io_alloc_async_data(req)) {
Jens Axboef499a022019-12-02 16:28:46 -07005356 ret = -ENOMEM;
5357 goto out;
5358 }
Jens Axboee8c2bc12020-08-15 18:44:09 -07005359 memcpy(req->async_data, &__io, sizeof(__io));
Jens Axboef8e85cf2019-11-23 14:24:24 -07005360 return -EAGAIN;
Jens Axboef499a022019-12-02 16:28:46 -07005361 }
Jens Axboef8e85cf2019-11-23 14:24:24 -07005362 if (ret == -ERESTARTSYS)
5363 ret = -EINTR;
Jens Axboef499a022019-12-02 16:28:46 -07005364out:
Jens Axboe4e88d6e2019-12-07 20:59:47 -07005365 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01005366 req_set_fail(req);
Pavel Begunkov889fca72021-02-10 00:03:09 +00005367 __io_req_complete(req, issue_flags, ret, 0);
Jens Axboef8e85cf2019-11-23 14:24:24 -07005368 return 0;
Jens Axboef8e85cf2019-11-23 14:24:24 -07005369}
YueHaibing469956e2020-03-04 15:53:52 +08005370#else /* !CONFIG_NET */
Jens Axboe99a10082021-02-19 09:35:19 -07005371#define IO_NETOP_FN(op) \
5372static int io_##op(struct io_kiocb *req, unsigned int issue_flags) \
5373{ \
5374 return -EOPNOTSUPP; \
Jens Axboef8e85cf2019-11-23 14:24:24 -07005375}
5376
Jens Axboe99a10082021-02-19 09:35:19 -07005377#define IO_NETOP_PREP(op) \
5378IO_NETOP_FN(op) \
5379static int io_##op##_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) \
5380{ \
5381 return -EOPNOTSUPP; \
5382} \
5383
5384#define IO_NETOP_PREP_ASYNC(op) \
5385IO_NETOP_PREP(op) \
5386static int io_##op##_prep_async(struct io_kiocb *req) \
5387{ \
5388 return -EOPNOTSUPP; \
YueHaibing469956e2020-03-04 15:53:52 +08005389}
5390
Jens Axboe99a10082021-02-19 09:35:19 -07005391IO_NETOP_PREP_ASYNC(sendmsg);
5392IO_NETOP_PREP_ASYNC(recvmsg);
5393IO_NETOP_PREP_ASYNC(connect);
5394IO_NETOP_PREP(accept);
5395IO_NETOP_FN(send);
5396IO_NETOP_FN(recv);
YueHaibing469956e2020-03-04 15:53:52 +08005397#endif /* CONFIG_NET */
Jens Axboe17f2fe32019-10-17 14:42:58 -06005398
Jens Axboed7718a92020-02-14 22:23:12 -07005399struct io_poll_table {
5400 struct poll_table_struct pt;
5401 struct io_kiocb *req;
Pavel Begunkov68b11e82021-07-20 10:50:43 +01005402 int nr_entries;
Jens Axboed7718a92020-02-14 22:23:12 -07005403 int error;
5404};
5405
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005406#define IO_POLL_CANCEL_FLAG BIT(31)
Pavel Begunkov4b702b72022-12-02 14:27:14 +00005407#define IO_POLL_RETRY_FLAG BIT(30)
5408#define IO_POLL_REF_MASK GENMASK(29, 0)
5409
5410/*
5411 * We usually have 1-2 refs taken, 128 is more than enough and we want to
5412 * maximise the margin between this amount and the moment when it overflows.
5413 */
5414#define IO_POLL_REF_BIAS 128
5415
5416static bool io_poll_get_ownership_slowpath(struct io_kiocb *req)
5417{
5418 int v;
5419
5420 /*
5421 * poll_refs are already elevated and we don't have much hope for
5422 * grabbing the ownership. Instead of incrementing set a retry flag
5423 * to notify the loop that there might have been some change.
5424 */
5425 v = atomic_fetch_or(IO_POLL_RETRY_FLAG, &req->poll_refs);
5426 if (v & IO_POLL_REF_MASK)
5427 return false;
5428 return !(atomic_fetch_inc(&req->poll_refs) & IO_POLL_REF_MASK);
5429}
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005430
5431/*
5432 * If refs part of ->poll_refs (see IO_POLL_REF_MASK) is 0, it's free. We can
5433 * bump it and acquire ownership. It's disallowed to modify requests while not
5434 * owning it, that prevents from races for enqueueing task_work's and b/w
5435 * arming poll and wakeups.
5436 */
5437static inline bool io_poll_get_ownership(struct io_kiocb *req)
5438{
Pavel Begunkov4b702b72022-12-02 14:27:14 +00005439 if (unlikely(atomic_read(&req->poll_refs) >= IO_POLL_REF_BIAS))
5440 return io_poll_get_ownership_slowpath(req);
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005441 return !(atomic_fetch_inc(&req->poll_refs) & IO_POLL_REF_MASK);
5442}
5443
5444static void io_poll_mark_cancelled(struct io_kiocb *req)
5445{
5446 atomic_or(IO_POLL_CANCEL_FLAG, &req->poll_refs);
5447}
5448
Pavel Begunkova85d7ac2022-08-29 14:30:15 +01005449static struct io_poll_iocb *io_poll_get_double(struct io_kiocb *req)
5450{
5451 /* pure poll stashes this in ->async_data, poll driven retry elsewhere */
5452 if (req->opcode == IORING_OP_POLL_ADD)
5453 return req->async_data;
5454 return req->apoll->double_poll;
5455}
5456
5457static struct io_poll_iocb *io_poll_get_single(struct io_kiocb *req)
5458{
5459 if (req->opcode == IORING_OP_POLL_ADD)
5460 return &req->poll;
5461 return &req->apoll->poll;
5462}
5463
5464static void io_poll_req_insert(struct io_kiocb *req)
5465{
5466 struct io_ring_ctx *ctx = req->ctx;
5467 struct hlist_head *list;
5468
5469 list = &ctx->cancel_hash[hash_long(req->user_data, ctx->cancel_hash_bits)];
5470 hlist_add_head(&req->hash_node, list);
5471}
5472
5473static void io_init_poll_iocb(struct io_poll_iocb *poll, __poll_t events,
5474 wait_queue_func_t wake_func)
5475{
5476 poll->head = NULL;
Pavel Begunkova85d7ac2022-08-29 14:30:15 +01005477#define IO_POLL_UNMASK (EPOLLERR|EPOLLHUP|EPOLLNVAL|EPOLLRDHUP)
5478 /* mask in events that we always want/need */
5479 poll->events = events | IO_POLL_UNMASK;
5480 INIT_LIST_HEAD(&poll->wait.entry);
5481 init_waitqueue_func_entry(&poll->wait, wake_func);
5482}
5483
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005484static inline void io_poll_remove_entry(struct io_poll_iocb *poll)
Jens Axboed7718a92020-02-14 22:23:12 -07005485{
Pavel Begunkove9d7ca02022-08-29 14:30:24 +01005486 struct wait_queue_head *head = smp_load_acquire(&poll->head);
Jens Axboed7718a92020-02-14 22:23:12 -07005487
Pavel Begunkove9d7ca02022-08-29 14:30:24 +01005488 if (head) {
5489 spin_lock_irq(&head->lock);
5490 list_del_init(&poll->wait.entry);
5491 poll->head = NULL;
5492 spin_unlock_irq(&head->lock);
5493 }
Jens Axboed7718a92020-02-14 22:23:12 -07005494}
5495
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005496static void io_poll_remove_entries(struct io_kiocb *req)
5497{
5498 struct io_poll_iocb *poll = io_poll_get_single(req);
5499 struct io_poll_iocb *poll_double = io_poll_get_double(req);
5500
Pavel Begunkove9d7ca02022-08-29 14:30:24 +01005501 /*
5502 * While we hold the waitqueue lock and the waitqueue is nonempty,
5503 * wake_up_pollfree() will wait for us. However, taking the waitqueue
5504 * lock in the first place can race with the waitqueue being freed.
5505 *
5506 * We solve this as eventpoll does: by taking advantage of the fact that
5507 * all users of wake_up_pollfree() will RCU-delay the actual free. If
5508 * we enter rcu_read_lock() and see that the pointer to the queue is
5509 * non-NULL, we can then lock it without the memory being freed out from
5510 * under us.
5511 *
5512 * Keep holding rcu_read_lock() as long as we hold the queue lock, in
5513 * case the caller deletes the entry from the queue, leaving it empty.
5514 * In that case, only RCU prevents the queue memory from being freed.
5515 */
5516 rcu_read_lock();
5517 io_poll_remove_entry(poll);
5518 if (poll_double)
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005519 io_poll_remove_entry(poll_double);
Pavel Begunkove9d7ca02022-08-29 14:30:24 +01005520 rcu_read_unlock();
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005521}
5522
5523/*
5524 * All poll tw should go through this. Checks for poll events, manages
5525 * references, does rewait, etc.
5526 *
5527 * Returns a negative error on failure. >0 when no action require, which is
5528 * either spurious wakeup or multishot CQE is served. 0 when it's done with
5529 * the request, then the mask is stored in req->result.
5530 */
5531static int io_poll_check_events(struct io_kiocb *req)
Jens Axboe74ce6ce2020-04-13 11:09:12 -06005532{
5533 struct io_ring_ctx *ctx = req->ctx;
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005534 struct io_poll_iocb *poll = io_poll_get_single(req);
5535 int v;
Jens Axboe74ce6ce2020-04-13 11:09:12 -06005536
Jens Axboe316319e2021-08-19 09:41:42 -06005537 /* req->task == current here, checking PF_EXITING is safe */
Pavel Begunkove09ee512021-07-01 13:26:05 +01005538 if (unlikely(req->task->flags & PF_EXITING))
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005539 io_poll_mark_cancelled(req);
Pavel Begunkove09ee512021-07-01 13:26:05 +01005540
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005541 do {
5542 v = atomic_read(&req->poll_refs);
Jens Axboe74ce6ce2020-04-13 11:09:12 -06005543
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005544 /* tw handler should be the owner, and so have some references */
5545 if (WARN_ON_ONCE(!(v & IO_POLL_REF_MASK)))
5546 return 0;
5547 if (v & IO_POLL_CANCEL_FLAG)
5548 return -ECANCELED;
Pavel Begunkovcd1981a2022-12-02 14:27:12 +00005549 /*
5550 * cqe.res contains only events of the first wake up
5551 * and all others are be lost. Redo vfs_poll() to get
5552 * up to date state.
5553 */
5554 if ((v & IO_POLL_REF_MASK) != 1)
5555 req->result = 0;
Pavel Begunkov4b702b72022-12-02 14:27:14 +00005556 if (v & IO_POLL_RETRY_FLAG) {
5557 req->result = 0;
5558 /*
5559 * We won't find new events that came in between
5560 * vfs_poll and the ref put unless we clear the
5561 * flag in advance.
5562 */
5563 atomic_andnot(IO_POLL_RETRY_FLAG, &req->poll_refs);
5564 v &= ~IO_POLL_RETRY_FLAG;
5565 }
Jens Axboe74ce6ce2020-04-13 11:09:12 -06005566
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005567 if (!req->result) {
5568 struct poll_table_struct pt = { ._key = poll->events };
Jens Axboe74ce6ce2020-04-13 11:09:12 -06005569
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005570 req->result = vfs_poll(req->file, &pt) & poll->events;
5571 }
Jens Axboe74ce6ce2020-04-13 11:09:12 -06005572
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005573 /* multishot, just fill an CQE and proceed */
5574 if (req->result && !(poll->events & EPOLLONESHOT)) {
5575 __poll_t mask = mangle_poll(req->result & poll->events);
5576 bool filled;
Jens Axboe18bceab2020-05-15 11:56:54 -06005577
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005578 spin_lock(&ctx->completion_lock);
5579 filled = io_fill_cqe_aux(ctx, req->user_data, mask,
5580 IORING_CQE_F_MORE);
5581 io_commit_cqring(ctx);
5582 spin_unlock(&ctx->completion_lock);
5583 if (unlikely(!filled))
5584 return -ECANCELED;
5585 io_cqring_ev_posted(ctx);
5586 } else if (req->result) {
5587 return 0;
5588 }
Jens Axboe18bceab2020-05-15 11:56:54 -06005589
Pavel Begunkov62321dc2022-12-02 14:27:11 +00005590 /* force the next iteration to vfs_poll() */
5591 req->result = 0;
5592
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005593 /*
5594 * Release all references, retry if someone tried to restart
5595 * task_work while we were executing it.
5596 */
Lin Madf4b1772022-12-02 14:27:15 +00005597 } while (atomic_sub_return(v & IO_POLL_REF_MASK, &req->poll_refs) &
5598 IO_POLL_REF_MASK);
Jens Axboe18bceab2020-05-15 11:56:54 -06005599
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005600 return 1;
Jens Axboe18bceab2020-05-15 11:56:54 -06005601}
5602
Pavel Begunkovf237c302021-08-18 12:42:46 +01005603static void io_poll_task_func(struct io_kiocb *req, bool *locked)
Jens Axboe18bceab2020-05-15 11:56:54 -06005604{
Jens Axboe6d816e02020-08-11 08:04:14 -06005605 struct io_ring_ctx *ctx = req->ctx;
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005606 int ret;
Jens Axboe18bceab2020-05-15 11:56:54 -06005607
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005608 ret = io_poll_check_events(req);
5609 if (ret > 0)
5610 return;
5611
5612 if (!ret) {
5613 req->result = mangle_poll(req->result & req->poll.events);
Pavel Begunkovdd221f462020-10-18 10:17:42 +01005614 } else {
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005615 req->result = ret;
5616 req_set_fail(req);
Pavel Begunkovdd221f462020-10-18 10:17:42 +01005617 }
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005618
5619 io_poll_remove_entries(req);
5620 spin_lock(&ctx->completion_lock);
5621 hash_del(&req->hash_node);
5622 spin_unlock(&ctx->completion_lock);
5623 io_req_complete_post(req, req->result, 0);
Jens Axboe18bceab2020-05-15 11:56:54 -06005624}
5625
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005626static void io_apoll_task_func(struct io_kiocb *req, bool *locked)
5627{
5628 struct io_ring_ctx *ctx = req->ctx;
5629 int ret;
5630
5631 ret = io_poll_check_events(req);
5632 if (ret > 0)
5633 return;
5634
5635 io_poll_remove_entries(req);
5636 spin_lock(&ctx->completion_lock);
5637 hash_del(&req->hash_node);
5638 spin_unlock(&ctx->completion_lock);
5639
5640 if (!ret)
5641 io_req_task_submit(req, locked);
5642 else
5643 io_req_complete_failed(req, ret);
5644}
5645
5646static void __io_poll_execute(struct io_kiocb *req, int mask)
5647{
5648 req->result = mask;
5649 if (req->opcode == IORING_OP_POLL_ADD)
5650 req->io_task_work.func = io_poll_task_func;
5651 else
5652 req->io_task_work.func = io_apoll_task_func;
5653
5654 trace_io_uring_task_add(req->ctx, req->opcode, req->user_data, mask);
5655 io_req_task_work_add(req);
5656}
5657
5658static inline void io_poll_execute(struct io_kiocb *req, int res)
5659{
5660 if (io_poll_get_ownership(req))
5661 __io_poll_execute(req, res);
5662}
5663
5664static void io_poll_cancel_req(struct io_kiocb *req)
5665{
5666 io_poll_mark_cancelled(req);
5667 /* kick tw, which should complete the request */
5668 io_poll_execute(req, 0);
5669}
5670
5671static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
5672 void *key)
Jens Axboe18bceab2020-05-15 11:56:54 -06005673{
5674 struct io_kiocb *req = wait->private;
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005675 struct io_poll_iocb *poll = container_of(wait, struct io_poll_iocb,
5676 wait);
Jens Axboe18bceab2020-05-15 11:56:54 -06005677 __poll_t mask = key_to_poll(key);
5678
Pavel Begunkove9d7ca02022-08-29 14:30:24 +01005679 if (unlikely(mask & POLLFREE)) {
5680 io_poll_mark_cancelled(req);
5681 /* we have to kick tw in case it's not already */
5682 io_poll_execute(req, 0);
5683
5684 /*
5685 * If the waitqueue is being freed early but someone is already
5686 * holds ownership over it, we have to tear down the request as
5687 * best we can. That means immediately removing the request from
5688 * its waitqueue and preventing all further accesses to the
5689 * waitqueue via the request.
5690 */
5691 list_del_init(&poll->wait.entry);
5692
5693 /*
5694 * Careful: this *must* be the last step, since as soon
5695 * as req->head is NULL'ed out, the request can be
5696 * completed and freed, since aio_poll_complete_work()
5697 * will no longer need to take the waitqueue lock.
5698 */
5699 smp_store_release(&poll->head, NULL);
5700 return 1;
5701 }
5702
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005703 /* for instances that support it check for an event match first */
Jens Axboe18bceab2020-05-15 11:56:54 -06005704 if (mask && !(mask & poll->events))
5705 return 0;
5706
Jens Axboeccf06b52022-12-23 07:04:49 -07005707 if (io_poll_get_ownership(req)) {
5708 /*
5709 * If we trigger a multishot poll off our own wakeup path,
5710 * disable multishot as there is a circular dependency between
5711 * CQ posting and triggering the event.
5712 */
5713 if (mask & EPOLL_URING_WAKE)
5714 poll->events |= EPOLLONESHOT;
5715
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005716 __io_poll_execute(req, mask);
Jens Axboeccf06b52022-12-23 07:04:49 -07005717 }
Jens Axboe18bceab2020-05-15 11:56:54 -06005718 return 1;
5719}
5720
Jens Axboe18bceab2020-05-15 11:56:54 -06005721static void __io_queue_proc(struct io_poll_iocb *poll, struct io_poll_table *pt,
Jens Axboe807abcb2020-07-17 17:09:27 -06005722 struct wait_queue_head *head,
5723 struct io_poll_iocb **poll_ptr)
Jens Axboe18bceab2020-05-15 11:56:54 -06005724{
5725 struct io_kiocb *req = pt->req;
5726
5727 /*
Pavel Begunkov68b11e82021-07-20 10:50:43 +01005728 * The file being polled uses multiple waitqueues for poll handling
5729 * (e.g. one for read, one for write). Setup a separate io_poll_iocb
5730 * if this happens.
Jens Axboe18bceab2020-05-15 11:56:54 -06005731 */
Pavel Begunkov68b11e82021-07-20 10:50:43 +01005732 if (unlikely(pt->nr_entries)) {
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005733 struct io_poll_iocb *first = poll;
Pavel Begunkov58852d42020-10-16 20:55:56 +01005734
Pavel Begunkov23a65db2021-08-17 20:28:11 +01005735 /* double add on the same waitqueue head, ignore */
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005736 if (first->head == head)
Pavel Begunkov23a65db2021-08-17 20:28:11 +01005737 return;
Jens Axboe18bceab2020-05-15 11:56:54 -06005738 /* already have a 2nd entry, fail a third attempt */
Jens Axboe807abcb2020-07-17 17:09:27 -06005739 if (*poll_ptr) {
Pavel Begunkov23a65db2021-08-17 20:28:11 +01005740 if ((*poll_ptr)->head == head)
5741 return;
Jens Axboe18bceab2020-05-15 11:56:54 -06005742 pt->error = -EINVAL;
5743 return;
5744 }
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005745
Jens Axboe18bceab2020-05-15 11:56:54 -06005746 poll = kmalloc(sizeof(*poll), GFP_ATOMIC);
5747 if (!poll) {
5748 pt->error = -ENOMEM;
5749 return;
5750 }
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005751 io_init_poll_iocb(poll, first->events, first->wait.func);
Jens Axboe807abcb2020-07-17 17:09:27 -06005752 *poll_ptr = poll;
Jens Axboe18bceab2020-05-15 11:56:54 -06005753 }
5754
Pavel Begunkov68b11e82021-07-20 10:50:43 +01005755 pt->nr_entries++;
Jens Axboe18bceab2020-05-15 11:56:54 -06005756 poll->head = head;
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005757 poll->wait.private = req;
Jiufei Xuea31eb4a2020-06-17 17:53:56 +08005758
5759 if (poll->events & EPOLLEXCLUSIVE)
5760 add_wait_queue_exclusive(head, &poll->wait);
5761 else
5762 add_wait_queue(head, &poll->wait);
Jens Axboe18bceab2020-05-15 11:56:54 -06005763}
5764
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005765static void io_poll_queue_proc(struct file *file, struct wait_queue_head *head,
5766 struct poll_table_struct *p)
5767{
5768 struct io_poll_table *pt = container_of(p, struct io_poll_table, pt);
5769
5770 __io_queue_proc(&pt->req->poll, pt, head,
5771 (struct io_poll_iocb **) &pt->req->async_data);
5772}
5773
5774static int __io_arm_poll_handler(struct io_kiocb *req,
5775 struct io_poll_iocb *poll,
5776 struct io_poll_table *ipt, __poll_t mask)
5777{
5778 struct io_ring_ctx *ctx = req->ctx;
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005779
5780 INIT_HLIST_NODE(&req->hash_node);
5781 io_init_poll_iocb(poll, mask, io_poll_wake);
5782 poll->file = req->file;
5783 poll->wait.private = req;
5784
5785 ipt->pt._key = mask;
5786 ipt->req = req;
5787 ipt->error = 0;
5788 ipt->nr_entries = 0;
5789
5790 /*
5791 * Take the ownership to delay any tw execution up until we're done
5792 * with poll arming. see io_poll_get_ownership().
5793 */
5794 atomic_set(&req->poll_refs, 1);
5795 mask = vfs_poll(req->file, &ipt->pt) & poll->events;
5796
5797 if (mask && (poll->events & EPOLLONESHOT)) {
5798 io_poll_remove_entries(req);
5799 /* no one else has access to the req, forget about the ref */
5800 return mask;
5801 }
5802 if (!mask && unlikely(ipt->error || !ipt->nr_entries)) {
5803 io_poll_remove_entries(req);
5804 if (!ipt->error)
5805 ipt->error = -EINVAL;
5806 return 0;
5807 }
5808
5809 spin_lock(&ctx->completion_lock);
5810 io_poll_req_insert(req);
5811 spin_unlock(&ctx->completion_lock);
5812
5813 if (mask) {
5814 /* can't multishot if failed, just queue the event we've got */
Pavel Begunkov182dc3a2022-08-29 14:30:23 +01005815 if (unlikely(ipt->error || !ipt->nr_entries)) {
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005816 poll->events |= EPOLLONESHOT;
Pavel Begunkov182dc3a2022-08-29 14:30:23 +01005817 ipt->error = 0;
5818 }
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005819 __io_poll_execute(req, mask);
5820 return 0;
5821 }
5822
5823 /*
Pavel Begunkov1d588492022-12-02 14:27:13 +00005824 * Try to release ownership. If we see a change of state, e.g.
5825 * poll was waken up, queue up a tw, it'll deal with it.
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005826 */
Pavel Begunkov1d588492022-12-02 14:27:13 +00005827 if (atomic_cmpxchg(&req->poll_refs, 1, 0) != 1)
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005828 __io_poll_execute(req, 0);
5829 return 0;
5830}
5831
Jens Axboe18bceab2020-05-15 11:56:54 -06005832static void io_async_queue_proc(struct file *file, struct wait_queue_head *head,
5833 struct poll_table_struct *p)
5834{
5835 struct io_poll_table *pt = container_of(p, struct io_poll_table, pt);
Jens Axboe807abcb2020-07-17 17:09:27 -06005836 struct async_poll *apoll = pt->req->apoll;
Jens Axboe18bceab2020-05-15 11:56:54 -06005837
Jens Axboe807abcb2020-07-17 17:09:27 -06005838 __io_queue_proc(&apoll->poll, pt, head, &apoll->double_poll);
Jens Axboe18bceab2020-05-15 11:56:54 -06005839}
5840
Olivier Langlois59b735a2021-06-22 05:17:39 -07005841enum {
5842 IO_APOLL_OK,
5843 IO_APOLL_ABORTED,
5844 IO_APOLL_READY
5845};
5846
5847static int io_arm_poll_handler(struct io_kiocb *req)
Jens Axboed7718a92020-02-14 22:23:12 -07005848{
5849 const struct io_op_def *def = &io_op_defs[req->opcode];
5850 struct io_ring_ctx *ctx = req->ctx;
5851 struct async_poll *apoll;
5852 struct io_poll_table ipt;
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005853 __poll_t mask = EPOLLONESHOT | POLLERR | POLLPRI;
5854 int ret;
Jens Axboed7718a92020-02-14 22:23:12 -07005855
5856 if (!req->file || !file_can_poll(req->file))
Olivier Langlois59b735a2021-06-22 05:17:39 -07005857 return IO_APOLL_ABORTED;
Jens Axboea79b13f2023-01-21 10:39:22 -07005858 if ((req->flags & (REQ_F_POLLED|REQ_F_PARTIAL_IO)) == REQ_F_POLLED)
Olivier Langlois59b735a2021-06-22 05:17:39 -07005859 return IO_APOLL_ABORTED;
Pavel Begunkovb2d9c3d2021-06-26 21:40:44 +01005860 if (!def->pollin && !def->pollout)
Olivier Langlois59b735a2021-06-22 05:17:39 -07005861 return IO_APOLL_ABORTED;
Pavel Begunkovb2d9c3d2021-06-26 21:40:44 +01005862
5863 if (def->pollin) {
Pavel Begunkovb2d9c3d2021-06-26 21:40:44 +01005864 mask |= POLLIN | POLLRDNORM;
5865
5866 /* If reading from MSG_ERRQUEUE using recvmsg, ignore POLLIN */
5867 if ((req->opcode == IORING_OP_RECVMSG) &&
5868 (req->sr_msg.msg_flags & MSG_ERRQUEUE))
5869 mask &= ~POLLIN;
5870 } else {
Pavel Begunkovb2d9c3d2021-06-26 21:40:44 +01005871 mask |= POLLOUT | POLLWRNORM;
5872 }
5873
Jens Axboea79b13f2023-01-21 10:39:22 -07005874 if (req->flags & REQ_F_POLLED)
5875 apoll = req->apoll;
5876 else
5877 apoll = kmalloc(sizeof(*apoll), GFP_ATOMIC);
Jens Axboed7718a92020-02-14 22:23:12 -07005878 if (unlikely(!apoll))
Olivier Langlois59b735a2021-06-22 05:17:39 -07005879 return IO_APOLL_ABORTED;
Jens Axboe807abcb2020-07-17 17:09:27 -06005880 apoll->double_poll = NULL;
Jens Axboed7718a92020-02-14 22:23:12 -07005881 req->apoll = apoll;
Pavel Begunkovb2d9c3d2021-06-26 21:40:44 +01005882 req->flags |= REQ_F_POLLED;
Jens Axboed7718a92020-02-14 22:23:12 -07005883 ipt.pt._qproc = io_async_queue_proc;
5884
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005885 ret = __io_arm_poll_handler(req, &apoll->poll, &ipt, mask);
Hao Xu41a51692021-08-12 15:47:02 +08005886 if (ret || ipt.error)
5887 return ret ? IO_APOLL_READY : IO_APOLL_ABORTED;
5888
Olivier Langlois236daeae2021-05-31 02:36:37 -04005889 trace_io_uring_poll_arm(ctx, req, req->opcode, req->user_data,
5890 mask, apoll->poll.events);
Olivier Langlois59b735a2021-06-22 05:17:39 -07005891 return IO_APOLL_OK;
Jens Axboed7718a92020-02-14 22:23:12 -07005892}
5893
Jens Axboe76e1b642020-09-26 15:05:03 -06005894/*
5895 * Returns true if we found and killed one or more poll requests
5896 */
Pavel Begunkov6b819282020-11-06 13:00:25 +00005897static bool io_poll_remove_all(struct io_ring_ctx *ctx, struct task_struct *tsk,
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01005898 bool cancel_all)
Jens Axboe221c5eb2019-01-17 09:41:58 -07005899{
Jens Axboe78076bb2019-12-04 19:56:40 -07005900 struct hlist_node *tmp;
Jens Axboe221c5eb2019-01-17 09:41:58 -07005901 struct io_kiocb *req;
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005902 bool found = false;
5903 int i;
Jens Axboe221c5eb2019-01-17 09:41:58 -07005904
Jens Axboe79ebeae2021-08-10 15:18:27 -06005905 spin_lock(&ctx->completion_lock);
Jens Axboe78076bb2019-12-04 19:56:40 -07005906 for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) {
5907 struct hlist_head *list;
5908
5909 list = &ctx->cancel_hash[i];
Jens Axboef3606e32020-09-22 08:18:24 -06005910 hlist_for_each_entry_safe(req, tmp, list, hash_node) {
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005911 if (io_match_task_safe(req, tsk, cancel_all)) {
Jens Axboe7524ec52022-08-29 14:30:20 +01005912 hlist_del_init(&req->hash_node);
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005913 io_poll_cancel_req(req);
5914 found = true;
5915 }
Jens Axboef3606e32020-09-22 08:18:24 -06005916 }
Jens Axboe221c5eb2019-01-17 09:41:58 -07005917 }
Jens Axboe79ebeae2021-08-10 15:18:27 -06005918 spin_unlock(&ctx->completion_lock);
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005919 return found;
Jens Axboe221c5eb2019-01-17 09:41:58 -07005920}
5921
Pavel Begunkov9ba5fac2021-04-14 13:38:35 +01005922static struct io_kiocb *io_poll_find(struct io_ring_ctx *ctx, __u64 sqe_addr,
5923 bool poll_only)
Pavel Begunkove07785b2021-04-01 15:43:57 +01005924 __must_hold(&ctx->completion_lock)
Jens Axboe47f46762019-11-09 17:43:02 -07005925{
Jens Axboe78076bb2019-12-04 19:56:40 -07005926 struct hlist_head *list;
Jens Axboe47f46762019-11-09 17:43:02 -07005927 struct io_kiocb *req;
5928
Jens Axboe78076bb2019-12-04 19:56:40 -07005929 list = &ctx->cancel_hash[hash_long(sqe_addr, ctx->cancel_hash_bits)];
5930 hlist_for_each_entry(req, list, hash_node) {
Jens Axboeb41e9852020-02-17 09:52:41 -07005931 if (sqe_addr != req->user_data)
5932 continue;
Pavel Begunkov9ba5fac2021-04-14 13:38:35 +01005933 if (poll_only && req->opcode != IORING_OP_POLL_ADD)
5934 continue;
Jens Axboeb2cb8052021-03-17 08:17:19 -06005935 return req;
Jens Axboe47f46762019-11-09 17:43:02 -07005936 }
Jens Axboeb2cb8052021-03-17 08:17:19 -06005937 return NULL;
Jens Axboe47f46762019-11-09 17:43:02 -07005938}
5939
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005940static bool io_poll_disarm(struct io_kiocb *req)
5941 __must_hold(&ctx->completion_lock)
5942{
5943 if (!io_poll_get_ownership(req))
5944 return false;
5945 io_poll_remove_entries(req);
5946 hash_del(&req->hash_node);
5947 return true;
5948}
5949
Pavel Begunkov9ba5fac2021-04-14 13:38:35 +01005950static int io_poll_cancel(struct io_ring_ctx *ctx, __u64 sqe_addr,
5951 bool poll_only)
Pavel Begunkove07785b2021-04-01 15:43:57 +01005952 __must_hold(&ctx->completion_lock)
Jens Axboeb2cb8052021-03-17 08:17:19 -06005953{
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005954 struct io_kiocb *req = io_poll_find(ctx, sqe_addr, poll_only);
Jens Axboeb2cb8052021-03-17 08:17:19 -06005955
Jens Axboeb2cb8052021-03-17 08:17:19 -06005956 if (!req)
5957 return -ENOENT;
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005958 io_poll_cancel_req(req);
5959 return 0;
Jens Axboe221c5eb2019-01-17 09:41:58 -07005960}
5961
Pavel Begunkov9096af32021-04-14 13:38:36 +01005962static __poll_t io_poll_parse_events(const struct io_uring_sqe *sqe,
5963 unsigned int flags)
5964{
5965 u32 events;
5966
5967 events = READ_ONCE(sqe->poll32_events);
5968#ifdef __BIG_ENDIAN
5969 events = swahw32(events);
5970#endif
5971 if (!(flags & IORING_POLL_ADD_MULTI))
5972 events |= EPOLLONESHOT;
5973 return demangle_poll(events) | (events & (EPOLLEXCLUSIVE|EPOLLONESHOT));
5974}
5975
Pavel Begunkovc5de0032021-04-14 13:38:37 +01005976static int io_poll_update_prep(struct io_kiocb *req,
Jens Axboe3529d8c2019-12-19 18:24:38 -07005977 const struct io_uring_sqe *sqe)
Jens Axboe221c5eb2019-01-17 09:41:58 -07005978{
Pavel Begunkovc5de0032021-04-14 13:38:37 +01005979 struct io_poll_update *upd = &req->poll_update;
5980 u32 flags;
5981
Jens Axboe221c5eb2019-01-17 09:41:58 -07005982 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
5983 return -EINVAL;
Pavel Begunkov26578cd2021-08-20 10:36:37 +01005984 if (sqe->ioprio || sqe->buf_index || sqe->splice_fd_in)
Pavel Begunkovc5de0032021-04-14 13:38:37 +01005985 return -EINVAL;
5986 flags = READ_ONCE(sqe->len);
5987 if (flags & ~(IORING_POLL_UPDATE_EVENTS | IORING_POLL_UPDATE_USER_DATA |
5988 IORING_POLL_ADD_MULTI))
5989 return -EINVAL;
5990 /* meaningless without update */
5991 if (flags == IORING_POLL_ADD_MULTI)
Jens Axboe221c5eb2019-01-17 09:41:58 -07005992 return -EINVAL;
5993
Pavel Begunkovc5de0032021-04-14 13:38:37 +01005994 upd->old_user_data = READ_ONCE(sqe->addr);
5995 upd->update_events = flags & IORING_POLL_UPDATE_EVENTS;
5996 upd->update_user_data = flags & IORING_POLL_UPDATE_USER_DATA;
Jens Axboe0969e782019-12-17 18:40:57 -07005997
Pavel Begunkovc5de0032021-04-14 13:38:37 +01005998 upd->new_user_data = READ_ONCE(sqe->off);
5999 if (!upd->update_user_data && upd->new_user_data)
6000 return -EINVAL;
6001 if (upd->update_events)
6002 upd->events = io_poll_parse_events(sqe, flags);
6003 else if (sqe->poll32_events)
6004 return -EINVAL;
Jens Axboe0969e782019-12-17 18:40:57 -07006005
Jens Axboe221c5eb2019-01-17 09:41:58 -07006006 return 0;
6007}
6008
Jens Axboe3529d8c2019-12-19 18:24:38 -07006009static int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboe221c5eb2019-01-17 09:41:58 -07006010{
6011 struct io_poll_iocb *poll = &req->poll;
Pavel Begunkovc5de0032021-04-14 13:38:37 +01006012 u32 flags;
Jens Axboe221c5eb2019-01-17 09:41:58 -07006013
6014 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
6015 return -EINVAL;
Pavel Begunkovc5de0032021-04-14 13:38:37 +01006016 if (sqe->ioprio || sqe->buf_index || sqe->off || sqe->addr)
Jens Axboe88e41cf2021-02-22 22:08:01 -07006017 return -EINVAL;
6018 flags = READ_ONCE(sqe->len);
Pavel Begunkovc5de0032021-04-14 13:38:37 +01006019 if (flags & ~IORING_POLL_ADD_MULTI)
Jens Axboe221c5eb2019-01-17 09:41:58 -07006020 return -EINVAL;
6021
Pavel Begunkov48dcd382021-08-15 10:40:18 +01006022 io_req_set_refcount(req);
Pavel Begunkovc5de0032021-04-14 13:38:37 +01006023 poll->events = io_poll_parse_events(sqe, flags);
Jens Axboe0969e782019-12-17 18:40:57 -07006024 return 0;
6025}
6026
Pavel Begunkov61e98202021-02-10 00:03:08 +00006027static int io_poll_add(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe0969e782019-12-17 18:40:57 -07006028{
6029 struct io_poll_iocb *poll = &req->poll;
Jens Axboe0969e782019-12-17 18:40:57 -07006030 struct io_poll_table ipt;
Pavel Begunkovf770fba2022-08-29 14:30:18 +01006031 int ret;
Jens Axboe0969e782019-12-17 18:40:57 -07006032
Jens Axboed7718a92020-02-14 22:23:12 -07006033 ipt.pt._qproc = io_poll_queue_proc;
Jens Axboe36703242019-07-25 10:20:18 -06006034
Pavel Begunkovf770fba2022-08-29 14:30:18 +01006035 ret = __io_arm_poll_handler(req, &req->poll, &ipt, poll->events);
Pavel Begunkov6c7259c2022-08-29 14:30:22 +01006036 if (!ret && ipt.error)
6037 req_set_fail(req);
Pavel Begunkovf770fba2022-08-29 14:30:18 +01006038 ret = ret ?: ipt.error;
6039 if (ret)
6040 __io_req_complete(req, issue_flags, ret, 0);
6041 return 0;
Jens Axboe221c5eb2019-01-17 09:41:58 -07006042}
6043
Pavel Begunkovc5de0032021-04-14 13:38:37 +01006044static int io_poll_update(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboeb69de282021-03-17 08:37:41 -06006045{
6046 struct io_ring_ctx *ctx = req->ctx;
6047 struct io_kiocb *preq;
Pavel Begunkov040e58f2022-08-29 14:30:14 +01006048 int ret2, ret = 0;
Jens Axboeb69de282021-03-17 08:37:41 -06006049
Jens Axboe79ebeae2021-08-10 15:18:27 -06006050 spin_lock(&ctx->completion_lock);
Pavel Begunkov9ba5fac2021-04-14 13:38:35 +01006051 preq = io_poll_find(ctx, req->poll_update.old_user_data, true);
Pavel Begunkovf770fba2022-08-29 14:30:18 +01006052 if (!preq || !io_poll_disarm(preq)) {
Pavel Begunkov040e58f2022-08-29 14:30:14 +01006053 spin_unlock(&ctx->completion_lock);
Pavel Begunkovf770fba2022-08-29 14:30:18 +01006054 ret = preq ? -EALREADY : -ENOENT;
Pavel Begunkov040e58f2022-08-29 14:30:14 +01006055 goto out;
Jens Axboeb69de282021-03-17 08:37:41 -06006056 }
Jens Axboe79ebeae2021-08-10 15:18:27 -06006057 spin_unlock(&ctx->completion_lock);
Jens Axboecb3b200e2021-04-06 09:49:31 -06006058
Pavel Begunkov040e58f2022-08-29 14:30:14 +01006059 if (req->poll_update.update_events || req->poll_update.update_user_data) {
6060 /* only mask one event flags, keep behavior flags */
6061 if (req->poll_update.update_events) {
6062 preq->poll.events &= ~0xffff;
6063 preq->poll.events |= req->poll_update.events & 0xffff;
6064 preq->poll.events |= IO_POLL_UNMASK;
6065 }
6066 if (req->poll_update.update_user_data)
6067 preq->user_data = req->poll_update.new_user_data;
6068
6069 ret2 = io_poll_add(preq, issue_flags);
6070 /* successfully updated, don't complete poll request */
6071 if (!ret2)
6072 goto out;
6073 }
6074 req_set_fail(preq);
6075 io_req_complete(preq, -ECANCELED);
6076out:
6077 if (ret < 0)
6078 req_set_fail(req);
Jens Axboeb69de282021-03-17 08:37:41 -06006079 /* complete update request, we're done with it */
6080 io_req_complete(req, ret);
Jens Axboeb69de282021-03-17 08:37:41 -06006081 return 0;
6082}
6083
Pavel Begunkovf237c302021-08-18 12:42:46 +01006084static void io_req_task_timeout(struct io_kiocb *req, bool *locked)
Jens Axboe89850fc2021-08-10 15:11:51 -06006085{
Jens Axboe89850fc2021-08-10 15:11:51 -06006086 req_set_fail(req);
Pavel Begunkov505657b2021-08-17 20:28:09 +01006087 io_req_complete_post(req, -ETIME, 0);
Jens Axboe89850fc2021-08-10 15:11:51 -06006088}
6089
Jens Axboe5262f562019-09-17 12:26:57 -06006090static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer)
6091{
Jens Axboead8a48a2019-11-15 08:49:11 -07006092 struct io_timeout_data *data = container_of(timer,
6093 struct io_timeout_data, timer);
6094 struct io_kiocb *req = data->req;
6095 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe5262f562019-09-17 12:26:57 -06006096 unsigned long flags;
6097
Jens Axboe89850fc2021-08-10 15:11:51 -06006098 spin_lock_irqsave(&ctx->timeout_lock, flags);
Pavel Begunkova71976f2020-10-10 18:34:11 +01006099 list_del_init(&req->timeout.list);
Pavel Begunkov01cec8c2020-07-30 18:43:50 +03006100 atomic_set(&req->ctx->cq_timeouts,
6101 atomic_read(&req->ctx->cq_timeouts) + 1);
Jens Axboe89850fc2021-08-10 15:11:51 -06006102 spin_unlock_irqrestore(&ctx->timeout_lock, flags);
Pavel Begunkov01cec8c2020-07-30 18:43:50 +03006103
Jens Axboe89850fc2021-08-10 15:11:51 -06006104 req->io_task_work.func = io_req_task_timeout;
6105 io_req_task_work_add(req);
Jens Axboe5262f562019-09-17 12:26:57 -06006106 return HRTIMER_NORESTART;
6107}
6108
Pavel Begunkovfbd15842020-11-30 19:11:15 +00006109static struct io_kiocb *io_timeout_extract(struct io_ring_ctx *ctx,
6110 __u64 user_data)
Jens Axboe89850fc2021-08-10 15:11:51 -06006111 __must_hold(&ctx->timeout_lock)
Jens Axboe47f46762019-11-09 17:43:02 -07006112{
Pavel Begunkovfbd15842020-11-30 19:11:15 +00006113 struct io_timeout_data *io;
Jens Axboef254ac02020-08-12 17:33:30 -06006114 struct io_kiocb *req;
Pavel Begunkovfd9c7bc2021-04-13 02:58:42 +01006115 bool found = false;
Jens Axboef254ac02020-08-12 17:33:30 -06006116
6117 list_for_each_entry(req, &ctx->timeout_list, timeout.list) {
Pavel Begunkovfd9c7bc2021-04-13 02:58:42 +01006118 found = user_data == req->user_data;
6119 if (found)
Jens Axboef254ac02020-08-12 17:33:30 -06006120 break;
Jens Axboef254ac02020-08-12 17:33:30 -06006121 }
Pavel Begunkovfd9c7bc2021-04-13 02:58:42 +01006122 if (!found)
6123 return ERR_PTR(-ENOENT);
Jens Axboef254ac02020-08-12 17:33:30 -06006124
Pavel Begunkovfbd15842020-11-30 19:11:15 +00006125 io = req->async_data;
Pavel Begunkovfd9c7bc2021-04-13 02:58:42 +01006126 if (hrtimer_try_to_cancel(&io->timer) == -1)
Pavel Begunkovfbd15842020-11-30 19:11:15 +00006127 return ERR_PTR(-EALREADY);
6128 list_del_init(&req->timeout.list);
6129 return req;
6130}
6131
6132static int io_timeout_cancel(struct io_ring_ctx *ctx, __u64 user_data)
Pavel Begunkovec3c3d02021-08-18 10:50:52 +01006133 __must_hold(&ctx->completion_lock)
Jens Axboe89850fc2021-08-10 15:11:51 -06006134 __must_hold(&ctx->timeout_lock)
Pavel Begunkovfbd15842020-11-30 19:11:15 +00006135{
6136 struct io_kiocb *req = io_timeout_extract(ctx, user_data);
6137
6138 if (IS_ERR(req))
6139 return PTR_ERR(req);
6140
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01006141 req_set_fail(req);
Pavel Begunkovb850d6d2022-08-29 14:30:13 +01006142 io_fill_cqe_req(req, -ECANCELED, 0);
Pavel Begunkov91c2f692021-08-11 19:28:28 +01006143 io_put_req_deferred(req);
Pavel Begunkovfbd15842020-11-30 19:11:15 +00006144 return 0;
Jens Axboef254ac02020-08-12 17:33:30 -06006145}
6146
Jens Axboe50c1df22021-08-27 17:11:06 -06006147static clockid_t io_timeout_get_clock(struct io_timeout_data *data)
6148{
6149 switch (data->flags & IORING_TIMEOUT_CLOCK_MASK) {
6150 case IORING_TIMEOUT_BOOTTIME:
6151 return CLOCK_BOOTTIME;
6152 case IORING_TIMEOUT_REALTIME:
6153 return CLOCK_REALTIME;
6154 default:
6155 /* can't happen, vetted at prep time */
6156 WARN_ON_ONCE(1);
6157 fallthrough;
6158 case 0:
6159 return CLOCK_MONOTONIC;
6160 }
6161}
6162
Pavel Begunkovf1042b62021-08-28 19:54:39 -06006163static int io_linked_timeout_update(struct io_ring_ctx *ctx, __u64 user_data,
6164 struct timespec64 *ts, enum hrtimer_mode mode)
6165 __must_hold(&ctx->timeout_lock)
6166{
6167 struct io_timeout_data *io;
6168 struct io_kiocb *req;
6169 bool found = false;
6170
6171 list_for_each_entry(req, &ctx->ltimeout_list, timeout.list) {
6172 found = user_data == req->user_data;
6173 if (found)
6174 break;
6175 }
6176 if (!found)
6177 return -ENOENT;
6178
6179 io = req->async_data;
6180 if (hrtimer_try_to_cancel(&io->timer) == -1)
6181 return -EALREADY;
6182 hrtimer_init(&io->timer, io_timeout_get_clock(io), mode);
6183 io->timer.function = io_link_timeout_fn;
6184 hrtimer_start(&io->timer, timespec64_to_ktime(*ts), mode);
6185 return 0;
6186}
6187
Pavel Begunkov9c8e11b2020-11-30 19:11:16 +00006188static int io_timeout_update(struct io_ring_ctx *ctx, __u64 user_data,
6189 struct timespec64 *ts, enum hrtimer_mode mode)
Jens Axboe89850fc2021-08-10 15:11:51 -06006190 __must_hold(&ctx->timeout_lock)
Pavel Begunkov9c8e11b2020-11-30 19:11:16 +00006191{
6192 struct io_kiocb *req = io_timeout_extract(ctx, user_data);
6193 struct io_timeout_data *data;
6194
6195 if (IS_ERR(req))
6196 return PTR_ERR(req);
6197
6198 req->timeout.off = 0; /* noseq */
6199 data = req->async_data;
6200 list_add_tail(&req->timeout.list, &ctx->timeout_list);
Jens Axboe50c1df22021-08-27 17:11:06 -06006201 hrtimer_init(&data->timer, io_timeout_get_clock(data), mode);
Pavel Begunkov9c8e11b2020-11-30 19:11:16 +00006202 data->timer.function = io_timeout_fn;
6203 hrtimer_start(&data->timer, timespec64_to_ktime(*ts), mode);
6204 return 0;
Jens Axboe47f46762019-11-09 17:43:02 -07006205}
6206
Jens Axboe3529d8c2019-12-19 18:24:38 -07006207static int io_timeout_remove_prep(struct io_kiocb *req,
6208 const struct io_uring_sqe *sqe)
Jens Axboeb29472e2019-12-17 18:50:29 -07006209{
Pavel Begunkov9c8e11b2020-11-30 19:11:16 +00006210 struct io_timeout_rem *tr = &req->timeout_rem;
6211
Jens Axboeb29472e2019-12-17 18:50:29 -07006212 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
6213 return -EINVAL;
Daniele Albano61710e42020-07-18 14:15:16 -06006214 if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT)))
6215 return -EINVAL;
Pavel Begunkov26578cd2021-08-20 10:36:37 +01006216 if (sqe->ioprio || sqe->buf_index || sqe->len || sqe->splice_fd_in)
Jens Axboeb29472e2019-12-17 18:50:29 -07006217 return -EINVAL;
6218
Pavel Begunkovf1042b62021-08-28 19:54:39 -06006219 tr->ltimeout = false;
Pavel Begunkov9c8e11b2020-11-30 19:11:16 +00006220 tr->addr = READ_ONCE(sqe->addr);
6221 tr->flags = READ_ONCE(sqe->timeout_flags);
Pavel Begunkovf1042b62021-08-28 19:54:39 -06006222 if (tr->flags & IORING_TIMEOUT_UPDATE_MASK) {
6223 if (hweight32(tr->flags & IORING_TIMEOUT_CLOCK_MASK) > 1)
6224 return -EINVAL;
6225 if (tr->flags & IORING_LINK_TIMEOUT_UPDATE)
6226 tr->ltimeout = true;
6227 if (tr->flags & ~(IORING_TIMEOUT_UPDATE_MASK|IORING_TIMEOUT_ABS))
Pavel Begunkov9c8e11b2020-11-30 19:11:16 +00006228 return -EINVAL;
6229 if (get_timespec64(&tr->ts, u64_to_user_ptr(sqe->addr2)))
6230 return -EFAULT;
6231 } else if (tr->flags) {
6232 /* timeout removal doesn't support flags */
6233 return -EINVAL;
6234 }
6235
Jens Axboeb29472e2019-12-17 18:50:29 -07006236 return 0;
6237}
6238
Pavel Begunkov8662dae2021-01-19 13:32:44 +00006239static inline enum hrtimer_mode io_translate_timeout_mode(unsigned int flags)
6240{
6241 return (flags & IORING_TIMEOUT_ABS) ? HRTIMER_MODE_ABS
6242 : HRTIMER_MODE_REL;
6243}
6244
Jens Axboe11365042019-10-16 09:08:32 -06006245/*
6246 * Remove or update an existing timeout command
6247 */
Pavel Begunkov61e98202021-02-10 00:03:08 +00006248static int io_timeout_remove(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe11365042019-10-16 09:08:32 -06006249{
Pavel Begunkov9c8e11b2020-11-30 19:11:16 +00006250 struct io_timeout_rem *tr = &req->timeout_rem;
Jens Axboe11365042019-10-16 09:08:32 -06006251 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe47f46762019-11-09 17:43:02 -07006252 int ret;
Jens Axboe11365042019-10-16 09:08:32 -06006253
Pavel Begunkovec3c3d02021-08-18 10:50:52 +01006254 if (!(req->timeout_rem.flags & IORING_TIMEOUT_UPDATE)) {
6255 spin_lock(&ctx->completion_lock);
6256 spin_lock_irq(&ctx->timeout_lock);
Pavel Begunkov9c8e11b2020-11-30 19:11:16 +00006257 ret = io_timeout_cancel(ctx, tr->addr);
Pavel Begunkovec3c3d02021-08-18 10:50:52 +01006258 spin_unlock_irq(&ctx->timeout_lock);
6259 spin_unlock(&ctx->completion_lock);
6260 } else {
Pavel Begunkovf1042b62021-08-28 19:54:39 -06006261 enum hrtimer_mode mode = io_translate_timeout_mode(tr->flags);
6262
Pavel Begunkovec3c3d02021-08-18 10:50:52 +01006263 spin_lock_irq(&ctx->timeout_lock);
Pavel Begunkovf1042b62021-08-28 19:54:39 -06006264 if (tr->ltimeout)
6265 ret = io_linked_timeout_update(ctx, tr->addr, &tr->ts, mode);
6266 else
6267 ret = io_timeout_update(ctx, tr->addr, &tr->ts, mode);
Pavel Begunkovec3c3d02021-08-18 10:50:52 +01006268 spin_unlock_irq(&ctx->timeout_lock);
6269 }
Jens Axboe11365042019-10-16 09:08:32 -06006270
Jens Axboe4e88d6e2019-12-07 20:59:47 -07006271 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01006272 req_set_fail(req);
Pavel Begunkov505657b2021-08-17 20:28:09 +01006273 io_req_complete_post(req, ret, 0);
Jens Axboe11365042019-10-16 09:08:32 -06006274 return 0;
Jens Axboe5262f562019-09-17 12:26:57 -06006275}
6276
Jens Axboe3529d8c2019-12-19 18:24:38 -07006277static int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe,
Jens Axboe2d283902019-12-04 11:08:05 -07006278 bool is_timeout_link)
Jens Axboe5262f562019-09-17 12:26:57 -06006279{
Jens Axboead8a48a2019-11-15 08:49:11 -07006280 struct io_timeout_data *data;
Jens Axboea41525a2019-10-15 16:48:15 -06006281 unsigned flags;
Pavel Begunkov56080b02020-05-26 20:34:04 +03006282 u32 off = READ_ONCE(sqe->off);
Jens Axboe5262f562019-09-17 12:26:57 -06006283
Jens Axboead8a48a2019-11-15 08:49:11 -07006284 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
Jens Axboe5262f562019-09-17 12:26:57 -06006285 return -EINVAL;
Pavel Begunkov26578cd2021-08-20 10:36:37 +01006286 if (sqe->ioprio || sqe->buf_index || sqe->len != 1 ||
6287 sqe->splice_fd_in)
Jens Axboea41525a2019-10-15 16:48:15 -06006288 return -EINVAL;
Pavel Begunkov56080b02020-05-26 20:34:04 +03006289 if (off && is_timeout_link)
Jens Axboe2d283902019-12-04 11:08:05 -07006290 return -EINVAL;
Jens Axboea41525a2019-10-15 16:48:15 -06006291 flags = READ_ONCE(sqe->timeout_flags);
Jens Axboe50c1df22021-08-27 17:11:06 -06006292 if (flags & ~(IORING_TIMEOUT_ABS | IORING_TIMEOUT_CLOCK_MASK))
6293 return -EINVAL;
6294 /* more than one clock specified is invalid, obviously */
6295 if (hweight32(flags & IORING_TIMEOUT_CLOCK_MASK) > 1)
Jens Axboe5262f562019-09-17 12:26:57 -06006296 return -EINVAL;
Arnd Bergmannbdf20072019-10-01 09:53:29 -06006297
Pavel Begunkovef9dd632021-08-28 19:54:38 -06006298 INIT_LIST_HEAD(&req->timeout.list);
Pavel Begunkovbfe68a22020-05-30 14:54:18 +03006299 req->timeout.off = off;
Pavel Begunkovf18ee4c2021-06-14 23:37:25 +01006300 if (unlikely(off && !req->ctx->off_timeout_used))
6301 req->ctx->off_timeout_used = true;
Jens Axboe26a61672019-12-20 09:02:01 -07006302
Jens Axboee8c2bc12020-08-15 18:44:09 -07006303 if (!req->async_data && io_alloc_async_data(req))
Jens Axboe26a61672019-12-20 09:02:01 -07006304 return -ENOMEM;
6305
Jens Axboee8c2bc12020-08-15 18:44:09 -07006306 data = req->async_data;
Jens Axboead8a48a2019-11-15 08:49:11 -07006307 data->req = req;
Jens Axboe50c1df22021-08-27 17:11:06 -06006308 data->flags = flags;
Jens Axboead8a48a2019-11-15 08:49:11 -07006309
6310 if (get_timespec64(&data->ts, u64_to_user_ptr(sqe->addr)))
Jens Axboe5262f562019-09-17 12:26:57 -06006311 return -EFAULT;
6312
Jens Axboeba7261a2022-04-08 11:08:58 -06006313 INIT_LIST_HEAD(&req->timeout.list);
Pavel Begunkov8662dae2021-01-19 13:32:44 +00006314 data->mode = io_translate_timeout_mode(flags);
Jens Axboe50c1df22021-08-27 17:11:06 -06006315 hrtimer_init(&data->timer, io_timeout_get_clock(data), data->mode);
Pavel Begunkovb97e7362021-08-15 10:40:23 +01006316
6317 if (is_timeout_link) {
6318 struct io_submit_link *link = &req->ctx->submit_state.link;
6319
6320 if (!link->head)
6321 return -EINVAL;
6322 if (link->last->opcode == IORING_OP_LINK_TIMEOUT)
6323 return -EINVAL;
Pavel Begunkov4d13d1a2021-08-15 10:40:24 +01006324 req->timeout.head = link->last;
6325 link->last->flags |= REQ_F_ARM_LTIMEOUT;
Pavel Begunkovb97e7362021-08-15 10:40:23 +01006326 }
Jens Axboead8a48a2019-11-15 08:49:11 -07006327 return 0;
6328}
6329
Pavel Begunkov61e98202021-02-10 00:03:08 +00006330static int io_timeout(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboead8a48a2019-11-15 08:49:11 -07006331{
Jens Axboead8a48a2019-11-15 08:49:11 -07006332 struct io_ring_ctx *ctx = req->ctx;
Jens Axboee8c2bc12020-08-15 18:44:09 -07006333 struct io_timeout_data *data = req->async_data;
Jens Axboead8a48a2019-11-15 08:49:11 -07006334 struct list_head *entry;
Pavel Begunkovbfe68a22020-05-30 14:54:18 +03006335 u32 tail, off = req->timeout.off;
Jens Axboead8a48a2019-11-15 08:49:11 -07006336
Jens Axboe89850fc2021-08-10 15:11:51 -06006337 spin_lock_irq(&ctx->timeout_lock);
Jens Axboe93bd25b2019-11-11 23:34:31 -07006338
Jens Axboe5262f562019-09-17 12:26:57 -06006339 /*
6340 * sqe->off holds how many events that need to occur for this
Jens Axboe93bd25b2019-11-11 23:34:31 -07006341 * timeout event to be satisfied. If it isn't set, then this is
6342 * a pure timeout request, sequence isn't used.
Jens Axboe5262f562019-09-17 12:26:57 -06006343 */
Pavel Begunkov8eb7e2d2020-06-29 13:13:02 +03006344 if (io_is_timeout_noseq(req)) {
Jens Axboe93bd25b2019-11-11 23:34:31 -07006345 entry = ctx->timeout_list.prev;
6346 goto add;
6347 }
Jens Axboe5262f562019-09-17 12:26:57 -06006348
Pavel Begunkovbfe68a22020-05-30 14:54:18 +03006349 tail = ctx->cached_cq_tail - atomic_read(&ctx->cq_timeouts);
6350 req->timeout.target_seq = tail + off;
Jens Axboe5262f562019-09-17 12:26:57 -06006351
Marcelo Diop-Gonzalezf0105052021-01-15 11:54:40 -05006352 /* Update the last seq here in case io_flush_timeouts() hasn't.
6353 * This is safe because ->completion_lock is held, and submissions
6354 * and completions are never mixed in the same ->completion_lock section.
6355 */
6356 ctx->cq_last_tm_flush = tail;
6357
Jens Axboe5262f562019-09-17 12:26:57 -06006358 /*
6359 * Insertion sort, ensuring the first entry in the list is always
6360 * the one we need first.
6361 */
Jens Axboe5262f562019-09-17 12:26:57 -06006362 list_for_each_prev(entry, &ctx->timeout_list) {
Pavel Begunkov135fcde2020-07-13 23:37:12 +03006363 struct io_kiocb *nxt = list_entry(entry, struct io_kiocb,
6364 timeout.list);
Jens Axboe5262f562019-09-17 12:26:57 -06006365
Pavel Begunkov8eb7e2d2020-06-29 13:13:02 +03006366 if (io_is_timeout_noseq(nxt))
Jens Axboe93bd25b2019-11-11 23:34:31 -07006367 continue;
Pavel Begunkovbfe68a22020-05-30 14:54:18 +03006368 /* nxt.seq is behind @tail, otherwise would've been completed */
6369 if (off >= nxt->timeout.target_seq - tail)
Jens Axboe5262f562019-09-17 12:26:57 -06006370 break;
6371 }
Jens Axboe93bd25b2019-11-11 23:34:31 -07006372add:
Pavel Begunkov135fcde2020-07-13 23:37:12 +03006373 list_add(&req->timeout.list, entry);
Jens Axboead8a48a2019-11-15 08:49:11 -07006374 data->timer.function = io_timeout_fn;
6375 hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), data->mode);
Jens Axboe89850fc2021-08-10 15:11:51 -06006376 spin_unlock_irq(&ctx->timeout_lock);
Jens Axboe5262f562019-09-17 12:26:57 -06006377 return 0;
6378}
6379
Pavel Begunkovf458dd842021-03-08 12:14:14 +00006380struct io_cancel_data {
6381 struct io_ring_ctx *ctx;
6382 u64 user_data;
6383};
6384
Jens Axboe62755e32019-10-28 21:49:21 -06006385static bool io_cancel_cb(struct io_wq_work *work, void *data)
Jens Axboede0617e2019-04-06 21:51:27 -06006386{
Jens Axboe62755e32019-10-28 21:49:21 -06006387 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
Pavel Begunkovf458dd842021-03-08 12:14:14 +00006388 struct io_cancel_data *cd = data;
Jens Axboede0617e2019-04-06 21:51:27 -06006389
Pavel Begunkovf458dd842021-03-08 12:14:14 +00006390 return req->ctx == cd->ctx && req->user_data == cd->user_data;
Jens Axboe62755e32019-10-28 21:49:21 -06006391}
6392
Pavel Begunkovf458dd842021-03-08 12:14:14 +00006393static int io_async_cancel_one(struct io_uring_task *tctx, u64 user_data,
6394 struct io_ring_ctx *ctx)
Jens Axboe62755e32019-10-28 21:49:21 -06006395{
Pavel Begunkovf458dd842021-03-08 12:14:14 +00006396 struct io_cancel_data data = { .ctx = ctx, .user_data = user_data, };
Jens Axboe62755e32019-10-28 21:49:21 -06006397 enum io_wq_cancel cancel_ret;
Jens Axboe62755e32019-10-28 21:49:21 -06006398 int ret = 0;
6399
Pavel Begunkovf458dd842021-03-08 12:14:14 +00006400 if (!tctx || !tctx->io_wq)
Jens Axboe5aa75ed2021-02-16 12:56:50 -07006401 return -ENOENT;
6402
Pavel Begunkovf458dd842021-03-08 12:14:14 +00006403 cancel_ret = io_wq_cancel_cb(tctx->io_wq, io_cancel_cb, &data, false);
Jens Axboe62755e32019-10-28 21:49:21 -06006404 switch (cancel_ret) {
6405 case IO_WQ_CANCEL_OK:
6406 ret = 0;
6407 break;
6408 case IO_WQ_CANCEL_RUNNING:
6409 ret = -EALREADY;
6410 break;
6411 case IO_WQ_CANCEL_NOTFOUND:
6412 ret = -ENOENT;
6413 break;
6414 }
6415
Jens Axboee977d6d2019-11-05 12:39:45 -07006416 return ret;
6417}
6418
Pavel Begunkov8cb01fa2021-08-15 10:40:22 +01006419static int io_try_cancel_userdata(struct io_kiocb *req, u64 sqe_addr)
Jens Axboe47f46762019-11-09 17:43:02 -07006420{
Pavel Begunkov8cb01fa2021-08-15 10:40:22 +01006421 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe47f46762019-11-09 17:43:02 -07006422 int ret;
6423
Pavel Begunkovdadebc32021-08-23 13:30:44 +01006424 WARN_ON_ONCE(!io_wq_current_is_worker() && req->task != current);
Pavel Begunkov8cb01fa2021-08-15 10:40:22 +01006425
Pavel Begunkovf458dd842021-03-08 12:14:14 +00006426 ret = io_async_cancel_one(req->task->io_uring, sqe_addr, ctx);
Pavel Begunkovdf9727a2021-04-01 15:43:59 +01006427 if (ret != -ENOENT)
Pavel Begunkov8cb01fa2021-08-15 10:40:22 +01006428 return ret;
Pavel Begunkov505657b2021-08-17 20:28:09 +01006429
6430 spin_lock(&ctx->completion_lock);
Jens Axboe79ebeae2021-08-10 15:18:27 -06006431 spin_lock_irq(&ctx->timeout_lock);
Jens Axboe47f46762019-11-09 17:43:02 -07006432 ret = io_timeout_cancel(ctx, sqe_addr);
Jens Axboe79ebeae2021-08-10 15:18:27 -06006433 spin_unlock_irq(&ctx->timeout_lock);
Jens Axboe47f46762019-11-09 17:43:02 -07006434 if (ret != -ENOENT)
Pavel Begunkov505657b2021-08-17 20:28:09 +01006435 goto out;
6436 ret = io_poll_cancel(ctx, sqe_addr, false);
6437out:
6438 spin_unlock(&ctx->completion_lock);
6439 return ret;
Jens Axboe47f46762019-11-09 17:43:02 -07006440}
6441
Jens Axboe3529d8c2019-12-19 18:24:38 -07006442static int io_async_cancel_prep(struct io_kiocb *req,
6443 const struct io_uring_sqe *sqe)
Jens Axboee977d6d2019-11-05 12:39:45 -07006444{
Jens Axboefbf23842019-12-17 18:45:56 -07006445 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
Jens Axboee977d6d2019-11-05 12:39:45 -07006446 return -EINVAL;
Daniele Albano61710e42020-07-18 14:15:16 -06006447 if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT)))
6448 return -EINVAL;
Pavel Begunkov26578cd2021-08-20 10:36:37 +01006449 if (sqe->ioprio || sqe->off || sqe->len || sqe->cancel_flags ||
6450 sqe->splice_fd_in)
Jens Axboee977d6d2019-11-05 12:39:45 -07006451 return -EINVAL;
6452
Jens Axboefbf23842019-12-17 18:45:56 -07006453 req->cancel.addr = READ_ONCE(sqe->addr);
6454 return 0;
6455}
6456
Pavel Begunkov61e98202021-02-10 00:03:08 +00006457static int io_async_cancel(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboefbf23842019-12-17 18:45:56 -07006458{
6459 struct io_ring_ctx *ctx = req->ctx;
Pavel Begunkov58f99372021-03-12 16:25:55 +00006460 u64 sqe_addr = req->cancel.addr;
6461 struct io_tctx_node *node;
6462 int ret;
Jens Axboefbf23842019-12-17 18:45:56 -07006463
Pavel Begunkov8cb01fa2021-08-15 10:40:22 +01006464 ret = io_try_cancel_userdata(req, sqe_addr);
Pavel Begunkov58f99372021-03-12 16:25:55 +00006465 if (ret != -ENOENT)
6466 goto done;
Pavel Begunkov58f99372021-03-12 16:25:55 +00006467
6468 /* slow path, try all io-wq's */
6469 io_ring_submit_lock(ctx, !(issue_flags & IO_URING_F_NONBLOCK));
6470 ret = -ENOENT;
6471 list_for_each_entry(node, &ctx->tctx_list, ctx_node) {
6472 struct io_uring_task *tctx = node->task->io_uring;
6473
Pavel Begunkov58f99372021-03-12 16:25:55 +00006474 ret = io_async_cancel_one(tctx, req->cancel.addr, ctx);
6475 if (ret != -ENOENT)
6476 break;
6477 }
6478 io_ring_submit_unlock(ctx, !(issue_flags & IO_URING_F_NONBLOCK));
Pavel Begunkov58f99372021-03-12 16:25:55 +00006479done:
Pavel Begunkov58f99372021-03-12 16:25:55 +00006480 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01006481 req_set_fail(req);
Pavel Begunkov505657b2021-08-17 20:28:09 +01006482 io_req_complete_post(req, ret, 0);
Jens Axboe62755e32019-10-28 21:49:21 -06006483 return 0;
6484}
6485
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00006486static int io_rsrc_update_prep(struct io_kiocb *req,
Jens Axboe05f3fb32019-12-09 11:22:50 -07006487 const struct io_uring_sqe *sqe)
6488{
Daniele Albano61710e42020-07-18 14:15:16 -06006489 if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT)))
6490 return -EINVAL;
Pavel Begunkov26578cd2021-08-20 10:36:37 +01006491 if (sqe->ioprio || sqe->rw_flags || sqe->splice_fd_in)
Jens Axboe05f3fb32019-12-09 11:22:50 -07006492 return -EINVAL;
6493
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00006494 req->rsrc_update.offset = READ_ONCE(sqe->off);
6495 req->rsrc_update.nr_args = READ_ONCE(sqe->len);
6496 if (!req->rsrc_update.nr_args)
Jens Axboe05f3fb32019-12-09 11:22:50 -07006497 return -EINVAL;
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00006498 req->rsrc_update.arg = READ_ONCE(sqe->addr);
Jens Axboe05f3fb32019-12-09 11:22:50 -07006499 return 0;
6500}
6501
Pavel Begunkov889fca72021-02-10 00:03:09 +00006502static int io_files_update(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe05f3fb32019-12-09 11:22:50 -07006503{
6504 struct io_ring_ctx *ctx = req->ctx;
Pavel Begunkovc3bdad02021-04-25 14:32:22 +01006505 struct io_uring_rsrc_update2 up;
Jens Axboe05f3fb32019-12-09 11:22:50 -07006506 int ret;
6507
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00006508 up.offset = req->rsrc_update.offset;
6509 up.data = req->rsrc_update.arg;
Pavel Begunkovc3bdad02021-04-25 14:32:22 +01006510 up.nr = 0;
6511 up.tags = 0;
Colin Ian King615cee42021-04-26 10:47:35 +01006512 up.resv = 0;
Dylan Yudaken7a7c9f92022-04-12 09:30:40 -07006513 up.resv2 = 0;
Jens Axboe05f3fb32019-12-09 11:22:50 -07006514
Jens Axboecdb31c22021-09-24 08:43:54 -06006515 io_ring_submit_lock(ctx, !(issue_flags & IO_URING_F_NONBLOCK));
Pavel Begunkovfdecb662021-04-25 14:32:20 +01006516 ret = __io_register_rsrc_update(ctx, IORING_RSRC_FILE,
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +01006517 &up, req->rsrc_update.nr_args);
Jens Axboecdb31c22021-09-24 08:43:54 -06006518 io_ring_submit_unlock(ctx, !(issue_flags & IO_URING_F_NONBLOCK));
Jens Axboe05f3fb32019-12-09 11:22:50 -07006519
6520 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01006521 req_set_fail(req);
Pavel Begunkov889fca72021-02-10 00:03:09 +00006522 __io_req_complete(req, issue_flags, ret, 0);
Jens Axboe05f3fb32019-12-09 11:22:50 -07006523 return 0;
6524}
6525
Pavel Begunkovbfe76552020-09-30 22:57:55 +03006526static int io_req_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboef67676d2019-12-02 11:03:47 -07006527{
Jens Axboed625c6e2019-12-17 19:53:05 -07006528 switch (req->opcode) {
Jens Axboee7815732019-12-17 19:45:06 -07006529 case IORING_OP_NOP:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03006530 return 0;
Jens Axboef67676d2019-12-02 11:03:47 -07006531 case IORING_OP_READV:
6532 case IORING_OP_READ_FIXED:
Jens Axboe3a6820f2019-12-22 15:19:35 -07006533 case IORING_OP_READ:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03006534 return io_read_prep(req, sqe);
Jens Axboef67676d2019-12-02 11:03:47 -07006535 case IORING_OP_WRITEV:
6536 case IORING_OP_WRITE_FIXED:
Jens Axboe3a6820f2019-12-22 15:19:35 -07006537 case IORING_OP_WRITE:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03006538 return io_write_prep(req, sqe);
Jens Axboe0969e782019-12-17 18:40:57 -07006539 case IORING_OP_POLL_ADD:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03006540 return io_poll_add_prep(req, sqe);
Jens Axboe0969e782019-12-17 18:40:57 -07006541 case IORING_OP_POLL_REMOVE:
Pavel Begunkovc5de0032021-04-14 13:38:37 +01006542 return io_poll_update_prep(req, sqe);
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07006543 case IORING_OP_FSYNC:
Pavel Begunkov1155c762021-02-18 18:29:38 +00006544 return io_fsync_prep(req, sqe);
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07006545 case IORING_OP_SYNC_FILE_RANGE:
Pavel Begunkov1155c762021-02-18 18:29:38 +00006546 return io_sfr_prep(req, sqe);
Jens Axboe03b12302019-12-02 18:50:25 -07006547 case IORING_OP_SENDMSG:
Jens Axboefddafac2020-01-04 20:19:44 -07006548 case IORING_OP_SEND:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03006549 return io_sendmsg_prep(req, sqe);
Jens Axboe03b12302019-12-02 18:50:25 -07006550 case IORING_OP_RECVMSG:
Jens Axboefddafac2020-01-04 20:19:44 -07006551 case IORING_OP_RECV:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03006552 return io_recvmsg_prep(req, sqe);
Jens Axboef499a022019-12-02 16:28:46 -07006553 case IORING_OP_CONNECT:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03006554 return io_connect_prep(req, sqe);
Jens Axboe2d283902019-12-04 11:08:05 -07006555 case IORING_OP_TIMEOUT:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03006556 return io_timeout_prep(req, sqe, false);
Jens Axboeb29472e2019-12-17 18:50:29 -07006557 case IORING_OP_TIMEOUT_REMOVE:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03006558 return io_timeout_remove_prep(req, sqe);
Jens Axboefbf23842019-12-17 18:45:56 -07006559 case IORING_OP_ASYNC_CANCEL:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03006560 return io_async_cancel_prep(req, sqe);
Jens Axboe2d283902019-12-04 11:08:05 -07006561 case IORING_OP_LINK_TIMEOUT:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03006562 return io_timeout_prep(req, sqe, true);
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07006563 case IORING_OP_ACCEPT:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03006564 return io_accept_prep(req, sqe);
Jens Axboed63d1b52019-12-10 10:38:56 -07006565 case IORING_OP_FALLOCATE:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03006566 return io_fallocate_prep(req, sqe);
Jens Axboe15b71ab2019-12-11 11:20:36 -07006567 case IORING_OP_OPENAT:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03006568 return io_openat_prep(req, sqe);
Jens Axboeb5dba592019-12-11 14:02:38 -07006569 case IORING_OP_CLOSE:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03006570 return io_close_prep(req, sqe);
Jens Axboe05f3fb32019-12-09 11:22:50 -07006571 case IORING_OP_FILES_UPDATE:
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00006572 return io_rsrc_update_prep(req, sqe);
Jens Axboeeddc7ef2019-12-13 21:18:10 -07006573 case IORING_OP_STATX:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03006574 return io_statx_prep(req, sqe);
Jens Axboe4840e412019-12-25 22:03:45 -07006575 case IORING_OP_FADVISE:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03006576 return io_fadvise_prep(req, sqe);
Jens Axboec1ca7572019-12-25 22:18:28 -07006577 case IORING_OP_MADVISE:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03006578 return io_madvise_prep(req, sqe);
Jens Axboecebdb982020-01-08 17:59:24 -07006579 case IORING_OP_OPENAT2:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03006580 return io_openat2_prep(req, sqe);
Jens Axboe3e4827b2020-01-08 15:18:09 -07006581 case IORING_OP_EPOLL_CTL:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03006582 return io_epoll_ctl_prep(req, sqe);
Pavel Begunkov7d67af22020-02-24 11:32:45 +03006583 case IORING_OP_SPLICE:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03006584 return io_splice_prep(req, sqe);
Jens Axboeddf0322d2020-02-23 16:41:33 -07006585 case IORING_OP_PROVIDE_BUFFERS:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03006586 return io_provide_buffers_prep(req, sqe);
Jens Axboe067524e2020-03-02 16:32:28 -07006587 case IORING_OP_REMOVE_BUFFERS:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03006588 return io_remove_buffers_prep(req, sqe);
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03006589 case IORING_OP_TEE:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03006590 return io_tee_prep(req, sqe);
Jens Axboe36f4fa62020-09-05 11:14:22 -06006591 case IORING_OP_SHUTDOWN:
6592 return io_shutdown_prep(req, sqe);
Jens Axboe80a261f2020-09-28 14:23:58 -06006593 case IORING_OP_RENAMEAT:
6594 return io_renameat_prep(req, sqe);
Jens Axboe14a11432020-09-28 14:27:37 -06006595 case IORING_OP_UNLINKAT:
6596 return io_unlinkat_prep(req, sqe);
Dmitry Kadasheve34a02d2021-07-08 13:34:45 +07006597 case IORING_OP_MKDIRAT:
6598 return io_mkdirat_prep(req, sqe);
Dmitry Kadashev7a8721f2021-07-08 13:34:46 +07006599 case IORING_OP_SYMLINKAT:
6600 return io_symlinkat_prep(req, sqe);
Dmitry Kadashevcf30da92021-07-08 13:34:47 +07006601 case IORING_OP_LINKAT:
6602 return io_linkat_prep(req, sqe);
Jens Axboef67676d2019-12-02 11:03:47 -07006603 }
6604
Pavel Begunkovbfe76552020-09-30 22:57:55 +03006605 printk_once(KERN_WARNING "io_uring: unhandled opcode %d\n",
6606 req->opcode);
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01006607 return -EINVAL;
Pavel Begunkovbfe76552020-09-30 22:57:55 +03006608}
6609
Pavel Begunkov93642ef2021-02-18 18:29:44 +00006610static int io_req_prep_async(struct io_kiocb *req)
Jens Axboedef596e2019-01-09 08:59:42 -07006611{
Pavel Begunkovb7e298d2021-02-28 22:35:19 +00006612 if (!io_op_defs[req->opcode].needs_async_setup)
6613 return 0;
6614 if (WARN_ON_ONCE(req->async_data))
6615 return -EFAULT;
6616 if (io_alloc_async_data(req))
6617 return -EAGAIN;
6618
Pavel Begunkov93642ef2021-02-18 18:29:44 +00006619 switch (req->opcode) {
6620 case IORING_OP_READV:
Pavel Begunkov93642ef2021-02-18 18:29:44 +00006621 return io_rw_prep_async(req, READ);
6622 case IORING_OP_WRITEV:
Pavel Begunkov93642ef2021-02-18 18:29:44 +00006623 return io_rw_prep_async(req, WRITE);
6624 case IORING_OP_SENDMSG:
Pavel Begunkov93642ef2021-02-18 18:29:44 +00006625 return io_sendmsg_prep_async(req);
6626 case IORING_OP_RECVMSG:
Pavel Begunkov93642ef2021-02-18 18:29:44 +00006627 return io_recvmsg_prep_async(req);
6628 case IORING_OP_CONNECT:
6629 return io_connect_prep_async(req);
6630 }
Pavel Begunkovb7e298d2021-02-28 22:35:19 +00006631 printk_once(KERN_WARNING "io_uring: prep_async() bad opcode %d\n",
6632 req->opcode);
6633 return -EFAULT;
Jens Axboedef596e2019-01-09 08:59:42 -07006634}
6635
Pavel Begunkov9cf7c102020-07-13 23:37:15 +03006636static u32 io_get_sequence(struct io_kiocb *req)
6637{
Pavel Begunkova3dbdf52021-06-17 18:14:05 +01006638 u32 seq = req->ctx->cached_sq_head;
Pavel Begunkov9cf7c102020-07-13 23:37:15 +03006639
Pavel Begunkova3dbdf52021-06-17 18:14:05 +01006640 /* need original cached_sq_head, but it was increased for each req */
6641 io_for_each_link(req, req)
6642 seq--;
6643 return seq;
Pavel Begunkov9cf7c102020-07-13 23:37:15 +03006644}
6645
Pavel Begunkov76cc33d2021-06-14 23:37:30 +01006646static bool io_drain_req(struct io_kiocb *req)
Jens Axboedef596e2019-01-09 08:59:42 -07006647{
Pavel Begunkov3c199662021-06-15 16:47:57 +01006648 struct io_kiocb *pos;
Jens Axboedef596e2019-01-09 08:59:42 -07006649 struct io_ring_ctx *ctx = req->ctx;
Pavel Begunkov27dc8332020-07-13 23:37:14 +03006650 struct io_defer_entry *de;
Jens Axboedef596e2019-01-09 08:59:42 -07006651 int ret;
Pavel Begunkov9cf7c102020-07-13 23:37:15 +03006652 u32 seq;
Jens Axboedef596e2019-01-09 08:59:42 -07006653
Pavel Begunkovb8ce1b92021-08-31 14:13:11 +01006654 if (req->flags & REQ_F_FAIL) {
6655 io_req_complete_fail_submit(req);
6656 return true;
6657 }
6658
Pavel Begunkov3c199662021-06-15 16:47:57 +01006659 /*
6660 * If we need to drain a request in the middle of a link, drain the
6661 * head request and the next request/link after the current link.
6662 * Considering sequential execution of links, IOSQE_IO_DRAIN will be
6663 * maintained for every request of our link.
6664 */
6665 if (ctx->drain_next) {
6666 req->flags |= REQ_F_IO_DRAIN;
6667 ctx->drain_next = false;
6668 }
6669 /* not interested in head, start from the first linked */
6670 io_for_each_link(pos, req->link) {
6671 if (pos->flags & REQ_F_IO_DRAIN) {
6672 ctx->drain_next = true;
6673 req->flags |= REQ_F_IO_DRAIN;
6674 break;
6675 }
6676 }
6677
Jens Axboedef596e2019-01-09 08:59:42 -07006678 /* Still need defer if there is pending req in defer list. */
Hao Xu1bd12b72021-11-25 17:21:02 +08006679 spin_lock(&ctx->completion_lock);
Pavel Begunkov9cf7c102020-07-13 23:37:15 +03006680 if (likely(list_empty_careful(&ctx->defer_list) &&
Pavel Begunkov10c66902021-06-15 16:47:56 +01006681 !(req->flags & REQ_F_IO_DRAIN))) {
Hao Xu1bd12b72021-11-25 17:21:02 +08006682 spin_unlock(&ctx->completion_lock);
Pavel Begunkov10c66902021-06-15 16:47:56 +01006683 ctx->drain_active = false;
Pavel Begunkov76cc33d2021-06-14 23:37:30 +01006684 return false;
Pavel Begunkov10c66902021-06-15 16:47:56 +01006685 }
Hao Xu1bd12b72021-11-25 17:21:02 +08006686 spin_unlock(&ctx->completion_lock);
Pavel Begunkov9cf7c102020-07-13 23:37:15 +03006687
6688 seq = io_get_sequence(req);
6689 /* Still a chance to pass the sequence check */
6690 if (!req_need_defer(req, seq) && list_empty_careful(&ctx->defer_list))
Pavel Begunkov76cc33d2021-06-14 23:37:30 +01006691 return false;
Jens Axboedef596e2019-01-09 08:59:42 -07006692
Pavel Begunkovb7e298d2021-02-28 22:35:19 +00006693 ret = io_req_prep_async(req);
Pavel Begunkovbe7053b2021-02-18 18:29:45 +00006694 if (ret)
Pavel Begunkov1b487732021-07-11 22:41:13 +01006695 goto fail;
Pavel Begunkovcbdcb432020-06-29 19:18:43 +03006696 io_prep_async_link(req);
Pavel Begunkov27dc8332020-07-13 23:37:14 +03006697 de = kmalloc(sizeof(*de), GFP_KERNEL);
Pavel Begunkov76cc33d2021-06-14 23:37:30 +01006698 if (!de) {
Pavel Begunkov1b487732021-07-11 22:41:13 +01006699 ret = -ENOMEM;
6700fail:
6701 io_req_complete_failed(req, ret);
Pavel Begunkov76cc33d2021-06-14 23:37:30 +01006702 return true;
6703 }
Jens Axboe31b51512019-01-18 22:56:34 -07006704
Jens Axboe79ebeae2021-08-10 15:18:27 -06006705 spin_lock(&ctx->completion_lock);
Pavel Begunkov9cf7c102020-07-13 23:37:15 +03006706 if (!req_need_defer(req, seq) && list_empty(&ctx->defer_list)) {
Jens Axboe79ebeae2021-08-10 15:18:27 -06006707 spin_unlock(&ctx->completion_lock);
Pavel Begunkov27dc8332020-07-13 23:37:14 +03006708 kfree(de);
Pavel Begunkovf237c302021-08-18 12:42:46 +01006709 io_queue_async_work(req, NULL);
Pavel Begunkov76cc33d2021-06-14 23:37:30 +01006710 return true;
Jens Axboe31b51512019-01-18 22:56:34 -07006711 }
6712
6713 trace_io_uring_defer(ctx, req, req->user_data);
Pavel Begunkov27dc8332020-07-13 23:37:14 +03006714 de->req = req;
Pavel Begunkov9cf7c102020-07-13 23:37:15 +03006715 de->seq = seq;
Pavel Begunkov27dc8332020-07-13 23:37:14 +03006716 list_add_tail(&de->list, &ctx->defer_list);
Jens Axboe79ebeae2021-08-10 15:18:27 -06006717 spin_unlock(&ctx->completion_lock);
Pavel Begunkov76cc33d2021-06-14 23:37:30 +01006718 return true;
Jens Axboe31b51512019-01-18 22:56:34 -07006719}
6720
Pavel Begunkov68fb8972021-03-19 17:22:41 +00006721static void io_clean_op(struct io_kiocb *req)
Pavel Begunkov99bc4c32020-02-07 22:04:45 +03006722{
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03006723 if (req->flags & REQ_F_BUFFER_SELECTED) {
6724 switch (req->opcode) {
6725 case IORING_OP_READV:
6726 case IORING_OP_READ_FIXED:
6727 case IORING_OP_READ:
Jens Axboebcda7ba2020-02-23 16:42:51 -07006728 kfree((void *)(unsigned long)req->rw.addr);
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03006729 break;
6730 case IORING_OP_RECVMSG:
6731 case IORING_OP_RECV:
Jens Axboe52de1fe2020-02-27 10:15:42 -07006732 kfree(req->sr_msg.kbuf);
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03006733 break;
6734 }
Pavel Begunkov99bc4c32020-02-07 22:04:45 +03006735 }
6736
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03006737 if (req->flags & REQ_F_NEED_CLEANUP) {
6738 switch (req->opcode) {
6739 case IORING_OP_READV:
6740 case IORING_OP_READ_FIXED:
6741 case IORING_OP_READ:
6742 case IORING_OP_WRITEV:
6743 case IORING_OP_WRITE_FIXED:
Jens Axboee8c2bc12020-08-15 18:44:09 -07006744 case IORING_OP_WRITE: {
6745 struct io_async_rw *io = req->async_data;
Pavel Begunkov1dacb4d2021-06-17 18:14:03 +01006746
6747 kfree(io->free_iovec);
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03006748 break;
Jens Axboee8c2bc12020-08-15 18:44:09 -07006749 }
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03006750 case IORING_OP_RECVMSG:
Jens Axboee8c2bc12020-08-15 18:44:09 -07006751 case IORING_OP_SENDMSG: {
6752 struct io_async_msghdr *io = req->async_data;
Pavel Begunkov257e84a2021-02-05 00:58:00 +00006753
6754 kfree(io->free_iov);
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03006755 break;
Jens Axboee8c2bc12020-08-15 18:44:09 -07006756 }
Jens Axboef3cd48502020-09-24 14:55:54 -06006757 case IORING_OP_OPENAT:
6758 case IORING_OP_OPENAT2:
6759 if (req->open.filename)
6760 putname(req->open.filename);
6761 break;
Jens Axboe80a261f2020-09-28 14:23:58 -06006762 case IORING_OP_RENAMEAT:
6763 putname(req->rename.oldpath);
6764 putname(req->rename.newpath);
6765 break;
Jens Axboe14a11432020-09-28 14:27:37 -06006766 case IORING_OP_UNLINKAT:
6767 putname(req->unlink.filename);
6768 break;
Dmitry Kadasheve34a02d2021-07-08 13:34:45 +07006769 case IORING_OP_MKDIRAT:
6770 putname(req->mkdir.filename);
6771 break;
Dmitry Kadashev7a8721f2021-07-08 13:34:46 +07006772 case IORING_OP_SYMLINKAT:
6773 putname(req->symlink.oldpath);
6774 putname(req->symlink.newpath);
6775 break;
Dmitry Kadashevcf30da92021-07-08 13:34:47 +07006776 case IORING_OP_LINKAT:
6777 putname(req->hardlink.oldpath);
6778 putname(req->hardlink.newpath);
6779 break;
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03006780 }
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03006781 }
Jens Axboe75652a302021-04-15 09:52:40 -06006782 if ((req->flags & REQ_F_POLLED) && req->apoll) {
6783 kfree(req->apoll->double_poll);
6784 kfree(req->apoll);
6785 req->apoll = NULL;
6786 }
Pavel Begunkov3a0a6902021-04-20 12:03:31 +01006787 if (req->flags & REQ_F_INFLIGHT) {
6788 struct io_uring_task *tctx = req->task->io_uring;
6789
6790 atomic_dec(&tctx->inflight_tracked);
Pavel Begunkov3a0a6902021-04-20 12:03:31 +01006791 }
Pavel Begunkovc8543572021-06-17 18:14:04 +01006792 if (req->flags & REQ_F_CREDS)
Pavel Begunkovb8e64b52021-06-17 18:14:02 +01006793 put_cred(req->creds);
Pavel Begunkovc8543572021-06-17 18:14:04 +01006794
6795 req->flags &= ~IO_REQ_CLEAN_FLAGS;
Pavel Begunkov99bc4c32020-02-07 22:04:45 +03006796}
6797
Pavel Begunkov889fca72021-02-10 00:03:09 +00006798static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboeedafcce2019-01-09 09:16:05 -07006799{
Jens Axboeedafcce2019-01-09 09:16:05 -07006800 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe5730b272021-02-27 15:57:30 -07006801 const struct cred *creds = NULL;
Jens Axboed625c6e2019-12-17 19:53:05 -07006802 int ret;
Jens Axboeedafcce2019-01-09 09:16:05 -07006803
Pavel Begunkovb8e64b52021-06-17 18:14:02 +01006804 if ((req->flags & REQ_F_CREDS) && req->creds != current_cred())
Pavel Begunkovc10d1f92021-06-17 18:14:01 +01006805 creds = override_creds(req->creds);
Jens Axboe5730b272021-02-27 15:57:30 -07006806
Jens Axboed625c6e2019-12-17 19:53:05 -07006807 switch (req->opcode) {
Jens Axboe2b188cc2019-01-07 10:46:33 -07006808 case IORING_OP_NOP:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006809 ret = io_nop(req, issue_flags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006810 break;
Jens Axboe2b188cc2019-01-07 10:46:33 -07006811 case IORING_OP_READV:
Jens Axboe3529d8c2019-12-19 18:24:38 -07006812 case IORING_OP_READ_FIXED:
Jens Axboe3a6820f2019-12-22 15:19:35 -07006813 case IORING_OP_READ:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006814 ret = io_read(req, issue_flags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006815 break;
6816 case IORING_OP_WRITEV:
Jens Axboe2b188cc2019-01-07 10:46:33 -07006817 case IORING_OP_WRITE_FIXED:
Jens Axboe3a6820f2019-12-22 15:19:35 -07006818 case IORING_OP_WRITE:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006819 ret = io_write(req, issue_flags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006820 break;
6821 case IORING_OP_FSYNC:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006822 ret = io_fsync(req, issue_flags);
Jackie Liuba5290c2019-10-09 09:19:59 +08006823 break;
Jens Axboe2b188cc2019-01-07 10:46:33 -07006824 case IORING_OP_POLL_ADD:
Pavel Begunkov61e98202021-02-10 00:03:08 +00006825 ret = io_poll_add(req, issue_flags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006826 break;
6827 case IORING_OP_POLL_REMOVE:
Pavel Begunkovc5de0032021-04-14 13:38:37 +01006828 ret = io_poll_update(req, issue_flags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006829 break;
Jens Axboeb76da702019-11-20 13:05:32 -07006830 case IORING_OP_SYNC_FILE_RANGE:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006831 ret = io_sync_file_range(req, issue_flags);
Jens Axboeb76da702019-11-20 13:05:32 -07006832 break;
6833 case IORING_OP_SENDMSG:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006834 ret = io_sendmsg(req, issue_flags);
Pavel Begunkov062d04d2020-10-10 18:34:12 +01006835 break;
Jens Axboefddafac2020-01-04 20:19:44 -07006836 case IORING_OP_SEND:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006837 ret = io_send(req, issue_flags);
Jens Axboeb76da702019-11-20 13:05:32 -07006838 break;
6839 case IORING_OP_RECVMSG:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006840 ret = io_recvmsg(req, issue_flags);
Pavel Begunkov062d04d2020-10-10 18:34:12 +01006841 break;
Jens Axboefddafac2020-01-04 20:19:44 -07006842 case IORING_OP_RECV:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006843 ret = io_recv(req, issue_flags);
Jens Axboeb76da702019-11-20 13:05:32 -07006844 break;
Jens Axboe561fb042019-10-24 07:25:42 -06006845 case IORING_OP_TIMEOUT:
Pavel Begunkov61e98202021-02-10 00:03:08 +00006846 ret = io_timeout(req, issue_flags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006847 break;
6848 case IORING_OP_TIMEOUT_REMOVE:
Pavel Begunkov61e98202021-02-10 00:03:08 +00006849 ret = io_timeout_remove(req, issue_flags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006850 break;
6851 case IORING_OP_ACCEPT:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006852 ret = io_accept(req, issue_flags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006853 break;
6854 case IORING_OP_CONNECT:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006855 ret = io_connect(req, issue_flags);
Jens Axboe31b51512019-01-18 22:56:34 -07006856 break;
6857 case IORING_OP_ASYNC_CANCEL:
Pavel Begunkov61e98202021-02-10 00:03:08 +00006858 ret = io_async_cancel(req, issue_flags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006859 break;
Jens Axboed63d1b52019-12-10 10:38:56 -07006860 case IORING_OP_FALLOCATE:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006861 ret = io_fallocate(req, issue_flags);
Jens Axboed63d1b52019-12-10 10:38:56 -07006862 break;
Jens Axboe15b71ab2019-12-11 11:20:36 -07006863 case IORING_OP_OPENAT:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006864 ret = io_openat(req, issue_flags);
Jens Axboe15b71ab2019-12-11 11:20:36 -07006865 break;
Jens Axboeb5dba592019-12-11 14:02:38 -07006866 case IORING_OP_CLOSE:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006867 ret = io_close(req, issue_flags);
Jens Axboeb5dba592019-12-11 14:02:38 -07006868 break;
Jens Axboe05f3fb32019-12-09 11:22:50 -07006869 case IORING_OP_FILES_UPDATE:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006870 ret = io_files_update(req, issue_flags);
Jens Axboe05f3fb32019-12-09 11:22:50 -07006871 break;
Jens Axboeeddc7ef2019-12-13 21:18:10 -07006872 case IORING_OP_STATX:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006873 ret = io_statx(req, issue_flags);
Jens Axboeeddc7ef2019-12-13 21:18:10 -07006874 break;
Jens Axboe4840e412019-12-25 22:03:45 -07006875 case IORING_OP_FADVISE:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006876 ret = io_fadvise(req, issue_flags);
Jens Axboe4840e412019-12-25 22:03:45 -07006877 break;
Jens Axboec1ca7572019-12-25 22:18:28 -07006878 case IORING_OP_MADVISE:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006879 ret = io_madvise(req, issue_flags);
Jens Axboec1ca7572019-12-25 22:18:28 -07006880 break;
Jens Axboecebdb982020-01-08 17:59:24 -07006881 case IORING_OP_OPENAT2:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006882 ret = io_openat2(req, issue_flags);
Jens Axboecebdb982020-01-08 17:59:24 -07006883 break;
Jens Axboe3e4827b2020-01-08 15:18:09 -07006884 case IORING_OP_EPOLL_CTL:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006885 ret = io_epoll_ctl(req, issue_flags);
Jens Axboe3e4827b2020-01-08 15:18:09 -07006886 break;
Pavel Begunkov7d67af22020-02-24 11:32:45 +03006887 case IORING_OP_SPLICE:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006888 ret = io_splice(req, issue_flags);
Pavel Begunkov7d67af22020-02-24 11:32:45 +03006889 break;
Jens Axboeddf0322d2020-02-23 16:41:33 -07006890 case IORING_OP_PROVIDE_BUFFERS:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006891 ret = io_provide_buffers(req, issue_flags);
Jens Axboeddf0322d2020-02-23 16:41:33 -07006892 break;
Jens Axboe067524e2020-03-02 16:32:28 -07006893 case IORING_OP_REMOVE_BUFFERS:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006894 ret = io_remove_buffers(req, issue_flags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006895 break;
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03006896 case IORING_OP_TEE:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006897 ret = io_tee(req, issue_flags);
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03006898 break;
Jens Axboe36f4fa62020-09-05 11:14:22 -06006899 case IORING_OP_SHUTDOWN:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006900 ret = io_shutdown(req, issue_flags);
Jens Axboe36f4fa62020-09-05 11:14:22 -06006901 break;
Jens Axboe80a261f2020-09-28 14:23:58 -06006902 case IORING_OP_RENAMEAT:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006903 ret = io_renameat(req, issue_flags);
Jens Axboe80a261f2020-09-28 14:23:58 -06006904 break;
Jens Axboe14a11432020-09-28 14:27:37 -06006905 case IORING_OP_UNLINKAT:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006906 ret = io_unlinkat(req, issue_flags);
Jens Axboe14a11432020-09-28 14:27:37 -06006907 break;
Dmitry Kadasheve34a02d2021-07-08 13:34:45 +07006908 case IORING_OP_MKDIRAT:
6909 ret = io_mkdirat(req, issue_flags);
6910 break;
Dmitry Kadashev7a8721f2021-07-08 13:34:46 +07006911 case IORING_OP_SYMLINKAT:
6912 ret = io_symlinkat(req, issue_flags);
6913 break;
Dmitry Kadashevcf30da92021-07-08 13:34:47 +07006914 case IORING_OP_LINKAT:
6915 ret = io_linkat(req, issue_flags);
6916 break;
Jens Axboe2b188cc2019-01-07 10:46:33 -07006917 default:
6918 ret = -EINVAL;
6919 break;
6920 }
Jens Axboe31b51512019-01-18 22:56:34 -07006921
Jens Axboe5730b272021-02-27 15:57:30 -07006922 if (creds)
6923 revert_creds(creds);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006924 if (ret)
6925 return ret;
Jens Axboeb5325762020-05-19 21:20:27 -06006926 /* If the op doesn't have a file, we're not polling for it */
Pavel Begunkovcb3d8972021-06-14 02:36:14 +01006927 if ((ctx->flags & IORING_SETUP_IOPOLL) && req->file)
6928 io_iopoll_req_issued(req);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006929
6930 return 0;
6931}
6932
Pavel Begunkovebc11b62021-08-09 13:04:05 +01006933static struct io_wq_work *io_wq_free_work(struct io_wq_work *work)
6934{
6935 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
6936
6937 req = io_put_req_find_next(req);
6938 return req ? &req->work : NULL;
6939}
6940
Pavel Begunkov5280f7e2021-02-04 13:52:08 +00006941static void io_wq_submit_work(struct io_wq_work *work)
Pavel Begunkovd4c81f32020-06-08 21:08:19 +03006942{
Jens Axboe2b188cc2019-01-07 10:46:33 -07006943 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
Pavel Begunkov6df1db62020-07-03 22:15:06 +03006944 struct io_kiocb *timeout;
Jens Axboe561fb042019-10-24 07:25:42 -06006945 int ret = 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07006946
Pavel Begunkov48dcd382021-08-15 10:40:18 +01006947 /* one will be dropped by ->io_free_work() after returning to io-wq */
6948 if (!(req->flags & REQ_F_REFCOUNT))
6949 __io_req_set_refcount(req, 2);
6950 else
6951 req_ref_get(req);
Pavel Begunkov5d5901a2021-08-11 19:28:29 +01006952
Pavel Begunkov6df1db62020-07-03 22:15:06 +03006953 timeout = io_prep_linked_timeout(req);
6954 if (timeout)
6955 io_queue_linked_timeout(timeout);
Pavel Begunkovd4c81f32020-06-08 21:08:19 +03006956
Pavel Begunkovdadebc32021-08-23 13:30:44 +01006957 /* either cancelled or io-wq is dying, so don't touch tctx->iowq */
Jens Axboe4014d942021-01-19 15:53:54 -07006958 if (work->flags & IO_WQ_WORK_CANCEL)
Jens Axboe561fb042019-10-24 07:25:42 -06006959 ret = -ECANCELED;
Jens Axboe31b51512019-01-18 22:56:34 -07006960
Jens Axboe561fb042019-10-24 07:25:42 -06006961 if (!ret) {
Jens Axboe561fb042019-10-24 07:25:42 -06006962 do {
Pavel Begunkov889fca72021-02-10 00:03:09 +00006963 ret = io_issue_sqe(req, 0);
Jens Axboe561fb042019-10-24 07:25:42 -06006964 /*
6965 * We can get EAGAIN for polled IO even though we're
6966 * forcing a sync submission from here, since we can't
6967 * wait for request slots on the block side.
6968 */
Pavel Begunkov51ebf1b2022-05-13 11:24:56 +01006969 if (ret != -EAGAIN || !(req->ctx->flags & IORING_SETUP_IOPOLL))
Jens Axboe561fb042019-10-24 07:25:42 -06006970 break;
6971 cond_resched();
6972 } while (1);
6973 }
Jens Axboe31b51512019-01-18 22:56:34 -07006974
Pavel Begunkova3df76982021-02-18 22:32:52 +00006975 /* avoid locking problems by failing it from a clean context */
Pavel Begunkov5d5901a2021-08-11 19:28:29 +01006976 if (ret)
Pavel Begunkova3df76982021-02-18 22:32:52 +00006977 io_req_task_queue_fail(req, ret);
Jens Axboe31b51512019-01-18 22:56:34 -07006978}
Jens Axboe2b188cc2019-01-07 10:46:33 -07006979
Pavel Begunkovaeca2412021-04-11 01:46:37 +01006980static inline struct io_fixed_file *io_fixed_file_slot(struct io_file_table *table,
Pavel Begunkov042b0d82021-08-09 13:04:01 +01006981 unsigned i)
Jens Axboe09bb8392019-03-13 12:39:28 -06006982{
Pavel Begunkov042b0d82021-08-09 13:04:01 +01006983 return &table->files[i];
Pavel Begunkovdafecf12021-02-28 22:35:11 +00006984}
6985
Jens Axboe09bb8392019-03-13 12:39:28 -06006986static inline struct file *io_file_from_index(struct io_ring_ctx *ctx,
6987 int index)
6988{
Pavel Begunkovaeca2412021-04-11 01:46:37 +01006989 struct io_fixed_file *slot = io_fixed_file_slot(&ctx->file_table, index);
Jens Axboe65e19f52019-10-26 07:20:21 -06006990
Pavel Begunkova04b0ac2021-04-01 15:44:04 +01006991 return (struct file *) (slot->file_ptr & FFS_MASK);
Jens Axboe65e19f52019-10-26 07:20:21 -06006992}
6993
Pavel Begunkova04b0ac2021-04-01 15:44:04 +01006994static void io_fixed_file_set(struct io_fixed_file *file_slot, struct file *file)
Pavel Begunkov9a321c92021-04-01 15:44:01 +01006995{
6996 unsigned long file_ptr = (unsigned long) file;
6997
Pavel Begunkovb191e2d2021-08-09 13:04:03 +01006998 if (__io_file_supports_nowait(file, READ))
Pavel Begunkov9a321c92021-04-01 15:44:01 +01006999 file_ptr |= FFS_ASYNC_READ;
Pavel Begunkovb191e2d2021-08-09 13:04:03 +01007000 if (__io_file_supports_nowait(file, WRITE))
Pavel Begunkov9a321c92021-04-01 15:44:01 +01007001 file_ptr |= FFS_ASYNC_WRITE;
7002 if (S_ISREG(file_inode(file)->i_mode))
7003 file_ptr |= FFS_ISREG;
Pavel Begunkova04b0ac2021-04-01 15:44:04 +01007004 file_slot->file_ptr = file_ptr;
Jens Axboe09bb8392019-03-13 12:39:28 -06007005}
7006
Pavel Begunkovac177052021-08-09 13:04:02 +01007007static inline struct file *io_file_get_fixed(struct io_ring_ctx *ctx,
7008 struct io_kiocb *req, int fd)
Pavel Begunkov8da11c12020-02-24 11:32:44 +03007009{
Pavel Begunkov8da11c12020-02-24 11:32:44 +03007010 struct file *file;
Pavel Begunkovac177052021-08-09 13:04:02 +01007011 unsigned long file_ptr;
Pavel Begunkov8da11c12020-02-24 11:32:44 +03007012
Pavel Begunkovac177052021-08-09 13:04:02 +01007013 if (unlikely((unsigned int)fd >= ctx->nr_user_files))
7014 return NULL;
7015 fd = array_index_nospec(fd, ctx->nr_user_files);
7016 file_ptr = io_fixed_file_slot(&ctx->file_table, fd)->file_ptr;
7017 file = (struct file *) (file_ptr & FFS_MASK);
7018 file_ptr &= ~FFS_MASK;
7019 /* mask in overlapping REQ_F and FFS bits */
Pavel Begunkovb191e2d2021-08-09 13:04:03 +01007020 req->flags |= (file_ptr << REQ_F_NOWAIT_READ_BIT);
Pavel Begunkovac177052021-08-09 13:04:02 +01007021 io_req_set_rsrc_node(req);
Pavel Begunkov8371adf2020-10-10 18:34:08 +01007022 return file;
Pavel Begunkov8da11c12020-02-24 11:32:44 +03007023}
7024
Pavel Begunkovac177052021-08-09 13:04:02 +01007025static struct file *io_file_get_normal(struct io_ring_ctx *ctx,
Pavel Begunkovac177052021-08-09 13:04:02 +01007026 struct io_kiocb *req, int fd)
7027{
Pavel Begunkov62906e82021-08-10 14:52:47 +01007028 struct file *file = fget(fd);
Pavel Begunkovac177052021-08-09 13:04:02 +01007029
7030 trace_io_uring_file_get(ctx, fd);
7031
7032 /* we don't allow fixed io_uring files */
7033 if (file && unlikely(file->f_op == &io_uring_fops))
7034 io_req_track_inflight(req);
7035 return file;
7036}
7037
7038static inline struct file *io_file_get(struct io_ring_ctx *ctx,
Pavel Begunkovac177052021-08-09 13:04:02 +01007039 struct io_kiocb *req, int fd, bool fixed)
7040{
7041 if (fixed)
7042 return io_file_get_fixed(ctx, req, fd);
7043 else
Pavel Begunkov62906e82021-08-10 14:52:47 +01007044 return io_file_get_normal(ctx, req, fd);
Pavel Begunkovac177052021-08-09 13:04:02 +01007045}
7046
Pavel Begunkovf237c302021-08-18 12:42:46 +01007047static void io_req_task_link_timeout(struct io_kiocb *req, bool *locked)
Jens Axboe89b263f2021-08-10 15:14:18 -06007048{
7049 struct io_kiocb *prev = req->timeout.prev;
Pavel Begunkov3d2a1e62021-11-26 14:38:14 +00007050 int ret = -ENOENT;
Jens Axboe89b263f2021-08-10 15:14:18 -06007051
7052 if (prev) {
Pavel Begunkov3d2a1e62021-11-26 14:38:14 +00007053 if (!(req->task->flags & PF_EXITING))
7054 ret = io_try_cancel_userdata(req, prev->user_data);
Pavel Begunkov505657b2021-08-17 20:28:09 +01007055 io_req_complete_post(req, ret ?: -ETIME, 0);
Jens Axboe89b263f2021-08-10 15:14:18 -06007056 io_put_req(prev);
Jens Axboe89b263f2021-08-10 15:14:18 -06007057 } else {
7058 io_req_complete_post(req, -ETIME, 0);
7059 }
7060}
7061
Jens Axboe2665abf2019-11-05 12:40:47 -07007062static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer)
7063{
Jens Axboead8a48a2019-11-15 08:49:11 -07007064 struct io_timeout_data *data = container_of(timer,
7065 struct io_timeout_data, timer);
Pavel Begunkov90cd7e42020-10-27 23:25:36 +00007066 struct io_kiocb *prev, *req = data->req;
Jens Axboe2665abf2019-11-05 12:40:47 -07007067 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe2665abf2019-11-05 12:40:47 -07007068 unsigned long flags;
Jens Axboe2665abf2019-11-05 12:40:47 -07007069
Jens Axboe89b263f2021-08-10 15:14:18 -06007070 spin_lock_irqsave(&ctx->timeout_lock, flags);
Pavel Begunkov90cd7e42020-10-27 23:25:36 +00007071 prev = req->timeout.head;
7072 req->timeout.head = NULL;
Jens Axboe2665abf2019-11-05 12:40:47 -07007073
7074 /*
7075 * We don't expect the list to be empty, that will only happen if we
7076 * race with the completion of the linked work.
7077 */
Pavel Begunkov447c19f2021-05-14 12:02:50 +01007078 if (prev) {
Pavel Begunkovf2f87372020-10-27 23:25:37 +00007079 io_remove_next_linked(prev);
Pavel Begunkov447c19f2021-05-14 12:02:50 +01007080 if (!req_ref_inc_not_zero(prev))
7081 prev = NULL;
7082 }
Pavel Begunkovef9dd632021-08-28 19:54:38 -06007083 list_del(&req->timeout.list);
Jens Axboe89b263f2021-08-10 15:14:18 -06007084 req->timeout.prev = prev;
7085 spin_unlock_irqrestore(&ctx->timeout_lock, flags);
Jens Axboe2665abf2019-11-05 12:40:47 -07007086
Jens Axboe89b263f2021-08-10 15:14:18 -06007087 req->io_task_work.func = io_req_task_link_timeout;
7088 io_req_task_work_add(req);
Jens Axboe2665abf2019-11-05 12:40:47 -07007089 return HRTIMER_NORESTART;
7090}
7091
Pavel Begunkovde968c12021-03-19 17:22:33 +00007092static void io_queue_linked_timeout(struct io_kiocb *req)
Jens Axboe2665abf2019-11-05 12:40:47 -07007093{
Pavel Begunkovde968c12021-03-19 17:22:33 +00007094 struct io_ring_ctx *ctx = req->ctx;
7095
Jens Axboe89b263f2021-08-10 15:14:18 -06007096 spin_lock_irq(&ctx->timeout_lock);
Jens Axboe76a46e02019-11-10 23:34:16 -07007097 /*
Pavel Begunkovf2f87372020-10-27 23:25:37 +00007098 * If the back reference is NULL, then our linked request finished
7099 * before we got a chance to setup the timer
Jens Axboe76a46e02019-11-10 23:34:16 -07007100 */
Pavel Begunkov90cd7e42020-10-27 23:25:36 +00007101 if (req->timeout.head) {
Jens Axboee8c2bc12020-08-15 18:44:09 -07007102 struct io_timeout_data *data = req->async_data;
Jens Axboe94ae5e72019-11-14 19:39:52 -07007103
Jens Axboead8a48a2019-11-15 08:49:11 -07007104 data->timer.function = io_link_timeout_fn;
7105 hrtimer_start(&data->timer, timespec64_to_ktime(data->ts),
7106 data->mode);
Pavel Begunkovef9dd632021-08-28 19:54:38 -06007107 list_add_tail(&req->timeout.list, &ctx->ltimeout_list);
Jens Axboe2665abf2019-11-05 12:40:47 -07007108 }
Jens Axboe89b263f2021-08-10 15:14:18 -06007109 spin_unlock_irq(&ctx->timeout_lock);
Jens Axboe2665abf2019-11-05 12:40:47 -07007110 /* drop submission reference */
Jens Axboe76a46e02019-11-10 23:34:16 -07007111 io_put_req(req);
Jens Axboe2665abf2019-11-05 12:40:47 -07007112}
7113
Pavel Begunkovc5eef2b2021-02-10 00:03:22 +00007114static void __io_queue_sqe(struct io_kiocb *req)
Pavel Begunkov282cdc82021-08-09 13:04:10 +01007115 __must_hold(&req->ctx->uring_lock)
Jens Axboe2b188cc2019-01-07 10:46:33 -07007116{
Pavel Begunkov906c6ca2021-08-15 10:40:26 +01007117 struct io_kiocb *linked_timeout;
Jens Axboee0c5c572019-03-12 10:18:47 -06007118 int ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07007119
Olivier Langlois59b735a2021-06-22 05:17:39 -07007120issue_sqe:
Pavel Begunkovc5eef2b2021-02-10 00:03:22 +00007121 ret = io_issue_sqe(req, IO_URING_F_NONBLOCK|IO_URING_F_COMPLETE_DEFER);
Jens Axboe491381ce2019-10-17 09:20:46 -06007122
7123 /*
7124 * We async punt it if the file wasn't marked NOWAIT, or if the file
7125 * doesn't support non-blocking read/write attempts
7126 */
Pavel Begunkov18400382021-03-19 17:22:34 +00007127 if (likely(!ret)) {
Pavel Begunkove342c802021-01-19 13:32:47 +00007128 if (req->flags & REQ_F_COMPLETE_INLINE) {
Pavel Begunkovc5eef2b2021-02-10 00:03:22 +00007129 struct io_ring_ctx *ctx = req->ctx;
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01007130 struct io_submit_state *state = &ctx->submit_state;
Jens Axboee65ef562019-03-12 10:16:44 -06007131
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01007132 state->compl_reqs[state->compl_nr++] = req;
7133 if (state->compl_nr == ARRAY_SIZE(state->compl_reqs))
Pavel Begunkov2a2758f2021-06-17 18:14:00 +01007134 io_submit_flush_completions(ctx);
Pavel Begunkov906c6ca2021-08-15 10:40:26 +01007135 return;
Pavel Begunkov0d63c142020-10-22 16:47:18 +01007136 }
Pavel Begunkov906c6ca2021-08-15 10:40:26 +01007137
7138 linked_timeout = io_prep_linked_timeout(req);
7139 if (linked_timeout)
7140 io_queue_linked_timeout(linked_timeout);
Pavel Begunkov18400382021-03-19 17:22:34 +00007141 } else if (ret == -EAGAIN && !(req->flags & REQ_F_NOWAIT)) {
Pavel Begunkov906c6ca2021-08-15 10:40:26 +01007142 linked_timeout = io_prep_linked_timeout(req);
7143
Olivier Langlois59b735a2021-06-22 05:17:39 -07007144 switch (io_arm_poll_handler(req)) {
7145 case IO_APOLL_READY:
Pavel Begunkov906c6ca2021-08-15 10:40:26 +01007146 if (linked_timeout)
Pavel Begunkov4ea672a2021-10-20 09:53:02 +01007147 io_queue_linked_timeout(linked_timeout);
Olivier Langlois59b735a2021-06-22 05:17:39 -07007148 goto issue_sqe;
7149 case IO_APOLL_ABORTED:
Pavel Begunkov18400382021-03-19 17:22:34 +00007150 /*
7151 * Queued up for async execution, worker will release
7152 * submit reference when the iocb is actually submitted.
7153 */
Pavel Begunkovf237c302021-08-18 12:42:46 +01007154 io_queue_async_work(req, NULL);
Olivier Langlois59b735a2021-06-22 05:17:39 -07007155 break;
Pavel Begunkov18400382021-03-19 17:22:34 +00007156 }
Pavel Begunkov906c6ca2021-08-15 10:40:26 +01007157
7158 if (linked_timeout)
7159 io_queue_linked_timeout(linked_timeout);
Pavel Begunkov0d63c142020-10-22 16:47:18 +01007160 } else {
Pavel Begunkovf41db2732021-02-28 22:35:12 +00007161 io_req_complete_failed(req, ret);
Jens Axboe9e645e112019-05-10 16:07:28 -06007162 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07007163}
7164
Pavel Begunkov441b8a72021-06-14 23:37:31 +01007165static inline void io_queue_sqe(struct io_kiocb *req)
Pavel Begunkov282cdc82021-08-09 13:04:10 +01007166 __must_hold(&req->ctx->uring_lock)
Jackie Liu4fe2c962019-09-09 20:50:40 +08007167{
Pavel Begunkov10c66902021-06-15 16:47:56 +01007168 if (unlikely(req->ctx->drain_active) && io_drain_req(req))
Pavel Begunkov76cc33d2021-06-14 23:37:30 +01007169 return;
Jackie Liu4fe2c962019-09-09 20:50:40 +08007170
Hao Xua8295b92021-08-27 17:46:09 +08007171 if (likely(!(req->flags & (REQ_F_FORCE_ASYNC | REQ_F_FAIL)))) {
Pavel Begunkovc5eef2b2021-02-10 00:03:22 +00007172 __io_queue_sqe(req);
Hao Xua8295b92021-08-27 17:46:09 +08007173 } else if (req->flags & REQ_F_FAIL) {
Pavel Begunkovc6d3d9c2021-08-31 14:13:10 +01007174 io_req_complete_fail_submit(req);
Pavel Begunkov76cc33d2021-06-14 23:37:30 +01007175 } else {
7176 int ret = io_req_prep_async(req);
7177
7178 if (unlikely(ret))
7179 io_req_complete_failed(req, ret);
7180 else
Pavel Begunkovf237c302021-08-18 12:42:46 +01007181 io_queue_async_work(req, NULL);
Jens Axboece35a472019-12-17 08:04:44 -07007182 }
Jackie Liu4fe2c962019-09-09 20:50:40 +08007183}
7184
Stefano Garzarella21b55db2020-08-27 16:58:30 +02007185/*
7186 * Check SQE restrictions (opcode and flags).
7187 *
7188 * Returns 'true' if SQE is allowed, 'false' otherwise.
7189 */
7190static inline bool io_check_restriction(struct io_ring_ctx *ctx,
7191 struct io_kiocb *req,
7192 unsigned int sqe_flags)
7193{
Pavel Begunkov4cfb25b2021-06-26 21:40:47 +01007194 if (likely(!ctx->restricted))
Stefano Garzarella21b55db2020-08-27 16:58:30 +02007195 return true;
7196
7197 if (!test_bit(req->opcode, ctx->restrictions.sqe_op))
7198 return false;
7199
7200 if ((sqe_flags & ctx->restrictions.sqe_flags_required) !=
7201 ctx->restrictions.sqe_flags_required)
7202 return false;
7203
7204 if (sqe_flags & ~(ctx->restrictions.sqe_flags_allowed |
7205 ctx->restrictions.sqe_flags_required))
7206 return false;
7207
7208 return true;
7209}
7210
Pavel Begunkovef4ff582020-04-12 02:05:05 +03007211static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,
Pavel Begunkov258b29a2021-02-10 00:03:10 +00007212 const struct io_uring_sqe *sqe)
Pavel Begunkov282cdc82021-08-09 13:04:10 +01007213 __must_hold(&ctx->uring_lock)
Pavel Begunkov0553b8b2020-04-08 08:58:45 +03007214{
Pavel Begunkov258b29a2021-02-10 00:03:10 +00007215 struct io_submit_state *state;
Pavel Begunkovef4ff582020-04-12 02:05:05 +03007216 unsigned int sqe_flags;
Jens Axboe003e8dc2021-03-06 09:22:27 -07007217 int personality, ret = 0;
Pavel Begunkovef4ff582020-04-12 02:05:05 +03007218
Pavel Begunkov864ea922021-08-09 13:04:08 +01007219 /* req is partially pre-initialised, see io_preinit_req() */
Pavel Begunkov0553b8b2020-04-08 08:58:45 +03007220 req->opcode = READ_ONCE(sqe->opcode);
Pavel Begunkov5be9ad12021-02-12 18:41:17 +00007221 /* same numerical values with corresponding REQ_F_*, safe to copy */
7222 req->flags = sqe_flags = READ_ONCE(sqe->flags);
Pavel Begunkov0553b8b2020-04-08 08:58:45 +03007223 req->user_data = READ_ONCE(sqe->user_data);
Pavel Begunkov0553b8b2020-04-08 08:58:45 +03007224 req->file = NULL;
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00007225 req->fixed_rsrc_refs = NULL;
Pavel Begunkov4dd28242020-06-15 10:33:13 +03007226 req->task = current;
Pavel Begunkovef4ff582020-04-12 02:05:05 +03007227
Pavel Begunkov5be9ad12021-02-12 18:41:17 +00007228 /* enforce forwards compatibility on users */
Pavel Begunkovdddca222021-04-27 16:13:52 +01007229 if (unlikely(sqe_flags & ~SQE_VALID_FLAGS))
Pavel Begunkov5be9ad12021-02-12 18:41:17 +00007230 return -EINVAL;
Pavel Begunkovef4ff582020-04-12 02:05:05 +03007231 if (unlikely(req->opcode >= IORING_OP_LAST))
7232 return -EINVAL;
Pavel Begunkov4cfb25b2021-06-26 21:40:47 +01007233 if (!io_check_restriction(ctx, req, sqe_flags))
Stefano Garzarella21b55db2020-08-27 16:58:30 +02007234 return -EACCES;
7235
Pavel Begunkovef4ff582020-04-12 02:05:05 +03007236 if ((sqe_flags & IOSQE_BUFFER_SELECT) &&
7237 !io_op_defs[req->opcode].buffer_select)
7238 return -EOPNOTSUPP;
Pavel Begunkov3c199662021-06-15 16:47:57 +01007239 if (unlikely(sqe_flags & IOSQE_IO_DRAIN))
7240 ctx->drain_active = true;
Pavel Begunkovef4ff582020-04-12 02:05:05 +03007241
Jens Axboe003e8dc2021-03-06 09:22:27 -07007242 personality = READ_ONCE(sqe->personality);
7243 if (personality) {
Pavel Begunkovc10d1f92021-06-17 18:14:01 +01007244 req->creds = xa_load(&ctx->personalities, personality);
7245 if (!req->creds)
Jens Axboe003e8dc2021-03-06 09:22:27 -07007246 return -EINVAL;
Pavel Begunkovc10d1f92021-06-17 18:14:01 +01007247 get_cred(req->creds);
Pavel Begunkovb8e64b52021-06-17 18:14:02 +01007248 req->flags |= REQ_F_CREDS;
Jens Axboe003e8dc2021-03-06 09:22:27 -07007249 }
Pavel Begunkov258b29a2021-02-10 00:03:10 +00007250 state = &ctx->submit_state;
Pavel Begunkovef4ff582020-04-12 02:05:05 +03007251
Jens Axboe27926b62020-10-28 09:33:23 -06007252 /*
7253 * Plug now if we have more than 1 IO left after this, and the target
7254 * is potentially a read/write to block based storage.
7255 */
7256 if (!state->plug_started && state->ios_left > 1 &&
7257 io_op_defs[req->opcode].plug) {
7258 blk_start_plug(&state->plug);
7259 state->plug_started = true;
7260 }
Jens Axboe63ff8222020-05-07 14:56:15 -06007261
Pavel Begunkovbd5bbda2020-11-20 15:50:51 +00007262 if (io_op_defs[req->opcode].needs_file) {
Pavel Begunkov62906e82021-08-10 14:52:47 +01007263 req->file = io_file_get(ctx, req, READ_ONCE(sqe->fd),
Pavel Begunkovac177052021-08-09 13:04:02 +01007264 (sqe_flags & IOSQE_FIXED_FILE));
Pavel Begunkovba13e232021-02-01 18:59:52 +00007265 if (unlikely(!req->file))
Pavel Begunkovbd5bbda2020-11-20 15:50:51 +00007266 ret = -EBADF;
7267 }
7268
Pavel Begunkov71b547c2020-10-10 18:34:09 +01007269 state->ios_left--;
7270 return ret;
Pavel Begunkov0553b8b2020-04-08 08:58:45 +03007271}
7272
Pavel Begunkova6b8cadc2021-02-18 18:29:41 +00007273static int io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,
Pavel Begunkova1ab7b32021-02-18 18:29:42 +00007274 const struct io_uring_sqe *sqe)
Pavel Begunkov282cdc82021-08-09 13:04:10 +01007275 __must_hold(&ctx->uring_lock)
Jens Axboe6c271ce2019-01-10 11:22:30 -07007276{
Pavel Begunkova1ab7b32021-02-18 18:29:42 +00007277 struct io_submit_link *link = &ctx->submit_state.link;
Jens Axboe6c271ce2019-01-10 11:22:30 -07007278 int ret;
7279
Pavel Begunkova6b8cadc2021-02-18 18:29:41 +00007280 ret = io_init_req(ctx, req, sqe);
7281 if (unlikely(ret)) {
7282fail_req:
Hao Xua8295b92021-08-27 17:46:09 +08007283 /* fail even hard links since we don't submit */
Pavel Begunkovde59bc12021-02-18 18:29:47 +00007284 if (link->head) {
Hao Xua8295b92021-08-27 17:46:09 +08007285 /*
7286 * we can judge a link req is failed or cancelled by if
7287 * REQ_F_FAIL is set, but the head is an exception since
7288 * it may be set REQ_F_FAIL because of other req's failure
7289 * so let's leverage req->result to distinguish if a head
7290 * is set REQ_F_FAIL because of its failure or other req's
7291 * failure so that we can set the correct ret code for it.
7292 * init result here to avoid affecting the normal path.
7293 */
7294 if (!(link->head->flags & REQ_F_FAIL))
7295 req_fail_link_node(link->head, -ECANCELED);
7296 } else if (!(req->flags & (REQ_F_LINK | REQ_F_HARDLINK))) {
7297 /*
7298 * the current req is a normal req, we should return
7299 * error and thus break the submittion loop.
7300 */
7301 io_req_complete_failed(req, ret);
7302 return ret;
Pavel Begunkovde59bc12021-02-18 18:29:47 +00007303 }
Hao Xua8295b92021-08-27 17:46:09 +08007304 req_fail_link_node(req, ret);
7305 } else {
7306 ret = io_req_prep(req, sqe);
7307 if (unlikely(ret))
7308 goto fail_req;
Pavel Begunkova6b8cadc2021-02-18 18:29:41 +00007309 }
Pavel Begunkov441b8a72021-06-14 23:37:31 +01007310
Pavel Begunkovbe7053b2021-02-18 18:29:45 +00007311 /* don't need @sqe from now on */
Olivier Langlois236daeae2021-05-31 02:36:37 -04007312 trace_io_uring_submit_sqe(ctx, req, req->opcode, req->user_data,
7313 req->flags, true,
7314 ctx->flags & IORING_SETUP_SQPOLL);
Pavel Begunkova6b8cadc2021-02-18 18:29:41 +00007315
Jens Axboe6c271ce2019-01-10 11:22:30 -07007316 /*
7317 * If we already have a head request, queue this one for async
7318 * submittal once the head completes. If we don't have a head but
7319 * IOSQE_IO_LINK is set in the sqe, start a new head. This one will be
7320 * submitted sync once the chain is complete. If none of those
7321 * conditions are true (normal request), then just queue it.
7322 */
7323 if (link->head) {
7324 struct io_kiocb *head = link->head;
7325
Hao Xua8295b92021-08-27 17:46:09 +08007326 if (!(req->flags & REQ_F_FAIL)) {
7327 ret = io_req_prep_async(req);
7328 if (unlikely(ret)) {
7329 req_fail_link_node(req, ret);
7330 if (!(head->flags & REQ_F_FAIL))
7331 req_fail_link_node(head, -ECANCELED);
7332 }
7333 }
Jens Axboe6c271ce2019-01-10 11:22:30 -07007334 trace_io_uring_link(ctx, req, head);
7335 link->last->link = req;
7336 link->last = req;
7337
7338 /* last request of a link, enqueue the link */
7339 if (!(req->flags & (REQ_F_LINK | REQ_F_HARDLINK))) {
7340 link->head = NULL;
Pavel Begunkov5e159202021-06-14 23:37:26 +01007341 io_queue_sqe(head);
Jens Axboe2b188cc2019-01-07 10:46:33 -07007342 }
Jackie Liu4fe2c962019-09-09 20:50:40 +08007343 } else {
Jens Axboe2b188cc2019-01-07 10:46:33 -07007344 if (req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) {
Jackie Liu4fe2c962019-09-09 20:50:40 +08007345 link->head = req;
7346 link->last = req;
7347 } else {
Pavel Begunkovbe7053b2021-02-18 18:29:45 +00007348 io_queue_sqe(req);
Jackie Liu4fe2c962019-09-09 20:50:40 +08007349 }
7350 }
7351
7352 return 0;
7353}
7354
7355/*
7356 * Batched submission is done, ensure local IO is flushed out.
7357 */
7358static void io_submit_state_end(struct io_submit_state *state,
7359 struct io_ring_ctx *ctx)
Pavel Begunkov1b4a51b2019-11-21 11:54:28 +03007360{
Pavel Begunkova1ab7b32021-02-18 18:29:42 +00007361 if (state->link.head)
Pavel Begunkovde59bc12021-02-18 18:29:47 +00007362 io_queue_sqe(state->link.head);
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01007363 if (state->compl_nr)
Pavel Begunkov2a2758f2021-06-17 18:14:00 +01007364 io_submit_flush_completions(ctx);
Jackie Liua197f662019-11-08 08:09:12 -07007365 if (state->plug_started)
Pavel Begunkov32fe5252019-12-17 22:26:58 +03007366 blk_finish_plug(&state->plug);
Jens Axboe9e645e112019-05-10 16:07:28 -06007367}
Pavel Begunkov32fe5252019-12-17 22:26:58 +03007368
Jens Axboe9e645e112019-05-10 16:07:28 -06007369/*
7370 * Start submission side cache.
Pavel Begunkov32fe5252019-12-17 22:26:58 +03007371 */
Jens Axboe9e645e112019-05-10 16:07:28 -06007372static void io_submit_state_start(struct io_submit_state *state,
Pavel Begunkov196be952019-11-07 01:41:06 +03007373 unsigned int max_ios)
Jens Axboe9e645e112019-05-10 16:07:28 -06007374{
7375 state->plug_started = false;
Jens Axboebcda7ba2020-02-23 16:42:51 -07007376 state->ios_left = max_ios;
Pavel Begunkova1ab7b32021-02-18 18:29:42 +00007377 /* set only head, no need to init link_last in advance */
7378 state->link.head = NULL;
Jens Axboe75c6a032020-01-28 10:15:23 -07007379}
7380
Jens Axboe193155c2020-02-22 23:22:19 -07007381static void io_commit_sqring(struct io_ring_ctx *ctx)
7382{
Jens Axboe75c6a032020-01-28 10:15:23 -07007383 struct io_rings *rings = ctx->rings;
7384
7385 /*
Jens Axboe193155c2020-02-22 23:22:19 -07007386 * Ensure any loads from the SQEs are done at this point,
Jens Axboe75c6a032020-01-28 10:15:23 -07007387 * since once we write the new head, the application could
7388 * write new data to them.
Pavel Begunkov6b47ee62020-01-18 20:22:41 +03007389 */
Pavel Begunkov8da11c12020-02-24 11:32:44 +03007390 smp_store_release(&rings->sq.head, ctx->cached_sq_head);
Jens Axboebcda7ba2020-02-23 16:42:51 -07007391}
7392
Jens Axboe9e645e112019-05-10 16:07:28 -06007393/*
Fam Zhengdd9ae8a2021-06-04 17:42:56 +01007394 * Fetch an sqe, if one is available. Note this returns a pointer to memory
Jens Axboe9e645e112019-05-10 16:07:28 -06007395 * that is mapped by userspace. This means that care needs to be taken to
7396 * ensure that reads are stable, as we cannot rely on userspace always
Jens Axboe78e19bb2019-11-06 15:21:34 -07007397 * being a good citizen. If members of the sqe are validated and then later
7398 * used, it's important that those reads are done through READ_ONCE() to
Pavel Begunkov2e6e1fd2019-12-05 16:15:45 +03007399 * prevent a re-load down the line.
Jens Axboe9e645e112019-05-10 16:07:28 -06007400 */
7401static const struct io_uring_sqe *io_get_sqe(struct io_ring_ctx *ctx)
Jens Axboe9e645e112019-05-10 16:07:28 -06007402{
Pavel Begunkovea5ab3b2021-05-16 22:58:09 +01007403 unsigned head, mask = ctx->sq_entries - 1;
Pavel Begunkov17d3aeb2021-06-14 23:37:23 +01007404 unsigned sq_idx = ctx->cached_sq_head++ & mask;
Jens Axboe9e645e112019-05-10 16:07:28 -06007405
7406 /*
7407 * The cached sq head (or cq tail) serves two purposes:
7408 *
7409 * 1) allows us to batch the cost of updating the user visible
Pavel Begunkov9d763772019-12-17 02:22:07 +03007410 * head updates.
Jens Axboe9e645e112019-05-10 16:07:28 -06007411 * 2) allows the kernel side to track the head on its own, even
Pavel Begunkov8cdf2192020-01-25 00:40:24 +03007412 * though the application is the one updating it.
7413 */
Pavel Begunkov17d3aeb2021-06-14 23:37:23 +01007414 head = READ_ONCE(ctx->sq_array[sq_idx]);
Pavel Begunkov8cdf2192020-01-25 00:40:24 +03007415 if (likely(head < ctx->sq_entries))
7416 return &ctx->sq_sqes[head];
7417
7418 /* drop invalid entries */
Pavel Begunkov15641e42021-06-14 23:37:24 +01007419 ctx->cq_extra--;
7420 WRITE_ONCE(ctx->rings->sq_dropped,
7421 READ_ONCE(ctx->rings->sq_dropped) + 1);
Pavel Begunkov711be032020-01-17 03:57:59 +03007422 return NULL;
7423}
Jens Axboeb7bb4f72019-12-15 22:13:43 -07007424
Jens Axboe0f212202020-09-13 13:09:39 -06007425static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr)
Pavel Begunkov282cdc82021-08-09 13:04:10 +01007426 __must_hold(&ctx->uring_lock)
Jens Axboe6c271ce2019-01-10 11:22:30 -07007427{
Pavel Begunkov46c4e162021-02-18 18:29:37 +00007428 int submitted = 0;
Jens Axboe6c271ce2019-01-10 11:22:30 -07007429
Pavel Begunkovee7d46d2019-12-30 21:24:45 +03007430 /* make sure SQ entry isn't read before tail */
7431 nr = min3(nr, ctx->sq_entries, io_sqring_entries(ctx));
Pavel Begunkov2b85edf2019-12-28 14:13:03 +03007432 if (!percpu_ref_tryget_many(&ctx->refs, nr))
7433 return -EAGAIN;
Pavel Begunkov9a108672021-08-27 11:55:01 +01007434 io_get_task_refs(nr);
Jens Axboe6c271ce2019-01-10 11:22:30 -07007435
Pavel Begunkovba88ff12021-02-10 00:03:11 +00007436 io_submit_state_start(&ctx->submit_state, nr);
Pavel Begunkov46c4e162021-02-18 18:29:37 +00007437 while (submitted < nr) {
Jens Axboe3529d8c2019-12-19 18:24:38 -07007438 const struct io_uring_sqe *sqe;
Pavel Begunkov196be952019-11-07 01:41:06 +03007439 struct io_kiocb *req;
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03007440
Pavel Begunkov258b29a2021-02-10 00:03:10 +00007441 req = io_alloc_req(ctx);
Pavel Begunkov196be952019-11-07 01:41:06 +03007442 if (unlikely(!req)) {
7443 if (!submitted)
7444 submitted = -EAGAIN;
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03007445 break;
Jens Axboe9e645e112019-05-10 16:07:28 -06007446 }
Pavel Begunkov4fccfcb2021-02-12 11:55:17 +00007447 sqe = io_get_sqe(ctx);
7448 if (unlikely(!sqe)) {
Hao Xu0c6e1d72021-08-26 01:58:56 +08007449 list_add(&req->inflight_entry, &ctx->submit_state.free_list);
Pavel Begunkov4fccfcb2021-02-12 11:55:17 +00007450 break;
7451 }
Jens Axboed3656342019-12-18 09:50:26 -07007452 /* will complete beyond this point, count as submitted */
7453 submitted++;
Pavel Begunkova1ab7b32021-02-18 18:29:42 +00007454 if (io_submit_sqe(ctx, req, sqe))
Jens Axboed3656342019-12-18 09:50:26 -07007455 break;
Jens Axboe6c271ce2019-01-10 11:22:30 -07007456 }
7457
Pavel Begunkov9466f432020-01-25 22:34:01 +03007458 if (unlikely(submitted != nr)) {
7459 int ref_used = (submitted == -EAGAIN) ? 0 : submitted;
Jens Axboed8a6df12020-10-15 16:24:45 -06007460 int unused = nr - ref_used;
Pavel Begunkov9466f432020-01-25 22:34:01 +03007461
Pavel Begunkov09899b12021-06-14 02:36:22 +01007462 current->io_uring->cached_refs += unused;
Jens Axboed8a6df12020-10-15 16:24:45 -06007463 percpu_ref_put_many(&ctx->refs, unused);
Pavel Begunkov9466f432020-01-25 22:34:01 +03007464 }
Jens Axboe6c271ce2019-01-10 11:22:30 -07007465
Pavel Begunkova1ab7b32021-02-18 18:29:42 +00007466 io_submit_state_end(&ctx->submit_state, ctx);
Pavel Begunkovae9428c2019-11-06 00:22:14 +03007467 /* Commit SQ ring head once we've consumed and submitted all SQEs */
7468 io_commit_sqring(ctx);
7469
Jens Axboe6c271ce2019-01-10 11:22:30 -07007470 return submitted;
7471}
7472
Pavel Begunkove4b6d902021-05-16 22:58:00 +01007473static inline bool io_sqd_events_pending(struct io_sq_data *sqd)
7474{
7475 return READ_ONCE(sqd->state);
7476}
7477
Xiaoguang Wang23b36282020-07-23 20:57:24 +08007478static inline void io_ring_set_wakeup_flag(struct io_ring_ctx *ctx)
7479{
7480 /* Tell userspace we may need a wakeup call */
Jens Axboe79ebeae2021-08-10 15:18:27 -06007481 spin_lock(&ctx->completion_lock);
Nadav Amit20c0b382021-08-07 17:13:42 -07007482 WRITE_ONCE(ctx->rings->sq_flags,
7483 ctx->rings->sq_flags | IORING_SQ_NEED_WAKEUP);
Jens Axboe79ebeae2021-08-10 15:18:27 -06007484 spin_unlock(&ctx->completion_lock);
Xiaoguang Wang23b36282020-07-23 20:57:24 +08007485}
7486
7487static inline void io_ring_clear_wakeup_flag(struct io_ring_ctx *ctx)
7488{
Jens Axboe79ebeae2021-08-10 15:18:27 -06007489 spin_lock(&ctx->completion_lock);
Nadav Amit20c0b382021-08-07 17:13:42 -07007490 WRITE_ONCE(ctx->rings->sq_flags,
7491 ctx->rings->sq_flags & ~IORING_SQ_NEED_WAKEUP);
Jens Axboe79ebeae2021-08-10 15:18:27 -06007492 spin_unlock(&ctx->completion_lock);
Xiaoguang Wang23b36282020-07-23 20:57:24 +08007493}
7494
Xiaoguang Wang08369242020-11-03 14:15:59 +08007495static int __io_sq_thread(struct io_ring_ctx *ctx, bool cap_entries)
Jens Axboe6c271ce2019-01-10 11:22:30 -07007496{
Jens Axboec8d1ba52020-09-14 11:07:26 -06007497 unsigned int to_submit;
Xiaoguang Wangbdcd3ea2020-02-25 22:12:08 +08007498 int ret = 0;
Jens Axboe6c271ce2019-01-10 11:22:30 -07007499
Jens Axboec8d1ba52020-09-14 11:07:26 -06007500 to_submit = io_sqring_entries(ctx);
Jens Axboee95eee22020-09-08 09:11:32 -06007501 /* if we're handling multiple rings, cap submit size for fairness */
Olivier Langlois4ce8ad92021-06-23 11:50:18 -07007502 if (cap_entries && to_submit > IORING_SQPOLL_CAP_ENTRIES_VALUE)
7503 to_submit = IORING_SQPOLL_CAP_ENTRIES_VALUE;
Jens Axboee95eee22020-09-08 09:11:32 -06007504
Xiaoguang Wang906a3c62020-11-12 14:56:00 +08007505 if (!list_empty(&ctx->iopoll_list) || to_submit) {
7506 unsigned nr_events = 0;
Pavel Begunkov948e1942021-06-24 15:09:55 +01007507 const struct cred *creds = NULL;
7508
7509 if (ctx->sq_creds != current_cred())
7510 creds = override_creds(ctx->sq_creds);
Xiaoguang Wang906a3c62020-11-12 14:56:00 +08007511
Xiaoguang Wang08369242020-11-03 14:15:59 +08007512 mutex_lock(&ctx->uring_lock);
Xiaoguang Wang906a3c62020-11-12 14:56:00 +08007513 if (!list_empty(&ctx->iopoll_list))
Pavel Begunkova8576af2021-08-15 10:40:21 +01007514 io_do_iopoll(ctx, &nr_events, 0);
Xiaoguang Wang906a3c62020-11-12 14:56:00 +08007515
Pavel Begunkov3b763ba2021-04-18 14:52:08 +01007516 /*
7517 * Don't submit if refs are dying, good for io_uring_register(),
7518 * but also it is relied upon by io_ring_exit_work()
7519 */
Pavel Begunkov0298ef92021-03-08 13:20:57 +00007520 if (to_submit && likely(!percpu_ref_is_dying(&ctx->refs)) &&
7521 !(ctx->flags & IORING_SETUP_R_DISABLED))
Xiaoguang Wang08369242020-11-03 14:15:59 +08007522 ret = io_submit_sqes(ctx, to_submit);
7523 mutex_unlock(&ctx->uring_lock);
Jens Axboe90554202020-09-03 12:12:41 -06007524
Pavel Begunkovacfb3812021-05-16 22:58:03 +01007525 if (to_submit && wq_has_sleeper(&ctx->sqo_sq_wait))
7526 wake_up(&ctx->sqo_sq_wait);
Pavel Begunkov948e1942021-06-24 15:09:55 +01007527 if (creds)
7528 revert_creds(creds);
Pavel Begunkovacfb3812021-05-16 22:58:03 +01007529 }
Jens Axboe90554202020-09-03 12:12:41 -06007530
Xiaoguang Wang08369242020-11-03 14:15:59 +08007531 return ret;
7532}
7533
7534static void io_sqd_update_thread_idle(struct io_sq_data *sqd)
7535{
7536 struct io_ring_ctx *ctx;
7537 unsigned sq_thread_idle = 0;
7538
Pavel Begunkovc9dca272021-03-10 13:13:55 +00007539 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list)
7540 sq_thread_idle = max(sq_thread_idle, ctx->sq_thread_idle);
Xiaoguang Wang08369242020-11-03 14:15:59 +08007541 sqd->sq_thread_idle = sq_thread_idle;
Jens Axboec8d1ba52020-09-14 11:07:26 -06007542}
7543
Pavel Begunkove4b6d902021-05-16 22:58:00 +01007544static bool io_sqd_handle_event(struct io_sq_data *sqd)
7545{
7546 bool did_sig = false;
7547 struct ksignal ksig;
7548
7549 if (test_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state) ||
7550 signal_pending(current)) {
7551 mutex_unlock(&sqd->lock);
7552 if (signal_pending(current))
7553 did_sig = get_signal(&ksig);
7554 cond_resched();
7555 mutex_lock(&sqd->lock);
7556 }
Pavel Begunkove4b6d902021-05-16 22:58:00 +01007557 return did_sig || test_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state);
7558}
7559
Jens Axboe6c271ce2019-01-10 11:22:30 -07007560static int io_sq_thread(void *data)
7561{
Jens Axboe69fb2132020-09-14 11:16:23 -06007562 struct io_sq_data *sqd = data;
7563 struct io_ring_ctx *ctx;
Xiaoguang Wanga0d92052020-11-12 14:55:59 +08007564 unsigned long timeout = 0;
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007565 char buf[TASK_COMM_LEN];
Xiaoguang Wang08369242020-11-03 14:15:59 +08007566 DEFINE_WAIT(wait);
Jens Axboe6c271ce2019-01-10 11:22:30 -07007567
Pavel Begunkov696ee882021-04-01 09:55:04 +01007568 snprintf(buf, sizeof(buf), "iou-sqp-%d", sqd->task_pid);
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007569 set_task_comm(current, buf);
Jens Axboe28cea78a2020-09-14 10:51:17 -06007570
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007571 if (sqd->sq_cpu != -1)
7572 set_cpus_allowed_ptr(current, cpumask_of(sqd->sq_cpu));
7573 else
7574 set_cpus_allowed_ptr(current, cpu_online_mask);
7575 current->flags |= PF_NO_SETAFFINITY;
7576
Pavel Begunkov09a6f4e2021-03-14 20:57:10 +00007577 mutex_lock(&sqd->lock);
Pavel Begunkove4b6d902021-05-16 22:58:00 +01007578 while (1) {
Pavel Begunkov1a924a82021-06-24 15:09:56 +01007579 bool cap_entries, sqt_spin = false;
Jens Axboec1edbf52019-11-10 16:56:04 -07007580
Pavel Begunkove4b6d902021-05-16 22:58:00 +01007581 if (io_sqd_events_pending(sqd) || signal_pending(current)) {
7582 if (io_sqd_handle_event(sqd))
Pavel Begunkovc7d95612021-04-13 11:43:00 +01007583 break;
Xiaoguang Wang08369242020-11-03 14:15:59 +08007584 timeout = jiffies + sqd->sq_thread_idle;
7585 }
Pavel Begunkove4b6d902021-05-16 22:58:00 +01007586
Jens Axboee95eee22020-09-08 09:11:32 -06007587 cap_entries = !list_is_singular(&sqd->ctx_list);
Jens Axboe69fb2132020-09-14 11:16:23 -06007588 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) {
Pavel Begunkov948e1942021-06-24 15:09:55 +01007589 int ret = __io_sq_thread(ctx, cap_entries);
Stefan Metzmacher7c30f36a2021-03-07 11:54:28 +01007590
Xiaoguang Wang08369242020-11-03 14:15:59 +08007591 if (!sqt_spin && (ret > 0 || !list_empty(&ctx->iopoll_list)))
7592 sqt_spin = true;
Jens Axboe6c271ce2019-01-10 11:22:30 -07007593 }
Pavel Begunkovdd432ea52021-06-26 21:40:45 +01007594 if (io_run_task_work())
7595 sqt_spin = true;
Jens Axboe6c271ce2019-01-10 11:22:30 -07007596
Xiaoguang Wang08369242020-11-03 14:15:59 +08007597 if (sqt_spin || !time_after(jiffies, timeout)) {
Jens Axboec8d1ba52020-09-14 11:07:26 -06007598 cond_resched();
Xiaoguang Wang08369242020-11-03 14:15:59 +08007599 if (sqt_spin)
7600 timeout = jiffies + sqd->sq_thread_idle;
7601 continue;
7602 }
7603
Xiaoguang Wang08369242020-11-03 14:15:59 +08007604 prepare_to_wait(&sqd->wait, &wait, TASK_INTERRUPTIBLE);
Pavel Begunkovdd432ea52021-06-26 21:40:45 +01007605 if (!io_sqd_events_pending(sqd) && !current->task_works) {
Pavel Begunkov1a924a82021-06-24 15:09:56 +01007606 bool needs_sched = true;
7607
Hao Xu724cb4f2021-04-21 23:19:11 +08007608 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) {
Pavel Begunkovaaa9f0f2021-05-16 22:58:01 +01007609 io_ring_set_wakeup_flag(ctx);
7610
Hao Xu724cb4f2021-04-21 23:19:11 +08007611 if ((ctx->flags & IORING_SETUP_IOPOLL) &&
7612 !list_empty_careful(&ctx->iopoll_list)) {
7613 needs_sched = false;
7614 break;
7615 }
7616 if (io_sqring_entries(ctx)) {
7617 needs_sched = false;
7618 break;
7619 }
7620 }
7621
7622 if (needs_sched) {
7623 mutex_unlock(&sqd->lock);
7624 schedule();
7625 mutex_lock(&sqd->lock);
7626 }
Jens Axboe69fb2132020-09-14 11:16:23 -06007627 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list)
7628 io_ring_clear_wakeup_flag(ctx);
Jens Axboe6c271ce2019-01-10 11:22:30 -07007629 }
Xiaoguang Wang08369242020-11-03 14:15:59 +08007630
7631 finish_wait(&sqd->wait, &wait);
7632 timeout = jiffies + sqd->sq_thread_idle;
Jens Axboe6c271ce2019-01-10 11:22:30 -07007633 }
7634
Pavel Begunkov78cc6872021-06-14 02:36:23 +01007635 io_uring_cancel_generic(true, sqd);
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007636 sqd->thread = NULL;
Jens Axboe05962f92021-03-06 13:58:48 -07007637 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list)
Jens Axboe5f3f26f2021-02-25 10:17:46 -07007638 io_ring_set_wakeup_flag(ctx);
Pavel Begunkov521d6a72021-03-11 23:29:38 +00007639 io_run_task_work();
Pavel Begunkov734551d2021-04-18 14:52:09 +01007640 mutex_unlock(&sqd->lock);
7641
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007642 complete(&sqd->exited);
7643 do_exit(0);
Jens Axboe6c271ce2019-01-10 11:22:30 -07007644}
7645
Jens Axboebda52162019-09-24 13:47:15 -06007646struct io_wait_queue {
7647 struct wait_queue_entry wq;
7648 struct io_ring_ctx *ctx;
Jens Axboe5fd46172021-08-06 14:04:31 -06007649 unsigned cq_tail;
Jens Axboebda52162019-09-24 13:47:15 -06007650 unsigned nr_timeouts;
7651};
7652
Pavel Begunkov6c503152021-01-04 20:36:36 +00007653static inline bool io_should_wake(struct io_wait_queue *iowq)
Jens Axboebda52162019-09-24 13:47:15 -06007654{
7655 struct io_ring_ctx *ctx = iowq->ctx;
Jens Axboe5fd46172021-08-06 14:04:31 -06007656 int dist = ctx->cached_cq_tail - (int) iowq->cq_tail;
Jens Axboebda52162019-09-24 13:47:15 -06007657
7658 /*
Brian Gianforcarod195a662019-12-13 03:09:50 -08007659 * Wake up if we have enough events, or if a timeout occurred since we
Jens Axboebda52162019-09-24 13:47:15 -06007660 * started waiting. For timeouts, we always want to return to userspace,
7661 * regardless of event count.
7662 */
Jens Axboe5fd46172021-08-06 14:04:31 -06007663 return dist >= 0 || atomic_read(&ctx->cq_timeouts) != iowq->nr_timeouts;
Jens Axboebda52162019-09-24 13:47:15 -06007664}
7665
7666static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode,
7667 int wake_flags, void *key)
7668{
7669 struct io_wait_queue *iowq = container_of(curr, struct io_wait_queue,
7670 wq);
7671
Pavel Begunkov6c503152021-01-04 20:36:36 +00007672 /*
7673 * Cannot safely flush overflowed CQEs from here, ensure we wake up
7674 * the task, and the next invocation will do it.
7675 */
Pavel Begunkov5ed7a372021-06-14 23:37:27 +01007676 if (io_should_wake(iowq) || test_bit(0, &iowq->ctx->check_cq_overflow))
Pavel Begunkov6c503152021-01-04 20:36:36 +00007677 return autoremove_wake_function(curr, mode, wake_flags, key);
7678 return -1;
Jens Axboebda52162019-09-24 13:47:15 -06007679}
7680
Jens Axboeaf9c1a42020-09-24 13:32:18 -06007681static int io_run_task_work_sig(void)
7682{
7683 if (io_run_task_work())
7684 return 1;
7685 if (!signal_pending(current))
7686 return 0;
Jens Axboe0b8cfa92021-03-21 14:16:08 -06007687 if (test_thread_flag(TIF_NOTIFY_SIGNAL))
Jens Axboe792ee0f62020-10-22 20:17:18 -06007688 return -ERESTARTSYS;
Jens Axboeaf9c1a42020-09-24 13:32:18 -06007689 return -EINTR;
7690}
7691
Pavel Begunkoveeb60b92021-02-04 13:51:58 +00007692/* when returns >0, the caller should retry */
7693static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx,
7694 struct io_wait_queue *iowq,
Pavel Begunkovc3222fd2023-01-05 10:49:15 +00007695 ktime_t *timeout)
Pavel Begunkoveeb60b92021-02-04 13:51:58 +00007696{
7697 int ret;
7698
7699 /* make sure we run task_work before checking for signals */
7700 ret = io_run_task_work_sig();
7701 if (ret || io_should_wake(iowq))
7702 return ret;
7703 /* let the caller flush overflows, retry */
Pavel Begunkov5ed7a372021-06-14 23:37:27 +01007704 if (test_bit(0, &ctx->check_cq_overflow))
Pavel Begunkoveeb60b92021-02-04 13:51:58 +00007705 return 1;
7706
Pavel Begunkovc3222fd2023-01-05 10:49:15 +00007707 if (!schedule_hrtimeout(timeout, HRTIMER_MODE_ABS))
Jens Axboe7c834372022-02-21 05:49:30 -07007708 return -ETIME;
7709 return 1;
Pavel Begunkoveeb60b92021-02-04 13:51:58 +00007710}
7711
Jens Axboe2b188cc2019-01-07 10:46:33 -07007712/*
7713 * Wait until events become available, if we don't already have some. The
7714 * application must reap them itself, as they reside on the shared cq ring.
7715 */
7716static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events,
Hao Xuc73ebb62020-11-03 10:54:37 +08007717 const sigset_t __user *sig, size_t sigsz,
7718 struct __kernel_timespec __user *uts)
Jens Axboe2b188cc2019-01-07 10:46:33 -07007719{
Pavel Begunkov902910992021-08-09 09:07:32 -06007720 struct io_wait_queue iowq;
Hristo Venev75b28af2019-08-26 17:23:46 +00007721 struct io_rings *rings = ctx->rings;
Jens Axboe7c834372022-02-21 05:49:30 -07007722 ktime_t timeout = KTIME_MAX;
Pavel Begunkovc1d5a222021-02-04 13:51:57 +00007723 int ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07007724
Jens Axboeb41e9852020-02-17 09:52:41 -07007725 do {
Pavel Begunkov90f67362021-08-09 20:18:12 +01007726 io_cqring_overflow_flush(ctx);
Pavel Begunkov6c503152021-01-04 20:36:36 +00007727 if (io_cqring_events(ctx) >= min_events)
Jens Axboeb41e9852020-02-17 09:52:41 -07007728 return 0;
Jens Axboe4c6e2772020-07-01 11:29:10 -06007729 if (!io_run_task_work())
Jens Axboeb41e9852020-02-17 09:52:41 -07007730 break;
Jens Axboeb41e9852020-02-17 09:52:41 -07007731 } while (1);
Jens Axboe2b188cc2019-01-07 10:46:33 -07007732
Xiaoguang Wang44df58d2021-09-14 22:38:52 +08007733 if (uts) {
7734 struct timespec64 ts;
7735
7736 if (get_timespec64(&ts, uts))
7737 return -EFAULT;
Jens Axboe7c834372022-02-21 05:49:30 -07007738 timeout = ktime_add_ns(timespec64_to_ktime(ts), ktime_get_ns());
Xiaoguang Wang44df58d2021-09-14 22:38:52 +08007739 }
7740
Jens Axboe2b188cc2019-01-07 10:46:33 -07007741 if (sig) {
Arnd Bergmann9e75ad52019-03-25 15:34:53 +01007742#ifdef CONFIG_COMPAT
7743 if (in_compat_syscall())
7744 ret = set_compat_user_sigmask((const compat_sigset_t __user *)sig,
Oleg Nesterovb7724342019-07-16 16:29:53 -07007745 sigsz);
Arnd Bergmann9e75ad52019-03-25 15:34:53 +01007746 else
7747#endif
Oleg Nesterovb7724342019-07-16 16:29:53 -07007748 ret = set_user_sigmask(sig, sigsz);
Arnd Bergmann9e75ad52019-03-25 15:34:53 +01007749
Jens Axboe2b188cc2019-01-07 10:46:33 -07007750 if (ret)
7751 return ret;
7752 }
7753
Pavel Begunkov902910992021-08-09 09:07:32 -06007754 init_waitqueue_func_entry(&iowq.wq, io_wake_function);
7755 iowq.wq.private = current;
7756 INIT_LIST_HEAD(&iowq.wq.entry);
7757 iowq.ctx = ctx;
Jens Axboebda52162019-09-24 13:47:15 -06007758 iowq.nr_timeouts = atomic_read(&ctx->cq_timeouts);
Jens Axboe5fd46172021-08-06 14:04:31 -06007759 iowq.cq_tail = READ_ONCE(ctx->rings->cq.head) + min_events;
Pavel Begunkov902910992021-08-09 09:07:32 -06007760
Dmitrii Dolgovc826bd72019-10-15 19:02:01 +02007761 trace_io_uring_cqring_wait(ctx, min_events);
Jens Axboebda52162019-09-24 13:47:15 -06007762 do {
Jens Axboeca0a2652021-03-04 17:15:48 -07007763 /* if we can't even flush overflow, don't wait for more */
Pavel Begunkov90f67362021-08-09 20:18:12 +01007764 if (!io_cqring_overflow_flush(ctx)) {
Jens Axboeca0a2652021-03-04 17:15:48 -07007765 ret = -EBUSY;
7766 break;
7767 }
Pavel Begunkov311997b2021-06-14 23:37:28 +01007768 prepare_to_wait_exclusive(&ctx->cq_wait, &iowq.wq,
Jens Axboebda52162019-09-24 13:47:15 -06007769 TASK_INTERRUPTIBLE);
Pavel Begunkovc3222fd2023-01-05 10:49:15 +00007770 ret = io_cqring_wait_schedule(ctx, &iowq, &timeout);
Pavel Begunkov311997b2021-06-14 23:37:28 +01007771 finish_wait(&ctx->cq_wait, &iowq.wq);
Jens Axboeca0a2652021-03-04 17:15:48 -07007772 cond_resched();
Pavel Begunkoveeb60b92021-02-04 13:51:58 +00007773 } while (ret > 0);
Jens Axboebda52162019-09-24 13:47:15 -06007774
Jens Axboeb7db41c2020-07-04 08:55:50 -06007775 restore_saved_sigmask_unless(ret == -EINTR);
Jens Axboe2b188cc2019-01-07 10:46:33 -07007776
Hristo Venev75b28af2019-08-26 17:23:46 +00007777 return READ_ONCE(rings->cq.head) == READ_ONCE(rings->cq.tail) ? ret : 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07007778}
7779
Pavel Begunkov9123c8f2021-06-14 02:36:20 +01007780static void io_free_page_table(void **table, size_t size)
Pavel Begunkov846a4ef2021-04-01 15:44:03 +01007781{
Pavel Begunkov9123c8f2021-06-14 02:36:20 +01007782 unsigned i, nr_tables = DIV_ROUND_UP(size, PAGE_SIZE);
Pavel Begunkov846a4ef2021-04-01 15:44:03 +01007783
7784 for (i = 0; i < nr_tables; i++)
Pavel Begunkov9123c8f2021-06-14 02:36:20 +01007785 kfree(table[i]);
7786 kfree(table);
7787}
7788
7789static void **io_alloc_page_table(size_t size)
7790{
7791 unsigned i, nr_tables = DIV_ROUND_UP(size, PAGE_SIZE);
7792 size_t init_size = size;
7793 void **table;
7794
Pavel Begunkov0bea96f2021-08-20 10:36:36 +01007795 table = kcalloc(nr_tables, sizeof(*table), GFP_KERNEL_ACCOUNT);
Pavel Begunkov9123c8f2021-06-14 02:36:20 +01007796 if (!table)
7797 return NULL;
7798
7799 for (i = 0; i < nr_tables; i++) {
Pavel Begunkov27f6b312021-06-15 13:20:13 +01007800 unsigned int this_size = min_t(size_t, size, PAGE_SIZE);
Pavel Begunkov9123c8f2021-06-14 02:36:20 +01007801
Pavel Begunkov0bea96f2021-08-20 10:36:36 +01007802 table[i] = kzalloc(this_size, GFP_KERNEL_ACCOUNT);
Pavel Begunkov9123c8f2021-06-14 02:36:20 +01007803 if (!table[i]) {
7804 io_free_page_table(table, init_size);
7805 return NULL;
7806 }
7807 size -= this_size;
7808 }
7809 return table;
Pavel Begunkov846a4ef2021-04-01 15:44:03 +01007810}
7811
Pavel Begunkov28a9fe22021-04-01 15:43:47 +01007812static void io_rsrc_node_destroy(struct io_rsrc_node *ref_node)
7813{
7814 percpu_ref_exit(&ref_node->refs);
7815 kfree(ref_node);
7816}
7817
Pavel Begunkovb9bd2be2021-08-09 09:09:47 -06007818static void io_rsrc_node_ref_zero(struct percpu_ref *ref)
7819{
7820 struct io_rsrc_node *node = container_of(ref, struct io_rsrc_node, refs);
7821 struct io_ring_ctx *ctx = node->rsrc_data->ctx;
7822 unsigned long flags;
7823 bool first_add = false;
Dylan Yudaken82cc3382022-01-21 04:38:56 -08007824 unsigned long delay = HZ;
Pavel Begunkovb9bd2be2021-08-09 09:09:47 -06007825
7826 spin_lock_irqsave(&ctx->rsrc_ref_lock, flags);
7827 node->done = true;
7828
Dylan Yudaken82cc3382022-01-21 04:38:56 -08007829 /* if we are mid-quiesce then do not delay */
7830 if (node->rsrc_data->quiesce)
7831 delay = 0;
7832
Pavel Begunkovb9bd2be2021-08-09 09:09:47 -06007833 while (!list_empty(&ctx->rsrc_ref_list)) {
7834 node = list_first_entry(&ctx->rsrc_ref_list,
7835 struct io_rsrc_node, node);
7836 /* recycle ref nodes in order */
7837 if (!node->done)
7838 break;
7839 list_del(&node->node);
7840 first_add |= llist_add(&node->llist, &ctx->rsrc_put_llist);
7841 }
7842 spin_unlock_irqrestore(&ctx->rsrc_ref_lock, flags);
7843
7844 if (first_add)
Dylan Yudaken82cc3382022-01-21 04:38:56 -08007845 mod_delayed_work(system_wq, &ctx->rsrc_put_work, delay);
Pavel Begunkovb9bd2be2021-08-09 09:09:47 -06007846}
7847
7848static struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx)
7849{
7850 struct io_rsrc_node *ref_node;
7851
7852 ref_node = kzalloc(sizeof(*ref_node), GFP_KERNEL);
7853 if (!ref_node)
7854 return NULL;
7855
7856 if (percpu_ref_init(&ref_node->refs, io_rsrc_node_ref_zero,
7857 0, GFP_KERNEL)) {
7858 kfree(ref_node);
7859 return NULL;
7860 }
7861 INIT_LIST_HEAD(&ref_node->node);
7862 INIT_LIST_HEAD(&ref_node->rsrc_list);
7863 ref_node->done = false;
7864 return ref_node;
7865}
7866
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01007867static void io_rsrc_node_switch(struct io_ring_ctx *ctx,
7868 struct io_rsrc_data *data_to_kill)
Pavel Begunkov1642b442020-12-30 21:34:14 +00007869{
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01007870 WARN_ON_ONCE(!ctx->rsrc_backup_node);
7871 WARN_ON_ONCE(data_to_kill && !ctx->rsrc_node);
Pavel Begunkov82fbcfa2021-04-01 15:43:43 +01007872
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01007873 if (data_to_kill) {
7874 struct io_rsrc_node *rsrc_node = ctx->rsrc_node;
Pavel Begunkov82fbcfa2021-04-01 15:43:43 +01007875
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01007876 rsrc_node->rsrc_data = data_to_kill;
Jens Axboe4956b9e2021-08-09 07:49:41 -06007877 spin_lock_irq(&ctx->rsrc_ref_lock);
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01007878 list_add_tail(&rsrc_node->node, &ctx->rsrc_ref_list);
Jens Axboe4956b9e2021-08-09 07:49:41 -06007879 spin_unlock_irq(&ctx->rsrc_ref_lock);
Pavel Begunkov82fbcfa2021-04-01 15:43:43 +01007880
Pavel Begunkov3e942492021-04-11 01:46:34 +01007881 atomic_inc(&data_to_kill->refs);
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01007882 percpu_ref_kill(&rsrc_node->refs);
7883 ctx->rsrc_node = NULL;
7884 }
7885
7886 if (!ctx->rsrc_node) {
7887 ctx->rsrc_node = ctx->rsrc_backup_node;
7888 ctx->rsrc_backup_node = NULL;
7889 }
Pavel Begunkov1642b442020-12-30 21:34:14 +00007890}
7891
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01007892static int io_rsrc_node_switch_start(struct io_ring_ctx *ctx)
Pavel Begunkov8dd03af2021-03-19 17:22:36 +00007893{
7894 if (ctx->rsrc_backup_node)
7895 return 0;
Pavel Begunkovb895c9a2021-04-01 15:43:40 +01007896 ctx->rsrc_backup_node = io_rsrc_node_alloc(ctx);
Pavel Begunkov8dd03af2021-03-19 17:22:36 +00007897 return ctx->rsrc_backup_node ? 0 : -ENOMEM;
7898}
7899
Pavel Begunkov40ae0ff2021-04-01 15:43:44 +01007900static int io_rsrc_ref_quiesce(struct io_rsrc_data *data, struct io_ring_ctx *ctx)
Hao Xu8bad28d2021-02-19 17:19:36 +08007901{
7902 int ret;
Xiaoguang Wang05589552020-03-31 14:05:18 +08007903
Pavel Begunkov215c3902021-04-01 15:43:48 +01007904 /* As we may drop ->uring_lock, other task may have started quiesce */
Hao Xu8bad28d2021-02-19 17:19:36 +08007905 if (data->quiesce)
7906 return -ENXIO;
7907
7908 data->quiesce = true;
Pavel Begunkov1ffc5422020-12-30 21:34:15 +00007909 do {
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01007910 ret = io_rsrc_node_switch_start(ctx);
Pavel Begunkov8dd03af2021-03-19 17:22:36 +00007911 if (ret)
Pavel Begunkovf2303b12021-02-20 18:03:49 +00007912 break;
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01007913 io_rsrc_node_switch(ctx, data);
7914
Pavel Begunkov3e942492021-04-11 01:46:34 +01007915 /* kill initial ref, already quiesced if zero */
7916 if (atomic_dec_and_test(&data->refs))
7917 break;
Jens Axboec018db42021-08-09 08:15:50 -06007918 mutex_unlock(&ctx->uring_lock);
Hao Xu8bad28d2021-02-19 17:19:36 +08007919 flush_delayed_work(&ctx->rsrc_put_work);
Pavel Begunkov1ffc5422020-12-30 21:34:15 +00007920 ret = wait_for_completion_interruptible(&data->done);
Jens Axboec018db42021-08-09 08:15:50 -06007921 if (!ret) {
7922 mutex_lock(&ctx->uring_lock);
Dylan Yudaken0d773aa2022-02-22 08:17:51 -08007923 if (atomic_read(&data->refs) > 0) {
7924 /*
7925 * it has been revived by another thread while
7926 * we were unlocked
7927 */
7928 mutex_unlock(&ctx->uring_lock);
7929 } else {
7930 break;
7931 }
Jens Axboec018db42021-08-09 08:15:50 -06007932 }
Jens Axboe05f3fb32019-12-09 11:22:50 -07007933
Pavel Begunkov3e942492021-04-11 01:46:34 +01007934 atomic_inc(&data->refs);
7935 /* wait for all works potentially completing data->done */
7936 flush_delayed_work(&ctx->rsrc_put_work);
Jens Axboecb5e1b82021-02-25 07:37:35 -07007937 reinit_completion(&data->done);
Pavel Begunkov8dd03af2021-03-19 17:22:36 +00007938
Hao Xu8bad28d2021-02-19 17:19:36 +08007939 ret = io_run_task_work_sig();
7940 mutex_lock(&ctx->uring_lock);
Pavel Begunkovf2303b12021-02-20 18:03:49 +00007941 } while (ret >= 0);
Hao Xu8bad28d2021-02-19 17:19:36 +08007942 data->quiesce = false;
7943
Hao Xu8bad28d2021-02-19 17:19:36 +08007944 return ret;
Bijan Mottahedehd7954b22021-01-15 17:37:50 +00007945}
7946
Pavel Begunkov2d091d62021-06-14 02:36:21 +01007947static u64 *io_get_tag_slot(struct io_rsrc_data *data, unsigned int idx)
7948{
7949 unsigned int off = idx & IO_RSRC_TAG_TABLE_MASK;
7950 unsigned int table_idx = idx >> IO_RSRC_TAG_TABLE_SHIFT;
7951
7952 return &data->tags[table_idx][off];
7953}
7954
Pavel Begunkov44b31f22021-04-25 14:32:16 +01007955static void io_rsrc_data_free(struct io_rsrc_data *data)
7956{
Pavel Begunkov2d091d62021-06-14 02:36:21 +01007957 size_t size = data->nr * sizeof(data->tags[0][0]);
7958
7959 if (data->tags)
7960 io_free_page_table((void **)data->tags, size);
Pavel Begunkov44b31f22021-04-25 14:32:16 +01007961 kfree(data);
7962}
7963
Pavel Begunkovd878c812021-06-14 02:36:18 +01007964static int io_rsrc_data_alloc(struct io_ring_ctx *ctx, rsrc_put_fn *do_put,
7965 u64 __user *utags, unsigned nr,
7966 struct io_rsrc_data **pdata)
Bijan Mottahedeh1ad555c2021-01-15 17:37:51 +00007967{
Pavel Begunkovb895c9a2021-04-01 15:43:40 +01007968 struct io_rsrc_data *data;
Pavel Begunkov2d091d62021-06-14 02:36:21 +01007969 int ret = -ENOMEM;
Pavel Begunkovd878c812021-06-14 02:36:18 +01007970 unsigned i;
Bijan Mottahedeh1ad555c2021-01-15 17:37:51 +00007971
7972 data = kzalloc(sizeof(*data), GFP_KERNEL);
7973 if (!data)
Pavel Begunkovd878c812021-06-14 02:36:18 +01007974 return -ENOMEM;
Pavel Begunkov2d091d62021-06-14 02:36:21 +01007975 data->tags = (u64 **)io_alloc_page_table(nr * sizeof(data->tags[0][0]));
Pavel Begunkovb60c8dc2021-04-25 14:32:18 +01007976 if (!data->tags) {
7977 kfree(data);
Pavel Begunkovd878c812021-06-14 02:36:18 +01007978 return -ENOMEM;
7979 }
Pavel Begunkov2d091d62021-06-14 02:36:21 +01007980
7981 data->nr = nr;
7982 data->ctx = ctx;
7983 data->do_put = do_put;
Pavel Begunkovd878c812021-06-14 02:36:18 +01007984 if (utags) {
Pavel Begunkov2d091d62021-06-14 02:36:21 +01007985 ret = -EFAULT;
Pavel Begunkovd878c812021-06-14 02:36:18 +01007986 for (i = 0; i < nr; i++) {
Colin Ian Kingfdd1dc32021-06-15 14:00:11 +01007987 u64 *tag_slot = io_get_tag_slot(data, i);
7988
7989 if (copy_from_user(tag_slot, &utags[i],
7990 sizeof(*tag_slot)))
Pavel Begunkov2d091d62021-06-14 02:36:21 +01007991 goto fail;
Pavel Begunkovd878c812021-06-14 02:36:18 +01007992 }
Pavel Begunkovb60c8dc2021-04-25 14:32:18 +01007993 }
7994
Pavel Begunkov3e942492021-04-11 01:46:34 +01007995 atomic_set(&data->refs, 1);
Bijan Mottahedeh1ad555c2021-01-15 17:37:51 +00007996 init_completion(&data->done);
Pavel Begunkovd878c812021-06-14 02:36:18 +01007997 *pdata = data;
7998 return 0;
Pavel Begunkov2d091d62021-06-14 02:36:21 +01007999fail:
8000 io_rsrc_data_free(data);
8001 return ret;
Bijan Mottahedeh1ad555c2021-01-15 17:37:51 +00008002}
8003
Pavel Begunkov9123c8f2021-06-14 02:36:20 +01008004static bool io_alloc_file_tables(struct io_file_table *table, unsigned nr_files)
8005{
Pavel Begunkov0bea96f2021-08-20 10:36:36 +01008006 table->files = kvcalloc(nr_files, sizeof(table->files[0]),
8007 GFP_KERNEL_ACCOUNT);
Pavel Begunkov9123c8f2021-06-14 02:36:20 +01008008 return !!table->files;
8009}
8010
Pavel Begunkov042b0d82021-08-09 13:04:01 +01008011static void io_free_file_tables(struct io_file_table *table)
Pavel Begunkov9123c8f2021-06-14 02:36:20 +01008012{
Pavel Begunkov042b0d82021-08-09 13:04:01 +01008013 kvfree(table->files);
Pavel Begunkov9123c8f2021-06-14 02:36:20 +01008014 table->files = NULL;
8015}
8016
Jens Axboe2b188cc2019-01-07 10:46:33 -07008017static void __io_sqe_files_unregister(struct io_ring_ctx *ctx)
8018{
8019#if defined(CONFIG_UNIX)
8020 if (ctx->ring_sock) {
8021 struct sock *sock = ctx->ring_sock->sk;
8022 struct sk_buff *skb;
8023
8024 while ((skb = skb_dequeue(&sock->sk_receive_queue)) != NULL)
8025 kfree_skb(skb);
8026 }
8027#else
8028 int i;
8029
8030 for (i = 0; i < ctx->nr_user_files; i++) {
8031 struct file *file;
8032
8033 file = io_file_from_index(ctx, i);
8034 if (file)
8035 fput(file);
8036 }
8037#endif
Pavel Begunkov042b0d82021-08-09 13:04:01 +01008038 io_free_file_tables(&ctx->file_table);
Pavel Begunkov44b31f22021-04-25 14:32:16 +01008039 io_rsrc_data_free(ctx->file_data);
Pavel Begunkovfff4db72021-04-25 14:32:15 +01008040 ctx->file_data = NULL;
8041 ctx->nr_user_files = 0;
Bijan Mottahedeh1ad555c2021-01-15 17:37:51 +00008042}
8043
Bijan Mottahedehd7954b22021-01-15 17:37:50 +00008044static int io_sqe_files_unregister(struct io_ring_ctx *ctx)
8045{
Pavel Begunkovb1e7cad2022-06-13 06:32:44 +01008046 unsigned nr = ctx->nr_user_files;
Bijan Mottahedehd7954b22021-01-15 17:37:50 +00008047 int ret;
8048
Pavel Begunkov08480402021-04-13 02:58:38 +01008049 if (!ctx->file_data)
Bijan Mottahedehd7954b22021-01-15 17:37:50 +00008050 return -ENXIO;
Pavel Begunkovb1e7cad2022-06-13 06:32:44 +01008051
8052 /*
8053 * Quiesce may unlock ->uring_lock, and while it's not held
8054 * prevent new requests using the table.
8055 */
8056 ctx->nr_user_files = 0;
Pavel Begunkov08480402021-04-13 02:58:38 +01008057 ret = io_rsrc_ref_quiesce(ctx->file_data, ctx);
Pavel Begunkovb1e7cad2022-06-13 06:32:44 +01008058 ctx->nr_user_files = nr;
Pavel Begunkov08480402021-04-13 02:58:38 +01008059 if (!ret)
8060 __io_sqe_files_unregister(ctx);
8061 return ret;
Jens Axboe6b063142019-01-10 22:13:58 -07008062}
8063
Jens Axboe37d1e2e2021-02-17 21:03:43 -07008064static void io_sq_thread_unpark(struct io_sq_data *sqd)
Pavel Begunkov09a6f4e2021-03-14 20:57:10 +00008065 __releases(&sqd->lock)
Jens Axboe37d1e2e2021-02-17 21:03:43 -07008066{
Pavel Begunkov521d6a72021-03-11 23:29:38 +00008067 WARN_ON_ONCE(sqd->thread == current);
8068
Pavel Begunkov9e138a42021-03-14 20:57:12 +00008069 /*
8070 * Do the dance but not conditional clear_bit() because it'd race with
8071 * other threads incrementing park_pending and setting the bit.
8072 */
Jens Axboe37d1e2e2021-02-17 21:03:43 -07008073 clear_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state);
Pavel Begunkov9e138a42021-03-14 20:57:12 +00008074 if (atomic_dec_return(&sqd->park_pending))
8075 set_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state);
Pavel Begunkov09a6f4e2021-03-14 20:57:10 +00008076 mutex_unlock(&sqd->lock);
Jens Axboe37d1e2e2021-02-17 21:03:43 -07008077}
8078
Jens Axboe86e0d672021-03-05 08:44:39 -07008079static void io_sq_thread_park(struct io_sq_data *sqd)
Pavel Begunkov09a6f4e2021-03-14 20:57:10 +00008080 __acquires(&sqd->lock)
Jens Axboe37d1e2e2021-02-17 21:03:43 -07008081{
Pavel Begunkov521d6a72021-03-11 23:29:38 +00008082 WARN_ON_ONCE(sqd->thread == current);
8083
Pavel Begunkov9e138a42021-03-14 20:57:12 +00008084 atomic_inc(&sqd->park_pending);
Jens Axboe37d1e2e2021-02-17 21:03:43 -07008085 set_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state);
Pavel Begunkov09a6f4e2021-03-14 20:57:10 +00008086 mutex_lock(&sqd->lock);
Jens Axboe05962f92021-03-06 13:58:48 -07008087 if (sqd->thread)
Jens Axboe86e0d672021-03-05 08:44:39 -07008088 wake_up_process(sqd->thread);
Jens Axboe37d1e2e2021-02-17 21:03:43 -07008089}
8090
8091static void io_sq_thread_stop(struct io_sq_data *sqd)
8092{
Pavel Begunkov521d6a72021-03-11 23:29:38 +00008093 WARN_ON_ONCE(sqd->thread == current);
Pavel Begunkov88885f62021-04-11 01:46:38 +01008094 WARN_ON_ONCE(test_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state));
Pavel Begunkov521d6a72021-03-11 23:29:38 +00008095
Jens Axboe05962f92021-03-06 13:58:48 -07008096 set_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state);
Pavel Begunkov88885f62021-04-11 01:46:38 +01008097 mutex_lock(&sqd->lock);
Jens Axboee8f98f242021-03-09 16:32:13 -07008098 if (sqd->thread)
8099 wake_up_process(sqd->thread);
Pavel Begunkov09a6f4e2021-03-14 20:57:10 +00008100 mutex_unlock(&sqd->lock);
Jens Axboe05962f92021-03-06 13:58:48 -07008101 wait_for_completion(&sqd->exited);
Jens Axboe37d1e2e2021-02-17 21:03:43 -07008102}
8103
Jens Axboe534ca6d2020-09-02 13:52:19 -06008104static void io_put_sq_data(struct io_sq_data *sqd)
Jens Axboe6c271ce2019-01-10 11:22:30 -07008105{
Jens Axboe534ca6d2020-09-02 13:52:19 -06008106 if (refcount_dec_and_test(&sqd->refs)) {
Pavel Begunkov9e138a42021-03-14 20:57:12 +00008107 WARN_ON_ONCE(atomic_read(&sqd->park_pending));
8108
Jens Axboe37d1e2e2021-02-17 21:03:43 -07008109 io_sq_thread_stop(sqd);
8110 kfree(sqd);
8111 }
8112}
8113
8114static void io_sq_thread_finish(struct io_ring_ctx *ctx)
8115{
8116 struct io_sq_data *sqd = ctx->sq_data;
8117
8118 if (sqd) {
Jens Axboe05962f92021-03-06 13:58:48 -07008119 io_sq_thread_park(sqd);
Pavel Begunkov521d6a72021-03-11 23:29:38 +00008120 list_del_init(&ctx->sqd_list);
Jens Axboe37d1e2e2021-02-17 21:03:43 -07008121 io_sqd_update_thread_idle(sqd);
Jens Axboe05962f92021-03-06 13:58:48 -07008122 io_sq_thread_unpark(sqd);
Jens Axboe37d1e2e2021-02-17 21:03:43 -07008123
8124 io_put_sq_data(sqd);
8125 ctx->sq_data = NULL;
Jens Axboe534ca6d2020-09-02 13:52:19 -06008126 }
8127}
8128
Jens Axboeaa061652020-09-02 14:50:27 -06008129static struct io_sq_data *io_attach_sq_data(struct io_uring_params *p)
8130{
8131 struct io_ring_ctx *ctx_attach;
8132 struct io_sq_data *sqd;
8133 struct fd f;
8134
8135 f = fdget(p->wq_fd);
8136 if (!f.file)
8137 return ERR_PTR(-ENXIO);
8138 if (f.file->f_op != &io_uring_fops) {
8139 fdput(f);
8140 return ERR_PTR(-EINVAL);
8141 }
8142
8143 ctx_attach = f.file->private_data;
8144 sqd = ctx_attach->sq_data;
8145 if (!sqd) {
8146 fdput(f);
8147 return ERR_PTR(-EINVAL);
8148 }
Jens Axboe5c2469e2021-03-11 10:17:56 -07008149 if (sqd->task_tgid != current->tgid) {
8150 fdput(f);
8151 return ERR_PTR(-EPERM);
8152 }
Jens Axboeaa061652020-09-02 14:50:27 -06008153
8154 refcount_inc(&sqd->refs);
8155 fdput(f);
8156 return sqd;
8157}
8158
Pavel Begunkov26984fb2021-03-11 23:29:37 +00008159static struct io_sq_data *io_get_sq_data(struct io_uring_params *p,
8160 bool *attached)
Jens Axboe534ca6d2020-09-02 13:52:19 -06008161{
8162 struct io_sq_data *sqd;
8163
Pavel Begunkov26984fb2021-03-11 23:29:37 +00008164 *attached = false;
Jens Axboe5c2469e2021-03-11 10:17:56 -07008165 if (p->flags & IORING_SETUP_ATTACH_WQ) {
8166 sqd = io_attach_sq_data(p);
Pavel Begunkov26984fb2021-03-11 23:29:37 +00008167 if (!IS_ERR(sqd)) {
8168 *attached = true;
Jens Axboe5c2469e2021-03-11 10:17:56 -07008169 return sqd;
Pavel Begunkov26984fb2021-03-11 23:29:37 +00008170 }
Jens Axboe5c2469e2021-03-11 10:17:56 -07008171 /* fall through for EPERM case, setup new sqd/task */
8172 if (PTR_ERR(sqd) != -EPERM)
8173 return sqd;
8174 }
Jens Axboeaa061652020-09-02 14:50:27 -06008175
Jens Axboe534ca6d2020-09-02 13:52:19 -06008176 sqd = kzalloc(sizeof(*sqd), GFP_KERNEL);
8177 if (!sqd)
8178 return ERR_PTR(-ENOMEM);
8179
Pavel Begunkov9e138a42021-03-14 20:57:12 +00008180 atomic_set(&sqd->park_pending, 0);
Jens Axboe534ca6d2020-09-02 13:52:19 -06008181 refcount_set(&sqd->refs, 1);
Jens Axboe69fb2132020-09-14 11:16:23 -06008182 INIT_LIST_HEAD(&sqd->ctx_list);
Pavel Begunkov09a6f4e2021-03-14 20:57:10 +00008183 mutex_init(&sqd->lock);
Jens Axboe534ca6d2020-09-02 13:52:19 -06008184 init_waitqueue_head(&sqd->wait);
Jens Axboe37d1e2e2021-02-17 21:03:43 -07008185 init_completion(&sqd->exited);
Jens Axboe534ca6d2020-09-02 13:52:19 -06008186 return sqd;
8187}
8188
Jens Axboe6b063142019-01-10 22:13:58 -07008189#if defined(CONFIG_UNIX)
Jens Axboe6b063142019-01-10 22:13:58 -07008190/*
8191 * Ensure the UNIX gc is aware of our file set, so we are certain that
8192 * the io_uring can be safely unregistered on process exit, even if we have
8193 * loops in the file referencing.
8194 */
8195static int __io_sqe_files_scm(struct io_ring_ctx *ctx, int nr, int offset)
8196{
8197 struct sock *sk = ctx->ring_sock->sk;
8198 struct scm_fp_list *fpl;
8199 struct sk_buff *skb;
Jens Axboe08a45172019-10-03 08:11:03 -06008200 int i, nr_files;
Jens Axboe6b063142019-01-10 22:13:58 -07008201
Jens Axboe6b063142019-01-10 22:13:58 -07008202 fpl = kzalloc(sizeof(*fpl), GFP_KERNEL);
8203 if (!fpl)
8204 return -ENOMEM;
8205
8206 skb = alloc_skb(0, GFP_KERNEL);
8207 if (!skb) {
8208 kfree(fpl);
8209 return -ENOMEM;
8210 }
8211
8212 skb->sk = sk;
Pavel Begunkov813d8fe2022-10-16 22:42:54 +01008213 skb->scm_io_uring = 1;
Jens Axboe6b063142019-01-10 22:13:58 -07008214
Jens Axboe08a45172019-10-03 08:11:03 -06008215 nr_files = 0;
Jens Axboe62e398b2021-02-21 16:19:37 -07008216 fpl->user = get_uid(current_user());
Jens Axboe6b063142019-01-10 22:13:58 -07008217 for (i = 0; i < nr; i++) {
Jens Axboe65e19f52019-10-26 07:20:21 -06008218 struct file *file = io_file_from_index(ctx, i + offset);
8219
8220 if (!file)
Jens Axboe08a45172019-10-03 08:11:03 -06008221 continue;
Jens Axboe65e19f52019-10-26 07:20:21 -06008222 fpl->fp[nr_files] = get_file(file);
Jens Axboe08a45172019-10-03 08:11:03 -06008223 unix_inflight(fpl->user, fpl->fp[nr_files]);
8224 nr_files++;
Jens Axboe6b063142019-01-10 22:13:58 -07008225 }
8226
Jens Axboe08a45172019-10-03 08:11:03 -06008227 if (nr_files) {
8228 fpl->max = SCM_MAX_FD;
8229 fpl->count = nr_files;
8230 UNIXCB(skb).fp = fpl;
Jens Axboe05f3fb32019-12-09 11:22:50 -07008231 skb->destructor = unix_destruct_scm;
Jens Axboe08a45172019-10-03 08:11:03 -06008232 refcount_add(skb->truesize, &sk->sk_wmem_alloc);
8233 skb_queue_head(&sk->sk_receive_queue, skb);
Jens Axboe6b063142019-01-10 22:13:58 -07008234
Pavel Begunkov285f5d72022-04-06 12:43:58 +01008235 for (i = 0; i < nr; i++) {
8236 struct file *file = io_file_from_index(ctx, i + offset);
8237
8238 if (file)
8239 fput(file);
8240 }
Jens Axboe08a45172019-10-03 08:11:03 -06008241 } else {
8242 kfree_skb(skb);
Pavel Begunkov0853bd62022-03-25 16:36:31 +00008243 free_uid(fpl->user);
Jens Axboe08a45172019-10-03 08:11:03 -06008244 kfree(fpl);
8245 }
Jens Axboe6b063142019-01-10 22:13:58 -07008246
8247 return 0;
8248}
8249
8250/*
8251 * If UNIX sockets are enabled, fd passing can cause a reference cycle which
8252 * causes regular reference counting to break down. We rely on the UNIX
8253 * garbage collection to take care of this problem for us.
8254 */
8255static int io_sqe_files_scm(struct io_ring_ctx *ctx)
8256{
8257 unsigned left, total;
8258 int ret = 0;
8259
8260 total = 0;
8261 left = ctx->nr_user_files;
8262 while (left) {
8263 unsigned this_files = min_t(unsigned, left, SCM_MAX_FD);
Jens Axboe6b063142019-01-10 22:13:58 -07008264
8265 ret = __io_sqe_files_scm(ctx, this_files, total);
8266 if (ret)
8267 break;
8268 left -= this_files;
8269 total += this_files;
8270 }
8271
8272 if (!ret)
8273 return 0;
8274
8275 while (total < ctx->nr_user_files) {
Jens Axboe65e19f52019-10-26 07:20:21 -06008276 struct file *file = io_file_from_index(ctx, total);
8277
8278 if (file)
8279 fput(file);
Jens Axboe6b063142019-01-10 22:13:58 -07008280 total++;
8281 }
8282
8283 return ret;
8284}
8285#else
8286static int io_sqe_files_scm(struct io_ring_ctx *ctx)
8287{
8288 return 0;
8289}
8290#endif
8291
Pavel Begunkov47e90392021-04-01 15:43:56 +01008292static void io_rsrc_file_put(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc)
Jens Axboec3a31e62019-10-03 13:59:56 -06008293{
Bijan Mottahedeh50238532021-01-15 17:37:45 +00008294 struct file *file = prsrc->file;
Jens Axboec3a31e62019-10-03 13:59:56 -06008295#if defined(CONFIG_UNIX)
Jens Axboec3a31e62019-10-03 13:59:56 -06008296 struct sock *sock = ctx->ring_sock->sk;
8297 struct sk_buff_head list, *head = &sock->sk_receive_queue;
8298 struct sk_buff *skb;
8299 int i;
8300
8301 __skb_queue_head_init(&list);
8302
8303 /*
8304 * Find the skb that holds this file in its SCM_RIGHTS. When found,
8305 * remove this entry and rearrange the file array.
8306 */
8307 skb = skb_dequeue(head);
8308 while (skb) {
8309 struct scm_fp_list *fp;
8310
8311 fp = UNIXCB(skb).fp;
8312 for (i = 0; i < fp->count; i++) {
8313 int left;
8314
8315 if (fp->fp[i] != file)
8316 continue;
8317
8318 unix_notinflight(fp->user, fp->fp[i]);
8319 left = fp->count - 1 - i;
8320 if (left) {
8321 memmove(&fp->fp[i], &fp->fp[i + 1],
8322 left * sizeof(struct file *));
8323 }
8324 fp->count--;
8325 if (!fp->count) {
8326 kfree_skb(skb);
8327 skb = NULL;
8328 } else {
8329 __skb_queue_tail(&list, skb);
8330 }
8331 fput(file);
8332 file = NULL;
8333 break;
8334 }
8335
8336 if (!file)
8337 break;
8338
8339 __skb_queue_tail(&list, skb);
8340
8341 skb = skb_dequeue(head);
8342 }
8343
8344 if (skb_peek(&list)) {
8345 spin_lock_irq(&head->lock);
8346 while ((skb = __skb_dequeue(&list)) != NULL)
8347 __skb_queue_tail(head, skb);
8348 spin_unlock_irq(&head->lock);
8349 }
8350#else
Jens Axboe05f3fb32019-12-09 11:22:50 -07008351 fput(file);
Jens Axboec3a31e62019-10-03 13:59:56 -06008352#endif
8353}
8354
Pavel Begunkovb895c9a2021-04-01 15:43:40 +01008355static void __io_rsrc_put_work(struct io_rsrc_node *ref_node)
Jens Axboe05f3fb32019-12-09 11:22:50 -07008356{
Pavel Begunkovb895c9a2021-04-01 15:43:40 +01008357 struct io_rsrc_data *rsrc_data = ref_node->rsrc_data;
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00008358 struct io_ring_ctx *ctx = rsrc_data->ctx;
8359 struct io_rsrc_put *prsrc, *tmp;
Xiaoguang Wang05589552020-03-31 14:05:18 +08008360
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00008361 list_for_each_entry_safe(prsrc, tmp, &ref_node->rsrc_list, list) {
8362 list_del(&prsrc->list);
Pavel Begunkovb60c8dc2021-04-25 14:32:18 +01008363
8364 if (prsrc->tag) {
8365 bool lock_ring = ctx->flags & IORING_SETUP_IOPOLL;
Pavel Begunkovb60c8dc2021-04-25 14:32:18 +01008366
8367 io_ring_submit_lock(ctx, lock_ring);
Jens Axboe79ebeae2021-08-10 15:18:27 -06008368 spin_lock(&ctx->completion_lock);
Pavel Begunkovb850d6d2022-08-29 14:30:13 +01008369 io_fill_cqe_aux(ctx, prsrc->tag, 0, 0);
Pavel Begunkovb60c8dc2021-04-25 14:32:18 +01008370 io_commit_cqring(ctx);
Jens Axboe79ebeae2021-08-10 15:18:27 -06008371 spin_unlock(&ctx->completion_lock);
Pavel Begunkovb60c8dc2021-04-25 14:32:18 +01008372 io_cqring_ev_posted(ctx);
8373 io_ring_submit_unlock(ctx, lock_ring);
8374 }
8375
Pavel Begunkov40ae0ff2021-04-01 15:43:44 +01008376 rsrc_data->do_put(ctx, prsrc);
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00008377 kfree(prsrc);
Jens Axboe05f3fb32019-12-09 11:22:50 -07008378 }
8379
Pavel Begunkov28a9fe22021-04-01 15:43:47 +01008380 io_rsrc_node_destroy(ref_node);
Pavel Begunkov3e942492021-04-11 01:46:34 +01008381 if (atomic_dec_and_test(&rsrc_data->refs))
8382 complete(&rsrc_data->done);
Jens Axboe05f3fb32019-12-09 11:22:50 -07008383}
8384
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00008385static void io_rsrc_put_work(struct work_struct *work)
Jens Axboe4a38aed22020-05-14 17:21:15 -06008386{
8387 struct io_ring_ctx *ctx;
8388 struct llist_node *node;
8389
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00008390 ctx = container_of(work, struct io_ring_ctx, rsrc_put_work.work);
8391 node = llist_del_all(&ctx->rsrc_put_llist);
Jens Axboe4a38aed22020-05-14 17:21:15 -06008392
8393 while (node) {
Pavel Begunkovb895c9a2021-04-01 15:43:40 +01008394 struct io_rsrc_node *ref_node;
Jens Axboe4a38aed22020-05-14 17:21:15 -06008395 struct llist_node *next = node->next;
8396
Pavel Begunkovb895c9a2021-04-01 15:43:40 +01008397 ref_node = llist_entry(node, struct io_rsrc_node, llist);
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00008398 __io_rsrc_put_work(ref_node);
Jens Axboe4a38aed22020-05-14 17:21:15 -06008399 node = next;
8400 }
8401}
8402
Jens Axboe05f3fb32019-12-09 11:22:50 -07008403static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg,
Pavel Begunkov792e3582021-04-25 14:32:21 +01008404 unsigned nr_args, u64 __user *tags)
Jens Axboe05f3fb32019-12-09 11:22:50 -07008405{
8406 __s32 __user *fds = (__s32 __user *) arg;
Jens Axboe05f3fb32019-12-09 11:22:50 -07008407 struct file *file;
Pavel Begunkovf3baed32021-04-01 15:43:42 +01008408 int fd, ret;
Pavel Begunkov846a4ef2021-04-01 15:44:03 +01008409 unsigned i;
Jens Axboe05f3fb32019-12-09 11:22:50 -07008410
8411 if (ctx->file_data)
8412 return -EBUSY;
8413 if (!nr_args)
8414 return -EINVAL;
8415 if (nr_args > IORING_MAX_FIXED_FILES)
8416 return -EMFILE;
Pavel Begunkov3a1b8a42021-08-20 10:36:35 +01008417 if (nr_args > rlimit(RLIMIT_NOFILE))
8418 return -EMFILE;
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01008419 ret = io_rsrc_node_switch_start(ctx);
Pavel Begunkovf3baed32021-04-01 15:43:42 +01008420 if (ret)
8421 return ret;
Pavel Begunkovd878c812021-06-14 02:36:18 +01008422 ret = io_rsrc_data_alloc(ctx, io_rsrc_file_put, tags, nr_args,
8423 &ctx->file_data);
8424 if (ret)
8425 return ret;
Jens Axboe05f3fb32019-12-09 11:22:50 -07008426
Pavel Begunkovf3baed32021-04-01 15:43:42 +01008427 ret = -ENOMEM;
Pavel Begunkovaeca2412021-04-11 01:46:37 +01008428 if (!io_alloc_file_tables(&ctx->file_table, nr_args))
Pavel Begunkov600cf3f2020-10-10 18:34:15 +01008429 goto out_free;
Jens Axboe05f3fb32019-12-09 11:22:50 -07008430
Jens Axboe05f3fb32019-12-09 11:22:50 -07008431 for (i = 0; i < nr_args; i++, ctx->nr_user_files++) {
Pavel Begunkovd878c812021-06-14 02:36:18 +01008432 if (copy_from_user(&fd, &fds[i], sizeof(fd))) {
Pavel Begunkov600cf3f2020-10-10 18:34:15 +01008433 ret = -EFAULT;
8434 goto out_fput;
Jens Axboe05f3fb32019-12-09 11:22:50 -07008435 }
Pavel Begunkov600cf3f2020-10-10 18:34:15 +01008436 /* allow sparse sets */
Pavel Begunkov792e3582021-04-25 14:32:21 +01008437 if (fd == -1) {
8438 ret = -EINVAL;
Pavel Begunkov2d091d62021-06-14 02:36:21 +01008439 if (unlikely(*io_get_tag_slot(ctx->file_data, i)))
Pavel Begunkov792e3582021-04-25 14:32:21 +01008440 goto out_fput;
Pavel Begunkov600cf3f2020-10-10 18:34:15 +01008441 continue;
Pavel Begunkov792e3582021-04-25 14:32:21 +01008442 }
Jens Axboe05f3fb32019-12-09 11:22:50 -07008443
Jens Axboe05f3fb32019-12-09 11:22:50 -07008444 file = fget(fd);
Jens Axboe05f3fb32019-12-09 11:22:50 -07008445 ret = -EBADF;
Pavel Begunkov792e3582021-04-25 14:32:21 +01008446 if (unlikely(!file))
Pavel Begunkov600cf3f2020-10-10 18:34:15 +01008447 goto out_fput;
Jens Axboe05f3fb32019-12-09 11:22:50 -07008448
8449 /*
8450 * Don't allow io_uring instances to be registered. If UNIX
8451 * isn't enabled, then this causes a reference cycle and this
8452 * instance can never get freed. If UNIX is enabled we'll
8453 * handle it just fine, but there's still no point in allowing
8454 * a ring fd as it doesn't support regular read/write anyway.
8455 */
8456 if (file->f_op == &io_uring_fops) {
8457 fput(file);
Pavel Begunkov600cf3f2020-10-10 18:34:15 +01008458 goto out_fput;
Jens Axboe05f3fb32019-12-09 11:22:50 -07008459 }
Pavel Begunkovaeca2412021-04-11 01:46:37 +01008460 io_fixed_file_set(io_fixed_file_slot(&ctx->file_table, i), file);
Jens Axboe05f3fb32019-12-09 11:22:50 -07008461 }
8462
Jens Axboe05f3fb32019-12-09 11:22:50 -07008463 ret = io_sqe_files_scm(ctx);
Xiaoguang Wang05589552020-03-31 14:05:18 +08008464 if (ret) {
Pavel Begunkov08480402021-04-13 02:58:38 +01008465 __io_sqe_files_unregister(ctx);
Xiaoguang Wang05589552020-03-31 14:05:18 +08008466 return ret;
8467 }
Jens Axboe05f3fb32019-12-09 11:22:50 -07008468
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01008469 io_rsrc_node_switch(ctx, NULL);
Jens Axboe05f3fb32019-12-09 11:22:50 -07008470 return ret;
Pavel Begunkov600cf3f2020-10-10 18:34:15 +01008471out_fput:
8472 for (i = 0; i < ctx->nr_user_files; i++) {
8473 file = io_file_from_index(ctx, i);
8474 if (file)
8475 fput(file);
8476 }
Pavel Begunkov042b0d82021-08-09 13:04:01 +01008477 io_free_file_tables(&ctx->file_table);
Pavel Begunkov600cf3f2020-10-10 18:34:15 +01008478 ctx->nr_user_files = 0;
Pavel Begunkov600cf3f2020-10-10 18:34:15 +01008479out_free:
Pavel Begunkov44b31f22021-04-25 14:32:16 +01008480 io_rsrc_data_free(ctx->file_data);
Jens Axboe55cbc252020-10-14 07:35:57 -06008481 ctx->file_data = NULL;
Jens Axboe05f3fb32019-12-09 11:22:50 -07008482 return ret;
8483}
8484
Jens Axboec3a31e62019-10-03 13:59:56 -06008485static int io_sqe_file_register(struct io_ring_ctx *ctx, struct file *file,
8486 int index)
8487{
8488#if defined(CONFIG_UNIX)
8489 struct sock *sock = ctx->ring_sock->sk;
8490 struct sk_buff_head *head = &sock->sk_receive_queue;
8491 struct sk_buff *skb;
8492
8493 /*
8494 * See if we can merge this file into an existing skb SCM_RIGHTS
8495 * file set. If there's no room, fall back to allocating a new skb
8496 * and filling it in.
8497 */
8498 spin_lock_irq(&head->lock);
8499 skb = skb_peek(head);
8500 if (skb) {
8501 struct scm_fp_list *fpl = UNIXCB(skb).fp;
8502
8503 if (fpl->count < SCM_MAX_FD) {
8504 __skb_unlink(skb, head);
8505 spin_unlock_irq(&head->lock);
8506 fpl->fp[fpl->count] = get_file(file);
8507 unix_inflight(fpl->user, fpl->fp[fpl->count]);
8508 fpl->count++;
8509 spin_lock_irq(&head->lock);
8510 __skb_queue_head(head, skb);
8511 } else {
8512 skb = NULL;
8513 }
8514 }
8515 spin_unlock_irq(&head->lock);
8516
8517 if (skb) {
8518 fput(file);
8519 return 0;
8520 }
8521
8522 return __io_sqe_files_scm(ctx, 1, index);
8523#else
8524 return 0;
8525#endif
8526}
8527
Pavel Begunkov9c7b0ba2021-09-14 16:12:52 +01008528static int io_queue_rsrc_removal(struct io_rsrc_data *data, unsigned idx,
8529 struct io_rsrc_node *node, void *rsrc)
8530{
Pavel Begunkov5218d5c2022-04-07 14:05:04 +01008531 u64 *tag_slot = io_get_tag_slot(data, idx);
Pavel Begunkov9c7b0ba2021-09-14 16:12:52 +01008532 struct io_rsrc_put *prsrc;
8533
8534 prsrc = kzalloc(sizeof(*prsrc), GFP_KERNEL);
8535 if (!prsrc)
8536 return -ENOMEM;
8537
Pavel Begunkov5218d5c2022-04-07 14:05:04 +01008538 prsrc->tag = *tag_slot;
8539 *tag_slot = 0;
Pavel Begunkov9c7b0ba2021-09-14 16:12:52 +01008540 prsrc->rsrc = rsrc;
8541 list_add(&prsrc->list, &node->rsrc_list);
8542 return 0;
8543}
8544
Pavel Begunkovb9445592021-08-25 12:25:45 +01008545static int io_install_fixed_file(struct io_kiocb *req, struct file *file,
8546 unsigned int issue_flags, u32 slot_index)
8547{
8548 struct io_ring_ctx *ctx = req->ctx;
8549 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
Pavel Begunkov9c7b0ba2021-09-14 16:12:52 +01008550 bool needs_switch = false;
Pavel Begunkovb9445592021-08-25 12:25:45 +01008551 struct io_fixed_file *file_slot;
8552 int ret = -EBADF;
8553
8554 io_ring_submit_lock(ctx, !force_nonblock);
8555 if (file->f_op == &io_uring_fops)
8556 goto err;
8557 ret = -ENXIO;
8558 if (!ctx->file_data)
8559 goto err;
8560 ret = -EINVAL;
8561 if (slot_index >= ctx->nr_user_files)
8562 goto err;
8563
8564 slot_index = array_index_nospec(slot_index, ctx->nr_user_files);
8565 file_slot = io_fixed_file_slot(&ctx->file_table, slot_index);
Pavel Begunkov9c7b0ba2021-09-14 16:12:52 +01008566
8567 if (file_slot->file_ptr) {
8568 struct file *old_file;
8569
8570 ret = io_rsrc_node_switch_start(ctx);
8571 if (ret)
8572 goto err;
8573
8574 old_file = (struct file *)(file_slot->file_ptr & FFS_MASK);
8575 ret = io_queue_rsrc_removal(ctx->file_data, slot_index,
8576 ctx->rsrc_node, old_file);
8577 if (ret)
8578 goto err;
8579 file_slot->file_ptr = 0;
8580 needs_switch = true;
8581 }
Pavel Begunkovb9445592021-08-25 12:25:45 +01008582
8583 *io_get_tag_slot(ctx->file_data, slot_index) = 0;
8584 io_fixed_file_set(file_slot, file);
8585 ret = io_sqe_file_register(ctx, file, slot_index);
8586 if (ret) {
8587 file_slot->file_ptr = 0;
8588 goto err;
8589 }
8590
8591 ret = 0;
8592err:
Pavel Begunkov9c7b0ba2021-09-14 16:12:52 +01008593 if (needs_switch)
8594 io_rsrc_node_switch(ctx, ctx->file_data);
Pavel Begunkovb9445592021-08-25 12:25:45 +01008595 io_ring_submit_unlock(ctx, !force_nonblock);
8596 if (ret)
8597 fput(file);
8598 return ret;
8599}
8600
Pavel Begunkov7df778b2021-09-24 20:04:29 +01008601static int io_close_fixed(struct io_kiocb *req, unsigned int issue_flags)
8602{
8603 unsigned int offset = req->close.file_slot - 1;
8604 struct io_ring_ctx *ctx = req->ctx;
8605 struct io_fixed_file *file_slot;
8606 struct file *file;
Pavel Begunkovb8ed0f72022-04-07 14:05:05 +01008607 int ret;
Pavel Begunkov7df778b2021-09-24 20:04:29 +01008608
8609 io_ring_submit_lock(ctx, !(issue_flags & IO_URING_F_NONBLOCK));
8610 ret = -ENXIO;
8611 if (unlikely(!ctx->file_data))
8612 goto out;
8613 ret = -EINVAL;
8614 if (offset >= ctx->nr_user_files)
8615 goto out;
8616 ret = io_rsrc_node_switch_start(ctx);
8617 if (ret)
8618 goto out;
8619
Pavel Begunkovb8ed0f72022-04-07 14:05:05 +01008620 offset = array_index_nospec(offset, ctx->nr_user_files);
8621 file_slot = io_fixed_file_slot(&ctx->file_table, offset);
Pavel Begunkov7df778b2021-09-24 20:04:29 +01008622 ret = -EBADF;
8623 if (!file_slot->file_ptr)
8624 goto out;
8625
8626 file = (struct file *)(file_slot->file_ptr & FFS_MASK);
8627 ret = io_queue_rsrc_removal(ctx->file_data, offset, ctx->rsrc_node, file);
8628 if (ret)
8629 goto out;
8630
8631 file_slot->file_ptr = 0;
8632 io_rsrc_node_switch(ctx, ctx->file_data);
8633 ret = 0;
8634out:
8635 io_ring_submit_unlock(ctx, !(issue_flags & IO_URING_F_NONBLOCK));
8636 return ret;
8637}
8638
Jens Axboe05f3fb32019-12-09 11:22:50 -07008639static int __io_sqe_files_update(struct io_ring_ctx *ctx,
Pavel Begunkovc3bdad02021-04-25 14:32:22 +01008640 struct io_uring_rsrc_update2 *up,
Jens Axboe05f3fb32019-12-09 11:22:50 -07008641 unsigned nr_args)
8642{
Pavel Begunkovc3bdad02021-04-25 14:32:22 +01008643 u64 __user *tags = u64_to_user_ptr(up->tags);
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +01008644 __s32 __user *fds = u64_to_user_ptr(up->data);
Pavel Begunkovb895c9a2021-04-01 15:43:40 +01008645 struct io_rsrc_data *data = ctx->file_data;
Pavel Begunkova04b0ac2021-04-01 15:44:04 +01008646 struct io_fixed_file *file_slot;
8647 struct file *file;
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +01008648 int fd, i, err = 0;
8649 unsigned int done;
Xiaoguang Wang05589552020-03-31 14:05:18 +08008650 bool needs_switch = false;
Jens Axboec3a31e62019-10-03 13:59:56 -06008651
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +01008652 if (!ctx->file_data)
8653 return -ENXIO;
8654 if (up->offset + nr_args > ctx->nr_user_files)
Jens Axboec3a31e62019-10-03 13:59:56 -06008655 return -EINVAL;
8656
Pavel Begunkov67973b92021-01-26 13:51:09 +00008657 for (done = 0; done < nr_args; done++) {
Pavel Begunkovc3bdad02021-04-25 14:32:22 +01008658 u64 tag = 0;
8659
8660 if ((tags && copy_from_user(&tag, &tags[done], sizeof(tag))) ||
8661 copy_from_user(&fd, &fds[done], sizeof(fd))) {
Jens Axboec3a31e62019-10-03 13:59:56 -06008662 err = -EFAULT;
8663 break;
8664 }
Pavel Begunkovc3bdad02021-04-25 14:32:22 +01008665 if ((fd == IORING_REGISTER_FILES_SKIP || fd == -1) && tag) {
8666 err = -EINVAL;
8667 break;
8668 }
noah4e0377a2021-01-26 15:23:28 -05008669 if (fd == IORING_REGISTER_FILES_SKIP)
8670 continue;
8671
Pavel Begunkov67973b92021-01-26 13:51:09 +00008672 i = array_index_nospec(up->offset + done, ctx->nr_user_files);
Pavel Begunkovaeca2412021-04-11 01:46:37 +01008673 file_slot = io_fixed_file_slot(&ctx->file_table, i);
Pavel Begunkovea64ec022021-02-04 13:52:07 +00008674
Pavel Begunkova04b0ac2021-04-01 15:44:04 +01008675 if (file_slot->file_ptr) {
8676 file = (struct file *)(file_slot->file_ptr & FFS_MASK);
Pavel Begunkovb8ed0f72022-04-07 14:05:05 +01008677 err = io_queue_rsrc_removal(data, i, ctx->rsrc_node, file);
Hillf Dantona5318d32020-03-23 17:47:15 +08008678 if (err)
8679 break;
Pavel Begunkova04b0ac2021-04-01 15:44:04 +01008680 file_slot->file_ptr = 0;
Xiaoguang Wang05589552020-03-31 14:05:18 +08008681 needs_switch = true;
Jens Axboec3a31e62019-10-03 13:59:56 -06008682 }
8683 if (fd != -1) {
Jens Axboec3a31e62019-10-03 13:59:56 -06008684 file = fget(fd);
8685 if (!file) {
8686 err = -EBADF;
8687 break;
8688 }
8689 /*
8690 * Don't allow io_uring instances to be registered. If
8691 * UNIX isn't enabled, then this causes a reference
8692 * cycle and this instance can never get freed. If UNIX
8693 * is enabled we'll handle it just fine, but there's
8694 * still no point in allowing a ring fd as it doesn't
8695 * support regular read/write anyway.
8696 */
8697 if (file->f_op == &io_uring_fops) {
8698 fput(file);
8699 err = -EBADF;
8700 break;
8701 }
Pavel Begunkov50c981b2022-04-06 12:43:57 +01008702 *io_get_tag_slot(data, i) = tag;
Pavel Begunkov9a321c92021-04-01 15:44:01 +01008703 io_fixed_file_set(file_slot, file);
Jens Axboec3a31e62019-10-03 13:59:56 -06008704 err = io_sqe_file_register(ctx, file, i);
Yang Yingliangf3bd9da2020-07-09 10:11:41 +00008705 if (err) {
Pavel Begunkova04b0ac2021-04-01 15:44:04 +01008706 file_slot->file_ptr = 0;
Yang Yingliangf3bd9da2020-07-09 10:11:41 +00008707 fput(file);
Jens Axboec3a31e62019-10-03 13:59:56 -06008708 break;
Yang Yingliangf3bd9da2020-07-09 10:11:41 +00008709 }
Jens Axboec3a31e62019-10-03 13:59:56 -06008710 }
Jens Axboe05f3fb32019-12-09 11:22:50 -07008711 }
8712
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01008713 if (needs_switch)
8714 io_rsrc_node_switch(ctx, data);
Jens Axboec3a31e62019-10-03 13:59:56 -06008715 return done ? done : err;
8716}
Xiaoguang Wang05589552020-03-31 14:05:18 +08008717
Jens Axboe685fe7f2021-03-08 09:37:51 -07008718static struct io_wq *io_init_wq_offload(struct io_ring_ctx *ctx,
8719 struct task_struct *task)
Pavel Begunkov24369c22020-01-28 03:15:48 +03008720{
Jens Axboee9418942021-02-19 12:33:30 -07008721 struct io_wq_hash *hash;
Pavel Begunkov24369c22020-01-28 03:15:48 +03008722 struct io_wq_data data;
Pavel Begunkov24369c22020-01-28 03:15:48 +03008723 unsigned int concurrency;
Pavel Begunkov24369c22020-01-28 03:15:48 +03008724
Yang Yingliang362a9e62021-07-20 16:38:05 +08008725 mutex_lock(&ctx->uring_lock);
Jens Axboee9418942021-02-19 12:33:30 -07008726 hash = ctx->hash_map;
8727 if (!hash) {
8728 hash = kzalloc(sizeof(*hash), GFP_KERNEL);
Yang Yingliang362a9e62021-07-20 16:38:05 +08008729 if (!hash) {
8730 mutex_unlock(&ctx->uring_lock);
Jens Axboee9418942021-02-19 12:33:30 -07008731 return ERR_PTR(-ENOMEM);
Yang Yingliang362a9e62021-07-20 16:38:05 +08008732 }
Jens Axboee9418942021-02-19 12:33:30 -07008733 refcount_set(&hash->refs, 1);
8734 init_waitqueue_head(&hash->wait);
8735 ctx->hash_map = hash;
8736 }
Yang Yingliang362a9e62021-07-20 16:38:05 +08008737 mutex_unlock(&ctx->uring_lock);
Jens Axboee9418942021-02-19 12:33:30 -07008738
8739 data.hash = hash;
Jens Axboe685fe7f2021-03-08 09:37:51 -07008740 data.task = task;
Pavel Begunkovebc11b62021-08-09 13:04:05 +01008741 data.free_work = io_wq_free_work;
Pavel Begunkovf5fa38c2020-06-08 21:08:20 +03008742 data.do_work = io_wq_submit_work;
Pavel Begunkov24369c22020-01-28 03:15:48 +03008743
Jens Axboed25e3a32021-02-16 11:41:41 -07008744 /* Do QD, or 4 * CPUS, whatever is smallest */
8745 concurrency = min(ctx->sq_entries, 4 * num_online_cpus());
Pavel Begunkov24369c22020-01-28 03:15:48 +03008746
Jens Axboe5aa75ed2021-02-16 12:56:50 -07008747 return io_wq_create(concurrency, &data);
Pavel Begunkov24369c22020-01-28 03:15:48 +03008748}
8749
Jens Axboe5aa75ed2021-02-16 12:56:50 -07008750static int io_uring_alloc_task_context(struct task_struct *task,
8751 struct io_ring_ctx *ctx)
Jens Axboe0f212202020-09-13 13:09:39 -06008752{
8753 struct io_uring_task *tctx;
Jens Axboed8a6df12020-10-15 16:24:45 -06008754 int ret;
Jens Axboe0f212202020-09-13 13:09:39 -06008755
Pavel Begunkov09899b12021-06-14 02:36:22 +01008756 tctx = kzalloc(sizeof(*tctx), GFP_KERNEL);
Jens Axboe0f212202020-09-13 13:09:39 -06008757 if (unlikely(!tctx))
8758 return -ENOMEM;
8759
Jens Axboed8a6df12020-10-15 16:24:45 -06008760 ret = percpu_counter_init(&tctx->inflight, 0, GFP_KERNEL);
8761 if (unlikely(ret)) {
8762 kfree(tctx);
8763 return ret;
8764 }
8765
Jens Axboe685fe7f2021-03-08 09:37:51 -07008766 tctx->io_wq = io_init_wq_offload(ctx, task);
Jens Axboe5aa75ed2021-02-16 12:56:50 -07008767 if (IS_ERR(tctx->io_wq)) {
8768 ret = PTR_ERR(tctx->io_wq);
8769 percpu_counter_destroy(&tctx->inflight);
8770 kfree(tctx);
8771 return ret;
8772 }
8773
Jens Axboe0f212202020-09-13 13:09:39 -06008774 xa_init(&tctx->xa);
8775 init_waitqueue_head(&tctx->wait);
Jens Axboefdaf0832020-10-30 09:37:30 -06008776 atomic_set(&tctx->in_idle, 0);
Pavel Begunkovb303fe22021-04-11 01:46:26 +01008777 atomic_set(&tctx->inflight_tracked, 0);
Jens Axboe0f212202020-09-13 13:09:39 -06008778 task->io_uring = tctx;
Jens Axboe7cbf1722021-02-10 00:03:20 +00008779 spin_lock_init(&tctx->task_lock);
8780 INIT_WQ_LIST(&tctx->task_list);
Jens Axboe7cbf1722021-02-10 00:03:20 +00008781 init_task_work(&tctx->task_work, tctx_task_work);
Jens Axboe0f212202020-09-13 13:09:39 -06008782 return 0;
8783}
8784
8785void __io_uring_free(struct task_struct *tsk)
8786{
8787 struct io_uring_task *tctx = tsk->io_uring;
8788
8789 WARN_ON_ONCE(!xa_empty(&tctx->xa));
Pavel Begunkovef8eaa42021-02-27 11:16:45 +00008790 WARN_ON_ONCE(tctx->io_wq);
Pavel Begunkov09899b12021-06-14 02:36:22 +01008791 WARN_ON_ONCE(tctx->cached_refs);
Pavel Begunkovef8eaa42021-02-27 11:16:45 +00008792
Jens Axboed8a6df12020-10-15 16:24:45 -06008793 percpu_counter_destroy(&tctx->inflight);
Jens Axboe0f212202020-09-13 13:09:39 -06008794 kfree(tctx);
8795 tsk->io_uring = NULL;
8796}
8797
Stefano Garzarella7e84e1c2020-08-27 16:58:31 +02008798static int io_sq_offload_create(struct io_ring_ctx *ctx,
8799 struct io_uring_params *p)
Jens Axboe2b188cc2019-01-07 10:46:33 -07008800{
8801 int ret;
8802
Jens Axboed25e3a32021-02-16 11:41:41 -07008803 /* Retain compatibility with failing for an invalid attach attempt */
8804 if ((ctx->flags & (IORING_SETUP_ATTACH_WQ | IORING_SETUP_SQPOLL)) ==
8805 IORING_SETUP_ATTACH_WQ) {
8806 struct fd f;
8807
8808 f = fdget(p->wq_fd);
8809 if (!f.file)
8810 return -ENXIO;
Jens Axboe0cc936f2021-07-22 17:08:07 -06008811 if (f.file->f_op != &io_uring_fops) {
8812 fdput(f);
Pavel Begunkovf2a48dd2021-04-20 12:03:33 +01008813 return -EINVAL;
Jens Axboe0cc936f2021-07-22 17:08:07 -06008814 }
8815 fdput(f);
Jens Axboed25e3a32021-02-16 11:41:41 -07008816 }
Jens Axboe6c271ce2019-01-10 11:22:30 -07008817 if (ctx->flags & IORING_SETUP_SQPOLL) {
Jens Axboe46fe18b2021-03-04 12:39:36 -07008818 struct task_struct *tsk;
Jens Axboe534ca6d2020-09-02 13:52:19 -06008819 struct io_sq_data *sqd;
Pavel Begunkov26984fb2021-03-11 23:29:37 +00008820 bool attached;
Jens Axboe534ca6d2020-09-02 13:52:19 -06008821
Pavel Begunkov26984fb2021-03-11 23:29:37 +00008822 sqd = io_get_sq_data(p, &attached);
Jens Axboe534ca6d2020-09-02 13:52:19 -06008823 if (IS_ERR(sqd)) {
8824 ret = PTR_ERR(sqd);
8825 goto err;
8826 }
Jens Axboe69fb2132020-09-14 11:16:23 -06008827
Stefan Metzmacher7c30f36a2021-03-07 11:54:28 +01008828 ctx->sq_creds = get_current_cred();
Jens Axboe534ca6d2020-09-02 13:52:19 -06008829 ctx->sq_data = sqd;
Jens Axboe917257d2019-04-13 09:28:55 -06008830 ctx->sq_thread_idle = msecs_to_jiffies(p->sq_thread_idle);
8831 if (!ctx->sq_thread_idle)
8832 ctx->sq_thread_idle = HZ;
8833
Pavel Begunkov78d7f6b2021-03-10 13:13:53 +00008834 io_sq_thread_park(sqd);
Pavel Begunkovde75a3d2021-03-18 11:54:35 +00008835 list_add(&ctx->sqd_list, &sqd->ctx_list);
8836 io_sqd_update_thread_idle(sqd);
Pavel Begunkov26984fb2021-03-11 23:29:37 +00008837 /* don't attach to a dying SQPOLL thread, would be racy */
Pavel Begunkovf2a48dd2021-04-20 12:03:33 +01008838 ret = (attached && !sqd->thread) ? -ENXIO : 0;
Pavel Begunkov78d7f6b2021-03-10 13:13:53 +00008839 io_sq_thread_unpark(sqd);
8840
Pavel Begunkovde75a3d2021-03-18 11:54:35 +00008841 if (ret < 0)
8842 goto err;
8843 if (attached)
Jens Axboe5aa75ed2021-02-16 12:56:50 -07008844 return 0;
Jens Axboeaa061652020-09-02 14:50:27 -06008845
Jens Axboe6c271ce2019-01-10 11:22:30 -07008846 if (p->flags & IORING_SETUP_SQ_AFF) {
Jens Axboe44a9bd12019-05-14 20:00:30 -06008847 int cpu = p->sq_thread_cpu;
Jens Axboe6c271ce2019-01-10 11:22:30 -07008848
Jens Axboe917257d2019-04-13 09:28:55 -06008849 ret = -EINVAL;
Pavel Begunkovf2a48dd2021-04-20 12:03:33 +01008850 if (cpu >= nr_cpu_ids || !cpu_online(cpu))
Jens Axboee8f98f242021-03-09 16:32:13 -07008851 goto err_sqpoll;
Jens Axboe37d1e2e2021-02-17 21:03:43 -07008852 sqd->sq_cpu = cpu;
Jens Axboe6c271ce2019-01-10 11:22:30 -07008853 } else {
Jens Axboe37d1e2e2021-02-17 21:03:43 -07008854 sqd->sq_cpu = -1;
Jens Axboe6c271ce2019-01-10 11:22:30 -07008855 }
Jens Axboe37d1e2e2021-02-17 21:03:43 -07008856
8857 sqd->task_pid = current->pid;
Jens Axboe5c2469e2021-03-11 10:17:56 -07008858 sqd->task_tgid = current->tgid;
Jens Axboe46fe18b2021-03-04 12:39:36 -07008859 tsk = create_io_thread(io_sq_thread, sqd, NUMA_NO_NODE);
8860 if (IS_ERR(tsk)) {
8861 ret = PTR_ERR(tsk);
Jens Axboee8f98f242021-03-09 16:32:13 -07008862 goto err_sqpoll;
Jens Axboe6c271ce2019-01-10 11:22:30 -07008863 }
Pavel Begunkov97a73a02021-03-08 17:30:54 +00008864
Jens Axboe46fe18b2021-03-04 12:39:36 -07008865 sqd->thread = tsk;
Pavel Begunkov97a73a02021-03-08 17:30:54 +00008866 ret = io_uring_alloc_task_context(tsk, ctx);
Jens Axboe46fe18b2021-03-04 12:39:36 -07008867 wake_up_new_task(tsk);
Jens Axboe0f212202020-09-13 13:09:39 -06008868 if (ret)
8869 goto err;
Jens Axboe6c271ce2019-01-10 11:22:30 -07008870 } else if (p->flags & IORING_SETUP_SQ_AFF) {
8871 /* Can't have SQ_AFF without SQPOLL */
8872 ret = -EINVAL;
8873 goto err;
8874 }
8875
Jens Axboe2b188cc2019-01-07 10:46:33 -07008876 return 0;
Pavel Begunkovf2a48dd2021-04-20 12:03:33 +01008877err_sqpoll:
8878 complete(&ctx->sq_data->exited);
Jens Axboe2b188cc2019-01-07 10:46:33 -07008879err:
Jens Axboe37d1e2e2021-02-17 21:03:43 -07008880 io_sq_thread_finish(ctx);
Jens Axboe2b188cc2019-01-07 10:46:33 -07008881 return ret;
8882}
8883
Bijan Mottahedeha087e2b2020-06-16 16:36:07 -07008884static inline void __io_unaccount_mem(struct user_struct *user,
8885 unsigned long nr_pages)
Jens Axboe2b188cc2019-01-07 10:46:33 -07008886{
8887 atomic_long_sub(nr_pages, &user->locked_vm);
8888}
8889
Bijan Mottahedeha087e2b2020-06-16 16:36:07 -07008890static inline int __io_account_mem(struct user_struct *user,
8891 unsigned long nr_pages)
Jens Axboe2b188cc2019-01-07 10:46:33 -07008892{
8893 unsigned long page_limit, cur_pages, new_pages;
8894
8895 /* Don't allow more pages than we can safely lock */
8896 page_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
8897
8898 do {
8899 cur_pages = atomic_long_read(&user->locked_vm);
8900 new_pages = cur_pages + nr_pages;
8901 if (new_pages > page_limit)
8902 return -ENOMEM;
8903 } while (atomic_long_cmpxchg(&user->locked_vm, cur_pages,
8904 new_pages) != cur_pages);
8905
8906 return 0;
8907}
8908
Jens Axboe26bfa89e2021-02-09 20:14:12 -07008909static void io_unaccount_mem(struct io_ring_ctx *ctx, unsigned long nr_pages)
Bijan Mottahedeha087e2b2020-06-16 16:36:07 -07008910{
Jens Axboe62e398b2021-02-21 16:19:37 -07008911 if (ctx->user)
Bijan Mottahedeha087e2b2020-06-16 16:36:07 -07008912 __io_unaccount_mem(ctx->user, nr_pages);
Bijan Mottahedeh30975822020-06-16 16:36:09 -07008913
Jens Axboe26bfa89e2021-02-09 20:14:12 -07008914 if (ctx->mm_account)
8915 atomic64_sub(nr_pages, &ctx->mm_account->pinned_vm);
Bijan Mottahedeha087e2b2020-06-16 16:36:07 -07008916}
8917
Jens Axboe26bfa89e2021-02-09 20:14:12 -07008918static int io_account_mem(struct io_ring_ctx *ctx, unsigned long nr_pages)
Bijan Mottahedeha087e2b2020-06-16 16:36:07 -07008919{
Bijan Mottahedeh30975822020-06-16 16:36:09 -07008920 int ret;
8921
Jens Axboe62e398b2021-02-21 16:19:37 -07008922 if (ctx->user) {
Bijan Mottahedeh30975822020-06-16 16:36:09 -07008923 ret = __io_account_mem(ctx->user, nr_pages);
8924 if (ret)
8925 return ret;
8926 }
8927
Jens Axboe26bfa89e2021-02-09 20:14:12 -07008928 if (ctx->mm_account)
8929 atomic64_add(nr_pages, &ctx->mm_account->pinned_vm);
Bijan Mottahedeha087e2b2020-06-16 16:36:07 -07008930
8931 return 0;
8932}
8933
Jens Axboe2b188cc2019-01-07 10:46:33 -07008934static void io_mem_free(void *ptr)
8935{
Mark Rutland52e04ef2019-04-30 17:30:21 +01008936 struct page *page;
Jens Axboe2b188cc2019-01-07 10:46:33 -07008937
Mark Rutland52e04ef2019-04-30 17:30:21 +01008938 if (!ptr)
8939 return;
8940
8941 page = virt_to_head_page(ptr);
Jens Axboe2b188cc2019-01-07 10:46:33 -07008942 if (put_page_testzero(page))
8943 free_compound_page(page);
8944}
8945
8946static void *io_mem_alloc(size_t size)
8947{
Shakeel Butt246dfbc2022-01-24 21:17:36 -08008948 gfp_t gfp = GFP_KERNEL_ACCOUNT | __GFP_ZERO | __GFP_NOWARN | __GFP_COMP;
Jens Axboe2b188cc2019-01-07 10:46:33 -07008949
Shakeel Butt246dfbc2022-01-24 21:17:36 -08008950 return (void *) __get_free_pages(gfp, get_order(size));
Jens Axboe2b188cc2019-01-07 10:46:33 -07008951}
8952
Hristo Venev75b28af2019-08-26 17:23:46 +00008953static unsigned long rings_size(unsigned sq_entries, unsigned cq_entries,
8954 size_t *sq_offset)
8955{
8956 struct io_rings *rings;
8957 size_t off, sq_array_size;
8958
8959 off = struct_size(rings, cqes, cq_entries);
8960 if (off == SIZE_MAX)
8961 return SIZE_MAX;
8962
8963#ifdef CONFIG_SMP
8964 off = ALIGN(off, SMP_CACHE_BYTES);
8965 if (off == 0)
8966 return SIZE_MAX;
8967#endif
8968
Dmitry Vyukovb36200f2020-07-11 11:31:11 +02008969 if (sq_offset)
8970 *sq_offset = off;
8971
Hristo Venev75b28af2019-08-26 17:23:46 +00008972 sq_array_size = array_size(sizeof(u32), sq_entries);
8973 if (sq_array_size == SIZE_MAX)
8974 return SIZE_MAX;
8975
8976 if (check_add_overflow(off, sq_array_size, &off))
8977 return SIZE_MAX;
8978
Hristo Venev75b28af2019-08-26 17:23:46 +00008979 return off;
8980}
8981
Pavel Begunkov41edf1a2021-04-25 14:32:23 +01008982static void io_buffer_unmap(struct io_ring_ctx *ctx, struct io_mapped_ubuf **slot)
Pavel Begunkov7f61a1e2021-04-11 01:46:35 +01008983{
Pavel Begunkov41edf1a2021-04-25 14:32:23 +01008984 struct io_mapped_ubuf *imu = *slot;
Pavel Begunkov7f61a1e2021-04-11 01:46:35 +01008985 unsigned int i;
8986
Pavel Begunkov62248432021-04-28 13:11:29 +01008987 if (imu != ctx->dummy_ubuf) {
8988 for (i = 0; i < imu->nr_bvecs; i++)
8989 unpin_user_page(imu->bvec[i].bv_page);
8990 if (imu->acct_pages)
8991 io_unaccount_mem(ctx, imu->acct_pages);
8992 kvfree(imu);
8993 }
Pavel Begunkov41edf1a2021-04-25 14:32:23 +01008994 *slot = NULL;
Pavel Begunkov7f61a1e2021-04-11 01:46:35 +01008995}
8996
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01008997static void io_rsrc_buf_put(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc)
8998{
Pavel Begunkov634d00d2021-04-25 14:32:26 +01008999 io_buffer_unmap(ctx, &prsrc->buf);
9000 prsrc->buf = NULL;
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01009001}
9002
9003static void __io_sqe_buffers_unregister(struct io_ring_ctx *ctx)
Jens Axboeedafcce2019-01-09 09:16:05 -07009004{
Pavel Begunkov7f61a1e2021-04-11 01:46:35 +01009005 unsigned int i;
Jens Axboeedafcce2019-01-09 09:16:05 -07009006
Pavel Begunkov7f61a1e2021-04-11 01:46:35 +01009007 for (i = 0; i < ctx->nr_user_bufs; i++)
9008 io_buffer_unmap(ctx, &ctx->user_bufs[i]);
Jens Axboeedafcce2019-01-09 09:16:05 -07009009 kfree(ctx->user_bufs);
Zqiangbb6659c2021-04-30 16:25:15 +08009010 io_rsrc_data_free(ctx->buf_data);
Jens Axboeedafcce2019-01-09 09:16:05 -07009011 ctx->user_bufs = NULL;
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01009012 ctx->buf_data = NULL;
Jens Axboeedafcce2019-01-09 09:16:05 -07009013 ctx->nr_user_bufs = 0;
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01009014}
9015
Jens Axboeedafcce2019-01-09 09:16:05 -07009016static int io_sqe_buffers_unregister(struct io_ring_ctx *ctx)
9017{
Pavel Begunkov91f5a602022-06-13 06:30:06 +01009018 unsigned nr = ctx->nr_user_bufs;
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01009019 int ret;
Jens Axboeedafcce2019-01-09 09:16:05 -07009020
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01009021 if (!ctx->buf_data)
Jens Axboeedafcce2019-01-09 09:16:05 -07009022 return -ENXIO;
9023
Pavel Begunkov91f5a602022-06-13 06:30:06 +01009024 /*
9025 * Quiesce may unlock ->uring_lock, and while it's not held
9026 * prevent new requests using the table.
9027 */
9028 ctx->nr_user_bufs = 0;
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01009029 ret = io_rsrc_ref_quiesce(ctx->buf_data, ctx);
Pavel Begunkov91f5a602022-06-13 06:30:06 +01009030 ctx->nr_user_bufs = nr;
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01009031 if (!ret)
9032 __io_sqe_buffers_unregister(ctx);
9033 return ret;
Jens Axboeedafcce2019-01-09 09:16:05 -07009034}
9035
9036static int io_copy_iov(struct io_ring_ctx *ctx, struct iovec *dst,
9037 void __user *arg, unsigned index)
9038{
9039 struct iovec __user *src;
9040
9041#ifdef CONFIG_COMPAT
9042 if (ctx->compat) {
9043 struct compat_iovec __user *ciovs;
9044 struct compat_iovec ciov;
9045
9046 ciovs = (struct compat_iovec __user *) arg;
9047 if (copy_from_user(&ciov, &ciovs[index], sizeof(ciov)))
9048 return -EFAULT;
9049
Jens Axboed55e5f52019-12-11 16:12:15 -07009050 dst->iov_base = u64_to_user_ptr((u64)ciov.iov_base);
Jens Axboeedafcce2019-01-09 09:16:05 -07009051 dst->iov_len = ciov.iov_len;
9052 return 0;
9053 }
9054#endif
9055 src = (struct iovec __user *) arg;
9056 if (copy_from_user(dst, &src[index], sizeof(*dst)))
9057 return -EFAULT;
9058 return 0;
9059}
9060
Jens Axboede293932020-09-17 16:19:16 -06009061/*
9062 * Not super efficient, but this is just a registration time. And we do cache
9063 * the last compound head, so generally we'll only do a full search if we don't
9064 * match that one.
9065 *
9066 * We check if the given compound head page has already been accounted, to
9067 * avoid double accounting it. This allows us to account the full size of the
9068 * page, not just the constituent pages of a huge page.
9069 */
9070static bool headpage_already_acct(struct io_ring_ctx *ctx, struct page **pages,
9071 int nr_pages, struct page *hpage)
9072{
9073 int i, j;
9074
9075 /* check current page array */
9076 for (i = 0; i < nr_pages; i++) {
9077 if (!PageCompound(pages[i]))
9078 continue;
9079 if (compound_head(pages[i]) == hpage)
9080 return true;
9081 }
9082
9083 /* check previously registered pages */
9084 for (i = 0; i < ctx->nr_user_bufs; i++) {
Pavel Begunkov41edf1a2021-04-25 14:32:23 +01009085 struct io_mapped_ubuf *imu = ctx->user_bufs[i];
Jens Axboede293932020-09-17 16:19:16 -06009086
9087 for (j = 0; j < imu->nr_bvecs; j++) {
9088 if (!PageCompound(imu->bvec[j].bv_page))
9089 continue;
9090 if (compound_head(imu->bvec[j].bv_page) == hpage)
9091 return true;
9092 }
9093 }
9094
9095 return false;
9096}
9097
9098static int io_buffer_account_pin(struct io_ring_ctx *ctx, struct page **pages,
9099 int nr_pages, struct io_mapped_ubuf *imu,
9100 struct page **last_hpage)
9101{
9102 int i, ret;
9103
Pavel Begunkov216e5832021-05-29 12:01:02 +01009104 imu->acct_pages = 0;
Jens Axboede293932020-09-17 16:19:16 -06009105 for (i = 0; i < nr_pages; i++) {
9106 if (!PageCompound(pages[i])) {
9107 imu->acct_pages++;
9108 } else {
9109 struct page *hpage;
9110
9111 hpage = compound_head(pages[i]);
9112 if (hpage == *last_hpage)
9113 continue;
9114 *last_hpage = hpage;
9115 if (headpage_already_acct(ctx, pages, i, hpage))
9116 continue;
9117 imu->acct_pages += page_size(hpage) >> PAGE_SHIFT;
9118 }
9119 }
9120
9121 if (!imu->acct_pages)
9122 return 0;
9123
Jens Axboe26bfa89e2021-02-09 20:14:12 -07009124 ret = io_account_mem(ctx, imu->acct_pages);
Jens Axboede293932020-09-17 16:19:16 -06009125 if (ret)
9126 imu->acct_pages = 0;
9127 return ret;
9128}
9129
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08009130static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov,
Pavel Begunkov41edf1a2021-04-25 14:32:23 +01009131 struct io_mapped_ubuf **pimu,
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08009132 struct page **last_hpage)
Jens Axboeedafcce2019-01-09 09:16:05 -07009133{
Pavel Begunkov41edf1a2021-04-25 14:32:23 +01009134 struct io_mapped_ubuf *imu = NULL;
Jens Axboeedafcce2019-01-09 09:16:05 -07009135 struct vm_area_struct **vmas = NULL;
9136 struct page **pages = NULL;
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08009137 unsigned long off, start, end, ubuf;
9138 size_t size;
9139 int ret, pret, nr_pages, i;
Jens Axboeedafcce2019-01-09 09:16:05 -07009140
Pavel Begunkov62248432021-04-28 13:11:29 +01009141 if (!iov->iov_base) {
9142 *pimu = ctx->dummy_ubuf;
9143 return 0;
9144 }
9145
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08009146 ubuf = (unsigned long) iov->iov_base;
9147 end = (ubuf + iov->iov_len + PAGE_SIZE - 1) >> PAGE_SHIFT;
9148 start = ubuf >> PAGE_SHIFT;
9149 nr_pages = end - start;
9150
Pavel Begunkov41edf1a2021-04-25 14:32:23 +01009151 *pimu = NULL;
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08009152 ret = -ENOMEM;
9153
9154 pages = kvmalloc_array(nr_pages, sizeof(struct page *), GFP_KERNEL);
9155 if (!pages)
9156 goto done;
9157
9158 vmas = kvmalloc_array(nr_pages, sizeof(struct vm_area_struct *),
9159 GFP_KERNEL);
9160 if (!vmas)
9161 goto done;
9162
Pavel Begunkov41edf1a2021-04-25 14:32:23 +01009163 imu = kvmalloc(struct_size(imu, bvec, nr_pages), GFP_KERNEL);
Pavel Begunkova2b41982021-04-26 00:16:31 +01009164 if (!imu)
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08009165 goto done;
9166
9167 ret = 0;
9168 mmap_read_lock(current->mm);
9169 pret = pin_user_pages(ubuf, nr_pages, FOLL_WRITE | FOLL_LONGTERM,
9170 pages, vmas);
9171 if (pret == nr_pages) {
9172 /* don't support file backed memory */
9173 for (i = 0; i < nr_pages; i++) {
9174 struct vm_area_struct *vma = vmas[i];
9175
Pavel Begunkov40dad762021-06-09 15:26:54 +01009176 if (vma_is_shmem(vma))
9177 continue;
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08009178 if (vma->vm_file &&
9179 !is_file_hugepages(vma->vm_file)) {
9180 ret = -EOPNOTSUPP;
9181 break;
9182 }
9183 }
9184 } else {
9185 ret = pret < 0 ? pret : -EFAULT;
9186 }
9187 mmap_read_unlock(current->mm);
9188 if (ret) {
9189 /*
9190 * if we did partial map, or found file backed vmas,
9191 * release any pages we did get
9192 */
9193 if (pret > 0)
9194 unpin_user_pages(pages, pret);
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08009195 goto done;
9196 }
9197
9198 ret = io_buffer_account_pin(ctx, pages, pret, imu, last_hpage);
9199 if (ret) {
9200 unpin_user_pages(pages, pret);
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08009201 goto done;
9202 }
9203
9204 off = ubuf & ~PAGE_MASK;
9205 size = iov->iov_len;
9206 for (i = 0; i < nr_pages; i++) {
9207 size_t vec_len;
9208
9209 vec_len = min_t(size_t, size, PAGE_SIZE - off);
9210 imu->bvec[i].bv_page = pages[i];
9211 imu->bvec[i].bv_len = vec_len;
9212 imu->bvec[i].bv_offset = off;
9213 off = 0;
9214 size -= vec_len;
9215 }
9216 /* store original address for later verification */
9217 imu->ubuf = ubuf;
Pavel Begunkov4751f532021-04-01 15:43:55 +01009218 imu->ubuf_end = ubuf + iov->iov_len;
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08009219 imu->nr_bvecs = nr_pages;
Pavel Begunkov41edf1a2021-04-25 14:32:23 +01009220 *pimu = imu;
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08009221 ret = 0;
9222done:
Pavel Begunkov41edf1a2021-04-25 14:32:23 +01009223 if (ret)
9224 kvfree(imu);
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08009225 kvfree(pages);
9226 kvfree(vmas);
9227 return ret;
9228}
9229
Bijan Mottahedeh2b358602021-01-06 12:39:11 -08009230static int io_buffers_map_alloc(struct io_ring_ctx *ctx, unsigned int nr_args)
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08009231{
Pavel Begunkov87094462021-04-11 01:46:36 +01009232 ctx->user_bufs = kcalloc(nr_args, sizeof(*ctx->user_bufs), GFP_KERNEL);
9233 return ctx->user_bufs ? 0 : -ENOMEM;
Bijan Mottahedeh2b358602021-01-06 12:39:11 -08009234}
9235
9236static int io_buffer_validate(struct iovec *iov)
9237{
Pavel Begunkov50e96982021-03-24 22:59:01 +00009238 unsigned long tmp, acct_len = iov->iov_len + (PAGE_SIZE - 1);
9239
Bijan Mottahedeh2b358602021-01-06 12:39:11 -08009240 /*
9241 * Don't impose further limits on the size and buffer
9242 * constraints here, we'll -EINVAL later when IO is
9243 * submitted if they are wrong.
9244 */
Pavel Begunkov62248432021-04-28 13:11:29 +01009245 if (!iov->iov_base)
9246 return iov->iov_len ? -EFAULT : 0;
9247 if (!iov->iov_len)
Bijan Mottahedeh2b358602021-01-06 12:39:11 -08009248 return -EFAULT;
9249
9250 /* arbitrary limit, but we need something */
9251 if (iov->iov_len > SZ_1G)
9252 return -EFAULT;
9253
Pavel Begunkov50e96982021-03-24 22:59:01 +00009254 if (check_add_overflow((unsigned long)iov->iov_base, acct_len, &tmp))
9255 return -EOVERFLOW;
9256
Bijan Mottahedeh2b358602021-01-06 12:39:11 -08009257 return 0;
9258}
9259
9260static int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg,
Pavel Begunkov634d00d2021-04-25 14:32:26 +01009261 unsigned int nr_args, u64 __user *tags)
Bijan Mottahedeh2b358602021-01-06 12:39:11 -08009262{
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01009263 struct page *last_hpage = NULL;
9264 struct io_rsrc_data *data;
Bijan Mottahedeh2b358602021-01-06 12:39:11 -08009265 int i, ret;
9266 struct iovec iov;
Bijan Mottahedeh2b358602021-01-06 12:39:11 -08009267
Pavel Begunkov87094462021-04-11 01:46:36 +01009268 if (ctx->user_bufs)
9269 return -EBUSY;
Pavel Begunkov489809e2021-05-14 12:06:44 +01009270 if (!nr_args || nr_args > IORING_MAX_REG_BUFFERS)
Pavel Begunkov87094462021-04-11 01:46:36 +01009271 return -EINVAL;
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01009272 ret = io_rsrc_node_switch_start(ctx);
Bijan Mottahedeh2b358602021-01-06 12:39:11 -08009273 if (ret)
9274 return ret;
Pavel Begunkovd878c812021-06-14 02:36:18 +01009275 ret = io_rsrc_data_alloc(ctx, io_rsrc_buf_put, tags, nr_args, &data);
9276 if (ret)
9277 return ret;
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01009278 ret = io_buffers_map_alloc(ctx, nr_args);
9279 if (ret) {
Zqiangbb6659c2021-04-30 16:25:15 +08009280 io_rsrc_data_free(data);
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01009281 return ret;
9282 }
Bijan Mottahedeh2b358602021-01-06 12:39:11 -08009283
Pavel Begunkov87094462021-04-11 01:46:36 +01009284 for (i = 0; i < nr_args; i++, ctx->nr_user_bufs++) {
Jens Axboeedafcce2019-01-09 09:16:05 -07009285 ret = io_copy_iov(ctx, &iov, arg, i);
9286 if (ret)
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08009287 break;
Bijan Mottahedeh2b358602021-01-06 12:39:11 -08009288 ret = io_buffer_validate(&iov);
9289 if (ret)
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08009290 break;
Pavel Begunkov2d091d62021-06-14 02:36:21 +01009291 if (!iov.iov_base && *io_get_tag_slot(data, i)) {
Colin Ian Kingcf3770e2021-04-29 11:46:02 +01009292 ret = -EINVAL;
9293 break;
9294 }
Jens Axboeedafcce2019-01-09 09:16:05 -07009295
Pavel Begunkov41edf1a2021-04-25 14:32:23 +01009296 ret = io_sqe_buffer_register(ctx, &iov, &ctx->user_bufs[i],
9297 &last_hpage);
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08009298 if (ret)
9299 break;
Jens Axboeedafcce2019-01-09 09:16:05 -07009300 }
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08009301
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01009302 WARN_ON_ONCE(ctx->buf_data);
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08009303
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01009304 ctx->buf_data = data;
9305 if (ret)
9306 __io_sqe_buffers_unregister(ctx);
9307 else
9308 io_rsrc_node_switch(ctx, NULL);
Jens Axboeedafcce2019-01-09 09:16:05 -07009309 return ret;
9310}
9311
Pavel Begunkov634d00d2021-04-25 14:32:26 +01009312static int __io_sqe_buffers_update(struct io_ring_ctx *ctx,
9313 struct io_uring_rsrc_update2 *up,
9314 unsigned int nr_args)
9315{
9316 u64 __user *tags = u64_to_user_ptr(up->tags);
9317 struct iovec iov, __user *iovs = u64_to_user_ptr(up->data);
Pavel Begunkov634d00d2021-04-25 14:32:26 +01009318 struct page *last_hpage = NULL;
9319 bool needs_switch = false;
9320 __u32 done;
9321 int i, err;
9322
9323 if (!ctx->buf_data)
9324 return -ENXIO;
9325 if (up->offset + nr_args > ctx->nr_user_bufs)
9326 return -EINVAL;
9327
9328 for (done = 0; done < nr_args; done++) {
Pavel Begunkov0b8c0e72021-04-26 15:17:38 +01009329 struct io_mapped_ubuf *imu;
9330 int offset = up->offset + done;
Pavel Begunkov634d00d2021-04-25 14:32:26 +01009331 u64 tag = 0;
9332
9333 err = io_copy_iov(ctx, &iov, iovs, done);
9334 if (err)
9335 break;
9336 if (tags && copy_from_user(&tag, &tags[done], sizeof(tag))) {
9337 err = -EFAULT;
9338 break;
9339 }
Pavel Begunkov0b8c0e72021-04-26 15:17:38 +01009340 err = io_buffer_validate(&iov);
9341 if (err)
9342 break;
Colin Ian Kingcf3770e2021-04-29 11:46:02 +01009343 if (!iov.iov_base && tag) {
9344 err = -EINVAL;
9345 break;
9346 }
Pavel Begunkov0b8c0e72021-04-26 15:17:38 +01009347 err = io_sqe_buffer_register(ctx, &iov, &imu, &last_hpage);
9348 if (err)
9349 break;
Pavel Begunkov634d00d2021-04-25 14:32:26 +01009350
Pavel Begunkov0b8c0e72021-04-26 15:17:38 +01009351 i = array_index_nospec(offset, ctx->nr_user_bufs);
Pavel Begunkov62248432021-04-28 13:11:29 +01009352 if (ctx->user_bufs[i] != ctx->dummy_ubuf) {
Pavel Begunkovb8ed0f72022-04-07 14:05:05 +01009353 err = io_queue_rsrc_removal(ctx->buf_data, i,
Pavel Begunkov0b8c0e72021-04-26 15:17:38 +01009354 ctx->rsrc_node, ctx->user_bufs[i]);
9355 if (unlikely(err)) {
9356 io_buffer_unmap(ctx, &imu);
Pavel Begunkov634d00d2021-04-25 14:32:26 +01009357 break;
Pavel Begunkov0b8c0e72021-04-26 15:17:38 +01009358 }
Pavel Begunkov634d00d2021-04-25 14:32:26 +01009359 ctx->user_bufs[i] = NULL;
9360 needs_switch = true;
9361 }
9362
Pavel Begunkov0b8c0e72021-04-26 15:17:38 +01009363 ctx->user_bufs[i] = imu;
Pavel Begunkov2d091d62021-06-14 02:36:21 +01009364 *io_get_tag_slot(ctx->buf_data, offset) = tag;
Pavel Begunkov634d00d2021-04-25 14:32:26 +01009365 }
9366
9367 if (needs_switch)
9368 io_rsrc_node_switch(ctx, ctx->buf_data);
9369 return done ? done : err;
9370}
9371
Jens Axboe9b402842019-04-11 11:45:41 -06009372static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg)
9373{
9374 __s32 __user *fds = arg;
9375 int fd;
9376
9377 if (ctx->cq_ev_fd)
9378 return -EBUSY;
9379
9380 if (copy_from_user(&fd, fds, sizeof(*fds)))
9381 return -EFAULT;
9382
9383 ctx->cq_ev_fd = eventfd_ctx_fdget(fd);
9384 if (IS_ERR(ctx->cq_ev_fd)) {
9385 int ret = PTR_ERR(ctx->cq_ev_fd);
Pavel Begunkovfe7e3252021-06-24 15:09:57 +01009386
Jens Axboe9b402842019-04-11 11:45:41 -06009387 ctx->cq_ev_fd = NULL;
9388 return ret;
9389 }
9390
9391 return 0;
9392}
9393
9394static int io_eventfd_unregister(struct io_ring_ctx *ctx)
9395{
9396 if (ctx->cq_ev_fd) {
9397 eventfd_ctx_put(ctx->cq_ev_fd);
9398 ctx->cq_ev_fd = NULL;
9399 return 0;
9400 }
9401
9402 return -ENXIO;
9403}
9404
Jens Axboe5a2e7452020-02-23 16:23:11 -07009405static void io_destroy_buffers(struct io_ring_ctx *ctx)
9406{
Jens Axboe9e15c3a2021-03-13 12:29:43 -07009407 struct io_buffer *buf;
9408 unsigned long index;
9409
Ye Bin2d447d32021-11-22 10:47:37 +08009410 xa_for_each(&ctx->io_buffers, index, buf)
Jens Axboe9e15c3a2021-03-13 12:29:43 -07009411 __io_remove_buffers(ctx, buf, index, -1U);
Jens Axboe5a2e7452020-02-23 16:23:11 -07009412}
9413
Pavel Begunkov72558342021-08-09 20:18:09 +01009414static void io_req_cache_free(struct list_head *list)
Jens Axboe1b4c3512021-02-10 00:03:19 +00009415{
Jens Axboe68e68ee2021-02-13 09:00:02 -07009416 struct io_kiocb *req, *nxt;
Jens Axboe1b4c3512021-02-10 00:03:19 +00009417
Pavel Begunkovbb943b82021-08-09 20:18:10 +01009418 list_for_each_entry_safe(req, nxt, list, inflight_entry) {
9419 list_del(&req->inflight_entry);
Jens Axboe1b4c3512021-02-10 00:03:19 +00009420 kmem_cache_free(req_cachep, req);
9421 }
9422}
9423
Jens Axboe4010fec2021-02-27 15:04:18 -07009424static void io_req_caches_free(struct io_ring_ctx *ctx)
Jens Axboe2b188cc2019-01-07 10:46:33 -07009425{
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01009426 struct io_submit_state *state = &ctx->submit_state;
Pavel Begunkovbf019da2021-02-10 00:03:17 +00009427
Jens Axboe9a4fdbd2021-02-13 09:09:44 -07009428 mutex_lock(&ctx->uring_lock);
9429
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01009430 if (state->free_reqs) {
9431 kmem_cache_free_bulk(req_cachep, state->free_reqs, state->reqs);
9432 state->free_reqs = 0;
Pavel Begunkov8e5c66c2021-02-22 11:45:55 +00009433 }
Jens Axboe9a4fdbd2021-02-13 09:09:44 -07009434
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01009435 io_flush_cached_locked_reqs(ctx, state);
9436 io_req_cache_free(&state->free_list);
Jens Axboe9a4fdbd2021-02-13 09:09:44 -07009437 mutex_unlock(&ctx->uring_lock);
9438}
9439
Pavel Begunkov43597aa2021-08-10 02:44:23 +01009440static void io_wait_rsrc_data(struct io_rsrc_data *data)
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01009441{
Pavel Begunkov43597aa2021-08-10 02:44:23 +01009442 if (data && !atomic_dec_and_test(&data->refs))
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01009443 wait_for_completion(&data->done);
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01009444}
9445
Jens Axboe2b188cc2019-01-07 10:46:33 -07009446static void io_ring_ctx_free(struct io_ring_ctx *ctx)
9447{
Jens Axboe37d1e2e2021-02-17 21:03:43 -07009448 io_sq_thread_finish(ctx);
Jens Axboe2aede0e2020-09-14 10:45:53 -06009449
Pavel Begunkov43597aa2021-08-10 02:44:23 +01009450 /* __io_rsrc_put_work() may need uring_lock to progress, wait w/o it */
9451 io_wait_rsrc_data(ctx->buf_data);
9452 io_wait_rsrc_data(ctx->file_data);
9453
Hao Xu8bad28d2021-02-19 17:19:36 +08009454 mutex_lock(&ctx->uring_lock);
Pavel Begunkov43597aa2021-08-10 02:44:23 +01009455 if (ctx->buf_data)
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01009456 __io_sqe_buffers_unregister(ctx);
Pavel Begunkov43597aa2021-08-10 02:44:23 +01009457 if (ctx->file_data)
Pavel Begunkov08480402021-04-13 02:58:38 +01009458 __io_sqe_files_unregister(ctx);
Pavel Begunkovc4ea0602021-04-01 15:43:58 +01009459 if (ctx->rings)
9460 __io_cqring_overflow_flush(ctx, true);
Hao Xu8bad28d2021-02-19 17:19:36 +08009461 mutex_unlock(&ctx->uring_lock);
Jens Axboe9b402842019-04-11 11:45:41 -06009462 io_eventfd_unregister(ctx);
Jens Axboe5a2e7452020-02-23 16:23:11 -07009463 io_destroy_buffers(ctx);
Pavel Begunkov07db2982021-04-20 12:03:32 +01009464 if (ctx->sq_creds)
9465 put_cred(ctx->sq_creds);
Jens Axboedef596e2019-01-09 08:59:42 -07009466
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01009467 /* there are no registered resources left, nobody uses it */
9468 if (ctx->rsrc_node)
9469 io_rsrc_node_destroy(ctx->rsrc_node);
Pavel Begunkov8dd03af2021-03-19 17:22:36 +00009470 if (ctx->rsrc_backup_node)
Pavel Begunkovb895c9a2021-04-01 15:43:40 +01009471 io_rsrc_node_destroy(ctx->rsrc_backup_node);
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01009472 flush_delayed_work(&ctx->rsrc_put_work);
9473
9474 WARN_ON_ONCE(!list_empty(&ctx->rsrc_ref_list));
9475 WARN_ON_ONCE(!llist_empty(&ctx->rsrc_put_llist));
Jens Axboe2b188cc2019-01-07 10:46:33 -07009476
9477#if defined(CONFIG_UNIX)
Eric Biggers355e8d22019-06-12 14:58:43 -07009478 if (ctx->ring_sock) {
9479 ctx->ring_sock->file = NULL; /* so that iput() is called */
Jens Axboe2b188cc2019-01-07 10:46:33 -07009480 sock_release(ctx->ring_sock);
Eric Biggers355e8d22019-06-12 14:58:43 -07009481 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07009482#endif
Pavel Begunkovef9dd632021-08-28 19:54:38 -06009483 WARN_ON_ONCE(!list_empty(&ctx->ltimeout_list));
Jens Axboe2b188cc2019-01-07 10:46:33 -07009484
Pavel Begunkovcd148d42022-10-16 22:42:55 +01009485 if (ctx->mm_account) {
9486 mmdrop(ctx->mm_account);
9487 ctx->mm_account = NULL;
9488 }
9489
Hristo Venev75b28af2019-08-26 17:23:46 +00009490 io_mem_free(ctx->rings);
Jens Axboe2b188cc2019-01-07 10:46:33 -07009491 io_mem_free(ctx->sq_sqes);
Jens Axboe2b188cc2019-01-07 10:46:33 -07009492
9493 percpu_ref_exit(&ctx->refs);
Jens Axboe2b188cc2019-01-07 10:46:33 -07009494 free_uid(ctx->user);
Jens Axboe4010fec2021-02-27 15:04:18 -07009495 io_req_caches_free(ctx);
Jens Axboee9418942021-02-19 12:33:30 -07009496 if (ctx->hash_map)
9497 io_wq_put_hash(ctx->hash_map);
Jens Axboe78076bb2019-12-04 19:56:40 -07009498 kfree(ctx->cancel_hash);
Pavel Begunkov62248432021-04-28 13:11:29 +01009499 kfree(ctx->dummy_ubuf);
Jens Axboe2b188cc2019-01-07 10:46:33 -07009500 kfree(ctx);
9501}
9502
9503static __poll_t io_uring_poll(struct file *file, poll_table *wait)
9504{
9505 struct io_ring_ctx *ctx = file->private_data;
9506 __poll_t mask = 0;
9507
Pavel Begunkov311997b2021-06-14 23:37:28 +01009508 poll_wait(file, &ctx->poll_wait, wait);
Stefan Bühler4f7067c2019-04-24 23:54:17 +02009509 /*
9510 * synchronizes with barrier from wq_has_sleeper call in
9511 * io_commit_cqring
9512 */
Jens Axboe2b188cc2019-01-07 10:46:33 -07009513 smp_rmb();
Jens Axboe90554202020-09-03 12:12:41 -06009514 if (!io_sqring_full(ctx))
Jens Axboe2b188cc2019-01-07 10:46:33 -07009515 mask |= EPOLLOUT | EPOLLWRNORM;
Hao Xued670c32021-02-05 16:34:21 +08009516
9517 /*
9518 * Don't flush cqring overflow list here, just do a simple check.
9519 * Otherwise there could possible be ABBA deadlock:
9520 * CPU0 CPU1
9521 * ---- ----
9522 * lock(&ctx->uring_lock);
9523 * lock(&ep->mtx);
9524 * lock(&ctx->uring_lock);
9525 * lock(&ep->mtx);
9526 *
9527 * Users may get EPOLLIN meanwhile seeing nothing in cqring, this
9528 * pushs them to do the flush.
9529 */
Pavel Begunkov5ed7a372021-06-14 23:37:27 +01009530 if (io_cqring_events(ctx) || test_bit(0, &ctx->check_cq_overflow))
Jens Axboe2b188cc2019-01-07 10:46:33 -07009531 mask |= EPOLLIN | EPOLLRDNORM;
9532
9533 return mask;
9534}
9535
Yejune Deng0bead8c2020-12-24 11:02:20 +08009536static int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id)
Jens Axboe071698e2020-01-28 10:04:42 -07009537{
Jens Axboe4379bf82021-02-15 13:40:22 -07009538 const struct cred *creds;
Jens Axboe071698e2020-01-28 10:04:42 -07009539
Matthew Wilcox (Oracle)61cf9372021-03-08 14:16:16 +00009540 creds = xa_erase(&ctx->personalities, id);
Jens Axboe4379bf82021-02-15 13:40:22 -07009541 if (creds) {
9542 put_cred(creds);
Yejune Deng0bead8c2020-12-24 11:02:20 +08009543 return 0;
Jens Axboe1e6fa522020-10-15 08:46:24 -06009544 }
Yejune Deng0bead8c2020-12-24 11:02:20 +08009545
9546 return -EINVAL;
9547}
9548
Pavel Begunkovd56d9382021-03-06 11:02:13 +00009549struct io_tctx_exit {
9550 struct callback_head task_work;
9551 struct completion completion;
Pavel Begunkovbaf186c2021-03-06 11:02:15 +00009552 struct io_ring_ctx *ctx;
Pavel Begunkovd56d9382021-03-06 11:02:13 +00009553};
9554
9555static void io_tctx_exit_cb(struct callback_head *cb)
9556{
9557 struct io_uring_task *tctx = current->io_uring;
9558 struct io_tctx_exit *work;
9559
9560 work = container_of(cb, struct io_tctx_exit, task_work);
9561 /*
9562 * When @in_idle, we're in cancellation and it's racy to remove the
9563 * node. It'll be removed by the end of cancellation, just ignore it.
Harshit Mogalapallif8955112022-12-06 01:38:32 -08009564 * tctx can be NULL if the queueing of this task_work raced with
9565 * work cancelation off the exec path.
Pavel Begunkovd56d9382021-03-06 11:02:13 +00009566 */
Harshit Mogalapallif8955112022-12-06 01:38:32 -08009567 if (tctx && !atomic_read(&tctx->in_idle))
Pavel Begunkoveef51da2021-06-14 02:36:15 +01009568 io_uring_del_tctx_node((unsigned long)work->ctx);
Pavel Begunkovd56d9382021-03-06 11:02:13 +00009569 complete(&work->completion);
9570}
9571
Pavel Begunkov28090c12021-04-25 23:34:45 +01009572static bool io_cancel_ctx_cb(struct io_wq_work *work, void *data)
9573{
9574 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
9575
9576 return req->ctx == data;
9577}
9578
Jens Axboe85faa7b2020-04-09 18:14:00 -06009579static void io_ring_exit_work(struct work_struct *work)
9580{
Pavel Begunkovd56d9382021-03-06 11:02:13 +00009581 struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx, exit_work);
Pavel Begunkovb5bb3a22021-03-06 11:02:16 +00009582 unsigned long timeout = jiffies + HZ * 60 * 5;
Pavel Begunkov58d3be22021-08-09 13:04:17 +01009583 unsigned long interval = HZ / 20;
Pavel Begunkovd56d9382021-03-06 11:02:13 +00009584 struct io_tctx_exit exit;
9585 struct io_tctx_node *node;
9586 int ret;
Jens Axboe85faa7b2020-04-09 18:14:00 -06009587
Jens Axboe56952e92020-06-17 15:00:04 -06009588 /*
9589 * If we're doing polled IO and end up having requests being
9590 * submitted async (out-of-line), then completions can come in while
9591 * we're waiting for refs to drop. We need to reap these manually,
9592 * as nobody else will be looking for them.
9593 */
Pavel Begunkovb2edc0a2020-07-07 16:36:22 +03009594 do {
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01009595 io_uring_try_cancel_requests(ctx, NULL, true);
Pavel Begunkov28090c12021-04-25 23:34:45 +01009596 if (ctx->sq_data) {
9597 struct io_sq_data *sqd = ctx->sq_data;
9598 struct task_struct *tsk;
9599
9600 io_sq_thread_park(sqd);
9601 tsk = sqd->thread;
9602 if (tsk && tsk->io_uring && tsk->io_uring->io_wq)
9603 io_wq_cancel_cb(tsk->io_uring->io_wq,
9604 io_cancel_ctx_cb, ctx, true);
9605 io_sq_thread_unpark(sqd);
9606 }
Pavel Begunkovb5bb3a22021-03-06 11:02:16 +00009607
Pavel Begunkov58d3be22021-08-09 13:04:17 +01009608 if (WARN_ON_ONCE(time_after(jiffies, timeout))) {
9609 /* there is little hope left, don't run it too often */
9610 interval = HZ * 60;
9611 }
9612 } while (!wait_for_completion_timeout(&ctx->ref_comp, interval));
Pavel Begunkovd56d9382021-03-06 11:02:13 +00009613
Pavel Begunkov7f006512021-04-14 13:38:34 +01009614 init_completion(&exit.completion);
9615 init_task_work(&exit.task_work, io_tctx_exit_cb);
9616 exit.ctx = ctx;
Pavel Begunkov89b50662021-04-01 15:43:50 +01009617 /*
9618 * Some may use context even when all refs and requests have been put,
9619 * and they are free to do so while still holding uring_lock or
Pavel Begunkov5b0a6ac2021-06-30 21:54:04 +01009620 * completion_lock, see io_req_task_submit(). Apart from other work,
Pavel Begunkov89b50662021-04-01 15:43:50 +01009621 * this lock/unlock section also waits them to finish.
9622 */
Pavel Begunkovd56d9382021-03-06 11:02:13 +00009623 mutex_lock(&ctx->uring_lock);
9624 while (!list_empty(&ctx->tctx_list)) {
Pavel Begunkovb5bb3a22021-03-06 11:02:16 +00009625 WARN_ON_ONCE(time_after(jiffies, timeout));
9626
Pavel Begunkovd56d9382021-03-06 11:02:13 +00009627 node = list_first_entry(&ctx->tctx_list, struct io_tctx_node,
9628 ctx_node);
Pavel Begunkov7f006512021-04-14 13:38:34 +01009629 /* don't spin on a single task if cancellation failed */
9630 list_rotate_left(&ctx->tctx_list);
Pavel Begunkovd56d9382021-03-06 11:02:13 +00009631 ret = task_work_add(node->task, &exit.task_work, TWA_SIGNAL);
9632 if (WARN_ON_ONCE(ret))
9633 continue;
9634 wake_up_process(node->task);
9635
9636 mutex_unlock(&ctx->uring_lock);
9637 wait_for_completion(&exit.completion);
Pavel Begunkovd56d9382021-03-06 11:02:13 +00009638 mutex_lock(&ctx->uring_lock);
9639 }
9640 mutex_unlock(&ctx->uring_lock);
Jens Axboe79ebeae2021-08-10 15:18:27 -06009641 spin_lock(&ctx->completion_lock);
9642 spin_unlock(&ctx->completion_lock);
Pavel Begunkovd56d9382021-03-06 11:02:13 +00009643
Jens Axboe85faa7b2020-04-09 18:14:00 -06009644 io_ring_ctx_free(ctx);
9645}
9646
Pavel Begunkov80c4cbd2021-03-25 18:32:43 +00009647/* Returns true if we found and killed one or more timeouts */
9648static bool io_kill_timeouts(struct io_ring_ctx *ctx, struct task_struct *tsk,
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01009649 bool cancel_all)
Pavel Begunkov80c4cbd2021-03-25 18:32:43 +00009650{
9651 struct io_kiocb *req, *tmp;
9652 int canceled = 0;
9653
Jens Axboe79ebeae2021-08-10 15:18:27 -06009654 spin_lock(&ctx->completion_lock);
9655 spin_lock_irq(&ctx->timeout_lock);
Pavel Begunkov80c4cbd2021-03-25 18:32:43 +00009656 list_for_each_entry_safe(req, tmp, &ctx->timeout_list, timeout.list) {
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01009657 if (io_match_task(req, tsk, cancel_all)) {
Pavel Begunkov80c4cbd2021-03-25 18:32:43 +00009658 io_kill_timeout(req, -ECANCELED);
9659 canceled++;
9660 }
9661 }
Jens Axboe79ebeae2021-08-10 15:18:27 -06009662 spin_unlock_irq(&ctx->timeout_lock);
Pavel Begunkov51520422021-03-29 11:39:29 +01009663 if (canceled != 0)
9664 io_commit_cqring(ctx);
Jens Axboe79ebeae2021-08-10 15:18:27 -06009665 spin_unlock(&ctx->completion_lock);
Pavel Begunkov80c4cbd2021-03-25 18:32:43 +00009666 if (canceled != 0)
9667 io_cqring_ev_posted(ctx);
9668 return canceled != 0;
9669}
9670
Jens Axboe2b188cc2019-01-07 10:46:33 -07009671static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx)
9672{
Matthew Wilcox (Oracle)61cf9372021-03-08 14:16:16 +00009673 unsigned long index;
9674 struct creds *creds;
9675
Jens Axboe2b188cc2019-01-07 10:46:33 -07009676 mutex_lock(&ctx->uring_lock);
9677 percpu_ref_kill(&ctx->refs);
Pavel Begunkov634578f2020-12-06 22:22:44 +00009678 if (ctx->rings)
Pavel Begunkov6c2450a2021-02-23 12:40:22 +00009679 __io_cqring_overflow_flush(ctx, true);
Matthew Wilcox (Oracle)61cf9372021-03-08 14:16:16 +00009680 xa_for_each(&ctx->personalities, index, creds)
9681 io_unregister_personality(ctx, index);
Jens Axboe2b188cc2019-01-07 10:46:33 -07009682 mutex_unlock(&ctx->uring_lock);
9683
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01009684 io_kill_timeouts(ctx, NULL, true);
9685 io_poll_remove_all(ctx, NULL, true);
Jens Axboe561fb042019-10-24 07:25:42 -06009686
Jens Axboe15dff282019-11-13 09:09:23 -07009687 /* if we failed setting up the ctx, we might not have any rings */
Pavel Begunkovb2edc0a2020-07-07 16:36:22 +03009688 io_iopoll_try_reap_events(ctx);
Jens Axboe309fc032020-07-10 09:13:34 -06009689
Jens Axboe86e2d692023-01-21 12:36:08 -07009690 /* drop cached put refs after potentially doing completions */
9691 if (current->io_uring)
9692 io_uring_drop_tctx_refs(current);
9693
Jens Axboe85faa7b2020-04-09 18:14:00 -06009694 INIT_WORK(&ctx->exit_work, io_ring_exit_work);
Jens Axboefc666772020-08-19 11:10:51 -06009695 /*
9696 * Use system_unbound_wq to avoid spawning tons of event kworkers
9697 * if we're exiting a ton of rings at the same time. It just adds
9698 * noise and overhead, there's no discernable change in runtime
9699 * over using system_wq.
9700 */
9701 queue_work(system_unbound_wq, &ctx->exit_work);
Jens Axboe2b188cc2019-01-07 10:46:33 -07009702}
9703
9704static int io_uring_release(struct inode *inode, struct file *file)
9705{
9706 struct io_ring_ctx *ctx = file->private_data;
9707
9708 file->private_data = NULL;
9709 io_ring_ctx_wait_and_kill(ctx);
9710 return 0;
9711}
9712
Pavel Begunkovf6edbab2020-11-06 13:00:26 +00009713struct io_task_cancel {
9714 struct task_struct *task;
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01009715 bool all;
Pavel Begunkovf6edbab2020-11-06 13:00:26 +00009716};
Pavel Begunkov67c4d9e2020-06-15 10:24:05 +03009717
Pavel Begunkovf6edbab2020-11-06 13:00:26 +00009718static bool io_cancel_task_cb(struct io_wq_work *work, void *data)
Jens Axboeb711d4e2020-08-16 08:23:05 -07009719{
Pavel Begunkov9a472ef2020-11-05 22:31:37 +00009720 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
Pavel Begunkovf6edbab2020-11-06 13:00:26 +00009721 struct io_task_cancel *cancel = data;
Pavel Begunkov9a472ef2020-11-05 22:31:37 +00009722
Pavel Begunkov1c939a52021-11-26 14:38:15 +00009723 return io_match_task_safe(req, cancel->task, cancel->all);
Jens Axboeb711d4e2020-08-16 08:23:05 -07009724}
9725
Pavel Begunkove1915f72021-03-11 23:29:35 +00009726static bool io_cancel_defer_files(struct io_ring_ctx *ctx,
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01009727 struct task_struct *task, bool cancel_all)
Pavel Begunkovb7ddce32020-09-06 00:45:14 +03009728{
Pavel Begunkove1915f72021-03-11 23:29:35 +00009729 struct io_defer_entry *de;
Pavel Begunkovb7ddce32020-09-06 00:45:14 +03009730 LIST_HEAD(list);
9731
Jens Axboe79ebeae2021-08-10 15:18:27 -06009732 spin_lock(&ctx->completion_lock);
Pavel Begunkovb7ddce32020-09-06 00:45:14 +03009733 list_for_each_entry_reverse(de, &ctx->defer_list, list) {
Pavel Begunkov1c939a52021-11-26 14:38:15 +00009734 if (io_match_task_safe(de->req, task, cancel_all)) {
Pavel Begunkovb7ddce32020-09-06 00:45:14 +03009735 list_cut_position(&list, &ctx->defer_list, &de->list);
9736 break;
9737 }
9738 }
Jens Axboe79ebeae2021-08-10 15:18:27 -06009739 spin_unlock(&ctx->completion_lock);
Pavel Begunkove1915f72021-03-11 23:29:35 +00009740 if (list_empty(&list))
9741 return false;
Pavel Begunkovb7ddce32020-09-06 00:45:14 +03009742
9743 while (!list_empty(&list)) {
9744 de = list_first_entry(&list, struct io_defer_entry, list);
9745 list_del_init(&de->list);
Pavel Begunkovf41db2732021-02-28 22:35:12 +00009746 io_req_complete_failed(de->req, -ECANCELED);
Pavel Begunkovb7ddce32020-09-06 00:45:14 +03009747 kfree(de);
9748 }
Pavel Begunkove1915f72021-03-11 23:29:35 +00009749 return true;
Pavel Begunkovb7ddce32020-09-06 00:45:14 +03009750}
9751
Pavel Begunkov1b007642021-03-06 11:02:17 +00009752static bool io_uring_try_cancel_iowq(struct io_ring_ctx *ctx)
9753{
9754 struct io_tctx_node *node;
9755 enum io_wq_cancel cret;
9756 bool ret = false;
9757
9758 mutex_lock(&ctx->uring_lock);
9759 list_for_each_entry(node, &ctx->tctx_list, ctx_node) {
9760 struct io_uring_task *tctx = node->task->io_uring;
9761
9762 /*
9763 * io_wq will stay alive while we hold uring_lock, because it's
9764 * killed after ctx nodes, which requires to take the lock.
9765 */
9766 if (!tctx || !tctx->io_wq)
9767 continue;
9768 cret = io_wq_cancel_cb(tctx->io_wq, io_cancel_ctx_cb, ctx, true);
9769 ret |= (cret != IO_WQ_CANCEL_NOTFOUND);
9770 }
9771 mutex_unlock(&ctx->uring_lock);
9772
9773 return ret;
9774}
9775
Pavel Begunkov9936c7c2021-02-04 13:51:56 +00009776static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
9777 struct task_struct *task,
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01009778 bool cancel_all)
Pavel Begunkov9936c7c2021-02-04 13:51:56 +00009779{
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01009780 struct io_task_cancel cancel = { .task = task, .all = cancel_all, };
Pavel Begunkov1b007642021-03-06 11:02:17 +00009781 struct io_uring_task *tctx = task ? task->io_uring : NULL;
Pavel Begunkov9936c7c2021-02-04 13:51:56 +00009782
9783 while (1) {
9784 enum io_wq_cancel cret;
9785 bool ret = false;
9786
Pavel Begunkov1b007642021-03-06 11:02:17 +00009787 if (!task) {
9788 ret |= io_uring_try_cancel_iowq(ctx);
9789 } else if (tctx && tctx->io_wq) {
9790 /*
9791 * Cancels requests of all rings, not only @ctx, but
9792 * it's fine as the task is in exit/exec.
9793 */
Jens Axboe5aa75ed2021-02-16 12:56:50 -07009794 cret = io_wq_cancel_cb(tctx->io_wq, io_cancel_task_cb,
Pavel Begunkov9936c7c2021-02-04 13:51:56 +00009795 &cancel, true);
9796 ret |= (cret != IO_WQ_CANCEL_NOTFOUND);
9797 }
9798
9799 /* SQPOLL thread does its own polling */
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01009800 if ((!(ctx->flags & IORING_SETUP_SQPOLL) && cancel_all) ||
Jens Axboed052d1d2021-03-11 10:49:20 -07009801 (ctx->sq_data && ctx->sq_data->thread == current)) {
Pavel Begunkov9936c7c2021-02-04 13:51:56 +00009802 while (!list_empty_careful(&ctx->iopoll_list)) {
9803 io_iopoll_try_reap_events(ctx);
9804 ret = true;
9805 }
9806 }
9807
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01009808 ret |= io_cancel_defer_files(ctx, task, cancel_all);
9809 ret |= io_poll_remove_all(ctx, task, cancel_all);
9810 ret |= io_kill_timeouts(ctx, task, cancel_all);
Pavel Begunkove5dc4802021-06-26 21:40:46 +01009811 if (task)
9812 ret |= io_run_task_work();
Pavel Begunkov9936c7c2021-02-04 13:51:56 +00009813 if (!ret)
9814 break;
9815 cond_resched();
9816 }
9817}
9818
Pavel Begunkoveef51da2021-06-14 02:36:15 +01009819static int __io_uring_add_tctx_node(struct io_ring_ctx *ctx)
Jens Axboe0f212202020-09-13 13:09:39 -06009820{
Matthew Wilcox (Oracle)236434c2020-10-09 13:49:52 +01009821 struct io_uring_task *tctx = current->io_uring;
Pavel Begunkov13bf43f2021-03-06 11:02:12 +00009822 struct io_tctx_node *node;
Pavel Begunkova528b042020-12-21 18:34:04 +00009823 int ret;
Matthew Wilcox (Oracle)236434c2020-10-09 13:49:52 +01009824
9825 if (unlikely(!tctx)) {
Jens Axboe5aa75ed2021-02-16 12:56:50 -07009826 ret = io_uring_alloc_task_context(current, ctx);
Jens Axboe0f212202020-09-13 13:09:39 -06009827 if (unlikely(ret))
9828 return ret;
Pavel Begunkove139a1e2021-10-19 23:43:46 +01009829
Matthew Wilcox (Oracle)236434c2020-10-09 13:49:52 +01009830 tctx = current->io_uring;
Pavel Begunkove139a1e2021-10-19 23:43:46 +01009831 if (ctx->iowq_limits_set) {
9832 unsigned int limits[2] = { ctx->iowq_limits[0],
9833 ctx->iowq_limits[1], };
9834
9835 ret = io_wq_max_workers(tctx->io_wq, limits);
9836 if (ret)
9837 return ret;
9838 }
Jens Axboe0f212202020-09-13 13:09:39 -06009839 }
Pavel Begunkovcf27f3b2021-03-19 17:22:31 +00009840 if (!xa_load(&tctx->xa, (unsigned long)ctx)) {
9841 node = kmalloc(sizeof(*node), GFP_KERNEL);
9842 if (!node)
9843 return -ENOMEM;
9844 node->ctx = ctx;
9845 node->task = current;
Jens Axboe0f212202020-09-13 13:09:39 -06009846
Pavel Begunkovcf27f3b2021-03-19 17:22:31 +00009847 ret = xa_err(xa_store(&tctx->xa, (unsigned long)ctx,
9848 node, GFP_KERNEL));
9849 if (ret) {
9850 kfree(node);
9851 return ret;
Jens Axboe0f212202020-09-13 13:09:39 -06009852 }
Pavel Begunkovcf27f3b2021-03-19 17:22:31 +00009853
9854 mutex_lock(&ctx->uring_lock);
9855 list_add(&node->ctx_node, &ctx->tctx_list);
9856 mutex_unlock(&ctx->uring_lock);
Jens Axboe0f212202020-09-13 13:09:39 -06009857 }
Pavel Begunkovcf27f3b2021-03-19 17:22:31 +00009858 tctx->last = ctx;
Jens Axboe0f212202020-09-13 13:09:39 -06009859 return 0;
9860}
9861
9862/*
Pavel Begunkovcf27f3b2021-03-19 17:22:31 +00009863 * Note that this task has used io_uring. We use it for cancelation purposes.
9864 */
Pavel Begunkoveef51da2021-06-14 02:36:15 +01009865static inline int io_uring_add_tctx_node(struct io_ring_ctx *ctx)
Pavel Begunkovcf27f3b2021-03-19 17:22:31 +00009866{
9867 struct io_uring_task *tctx = current->io_uring;
9868
9869 if (likely(tctx && tctx->last == ctx))
9870 return 0;
Pavel Begunkoveef51da2021-06-14 02:36:15 +01009871 return __io_uring_add_tctx_node(ctx);
Pavel Begunkovcf27f3b2021-03-19 17:22:31 +00009872}
9873
9874/*
Jens Axboe0f212202020-09-13 13:09:39 -06009875 * Remove this io_uring_file -> task mapping.
9876 */
Pavel Begunkoveef51da2021-06-14 02:36:15 +01009877static void io_uring_del_tctx_node(unsigned long index)
Jens Axboe0f212202020-09-13 13:09:39 -06009878{
9879 struct io_uring_task *tctx = current->io_uring;
Pavel Begunkov13bf43f2021-03-06 11:02:12 +00009880 struct io_tctx_node *node;
Pavel Begunkov29412672021-03-06 11:02:11 +00009881
Pavel Begunkoveebd2e32021-03-06 11:02:14 +00009882 if (!tctx)
9883 return;
Pavel Begunkov13bf43f2021-03-06 11:02:12 +00009884 node = xa_erase(&tctx->xa, index);
9885 if (!node)
Pavel Begunkov29412672021-03-06 11:02:11 +00009886 return;
Jens Axboe0f212202020-09-13 13:09:39 -06009887
Pavel Begunkov13bf43f2021-03-06 11:02:12 +00009888 WARN_ON_ONCE(current != node->task);
9889 WARN_ON_ONCE(list_empty(&node->ctx_node));
9890
9891 mutex_lock(&node->ctx->uring_lock);
9892 list_del(&node->ctx_node);
9893 mutex_unlock(&node->ctx->uring_lock);
9894
Pavel Begunkovbaf186c2021-03-06 11:02:15 +00009895 if (tctx->last == node->ctx)
Jens Axboe0f212202020-09-13 13:09:39 -06009896 tctx->last = NULL;
Pavel Begunkov13bf43f2021-03-06 11:02:12 +00009897 kfree(node);
Jens Axboe0f212202020-09-13 13:09:39 -06009898}
9899
Pavel Begunkov8452d4a2021-02-27 11:16:46 +00009900static void io_uring_clean_tctx(struct io_uring_task *tctx)
Pavel Begunkovde7f1d92021-01-04 20:43:29 +00009901{
Pavel Begunkovba5ef6d2021-05-20 13:21:20 +01009902 struct io_wq *wq = tctx->io_wq;
Pavel Begunkov13bf43f2021-03-06 11:02:12 +00009903 struct io_tctx_node *node;
Pavel Begunkovde7f1d92021-01-04 20:43:29 +00009904 unsigned long index;
9905
Jens Axboe8bab4c02021-09-24 07:12:27 -06009906 xa_for_each(&tctx->xa, index, node) {
Pavel Begunkoveef51da2021-06-14 02:36:15 +01009907 io_uring_del_tctx_node(index);
Jens Axboe8bab4c02021-09-24 07:12:27 -06009908 cond_resched();
9909 }
Marco Elverb16ef422021-05-27 11:25:48 +02009910 if (wq) {
9911 /*
9912 * Must be after io_uring_del_task_file() (removes nodes under
9913 * uring_lock) to avoid race with io_uring_try_cancel_iowq().
9914 */
Pavel Begunkovba5ef6d2021-05-20 13:21:20 +01009915 io_wq_put_and_exit(wq);
Pavel Begunkovdadebc32021-08-23 13:30:44 +01009916 tctx->io_wq = NULL;
Marco Elverb16ef422021-05-27 11:25:48 +02009917 }
Pavel Begunkovde7f1d92021-01-04 20:43:29 +00009918}
9919
Pavel Begunkov3f48cf12021-04-11 01:46:27 +01009920static s64 tctx_inflight(struct io_uring_task *tctx, bool tracked)
Pavel Begunkov521d6a72021-03-11 23:29:38 +00009921{
Pavel Begunkov3f48cf12021-04-11 01:46:27 +01009922 if (tracked)
9923 return atomic_read(&tctx->inflight_tracked);
Pavel Begunkov521d6a72021-03-11 23:29:38 +00009924 return percpu_counter_sum(&tctx->inflight);
9925}
9926
Pavel Begunkov78cc6872021-06-14 02:36:23 +01009927/*
9928 * Find any io_uring ctx that this task has registered or done IO on, and cancel
Jens Axboe8e129762021-12-09 08:54:29 -07009929 * requests. @sqd should be not-null IFF it's an SQPOLL thread cancellation.
Pavel Begunkov78cc6872021-06-14 02:36:23 +01009930 */
9931static void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd)
Pavel Begunkov0e9ddb32021-02-07 22:34:26 +00009932{
Pavel Begunkov521d6a72021-03-11 23:29:38 +00009933 struct io_uring_task *tctx = current->io_uring;
Pavel Begunkov734551d2021-04-18 14:52:09 +01009934 struct io_ring_ctx *ctx;
Jens Axboefdaf0832020-10-30 09:37:30 -06009935 s64 inflight;
Pavel Begunkov0e9ddb32021-02-07 22:34:26 +00009936 DEFINE_WAIT(wait);
Jens Axboefdaf0832020-10-30 09:37:30 -06009937
Pavel Begunkov78cc6872021-06-14 02:36:23 +01009938 WARN_ON_ONCE(sqd && sqd->thread != current);
9939
Palash Oswal6d042ff2021-04-27 18:21:49 +05309940 if (!current->io_uring)
9941 return;
Pavel Begunkov17a91052021-05-23 15:48:39 +01009942 if (tctx->io_wq)
9943 io_wq_exit_start(tctx->io_wq);
9944
Jens Axboefdaf0832020-10-30 09:37:30 -06009945 atomic_inc(&tctx->in_idle);
Jens Axboed8a6df12020-10-15 16:24:45 -06009946 do {
Pavel Begunkove9dbe222021-08-09 13:04:20 +01009947 io_uring_drop_tctx_refs(current);
Jens Axboe0f212202020-09-13 13:09:39 -06009948 /* read completions before cancelations */
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01009949 inflight = tctx_inflight(tctx, !cancel_all);
Jens Axboed8a6df12020-10-15 16:24:45 -06009950 if (!inflight)
9951 break;
Jens Axboe0f212202020-09-13 13:09:39 -06009952
Pavel Begunkov78cc6872021-06-14 02:36:23 +01009953 if (!sqd) {
9954 struct io_tctx_node *node;
9955 unsigned long index;
9956
9957 xa_for_each(&tctx->xa, index, node) {
9958 /* sqpoll task will cancel all its requests */
9959 if (node->ctx->sq_data)
9960 continue;
9961 io_uring_try_cancel_requests(node->ctx, current,
9962 cancel_all);
9963 }
9964 } else {
9965 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list)
9966 io_uring_try_cancel_requests(ctx, current,
9967 cancel_all);
9968 }
9969
Jens Axboe8e129762021-12-09 08:54:29 -07009970 prepare_to_wait(&tctx->wait, &wait, TASK_INTERRUPTIBLE);
9971 io_run_task_work();
Pavel Begunkove9dbe222021-08-09 13:04:20 +01009972 io_uring_drop_tctx_refs(current);
Jens Axboe8e129762021-12-09 08:54:29 -07009973
Jens Axboe0f212202020-09-13 13:09:39 -06009974 /*
Pavel Begunkova1bb3cd2021-01-26 15:28:26 +00009975 * If we've seen completions, retry without waiting. This
9976 * avoids a race where a completion comes in before we did
9977 * prepare_to_wait().
Jens Axboe0f212202020-09-13 13:09:39 -06009978 */
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01009979 if (inflight == tctx_inflight(tctx, !cancel_all))
Pavel Begunkova1bb3cd2021-01-26 15:28:26 +00009980 schedule();
Pavel Begunkovf57555e2020-12-20 13:21:44 +00009981 finish_wait(&tctx->wait, &wait);
Jens Axboed8a6df12020-10-15 16:24:45 -06009982 } while (1);
Pavel Begunkovde7f1d92021-01-04 20:43:29 +00009983
Pavel Begunkov8452d4a2021-02-27 11:16:46 +00009984 io_uring_clean_tctx(tctx);
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01009985 if (cancel_all) {
Pavel Begunkovb168b1a2022-01-09 00:53:22 +00009986 /*
9987 * We shouldn't run task_works after cancel, so just leave
9988 * ->in_idle set for normal exit.
9989 */
9990 atomic_dec(&tctx->in_idle);
Pavel Begunkov3f48cf12021-04-11 01:46:27 +01009991 /* for exec all current's requests should be gone, kill tctx */
9992 __io_uring_free(current);
9993 }
Pavel Begunkov44e728b2020-06-15 10:24:04 +03009994}
9995
Hao Xuf552a272021-08-12 12:14:35 +08009996void __io_uring_cancel(bool cancel_all)
Pavel Begunkov78cc6872021-06-14 02:36:23 +01009997{
Hao Xuf552a272021-08-12 12:14:35 +08009998 io_uring_cancel_generic(cancel_all, NULL);
Pavel Begunkov78cc6872021-06-14 02:36:23 +01009999}
10000
Roman Penyaev6c5c2402019-11-28 12:53:22 +010010001static void *io_uring_validate_mmap_request(struct file *file,
10002 loff_t pgoff, size_t sz)
Jens Axboe2b188cc2019-01-07 10:46:33 -070010003{
Jens Axboe2b188cc2019-01-07 10:46:33 -070010004 struct io_ring_ctx *ctx = file->private_data;
Roman Penyaev6c5c2402019-11-28 12:53:22 +010010005 loff_t offset = pgoff << PAGE_SHIFT;
Jens Axboe2b188cc2019-01-07 10:46:33 -070010006 struct page *page;
10007 void *ptr;
10008
10009 switch (offset) {
10010 case IORING_OFF_SQ_RING:
Hristo Venev75b28af2019-08-26 17:23:46 +000010011 case IORING_OFF_CQ_RING:
10012 ptr = ctx->rings;
Jens Axboe2b188cc2019-01-07 10:46:33 -070010013 break;
10014 case IORING_OFF_SQES:
10015 ptr = ctx->sq_sqes;
10016 break;
Jens Axboe2b188cc2019-01-07 10:46:33 -070010017 default:
Roman Penyaev6c5c2402019-11-28 12:53:22 +010010018 return ERR_PTR(-EINVAL);
Jens Axboe2b188cc2019-01-07 10:46:33 -070010019 }
10020
10021 page = virt_to_head_page(ptr);
Matthew Wilcox (Oracle)a50b8542019-09-23 15:34:25 -070010022 if (sz > page_size(page))
Roman Penyaev6c5c2402019-11-28 12:53:22 +010010023 return ERR_PTR(-EINVAL);
10024
10025 return ptr;
10026}
10027
10028#ifdef CONFIG_MMU
10029
10030static int io_uring_mmap(struct file *file, struct vm_area_struct *vma)
10031{
10032 size_t sz = vma->vm_end - vma->vm_start;
10033 unsigned long pfn;
10034 void *ptr;
10035
10036 ptr = io_uring_validate_mmap_request(file, vma->vm_pgoff, sz);
10037 if (IS_ERR(ptr))
10038 return PTR_ERR(ptr);
Jens Axboe2b188cc2019-01-07 10:46:33 -070010039
10040 pfn = virt_to_phys(ptr) >> PAGE_SHIFT;
10041 return remap_pfn_range(vma, vma->vm_start, pfn, sz, vma->vm_page_prot);
10042}
10043
Roman Penyaev6c5c2402019-11-28 12:53:22 +010010044#else /* !CONFIG_MMU */
10045
10046static int io_uring_mmap(struct file *file, struct vm_area_struct *vma)
10047{
10048 return vma->vm_flags & (VM_SHARED | VM_MAYSHARE) ? 0 : -EINVAL;
10049}
10050
10051static unsigned int io_uring_nommu_mmap_capabilities(struct file *file)
10052{
10053 return NOMMU_MAP_DIRECT | NOMMU_MAP_READ | NOMMU_MAP_WRITE;
10054}
10055
10056static unsigned long io_uring_nommu_get_unmapped_area(struct file *file,
10057 unsigned long addr, unsigned long len,
10058 unsigned long pgoff, unsigned long flags)
10059{
10060 void *ptr;
10061
10062 ptr = io_uring_validate_mmap_request(file, pgoff, len);
10063 if (IS_ERR(ptr))
10064 return PTR_ERR(ptr);
10065
10066 return (unsigned long) ptr;
10067}
10068
10069#endif /* !CONFIG_MMU */
10070
Pavel Begunkovd9d05212021-01-08 20:57:25 +000010071static int io_sqpoll_wait_sq(struct io_ring_ctx *ctx)
Jens Axboe90554202020-09-03 12:12:41 -060010072{
10073 DEFINE_WAIT(wait);
10074
10075 do {
10076 if (!io_sqring_full(ctx))
10077 break;
Jens Axboe90554202020-09-03 12:12:41 -060010078 prepare_to_wait(&ctx->sqo_sq_wait, &wait, TASK_INTERRUPTIBLE);
10079
10080 if (!io_sqring_full(ctx))
10081 break;
Jens Axboe90554202020-09-03 12:12:41 -060010082 schedule();
10083 } while (!signal_pending(current));
10084
10085 finish_wait(&ctx->sqo_sq_wait, &wait);
Yang Li51993282021-03-09 14:30:41 +080010086 return 0;
Jens Axboe90554202020-09-03 12:12:41 -060010087}
10088
Hao Xuc73ebb62020-11-03 10:54:37 +080010089static int io_get_ext_arg(unsigned flags, const void __user *argp, size_t *argsz,
10090 struct __kernel_timespec __user **ts,
10091 const sigset_t __user **sig)
10092{
10093 struct io_uring_getevents_arg arg;
10094
10095 /*
10096 * If EXT_ARG isn't set, then we have no timespec and the argp pointer
10097 * is just a pointer to the sigset_t.
10098 */
10099 if (!(flags & IORING_ENTER_EXT_ARG)) {
10100 *sig = (const sigset_t __user *) argp;
10101 *ts = NULL;
10102 return 0;
10103 }
10104
10105 /*
10106 * EXT_ARG is set - ensure we agree on the size of it and copy in our
10107 * timespec and sigset_t pointers if good.
10108 */
10109 if (*argsz != sizeof(arg))
10110 return -EINVAL;
10111 if (copy_from_user(&arg, argp, sizeof(arg)))
10112 return -EFAULT;
Dylan Yudaken99475482022-04-12 09:30:42 -070010113 if (arg.pad)
10114 return -EINVAL;
Hao Xuc73ebb62020-11-03 10:54:37 +080010115 *sig = u64_to_user_ptr(arg.sigmask);
10116 *argsz = arg.sigmask_sz;
10117 *ts = u64_to_user_ptr(arg.ts);
10118 return 0;
10119}
10120
Jens Axboe2b188cc2019-01-07 10:46:33 -070010121SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
Hao Xuc73ebb62020-11-03 10:54:37 +080010122 u32, min_complete, u32, flags, const void __user *, argp,
10123 size_t, argsz)
Jens Axboe2b188cc2019-01-07 10:46:33 -070010124{
10125 struct io_ring_ctx *ctx;
Jens Axboe2b188cc2019-01-07 10:46:33 -070010126 int submitted = 0;
10127 struct fd f;
Pavel Begunkov33f993d2021-03-19 17:22:30 +000010128 long ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -070010129
Jens Axboe4c6e2772020-07-01 11:29:10 -060010130 io_run_task_work();
Jens Axboeb41e9852020-02-17 09:52:41 -070010131
Pavel Begunkov33f993d2021-03-19 17:22:30 +000010132 if (unlikely(flags & ~(IORING_ENTER_GETEVENTS | IORING_ENTER_SQ_WAKEUP |
10133 IORING_ENTER_SQ_WAIT | IORING_ENTER_EXT_ARG)))
Jens Axboe2b188cc2019-01-07 10:46:33 -070010134 return -EINVAL;
10135
10136 f = fdget(fd);
Pavel Begunkov33f993d2021-03-19 17:22:30 +000010137 if (unlikely(!f.file))
Jens Axboe2b188cc2019-01-07 10:46:33 -070010138 return -EBADF;
10139
10140 ret = -EOPNOTSUPP;
Pavel Begunkov33f993d2021-03-19 17:22:30 +000010141 if (unlikely(f.file->f_op != &io_uring_fops))
Jens Axboe2b188cc2019-01-07 10:46:33 -070010142 goto out_fput;
10143
10144 ret = -ENXIO;
10145 ctx = f.file->private_data;
Pavel Begunkov33f993d2021-03-19 17:22:30 +000010146 if (unlikely(!percpu_ref_tryget(&ctx->refs)))
Jens Axboe2b188cc2019-01-07 10:46:33 -070010147 goto out_fput;
10148
Stefano Garzarella7e84e1c2020-08-27 16:58:31 +020010149 ret = -EBADFD;
Pavel Begunkov33f993d2021-03-19 17:22:30 +000010150 if (unlikely(ctx->flags & IORING_SETUP_R_DISABLED))
Stefano Garzarella7e84e1c2020-08-27 16:58:31 +020010151 goto out;
10152
Jens Axboe6c271ce2019-01-10 11:22:30 -070010153 /*
10154 * For SQ polling, the thread will do all submissions and completions.
10155 * Just return the requested submit count, and wake the thread if
10156 * we were asked to.
10157 */
Jens Axboeb2a9ead2019-09-12 14:19:16 -060010158 ret = 0;
Jens Axboe6c271ce2019-01-10 11:22:30 -070010159 if (ctx->flags & IORING_SETUP_SQPOLL) {
Pavel Begunkov90f67362021-08-09 20:18:12 +010010160 io_cqring_overflow_flush(ctx);
Pavel Begunkov89448c42020-12-17 00:24:39 +000010161
Jens Axboe21f96522021-08-14 09:04:40 -060010162 if (unlikely(ctx->sq_data->thread == NULL)) {
10163 ret = -EOWNERDEAD;
Stefan Metzmacher04147482021-03-07 11:54:29 +010010164 goto out;
Jens Axboe21f96522021-08-14 09:04:40 -060010165 }
Jens Axboe6c271ce2019-01-10 11:22:30 -070010166 if (flags & IORING_ENTER_SQ_WAKEUP)
Jens Axboe534ca6d2020-09-02 13:52:19 -060010167 wake_up(&ctx->sq_data->wait);
Pavel Begunkovd9d05212021-01-08 20:57:25 +000010168 if (flags & IORING_ENTER_SQ_WAIT) {
10169 ret = io_sqpoll_wait_sq(ctx);
10170 if (ret)
10171 goto out;
10172 }
Jens Axboe6c271ce2019-01-10 11:22:30 -070010173 submitted = to_submit;
Jens Axboeb2a9ead2019-09-12 14:19:16 -060010174 } else if (to_submit) {
Pavel Begunkoveef51da2021-06-14 02:36:15 +010010175 ret = io_uring_add_tctx_node(ctx);
Jens Axboe0f212202020-09-13 13:09:39 -060010176 if (unlikely(ret))
10177 goto out;
Jens Axboe2b188cc2019-01-07 10:46:33 -070010178 mutex_lock(&ctx->uring_lock);
Jens Axboe0f212202020-09-13 13:09:39 -060010179 submitted = io_submit_sqes(ctx, to_submit);
Jens Axboe2b188cc2019-01-07 10:46:33 -070010180 mutex_unlock(&ctx->uring_lock);
Pavel Begunkov7c504e652019-12-18 19:53:45 +030010181
10182 if (submitted != to_submit)
10183 goto out;
Jens Axboe2b188cc2019-01-07 10:46:33 -070010184 }
10185 if (flags & IORING_ENTER_GETEVENTS) {
Hao Xuc73ebb62020-11-03 10:54:37 +080010186 const sigset_t __user *sig;
10187 struct __kernel_timespec __user *ts;
10188
10189 ret = io_get_ext_arg(flags, argp, &argsz, &ts, &sig);
10190 if (unlikely(ret))
10191 goto out;
10192
Jens Axboe2b188cc2019-01-07 10:46:33 -070010193 min_complete = min(min_complete, ctx->cq_entries);
10194
Xiaoguang Wang32b22442020-03-11 09:26:09 +080010195 /*
10196 * When SETUP_IOPOLL and SETUP_SQPOLL are both enabled, user
10197 * space applications don't need to do io completion events
10198 * polling again, they can rely on io_sq_thread to do polling
10199 * work, which can reduce cpu usage and uring_lock contention.
10200 */
10201 if (ctx->flags & IORING_SETUP_IOPOLL &&
10202 !(ctx->flags & IORING_SETUP_SQPOLL)) {
Pavel Begunkov7668b922020-07-07 16:36:21 +030010203 ret = io_iopoll_check(ctx, min_complete);
Jens Axboedef596e2019-01-09 08:59:42 -070010204 } else {
Hao Xuc73ebb62020-11-03 10:54:37 +080010205 ret = io_cqring_wait(ctx, min_complete, sig, argsz, ts);
Jens Axboedef596e2019-01-09 08:59:42 -070010206 }
Jens Axboe2b188cc2019-01-07 10:46:33 -070010207 }
10208
Pavel Begunkov7c504e652019-12-18 19:53:45 +030010209out:
Pavel Begunkov6805b322019-10-08 02:18:42 +030010210 percpu_ref_put(&ctx->refs);
Jens Axboe2b188cc2019-01-07 10:46:33 -070010211out_fput:
10212 fdput(f);
10213 return submitted ? submitted : ret;
10214}
10215
Tobias Klauserbebdb652020-02-26 18:38:32 +010010216#ifdef CONFIG_PROC_FS
Matthew Wilcox (Oracle)61cf9372021-03-08 14:16:16 +000010217static int io_uring_show_cred(struct seq_file *m, unsigned int id,
10218 const struct cred *cred)
Jens Axboe87ce9552020-01-30 08:25:34 -070010219{
Jens Axboe87ce9552020-01-30 08:25:34 -070010220 struct user_namespace *uns = seq_user_ns(m);
10221 struct group_info *gi;
10222 kernel_cap_t cap;
10223 unsigned __capi;
10224 int g;
10225
10226 seq_printf(m, "%5d\n", id);
10227 seq_put_decimal_ull(m, "\tUid:\t", from_kuid_munged(uns, cred->uid));
10228 seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->euid));
10229 seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->suid));
10230 seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->fsuid));
10231 seq_put_decimal_ull(m, "\n\tGid:\t", from_kgid_munged(uns, cred->gid));
10232 seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->egid));
10233 seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->sgid));
10234 seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->fsgid));
10235 seq_puts(m, "\n\tGroups:\t");
10236 gi = cred->group_info;
10237 for (g = 0; g < gi->ngroups; g++) {
10238 seq_put_decimal_ull(m, g ? " " : "",
10239 from_kgid_munged(uns, gi->gid[g]));
10240 }
10241 seq_puts(m, "\n\tCapEff:\t");
10242 cap = cred->cap_effective;
10243 CAP_FOR_EACH_U32(__capi)
10244 seq_put_hex_ll(m, NULL, cap.cap[CAP_LAST_U32 - __capi], 8);
10245 seq_putc(m, '\n');
10246 return 0;
10247}
10248
10249static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m)
10250{
Joseph Qidbbe9c62020-09-29 09:01:22 -060010251 struct io_sq_data *sq = NULL;
Jens Axboefad8e0d2020-09-28 08:57:48 -060010252 bool has_lock;
Jens Axboe87ce9552020-01-30 08:25:34 -070010253 int i;
10254
Jens Axboefad8e0d2020-09-28 08:57:48 -060010255 /*
10256 * Avoid ABBA deadlock between the seq lock and the io_uring mutex,
10257 * since fdinfo case grabs it in the opposite direction of normal use
10258 * cases. If we fail to get the lock, we just don't iterate any
10259 * structures that could be going away outside the io_uring mutex.
10260 */
10261 has_lock = mutex_trylock(&ctx->uring_lock);
10262
Jens Axboe5f3f26f2021-02-25 10:17:46 -070010263 if (has_lock && (ctx->flags & IORING_SETUP_SQPOLL)) {
Joseph Qidbbe9c62020-09-29 09:01:22 -060010264 sq = ctx->sq_data;
Jens Axboe5f3f26f2021-02-25 10:17:46 -070010265 if (!sq->thread)
10266 sq = NULL;
10267 }
Joseph Qidbbe9c62020-09-29 09:01:22 -060010268
10269 seq_printf(m, "SqThread:\t%d\n", sq ? task_pid_nr(sq->thread) : -1);
10270 seq_printf(m, "SqThreadCpu:\t%d\n", sq ? task_cpu(sq->thread) : -1);
Jens Axboe87ce9552020-01-30 08:25:34 -070010271 seq_printf(m, "UserFiles:\t%u\n", ctx->nr_user_files);
Jens Axboefad8e0d2020-09-28 08:57:48 -060010272 for (i = 0; has_lock && i < ctx->nr_user_files; i++) {
Jens Axboe7b29f922021-03-12 08:30:14 -070010273 struct file *f = io_file_from_index(ctx, i);
Jens Axboe87ce9552020-01-30 08:25:34 -070010274
Jens Axboe87ce9552020-01-30 08:25:34 -070010275 if (f)
10276 seq_printf(m, "%5u: %s\n", i, file_dentry(f)->d_iname);
10277 else
10278 seq_printf(m, "%5u: <none>\n", i);
10279 }
10280 seq_printf(m, "UserBufs:\t%u\n", ctx->nr_user_bufs);
Jens Axboefad8e0d2020-09-28 08:57:48 -060010281 for (i = 0; has_lock && i < ctx->nr_user_bufs; i++) {
Pavel Begunkov41edf1a2021-04-25 14:32:23 +010010282 struct io_mapped_ubuf *buf = ctx->user_bufs[i];
Pavel Begunkov4751f532021-04-01 15:43:55 +010010283 unsigned int len = buf->ubuf_end - buf->ubuf;
Jens Axboe87ce9552020-01-30 08:25:34 -070010284
Pavel Begunkov4751f532021-04-01 15:43:55 +010010285 seq_printf(m, "%5u: 0x%llx/%u\n", i, buf->ubuf, len);
Jens Axboe87ce9552020-01-30 08:25:34 -070010286 }
Matthew Wilcox (Oracle)61cf9372021-03-08 14:16:16 +000010287 if (has_lock && !xa_empty(&ctx->personalities)) {
10288 unsigned long index;
10289 const struct cred *cred;
10290
Jens Axboe87ce9552020-01-30 08:25:34 -070010291 seq_printf(m, "Personalities:\n");
Matthew Wilcox (Oracle)61cf9372021-03-08 14:16:16 +000010292 xa_for_each(&ctx->personalities, index, cred)
10293 io_uring_show_cred(m, index, cred);
Jens Axboe87ce9552020-01-30 08:25:34 -070010294 }
Jens Axboed7718a92020-02-14 22:23:12 -070010295 seq_printf(m, "PollList:\n");
Jens Axboe79ebeae2021-08-10 15:18:27 -060010296 spin_lock(&ctx->completion_lock);
Jens Axboed7718a92020-02-14 22:23:12 -070010297 for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) {
10298 struct hlist_head *list = &ctx->cancel_hash[i];
10299 struct io_kiocb *req;
10300
10301 hlist_for_each_entry(req, list, hash_node)
10302 seq_printf(m, " op=%d, task_works=%d\n", req->opcode,
10303 req->task->task_works != NULL);
10304 }
Jens Axboe79ebeae2021-08-10 15:18:27 -060010305 spin_unlock(&ctx->completion_lock);
Jens Axboefad8e0d2020-09-28 08:57:48 -060010306 if (has_lock)
10307 mutex_unlock(&ctx->uring_lock);
Jens Axboe87ce9552020-01-30 08:25:34 -070010308}
10309
10310static void io_uring_show_fdinfo(struct seq_file *m, struct file *f)
10311{
10312 struct io_ring_ctx *ctx = f->private_data;
10313
10314 if (percpu_ref_tryget(&ctx->refs)) {
10315 __io_uring_show_fdinfo(ctx, m);
10316 percpu_ref_put(&ctx->refs);
10317 }
10318}
Tobias Klauserbebdb652020-02-26 18:38:32 +010010319#endif
Jens Axboe87ce9552020-01-30 08:25:34 -070010320
Jens Axboe2b188cc2019-01-07 10:46:33 -070010321static const struct file_operations io_uring_fops = {
10322 .release = io_uring_release,
10323 .mmap = io_uring_mmap,
Roman Penyaev6c5c2402019-11-28 12:53:22 +010010324#ifndef CONFIG_MMU
10325 .get_unmapped_area = io_uring_nommu_get_unmapped_area,
10326 .mmap_capabilities = io_uring_nommu_mmap_capabilities,
10327#endif
Jens Axboe2b188cc2019-01-07 10:46:33 -070010328 .poll = io_uring_poll,
Tobias Klauserbebdb652020-02-26 18:38:32 +010010329#ifdef CONFIG_PROC_FS
Jens Axboe87ce9552020-01-30 08:25:34 -070010330 .show_fdinfo = io_uring_show_fdinfo,
Tobias Klauserbebdb652020-02-26 18:38:32 +010010331#endif
Jens Axboe2b188cc2019-01-07 10:46:33 -070010332};
10333
10334static int io_allocate_scq_urings(struct io_ring_ctx *ctx,
10335 struct io_uring_params *p)
10336{
Hristo Venev75b28af2019-08-26 17:23:46 +000010337 struct io_rings *rings;
10338 size_t size, sq_array_offset;
Jens Axboe2b188cc2019-01-07 10:46:33 -070010339
Jens Axboebd740482020-08-05 12:58:23 -060010340 /* make sure these are sane, as we already accounted them */
10341 ctx->sq_entries = p->sq_entries;
10342 ctx->cq_entries = p->cq_entries;
10343
Hristo Venev75b28af2019-08-26 17:23:46 +000010344 size = rings_size(p->sq_entries, p->cq_entries, &sq_array_offset);
10345 if (size == SIZE_MAX)
10346 return -EOVERFLOW;
10347
10348 rings = io_mem_alloc(size);
10349 if (!rings)
Jens Axboe2b188cc2019-01-07 10:46:33 -070010350 return -ENOMEM;
10351
Hristo Venev75b28af2019-08-26 17:23:46 +000010352 ctx->rings = rings;
10353 ctx->sq_array = (u32 *)((char *)rings + sq_array_offset);
10354 rings->sq_ring_mask = p->sq_entries - 1;
10355 rings->cq_ring_mask = p->cq_entries - 1;
10356 rings->sq_ring_entries = p->sq_entries;
10357 rings->cq_ring_entries = p->cq_entries;
Jens Axboe2b188cc2019-01-07 10:46:33 -070010358
10359 size = array_size(sizeof(struct io_uring_sqe), p->sq_entries);
Jens Axboeeb065d32019-11-20 09:26:29 -070010360 if (size == SIZE_MAX) {
10361 io_mem_free(ctx->rings);
10362 ctx->rings = NULL;
Jens Axboe2b188cc2019-01-07 10:46:33 -070010363 return -EOVERFLOW;
Jens Axboeeb065d32019-11-20 09:26:29 -070010364 }
Jens Axboe2b188cc2019-01-07 10:46:33 -070010365
10366 ctx->sq_sqes = io_mem_alloc(size);
Jens Axboeeb065d32019-11-20 09:26:29 -070010367 if (!ctx->sq_sqes) {
10368 io_mem_free(ctx->rings);
10369 ctx->rings = NULL;
Jens Axboe2b188cc2019-01-07 10:46:33 -070010370 return -ENOMEM;
Jens Axboeeb065d32019-11-20 09:26:29 -070010371 }
Jens Axboe2b188cc2019-01-07 10:46:33 -070010372
Jens Axboe2b188cc2019-01-07 10:46:33 -070010373 return 0;
10374}
10375
Pavel Begunkov9faadcc2020-12-21 18:34:05 +000010376static int io_uring_install_fd(struct io_ring_ctx *ctx, struct file *file)
10377{
10378 int ret, fd;
10379
10380 fd = get_unused_fd_flags(O_RDWR | O_CLOEXEC);
10381 if (fd < 0)
10382 return fd;
10383
Pavel Begunkoveef51da2021-06-14 02:36:15 +010010384 ret = io_uring_add_tctx_node(ctx);
Pavel Begunkov9faadcc2020-12-21 18:34:05 +000010385 if (ret) {
10386 put_unused_fd(fd);
10387 return ret;
10388 }
10389 fd_install(fd, file);
10390 return fd;
10391}
10392
Jens Axboe2b188cc2019-01-07 10:46:33 -070010393/*
10394 * Allocate an anonymous fd, this is what constitutes the application
10395 * visible backing of an io_uring instance. The application mmaps this
10396 * fd to gain access to the SQ/CQ ring details. If UNIX sockets are enabled,
10397 * we have to tie this fd to a socket for file garbage collection purposes.
10398 */
Pavel Begunkov9faadcc2020-12-21 18:34:05 +000010399static struct file *io_uring_get_file(struct io_ring_ctx *ctx)
Jens Axboe2b188cc2019-01-07 10:46:33 -070010400{
10401 struct file *file;
Jens Axboe2b188cc2019-01-07 10:46:33 -070010402#if defined(CONFIG_UNIX)
Pavel Begunkov9faadcc2020-12-21 18:34:05 +000010403 int ret;
10404
Jens Axboe2b188cc2019-01-07 10:46:33 -070010405 ret = sock_create_kern(&init_net, PF_UNIX, SOCK_RAW, IPPROTO_IP,
10406 &ctx->ring_sock);
10407 if (ret)
Pavel Begunkov9faadcc2020-12-21 18:34:05 +000010408 return ERR_PTR(ret);
Jens Axboe2b188cc2019-01-07 10:46:33 -070010409#endif
10410
Jens Axboe2b188cc2019-01-07 10:46:33 -070010411 file = anon_inode_getfile("[io_uring]", &io_uring_fops, ctx,
10412 O_RDWR | O_CLOEXEC);
Pavel Begunkov9faadcc2020-12-21 18:34:05 +000010413#if defined(CONFIG_UNIX)
Jens Axboe2b188cc2019-01-07 10:46:33 -070010414 if (IS_ERR(file)) {
Pavel Begunkov9faadcc2020-12-21 18:34:05 +000010415 sock_release(ctx->ring_sock);
10416 ctx->ring_sock = NULL;
10417 } else {
10418 ctx->ring_sock->file = file;
Jens Axboe2b188cc2019-01-07 10:46:33 -070010419 }
Jens Axboe2b188cc2019-01-07 10:46:33 -070010420#endif
Pavel Begunkov9faadcc2020-12-21 18:34:05 +000010421 return file;
Jens Axboe2b188cc2019-01-07 10:46:33 -070010422}
10423
Xiaoguang Wang7f136572020-05-05 16:28:53 +080010424static int io_uring_create(unsigned entries, struct io_uring_params *p,
10425 struct io_uring_params __user *params)
Jens Axboe2b188cc2019-01-07 10:46:33 -070010426{
Jens Axboe2b188cc2019-01-07 10:46:33 -070010427 struct io_ring_ctx *ctx;
Pavel Begunkov9faadcc2020-12-21 18:34:05 +000010428 struct file *file;
Jens Axboe2b188cc2019-01-07 10:46:33 -070010429 int ret;
10430
Jens Axboe8110c1a2019-12-28 15:39:54 -070010431 if (!entries)
Jens Axboe2b188cc2019-01-07 10:46:33 -070010432 return -EINVAL;
Jens Axboe8110c1a2019-12-28 15:39:54 -070010433 if (entries > IORING_MAX_ENTRIES) {
10434 if (!(p->flags & IORING_SETUP_CLAMP))
10435 return -EINVAL;
10436 entries = IORING_MAX_ENTRIES;
10437 }
Jens Axboe2b188cc2019-01-07 10:46:33 -070010438
10439 /*
10440 * Use twice as many entries for the CQ ring. It's possible for the
10441 * application to drive a higher depth than the size of the SQ ring,
10442 * since the sqes are only used at submission time. This allows for
Jens Axboe33a107f2019-10-04 12:10:03 -060010443 * some flexibility in overcommitting a bit. If the application has
10444 * set IORING_SETUP_CQSIZE, it will have passed in the desired number
10445 * of CQ ring entries manually.
Jens Axboe2b188cc2019-01-07 10:46:33 -070010446 */
10447 p->sq_entries = roundup_pow_of_two(entries);
Jens Axboe33a107f2019-10-04 12:10:03 -060010448 if (p->flags & IORING_SETUP_CQSIZE) {
10449 /*
10450 * If IORING_SETUP_CQSIZE is set, we do the same roundup
10451 * to a power-of-two, if it isn't already. We do NOT impose
10452 * any cq vs sq ring sizing.
10453 */
Joseph Qieb2667b32020-11-24 15:03:03 +080010454 if (!p->cq_entries)
Jens Axboe33a107f2019-10-04 12:10:03 -060010455 return -EINVAL;
Jens Axboe8110c1a2019-12-28 15:39:54 -070010456 if (p->cq_entries > IORING_MAX_CQ_ENTRIES) {
10457 if (!(p->flags & IORING_SETUP_CLAMP))
10458 return -EINVAL;
10459 p->cq_entries = IORING_MAX_CQ_ENTRIES;
10460 }
Joseph Qieb2667b32020-11-24 15:03:03 +080010461 p->cq_entries = roundup_pow_of_two(p->cq_entries);
10462 if (p->cq_entries < p->sq_entries)
10463 return -EINVAL;
Jens Axboe33a107f2019-10-04 12:10:03 -060010464 } else {
10465 p->cq_entries = 2 * p->sq_entries;
10466 }
Jens Axboe2b188cc2019-01-07 10:46:33 -070010467
Jens Axboe2b188cc2019-01-07 10:46:33 -070010468 ctx = io_ring_ctx_alloc(p);
Jens Axboe62e398b2021-02-21 16:19:37 -070010469 if (!ctx)
Jens Axboe2b188cc2019-01-07 10:46:33 -070010470 return -ENOMEM;
Jens Axboe2b188cc2019-01-07 10:46:33 -070010471 ctx->compat = in_compat_syscall();
Jens Axboe62e398b2021-02-21 16:19:37 -070010472 if (!capable(CAP_IPC_LOCK))
10473 ctx->user = get_uid(current_user());
Jens Axboe2aede0e2020-09-14 10:45:53 -060010474
10475 /*
10476 * This is just grabbed for accounting purposes. When a process exits,
10477 * the mm is exited and dropped before the files, hence we need to hang
10478 * on to this mm purely for the purposes of being able to unaccount
10479 * memory (locked/pinned vm). It's not used for anything else.
10480 */
Jens Axboe6b7898e2020-08-25 07:58:00 -060010481 mmgrab(current->mm);
Jens Axboe2aede0e2020-09-14 10:45:53 -060010482 ctx->mm_account = current->mm;
Jens Axboe6b7898e2020-08-25 07:58:00 -060010483
Jens Axboe2b188cc2019-01-07 10:46:33 -070010484 ret = io_allocate_scq_urings(ctx, p);
10485 if (ret)
10486 goto err;
10487
Stefano Garzarella7e84e1c2020-08-27 16:58:31 +020010488 ret = io_sq_offload_create(ctx, p);
Jens Axboe2b188cc2019-01-07 10:46:33 -070010489 if (ret)
10490 goto err;
Pavel Begunkoveae071c2021-04-25 14:32:24 +010010491 /* always set a rsrc node */
Pavel Begunkov47b228c2021-04-29 11:46:48 +010010492 ret = io_rsrc_node_switch_start(ctx);
10493 if (ret)
10494 goto err;
Pavel Begunkoveae071c2021-04-25 14:32:24 +010010495 io_rsrc_node_switch(ctx, NULL);
Jens Axboe2b188cc2019-01-07 10:46:33 -070010496
Jens Axboe2b188cc2019-01-07 10:46:33 -070010497 memset(&p->sq_off, 0, sizeof(p->sq_off));
Hristo Venev75b28af2019-08-26 17:23:46 +000010498 p->sq_off.head = offsetof(struct io_rings, sq.head);
10499 p->sq_off.tail = offsetof(struct io_rings, sq.tail);
10500 p->sq_off.ring_mask = offsetof(struct io_rings, sq_ring_mask);
10501 p->sq_off.ring_entries = offsetof(struct io_rings, sq_ring_entries);
10502 p->sq_off.flags = offsetof(struct io_rings, sq_flags);
10503 p->sq_off.dropped = offsetof(struct io_rings, sq_dropped);
10504 p->sq_off.array = (char *)ctx->sq_array - (char *)ctx->rings;
Jens Axboe2b188cc2019-01-07 10:46:33 -070010505
10506 memset(&p->cq_off, 0, sizeof(p->cq_off));
Hristo Venev75b28af2019-08-26 17:23:46 +000010507 p->cq_off.head = offsetof(struct io_rings, cq.head);
10508 p->cq_off.tail = offsetof(struct io_rings, cq.tail);
10509 p->cq_off.ring_mask = offsetof(struct io_rings, cq_ring_mask);
10510 p->cq_off.ring_entries = offsetof(struct io_rings, cq_ring_entries);
10511 p->cq_off.overflow = offsetof(struct io_rings, cq_overflow);
10512 p->cq_off.cqes = offsetof(struct io_rings, cqes);
Stefano Garzarella0d9b5b32020-05-15 18:38:04 +020010513 p->cq_off.flags = offsetof(struct io_rings, cq_flags);
Jens Axboeac90f242019-09-06 10:26:21 -060010514
Xiaoguang Wang7f136572020-05-05 16:28:53 +080010515 p->features = IORING_FEAT_SINGLE_MMAP | IORING_FEAT_NODROP |
10516 IORING_FEAT_SUBMIT_STABLE | IORING_FEAT_RW_CUR_POS |
Jiufei Xue5769a352020-06-17 17:53:55 +080010517 IORING_FEAT_CUR_PERSONALITY | IORING_FEAT_FAST_POLL |
Hao Xuc73ebb62020-11-03 10:54:37 +080010518 IORING_FEAT_POLL_32BITS | IORING_FEAT_SQPOLL_NONFIXED |
Pavel Begunkov96905572021-06-10 16:37:38 +010010519 IORING_FEAT_EXT_ARG | IORING_FEAT_NATIVE_WORKERS |
10520 IORING_FEAT_RSRC_TAGS;
Xiaoguang Wang7f136572020-05-05 16:28:53 +080010521
10522 if (copy_to_user(params, p, sizeof(*p))) {
10523 ret = -EFAULT;
10524 goto err;
10525 }
Jens Axboed1719f72020-07-30 13:43:53 -060010526
Pavel Begunkov9faadcc2020-12-21 18:34:05 +000010527 file = io_uring_get_file(ctx);
10528 if (IS_ERR(file)) {
10529 ret = PTR_ERR(file);
10530 goto err;
10531 }
10532
Jens Axboed1719f72020-07-30 13:43:53 -060010533 /*
Jens Axboe044c1ab2019-10-28 09:15:33 -060010534 * Install ring fd as the very last thing, so we don't risk someone
10535 * having closed it before we finish setup
10536 */
Pavel Begunkov9faadcc2020-12-21 18:34:05 +000010537 ret = io_uring_install_fd(ctx, file);
10538 if (ret < 0) {
10539 /* fput will clean it up */
10540 fput(file);
10541 return ret;
10542 }
Jens Axboe044c1ab2019-10-28 09:15:33 -060010543
Dmitrii Dolgovc826bd72019-10-15 19:02:01 +020010544 trace_io_uring_create(ret, ctx, p->sq_entries, p->cq_entries, p->flags);
Jens Axboe2b188cc2019-01-07 10:46:33 -070010545 return ret;
10546err:
10547 io_ring_ctx_wait_and_kill(ctx);
10548 return ret;
10549}
10550
10551/*
10552 * Sets up an aio uring context, and returns the fd. Applications asks for a
10553 * ring size, we return the actual sq/cq ring sizes (among other things) in the
10554 * params structure passed in.
10555 */
10556static long io_uring_setup(u32 entries, struct io_uring_params __user *params)
10557{
10558 struct io_uring_params p;
Jens Axboe2b188cc2019-01-07 10:46:33 -070010559 int i;
10560
10561 if (copy_from_user(&p, params, sizeof(p)))
10562 return -EFAULT;
10563 for (i = 0; i < ARRAY_SIZE(p.resv); i++) {
10564 if (p.resv[i])
10565 return -EINVAL;
10566 }
10567
Jens Axboe6c271ce2019-01-10 11:22:30 -070010568 if (p.flags & ~(IORING_SETUP_IOPOLL | IORING_SETUP_SQPOLL |
Jens Axboe8110c1a2019-12-28 15:39:54 -070010569 IORING_SETUP_SQ_AFF | IORING_SETUP_CQSIZE |
Stefano Garzarella7e84e1c2020-08-27 16:58:31 +020010570 IORING_SETUP_CLAMP | IORING_SETUP_ATTACH_WQ |
10571 IORING_SETUP_R_DISABLED))
Jens Axboe2b188cc2019-01-07 10:46:33 -070010572 return -EINVAL;
10573
Xiaoguang Wang7f136572020-05-05 16:28:53 +080010574 return io_uring_create(entries, &p, params);
Jens Axboe2b188cc2019-01-07 10:46:33 -070010575}
10576
10577SYSCALL_DEFINE2(io_uring_setup, u32, entries,
10578 struct io_uring_params __user *, params)
10579{
10580 return io_uring_setup(entries, params);
10581}
10582
Jens Axboe66f4af92020-01-16 15:36:52 -070010583static int io_probe(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args)
10584{
10585 struct io_uring_probe *p;
10586 size_t size;
10587 int i, ret;
10588
10589 size = struct_size(p, ops, nr_args);
10590 if (size == SIZE_MAX)
10591 return -EOVERFLOW;
10592 p = kzalloc(size, GFP_KERNEL);
10593 if (!p)
10594 return -ENOMEM;
10595
10596 ret = -EFAULT;
10597 if (copy_from_user(p, arg, size))
10598 goto out;
10599 ret = -EINVAL;
10600 if (memchr_inv(p, 0, size))
10601 goto out;
10602
10603 p->last_op = IORING_OP_LAST - 1;
10604 if (nr_args > IORING_OP_LAST)
10605 nr_args = IORING_OP_LAST;
10606
10607 for (i = 0; i < nr_args; i++) {
10608 p->ops[i].op = i;
10609 if (!io_op_defs[i].not_supported)
10610 p->ops[i].flags = IO_URING_OP_SUPPORTED;
10611 }
10612 p->ops_len = i;
10613
10614 ret = 0;
10615 if (copy_to_user(arg, p, size))
10616 ret = -EFAULT;
10617out:
10618 kfree(p);
10619 return ret;
10620}
10621
Jens Axboe071698e2020-01-28 10:04:42 -070010622static int io_register_personality(struct io_ring_ctx *ctx)
10623{
Jens Axboe4379bf82021-02-15 13:40:22 -070010624 const struct cred *creds;
Matthew Wilcox (Oracle)61cf9372021-03-08 14:16:16 +000010625 u32 id;
Jens Axboe1e6fa522020-10-15 08:46:24 -060010626 int ret;
Jens Axboe071698e2020-01-28 10:04:42 -070010627
Jens Axboe4379bf82021-02-15 13:40:22 -070010628 creds = get_current_cred();
Jens Axboe1e6fa522020-10-15 08:46:24 -060010629
Matthew Wilcox (Oracle)61cf9372021-03-08 14:16:16 +000010630 ret = xa_alloc_cyclic(&ctx->personalities, &id, (void *)creds,
10631 XA_LIMIT(0, USHRT_MAX), &ctx->pers_next, GFP_KERNEL);
Jens Axboea30f8952021-08-20 14:53:59 -060010632 if (ret < 0) {
10633 put_cred(creds);
10634 return ret;
10635 }
10636 return id;
Jens Axboe071698e2020-01-28 10:04:42 -070010637}
10638
Stefano Garzarella21b55db2020-08-27 16:58:30 +020010639static int io_register_restrictions(struct io_ring_ctx *ctx, void __user *arg,
10640 unsigned int nr_args)
10641{
10642 struct io_uring_restriction *res;
10643 size_t size;
10644 int i, ret;
10645
Stefano Garzarella7e84e1c2020-08-27 16:58:31 +020010646 /* Restrictions allowed only if rings started disabled */
10647 if (!(ctx->flags & IORING_SETUP_R_DISABLED))
10648 return -EBADFD;
10649
Stefano Garzarella21b55db2020-08-27 16:58:30 +020010650 /* We allow only a single restrictions registration */
Stefano Garzarella7e84e1c2020-08-27 16:58:31 +020010651 if (ctx->restrictions.registered)
Stefano Garzarella21b55db2020-08-27 16:58:30 +020010652 return -EBUSY;
10653
10654 if (!arg || nr_args > IORING_MAX_RESTRICTIONS)
10655 return -EINVAL;
10656
10657 size = array_size(nr_args, sizeof(*res));
10658 if (size == SIZE_MAX)
10659 return -EOVERFLOW;
10660
10661 res = memdup_user(arg, size);
10662 if (IS_ERR(res))
10663 return PTR_ERR(res);
10664
10665 ret = 0;
10666
10667 for (i = 0; i < nr_args; i++) {
10668 switch (res[i].opcode) {
10669 case IORING_RESTRICTION_REGISTER_OP:
10670 if (res[i].register_op >= IORING_REGISTER_LAST) {
10671 ret = -EINVAL;
10672 goto out;
10673 }
10674
10675 __set_bit(res[i].register_op,
10676 ctx->restrictions.register_op);
10677 break;
10678 case IORING_RESTRICTION_SQE_OP:
10679 if (res[i].sqe_op >= IORING_OP_LAST) {
10680 ret = -EINVAL;
10681 goto out;
10682 }
10683
10684 __set_bit(res[i].sqe_op, ctx->restrictions.sqe_op);
10685 break;
10686 case IORING_RESTRICTION_SQE_FLAGS_ALLOWED:
10687 ctx->restrictions.sqe_flags_allowed = res[i].sqe_flags;
10688 break;
10689 case IORING_RESTRICTION_SQE_FLAGS_REQUIRED:
10690 ctx->restrictions.sqe_flags_required = res[i].sqe_flags;
10691 break;
10692 default:
10693 ret = -EINVAL;
10694 goto out;
10695 }
10696 }
10697
10698out:
10699 /* Reset all restrictions if an error happened */
10700 if (ret != 0)
10701 memset(&ctx->restrictions, 0, sizeof(ctx->restrictions));
10702 else
Stefano Garzarella7e84e1c2020-08-27 16:58:31 +020010703 ctx->restrictions.registered = true;
Stefano Garzarella21b55db2020-08-27 16:58:30 +020010704
10705 kfree(res);
10706 return ret;
10707}
10708
Stefano Garzarella7e84e1c2020-08-27 16:58:31 +020010709static int io_register_enable_rings(struct io_ring_ctx *ctx)
10710{
10711 if (!(ctx->flags & IORING_SETUP_R_DISABLED))
10712 return -EBADFD;
10713
10714 if (ctx->restrictions.registered)
10715 ctx->restricted = 1;
10716
Pavel Begunkov0298ef92021-03-08 13:20:57 +000010717 ctx->flags &= ~IORING_SETUP_R_DISABLED;
10718 if (ctx->sq_data && wq_has_sleeper(&ctx->sq_data->wait))
10719 wake_up(&ctx->sq_data->wait);
Stefano Garzarella7e84e1c2020-08-27 16:58:31 +020010720 return 0;
10721}
10722
Pavel Begunkovfdecb662021-04-25 14:32:20 +010010723static int __io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned type,
Pavel Begunkovc3bdad02021-04-25 14:32:22 +010010724 struct io_uring_rsrc_update2 *up,
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +010010725 unsigned nr_args)
10726{
10727 __u32 tmp;
10728 int err;
10729
10730 if (check_add_overflow(up->offset, nr_args, &tmp))
10731 return -EOVERFLOW;
10732 err = io_rsrc_node_switch_start(ctx);
10733 if (err)
10734 return err;
10735
Pavel Begunkovfdecb662021-04-25 14:32:20 +010010736 switch (type) {
10737 case IORING_RSRC_FILE:
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +010010738 return __io_sqe_files_update(ctx, up, nr_args);
Pavel Begunkov634d00d2021-04-25 14:32:26 +010010739 case IORING_RSRC_BUFFER:
10740 return __io_sqe_buffers_update(ctx, up, nr_args);
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +010010741 }
10742 return -EINVAL;
10743}
10744
Pavel Begunkovc3bdad02021-04-25 14:32:22 +010010745static int io_register_files_update(struct io_ring_ctx *ctx, void __user *arg,
10746 unsigned nr_args)
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +010010747{
Pavel Begunkovc3bdad02021-04-25 14:32:22 +010010748 struct io_uring_rsrc_update2 up;
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +010010749
10750 if (!nr_args)
10751 return -EINVAL;
Pavel Begunkovc3bdad02021-04-25 14:32:22 +010010752 memset(&up, 0, sizeof(up));
10753 if (copy_from_user(&up, arg, sizeof(struct io_uring_rsrc_update)))
10754 return -EFAULT;
Dylan Yudaken7a7c9f92022-04-12 09:30:40 -070010755 if (up.resv || up.resv2)
Dylan Yudaken22aa1592022-04-12 09:30:39 -070010756 return -EINVAL;
Pavel Begunkovc3bdad02021-04-25 14:32:22 +010010757 return __io_register_rsrc_update(ctx, IORING_RSRC_FILE, &up, nr_args);
10758}
10759
10760static int io_register_rsrc_update(struct io_ring_ctx *ctx, void __user *arg,
Pavel Begunkov992da012021-06-10 16:37:37 +010010761 unsigned size, unsigned type)
Pavel Begunkovc3bdad02021-04-25 14:32:22 +010010762{
10763 struct io_uring_rsrc_update2 up;
10764
10765 if (size != sizeof(up))
10766 return -EINVAL;
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +010010767 if (copy_from_user(&up, arg, sizeof(up)))
10768 return -EFAULT;
Dylan Yudaken7a7c9f92022-04-12 09:30:40 -070010769 if (!up.nr || up.resv || up.resv2)
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +010010770 return -EINVAL;
Pavel Begunkov992da012021-06-10 16:37:37 +010010771 return __io_register_rsrc_update(ctx, type, &up, up.nr);
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +010010772}
10773
Pavel Begunkov792e3582021-04-25 14:32:21 +010010774static int io_register_rsrc(struct io_ring_ctx *ctx, void __user *arg,
Pavel Begunkov992da012021-06-10 16:37:37 +010010775 unsigned int size, unsigned int type)
Pavel Begunkov792e3582021-04-25 14:32:21 +010010776{
10777 struct io_uring_rsrc_register rr;
10778
10779 /* keep it extendible */
10780 if (size != sizeof(rr))
10781 return -EINVAL;
10782
10783 memset(&rr, 0, sizeof(rr));
10784 if (copy_from_user(&rr, arg, size))
10785 return -EFAULT;
Pavel Begunkov992da012021-06-10 16:37:37 +010010786 if (!rr.nr || rr.resv || rr.resv2)
Pavel Begunkov792e3582021-04-25 14:32:21 +010010787 return -EINVAL;
10788
Pavel Begunkov992da012021-06-10 16:37:37 +010010789 switch (type) {
Pavel Begunkov792e3582021-04-25 14:32:21 +010010790 case IORING_RSRC_FILE:
10791 return io_sqe_files_register(ctx, u64_to_user_ptr(rr.data),
10792 rr.nr, u64_to_user_ptr(rr.tags));
Pavel Begunkov634d00d2021-04-25 14:32:26 +010010793 case IORING_RSRC_BUFFER:
10794 return io_sqe_buffers_register(ctx, u64_to_user_ptr(rr.data),
10795 rr.nr, u64_to_user_ptr(rr.tags));
Pavel Begunkov792e3582021-04-25 14:32:21 +010010796 }
10797 return -EINVAL;
10798}
10799
Jens Axboefe764212021-06-17 10:19:54 -060010800static int io_register_iowq_aff(struct io_ring_ctx *ctx, void __user *arg,
10801 unsigned len)
10802{
10803 struct io_uring_task *tctx = current->io_uring;
10804 cpumask_var_t new_mask;
10805 int ret;
10806
10807 if (!tctx || !tctx->io_wq)
10808 return -EINVAL;
10809
10810 if (!alloc_cpumask_var(&new_mask, GFP_KERNEL))
10811 return -ENOMEM;
10812
10813 cpumask_clear(new_mask);
10814 if (len > cpumask_size())
10815 len = cpumask_size();
10816
Eugene Syromiatnikovfe223dd2022-04-06 13:55:33 +020010817 if (in_compat_syscall()) {
10818 ret = compat_get_bitmap(cpumask_bits(new_mask),
10819 (const compat_ulong_t __user *)arg,
10820 len * 8 /* CHAR_BIT */);
10821 } else {
10822 ret = copy_from_user(new_mask, arg, len);
10823 }
10824
10825 if (ret) {
Jens Axboefe764212021-06-17 10:19:54 -060010826 free_cpumask_var(new_mask);
10827 return -EFAULT;
10828 }
10829
10830 ret = io_wq_cpu_affinity(tctx->io_wq, new_mask);
10831 free_cpumask_var(new_mask);
10832 return ret;
10833}
10834
10835static int io_unregister_iowq_aff(struct io_ring_ctx *ctx)
10836{
10837 struct io_uring_task *tctx = current->io_uring;
10838
10839 if (!tctx || !tctx->io_wq)
10840 return -EINVAL;
10841
10842 return io_wq_cpu_affinity(tctx->io_wq, NULL);
10843}
10844
Jens Axboe2e480052021-08-27 11:33:19 -060010845static int io_register_iowq_max_workers(struct io_ring_ctx *ctx,
10846 void __user *arg)
Pavel Begunkovb22fa622021-10-21 13:20:29 +010010847 __must_hold(&ctx->uring_lock)
Jens Axboe2e480052021-08-27 11:33:19 -060010848{
Pavel Begunkovb22fa622021-10-21 13:20:29 +010010849 struct io_tctx_node *node;
Jens Axboefa846932021-09-01 14:15:59 -060010850 struct io_uring_task *tctx = NULL;
10851 struct io_sq_data *sqd = NULL;
Jens Axboe2e480052021-08-27 11:33:19 -060010852 __u32 new_count[2];
10853 int i, ret;
10854
Jens Axboe2e480052021-08-27 11:33:19 -060010855 if (copy_from_user(new_count, arg, sizeof(new_count)))
10856 return -EFAULT;
10857 for (i = 0; i < ARRAY_SIZE(new_count); i++)
10858 if (new_count[i] > INT_MAX)
10859 return -EINVAL;
10860
Jens Axboefa846932021-09-01 14:15:59 -060010861 if (ctx->flags & IORING_SETUP_SQPOLL) {
10862 sqd = ctx->sq_data;
10863 if (sqd) {
Jens Axboe009ad9f2021-09-08 19:07:26 -060010864 /*
10865 * Observe the correct sqd->lock -> ctx->uring_lock
10866 * ordering. Fine to drop uring_lock here, we hold
10867 * a ref to the ctx.
10868 */
Jens Axboe41d3a6b2021-09-13 13:08:51 -060010869 refcount_inc(&sqd->refs);
Jens Axboe009ad9f2021-09-08 19:07:26 -060010870 mutex_unlock(&ctx->uring_lock);
Jens Axboefa846932021-09-01 14:15:59 -060010871 mutex_lock(&sqd->lock);
Jens Axboe009ad9f2021-09-08 19:07:26 -060010872 mutex_lock(&ctx->uring_lock);
Jens Axboe41d3a6b2021-09-13 13:08:51 -060010873 if (sqd->thread)
10874 tctx = sqd->thread->io_uring;
Jens Axboefa846932021-09-01 14:15:59 -060010875 }
10876 } else {
10877 tctx = current->io_uring;
10878 }
10879
Pavel Begunkove139a1e2021-10-19 23:43:46 +010010880 BUILD_BUG_ON(sizeof(new_count) != sizeof(ctx->iowq_limits));
Jens Axboefa846932021-09-01 14:15:59 -060010881
Pavel Begunkov4cac4872021-11-08 15:10:03 +000010882 for (i = 0; i < ARRAY_SIZE(new_count); i++)
10883 if (new_count[i])
10884 ctx->iowq_limits[i] = new_count[i];
Pavel Begunkove139a1e2021-10-19 23:43:46 +010010885 ctx->iowq_limits_set = true;
10886
10887 ret = -EINVAL;
10888 if (tctx && tctx->io_wq) {
10889 ret = io_wq_max_workers(tctx->io_wq, new_count);
10890 if (ret)
10891 goto err;
10892 } else {
10893 memset(new_count, 0, sizeof(new_count));
10894 }
Jens Axboefa846932021-09-01 14:15:59 -060010895
Jens Axboe41d3a6b2021-09-13 13:08:51 -060010896 if (sqd) {
Jens Axboefa846932021-09-01 14:15:59 -060010897 mutex_unlock(&sqd->lock);
Jens Axboe41d3a6b2021-09-13 13:08:51 -060010898 io_put_sq_data(sqd);
10899 }
Jens Axboe2e480052021-08-27 11:33:19 -060010900
10901 if (copy_to_user(arg, new_count, sizeof(new_count)))
10902 return -EFAULT;
10903
Pavel Begunkovb22fa622021-10-21 13:20:29 +010010904 /* that's it for SQPOLL, only the SQPOLL task creates requests */
10905 if (sqd)
10906 return 0;
10907
10908 /* now propagate the restriction to all registered users */
10909 list_for_each_entry(node, &ctx->tctx_list, ctx_node) {
10910 struct io_uring_task *tctx = node->task->io_uring;
10911
10912 if (WARN_ON_ONCE(!tctx->io_wq))
10913 continue;
10914
10915 for (i = 0; i < ARRAY_SIZE(new_count); i++)
10916 new_count[i] = ctx->iowq_limits[i];
10917 /* ignore errors, it always returns zero anyway */
10918 (void)io_wq_max_workers(tctx->io_wq, new_count);
10919 }
Jens Axboe2e480052021-08-27 11:33:19 -060010920 return 0;
Jens Axboefa846932021-09-01 14:15:59 -060010921err:
Jens Axboe41d3a6b2021-09-13 13:08:51 -060010922 if (sqd) {
Jens Axboefa846932021-09-01 14:15:59 -060010923 mutex_unlock(&sqd->lock);
Jens Axboe41d3a6b2021-09-13 13:08:51 -060010924 io_put_sq_data(sqd);
10925 }
Jens Axboefa846932021-09-01 14:15:59 -060010926 return ret;
Jens Axboe2e480052021-08-27 11:33:19 -060010927}
10928
Jens Axboe071698e2020-01-28 10:04:42 -070010929static bool io_register_op_must_quiesce(int op)
10930{
10931 switch (op) {
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +010010932 case IORING_REGISTER_BUFFERS:
10933 case IORING_UNREGISTER_BUFFERS:
Pavel Begunkovf4f7d212021-04-01 15:44:02 +010010934 case IORING_REGISTER_FILES:
Jens Axboe071698e2020-01-28 10:04:42 -070010935 case IORING_UNREGISTER_FILES:
10936 case IORING_REGISTER_FILES_UPDATE:
10937 case IORING_REGISTER_PROBE:
10938 case IORING_REGISTER_PERSONALITY:
10939 case IORING_UNREGISTER_PERSONALITY:
Pavel Begunkov992da012021-06-10 16:37:37 +010010940 case IORING_REGISTER_FILES2:
10941 case IORING_REGISTER_FILES_UPDATE2:
10942 case IORING_REGISTER_BUFFERS2:
10943 case IORING_REGISTER_BUFFERS_UPDATE:
Jens Axboefe764212021-06-17 10:19:54 -060010944 case IORING_REGISTER_IOWQ_AFF:
10945 case IORING_UNREGISTER_IOWQ_AFF:
Jens Axboe2e480052021-08-27 11:33:19 -060010946 case IORING_REGISTER_IOWQ_MAX_WORKERS:
Jens Axboe071698e2020-01-28 10:04:42 -070010947 return false;
10948 default:
10949 return true;
10950 }
10951}
10952
Pavel Begunkove73c5c72021-08-09 13:04:12 +010010953static int io_ctx_quiesce(struct io_ring_ctx *ctx)
10954{
10955 long ret;
10956
10957 percpu_ref_kill(&ctx->refs);
10958
10959 /*
10960 * Drop uring mutex before waiting for references to exit. If another
10961 * thread is currently inside io_uring_enter() it might need to grab the
10962 * uring_lock to make progress. If we hold it here across the drain
10963 * wait, then we can deadlock. It's safe to drop the mutex here, since
10964 * no new references will come in after we've killed the percpu ref.
10965 */
10966 mutex_unlock(&ctx->uring_lock);
10967 do {
10968 ret = wait_for_completion_interruptible(&ctx->ref_comp);
10969 if (!ret)
10970 break;
10971 ret = io_run_task_work_sig();
10972 } while (ret >= 0);
10973 mutex_lock(&ctx->uring_lock);
10974
10975 if (ret)
10976 io_refs_resurrect(&ctx->refs, &ctx->ref_comp);
10977 return ret;
10978}
10979
Jens Axboeedafcce2019-01-09 09:16:05 -070010980static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,
10981 void __user *arg, unsigned nr_args)
Jens Axboeb19062a2019-04-15 10:49:38 -060010982 __releases(ctx->uring_lock)
10983 __acquires(ctx->uring_lock)
Jens Axboeedafcce2019-01-09 09:16:05 -070010984{
10985 int ret;
10986
Jens Axboe35fa71a2019-04-22 10:23:23 -060010987 /*
10988 * We're inside the ring mutex, if the ref is already dying, then
10989 * someone else killed the ctx or is already going through
10990 * io_uring_register().
10991 */
10992 if (percpu_ref_is_dying(&ctx->refs))
10993 return -ENXIO;
10994
Pavel Begunkov75c40212021-04-15 13:07:40 +010010995 if (ctx->restricted) {
Pavel Begunkov75c40212021-04-15 13:07:40 +010010996 opcode = array_index_nospec(opcode, IORING_REGISTER_LAST);
10997 if (!test_bit(opcode, ctx->restrictions.register_op))
10998 return -EACCES;
10999 }
11000
Jens Axboe071698e2020-01-28 10:04:42 -070011001 if (io_register_op_must_quiesce(opcode)) {
Pavel Begunkove73c5c72021-08-09 13:04:12 +010011002 ret = io_ctx_quiesce(ctx);
11003 if (ret)
Pavel Begunkovf70865d2021-04-11 01:46:40 +010011004 return ret;
Jens Axboe05f3fb32019-12-09 11:22:50 -070011005 }
Jens Axboeedafcce2019-01-09 09:16:05 -070011006
11007 switch (opcode) {
11008 case IORING_REGISTER_BUFFERS:
Pavel Begunkov634d00d2021-04-25 14:32:26 +010011009 ret = io_sqe_buffers_register(ctx, arg, nr_args, NULL);
Jens Axboeedafcce2019-01-09 09:16:05 -070011010 break;
11011 case IORING_UNREGISTER_BUFFERS:
11012 ret = -EINVAL;
11013 if (arg || nr_args)
11014 break;
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -080011015 ret = io_sqe_buffers_unregister(ctx);
Jens Axboeedafcce2019-01-09 09:16:05 -070011016 break;
Jens Axboe6b063142019-01-10 22:13:58 -070011017 case IORING_REGISTER_FILES:
Pavel Begunkov792e3582021-04-25 14:32:21 +010011018 ret = io_sqe_files_register(ctx, arg, nr_args, NULL);
Jens Axboe6b063142019-01-10 22:13:58 -070011019 break;
11020 case IORING_UNREGISTER_FILES:
11021 ret = -EINVAL;
11022 if (arg || nr_args)
11023 break;
11024 ret = io_sqe_files_unregister(ctx);
11025 break;
Jens Axboec3a31e62019-10-03 13:59:56 -060011026 case IORING_REGISTER_FILES_UPDATE:
Pavel Begunkovc3bdad02021-04-25 14:32:22 +010011027 ret = io_register_files_update(ctx, arg, nr_args);
Jens Axboec3a31e62019-10-03 13:59:56 -060011028 break;
Jens Axboe9b402842019-04-11 11:45:41 -060011029 case IORING_REGISTER_EVENTFD:
Jens Axboef2842ab2020-01-08 11:04:00 -070011030 case IORING_REGISTER_EVENTFD_ASYNC:
Jens Axboe9b402842019-04-11 11:45:41 -060011031 ret = -EINVAL;
11032 if (nr_args != 1)
11033 break;
11034 ret = io_eventfd_register(ctx, arg);
Jens Axboef2842ab2020-01-08 11:04:00 -070011035 if (ret)
11036 break;
11037 if (opcode == IORING_REGISTER_EVENTFD_ASYNC)
11038 ctx->eventfd_async = 1;
11039 else
11040 ctx->eventfd_async = 0;
Jens Axboe9b402842019-04-11 11:45:41 -060011041 break;
11042 case IORING_UNREGISTER_EVENTFD:
11043 ret = -EINVAL;
11044 if (arg || nr_args)
11045 break;
11046 ret = io_eventfd_unregister(ctx);
11047 break;
Jens Axboe66f4af92020-01-16 15:36:52 -070011048 case IORING_REGISTER_PROBE:
11049 ret = -EINVAL;
11050 if (!arg || nr_args > 256)
11051 break;
11052 ret = io_probe(ctx, arg, nr_args);
11053 break;
Jens Axboe071698e2020-01-28 10:04:42 -070011054 case IORING_REGISTER_PERSONALITY:
11055 ret = -EINVAL;
11056 if (arg || nr_args)
11057 break;
11058 ret = io_register_personality(ctx);
11059 break;
11060 case IORING_UNREGISTER_PERSONALITY:
11061 ret = -EINVAL;
11062 if (arg)
11063 break;
11064 ret = io_unregister_personality(ctx, nr_args);
11065 break;
Stefano Garzarella7e84e1c2020-08-27 16:58:31 +020011066 case IORING_REGISTER_ENABLE_RINGS:
11067 ret = -EINVAL;
11068 if (arg || nr_args)
11069 break;
11070 ret = io_register_enable_rings(ctx);
11071 break;
Stefano Garzarella21b55db2020-08-27 16:58:30 +020011072 case IORING_REGISTER_RESTRICTIONS:
11073 ret = io_register_restrictions(ctx, arg, nr_args);
11074 break;
Pavel Begunkov992da012021-06-10 16:37:37 +010011075 case IORING_REGISTER_FILES2:
11076 ret = io_register_rsrc(ctx, arg, nr_args, IORING_RSRC_FILE);
Pavel Begunkov792e3582021-04-25 14:32:21 +010011077 break;
Pavel Begunkov992da012021-06-10 16:37:37 +010011078 case IORING_REGISTER_FILES_UPDATE2:
11079 ret = io_register_rsrc_update(ctx, arg, nr_args,
11080 IORING_RSRC_FILE);
11081 break;
11082 case IORING_REGISTER_BUFFERS2:
11083 ret = io_register_rsrc(ctx, arg, nr_args, IORING_RSRC_BUFFER);
11084 break;
11085 case IORING_REGISTER_BUFFERS_UPDATE:
11086 ret = io_register_rsrc_update(ctx, arg, nr_args,
11087 IORING_RSRC_BUFFER);
Pavel Begunkovc3bdad02021-04-25 14:32:22 +010011088 break;
Jens Axboefe764212021-06-17 10:19:54 -060011089 case IORING_REGISTER_IOWQ_AFF:
11090 ret = -EINVAL;
11091 if (!arg || !nr_args)
11092 break;
11093 ret = io_register_iowq_aff(ctx, arg, nr_args);
11094 break;
11095 case IORING_UNREGISTER_IOWQ_AFF:
11096 ret = -EINVAL;
11097 if (arg || nr_args)
11098 break;
11099 ret = io_unregister_iowq_aff(ctx);
11100 break;
Jens Axboe2e480052021-08-27 11:33:19 -060011101 case IORING_REGISTER_IOWQ_MAX_WORKERS:
11102 ret = -EINVAL;
11103 if (!arg || nr_args != 2)
11104 break;
11105 ret = io_register_iowq_max_workers(ctx, arg);
11106 break;
Jens Axboeedafcce2019-01-09 09:16:05 -070011107 default:
11108 ret = -EINVAL;
11109 break;
11110 }
11111
Jens Axboe071698e2020-01-28 10:04:42 -070011112 if (io_register_op_must_quiesce(opcode)) {
Jens Axboe05f3fb32019-12-09 11:22:50 -070011113 /* bring the ctx back to life */
Jens Axboe05f3fb32019-12-09 11:22:50 -070011114 percpu_ref_reinit(&ctx->refs);
Jens Axboe0f158b42020-05-14 17:18:39 -060011115 reinit_completion(&ctx->ref_comp);
Jens Axboe05f3fb32019-12-09 11:22:50 -070011116 }
Jens Axboeedafcce2019-01-09 09:16:05 -070011117 return ret;
11118}
11119
11120SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode,
11121 void __user *, arg, unsigned int, nr_args)
11122{
11123 struct io_ring_ctx *ctx;
11124 long ret = -EBADF;
11125 struct fd f;
11126
Jens Axboef9309dc2022-12-23 06:37:08 -070011127 if (opcode >= IORING_REGISTER_LAST)
11128 return -EINVAL;
11129
Jens Axboeedafcce2019-01-09 09:16:05 -070011130 f = fdget(fd);
11131 if (!f.file)
11132 return -EBADF;
11133
11134 ret = -EOPNOTSUPP;
11135 if (f.file->f_op != &io_uring_fops)
11136 goto out_fput;
11137
11138 ctx = f.file->private_data;
11139
Pavel Begunkovb6c23dd2021-02-20 15:17:18 +000011140 io_run_task_work();
11141
Jens Axboeedafcce2019-01-09 09:16:05 -070011142 mutex_lock(&ctx->uring_lock);
11143 ret = __io_uring_register(ctx, opcode, arg, nr_args);
11144 mutex_unlock(&ctx->uring_lock);
Dmitrii Dolgovc826bd72019-10-15 19:02:01 +020011145 trace_io_uring_register(ctx, opcode, ctx->nr_user_files, ctx->nr_user_bufs,
11146 ctx->cq_ev_fd != NULL, ret);
Jens Axboeedafcce2019-01-09 09:16:05 -070011147out_fput:
11148 fdput(f);
11149 return ret;
11150}
11151
Jens Axboe2b188cc2019-01-07 10:46:33 -070011152static int __init io_uring_init(void)
11153{
Stefan Metzmacherd7f62e82020-01-29 14:39:41 +010011154#define __BUILD_BUG_VERIFY_ELEMENT(stype, eoffset, etype, ename) do { \
11155 BUILD_BUG_ON(offsetof(stype, ename) != eoffset); \
11156 BUILD_BUG_ON(sizeof(etype) != sizeof_field(stype, ename)); \
11157} while (0)
11158
11159#define BUILD_BUG_SQE_ELEM(eoffset, etype, ename) \
11160 __BUILD_BUG_VERIFY_ELEMENT(struct io_uring_sqe, eoffset, etype, ename)
11161 BUILD_BUG_ON(sizeof(struct io_uring_sqe) != 64);
11162 BUILD_BUG_SQE_ELEM(0, __u8, opcode);
11163 BUILD_BUG_SQE_ELEM(1, __u8, flags);
11164 BUILD_BUG_SQE_ELEM(2, __u16, ioprio);
11165 BUILD_BUG_SQE_ELEM(4, __s32, fd);
11166 BUILD_BUG_SQE_ELEM(8, __u64, off);
11167 BUILD_BUG_SQE_ELEM(8, __u64, addr2);
11168 BUILD_BUG_SQE_ELEM(16, __u64, addr);
Pavel Begunkov7d67af22020-02-24 11:32:45 +030011169 BUILD_BUG_SQE_ELEM(16, __u64, splice_off_in);
Stefan Metzmacherd7f62e82020-01-29 14:39:41 +010011170 BUILD_BUG_SQE_ELEM(24, __u32, len);
11171 BUILD_BUG_SQE_ELEM(28, __kernel_rwf_t, rw_flags);
11172 BUILD_BUG_SQE_ELEM(28, /* compat */ int, rw_flags);
11173 BUILD_BUG_SQE_ELEM(28, /* compat */ __u32, rw_flags);
11174 BUILD_BUG_SQE_ELEM(28, __u32, fsync_flags);
Jiufei Xue5769a352020-06-17 17:53:55 +080011175 BUILD_BUG_SQE_ELEM(28, /* compat */ __u16, poll_events);
11176 BUILD_BUG_SQE_ELEM(28, __u32, poll32_events);
Stefan Metzmacherd7f62e82020-01-29 14:39:41 +010011177 BUILD_BUG_SQE_ELEM(28, __u32, sync_range_flags);
11178 BUILD_BUG_SQE_ELEM(28, __u32, msg_flags);
11179 BUILD_BUG_SQE_ELEM(28, __u32, timeout_flags);
11180 BUILD_BUG_SQE_ELEM(28, __u32, accept_flags);
11181 BUILD_BUG_SQE_ELEM(28, __u32, cancel_flags);
11182 BUILD_BUG_SQE_ELEM(28, __u32, open_flags);
11183 BUILD_BUG_SQE_ELEM(28, __u32, statx_flags);
11184 BUILD_BUG_SQE_ELEM(28, __u32, fadvise_advice);
Pavel Begunkov7d67af22020-02-24 11:32:45 +030011185 BUILD_BUG_SQE_ELEM(28, __u32, splice_flags);
Stefan Metzmacherd7f62e82020-01-29 14:39:41 +010011186 BUILD_BUG_SQE_ELEM(32, __u64, user_data);
11187 BUILD_BUG_SQE_ELEM(40, __u16, buf_index);
Pavel Begunkov16340ea2021-06-24 15:09:58 +010011188 BUILD_BUG_SQE_ELEM(40, __u16, buf_group);
Stefan Metzmacherd7f62e82020-01-29 14:39:41 +010011189 BUILD_BUG_SQE_ELEM(42, __u16, personality);
Pavel Begunkov7d67af22020-02-24 11:32:45 +030011190 BUILD_BUG_SQE_ELEM(44, __s32, splice_fd_in);
Pavel Begunkovb9445592021-08-25 12:25:45 +010011191 BUILD_BUG_SQE_ELEM(44, __u32, file_index);
Stefan Metzmacherd7f62e82020-01-29 14:39:41 +010011192
Pavel Begunkovb0d658ec2021-04-27 16:13:53 +010011193 BUILD_BUG_ON(sizeof(struct io_uring_files_update) !=
11194 sizeof(struct io_uring_rsrc_update));
11195 BUILD_BUG_ON(sizeof(struct io_uring_rsrc_update) >
11196 sizeof(struct io_uring_rsrc_update2));
Pavel Begunkov90499ad2021-08-25 20:51:40 +010011197
11198 /* ->buf_index is u16 */
11199 BUILD_BUG_ON(IORING_MAX_REG_BUFFERS >= (1u << 16));
11200
Pavel Begunkovb0d658ec2021-04-27 16:13:53 +010011201 /* should fit into one byte */
11202 BUILD_BUG_ON(SQE_VALID_FLAGS >= (1 << 8));
11203
Jens Axboed3656342019-12-18 09:50:26 -070011204 BUILD_BUG_ON(ARRAY_SIZE(io_op_defs) != IORING_OP_LAST);
Hao Xu32c2d332021-09-07 11:22:43 +080011205 BUILD_BUG_ON(__REQ_F_LAST_BIT > 8 * sizeof(int));
Pavel Begunkov16340ea2021-06-24 15:09:58 +010011206
Jens Axboe91f245d2021-02-09 13:48:50 -070011207 req_cachep = KMEM_CACHE(io_kiocb, SLAB_HWCACHE_ALIGN | SLAB_PANIC |
11208 SLAB_ACCOUNT);
Jens Axboe2b188cc2019-01-07 10:46:33 -070011209 return 0;
11210};
11211__initcall(io_uring_init);