blob: 0e3fc80fee05a14e0124926edc194ed44810d8fb [file] [log] [blame]
Jens Axboe2b188cc2019-01-07 10:46:33 -07001// SPDX-License-Identifier: GPL-2.0
2/*
3 * Shared application/kernel submission and completion ring pairs, for
4 * supporting fast/efficient IO.
5 *
6 * A note on the read/write ordering memory barriers that are matched between
Stefan Bühler1e84b972019-04-24 23:54:16 +02007 * the application and kernel side.
8 *
9 * After the application reads the CQ ring tail, it must use an
10 * appropriate smp_rmb() to pair with the smp_wmb() the kernel uses
11 * before writing the tail (using smp_load_acquire to read the tail will
12 * do). It also needs a smp_mb() before updating CQ head (ordering the
13 * entry load(s) with the head store), pairing with an implicit barrier
Pavel Begunkovd068b502021-05-16 22:58:11 +010014 * through a control-dependency in io_get_cqe (smp_store_release to
Stefan Bühler1e84b972019-04-24 23:54:16 +020015 * store head will do). Failure to do so could lead to reading invalid
16 * CQ entries.
17 *
18 * Likewise, the application must use an appropriate smp_wmb() before
19 * writing the SQ tail (ordering SQ entry stores with the tail store),
20 * which pairs with smp_load_acquire in io_get_sqring (smp_store_release
21 * to store the tail will do). And it needs a barrier ordering the SQ
22 * head load before writing new SQ entries (smp_load_acquire to read
23 * head will do).
24 *
25 * When using the SQ poll thread (IORING_SETUP_SQPOLL), the application
26 * needs to check the SQ flags for IORING_SQ_NEED_WAKEUP *after*
27 * updating the SQ tail; a full memory barrier smp_mb() is needed
28 * between.
Jens Axboe2b188cc2019-01-07 10:46:33 -070029 *
30 * Also see the examples in the liburing library:
31 *
32 * git://git.kernel.dk/liburing
33 *
34 * io_uring also uses READ/WRITE_ONCE() for _any_ store or load that happens
35 * from data shared between the kernel and application. This is done both
36 * for ordering purposes, but also to ensure that once a value is loaded from
37 * data that the application could potentially modify, it remains stable.
38 *
39 * Copyright (C) 2018-2019 Jens Axboe
Christoph Hellwigc992fe22019-01-11 09:43:02 -070040 * Copyright (c) 2018-2019 Christoph Hellwig
Jens Axboe2b188cc2019-01-07 10:46:33 -070041 */
42#include <linux/kernel.h>
43#include <linux/init.h>
44#include <linux/errno.h>
45#include <linux/syscalls.h>
46#include <linux/compat.h>
Jens Axboe52de1fe2020-02-27 10:15:42 -070047#include <net/compat.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070048#include <linux/refcount.h>
49#include <linux/uio.h>
Pavel Begunkov6b47ee62020-01-18 20:22:41 +030050#include <linux/bits.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070051
52#include <linux/sched/signal.h>
53#include <linux/fs.h>
54#include <linux/file.h>
55#include <linux/fdtable.h>
56#include <linux/mm.h>
57#include <linux/mman.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070058#include <linux/percpu.h>
59#include <linux/slab.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070060#include <linux/blkdev.h>
Jens Axboeedafcce2019-01-09 09:16:05 -070061#include <linux/bvec.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070062#include <linux/net.h>
63#include <net/sock.h>
64#include <net/af_unix.h>
Jens Axboe6b063142019-01-10 22:13:58 -070065#include <net/scm.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070066#include <linux/anon_inodes.h>
67#include <linux/sched/mm.h>
68#include <linux/uaccess.h>
69#include <linux/nospec.h>
Jens Axboeedafcce2019-01-09 09:16:05 -070070#include <linux/sizes.h>
71#include <linux/hugetlb.h>
Jens Axboeaa4c3962019-11-29 10:14:00 -070072#include <linux/highmem.h>
Jens Axboe15b71ab2019-12-11 11:20:36 -070073#include <linux/namei.h>
74#include <linux/fsnotify.h>
Jens Axboe4840e412019-12-25 22:03:45 -070075#include <linux/fadvise.h>
Jens Axboe3e4827b2020-01-08 15:18:09 -070076#include <linux/eventpoll.h>
Pavel Begunkov7d67af22020-02-24 11:32:45 +030077#include <linux/splice.h>
Jens Axboeb41e9852020-02-17 09:52:41 -070078#include <linux/task_work.h>
Jens Axboebcf5a062020-05-22 09:24:42 -060079#include <linux/pagemap.h>
Jens Axboe0f212202020-09-13 13:09:39 -060080#include <linux/io_uring.h>
Nadav Amitef98eb02021-08-07 17:13:41 -070081#include <linux/tracehook.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070082
Dmitrii Dolgovc826bd72019-10-15 19:02:01 +020083#define CREATE_TRACE_POINTS
84#include <trace/events/io_uring.h>
85
Jens Axboe2b188cc2019-01-07 10:46:33 -070086#include <uapi/linux/io_uring.h>
87
88#include "internal.h"
Jens Axboe561fb042019-10-24 07:25:42 -060089#include "io-wq.h"
Jens Axboe2b188cc2019-01-07 10:46:33 -070090
Daniel Xu5277dea2019-09-14 14:23:45 -070091#define IORING_MAX_ENTRIES 32768
Jens Axboe33a107f2019-10-04 12:10:03 -060092#define IORING_MAX_CQ_ENTRIES (2 * IORING_MAX_ENTRIES)
Olivier Langlois4ce8ad92021-06-23 11:50:18 -070093#define IORING_SQPOLL_CAP_ENTRIES_VALUE 8
Jens Axboe65e19f52019-10-26 07:20:21 -060094
wangyangbo187f08c2021-08-19 13:56:57 +080095/* only define max */
Pavel Begunkov042b0d82021-08-09 13:04:01 +010096#define IORING_MAX_FIXED_FILES (1U << 15)
Stefano Garzarella21b55db2020-08-27 16:58:30 +020097#define IORING_MAX_RESTRICTIONS (IORING_RESTRICTION_LAST + \
98 IORING_REGISTER_LAST + IORING_OP_LAST)
Jens Axboe2b188cc2019-01-07 10:46:33 -070099
wangyangbo187f08c2021-08-19 13:56:57 +0800100#define IO_RSRC_TAG_TABLE_SHIFT (PAGE_SHIFT - 3)
Pavel Begunkov2d091d62021-06-14 02:36:21 +0100101#define IO_RSRC_TAG_TABLE_MAX (1U << IO_RSRC_TAG_TABLE_SHIFT)
102#define IO_RSRC_TAG_TABLE_MASK (IO_RSRC_TAG_TABLE_MAX - 1)
103
Pavel Begunkov489809e2021-05-14 12:06:44 +0100104#define IORING_MAX_REG_BUFFERS (1U << 14)
105
Pavel Begunkovb16fed662021-02-18 18:29:40 +0000106#define SQE_VALID_FLAGS (IOSQE_FIXED_FILE|IOSQE_IO_DRAIN|IOSQE_IO_LINK| \
107 IOSQE_IO_HARDLINK | IOSQE_ASYNC | \
108 IOSQE_BUFFER_SELECT)
Pavel Begunkovc8543572021-06-17 18:14:04 +0100109#define IO_REQ_CLEAN_FLAGS (REQ_F_BUFFER_SELECTED | REQ_F_NEED_CLEANUP | \
110 REQ_F_POLLED | REQ_F_INFLIGHT | REQ_F_CREDS)
Pavel Begunkovb16fed662021-02-18 18:29:40 +0000111
Pavel Begunkov09899b12021-06-14 02:36:22 +0100112#define IO_TCTX_REFS_CACHE_NR (1U << 10)
113
Jens Axboe2b188cc2019-01-07 10:46:33 -0700114struct io_uring {
115 u32 head ____cacheline_aligned_in_smp;
116 u32 tail ____cacheline_aligned_in_smp;
117};
118
Stefan Bühler1e84b972019-04-24 23:54:16 +0200119/*
Hristo Venev75b28af2019-08-26 17:23:46 +0000120 * This data is shared with the application through the mmap at offsets
121 * IORING_OFF_SQ_RING and IORING_OFF_CQ_RING.
Stefan Bühler1e84b972019-04-24 23:54:16 +0200122 *
123 * The offsets to the member fields are published through struct
124 * io_sqring_offsets when calling io_uring_setup.
125 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000126struct io_rings {
Stefan Bühler1e84b972019-04-24 23:54:16 +0200127 /*
128 * Head and tail offsets into the ring; the offsets need to be
129 * masked to get valid indices.
130 *
Hristo Venev75b28af2019-08-26 17:23:46 +0000131 * The kernel controls head of the sq ring and the tail of the cq ring,
132 * and the application controls tail of the sq ring and the head of the
133 * cq ring.
Stefan Bühler1e84b972019-04-24 23:54:16 +0200134 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000135 struct io_uring sq, cq;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200136 /*
Hristo Venev75b28af2019-08-26 17:23:46 +0000137 * Bitmasks to apply to head and tail offsets (constant, equals
Stefan Bühler1e84b972019-04-24 23:54:16 +0200138 * ring_entries - 1)
139 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000140 u32 sq_ring_mask, cq_ring_mask;
141 /* Ring sizes (constant, power of 2) */
142 u32 sq_ring_entries, cq_ring_entries;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200143 /*
144 * Number of invalid entries dropped by the kernel due to
145 * invalid index stored in array
146 *
147 * Written by the kernel, shouldn't be modified by the
148 * application (i.e. get number of "new events" by comparing to
149 * cached value).
150 *
151 * After a new SQ head value was read by the application this
152 * counter includes all submissions that were dropped reaching
153 * the new SQ head (and possibly more).
154 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000155 u32 sq_dropped;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200156 /*
Stefano Garzarella0d9b5b32020-05-15 18:38:04 +0200157 * Runtime SQ flags
Stefan Bühler1e84b972019-04-24 23:54:16 +0200158 *
159 * Written by the kernel, shouldn't be modified by the
160 * application.
161 *
162 * The application needs a full memory barrier before checking
163 * for IORING_SQ_NEED_WAKEUP after updating the sq tail.
164 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000165 u32 sq_flags;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200166 /*
Stefano Garzarella0d9b5b32020-05-15 18:38:04 +0200167 * Runtime CQ flags
168 *
169 * Written by the application, shouldn't be modified by the
170 * kernel.
171 */
Pavel Begunkovfe7e3252021-06-24 15:09:57 +0100172 u32 cq_flags;
Stefano Garzarella0d9b5b32020-05-15 18:38:04 +0200173 /*
Stefan Bühler1e84b972019-04-24 23:54:16 +0200174 * Number of completion events lost because the queue was full;
175 * this should be avoided by the application by making sure
LimingWu0b4295b2019-12-05 20:18:18 +0800176 * there are not more requests pending than there is space in
Stefan Bühler1e84b972019-04-24 23:54:16 +0200177 * the completion queue.
178 *
179 * Written by the kernel, shouldn't be modified by the
180 * application (i.e. get number of "new events" by comparing to
181 * cached value).
182 *
183 * As completion events come in out of order this counter is not
184 * ordered with any other data.
185 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000186 u32 cq_overflow;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200187 /*
188 * Ring buffer of completion events.
189 *
190 * The kernel writes completion events fresh every time they are
191 * produced, so the application is allowed to modify pending
192 * entries.
193 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000194 struct io_uring_cqe cqes[] ____cacheline_aligned_in_smp;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700195};
196
Pavel Begunkov45d189c2021-02-10 00:03:07 +0000197enum io_uring_cmd_flags {
198 IO_URING_F_NONBLOCK = 1,
Pavel Begunkov889fca72021-02-10 00:03:09 +0000199 IO_URING_F_COMPLETE_DEFER = 2,
Pavel Begunkov45d189c2021-02-10 00:03:07 +0000200};
201
Jens Axboeedafcce2019-01-09 09:16:05 -0700202struct io_mapped_ubuf {
203 u64 ubuf;
Pavel Begunkov4751f532021-04-01 15:43:55 +0100204 u64 ubuf_end;
Jens Axboeedafcce2019-01-09 09:16:05 -0700205 unsigned int nr_bvecs;
Jens Axboede293932020-09-17 16:19:16 -0600206 unsigned long acct_pages;
Pavel Begunkov41edf1a2021-04-25 14:32:23 +0100207 struct bio_vec bvec[];
Jens Axboeedafcce2019-01-09 09:16:05 -0700208};
209
Bijan Mottahedeh50238532021-01-15 17:37:45 +0000210struct io_ring_ctx;
211
Pavel Begunkov6c2450a2021-02-23 12:40:22 +0000212struct io_overflow_cqe {
213 struct io_uring_cqe cqe;
214 struct list_head list;
215};
216
Pavel Begunkova04b0ac2021-04-01 15:44:04 +0100217struct io_fixed_file {
218 /* file * with additional FFS_* flags */
219 unsigned long file_ptr;
220};
221
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +0000222struct io_rsrc_put {
223 struct list_head list;
Pavel Begunkovb60c8dc2021-04-25 14:32:18 +0100224 u64 tag;
Bijan Mottahedeh50238532021-01-15 17:37:45 +0000225 union {
226 void *rsrc;
227 struct file *file;
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +0100228 struct io_mapped_ubuf *buf;
Bijan Mottahedeh50238532021-01-15 17:37:45 +0000229 };
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +0000230};
231
Pavel Begunkovaeca2412021-04-11 01:46:37 +0100232struct io_file_table {
Pavel Begunkov042b0d82021-08-09 13:04:01 +0100233 struct io_fixed_file *files;
Jens Axboe31b51512019-01-18 22:56:34 -0700234};
235
Pavel Begunkovb895c9a2021-04-01 15:43:40 +0100236struct io_rsrc_node {
Xiaoguang Wang05589552020-03-31 14:05:18 +0800237 struct percpu_ref refs;
238 struct list_head node;
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +0000239 struct list_head rsrc_list;
Pavel Begunkovb895c9a2021-04-01 15:43:40 +0100240 struct io_rsrc_data *rsrc_data;
Jens Axboe4a38aed22020-05-14 17:21:15 -0600241 struct llist_node llist;
Pavel Begunkove2978222020-11-18 14:56:26 +0000242 bool done;
Xiaoguang Wang05589552020-03-31 14:05:18 +0800243};
244
Pavel Begunkov40ae0ff2021-04-01 15:43:44 +0100245typedef void (rsrc_put_fn)(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc);
246
Pavel Begunkovb895c9a2021-04-01 15:43:40 +0100247struct io_rsrc_data {
Jens Axboe05f3fb32019-12-09 11:22:50 -0700248 struct io_ring_ctx *ctx;
249
Pavel Begunkov2d091d62021-06-14 02:36:21 +0100250 u64 **tags;
251 unsigned int nr;
Pavel Begunkov40ae0ff2021-04-01 15:43:44 +0100252 rsrc_put_fn *do_put;
Pavel Begunkov3e942492021-04-11 01:46:34 +0100253 atomic_t refs;
Jens Axboe05f3fb32019-12-09 11:22:50 -0700254 struct completion done;
Hao Xu8bad28d2021-02-19 17:19:36 +0800255 bool quiesce;
Jens Axboe05f3fb32019-12-09 11:22:50 -0700256};
257
Jens Axboe5a2e7452020-02-23 16:23:11 -0700258struct io_buffer {
259 struct list_head list;
260 __u64 addr;
Thadeu Lima de Souza Cascardod1f82802021-05-05 09:47:06 -0300261 __u32 len;
Jens Axboe5a2e7452020-02-23 16:23:11 -0700262 __u16 bid;
263};
264
Stefano Garzarella21b55db2020-08-27 16:58:30 +0200265struct io_restriction {
266 DECLARE_BITMAP(register_op, IORING_REGISTER_LAST);
267 DECLARE_BITMAP(sqe_op, IORING_OP_LAST);
268 u8 sqe_flags_allowed;
269 u8 sqe_flags_required;
Stefano Garzarella7e84e1c2020-08-27 16:58:31 +0200270 bool registered;
Stefano Garzarella21b55db2020-08-27 16:58:30 +0200271};
272
Jens Axboe37d1e2e2021-02-17 21:03:43 -0700273enum {
274 IO_SQ_THREAD_SHOULD_STOP = 0,
275 IO_SQ_THREAD_SHOULD_PARK,
276};
277
Jens Axboe534ca6d2020-09-02 13:52:19 -0600278struct io_sq_data {
279 refcount_t refs;
Pavel Begunkov9e138a42021-03-14 20:57:12 +0000280 atomic_t park_pending;
Pavel Begunkov09a6f4e2021-03-14 20:57:10 +0000281 struct mutex lock;
Jens Axboe69fb2132020-09-14 11:16:23 -0600282
283 /* ctx's that are using this sqd */
284 struct list_head ctx_list;
Jens Axboe69fb2132020-09-14 11:16:23 -0600285
Jens Axboe534ca6d2020-09-02 13:52:19 -0600286 struct task_struct *thread;
287 struct wait_queue_head wait;
Xiaoguang Wang08369242020-11-03 14:15:59 +0800288
289 unsigned sq_thread_idle;
Jens Axboe37d1e2e2021-02-17 21:03:43 -0700290 int sq_cpu;
291 pid_t task_pid;
Jens Axboe5c2469e2021-03-11 10:17:56 -0700292 pid_t task_tgid;
Jens Axboe37d1e2e2021-02-17 21:03:43 -0700293
294 unsigned long state;
Jens Axboe37d1e2e2021-02-17 21:03:43 -0700295 struct completion exited;
Jens Axboe534ca6d2020-09-02 13:52:19 -0600296};
297
Pavel Begunkov6dd0be12021-02-10 00:03:13 +0000298#define IO_COMPL_BATCH 32
Pavel Begunkov6ff119a2021-02-10 00:03:18 +0000299#define IO_REQ_CACHE_SIZE 32
Pavel Begunkovbf019da2021-02-10 00:03:17 +0000300#define IO_REQ_ALLOC_BATCH 8
Pavel Begunkov258b29a2021-02-10 00:03:10 +0000301
Pavel Begunkova1ab7b32021-02-18 18:29:42 +0000302struct io_submit_link {
303 struct io_kiocb *head;
304 struct io_kiocb *last;
305};
306
Pavel Begunkov258b29a2021-02-10 00:03:10 +0000307struct io_submit_state {
308 struct blk_plug plug;
Pavel Begunkova1ab7b32021-02-18 18:29:42 +0000309 struct io_submit_link link;
Pavel Begunkov258b29a2021-02-10 00:03:10 +0000310
311 /*
312 * io_kiocb alloc cache
313 */
Pavel Begunkovbf019da2021-02-10 00:03:17 +0000314 void *reqs[IO_REQ_CACHE_SIZE];
Pavel Begunkov258b29a2021-02-10 00:03:10 +0000315 unsigned int free_reqs;
316
317 bool plug_started;
318
319 /*
320 * Batch completion logic
321 */
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +0100322 struct io_kiocb *compl_reqs[IO_COMPL_BATCH];
323 unsigned int compl_nr;
324 /* inline/task_work completion list, under ->uring_lock */
325 struct list_head free_list;
Pavel Begunkov258b29a2021-02-10 00:03:10 +0000326
Pavel Begunkov258b29a2021-02-10 00:03:10 +0000327 unsigned int ios_left;
328};
329
Jens Axboe2b188cc2019-01-07 10:46:33 -0700330struct io_ring_ctx {
Pavel Begunkovb52ecf82021-06-14 23:37:21 +0100331 /* const or read-mostly hot data */
Jens Axboe2b188cc2019-01-07 10:46:33 -0700332 struct {
333 struct percpu_ref refs;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700334
Pavel Begunkovb52ecf82021-06-14 23:37:21 +0100335 struct io_rings *rings;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700336 unsigned int flags;
Randy Dunlape1d85332020-02-05 20:57:10 -0800337 unsigned int compat: 1;
Randy Dunlape1d85332020-02-05 20:57:10 -0800338 unsigned int drain_next: 1;
339 unsigned int eventfd_async: 1;
Stefano Garzarella21b55db2020-08-27 16:58:30 +0200340 unsigned int restricted: 1;
Pavel Begunkovf18ee4c2021-06-14 23:37:25 +0100341 unsigned int off_timeout_used: 1;
Pavel Begunkov10c66902021-06-15 16:47:56 +0100342 unsigned int drain_active: 1;
Pavel Begunkovb52ecf82021-06-14 23:37:21 +0100343 } ____cacheline_aligned_in_smp;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700344
Pavel Begunkov7f1129d2021-06-14 23:37:22 +0100345 /* submission data */
Pavel Begunkovb52ecf82021-06-14 23:37:21 +0100346 struct {
Pavel Begunkov0499e582021-06-14 23:37:29 +0100347 struct mutex uring_lock;
348
Hristo Venev75b28af2019-08-26 17:23:46 +0000349 /*
350 * Ring buffer of indices into array of io_uring_sqe, which is
351 * mmapped by the application using the IORING_OFF_SQES offset.
352 *
353 * This indirection could e.g. be used to assign fixed
354 * io_uring_sqe entries to operations and only submit them to
355 * the queue when needed.
356 *
357 * The kernel modifies neither the indices array nor the entries
358 * array.
359 */
360 u32 *sq_array;
Pavel Begunkovc7af47c2021-06-14 23:37:20 +0100361 struct io_uring_sqe *sq_sqes;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700362 unsigned cached_sq_head;
363 unsigned sq_entries;
Jens Axboede0617e2019-04-06 21:51:27 -0600364 struct list_head defer_list;
Pavel Begunkov7f1129d2021-06-14 23:37:22 +0100365
366 /*
367 * Fixed resources fast path, should be accessed only under
368 * uring_lock, and updated through io_uring_register(2)
369 */
370 struct io_rsrc_node *rsrc_node;
371 struct io_file_table file_table;
372 unsigned nr_user_files;
373 unsigned nr_user_bufs;
374 struct io_mapped_ubuf **user_bufs;
375
376 struct io_submit_state submit_state;
Jens Axboe5262f562019-09-17 12:26:57 -0600377 struct list_head timeout_list;
Pavel Begunkovef9dd632021-08-28 19:54:38 -0600378 struct list_head ltimeout_list;
Jens Axboe1d7bb1d2019-11-06 11:31:17 -0700379 struct list_head cq_overflow_list;
Pavel Begunkov7f1129d2021-06-14 23:37:22 +0100380 struct xarray io_buffers;
381 struct xarray personalities;
382 u32 pers_next;
383 unsigned sq_thread_idle;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700384 } ____cacheline_aligned_in_smp;
385
Pavel Begunkovd0acdee2021-05-16 22:58:12 +0100386 /* IRQ completion list, under ->completion_lock */
387 struct list_head locked_free_list;
388 unsigned int locked_free_nr;
Jens Axboe3c1a2ea2021-02-11 10:48:03 -0700389
Stefan Metzmacher7c30f36a2021-03-07 11:54:28 +0100390 const struct cred *sq_creds; /* cred used for __io_sq_thread() */
Jens Axboe534ca6d2020-09-02 13:52:19 -0600391 struct io_sq_data *sq_data; /* if using sq thread polling */
392
Jens Axboe90554202020-09-03 12:12:41 -0600393 struct wait_queue_head sqo_sq_wait;
Jens Axboe69fb2132020-09-14 11:16:23 -0600394 struct list_head sqd_list;
Hristo Venev75b28af2019-08-26 17:23:46 +0000395
Pavel Begunkov5ed7a372021-06-14 23:37:27 +0100396 unsigned long check_cq_overflow;
397
Jens Axboe206aefd2019-11-07 18:27:42 -0700398 struct {
399 unsigned cached_cq_tail;
400 unsigned cq_entries;
Jens Axboe206aefd2019-11-07 18:27:42 -0700401 struct eventfd_ctx *cq_ev_fd;
Pavel Begunkov0499e582021-06-14 23:37:29 +0100402 struct wait_queue_head poll_wait;
403 struct wait_queue_head cq_wait;
404 unsigned cq_extra;
405 atomic_t cq_timeouts;
Pavel Begunkov0499e582021-06-14 23:37:29 +0100406 unsigned cq_last_tm_flush;
Jens Axboe206aefd2019-11-07 18:27:42 -0700407 } ____cacheline_aligned_in_smp;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700408
409 struct {
Jens Axboe2b188cc2019-01-07 10:46:33 -0700410 spinlock_t completion_lock;
Jens Axboee94f1412019-12-19 12:06:02 -0700411
Jens Axboe89850fc2021-08-10 15:11:51 -0600412 spinlock_t timeout_lock;
413
Jens Axboedef596e2019-01-09 08:59:42 -0700414 /*
Pavel Begunkov540e32a2020-07-13 23:37:09 +0300415 * ->iopoll_list is protected by the ctx->uring_lock for
Jens Axboedef596e2019-01-09 08:59:42 -0700416 * io_uring instances that don't use IORING_SETUP_SQPOLL.
417 * For SQPOLL, only the single threaded io_sq_thread() will
418 * manipulate the list, hence no extra locking is needed there.
419 */
Pavel Begunkov540e32a2020-07-13 23:37:09 +0300420 struct list_head iopoll_list;
Jens Axboe78076bb2019-12-04 19:56:40 -0700421 struct hlist_head *cancel_hash;
422 unsigned cancel_hash_bits;
Hao Xu915b3dd2021-06-28 05:37:30 +0800423 bool poll_multi_queue;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700424 } ____cacheline_aligned_in_smp;
Jens Axboe85faa7b2020-04-09 18:14:00 -0600425
Stefano Garzarella21b55db2020-08-27 16:58:30 +0200426 struct io_restriction restrictions;
Jens Axboe3c1a2ea2021-02-11 10:48:03 -0700427
Pavel Begunkovb13a8912021-05-16 22:58:07 +0100428 /* slow path rsrc auxilary data, used by update/register */
429 struct {
430 struct io_rsrc_node *rsrc_backup_node;
431 struct io_mapped_ubuf *dummy_ubuf;
432 struct io_rsrc_data *file_data;
433 struct io_rsrc_data *buf_data;
434
435 struct delayed_work rsrc_put_work;
436 struct llist_head rsrc_put_llist;
437 struct list_head rsrc_ref_list;
438 spinlock_t rsrc_ref_lock;
439 };
440
Jens Axboe3c1a2ea2021-02-11 10:48:03 -0700441 /* Keep this last, we don't need it for the fast path */
Pavel Begunkovb986af72021-05-16 22:58:06 +0100442 struct {
443 #if defined(CONFIG_UNIX)
444 struct socket *ring_sock;
445 #endif
446 /* hashed buffered write serialization */
447 struct io_wq_hash *hash_map;
448
449 /* Only used for accounting purposes */
450 struct user_struct *user;
451 struct mm_struct *mm_account;
452
453 /* ctx exit and cancelation */
Pavel Begunkov9011bf92021-06-30 21:54:03 +0100454 struct llist_head fallback_llist;
455 struct delayed_work fallback_work;
Pavel Begunkovb986af72021-05-16 22:58:06 +0100456 struct work_struct exit_work;
457 struct list_head tctx_list;
458 struct completion ref_comp;
Pavel Begunkove139a1e2021-10-19 23:43:46 +0100459 u32 iowq_limits[2];
460 bool iowq_limits_set;
Pavel Begunkovb986af72021-05-16 22:58:06 +0100461 };
Jens Axboe2b188cc2019-01-07 10:46:33 -0700462};
463
Stefan Metzmacher53e043b2021-03-15 12:56:56 +0100464struct io_uring_task {
465 /* submission side */
Pavel Begunkov09899b12021-06-14 02:36:22 +0100466 int cached_refs;
Stefan Metzmacher53e043b2021-03-15 12:56:56 +0100467 struct xarray xa;
468 struct wait_queue_head wait;
Stefan Metzmacheree53fb22021-03-15 12:56:57 +0100469 const struct io_ring_ctx *last;
470 struct io_wq *io_wq;
Stefan Metzmacher53e043b2021-03-15 12:56:56 +0100471 struct percpu_counter inflight;
Pavel Begunkovb303fe22021-04-11 01:46:26 +0100472 atomic_t inflight_tracked;
Stefan Metzmacher53e043b2021-03-15 12:56:56 +0100473 atomic_t in_idle;
Stefan Metzmacher53e043b2021-03-15 12:56:56 +0100474
475 spinlock_t task_lock;
476 struct io_wq_work_list task_list;
Stefan Metzmacher53e043b2021-03-15 12:56:56 +0100477 struct callback_head task_work;
Pavel Begunkov6294f362021-08-10 17:53:55 +0100478 bool task_running;
Stefan Metzmacher53e043b2021-03-15 12:56:56 +0100479};
480
Jens Axboe09bb8392019-03-13 12:39:28 -0600481/*
482 * First field must be the file pointer in all the
483 * iocb unions! See also 'struct kiocb' in <linux/fs.h>
484 */
Jens Axboe221c5eb2019-01-17 09:41:58 -0700485struct io_poll_iocb {
486 struct file *file;
Pavel Begunkov018043b2020-10-27 23:17:18 +0000487 struct wait_queue_head *head;
Jens Axboe221c5eb2019-01-17 09:41:58 -0700488 __poll_t events;
Jens Axboe392edb42019-12-09 17:52:20 -0700489 struct wait_queue_entry wait;
Jens Axboe221c5eb2019-01-17 09:41:58 -0700490};
491
Pavel Begunkov9d805892021-04-13 02:58:40 +0100492struct io_poll_update {
Pavel Begunkov018043b2020-10-27 23:17:18 +0000493 struct file *file;
Pavel Begunkov9d805892021-04-13 02:58:40 +0100494 u64 old_user_data;
495 u64 new_user_data;
496 __poll_t events;
Jens Axboeb69de282021-03-17 08:37:41 -0600497 bool update_events;
498 bool update_user_data;
Pavel Begunkov018043b2020-10-27 23:17:18 +0000499};
500
Jens Axboeb5dba592019-12-11 14:02:38 -0700501struct io_close {
502 struct file *file;
Jens Axboeb5dba592019-12-11 14:02:38 -0700503 int fd;
Pavel Begunkov7df778b2021-09-24 20:04:29 +0100504 u32 file_slot;
Jens Axboeb5dba592019-12-11 14:02:38 -0700505};
506
Jens Axboead8a48a2019-11-15 08:49:11 -0700507struct io_timeout_data {
508 struct io_kiocb *req;
509 struct hrtimer timer;
510 struct timespec64 ts;
511 enum hrtimer_mode mode;
Jens Axboe50c1df22021-08-27 17:11:06 -0600512 u32 flags;
Jens Axboead8a48a2019-11-15 08:49:11 -0700513};
514
Jens Axboe8ed8d3c2019-12-16 11:55:28 -0700515struct io_accept {
516 struct file *file;
517 struct sockaddr __user *addr;
518 int __user *addr_len;
519 int flags;
Pavel Begunkovaaa4db12021-08-25 12:25:47 +0100520 u32 file_slot;
Jens Axboe09952e32020-03-19 20:16:56 -0600521 unsigned long nofile;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -0700522};
523
524struct io_sync {
525 struct file *file;
526 loff_t len;
527 loff_t off;
528 int flags;
Jens Axboed63d1b52019-12-10 10:38:56 -0700529 int mode;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -0700530};
531
Jens Axboefbf23842019-12-17 18:45:56 -0700532struct io_cancel {
533 struct file *file;
534 u64 addr;
535};
536
Jens Axboeb29472e2019-12-17 18:50:29 -0700537struct io_timeout {
538 struct file *file;
Pavel Begunkovbfe68a22020-05-30 14:54:18 +0300539 u32 off;
540 u32 target_seq;
Pavel Begunkov135fcde2020-07-13 23:37:12 +0300541 struct list_head list;
Pavel Begunkov90cd7e42020-10-27 23:25:36 +0000542 /* head of the link, used by linked timeouts only */
543 struct io_kiocb *head;
Jens Axboe89b263f2021-08-10 15:14:18 -0600544 /* for linked completions */
545 struct io_kiocb *prev;
Jens Axboeb29472e2019-12-17 18:50:29 -0700546};
547
Pavel Begunkov0bdf7a22020-10-10 18:34:10 +0100548struct io_timeout_rem {
549 struct file *file;
550 u64 addr;
Pavel Begunkov9c8e11b2020-11-30 19:11:16 +0000551
552 /* timeout update */
553 struct timespec64 ts;
554 u32 flags;
Pavel Begunkovf1042b62021-08-28 19:54:39 -0600555 bool ltimeout;
Pavel Begunkov0bdf7a22020-10-10 18:34:10 +0100556};
557
Jens Axboe9adbd452019-12-20 08:45:55 -0700558struct io_rw {
559 /* NOTE: kiocb has the file as the first member, so don't do it here */
560 struct kiocb kiocb;
561 u64 addr;
562 u64 len;
563};
564
Jens Axboe3fbb51c2019-12-20 08:51:52 -0700565struct io_connect {
566 struct file *file;
567 struct sockaddr __user *addr;
568 int addr_len;
569};
570
Jens Axboee47293f2019-12-20 08:58:21 -0700571struct io_sr_msg {
572 struct file *file;
Jens Axboefddafac2020-01-04 20:19:44 -0700573 union {
Pavel Begunkov4af34172021-04-11 01:46:30 +0100574 struct compat_msghdr __user *umsg_compat;
575 struct user_msghdr __user *umsg;
576 void __user *buf;
Jens Axboefddafac2020-01-04 20:19:44 -0700577 };
Jens Axboee47293f2019-12-20 08:58:21 -0700578 int msg_flags;
Jens Axboebcda7ba2020-02-23 16:42:51 -0700579 int bgid;
Jens Axboefddafac2020-01-04 20:19:44 -0700580 size_t len;
Jens Axboebcda7ba2020-02-23 16:42:51 -0700581 struct io_buffer *kbuf;
Jens Axboee47293f2019-12-20 08:58:21 -0700582};
583
Jens Axboe15b71ab2019-12-11 11:20:36 -0700584struct io_open {
585 struct file *file;
586 int dfd;
Pavel Begunkovb9445592021-08-25 12:25:45 +0100587 u32 file_slot;
Jens Axboe15b71ab2019-12-11 11:20:36 -0700588 struct filename *filename;
Jens Axboec12cedf2020-01-08 17:41:21 -0700589 struct open_how how;
Jens Axboe4022e7a2020-03-19 19:23:18 -0600590 unsigned long nofile;
Jens Axboe15b71ab2019-12-11 11:20:36 -0700591};
592
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +0000593struct io_rsrc_update {
Jens Axboe05f3fb32019-12-09 11:22:50 -0700594 struct file *file;
595 u64 arg;
596 u32 nr_args;
597 u32 offset;
598};
599
Jens Axboe4840e412019-12-25 22:03:45 -0700600struct io_fadvise {
601 struct file *file;
602 u64 offset;
603 u32 len;
604 u32 advice;
605};
606
Jens Axboec1ca7572019-12-25 22:18:28 -0700607struct io_madvise {
608 struct file *file;
609 u64 addr;
610 u32 len;
611 u32 advice;
612};
613
Jens Axboe3e4827b2020-01-08 15:18:09 -0700614struct io_epoll {
615 struct file *file;
616 int epfd;
617 int op;
618 int fd;
619 struct epoll_event event;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700620};
621
Pavel Begunkov7d67af22020-02-24 11:32:45 +0300622struct io_splice {
623 struct file *file_out;
Pavel Begunkov7d67af22020-02-24 11:32:45 +0300624 loff_t off_out;
625 loff_t off_in;
626 u64 len;
Jens Axboeae6cba32022-03-29 10:59:20 -0600627 int splice_fd_in;
Pavel Begunkov7d67af22020-02-24 11:32:45 +0300628 unsigned int flags;
629};
630
Jens Axboeddf0322d2020-02-23 16:41:33 -0700631struct io_provide_buf {
632 struct file *file;
633 __u64 addr;
Pavel Begunkov38134ad2021-04-15 13:07:39 +0100634 __u32 len;
Jens Axboeddf0322d2020-02-23 16:41:33 -0700635 __u32 bgid;
636 __u16 nbufs;
637 __u16 bid;
638};
639
Bijan Mottahedeh1d9e1282020-05-22 21:31:16 -0700640struct io_statx {
641 struct file *file;
642 int dfd;
643 unsigned int mask;
644 unsigned int flags;
Bijan Mottahedehe62753e2020-05-22 21:31:18 -0700645 const char __user *filename;
Bijan Mottahedeh1d9e1282020-05-22 21:31:16 -0700646 struct statx __user *buffer;
647};
648
Jens Axboe36f4fa62020-09-05 11:14:22 -0600649struct io_shutdown {
650 struct file *file;
651 int how;
652};
653
Jens Axboe80a261f2020-09-28 14:23:58 -0600654struct io_rename {
655 struct file *file;
656 int old_dfd;
657 int new_dfd;
658 struct filename *oldpath;
659 struct filename *newpath;
660 int flags;
661};
662
Jens Axboe14a11432020-09-28 14:27:37 -0600663struct io_unlink {
664 struct file *file;
665 int dfd;
666 int flags;
667 struct filename *filename;
668};
669
Dmitry Kadasheve34a02d2021-07-08 13:34:45 +0700670struct io_mkdir {
671 struct file *file;
672 int dfd;
673 umode_t mode;
674 struct filename *filename;
675};
676
Dmitry Kadashev7a8721f2021-07-08 13:34:46 +0700677struct io_symlink {
678 struct file *file;
679 int new_dfd;
680 struct filename *oldpath;
681 struct filename *newpath;
682};
683
Dmitry Kadashevcf30da92021-07-08 13:34:47 +0700684struct io_hardlink {
685 struct file *file;
686 int old_dfd;
687 int new_dfd;
688 struct filename *oldpath;
689 struct filename *newpath;
690 int flags;
691};
692
Pavel Begunkov3ca405e2020-07-13 23:37:08 +0300693struct io_completion {
694 struct file *file;
Pavel Begunkov8c3f9cd2021-02-28 22:35:15 +0000695 u32 cflags;
Pavel Begunkov3ca405e2020-07-13 23:37:08 +0300696};
697
Jens Axboef499a022019-12-02 16:28:46 -0700698struct io_async_connect {
699 struct sockaddr_storage address;
700};
701
Jens Axboe03b12302019-12-02 18:50:25 -0700702struct io_async_msghdr {
703 struct iovec fast_iov[UIO_FASTIOV];
Pavel Begunkov257e84a2021-02-05 00:58:00 +0000704 /* points to an allocated iov, if NULL we use fast_iov instead */
705 struct iovec *free_iov;
Jens Axboe03b12302019-12-02 18:50:25 -0700706 struct sockaddr __user *uaddr;
707 struct msghdr msg;
Jens Axboeb5379162020-02-09 11:29:15 -0700708 struct sockaddr_storage addr;
Jens Axboe03b12302019-12-02 18:50:25 -0700709};
710
Jens Axboef67676d2019-12-02 11:03:47 -0700711struct io_async_rw {
712 struct iovec fast_iov[UIO_FASTIOV];
Jens Axboeff6165b2020-08-13 09:47:43 -0600713 const struct iovec *free_iovec;
714 struct iov_iter iter;
Jens Axboecd658692021-09-10 11:19:14 -0600715 struct iov_iter_state iter_state;
Jens Axboe227c0c92020-08-13 11:51:40 -0600716 size_t bytes_done;
Jens Axboebcf5a062020-05-22 09:24:42 -0600717 struct wait_page_queue wpq;
Jens Axboef67676d2019-12-02 11:03:47 -0700718};
719
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300720enum {
721 REQ_F_FIXED_FILE_BIT = IOSQE_FIXED_FILE_BIT,
722 REQ_F_IO_DRAIN_BIT = IOSQE_IO_DRAIN_BIT,
723 REQ_F_LINK_BIT = IOSQE_IO_LINK_BIT,
724 REQ_F_HARDLINK_BIT = IOSQE_IO_HARDLINK_BIT,
725 REQ_F_FORCE_ASYNC_BIT = IOSQE_ASYNC_BIT,
Jens Axboebcda7ba2020-02-23 16:42:51 -0700726 REQ_F_BUFFER_SELECT_BIT = IOSQE_BUFFER_SELECT_BIT,
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300727
Pavel Begunkovdddca222021-04-27 16:13:52 +0100728 /* first byte is taken by user flags, shift it to not overlap */
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +0100729 REQ_F_FAIL_BIT = 8,
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300730 REQ_F_INFLIGHT_BIT,
731 REQ_F_CUR_POS_BIT,
732 REQ_F_NOWAIT_BIT,
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300733 REQ_F_LINK_TIMEOUT_BIT,
Pavel Begunkov99bc4c32020-02-07 22:04:45 +0300734 REQ_F_NEED_CLEANUP_BIT,
Jens Axboed7718a92020-02-14 22:23:12 -0700735 REQ_F_POLLED_BIT,
Jens Axboebcda7ba2020-02-23 16:42:51 -0700736 REQ_F_BUFFER_SELECTED_BIT,
Pavel Begunkove342c802021-01-19 13:32:47 +0000737 REQ_F_COMPLETE_INLINE_BIT,
Jens Axboe230d50d2021-04-01 20:41:15 -0600738 REQ_F_REISSUE_BIT,
Pavel Begunkovb8e64b52021-06-17 18:14:02 +0100739 REQ_F_CREDS_BIT,
Pavel Begunkov20e60a32021-08-11 19:28:30 +0100740 REQ_F_REFCOUNT_BIT,
Pavel Begunkov4d13d1a2021-08-15 10:40:24 +0100741 REQ_F_ARM_LTIMEOUT_BIT,
Jens Axboe7b29f922021-03-12 08:30:14 -0700742 /* keep async read/write and isreg together and in order */
Pavel Begunkovb191e2d2021-08-09 13:04:03 +0100743 REQ_F_NOWAIT_READ_BIT,
744 REQ_F_NOWAIT_WRITE_BIT,
Jens Axboe7b29f922021-03-12 08:30:14 -0700745 REQ_F_ISREG_BIT,
Jens Axboe84557872020-03-03 15:28:17 -0700746
747 /* not a real bit, just to check we're not overflowing the space */
748 __REQ_F_LAST_BIT,
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300749};
750
751enum {
752 /* ctx owns file */
753 REQ_F_FIXED_FILE = BIT(REQ_F_FIXED_FILE_BIT),
754 /* drain existing IO first */
755 REQ_F_IO_DRAIN = BIT(REQ_F_IO_DRAIN_BIT),
756 /* linked sqes */
757 REQ_F_LINK = BIT(REQ_F_LINK_BIT),
758 /* doesn't sever on completion < 0 */
759 REQ_F_HARDLINK = BIT(REQ_F_HARDLINK_BIT),
760 /* IOSQE_ASYNC */
761 REQ_F_FORCE_ASYNC = BIT(REQ_F_FORCE_ASYNC_BIT),
Jens Axboebcda7ba2020-02-23 16:42:51 -0700762 /* IOSQE_BUFFER_SELECT */
763 REQ_F_BUFFER_SELECT = BIT(REQ_F_BUFFER_SELECT_BIT),
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300764
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300765 /* fail rest of links */
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +0100766 REQ_F_FAIL = BIT(REQ_F_FAIL_BIT),
Pavel Begunkovb05a1bc2021-03-04 13:59:24 +0000767 /* on inflight list, should be cancelled and waited on exit reliably */
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300768 REQ_F_INFLIGHT = BIT(REQ_F_INFLIGHT_BIT),
769 /* read/write uses file position */
770 REQ_F_CUR_POS = BIT(REQ_F_CUR_POS_BIT),
771 /* must not punt to workers */
772 REQ_F_NOWAIT = BIT(REQ_F_NOWAIT_BIT),
Pavel Begunkov900fad42020-10-19 16:39:16 +0100773 /* has or had linked timeout */
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300774 REQ_F_LINK_TIMEOUT = BIT(REQ_F_LINK_TIMEOUT_BIT),
Pavel Begunkov99bc4c32020-02-07 22:04:45 +0300775 /* needs cleanup */
776 REQ_F_NEED_CLEANUP = BIT(REQ_F_NEED_CLEANUP_BIT),
Jens Axboed7718a92020-02-14 22:23:12 -0700777 /* already went through poll handler */
778 REQ_F_POLLED = BIT(REQ_F_POLLED_BIT),
Jens Axboebcda7ba2020-02-23 16:42:51 -0700779 /* buffer already selected */
780 REQ_F_BUFFER_SELECTED = BIT(REQ_F_BUFFER_SELECTED_BIT),
Pavel Begunkove342c802021-01-19 13:32:47 +0000781 /* completion is deferred through io_comp_state */
782 REQ_F_COMPLETE_INLINE = BIT(REQ_F_COMPLETE_INLINE_BIT),
Jens Axboe230d50d2021-04-01 20:41:15 -0600783 /* caller should reissue async */
784 REQ_F_REISSUE = BIT(REQ_F_REISSUE_BIT),
Jens Axboe7b29f922021-03-12 08:30:14 -0700785 /* supports async reads */
Pavel Begunkovb191e2d2021-08-09 13:04:03 +0100786 REQ_F_NOWAIT_READ = BIT(REQ_F_NOWAIT_READ_BIT),
Jens Axboe7b29f922021-03-12 08:30:14 -0700787 /* supports async writes */
Pavel Begunkovb191e2d2021-08-09 13:04:03 +0100788 REQ_F_NOWAIT_WRITE = BIT(REQ_F_NOWAIT_WRITE_BIT),
Jens Axboe7b29f922021-03-12 08:30:14 -0700789 /* regular file */
790 REQ_F_ISREG = BIT(REQ_F_ISREG_BIT),
Pavel Begunkovb8e64b52021-06-17 18:14:02 +0100791 /* has creds assigned */
792 REQ_F_CREDS = BIT(REQ_F_CREDS_BIT),
Pavel Begunkov20e60a32021-08-11 19:28:30 +0100793 /* skip refcounting if not set */
794 REQ_F_REFCOUNT = BIT(REQ_F_REFCOUNT_BIT),
Pavel Begunkov4d13d1a2021-08-15 10:40:24 +0100795 /* there is a linked timeout that has to be armed */
796 REQ_F_ARM_LTIMEOUT = BIT(REQ_F_ARM_LTIMEOUT_BIT),
Jens Axboed7718a92020-02-14 22:23:12 -0700797};
798
799struct async_poll {
800 struct io_poll_iocb poll;
Jens Axboe807abcb2020-07-17 17:09:27 -0600801 struct io_poll_iocb *double_poll;
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300802};
803
Pavel Begunkovf237c302021-08-18 12:42:46 +0100804typedef void (*io_req_tw_func_t)(struct io_kiocb *req, bool *locked);
Pavel Begunkov5b0a6ac2021-06-30 21:54:04 +0100805
Jens Axboe7cbf1722021-02-10 00:03:20 +0000806struct io_task_work {
Pavel Begunkov5b0a6ac2021-06-30 21:54:04 +0100807 union {
808 struct io_wq_work_node node;
809 struct llist_node fallback_node;
810 };
811 io_req_tw_func_t func;
Jens Axboe7cbf1722021-02-10 00:03:20 +0000812};
813
Pavel Begunkov992da012021-06-10 16:37:37 +0100814enum {
815 IORING_RSRC_FILE = 0,
816 IORING_RSRC_BUFFER = 1,
817};
818
Jens Axboe09bb8392019-03-13 12:39:28 -0600819/*
820 * NOTE! Each of the iocb union members has the file pointer
821 * as the first entry in their struct definition. So you can
822 * access the file pointer through any of the sub-structs,
823 * or directly as just 'ki_filp' in this struct.
824 */
Jens Axboe2b188cc2019-01-07 10:46:33 -0700825struct io_kiocb {
Jens Axboe221c5eb2019-01-17 09:41:58 -0700826 union {
Jens Axboe09bb8392019-03-13 12:39:28 -0600827 struct file *file;
Jens Axboe9adbd452019-12-20 08:45:55 -0700828 struct io_rw rw;
Jens Axboe221c5eb2019-01-17 09:41:58 -0700829 struct io_poll_iocb poll;
Pavel Begunkov9d805892021-04-13 02:58:40 +0100830 struct io_poll_update poll_update;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -0700831 struct io_accept accept;
832 struct io_sync sync;
Jens Axboefbf23842019-12-17 18:45:56 -0700833 struct io_cancel cancel;
Jens Axboeb29472e2019-12-17 18:50:29 -0700834 struct io_timeout timeout;
Pavel Begunkov0bdf7a22020-10-10 18:34:10 +0100835 struct io_timeout_rem timeout_rem;
Jens Axboe3fbb51c2019-12-20 08:51:52 -0700836 struct io_connect connect;
Jens Axboee47293f2019-12-20 08:58:21 -0700837 struct io_sr_msg sr_msg;
Jens Axboe15b71ab2019-12-11 11:20:36 -0700838 struct io_open open;
Jens Axboeb5dba592019-12-11 14:02:38 -0700839 struct io_close close;
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +0000840 struct io_rsrc_update rsrc_update;
Jens Axboe4840e412019-12-25 22:03:45 -0700841 struct io_fadvise fadvise;
Jens Axboec1ca7572019-12-25 22:18:28 -0700842 struct io_madvise madvise;
Jens Axboe3e4827b2020-01-08 15:18:09 -0700843 struct io_epoll epoll;
Pavel Begunkov7d67af22020-02-24 11:32:45 +0300844 struct io_splice splice;
Jens Axboeddf0322d2020-02-23 16:41:33 -0700845 struct io_provide_buf pbuf;
Bijan Mottahedeh1d9e1282020-05-22 21:31:16 -0700846 struct io_statx statx;
Jens Axboe36f4fa62020-09-05 11:14:22 -0600847 struct io_shutdown shutdown;
Jens Axboe80a261f2020-09-28 14:23:58 -0600848 struct io_rename rename;
Jens Axboe14a11432020-09-28 14:27:37 -0600849 struct io_unlink unlink;
Dmitry Kadasheve34a02d2021-07-08 13:34:45 +0700850 struct io_mkdir mkdir;
Dmitry Kadashev7a8721f2021-07-08 13:34:46 +0700851 struct io_symlink symlink;
Dmitry Kadashevcf30da92021-07-08 13:34:47 +0700852 struct io_hardlink hardlink;
Pavel Begunkov3ca405e2020-07-13 23:37:08 +0300853 /* use only after cleaning per-op data, see io_clean_op() */
854 struct io_completion compl;
Jens Axboe221c5eb2019-01-17 09:41:58 -0700855 };
Jens Axboe2b188cc2019-01-07 10:46:33 -0700856
Jens Axboee8c2bc12020-08-15 18:44:09 -0700857 /* opcode allocated if it needs to store data for async defer */
858 void *async_data;
Jens Axboed625c6e2019-12-17 19:53:05 -0700859 u8 opcode;
Xiaoguang Wang65a65432020-06-11 23:39:36 +0800860 /* polled IO has completed */
861 u8 iopoll_completed;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700862
Bijan Mottahedeh4f4eeba2020-05-19 14:52:49 -0700863 u16 buf_index;
Pavel Begunkov9cf7c102020-07-13 23:37:15 +0300864 u32 result;
Bijan Mottahedeh4f4eeba2020-05-19 14:52:49 -0700865
Pavel Begunkov010e8e62020-07-30 18:43:45 +0300866 struct io_ring_ctx *ctx;
867 unsigned int flags;
Jens Axboeabc54d62021-02-24 13:32:30 -0700868 atomic_t refs;
Pavel Begunkov010e8e62020-07-30 18:43:45 +0300869 struct task_struct *task;
870 u64 user_data;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700871
Pavel Begunkovf2f87372020-10-27 23:25:37 +0000872 struct io_kiocb *link;
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +0000873 struct percpu_ref *fixed_rsrc_refs;
Jens Axboed7718a92020-02-14 22:23:12 -0700874
Pavel Begunkovb303fe22021-04-11 01:46:26 +0100875 /* used with ctx->iopoll_list with reads/writes */
Pavel Begunkov010e8e62020-07-30 18:43:45 +0300876 struct list_head inflight_entry;
Pavel Begunkov5b0a6ac2021-06-30 21:54:04 +0100877 struct io_task_work io_task_work;
Pavel Begunkov010e8e62020-07-30 18:43:45 +0300878 /* for polled requests, i.e. IORING_OP_POLL_ADD and async armed poll */
879 struct hlist_node hash_node;
880 struct async_poll *apoll;
881 struct io_wq_work work;
Pavel Begunkovfe7e3252021-06-24 15:09:57 +0100882 const struct cred *creds;
Pavel Begunkovc10d1f92021-06-17 18:14:01 +0100883
Pavel Begunkoveae071c2021-04-25 14:32:24 +0100884 /* store used ubuf, so we can prevent reloading */
885 struct io_mapped_ubuf *imu;
Pavel Begunkovf770fba2022-08-29 14:30:18 +0100886 /* stores selected buf, valid IFF REQ_F_BUFFER_SELECTED is set */
887 struct io_buffer *kbuf;
888 atomic_t poll_refs;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700889};
890
Pavel Begunkov13bf43f2021-03-06 11:02:12 +0000891struct io_tctx_node {
892 struct list_head ctx_node;
893 struct task_struct *task;
Pavel Begunkov13bf43f2021-03-06 11:02:12 +0000894 struct io_ring_ctx *ctx;
895};
896
Pavel Begunkov27dc8332020-07-13 23:37:14 +0300897struct io_defer_entry {
898 struct list_head list;
899 struct io_kiocb *req;
Pavel Begunkov9cf7c102020-07-13 23:37:15 +0300900 u32 seq;
Pavel Begunkov27dc8332020-07-13 23:37:14 +0300901};
902
Jens Axboed3656342019-12-18 09:50:26 -0700903struct io_op_def {
Jens Axboed3656342019-12-18 09:50:26 -0700904 /* needs req->file assigned */
905 unsigned needs_file : 1;
Jens Axboed3656342019-12-18 09:50:26 -0700906 /* hash wq insertion if file is a regular file */
907 unsigned hash_reg_file : 1;
908 /* unbound wq insertion if file is a non-regular file */
909 unsigned unbound_nonreg_file : 1;
Jens Axboe66f4af92020-01-16 15:36:52 -0700910 /* opcode is not supported by this kernel */
911 unsigned not_supported : 1;
Jens Axboe8a727582020-02-20 09:59:44 -0700912 /* set if opcode supports polled "wait" */
913 unsigned pollin : 1;
914 unsigned pollout : 1;
Jens Axboebcda7ba2020-02-23 16:42:51 -0700915 /* op supports buffer selection */
916 unsigned buffer_select : 1;
Pavel Begunkov26f05052021-02-28 22:35:18 +0000917 /* do prep async if is going to be punted */
918 unsigned needs_async_setup : 1;
Jens Axboe27926b62020-10-28 09:33:23 -0600919 /* should block plug */
920 unsigned plug : 1;
Jens Axboee8c2bc12020-08-15 18:44:09 -0700921 /* size of async data needed, if any */
922 unsigned short async_size;
Jens Axboed3656342019-12-18 09:50:26 -0700923};
924
Jens Axboe09186822020-10-13 15:01:40 -0600925static const struct io_op_def io_op_defs[] = {
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300926 [IORING_OP_NOP] = {},
927 [IORING_OP_READV] = {
Jens Axboed3656342019-12-18 09:50:26 -0700928 .needs_file = 1,
929 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -0700930 .pollin = 1,
Jens Axboe4d954c22020-02-27 07:31:19 -0700931 .buffer_select = 1,
Pavel Begunkov26f05052021-02-28 22:35:18 +0000932 .needs_async_setup = 1,
Jens Axboe27926b62020-10-28 09:33:23 -0600933 .plug = 1,
Jens Axboee8c2bc12020-08-15 18:44:09 -0700934 .async_size = sizeof(struct io_async_rw),
Jens Axboed3656342019-12-18 09:50:26 -0700935 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300936 [IORING_OP_WRITEV] = {
Jens Axboed3656342019-12-18 09:50:26 -0700937 .needs_file = 1,
938 .hash_reg_file = 1,
939 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -0700940 .pollout = 1,
Pavel Begunkov26f05052021-02-28 22:35:18 +0000941 .needs_async_setup = 1,
Jens Axboe27926b62020-10-28 09:33:23 -0600942 .plug = 1,
Jens Axboee8c2bc12020-08-15 18:44:09 -0700943 .async_size = sizeof(struct io_async_rw),
Jens Axboed3656342019-12-18 09:50:26 -0700944 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300945 [IORING_OP_FSYNC] = {
Jens Axboed3656342019-12-18 09:50:26 -0700946 .needs_file = 1,
947 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300948 [IORING_OP_READ_FIXED] = {
Jens Axboed3656342019-12-18 09:50:26 -0700949 .needs_file = 1,
950 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -0700951 .pollin = 1,
Jens Axboe27926b62020-10-28 09:33:23 -0600952 .plug = 1,
Jens Axboee8c2bc12020-08-15 18:44:09 -0700953 .async_size = sizeof(struct io_async_rw),
Jens Axboed3656342019-12-18 09:50:26 -0700954 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300955 [IORING_OP_WRITE_FIXED] = {
Jens Axboed3656342019-12-18 09:50:26 -0700956 .needs_file = 1,
957 .hash_reg_file = 1,
958 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -0700959 .pollout = 1,
Jens Axboe27926b62020-10-28 09:33:23 -0600960 .plug = 1,
Jens Axboee8c2bc12020-08-15 18:44:09 -0700961 .async_size = sizeof(struct io_async_rw),
Jens Axboed3656342019-12-18 09:50:26 -0700962 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300963 [IORING_OP_POLL_ADD] = {
Jens Axboed3656342019-12-18 09:50:26 -0700964 .needs_file = 1,
965 .unbound_nonreg_file = 1,
966 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300967 [IORING_OP_POLL_REMOVE] = {},
968 [IORING_OP_SYNC_FILE_RANGE] = {
Jens Axboed3656342019-12-18 09:50:26 -0700969 .needs_file = 1,
970 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300971 [IORING_OP_SENDMSG] = {
Jens Axboed3656342019-12-18 09:50:26 -0700972 .needs_file = 1,
973 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -0700974 .pollout = 1,
Pavel Begunkov26f05052021-02-28 22:35:18 +0000975 .needs_async_setup = 1,
Jens Axboee8c2bc12020-08-15 18:44:09 -0700976 .async_size = sizeof(struct io_async_msghdr),
Jens Axboed3656342019-12-18 09:50:26 -0700977 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300978 [IORING_OP_RECVMSG] = {
Jens Axboed3656342019-12-18 09:50:26 -0700979 .needs_file = 1,
980 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -0700981 .pollin = 1,
Jens Axboe52de1fe2020-02-27 10:15:42 -0700982 .buffer_select = 1,
Pavel Begunkov26f05052021-02-28 22:35:18 +0000983 .needs_async_setup = 1,
Jens Axboee8c2bc12020-08-15 18:44:09 -0700984 .async_size = sizeof(struct io_async_msghdr),
Jens Axboed3656342019-12-18 09:50:26 -0700985 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300986 [IORING_OP_TIMEOUT] = {
Jens Axboee8c2bc12020-08-15 18:44:09 -0700987 .async_size = sizeof(struct io_timeout_data),
Jens Axboed3656342019-12-18 09:50:26 -0700988 },
Pavel Begunkov9c8e11b2020-11-30 19:11:16 +0000989 [IORING_OP_TIMEOUT_REMOVE] = {
990 /* used by timeout updates' prep() */
Pavel Begunkov9c8e11b2020-11-30 19:11:16 +0000991 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300992 [IORING_OP_ACCEPT] = {
Jens Axboed3656342019-12-18 09:50:26 -0700993 .needs_file = 1,
994 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -0700995 .pollin = 1,
Jens Axboed3656342019-12-18 09:50:26 -0700996 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300997 [IORING_OP_ASYNC_CANCEL] = {},
998 [IORING_OP_LINK_TIMEOUT] = {
Jens Axboee8c2bc12020-08-15 18:44:09 -0700999 .async_size = sizeof(struct io_timeout_data),
Jens Axboed3656342019-12-18 09:50:26 -07001000 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +03001001 [IORING_OP_CONNECT] = {
Jens Axboed3656342019-12-18 09:50:26 -07001002 .needs_file = 1,
1003 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -07001004 .pollout = 1,
Pavel Begunkov26f05052021-02-28 22:35:18 +00001005 .needs_async_setup = 1,
Jens Axboee8c2bc12020-08-15 18:44:09 -07001006 .async_size = sizeof(struct io_async_connect),
Jens Axboed3656342019-12-18 09:50:26 -07001007 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +03001008 [IORING_OP_FALLOCATE] = {
Jens Axboed3656342019-12-18 09:50:26 -07001009 .needs_file = 1,
1010 },
Jens Axboe44526be2021-02-15 13:32:18 -07001011 [IORING_OP_OPENAT] = {},
1012 [IORING_OP_CLOSE] = {},
1013 [IORING_OP_FILES_UPDATE] = {},
1014 [IORING_OP_STATX] = {},
Pavel Begunkov0463b6c2020-01-18 21:35:38 +03001015 [IORING_OP_READ] = {
Jens Axboe3a6820f2019-12-22 15:19:35 -07001016 .needs_file = 1,
1017 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -07001018 .pollin = 1,
Jens Axboebcda7ba2020-02-23 16:42:51 -07001019 .buffer_select = 1,
Jens Axboe27926b62020-10-28 09:33:23 -06001020 .plug = 1,
Jens Axboee8c2bc12020-08-15 18:44:09 -07001021 .async_size = sizeof(struct io_async_rw),
Jens Axboe3a6820f2019-12-22 15:19:35 -07001022 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +03001023 [IORING_OP_WRITE] = {
Jens Axboe3a6820f2019-12-22 15:19:35 -07001024 .needs_file = 1,
Jens Axboe7b3188e2021-08-30 19:37:41 -06001025 .hash_reg_file = 1,
Jens Axboe3a6820f2019-12-22 15:19:35 -07001026 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -07001027 .pollout = 1,
Jens Axboe27926b62020-10-28 09:33:23 -06001028 .plug = 1,
Jens Axboee8c2bc12020-08-15 18:44:09 -07001029 .async_size = sizeof(struct io_async_rw),
Jens Axboe3a6820f2019-12-22 15:19:35 -07001030 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +03001031 [IORING_OP_FADVISE] = {
Jens Axboe4840e412019-12-25 22:03:45 -07001032 .needs_file = 1,
1033 },
Jens Axboe44526be2021-02-15 13:32:18 -07001034 [IORING_OP_MADVISE] = {},
Pavel Begunkov0463b6c2020-01-18 21:35:38 +03001035 [IORING_OP_SEND] = {
Jens Axboefddafac2020-01-04 20:19:44 -07001036 .needs_file = 1,
1037 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -07001038 .pollout = 1,
Jens Axboefddafac2020-01-04 20:19:44 -07001039 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +03001040 [IORING_OP_RECV] = {
Jens Axboefddafac2020-01-04 20:19:44 -07001041 .needs_file = 1,
1042 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -07001043 .pollin = 1,
Jens Axboebcda7ba2020-02-23 16:42:51 -07001044 .buffer_select = 1,
Jens Axboefddafac2020-01-04 20:19:44 -07001045 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +03001046 [IORING_OP_OPENAT2] = {
Jens Axboecebdb982020-01-08 17:59:24 -07001047 },
Jens Axboe3e4827b2020-01-08 15:18:09 -07001048 [IORING_OP_EPOLL_CTL] = {
1049 .unbound_nonreg_file = 1,
Jens Axboe3e4827b2020-01-08 15:18:09 -07001050 },
Pavel Begunkov7d67af22020-02-24 11:32:45 +03001051 [IORING_OP_SPLICE] = {
1052 .needs_file = 1,
1053 .hash_reg_file = 1,
1054 .unbound_nonreg_file = 1,
Jens Axboeddf0322d2020-02-23 16:41:33 -07001055 },
1056 [IORING_OP_PROVIDE_BUFFERS] = {},
Jens Axboe067524e2020-03-02 16:32:28 -07001057 [IORING_OP_REMOVE_BUFFERS] = {},
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03001058 [IORING_OP_TEE] = {
1059 .needs_file = 1,
1060 .hash_reg_file = 1,
1061 .unbound_nonreg_file = 1,
1062 },
Jens Axboe36f4fa62020-09-05 11:14:22 -06001063 [IORING_OP_SHUTDOWN] = {
1064 .needs_file = 1,
1065 },
Jens Axboe44526be2021-02-15 13:32:18 -07001066 [IORING_OP_RENAMEAT] = {},
1067 [IORING_OP_UNLINKAT] = {},
Dmitry Kadasheve34a02d2021-07-08 13:34:45 +07001068 [IORING_OP_MKDIRAT] = {},
Dmitry Kadashev7a8721f2021-07-08 13:34:46 +07001069 [IORING_OP_SYMLINKAT] = {},
Dmitry Kadashevcf30da92021-07-08 13:34:47 +07001070 [IORING_OP_LINKAT] = {},
Jens Axboed3656342019-12-18 09:50:26 -07001071};
1072
Pavel Begunkov0756a862021-08-15 10:40:25 +01001073/* requests with any of those set should undergo io_disarm_next() */
1074#define IO_DISARM_MASK (REQ_F_ARM_LTIMEOUT | REQ_F_LINK_TIMEOUT | REQ_F_FAIL)
1075
Pavel Begunkov7a612352021-03-09 00:37:59 +00001076static bool io_disarm_next(struct io_kiocb *req);
Pavel Begunkoveef51da2021-06-14 02:36:15 +01001077static void io_uring_del_tctx_node(unsigned long index);
Pavel Begunkov9936c7c2021-02-04 13:51:56 +00001078static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
1079 struct task_struct *task,
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01001080 bool cancel_all);
Pavel Begunkov78cc6872021-06-14 02:36:23 +01001081static void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd);
Pavel Begunkov1ffc5422020-12-30 21:34:15 +00001082
Pavel Begunkovb850d6d2022-08-29 14:30:13 +01001083static void io_fill_cqe_req(struct io_kiocb *req, s32 res, u32 cflags);
1084
Jackie Liuec9c02a2019-11-08 23:50:36 +08001085static void io_put_req(struct io_kiocb *req);
Pavel Begunkov91c2f692021-08-11 19:28:28 +01001086static void io_put_req_deferred(struct io_kiocb *req);
Jens Axboec7dae4b2021-02-09 19:53:37 -07001087static void io_dismantle_req(struct io_kiocb *req);
Jens Axboe94ae5e72019-11-14 19:39:52 -07001088static void io_queue_linked_timeout(struct io_kiocb *req);
Pavel Begunkovfdecb662021-04-25 14:32:20 +01001089static int __io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned type,
Pavel Begunkovc3bdad02021-04-25 14:32:22 +01001090 struct io_uring_rsrc_update2 *up,
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +01001091 unsigned nr_args);
Pavel Begunkov68fb8972021-03-19 17:22:41 +00001092static void io_clean_op(struct io_kiocb *req);
Pavel Begunkovac177052021-08-09 13:04:02 +01001093static struct file *io_file_get(struct io_ring_ctx *ctx,
Pavel Begunkov8371adf2020-10-10 18:34:08 +01001094 struct io_kiocb *req, int fd, bool fixed);
Pavel Begunkovc5eef2b2021-02-10 00:03:22 +00001095static void __io_queue_sqe(struct io_kiocb *req);
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00001096static void io_rsrc_put_work(struct work_struct *work);
Jens Axboede0617e2019-04-06 21:51:27 -06001097
Pavel Begunkov907d1df2021-01-26 23:35:10 +00001098static void io_req_task_queue(struct io_kiocb *req);
Pavel Begunkov2a2758f2021-06-17 18:14:00 +01001099static void io_submit_flush_completions(struct io_ring_ctx *ctx);
Pavel Begunkov179ae0d2021-02-28 22:35:20 +00001100static int io_req_prep_async(struct io_kiocb *req);
Jens Axboe9a56a232019-01-09 09:06:50 -07001101
Pavel Begunkovb9445592021-08-25 12:25:45 +01001102static int io_install_fixed_file(struct io_kiocb *req, struct file *file,
1103 unsigned int issue_flags, u32 slot_index);
Pavel Begunkov7df778b2021-09-24 20:04:29 +01001104static int io_close_fixed(struct io_kiocb *req, unsigned int issue_flags);
1105
Pavel Begunkovf1042b62021-08-28 19:54:39 -06001106static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer);
Pavel Begunkovb9445592021-08-25 12:25:45 +01001107
Jens Axboe2b188cc2019-01-07 10:46:33 -07001108static struct kmem_cache *req_cachep;
1109
Jens Axboe09186822020-10-13 15:01:40 -06001110static const struct file_operations io_uring_fops;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001111
1112struct sock *io_uring_get_socket(struct file *file)
1113{
1114#if defined(CONFIG_UNIX)
1115 if (file->f_op == &io_uring_fops) {
1116 struct io_ring_ctx *ctx = file->private_data;
1117
1118 return ctx->ring_sock->sk;
1119 }
1120#endif
1121 return NULL;
1122}
1123EXPORT_SYMBOL(io_uring_get_socket);
1124
Pavel Begunkovf237c302021-08-18 12:42:46 +01001125static inline void io_tw_lock(struct io_ring_ctx *ctx, bool *locked)
1126{
1127 if (!*locked) {
1128 mutex_lock(&ctx->uring_lock);
1129 *locked = true;
1130 }
1131}
1132
Pavel Begunkovf2f87372020-10-27 23:25:37 +00001133#define io_for_each_link(pos, head) \
1134 for (pos = (head); pos; pos = pos->link)
1135
Pavel Begunkov21c843d2021-08-11 19:28:27 +01001136/*
1137 * Shamelessly stolen from the mm implementation of page reference checking,
1138 * see commit f958d7b528b1 for details.
1139 */
1140#define req_ref_zero_or_close_to_overflow(req) \
1141 ((unsigned int) atomic_read(&(req->refs)) + 127u <= 127u)
1142
1143static inline bool req_ref_inc_not_zero(struct io_kiocb *req)
1144{
Pavel Begunkov20e60a32021-08-11 19:28:30 +01001145 WARN_ON_ONCE(!(req->flags & REQ_F_REFCOUNT));
Pavel Begunkov21c843d2021-08-11 19:28:27 +01001146 return atomic_inc_not_zero(&req->refs);
1147}
1148
Pavel Begunkov21c843d2021-08-11 19:28:27 +01001149static inline bool req_ref_put_and_test(struct io_kiocb *req)
1150{
Pavel Begunkov20e60a32021-08-11 19:28:30 +01001151 if (likely(!(req->flags & REQ_F_REFCOUNT)))
1152 return true;
1153
Pavel Begunkov21c843d2021-08-11 19:28:27 +01001154 WARN_ON_ONCE(req_ref_zero_or_close_to_overflow(req));
1155 return atomic_dec_and_test(&req->refs);
1156}
1157
Pavel Begunkov21c843d2021-08-11 19:28:27 +01001158static inline void req_ref_get(struct io_kiocb *req)
1159{
Pavel Begunkov20e60a32021-08-11 19:28:30 +01001160 WARN_ON_ONCE(!(req->flags & REQ_F_REFCOUNT));
Pavel Begunkov21c843d2021-08-11 19:28:27 +01001161 WARN_ON_ONCE(req_ref_zero_or_close_to_overflow(req));
1162 atomic_inc(&req->refs);
1163}
1164
Pavel Begunkov48dcd382021-08-15 10:40:18 +01001165static inline void __io_req_set_refcount(struct io_kiocb *req, int nr)
Pavel Begunkov20e60a32021-08-11 19:28:30 +01001166{
1167 if (!(req->flags & REQ_F_REFCOUNT)) {
1168 req->flags |= REQ_F_REFCOUNT;
Pavel Begunkov48dcd382021-08-15 10:40:18 +01001169 atomic_set(&req->refs, nr);
Pavel Begunkov20e60a32021-08-11 19:28:30 +01001170 }
1171}
1172
Pavel Begunkov48dcd382021-08-15 10:40:18 +01001173static inline void io_req_set_refcount(struct io_kiocb *req)
1174{
1175 __io_req_set_refcount(req, 1);
1176}
1177
Pavel Begunkovb895c9a2021-04-01 15:43:40 +01001178static inline void io_req_set_rsrc_node(struct io_kiocb *req)
Jens Axboec40f6372020-06-25 15:39:59 -06001179{
Pavel Begunkov36f72fe2020-11-18 19:57:26 +00001180 struct io_ring_ctx *ctx = req->ctx;
1181
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00001182 if (!req->fixed_rsrc_refs) {
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01001183 req->fixed_rsrc_refs = &ctx->rsrc_node->refs;
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00001184 percpu_ref_get(req->fixed_rsrc_refs);
Pavel Begunkov36f72fe2020-11-18 19:57:26 +00001185 }
1186}
1187
Pavel Begunkovf70865d2021-04-11 01:46:40 +01001188static void io_refs_resurrect(struct percpu_ref *ref, struct completion *compl)
1189{
1190 bool got = percpu_ref_tryget(ref);
1191
1192 /* already at zero, wait for ->release() */
1193 if (!got)
1194 wait_for_completion(compl);
1195 percpu_ref_resurrect(ref);
1196 if (got)
1197 percpu_ref_put(ref);
1198}
1199
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01001200static bool io_match_task(struct io_kiocb *head, struct task_struct *task,
1201 bool cancel_all)
Pavel Begunkov1c939a52021-11-26 14:38:15 +00001202 __must_hold(&req->ctx->timeout_lock)
Pavel Begunkov08d23632020-11-06 13:00:22 +00001203{
1204 struct io_kiocb *req;
1205
Pavel Begunkov68207682021-03-22 01:58:25 +00001206 if (task && head->task != task)
Pavel Begunkov08d23632020-11-06 13:00:22 +00001207 return false;
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01001208 if (cancel_all)
Pavel Begunkov08d23632020-11-06 13:00:22 +00001209 return true;
1210
1211 io_for_each_link(req, head) {
Pavel Begunkovb05a1bc2021-03-04 13:59:24 +00001212 if (req->flags & REQ_F_INFLIGHT)
Jens Axboe02a13672021-01-23 15:49:31 -07001213 return true;
Pavel Begunkov08d23632020-11-06 13:00:22 +00001214 }
1215 return false;
1216}
1217
Pavel Begunkov1c939a52021-11-26 14:38:15 +00001218static bool io_match_linked(struct io_kiocb *head)
1219{
1220 struct io_kiocb *req;
1221
1222 io_for_each_link(req, head) {
1223 if (req->flags & REQ_F_INFLIGHT)
1224 return true;
1225 }
1226 return false;
1227}
1228
1229/*
1230 * As io_match_task() but protected against racing with linked timeouts.
1231 * User must not hold timeout_lock.
1232 */
1233static bool io_match_task_safe(struct io_kiocb *head, struct task_struct *task,
1234 bool cancel_all)
1235{
1236 bool matched;
1237
1238 if (task && head->task != task)
1239 return false;
1240 if (cancel_all)
1241 return true;
1242
1243 if (head->flags & REQ_F_LINK_TIMEOUT) {
1244 struct io_ring_ctx *ctx = head->ctx;
1245
1246 /* protect against races with linked timeouts */
1247 spin_lock_irq(&ctx->timeout_lock);
1248 matched = io_match_linked(head);
1249 spin_unlock_irq(&ctx->timeout_lock);
1250 } else {
1251 matched = io_match_linked(head);
1252 }
1253 return matched;
1254}
1255
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01001256static inline void req_set_fail(struct io_kiocb *req)
Jens Axboec40f6372020-06-25 15:39:59 -06001257{
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01001258 req->flags |= REQ_F_FAIL;
Jens Axboec40f6372020-06-25 15:39:59 -06001259}
Jens Axboe4a38aed22020-05-14 17:21:15 -06001260
Hao Xua8295b92021-08-27 17:46:09 +08001261static inline void req_fail_link_node(struct io_kiocb *req, int res)
1262{
1263 req_set_fail(req);
1264 req->result = res;
1265}
1266
Jens Axboe2b188cc2019-01-07 10:46:33 -07001267static void io_ring_ctx_ref_free(struct percpu_ref *ref)
1268{
1269 struct io_ring_ctx *ctx = container_of(ref, struct io_ring_ctx, refs);
1270
Jens Axboe0f158b42020-05-14 17:18:39 -06001271 complete(&ctx->ref_comp);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001272}
1273
Pavel Begunkov8eb7e2d2020-06-29 13:13:02 +03001274static inline bool io_is_timeout_noseq(struct io_kiocb *req)
1275{
1276 return !req->timeout.off;
1277}
1278
Pavel Begunkovf56165e2021-08-09 20:18:07 +01001279static void io_fallback_req_func(struct work_struct *work)
1280{
1281 struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx,
1282 fallback_work.work);
1283 struct llist_node *node = llist_del_all(&ctx->fallback_llist);
1284 struct io_kiocb *req, *tmp;
Pavel Begunkovf237c302021-08-18 12:42:46 +01001285 bool locked = false;
Pavel Begunkovf56165e2021-08-09 20:18:07 +01001286
1287 percpu_ref_get(&ctx->refs);
1288 llist_for_each_entry_safe(req, tmp, node, io_task_work.fallback_node)
Pavel Begunkovf237c302021-08-18 12:42:46 +01001289 req->io_task_work.func(req, &locked);
Pavel Begunkov5636c002021-08-18 12:42:45 +01001290
Pavel Begunkovf237c302021-08-18 12:42:46 +01001291 if (locked) {
1292 if (ctx->submit_state.compl_nr)
1293 io_submit_flush_completions(ctx);
1294 mutex_unlock(&ctx->uring_lock);
1295 }
Pavel Begunkovf56165e2021-08-09 20:18:07 +01001296 percpu_ref_put(&ctx->refs);
Pavel Begunkovf237c302021-08-18 12:42:46 +01001297
Pavel Begunkovf56165e2021-08-09 20:18:07 +01001298}
1299
Jens Axboe2b188cc2019-01-07 10:46:33 -07001300static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
1301{
1302 struct io_ring_ctx *ctx;
Jens Axboe78076bb2019-12-04 19:56:40 -07001303 int hash_bits;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001304
1305 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
1306 if (!ctx)
1307 return NULL;
1308
Jens Axboe78076bb2019-12-04 19:56:40 -07001309 /*
1310 * Use 5 bits less than the max cq entries, that should give us around
1311 * 32 entries per hash list if totally full and uniformly spread.
1312 */
1313 hash_bits = ilog2(p->cq_entries);
1314 hash_bits -= 5;
1315 if (hash_bits <= 0)
1316 hash_bits = 1;
1317 ctx->cancel_hash_bits = hash_bits;
1318 ctx->cancel_hash = kmalloc((1U << hash_bits) * sizeof(struct hlist_head),
1319 GFP_KERNEL);
1320 if (!ctx->cancel_hash)
1321 goto err;
1322 __hash_init(ctx->cancel_hash, 1U << hash_bits);
1323
Pavel Begunkov62248432021-04-28 13:11:29 +01001324 ctx->dummy_ubuf = kzalloc(sizeof(*ctx->dummy_ubuf), GFP_KERNEL);
1325 if (!ctx->dummy_ubuf)
1326 goto err;
1327 /* set invalid range, so io_import_fixed() fails meeting it */
1328 ctx->dummy_ubuf->ubuf = -1UL;
1329
Roman Gushchin21482892019-05-07 10:01:48 -07001330 if (percpu_ref_init(&ctx->refs, io_ring_ctx_ref_free,
Jens Axboe206aefd2019-11-07 18:27:42 -07001331 PERCPU_REF_ALLOW_REINIT, GFP_KERNEL))
1332 goto err;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001333
1334 ctx->flags = p->flags;
Jens Axboe90554202020-09-03 12:12:41 -06001335 init_waitqueue_head(&ctx->sqo_sq_wait);
Jens Axboe69fb2132020-09-14 11:16:23 -06001336 INIT_LIST_HEAD(&ctx->sqd_list);
Pavel Begunkov311997b2021-06-14 23:37:28 +01001337 init_waitqueue_head(&ctx->poll_wait);
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001338 INIT_LIST_HEAD(&ctx->cq_overflow_list);
Jens Axboe0f158b42020-05-14 17:18:39 -06001339 init_completion(&ctx->ref_comp);
Jens Axboe9e15c3a2021-03-13 12:29:43 -07001340 xa_init_flags(&ctx->io_buffers, XA_FLAGS_ALLOC1);
Matthew Wilcox (Oracle)61cf9372021-03-08 14:16:16 +00001341 xa_init_flags(&ctx->personalities, XA_FLAGS_ALLOC1);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001342 mutex_init(&ctx->uring_lock);
Pavel Begunkov311997b2021-06-14 23:37:28 +01001343 init_waitqueue_head(&ctx->cq_wait);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001344 spin_lock_init(&ctx->completion_lock);
Jens Axboe89850fc2021-08-10 15:11:51 -06001345 spin_lock_init(&ctx->timeout_lock);
Pavel Begunkov540e32a2020-07-13 23:37:09 +03001346 INIT_LIST_HEAD(&ctx->iopoll_list);
Jens Axboede0617e2019-04-06 21:51:27 -06001347 INIT_LIST_HEAD(&ctx->defer_list);
Jens Axboe5262f562019-09-17 12:26:57 -06001348 INIT_LIST_HEAD(&ctx->timeout_list);
Pavel Begunkovef9dd632021-08-28 19:54:38 -06001349 INIT_LIST_HEAD(&ctx->ltimeout_list);
Bijan Mottahedehd67d2262021-01-15 17:37:46 +00001350 spin_lock_init(&ctx->rsrc_ref_lock);
1351 INIT_LIST_HEAD(&ctx->rsrc_ref_list);
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00001352 INIT_DELAYED_WORK(&ctx->rsrc_put_work, io_rsrc_put_work);
1353 init_llist_head(&ctx->rsrc_put_llist);
Pavel Begunkov13bf43f2021-03-06 11:02:12 +00001354 INIT_LIST_HEAD(&ctx->tctx_list);
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01001355 INIT_LIST_HEAD(&ctx->submit_state.free_list);
Pavel Begunkovd0acdee2021-05-16 22:58:12 +01001356 INIT_LIST_HEAD(&ctx->locked_free_list);
Pavel Begunkov9011bf92021-06-30 21:54:03 +01001357 INIT_DELAYED_WORK(&ctx->fallback_work, io_fallback_req_func);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001358 return ctx;
Jens Axboe206aefd2019-11-07 18:27:42 -07001359err:
Pavel Begunkov62248432021-04-28 13:11:29 +01001360 kfree(ctx->dummy_ubuf);
Jens Axboe78076bb2019-12-04 19:56:40 -07001361 kfree(ctx->cancel_hash);
Jens Axboe206aefd2019-11-07 18:27:42 -07001362 kfree(ctx);
1363 return NULL;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001364}
1365
Pavel Begunkov8f6ed492021-05-16 22:58:10 +01001366static void io_account_cq_overflow(struct io_ring_ctx *ctx)
1367{
1368 struct io_rings *r = ctx->rings;
1369
1370 WRITE_ONCE(r->cq_overflow, READ_ONCE(r->cq_overflow) + 1);
1371 ctx->cq_extra--;
1372}
1373
Pavel Begunkov9cf7c102020-07-13 23:37:15 +03001374static bool req_need_defer(struct io_kiocb *req, u32 seq)
Jens Axboede0617e2019-04-06 21:51:27 -06001375{
Jens Axboe2bc99302020-07-09 09:43:27 -06001376 if (unlikely(req->flags & REQ_F_IO_DRAIN)) {
1377 struct io_ring_ctx *ctx = req->ctx;
Jackie Liua197f662019-11-08 08:09:12 -07001378
Pavel Begunkov8f6ed492021-05-16 22:58:10 +01001379 return seq + READ_ONCE(ctx->cq_extra) != ctx->cached_cq_tail;
Jens Axboe2bc99302020-07-09 09:43:27 -06001380 }
Jens Axboe7adf4ea2019-10-10 21:42:58 -06001381
Bob Liu9d858b22019-11-13 18:06:25 +08001382 return false;
Jens Axboe7adf4ea2019-10-10 21:42:58 -06001383}
1384
Pavel Begunkovc97d8a02021-08-09 13:04:04 +01001385#define FFS_ASYNC_READ 0x1UL
1386#define FFS_ASYNC_WRITE 0x2UL
1387#ifdef CONFIG_64BIT
1388#define FFS_ISREG 0x4UL
1389#else
1390#define FFS_ISREG 0x0UL
1391#endif
1392#define FFS_MASK ~(FFS_ASYNC_READ|FFS_ASYNC_WRITE|FFS_ISREG)
1393
1394static inline bool io_req_ffs_set(struct io_kiocb *req)
1395{
1396 return IS_ENABLED(CONFIG_64BIT) && (req->flags & REQ_F_FIXED_FILE);
1397}
1398
Pavel Begunkovce3d5aa2021-02-01 18:59:55 +00001399static void io_req_track_inflight(struct io_kiocb *req)
1400{
Pavel Begunkovce3d5aa2021-02-01 18:59:55 +00001401 if (!(req->flags & REQ_F_INFLIGHT)) {
Pavel Begunkovce3d5aa2021-02-01 18:59:55 +00001402 req->flags |= REQ_F_INFLIGHT;
Jens Axboe3746d622022-06-23 11:06:43 -06001403 atomic_inc(&req->task->io_uring->inflight_tracked);
Pavel Begunkovce3d5aa2021-02-01 18:59:55 +00001404 }
1405}
1406
Pavel Begunkovfd08e532021-08-11 19:28:31 +01001407static struct io_kiocb *__io_prep_linked_timeout(struct io_kiocb *req)
1408{
Pavel Begunkov906c6ca2021-08-15 10:40:26 +01001409 if (WARN_ON_ONCE(!req->link))
1410 return NULL;
1411
Pavel Begunkov4d13d1a2021-08-15 10:40:24 +01001412 req->flags &= ~REQ_F_ARM_LTIMEOUT;
1413 req->flags |= REQ_F_LINK_TIMEOUT;
Pavel Begunkovfd08e532021-08-11 19:28:31 +01001414
1415 /* linked timeouts should have two refs once prep'ed */
Pavel Begunkov48dcd382021-08-15 10:40:18 +01001416 io_req_set_refcount(req);
Pavel Begunkov4d13d1a2021-08-15 10:40:24 +01001417 __io_req_set_refcount(req->link, 2);
1418 return req->link;
Pavel Begunkovfd08e532021-08-11 19:28:31 +01001419}
1420
1421static inline struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req)
1422{
Pavel Begunkov4d13d1a2021-08-15 10:40:24 +01001423 if (likely(!(req->flags & REQ_F_ARM_LTIMEOUT)))
Pavel Begunkovfd08e532021-08-11 19:28:31 +01001424 return NULL;
1425 return __io_prep_linked_timeout(req);
1426}
1427
Pavel Begunkovcbdcb432020-06-29 19:18:43 +03001428static void io_prep_async_work(struct io_kiocb *req)
Jens Axboe561fb042019-10-24 07:25:42 -06001429{
Jens Axboed3656342019-12-18 09:50:26 -07001430 const struct io_op_def *def = &io_op_defs[req->opcode];
Pavel Begunkov23329512020-10-10 18:34:06 +01001431 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe54a91f32019-09-10 09:15:04 -06001432
Pavel Begunkovb8e64b52021-06-17 18:14:02 +01001433 if (!(req->flags & REQ_F_CREDS)) {
1434 req->flags |= REQ_F_CREDS;
Pavel Begunkovc10d1f92021-06-17 18:14:01 +01001435 req->creds = get_current_cred();
Pavel Begunkovb8e64b52021-06-17 18:14:02 +01001436 }
Jens Axboe003e8dc2021-03-06 09:22:27 -07001437
Pavel Begunkove1d675d2021-03-22 01:58:29 +00001438 req->work.list.next = NULL;
1439 req->work.flags = 0;
Pavel Begunkovfeaadc42020-10-22 16:47:16 +01001440 if (req->flags & REQ_F_FORCE_ASYNC)
1441 req->work.flags |= IO_WQ_WORK_CONCURRENT;
1442
Jens Axboed3656342019-12-18 09:50:26 -07001443 if (req->flags & REQ_F_ISREG) {
Pavel Begunkov23329512020-10-10 18:34:06 +01001444 if (def->hash_reg_file || (ctx->flags & IORING_SETUP_IOPOLL))
Pavel Begunkov8766dd52020-03-14 00:31:04 +03001445 io_wq_hash_work(&req->work, file_inode(req->file));
Jens Axboe4b982bd2021-04-01 08:38:34 -06001446 } else if (!req->file || !S_ISBLK(file_inode(req->file)->i_mode)) {
Jens Axboed3656342019-12-18 09:50:26 -07001447 if (def->unbound_nonreg_file)
Jens Axboe3529d8c2019-12-19 18:24:38 -07001448 req->work.flags |= IO_WQ_WORK_UNBOUND;
Jens Axboe54a91f32019-09-10 09:15:04 -06001449 }
Jens Axboe561fb042019-10-24 07:25:42 -06001450}
1451
Pavel Begunkovcbdcb432020-06-29 19:18:43 +03001452static void io_prep_async_link(struct io_kiocb *req)
1453{
1454 struct io_kiocb *cur;
1455
Pavel Begunkov44eff402021-07-26 14:14:31 +01001456 if (req->flags & REQ_F_LINK_TIMEOUT) {
1457 struct io_ring_ctx *ctx = req->ctx;
1458
Pavel Begunkov09eb40f2021-11-23 01:45:35 +00001459 spin_lock_irq(&ctx->timeout_lock);
Pavel Begunkov44eff402021-07-26 14:14:31 +01001460 io_for_each_link(cur, req)
1461 io_prep_async_work(cur);
Pavel Begunkov09eb40f2021-11-23 01:45:35 +00001462 spin_unlock_irq(&ctx->timeout_lock);
Pavel Begunkov44eff402021-07-26 14:14:31 +01001463 } else {
1464 io_for_each_link(cur, req)
1465 io_prep_async_work(cur);
1466 }
Pavel Begunkovcbdcb432020-06-29 19:18:43 +03001467}
1468
Pavel Begunkovf237c302021-08-18 12:42:46 +01001469static void io_queue_async_work(struct io_kiocb *req, bool *locked)
Jens Axboe561fb042019-10-24 07:25:42 -06001470{
Jackie Liua197f662019-11-08 08:09:12 -07001471 struct io_ring_ctx *ctx = req->ctx;
Pavel Begunkovcbdcb432020-06-29 19:18:43 +03001472 struct io_kiocb *link = io_prep_linked_timeout(req);
Jens Axboe5aa75ed2021-02-16 12:56:50 -07001473 struct io_uring_task *tctx = req->task->io_uring;
Jens Axboe561fb042019-10-24 07:25:42 -06001474
Pavel Begunkovf237c302021-08-18 12:42:46 +01001475 /* must not take the lock, NULL it as a precaution */
1476 locked = NULL;
1477
Jens Axboe3bfe6102021-02-16 14:15:30 -07001478 BUG_ON(!tctx);
1479 BUG_ON(!tctx->io_wq);
Jens Axboe561fb042019-10-24 07:25:42 -06001480
Pavel Begunkovcbdcb432020-06-29 19:18:43 +03001481 /* init ->work of the whole link before punting */
1482 io_prep_async_link(req);
Jens Axboe991468d2021-07-23 11:53:54 -06001483
1484 /*
1485 * Not expected to happen, but if we do have a bug where this _can_
1486 * happen, catch it here and ensure the request is marked as
1487 * canceled. That will make io-wq go through the usual work cancel
1488 * procedure rather than attempt to run this request (or create a new
1489 * worker for it).
1490 */
1491 if (WARN_ON_ONCE(!same_thread_group(req->task, current)))
1492 req->work.flags |= IO_WQ_WORK_CANCEL;
1493
Pavel Begunkovd07f1e8a2021-03-22 01:45:58 +00001494 trace_io_uring_queue_async_work(ctx, io_wq_is_hashed(&req->work), req,
1495 &req->work, req->flags);
Pavel Begunkovebf93662021-03-01 18:20:47 +00001496 io_wq_enqueue(tctx->io_wq, &req->work);
Jens Axboe7271ef32020-08-10 09:55:22 -06001497 if (link)
1498 io_queue_linked_timeout(link);
Pavel Begunkovcbdcb432020-06-29 19:18:43 +03001499}
1500
Pavel Begunkov1ee41602021-03-25 18:32:42 +00001501static void io_kill_timeout(struct io_kiocb *req, int status)
Pavel Begunkov8c855882021-04-13 02:58:41 +01001502 __must_hold(&req->ctx->completion_lock)
Jens Axboe89850fc2021-08-10 15:11:51 -06001503 __must_hold(&req->ctx->timeout_lock)
Jens Axboe5262f562019-09-17 12:26:57 -06001504{
Jens Axboee8c2bc12020-08-15 18:44:09 -07001505 struct io_timeout_data *io = req->async_data;
Jens Axboe5262f562019-09-17 12:26:57 -06001506
Pavel Begunkovfd9c7bc2021-04-13 02:58:42 +01001507 if (hrtimer_try_to_cancel(&io->timer) != -1) {
Pavel Begunkov2ae2eb92021-09-09 13:56:27 +01001508 if (status)
1509 req_set_fail(req);
Pavel Begunkov01cec8c2020-07-30 18:43:50 +03001510 atomic_set(&req->ctx->cq_timeouts,
1511 atomic_read(&req->ctx->cq_timeouts) + 1);
Pavel Begunkov135fcde2020-07-13 23:37:12 +03001512 list_del_init(&req->timeout.list);
Pavel Begunkovb850d6d2022-08-29 14:30:13 +01001513 io_fill_cqe_req(req, status, 0);
Pavel Begunkov91c2f692021-08-11 19:28:28 +01001514 io_put_req_deferred(req);
Jens Axboe5262f562019-09-17 12:26:57 -06001515 }
1516}
1517
Pavel Begunkov441b8a72021-06-14 23:37:31 +01001518static void io_queue_deferred(struct io_ring_ctx *ctx)
Pavel Begunkov04518942020-05-26 20:34:05 +03001519{
Pavel Begunkov441b8a72021-06-14 23:37:31 +01001520 while (!list_empty(&ctx->defer_list)) {
Pavel Begunkov27dc8332020-07-13 23:37:14 +03001521 struct io_defer_entry *de = list_first_entry(&ctx->defer_list,
1522 struct io_defer_entry, list);
Pavel Begunkov04518942020-05-26 20:34:05 +03001523
Pavel Begunkov9cf7c102020-07-13 23:37:15 +03001524 if (req_need_defer(de->req, de->seq))
Pavel Begunkov04518942020-05-26 20:34:05 +03001525 break;
Pavel Begunkov27dc8332020-07-13 23:37:14 +03001526 list_del_init(&de->list);
Pavel Begunkov907d1df2021-01-26 23:35:10 +00001527 io_req_task_queue(de->req);
Pavel Begunkov27dc8332020-07-13 23:37:14 +03001528 kfree(de);
Pavel Begunkov441b8a72021-06-14 23:37:31 +01001529 }
Pavel Begunkov04518942020-05-26 20:34:05 +03001530}
1531
Pavel Begunkov360428f2020-05-30 14:54:17 +03001532static void io_flush_timeouts(struct io_ring_ctx *ctx)
Jens Axboe89850fc2021-08-10 15:11:51 -06001533 __must_hold(&ctx->completion_lock)
Pavel Begunkov360428f2020-05-30 14:54:17 +03001534{
Pavel Begunkov441b8a72021-06-14 23:37:31 +01001535 u32 seq = ctx->cached_cq_tail - atomic_read(&ctx->cq_timeouts);
Jens Axboeba7261a2022-04-08 11:08:58 -06001536 struct io_kiocb *req, *tmp;
Marcelo Diop-Gonzalezf0105052021-01-15 11:54:40 -05001537
Jens Axboe79ebeae2021-08-10 15:18:27 -06001538 spin_lock_irq(&ctx->timeout_lock);
Jens Axboeba7261a2022-04-08 11:08:58 -06001539 list_for_each_entry_safe(req, tmp, &ctx->timeout_list, timeout.list) {
Marcelo Diop-Gonzalezf0105052021-01-15 11:54:40 -05001540 u32 events_needed, events_got;
Pavel Begunkov360428f2020-05-30 14:54:17 +03001541
Pavel Begunkov8eb7e2d2020-06-29 13:13:02 +03001542 if (io_is_timeout_noseq(req))
Pavel Begunkov360428f2020-05-30 14:54:17 +03001543 break;
Marcelo Diop-Gonzalezf0105052021-01-15 11:54:40 -05001544
1545 /*
1546 * Since seq can easily wrap around over time, subtract
1547 * the last seq at which timeouts were flushed before comparing.
1548 * Assuming not more than 2^31-1 events have happened since,
1549 * these subtractions won't have wrapped, so we can check if
1550 * target is in [last_seq, current_seq] by comparing the two.
1551 */
1552 events_needed = req->timeout.target_seq - ctx->cq_last_tm_flush;
1553 events_got = seq - ctx->cq_last_tm_flush;
1554 if (events_got < events_needed)
Pavel Begunkov360428f2020-05-30 14:54:17 +03001555 break;
Pavel Begunkovbfe68a22020-05-30 14:54:18 +03001556
Pavel Begunkov1ee41602021-03-25 18:32:42 +00001557 io_kill_timeout(req, 0);
Pavel Begunkovf18ee4c2021-06-14 23:37:25 +01001558 }
Marcelo Diop-Gonzalezf0105052021-01-15 11:54:40 -05001559 ctx->cq_last_tm_flush = seq;
Jens Axboe79ebeae2021-08-10 15:18:27 -06001560 spin_unlock_irq(&ctx->timeout_lock);
Pavel Begunkov360428f2020-05-30 14:54:17 +03001561}
1562
Pavel Begunkov2335f6f2021-06-15 16:47:58 +01001563static void __io_commit_cqring_flush(struct io_ring_ctx *ctx)
Jens Axboede0617e2019-04-06 21:51:27 -06001564{
Pavel Begunkov2335f6f2021-06-15 16:47:58 +01001565 if (ctx->off_timeout_used)
1566 io_flush_timeouts(ctx);
1567 if (ctx->drain_active)
1568 io_queue_deferred(ctx);
1569}
1570
1571static inline void io_commit_cqring(struct io_ring_ctx *ctx)
1572{
1573 if (unlikely(ctx->off_timeout_used || ctx->drain_active))
1574 __io_commit_cqring_flush(ctx);
Pavel Begunkovec30e042021-01-19 13:32:38 +00001575 /* order cqe stores with ring update */
1576 smp_store_release(&ctx->rings->cq.tail, ctx->cached_cq_tail);
Jens Axboede0617e2019-04-06 21:51:27 -06001577}
1578
Jens Axboe90554202020-09-03 12:12:41 -06001579static inline bool io_sqring_full(struct io_ring_ctx *ctx)
1580{
1581 struct io_rings *r = ctx->rings;
1582
Pavel Begunkova566c552021-05-16 22:58:08 +01001583 return READ_ONCE(r->sq.tail) - ctx->cached_sq_head == ctx->sq_entries;
Jens Axboe90554202020-09-03 12:12:41 -06001584}
1585
Pavel Begunkov888aae22021-01-19 13:32:39 +00001586static inline unsigned int __io_cqring_events(struct io_ring_ctx *ctx)
1587{
1588 return ctx->cached_cq_tail - READ_ONCE(ctx->rings->cq.head);
1589}
1590
Pavel Begunkovd068b502021-05-16 22:58:11 +01001591static inline struct io_uring_cqe *io_get_cqe(struct io_ring_ctx *ctx)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001592{
Hristo Venev75b28af2019-08-26 17:23:46 +00001593 struct io_rings *rings = ctx->rings;
Pavel Begunkovea5ab3b2021-05-16 22:58:09 +01001594 unsigned tail, mask = ctx->cq_entries - 1;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001595
Stefan Bühler115e12e2019-04-24 23:54:18 +02001596 /*
1597 * writes to the cq entry need to come after reading head; the
1598 * control dependency is enough as we're using WRITE_ONCE to
1599 * fill the cq entry
1600 */
Pavel Begunkova566c552021-05-16 22:58:08 +01001601 if (__io_cqring_events(ctx) == ctx->cq_entries)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001602 return NULL;
1603
Pavel Begunkov888aae22021-01-19 13:32:39 +00001604 tail = ctx->cached_cq_tail++;
Pavel Begunkovea5ab3b2021-05-16 22:58:09 +01001605 return &rings->cqes[tail & mask];
Jens Axboe2b188cc2019-01-07 10:46:33 -07001606}
1607
Jens Axboef2842ab2020-01-08 11:04:00 -07001608static inline bool io_should_trigger_evfd(struct io_ring_ctx *ctx)
1609{
Pavel Begunkov44c769d2021-04-11 01:46:31 +01001610 if (likely(!ctx->cq_ev_fd))
Jens Axboef0b493e2020-02-01 21:30:11 -07001611 return false;
Stefano Garzarella7e55a192020-05-15 18:38:05 +02001612 if (READ_ONCE(ctx->rings->cq_flags) & IORING_CQ_EVENTFD_DISABLED)
1613 return false;
Pavel Begunkov44c769d2021-04-11 01:46:31 +01001614 return !ctx->eventfd_async || io_wq_current_is_worker();
Jens Axboef2842ab2020-01-08 11:04:00 -07001615}
1616
Jens Axboe2c5d7632021-08-21 07:21:19 -06001617/*
1618 * This should only get called when at least one event has been posted.
1619 * Some applications rely on the eventfd notification count only changing
1620 * IFF a new CQE has been added to the CQ ring. There's no depedency on
1621 * 1:1 relationship between how many times this function is called (and
1622 * hence the eventfd count) and number of CQEs posted to the CQ ring.
1623 */
Jens Axboeb41e9852020-02-17 09:52:41 -07001624static void io_cqring_ev_posted(struct io_ring_ctx *ctx)
Jens Axboe8c838782019-03-12 15:48:16 -06001625{
Jens Axboe5fd46172021-08-06 14:04:31 -06001626 /*
1627 * wake_up_all() may seem excessive, but io_wake_function() and
1628 * io_should_wake() handle the termination of the loop and only
1629 * wake as many waiters as we need to.
1630 */
1631 if (wq_has_sleeper(&ctx->cq_wait))
1632 wake_up_all(&ctx->cq_wait);
Jens Axboe534ca6d2020-09-02 13:52:19 -06001633 if (ctx->sq_data && waitqueue_active(&ctx->sq_data->wait))
1634 wake_up(&ctx->sq_data->wait);
Jens Axboeb41e9852020-02-17 09:52:41 -07001635 if (io_should_trigger_evfd(ctx))
Jens Axboe9b402842019-04-11 11:45:41 -06001636 eventfd_signal(ctx->cq_ev_fd, 1);
Pavel Begunkov3f008382021-10-01 10:39:33 +01001637 if (waitqueue_active(&ctx->poll_wait))
Pavel Begunkov311997b2021-06-14 23:37:28 +01001638 wake_up_interruptible(&ctx->poll_wait);
Jens Axboe8c838782019-03-12 15:48:16 -06001639}
1640
Pavel Begunkov80c18e42021-01-07 03:15:41 +00001641static void io_cqring_ev_posted_iopoll(struct io_ring_ctx *ctx)
1642{
Pavel Begunkovc57a91fb2021-09-08 20:49:17 +01001643 /* see waitqueue_active() comment */
1644 smp_mb();
1645
Pavel Begunkov80c18e42021-01-07 03:15:41 +00001646 if (ctx->flags & IORING_SETUP_SQPOLL) {
Pavel Begunkovc57a91fb2021-09-08 20:49:17 +01001647 if (waitqueue_active(&ctx->cq_wait))
Jens Axboe5fd46172021-08-06 14:04:31 -06001648 wake_up_all(&ctx->cq_wait);
Pavel Begunkov80c18e42021-01-07 03:15:41 +00001649 }
1650 if (io_should_trigger_evfd(ctx))
1651 eventfd_signal(ctx->cq_ev_fd, 1);
Pavel Begunkov3f008382021-10-01 10:39:33 +01001652 if (waitqueue_active(&ctx->poll_wait))
Pavel Begunkov311997b2021-06-14 23:37:28 +01001653 wake_up_interruptible(&ctx->poll_wait);
Pavel Begunkov80c18e42021-01-07 03:15:41 +00001654}
1655
Jens Axboec4a2ed72019-11-21 21:01:26 -07001656/* Returns true if there are no backlogged entries after the flush */
Pavel Begunkov6c2450a2021-02-23 12:40:22 +00001657static bool __io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001658{
Jens Axboeb18032b2021-01-24 16:58:56 -07001659 bool all_flushed, posted;
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001660
Pavel Begunkova566c552021-05-16 22:58:08 +01001661 if (!force && __io_cqring_events(ctx) == ctx->cq_entries)
Pavel Begunkove23de152020-12-17 00:24:37 +00001662 return false;
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001663
Jens Axboeb18032b2021-01-24 16:58:56 -07001664 posted = false;
Jens Axboe79ebeae2021-08-10 15:18:27 -06001665 spin_lock(&ctx->completion_lock);
Pavel Begunkov6c2450a2021-02-23 12:40:22 +00001666 while (!list_empty(&ctx->cq_overflow_list)) {
Pavel Begunkovd068b502021-05-16 22:58:11 +01001667 struct io_uring_cqe *cqe = io_get_cqe(ctx);
Pavel Begunkov6c2450a2021-02-23 12:40:22 +00001668 struct io_overflow_cqe *ocqe;
Jens Axboee6c8aa92020-09-28 13:10:13 -06001669
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001670 if (!cqe && !force)
1671 break;
Pavel Begunkov6c2450a2021-02-23 12:40:22 +00001672 ocqe = list_first_entry(&ctx->cq_overflow_list,
1673 struct io_overflow_cqe, list);
1674 if (cqe)
1675 memcpy(cqe, &ocqe->cqe, sizeof(*cqe));
1676 else
Pavel Begunkov8f6ed492021-05-16 22:58:10 +01001677 io_account_cq_overflow(ctx);
1678
Jens Axboeb18032b2021-01-24 16:58:56 -07001679 posted = true;
Pavel Begunkov6c2450a2021-02-23 12:40:22 +00001680 list_del(&ocqe->list);
1681 kfree(ocqe);
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001682 }
1683
Pavel Begunkov09e88402020-12-17 00:24:38 +00001684 all_flushed = list_empty(&ctx->cq_overflow_list);
1685 if (all_flushed) {
Pavel Begunkov5ed7a372021-06-14 23:37:27 +01001686 clear_bit(0, &ctx->check_cq_overflow);
Nadav Amit20c0b382021-08-07 17:13:42 -07001687 WRITE_ONCE(ctx->rings->sq_flags,
1688 ctx->rings->sq_flags & ~IORING_SQ_CQ_OVERFLOW);
Pavel Begunkov09e88402020-12-17 00:24:38 +00001689 }
Pavel Begunkov46930142020-07-30 18:43:49 +03001690
Jens Axboeb18032b2021-01-24 16:58:56 -07001691 if (posted)
1692 io_commit_cqring(ctx);
Jens Axboe79ebeae2021-08-10 15:18:27 -06001693 spin_unlock(&ctx->completion_lock);
Jens Axboeb18032b2021-01-24 16:58:56 -07001694 if (posted)
1695 io_cqring_ev_posted(ctx);
Pavel Begunkov09e88402020-12-17 00:24:38 +00001696 return all_flushed;
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001697}
1698
Pavel Begunkov90f67362021-08-09 20:18:12 +01001699static bool io_cqring_overflow_flush(struct io_ring_ctx *ctx)
Pavel Begunkov6c503152021-01-04 20:36:36 +00001700{
Jens Axboeca0a2652021-03-04 17:15:48 -07001701 bool ret = true;
1702
Pavel Begunkov5ed7a372021-06-14 23:37:27 +01001703 if (test_bit(0, &ctx->check_cq_overflow)) {
Pavel Begunkov6c503152021-01-04 20:36:36 +00001704 /* iopoll syncs against uring_lock, not completion_lock */
1705 if (ctx->flags & IORING_SETUP_IOPOLL)
1706 mutex_lock(&ctx->uring_lock);
Pavel Begunkov90f67362021-08-09 20:18:12 +01001707 ret = __io_cqring_overflow_flush(ctx, false);
Pavel Begunkov6c503152021-01-04 20:36:36 +00001708 if (ctx->flags & IORING_SETUP_IOPOLL)
1709 mutex_unlock(&ctx->uring_lock);
1710 }
Jens Axboeca0a2652021-03-04 17:15:48 -07001711
1712 return ret;
Pavel Begunkov6c503152021-01-04 20:36:36 +00001713}
1714
Pavel Begunkov6a290a12021-08-09 13:04:13 +01001715/* must to be called somewhat shortly after putting a request */
1716static inline void io_put_task(struct task_struct *task, int nr)
1717{
1718 struct io_uring_task *tctx = task->io_uring;
1719
Pavel Begunkove98e49b2021-08-18 17:01:43 +01001720 if (likely(task == current)) {
1721 tctx->cached_refs += nr;
1722 } else {
1723 percpu_counter_sub(&tctx->inflight, nr);
1724 if (unlikely(atomic_read(&tctx->in_idle)))
1725 wake_up(&tctx->wait);
1726 put_task_struct_many(task, nr);
1727 }
Pavel Begunkov6a290a12021-08-09 13:04:13 +01001728}
1729
Pavel Begunkov9a108672021-08-27 11:55:01 +01001730static void io_task_refs_refill(struct io_uring_task *tctx)
1731{
1732 unsigned int refill = -tctx->cached_refs + IO_TCTX_REFS_CACHE_NR;
1733
1734 percpu_counter_add(&tctx->inflight, refill);
1735 refcount_add(refill, &current->usage);
1736 tctx->cached_refs += refill;
1737}
1738
1739static inline void io_get_task_refs(int nr)
1740{
1741 struct io_uring_task *tctx = current->io_uring;
1742
1743 tctx->cached_refs -= nr;
1744 if (unlikely(tctx->cached_refs < 0))
1745 io_task_refs_refill(tctx);
1746}
1747
Pavel Begunkovb168b1a2022-01-09 00:53:22 +00001748static __cold void io_uring_drop_tctx_refs(struct task_struct *task)
1749{
1750 struct io_uring_task *tctx = task->io_uring;
1751 unsigned int refs = tctx->cached_refs;
1752
1753 if (refs) {
1754 tctx->cached_refs = 0;
1755 percpu_counter_sub(&tctx->inflight, refs);
1756 put_task_struct_many(task, refs);
1757 }
1758}
1759
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01001760static bool io_cqring_event_overflow(struct io_ring_ctx *ctx, u64 user_data,
Pavel Begunkov5c0ea4c2022-08-29 14:30:12 +01001761 s32 res, u32 cflags)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001762{
Pavel Begunkovcce4b8b2021-04-13 02:58:44 +01001763 struct io_overflow_cqe *ocqe;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001764
Pavel Begunkovcce4b8b2021-04-13 02:58:44 +01001765 ocqe = kmalloc(sizeof(*ocqe), GFP_ATOMIC | __GFP_ACCOUNT);
1766 if (!ocqe) {
1767 /*
1768 * If we're in ring overflow flush mode, or in task cancel mode,
1769 * or cannot allocate an overflow entry, then we need to drop it
1770 * on the floor.
1771 */
Pavel Begunkov8f6ed492021-05-16 22:58:10 +01001772 io_account_cq_overflow(ctx);
Pavel Begunkovcce4b8b2021-04-13 02:58:44 +01001773 return false;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001774 }
Pavel Begunkovcce4b8b2021-04-13 02:58:44 +01001775 if (list_empty(&ctx->cq_overflow_list)) {
Pavel Begunkov5ed7a372021-06-14 23:37:27 +01001776 set_bit(0, &ctx->check_cq_overflow);
Nadav Amit20c0b382021-08-07 17:13:42 -07001777 WRITE_ONCE(ctx->rings->sq_flags,
1778 ctx->rings->sq_flags | IORING_SQ_CQ_OVERFLOW);
1779
Pavel Begunkovcce4b8b2021-04-13 02:58:44 +01001780 }
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01001781 ocqe->cqe.user_data = user_data;
Pavel Begunkovcce4b8b2021-04-13 02:58:44 +01001782 ocqe->cqe.res = res;
1783 ocqe->cqe.flags = cflags;
1784 list_add_tail(&ocqe->list, &ctx->cq_overflow_list);
1785 return true;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001786}
1787
Pavel Begunkovb850d6d2022-08-29 14:30:13 +01001788static inline bool __io_fill_cqe(struct io_ring_ctx *ctx, u64 user_data,
1789 s32 res, u32 cflags)
Pavel Begunkov8d133262021-04-11 01:46:33 +01001790{
Jens Axboe2b188cc2019-01-07 10:46:33 -07001791 struct io_uring_cqe *cqe;
1792
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01001793 trace_io_uring_complete(ctx, user_data, res, cflags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001794
1795 /*
1796 * If we can't get a cq entry, userspace overflowed the
1797 * submission (by quite a lot). Increment the overflow count in
1798 * the ring.
1799 */
Pavel Begunkovd068b502021-05-16 22:58:11 +01001800 cqe = io_get_cqe(ctx);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001801 if (likely(cqe)) {
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01001802 WRITE_ONCE(cqe->user_data, user_data);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001803 WRITE_ONCE(cqe->res, res);
1804 WRITE_ONCE(cqe->flags, cflags);
Pavel Begunkov8d133262021-04-11 01:46:33 +01001805 return true;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001806 }
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01001807 return io_cqring_event_overflow(ctx, user_data, res, cflags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001808}
1809
Pavel Begunkovb850d6d2022-08-29 14:30:13 +01001810static noinline void io_fill_cqe_req(struct io_kiocb *req, s32 res, u32 cflags)
Jens Axboebcda7ba2020-02-23 16:42:51 -07001811{
Pavel Begunkovb850d6d2022-08-29 14:30:13 +01001812 __io_fill_cqe(req->ctx, req->user_data, res, cflags);
1813}
1814
1815static noinline bool io_fill_cqe_aux(struct io_ring_ctx *ctx, u64 user_data,
1816 s32 res, u32 cflags)
1817{
1818 ctx->cq_extra++;
1819 return __io_fill_cqe(ctx, user_data, res, cflags);
Jens Axboebcda7ba2020-02-23 16:42:51 -07001820}
1821
Pavel Begunkov5c0ea4c2022-08-29 14:30:12 +01001822static void io_req_complete_post(struct io_kiocb *req, s32 res,
1823 u32 cflags)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001824{
Jens Axboe78e19bb2019-11-06 15:21:34 -07001825 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001826
Jens Axboe79ebeae2021-08-10 15:18:27 -06001827 spin_lock(&ctx->completion_lock);
Pavel Begunkovb850d6d2022-08-29 14:30:13 +01001828 __io_fill_cqe(ctx, req->user_data, res, cflags);
Jens Axboec7dae4b2021-02-09 19:53:37 -07001829 /*
1830 * If we're the last reference to this request, add to our locked
1831 * free_list cache.
1832 */
Jens Axboede9b4cc2021-02-24 13:28:27 -07001833 if (req_ref_put_and_test(req)) {
Pavel Begunkov7a612352021-03-09 00:37:59 +00001834 if (req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) {
Pavel Begunkov0756a862021-08-15 10:40:25 +01001835 if (req->flags & IO_DISARM_MASK)
Pavel Begunkov7a612352021-03-09 00:37:59 +00001836 io_disarm_next(req);
1837 if (req->link) {
1838 io_req_task_queue(req->link);
1839 req->link = NULL;
1840 }
1841 }
Jens Axboec7dae4b2021-02-09 19:53:37 -07001842 io_dismantle_req(req);
1843 io_put_task(req->task, 1);
Pavel Begunkovbb943b82021-08-09 20:18:10 +01001844 list_add(&req->inflight_entry, &ctx->locked_free_list);
Pavel Begunkovd0acdee2021-05-16 22:58:12 +01001845 ctx->locked_free_nr++;
Pavel Begunkov180f8292021-03-14 20:57:09 +00001846 } else {
1847 if (!percpu_ref_tryget(&ctx->refs))
1848 req = NULL;
1849 }
Pavel Begunkov7a612352021-03-09 00:37:59 +00001850 io_commit_cqring(ctx);
Jens Axboe79ebeae2021-08-10 15:18:27 -06001851 spin_unlock(&ctx->completion_lock);
Pavel Begunkov7a612352021-03-09 00:37:59 +00001852
Pavel Begunkov180f8292021-03-14 20:57:09 +00001853 if (req) {
1854 io_cqring_ev_posted(ctx);
Jens Axboec7dae4b2021-02-09 19:53:37 -07001855 percpu_ref_put(&ctx->refs);
Pavel Begunkov180f8292021-03-14 20:57:09 +00001856 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07001857}
1858
Jens Axboe4e3d9ff2021-04-15 17:44:34 -06001859static inline bool io_req_needs_clean(struct io_kiocb *req)
1860{
Pavel Begunkovc8543572021-06-17 18:14:04 +01001861 return req->flags & IO_REQ_CLEAN_FLAGS;
Jens Axboe4e3d9ff2021-04-15 17:44:34 -06001862}
1863
Pavel Begunkov5c0ea4c2022-08-29 14:30:12 +01001864static inline void io_req_complete_state(struct io_kiocb *req, s32 res,
1865 u32 cflags)
Jens Axboebcda7ba2020-02-23 16:42:51 -07001866{
Jens Axboe4e3d9ff2021-04-15 17:44:34 -06001867 if (io_req_needs_clean(req))
Pavel Begunkov68fb8972021-03-19 17:22:41 +00001868 io_clean_op(req);
Pavel Begunkova38d68d2021-01-19 13:32:45 +00001869 req->result = res;
1870 req->compl.cflags = cflags;
Pavel Begunkove342c802021-01-19 13:32:47 +00001871 req->flags |= REQ_F_COMPLETE_INLINE;
Jens Axboee1e16092020-06-22 09:17:17 -06001872}
Jens Axboe2b188cc2019-01-07 10:46:33 -07001873
Pavel Begunkov889fca72021-02-10 00:03:09 +00001874static inline void __io_req_complete(struct io_kiocb *req, unsigned issue_flags,
Pavel Begunkov5c0ea4c2022-08-29 14:30:12 +01001875 s32 res, u32 cflags)
Pavel Begunkova38d68d2021-01-19 13:32:45 +00001876{
Pavel Begunkov889fca72021-02-10 00:03:09 +00001877 if (issue_flags & IO_URING_F_COMPLETE_DEFER)
1878 io_req_complete_state(req, res, cflags);
Pavel Begunkova38d68d2021-01-19 13:32:45 +00001879 else
Jens Axboec7dae4b2021-02-09 19:53:37 -07001880 io_req_complete_post(req, res, cflags);
Pavel Begunkova38d68d2021-01-19 13:32:45 +00001881}
Jens Axboebcda7ba2020-02-23 16:42:51 -07001882
Pavel Begunkov5c0ea4c2022-08-29 14:30:12 +01001883static inline void io_req_complete(struct io_kiocb *req, s32 res)
Jens Axboee1e16092020-06-22 09:17:17 -06001884{
Pavel Begunkov889fca72021-02-10 00:03:09 +00001885 __io_req_complete(req, 0, res, 0);
Jens Axboebcda7ba2020-02-23 16:42:51 -07001886}
1887
Pavel Begunkov5c0ea4c2022-08-29 14:30:12 +01001888static void io_req_complete_failed(struct io_kiocb *req, s32 res)
Pavel Begunkovf41db2732021-02-28 22:35:12 +00001889{
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01001890 req_set_fail(req);
Pavel Begunkovf41db2732021-02-28 22:35:12 +00001891 io_req_complete_post(req, res, 0);
1892}
1893
Pavel Begunkovc6d3d9c2021-08-31 14:13:10 +01001894static void io_req_complete_fail_submit(struct io_kiocb *req)
1895{
1896 /*
1897 * We don't submit, fail them all, for that replace hardlinks with
1898 * normal links. Extra REQ_F_LINK is tolerated.
1899 */
1900 req->flags &= ~REQ_F_HARDLINK;
1901 req->flags |= REQ_F_LINK;
1902 io_req_complete_failed(req, req->result);
1903}
1904
Pavel Begunkov864ea922021-08-09 13:04:08 +01001905/*
1906 * Don't initialise the fields below on every allocation, but do that in
1907 * advance and keep them valid across allocations.
1908 */
1909static void io_preinit_req(struct io_kiocb *req, struct io_ring_ctx *ctx)
1910{
1911 req->ctx = ctx;
1912 req->link = NULL;
1913 req->async_data = NULL;
1914 /* not necessary, but safer to zero */
1915 req->result = 0;
1916}
1917
Pavel Begunkovdac7a092021-03-19 17:22:39 +00001918static void io_flush_cached_locked_reqs(struct io_ring_ctx *ctx,
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01001919 struct io_submit_state *state)
Pavel Begunkovdac7a092021-03-19 17:22:39 +00001920{
Jens Axboe79ebeae2021-08-10 15:18:27 -06001921 spin_lock(&ctx->completion_lock);
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01001922 list_splice_init(&ctx->locked_free_list, &state->free_list);
Pavel Begunkovd0acdee2021-05-16 22:58:12 +01001923 ctx->locked_free_nr = 0;
Jens Axboe79ebeae2021-08-10 15:18:27 -06001924 spin_unlock(&ctx->completion_lock);
Pavel Begunkovdac7a092021-03-19 17:22:39 +00001925}
1926
Pavel Begunkovdd78f492021-03-19 17:22:35 +00001927/* Returns true IFF there are requests in the cache */
Jens Axboec7dae4b2021-02-09 19:53:37 -07001928static bool io_flush_cached_reqs(struct io_ring_ctx *ctx)
Pavel Begunkove5d1bc02021-02-10 00:03:23 +00001929{
Jens Axboec7dae4b2021-02-09 19:53:37 -07001930 struct io_submit_state *state = &ctx->submit_state;
Pavel Begunkovdd78f492021-03-19 17:22:35 +00001931 int nr;
Pavel Begunkove5d1bc02021-02-10 00:03:23 +00001932
Jens Axboec7dae4b2021-02-09 19:53:37 -07001933 /*
1934 * If we have more than a batch's worth of requests in our IRQ side
1935 * locked cache, grab the lock and move them over to our submission
1936 * side cache.
1937 */
Pavel Begunkovd0acdee2021-05-16 22:58:12 +01001938 if (READ_ONCE(ctx->locked_free_nr) > IO_COMPL_BATCH)
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01001939 io_flush_cached_locked_reqs(ctx, state);
Jens Axboec7dae4b2021-02-09 19:53:37 -07001940
Pavel Begunkovdd78f492021-03-19 17:22:35 +00001941 nr = state->free_reqs;
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01001942 while (!list_empty(&state->free_list)) {
1943 struct io_kiocb *req = list_first_entry(&state->free_list,
Pavel Begunkovbb943b82021-08-09 20:18:10 +01001944 struct io_kiocb, inflight_entry);
Pavel Begunkovdd78f492021-03-19 17:22:35 +00001945
Pavel Begunkovbb943b82021-08-09 20:18:10 +01001946 list_del(&req->inflight_entry);
Pavel Begunkovdd78f492021-03-19 17:22:35 +00001947 state->reqs[nr++] = req;
1948 if (nr == ARRAY_SIZE(state->reqs))
Pavel Begunkove5d1bc02021-02-10 00:03:23 +00001949 break;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001950 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07001951
Pavel Begunkovdd78f492021-03-19 17:22:35 +00001952 state->free_reqs = nr;
1953 return nr != 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001954}
1955
Pavel Begunkov5d5901a2021-08-11 19:28:29 +01001956/*
1957 * A request might get retired back into the request caches even before opcode
1958 * handlers and io_issue_sqe() are done with it, e.g. inline completion path.
1959 * Because of that, io_alloc_req() should be called only under ->uring_lock
1960 * and with extra caution to not get a request that is still worked on.
1961 */
Pavel Begunkov258b29a2021-02-10 00:03:10 +00001962static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx)
Pavel Begunkov5d5901a2021-08-11 19:28:29 +01001963 __must_hold(&ctx->uring_lock)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001964{
Pavel Begunkov258b29a2021-02-10 00:03:10 +00001965 struct io_submit_state *state = &ctx->submit_state;
Pavel Begunkov864ea922021-08-09 13:04:08 +01001966 gfp_t gfp = GFP_KERNEL | __GFP_NOWARN;
1967 int ret, i;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001968
Pavel Begunkovfe7e3252021-06-24 15:09:57 +01001969 BUILD_BUG_ON(ARRAY_SIZE(state->reqs) < IO_REQ_ALLOC_BATCH);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001970
Pavel Begunkov864ea922021-08-09 13:04:08 +01001971 if (likely(state->free_reqs || io_flush_cached_reqs(ctx)))
1972 goto got_req;
Jens Axboe2579f912019-01-09 09:10:43 -07001973
Pavel Begunkov864ea922021-08-09 13:04:08 +01001974 ret = kmem_cache_alloc_bulk(req_cachep, gfp, IO_REQ_ALLOC_BATCH,
1975 state->reqs);
Pavel Begunkove5d1bc02021-02-10 00:03:23 +00001976
Pavel Begunkov864ea922021-08-09 13:04:08 +01001977 /*
1978 * Bulk alloc is all-or-nothing. If we fail to get a batch,
1979 * retry single alloc to be on the safe side.
1980 */
1981 if (unlikely(ret <= 0)) {
1982 state->reqs[0] = kmem_cache_alloc(req_cachep, gfp);
1983 if (!state->reqs[0])
1984 return NULL;
1985 ret = 1;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001986 }
Pavel Begunkov864ea922021-08-09 13:04:08 +01001987
1988 for (i = 0; i < ret; i++)
1989 io_preinit_req(state->reqs[i], ctx);
1990 state->free_reqs = ret;
Pavel Begunkove5d1bc02021-02-10 00:03:23 +00001991got_req:
Pavel Begunkov291b2822020-09-30 22:57:01 +03001992 state->free_reqs--;
1993 return state->reqs[state->free_reqs];
Jens Axboe2b188cc2019-01-07 10:46:33 -07001994}
1995
Pavel Begunkove1d767f2021-03-19 17:22:43 +00001996static inline void io_put_file(struct file *file)
Pavel Begunkov8da11c12020-02-24 11:32:44 +03001997{
Pavel Begunkove1d767f2021-03-19 17:22:43 +00001998 if (file)
Pavel Begunkov8da11c12020-02-24 11:32:44 +03001999 fput(file);
2000}
2001
Pavel Begunkov4edf20f2020-10-13 09:43:59 +01002002static void io_dismantle_req(struct io_kiocb *req)
Jens Axboe2b188cc2019-01-07 10:46:33 -07002003{
Pavel Begunkov094bae42021-03-19 17:22:42 +00002004 unsigned int flags = req->flags;
Pavel Begunkov929a3af2020-02-19 00:19:09 +03002005
Pavel Begunkov3a0a6902021-04-20 12:03:31 +01002006 if (io_req_needs_clean(req))
2007 io_clean_op(req);
Pavel Begunkove1d767f2021-03-19 17:22:43 +00002008 if (!(flags & REQ_F_FIXED_FILE))
2009 io_put_file(req->file);
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00002010 if (req->fixed_rsrc_refs)
2011 percpu_ref_put(req->fixed_rsrc_refs);
Pavel Begunkov99ebe4e2021-06-26 21:40:49 +01002012 if (req->async_data) {
Pavel Begunkov094bae42021-03-19 17:22:42 +00002013 kfree(req->async_data);
Pavel Begunkov99ebe4e2021-06-26 21:40:49 +01002014 req->async_data = NULL;
2015 }
Pavel Begunkove6543a82020-06-28 12:52:30 +03002016}
Pavel Begunkov2b85edf2019-12-28 14:13:03 +03002017
Pavel Begunkov216578e2020-10-13 09:44:00 +01002018static void __io_free_req(struct io_kiocb *req)
Pavel Begunkove6543a82020-06-28 12:52:30 +03002019{
Jens Axboe51a4cc12020-08-10 10:55:56 -06002020 struct io_ring_ctx *ctx = req->ctx;
Pavel Begunkovecfc5172020-06-29 13:13:03 +03002021
Pavel Begunkov216578e2020-10-13 09:44:00 +01002022 io_dismantle_req(req);
Pavel Begunkov7c660732021-01-25 11:42:21 +00002023 io_put_task(req->task, 1);
Pavel Begunkove6543a82020-06-28 12:52:30 +03002024
Jens Axboe79ebeae2021-08-10 15:18:27 -06002025 spin_lock(&ctx->completion_lock);
Pavel Begunkovbb943b82021-08-09 20:18:10 +01002026 list_add(&req->inflight_entry, &ctx->locked_free_list);
Pavel Begunkovc34b0252021-08-09 20:18:08 +01002027 ctx->locked_free_nr++;
Jens Axboe79ebeae2021-08-10 15:18:27 -06002028 spin_unlock(&ctx->completion_lock);
Pavel Begunkovc34b0252021-08-09 20:18:08 +01002029
Pavel Begunkovecfc5172020-06-29 13:13:03 +03002030 percpu_ref_put(&ctx->refs);
Jens Axboee65ef562019-03-12 10:16:44 -06002031}
2032
Pavel Begunkovf2f87372020-10-27 23:25:37 +00002033static inline void io_remove_next_linked(struct io_kiocb *req)
2034{
2035 struct io_kiocb *nxt = req->link;
2036
2037 req->link = nxt->link;
2038 nxt->link = NULL;
2039}
2040
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00002041static bool io_kill_linked_timeout(struct io_kiocb *req)
2042 __must_hold(&req->ctx->completion_lock)
Jens Axboe89b263f2021-08-10 15:14:18 -06002043 __must_hold(&req->ctx->timeout_lock)
Jens Axboe9e645e112019-05-10 16:07:28 -06002044{
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00002045 struct io_kiocb *link = req->link;
Pavel Begunkovf2f87372020-10-27 23:25:37 +00002046
Pavel Begunkovb97e7362021-08-15 10:40:23 +01002047 if (link && link->opcode == IORING_OP_LINK_TIMEOUT) {
Pavel Begunkovc9abd7a2020-10-22 16:43:11 +01002048 struct io_timeout_data *io = link->async_data;
Pavel Begunkov7c86ffe2020-06-29 13:12:59 +03002049
Pavel Begunkovf2f87372020-10-27 23:25:37 +00002050 io_remove_next_linked(req);
Pavel Begunkov90cd7e42020-10-27 23:25:36 +00002051 link->timeout.head = NULL;
Pavel Begunkovfd9c7bc2021-04-13 02:58:42 +01002052 if (hrtimer_try_to_cancel(&io->timer) != -1) {
Pavel Begunkovef9dd632021-08-28 19:54:38 -06002053 list_del(&link->timeout.list);
Pavel Begunkovb850d6d2022-08-29 14:30:13 +01002054 io_fill_cqe_req(link, -ECANCELED, 0);
Pavel Begunkov91c2f692021-08-11 19:28:28 +01002055 io_put_req_deferred(link);
Pavel Begunkovd4729fb2021-03-22 01:58:24 +00002056 return true;
Pavel Begunkovc9abd7a2020-10-22 16:43:11 +01002057 }
2058 }
Pavel Begunkovd4729fb2021-03-22 01:58:24 +00002059 return false;
Pavel Begunkov7c86ffe2020-06-29 13:12:59 +03002060}
2061
Pavel Begunkovd148ca42020-10-18 10:17:39 +01002062static void io_fail_links(struct io_kiocb *req)
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00002063 __must_hold(&req->ctx->completion_lock)
Jens Axboe9e645e112019-05-10 16:07:28 -06002064{
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00002065 struct io_kiocb *nxt, *link = req->link;
Jens Axboe9e645e112019-05-10 16:07:28 -06002066
Pavel Begunkovf2f87372020-10-27 23:25:37 +00002067 req->link = NULL;
Pavel Begunkovf2f87372020-10-27 23:25:37 +00002068 while (link) {
Hao Xua8295b92021-08-27 17:46:09 +08002069 long res = -ECANCELED;
2070
2071 if (link->flags & REQ_F_FAIL)
2072 res = link->result;
2073
Pavel Begunkovf2f87372020-10-27 23:25:37 +00002074 nxt = link->link;
2075 link->link = NULL;
2076
Dmitrii Dolgovc826bd72019-10-15 19:02:01 +02002077 trace_io_uring_fail_link(req, link);
Pavel Begunkovb850d6d2022-08-29 14:30:13 +01002078 io_fill_cqe_req(link, res, 0);
Pavel Begunkov91c2f692021-08-11 19:28:28 +01002079 io_put_req_deferred(link);
Pavel Begunkovf2f87372020-10-27 23:25:37 +00002080 link = nxt;
Jens Axboe9e645e112019-05-10 16:07:28 -06002081 }
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00002082}
Pavel Begunkov7c86ffe2020-06-29 13:12:59 +03002083
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00002084static bool io_disarm_next(struct io_kiocb *req)
2085 __must_hold(&req->ctx->completion_lock)
2086{
2087 bool posted = false;
2088
Pavel Begunkov0756a862021-08-15 10:40:25 +01002089 if (req->flags & REQ_F_ARM_LTIMEOUT) {
2090 struct io_kiocb *link = req->link;
2091
Pavel Begunkov906c6ca2021-08-15 10:40:26 +01002092 req->flags &= ~REQ_F_ARM_LTIMEOUT;
Pavel Begunkov0756a862021-08-15 10:40:25 +01002093 if (link && link->opcode == IORING_OP_LINK_TIMEOUT) {
2094 io_remove_next_linked(req);
Pavel Begunkovb850d6d2022-08-29 14:30:13 +01002095 io_fill_cqe_req(link, -ECANCELED, 0);
Pavel Begunkov0756a862021-08-15 10:40:25 +01002096 io_put_req_deferred(link);
2097 posted = true;
2098 }
2099 } else if (req->flags & REQ_F_LINK_TIMEOUT) {
Jens Axboe89b263f2021-08-10 15:14:18 -06002100 struct io_ring_ctx *ctx = req->ctx;
2101
2102 spin_lock_irq(&ctx->timeout_lock);
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00002103 posted = io_kill_linked_timeout(req);
Jens Axboe89b263f2021-08-10 15:14:18 -06002104 spin_unlock_irq(&ctx->timeout_lock);
2105 }
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01002106 if (unlikely((req->flags & REQ_F_FAIL) &&
Pavel Begunkove4335ed2021-04-11 01:46:39 +01002107 !(req->flags & REQ_F_HARDLINK))) {
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00002108 posted |= (req->link != NULL);
2109 io_fail_links(req);
2110 }
2111 return posted;
Jens Axboe9e645e112019-05-10 16:07:28 -06002112}
2113
Pavel Begunkov3fa5e0f2020-06-30 15:20:43 +03002114static struct io_kiocb *__io_req_find_next(struct io_kiocb *req)
Jens Axboe9e645e112019-05-10 16:07:28 -06002115{
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00002116 struct io_kiocb *nxt;
Jens Axboe2665abf2019-11-05 12:40:47 -07002117
Jens Axboe9e645e112019-05-10 16:07:28 -06002118 /*
2119 * If LINK is set, we have dependent requests in this chain. If we
2120 * didn't fail this request, queue the first one up, moving any other
2121 * dependencies to the next request. In case of failure, fail the rest
2122 * of the chain.
2123 */
Pavel Begunkov0756a862021-08-15 10:40:25 +01002124 if (req->flags & IO_DISARM_MASK) {
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00002125 struct io_ring_ctx *ctx = req->ctx;
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00002126 bool posted;
Pavel Begunkovf2f87372020-10-27 23:25:37 +00002127
Jens Axboe79ebeae2021-08-10 15:18:27 -06002128 spin_lock(&ctx->completion_lock);
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00002129 posted = io_disarm_next(req);
2130 if (posted)
2131 io_commit_cqring(req->ctx);
Jens Axboe79ebeae2021-08-10 15:18:27 -06002132 spin_unlock(&ctx->completion_lock);
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00002133 if (posted)
2134 io_cqring_ev_posted(ctx);
Pavel Begunkovf2f87372020-10-27 23:25:37 +00002135 }
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00002136 nxt = req->link;
2137 req->link = NULL;
2138 return nxt;
Jens Axboe4d7dd462019-11-20 13:03:52 -07002139}
Jens Axboe2665abf2019-11-05 12:40:47 -07002140
Pavel Begunkovf2f87372020-10-27 23:25:37 +00002141static inline struct io_kiocb *io_req_find_next(struct io_kiocb *req)
Pavel Begunkov3fa5e0f2020-06-30 15:20:43 +03002142{
Pavel Begunkovcdbff982021-02-12 18:41:16 +00002143 if (likely(!(req->flags & (REQ_F_LINK|REQ_F_HARDLINK))))
Pavel Begunkov3fa5e0f2020-06-30 15:20:43 +03002144 return NULL;
2145 return __io_req_find_next(req);
2146}
2147
Pavel Begunkovf237c302021-08-18 12:42:46 +01002148static void ctx_flush_and_put(struct io_ring_ctx *ctx, bool *locked)
Pavel Begunkov2c323952021-02-28 22:04:53 +00002149{
2150 if (!ctx)
2151 return;
Pavel Begunkovf237c302021-08-18 12:42:46 +01002152 if (*locked) {
Hao Xu99c8bc52021-08-21 06:19:54 +08002153 if (ctx->submit_state.compl_nr)
2154 io_submit_flush_completions(ctx);
Pavel Begunkov2c323952021-02-28 22:04:53 +00002155 mutex_unlock(&ctx->uring_lock);
Pavel Begunkovf237c302021-08-18 12:42:46 +01002156 *locked = false;
Pavel Begunkov2c323952021-02-28 22:04:53 +00002157 }
2158 percpu_ref_put(&ctx->refs);
2159}
2160
Jens Axboe7cbf1722021-02-10 00:03:20 +00002161static void tctx_task_work(struct callback_head *cb)
2162{
Pavel Begunkovf237c302021-08-18 12:42:46 +01002163 bool locked = false;
Pavel Begunkovebd0df22021-06-17 18:14:07 +01002164 struct io_ring_ctx *ctx = NULL;
Pavel Begunkov3f184072021-06-17 18:14:06 +01002165 struct io_uring_task *tctx = container_of(cb, struct io_uring_task,
2166 task_work);
Jens Axboe7cbf1722021-02-10 00:03:20 +00002167
Pavel Begunkov16f72072021-06-17 18:14:09 +01002168 while (1) {
Pavel Begunkov3f184072021-06-17 18:14:06 +01002169 struct io_wq_work_node *node;
2170
Pavel Begunkov8d4ad412021-09-02 00:38:23 +01002171 if (!tctx->task_list.first && locked && ctx->submit_state.compl_nr)
2172 io_submit_flush_completions(ctx);
2173
Pavel Begunkov3f184072021-06-17 18:14:06 +01002174 spin_lock_irq(&tctx->task_lock);
Pavel Begunkovc6538be2021-06-17 18:14:08 +01002175 node = tctx->task_list.first;
Pavel Begunkov3f184072021-06-17 18:14:06 +01002176 INIT_WQ_LIST(&tctx->task_list);
Pavel Begunkov6294f362021-08-10 17:53:55 +01002177 if (!node)
2178 tctx->task_running = false;
Pavel Begunkov3f184072021-06-17 18:14:06 +01002179 spin_unlock_irq(&tctx->task_lock);
Pavel Begunkov6294f362021-08-10 17:53:55 +01002180 if (!node)
2181 break;
Pavel Begunkov3f184072021-06-17 18:14:06 +01002182
Pavel Begunkov6294f362021-08-10 17:53:55 +01002183 do {
Pavel Begunkov3f184072021-06-17 18:14:06 +01002184 struct io_wq_work_node *next = node->next;
2185 struct io_kiocb *req = container_of(node, struct io_kiocb,
2186 io_task_work.node);
2187
2188 if (req->ctx != ctx) {
Pavel Begunkovf237c302021-08-18 12:42:46 +01002189 ctx_flush_and_put(ctx, &locked);
Pavel Begunkov3f184072021-06-17 18:14:06 +01002190 ctx = req->ctx;
Pavel Begunkov126180b2021-08-18 12:42:47 +01002191 /* if not contended, grab and improve batching */
2192 locked = mutex_trylock(&ctx->uring_lock);
Pavel Begunkov3f184072021-06-17 18:14:06 +01002193 percpu_ref_get(&ctx->refs);
2194 }
Pavel Begunkovf237c302021-08-18 12:42:46 +01002195 req->io_task_work.func(req, &locked);
Pavel Begunkov3f184072021-06-17 18:14:06 +01002196 node = next;
Pavel Begunkov6294f362021-08-10 17:53:55 +01002197 } while (node);
2198
Jens Axboe7cbf1722021-02-10 00:03:20 +00002199 cond_resched();
Pavel Begunkov3f184072021-06-17 18:14:06 +01002200 }
Pavel Begunkovebd0df22021-06-17 18:14:07 +01002201
Pavel Begunkovf237c302021-08-18 12:42:46 +01002202 ctx_flush_and_put(ctx, &locked);
Pavel Begunkovb168b1a2022-01-09 00:53:22 +00002203
2204 /* relaxed read is enough as only the task itself sets ->in_idle */
2205 if (unlikely(atomic_read(&tctx->in_idle)))
2206 io_uring_drop_tctx_refs(current);
Jens Axboe7cbf1722021-02-10 00:03:20 +00002207}
2208
Pavel Begunkove09ee512021-07-01 13:26:05 +01002209static void io_req_task_work_add(struct io_kiocb *req)
Jens Axboe7cbf1722021-02-10 00:03:20 +00002210{
Pavel Begunkovc15b79d2021-03-19 17:22:44 +00002211 struct task_struct *tsk = req->task;
Jens Axboe7cbf1722021-02-10 00:03:20 +00002212 struct io_uring_task *tctx = tsk->io_uring;
Pavel Begunkovc15b79d2021-03-19 17:22:44 +00002213 enum task_work_notify_mode notify;
Pavel Begunkove09ee512021-07-01 13:26:05 +01002214 struct io_wq_work_node *node;
Jens Axboe0b81e802021-02-16 10:33:53 -07002215 unsigned long flags;
Pavel Begunkov6294f362021-08-10 17:53:55 +01002216 bool running;
Jens Axboe7cbf1722021-02-10 00:03:20 +00002217
2218 WARN_ON_ONCE(!tctx);
2219
Jens Axboe0b81e802021-02-16 10:33:53 -07002220 spin_lock_irqsave(&tctx->task_lock, flags);
Jens Axboe7cbf1722021-02-10 00:03:20 +00002221 wq_list_add_tail(&req->io_task_work.node, &tctx->task_list);
Pavel Begunkov6294f362021-08-10 17:53:55 +01002222 running = tctx->task_running;
2223 if (!running)
2224 tctx->task_running = true;
Jens Axboe0b81e802021-02-16 10:33:53 -07002225 spin_unlock_irqrestore(&tctx->task_lock, flags);
Jens Axboe7cbf1722021-02-10 00:03:20 +00002226
2227 /* task_work already pending, we're done */
Pavel Begunkov6294f362021-08-10 17:53:55 +01002228 if (running)
Pavel Begunkove09ee512021-07-01 13:26:05 +01002229 return;
Jens Axboe7cbf1722021-02-10 00:03:20 +00002230
Pavel Begunkovc15b79d2021-03-19 17:22:44 +00002231 /*
2232 * SQPOLL kernel thread doesn't need notification, just a wakeup. For
2233 * all other cases, use TWA_SIGNAL unconditionally to ensure we're
2234 * processing task_work. There's no reliable way to tell if TWA_RESUME
2235 * will do the job.
2236 */
2237 notify = (req->ctx->flags & IORING_SETUP_SQPOLL) ? TWA_NONE : TWA_SIGNAL;
Pavel Begunkovc15b79d2021-03-19 17:22:44 +00002238 if (!task_work_add(tsk, &tctx->task_work, notify)) {
2239 wake_up_process(tsk);
Pavel Begunkove09ee512021-07-01 13:26:05 +01002240 return;
Pavel Begunkovc15b79d2021-03-19 17:22:44 +00002241 }
Pavel Begunkov2215bed2021-08-09 13:04:06 +01002242
Pavel Begunkove09ee512021-07-01 13:26:05 +01002243 spin_lock_irqsave(&tctx->task_lock, flags);
Pavel Begunkov6294f362021-08-10 17:53:55 +01002244 tctx->task_running = false;
Pavel Begunkove09ee512021-07-01 13:26:05 +01002245 node = tctx->task_list.first;
2246 INIT_WQ_LIST(&tctx->task_list);
2247 spin_unlock_irqrestore(&tctx->task_lock, flags);
Jens Axboe7cbf1722021-02-10 00:03:20 +00002248
Pavel Begunkove09ee512021-07-01 13:26:05 +01002249 while (node) {
2250 req = container_of(node, struct io_kiocb, io_task_work.node);
2251 node = node->next;
2252 if (llist_add(&req->io_task_work.fallback_node,
2253 &req->ctx->fallback_llist))
2254 schedule_delayed_work(&req->ctx->fallback_work, 1);
2255 }
Pavel Begunkoveab30c42021-01-19 13:32:42 +00002256}
2257
Pavel Begunkovf237c302021-08-18 12:42:46 +01002258static void io_req_task_cancel(struct io_kiocb *req, bool *locked)
Jens Axboec40f6372020-06-25 15:39:59 -06002259{
Jens Axboe87ceb6a2020-09-14 08:20:12 -06002260 struct io_ring_ctx *ctx = req->ctx;
Jens Axboec40f6372020-06-25 15:39:59 -06002261
Pavel Begunkovb18a1a42021-08-25 20:51:39 +01002262 /* not needed for normal modes, but SQPOLL depends on it */
Pavel Begunkovf237c302021-08-18 12:42:46 +01002263 io_tw_lock(ctx, locked);
Pavel Begunkov25935532021-03-19 17:22:40 +00002264 io_req_complete_failed(req, req->result);
Jens Axboec40f6372020-06-25 15:39:59 -06002265}
2266
Pavel Begunkovf237c302021-08-18 12:42:46 +01002267static void io_req_task_submit(struct io_kiocb *req, bool *locked)
Jens Axboec40f6372020-06-25 15:39:59 -06002268{
2269 struct io_ring_ctx *ctx = req->ctx;
2270
Pavel Begunkovf237c302021-08-18 12:42:46 +01002271 io_tw_lock(ctx, locked);
Jens Axboe316319e2021-08-19 09:41:42 -06002272 /* req->task == current here, checking PF_EXITING is safe */
Pavel Begunkovaf066f32021-08-09 13:04:19 +01002273 if (likely(!(req->task->flags & PF_EXITING)))
Pavel Begunkovc5eef2b2021-02-10 00:03:22 +00002274 __io_queue_sqe(req);
Pavel Begunkov81b6d052021-01-04 20:36:35 +00002275 else
Pavel Begunkov25935532021-03-19 17:22:40 +00002276 io_req_complete_failed(req, -EFAULT);
Jens Axboe9e645e112019-05-10 16:07:28 -06002277}
2278
Pavel Begunkova3df76982021-02-18 22:32:52 +00002279static void io_req_task_queue_fail(struct io_kiocb *req, int ret)
2280{
Pavel Begunkova3df76982021-02-18 22:32:52 +00002281 req->result = ret;
Pavel Begunkov5b0a6ac2021-06-30 21:54:04 +01002282 req->io_task_work.func = io_req_task_cancel;
Pavel Begunkove09ee512021-07-01 13:26:05 +01002283 io_req_task_work_add(req);
Pavel Begunkova3df76982021-02-18 22:32:52 +00002284}
2285
Pavel Begunkov2c4b8eb2021-02-28 22:35:10 +00002286static void io_req_task_queue(struct io_kiocb *req)
2287{
Pavel Begunkov5b0a6ac2021-06-30 21:54:04 +01002288 req->io_task_work.func = io_req_task_submit;
Pavel Begunkove09ee512021-07-01 13:26:05 +01002289 io_req_task_work_add(req);
Pavel Begunkov2c4b8eb2021-02-28 22:35:10 +00002290}
2291
Jens Axboe773af692021-07-27 10:25:55 -06002292static void io_req_task_queue_reissue(struct io_kiocb *req)
2293{
2294 req->io_task_work.func = io_queue_async_work;
2295 io_req_task_work_add(req);
2296}
2297
Pavel Begunkovf2f87372020-10-27 23:25:37 +00002298static inline void io_queue_next(struct io_kiocb *req)
Jackie Liuc69f8db2019-11-09 11:00:08 +08002299{
Pavel Begunkov9b5f7bd2020-06-29 13:13:00 +03002300 struct io_kiocb *nxt = io_req_find_next(req);
Pavel Begunkov944e58b2019-11-21 23:21:01 +03002301
Pavel Begunkov906a8c32020-06-27 14:04:55 +03002302 if (nxt)
2303 io_req_task_queue(nxt);
Jackie Liuc69f8db2019-11-09 11:00:08 +08002304}
2305
Jens Axboe9e645e112019-05-10 16:07:28 -06002306static void io_free_req(struct io_kiocb *req)
2307{
Pavel Begunkovc3524382020-06-28 12:52:32 +03002308 io_queue_next(req);
Jens Axboe9e645e112019-05-10 16:07:28 -06002309 __io_free_req(req);
Jens Axboee65ef562019-03-12 10:16:44 -06002310}
2311
Pavel Begunkovf237c302021-08-18 12:42:46 +01002312static void io_free_req_work(struct io_kiocb *req, bool *locked)
2313{
2314 io_free_req(req);
2315}
2316
Pavel Begunkov2d6500d2020-06-28 12:52:33 +03002317struct req_batch {
Pavel Begunkov5af1d132020-07-18 11:32:52 +03002318 struct task_struct *task;
2319 int task_refs;
Jens Axboe1b4c3512021-02-10 00:03:19 +00002320 int ctx_refs;
Pavel Begunkov2d6500d2020-06-28 12:52:33 +03002321};
2322
Pavel Begunkov5af1d132020-07-18 11:32:52 +03002323static inline void io_init_req_batch(struct req_batch *rb)
Pavel Begunkov7a743e22020-03-03 21:33:13 +03002324{
Pavel Begunkov5af1d132020-07-18 11:32:52 +03002325 rb->task_refs = 0;
Pavel Begunkov9ae72462021-02-10 00:03:16 +00002326 rb->ctx_refs = 0;
Pavel Begunkov5af1d132020-07-18 11:32:52 +03002327 rb->task = NULL;
2328}
Pavel Begunkov8766dd52020-03-14 00:31:04 +03002329
Pavel Begunkov2d6500d2020-06-28 12:52:33 +03002330static void io_req_free_batch_finish(struct io_ring_ctx *ctx,
2331 struct req_batch *rb)
2332{
Pavel Begunkov9ae72462021-02-10 00:03:16 +00002333 if (rb->ctx_refs)
2334 percpu_ref_put_many(&ctx->refs, rb->ctx_refs);
Pavel Begunkove98e49b2021-08-18 17:01:43 +01002335 if (rb->task)
Pavel Begunkove9dbe222021-08-09 13:04:20 +01002336 io_put_task(rb->task, rb->task_refs);
Pavel Begunkov2d6500d2020-06-28 12:52:33 +03002337}
2338
Pavel Begunkov6ff119a2021-02-10 00:03:18 +00002339static void io_req_free_batch(struct req_batch *rb, struct io_kiocb *req,
2340 struct io_submit_state *state)
Pavel Begunkov2d6500d2020-06-28 12:52:33 +03002341{
Pavel Begunkovf2f87372020-10-27 23:25:37 +00002342 io_queue_next(req);
Pavel Begunkov96670652021-03-19 17:22:32 +00002343 io_dismantle_req(req);
Pavel Begunkov2d6500d2020-06-28 12:52:33 +03002344
Jens Axboee3bc8e92020-09-24 08:45:57 -06002345 if (req->task != rb->task) {
Pavel Begunkov7c660732021-01-25 11:42:21 +00002346 if (rb->task)
2347 io_put_task(rb->task, rb->task_refs);
Jens Axboee3bc8e92020-09-24 08:45:57 -06002348 rb->task = req->task;
2349 rb->task_refs = 0;
Pavel Begunkov5af1d132020-07-18 11:32:52 +03002350 }
Jens Axboee3bc8e92020-09-24 08:45:57 -06002351 rb->task_refs++;
Pavel Begunkov9ae72462021-02-10 00:03:16 +00002352 rb->ctx_refs++;
Pavel Begunkov5af1d132020-07-18 11:32:52 +03002353
Pavel Begunkovbd759042021-02-12 03:23:50 +00002354 if (state->free_reqs != ARRAY_SIZE(state->reqs))
Pavel Begunkov6ff119a2021-02-10 00:03:18 +00002355 state->reqs[state->free_reqs++] = req;
Pavel Begunkovbd759042021-02-12 03:23:50 +00002356 else
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01002357 list_add(&req->inflight_entry, &state->free_list);
Pavel Begunkov7a743e22020-03-03 21:33:13 +03002358}
2359
Pavel Begunkov2a2758f2021-06-17 18:14:00 +01002360static void io_submit_flush_completions(struct io_ring_ctx *ctx)
Jens Axboea141dd82021-08-12 12:48:34 -06002361 __must_hold(&ctx->uring_lock)
Pavel Begunkov905c1722021-02-10 00:03:14 +00002362{
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01002363 struct io_submit_state *state = &ctx->submit_state;
2364 int i, nr = state->compl_nr;
Pavel Begunkov905c1722021-02-10 00:03:14 +00002365 struct req_batch rb;
2366
Jens Axboe79ebeae2021-08-10 15:18:27 -06002367 spin_lock(&ctx->completion_lock);
Pavel Begunkov905c1722021-02-10 00:03:14 +00002368 for (i = 0; i < nr; i++) {
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01002369 struct io_kiocb *req = state->compl_reqs[i];
Pavel Begunkov5182ed22021-06-26 21:40:48 +01002370
Pavel Begunkovb850d6d2022-08-29 14:30:13 +01002371 __io_fill_cqe(ctx, req->user_data, req->result,
2372 req->compl.cflags);
Pavel Begunkov905c1722021-02-10 00:03:14 +00002373 }
2374 io_commit_cqring(ctx);
Jens Axboe79ebeae2021-08-10 15:18:27 -06002375 spin_unlock(&ctx->completion_lock);
Pavel Begunkov905c1722021-02-10 00:03:14 +00002376 io_cqring_ev_posted(ctx);
Pavel Begunkov5182ed22021-06-26 21:40:48 +01002377
2378 io_init_req_batch(&rb);
Pavel Begunkov905c1722021-02-10 00:03:14 +00002379 for (i = 0; i < nr; i++) {
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01002380 struct io_kiocb *req = state->compl_reqs[i];
Pavel Begunkov905c1722021-02-10 00:03:14 +00002381
Pavel Begunkov91c2f692021-08-11 19:28:28 +01002382 if (req_ref_put_and_test(req))
Pavel Begunkov6ff119a2021-02-10 00:03:18 +00002383 io_req_free_batch(&rb, req, &ctx->submit_state);
Pavel Begunkov905c1722021-02-10 00:03:14 +00002384 }
2385
2386 io_req_free_batch_finish(ctx, &rb);
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01002387 state->compl_nr = 0;
Jens Axboee65ef562019-03-12 10:16:44 -06002388}
2389
Jens Axboeba816ad2019-09-28 11:36:45 -06002390/*
2391 * Drop reference to request, return next in chain (if there is one) if this
2392 * was the last reference to this request.
2393 */
Pavel Begunkov0d850352021-03-19 17:22:37 +00002394static inline struct io_kiocb *io_put_req_find_next(struct io_kiocb *req)
Jens Axboee65ef562019-03-12 10:16:44 -06002395{
Pavel Begunkov9b5f7bd2020-06-29 13:13:00 +03002396 struct io_kiocb *nxt = NULL;
2397
Jens Axboede9b4cc2021-02-24 13:28:27 -07002398 if (req_ref_put_and_test(req)) {
Pavel Begunkov9b5f7bd2020-06-29 13:13:00 +03002399 nxt = io_req_find_next(req);
Jens Axboe4d7dd462019-11-20 13:03:52 -07002400 __io_free_req(req);
Jens Axboe2a44f462020-02-25 13:25:41 -07002401 }
Pavel Begunkov9b5f7bd2020-06-29 13:13:00 +03002402 return nxt;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002403}
2404
Pavel Begunkov0d850352021-03-19 17:22:37 +00002405static inline void io_put_req(struct io_kiocb *req)
Jens Axboe2b188cc2019-01-07 10:46:33 -07002406{
Jens Axboede9b4cc2021-02-24 13:28:27 -07002407 if (req_ref_put_and_test(req))
Jens Axboedef596e2019-01-09 08:59:42 -07002408 io_free_req(req);
2409}
2410
Pavel Begunkov91c2f692021-08-11 19:28:28 +01002411static inline void io_put_req_deferred(struct io_kiocb *req)
Pavel Begunkov216578e2020-10-13 09:44:00 +01002412{
Pavel Begunkov91c2f692021-08-11 19:28:28 +01002413 if (req_ref_put_and_test(req)) {
Pavel Begunkovf237c302021-08-18 12:42:46 +01002414 req->io_task_work.func = io_free_req_work;
Pavel Begunkov543af3a2021-08-09 13:04:15 +01002415 io_req_task_work_add(req);
2416 }
Pavel Begunkov216578e2020-10-13 09:44:00 +01002417}
2418
Pavel Begunkov6c503152021-01-04 20:36:36 +00002419static unsigned io_cqring_events(struct io_ring_ctx *ctx)
Jens Axboea3a0e432019-08-20 11:03:11 -06002420{
2421 /* See comment at the top of this file */
2422 smp_rmb();
Pavel Begunkove23de152020-12-17 00:24:37 +00002423 return __io_cqring_events(ctx);
Jens Axboea3a0e432019-08-20 11:03:11 -06002424}
2425
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03002426static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx)
2427{
2428 struct io_rings *rings = ctx->rings;
2429
2430 /* make sure SQ entry isn't read before tail */
2431 return smp_load_acquire(&rings->sq.tail) - ctx->cached_sq_head;
2432}
2433
Pavel Begunkov8ff069b2020-07-16 23:28:04 +03002434static unsigned int io_put_kbuf(struct io_kiocb *req, struct io_buffer *kbuf)
Jens Axboee94f1412019-12-19 12:06:02 -07002435{
Pavel Begunkov8ff069b2020-07-16 23:28:04 +03002436 unsigned int cflags;
Jens Axboee94f1412019-12-19 12:06:02 -07002437
Jens Axboebcda7ba2020-02-23 16:42:51 -07002438 cflags = kbuf->bid << IORING_CQE_BUFFER_SHIFT;
2439 cflags |= IORING_CQE_F_BUFFER;
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03002440 req->flags &= ~REQ_F_BUFFER_SELECTED;
Jens Axboebcda7ba2020-02-23 16:42:51 -07002441 kfree(kbuf);
2442 return cflags;
2443}
2444
Pavel Begunkov8ff069b2020-07-16 23:28:04 +03002445static inline unsigned int io_put_rw_kbuf(struct io_kiocb *req)
2446{
2447 struct io_buffer *kbuf;
2448
Pavel Begunkovae421d92021-08-17 20:28:08 +01002449 if (likely(!(req->flags & REQ_F_BUFFER_SELECTED)))
2450 return 0;
Pavel Begunkov8ff069b2020-07-16 23:28:04 +03002451 kbuf = (struct io_buffer *) (unsigned long) req->rw.addr;
2452 return io_put_kbuf(req, kbuf);
2453}
2454
Jens Axboe4c6e2772020-07-01 11:29:10 -06002455static inline bool io_run_task_work(void)
2456{
Nadav Amitef98eb02021-08-07 17:13:41 -07002457 if (test_thread_flag(TIF_NOTIFY_SIGNAL) || current->task_works) {
Jens Axboe4c6e2772020-07-01 11:29:10 -06002458 __set_current_state(TASK_RUNNING);
Nadav Amitef98eb02021-08-07 17:13:41 -07002459 tracehook_notify_signal();
Jens Axboe4c6e2772020-07-01 11:29:10 -06002460 return true;
2461 }
2462
2463 return false;
2464}
2465
Jens Axboedef596e2019-01-09 08:59:42 -07002466/*
2467 * Find and free completed poll iocbs
2468 */
2469static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events,
Pavel Begunkova8576af2021-08-15 10:40:21 +01002470 struct list_head *done)
Jens Axboedef596e2019-01-09 08:59:42 -07002471{
Jens Axboe8237e042019-12-28 10:48:22 -07002472 struct req_batch rb;
Jens Axboedef596e2019-01-09 08:59:42 -07002473 struct io_kiocb *req;
Xiaoguang Wangbbde0172020-06-16 02:06:38 +08002474
2475 /* order with ->result store in io_complete_rw_iopoll() */
2476 smp_rmb();
Jens Axboedef596e2019-01-09 08:59:42 -07002477
Pavel Begunkov5af1d132020-07-18 11:32:52 +03002478 io_init_req_batch(&rb);
Jens Axboedef596e2019-01-09 08:59:42 -07002479 while (!list_empty(done)) {
Pavel Begunkovd21ffe72020-07-13 23:37:10 +03002480 req = list_first_entry(done, struct io_kiocb, inflight_entry);
Pavel Begunkovd21ffe72020-07-13 23:37:10 +03002481 list_del(&req->inflight_entry);
Jens Axboedef596e2019-01-09 08:59:42 -07002482
Pavel Begunkovb850d6d2022-08-29 14:30:13 +01002483 io_fill_cqe_req(req, req->result, io_put_rw_kbuf(req));
Jens Axboedef596e2019-01-09 08:59:42 -07002484 (*nr_events)++;
2485
Jens Axboede9b4cc2021-02-24 13:28:27 -07002486 if (req_ref_put_and_test(req))
Pavel Begunkov6ff119a2021-02-10 00:03:18 +00002487 io_req_free_batch(&rb, req, &ctx->submit_state);
Jens Axboedef596e2019-01-09 08:59:42 -07002488 }
Jens Axboedef596e2019-01-09 08:59:42 -07002489
Jens Axboe09bb8392019-03-13 12:39:28 -06002490 io_commit_cqring(ctx);
Pavel Begunkov80c18e42021-01-07 03:15:41 +00002491 io_cqring_ev_posted_iopoll(ctx);
Pavel Begunkov2d6500d2020-06-28 12:52:33 +03002492 io_req_free_batch_finish(ctx, &rb);
Bijan Mottahedeh581f9812020-04-03 13:51:33 -07002493}
2494
Jens Axboedef596e2019-01-09 08:59:42 -07002495static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events,
Pavel Begunkova8576af2021-08-15 10:40:21 +01002496 long min)
Jens Axboedef596e2019-01-09 08:59:42 -07002497{
2498 struct io_kiocb *req, *tmp;
2499 LIST_HEAD(done);
2500 bool spin;
Jens Axboedef596e2019-01-09 08:59:42 -07002501
2502 /*
2503 * Only spin for completions if we don't have multiple devices hanging
2504 * off our complete list, and we're under the requested amount.
2505 */
Hao Xu915b3dd2021-06-28 05:37:30 +08002506 spin = !ctx->poll_multi_queue && *nr_events < min;
Jens Axboedef596e2019-01-09 08:59:42 -07002507
Pavel Begunkovd21ffe72020-07-13 23:37:10 +03002508 list_for_each_entry_safe(req, tmp, &ctx->iopoll_list, inflight_entry) {
Jens Axboe9adbd452019-12-20 08:45:55 -07002509 struct kiocb *kiocb = &req->rw.kiocb;
Pavel Begunkova2416e12021-08-09 13:04:09 +01002510 int ret;
Jens Axboedef596e2019-01-09 08:59:42 -07002511
2512 /*
Bijan Mottahedeh581f9812020-04-03 13:51:33 -07002513 * Move completed and retryable entries to our local lists.
2514 * If we find a request that requires polling, break out
2515 * and complete those lists first, if we have entries there.
Jens Axboedef596e2019-01-09 08:59:42 -07002516 */
Xiaoguang Wang65a65432020-06-11 23:39:36 +08002517 if (READ_ONCE(req->iopoll_completed)) {
Pavel Begunkovd21ffe72020-07-13 23:37:10 +03002518 list_move_tail(&req->inflight_entry, &done);
Jens Axboedef596e2019-01-09 08:59:42 -07002519 continue;
2520 }
2521 if (!list_empty(&done))
2522 break;
2523
2524 ret = kiocb->ki_filp->f_op->iopoll(kiocb, spin);
Pavel Begunkova2416e12021-08-09 13:04:09 +01002525 if (unlikely(ret < 0))
2526 return ret;
2527 else if (ret)
2528 spin = false;
Jens Axboedef596e2019-01-09 08:59:42 -07002529
Pavel Begunkov3aadc232020-07-06 17:59:29 +03002530 /* iopoll may have completed current req */
2531 if (READ_ONCE(req->iopoll_completed))
Pavel Begunkovd21ffe72020-07-13 23:37:10 +03002532 list_move_tail(&req->inflight_entry, &done);
Jens Axboedef596e2019-01-09 08:59:42 -07002533 }
2534
2535 if (!list_empty(&done))
Pavel Begunkova8576af2021-08-15 10:40:21 +01002536 io_iopoll_complete(ctx, nr_events, &done);
Jens Axboedef596e2019-01-09 08:59:42 -07002537
Pavel Begunkova2416e12021-08-09 13:04:09 +01002538 return 0;
Jens Axboedef596e2019-01-09 08:59:42 -07002539}
2540
2541/*
Jens Axboedef596e2019-01-09 08:59:42 -07002542 * We can't just wait for polled events to come to us, we have to actively
2543 * find and complete them.
2544 */
Pavel Begunkovb2edc0a2020-07-07 16:36:22 +03002545static void io_iopoll_try_reap_events(struct io_ring_ctx *ctx)
Jens Axboedef596e2019-01-09 08:59:42 -07002546{
2547 if (!(ctx->flags & IORING_SETUP_IOPOLL))
2548 return;
2549
2550 mutex_lock(&ctx->uring_lock);
Pavel Begunkov540e32a2020-07-13 23:37:09 +03002551 while (!list_empty(&ctx->iopoll_list)) {
Jens Axboedef596e2019-01-09 08:59:42 -07002552 unsigned int nr_events = 0;
2553
Pavel Begunkova8576af2021-08-15 10:40:21 +01002554 io_do_iopoll(ctx, &nr_events, 0);
Jens Axboe08f54392019-08-21 22:19:11 -06002555
Pavel Begunkovb2edc0a2020-07-07 16:36:22 +03002556 /* let it sleep and repeat later if can't complete a request */
2557 if (nr_events == 0)
2558 break;
Jens Axboe08f54392019-08-21 22:19:11 -06002559 /*
2560 * Ensure we allow local-to-the-cpu processing to take place,
2561 * in this case we need to ensure that we reap all events.
Pavel Begunkov3fcee5a2020-07-06 17:59:31 +03002562 * Also let task_work, etc. to progress by releasing the mutex
Jens Axboe08f54392019-08-21 22:19:11 -06002563 */
Pavel Begunkov3fcee5a2020-07-06 17:59:31 +03002564 if (need_resched()) {
2565 mutex_unlock(&ctx->uring_lock);
2566 cond_resched();
2567 mutex_lock(&ctx->uring_lock);
2568 }
Jens Axboedef596e2019-01-09 08:59:42 -07002569 }
2570 mutex_unlock(&ctx->uring_lock);
2571}
2572
Pavel Begunkov7668b922020-07-07 16:36:21 +03002573static int io_iopoll_check(struct io_ring_ctx *ctx, long min)
Jens Axboedef596e2019-01-09 08:59:42 -07002574{
Pavel Begunkov7668b922020-07-07 16:36:21 +03002575 unsigned int nr_events = 0;
Pavel Begunkove9979b32021-04-13 02:58:45 +01002576 int ret = 0;
Jens Axboedef596e2019-01-09 08:59:42 -07002577
Xiaoguang Wangc7849be2020-02-22 14:46:05 +08002578 /*
2579 * We disallow the app entering submit/complete with polling, but we
2580 * still need to lock the ring to prevent racing with polled issue
2581 * that got punted to a workqueue.
2582 */
2583 mutex_lock(&ctx->uring_lock);
Pavel Begunkovf39c8a52021-04-13 02:58:46 +01002584 /*
2585 * Don't enter poll loop if we already have events pending.
2586 * If we do, we can potentially be spinning for commands that
2587 * already triggered a CQE (eg in error).
2588 */
Pavel Begunkov5ed7a372021-06-14 23:37:27 +01002589 if (test_bit(0, &ctx->check_cq_overflow))
Pavel Begunkovf39c8a52021-04-13 02:58:46 +01002590 __io_cqring_overflow_flush(ctx, false);
2591 if (io_cqring_events(ctx))
2592 goto out;
Jens Axboedef596e2019-01-09 08:59:42 -07002593 do {
Jens Axboe500f9fb2019-08-19 12:15:59 -06002594 /*
2595 * If a submit got punted to a workqueue, we can have the
2596 * application entering polling for a command before it gets
2597 * issued. That app will hold the uring_lock for the duration
2598 * of the poll right here, so we need to take a breather every
2599 * now and then to ensure that the issue has a chance to add
2600 * the poll to the issued list. Otherwise we can spin here
2601 * forever, while the workqueue is stuck trying to acquire the
2602 * very same mutex.
2603 */
Pavel Begunkove9979b32021-04-13 02:58:45 +01002604 if (list_empty(&ctx->iopoll_list)) {
Pavel Begunkov8f487ef2021-07-08 13:37:06 +01002605 u32 tail = ctx->cached_cq_tail;
2606
Jens Axboe500f9fb2019-08-19 12:15:59 -06002607 mutex_unlock(&ctx->uring_lock);
Jens Axboe4c6e2772020-07-01 11:29:10 -06002608 io_run_task_work();
Jens Axboe500f9fb2019-08-19 12:15:59 -06002609 mutex_lock(&ctx->uring_lock);
Pavel Begunkove9979b32021-04-13 02:58:45 +01002610
Pavel Begunkov8f487ef2021-07-08 13:37:06 +01002611 /* some requests don't go through iopoll_list */
2612 if (tail != ctx->cached_cq_tail ||
2613 list_empty(&ctx->iopoll_list))
Pavel Begunkove9979b32021-04-13 02:58:45 +01002614 break;
Jens Axboe500f9fb2019-08-19 12:15:59 -06002615 }
Pavel Begunkova8576af2021-08-15 10:40:21 +01002616 ret = io_do_iopoll(ctx, &nr_events, min);
Pavel Begunkovf39c8a52021-04-13 02:58:46 +01002617 } while (!ret && nr_events < min && !need_resched());
2618out:
Jens Axboe500f9fb2019-08-19 12:15:59 -06002619 mutex_unlock(&ctx->uring_lock);
Jens Axboedef596e2019-01-09 08:59:42 -07002620 return ret;
2621}
2622
Jens Axboe491381ce2019-10-17 09:20:46 -06002623static void kiocb_end_write(struct io_kiocb *req)
Jens Axboe2b188cc2019-01-07 10:46:33 -07002624{
Jens Axboe491381ce2019-10-17 09:20:46 -06002625 /*
2626 * Tell lockdep we inherited freeze protection from submission
2627 * thread.
2628 */
2629 if (req->flags & REQ_F_ISREG) {
Pavel Begunkov1c986792021-03-22 01:58:31 +00002630 struct super_block *sb = file_inode(req->file)->i_sb;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002631
Pavel Begunkov1c986792021-03-22 01:58:31 +00002632 __sb_writers_acquired(sb, SB_FREEZE_WRITE);
2633 sb_end_write(sb);
Jens Axboe2b188cc2019-01-07 10:46:33 -07002634 }
2635}
2636
Jens Axboeb63534c2020-06-04 11:28:00 -06002637#ifdef CONFIG_BLOCK
Pavel Begunkovdc2a6e92021-01-19 13:32:35 +00002638static bool io_resubmit_prep(struct io_kiocb *req)
Jens Axboeb63534c2020-06-04 11:28:00 -06002639{
Pavel Begunkovab454432021-03-22 01:58:33 +00002640 struct io_async_rw *rw = req->async_data;
Jens Axboeb63534c2020-06-04 11:28:00 -06002641
Pavel Begunkovab454432021-03-22 01:58:33 +00002642 if (!rw)
2643 return !io_req_prep_async(req);
Jens Axboecd658692021-09-10 11:19:14 -06002644 iov_iter_restore(&rw->iter, &rw->iter_state);
Pavel Begunkovab454432021-03-22 01:58:33 +00002645 return true;
Jens Axboeb63534c2020-06-04 11:28:00 -06002646}
Jens Axboeb63534c2020-06-04 11:28:00 -06002647
Jens Axboe3e6a0d32021-03-01 13:56:00 -07002648static bool io_rw_should_reissue(struct io_kiocb *req)
Jens Axboeb63534c2020-06-04 11:28:00 -06002649{
Jens Axboe355afae2020-09-02 09:30:31 -06002650 umode_t mode = file_inode(req->file)->i_mode;
Jens Axboe3e6a0d32021-03-01 13:56:00 -07002651 struct io_ring_ctx *ctx = req->ctx;
Jens Axboeb63534c2020-06-04 11:28:00 -06002652
Jens Axboe355afae2020-09-02 09:30:31 -06002653 if (!S_ISBLK(mode) && !S_ISREG(mode))
2654 return false;
Jens Axboe3e6a0d32021-03-01 13:56:00 -07002655 if ((req->flags & REQ_F_NOWAIT) || (io_wq_current_is_worker() &&
2656 !(ctx->flags & IORING_SETUP_IOPOLL)))
Jens Axboeb63534c2020-06-04 11:28:00 -06002657 return false;
Jens Axboe7c977a52021-02-23 19:17:35 -07002658 /*
2659 * If ref is dying, we might be running poll reap from the exit work.
2660 * Don't attempt to reissue from that path, just let it fail with
2661 * -EAGAIN.
2662 */
Jens Axboe3e6a0d32021-03-01 13:56:00 -07002663 if (percpu_ref_is_dying(&ctx->refs))
2664 return false;
Jens Axboeef046882021-07-27 10:50:31 -06002665 /*
2666 * Play it safe and assume not safe to re-import and reissue if we're
2667 * not in the original thread group (or in task context).
2668 */
2669 if (!same_thread_group(req->task, current) || !in_task())
2670 return false;
Jens Axboe3e6a0d32021-03-01 13:56:00 -07002671 return true;
2672}
Jens Axboee82ad482021-04-02 19:45:34 -06002673#else
Jens Axboea1ff1e32021-04-12 06:40:02 -06002674static bool io_resubmit_prep(struct io_kiocb *req)
2675{
2676 return false;
2677}
Jens Axboee82ad482021-04-02 19:45:34 -06002678static bool io_rw_should_reissue(struct io_kiocb *req)
2679{
2680 return false;
2681}
Jens Axboe3e6a0d32021-03-01 13:56:00 -07002682#endif
2683
Jens Axboe8ef12ef2021-08-10 15:15:25 -06002684static bool __io_complete_rw_common(struct io_kiocb *req, long res)
Jens Axboea1d7c392020-06-22 11:09:46 -06002685{
Jens Axboedf1ec532022-03-20 13:08:38 -06002686 if (req->rw.kiocb.ki_flags & IOCB_WRITE) {
Pavel Begunkovb65c1282021-03-22 01:45:59 +00002687 kiocb_end_write(req);
Jens Axboedf1ec532022-03-20 13:08:38 -06002688 fsnotify_modify(req->file);
2689 } else {
2690 fsnotify_access(req->file);
2691 }
Pavel Begunkov9532b992021-03-22 01:58:34 +00002692 if (res != req->result) {
2693 if ((res == -EAGAIN || res == -EOPNOTSUPP) &&
2694 io_rw_should_reissue(req)) {
2695 req->flags |= REQ_F_REISSUE;
Jens Axboe8ef12ef2021-08-10 15:15:25 -06002696 return true;
Pavel Begunkov9532b992021-03-22 01:58:34 +00002697 }
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01002698 req_set_fail(req);
Jens Axboe8ef12ef2021-08-10 15:15:25 -06002699 req->result = res;
Pavel Begunkov9532b992021-03-22 01:58:34 +00002700 }
Jens Axboe8ef12ef2021-08-10 15:15:25 -06002701 return false;
2702}
2703
Pavel Begunkovd6b7efc2022-10-16 22:42:57 +01002704static inline int io_fixup_rw_res(struct io_kiocb *req, unsigned res)
Pavel Begunkove8574572022-10-16 22:42:56 +01002705{
2706 struct io_async_rw *io = req->async_data;
2707
2708 /* add previously done IO, if any */
2709 if (io && io->bytes_done > 0) {
2710 if (res < 0)
2711 res = io->bytes_done;
2712 else
2713 res += io->bytes_done;
2714 }
2715 return res;
2716}
2717
Pavel Begunkovf237c302021-08-18 12:42:46 +01002718static void io_req_task_complete(struct io_kiocb *req, bool *locked)
Jens Axboe8ef12ef2021-08-10 15:15:25 -06002719{
Pavel Begunkov126180b2021-08-18 12:42:47 +01002720 unsigned int cflags = io_put_rw_kbuf(req);
Pavel Begunkov5c0ea4c2022-08-29 14:30:12 +01002721 int res = req->result;
Pavel Begunkov126180b2021-08-18 12:42:47 +01002722
2723 if (*locked) {
2724 struct io_ring_ctx *ctx = req->ctx;
2725 struct io_submit_state *state = &ctx->submit_state;
2726
2727 io_req_complete_state(req, res, cflags);
2728 state->compl_reqs[state->compl_nr++] = req;
2729 if (state->compl_nr == ARRAY_SIZE(state->compl_reqs))
2730 io_submit_flush_completions(ctx);
2731 } else {
2732 io_req_complete_post(req, res, cflags);
2733 }
Jens Axboe8ef12ef2021-08-10 15:15:25 -06002734}
2735
2736static void __io_complete_rw(struct io_kiocb *req, long res, long res2,
2737 unsigned int issue_flags)
2738{
2739 if (__io_complete_rw_common(req, res))
2740 return;
Pavel Begunkove8574572022-10-16 22:42:56 +01002741 __io_req_complete(req, issue_flags, io_fixup_rw_res(req, res), io_put_rw_kbuf(req));
Jens Axboeba816ad2019-09-28 11:36:45 -06002742}
2743
2744static void io_complete_rw(struct kiocb *kiocb, long res, long res2)
2745{
Jens Axboe9adbd452019-12-20 08:45:55 -07002746 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
Jens Axboeba816ad2019-09-28 11:36:45 -06002747
Jens Axboe8ef12ef2021-08-10 15:15:25 -06002748 if (__io_complete_rw_common(req, res))
2749 return;
Pavel Begunkove8574572022-10-16 22:42:56 +01002750 req->result = io_fixup_rw_res(req, res);
Jens Axboe8ef12ef2021-08-10 15:15:25 -06002751 req->io_task_work.func = io_req_task_complete;
2752 io_req_task_work_add(req);
Jens Axboe2b188cc2019-01-07 10:46:33 -07002753}
2754
Jens Axboedef596e2019-01-09 08:59:42 -07002755static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2)
2756{
Jens Axboe9adbd452019-12-20 08:45:55 -07002757 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
Jens Axboedef596e2019-01-09 08:59:42 -07002758
Jens Axboe491381ce2019-10-17 09:20:46 -06002759 if (kiocb->ki_flags & IOCB_WRITE)
2760 kiocb_end_write(req);
Pavel Begunkov9532b992021-03-22 01:58:34 +00002761 if (unlikely(res != req->result)) {
Pavel Begunkovb66ceaf2021-09-15 11:00:05 +01002762 if (res == -EAGAIN && io_rw_should_reissue(req)) {
2763 req->flags |= REQ_F_REISSUE;
2764 return;
Pavel Begunkov9532b992021-03-22 01:58:34 +00002765 }
Pavel Begunkov8c130822021-03-22 01:58:32 +00002766 }
Xiaoguang Wangbbde0172020-06-16 02:06:38 +08002767
2768 WRITE_ONCE(req->result, res);
Jens Axboeb9b0e0d2021-02-23 08:18:36 -07002769 /* order with io_iopoll_complete() checking ->result */
Pavel Begunkovcd664b02020-06-25 12:37:10 +03002770 smp_wmb();
2771 WRITE_ONCE(req->iopoll_completed, 1);
Jens Axboedef596e2019-01-09 08:59:42 -07002772}
2773
2774/*
2775 * After the iocb has been issued, it's safe to be found on the poll list.
2776 * Adding the kiocb to the list AFTER submission ensures that we don't
Pavel Begunkovf39c8a52021-04-13 02:58:46 +01002777 * find it from a io_do_iopoll() thread before the issuer is done
Jens Axboedef596e2019-01-09 08:59:42 -07002778 * accessing the kiocb cookie.
2779 */
Pavel Begunkovcb3d8972021-06-14 02:36:14 +01002780static void io_iopoll_req_issued(struct io_kiocb *req)
Jens Axboedef596e2019-01-09 08:59:42 -07002781{
2782 struct io_ring_ctx *ctx = req->ctx;
Pavel Begunkovcb3d8972021-06-14 02:36:14 +01002783 const bool in_async = io_wq_current_is_worker();
2784
2785 /* workqueue context doesn't hold uring_lock, grab it now */
2786 if (unlikely(in_async))
2787 mutex_lock(&ctx->uring_lock);
Jens Axboedef596e2019-01-09 08:59:42 -07002788
2789 /*
2790 * Track whether we have multiple files in our lists. This will impact
2791 * how we do polling eventually, not spinning if we're on potentially
2792 * different devices.
2793 */
Pavel Begunkov540e32a2020-07-13 23:37:09 +03002794 if (list_empty(&ctx->iopoll_list)) {
Hao Xu915b3dd2021-06-28 05:37:30 +08002795 ctx->poll_multi_queue = false;
2796 } else if (!ctx->poll_multi_queue) {
Jens Axboedef596e2019-01-09 08:59:42 -07002797 struct io_kiocb *list_req;
Hao Xu915b3dd2021-06-28 05:37:30 +08002798 unsigned int queue_num0, queue_num1;
Jens Axboedef596e2019-01-09 08:59:42 -07002799
Pavel Begunkov540e32a2020-07-13 23:37:09 +03002800 list_req = list_first_entry(&ctx->iopoll_list, struct io_kiocb,
Pavel Begunkovd21ffe72020-07-13 23:37:10 +03002801 inflight_entry);
Hao Xu915b3dd2021-06-28 05:37:30 +08002802
2803 if (list_req->file != req->file) {
2804 ctx->poll_multi_queue = true;
2805 } else {
2806 queue_num0 = blk_qc_t_to_queue_num(list_req->rw.kiocb.ki_cookie);
2807 queue_num1 = blk_qc_t_to_queue_num(req->rw.kiocb.ki_cookie);
2808 if (queue_num0 != queue_num1)
2809 ctx->poll_multi_queue = true;
2810 }
Jens Axboedef596e2019-01-09 08:59:42 -07002811 }
2812
2813 /*
2814 * For fast devices, IO may have already completed. If it has, add
2815 * it to the front so we find it first.
2816 */
Xiaoguang Wang65a65432020-06-11 23:39:36 +08002817 if (READ_ONCE(req->iopoll_completed))
Pavel Begunkovd21ffe72020-07-13 23:37:10 +03002818 list_add(&req->inflight_entry, &ctx->iopoll_list);
Jens Axboedef596e2019-01-09 08:59:42 -07002819 else
Pavel Begunkovd21ffe72020-07-13 23:37:10 +03002820 list_add_tail(&req->inflight_entry, &ctx->iopoll_list);
Xiaoguang Wangbdcd3ea2020-02-25 22:12:08 +08002821
Pavel Begunkovcb3d8972021-06-14 02:36:14 +01002822 if (unlikely(in_async)) {
2823 /*
2824 * If IORING_SETUP_SQPOLL is enabled, sqes are either handle
2825 * in sq thread task context or in io worker task context. If
2826 * current task context is sq thread, we don't need to check
2827 * whether should wake up sq thread.
2828 */
2829 if ((ctx->flags & IORING_SETUP_SQPOLL) &&
2830 wq_has_sleeper(&ctx->sq_data->wait))
2831 wake_up(&ctx->sq_data->wait);
2832
2833 mutex_unlock(&ctx->uring_lock);
2834 }
Jens Axboedef596e2019-01-09 08:59:42 -07002835}
2836
Jens Axboe4503b762020-06-01 10:00:27 -06002837static bool io_bdev_nowait(struct block_device *bdev)
2838{
Jeffle Xu9ba0d0c2020-10-19 16:59:42 +08002839 return !bdev || blk_queue_nowait(bdev_get_queue(bdev));
Jens Axboe4503b762020-06-01 10:00:27 -06002840}
2841
Jens Axboe2b188cc2019-01-07 10:46:33 -07002842/*
2843 * If we tracked the file through the SCM inflight mechanism, we could support
2844 * any file. For now, just ensure that anything potentially problematic is done
2845 * inline.
2846 */
Pavel Begunkovb191e2d2021-08-09 13:04:03 +01002847static bool __io_file_supports_nowait(struct file *file, int rw)
Jens Axboe2b188cc2019-01-07 10:46:33 -07002848{
2849 umode_t mode = file_inode(file)->i_mode;
2850
Jens Axboe4503b762020-06-01 10:00:27 -06002851 if (S_ISBLK(mode)) {
Christoph Hellwig4e7b5672020-11-23 13:38:40 +01002852 if (IS_ENABLED(CONFIG_BLOCK) &&
2853 io_bdev_nowait(I_BDEV(file->f_mapping->host)))
Jens Axboe4503b762020-06-01 10:00:27 -06002854 return true;
2855 return false;
2856 }
Pavel Begunkov976517f2021-06-09 12:07:25 +01002857 if (S_ISSOCK(mode))
Jens Axboe2b188cc2019-01-07 10:46:33 -07002858 return true;
Jens Axboe4503b762020-06-01 10:00:27 -06002859 if (S_ISREG(mode)) {
Christoph Hellwig4e7b5672020-11-23 13:38:40 +01002860 if (IS_ENABLED(CONFIG_BLOCK) &&
2861 io_bdev_nowait(file->f_inode->i_sb->s_bdev) &&
Jens Axboe4503b762020-06-01 10:00:27 -06002862 file->f_op != &io_uring_fops)
2863 return true;
2864 return false;
2865 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07002866
Jens Axboec5b85622020-06-09 19:23:05 -06002867 /* any ->read/write should understand O_NONBLOCK */
2868 if (file->f_flags & O_NONBLOCK)
2869 return true;
2870
Jens Axboeaf197f52020-04-28 13:15:06 -06002871 if (!(file->f_mode & FMODE_NOWAIT))
2872 return false;
2873
2874 if (rw == READ)
2875 return file->f_op->read_iter != NULL;
2876
2877 return file->f_op->write_iter != NULL;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002878}
2879
Pavel Begunkovb191e2d2021-08-09 13:04:03 +01002880static bool io_file_supports_nowait(struct io_kiocb *req, int rw)
Jens Axboe7b29f922021-03-12 08:30:14 -07002881{
Pavel Begunkovb191e2d2021-08-09 13:04:03 +01002882 if (rw == READ && (req->flags & REQ_F_NOWAIT_READ))
Jens Axboe7b29f922021-03-12 08:30:14 -07002883 return true;
Pavel Begunkovb191e2d2021-08-09 13:04:03 +01002884 else if (rw == WRITE && (req->flags & REQ_F_NOWAIT_WRITE))
Jens Axboe7b29f922021-03-12 08:30:14 -07002885 return true;
2886
Pavel Begunkovb191e2d2021-08-09 13:04:03 +01002887 return __io_file_supports_nowait(req->file, rw);
Jens Axboe7b29f922021-03-12 08:30:14 -07002888}
2889
Jens Axboe5d329e12021-09-14 11:08:37 -06002890static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,
2891 int rw)
Jens Axboe2b188cc2019-01-07 10:46:33 -07002892{
Jens Axboedef596e2019-01-09 08:59:42 -07002893 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe9adbd452019-12-20 08:45:55 -07002894 struct kiocb *kiocb = &req->rw.kiocb;
Pavel Begunkov75c668c2021-02-04 13:52:05 +00002895 struct file *file = req->file;
Jens Axboe09bb8392019-03-13 12:39:28 -06002896 unsigned ioprio;
2897 int ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002898
Pavel Begunkovc97d8a02021-08-09 13:04:04 +01002899 if (!io_req_ffs_set(req) && S_ISREG(file_inode(file)->i_mode))
Jens Axboe491381ce2019-10-17 09:20:46 -06002900 req->flags |= REQ_F_ISREG;
2901
Jens Axboe2b188cc2019-01-07 10:46:33 -07002902 kiocb->ki_pos = READ_ONCE(sqe->off);
Jens Axboe20fb0dc2021-12-22 20:26:56 -07002903 if (kiocb->ki_pos == -1) {
2904 if (!(file->f_mode & FMODE_STREAM)) {
2905 req->flags |= REQ_F_CUR_POS;
2906 kiocb->ki_pos = file->f_pos;
2907 } else {
2908 kiocb->ki_pos = 0;
2909 }
Jens Axboeba042912019-12-25 16:33:42 -07002910 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07002911 kiocb->ki_hint = ki_hint_validate(file_write_hint(kiocb->ki_filp));
Pavel Begunkov3e577dc2020-02-01 03:58:42 +03002912 kiocb->ki_flags = iocb_flags(kiocb->ki_filp);
2913 ret = kiocb_set_rw_flags(kiocb, READ_ONCE(sqe->rw_flags));
2914 if (unlikely(ret))
2915 return ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002916
Jens Axboe5d329e12021-09-14 11:08:37 -06002917 /*
2918 * If the file is marked O_NONBLOCK, still allow retry for it if it
2919 * supports async. Otherwise it's impossible to use O_NONBLOCK files
2920 * reliably. If not, or it IOCB_NOWAIT is set, don't retry.
2921 */
2922 if ((kiocb->ki_flags & IOCB_NOWAIT) ||
2923 ((file->f_flags & O_NONBLOCK) && !io_file_supports_nowait(req, rw)))
Pavel Begunkov75c668c2021-02-04 13:52:05 +00002924 req->flags |= REQ_F_NOWAIT;
2925
Jens Axboe2b188cc2019-01-07 10:46:33 -07002926 ioprio = READ_ONCE(sqe->ioprio);
2927 if (ioprio) {
2928 ret = ioprio_check_cap(ioprio);
2929 if (ret)
Jens Axboe09bb8392019-03-13 12:39:28 -06002930 return ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002931
2932 kiocb->ki_ioprio = ioprio;
2933 } else
2934 kiocb->ki_ioprio = get_current_ioprio();
2935
Jens Axboedef596e2019-01-09 08:59:42 -07002936 if (ctx->flags & IORING_SETUP_IOPOLL) {
Jens Axboedef596e2019-01-09 08:59:42 -07002937 if (!(kiocb->ki_flags & IOCB_DIRECT) ||
2938 !kiocb->ki_filp->f_op->iopoll)
Jens Axboe09bb8392019-03-13 12:39:28 -06002939 return -EOPNOTSUPP;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002940
Jens Axboe394918e2021-03-08 11:40:23 -07002941 kiocb->ki_flags |= IOCB_HIPRI | IOCB_ALLOC_CACHE;
Jens Axboedef596e2019-01-09 08:59:42 -07002942 kiocb->ki_complete = io_complete_rw_iopoll;
Xiaoguang Wang65a65432020-06-11 23:39:36 +08002943 req->iopoll_completed = 0;
Jens Axboedef596e2019-01-09 08:59:42 -07002944 } else {
Jens Axboe09bb8392019-03-13 12:39:28 -06002945 if (kiocb->ki_flags & IOCB_HIPRI)
2946 return -EINVAL;
Jens Axboedef596e2019-01-09 08:59:42 -07002947 kiocb->ki_complete = io_complete_rw;
2948 }
Jens Axboe9adbd452019-12-20 08:45:55 -07002949
Pavel Begunkovea512d52022-06-09 08:34:35 +01002950 /* used for fixed read/write too - just read unconditionally */
2951 req->buf_index = READ_ONCE(sqe->buf_index);
2952 req->imu = NULL;
2953
Pavel Begunkoveae071c2021-04-25 14:32:24 +01002954 if (req->opcode == IORING_OP_READ_FIXED ||
2955 req->opcode == IORING_OP_WRITE_FIXED) {
Pavel Begunkovea512d52022-06-09 08:34:35 +01002956 struct io_ring_ctx *ctx = req->ctx;
2957 u16 index;
2958
2959 if (unlikely(req->buf_index >= ctx->nr_user_bufs))
2960 return -EFAULT;
2961 index = array_index_nospec(req->buf_index, ctx->nr_user_bufs);
2962 req->imu = ctx->user_bufs[index];
Pavel Begunkoveae071c2021-04-25 14:32:24 +01002963 io_req_set_rsrc_node(req);
2964 }
2965
Jens Axboe3529d8c2019-12-19 18:24:38 -07002966 req->rw.addr = READ_ONCE(sqe->addr);
2967 req->rw.len = READ_ONCE(sqe->len);
Jens Axboe2b188cc2019-01-07 10:46:33 -07002968 return 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002969}
2970
2971static inline void io_rw_done(struct kiocb *kiocb, ssize_t ret)
2972{
2973 switch (ret) {
2974 case -EIOCBQUEUED:
2975 break;
2976 case -ERESTARTSYS:
2977 case -ERESTARTNOINTR:
2978 case -ERESTARTNOHAND:
2979 case -ERESTART_RESTARTBLOCK:
2980 /*
2981 * We can't just restart the syscall, since previously
2982 * submitted sqes may already be in progress. Just fail this
2983 * IO with EINTR.
2984 */
2985 ret = -EINTR;
Gustavo A. R. Silvadf561f662020-08-23 17:36:59 -05002986 fallthrough;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002987 default:
2988 kiocb->ki_complete(kiocb, ret, 0);
2989 }
2990}
2991
Jens Axboea1d7c392020-06-22 11:09:46 -06002992static void kiocb_done(struct kiocb *kiocb, ssize_t ret,
Pavel Begunkov889fca72021-02-10 00:03:09 +00002993 unsigned int issue_flags)
Jens Axboeba816ad2019-09-28 11:36:45 -06002994{
Jens Axboeba042912019-12-25 16:33:42 -07002995 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
Jens Axboe227c0c92020-08-13 11:51:40 -06002996
Jens Axboeba042912019-12-25 16:33:42 -07002997 if (req->flags & REQ_F_CUR_POS)
2998 req->file->f_pos = kiocb->ki_pos;
Pavel Begunkovb66ceaf2021-09-15 11:00:05 +01002999 if (ret >= 0 && (kiocb->ki_complete == io_complete_rw))
Pavel Begunkov889fca72021-02-10 00:03:09 +00003000 __io_complete_rw(req, ret, 0, issue_flags);
Jens Axboeba816ad2019-09-28 11:36:45 -06003001 else
3002 io_rw_done(kiocb, ret);
Pavel Begunkov97284632021-04-08 19:28:03 +01003003
Pavel Begunkovb66ceaf2021-09-15 11:00:05 +01003004 if (req->flags & REQ_F_REISSUE) {
Pavel Begunkov97284632021-04-08 19:28:03 +01003005 req->flags &= ~REQ_F_REISSUE;
Jens Axboea7be7c22021-04-15 11:31:14 -06003006 if (io_resubmit_prep(req)) {
Jens Axboe773af692021-07-27 10:25:55 -06003007 io_req_task_queue_reissue(req);
Pavel Begunkov8c130822021-03-22 01:58:32 +00003008 } else {
Pavel Begunkovb66ceaf2021-09-15 11:00:05 +01003009 unsigned int cflags = io_put_rw_kbuf(req);
3010 struct io_ring_ctx *ctx = req->ctx;
3011
Pavel Begunkove8574572022-10-16 22:42:56 +01003012 ret = io_fixup_rw_res(req, ret);
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01003013 req_set_fail(req);
Hao Xu14cfbb72021-10-14 22:04:00 +08003014 if (!(issue_flags & IO_URING_F_NONBLOCK)) {
Pavel Begunkovb66ceaf2021-09-15 11:00:05 +01003015 mutex_lock(&ctx->uring_lock);
3016 __io_req_complete(req, issue_flags, ret, cflags);
3017 mutex_unlock(&ctx->uring_lock);
3018 } else {
3019 __io_req_complete(req, issue_flags, ret, cflags);
3020 }
Pavel Begunkov97284632021-04-08 19:28:03 +01003021 }
3022 }
Jens Axboeba816ad2019-09-28 11:36:45 -06003023}
3024
Pavel Begunkoveae071c2021-04-25 14:32:24 +01003025static int __io_import_fixed(struct io_kiocb *req, int rw, struct iov_iter *iter,
3026 struct io_mapped_ubuf *imu)
Jens Axboeedafcce2019-01-09 09:16:05 -07003027{
Jens Axboe9adbd452019-12-20 08:45:55 -07003028 size_t len = req->rw.len;
Pavel Begunkov75769e32021-04-01 15:43:54 +01003029 u64 buf_end, buf_addr = req->rw.addr;
Jens Axboeedafcce2019-01-09 09:16:05 -07003030 size_t offset;
Jens Axboeedafcce2019-01-09 09:16:05 -07003031
Pavel Begunkov75769e32021-04-01 15:43:54 +01003032 if (unlikely(check_add_overflow(buf_addr, (u64)len, &buf_end)))
Jens Axboeedafcce2019-01-09 09:16:05 -07003033 return -EFAULT;
3034 /* not inside the mapped region */
Pavel Begunkov4751f532021-04-01 15:43:55 +01003035 if (unlikely(buf_addr < imu->ubuf || buf_end > imu->ubuf_end))
Jens Axboeedafcce2019-01-09 09:16:05 -07003036 return -EFAULT;
3037
3038 /*
3039 * May not be a start of buffer, set size appropriately
3040 * and advance us to the beginning.
3041 */
3042 offset = buf_addr - imu->ubuf;
3043 iov_iter_bvec(iter, rw, imu->bvec, imu->nr_bvecs, offset + len);
Jens Axboebd11b3a2019-07-20 08:37:31 -06003044
3045 if (offset) {
3046 /*
3047 * Don't use iov_iter_advance() here, as it's really slow for
3048 * using the latter parts of a big fixed buffer - it iterates
3049 * over each segment manually. We can cheat a bit here, because
3050 * we know that:
3051 *
3052 * 1) it's a BVEC iter, we set it up
3053 * 2) all bvecs are PAGE_SIZE in size, except potentially the
3054 * first and last bvec
3055 *
3056 * So just find our index, and adjust the iterator afterwards.
3057 * If the offset is within the first bvec (or the whole first
3058 * bvec, just use iov_iter_advance(). This makes it easier
3059 * since we can just skip the first segment, which may not
3060 * be PAGE_SIZE aligned.
3061 */
3062 const struct bio_vec *bvec = imu->bvec;
3063
3064 if (offset <= bvec->bv_len) {
3065 iov_iter_advance(iter, offset);
3066 } else {
3067 unsigned long seg_skip;
3068
3069 /* skip first vec */
3070 offset -= bvec->bv_len;
3071 seg_skip = 1 + (offset >> PAGE_SHIFT);
3072
3073 iter->bvec = bvec + seg_skip;
3074 iter->nr_segs -= seg_skip;
Aleix Roca Nonell99c79f62019-08-15 14:03:22 +02003075 iter->count -= bvec->bv_len + offset;
Jens Axboebd11b3a2019-07-20 08:37:31 -06003076 iter->iov_offset = offset & ~PAGE_MASK;
Jens Axboebd11b3a2019-07-20 08:37:31 -06003077 }
3078 }
3079
Pavel Begunkov847595d2021-02-04 13:52:06 +00003080 return 0;
Jens Axboeedafcce2019-01-09 09:16:05 -07003081}
3082
Pavel Begunkoveae071c2021-04-25 14:32:24 +01003083static int io_import_fixed(struct io_kiocb *req, int rw, struct iov_iter *iter)
3084{
Pavel Begunkovea512d52022-06-09 08:34:35 +01003085 if (WARN_ON_ONCE(!req->imu))
3086 return -EFAULT;
3087 return __io_import_fixed(req, rw, iter, req->imu);
Pavel Begunkoveae071c2021-04-25 14:32:24 +01003088}
3089
Jens Axboebcda7ba2020-02-23 16:42:51 -07003090static void io_ring_submit_unlock(struct io_ring_ctx *ctx, bool needs_lock)
3091{
3092 if (needs_lock)
3093 mutex_unlock(&ctx->uring_lock);
3094}
3095
3096static void io_ring_submit_lock(struct io_ring_ctx *ctx, bool needs_lock)
3097{
3098 /*
3099 * "Normal" inline submissions always hold the uring_lock, since we
3100 * grab it from the system call. Same is true for the SQPOLL offload.
3101 * The only exception is when we've detached the request and issue it
3102 * from an async worker thread, grab the lock for that case.
3103 */
3104 if (needs_lock)
3105 mutex_lock(&ctx->uring_lock);
3106}
3107
3108static struct io_buffer *io_buffer_select(struct io_kiocb *req, size_t *len,
3109 int bgid, struct io_buffer *kbuf,
3110 bool needs_lock)
3111{
3112 struct io_buffer *head;
3113
3114 if (req->flags & REQ_F_BUFFER_SELECTED)
3115 return kbuf;
3116
3117 io_ring_submit_lock(req->ctx, needs_lock);
3118
3119 lockdep_assert_held(&req->ctx->uring_lock);
3120
Jens Axboe9e15c3a2021-03-13 12:29:43 -07003121 head = xa_load(&req->ctx->io_buffers, bgid);
Jens Axboebcda7ba2020-02-23 16:42:51 -07003122 if (head) {
3123 if (!list_empty(&head->list)) {
3124 kbuf = list_last_entry(&head->list, struct io_buffer,
3125 list);
3126 list_del(&kbuf->list);
3127 } else {
3128 kbuf = head;
Jens Axboe9e15c3a2021-03-13 12:29:43 -07003129 xa_erase(&req->ctx->io_buffers, bgid);
Jens Axboebcda7ba2020-02-23 16:42:51 -07003130 }
3131 if (*len > kbuf->len)
3132 *len = kbuf->len;
3133 } else {
3134 kbuf = ERR_PTR(-ENOBUFS);
3135 }
3136
3137 io_ring_submit_unlock(req->ctx, needs_lock);
3138
3139 return kbuf;
3140}
3141
Jens Axboe4d954c22020-02-27 07:31:19 -07003142static void __user *io_rw_buffer_select(struct io_kiocb *req, size_t *len,
3143 bool needs_lock)
3144{
3145 struct io_buffer *kbuf;
Bijan Mottahedeh4f4eeba2020-05-19 14:52:49 -07003146 u16 bgid;
Jens Axboe4d954c22020-02-27 07:31:19 -07003147
3148 kbuf = (struct io_buffer *) (unsigned long) req->rw.addr;
Bijan Mottahedeh4f4eeba2020-05-19 14:52:49 -07003149 bgid = req->buf_index;
Jens Axboe4d954c22020-02-27 07:31:19 -07003150 kbuf = io_buffer_select(req, len, bgid, kbuf, needs_lock);
3151 if (IS_ERR(kbuf))
3152 return kbuf;
3153 req->rw.addr = (u64) (unsigned long) kbuf;
3154 req->flags |= REQ_F_BUFFER_SELECTED;
3155 return u64_to_user_ptr(kbuf->addr);
3156}
3157
3158#ifdef CONFIG_COMPAT
3159static ssize_t io_compat_import(struct io_kiocb *req, struct iovec *iov,
3160 bool needs_lock)
3161{
3162 struct compat_iovec __user *uiov;
3163 compat_ssize_t clen;
3164 void __user *buf;
3165 ssize_t len;
3166
3167 uiov = u64_to_user_ptr(req->rw.addr);
3168 if (!access_ok(uiov, sizeof(*uiov)))
3169 return -EFAULT;
3170 if (__get_user(clen, &uiov->iov_len))
3171 return -EFAULT;
3172 if (clen < 0)
3173 return -EINVAL;
3174
3175 len = clen;
3176 buf = io_rw_buffer_select(req, &len, needs_lock);
3177 if (IS_ERR(buf))
3178 return PTR_ERR(buf);
3179 iov[0].iov_base = buf;
3180 iov[0].iov_len = (compat_size_t) len;
3181 return 0;
3182}
3183#endif
3184
3185static ssize_t __io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov,
3186 bool needs_lock)
3187{
3188 struct iovec __user *uiov = u64_to_user_ptr(req->rw.addr);
3189 void __user *buf;
3190 ssize_t len;
3191
3192 if (copy_from_user(iov, uiov, sizeof(*uiov)))
3193 return -EFAULT;
3194
3195 len = iov[0].iov_len;
3196 if (len < 0)
3197 return -EINVAL;
3198 buf = io_rw_buffer_select(req, &len, needs_lock);
3199 if (IS_ERR(buf))
3200 return PTR_ERR(buf);
3201 iov[0].iov_base = buf;
3202 iov[0].iov_len = len;
3203 return 0;
3204}
3205
3206static ssize_t io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov,
3207 bool needs_lock)
3208{
Jens Axboedddb3e22020-06-04 11:27:01 -06003209 if (req->flags & REQ_F_BUFFER_SELECTED) {
3210 struct io_buffer *kbuf;
3211
3212 kbuf = (struct io_buffer *) (unsigned long) req->rw.addr;
3213 iov[0].iov_base = u64_to_user_ptr(kbuf->addr);
3214 iov[0].iov_len = kbuf->len;
Jens Axboe4d954c22020-02-27 07:31:19 -07003215 return 0;
Jens Axboedddb3e22020-06-04 11:27:01 -06003216 }
Pavel Begunkovdd201662020-12-19 03:15:43 +00003217 if (req->rw.len != 1)
Jens Axboe4d954c22020-02-27 07:31:19 -07003218 return -EINVAL;
3219
3220#ifdef CONFIG_COMPAT
3221 if (req->ctx->compat)
3222 return io_compat_import(req, iov, needs_lock);
3223#endif
3224
3225 return __io_iov_buffer_select(req, iov, needs_lock);
3226}
3227
Pavel Begunkov847595d2021-02-04 13:52:06 +00003228static int io_import_iovec(int rw, struct io_kiocb *req, struct iovec **iovec,
3229 struct iov_iter *iter, bool needs_lock)
Jens Axboe2b188cc2019-01-07 10:46:33 -07003230{
Jens Axboe9adbd452019-12-20 08:45:55 -07003231 void __user *buf = u64_to_user_ptr(req->rw.addr);
3232 size_t sqe_len = req->rw.len;
Pavel Begunkov847595d2021-02-04 13:52:06 +00003233 u8 opcode = req->opcode;
Jens Axboe4d954c22020-02-27 07:31:19 -07003234 ssize_t ret;
Jens Axboeedafcce2019-01-09 09:16:05 -07003235
Pavel Begunkov7d009162019-11-25 23:14:40 +03003236 if (opcode == IORING_OP_READ_FIXED || opcode == IORING_OP_WRITE_FIXED) {
Jens Axboeedafcce2019-01-09 09:16:05 -07003237 *iovec = NULL;
Jens Axboe9adbd452019-12-20 08:45:55 -07003238 return io_import_fixed(req, rw, iter);
Jens Axboeedafcce2019-01-09 09:16:05 -07003239 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07003240
Jens Axboebcda7ba2020-02-23 16:42:51 -07003241 /* buffer index only valid with fixed read/write, or buffer select */
Bijan Mottahedeh4f4eeba2020-05-19 14:52:49 -07003242 if (req->buf_index && !(req->flags & REQ_F_BUFFER_SELECT))
Jens Axboe9adbd452019-12-20 08:45:55 -07003243 return -EINVAL;
3244
Jens Axboe3a6820f2019-12-22 15:19:35 -07003245 if (opcode == IORING_OP_READ || opcode == IORING_OP_WRITE) {
Jens Axboebcda7ba2020-02-23 16:42:51 -07003246 if (req->flags & REQ_F_BUFFER_SELECT) {
Jens Axboe4d954c22020-02-27 07:31:19 -07003247 buf = io_rw_buffer_select(req, &sqe_len, needs_lock);
Pavel Begunkov867a23e2020-08-20 11:34:39 +03003248 if (IS_ERR(buf))
Jens Axboe4d954c22020-02-27 07:31:19 -07003249 return PTR_ERR(buf);
Jens Axboe3f9d6442020-03-11 12:27:04 -06003250 req->rw.len = sqe_len;
Jens Axboebcda7ba2020-02-23 16:42:51 -07003251 }
3252
Jens Axboe3a6820f2019-12-22 15:19:35 -07003253 ret = import_single_range(rw, buf, sqe_len, *iovec, iter);
3254 *iovec = NULL;
David Laight10fc72e2020-11-07 13:16:25 +00003255 return ret;
Jens Axboe3a6820f2019-12-22 15:19:35 -07003256 }
3257
Jens Axboe4d954c22020-02-27 07:31:19 -07003258 if (req->flags & REQ_F_BUFFER_SELECT) {
3259 ret = io_iov_buffer_select(req, *iovec, needs_lock);
Pavel Begunkov847595d2021-02-04 13:52:06 +00003260 if (!ret)
3261 iov_iter_init(iter, rw, *iovec, 1, (*iovec)->iov_len);
Jens Axboe4d954c22020-02-27 07:31:19 -07003262 *iovec = NULL;
3263 return ret;
3264 }
3265
Christoph Hellwig89cd35c2020-09-25 06:51:41 +02003266 return __import_iovec(rw, buf, sqe_len, UIO_FASTIOV, iovec, iter,
3267 req->ctx->compat);
Jens Axboe2b188cc2019-01-07 10:46:33 -07003268}
3269
Jens Axboe0fef9482020-08-26 10:36:20 -06003270static inline loff_t *io_kiocb_ppos(struct kiocb *kiocb)
3271{
Pavel Begunkov5b09e372020-09-30 22:57:15 +03003272 return (kiocb->ki_filp->f_mode & FMODE_STREAM) ? NULL : &kiocb->ki_pos;
Jens Axboe0fef9482020-08-26 10:36:20 -06003273}
3274
Jens Axboe32960612019-09-23 11:05:34 -06003275/*
3276 * For files that don't have ->read_iter() and ->write_iter(), handle them
3277 * by looping over ->read() or ->write() manually.
3278 */
Jens Axboe4017eb92020-10-22 14:14:12 -06003279static ssize_t loop_rw_iter(int rw, struct io_kiocb *req, struct iov_iter *iter)
Jens Axboe32960612019-09-23 11:05:34 -06003280{
Jens Axboe4017eb92020-10-22 14:14:12 -06003281 struct kiocb *kiocb = &req->rw.kiocb;
3282 struct file *file = req->file;
Jens Axboe32960612019-09-23 11:05:34 -06003283 ssize_t ret = 0;
3284
3285 /*
3286 * Don't support polled IO through this interface, and we can't
3287 * support non-blocking either. For the latter, this just causes
3288 * the kiocb to be handled from an async context.
3289 */
3290 if (kiocb->ki_flags & IOCB_HIPRI)
3291 return -EOPNOTSUPP;
3292 if (kiocb->ki_flags & IOCB_NOWAIT)
3293 return -EAGAIN;
3294
3295 while (iov_iter_count(iter)) {
Pavel Begunkov311ae9e2019-11-24 11:58:24 +03003296 struct iovec iovec;
Jens Axboe32960612019-09-23 11:05:34 -06003297 ssize_t nr;
3298
Pavel Begunkov311ae9e2019-11-24 11:58:24 +03003299 if (!iov_iter_is_bvec(iter)) {
3300 iovec = iov_iter_iovec(iter);
3301 } else {
Jens Axboe4017eb92020-10-22 14:14:12 -06003302 iovec.iov_base = u64_to_user_ptr(req->rw.addr);
3303 iovec.iov_len = req->rw.len;
Pavel Begunkov311ae9e2019-11-24 11:58:24 +03003304 }
3305
Jens Axboe32960612019-09-23 11:05:34 -06003306 if (rw == READ) {
3307 nr = file->f_op->read(file, iovec.iov_base,
Jens Axboe0fef9482020-08-26 10:36:20 -06003308 iovec.iov_len, io_kiocb_ppos(kiocb));
Jens Axboe32960612019-09-23 11:05:34 -06003309 } else {
3310 nr = file->f_op->write(file, iovec.iov_base,
Jens Axboe0fef9482020-08-26 10:36:20 -06003311 iovec.iov_len, io_kiocb_ppos(kiocb));
Jens Axboe32960612019-09-23 11:05:34 -06003312 }
3313
3314 if (nr < 0) {
3315 if (!ret)
3316 ret = nr;
3317 break;
3318 }
Jens Axboe109dda42022-03-18 11:28:13 -06003319 ret += nr;
Jens Axboe16c8d2d2021-09-12 06:45:07 -06003320 if (!iov_iter_is_bvec(iter)) {
3321 iov_iter_advance(iter, nr);
3322 } else {
Jens Axboe16c8d2d2021-09-12 06:45:07 -06003323 req->rw.addr += nr;
Jens Axboe109dda42022-03-18 11:28:13 -06003324 req->rw.len -= nr;
3325 if (!req->rw.len)
3326 break;
Jens Axboe16c8d2d2021-09-12 06:45:07 -06003327 }
Jens Axboe32960612019-09-23 11:05:34 -06003328 if (nr != iovec.iov_len)
3329 break;
Jens Axboe32960612019-09-23 11:05:34 -06003330 }
3331
3332 return ret;
3333}
3334
Jens Axboeff6165b2020-08-13 09:47:43 -06003335static void io_req_map_rw(struct io_kiocb *req, const struct iovec *iovec,
3336 const struct iovec *fast_iov, struct iov_iter *iter)
Jens Axboef67676d2019-12-02 11:03:47 -07003337{
Jens Axboee8c2bc12020-08-15 18:44:09 -07003338 struct io_async_rw *rw = req->async_data;
Pavel Begunkovb64e3442020-07-13 22:59:18 +03003339
Jens Axboeff6165b2020-08-13 09:47:43 -06003340 memcpy(&rw->iter, iter, sizeof(*iter));
Pavel Begunkovafb87652020-09-06 00:45:46 +03003341 rw->free_iovec = iovec;
Jens Axboe227c0c92020-08-13 11:51:40 -06003342 rw->bytes_done = 0;
Jens Axboeff6165b2020-08-13 09:47:43 -06003343 /* can only be fixed buffers, no need to do anything */
Pavel Begunkov9c3a2052020-11-23 23:20:27 +00003344 if (iov_iter_is_bvec(iter))
Jens Axboeff6165b2020-08-13 09:47:43 -06003345 return;
Pavel Begunkovb64e3442020-07-13 22:59:18 +03003346 if (!iovec) {
Jens Axboeff6165b2020-08-13 09:47:43 -06003347 unsigned iov_off = 0;
3348
3349 rw->iter.iov = rw->fast_iov;
3350 if (iter->iov != fast_iov) {
3351 iov_off = iter->iov - fast_iov;
3352 rw->iter.iov += iov_off;
3353 }
3354 if (rw->fast_iov != fast_iov)
3355 memcpy(rw->fast_iov + iov_off, fast_iov + iov_off,
Xiaoguang Wang45097da2020-04-08 22:29:58 +08003356 sizeof(struct iovec) * iter->nr_segs);
Pavel Begunkov99bc4c32020-02-07 22:04:45 +03003357 } else {
3358 req->flags |= REQ_F_NEED_CLEANUP;
Jens Axboef67676d2019-12-02 11:03:47 -07003359 }
3360}
3361
Pavel Begunkov6cb78682021-02-28 22:35:17 +00003362static inline int io_alloc_async_data(struct io_kiocb *req)
Xiaoguang Wang3d9932a2020-03-27 15:36:52 +08003363{
Jens Axboee8c2bc12020-08-15 18:44:09 -07003364 WARN_ON_ONCE(!io_op_defs[req->opcode].async_size);
3365 req->async_data = kmalloc(io_op_defs[req->opcode].async_size, GFP_KERNEL);
3366 return req->async_data == NULL;
Xiaoguang Wang3d9932a2020-03-27 15:36:52 +08003367}
3368
Jens Axboeff6165b2020-08-13 09:47:43 -06003369static int io_setup_async_rw(struct io_kiocb *req, const struct iovec *iovec,
3370 const struct iovec *fast_iov,
Jens Axboe227c0c92020-08-13 11:51:40 -06003371 struct iov_iter *iter, bool force)
Jens Axboeb7bb4f72019-12-15 22:13:43 -07003372{
Pavel Begunkov26f05052021-02-28 22:35:18 +00003373 if (!force && !io_op_defs[req->opcode].needs_async_setup)
Jens Axboe74566df2020-01-13 19:23:24 -07003374 return 0;
Jens Axboee8c2bc12020-08-15 18:44:09 -07003375 if (!req->async_data) {
Jens Axboecd658692021-09-10 11:19:14 -06003376 struct io_async_rw *iorw;
3377
Pavel Begunkov6cb78682021-02-28 22:35:17 +00003378 if (io_alloc_async_data(req)) {
Pavel Begunkov6bf985d2021-02-04 13:52:01 +00003379 kfree(iovec);
Jens Axboe5d204bc2020-01-31 12:06:52 -07003380 return -ENOMEM;
Pavel Begunkov6bf985d2021-02-04 13:52:01 +00003381 }
Jens Axboeb7bb4f72019-12-15 22:13:43 -07003382
Jens Axboeff6165b2020-08-13 09:47:43 -06003383 io_req_map_rw(req, iovec, fast_iov, iter);
Jens Axboecd658692021-09-10 11:19:14 -06003384 iorw = req->async_data;
3385 /* we've copied and mapped the iter, ensure state is saved */
3386 iov_iter_save_state(&iorw->iter, &iorw->iter_state);
Jens Axboe5d204bc2020-01-31 12:06:52 -07003387 }
Jens Axboeb7bb4f72019-12-15 22:13:43 -07003388 return 0;
Jens Axboef67676d2019-12-02 11:03:47 -07003389}
3390
Pavel Begunkov73debe62020-09-30 22:57:54 +03003391static inline int io_rw_prep_async(struct io_kiocb *req, int rw)
Pavel Begunkovc3e330a2020-07-13 22:59:19 +03003392{
Jens Axboee8c2bc12020-08-15 18:44:09 -07003393 struct io_async_rw *iorw = req->async_data;
Pavel Begunkovf4bff102020-09-06 00:45:45 +03003394 struct iovec *iov = iorw->fast_iov;
Pavel Begunkov847595d2021-02-04 13:52:06 +00003395 int ret;
Pavel Begunkovc3e330a2020-07-13 22:59:19 +03003396
Pavel Begunkov2846c482020-11-07 13:16:27 +00003397 ret = io_import_iovec(rw, req, &iov, &iorw->iter, false);
Pavel Begunkovc3e330a2020-07-13 22:59:19 +03003398 if (unlikely(ret < 0))
3399 return ret;
3400
Pavel Begunkovab0b1962020-09-06 00:45:47 +03003401 iorw->bytes_done = 0;
3402 iorw->free_iovec = iov;
3403 if (iov)
3404 req->flags |= REQ_F_NEED_CLEANUP;
Jens Axboecd658692021-09-10 11:19:14 -06003405 iov_iter_save_state(&iorw->iter, &iorw->iter_state);
Pavel Begunkovc3e330a2020-07-13 22:59:19 +03003406 return 0;
3407}
3408
Pavel Begunkov73debe62020-09-30 22:57:54 +03003409static int io_read_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboef67676d2019-12-02 11:03:47 -07003410{
Jens Axboe3529d8c2019-12-19 18:24:38 -07003411 if (unlikely(!(req->file->f_mode & FMODE_READ)))
3412 return -EBADF;
Jens Axboe5d329e12021-09-14 11:08:37 -06003413 return io_prep_rw(req, sqe, READ);
Jens Axboef67676d2019-12-02 11:03:47 -07003414}
3415
Jens Axboec1dd91d2020-08-03 16:43:59 -06003416/*
3417 * This is our waitqueue callback handler, registered through lock_page_async()
3418 * when we initially tried to do the IO with the iocb armed our waitqueue.
3419 * This gets called when the page is unlocked, and we generally expect that to
3420 * happen when the page IO is completed and the page is now uptodate. This will
3421 * queue a task_work based retry of the operation, attempting to copy the data
3422 * again. If the latter fails because the page was NOT uptodate, then we will
3423 * do a thread based blocking retry of the operation. That's the unexpected
3424 * slow path.
3425 */
Jens Axboebcf5a062020-05-22 09:24:42 -06003426static int io_async_buf_func(struct wait_queue_entry *wait, unsigned mode,
3427 int sync, void *arg)
3428{
3429 struct wait_page_queue *wpq;
3430 struct io_kiocb *req = wait->private;
Jens Axboebcf5a062020-05-22 09:24:42 -06003431 struct wait_page_key *key = arg;
Jens Axboebcf5a062020-05-22 09:24:42 -06003432
3433 wpq = container_of(wait, struct wait_page_queue, wait);
3434
Linus Torvaldscdc8fcb2020-08-03 13:01:22 -07003435 if (!wake_page_match(wpq, key))
3436 return 0;
3437
Hao Xuc8d317a2020-09-29 20:00:45 +08003438 req->rw.kiocb.ki_flags &= ~IOCB_WAITQ;
Jens Axboebcf5a062020-05-22 09:24:42 -06003439 list_del_init(&wait->entry);
Pavel Begunkov921b9052021-02-12 03:23:53 +00003440 io_req_task_queue(req);
Jens Axboebcf5a062020-05-22 09:24:42 -06003441 return 1;
3442}
3443
Jens Axboec1dd91d2020-08-03 16:43:59 -06003444/*
3445 * This controls whether a given IO request should be armed for async page
3446 * based retry. If we return false here, the request is handed to the async
3447 * worker threads for retry. If we're doing buffered reads on a regular file,
3448 * we prepare a private wait_page_queue entry and retry the operation. This
3449 * will either succeed because the page is now uptodate and unlocked, or it
3450 * will register a callback when the page is unlocked at IO completion. Through
3451 * that callback, io_uring uses task_work to setup a retry of the operation.
3452 * That retry will attempt the buffered read again. The retry will generally
3453 * succeed, or in rare cases where it fails, we then fall back to using the
3454 * async worker threads for a blocking retry.
3455 */
Jens Axboe227c0c92020-08-13 11:51:40 -06003456static bool io_rw_should_retry(struct io_kiocb *req)
Jens Axboebcf5a062020-05-22 09:24:42 -06003457{
Jens Axboee8c2bc12020-08-15 18:44:09 -07003458 struct io_async_rw *rw = req->async_data;
3459 struct wait_page_queue *wait = &rw->wpq;
Jens Axboebcf5a062020-05-22 09:24:42 -06003460 struct kiocb *kiocb = &req->rw.kiocb;
Jens Axboebcf5a062020-05-22 09:24:42 -06003461
3462 /* never retry for NOWAIT, we just complete with -EAGAIN */
3463 if (req->flags & REQ_F_NOWAIT)
3464 return false;
3465
Jens Axboe227c0c92020-08-13 11:51:40 -06003466 /* Only for buffered IO */
Jens Axboe3b2a4432020-08-16 10:58:43 -07003467 if (kiocb->ki_flags & (IOCB_DIRECT | IOCB_HIPRI))
Jens Axboebcf5a062020-05-22 09:24:42 -06003468 return false;
Jens Axboe3b2a4432020-08-16 10:58:43 -07003469
Jens Axboebcf5a062020-05-22 09:24:42 -06003470 /*
3471 * just use poll if we can, and don't attempt if the fs doesn't
3472 * support callback based unlocks
3473 */
3474 if (file_can_poll(req->file) || !(req->file->f_mode & FMODE_BUF_RASYNC))
3475 return false;
3476
Jens Axboe3b2a4432020-08-16 10:58:43 -07003477 wait->wait.func = io_async_buf_func;
3478 wait->wait.private = req;
3479 wait->wait.flags = 0;
3480 INIT_LIST_HEAD(&wait->wait.entry);
3481 kiocb->ki_flags |= IOCB_WAITQ;
Hao Xuc8d317a2020-09-29 20:00:45 +08003482 kiocb->ki_flags &= ~IOCB_NOWAIT;
Jens Axboe3b2a4432020-08-16 10:58:43 -07003483 kiocb->ki_waitq = wait;
Jens Axboe3b2a4432020-08-16 10:58:43 -07003484 return true;
Jens Axboebcf5a062020-05-22 09:24:42 -06003485}
3486
Pavel Begunkovaeab9502021-06-14 02:36:24 +01003487static inline int io_iter_do_read(struct io_kiocb *req, struct iov_iter *iter)
Jens Axboebcf5a062020-05-22 09:24:42 -06003488{
3489 if (req->file->f_op->read_iter)
3490 return call_read_iter(req->file, &req->rw.kiocb, iter);
Guoyu Huang2dd21112020-08-05 03:53:50 -07003491 else if (req->file->f_op->read)
Jens Axboe4017eb92020-10-22 14:14:12 -06003492 return loop_rw_iter(READ, req, iter);
Guoyu Huang2dd21112020-08-05 03:53:50 -07003493 else
3494 return -EINVAL;
Jens Axboebcf5a062020-05-22 09:24:42 -06003495}
3496
Ming Lei7db30432021-08-21 23:07:51 +08003497static bool need_read_all(struct io_kiocb *req)
3498{
3499 return req->flags & REQ_F_ISREG ||
3500 S_ISBLK(file_inode(req->file)->i_mode);
3501}
3502
Pavel Begunkov889fca72021-02-10 00:03:09 +00003503static int io_read(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe2b188cc2019-01-07 10:46:33 -07003504{
3505 struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
Jens Axboe9adbd452019-12-20 08:45:55 -07003506 struct kiocb *kiocb = &req->rw.kiocb;
Jens Axboeff6165b2020-08-13 09:47:43 -06003507 struct iov_iter __iter, *iter = &__iter;
Jens Axboee8c2bc12020-08-15 18:44:09 -07003508 struct io_async_rw *rw = req->async_data;
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003509 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
Jens Axboecd658692021-09-10 11:19:14 -06003510 struct iov_iter_state __state, *state;
3511 ssize_t ret, ret2;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003512
Pavel Begunkov2846c482020-11-07 13:16:27 +00003513 if (rw) {
Jens Axboee8c2bc12020-08-15 18:44:09 -07003514 iter = &rw->iter;
Jens Axboecd658692021-09-10 11:19:14 -06003515 state = &rw->iter_state;
3516 /*
3517 * We come here from an earlier attempt, restore our state to
3518 * match in case it doesn't. It's cheap enough that we don't
3519 * need to make this conditional.
3520 */
3521 iov_iter_restore(iter, state);
Pavel Begunkov2846c482020-11-07 13:16:27 +00003522 iovec = NULL;
3523 } else {
3524 ret = io_import_iovec(READ, req, &iovec, iter, !force_nonblock);
3525 if (ret < 0)
3526 return ret;
Jens Axboecd658692021-09-10 11:19:14 -06003527 state = &__state;
3528 iov_iter_save_state(iter, state);
Pavel Begunkov2846c482020-11-07 13:16:27 +00003529 }
Jens Axboecd658692021-09-10 11:19:14 -06003530 req->result = iov_iter_count(iter);
Jens Axboe2b188cc2019-01-07 10:46:33 -07003531
Jens Axboefd6c2e42019-12-18 12:19:41 -07003532 /* Ensure we clear previously set non-block flag */
3533 if (!force_nonblock)
Jens Axboe29de5f62020-02-20 09:56:08 -07003534 kiocb->ki_flags &= ~IOCB_NOWAIT;
Pavel Begunkova88fc402020-09-30 22:57:53 +03003535 else
3536 kiocb->ki_flags |= IOCB_NOWAIT;
3537
Pavel Begunkov24c74672020-06-21 13:09:51 +03003538 /* If the file doesn't support async, just async punt */
Pavel Begunkovb191e2d2021-08-09 13:04:03 +01003539 if (force_nonblock && !io_file_supports_nowait(req, READ)) {
Pavel Begunkov6713e7a2021-02-04 13:51:59 +00003540 ret = io_setup_async_rw(req, iovec, inline_vecs, iter, true);
Pavel Begunkov6bf985d2021-02-04 13:52:01 +00003541 return ret ?: -EAGAIN;
Pavel Begunkov6713e7a2021-02-04 13:51:59 +00003542 }
Jens Axboe9e645e112019-05-10 16:07:28 -06003543
Jens Axboecd658692021-09-10 11:19:14 -06003544 ret = rw_verify_area(READ, req->file, io_kiocb_ppos(kiocb), req->result);
Pavel Begunkov5ea5dd42021-02-04 13:52:03 +00003545 if (unlikely(ret)) {
3546 kfree(iovec);
3547 return ret;
3548 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07003549
Jens Axboe227c0c92020-08-13 11:51:40 -06003550 ret = io_iter_do_read(req, iter);
Jens Axboe32960612019-09-23 11:05:34 -06003551
Jens Axboe230d50d2021-04-01 20:41:15 -06003552 if (ret == -EAGAIN || (req->flags & REQ_F_REISSUE)) {
Pavel Begunkov6ad7f232021-04-08 01:54:39 +01003553 req->flags &= ~REQ_F_REISSUE;
Jens Axboeeefdf302020-08-27 16:40:19 -06003554 /* IOPOLL retry should happen for io-wq threads */
3555 if (!force_nonblock && !(req->ctx->flags & IORING_SETUP_IOPOLL))
Jens Axboef91daf52020-08-15 15:58:42 -07003556 goto done;
Pavel Begunkov75c668c2021-02-04 13:52:05 +00003557 /* no retry on NONBLOCK nor RWF_NOWAIT */
3558 if (req->flags & REQ_F_NOWAIT)
Jens Axboe355afae2020-09-02 09:30:31 -06003559 goto done;
Jens Axboef38c7e32020-09-25 15:23:43 -06003560 ret = 0;
Jens Axboe230d50d2021-04-01 20:41:15 -06003561 } else if (ret == -EIOCBQUEUED) {
3562 goto out_free;
Jens Axboecd658692021-09-10 11:19:14 -06003563 } else if (ret <= 0 || ret == req->result || !force_nonblock ||
Ming Lei7db30432021-08-21 23:07:51 +08003564 (req->flags & REQ_F_NOWAIT) || !need_read_all(req)) {
Pavel Begunkov7335e3b2021-02-04 13:52:02 +00003565 /* read all, failed, already did sync or don't want to retry */
Jens Axboe00d23d52020-08-25 12:59:22 -06003566 goto done;
Jens Axboe227c0c92020-08-13 11:51:40 -06003567 }
3568
Jens Axboecd658692021-09-10 11:19:14 -06003569 /*
3570 * Don't depend on the iter state matching what was consumed, or being
3571 * untouched in case of error. Restore it and we'll advance it
3572 * manually if we need to.
3573 */
3574 iov_iter_restore(iter, state);
3575
Jens Axboe227c0c92020-08-13 11:51:40 -06003576 ret2 = io_setup_async_rw(req, iovec, inline_vecs, iter, true);
Pavel Begunkov6bf985d2021-02-04 13:52:01 +00003577 if (ret2)
3578 return ret2;
3579
Pavel Begunkovfe1cdd52021-02-17 21:02:36 +00003580 iovec = NULL;
Jens Axboee8c2bc12020-08-15 18:44:09 -07003581 rw = req->async_data;
Jens Axboecd658692021-09-10 11:19:14 -06003582 /*
3583 * Now use our persistent iterator and state, if we aren't already.
3584 * We've restored and mapped the iter to match.
3585 */
3586 if (iter != &rw->iter) {
3587 iter = &rw->iter;
3588 state = &rw->iter_state;
3589 }
Jens Axboe227c0c92020-08-13 11:51:40 -06003590
Pavel Begunkovb23df912021-02-04 13:52:04 +00003591 do {
Jens Axboecd658692021-09-10 11:19:14 -06003592 /*
3593 * We end up here because of a partial read, either from
3594 * above or inside this loop. Advance the iter by the bytes
3595 * that were consumed.
3596 */
3597 iov_iter_advance(iter, ret);
3598 if (!iov_iter_count(iter))
3599 break;
Pavel Begunkovb23df912021-02-04 13:52:04 +00003600 rw->bytes_done += ret;
Jens Axboecd658692021-09-10 11:19:14 -06003601 iov_iter_save_state(iter, state);
3602
Pavel Begunkovb23df912021-02-04 13:52:04 +00003603 /* if we can retry, do so with the callbacks armed */
3604 if (!io_rw_should_retry(req)) {
3605 kiocb->ki_flags &= ~IOCB_WAITQ;
3606 return -EAGAIN;
3607 }
3608
Pavel Begunkov98aada62022-10-16 22:42:58 +01003609 req->result = iov_iter_count(iter);
Pavel Begunkovb23df912021-02-04 13:52:04 +00003610 /*
3611 * Now retry read with the IOCB_WAITQ parts set in the iocb. If
3612 * we get -EIOCBQUEUED, then we'll get a notification when the
3613 * desired page gets unlocked. We can also get a partial read
3614 * here, and if we do, then just retry at the new offset.
3615 */
3616 ret = io_iter_do_read(req, iter);
3617 if (ret == -EIOCBQUEUED)
3618 return 0;
Jens Axboe227c0c92020-08-13 11:51:40 -06003619 /* we got some bytes, but not all. retry. */
Jens Axboeb5b0ecb2021-03-04 21:02:58 -07003620 kiocb->ki_flags &= ~IOCB_WAITQ;
Jens Axboecd658692021-09-10 11:19:14 -06003621 iov_iter_restore(iter, state);
3622 } while (ret > 0);
Jens Axboe227c0c92020-08-13 11:51:40 -06003623done:
Pavel Begunkov889fca72021-02-10 00:03:09 +00003624 kiocb_done(kiocb, ret, issue_flags);
Pavel Begunkovfe1cdd52021-02-17 21:02:36 +00003625out_free:
3626 /* it's faster to check here then delegate to kfree */
3627 if (iovec)
3628 kfree(iovec);
Pavel Begunkov5ea5dd42021-02-04 13:52:03 +00003629 return 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003630}
3631
Pavel Begunkov73debe62020-09-30 22:57:54 +03003632static int io_write_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboef67676d2019-12-02 11:03:47 -07003633{
Jens Axboe3529d8c2019-12-19 18:24:38 -07003634 if (unlikely(!(req->file->f_mode & FMODE_WRITE)))
3635 return -EBADF;
Jens Axboe5d329e12021-09-14 11:08:37 -06003636 return io_prep_rw(req, sqe, WRITE);
Jens Axboef67676d2019-12-02 11:03:47 -07003637}
3638
Pavel Begunkov889fca72021-02-10 00:03:09 +00003639static int io_write(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe2b188cc2019-01-07 10:46:33 -07003640{
3641 struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
Jens Axboe9adbd452019-12-20 08:45:55 -07003642 struct kiocb *kiocb = &req->rw.kiocb;
Jens Axboeff6165b2020-08-13 09:47:43 -06003643 struct iov_iter __iter, *iter = &__iter;
Jens Axboee8c2bc12020-08-15 18:44:09 -07003644 struct io_async_rw *rw = req->async_data;
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003645 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
Jens Axboecd658692021-09-10 11:19:14 -06003646 struct iov_iter_state __state, *state;
3647 ssize_t ret, ret2;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003648
Pavel Begunkov2846c482020-11-07 13:16:27 +00003649 if (rw) {
Jens Axboee8c2bc12020-08-15 18:44:09 -07003650 iter = &rw->iter;
Jens Axboecd658692021-09-10 11:19:14 -06003651 state = &rw->iter_state;
3652 iov_iter_restore(iter, state);
Pavel Begunkov2846c482020-11-07 13:16:27 +00003653 iovec = NULL;
3654 } else {
3655 ret = io_import_iovec(WRITE, req, &iovec, iter, !force_nonblock);
3656 if (ret < 0)
3657 return ret;
Jens Axboecd658692021-09-10 11:19:14 -06003658 state = &__state;
3659 iov_iter_save_state(iter, state);
Pavel Begunkov2846c482020-11-07 13:16:27 +00003660 }
Jens Axboecd658692021-09-10 11:19:14 -06003661 req->result = iov_iter_count(iter);
Jens Axboe2b188cc2019-01-07 10:46:33 -07003662
Jens Axboefd6c2e42019-12-18 12:19:41 -07003663 /* Ensure we clear previously set non-block flag */
3664 if (!force_nonblock)
Pavel Begunkova88fc402020-09-30 22:57:53 +03003665 kiocb->ki_flags &= ~IOCB_NOWAIT;
3666 else
3667 kiocb->ki_flags |= IOCB_NOWAIT;
Jens Axboefd6c2e42019-12-18 12:19:41 -07003668
Pavel Begunkov24c74672020-06-21 13:09:51 +03003669 /* If the file doesn't support async, just async punt */
Pavel Begunkovb191e2d2021-08-09 13:04:03 +01003670 if (force_nonblock && !io_file_supports_nowait(req, WRITE))
Jens Axboef67676d2019-12-02 11:03:47 -07003671 goto copy_iov;
Jens Axboef67676d2019-12-02 11:03:47 -07003672
Jens Axboe10d59342019-12-09 20:16:22 -07003673 /* file path doesn't support NOWAIT for non-direct_IO */
3674 if (force_nonblock && !(kiocb->ki_flags & IOCB_DIRECT) &&
3675 (req->flags & REQ_F_ISREG))
Jens Axboef67676d2019-12-02 11:03:47 -07003676 goto copy_iov;
Jens Axboe9e645e112019-05-10 16:07:28 -06003677
Jens Axboecd658692021-09-10 11:19:14 -06003678 ret = rw_verify_area(WRITE, req->file, io_kiocb_ppos(kiocb), req->result);
Pavel Begunkovfa15baf2020-08-01 13:50:02 +03003679 if (unlikely(ret))
3680 goto out_free;
Roman Penyaev9bf79332019-03-25 20:09:24 +01003681
Pavel Begunkovfa15baf2020-08-01 13:50:02 +03003682 /*
3683 * Open-code file_start_write here to grab freeze protection,
3684 * which will be released by another thread in
3685 * io_complete_rw(). Fool lockdep by telling it the lock got
3686 * released so that it doesn't complain about the held lock when
3687 * we return to userspace.
3688 */
3689 if (req->flags & REQ_F_ISREG) {
Darrick J. Wong8a3c84b2020-11-10 16:50:21 -08003690 sb_start_write(file_inode(req->file)->i_sb);
Pavel Begunkovfa15baf2020-08-01 13:50:02 +03003691 __sb_writers_release(file_inode(req->file)->i_sb,
3692 SB_FREEZE_WRITE);
3693 }
3694 kiocb->ki_flags |= IOCB_WRITE;
Roman Penyaev9bf79332019-03-25 20:09:24 +01003695
Pavel Begunkovfa15baf2020-08-01 13:50:02 +03003696 if (req->file->f_op->write_iter)
Jens Axboeff6165b2020-08-13 09:47:43 -06003697 ret2 = call_write_iter(req->file, kiocb, iter);
Guoyu Huang2dd21112020-08-05 03:53:50 -07003698 else if (req->file->f_op->write)
Jens Axboe4017eb92020-10-22 14:14:12 -06003699 ret2 = loop_rw_iter(WRITE, req, iter);
Guoyu Huang2dd21112020-08-05 03:53:50 -07003700 else
3701 ret2 = -EINVAL;
Jens Axboe4ed734b2020-03-20 11:23:41 -06003702
Pavel Begunkov6ad7f232021-04-08 01:54:39 +01003703 if (req->flags & REQ_F_REISSUE) {
3704 req->flags &= ~REQ_F_REISSUE;
Jens Axboe230d50d2021-04-01 20:41:15 -06003705 ret2 = -EAGAIN;
Pavel Begunkov6ad7f232021-04-08 01:54:39 +01003706 }
Jens Axboe230d50d2021-04-01 20:41:15 -06003707
Pavel Begunkovfa15baf2020-08-01 13:50:02 +03003708 /*
3709 * Raw bdev writes will return -EOPNOTSUPP for IOCB_NOWAIT. Just
3710 * retry them without IOCB_NOWAIT.
3711 */
3712 if (ret2 == -EOPNOTSUPP && (kiocb->ki_flags & IOCB_NOWAIT))
3713 ret2 = -EAGAIN;
Pavel Begunkov75c668c2021-02-04 13:52:05 +00003714 /* no retry on NONBLOCK nor RWF_NOWAIT */
3715 if (ret2 == -EAGAIN && (req->flags & REQ_F_NOWAIT))
Jens Axboe355afae2020-09-02 09:30:31 -06003716 goto done;
Pavel Begunkovfa15baf2020-08-01 13:50:02 +03003717 if (!force_nonblock || ret2 != -EAGAIN) {
Jens Axboeeefdf302020-08-27 16:40:19 -06003718 /* IOPOLL retry should happen for io-wq threads */
3719 if ((req->ctx->flags & IORING_SETUP_IOPOLL) && ret2 == -EAGAIN)
3720 goto copy_iov;
Jens Axboe355afae2020-09-02 09:30:31 -06003721done:
Pavel Begunkov889fca72021-02-10 00:03:09 +00003722 kiocb_done(kiocb, ret2, issue_flags);
Pavel Begunkovfa15baf2020-08-01 13:50:02 +03003723 } else {
Jens Axboef67676d2019-12-02 11:03:47 -07003724copy_iov:
Jens Axboecd658692021-09-10 11:19:14 -06003725 iov_iter_restore(iter, state);
Jens Axboe227c0c92020-08-13 11:51:40 -06003726 ret = io_setup_async_rw(req, iovec, inline_vecs, iter, false);
Jens Axboe295219a2022-08-25 10:19:08 -06003727 if (!ret) {
3728 if (kiocb->ki_flags & IOCB_WRITE)
3729 kiocb_end_write(req);
3730 return -EAGAIN;
3731 }
3732 return ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003733 }
Jens Axboe31b51512019-01-18 22:56:34 -07003734out_free:
Pavel Begunkovf261c162020-08-20 11:34:10 +03003735 /* it's reportedly faster than delegating the null check to kfree() */
Pavel Begunkov252917c2020-07-13 22:59:20 +03003736 if (iovec)
Xiaoguang Wang6f2cc162020-06-18 15:01:56 +08003737 kfree(iovec);
Jens Axboe2b188cc2019-01-07 10:46:33 -07003738 return ret;
3739}
3740
Jens Axboe80a261f2020-09-28 14:23:58 -06003741static int io_renameat_prep(struct io_kiocb *req,
3742 const struct io_uring_sqe *sqe)
3743{
3744 struct io_rename *ren = &req->rename;
3745 const char __user *oldf, *newf;
3746
Jens Axboeed7eb252021-06-23 09:04:13 -06003747 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3748 return -EINVAL;
Pavel Begunkov26578cd2021-08-20 10:36:37 +01003749 if (sqe->ioprio || sqe->buf_index || sqe->splice_fd_in)
Jens Axboeed7eb252021-06-23 09:04:13 -06003750 return -EINVAL;
Jens Axboe80a261f2020-09-28 14:23:58 -06003751 if (unlikely(req->flags & REQ_F_FIXED_FILE))
3752 return -EBADF;
3753
3754 ren->old_dfd = READ_ONCE(sqe->fd);
3755 oldf = u64_to_user_ptr(READ_ONCE(sqe->addr));
3756 newf = u64_to_user_ptr(READ_ONCE(sqe->addr2));
3757 ren->new_dfd = READ_ONCE(sqe->len);
3758 ren->flags = READ_ONCE(sqe->rename_flags);
3759
3760 ren->oldpath = getname(oldf);
3761 if (IS_ERR(ren->oldpath))
3762 return PTR_ERR(ren->oldpath);
3763
3764 ren->newpath = getname(newf);
3765 if (IS_ERR(ren->newpath)) {
3766 putname(ren->oldpath);
3767 return PTR_ERR(ren->newpath);
3768 }
3769
3770 req->flags |= REQ_F_NEED_CLEANUP;
3771 return 0;
3772}
3773
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003774static int io_renameat(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe80a261f2020-09-28 14:23:58 -06003775{
3776 struct io_rename *ren = &req->rename;
3777 int ret;
3778
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003779 if (issue_flags & IO_URING_F_NONBLOCK)
Jens Axboe80a261f2020-09-28 14:23:58 -06003780 return -EAGAIN;
3781
3782 ret = do_renameat2(ren->old_dfd, ren->oldpath, ren->new_dfd,
3783 ren->newpath, ren->flags);
3784
3785 req->flags &= ~REQ_F_NEED_CLEANUP;
3786 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01003787 req_set_fail(req);
Jens Axboe80a261f2020-09-28 14:23:58 -06003788 io_req_complete(req, ret);
3789 return 0;
3790}
3791
Jens Axboe14a11432020-09-28 14:27:37 -06003792static int io_unlinkat_prep(struct io_kiocb *req,
3793 const struct io_uring_sqe *sqe)
3794{
3795 struct io_unlink *un = &req->unlink;
3796 const char __user *fname;
3797
Jens Axboe22634bc2021-06-23 09:07:45 -06003798 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3799 return -EINVAL;
Pavel Begunkov26578cd2021-08-20 10:36:37 +01003800 if (sqe->ioprio || sqe->off || sqe->len || sqe->buf_index ||
3801 sqe->splice_fd_in)
Jens Axboe22634bc2021-06-23 09:07:45 -06003802 return -EINVAL;
Jens Axboe14a11432020-09-28 14:27:37 -06003803 if (unlikely(req->flags & REQ_F_FIXED_FILE))
3804 return -EBADF;
3805
3806 un->dfd = READ_ONCE(sqe->fd);
3807
3808 un->flags = READ_ONCE(sqe->unlink_flags);
3809 if (un->flags & ~AT_REMOVEDIR)
3810 return -EINVAL;
3811
3812 fname = u64_to_user_ptr(READ_ONCE(sqe->addr));
3813 un->filename = getname(fname);
3814 if (IS_ERR(un->filename))
3815 return PTR_ERR(un->filename);
3816
3817 req->flags |= REQ_F_NEED_CLEANUP;
3818 return 0;
3819}
3820
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003821static int io_unlinkat(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe14a11432020-09-28 14:27:37 -06003822{
3823 struct io_unlink *un = &req->unlink;
3824 int ret;
3825
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003826 if (issue_flags & IO_URING_F_NONBLOCK)
Jens Axboe14a11432020-09-28 14:27:37 -06003827 return -EAGAIN;
3828
3829 if (un->flags & AT_REMOVEDIR)
3830 ret = do_rmdir(un->dfd, un->filename);
3831 else
3832 ret = do_unlinkat(un->dfd, un->filename);
3833
3834 req->flags &= ~REQ_F_NEED_CLEANUP;
3835 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01003836 req_set_fail(req);
Jens Axboe14a11432020-09-28 14:27:37 -06003837 io_req_complete(req, ret);
3838 return 0;
3839}
3840
Dmitry Kadasheve34a02d2021-07-08 13:34:45 +07003841static int io_mkdirat_prep(struct io_kiocb *req,
3842 const struct io_uring_sqe *sqe)
3843{
3844 struct io_mkdir *mkd = &req->mkdir;
3845 const char __user *fname;
3846
3847 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3848 return -EINVAL;
3849 if (sqe->ioprio || sqe->off || sqe->rw_flags || sqe->buf_index ||
3850 sqe->splice_fd_in)
3851 return -EINVAL;
3852 if (unlikely(req->flags & REQ_F_FIXED_FILE))
3853 return -EBADF;
3854
3855 mkd->dfd = READ_ONCE(sqe->fd);
3856 mkd->mode = READ_ONCE(sqe->len);
3857
3858 fname = u64_to_user_ptr(READ_ONCE(sqe->addr));
3859 mkd->filename = getname(fname);
3860 if (IS_ERR(mkd->filename))
3861 return PTR_ERR(mkd->filename);
3862
3863 req->flags |= REQ_F_NEED_CLEANUP;
3864 return 0;
3865}
3866
3867static int io_mkdirat(struct io_kiocb *req, int issue_flags)
3868{
3869 struct io_mkdir *mkd = &req->mkdir;
3870 int ret;
3871
3872 if (issue_flags & IO_URING_F_NONBLOCK)
3873 return -EAGAIN;
3874
3875 ret = do_mkdirat(mkd->dfd, mkd->filename, mkd->mode);
3876
3877 req->flags &= ~REQ_F_NEED_CLEANUP;
3878 if (ret < 0)
3879 req_set_fail(req);
3880 io_req_complete(req, ret);
3881 return 0;
3882}
3883
Dmitry Kadashev7a8721f2021-07-08 13:34:46 +07003884static int io_symlinkat_prep(struct io_kiocb *req,
3885 const struct io_uring_sqe *sqe)
3886{
3887 struct io_symlink *sl = &req->symlink;
3888 const char __user *oldpath, *newpath;
3889
3890 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3891 return -EINVAL;
3892 if (sqe->ioprio || sqe->len || sqe->rw_flags || sqe->buf_index ||
3893 sqe->splice_fd_in)
3894 return -EINVAL;
3895 if (unlikely(req->flags & REQ_F_FIXED_FILE))
3896 return -EBADF;
3897
3898 sl->new_dfd = READ_ONCE(sqe->fd);
3899 oldpath = u64_to_user_ptr(READ_ONCE(sqe->addr));
3900 newpath = u64_to_user_ptr(READ_ONCE(sqe->addr2));
3901
3902 sl->oldpath = getname(oldpath);
3903 if (IS_ERR(sl->oldpath))
3904 return PTR_ERR(sl->oldpath);
3905
3906 sl->newpath = getname(newpath);
3907 if (IS_ERR(sl->newpath)) {
3908 putname(sl->oldpath);
3909 return PTR_ERR(sl->newpath);
3910 }
3911
3912 req->flags |= REQ_F_NEED_CLEANUP;
3913 return 0;
3914}
3915
3916static int io_symlinkat(struct io_kiocb *req, int issue_flags)
3917{
3918 struct io_symlink *sl = &req->symlink;
3919 int ret;
3920
3921 if (issue_flags & IO_URING_F_NONBLOCK)
3922 return -EAGAIN;
3923
3924 ret = do_symlinkat(sl->oldpath, sl->new_dfd, sl->newpath);
3925
3926 req->flags &= ~REQ_F_NEED_CLEANUP;
3927 if (ret < 0)
3928 req_set_fail(req);
3929 io_req_complete(req, ret);
3930 return 0;
3931}
3932
Dmitry Kadashevcf30da92021-07-08 13:34:47 +07003933static int io_linkat_prep(struct io_kiocb *req,
3934 const struct io_uring_sqe *sqe)
3935{
3936 struct io_hardlink *lnk = &req->hardlink;
3937 const char __user *oldf, *newf;
3938
3939 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3940 return -EINVAL;
3941 if (sqe->ioprio || sqe->rw_flags || sqe->buf_index || sqe->splice_fd_in)
3942 return -EINVAL;
3943 if (unlikely(req->flags & REQ_F_FIXED_FILE))
3944 return -EBADF;
3945
3946 lnk->old_dfd = READ_ONCE(sqe->fd);
3947 lnk->new_dfd = READ_ONCE(sqe->len);
3948 oldf = u64_to_user_ptr(READ_ONCE(sqe->addr));
3949 newf = u64_to_user_ptr(READ_ONCE(sqe->addr2));
3950 lnk->flags = READ_ONCE(sqe->hardlink_flags);
3951
3952 lnk->oldpath = getname(oldf);
3953 if (IS_ERR(lnk->oldpath))
3954 return PTR_ERR(lnk->oldpath);
3955
3956 lnk->newpath = getname(newf);
3957 if (IS_ERR(lnk->newpath)) {
3958 putname(lnk->oldpath);
3959 return PTR_ERR(lnk->newpath);
3960 }
3961
3962 req->flags |= REQ_F_NEED_CLEANUP;
3963 return 0;
3964}
3965
3966static int io_linkat(struct io_kiocb *req, int issue_flags)
3967{
3968 struct io_hardlink *lnk = &req->hardlink;
3969 int ret;
3970
3971 if (issue_flags & IO_URING_F_NONBLOCK)
3972 return -EAGAIN;
3973
3974 ret = do_linkat(lnk->old_dfd, lnk->oldpath, lnk->new_dfd,
3975 lnk->newpath, lnk->flags);
3976
3977 req->flags &= ~REQ_F_NEED_CLEANUP;
3978 if (ret < 0)
3979 req_set_fail(req);
3980 io_req_complete(req, ret);
3981 return 0;
3982}
3983
Jens Axboe36f4fa62020-09-05 11:14:22 -06003984static int io_shutdown_prep(struct io_kiocb *req,
3985 const struct io_uring_sqe *sqe)
3986{
3987#if defined(CONFIG_NET)
3988 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3989 return -EINVAL;
Pavel Begunkov26578cd2021-08-20 10:36:37 +01003990 if (unlikely(sqe->ioprio || sqe->off || sqe->addr || sqe->rw_flags ||
3991 sqe->buf_index || sqe->splice_fd_in))
Jens Axboe36f4fa62020-09-05 11:14:22 -06003992 return -EINVAL;
3993
3994 req->shutdown.how = READ_ONCE(sqe->len);
3995 return 0;
3996#else
3997 return -EOPNOTSUPP;
3998#endif
3999}
4000
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004001static int io_shutdown(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe36f4fa62020-09-05 11:14:22 -06004002{
4003#if defined(CONFIG_NET)
4004 struct socket *sock;
4005 int ret;
4006
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004007 if (issue_flags & IO_URING_F_NONBLOCK)
Jens Axboe36f4fa62020-09-05 11:14:22 -06004008 return -EAGAIN;
4009
Linus Torvalds48aba792020-12-16 12:44:05 -08004010 sock = sock_from_file(req->file);
Jens Axboe36f4fa62020-09-05 11:14:22 -06004011 if (unlikely(!sock))
Linus Torvalds48aba792020-12-16 12:44:05 -08004012 return -ENOTSOCK;
Jens Axboe36f4fa62020-09-05 11:14:22 -06004013
4014 ret = __sys_shutdown_sock(sock, req->shutdown.how);
Jens Axboea1464682020-12-14 20:57:27 -07004015 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004016 req_set_fail(req);
Jens Axboe36f4fa62020-09-05 11:14:22 -06004017 io_req_complete(req, ret);
4018 return 0;
4019#else
4020 return -EOPNOTSUPP;
4021#endif
4022}
4023
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03004024static int __io_splice_prep(struct io_kiocb *req,
4025 const struct io_uring_sqe *sqe)
Pavel Begunkov7d67af22020-02-24 11:32:45 +03004026{
Pavel Begunkovfe7e3252021-06-24 15:09:57 +01004027 struct io_splice *sp = &req->splice;
Pavel Begunkov7d67af22020-02-24 11:32:45 +03004028 unsigned int valid_flags = SPLICE_F_FD_IN_FIXED | SPLICE_F_ALL;
Pavel Begunkov7d67af22020-02-24 11:32:45 +03004029
Pavel Begunkov3232dd02020-06-03 18:03:22 +03004030 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4031 return -EINVAL;
Pavel Begunkov7d67af22020-02-24 11:32:45 +03004032
Pavel Begunkov7d67af22020-02-24 11:32:45 +03004033 sp->len = READ_ONCE(sqe->len);
4034 sp->flags = READ_ONCE(sqe->splice_flags);
Pavel Begunkov7d67af22020-02-24 11:32:45 +03004035 if (unlikely(sp->flags & ~valid_flags))
4036 return -EINVAL;
Jens Axboeae6cba32022-03-29 10:59:20 -06004037 sp->splice_fd_in = READ_ONCE(sqe->splice_fd_in);
Pavel Begunkov7d67af22020-02-24 11:32:45 +03004038 return 0;
4039}
4040
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03004041static int io_tee_prep(struct io_kiocb *req,
4042 const struct io_uring_sqe *sqe)
4043{
4044 if (READ_ONCE(sqe->splice_off_in) || READ_ONCE(sqe->off))
4045 return -EINVAL;
4046 return __io_splice_prep(req, sqe);
4047}
4048
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004049static int io_tee(struct io_kiocb *req, unsigned int issue_flags)
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03004050{
4051 struct io_splice *sp = &req->splice;
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03004052 struct file *out = sp->file_out;
4053 unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED;
Jens Axboeae6cba32022-03-29 10:59:20 -06004054 struct file *in;
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03004055 long ret = 0;
4056
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004057 if (issue_flags & IO_URING_F_NONBLOCK)
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03004058 return -EAGAIN;
Jens Axboeae6cba32022-03-29 10:59:20 -06004059
4060 in = io_file_get(req->ctx, req, sp->splice_fd_in,
4061 (sp->flags & SPLICE_F_FD_IN_FIXED));
4062 if (!in) {
4063 ret = -EBADF;
4064 goto done;
4065 }
4066
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03004067 if (sp->len)
4068 ret = do_tee(in, out, sp->len, flags);
4069
Pavel Begunkove1d767f2021-03-19 17:22:43 +00004070 if (!(sp->flags & SPLICE_F_FD_IN_FIXED))
4071 io_put_file(in);
Jens Axboeae6cba32022-03-29 10:59:20 -06004072done:
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03004073 if (ret != sp->len)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004074 req_set_fail(req);
Jens Axboee1e16092020-06-22 09:17:17 -06004075 io_req_complete(req, ret);
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03004076 return 0;
4077}
4078
4079static int io_splice_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4080{
Pavel Begunkovfe7e3252021-06-24 15:09:57 +01004081 struct io_splice *sp = &req->splice;
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03004082
4083 sp->off_in = READ_ONCE(sqe->splice_off_in);
4084 sp->off_out = READ_ONCE(sqe->off);
4085 return __io_splice_prep(req, sqe);
4086}
4087
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004088static int io_splice(struct io_kiocb *req, unsigned int issue_flags)
Pavel Begunkov7d67af22020-02-24 11:32:45 +03004089{
4090 struct io_splice *sp = &req->splice;
Pavel Begunkov7d67af22020-02-24 11:32:45 +03004091 struct file *out = sp->file_out;
4092 unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED;
4093 loff_t *poff_in, *poff_out;
Jens Axboeae6cba32022-03-29 10:59:20 -06004094 struct file *in;
Pavel Begunkovc9687422020-05-04 23:00:54 +03004095 long ret = 0;
Pavel Begunkov7d67af22020-02-24 11:32:45 +03004096
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004097 if (issue_flags & IO_URING_F_NONBLOCK)
Pavel Begunkov2fb3e822020-05-01 17:09:38 +03004098 return -EAGAIN;
Pavel Begunkov7d67af22020-02-24 11:32:45 +03004099
Jens Axboeae6cba32022-03-29 10:59:20 -06004100 in = io_file_get(req->ctx, req, sp->splice_fd_in,
4101 (sp->flags & SPLICE_F_FD_IN_FIXED));
4102 if (!in) {
4103 ret = -EBADF;
4104 goto done;
4105 }
4106
Pavel Begunkov7d67af22020-02-24 11:32:45 +03004107 poff_in = (sp->off_in == -1) ? NULL : &sp->off_in;
4108 poff_out = (sp->off_out == -1) ? NULL : &sp->off_out;
Pavel Begunkovc9687422020-05-04 23:00:54 +03004109
Jens Axboe948a7742020-05-17 14:21:38 -06004110 if (sp->len)
Pavel Begunkovc9687422020-05-04 23:00:54 +03004111 ret = do_splice(in, poff_in, out, poff_out, sp->len, flags);
Pavel Begunkov7d67af22020-02-24 11:32:45 +03004112
Pavel Begunkove1d767f2021-03-19 17:22:43 +00004113 if (!(sp->flags & SPLICE_F_FD_IN_FIXED))
4114 io_put_file(in);
Jens Axboeae6cba32022-03-29 10:59:20 -06004115done:
Pavel Begunkov7d67af22020-02-24 11:32:45 +03004116 if (ret != sp->len)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004117 req_set_fail(req);
Jens Axboee1e16092020-06-22 09:17:17 -06004118 io_req_complete(req, ret);
Pavel Begunkov7d67af22020-02-24 11:32:45 +03004119 return 0;
4120}
4121
Jens Axboe2b188cc2019-01-07 10:46:33 -07004122/*
4123 * IORING_OP_NOP just posts a completion event, nothing else.
4124 */
Pavel Begunkov889fca72021-02-10 00:03:09 +00004125static int io_nop(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe2b188cc2019-01-07 10:46:33 -07004126{
4127 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe2b188cc2019-01-07 10:46:33 -07004128
Jens Axboedef596e2019-01-09 08:59:42 -07004129 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
4130 return -EINVAL;
4131
Pavel Begunkov889fca72021-02-10 00:03:09 +00004132 __io_req_complete(req, issue_flags, 0, 0);
Jens Axboe2b188cc2019-01-07 10:46:33 -07004133 return 0;
4134}
4135
Pavel Begunkov1155c762021-02-18 18:29:38 +00004136static int io_fsync_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Christoph Hellwigc992fe22019-01-11 09:43:02 -07004137{
Jens Axboe6b063142019-01-10 22:13:58 -07004138 struct io_ring_ctx *ctx = req->ctx;
Christoph Hellwigc992fe22019-01-11 09:43:02 -07004139
Jens Axboe6b063142019-01-10 22:13:58 -07004140 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
Jens Axboedef596e2019-01-09 08:59:42 -07004141 return -EINVAL;
Pavel Begunkov26578cd2021-08-20 10:36:37 +01004142 if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index ||
4143 sqe->splice_fd_in))
Christoph Hellwigc992fe22019-01-11 09:43:02 -07004144 return -EINVAL;
4145
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004146 req->sync.flags = READ_ONCE(sqe->fsync_flags);
4147 if (unlikely(req->sync.flags & ~IORING_FSYNC_DATASYNC))
4148 return -EINVAL;
4149
4150 req->sync.off = READ_ONCE(sqe->off);
4151 req->sync.len = READ_ONCE(sqe->len);
Christoph Hellwigc992fe22019-01-11 09:43:02 -07004152 return 0;
4153}
4154
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004155static int io_fsync(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe78912932020-01-14 22:09:06 -07004156{
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004157 loff_t end = req->sync.off + req->sync.len;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004158 int ret;
4159
Pavel Begunkovac45abc2020-06-08 21:08:18 +03004160 /* fsync always requires a blocking context */
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004161 if (issue_flags & IO_URING_F_NONBLOCK)
Pavel Begunkovac45abc2020-06-08 21:08:18 +03004162 return -EAGAIN;
4163
Jens Axboe9adbd452019-12-20 08:45:55 -07004164 ret = vfs_fsync_range(req->file, req->sync.off,
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004165 end > 0 ? end : LLONG_MAX,
4166 req->sync.flags & IORING_FSYNC_DATASYNC);
4167 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004168 req_set_fail(req);
Jens Axboee1e16092020-06-22 09:17:17 -06004169 io_req_complete(req, ret);
Christoph Hellwigc992fe22019-01-11 09:43:02 -07004170 return 0;
4171}
4172
Jens Axboed63d1b52019-12-10 10:38:56 -07004173static int io_fallocate_prep(struct io_kiocb *req,
4174 const struct io_uring_sqe *sqe)
4175{
Pavel Begunkov26578cd2021-08-20 10:36:37 +01004176 if (sqe->ioprio || sqe->buf_index || sqe->rw_flags ||
4177 sqe->splice_fd_in)
Jens Axboed63d1b52019-12-10 10:38:56 -07004178 return -EINVAL;
Pavel Begunkov3232dd02020-06-03 18:03:22 +03004179 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4180 return -EINVAL;
Jens Axboed63d1b52019-12-10 10:38:56 -07004181
4182 req->sync.off = READ_ONCE(sqe->off);
4183 req->sync.len = READ_ONCE(sqe->addr);
4184 req->sync.mode = READ_ONCE(sqe->len);
4185 return 0;
4186}
4187
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004188static int io_fallocate(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboed63d1b52019-12-10 10:38:56 -07004189{
Pavel Begunkovac45abc2020-06-08 21:08:18 +03004190 int ret;
Jens Axboed63d1b52019-12-10 10:38:56 -07004191
Pavel Begunkovac45abc2020-06-08 21:08:18 +03004192 /* fallocate always requiring blocking context */
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004193 if (issue_flags & IO_URING_F_NONBLOCK)
Pavel Begunkovac45abc2020-06-08 21:08:18 +03004194 return -EAGAIN;
Pavel Begunkovac45abc2020-06-08 21:08:18 +03004195 ret = vfs_fallocate(req->file, req->sync.mode, req->sync.off,
4196 req->sync.len);
Pavel Begunkovac45abc2020-06-08 21:08:18 +03004197 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004198 req_set_fail(req);
Jens Axboedf1ec532022-03-20 13:08:38 -06004199 else
4200 fsnotify_modify(req->file);
Jens Axboee1e16092020-06-22 09:17:17 -06004201 io_req_complete(req, ret);
Jens Axboed63d1b52019-12-10 10:38:56 -07004202 return 0;
4203}
4204
Pavel Begunkovec65fea2020-06-03 18:03:24 +03004205static int __io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboe15b71ab2019-12-11 11:20:36 -07004206{
Jens Axboef8748882020-01-08 17:47:02 -07004207 const char __user *fname;
Jens Axboe15b71ab2019-12-11 11:20:36 -07004208 int ret;
4209
Pavel Begunkovd3fddf62021-08-09 13:04:16 +01004210 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4211 return -EINVAL;
Pavel Begunkovb9445592021-08-25 12:25:45 +01004212 if (unlikely(sqe->ioprio || sqe->buf_index))
Jens Axboe15b71ab2019-12-11 11:20:36 -07004213 return -EINVAL;
Pavel Begunkovec65fea2020-06-03 18:03:24 +03004214 if (unlikely(req->flags & REQ_F_FIXED_FILE))
Jens Axboecf3040c2020-02-06 21:31:40 -07004215 return -EBADF;
Jens Axboe15b71ab2019-12-11 11:20:36 -07004216
Pavel Begunkovec65fea2020-06-03 18:03:24 +03004217 /* open.how should be already initialised */
4218 if (!(req->open.how.flags & O_PATH) && force_o_largefile())
Jens Axboe08a1d26eb2020-04-08 09:20:54 -06004219 req->open.how.flags |= O_LARGEFILE;
Jens Axboe15b71ab2019-12-11 11:20:36 -07004220
Pavel Begunkov25e72d12020-06-03 18:03:23 +03004221 req->open.dfd = READ_ONCE(sqe->fd);
4222 fname = u64_to_user_ptr(READ_ONCE(sqe->addr));
Jens Axboef8748882020-01-08 17:47:02 -07004223 req->open.filename = getname(fname);
Jens Axboe15b71ab2019-12-11 11:20:36 -07004224 if (IS_ERR(req->open.filename)) {
4225 ret = PTR_ERR(req->open.filename);
4226 req->open.filename = NULL;
4227 return ret;
4228 }
Pavel Begunkovb9445592021-08-25 12:25:45 +01004229
4230 req->open.file_slot = READ_ONCE(sqe->file_index);
4231 if (req->open.file_slot && (req->open.how.flags & O_CLOEXEC))
4232 return -EINVAL;
4233
Jens Axboe4022e7a2020-03-19 19:23:18 -06004234 req->open.nofile = rlimit(RLIMIT_NOFILE);
Pavel Begunkov8fef80b2020-02-07 23:59:53 +03004235 req->flags |= REQ_F_NEED_CLEANUP;
Jens Axboe15b71ab2019-12-11 11:20:36 -07004236 return 0;
4237}
4238
Pavel Begunkovec65fea2020-06-03 18:03:24 +03004239static int io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4240{
Pavel Begunkovd3fddf62021-08-09 13:04:16 +01004241 u64 mode = READ_ONCE(sqe->len);
4242 u64 flags = READ_ONCE(sqe->open_flags);
Pavel Begunkovec65fea2020-06-03 18:03:24 +03004243
Pavel Begunkovec65fea2020-06-03 18:03:24 +03004244 req->open.how = build_open_how(flags, mode);
4245 return __io_openat_prep(req, sqe);
4246}
4247
Jens Axboecebdb982020-01-08 17:59:24 -07004248static int io_openat2_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4249{
4250 struct open_how __user *how;
Jens Axboecebdb982020-01-08 17:59:24 -07004251 size_t len;
4252 int ret;
4253
Jens Axboecebdb982020-01-08 17:59:24 -07004254 how = u64_to_user_ptr(READ_ONCE(sqe->addr2));
4255 len = READ_ONCE(sqe->len);
Jens Axboecebdb982020-01-08 17:59:24 -07004256 if (len < OPEN_HOW_SIZE_VER0)
4257 return -EINVAL;
4258
4259 ret = copy_struct_from_user(&req->open.how, sizeof(req->open.how), how,
4260 len);
4261 if (ret)
4262 return ret;
4263
Pavel Begunkovec65fea2020-06-03 18:03:24 +03004264 return __io_openat_prep(req, sqe);
Jens Axboecebdb982020-01-08 17:59:24 -07004265}
4266
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004267static int io_openat2(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe15b71ab2019-12-11 11:20:36 -07004268{
4269 struct open_flags op;
Jens Axboe15b71ab2019-12-11 11:20:36 -07004270 struct file *file;
Pavel Begunkovb9445592021-08-25 12:25:45 +01004271 bool resolve_nonblock, nonblock_set;
4272 bool fixed = !!req->open.file_slot;
Jens Axboe15b71ab2019-12-11 11:20:36 -07004273 int ret;
4274
Jens Axboecebdb982020-01-08 17:59:24 -07004275 ret = build_open_flags(&req->open.how, &op);
Jens Axboe15b71ab2019-12-11 11:20:36 -07004276 if (ret)
4277 goto err;
Jens Axboe3a81fd02020-12-10 12:25:36 -07004278 nonblock_set = op.open_flag & O_NONBLOCK;
4279 resolve_nonblock = req->open.how.resolve & RESOLVE_CACHED;
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004280 if (issue_flags & IO_URING_F_NONBLOCK) {
Jens Axboe3a81fd02020-12-10 12:25:36 -07004281 /*
4282 * Don't bother trying for O_TRUNC, O_CREAT, or O_TMPFILE open,
4283 * it'll always -EAGAIN
4284 */
4285 if (req->open.how.flags & (O_TRUNC | O_CREAT | O_TMPFILE))
4286 return -EAGAIN;
4287 op.lookup_flags |= LOOKUP_CACHED;
4288 op.open_flag |= O_NONBLOCK;
4289 }
Jens Axboe15b71ab2019-12-11 11:20:36 -07004290
Pavel Begunkovb9445592021-08-25 12:25:45 +01004291 if (!fixed) {
4292 ret = __get_unused_fd_flags(req->open.how.flags, req->open.nofile);
4293 if (ret < 0)
4294 goto err;
4295 }
Jens Axboe15b71ab2019-12-11 11:20:36 -07004296
4297 file = do_filp_open(req->open.dfd, req->open.filename, &op);
Pavel Begunkov12dcb58a2021-06-24 15:10:00 +01004298 if (IS_ERR(file)) {
Jens Axboe3a81fd02020-12-10 12:25:36 -07004299 /*
Pavel Begunkov12dcb58a2021-06-24 15:10:00 +01004300 * We could hang on to this 'fd' on retrying, but seems like
4301 * marginal gain for something that is now known to be a slower
4302 * path. So just put it, and we'll get a new one when we retry.
Jens Axboe3a81fd02020-12-10 12:25:36 -07004303 */
Pavel Begunkovb9445592021-08-25 12:25:45 +01004304 if (!fixed)
4305 put_unused_fd(ret);
Pavel Begunkov12dcb58a2021-06-24 15:10:00 +01004306
4307 ret = PTR_ERR(file);
4308 /* only retry if RESOLVE_CACHED wasn't already set by application */
4309 if (ret == -EAGAIN &&
4310 (!resolve_nonblock && (issue_flags & IO_URING_F_NONBLOCK)))
4311 return -EAGAIN;
4312 goto err;
Jens Axboe3a81fd02020-12-10 12:25:36 -07004313 }
4314
Pavel Begunkov12dcb58a2021-06-24 15:10:00 +01004315 if ((issue_flags & IO_URING_F_NONBLOCK) && !nonblock_set)
4316 file->f_flags &= ~O_NONBLOCK;
4317 fsnotify_open(file);
Pavel Begunkovb9445592021-08-25 12:25:45 +01004318
4319 if (!fixed)
4320 fd_install(ret, file);
4321 else
4322 ret = io_install_fixed_file(req, file, issue_flags,
4323 req->open.file_slot - 1);
Jens Axboe15b71ab2019-12-11 11:20:36 -07004324err:
4325 putname(req->open.filename);
Pavel Begunkov8fef80b2020-02-07 23:59:53 +03004326 req->flags &= ~REQ_F_NEED_CLEANUP;
Jens Axboe15b71ab2019-12-11 11:20:36 -07004327 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004328 req_set_fail(req);
Pavel Begunkov0bdf3392021-04-11 01:46:29 +01004329 __io_req_complete(req, issue_flags, ret, 0);
Jens Axboe15b71ab2019-12-11 11:20:36 -07004330 return 0;
4331}
4332
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004333static int io_openat(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboecebdb982020-01-08 17:59:24 -07004334{
Pavel Begunkove45cff52021-02-28 22:35:14 +00004335 return io_openat2(req, issue_flags);
Jens Axboecebdb982020-01-08 17:59:24 -07004336}
4337
Jens Axboe067524e2020-03-02 16:32:28 -07004338static int io_remove_buffers_prep(struct io_kiocb *req,
4339 const struct io_uring_sqe *sqe)
4340{
4341 struct io_provide_buf *p = &req->pbuf;
4342 u64 tmp;
4343
Pavel Begunkov26578cd2021-08-20 10:36:37 +01004344 if (sqe->ioprio || sqe->rw_flags || sqe->addr || sqe->len || sqe->off ||
4345 sqe->splice_fd_in)
Jens Axboe067524e2020-03-02 16:32:28 -07004346 return -EINVAL;
4347
4348 tmp = READ_ONCE(sqe->fd);
4349 if (!tmp || tmp > USHRT_MAX)
4350 return -EINVAL;
4351
4352 memset(p, 0, sizeof(*p));
4353 p->nbufs = tmp;
4354 p->bgid = READ_ONCE(sqe->buf_group);
4355 return 0;
4356}
4357
4358static int __io_remove_buffers(struct io_ring_ctx *ctx, struct io_buffer *buf,
4359 int bgid, unsigned nbufs)
4360{
4361 unsigned i = 0;
4362
4363 /* shouldn't happen */
4364 if (!nbufs)
4365 return 0;
4366
4367 /* the head kbuf is the list itself */
4368 while (!list_empty(&buf->list)) {
4369 struct io_buffer *nxt;
4370
4371 nxt = list_first_entry(&buf->list, struct io_buffer, list);
4372 list_del(&nxt->list);
4373 kfree(nxt);
4374 if (++i == nbufs)
4375 return i;
Ye Bin2d447d32021-11-22 10:47:37 +08004376 cond_resched();
Jens Axboe067524e2020-03-02 16:32:28 -07004377 }
4378 i++;
4379 kfree(buf);
Jens Axboe9e15c3a2021-03-13 12:29:43 -07004380 xa_erase(&ctx->io_buffers, bgid);
Jens Axboe067524e2020-03-02 16:32:28 -07004381
4382 return i;
4383}
4384
Pavel Begunkov889fca72021-02-10 00:03:09 +00004385static int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe067524e2020-03-02 16:32:28 -07004386{
4387 struct io_provide_buf *p = &req->pbuf;
4388 struct io_ring_ctx *ctx = req->ctx;
4389 struct io_buffer *head;
4390 int ret = 0;
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004391 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
Jens Axboe067524e2020-03-02 16:32:28 -07004392
4393 io_ring_submit_lock(ctx, !force_nonblock);
4394
4395 lockdep_assert_held(&ctx->uring_lock);
4396
4397 ret = -ENOENT;
Jens Axboe9e15c3a2021-03-13 12:29:43 -07004398 head = xa_load(&ctx->io_buffers, p->bgid);
Jens Axboe067524e2020-03-02 16:32:28 -07004399 if (head)
4400 ret = __io_remove_buffers(ctx, head, p->bgid, p->nbufs);
Jens Axboe067524e2020-03-02 16:32:28 -07004401 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004402 req_set_fail(req);
Pavel Begunkov31bff9a2020-12-06 22:22:43 +00004403
Pavel Begunkov9fb8cb42021-02-28 22:35:13 +00004404 /* complete before unlock, IOPOLL may need the lock */
4405 __io_req_complete(req, issue_flags, ret, 0);
4406 io_ring_submit_unlock(ctx, !force_nonblock);
Jens Axboe067524e2020-03-02 16:32:28 -07004407 return 0;
4408}
4409
Jens Axboeddf0322d2020-02-23 16:41:33 -07004410static int io_provide_buffers_prep(struct io_kiocb *req,
4411 const struct io_uring_sqe *sqe)
4412{
Pavel Begunkov38134ad2021-04-15 13:07:39 +01004413 unsigned long size, tmp_check;
Jens Axboeddf0322d2020-02-23 16:41:33 -07004414 struct io_provide_buf *p = &req->pbuf;
4415 u64 tmp;
4416
Pavel Begunkov26578cd2021-08-20 10:36:37 +01004417 if (sqe->ioprio || sqe->rw_flags || sqe->splice_fd_in)
Jens Axboeddf0322d2020-02-23 16:41:33 -07004418 return -EINVAL;
4419
4420 tmp = READ_ONCE(sqe->fd);
4421 if (!tmp || tmp > USHRT_MAX)
4422 return -E2BIG;
4423 p->nbufs = tmp;
4424 p->addr = READ_ONCE(sqe->addr);
4425 p->len = READ_ONCE(sqe->len);
4426
Pavel Begunkov38134ad2021-04-15 13:07:39 +01004427 if (check_mul_overflow((unsigned long)p->len, (unsigned long)p->nbufs,
4428 &size))
4429 return -EOVERFLOW;
4430 if (check_add_overflow((unsigned long)p->addr, size, &tmp_check))
4431 return -EOVERFLOW;
4432
Pavel Begunkovd81269f2021-03-19 10:21:19 +00004433 size = (unsigned long)p->len * p->nbufs;
4434 if (!access_ok(u64_to_user_ptr(p->addr), size))
Jens Axboeddf0322d2020-02-23 16:41:33 -07004435 return -EFAULT;
4436
4437 p->bgid = READ_ONCE(sqe->buf_group);
4438 tmp = READ_ONCE(sqe->off);
4439 if (tmp > USHRT_MAX)
4440 return -E2BIG;
4441 p->bid = tmp;
4442 return 0;
4443}
4444
4445static int io_add_buffers(struct io_provide_buf *pbuf, struct io_buffer **head)
4446{
4447 struct io_buffer *buf;
4448 u64 addr = pbuf->addr;
4449 int i, bid = pbuf->bid;
4450
4451 for (i = 0; i < pbuf->nbufs; i++) {
Jens Axboe9990da92021-09-24 07:39:08 -06004452 buf = kmalloc(sizeof(*buf), GFP_KERNEL_ACCOUNT);
Jens Axboeddf0322d2020-02-23 16:41:33 -07004453 if (!buf)
4454 break;
4455
4456 buf->addr = addr;
Thadeu Lima de Souza Cascardod1f82802021-05-05 09:47:06 -03004457 buf->len = min_t(__u32, pbuf->len, MAX_RW_COUNT);
Jens Axboeddf0322d2020-02-23 16:41:33 -07004458 buf->bid = bid;
4459 addr += pbuf->len;
4460 bid++;
4461 if (!*head) {
4462 INIT_LIST_HEAD(&buf->list);
4463 *head = buf;
4464 } else {
4465 list_add_tail(&buf->list, &(*head)->list);
4466 }
Eric Dumazetc718ea42022-02-14 20:10:03 -08004467 cond_resched();
Jens Axboeddf0322d2020-02-23 16:41:33 -07004468 }
4469
4470 return i ? i : -ENOMEM;
4471}
4472
Pavel Begunkov889fca72021-02-10 00:03:09 +00004473static int io_provide_buffers(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboeddf0322d2020-02-23 16:41:33 -07004474{
4475 struct io_provide_buf *p = &req->pbuf;
4476 struct io_ring_ctx *ctx = req->ctx;
4477 struct io_buffer *head, *list;
4478 int ret = 0;
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004479 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
Jens Axboeddf0322d2020-02-23 16:41:33 -07004480
4481 io_ring_submit_lock(ctx, !force_nonblock);
4482
4483 lockdep_assert_held(&ctx->uring_lock);
4484
Jens Axboe9e15c3a2021-03-13 12:29:43 -07004485 list = head = xa_load(&ctx->io_buffers, p->bgid);
Jens Axboeddf0322d2020-02-23 16:41:33 -07004486
4487 ret = io_add_buffers(p, &head);
Jens Axboe9e15c3a2021-03-13 12:29:43 -07004488 if (ret >= 0 && !list) {
Pavel Begunkovfa304062022-08-04 15:13:46 +01004489 ret = xa_insert(&ctx->io_buffers, p->bgid, head,
4490 GFP_KERNEL_ACCOUNT);
Jens Axboe9e15c3a2021-03-13 12:29:43 -07004491 if (ret < 0)
Jens Axboe067524e2020-03-02 16:32:28 -07004492 __io_remove_buffers(ctx, head, p->bgid, -1U);
Jens Axboeddf0322d2020-02-23 16:41:33 -07004493 }
Jens Axboeddf0322d2020-02-23 16:41:33 -07004494 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004495 req_set_fail(req);
Pavel Begunkov9fb8cb42021-02-28 22:35:13 +00004496 /* complete before unlock, IOPOLL may need the lock */
4497 __io_req_complete(req, issue_flags, ret, 0);
4498 io_ring_submit_unlock(ctx, !force_nonblock);
Jens Axboeddf0322d2020-02-23 16:41:33 -07004499 return 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07004500}
4501
Jens Axboe3e4827b2020-01-08 15:18:09 -07004502static int io_epoll_ctl_prep(struct io_kiocb *req,
4503 const struct io_uring_sqe *sqe)
4504{
4505#if defined(CONFIG_EPOLL)
Pavel Begunkov26578cd2021-08-20 10:36:37 +01004506 if (sqe->ioprio || sqe->buf_index || sqe->splice_fd_in)
Jens Axboe3e4827b2020-01-08 15:18:09 -07004507 return -EINVAL;
Pavel Begunkov2d74d042021-05-14 12:05:46 +01004508 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
Pavel Begunkov3232dd02020-06-03 18:03:22 +03004509 return -EINVAL;
Jens Axboe3e4827b2020-01-08 15:18:09 -07004510
4511 req->epoll.epfd = READ_ONCE(sqe->fd);
4512 req->epoll.op = READ_ONCE(sqe->len);
4513 req->epoll.fd = READ_ONCE(sqe->off);
4514
4515 if (ep_op_has_event(req->epoll.op)) {
4516 struct epoll_event __user *ev;
4517
4518 ev = u64_to_user_ptr(READ_ONCE(sqe->addr));
4519 if (copy_from_user(&req->epoll.event, ev, sizeof(*ev)))
4520 return -EFAULT;
4521 }
4522
4523 return 0;
4524#else
4525 return -EOPNOTSUPP;
4526#endif
4527}
4528
Pavel Begunkov889fca72021-02-10 00:03:09 +00004529static int io_epoll_ctl(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe3e4827b2020-01-08 15:18:09 -07004530{
4531#if defined(CONFIG_EPOLL)
4532 struct io_epoll *ie = &req->epoll;
4533 int ret;
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004534 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
Jens Axboe3e4827b2020-01-08 15:18:09 -07004535
4536 ret = do_epoll_ctl(ie->epfd, ie->op, ie->fd, &ie->event, force_nonblock);
4537 if (force_nonblock && ret == -EAGAIN)
4538 return -EAGAIN;
4539
4540 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004541 req_set_fail(req);
Pavel Begunkov889fca72021-02-10 00:03:09 +00004542 __io_req_complete(req, issue_flags, ret, 0);
Jens Axboe3e4827b2020-01-08 15:18:09 -07004543 return 0;
4544#else
4545 return -EOPNOTSUPP;
4546#endif
4547}
4548
Jens Axboec1ca7572019-12-25 22:18:28 -07004549static int io_madvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4550{
4551#if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU)
Pavel Begunkov26578cd2021-08-20 10:36:37 +01004552 if (sqe->ioprio || sqe->buf_index || sqe->off || sqe->splice_fd_in)
Jens Axboec1ca7572019-12-25 22:18:28 -07004553 return -EINVAL;
Pavel Begunkov3232dd02020-06-03 18:03:22 +03004554 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4555 return -EINVAL;
Jens Axboec1ca7572019-12-25 22:18:28 -07004556
4557 req->madvise.addr = READ_ONCE(sqe->addr);
4558 req->madvise.len = READ_ONCE(sqe->len);
4559 req->madvise.advice = READ_ONCE(sqe->fadvise_advice);
4560 return 0;
4561#else
4562 return -EOPNOTSUPP;
4563#endif
4564}
4565
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004566static int io_madvise(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboec1ca7572019-12-25 22:18:28 -07004567{
4568#if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU)
4569 struct io_madvise *ma = &req->madvise;
4570 int ret;
4571
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004572 if (issue_flags & IO_URING_F_NONBLOCK)
Jens Axboec1ca7572019-12-25 22:18:28 -07004573 return -EAGAIN;
4574
Minchan Kim0726b012020-10-17 16:14:50 -07004575 ret = do_madvise(current->mm, ma->addr, ma->len, ma->advice);
Jens Axboec1ca7572019-12-25 22:18:28 -07004576 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004577 req_set_fail(req);
Jens Axboee1e16092020-06-22 09:17:17 -06004578 io_req_complete(req, ret);
Jens Axboec1ca7572019-12-25 22:18:28 -07004579 return 0;
4580#else
4581 return -EOPNOTSUPP;
4582#endif
4583}
4584
Jens Axboe4840e412019-12-25 22:03:45 -07004585static int io_fadvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4586{
Pavel Begunkov26578cd2021-08-20 10:36:37 +01004587 if (sqe->ioprio || sqe->buf_index || sqe->addr || sqe->splice_fd_in)
Jens Axboe4840e412019-12-25 22:03:45 -07004588 return -EINVAL;
Pavel Begunkov3232dd02020-06-03 18:03:22 +03004589 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4590 return -EINVAL;
Jens Axboe4840e412019-12-25 22:03:45 -07004591
4592 req->fadvise.offset = READ_ONCE(sqe->off);
4593 req->fadvise.len = READ_ONCE(sqe->len);
4594 req->fadvise.advice = READ_ONCE(sqe->fadvise_advice);
4595 return 0;
4596}
4597
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004598static int io_fadvise(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe4840e412019-12-25 22:03:45 -07004599{
4600 struct io_fadvise *fa = &req->fadvise;
4601 int ret;
4602
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004603 if (issue_flags & IO_URING_F_NONBLOCK) {
Jens Axboe3e694262020-02-01 09:22:49 -07004604 switch (fa->advice) {
4605 case POSIX_FADV_NORMAL:
4606 case POSIX_FADV_RANDOM:
4607 case POSIX_FADV_SEQUENTIAL:
4608 break;
4609 default:
4610 return -EAGAIN;
4611 }
4612 }
Jens Axboe4840e412019-12-25 22:03:45 -07004613
4614 ret = vfs_fadvise(req->file, fa->offset, fa->len, fa->advice);
4615 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004616 req_set_fail(req);
Pavel Begunkov0bdf3392021-04-11 01:46:29 +01004617 __io_req_complete(req, issue_flags, ret, 0);
Jens Axboe4840e412019-12-25 22:03:45 -07004618 return 0;
4619}
4620
Jens Axboeeddc7ef2019-12-13 21:18:10 -07004621static int io_statx_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4622{
Pavel Begunkov2d74d042021-05-14 12:05:46 +01004623 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
Pavel Begunkov3232dd02020-06-03 18:03:22 +03004624 return -EINVAL;
Pavel Begunkov26578cd2021-08-20 10:36:37 +01004625 if (sqe->ioprio || sqe->buf_index || sqe->splice_fd_in)
Jens Axboeeddc7ef2019-12-13 21:18:10 -07004626 return -EINVAL;
Pavel Begunkov9c280f92020-04-08 08:58:46 +03004627 if (req->flags & REQ_F_FIXED_FILE)
Jens Axboecf3040c2020-02-06 21:31:40 -07004628 return -EBADF;
Jens Axboeeddc7ef2019-12-13 21:18:10 -07004629
Bijan Mottahedeh1d9e1282020-05-22 21:31:16 -07004630 req->statx.dfd = READ_ONCE(sqe->fd);
4631 req->statx.mask = READ_ONCE(sqe->len);
Bijan Mottahedehe62753e2020-05-22 21:31:18 -07004632 req->statx.filename = u64_to_user_ptr(READ_ONCE(sqe->addr));
Bijan Mottahedeh1d9e1282020-05-22 21:31:16 -07004633 req->statx.buffer = u64_to_user_ptr(READ_ONCE(sqe->addr2));
4634 req->statx.flags = READ_ONCE(sqe->statx_flags);
Jens Axboeeddc7ef2019-12-13 21:18:10 -07004635
Jens Axboeeddc7ef2019-12-13 21:18:10 -07004636 return 0;
4637}
4638
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004639static int io_statx(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboeeddc7ef2019-12-13 21:18:10 -07004640{
Bijan Mottahedeh1d9e1282020-05-22 21:31:16 -07004641 struct io_statx *ctx = &req->statx;
Jens Axboeeddc7ef2019-12-13 21:18:10 -07004642 int ret;
4643
Pavel Begunkov59d70012021-03-22 01:58:30 +00004644 if (issue_flags & IO_URING_F_NONBLOCK)
Jens Axboeeddc7ef2019-12-13 21:18:10 -07004645 return -EAGAIN;
4646
Bijan Mottahedehe62753e2020-05-22 21:31:18 -07004647 ret = do_statx(ctx->dfd, ctx->filename, ctx->flags, ctx->mask,
4648 ctx->buffer);
Jens Axboeeddc7ef2019-12-13 21:18:10 -07004649
Jens Axboeeddc7ef2019-12-13 21:18:10 -07004650 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004651 req_set_fail(req);
Jens Axboee1e16092020-06-22 09:17:17 -06004652 io_req_complete(req, ret);
Jens Axboeeddc7ef2019-12-13 21:18:10 -07004653 return 0;
4654}
4655
Jens Axboeb5dba592019-12-11 14:02:38 -07004656static int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4657{
Jens Axboe14587a462020-09-05 11:36:08 -06004658 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
Pavel Begunkov3232dd02020-06-03 18:03:22 +03004659 return -EINVAL;
Jens Axboeb5dba592019-12-11 14:02:38 -07004660 if (sqe->ioprio || sqe->off || sqe->addr || sqe->len ||
Pavel Begunkov7df778b2021-09-24 20:04:29 +01004661 sqe->rw_flags || sqe->buf_index)
Jens Axboeb5dba592019-12-11 14:02:38 -07004662 return -EINVAL;
Pavel Begunkov9c280f92020-04-08 08:58:46 +03004663 if (req->flags & REQ_F_FIXED_FILE)
Jens Axboecf3040c2020-02-06 21:31:40 -07004664 return -EBADF;
Jens Axboeb5dba592019-12-11 14:02:38 -07004665
4666 req->close.fd = READ_ONCE(sqe->fd);
Pavel Begunkov7df778b2021-09-24 20:04:29 +01004667 req->close.file_slot = READ_ONCE(sqe->file_index);
4668 if (req->close.file_slot && req->close.fd)
4669 return -EINVAL;
4670
Jens Axboeb5dba592019-12-11 14:02:38 -07004671 return 0;
4672}
4673
Pavel Begunkov889fca72021-02-10 00:03:09 +00004674static int io_close(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboeb5dba592019-12-11 14:02:38 -07004675{
Jens Axboe9eac1902021-01-19 15:50:37 -07004676 struct files_struct *files = current->files;
Pavel Begunkov3af73b22020-06-08 21:08:17 +03004677 struct io_close *close = &req->close;
Jens Axboe9eac1902021-01-19 15:50:37 -07004678 struct fdtable *fdt;
Pavel Begunkova1fde922021-04-11 01:46:28 +01004679 struct file *file = NULL;
4680 int ret = -EBADF;
Jens Axboeb5dba592019-12-11 14:02:38 -07004681
Pavel Begunkov7df778b2021-09-24 20:04:29 +01004682 if (req->close.file_slot) {
4683 ret = io_close_fixed(req, issue_flags);
4684 goto err;
4685 }
4686
Jens Axboe9eac1902021-01-19 15:50:37 -07004687 spin_lock(&files->file_lock);
4688 fdt = files_fdtable(files);
4689 if (close->fd >= fdt->max_fds) {
4690 spin_unlock(&files->file_lock);
4691 goto err;
4692 }
4693 file = fdt->fd[close->fd];
Pavel Begunkova1fde922021-04-11 01:46:28 +01004694 if (!file || file->f_op == &io_uring_fops) {
Jens Axboe9eac1902021-01-19 15:50:37 -07004695 spin_unlock(&files->file_lock);
4696 file = NULL;
4697 goto err;
Pavel Begunkov3af73b22020-06-08 21:08:17 +03004698 }
Jens Axboeb5dba592019-12-11 14:02:38 -07004699
4700 /* if the file has a flush method, be safe and punt to async */
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004701 if (file->f_op->flush && (issue_flags & IO_URING_F_NONBLOCK)) {
Jens Axboe9eac1902021-01-19 15:50:37 -07004702 spin_unlock(&files->file_lock);
Pavel Begunkov0bf0eef2020-05-26 20:34:06 +03004703 return -EAGAIN;
Pavel Begunkova2100672020-03-02 23:45:16 +03004704 }
Jens Axboeb5dba592019-12-11 14:02:38 -07004705
Jens Axboe9eac1902021-01-19 15:50:37 -07004706 ret = __close_fd_get_file(close->fd, &file);
4707 spin_unlock(&files->file_lock);
4708 if (ret < 0) {
4709 if (ret == -ENOENT)
4710 ret = -EBADF;
4711 goto err;
4712 }
4713
Pavel Begunkov3af73b22020-06-08 21:08:17 +03004714 /* No ->flush() or already async, safely close from here */
Jens Axboe9eac1902021-01-19 15:50:37 -07004715 ret = filp_close(file, current->files);
4716err:
Pavel Begunkov3af73b22020-06-08 21:08:17 +03004717 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004718 req_set_fail(req);
Jens Axboe9eac1902021-01-19 15:50:37 -07004719 if (file)
4720 fput(file);
Pavel Begunkov889fca72021-02-10 00:03:09 +00004721 __io_req_complete(req, issue_flags, ret, 0);
Jens Axboe1a417f42020-01-31 17:16:48 -07004722 return 0;
Jens Axboeb5dba592019-12-11 14:02:38 -07004723}
4724
Pavel Begunkov1155c762021-02-18 18:29:38 +00004725static int io_sfr_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboe5d17b4a2019-04-09 14:56:44 -06004726{
4727 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe5d17b4a2019-04-09 14:56:44 -06004728
Jens Axboe5d17b4a2019-04-09 14:56:44 -06004729 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
4730 return -EINVAL;
Pavel Begunkov26578cd2021-08-20 10:36:37 +01004731 if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index ||
4732 sqe->splice_fd_in))
Jens Axboe5d17b4a2019-04-09 14:56:44 -06004733 return -EINVAL;
4734
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004735 req->sync.off = READ_ONCE(sqe->off);
4736 req->sync.len = READ_ONCE(sqe->len);
4737 req->sync.flags = READ_ONCE(sqe->sync_range_flags);
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004738 return 0;
4739}
4740
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004741static int io_sync_file_range(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004742{
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004743 int ret;
4744
Pavel Begunkovac45abc2020-06-08 21:08:18 +03004745 /* sync_file_range always requires a blocking context */
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004746 if (issue_flags & IO_URING_F_NONBLOCK)
Pavel Begunkovac45abc2020-06-08 21:08:18 +03004747 return -EAGAIN;
4748
Jens Axboe9adbd452019-12-20 08:45:55 -07004749 ret = sync_file_range(req->file, req->sync.off, req->sync.len,
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004750 req->sync.flags);
4751 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004752 req_set_fail(req);
Jens Axboee1e16092020-06-22 09:17:17 -06004753 io_req_complete(req, ret);
Jens Axboe5d17b4a2019-04-09 14:56:44 -06004754 return 0;
4755}
4756
YueHaibing469956e2020-03-04 15:53:52 +08004757#if defined(CONFIG_NET)
Pavel Begunkov02d27d82020-02-28 10:36:36 +03004758static int io_setup_async_msg(struct io_kiocb *req,
4759 struct io_async_msghdr *kmsg)
4760{
Jens Axboee8c2bc12020-08-15 18:44:09 -07004761 struct io_async_msghdr *async_msg = req->async_data;
4762
4763 if (async_msg)
Pavel Begunkov02d27d82020-02-28 10:36:36 +03004764 return -EAGAIN;
Jens Axboee8c2bc12020-08-15 18:44:09 -07004765 if (io_alloc_async_data(req)) {
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004766 kfree(kmsg->free_iov);
Pavel Begunkov02d27d82020-02-28 10:36:36 +03004767 return -ENOMEM;
4768 }
Jens Axboee8c2bc12020-08-15 18:44:09 -07004769 async_msg = req->async_data;
Pavel Begunkov02d27d82020-02-28 10:36:36 +03004770 req->flags |= REQ_F_NEED_CLEANUP;
Jens Axboee8c2bc12020-08-15 18:44:09 -07004771 memcpy(async_msg, kmsg, sizeof(*kmsg));
Pavel Begunkovf9dc33f2022-09-29 22:23:18 +01004772 if (async_msg->msg.msg_name)
4773 async_msg->msg.msg_name = &async_msg->addr;
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004774 /* if were using fast_iov, set it to the new one */
4775 if (!async_msg->free_iov)
4776 async_msg->msg.msg_iter.iov = async_msg->fast_iov;
4777
Pavel Begunkov02d27d82020-02-28 10:36:36 +03004778 return -EAGAIN;
4779}
4780
Pavel Begunkov2ae523e2020-07-12 20:41:06 +03004781static int io_sendmsg_copy_hdr(struct io_kiocb *req,
4782 struct io_async_msghdr *iomsg)
4783{
Pavel Begunkov2ae523e2020-07-12 20:41:06 +03004784 iomsg->msg.msg_name = &iomsg->addr;
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004785 iomsg->free_iov = iomsg->fast_iov;
Pavel Begunkov2ae523e2020-07-12 20:41:06 +03004786 return sendmsg_copy_msghdr(&iomsg->msg, req->sr_msg.umsg,
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004787 req->sr_msg.msg_flags, &iomsg->free_iov);
Pavel Begunkov2ae523e2020-07-12 20:41:06 +03004788}
4789
Pavel Begunkov93642ef2021-02-18 18:29:44 +00004790static int io_sendmsg_prep_async(struct io_kiocb *req)
4791{
4792 int ret;
4793
Pavel Begunkov93642ef2021-02-18 18:29:44 +00004794 ret = io_sendmsg_copy_hdr(req, req->async_data);
4795 if (!ret)
4796 req->flags |= REQ_F_NEED_CLEANUP;
4797 return ret;
4798}
4799
Jens Axboe3529d8c2019-12-19 18:24:38 -07004800static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboeaa1fa282019-04-19 13:38:09 -06004801{
Jens Axboee47293f2019-12-20 08:58:21 -07004802 struct io_sr_msg *sr = &req->sr_msg;
Jens Axboe03b12302019-12-02 18:50:25 -07004803
Pavel Begunkovd2b6f482020-06-03 18:03:25 +03004804 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4805 return -EINVAL;
Jens Axboe79c10cb2022-04-26 19:34:11 -06004806 if (unlikely(sqe->addr2 || sqe->file_index))
4807 return -EINVAL;
Jens Axboe50fefe52022-06-30 14:42:05 -06004808 if (unlikely(sqe->addr2 || sqe->file_index || sqe->ioprio))
4809 return -EINVAL;
Pavel Begunkovd2b6f482020-06-03 18:03:25 +03004810
Pavel Begunkov270a5942020-07-12 20:41:04 +03004811 sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
Jens Axboefddafac2020-01-04 20:19:44 -07004812 sr->len = READ_ONCE(sqe->len);
Pavel Begunkov04411802021-04-01 15:44:00 +01004813 sr->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL;
4814 if (sr->msg_flags & MSG_DONTWAIT)
4815 req->flags |= REQ_F_NOWAIT;
Jens Axboe3529d8c2019-12-19 18:24:38 -07004816
Jens Axboed8768362020-02-27 14:17:49 -07004817#ifdef CONFIG_COMPAT
4818 if (req->ctx->compat)
4819 sr->msg_flags |= MSG_CMSG_COMPAT;
4820#endif
Pavel Begunkov93642ef2021-02-18 18:29:44 +00004821 return 0;
Jens Axboe03b12302019-12-02 18:50:25 -07004822}
4823
Pavel Begunkov889fca72021-02-10 00:03:09 +00004824static int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe03b12302019-12-02 18:50:25 -07004825{
Pavel Begunkov6b754c82020-07-16 23:28:00 +03004826 struct io_async_msghdr iomsg, *kmsg;
Jens Axboe03b12302019-12-02 18:50:25 -07004827 struct socket *sock;
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004828 unsigned flags;
Stefan Metzmacher00312752021-03-20 20:33:36 +01004829 int min_ret = 0;
Jens Axboe03b12302019-12-02 18:50:25 -07004830 int ret;
4831
Florent Revestdba4a922020-12-04 12:36:04 +01004832 sock = sock_from_file(req->file);
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004833 if (unlikely(!sock))
Florent Revestdba4a922020-12-04 12:36:04 +01004834 return -ENOTSOCK;
Jens Axboe03b12302019-12-02 18:50:25 -07004835
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004836 kmsg = req->async_data;
4837 if (!kmsg) {
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004838 ret = io_sendmsg_copy_hdr(req, &iomsg);
Jens Axboefddafac2020-01-04 20:19:44 -07004839 if (ret)
4840 return ret;
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004841 kmsg = &iomsg;
Jens Axboefddafac2020-01-04 20:19:44 -07004842 }
4843
Pavel Begunkov04411802021-04-01 15:44:00 +01004844 flags = req->sr_msg.msg_flags;
4845 if (issue_flags & IO_URING_F_NONBLOCK)
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004846 flags |= MSG_DONTWAIT;
Stefan Metzmacher00312752021-03-20 20:33:36 +01004847 if (flags & MSG_WAITALL)
4848 min_ret = iov_iter_count(&kmsg->msg.msg_iter);
4849
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004850 ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags);
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004851 if ((issue_flags & IO_URING_F_NONBLOCK) && ret == -EAGAIN)
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004852 return io_setup_async_msg(req, kmsg);
4853 if (ret == -ERESTARTSYS)
4854 ret = -EINTR;
4855
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004856 /* fast path, check for non-NULL to avoid function call */
4857 if (kmsg->free_iov)
4858 kfree(kmsg->free_iov);
Jens Axboe03b12302019-12-02 18:50:25 -07004859 req->flags &= ~REQ_F_NEED_CLEANUP;
Stefan Metzmacher00312752021-03-20 20:33:36 +01004860 if (ret < min_ret)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004861 req_set_fail(req);
Pavel Begunkov889fca72021-02-10 00:03:09 +00004862 __io_req_complete(req, issue_flags, ret, 0);
Jens Axboefddafac2020-01-04 20:19:44 -07004863 return 0;
Jens Axboefddafac2020-01-04 20:19:44 -07004864}
4865
Pavel Begunkov889fca72021-02-10 00:03:09 +00004866static int io_send(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe03b12302019-12-02 18:50:25 -07004867{
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004868 struct io_sr_msg *sr = &req->sr_msg;
4869 struct msghdr msg;
4870 struct iovec iov;
Jens Axboe03b12302019-12-02 18:50:25 -07004871 struct socket *sock;
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004872 unsigned flags;
Stefan Metzmacher00312752021-03-20 20:33:36 +01004873 int min_ret = 0;
Jens Axboe03b12302019-12-02 18:50:25 -07004874 int ret;
4875
Florent Revestdba4a922020-12-04 12:36:04 +01004876 sock = sock_from_file(req->file);
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004877 if (unlikely(!sock))
Florent Revestdba4a922020-12-04 12:36:04 +01004878 return -ENOTSOCK;
Jens Axboe03b12302019-12-02 18:50:25 -07004879
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004880 ret = import_single_range(WRITE, sr->buf, sr->len, &iov, &msg.msg_iter);
4881 if (unlikely(ret))
Zheng Bin14db8412020-09-09 20:12:37 +08004882 return ret;
Jens Axboe03b12302019-12-02 18:50:25 -07004883
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004884 msg.msg_name = NULL;
4885 msg.msg_control = NULL;
4886 msg.msg_controllen = 0;
4887 msg.msg_namelen = 0;
Jens Axboe03b12302019-12-02 18:50:25 -07004888
Pavel Begunkov04411802021-04-01 15:44:00 +01004889 flags = req->sr_msg.msg_flags;
4890 if (issue_flags & IO_URING_F_NONBLOCK)
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004891 flags |= MSG_DONTWAIT;
Stefan Metzmacher00312752021-03-20 20:33:36 +01004892 if (flags & MSG_WAITALL)
4893 min_ret = iov_iter_count(&msg.msg_iter);
4894
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004895 msg.msg_flags = flags;
4896 ret = sock_sendmsg(sock, &msg);
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004897 if ((issue_flags & IO_URING_F_NONBLOCK) && ret == -EAGAIN)
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004898 return -EAGAIN;
4899 if (ret == -ERESTARTSYS)
4900 ret = -EINTR;
Jens Axboe03b12302019-12-02 18:50:25 -07004901
Stefan Metzmacher00312752021-03-20 20:33:36 +01004902 if (ret < min_ret)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004903 req_set_fail(req);
Pavel Begunkov889fca72021-02-10 00:03:09 +00004904 __io_req_complete(req, issue_flags, ret, 0);
Jens Axboe03b12302019-12-02 18:50:25 -07004905 return 0;
Jens Axboe03b12302019-12-02 18:50:25 -07004906}
4907
Pavel Begunkov1400e692020-07-12 20:41:05 +03004908static int __io_recvmsg_copy_hdr(struct io_kiocb *req,
4909 struct io_async_msghdr *iomsg)
Jens Axboe52de1fe2020-02-27 10:15:42 -07004910{
4911 struct io_sr_msg *sr = &req->sr_msg;
4912 struct iovec __user *uiov;
4913 size_t iov_len;
4914 int ret;
4915
Pavel Begunkov1400e692020-07-12 20:41:05 +03004916 ret = __copy_msghdr_from_user(&iomsg->msg, sr->umsg,
4917 &iomsg->uaddr, &uiov, &iov_len);
Jens Axboe52de1fe2020-02-27 10:15:42 -07004918 if (ret)
4919 return ret;
4920
4921 if (req->flags & REQ_F_BUFFER_SELECT) {
4922 if (iov_len > 1)
4923 return -EINVAL;
Pavel Begunkov5476dfe2021-02-05 00:57:59 +00004924 if (copy_from_user(iomsg->fast_iov, uiov, sizeof(*uiov)))
Jens Axboe52de1fe2020-02-27 10:15:42 -07004925 return -EFAULT;
Pavel Begunkov5476dfe2021-02-05 00:57:59 +00004926 sr->len = iomsg->fast_iov[0].iov_len;
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004927 iomsg->free_iov = NULL;
Jens Axboe52de1fe2020-02-27 10:15:42 -07004928 } else {
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004929 iomsg->free_iov = iomsg->fast_iov;
Christoph Hellwig89cd35c2020-09-25 06:51:41 +02004930 ret = __import_iovec(READ, uiov, iov_len, UIO_FASTIOV,
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004931 &iomsg->free_iov, &iomsg->msg.msg_iter,
Christoph Hellwig89cd35c2020-09-25 06:51:41 +02004932 false);
Jens Axboe52de1fe2020-02-27 10:15:42 -07004933 if (ret > 0)
4934 ret = 0;
4935 }
4936
4937 return ret;
4938}
4939
4940#ifdef CONFIG_COMPAT
4941static int __io_compat_recvmsg_copy_hdr(struct io_kiocb *req,
Pavel Begunkov1400e692020-07-12 20:41:05 +03004942 struct io_async_msghdr *iomsg)
Jens Axboe52de1fe2020-02-27 10:15:42 -07004943{
Jens Axboe52de1fe2020-02-27 10:15:42 -07004944 struct io_sr_msg *sr = &req->sr_msg;
4945 struct compat_iovec __user *uiov;
4946 compat_uptr_t ptr;
4947 compat_size_t len;
4948 int ret;
4949
Pavel Begunkov4af34172021-04-11 01:46:30 +01004950 ret = __get_compat_msghdr(&iomsg->msg, sr->umsg_compat, &iomsg->uaddr,
4951 &ptr, &len);
Jens Axboe52de1fe2020-02-27 10:15:42 -07004952 if (ret)
4953 return ret;
4954
4955 uiov = compat_ptr(ptr);
4956 if (req->flags & REQ_F_BUFFER_SELECT) {
4957 compat_ssize_t clen;
4958
4959 if (len > 1)
4960 return -EINVAL;
4961 if (!access_ok(uiov, sizeof(*uiov)))
4962 return -EFAULT;
4963 if (__get_user(clen, &uiov->iov_len))
4964 return -EFAULT;
4965 if (clen < 0)
4966 return -EINVAL;
Pavel Begunkov2d280bc2020-11-29 18:33:32 +00004967 sr->len = clen;
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004968 iomsg->free_iov = NULL;
Jens Axboe52de1fe2020-02-27 10:15:42 -07004969 } else {
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004970 iomsg->free_iov = iomsg->fast_iov;
Christoph Hellwig89cd35c2020-09-25 06:51:41 +02004971 ret = __import_iovec(READ, (struct iovec __user *)uiov, len,
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004972 UIO_FASTIOV, &iomsg->free_iov,
Christoph Hellwig89cd35c2020-09-25 06:51:41 +02004973 &iomsg->msg.msg_iter, true);
Jens Axboe52de1fe2020-02-27 10:15:42 -07004974 if (ret < 0)
4975 return ret;
4976 }
4977
4978 return 0;
4979}
Jens Axboe03b12302019-12-02 18:50:25 -07004980#endif
Jens Axboe52de1fe2020-02-27 10:15:42 -07004981
Pavel Begunkov1400e692020-07-12 20:41:05 +03004982static int io_recvmsg_copy_hdr(struct io_kiocb *req,
4983 struct io_async_msghdr *iomsg)
Jens Axboe52de1fe2020-02-27 10:15:42 -07004984{
Pavel Begunkov1400e692020-07-12 20:41:05 +03004985 iomsg->msg.msg_name = &iomsg->addr;
Jens Axboe52de1fe2020-02-27 10:15:42 -07004986
4987#ifdef CONFIG_COMPAT
4988 if (req->ctx->compat)
Pavel Begunkov1400e692020-07-12 20:41:05 +03004989 return __io_compat_recvmsg_copy_hdr(req, iomsg);
Jens Axboe52de1fe2020-02-27 10:15:42 -07004990#endif
4991
Pavel Begunkov1400e692020-07-12 20:41:05 +03004992 return __io_recvmsg_copy_hdr(req, iomsg);
Jens Axboe52de1fe2020-02-27 10:15:42 -07004993}
4994
Jens Axboebcda7ba2020-02-23 16:42:51 -07004995static struct io_buffer *io_recv_buffer_select(struct io_kiocb *req,
Pavel Begunkov7fbb1b52020-07-16 23:28:05 +03004996 bool needs_lock)
Jens Axboebcda7ba2020-02-23 16:42:51 -07004997{
4998 struct io_sr_msg *sr = &req->sr_msg;
4999 struct io_buffer *kbuf;
5000
Jens Axboebcda7ba2020-02-23 16:42:51 -07005001 kbuf = io_buffer_select(req, &sr->len, sr->bgid, sr->kbuf, needs_lock);
5002 if (IS_ERR(kbuf))
5003 return kbuf;
5004
5005 sr->kbuf = kbuf;
5006 req->flags |= REQ_F_BUFFER_SELECTED;
Jens Axboebcda7ba2020-02-23 16:42:51 -07005007 return kbuf;
Jens Axboe03b12302019-12-02 18:50:25 -07005008}
5009
Pavel Begunkov7fbb1b52020-07-16 23:28:05 +03005010static inline unsigned int io_put_recv_kbuf(struct io_kiocb *req)
5011{
5012 return io_put_kbuf(req, req->sr_msg.kbuf);
5013}
5014
Pavel Begunkov93642ef2021-02-18 18:29:44 +00005015static int io_recvmsg_prep_async(struct io_kiocb *req)
Jens Axboe03b12302019-12-02 18:50:25 -07005016{
Pavel Begunkov99bc4c32020-02-07 22:04:45 +03005017 int ret;
Jens Axboe06b76d42019-12-19 14:44:26 -07005018
Pavel Begunkov93642ef2021-02-18 18:29:44 +00005019 ret = io_recvmsg_copy_hdr(req, req->async_data);
5020 if (!ret)
5021 req->flags |= REQ_F_NEED_CLEANUP;
5022 return ret;
5023}
5024
5025static int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
5026{
5027 struct io_sr_msg *sr = &req->sr_msg;
5028
Pavel Begunkovd2b6f482020-06-03 18:03:25 +03005029 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
5030 return -EINVAL;
Jens Axboe37811e42022-04-26 19:34:57 -06005031 if (unlikely(sqe->addr2 || sqe->file_index))
5032 return -EINVAL;
Jens Axboe50fefe52022-06-30 14:42:05 -06005033 if (unlikely(sqe->addr2 || sqe->file_index || sqe->ioprio))
5034 return -EINVAL;
Pavel Begunkovd2b6f482020-06-03 18:03:25 +03005035
Pavel Begunkov270a5942020-07-12 20:41:04 +03005036 sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
Jens Axboe0b7b21e2020-01-31 08:34:59 -07005037 sr->len = READ_ONCE(sqe->len);
Jens Axboebcda7ba2020-02-23 16:42:51 -07005038 sr->bgid = READ_ONCE(sqe->buf_group);
Pavel Begunkov04411802021-04-01 15:44:00 +01005039 sr->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL;
5040 if (sr->msg_flags & MSG_DONTWAIT)
5041 req->flags |= REQ_F_NOWAIT;
Jens Axboe3529d8c2019-12-19 18:24:38 -07005042
Jens Axboed8768362020-02-27 14:17:49 -07005043#ifdef CONFIG_COMPAT
5044 if (req->ctx->compat)
5045 sr->msg_flags |= MSG_CMSG_COMPAT;
5046#endif
Pavel Begunkov93642ef2021-02-18 18:29:44 +00005047 return 0;
Jens Axboe03b12302019-12-02 18:50:25 -07005048}
5049
Pavel Begunkov889fca72021-02-10 00:03:09 +00005050static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe03b12302019-12-02 18:50:25 -07005051{
Pavel Begunkov6b754c82020-07-16 23:28:00 +03005052 struct io_async_msghdr iomsg, *kmsg;
Jens Axboe0fa03c62019-04-19 13:34:07 -06005053 struct socket *sock;
Pavel Begunkov7fbb1b52020-07-16 23:28:05 +03005054 struct io_buffer *kbuf;
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03005055 unsigned flags;
Stefan Metzmacher00312752021-03-20 20:33:36 +01005056 int min_ret = 0;
Jens Axboe52de1fe2020-02-27 10:15:42 -07005057 int ret, cflags = 0;
Pavel Begunkov45d189c2021-02-10 00:03:07 +00005058 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
Jens Axboe0fa03c62019-04-19 13:34:07 -06005059
Florent Revestdba4a922020-12-04 12:36:04 +01005060 sock = sock_from_file(req->file);
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03005061 if (unlikely(!sock))
Florent Revestdba4a922020-12-04 12:36:04 +01005062 return -ENOTSOCK;
Jens Axboe0fa03c62019-04-19 13:34:07 -06005063
Pavel Begunkov257e84a2021-02-05 00:58:00 +00005064 kmsg = req->async_data;
5065 if (!kmsg) {
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03005066 ret = io_recvmsg_copy_hdr(req, &iomsg);
5067 if (ret)
Pavel Begunkov681fda82020-07-15 22:20:45 +03005068 return ret;
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03005069 kmsg = &iomsg;
Jens Axboe0fa03c62019-04-19 13:34:07 -06005070 }
5071
Pavel Begunkovbc02ef32020-07-16 23:28:03 +03005072 if (req->flags & REQ_F_BUFFER_SELECT) {
Pavel Begunkov7fbb1b52020-07-16 23:28:05 +03005073 kbuf = io_recv_buffer_select(req, !force_nonblock);
Pavel Begunkovbc02ef32020-07-16 23:28:03 +03005074 if (IS_ERR(kbuf))
5075 return PTR_ERR(kbuf);
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03005076 kmsg->fast_iov[0].iov_base = u64_to_user_ptr(kbuf->addr);
Pavel Begunkov5476dfe2021-02-05 00:57:59 +00005077 kmsg->fast_iov[0].iov_len = req->sr_msg.len;
5078 iov_iter_init(&kmsg->msg.msg_iter, READ, kmsg->fast_iov,
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03005079 1, req->sr_msg.len);
5080 }
5081
Pavel Begunkov04411802021-04-01 15:44:00 +01005082 flags = req->sr_msg.msg_flags;
5083 if (force_nonblock)
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03005084 flags |= MSG_DONTWAIT;
Stefan Metzmacher00312752021-03-20 20:33:36 +01005085 if (flags & MSG_WAITALL)
5086 min_ret = iov_iter_count(&kmsg->msg.msg_iter);
5087
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03005088 ret = __sys_recvmsg_sock(sock, &kmsg->msg, req->sr_msg.umsg,
5089 kmsg->uaddr, flags);
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03005090 if (force_nonblock && ret == -EAGAIN)
5091 return io_setup_async_msg(req, kmsg);
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03005092 if (ret == -ERESTARTSYS)
5093 ret = -EINTR;
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03005094
Pavel Begunkov7fbb1b52020-07-16 23:28:05 +03005095 if (req->flags & REQ_F_BUFFER_SELECTED)
5096 cflags = io_put_recv_kbuf(req);
Pavel Begunkov257e84a2021-02-05 00:58:00 +00005097 /* fast path, check for non-NULL to avoid function call */
5098 if (kmsg->free_iov)
5099 kfree(kmsg->free_iov);
Pavel Begunkov99bc4c32020-02-07 22:04:45 +03005100 req->flags &= ~REQ_F_NEED_CLEANUP;
Stefan Metzmacher00312752021-03-20 20:33:36 +01005101 if (ret < min_ret || ((flags & MSG_WAITALL) && (kmsg->msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))))
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01005102 req_set_fail(req);
Pavel Begunkov889fca72021-02-10 00:03:09 +00005103 __io_req_complete(req, issue_flags, ret, cflags);
Jens Axboe0fa03c62019-04-19 13:34:07 -06005104 return 0;
Jens Axboe0fa03c62019-04-19 13:34:07 -06005105}
5106
Pavel Begunkov889fca72021-02-10 00:03:09 +00005107static int io_recv(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboefddafac2020-01-04 20:19:44 -07005108{
Pavel Begunkov6b754c82020-07-16 23:28:00 +03005109 struct io_buffer *kbuf;
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03005110 struct io_sr_msg *sr = &req->sr_msg;
5111 struct msghdr msg;
5112 void __user *buf = sr->buf;
Jens Axboefddafac2020-01-04 20:19:44 -07005113 struct socket *sock;
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03005114 struct iovec iov;
5115 unsigned flags;
Stefan Metzmacher00312752021-03-20 20:33:36 +01005116 int min_ret = 0;
Jens Axboebcda7ba2020-02-23 16:42:51 -07005117 int ret, cflags = 0;
Pavel Begunkov45d189c2021-02-10 00:03:07 +00005118 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
Jens Axboefddafac2020-01-04 20:19:44 -07005119
Florent Revestdba4a922020-12-04 12:36:04 +01005120 sock = sock_from_file(req->file);
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03005121 if (unlikely(!sock))
Florent Revestdba4a922020-12-04 12:36:04 +01005122 return -ENOTSOCK;
Jens Axboefddafac2020-01-04 20:19:44 -07005123
Pavel Begunkovbc02ef32020-07-16 23:28:03 +03005124 if (req->flags & REQ_F_BUFFER_SELECT) {
Pavel Begunkov7fbb1b52020-07-16 23:28:05 +03005125 kbuf = io_recv_buffer_select(req, !force_nonblock);
Jens Axboebcda7ba2020-02-23 16:42:51 -07005126 if (IS_ERR(kbuf))
5127 return PTR_ERR(kbuf);
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03005128 buf = u64_to_user_ptr(kbuf->addr);
Jens Axboefddafac2020-01-04 20:19:44 -07005129 }
5130
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03005131 ret = import_single_range(READ, buf, sr->len, &iov, &msg.msg_iter);
Pavel Begunkov14c32ee2020-07-16 23:28:01 +03005132 if (unlikely(ret))
5133 goto out_free;
Jens Axboefddafac2020-01-04 20:19:44 -07005134
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03005135 msg.msg_name = NULL;
5136 msg.msg_control = NULL;
5137 msg.msg_controllen = 0;
5138 msg.msg_namelen = 0;
5139 msg.msg_iocb = NULL;
5140 msg.msg_flags = 0;
5141
Pavel Begunkov04411802021-04-01 15:44:00 +01005142 flags = req->sr_msg.msg_flags;
5143 if (force_nonblock)
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03005144 flags |= MSG_DONTWAIT;
Stefan Metzmacher00312752021-03-20 20:33:36 +01005145 if (flags & MSG_WAITALL)
5146 min_ret = iov_iter_count(&msg.msg_iter);
5147
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03005148 ret = sock_recvmsg(sock, &msg, flags);
5149 if (force_nonblock && ret == -EAGAIN)
5150 return -EAGAIN;
5151 if (ret == -ERESTARTSYS)
5152 ret = -EINTR;
Pavel Begunkov14c32ee2020-07-16 23:28:01 +03005153out_free:
Pavel Begunkov7fbb1b52020-07-16 23:28:05 +03005154 if (req->flags & REQ_F_BUFFER_SELECTED)
5155 cflags = io_put_recv_kbuf(req);
Stefan Metzmacher00312752021-03-20 20:33:36 +01005156 if (ret < min_ret || ((flags & MSG_WAITALL) && (msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))))
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01005157 req_set_fail(req);
Pavel Begunkov889fca72021-02-10 00:03:09 +00005158 __io_req_complete(req, issue_flags, ret, cflags);
Jens Axboefddafac2020-01-04 20:19:44 -07005159 return 0;
Jens Axboefddafac2020-01-04 20:19:44 -07005160}
5161
Jens Axboe3529d8c2019-12-19 18:24:38 -07005162static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboe17f2fe32019-10-17 14:42:58 -06005163{
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07005164 struct io_accept *accept = &req->accept;
5165
Jens Axboe14587a462020-09-05 11:36:08 -06005166 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
Jens Axboe17f2fe32019-10-17 14:42:58 -06005167 return -EINVAL;
Pavel Begunkovaaa4db12021-08-25 12:25:47 +01005168 if (sqe->ioprio || sqe->len || sqe->buf_index)
Jens Axboe17f2fe32019-10-17 14:42:58 -06005169 return -EINVAL;
5170
Jens Axboed55e5f52019-12-11 16:12:15 -07005171 accept->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));
5172 accept->addr_len = u64_to_user_ptr(READ_ONCE(sqe->addr2));
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07005173 accept->flags = READ_ONCE(sqe->accept_flags);
Jens Axboe09952e32020-03-19 20:16:56 -06005174 accept->nofile = rlimit(RLIMIT_NOFILE);
Pavel Begunkova7083ad2021-08-25 12:25:46 +01005175
Pavel Begunkovaaa4db12021-08-25 12:25:47 +01005176 accept->file_slot = READ_ONCE(sqe->file_index);
Jens Axboe13239762022-03-14 17:26:19 -06005177 if (accept->file_slot && (accept->flags & SOCK_CLOEXEC))
Pavel Begunkovaaa4db12021-08-25 12:25:47 +01005178 return -EINVAL;
Pavel Begunkova7083ad2021-08-25 12:25:46 +01005179 if (accept->flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK))
5180 return -EINVAL;
5181 if (SOCK_NONBLOCK != O_NONBLOCK && (accept->flags & SOCK_NONBLOCK))
5182 accept->flags = (accept->flags & ~SOCK_NONBLOCK) | O_NONBLOCK;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07005183 return 0;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07005184}
Jens Axboe17f2fe32019-10-17 14:42:58 -06005185
Pavel Begunkov889fca72021-02-10 00:03:09 +00005186static int io_accept(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07005187{
5188 struct io_accept *accept = &req->accept;
Pavel Begunkov45d189c2021-02-10 00:03:07 +00005189 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
Pavel Begunkovac45abc2020-06-08 21:08:18 +03005190 unsigned int file_flags = force_nonblock ? O_NONBLOCK : 0;
Pavel Begunkovaaa4db12021-08-25 12:25:47 +01005191 bool fixed = !!accept->file_slot;
Pavel Begunkova7083ad2021-08-25 12:25:46 +01005192 struct file *file;
5193 int ret, fd;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07005194
Jiufei Xuee697dee2020-06-10 13:41:59 +08005195 if (req->file->f_flags & O_NONBLOCK)
5196 req->flags |= REQ_F_NOWAIT;
5197
Pavel Begunkovaaa4db12021-08-25 12:25:47 +01005198 if (!fixed) {
5199 fd = __get_unused_fd_flags(accept->flags, accept->nofile);
5200 if (unlikely(fd < 0))
5201 return fd;
5202 }
Pavel Begunkova7083ad2021-08-25 12:25:46 +01005203 file = do_accept(req->file, file_flags, accept->addr, accept->addr_len,
5204 accept->flags);
5205 if (IS_ERR(file)) {
Pavel Begunkovaaa4db12021-08-25 12:25:47 +01005206 if (!fixed)
5207 put_unused_fd(fd);
Pavel Begunkova7083ad2021-08-25 12:25:46 +01005208 ret = PTR_ERR(file);
5209 if (ret == -EAGAIN && force_nonblock)
5210 return -EAGAIN;
Pavel Begunkovac45abc2020-06-08 21:08:18 +03005211 if (ret == -ERESTARTSYS)
5212 ret = -EINTR;
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01005213 req_set_fail(req);
Pavel Begunkovaaa4db12021-08-25 12:25:47 +01005214 } else if (!fixed) {
Pavel Begunkova7083ad2021-08-25 12:25:46 +01005215 fd_install(fd, file);
5216 ret = fd;
Pavel Begunkovaaa4db12021-08-25 12:25:47 +01005217 } else {
5218 ret = io_install_fixed_file(req, file, issue_flags,
5219 accept->file_slot - 1);
Pavel Begunkovac45abc2020-06-08 21:08:18 +03005220 }
Pavel Begunkov889fca72021-02-10 00:03:09 +00005221 __io_req_complete(req, issue_flags, ret, 0);
Jens Axboe17f2fe32019-10-17 14:42:58 -06005222 return 0;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07005223}
5224
Pavel Begunkov93642ef2021-02-18 18:29:44 +00005225static int io_connect_prep_async(struct io_kiocb *req)
5226{
5227 struct io_async_connect *io = req->async_data;
5228 struct io_connect *conn = &req->connect;
5229
5230 return move_addr_to_kernel(conn->addr, conn->addr_len, &io->address);
5231}
5232
Jens Axboe3529d8c2019-12-19 18:24:38 -07005233static int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboef499a022019-12-02 16:28:46 -07005234{
Jens Axboe3529d8c2019-12-19 18:24:38 -07005235 struct io_connect *conn = &req->connect;
Jens Axboef499a022019-12-02 16:28:46 -07005236
Jens Axboe14587a462020-09-05 11:36:08 -06005237 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
Jens Axboe3fbb51c2019-12-20 08:51:52 -07005238 return -EINVAL;
Pavel Begunkov26578cd2021-08-20 10:36:37 +01005239 if (sqe->ioprio || sqe->len || sqe->buf_index || sqe->rw_flags ||
5240 sqe->splice_fd_in)
Jens Axboe3fbb51c2019-12-20 08:51:52 -07005241 return -EINVAL;
5242
Jens Axboe3529d8c2019-12-19 18:24:38 -07005243 conn->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));
5244 conn->addr_len = READ_ONCE(sqe->addr2);
Pavel Begunkov93642ef2021-02-18 18:29:44 +00005245 return 0;
Jens Axboef499a022019-12-02 16:28:46 -07005246}
5247
Pavel Begunkov889fca72021-02-10 00:03:09 +00005248static int io_connect(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboef8e85cf2019-11-23 14:24:24 -07005249{
Jens Axboee8c2bc12020-08-15 18:44:09 -07005250 struct io_async_connect __io, *io;
Jens Axboef8e85cf2019-11-23 14:24:24 -07005251 unsigned file_flags;
Jens Axboe3fbb51c2019-12-20 08:51:52 -07005252 int ret;
Pavel Begunkov45d189c2021-02-10 00:03:07 +00005253 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
Jens Axboef8e85cf2019-11-23 14:24:24 -07005254
Jens Axboee8c2bc12020-08-15 18:44:09 -07005255 if (req->async_data) {
5256 io = req->async_data;
Jens Axboef499a022019-12-02 16:28:46 -07005257 } else {
Jens Axboe3529d8c2019-12-19 18:24:38 -07005258 ret = move_addr_to_kernel(req->connect.addr,
5259 req->connect.addr_len,
Jens Axboee8c2bc12020-08-15 18:44:09 -07005260 &__io.address);
Jens Axboef499a022019-12-02 16:28:46 -07005261 if (ret)
5262 goto out;
5263 io = &__io;
5264 }
5265
Jens Axboe3fbb51c2019-12-20 08:51:52 -07005266 file_flags = force_nonblock ? O_NONBLOCK : 0;
5267
Jens Axboee8c2bc12020-08-15 18:44:09 -07005268 ret = __sys_connect_file(req->file, &io->address,
Jens Axboe3fbb51c2019-12-20 08:51:52 -07005269 req->connect.addr_len, file_flags);
Jens Axboe87f80d62019-12-03 11:23:54 -07005270 if ((ret == -EAGAIN || ret == -EINPROGRESS) && force_nonblock) {
Jens Axboee8c2bc12020-08-15 18:44:09 -07005271 if (req->async_data)
Jens Axboeb7bb4f72019-12-15 22:13:43 -07005272 return -EAGAIN;
Jens Axboee8c2bc12020-08-15 18:44:09 -07005273 if (io_alloc_async_data(req)) {
Jens Axboef499a022019-12-02 16:28:46 -07005274 ret = -ENOMEM;
5275 goto out;
5276 }
Jens Axboee8c2bc12020-08-15 18:44:09 -07005277 memcpy(req->async_data, &__io, sizeof(__io));
Jens Axboef8e85cf2019-11-23 14:24:24 -07005278 return -EAGAIN;
Jens Axboef499a022019-12-02 16:28:46 -07005279 }
Jens Axboef8e85cf2019-11-23 14:24:24 -07005280 if (ret == -ERESTARTSYS)
5281 ret = -EINTR;
Jens Axboef499a022019-12-02 16:28:46 -07005282out:
Jens Axboe4e88d6e2019-12-07 20:59:47 -07005283 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01005284 req_set_fail(req);
Pavel Begunkov889fca72021-02-10 00:03:09 +00005285 __io_req_complete(req, issue_flags, ret, 0);
Jens Axboef8e85cf2019-11-23 14:24:24 -07005286 return 0;
Jens Axboef8e85cf2019-11-23 14:24:24 -07005287}
YueHaibing469956e2020-03-04 15:53:52 +08005288#else /* !CONFIG_NET */
Jens Axboe99a10082021-02-19 09:35:19 -07005289#define IO_NETOP_FN(op) \
5290static int io_##op(struct io_kiocb *req, unsigned int issue_flags) \
5291{ \
5292 return -EOPNOTSUPP; \
Jens Axboef8e85cf2019-11-23 14:24:24 -07005293}
5294
Jens Axboe99a10082021-02-19 09:35:19 -07005295#define IO_NETOP_PREP(op) \
5296IO_NETOP_FN(op) \
5297static int io_##op##_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) \
5298{ \
5299 return -EOPNOTSUPP; \
5300} \
5301
5302#define IO_NETOP_PREP_ASYNC(op) \
5303IO_NETOP_PREP(op) \
5304static int io_##op##_prep_async(struct io_kiocb *req) \
5305{ \
5306 return -EOPNOTSUPP; \
YueHaibing469956e2020-03-04 15:53:52 +08005307}
5308
Jens Axboe99a10082021-02-19 09:35:19 -07005309IO_NETOP_PREP_ASYNC(sendmsg);
5310IO_NETOP_PREP_ASYNC(recvmsg);
5311IO_NETOP_PREP_ASYNC(connect);
5312IO_NETOP_PREP(accept);
5313IO_NETOP_FN(send);
5314IO_NETOP_FN(recv);
YueHaibing469956e2020-03-04 15:53:52 +08005315#endif /* CONFIG_NET */
Jens Axboe17f2fe32019-10-17 14:42:58 -06005316
Jens Axboed7718a92020-02-14 22:23:12 -07005317struct io_poll_table {
5318 struct poll_table_struct pt;
5319 struct io_kiocb *req;
Pavel Begunkov68b11e82021-07-20 10:50:43 +01005320 int nr_entries;
Jens Axboed7718a92020-02-14 22:23:12 -07005321 int error;
5322};
5323
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005324#define IO_POLL_CANCEL_FLAG BIT(31)
Jens Axboec41e79a2022-08-29 14:30:21 +01005325#define IO_POLL_REF_MASK GENMASK(30, 0)
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005326
5327/*
5328 * If refs part of ->poll_refs (see IO_POLL_REF_MASK) is 0, it's free. We can
5329 * bump it and acquire ownership. It's disallowed to modify requests while not
5330 * owning it, that prevents from races for enqueueing task_work's and b/w
5331 * arming poll and wakeups.
5332 */
5333static inline bool io_poll_get_ownership(struct io_kiocb *req)
5334{
5335 return !(atomic_fetch_inc(&req->poll_refs) & IO_POLL_REF_MASK);
5336}
5337
5338static void io_poll_mark_cancelled(struct io_kiocb *req)
5339{
5340 atomic_or(IO_POLL_CANCEL_FLAG, &req->poll_refs);
5341}
5342
Pavel Begunkova85d7ac2022-08-29 14:30:15 +01005343static struct io_poll_iocb *io_poll_get_double(struct io_kiocb *req)
5344{
5345 /* pure poll stashes this in ->async_data, poll driven retry elsewhere */
5346 if (req->opcode == IORING_OP_POLL_ADD)
5347 return req->async_data;
5348 return req->apoll->double_poll;
5349}
5350
5351static struct io_poll_iocb *io_poll_get_single(struct io_kiocb *req)
5352{
5353 if (req->opcode == IORING_OP_POLL_ADD)
5354 return &req->poll;
5355 return &req->apoll->poll;
5356}
5357
5358static void io_poll_req_insert(struct io_kiocb *req)
5359{
5360 struct io_ring_ctx *ctx = req->ctx;
5361 struct hlist_head *list;
5362
5363 list = &ctx->cancel_hash[hash_long(req->user_data, ctx->cancel_hash_bits)];
5364 hlist_add_head(&req->hash_node, list);
5365}
5366
5367static void io_init_poll_iocb(struct io_poll_iocb *poll, __poll_t events,
5368 wait_queue_func_t wake_func)
5369{
5370 poll->head = NULL;
Pavel Begunkova85d7ac2022-08-29 14:30:15 +01005371#define IO_POLL_UNMASK (EPOLLERR|EPOLLHUP|EPOLLNVAL|EPOLLRDHUP)
5372 /* mask in events that we always want/need */
5373 poll->events = events | IO_POLL_UNMASK;
5374 INIT_LIST_HEAD(&poll->wait.entry);
5375 init_waitqueue_func_entry(&poll->wait, wake_func);
5376}
5377
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005378static inline void io_poll_remove_entry(struct io_poll_iocb *poll)
Jens Axboed7718a92020-02-14 22:23:12 -07005379{
Pavel Begunkove9d7ca02022-08-29 14:30:24 +01005380 struct wait_queue_head *head = smp_load_acquire(&poll->head);
Jens Axboed7718a92020-02-14 22:23:12 -07005381
Pavel Begunkove9d7ca02022-08-29 14:30:24 +01005382 if (head) {
5383 spin_lock_irq(&head->lock);
5384 list_del_init(&poll->wait.entry);
5385 poll->head = NULL;
5386 spin_unlock_irq(&head->lock);
5387 }
Jens Axboed7718a92020-02-14 22:23:12 -07005388}
5389
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005390static void io_poll_remove_entries(struct io_kiocb *req)
5391{
5392 struct io_poll_iocb *poll = io_poll_get_single(req);
5393 struct io_poll_iocb *poll_double = io_poll_get_double(req);
5394
Pavel Begunkove9d7ca02022-08-29 14:30:24 +01005395 /*
5396 * While we hold the waitqueue lock and the waitqueue is nonempty,
5397 * wake_up_pollfree() will wait for us. However, taking the waitqueue
5398 * lock in the first place can race with the waitqueue being freed.
5399 *
5400 * We solve this as eventpoll does: by taking advantage of the fact that
5401 * all users of wake_up_pollfree() will RCU-delay the actual free. If
5402 * we enter rcu_read_lock() and see that the pointer to the queue is
5403 * non-NULL, we can then lock it without the memory being freed out from
5404 * under us.
5405 *
5406 * Keep holding rcu_read_lock() as long as we hold the queue lock, in
5407 * case the caller deletes the entry from the queue, leaving it empty.
5408 * In that case, only RCU prevents the queue memory from being freed.
5409 */
5410 rcu_read_lock();
5411 io_poll_remove_entry(poll);
5412 if (poll_double)
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005413 io_poll_remove_entry(poll_double);
Pavel Begunkove9d7ca02022-08-29 14:30:24 +01005414 rcu_read_unlock();
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005415}
5416
5417/*
5418 * All poll tw should go through this. Checks for poll events, manages
5419 * references, does rewait, etc.
5420 *
5421 * Returns a negative error on failure. >0 when no action require, which is
5422 * either spurious wakeup or multishot CQE is served. 0 when it's done with
5423 * the request, then the mask is stored in req->result.
5424 */
5425static int io_poll_check_events(struct io_kiocb *req)
Jens Axboe74ce6ce2020-04-13 11:09:12 -06005426{
5427 struct io_ring_ctx *ctx = req->ctx;
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005428 struct io_poll_iocb *poll = io_poll_get_single(req);
5429 int v;
Jens Axboe74ce6ce2020-04-13 11:09:12 -06005430
Jens Axboe316319e2021-08-19 09:41:42 -06005431 /* req->task == current here, checking PF_EXITING is safe */
Pavel Begunkove09ee512021-07-01 13:26:05 +01005432 if (unlikely(req->task->flags & PF_EXITING))
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005433 io_poll_mark_cancelled(req);
Pavel Begunkove09ee512021-07-01 13:26:05 +01005434
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005435 do {
5436 v = atomic_read(&req->poll_refs);
Jens Axboe74ce6ce2020-04-13 11:09:12 -06005437
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005438 /* tw handler should be the owner, and so have some references */
5439 if (WARN_ON_ONCE(!(v & IO_POLL_REF_MASK)))
5440 return 0;
5441 if (v & IO_POLL_CANCEL_FLAG)
5442 return -ECANCELED;
Pavel Begunkovcd1981a2022-12-02 14:27:12 +00005443 /*
5444 * cqe.res contains only events of the first wake up
5445 * and all others are be lost. Redo vfs_poll() to get
5446 * up to date state.
5447 */
5448 if ((v & IO_POLL_REF_MASK) != 1)
5449 req->result = 0;
Jens Axboe74ce6ce2020-04-13 11:09:12 -06005450
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005451 if (!req->result) {
5452 struct poll_table_struct pt = { ._key = poll->events };
Jens Axboe74ce6ce2020-04-13 11:09:12 -06005453
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005454 req->result = vfs_poll(req->file, &pt) & poll->events;
5455 }
Jens Axboe74ce6ce2020-04-13 11:09:12 -06005456
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005457 /* multishot, just fill an CQE and proceed */
5458 if (req->result && !(poll->events & EPOLLONESHOT)) {
5459 __poll_t mask = mangle_poll(req->result & poll->events);
5460 bool filled;
Jens Axboe18bceab2020-05-15 11:56:54 -06005461
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005462 spin_lock(&ctx->completion_lock);
5463 filled = io_fill_cqe_aux(ctx, req->user_data, mask,
5464 IORING_CQE_F_MORE);
5465 io_commit_cqring(ctx);
5466 spin_unlock(&ctx->completion_lock);
5467 if (unlikely(!filled))
5468 return -ECANCELED;
5469 io_cqring_ev_posted(ctx);
5470 } else if (req->result) {
5471 return 0;
5472 }
Jens Axboe18bceab2020-05-15 11:56:54 -06005473
Pavel Begunkov62321dc2022-12-02 14:27:11 +00005474 /* force the next iteration to vfs_poll() */
5475 req->result = 0;
5476
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005477 /*
5478 * Release all references, retry if someone tried to restart
5479 * task_work while we were executing it.
5480 */
5481 } while (atomic_sub_return(v & IO_POLL_REF_MASK, &req->poll_refs));
Jens Axboe18bceab2020-05-15 11:56:54 -06005482
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005483 return 1;
Jens Axboe18bceab2020-05-15 11:56:54 -06005484}
5485
Pavel Begunkovf237c302021-08-18 12:42:46 +01005486static void io_poll_task_func(struct io_kiocb *req, bool *locked)
Jens Axboe18bceab2020-05-15 11:56:54 -06005487{
Jens Axboe6d816e02020-08-11 08:04:14 -06005488 struct io_ring_ctx *ctx = req->ctx;
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005489 int ret;
Jens Axboe18bceab2020-05-15 11:56:54 -06005490
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005491 ret = io_poll_check_events(req);
5492 if (ret > 0)
5493 return;
5494
5495 if (!ret) {
5496 req->result = mangle_poll(req->result & req->poll.events);
Pavel Begunkovdd221f462020-10-18 10:17:42 +01005497 } else {
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005498 req->result = ret;
5499 req_set_fail(req);
Pavel Begunkovdd221f462020-10-18 10:17:42 +01005500 }
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005501
5502 io_poll_remove_entries(req);
5503 spin_lock(&ctx->completion_lock);
5504 hash_del(&req->hash_node);
5505 spin_unlock(&ctx->completion_lock);
5506 io_req_complete_post(req, req->result, 0);
Jens Axboe18bceab2020-05-15 11:56:54 -06005507}
5508
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005509static void io_apoll_task_func(struct io_kiocb *req, bool *locked)
5510{
5511 struct io_ring_ctx *ctx = req->ctx;
5512 int ret;
5513
5514 ret = io_poll_check_events(req);
5515 if (ret > 0)
5516 return;
5517
5518 io_poll_remove_entries(req);
5519 spin_lock(&ctx->completion_lock);
5520 hash_del(&req->hash_node);
5521 spin_unlock(&ctx->completion_lock);
5522
5523 if (!ret)
5524 io_req_task_submit(req, locked);
5525 else
5526 io_req_complete_failed(req, ret);
5527}
5528
5529static void __io_poll_execute(struct io_kiocb *req, int mask)
5530{
5531 req->result = mask;
5532 if (req->opcode == IORING_OP_POLL_ADD)
5533 req->io_task_work.func = io_poll_task_func;
5534 else
5535 req->io_task_work.func = io_apoll_task_func;
5536
5537 trace_io_uring_task_add(req->ctx, req->opcode, req->user_data, mask);
5538 io_req_task_work_add(req);
5539}
5540
5541static inline void io_poll_execute(struct io_kiocb *req, int res)
5542{
5543 if (io_poll_get_ownership(req))
5544 __io_poll_execute(req, res);
5545}
5546
5547static void io_poll_cancel_req(struct io_kiocb *req)
5548{
5549 io_poll_mark_cancelled(req);
5550 /* kick tw, which should complete the request */
5551 io_poll_execute(req, 0);
5552}
5553
5554static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
5555 void *key)
Jens Axboe18bceab2020-05-15 11:56:54 -06005556{
5557 struct io_kiocb *req = wait->private;
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005558 struct io_poll_iocb *poll = container_of(wait, struct io_poll_iocb,
5559 wait);
Jens Axboe18bceab2020-05-15 11:56:54 -06005560 __poll_t mask = key_to_poll(key);
5561
Pavel Begunkove9d7ca02022-08-29 14:30:24 +01005562 if (unlikely(mask & POLLFREE)) {
5563 io_poll_mark_cancelled(req);
5564 /* we have to kick tw in case it's not already */
5565 io_poll_execute(req, 0);
5566
5567 /*
5568 * If the waitqueue is being freed early but someone is already
5569 * holds ownership over it, we have to tear down the request as
5570 * best we can. That means immediately removing the request from
5571 * its waitqueue and preventing all further accesses to the
5572 * waitqueue via the request.
5573 */
5574 list_del_init(&poll->wait.entry);
5575
5576 /*
5577 * Careful: this *must* be the last step, since as soon
5578 * as req->head is NULL'ed out, the request can be
5579 * completed and freed, since aio_poll_complete_work()
5580 * will no longer need to take the waitqueue lock.
5581 */
5582 smp_store_release(&poll->head, NULL);
5583 return 1;
5584 }
5585
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005586 /* for instances that support it check for an event match first */
Jens Axboe18bceab2020-05-15 11:56:54 -06005587 if (mask && !(mask & poll->events))
5588 return 0;
5589
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005590 if (io_poll_get_ownership(req))
5591 __io_poll_execute(req, mask);
Jens Axboe18bceab2020-05-15 11:56:54 -06005592 return 1;
5593}
5594
Jens Axboe18bceab2020-05-15 11:56:54 -06005595static void __io_queue_proc(struct io_poll_iocb *poll, struct io_poll_table *pt,
Jens Axboe807abcb2020-07-17 17:09:27 -06005596 struct wait_queue_head *head,
5597 struct io_poll_iocb **poll_ptr)
Jens Axboe18bceab2020-05-15 11:56:54 -06005598{
5599 struct io_kiocb *req = pt->req;
5600
5601 /*
Pavel Begunkov68b11e82021-07-20 10:50:43 +01005602 * The file being polled uses multiple waitqueues for poll handling
5603 * (e.g. one for read, one for write). Setup a separate io_poll_iocb
5604 * if this happens.
Jens Axboe18bceab2020-05-15 11:56:54 -06005605 */
Pavel Begunkov68b11e82021-07-20 10:50:43 +01005606 if (unlikely(pt->nr_entries)) {
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005607 struct io_poll_iocb *first = poll;
Pavel Begunkov58852d42020-10-16 20:55:56 +01005608
Pavel Begunkov23a65db2021-08-17 20:28:11 +01005609 /* double add on the same waitqueue head, ignore */
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005610 if (first->head == head)
Pavel Begunkov23a65db2021-08-17 20:28:11 +01005611 return;
Jens Axboe18bceab2020-05-15 11:56:54 -06005612 /* already have a 2nd entry, fail a third attempt */
Jens Axboe807abcb2020-07-17 17:09:27 -06005613 if (*poll_ptr) {
Pavel Begunkov23a65db2021-08-17 20:28:11 +01005614 if ((*poll_ptr)->head == head)
5615 return;
Jens Axboe18bceab2020-05-15 11:56:54 -06005616 pt->error = -EINVAL;
5617 return;
5618 }
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005619
Jens Axboe18bceab2020-05-15 11:56:54 -06005620 poll = kmalloc(sizeof(*poll), GFP_ATOMIC);
5621 if (!poll) {
5622 pt->error = -ENOMEM;
5623 return;
5624 }
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005625 io_init_poll_iocb(poll, first->events, first->wait.func);
Jens Axboe807abcb2020-07-17 17:09:27 -06005626 *poll_ptr = poll;
Jens Axboe18bceab2020-05-15 11:56:54 -06005627 }
5628
Pavel Begunkov68b11e82021-07-20 10:50:43 +01005629 pt->nr_entries++;
Jens Axboe18bceab2020-05-15 11:56:54 -06005630 poll->head = head;
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005631 poll->wait.private = req;
Jiufei Xuea31eb4a2020-06-17 17:53:56 +08005632
5633 if (poll->events & EPOLLEXCLUSIVE)
5634 add_wait_queue_exclusive(head, &poll->wait);
5635 else
5636 add_wait_queue(head, &poll->wait);
Jens Axboe18bceab2020-05-15 11:56:54 -06005637}
5638
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005639static void io_poll_queue_proc(struct file *file, struct wait_queue_head *head,
5640 struct poll_table_struct *p)
5641{
5642 struct io_poll_table *pt = container_of(p, struct io_poll_table, pt);
5643
5644 __io_queue_proc(&pt->req->poll, pt, head,
5645 (struct io_poll_iocb **) &pt->req->async_data);
5646}
5647
5648static int __io_arm_poll_handler(struct io_kiocb *req,
5649 struct io_poll_iocb *poll,
5650 struct io_poll_table *ipt, __poll_t mask)
5651{
5652 struct io_ring_ctx *ctx = req->ctx;
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005653
5654 INIT_HLIST_NODE(&req->hash_node);
5655 io_init_poll_iocb(poll, mask, io_poll_wake);
5656 poll->file = req->file;
5657 poll->wait.private = req;
5658
5659 ipt->pt._key = mask;
5660 ipt->req = req;
5661 ipt->error = 0;
5662 ipt->nr_entries = 0;
5663
5664 /*
5665 * Take the ownership to delay any tw execution up until we're done
5666 * with poll arming. see io_poll_get_ownership().
5667 */
5668 atomic_set(&req->poll_refs, 1);
5669 mask = vfs_poll(req->file, &ipt->pt) & poll->events;
5670
5671 if (mask && (poll->events & EPOLLONESHOT)) {
5672 io_poll_remove_entries(req);
5673 /* no one else has access to the req, forget about the ref */
5674 return mask;
5675 }
5676 if (!mask && unlikely(ipt->error || !ipt->nr_entries)) {
5677 io_poll_remove_entries(req);
5678 if (!ipt->error)
5679 ipt->error = -EINVAL;
5680 return 0;
5681 }
5682
5683 spin_lock(&ctx->completion_lock);
5684 io_poll_req_insert(req);
5685 spin_unlock(&ctx->completion_lock);
5686
5687 if (mask) {
5688 /* can't multishot if failed, just queue the event we've got */
Pavel Begunkov182dc3a2022-08-29 14:30:23 +01005689 if (unlikely(ipt->error || !ipt->nr_entries)) {
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005690 poll->events |= EPOLLONESHOT;
Pavel Begunkov182dc3a2022-08-29 14:30:23 +01005691 ipt->error = 0;
5692 }
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005693 __io_poll_execute(req, mask);
5694 return 0;
5695 }
5696
5697 /*
Pavel Begunkov1d588492022-12-02 14:27:13 +00005698 * Try to release ownership. If we see a change of state, e.g.
5699 * poll was waken up, queue up a tw, it'll deal with it.
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005700 */
Pavel Begunkov1d588492022-12-02 14:27:13 +00005701 if (atomic_cmpxchg(&req->poll_refs, 1, 0) != 1)
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005702 __io_poll_execute(req, 0);
5703 return 0;
5704}
5705
Jens Axboe18bceab2020-05-15 11:56:54 -06005706static void io_async_queue_proc(struct file *file, struct wait_queue_head *head,
5707 struct poll_table_struct *p)
5708{
5709 struct io_poll_table *pt = container_of(p, struct io_poll_table, pt);
Jens Axboe807abcb2020-07-17 17:09:27 -06005710 struct async_poll *apoll = pt->req->apoll;
Jens Axboe18bceab2020-05-15 11:56:54 -06005711
Jens Axboe807abcb2020-07-17 17:09:27 -06005712 __io_queue_proc(&apoll->poll, pt, head, &apoll->double_poll);
Jens Axboe18bceab2020-05-15 11:56:54 -06005713}
5714
Olivier Langlois59b735a2021-06-22 05:17:39 -07005715enum {
5716 IO_APOLL_OK,
5717 IO_APOLL_ABORTED,
5718 IO_APOLL_READY
5719};
5720
5721static int io_arm_poll_handler(struct io_kiocb *req)
Jens Axboed7718a92020-02-14 22:23:12 -07005722{
5723 const struct io_op_def *def = &io_op_defs[req->opcode];
5724 struct io_ring_ctx *ctx = req->ctx;
5725 struct async_poll *apoll;
5726 struct io_poll_table ipt;
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005727 __poll_t mask = EPOLLONESHOT | POLLERR | POLLPRI;
5728 int ret;
Jens Axboed7718a92020-02-14 22:23:12 -07005729
5730 if (!req->file || !file_can_poll(req->file))
Olivier Langlois59b735a2021-06-22 05:17:39 -07005731 return IO_APOLL_ABORTED;
Pavel Begunkov24c74672020-06-21 13:09:51 +03005732 if (req->flags & REQ_F_POLLED)
Olivier Langlois59b735a2021-06-22 05:17:39 -07005733 return IO_APOLL_ABORTED;
Pavel Begunkovb2d9c3d2021-06-26 21:40:44 +01005734 if (!def->pollin && !def->pollout)
Olivier Langlois59b735a2021-06-22 05:17:39 -07005735 return IO_APOLL_ABORTED;
Pavel Begunkovb2d9c3d2021-06-26 21:40:44 +01005736
5737 if (def->pollin) {
Pavel Begunkovb2d9c3d2021-06-26 21:40:44 +01005738 mask |= POLLIN | POLLRDNORM;
5739
5740 /* If reading from MSG_ERRQUEUE using recvmsg, ignore POLLIN */
5741 if ((req->opcode == IORING_OP_RECVMSG) &&
5742 (req->sr_msg.msg_flags & MSG_ERRQUEUE))
5743 mask &= ~POLLIN;
5744 } else {
Pavel Begunkovb2d9c3d2021-06-26 21:40:44 +01005745 mask |= POLLOUT | POLLWRNORM;
5746 }
5747
Jens Axboed7718a92020-02-14 22:23:12 -07005748 apoll = kmalloc(sizeof(*apoll), GFP_ATOMIC);
5749 if (unlikely(!apoll))
Olivier Langlois59b735a2021-06-22 05:17:39 -07005750 return IO_APOLL_ABORTED;
Jens Axboe807abcb2020-07-17 17:09:27 -06005751 apoll->double_poll = NULL;
Jens Axboed7718a92020-02-14 22:23:12 -07005752 req->apoll = apoll;
Pavel Begunkovb2d9c3d2021-06-26 21:40:44 +01005753 req->flags |= REQ_F_POLLED;
Jens Axboed7718a92020-02-14 22:23:12 -07005754 ipt.pt._qproc = io_async_queue_proc;
5755
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005756 ret = __io_arm_poll_handler(req, &apoll->poll, &ipt, mask);
Hao Xu41a51692021-08-12 15:47:02 +08005757 if (ret || ipt.error)
5758 return ret ? IO_APOLL_READY : IO_APOLL_ABORTED;
5759
Olivier Langlois236daeae2021-05-31 02:36:37 -04005760 trace_io_uring_poll_arm(ctx, req, req->opcode, req->user_data,
5761 mask, apoll->poll.events);
Olivier Langlois59b735a2021-06-22 05:17:39 -07005762 return IO_APOLL_OK;
Jens Axboed7718a92020-02-14 22:23:12 -07005763}
5764
Jens Axboe76e1b642020-09-26 15:05:03 -06005765/*
5766 * Returns true if we found and killed one or more poll requests
5767 */
Pavel Begunkov6b819282020-11-06 13:00:25 +00005768static bool io_poll_remove_all(struct io_ring_ctx *ctx, struct task_struct *tsk,
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01005769 bool cancel_all)
Jens Axboe221c5eb2019-01-17 09:41:58 -07005770{
Jens Axboe78076bb2019-12-04 19:56:40 -07005771 struct hlist_node *tmp;
Jens Axboe221c5eb2019-01-17 09:41:58 -07005772 struct io_kiocb *req;
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005773 bool found = false;
5774 int i;
Jens Axboe221c5eb2019-01-17 09:41:58 -07005775
Jens Axboe79ebeae2021-08-10 15:18:27 -06005776 spin_lock(&ctx->completion_lock);
Jens Axboe78076bb2019-12-04 19:56:40 -07005777 for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) {
5778 struct hlist_head *list;
5779
5780 list = &ctx->cancel_hash[i];
Jens Axboef3606e32020-09-22 08:18:24 -06005781 hlist_for_each_entry_safe(req, tmp, list, hash_node) {
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005782 if (io_match_task_safe(req, tsk, cancel_all)) {
Jens Axboe7524ec52022-08-29 14:30:20 +01005783 hlist_del_init(&req->hash_node);
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005784 io_poll_cancel_req(req);
5785 found = true;
5786 }
Jens Axboef3606e32020-09-22 08:18:24 -06005787 }
Jens Axboe221c5eb2019-01-17 09:41:58 -07005788 }
Jens Axboe79ebeae2021-08-10 15:18:27 -06005789 spin_unlock(&ctx->completion_lock);
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005790 return found;
Jens Axboe221c5eb2019-01-17 09:41:58 -07005791}
5792
Pavel Begunkov9ba5fac2021-04-14 13:38:35 +01005793static struct io_kiocb *io_poll_find(struct io_ring_ctx *ctx, __u64 sqe_addr,
5794 bool poll_only)
Pavel Begunkove07785b2021-04-01 15:43:57 +01005795 __must_hold(&ctx->completion_lock)
Jens Axboe47f46762019-11-09 17:43:02 -07005796{
Jens Axboe78076bb2019-12-04 19:56:40 -07005797 struct hlist_head *list;
Jens Axboe47f46762019-11-09 17:43:02 -07005798 struct io_kiocb *req;
5799
Jens Axboe78076bb2019-12-04 19:56:40 -07005800 list = &ctx->cancel_hash[hash_long(sqe_addr, ctx->cancel_hash_bits)];
5801 hlist_for_each_entry(req, list, hash_node) {
Jens Axboeb41e9852020-02-17 09:52:41 -07005802 if (sqe_addr != req->user_data)
5803 continue;
Pavel Begunkov9ba5fac2021-04-14 13:38:35 +01005804 if (poll_only && req->opcode != IORING_OP_POLL_ADD)
5805 continue;
Jens Axboeb2cb8052021-03-17 08:17:19 -06005806 return req;
Jens Axboe47f46762019-11-09 17:43:02 -07005807 }
Jens Axboeb2cb8052021-03-17 08:17:19 -06005808 return NULL;
Jens Axboe47f46762019-11-09 17:43:02 -07005809}
5810
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005811static bool io_poll_disarm(struct io_kiocb *req)
5812 __must_hold(&ctx->completion_lock)
5813{
5814 if (!io_poll_get_ownership(req))
5815 return false;
5816 io_poll_remove_entries(req);
5817 hash_del(&req->hash_node);
5818 return true;
5819}
5820
Pavel Begunkov9ba5fac2021-04-14 13:38:35 +01005821static int io_poll_cancel(struct io_ring_ctx *ctx, __u64 sqe_addr,
5822 bool poll_only)
Pavel Begunkove07785b2021-04-01 15:43:57 +01005823 __must_hold(&ctx->completion_lock)
Jens Axboeb2cb8052021-03-17 08:17:19 -06005824{
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005825 struct io_kiocb *req = io_poll_find(ctx, sqe_addr, poll_only);
Jens Axboeb2cb8052021-03-17 08:17:19 -06005826
Jens Axboeb2cb8052021-03-17 08:17:19 -06005827 if (!req)
5828 return -ENOENT;
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005829 io_poll_cancel_req(req);
5830 return 0;
Jens Axboe221c5eb2019-01-17 09:41:58 -07005831}
5832
Pavel Begunkov9096af32021-04-14 13:38:36 +01005833static __poll_t io_poll_parse_events(const struct io_uring_sqe *sqe,
5834 unsigned int flags)
5835{
5836 u32 events;
5837
5838 events = READ_ONCE(sqe->poll32_events);
5839#ifdef __BIG_ENDIAN
5840 events = swahw32(events);
5841#endif
5842 if (!(flags & IORING_POLL_ADD_MULTI))
5843 events |= EPOLLONESHOT;
5844 return demangle_poll(events) | (events & (EPOLLEXCLUSIVE|EPOLLONESHOT));
5845}
5846
Pavel Begunkovc5de0032021-04-14 13:38:37 +01005847static int io_poll_update_prep(struct io_kiocb *req,
Jens Axboe3529d8c2019-12-19 18:24:38 -07005848 const struct io_uring_sqe *sqe)
Jens Axboe221c5eb2019-01-17 09:41:58 -07005849{
Pavel Begunkovc5de0032021-04-14 13:38:37 +01005850 struct io_poll_update *upd = &req->poll_update;
5851 u32 flags;
5852
Jens Axboe221c5eb2019-01-17 09:41:58 -07005853 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
5854 return -EINVAL;
Pavel Begunkov26578cd2021-08-20 10:36:37 +01005855 if (sqe->ioprio || sqe->buf_index || sqe->splice_fd_in)
Pavel Begunkovc5de0032021-04-14 13:38:37 +01005856 return -EINVAL;
5857 flags = READ_ONCE(sqe->len);
5858 if (flags & ~(IORING_POLL_UPDATE_EVENTS | IORING_POLL_UPDATE_USER_DATA |
5859 IORING_POLL_ADD_MULTI))
5860 return -EINVAL;
5861 /* meaningless without update */
5862 if (flags == IORING_POLL_ADD_MULTI)
Jens Axboe221c5eb2019-01-17 09:41:58 -07005863 return -EINVAL;
5864
Pavel Begunkovc5de0032021-04-14 13:38:37 +01005865 upd->old_user_data = READ_ONCE(sqe->addr);
5866 upd->update_events = flags & IORING_POLL_UPDATE_EVENTS;
5867 upd->update_user_data = flags & IORING_POLL_UPDATE_USER_DATA;
Jens Axboe0969e782019-12-17 18:40:57 -07005868
Pavel Begunkovc5de0032021-04-14 13:38:37 +01005869 upd->new_user_data = READ_ONCE(sqe->off);
5870 if (!upd->update_user_data && upd->new_user_data)
5871 return -EINVAL;
5872 if (upd->update_events)
5873 upd->events = io_poll_parse_events(sqe, flags);
5874 else if (sqe->poll32_events)
5875 return -EINVAL;
Jens Axboe0969e782019-12-17 18:40:57 -07005876
Jens Axboe221c5eb2019-01-17 09:41:58 -07005877 return 0;
5878}
5879
Jens Axboe3529d8c2019-12-19 18:24:38 -07005880static int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboe221c5eb2019-01-17 09:41:58 -07005881{
5882 struct io_poll_iocb *poll = &req->poll;
Pavel Begunkovc5de0032021-04-14 13:38:37 +01005883 u32 flags;
Jens Axboe221c5eb2019-01-17 09:41:58 -07005884
5885 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
5886 return -EINVAL;
Pavel Begunkovc5de0032021-04-14 13:38:37 +01005887 if (sqe->ioprio || sqe->buf_index || sqe->off || sqe->addr)
Jens Axboe88e41cf2021-02-22 22:08:01 -07005888 return -EINVAL;
5889 flags = READ_ONCE(sqe->len);
Pavel Begunkovc5de0032021-04-14 13:38:37 +01005890 if (flags & ~IORING_POLL_ADD_MULTI)
Jens Axboe221c5eb2019-01-17 09:41:58 -07005891 return -EINVAL;
5892
Pavel Begunkov48dcd382021-08-15 10:40:18 +01005893 io_req_set_refcount(req);
Pavel Begunkovc5de0032021-04-14 13:38:37 +01005894 poll->events = io_poll_parse_events(sqe, flags);
Jens Axboe0969e782019-12-17 18:40:57 -07005895 return 0;
5896}
5897
Pavel Begunkov61e98202021-02-10 00:03:08 +00005898static int io_poll_add(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe0969e782019-12-17 18:40:57 -07005899{
5900 struct io_poll_iocb *poll = &req->poll;
Jens Axboe0969e782019-12-17 18:40:57 -07005901 struct io_poll_table ipt;
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005902 int ret;
Jens Axboe0969e782019-12-17 18:40:57 -07005903
Jens Axboed7718a92020-02-14 22:23:12 -07005904 ipt.pt._qproc = io_poll_queue_proc;
Jens Axboe36703242019-07-25 10:20:18 -06005905
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005906 ret = __io_arm_poll_handler(req, &req->poll, &ipt, poll->events);
Pavel Begunkov6c7259c2022-08-29 14:30:22 +01005907 if (!ret && ipt.error)
5908 req_set_fail(req);
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005909 ret = ret ?: ipt.error;
5910 if (ret)
5911 __io_req_complete(req, issue_flags, ret, 0);
5912 return 0;
Jens Axboe221c5eb2019-01-17 09:41:58 -07005913}
5914
Pavel Begunkovc5de0032021-04-14 13:38:37 +01005915static int io_poll_update(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboeb69de282021-03-17 08:37:41 -06005916{
5917 struct io_ring_ctx *ctx = req->ctx;
5918 struct io_kiocb *preq;
Pavel Begunkov040e58f2022-08-29 14:30:14 +01005919 int ret2, ret = 0;
Jens Axboeb69de282021-03-17 08:37:41 -06005920
Jens Axboe79ebeae2021-08-10 15:18:27 -06005921 spin_lock(&ctx->completion_lock);
Pavel Begunkov9ba5fac2021-04-14 13:38:35 +01005922 preq = io_poll_find(ctx, req->poll_update.old_user_data, true);
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005923 if (!preq || !io_poll_disarm(preq)) {
Pavel Begunkov040e58f2022-08-29 14:30:14 +01005924 spin_unlock(&ctx->completion_lock);
Pavel Begunkovf770fba2022-08-29 14:30:18 +01005925 ret = preq ? -EALREADY : -ENOENT;
Pavel Begunkov040e58f2022-08-29 14:30:14 +01005926 goto out;
Jens Axboeb69de282021-03-17 08:37:41 -06005927 }
Jens Axboe79ebeae2021-08-10 15:18:27 -06005928 spin_unlock(&ctx->completion_lock);
Jens Axboecb3b200e2021-04-06 09:49:31 -06005929
Pavel Begunkov040e58f2022-08-29 14:30:14 +01005930 if (req->poll_update.update_events || req->poll_update.update_user_data) {
5931 /* only mask one event flags, keep behavior flags */
5932 if (req->poll_update.update_events) {
5933 preq->poll.events &= ~0xffff;
5934 preq->poll.events |= req->poll_update.events & 0xffff;
5935 preq->poll.events |= IO_POLL_UNMASK;
5936 }
5937 if (req->poll_update.update_user_data)
5938 preq->user_data = req->poll_update.new_user_data;
5939
5940 ret2 = io_poll_add(preq, issue_flags);
5941 /* successfully updated, don't complete poll request */
5942 if (!ret2)
5943 goto out;
5944 }
5945 req_set_fail(preq);
5946 io_req_complete(preq, -ECANCELED);
5947out:
5948 if (ret < 0)
5949 req_set_fail(req);
Jens Axboeb69de282021-03-17 08:37:41 -06005950 /* complete update request, we're done with it */
5951 io_req_complete(req, ret);
Jens Axboeb69de282021-03-17 08:37:41 -06005952 return 0;
5953}
5954
Pavel Begunkovf237c302021-08-18 12:42:46 +01005955static void io_req_task_timeout(struct io_kiocb *req, bool *locked)
Jens Axboe89850fc2021-08-10 15:11:51 -06005956{
Jens Axboe89850fc2021-08-10 15:11:51 -06005957 req_set_fail(req);
Pavel Begunkov505657b2021-08-17 20:28:09 +01005958 io_req_complete_post(req, -ETIME, 0);
Jens Axboe89850fc2021-08-10 15:11:51 -06005959}
5960
Jens Axboe5262f562019-09-17 12:26:57 -06005961static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer)
5962{
Jens Axboead8a48a2019-11-15 08:49:11 -07005963 struct io_timeout_data *data = container_of(timer,
5964 struct io_timeout_data, timer);
5965 struct io_kiocb *req = data->req;
5966 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe5262f562019-09-17 12:26:57 -06005967 unsigned long flags;
5968
Jens Axboe89850fc2021-08-10 15:11:51 -06005969 spin_lock_irqsave(&ctx->timeout_lock, flags);
Pavel Begunkova71976f2020-10-10 18:34:11 +01005970 list_del_init(&req->timeout.list);
Pavel Begunkov01cec8c2020-07-30 18:43:50 +03005971 atomic_set(&req->ctx->cq_timeouts,
5972 atomic_read(&req->ctx->cq_timeouts) + 1);
Jens Axboe89850fc2021-08-10 15:11:51 -06005973 spin_unlock_irqrestore(&ctx->timeout_lock, flags);
Pavel Begunkov01cec8c2020-07-30 18:43:50 +03005974
Jens Axboe89850fc2021-08-10 15:11:51 -06005975 req->io_task_work.func = io_req_task_timeout;
5976 io_req_task_work_add(req);
Jens Axboe5262f562019-09-17 12:26:57 -06005977 return HRTIMER_NORESTART;
5978}
5979
Pavel Begunkovfbd15842020-11-30 19:11:15 +00005980static struct io_kiocb *io_timeout_extract(struct io_ring_ctx *ctx,
5981 __u64 user_data)
Jens Axboe89850fc2021-08-10 15:11:51 -06005982 __must_hold(&ctx->timeout_lock)
Jens Axboe47f46762019-11-09 17:43:02 -07005983{
Pavel Begunkovfbd15842020-11-30 19:11:15 +00005984 struct io_timeout_data *io;
Jens Axboef254ac02020-08-12 17:33:30 -06005985 struct io_kiocb *req;
Pavel Begunkovfd9c7bc2021-04-13 02:58:42 +01005986 bool found = false;
Jens Axboef254ac02020-08-12 17:33:30 -06005987
5988 list_for_each_entry(req, &ctx->timeout_list, timeout.list) {
Pavel Begunkovfd9c7bc2021-04-13 02:58:42 +01005989 found = user_data == req->user_data;
5990 if (found)
Jens Axboef254ac02020-08-12 17:33:30 -06005991 break;
Jens Axboef254ac02020-08-12 17:33:30 -06005992 }
Pavel Begunkovfd9c7bc2021-04-13 02:58:42 +01005993 if (!found)
5994 return ERR_PTR(-ENOENT);
Jens Axboef254ac02020-08-12 17:33:30 -06005995
Pavel Begunkovfbd15842020-11-30 19:11:15 +00005996 io = req->async_data;
Pavel Begunkovfd9c7bc2021-04-13 02:58:42 +01005997 if (hrtimer_try_to_cancel(&io->timer) == -1)
Pavel Begunkovfbd15842020-11-30 19:11:15 +00005998 return ERR_PTR(-EALREADY);
5999 list_del_init(&req->timeout.list);
6000 return req;
6001}
6002
6003static int io_timeout_cancel(struct io_ring_ctx *ctx, __u64 user_data)
Pavel Begunkovec3c3d02021-08-18 10:50:52 +01006004 __must_hold(&ctx->completion_lock)
Jens Axboe89850fc2021-08-10 15:11:51 -06006005 __must_hold(&ctx->timeout_lock)
Pavel Begunkovfbd15842020-11-30 19:11:15 +00006006{
6007 struct io_kiocb *req = io_timeout_extract(ctx, user_data);
6008
6009 if (IS_ERR(req))
6010 return PTR_ERR(req);
6011
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01006012 req_set_fail(req);
Pavel Begunkovb850d6d2022-08-29 14:30:13 +01006013 io_fill_cqe_req(req, -ECANCELED, 0);
Pavel Begunkov91c2f692021-08-11 19:28:28 +01006014 io_put_req_deferred(req);
Pavel Begunkovfbd15842020-11-30 19:11:15 +00006015 return 0;
Jens Axboef254ac02020-08-12 17:33:30 -06006016}
6017
Jens Axboe50c1df22021-08-27 17:11:06 -06006018static clockid_t io_timeout_get_clock(struct io_timeout_data *data)
6019{
6020 switch (data->flags & IORING_TIMEOUT_CLOCK_MASK) {
6021 case IORING_TIMEOUT_BOOTTIME:
6022 return CLOCK_BOOTTIME;
6023 case IORING_TIMEOUT_REALTIME:
6024 return CLOCK_REALTIME;
6025 default:
6026 /* can't happen, vetted at prep time */
6027 WARN_ON_ONCE(1);
6028 fallthrough;
6029 case 0:
6030 return CLOCK_MONOTONIC;
6031 }
6032}
6033
Pavel Begunkovf1042b62021-08-28 19:54:39 -06006034static int io_linked_timeout_update(struct io_ring_ctx *ctx, __u64 user_data,
6035 struct timespec64 *ts, enum hrtimer_mode mode)
6036 __must_hold(&ctx->timeout_lock)
6037{
6038 struct io_timeout_data *io;
6039 struct io_kiocb *req;
6040 bool found = false;
6041
6042 list_for_each_entry(req, &ctx->ltimeout_list, timeout.list) {
6043 found = user_data == req->user_data;
6044 if (found)
6045 break;
6046 }
6047 if (!found)
6048 return -ENOENT;
6049
6050 io = req->async_data;
6051 if (hrtimer_try_to_cancel(&io->timer) == -1)
6052 return -EALREADY;
6053 hrtimer_init(&io->timer, io_timeout_get_clock(io), mode);
6054 io->timer.function = io_link_timeout_fn;
6055 hrtimer_start(&io->timer, timespec64_to_ktime(*ts), mode);
6056 return 0;
6057}
6058
Pavel Begunkov9c8e11b2020-11-30 19:11:16 +00006059static int io_timeout_update(struct io_ring_ctx *ctx, __u64 user_data,
6060 struct timespec64 *ts, enum hrtimer_mode mode)
Jens Axboe89850fc2021-08-10 15:11:51 -06006061 __must_hold(&ctx->timeout_lock)
Pavel Begunkov9c8e11b2020-11-30 19:11:16 +00006062{
6063 struct io_kiocb *req = io_timeout_extract(ctx, user_data);
6064 struct io_timeout_data *data;
6065
6066 if (IS_ERR(req))
6067 return PTR_ERR(req);
6068
6069 req->timeout.off = 0; /* noseq */
6070 data = req->async_data;
6071 list_add_tail(&req->timeout.list, &ctx->timeout_list);
Jens Axboe50c1df22021-08-27 17:11:06 -06006072 hrtimer_init(&data->timer, io_timeout_get_clock(data), mode);
Pavel Begunkov9c8e11b2020-11-30 19:11:16 +00006073 data->timer.function = io_timeout_fn;
6074 hrtimer_start(&data->timer, timespec64_to_ktime(*ts), mode);
6075 return 0;
Jens Axboe47f46762019-11-09 17:43:02 -07006076}
6077
Jens Axboe3529d8c2019-12-19 18:24:38 -07006078static int io_timeout_remove_prep(struct io_kiocb *req,
6079 const struct io_uring_sqe *sqe)
Jens Axboeb29472e2019-12-17 18:50:29 -07006080{
Pavel Begunkov9c8e11b2020-11-30 19:11:16 +00006081 struct io_timeout_rem *tr = &req->timeout_rem;
6082
Jens Axboeb29472e2019-12-17 18:50:29 -07006083 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
6084 return -EINVAL;
Daniele Albano61710e42020-07-18 14:15:16 -06006085 if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT)))
6086 return -EINVAL;
Pavel Begunkov26578cd2021-08-20 10:36:37 +01006087 if (sqe->ioprio || sqe->buf_index || sqe->len || sqe->splice_fd_in)
Jens Axboeb29472e2019-12-17 18:50:29 -07006088 return -EINVAL;
6089
Pavel Begunkovf1042b62021-08-28 19:54:39 -06006090 tr->ltimeout = false;
Pavel Begunkov9c8e11b2020-11-30 19:11:16 +00006091 tr->addr = READ_ONCE(sqe->addr);
6092 tr->flags = READ_ONCE(sqe->timeout_flags);
Pavel Begunkovf1042b62021-08-28 19:54:39 -06006093 if (tr->flags & IORING_TIMEOUT_UPDATE_MASK) {
6094 if (hweight32(tr->flags & IORING_TIMEOUT_CLOCK_MASK) > 1)
6095 return -EINVAL;
6096 if (tr->flags & IORING_LINK_TIMEOUT_UPDATE)
6097 tr->ltimeout = true;
6098 if (tr->flags & ~(IORING_TIMEOUT_UPDATE_MASK|IORING_TIMEOUT_ABS))
Pavel Begunkov9c8e11b2020-11-30 19:11:16 +00006099 return -EINVAL;
6100 if (get_timespec64(&tr->ts, u64_to_user_ptr(sqe->addr2)))
6101 return -EFAULT;
6102 } else if (tr->flags) {
6103 /* timeout removal doesn't support flags */
6104 return -EINVAL;
6105 }
6106
Jens Axboeb29472e2019-12-17 18:50:29 -07006107 return 0;
6108}
6109
Pavel Begunkov8662dae2021-01-19 13:32:44 +00006110static inline enum hrtimer_mode io_translate_timeout_mode(unsigned int flags)
6111{
6112 return (flags & IORING_TIMEOUT_ABS) ? HRTIMER_MODE_ABS
6113 : HRTIMER_MODE_REL;
6114}
6115
Jens Axboe11365042019-10-16 09:08:32 -06006116/*
6117 * Remove or update an existing timeout command
6118 */
Pavel Begunkov61e98202021-02-10 00:03:08 +00006119static int io_timeout_remove(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe11365042019-10-16 09:08:32 -06006120{
Pavel Begunkov9c8e11b2020-11-30 19:11:16 +00006121 struct io_timeout_rem *tr = &req->timeout_rem;
Jens Axboe11365042019-10-16 09:08:32 -06006122 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe47f46762019-11-09 17:43:02 -07006123 int ret;
Jens Axboe11365042019-10-16 09:08:32 -06006124
Pavel Begunkovec3c3d02021-08-18 10:50:52 +01006125 if (!(req->timeout_rem.flags & IORING_TIMEOUT_UPDATE)) {
6126 spin_lock(&ctx->completion_lock);
6127 spin_lock_irq(&ctx->timeout_lock);
Pavel Begunkov9c8e11b2020-11-30 19:11:16 +00006128 ret = io_timeout_cancel(ctx, tr->addr);
Pavel Begunkovec3c3d02021-08-18 10:50:52 +01006129 spin_unlock_irq(&ctx->timeout_lock);
6130 spin_unlock(&ctx->completion_lock);
6131 } else {
Pavel Begunkovf1042b62021-08-28 19:54:39 -06006132 enum hrtimer_mode mode = io_translate_timeout_mode(tr->flags);
6133
Pavel Begunkovec3c3d02021-08-18 10:50:52 +01006134 spin_lock_irq(&ctx->timeout_lock);
Pavel Begunkovf1042b62021-08-28 19:54:39 -06006135 if (tr->ltimeout)
6136 ret = io_linked_timeout_update(ctx, tr->addr, &tr->ts, mode);
6137 else
6138 ret = io_timeout_update(ctx, tr->addr, &tr->ts, mode);
Pavel Begunkovec3c3d02021-08-18 10:50:52 +01006139 spin_unlock_irq(&ctx->timeout_lock);
6140 }
Jens Axboe11365042019-10-16 09:08:32 -06006141
Jens Axboe4e88d6e2019-12-07 20:59:47 -07006142 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01006143 req_set_fail(req);
Pavel Begunkov505657b2021-08-17 20:28:09 +01006144 io_req_complete_post(req, ret, 0);
Jens Axboe11365042019-10-16 09:08:32 -06006145 return 0;
Jens Axboe5262f562019-09-17 12:26:57 -06006146}
6147
Jens Axboe3529d8c2019-12-19 18:24:38 -07006148static int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe,
Jens Axboe2d283902019-12-04 11:08:05 -07006149 bool is_timeout_link)
Jens Axboe5262f562019-09-17 12:26:57 -06006150{
Jens Axboead8a48a2019-11-15 08:49:11 -07006151 struct io_timeout_data *data;
Jens Axboea41525a2019-10-15 16:48:15 -06006152 unsigned flags;
Pavel Begunkov56080b02020-05-26 20:34:04 +03006153 u32 off = READ_ONCE(sqe->off);
Jens Axboe5262f562019-09-17 12:26:57 -06006154
Jens Axboead8a48a2019-11-15 08:49:11 -07006155 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
Jens Axboe5262f562019-09-17 12:26:57 -06006156 return -EINVAL;
Pavel Begunkov26578cd2021-08-20 10:36:37 +01006157 if (sqe->ioprio || sqe->buf_index || sqe->len != 1 ||
6158 sqe->splice_fd_in)
Jens Axboea41525a2019-10-15 16:48:15 -06006159 return -EINVAL;
Pavel Begunkov56080b02020-05-26 20:34:04 +03006160 if (off && is_timeout_link)
Jens Axboe2d283902019-12-04 11:08:05 -07006161 return -EINVAL;
Jens Axboea41525a2019-10-15 16:48:15 -06006162 flags = READ_ONCE(sqe->timeout_flags);
Jens Axboe50c1df22021-08-27 17:11:06 -06006163 if (flags & ~(IORING_TIMEOUT_ABS | IORING_TIMEOUT_CLOCK_MASK))
6164 return -EINVAL;
6165 /* more than one clock specified is invalid, obviously */
6166 if (hweight32(flags & IORING_TIMEOUT_CLOCK_MASK) > 1)
Jens Axboe5262f562019-09-17 12:26:57 -06006167 return -EINVAL;
Arnd Bergmannbdf20072019-10-01 09:53:29 -06006168
Pavel Begunkovef9dd632021-08-28 19:54:38 -06006169 INIT_LIST_HEAD(&req->timeout.list);
Pavel Begunkovbfe68a22020-05-30 14:54:18 +03006170 req->timeout.off = off;
Pavel Begunkovf18ee4c2021-06-14 23:37:25 +01006171 if (unlikely(off && !req->ctx->off_timeout_used))
6172 req->ctx->off_timeout_used = true;
Jens Axboe26a61672019-12-20 09:02:01 -07006173
Jens Axboee8c2bc12020-08-15 18:44:09 -07006174 if (!req->async_data && io_alloc_async_data(req))
Jens Axboe26a61672019-12-20 09:02:01 -07006175 return -ENOMEM;
6176
Jens Axboee8c2bc12020-08-15 18:44:09 -07006177 data = req->async_data;
Jens Axboead8a48a2019-11-15 08:49:11 -07006178 data->req = req;
Jens Axboe50c1df22021-08-27 17:11:06 -06006179 data->flags = flags;
Jens Axboead8a48a2019-11-15 08:49:11 -07006180
6181 if (get_timespec64(&data->ts, u64_to_user_ptr(sqe->addr)))
Jens Axboe5262f562019-09-17 12:26:57 -06006182 return -EFAULT;
6183
Jens Axboeba7261a2022-04-08 11:08:58 -06006184 INIT_LIST_HEAD(&req->timeout.list);
Pavel Begunkov8662dae2021-01-19 13:32:44 +00006185 data->mode = io_translate_timeout_mode(flags);
Jens Axboe50c1df22021-08-27 17:11:06 -06006186 hrtimer_init(&data->timer, io_timeout_get_clock(data), data->mode);
Pavel Begunkovb97e7362021-08-15 10:40:23 +01006187
6188 if (is_timeout_link) {
6189 struct io_submit_link *link = &req->ctx->submit_state.link;
6190
6191 if (!link->head)
6192 return -EINVAL;
6193 if (link->last->opcode == IORING_OP_LINK_TIMEOUT)
6194 return -EINVAL;
Pavel Begunkov4d13d1a2021-08-15 10:40:24 +01006195 req->timeout.head = link->last;
6196 link->last->flags |= REQ_F_ARM_LTIMEOUT;
Pavel Begunkovb97e7362021-08-15 10:40:23 +01006197 }
Jens Axboead8a48a2019-11-15 08:49:11 -07006198 return 0;
6199}
6200
Pavel Begunkov61e98202021-02-10 00:03:08 +00006201static int io_timeout(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboead8a48a2019-11-15 08:49:11 -07006202{
Jens Axboead8a48a2019-11-15 08:49:11 -07006203 struct io_ring_ctx *ctx = req->ctx;
Jens Axboee8c2bc12020-08-15 18:44:09 -07006204 struct io_timeout_data *data = req->async_data;
Jens Axboead8a48a2019-11-15 08:49:11 -07006205 struct list_head *entry;
Pavel Begunkovbfe68a22020-05-30 14:54:18 +03006206 u32 tail, off = req->timeout.off;
Jens Axboead8a48a2019-11-15 08:49:11 -07006207
Jens Axboe89850fc2021-08-10 15:11:51 -06006208 spin_lock_irq(&ctx->timeout_lock);
Jens Axboe93bd25b2019-11-11 23:34:31 -07006209
Jens Axboe5262f562019-09-17 12:26:57 -06006210 /*
6211 * sqe->off holds how many events that need to occur for this
Jens Axboe93bd25b2019-11-11 23:34:31 -07006212 * timeout event to be satisfied. If it isn't set, then this is
6213 * a pure timeout request, sequence isn't used.
Jens Axboe5262f562019-09-17 12:26:57 -06006214 */
Pavel Begunkov8eb7e2d2020-06-29 13:13:02 +03006215 if (io_is_timeout_noseq(req)) {
Jens Axboe93bd25b2019-11-11 23:34:31 -07006216 entry = ctx->timeout_list.prev;
6217 goto add;
6218 }
Jens Axboe5262f562019-09-17 12:26:57 -06006219
Pavel Begunkovbfe68a22020-05-30 14:54:18 +03006220 tail = ctx->cached_cq_tail - atomic_read(&ctx->cq_timeouts);
6221 req->timeout.target_seq = tail + off;
Jens Axboe5262f562019-09-17 12:26:57 -06006222
Marcelo Diop-Gonzalezf0105052021-01-15 11:54:40 -05006223 /* Update the last seq here in case io_flush_timeouts() hasn't.
6224 * This is safe because ->completion_lock is held, and submissions
6225 * and completions are never mixed in the same ->completion_lock section.
6226 */
6227 ctx->cq_last_tm_flush = tail;
6228
Jens Axboe5262f562019-09-17 12:26:57 -06006229 /*
6230 * Insertion sort, ensuring the first entry in the list is always
6231 * the one we need first.
6232 */
Jens Axboe5262f562019-09-17 12:26:57 -06006233 list_for_each_prev(entry, &ctx->timeout_list) {
Pavel Begunkov135fcde2020-07-13 23:37:12 +03006234 struct io_kiocb *nxt = list_entry(entry, struct io_kiocb,
6235 timeout.list);
Jens Axboe5262f562019-09-17 12:26:57 -06006236
Pavel Begunkov8eb7e2d2020-06-29 13:13:02 +03006237 if (io_is_timeout_noseq(nxt))
Jens Axboe93bd25b2019-11-11 23:34:31 -07006238 continue;
Pavel Begunkovbfe68a22020-05-30 14:54:18 +03006239 /* nxt.seq is behind @tail, otherwise would've been completed */
6240 if (off >= nxt->timeout.target_seq - tail)
Jens Axboe5262f562019-09-17 12:26:57 -06006241 break;
6242 }
Jens Axboe93bd25b2019-11-11 23:34:31 -07006243add:
Pavel Begunkov135fcde2020-07-13 23:37:12 +03006244 list_add(&req->timeout.list, entry);
Jens Axboead8a48a2019-11-15 08:49:11 -07006245 data->timer.function = io_timeout_fn;
6246 hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), data->mode);
Jens Axboe89850fc2021-08-10 15:11:51 -06006247 spin_unlock_irq(&ctx->timeout_lock);
Jens Axboe5262f562019-09-17 12:26:57 -06006248 return 0;
6249}
6250
Pavel Begunkovf458dd842021-03-08 12:14:14 +00006251struct io_cancel_data {
6252 struct io_ring_ctx *ctx;
6253 u64 user_data;
6254};
6255
Jens Axboe62755e32019-10-28 21:49:21 -06006256static bool io_cancel_cb(struct io_wq_work *work, void *data)
Jens Axboede0617e2019-04-06 21:51:27 -06006257{
Jens Axboe62755e32019-10-28 21:49:21 -06006258 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
Pavel Begunkovf458dd842021-03-08 12:14:14 +00006259 struct io_cancel_data *cd = data;
Jens Axboede0617e2019-04-06 21:51:27 -06006260
Pavel Begunkovf458dd842021-03-08 12:14:14 +00006261 return req->ctx == cd->ctx && req->user_data == cd->user_data;
Jens Axboe62755e32019-10-28 21:49:21 -06006262}
6263
Pavel Begunkovf458dd842021-03-08 12:14:14 +00006264static int io_async_cancel_one(struct io_uring_task *tctx, u64 user_data,
6265 struct io_ring_ctx *ctx)
Jens Axboe62755e32019-10-28 21:49:21 -06006266{
Pavel Begunkovf458dd842021-03-08 12:14:14 +00006267 struct io_cancel_data data = { .ctx = ctx, .user_data = user_data, };
Jens Axboe62755e32019-10-28 21:49:21 -06006268 enum io_wq_cancel cancel_ret;
Jens Axboe62755e32019-10-28 21:49:21 -06006269 int ret = 0;
6270
Pavel Begunkovf458dd842021-03-08 12:14:14 +00006271 if (!tctx || !tctx->io_wq)
Jens Axboe5aa75ed2021-02-16 12:56:50 -07006272 return -ENOENT;
6273
Pavel Begunkovf458dd842021-03-08 12:14:14 +00006274 cancel_ret = io_wq_cancel_cb(tctx->io_wq, io_cancel_cb, &data, false);
Jens Axboe62755e32019-10-28 21:49:21 -06006275 switch (cancel_ret) {
6276 case IO_WQ_CANCEL_OK:
6277 ret = 0;
6278 break;
6279 case IO_WQ_CANCEL_RUNNING:
6280 ret = -EALREADY;
6281 break;
6282 case IO_WQ_CANCEL_NOTFOUND:
6283 ret = -ENOENT;
6284 break;
6285 }
6286
Jens Axboee977d6d2019-11-05 12:39:45 -07006287 return ret;
6288}
6289
Pavel Begunkov8cb01fa2021-08-15 10:40:22 +01006290static int io_try_cancel_userdata(struct io_kiocb *req, u64 sqe_addr)
Jens Axboe47f46762019-11-09 17:43:02 -07006291{
Pavel Begunkov8cb01fa2021-08-15 10:40:22 +01006292 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe47f46762019-11-09 17:43:02 -07006293 int ret;
6294
Pavel Begunkovdadebc32021-08-23 13:30:44 +01006295 WARN_ON_ONCE(!io_wq_current_is_worker() && req->task != current);
Pavel Begunkov8cb01fa2021-08-15 10:40:22 +01006296
Pavel Begunkovf458dd842021-03-08 12:14:14 +00006297 ret = io_async_cancel_one(req->task->io_uring, sqe_addr, ctx);
Pavel Begunkovdf9727a2021-04-01 15:43:59 +01006298 if (ret != -ENOENT)
Pavel Begunkov8cb01fa2021-08-15 10:40:22 +01006299 return ret;
Pavel Begunkov505657b2021-08-17 20:28:09 +01006300
6301 spin_lock(&ctx->completion_lock);
Jens Axboe79ebeae2021-08-10 15:18:27 -06006302 spin_lock_irq(&ctx->timeout_lock);
Jens Axboe47f46762019-11-09 17:43:02 -07006303 ret = io_timeout_cancel(ctx, sqe_addr);
Jens Axboe79ebeae2021-08-10 15:18:27 -06006304 spin_unlock_irq(&ctx->timeout_lock);
Jens Axboe47f46762019-11-09 17:43:02 -07006305 if (ret != -ENOENT)
Pavel Begunkov505657b2021-08-17 20:28:09 +01006306 goto out;
6307 ret = io_poll_cancel(ctx, sqe_addr, false);
6308out:
6309 spin_unlock(&ctx->completion_lock);
6310 return ret;
Jens Axboe47f46762019-11-09 17:43:02 -07006311}
6312
Jens Axboe3529d8c2019-12-19 18:24:38 -07006313static int io_async_cancel_prep(struct io_kiocb *req,
6314 const struct io_uring_sqe *sqe)
Jens Axboee977d6d2019-11-05 12:39:45 -07006315{
Jens Axboefbf23842019-12-17 18:45:56 -07006316 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
Jens Axboee977d6d2019-11-05 12:39:45 -07006317 return -EINVAL;
Daniele Albano61710e42020-07-18 14:15:16 -06006318 if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT)))
6319 return -EINVAL;
Pavel Begunkov26578cd2021-08-20 10:36:37 +01006320 if (sqe->ioprio || sqe->off || sqe->len || sqe->cancel_flags ||
6321 sqe->splice_fd_in)
Jens Axboee977d6d2019-11-05 12:39:45 -07006322 return -EINVAL;
6323
Jens Axboefbf23842019-12-17 18:45:56 -07006324 req->cancel.addr = READ_ONCE(sqe->addr);
6325 return 0;
6326}
6327
Pavel Begunkov61e98202021-02-10 00:03:08 +00006328static int io_async_cancel(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboefbf23842019-12-17 18:45:56 -07006329{
6330 struct io_ring_ctx *ctx = req->ctx;
Pavel Begunkov58f99372021-03-12 16:25:55 +00006331 u64 sqe_addr = req->cancel.addr;
6332 struct io_tctx_node *node;
6333 int ret;
Jens Axboefbf23842019-12-17 18:45:56 -07006334
Pavel Begunkov8cb01fa2021-08-15 10:40:22 +01006335 ret = io_try_cancel_userdata(req, sqe_addr);
Pavel Begunkov58f99372021-03-12 16:25:55 +00006336 if (ret != -ENOENT)
6337 goto done;
Pavel Begunkov58f99372021-03-12 16:25:55 +00006338
6339 /* slow path, try all io-wq's */
6340 io_ring_submit_lock(ctx, !(issue_flags & IO_URING_F_NONBLOCK));
6341 ret = -ENOENT;
6342 list_for_each_entry(node, &ctx->tctx_list, ctx_node) {
6343 struct io_uring_task *tctx = node->task->io_uring;
6344
Pavel Begunkov58f99372021-03-12 16:25:55 +00006345 ret = io_async_cancel_one(tctx, req->cancel.addr, ctx);
6346 if (ret != -ENOENT)
6347 break;
6348 }
6349 io_ring_submit_unlock(ctx, !(issue_flags & IO_URING_F_NONBLOCK));
Pavel Begunkov58f99372021-03-12 16:25:55 +00006350done:
Pavel Begunkov58f99372021-03-12 16:25:55 +00006351 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01006352 req_set_fail(req);
Pavel Begunkov505657b2021-08-17 20:28:09 +01006353 io_req_complete_post(req, ret, 0);
Jens Axboe62755e32019-10-28 21:49:21 -06006354 return 0;
6355}
6356
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00006357static int io_rsrc_update_prep(struct io_kiocb *req,
Jens Axboe05f3fb32019-12-09 11:22:50 -07006358 const struct io_uring_sqe *sqe)
6359{
Daniele Albano61710e42020-07-18 14:15:16 -06006360 if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT)))
6361 return -EINVAL;
Pavel Begunkov26578cd2021-08-20 10:36:37 +01006362 if (sqe->ioprio || sqe->rw_flags || sqe->splice_fd_in)
Jens Axboe05f3fb32019-12-09 11:22:50 -07006363 return -EINVAL;
6364
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00006365 req->rsrc_update.offset = READ_ONCE(sqe->off);
6366 req->rsrc_update.nr_args = READ_ONCE(sqe->len);
6367 if (!req->rsrc_update.nr_args)
Jens Axboe05f3fb32019-12-09 11:22:50 -07006368 return -EINVAL;
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00006369 req->rsrc_update.arg = READ_ONCE(sqe->addr);
Jens Axboe05f3fb32019-12-09 11:22:50 -07006370 return 0;
6371}
6372
Pavel Begunkov889fca72021-02-10 00:03:09 +00006373static int io_files_update(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe05f3fb32019-12-09 11:22:50 -07006374{
6375 struct io_ring_ctx *ctx = req->ctx;
Pavel Begunkovc3bdad02021-04-25 14:32:22 +01006376 struct io_uring_rsrc_update2 up;
Jens Axboe05f3fb32019-12-09 11:22:50 -07006377 int ret;
6378
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00006379 up.offset = req->rsrc_update.offset;
6380 up.data = req->rsrc_update.arg;
Pavel Begunkovc3bdad02021-04-25 14:32:22 +01006381 up.nr = 0;
6382 up.tags = 0;
Colin Ian King615cee42021-04-26 10:47:35 +01006383 up.resv = 0;
Dylan Yudaken7a7c9f92022-04-12 09:30:40 -07006384 up.resv2 = 0;
Jens Axboe05f3fb32019-12-09 11:22:50 -07006385
Jens Axboecdb31c22021-09-24 08:43:54 -06006386 io_ring_submit_lock(ctx, !(issue_flags & IO_URING_F_NONBLOCK));
Pavel Begunkovfdecb662021-04-25 14:32:20 +01006387 ret = __io_register_rsrc_update(ctx, IORING_RSRC_FILE,
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +01006388 &up, req->rsrc_update.nr_args);
Jens Axboecdb31c22021-09-24 08:43:54 -06006389 io_ring_submit_unlock(ctx, !(issue_flags & IO_URING_F_NONBLOCK));
Jens Axboe05f3fb32019-12-09 11:22:50 -07006390
6391 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01006392 req_set_fail(req);
Pavel Begunkov889fca72021-02-10 00:03:09 +00006393 __io_req_complete(req, issue_flags, ret, 0);
Jens Axboe05f3fb32019-12-09 11:22:50 -07006394 return 0;
6395}
6396
Pavel Begunkovbfe76552020-09-30 22:57:55 +03006397static int io_req_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboef67676d2019-12-02 11:03:47 -07006398{
Jens Axboed625c6e2019-12-17 19:53:05 -07006399 switch (req->opcode) {
Jens Axboee7815732019-12-17 19:45:06 -07006400 case IORING_OP_NOP:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03006401 return 0;
Jens Axboef67676d2019-12-02 11:03:47 -07006402 case IORING_OP_READV:
6403 case IORING_OP_READ_FIXED:
Jens Axboe3a6820f2019-12-22 15:19:35 -07006404 case IORING_OP_READ:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03006405 return io_read_prep(req, sqe);
Jens Axboef67676d2019-12-02 11:03:47 -07006406 case IORING_OP_WRITEV:
6407 case IORING_OP_WRITE_FIXED:
Jens Axboe3a6820f2019-12-22 15:19:35 -07006408 case IORING_OP_WRITE:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03006409 return io_write_prep(req, sqe);
Jens Axboe0969e782019-12-17 18:40:57 -07006410 case IORING_OP_POLL_ADD:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03006411 return io_poll_add_prep(req, sqe);
Jens Axboe0969e782019-12-17 18:40:57 -07006412 case IORING_OP_POLL_REMOVE:
Pavel Begunkovc5de0032021-04-14 13:38:37 +01006413 return io_poll_update_prep(req, sqe);
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07006414 case IORING_OP_FSYNC:
Pavel Begunkov1155c762021-02-18 18:29:38 +00006415 return io_fsync_prep(req, sqe);
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07006416 case IORING_OP_SYNC_FILE_RANGE:
Pavel Begunkov1155c762021-02-18 18:29:38 +00006417 return io_sfr_prep(req, sqe);
Jens Axboe03b12302019-12-02 18:50:25 -07006418 case IORING_OP_SENDMSG:
Jens Axboefddafac2020-01-04 20:19:44 -07006419 case IORING_OP_SEND:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03006420 return io_sendmsg_prep(req, sqe);
Jens Axboe03b12302019-12-02 18:50:25 -07006421 case IORING_OP_RECVMSG:
Jens Axboefddafac2020-01-04 20:19:44 -07006422 case IORING_OP_RECV:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03006423 return io_recvmsg_prep(req, sqe);
Jens Axboef499a022019-12-02 16:28:46 -07006424 case IORING_OP_CONNECT:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03006425 return io_connect_prep(req, sqe);
Jens Axboe2d283902019-12-04 11:08:05 -07006426 case IORING_OP_TIMEOUT:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03006427 return io_timeout_prep(req, sqe, false);
Jens Axboeb29472e2019-12-17 18:50:29 -07006428 case IORING_OP_TIMEOUT_REMOVE:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03006429 return io_timeout_remove_prep(req, sqe);
Jens Axboefbf23842019-12-17 18:45:56 -07006430 case IORING_OP_ASYNC_CANCEL:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03006431 return io_async_cancel_prep(req, sqe);
Jens Axboe2d283902019-12-04 11:08:05 -07006432 case IORING_OP_LINK_TIMEOUT:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03006433 return io_timeout_prep(req, sqe, true);
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07006434 case IORING_OP_ACCEPT:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03006435 return io_accept_prep(req, sqe);
Jens Axboed63d1b52019-12-10 10:38:56 -07006436 case IORING_OP_FALLOCATE:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03006437 return io_fallocate_prep(req, sqe);
Jens Axboe15b71ab2019-12-11 11:20:36 -07006438 case IORING_OP_OPENAT:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03006439 return io_openat_prep(req, sqe);
Jens Axboeb5dba592019-12-11 14:02:38 -07006440 case IORING_OP_CLOSE:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03006441 return io_close_prep(req, sqe);
Jens Axboe05f3fb32019-12-09 11:22:50 -07006442 case IORING_OP_FILES_UPDATE:
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00006443 return io_rsrc_update_prep(req, sqe);
Jens Axboeeddc7ef2019-12-13 21:18:10 -07006444 case IORING_OP_STATX:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03006445 return io_statx_prep(req, sqe);
Jens Axboe4840e412019-12-25 22:03:45 -07006446 case IORING_OP_FADVISE:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03006447 return io_fadvise_prep(req, sqe);
Jens Axboec1ca7572019-12-25 22:18:28 -07006448 case IORING_OP_MADVISE:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03006449 return io_madvise_prep(req, sqe);
Jens Axboecebdb982020-01-08 17:59:24 -07006450 case IORING_OP_OPENAT2:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03006451 return io_openat2_prep(req, sqe);
Jens Axboe3e4827b2020-01-08 15:18:09 -07006452 case IORING_OP_EPOLL_CTL:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03006453 return io_epoll_ctl_prep(req, sqe);
Pavel Begunkov7d67af22020-02-24 11:32:45 +03006454 case IORING_OP_SPLICE:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03006455 return io_splice_prep(req, sqe);
Jens Axboeddf0322d2020-02-23 16:41:33 -07006456 case IORING_OP_PROVIDE_BUFFERS:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03006457 return io_provide_buffers_prep(req, sqe);
Jens Axboe067524e2020-03-02 16:32:28 -07006458 case IORING_OP_REMOVE_BUFFERS:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03006459 return io_remove_buffers_prep(req, sqe);
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03006460 case IORING_OP_TEE:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03006461 return io_tee_prep(req, sqe);
Jens Axboe36f4fa62020-09-05 11:14:22 -06006462 case IORING_OP_SHUTDOWN:
6463 return io_shutdown_prep(req, sqe);
Jens Axboe80a261f2020-09-28 14:23:58 -06006464 case IORING_OP_RENAMEAT:
6465 return io_renameat_prep(req, sqe);
Jens Axboe14a11432020-09-28 14:27:37 -06006466 case IORING_OP_UNLINKAT:
6467 return io_unlinkat_prep(req, sqe);
Dmitry Kadasheve34a02d2021-07-08 13:34:45 +07006468 case IORING_OP_MKDIRAT:
6469 return io_mkdirat_prep(req, sqe);
Dmitry Kadashev7a8721f2021-07-08 13:34:46 +07006470 case IORING_OP_SYMLINKAT:
6471 return io_symlinkat_prep(req, sqe);
Dmitry Kadashevcf30da92021-07-08 13:34:47 +07006472 case IORING_OP_LINKAT:
6473 return io_linkat_prep(req, sqe);
Jens Axboef67676d2019-12-02 11:03:47 -07006474 }
6475
Pavel Begunkovbfe76552020-09-30 22:57:55 +03006476 printk_once(KERN_WARNING "io_uring: unhandled opcode %d\n",
6477 req->opcode);
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01006478 return -EINVAL;
Pavel Begunkovbfe76552020-09-30 22:57:55 +03006479}
6480
Pavel Begunkov93642ef2021-02-18 18:29:44 +00006481static int io_req_prep_async(struct io_kiocb *req)
Jens Axboedef596e2019-01-09 08:59:42 -07006482{
Pavel Begunkovb7e298d2021-02-28 22:35:19 +00006483 if (!io_op_defs[req->opcode].needs_async_setup)
6484 return 0;
6485 if (WARN_ON_ONCE(req->async_data))
6486 return -EFAULT;
6487 if (io_alloc_async_data(req))
6488 return -EAGAIN;
6489
Pavel Begunkov93642ef2021-02-18 18:29:44 +00006490 switch (req->opcode) {
6491 case IORING_OP_READV:
Pavel Begunkov93642ef2021-02-18 18:29:44 +00006492 return io_rw_prep_async(req, READ);
6493 case IORING_OP_WRITEV:
Pavel Begunkov93642ef2021-02-18 18:29:44 +00006494 return io_rw_prep_async(req, WRITE);
6495 case IORING_OP_SENDMSG:
Pavel Begunkov93642ef2021-02-18 18:29:44 +00006496 return io_sendmsg_prep_async(req);
6497 case IORING_OP_RECVMSG:
Pavel Begunkov93642ef2021-02-18 18:29:44 +00006498 return io_recvmsg_prep_async(req);
6499 case IORING_OP_CONNECT:
6500 return io_connect_prep_async(req);
6501 }
Pavel Begunkovb7e298d2021-02-28 22:35:19 +00006502 printk_once(KERN_WARNING "io_uring: prep_async() bad opcode %d\n",
6503 req->opcode);
6504 return -EFAULT;
Jens Axboedef596e2019-01-09 08:59:42 -07006505}
6506
Pavel Begunkov9cf7c102020-07-13 23:37:15 +03006507static u32 io_get_sequence(struct io_kiocb *req)
6508{
Pavel Begunkova3dbdf52021-06-17 18:14:05 +01006509 u32 seq = req->ctx->cached_sq_head;
Pavel Begunkov9cf7c102020-07-13 23:37:15 +03006510
Pavel Begunkova3dbdf52021-06-17 18:14:05 +01006511 /* need original cached_sq_head, but it was increased for each req */
6512 io_for_each_link(req, req)
6513 seq--;
6514 return seq;
Pavel Begunkov9cf7c102020-07-13 23:37:15 +03006515}
6516
Pavel Begunkov76cc33d2021-06-14 23:37:30 +01006517static bool io_drain_req(struct io_kiocb *req)
Jens Axboedef596e2019-01-09 08:59:42 -07006518{
Pavel Begunkov3c199662021-06-15 16:47:57 +01006519 struct io_kiocb *pos;
Jens Axboedef596e2019-01-09 08:59:42 -07006520 struct io_ring_ctx *ctx = req->ctx;
Pavel Begunkov27dc8332020-07-13 23:37:14 +03006521 struct io_defer_entry *de;
Jens Axboedef596e2019-01-09 08:59:42 -07006522 int ret;
Pavel Begunkov9cf7c102020-07-13 23:37:15 +03006523 u32 seq;
Jens Axboedef596e2019-01-09 08:59:42 -07006524
Pavel Begunkovb8ce1b92021-08-31 14:13:11 +01006525 if (req->flags & REQ_F_FAIL) {
6526 io_req_complete_fail_submit(req);
6527 return true;
6528 }
6529
Pavel Begunkov3c199662021-06-15 16:47:57 +01006530 /*
6531 * If we need to drain a request in the middle of a link, drain the
6532 * head request and the next request/link after the current link.
6533 * Considering sequential execution of links, IOSQE_IO_DRAIN will be
6534 * maintained for every request of our link.
6535 */
6536 if (ctx->drain_next) {
6537 req->flags |= REQ_F_IO_DRAIN;
6538 ctx->drain_next = false;
6539 }
6540 /* not interested in head, start from the first linked */
6541 io_for_each_link(pos, req->link) {
6542 if (pos->flags & REQ_F_IO_DRAIN) {
6543 ctx->drain_next = true;
6544 req->flags |= REQ_F_IO_DRAIN;
6545 break;
6546 }
6547 }
6548
Jens Axboedef596e2019-01-09 08:59:42 -07006549 /* Still need defer if there is pending req in defer list. */
Hao Xu1bd12b72021-11-25 17:21:02 +08006550 spin_lock(&ctx->completion_lock);
Pavel Begunkov9cf7c102020-07-13 23:37:15 +03006551 if (likely(list_empty_careful(&ctx->defer_list) &&
Pavel Begunkov10c66902021-06-15 16:47:56 +01006552 !(req->flags & REQ_F_IO_DRAIN))) {
Hao Xu1bd12b72021-11-25 17:21:02 +08006553 spin_unlock(&ctx->completion_lock);
Pavel Begunkov10c66902021-06-15 16:47:56 +01006554 ctx->drain_active = false;
Pavel Begunkov76cc33d2021-06-14 23:37:30 +01006555 return false;
Pavel Begunkov10c66902021-06-15 16:47:56 +01006556 }
Hao Xu1bd12b72021-11-25 17:21:02 +08006557 spin_unlock(&ctx->completion_lock);
Pavel Begunkov9cf7c102020-07-13 23:37:15 +03006558
6559 seq = io_get_sequence(req);
6560 /* Still a chance to pass the sequence check */
6561 if (!req_need_defer(req, seq) && list_empty_careful(&ctx->defer_list))
Pavel Begunkov76cc33d2021-06-14 23:37:30 +01006562 return false;
Jens Axboedef596e2019-01-09 08:59:42 -07006563
Pavel Begunkovb7e298d2021-02-28 22:35:19 +00006564 ret = io_req_prep_async(req);
Pavel Begunkovbe7053b2021-02-18 18:29:45 +00006565 if (ret)
Pavel Begunkov1b487732021-07-11 22:41:13 +01006566 goto fail;
Pavel Begunkovcbdcb432020-06-29 19:18:43 +03006567 io_prep_async_link(req);
Pavel Begunkov27dc8332020-07-13 23:37:14 +03006568 de = kmalloc(sizeof(*de), GFP_KERNEL);
Pavel Begunkov76cc33d2021-06-14 23:37:30 +01006569 if (!de) {
Pavel Begunkov1b487732021-07-11 22:41:13 +01006570 ret = -ENOMEM;
6571fail:
6572 io_req_complete_failed(req, ret);
Pavel Begunkov76cc33d2021-06-14 23:37:30 +01006573 return true;
6574 }
Jens Axboe31b51512019-01-18 22:56:34 -07006575
Jens Axboe79ebeae2021-08-10 15:18:27 -06006576 spin_lock(&ctx->completion_lock);
Pavel Begunkov9cf7c102020-07-13 23:37:15 +03006577 if (!req_need_defer(req, seq) && list_empty(&ctx->defer_list)) {
Jens Axboe79ebeae2021-08-10 15:18:27 -06006578 spin_unlock(&ctx->completion_lock);
Pavel Begunkov27dc8332020-07-13 23:37:14 +03006579 kfree(de);
Pavel Begunkovf237c302021-08-18 12:42:46 +01006580 io_queue_async_work(req, NULL);
Pavel Begunkov76cc33d2021-06-14 23:37:30 +01006581 return true;
Jens Axboe31b51512019-01-18 22:56:34 -07006582 }
6583
6584 trace_io_uring_defer(ctx, req, req->user_data);
Pavel Begunkov27dc8332020-07-13 23:37:14 +03006585 de->req = req;
Pavel Begunkov9cf7c102020-07-13 23:37:15 +03006586 de->seq = seq;
Pavel Begunkov27dc8332020-07-13 23:37:14 +03006587 list_add_tail(&de->list, &ctx->defer_list);
Jens Axboe79ebeae2021-08-10 15:18:27 -06006588 spin_unlock(&ctx->completion_lock);
Pavel Begunkov76cc33d2021-06-14 23:37:30 +01006589 return true;
Jens Axboe31b51512019-01-18 22:56:34 -07006590}
6591
Pavel Begunkov68fb8972021-03-19 17:22:41 +00006592static void io_clean_op(struct io_kiocb *req)
Pavel Begunkov99bc4c32020-02-07 22:04:45 +03006593{
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03006594 if (req->flags & REQ_F_BUFFER_SELECTED) {
6595 switch (req->opcode) {
6596 case IORING_OP_READV:
6597 case IORING_OP_READ_FIXED:
6598 case IORING_OP_READ:
Jens Axboebcda7ba2020-02-23 16:42:51 -07006599 kfree((void *)(unsigned long)req->rw.addr);
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03006600 break;
6601 case IORING_OP_RECVMSG:
6602 case IORING_OP_RECV:
Jens Axboe52de1fe2020-02-27 10:15:42 -07006603 kfree(req->sr_msg.kbuf);
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03006604 break;
6605 }
Pavel Begunkov99bc4c32020-02-07 22:04:45 +03006606 }
6607
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03006608 if (req->flags & REQ_F_NEED_CLEANUP) {
6609 switch (req->opcode) {
6610 case IORING_OP_READV:
6611 case IORING_OP_READ_FIXED:
6612 case IORING_OP_READ:
6613 case IORING_OP_WRITEV:
6614 case IORING_OP_WRITE_FIXED:
Jens Axboee8c2bc12020-08-15 18:44:09 -07006615 case IORING_OP_WRITE: {
6616 struct io_async_rw *io = req->async_data;
Pavel Begunkov1dacb4d2021-06-17 18:14:03 +01006617
6618 kfree(io->free_iovec);
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03006619 break;
Jens Axboee8c2bc12020-08-15 18:44:09 -07006620 }
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03006621 case IORING_OP_RECVMSG:
Jens Axboee8c2bc12020-08-15 18:44:09 -07006622 case IORING_OP_SENDMSG: {
6623 struct io_async_msghdr *io = req->async_data;
Pavel Begunkov257e84a2021-02-05 00:58:00 +00006624
6625 kfree(io->free_iov);
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03006626 break;
Jens Axboee8c2bc12020-08-15 18:44:09 -07006627 }
Jens Axboef3cd48502020-09-24 14:55:54 -06006628 case IORING_OP_OPENAT:
6629 case IORING_OP_OPENAT2:
6630 if (req->open.filename)
6631 putname(req->open.filename);
6632 break;
Jens Axboe80a261f2020-09-28 14:23:58 -06006633 case IORING_OP_RENAMEAT:
6634 putname(req->rename.oldpath);
6635 putname(req->rename.newpath);
6636 break;
Jens Axboe14a11432020-09-28 14:27:37 -06006637 case IORING_OP_UNLINKAT:
6638 putname(req->unlink.filename);
6639 break;
Dmitry Kadasheve34a02d2021-07-08 13:34:45 +07006640 case IORING_OP_MKDIRAT:
6641 putname(req->mkdir.filename);
6642 break;
Dmitry Kadashev7a8721f2021-07-08 13:34:46 +07006643 case IORING_OP_SYMLINKAT:
6644 putname(req->symlink.oldpath);
6645 putname(req->symlink.newpath);
6646 break;
Dmitry Kadashevcf30da92021-07-08 13:34:47 +07006647 case IORING_OP_LINKAT:
6648 putname(req->hardlink.oldpath);
6649 putname(req->hardlink.newpath);
6650 break;
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03006651 }
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03006652 }
Jens Axboe75652a302021-04-15 09:52:40 -06006653 if ((req->flags & REQ_F_POLLED) && req->apoll) {
6654 kfree(req->apoll->double_poll);
6655 kfree(req->apoll);
6656 req->apoll = NULL;
6657 }
Pavel Begunkov3a0a6902021-04-20 12:03:31 +01006658 if (req->flags & REQ_F_INFLIGHT) {
6659 struct io_uring_task *tctx = req->task->io_uring;
6660
6661 atomic_dec(&tctx->inflight_tracked);
Pavel Begunkov3a0a6902021-04-20 12:03:31 +01006662 }
Pavel Begunkovc8543572021-06-17 18:14:04 +01006663 if (req->flags & REQ_F_CREDS)
Pavel Begunkovb8e64b52021-06-17 18:14:02 +01006664 put_cred(req->creds);
Pavel Begunkovc8543572021-06-17 18:14:04 +01006665
6666 req->flags &= ~IO_REQ_CLEAN_FLAGS;
Pavel Begunkov99bc4c32020-02-07 22:04:45 +03006667}
6668
Pavel Begunkov889fca72021-02-10 00:03:09 +00006669static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboeedafcce2019-01-09 09:16:05 -07006670{
Jens Axboeedafcce2019-01-09 09:16:05 -07006671 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe5730b272021-02-27 15:57:30 -07006672 const struct cred *creds = NULL;
Jens Axboed625c6e2019-12-17 19:53:05 -07006673 int ret;
Jens Axboeedafcce2019-01-09 09:16:05 -07006674
Pavel Begunkovb8e64b52021-06-17 18:14:02 +01006675 if ((req->flags & REQ_F_CREDS) && req->creds != current_cred())
Pavel Begunkovc10d1f92021-06-17 18:14:01 +01006676 creds = override_creds(req->creds);
Jens Axboe5730b272021-02-27 15:57:30 -07006677
Jens Axboed625c6e2019-12-17 19:53:05 -07006678 switch (req->opcode) {
Jens Axboe2b188cc2019-01-07 10:46:33 -07006679 case IORING_OP_NOP:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006680 ret = io_nop(req, issue_flags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006681 break;
Jens Axboe2b188cc2019-01-07 10:46:33 -07006682 case IORING_OP_READV:
Jens Axboe3529d8c2019-12-19 18:24:38 -07006683 case IORING_OP_READ_FIXED:
Jens Axboe3a6820f2019-12-22 15:19:35 -07006684 case IORING_OP_READ:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006685 ret = io_read(req, issue_flags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006686 break;
6687 case IORING_OP_WRITEV:
Jens Axboe2b188cc2019-01-07 10:46:33 -07006688 case IORING_OP_WRITE_FIXED:
Jens Axboe3a6820f2019-12-22 15:19:35 -07006689 case IORING_OP_WRITE:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006690 ret = io_write(req, issue_flags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006691 break;
6692 case IORING_OP_FSYNC:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006693 ret = io_fsync(req, issue_flags);
Jackie Liuba5290c2019-10-09 09:19:59 +08006694 break;
Jens Axboe2b188cc2019-01-07 10:46:33 -07006695 case IORING_OP_POLL_ADD:
Pavel Begunkov61e98202021-02-10 00:03:08 +00006696 ret = io_poll_add(req, issue_flags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006697 break;
6698 case IORING_OP_POLL_REMOVE:
Pavel Begunkovc5de0032021-04-14 13:38:37 +01006699 ret = io_poll_update(req, issue_flags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006700 break;
Jens Axboeb76da702019-11-20 13:05:32 -07006701 case IORING_OP_SYNC_FILE_RANGE:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006702 ret = io_sync_file_range(req, issue_flags);
Jens Axboeb76da702019-11-20 13:05:32 -07006703 break;
6704 case IORING_OP_SENDMSG:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006705 ret = io_sendmsg(req, issue_flags);
Pavel Begunkov062d04d2020-10-10 18:34:12 +01006706 break;
Jens Axboefddafac2020-01-04 20:19:44 -07006707 case IORING_OP_SEND:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006708 ret = io_send(req, issue_flags);
Jens Axboeb76da702019-11-20 13:05:32 -07006709 break;
6710 case IORING_OP_RECVMSG:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006711 ret = io_recvmsg(req, issue_flags);
Pavel Begunkov062d04d2020-10-10 18:34:12 +01006712 break;
Jens Axboefddafac2020-01-04 20:19:44 -07006713 case IORING_OP_RECV:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006714 ret = io_recv(req, issue_flags);
Jens Axboeb76da702019-11-20 13:05:32 -07006715 break;
Jens Axboe561fb042019-10-24 07:25:42 -06006716 case IORING_OP_TIMEOUT:
Pavel Begunkov61e98202021-02-10 00:03:08 +00006717 ret = io_timeout(req, issue_flags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006718 break;
6719 case IORING_OP_TIMEOUT_REMOVE:
Pavel Begunkov61e98202021-02-10 00:03:08 +00006720 ret = io_timeout_remove(req, issue_flags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006721 break;
6722 case IORING_OP_ACCEPT:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006723 ret = io_accept(req, issue_flags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006724 break;
6725 case IORING_OP_CONNECT:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006726 ret = io_connect(req, issue_flags);
Jens Axboe31b51512019-01-18 22:56:34 -07006727 break;
6728 case IORING_OP_ASYNC_CANCEL:
Pavel Begunkov61e98202021-02-10 00:03:08 +00006729 ret = io_async_cancel(req, issue_flags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006730 break;
Jens Axboed63d1b52019-12-10 10:38:56 -07006731 case IORING_OP_FALLOCATE:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006732 ret = io_fallocate(req, issue_flags);
Jens Axboed63d1b52019-12-10 10:38:56 -07006733 break;
Jens Axboe15b71ab2019-12-11 11:20:36 -07006734 case IORING_OP_OPENAT:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006735 ret = io_openat(req, issue_flags);
Jens Axboe15b71ab2019-12-11 11:20:36 -07006736 break;
Jens Axboeb5dba592019-12-11 14:02:38 -07006737 case IORING_OP_CLOSE:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006738 ret = io_close(req, issue_flags);
Jens Axboeb5dba592019-12-11 14:02:38 -07006739 break;
Jens Axboe05f3fb32019-12-09 11:22:50 -07006740 case IORING_OP_FILES_UPDATE:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006741 ret = io_files_update(req, issue_flags);
Jens Axboe05f3fb32019-12-09 11:22:50 -07006742 break;
Jens Axboeeddc7ef2019-12-13 21:18:10 -07006743 case IORING_OP_STATX:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006744 ret = io_statx(req, issue_flags);
Jens Axboeeddc7ef2019-12-13 21:18:10 -07006745 break;
Jens Axboe4840e412019-12-25 22:03:45 -07006746 case IORING_OP_FADVISE:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006747 ret = io_fadvise(req, issue_flags);
Jens Axboe4840e412019-12-25 22:03:45 -07006748 break;
Jens Axboec1ca7572019-12-25 22:18:28 -07006749 case IORING_OP_MADVISE:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006750 ret = io_madvise(req, issue_flags);
Jens Axboec1ca7572019-12-25 22:18:28 -07006751 break;
Jens Axboecebdb982020-01-08 17:59:24 -07006752 case IORING_OP_OPENAT2:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006753 ret = io_openat2(req, issue_flags);
Jens Axboecebdb982020-01-08 17:59:24 -07006754 break;
Jens Axboe3e4827b2020-01-08 15:18:09 -07006755 case IORING_OP_EPOLL_CTL:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006756 ret = io_epoll_ctl(req, issue_flags);
Jens Axboe3e4827b2020-01-08 15:18:09 -07006757 break;
Pavel Begunkov7d67af22020-02-24 11:32:45 +03006758 case IORING_OP_SPLICE:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006759 ret = io_splice(req, issue_flags);
Pavel Begunkov7d67af22020-02-24 11:32:45 +03006760 break;
Jens Axboeddf0322d2020-02-23 16:41:33 -07006761 case IORING_OP_PROVIDE_BUFFERS:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006762 ret = io_provide_buffers(req, issue_flags);
Jens Axboeddf0322d2020-02-23 16:41:33 -07006763 break;
Jens Axboe067524e2020-03-02 16:32:28 -07006764 case IORING_OP_REMOVE_BUFFERS:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006765 ret = io_remove_buffers(req, issue_flags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006766 break;
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03006767 case IORING_OP_TEE:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006768 ret = io_tee(req, issue_flags);
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03006769 break;
Jens Axboe36f4fa62020-09-05 11:14:22 -06006770 case IORING_OP_SHUTDOWN:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006771 ret = io_shutdown(req, issue_flags);
Jens Axboe36f4fa62020-09-05 11:14:22 -06006772 break;
Jens Axboe80a261f2020-09-28 14:23:58 -06006773 case IORING_OP_RENAMEAT:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006774 ret = io_renameat(req, issue_flags);
Jens Axboe80a261f2020-09-28 14:23:58 -06006775 break;
Jens Axboe14a11432020-09-28 14:27:37 -06006776 case IORING_OP_UNLINKAT:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006777 ret = io_unlinkat(req, issue_flags);
Jens Axboe14a11432020-09-28 14:27:37 -06006778 break;
Dmitry Kadasheve34a02d2021-07-08 13:34:45 +07006779 case IORING_OP_MKDIRAT:
6780 ret = io_mkdirat(req, issue_flags);
6781 break;
Dmitry Kadashev7a8721f2021-07-08 13:34:46 +07006782 case IORING_OP_SYMLINKAT:
6783 ret = io_symlinkat(req, issue_flags);
6784 break;
Dmitry Kadashevcf30da92021-07-08 13:34:47 +07006785 case IORING_OP_LINKAT:
6786 ret = io_linkat(req, issue_flags);
6787 break;
Jens Axboe2b188cc2019-01-07 10:46:33 -07006788 default:
6789 ret = -EINVAL;
6790 break;
6791 }
Jens Axboe31b51512019-01-18 22:56:34 -07006792
Jens Axboe5730b272021-02-27 15:57:30 -07006793 if (creds)
6794 revert_creds(creds);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006795 if (ret)
6796 return ret;
Jens Axboeb5325762020-05-19 21:20:27 -06006797 /* If the op doesn't have a file, we're not polling for it */
Pavel Begunkovcb3d8972021-06-14 02:36:14 +01006798 if ((ctx->flags & IORING_SETUP_IOPOLL) && req->file)
6799 io_iopoll_req_issued(req);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006800
6801 return 0;
6802}
6803
Pavel Begunkovebc11b62021-08-09 13:04:05 +01006804static struct io_wq_work *io_wq_free_work(struct io_wq_work *work)
6805{
6806 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
6807
6808 req = io_put_req_find_next(req);
6809 return req ? &req->work : NULL;
6810}
6811
Pavel Begunkov5280f7e2021-02-04 13:52:08 +00006812static void io_wq_submit_work(struct io_wq_work *work)
Pavel Begunkovd4c81f32020-06-08 21:08:19 +03006813{
Jens Axboe2b188cc2019-01-07 10:46:33 -07006814 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
Pavel Begunkov6df1db62020-07-03 22:15:06 +03006815 struct io_kiocb *timeout;
Jens Axboe561fb042019-10-24 07:25:42 -06006816 int ret = 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07006817
Pavel Begunkov48dcd382021-08-15 10:40:18 +01006818 /* one will be dropped by ->io_free_work() after returning to io-wq */
6819 if (!(req->flags & REQ_F_REFCOUNT))
6820 __io_req_set_refcount(req, 2);
6821 else
6822 req_ref_get(req);
Pavel Begunkov5d5901a2021-08-11 19:28:29 +01006823
Pavel Begunkov6df1db62020-07-03 22:15:06 +03006824 timeout = io_prep_linked_timeout(req);
6825 if (timeout)
6826 io_queue_linked_timeout(timeout);
Pavel Begunkovd4c81f32020-06-08 21:08:19 +03006827
Pavel Begunkovdadebc32021-08-23 13:30:44 +01006828 /* either cancelled or io-wq is dying, so don't touch tctx->iowq */
Jens Axboe4014d942021-01-19 15:53:54 -07006829 if (work->flags & IO_WQ_WORK_CANCEL)
Jens Axboe561fb042019-10-24 07:25:42 -06006830 ret = -ECANCELED;
Jens Axboe31b51512019-01-18 22:56:34 -07006831
Jens Axboe561fb042019-10-24 07:25:42 -06006832 if (!ret) {
Jens Axboe561fb042019-10-24 07:25:42 -06006833 do {
Pavel Begunkov889fca72021-02-10 00:03:09 +00006834 ret = io_issue_sqe(req, 0);
Jens Axboe561fb042019-10-24 07:25:42 -06006835 /*
6836 * We can get EAGAIN for polled IO even though we're
6837 * forcing a sync submission from here, since we can't
6838 * wait for request slots on the block side.
6839 */
Pavel Begunkov51ebf1b2022-05-13 11:24:56 +01006840 if (ret != -EAGAIN || !(req->ctx->flags & IORING_SETUP_IOPOLL))
Jens Axboe561fb042019-10-24 07:25:42 -06006841 break;
6842 cond_resched();
6843 } while (1);
6844 }
Jens Axboe31b51512019-01-18 22:56:34 -07006845
Pavel Begunkova3df76982021-02-18 22:32:52 +00006846 /* avoid locking problems by failing it from a clean context */
Pavel Begunkov5d5901a2021-08-11 19:28:29 +01006847 if (ret)
Pavel Begunkova3df76982021-02-18 22:32:52 +00006848 io_req_task_queue_fail(req, ret);
Jens Axboe31b51512019-01-18 22:56:34 -07006849}
Jens Axboe2b188cc2019-01-07 10:46:33 -07006850
Pavel Begunkovaeca2412021-04-11 01:46:37 +01006851static inline struct io_fixed_file *io_fixed_file_slot(struct io_file_table *table,
Pavel Begunkov042b0d82021-08-09 13:04:01 +01006852 unsigned i)
Jens Axboe09bb8392019-03-13 12:39:28 -06006853{
Pavel Begunkov042b0d82021-08-09 13:04:01 +01006854 return &table->files[i];
Pavel Begunkovdafecf12021-02-28 22:35:11 +00006855}
6856
Jens Axboe09bb8392019-03-13 12:39:28 -06006857static inline struct file *io_file_from_index(struct io_ring_ctx *ctx,
6858 int index)
6859{
Pavel Begunkovaeca2412021-04-11 01:46:37 +01006860 struct io_fixed_file *slot = io_fixed_file_slot(&ctx->file_table, index);
Jens Axboe65e19f52019-10-26 07:20:21 -06006861
Pavel Begunkova04b0ac2021-04-01 15:44:04 +01006862 return (struct file *) (slot->file_ptr & FFS_MASK);
Jens Axboe65e19f52019-10-26 07:20:21 -06006863}
6864
Pavel Begunkova04b0ac2021-04-01 15:44:04 +01006865static void io_fixed_file_set(struct io_fixed_file *file_slot, struct file *file)
Pavel Begunkov9a321c92021-04-01 15:44:01 +01006866{
6867 unsigned long file_ptr = (unsigned long) file;
6868
Pavel Begunkovb191e2d2021-08-09 13:04:03 +01006869 if (__io_file_supports_nowait(file, READ))
Pavel Begunkov9a321c92021-04-01 15:44:01 +01006870 file_ptr |= FFS_ASYNC_READ;
Pavel Begunkovb191e2d2021-08-09 13:04:03 +01006871 if (__io_file_supports_nowait(file, WRITE))
Pavel Begunkov9a321c92021-04-01 15:44:01 +01006872 file_ptr |= FFS_ASYNC_WRITE;
6873 if (S_ISREG(file_inode(file)->i_mode))
6874 file_ptr |= FFS_ISREG;
Pavel Begunkova04b0ac2021-04-01 15:44:04 +01006875 file_slot->file_ptr = file_ptr;
Jens Axboe09bb8392019-03-13 12:39:28 -06006876}
6877
Pavel Begunkovac177052021-08-09 13:04:02 +01006878static inline struct file *io_file_get_fixed(struct io_ring_ctx *ctx,
6879 struct io_kiocb *req, int fd)
Pavel Begunkov8da11c12020-02-24 11:32:44 +03006880{
Pavel Begunkov8da11c12020-02-24 11:32:44 +03006881 struct file *file;
Pavel Begunkovac177052021-08-09 13:04:02 +01006882 unsigned long file_ptr;
Pavel Begunkov8da11c12020-02-24 11:32:44 +03006883
Pavel Begunkovac177052021-08-09 13:04:02 +01006884 if (unlikely((unsigned int)fd >= ctx->nr_user_files))
6885 return NULL;
6886 fd = array_index_nospec(fd, ctx->nr_user_files);
6887 file_ptr = io_fixed_file_slot(&ctx->file_table, fd)->file_ptr;
6888 file = (struct file *) (file_ptr & FFS_MASK);
6889 file_ptr &= ~FFS_MASK;
6890 /* mask in overlapping REQ_F and FFS bits */
Pavel Begunkovb191e2d2021-08-09 13:04:03 +01006891 req->flags |= (file_ptr << REQ_F_NOWAIT_READ_BIT);
Pavel Begunkovac177052021-08-09 13:04:02 +01006892 io_req_set_rsrc_node(req);
Pavel Begunkov8371adf2020-10-10 18:34:08 +01006893 return file;
Pavel Begunkov8da11c12020-02-24 11:32:44 +03006894}
6895
Pavel Begunkovac177052021-08-09 13:04:02 +01006896static struct file *io_file_get_normal(struct io_ring_ctx *ctx,
Pavel Begunkovac177052021-08-09 13:04:02 +01006897 struct io_kiocb *req, int fd)
6898{
Pavel Begunkov62906e82021-08-10 14:52:47 +01006899 struct file *file = fget(fd);
Pavel Begunkovac177052021-08-09 13:04:02 +01006900
6901 trace_io_uring_file_get(ctx, fd);
6902
6903 /* we don't allow fixed io_uring files */
6904 if (file && unlikely(file->f_op == &io_uring_fops))
6905 io_req_track_inflight(req);
6906 return file;
6907}
6908
6909static inline struct file *io_file_get(struct io_ring_ctx *ctx,
Pavel Begunkovac177052021-08-09 13:04:02 +01006910 struct io_kiocb *req, int fd, bool fixed)
6911{
6912 if (fixed)
6913 return io_file_get_fixed(ctx, req, fd);
6914 else
Pavel Begunkov62906e82021-08-10 14:52:47 +01006915 return io_file_get_normal(ctx, req, fd);
Pavel Begunkovac177052021-08-09 13:04:02 +01006916}
6917
Pavel Begunkovf237c302021-08-18 12:42:46 +01006918static void io_req_task_link_timeout(struct io_kiocb *req, bool *locked)
Jens Axboe89b263f2021-08-10 15:14:18 -06006919{
6920 struct io_kiocb *prev = req->timeout.prev;
Pavel Begunkov3d2a1e62021-11-26 14:38:14 +00006921 int ret = -ENOENT;
Jens Axboe89b263f2021-08-10 15:14:18 -06006922
6923 if (prev) {
Pavel Begunkov3d2a1e62021-11-26 14:38:14 +00006924 if (!(req->task->flags & PF_EXITING))
6925 ret = io_try_cancel_userdata(req, prev->user_data);
Pavel Begunkov505657b2021-08-17 20:28:09 +01006926 io_req_complete_post(req, ret ?: -ETIME, 0);
Jens Axboe89b263f2021-08-10 15:14:18 -06006927 io_put_req(prev);
Jens Axboe89b263f2021-08-10 15:14:18 -06006928 } else {
6929 io_req_complete_post(req, -ETIME, 0);
6930 }
6931}
6932
Jens Axboe2665abf2019-11-05 12:40:47 -07006933static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer)
6934{
Jens Axboead8a48a2019-11-15 08:49:11 -07006935 struct io_timeout_data *data = container_of(timer,
6936 struct io_timeout_data, timer);
Pavel Begunkov90cd7e42020-10-27 23:25:36 +00006937 struct io_kiocb *prev, *req = data->req;
Jens Axboe2665abf2019-11-05 12:40:47 -07006938 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe2665abf2019-11-05 12:40:47 -07006939 unsigned long flags;
Jens Axboe2665abf2019-11-05 12:40:47 -07006940
Jens Axboe89b263f2021-08-10 15:14:18 -06006941 spin_lock_irqsave(&ctx->timeout_lock, flags);
Pavel Begunkov90cd7e42020-10-27 23:25:36 +00006942 prev = req->timeout.head;
6943 req->timeout.head = NULL;
Jens Axboe2665abf2019-11-05 12:40:47 -07006944
6945 /*
6946 * We don't expect the list to be empty, that will only happen if we
6947 * race with the completion of the linked work.
6948 */
Pavel Begunkov447c19f2021-05-14 12:02:50 +01006949 if (prev) {
Pavel Begunkovf2f87372020-10-27 23:25:37 +00006950 io_remove_next_linked(prev);
Pavel Begunkov447c19f2021-05-14 12:02:50 +01006951 if (!req_ref_inc_not_zero(prev))
6952 prev = NULL;
6953 }
Pavel Begunkovef9dd632021-08-28 19:54:38 -06006954 list_del(&req->timeout.list);
Jens Axboe89b263f2021-08-10 15:14:18 -06006955 req->timeout.prev = prev;
6956 spin_unlock_irqrestore(&ctx->timeout_lock, flags);
Jens Axboe2665abf2019-11-05 12:40:47 -07006957
Jens Axboe89b263f2021-08-10 15:14:18 -06006958 req->io_task_work.func = io_req_task_link_timeout;
6959 io_req_task_work_add(req);
Jens Axboe2665abf2019-11-05 12:40:47 -07006960 return HRTIMER_NORESTART;
6961}
6962
Pavel Begunkovde968c12021-03-19 17:22:33 +00006963static void io_queue_linked_timeout(struct io_kiocb *req)
Jens Axboe2665abf2019-11-05 12:40:47 -07006964{
Pavel Begunkovde968c12021-03-19 17:22:33 +00006965 struct io_ring_ctx *ctx = req->ctx;
6966
Jens Axboe89b263f2021-08-10 15:14:18 -06006967 spin_lock_irq(&ctx->timeout_lock);
Jens Axboe76a46e02019-11-10 23:34:16 -07006968 /*
Pavel Begunkovf2f87372020-10-27 23:25:37 +00006969 * If the back reference is NULL, then our linked request finished
6970 * before we got a chance to setup the timer
Jens Axboe76a46e02019-11-10 23:34:16 -07006971 */
Pavel Begunkov90cd7e42020-10-27 23:25:36 +00006972 if (req->timeout.head) {
Jens Axboee8c2bc12020-08-15 18:44:09 -07006973 struct io_timeout_data *data = req->async_data;
Jens Axboe94ae5e72019-11-14 19:39:52 -07006974
Jens Axboead8a48a2019-11-15 08:49:11 -07006975 data->timer.function = io_link_timeout_fn;
6976 hrtimer_start(&data->timer, timespec64_to_ktime(data->ts),
6977 data->mode);
Pavel Begunkovef9dd632021-08-28 19:54:38 -06006978 list_add_tail(&req->timeout.list, &ctx->ltimeout_list);
Jens Axboe2665abf2019-11-05 12:40:47 -07006979 }
Jens Axboe89b263f2021-08-10 15:14:18 -06006980 spin_unlock_irq(&ctx->timeout_lock);
Jens Axboe2665abf2019-11-05 12:40:47 -07006981 /* drop submission reference */
Jens Axboe76a46e02019-11-10 23:34:16 -07006982 io_put_req(req);
Jens Axboe2665abf2019-11-05 12:40:47 -07006983}
6984
Pavel Begunkovc5eef2b2021-02-10 00:03:22 +00006985static void __io_queue_sqe(struct io_kiocb *req)
Pavel Begunkov282cdc82021-08-09 13:04:10 +01006986 __must_hold(&req->ctx->uring_lock)
Jens Axboe2b188cc2019-01-07 10:46:33 -07006987{
Pavel Begunkov906c6ca2021-08-15 10:40:26 +01006988 struct io_kiocb *linked_timeout;
Jens Axboee0c5c572019-03-12 10:18:47 -06006989 int ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07006990
Olivier Langlois59b735a2021-06-22 05:17:39 -07006991issue_sqe:
Pavel Begunkovc5eef2b2021-02-10 00:03:22 +00006992 ret = io_issue_sqe(req, IO_URING_F_NONBLOCK|IO_URING_F_COMPLETE_DEFER);
Jens Axboe491381ce2019-10-17 09:20:46 -06006993
6994 /*
6995 * We async punt it if the file wasn't marked NOWAIT, or if the file
6996 * doesn't support non-blocking read/write attempts
6997 */
Pavel Begunkov18400382021-03-19 17:22:34 +00006998 if (likely(!ret)) {
Pavel Begunkove342c802021-01-19 13:32:47 +00006999 if (req->flags & REQ_F_COMPLETE_INLINE) {
Pavel Begunkovc5eef2b2021-02-10 00:03:22 +00007000 struct io_ring_ctx *ctx = req->ctx;
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01007001 struct io_submit_state *state = &ctx->submit_state;
Jens Axboee65ef562019-03-12 10:16:44 -06007002
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01007003 state->compl_reqs[state->compl_nr++] = req;
7004 if (state->compl_nr == ARRAY_SIZE(state->compl_reqs))
Pavel Begunkov2a2758f2021-06-17 18:14:00 +01007005 io_submit_flush_completions(ctx);
Pavel Begunkov906c6ca2021-08-15 10:40:26 +01007006 return;
Pavel Begunkov0d63c142020-10-22 16:47:18 +01007007 }
Pavel Begunkov906c6ca2021-08-15 10:40:26 +01007008
7009 linked_timeout = io_prep_linked_timeout(req);
7010 if (linked_timeout)
7011 io_queue_linked_timeout(linked_timeout);
Pavel Begunkov18400382021-03-19 17:22:34 +00007012 } else if (ret == -EAGAIN && !(req->flags & REQ_F_NOWAIT)) {
Pavel Begunkov906c6ca2021-08-15 10:40:26 +01007013 linked_timeout = io_prep_linked_timeout(req);
7014
Olivier Langlois59b735a2021-06-22 05:17:39 -07007015 switch (io_arm_poll_handler(req)) {
7016 case IO_APOLL_READY:
Pavel Begunkov906c6ca2021-08-15 10:40:26 +01007017 if (linked_timeout)
Pavel Begunkov4ea672a2021-10-20 09:53:02 +01007018 io_queue_linked_timeout(linked_timeout);
Olivier Langlois59b735a2021-06-22 05:17:39 -07007019 goto issue_sqe;
7020 case IO_APOLL_ABORTED:
Pavel Begunkov18400382021-03-19 17:22:34 +00007021 /*
7022 * Queued up for async execution, worker will release
7023 * submit reference when the iocb is actually submitted.
7024 */
Pavel Begunkovf237c302021-08-18 12:42:46 +01007025 io_queue_async_work(req, NULL);
Olivier Langlois59b735a2021-06-22 05:17:39 -07007026 break;
Pavel Begunkov18400382021-03-19 17:22:34 +00007027 }
Pavel Begunkov906c6ca2021-08-15 10:40:26 +01007028
7029 if (linked_timeout)
7030 io_queue_linked_timeout(linked_timeout);
Pavel Begunkov0d63c142020-10-22 16:47:18 +01007031 } else {
Pavel Begunkovf41db2732021-02-28 22:35:12 +00007032 io_req_complete_failed(req, ret);
Jens Axboe9e645e112019-05-10 16:07:28 -06007033 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07007034}
7035
Pavel Begunkov441b8a72021-06-14 23:37:31 +01007036static inline void io_queue_sqe(struct io_kiocb *req)
Pavel Begunkov282cdc82021-08-09 13:04:10 +01007037 __must_hold(&req->ctx->uring_lock)
Jackie Liu4fe2c962019-09-09 20:50:40 +08007038{
Pavel Begunkov10c66902021-06-15 16:47:56 +01007039 if (unlikely(req->ctx->drain_active) && io_drain_req(req))
Pavel Begunkov76cc33d2021-06-14 23:37:30 +01007040 return;
Jackie Liu4fe2c962019-09-09 20:50:40 +08007041
Hao Xua8295b92021-08-27 17:46:09 +08007042 if (likely(!(req->flags & (REQ_F_FORCE_ASYNC | REQ_F_FAIL)))) {
Pavel Begunkovc5eef2b2021-02-10 00:03:22 +00007043 __io_queue_sqe(req);
Hao Xua8295b92021-08-27 17:46:09 +08007044 } else if (req->flags & REQ_F_FAIL) {
Pavel Begunkovc6d3d9c2021-08-31 14:13:10 +01007045 io_req_complete_fail_submit(req);
Pavel Begunkov76cc33d2021-06-14 23:37:30 +01007046 } else {
7047 int ret = io_req_prep_async(req);
7048
7049 if (unlikely(ret))
7050 io_req_complete_failed(req, ret);
7051 else
Pavel Begunkovf237c302021-08-18 12:42:46 +01007052 io_queue_async_work(req, NULL);
Jens Axboece35a472019-12-17 08:04:44 -07007053 }
Jackie Liu4fe2c962019-09-09 20:50:40 +08007054}
7055
Stefano Garzarella21b55db2020-08-27 16:58:30 +02007056/*
7057 * Check SQE restrictions (opcode and flags).
7058 *
7059 * Returns 'true' if SQE is allowed, 'false' otherwise.
7060 */
7061static inline bool io_check_restriction(struct io_ring_ctx *ctx,
7062 struct io_kiocb *req,
7063 unsigned int sqe_flags)
7064{
Pavel Begunkov4cfb25b2021-06-26 21:40:47 +01007065 if (likely(!ctx->restricted))
Stefano Garzarella21b55db2020-08-27 16:58:30 +02007066 return true;
7067
7068 if (!test_bit(req->opcode, ctx->restrictions.sqe_op))
7069 return false;
7070
7071 if ((sqe_flags & ctx->restrictions.sqe_flags_required) !=
7072 ctx->restrictions.sqe_flags_required)
7073 return false;
7074
7075 if (sqe_flags & ~(ctx->restrictions.sqe_flags_allowed |
7076 ctx->restrictions.sqe_flags_required))
7077 return false;
7078
7079 return true;
7080}
7081
Pavel Begunkovef4ff582020-04-12 02:05:05 +03007082static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,
Pavel Begunkov258b29a2021-02-10 00:03:10 +00007083 const struct io_uring_sqe *sqe)
Pavel Begunkov282cdc82021-08-09 13:04:10 +01007084 __must_hold(&ctx->uring_lock)
Pavel Begunkov0553b8b2020-04-08 08:58:45 +03007085{
Pavel Begunkov258b29a2021-02-10 00:03:10 +00007086 struct io_submit_state *state;
Pavel Begunkovef4ff582020-04-12 02:05:05 +03007087 unsigned int sqe_flags;
Jens Axboe003e8dc2021-03-06 09:22:27 -07007088 int personality, ret = 0;
Pavel Begunkovef4ff582020-04-12 02:05:05 +03007089
Pavel Begunkov864ea922021-08-09 13:04:08 +01007090 /* req is partially pre-initialised, see io_preinit_req() */
Pavel Begunkov0553b8b2020-04-08 08:58:45 +03007091 req->opcode = READ_ONCE(sqe->opcode);
Pavel Begunkov5be9ad12021-02-12 18:41:17 +00007092 /* same numerical values with corresponding REQ_F_*, safe to copy */
7093 req->flags = sqe_flags = READ_ONCE(sqe->flags);
Pavel Begunkov0553b8b2020-04-08 08:58:45 +03007094 req->user_data = READ_ONCE(sqe->user_data);
Pavel Begunkov0553b8b2020-04-08 08:58:45 +03007095 req->file = NULL;
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00007096 req->fixed_rsrc_refs = NULL;
Pavel Begunkov4dd28242020-06-15 10:33:13 +03007097 req->task = current;
Pavel Begunkovef4ff582020-04-12 02:05:05 +03007098
Pavel Begunkov5be9ad12021-02-12 18:41:17 +00007099 /* enforce forwards compatibility on users */
Pavel Begunkovdddca222021-04-27 16:13:52 +01007100 if (unlikely(sqe_flags & ~SQE_VALID_FLAGS))
Pavel Begunkov5be9ad12021-02-12 18:41:17 +00007101 return -EINVAL;
Pavel Begunkovef4ff582020-04-12 02:05:05 +03007102 if (unlikely(req->opcode >= IORING_OP_LAST))
7103 return -EINVAL;
Pavel Begunkov4cfb25b2021-06-26 21:40:47 +01007104 if (!io_check_restriction(ctx, req, sqe_flags))
Stefano Garzarella21b55db2020-08-27 16:58:30 +02007105 return -EACCES;
7106
Pavel Begunkovef4ff582020-04-12 02:05:05 +03007107 if ((sqe_flags & IOSQE_BUFFER_SELECT) &&
7108 !io_op_defs[req->opcode].buffer_select)
7109 return -EOPNOTSUPP;
Pavel Begunkov3c199662021-06-15 16:47:57 +01007110 if (unlikely(sqe_flags & IOSQE_IO_DRAIN))
7111 ctx->drain_active = true;
Pavel Begunkovef4ff582020-04-12 02:05:05 +03007112
Jens Axboe003e8dc2021-03-06 09:22:27 -07007113 personality = READ_ONCE(sqe->personality);
7114 if (personality) {
Pavel Begunkovc10d1f92021-06-17 18:14:01 +01007115 req->creds = xa_load(&ctx->personalities, personality);
7116 if (!req->creds)
Jens Axboe003e8dc2021-03-06 09:22:27 -07007117 return -EINVAL;
Pavel Begunkovc10d1f92021-06-17 18:14:01 +01007118 get_cred(req->creds);
Pavel Begunkovb8e64b52021-06-17 18:14:02 +01007119 req->flags |= REQ_F_CREDS;
Jens Axboe003e8dc2021-03-06 09:22:27 -07007120 }
Pavel Begunkov258b29a2021-02-10 00:03:10 +00007121 state = &ctx->submit_state;
Pavel Begunkovef4ff582020-04-12 02:05:05 +03007122
Jens Axboe27926b62020-10-28 09:33:23 -06007123 /*
7124 * Plug now if we have more than 1 IO left after this, and the target
7125 * is potentially a read/write to block based storage.
7126 */
7127 if (!state->plug_started && state->ios_left > 1 &&
7128 io_op_defs[req->opcode].plug) {
7129 blk_start_plug(&state->plug);
7130 state->plug_started = true;
7131 }
Jens Axboe63ff8222020-05-07 14:56:15 -06007132
Pavel Begunkovbd5bbda2020-11-20 15:50:51 +00007133 if (io_op_defs[req->opcode].needs_file) {
Pavel Begunkov62906e82021-08-10 14:52:47 +01007134 req->file = io_file_get(ctx, req, READ_ONCE(sqe->fd),
Pavel Begunkovac177052021-08-09 13:04:02 +01007135 (sqe_flags & IOSQE_FIXED_FILE));
Pavel Begunkovba13e232021-02-01 18:59:52 +00007136 if (unlikely(!req->file))
Pavel Begunkovbd5bbda2020-11-20 15:50:51 +00007137 ret = -EBADF;
7138 }
7139
Pavel Begunkov71b547c2020-10-10 18:34:09 +01007140 state->ios_left--;
7141 return ret;
Pavel Begunkov0553b8b2020-04-08 08:58:45 +03007142}
7143
Pavel Begunkova6b8cadc2021-02-18 18:29:41 +00007144static int io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,
Pavel Begunkova1ab7b32021-02-18 18:29:42 +00007145 const struct io_uring_sqe *sqe)
Pavel Begunkov282cdc82021-08-09 13:04:10 +01007146 __must_hold(&ctx->uring_lock)
Jens Axboe6c271ce2019-01-10 11:22:30 -07007147{
Pavel Begunkova1ab7b32021-02-18 18:29:42 +00007148 struct io_submit_link *link = &ctx->submit_state.link;
Jens Axboe6c271ce2019-01-10 11:22:30 -07007149 int ret;
7150
Pavel Begunkova6b8cadc2021-02-18 18:29:41 +00007151 ret = io_init_req(ctx, req, sqe);
7152 if (unlikely(ret)) {
7153fail_req:
Hao Xua8295b92021-08-27 17:46:09 +08007154 /* fail even hard links since we don't submit */
Pavel Begunkovde59bc12021-02-18 18:29:47 +00007155 if (link->head) {
Hao Xua8295b92021-08-27 17:46:09 +08007156 /*
7157 * we can judge a link req is failed or cancelled by if
7158 * REQ_F_FAIL is set, but the head is an exception since
7159 * it may be set REQ_F_FAIL because of other req's failure
7160 * so let's leverage req->result to distinguish if a head
7161 * is set REQ_F_FAIL because of its failure or other req's
7162 * failure so that we can set the correct ret code for it.
7163 * init result here to avoid affecting the normal path.
7164 */
7165 if (!(link->head->flags & REQ_F_FAIL))
7166 req_fail_link_node(link->head, -ECANCELED);
7167 } else if (!(req->flags & (REQ_F_LINK | REQ_F_HARDLINK))) {
7168 /*
7169 * the current req is a normal req, we should return
7170 * error and thus break the submittion loop.
7171 */
7172 io_req_complete_failed(req, ret);
7173 return ret;
Pavel Begunkovde59bc12021-02-18 18:29:47 +00007174 }
Hao Xua8295b92021-08-27 17:46:09 +08007175 req_fail_link_node(req, ret);
7176 } else {
7177 ret = io_req_prep(req, sqe);
7178 if (unlikely(ret))
7179 goto fail_req;
Pavel Begunkova6b8cadc2021-02-18 18:29:41 +00007180 }
Pavel Begunkov441b8a72021-06-14 23:37:31 +01007181
Pavel Begunkovbe7053b2021-02-18 18:29:45 +00007182 /* don't need @sqe from now on */
Olivier Langlois236daeae2021-05-31 02:36:37 -04007183 trace_io_uring_submit_sqe(ctx, req, req->opcode, req->user_data,
7184 req->flags, true,
7185 ctx->flags & IORING_SETUP_SQPOLL);
Pavel Begunkova6b8cadc2021-02-18 18:29:41 +00007186
Jens Axboe6c271ce2019-01-10 11:22:30 -07007187 /*
7188 * If we already have a head request, queue this one for async
7189 * submittal once the head completes. If we don't have a head but
7190 * IOSQE_IO_LINK is set in the sqe, start a new head. This one will be
7191 * submitted sync once the chain is complete. If none of those
7192 * conditions are true (normal request), then just queue it.
7193 */
7194 if (link->head) {
7195 struct io_kiocb *head = link->head;
7196
Hao Xua8295b92021-08-27 17:46:09 +08007197 if (!(req->flags & REQ_F_FAIL)) {
7198 ret = io_req_prep_async(req);
7199 if (unlikely(ret)) {
7200 req_fail_link_node(req, ret);
7201 if (!(head->flags & REQ_F_FAIL))
7202 req_fail_link_node(head, -ECANCELED);
7203 }
7204 }
Jens Axboe6c271ce2019-01-10 11:22:30 -07007205 trace_io_uring_link(ctx, req, head);
7206 link->last->link = req;
7207 link->last = req;
7208
7209 /* last request of a link, enqueue the link */
7210 if (!(req->flags & (REQ_F_LINK | REQ_F_HARDLINK))) {
7211 link->head = NULL;
Pavel Begunkov5e159202021-06-14 23:37:26 +01007212 io_queue_sqe(head);
Jens Axboe2b188cc2019-01-07 10:46:33 -07007213 }
Jackie Liu4fe2c962019-09-09 20:50:40 +08007214 } else {
Jens Axboe2b188cc2019-01-07 10:46:33 -07007215 if (req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) {
Jackie Liu4fe2c962019-09-09 20:50:40 +08007216 link->head = req;
7217 link->last = req;
7218 } else {
Pavel Begunkovbe7053b2021-02-18 18:29:45 +00007219 io_queue_sqe(req);
Jackie Liu4fe2c962019-09-09 20:50:40 +08007220 }
7221 }
7222
7223 return 0;
7224}
7225
7226/*
7227 * Batched submission is done, ensure local IO is flushed out.
7228 */
7229static void io_submit_state_end(struct io_submit_state *state,
7230 struct io_ring_ctx *ctx)
Pavel Begunkov1b4a51b2019-11-21 11:54:28 +03007231{
Pavel Begunkova1ab7b32021-02-18 18:29:42 +00007232 if (state->link.head)
Pavel Begunkovde59bc12021-02-18 18:29:47 +00007233 io_queue_sqe(state->link.head);
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01007234 if (state->compl_nr)
Pavel Begunkov2a2758f2021-06-17 18:14:00 +01007235 io_submit_flush_completions(ctx);
Jackie Liua197f662019-11-08 08:09:12 -07007236 if (state->plug_started)
Pavel Begunkov32fe5252019-12-17 22:26:58 +03007237 blk_finish_plug(&state->plug);
Jens Axboe9e645e112019-05-10 16:07:28 -06007238}
Pavel Begunkov32fe5252019-12-17 22:26:58 +03007239
Jens Axboe9e645e112019-05-10 16:07:28 -06007240/*
7241 * Start submission side cache.
Pavel Begunkov32fe5252019-12-17 22:26:58 +03007242 */
Jens Axboe9e645e112019-05-10 16:07:28 -06007243static void io_submit_state_start(struct io_submit_state *state,
Pavel Begunkov196be952019-11-07 01:41:06 +03007244 unsigned int max_ios)
Jens Axboe9e645e112019-05-10 16:07:28 -06007245{
7246 state->plug_started = false;
Jens Axboebcda7ba2020-02-23 16:42:51 -07007247 state->ios_left = max_ios;
Pavel Begunkova1ab7b32021-02-18 18:29:42 +00007248 /* set only head, no need to init link_last in advance */
7249 state->link.head = NULL;
Jens Axboe75c6a032020-01-28 10:15:23 -07007250}
7251
Jens Axboe193155c2020-02-22 23:22:19 -07007252static void io_commit_sqring(struct io_ring_ctx *ctx)
7253{
Jens Axboe75c6a032020-01-28 10:15:23 -07007254 struct io_rings *rings = ctx->rings;
7255
7256 /*
Jens Axboe193155c2020-02-22 23:22:19 -07007257 * Ensure any loads from the SQEs are done at this point,
Jens Axboe75c6a032020-01-28 10:15:23 -07007258 * since once we write the new head, the application could
7259 * write new data to them.
Pavel Begunkov6b47ee62020-01-18 20:22:41 +03007260 */
Pavel Begunkov8da11c12020-02-24 11:32:44 +03007261 smp_store_release(&rings->sq.head, ctx->cached_sq_head);
Jens Axboebcda7ba2020-02-23 16:42:51 -07007262}
7263
Jens Axboe9e645e112019-05-10 16:07:28 -06007264/*
Fam Zhengdd9ae8a2021-06-04 17:42:56 +01007265 * Fetch an sqe, if one is available. Note this returns a pointer to memory
Jens Axboe9e645e112019-05-10 16:07:28 -06007266 * that is mapped by userspace. This means that care needs to be taken to
7267 * ensure that reads are stable, as we cannot rely on userspace always
Jens Axboe78e19bb2019-11-06 15:21:34 -07007268 * being a good citizen. If members of the sqe are validated and then later
7269 * used, it's important that those reads are done through READ_ONCE() to
Pavel Begunkov2e6e1fd2019-12-05 16:15:45 +03007270 * prevent a re-load down the line.
Jens Axboe9e645e112019-05-10 16:07:28 -06007271 */
7272static const struct io_uring_sqe *io_get_sqe(struct io_ring_ctx *ctx)
Jens Axboe9e645e112019-05-10 16:07:28 -06007273{
Pavel Begunkovea5ab3b2021-05-16 22:58:09 +01007274 unsigned head, mask = ctx->sq_entries - 1;
Pavel Begunkov17d3aeb2021-06-14 23:37:23 +01007275 unsigned sq_idx = ctx->cached_sq_head++ & mask;
Jens Axboe9e645e112019-05-10 16:07:28 -06007276
7277 /*
7278 * The cached sq head (or cq tail) serves two purposes:
7279 *
7280 * 1) allows us to batch the cost of updating the user visible
Pavel Begunkov9d763772019-12-17 02:22:07 +03007281 * head updates.
Jens Axboe9e645e112019-05-10 16:07:28 -06007282 * 2) allows the kernel side to track the head on its own, even
Pavel Begunkov8cdf2192020-01-25 00:40:24 +03007283 * though the application is the one updating it.
7284 */
Pavel Begunkov17d3aeb2021-06-14 23:37:23 +01007285 head = READ_ONCE(ctx->sq_array[sq_idx]);
Pavel Begunkov8cdf2192020-01-25 00:40:24 +03007286 if (likely(head < ctx->sq_entries))
7287 return &ctx->sq_sqes[head];
7288
7289 /* drop invalid entries */
Pavel Begunkov15641e42021-06-14 23:37:24 +01007290 ctx->cq_extra--;
7291 WRITE_ONCE(ctx->rings->sq_dropped,
7292 READ_ONCE(ctx->rings->sq_dropped) + 1);
Pavel Begunkov711be032020-01-17 03:57:59 +03007293 return NULL;
7294}
Jens Axboeb7bb4f72019-12-15 22:13:43 -07007295
Jens Axboe0f212202020-09-13 13:09:39 -06007296static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr)
Pavel Begunkov282cdc82021-08-09 13:04:10 +01007297 __must_hold(&ctx->uring_lock)
Jens Axboe6c271ce2019-01-10 11:22:30 -07007298{
Pavel Begunkov46c4e162021-02-18 18:29:37 +00007299 int submitted = 0;
Jens Axboe6c271ce2019-01-10 11:22:30 -07007300
Pavel Begunkovee7d46d2019-12-30 21:24:45 +03007301 /* make sure SQ entry isn't read before tail */
7302 nr = min3(nr, ctx->sq_entries, io_sqring_entries(ctx));
Pavel Begunkov2b85edf2019-12-28 14:13:03 +03007303 if (!percpu_ref_tryget_many(&ctx->refs, nr))
7304 return -EAGAIN;
Pavel Begunkov9a108672021-08-27 11:55:01 +01007305 io_get_task_refs(nr);
Jens Axboe6c271ce2019-01-10 11:22:30 -07007306
Pavel Begunkovba88ff12021-02-10 00:03:11 +00007307 io_submit_state_start(&ctx->submit_state, nr);
Pavel Begunkov46c4e162021-02-18 18:29:37 +00007308 while (submitted < nr) {
Jens Axboe3529d8c2019-12-19 18:24:38 -07007309 const struct io_uring_sqe *sqe;
Pavel Begunkov196be952019-11-07 01:41:06 +03007310 struct io_kiocb *req;
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03007311
Pavel Begunkov258b29a2021-02-10 00:03:10 +00007312 req = io_alloc_req(ctx);
Pavel Begunkov196be952019-11-07 01:41:06 +03007313 if (unlikely(!req)) {
7314 if (!submitted)
7315 submitted = -EAGAIN;
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03007316 break;
Jens Axboe9e645e112019-05-10 16:07:28 -06007317 }
Pavel Begunkov4fccfcb2021-02-12 11:55:17 +00007318 sqe = io_get_sqe(ctx);
7319 if (unlikely(!sqe)) {
Hao Xu0c6e1d72021-08-26 01:58:56 +08007320 list_add(&req->inflight_entry, &ctx->submit_state.free_list);
Pavel Begunkov4fccfcb2021-02-12 11:55:17 +00007321 break;
7322 }
Jens Axboed3656342019-12-18 09:50:26 -07007323 /* will complete beyond this point, count as submitted */
7324 submitted++;
Pavel Begunkova1ab7b32021-02-18 18:29:42 +00007325 if (io_submit_sqe(ctx, req, sqe))
Jens Axboed3656342019-12-18 09:50:26 -07007326 break;
Jens Axboe6c271ce2019-01-10 11:22:30 -07007327 }
7328
Pavel Begunkov9466f432020-01-25 22:34:01 +03007329 if (unlikely(submitted != nr)) {
7330 int ref_used = (submitted == -EAGAIN) ? 0 : submitted;
Jens Axboed8a6df12020-10-15 16:24:45 -06007331 int unused = nr - ref_used;
Pavel Begunkov9466f432020-01-25 22:34:01 +03007332
Pavel Begunkov09899b12021-06-14 02:36:22 +01007333 current->io_uring->cached_refs += unused;
Jens Axboed8a6df12020-10-15 16:24:45 -06007334 percpu_ref_put_many(&ctx->refs, unused);
Pavel Begunkov9466f432020-01-25 22:34:01 +03007335 }
Jens Axboe6c271ce2019-01-10 11:22:30 -07007336
Pavel Begunkova1ab7b32021-02-18 18:29:42 +00007337 io_submit_state_end(&ctx->submit_state, ctx);
Pavel Begunkovae9428c2019-11-06 00:22:14 +03007338 /* Commit SQ ring head once we've consumed and submitted all SQEs */
7339 io_commit_sqring(ctx);
7340
Jens Axboe6c271ce2019-01-10 11:22:30 -07007341 return submitted;
7342}
7343
Pavel Begunkove4b6d902021-05-16 22:58:00 +01007344static inline bool io_sqd_events_pending(struct io_sq_data *sqd)
7345{
7346 return READ_ONCE(sqd->state);
7347}
7348
Xiaoguang Wang23b36282020-07-23 20:57:24 +08007349static inline void io_ring_set_wakeup_flag(struct io_ring_ctx *ctx)
7350{
7351 /* Tell userspace we may need a wakeup call */
Jens Axboe79ebeae2021-08-10 15:18:27 -06007352 spin_lock(&ctx->completion_lock);
Nadav Amit20c0b382021-08-07 17:13:42 -07007353 WRITE_ONCE(ctx->rings->sq_flags,
7354 ctx->rings->sq_flags | IORING_SQ_NEED_WAKEUP);
Jens Axboe79ebeae2021-08-10 15:18:27 -06007355 spin_unlock(&ctx->completion_lock);
Xiaoguang Wang23b36282020-07-23 20:57:24 +08007356}
7357
7358static inline void io_ring_clear_wakeup_flag(struct io_ring_ctx *ctx)
7359{
Jens Axboe79ebeae2021-08-10 15:18:27 -06007360 spin_lock(&ctx->completion_lock);
Nadav Amit20c0b382021-08-07 17:13:42 -07007361 WRITE_ONCE(ctx->rings->sq_flags,
7362 ctx->rings->sq_flags & ~IORING_SQ_NEED_WAKEUP);
Jens Axboe79ebeae2021-08-10 15:18:27 -06007363 spin_unlock(&ctx->completion_lock);
Xiaoguang Wang23b36282020-07-23 20:57:24 +08007364}
7365
Xiaoguang Wang08369242020-11-03 14:15:59 +08007366static int __io_sq_thread(struct io_ring_ctx *ctx, bool cap_entries)
Jens Axboe6c271ce2019-01-10 11:22:30 -07007367{
Jens Axboec8d1ba52020-09-14 11:07:26 -06007368 unsigned int to_submit;
Xiaoguang Wangbdcd3ea2020-02-25 22:12:08 +08007369 int ret = 0;
Jens Axboe6c271ce2019-01-10 11:22:30 -07007370
Jens Axboec8d1ba52020-09-14 11:07:26 -06007371 to_submit = io_sqring_entries(ctx);
Jens Axboee95eee22020-09-08 09:11:32 -06007372 /* if we're handling multiple rings, cap submit size for fairness */
Olivier Langlois4ce8ad92021-06-23 11:50:18 -07007373 if (cap_entries && to_submit > IORING_SQPOLL_CAP_ENTRIES_VALUE)
7374 to_submit = IORING_SQPOLL_CAP_ENTRIES_VALUE;
Jens Axboee95eee22020-09-08 09:11:32 -06007375
Xiaoguang Wang906a3c62020-11-12 14:56:00 +08007376 if (!list_empty(&ctx->iopoll_list) || to_submit) {
7377 unsigned nr_events = 0;
Pavel Begunkov948e1942021-06-24 15:09:55 +01007378 const struct cred *creds = NULL;
7379
7380 if (ctx->sq_creds != current_cred())
7381 creds = override_creds(ctx->sq_creds);
Xiaoguang Wang906a3c62020-11-12 14:56:00 +08007382
Xiaoguang Wang08369242020-11-03 14:15:59 +08007383 mutex_lock(&ctx->uring_lock);
Xiaoguang Wang906a3c62020-11-12 14:56:00 +08007384 if (!list_empty(&ctx->iopoll_list))
Pavel Begunkova8576af2021-08-15 10:40:21 +01007385 io_do_iopoll(ctx, &nr_events, 0);
Xiaoguang Wang906a3c62020-11-12 14:56:00 +08007386
Pavel Begunkov3b763ba2021-04-18 14:52:08 +01007387 /*
7388 * Don't submit if refs are dying, good for io_uring_register(),
7389 * but also it is relied upon by io_ring_exit_work()
7390 */
Pavel Begunkov0298ef92021-03-08 13:20:57 +00007391 if (to_submit && likely(!percpu_ref_is_dying(&ctx->refs)) &&
7392 !(ctx->flags & IORING_SETUP_R_DISABLED))
Xiaoguang Wang08369242020-11-03 14:15:59 +08007393 ret = io_submit_sqes(ctx, to_submit);
7394 mutex_unlock(&ctx->uring_lock);
Jens Axboe90554202020-09-03 12:12:41 -06007395
Pavel Begunkovacfb3812021-05-16 22:58:03 +01007396 if (to_submit && wq_has_sleeper(&ctx->sqo_sq_wait))
7397 wake_up(&ctx->sqo_sq_wait);
Pavel Begunkov948e1942021-06-24 15:09:55 +01007398 if (creds)
7399 revert_creds(creds);
Pavel Begunkovacfb3812021-05-16 22:58:03 +01007400 }
Jens Axboe90554202020-09-03 12:12:41 -06007401
Xiaoguang Wang08369242020-11-03 14:15:59 +08007402 return ret;
7403}
7404
7405static void io_sqd_update_thread_idle(struct io_sq_data *sqd)
7406{
7407 struct io_ring_ctx *ctx;
7408 unsigned sq_thread_idle = 0;
7409
Pavel Begunkovc9dca272021-03-10 13:13:55 +00007410 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list)
7411 sq_thread_idle = max(sq_thread_idle, ctx->sq_thread_idle);
Xiaoguang Wang08369242020-11-03 14:15:59 +08007412 sqd->sq_thread_idle = sq_thread_idle;
Jens Axboec8d1ba52020-09-14 11:07:26 -06007413}
7414
Pavel Begunkove4b6d902021-05-16 22:58:00 +01007415static bool io_sqd_handle_event(struct io_sq_data *sqd)
7416{
7417 bool did_sig = false;
7418 struct ksignal ksig;
7419
7420 if (test_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state) ||
7421 signal_pending(current)) {
7422 mutex_unlock(&sqd->lock);
7423 if (signal_pending(current))
7424 did_sig = get_signal(&ksig);
7425 cond_resched();
7426 mutex_lock(&sqd->lock);
7427 }
Pavel Begunkove4b6d902021-05-16 22:58:00 +01007428 return did_sig || test_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state);
7429}
7430
Jens Axboe6c271ce2019-01-10 11:22:30 -07007431static int io_sq_thread(void *data)
7432{
Jens Axboe69fb2132020-09-14 11:16:23 -06007433 struct io_sq_data *sqd = data;
7434 struct io_ring_ctx *ctx;
Xiaoguang Wanga0d92052020-11-12 14:55:59 +08007435 unsigned long timeout = 0;
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007436 char buf[TASK_COMM_LEN];
Xiaoguang Wang08369242020-11-03 14:15:59 +08007437 DEFINE_WAIT(wait);
Jens Axboe6c271ce2019-01-10 11:22:30 -07007438
Pavel Begunkov696ee882021-04-01 09:55:04 +01007439 snprintf(buf, sizeof(buf), "iou-sqp-%d", sqd->task_pid);
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007440 set_task_comm(current, buf);
Jens Axboe28cea78a2020-09-14 10:51:17 -06007441
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007442 if (sqd->sq_cpu != -1)
7443 set_cpus_allowed_ptr(current, cpumask_of(sqd->sq_cpu));
7444 else
7445 set_cpus_allowed_ptr(current, cpu_online_mask);
7446 current->flags |= PF_NO_SETAFFINITY;
7447
Pavel Begunkov09a6f4e2021-03-14 20:57:10 +00007448 mutex_lock(&sqd->lock);
Pavel Begunkove4b6d902021-05-16 22:58:00 +01007449 while (1) {
Pavel Begunkov1a924a82021-06-24 15:09:56 +01007450 bool cap_entries, sqt_spin = false;
Jens Axboec1edbf52019-11-10 16:56:04 -07007451
Pavel Begunkove4b6d902021-05-16 22:58:00 +01007452 if (io_sqd_events_pending(sqd) || signal_pending(current)) {
7453 if (io_sqd_handle_event(sqd))
Pavel Begunkovc7d95612021-04-13 11:43:00 +01007454 break;
Xiaoguang Wang08369242020-11-03 14:15:59 +08007455 timeout = jiffies + sqd->sq_thread_idle;
7456 }
Pavel Begunkove4b6d902021-05-16 22:58:00 +01007457
Jens Axboee95eee22020-09-08 09:11:32 -06007458 cap_entries = !list_is_singular(&sqd->ctx_list);
Jens Axboe69fb2132020-09-14 11:16:23 -06007459 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) {
Pavel Begunkov948e1942021-06-24 15:09:55 +01007460 int ret = __io_sq_thread(ctx, cap_entries);
Stefan Metzmacher7c30f36a2021-03-07 11:54:28 +01007461
Xiaoguang Wang08369242020-11-03 14:15:59 +08007462 if (!sqt_spin && (ret > 0 || !list_empty(&ctx->iopoll_list)))
7463 sqt_spin = true;
Jens Axboe6c271ce2019-01-10 11:22:30 -07007464 }
Pavel Begunkovdd432ea52021-06-26 21:40:45 +01007465 if (io_run_task_work())
7466 sqt_spin = true;
Jens Axboe6c271ce2019-01-10 11:22:30 -07007467
Xiaoguang Wang08369242020-11-03 14:15:59 +08007468 if (sqt_spin || !time_after(jiffies, timeout)) {
Jens Axboec8d1ba52020-09-14 11:07:26 -06007469 cond_resched();
Xiaoguang Wang08369242020-11-03 14:15:59 +08007470 if (sqt_spin)
7471 timeout = jiffies + sqd->sq_thread_idle;
7472 continue;
7473 }
7474
Xiaoguang Wang08369242020-11-03 14:15:59 +08007475 prepare_to_wait(&sqd->wait, &wait, TASK_INTERRUPTIBLE);
Pavel Begunkovdd432ea52021-06-26 21:40:45 +01007476 if (!io_sqd_events_pending(sqd) && !current->task_works) {
Pavel Begunkov1a924a82021-06-24 15:09:56 +01007477 bool needs_sched = true;
7478
Hao Xu724cb4f2021-04-21 23:19:11 +08007479 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) {
Pavel Begunkovaaa9f0f2021-05-16 22:58:01 +01007480 io_ring_set_wakeup_flag(ctx);
7481
Hao Xu724cb4f2021-04-21 23:19:11 +08007482 if ((ctx->flags & IORING_SETUP_IOPOLL) &&
7483 !list_empty_careful(&ctx->iopoll_list)) {
7484 needs_sched = false;
7485 break;
7486 }
7487 if (io_sqring_entries(ctx)) {
7488 needs_sched = false;
7489 break;
7490 }
7491 }
7492
7493 if (needs_sched) {
7494 mutex_unlock(&sqd->lock);
7495 schedule();
7496 mutex_lock(&sqd->lock);
7497 }
Jens Axboe69fb2132020-09-14 11:16:23 -06007498 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list)
7499 io_ring_clear_wakeup_flag(ctx);
Jens Axboe6c271ce2019-01-10 11:22:30 -07007500 }
Xiaoguang Wang08369242020-11-03 14:15:59 +08007501
7502 finish_wait(&sqd->wait, &wait);
7503 timeout = jiffies + sqd->sq_thread_idle;
Jens Axboe6c271ce2019-01-10 11:22:30 -07007504 }
7505
Pavel Begunkov78cc6872021-06-14 02:36:23 +01007506 io_uring_cancel_generic(true, sqd);
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007507 sqd->thread = NULL;
Jens Axboe05962f92021-03-06 13:58:48 -07007508 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list)
Jens Axboe5f3f26f2021-02-25 10:17:46 -07007509 io_ring_set_wakeup_flag(ctx);
Pavel Begunkov521d6a72021-03-11 23:29:38 +00007510 io_run_task_work();
Pavel Begunkov734551d2021-04-18 14:52:09 +01007511 mutex_unlock(&sqd->lock);
7512
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007513 complete(&sqd->exited);
7514 do_exit(0);
Jens Axboe6c271ce2019-01-10 11:22:30 -07007515}
7516
Jens Axboebda52162019-09-24 13:47:15 -06007517struct io_wait_queue {
7518 struct wait_queue_entry wq;
7519 struct io_ring_ctx *ctx;
Jens Axboe5fd46172021-08-06 14:04:31 -06007520 unsigned cq_tail;
Jens Axboebda52162019-09-24 13:47:15 -06007521 unsigned nr_timeouts;
7522};
7523
Pavel Begunkov6c503152021-01-04 20:36:36 +00007524static inline bool io_should_wake(struct io_wait_queue *iowq)
Jens Axboebda52162019-09-24 13:47:15 -06007525{
7526 struct io_ring_ctx *ctx = iowq->ctx;
Jens Axboe5fd46172021-08-06 14:04:31 -06007527 int dist = ctx->cached_cq_tail - (int) iowq->cq_tail;
Jens Axboebda52162019-09-24 13:47:15 -06007528
7529 /*
Brian Gianforcarod195a662019-12-13 03:09:50 -08007530 * Wake up if we have enough events, or if a timeout occurred since we
Jens Axboebda52162019-09-24 13:47:15 -06007531 * started waiting. For timeouts, we always want to return to userspace,
7532 * regardless of event count.
7533 */
Jens Axboe5fd46172021-08-06 14:04:31 -06007534 return dist >= 0 || atomic_read(&ctx->cq_timeouts) != iowq->nr_timeouts;
Jens Axboebda52162019-09-24 13:47:15 -06007535}
7536
7537static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode,
7538 int wake_flags, void *key)
7539{
7540 struct io_wait_queue *iowq = container_of(curr, struct io_wait_queue,
7541 wq);
7542
Pavel Begunkov6c503152021-01-04 20:36:36 +00007543 /*
7544 * Cannot safely flush overflowed CQEs from here, ensure we wake up
7545 * the task, and the next invocation will do it.
7546 */
Pavel Begunkov5ed7a372021-06-14 23:37:27 +01007547 if (io_should_wake(iowq) || test_bit(0, &iowq->ctx->check_cq_overflow))
Pavel Begunkov6c503152021-01-04 20:36:36 +00007548 return autoremove_wake_function(curr, mode, wake_flags, key);
7549 return -1;
Jens Axboebda52162019-09-24 13:47:15 -06007550}
7551
Jens Axboeaf9c1a42020-09-24 13:32:18 -06007552static int io_run_task_work_sig(void)
7553{
7554 if (io_run_task_work())
7555 return 1;
7556 if (!signal_pending(current))
7557 return 0;
Jens Axboe0b8cfa92021-03-21 14:16:08 -06007558 if (test_thread_flag(TIF_NOTIFY_SIGNAL))
Jens Axboe792ee0f62020-10-22 20:17:18 -06007559 return -ERESTARTSYS;
Jens Axboeaf9c1a42020-09-24 13:32:18 -06007560 return -EINTR;
7561}
7562
Pavel Begunkoveeb60b92021-02-04 13:51:58 +00007563/* when returns >0, the caller should retry */
7564static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx,
7565 struct io_wait_queue *iowq,
Jens Axboe7c834372022-02-21 05:49:30 -07007566 ktime_t timeout)
Pavel Begunkoveeb60b92021-02-04 13:51:58 +00007567{
7568 int ret;
7569
7570 /* make sure we run task_work before checking for signals */
7571 ret = io_run_task_work_sig();
7572 if (ret || io_should_wake(iowq))
7573 return ret;
7574 /* let the caller flush overflows, retry */
Pavel Begunkov5ed7a372021-06-14 23:37:27 +01007575 if (test_bit(0, &ctx->check_cq_overflow))
Pavel Begunkoveeb60b92021-02-04 13:51:58 +00007576 return 1;
7577
Jens Axboe7c834372022-02-21 05:49:30 -07007578 if (!schedule_hrtimeout(&timeout, HRTIMER_MODE_ABS))
7579 return -ETIME;
7580 return 1;
Pavel Begunkoveeb60b92021-02-04 13:51:58 +00007581}
7582
Jens Axboe2b188cc2019-01-07 10:46:33 -07007583/*
7584 * Wait until events become available, if we don't already have some. The
7585 * application must reap them itself, as they reside on the shared cq ring.
7586 */
7587static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events,
Hao Xuc73ebb62020-11-03 10:54:37 +08007588 const sigset_t __user *sig, size_t sigsz,
7589 struct __kernel_timespec __user *uts)
Jens Axboe2b188cc2019-01-07 10:46:33 -07007590{
Pavel Begunkov902910992021-08-09 09:07:32 -06007591 struct io_wait_queue iowq;
Hristo Venev75b28af2019-08-26 17:23:46 +00007592 struct io_rings *rings = ctx->rings;
Jens Axboe7c834372022-02-21 05:49:30 -07007593 ktime_t timeout = KTIME_MAX;
Pavel Begunkovc1d5a222021-02-04 13:51:57 +00007594 int ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07007595
Jens Axboeb41e9852020-02-17 09:52:41 -07007596 do {
Pavel Begunkov90f67362021-08-09 20:18:12 +01007597 io_cqring_overflow_flush(ctx);
Pavel Begunkov6c503152021-01-04 20:36:36 +00007598 if (io_cqring_events(ctx) >= min_events)
Jens Axboeb41e9852020-02-17 09:52:41 -07007599 return 0;
Jens Axboe4c6e2772020-07-01 11:29:10 -06007600 if (!io_run_task_work())
Jens Axboeb41e9852020-02-17 09:52:41 -07007601 break;
Jens Axboeb41e9852020-02-17 09:52:41 -07007602 } while (1);
Jens Axboe2b188cc2019-01-07 10:46:33 -07007603
Xiaoguang Wang44df58d2021-09-14 22:38:52 +08007604 if (uts) {
7605 struct timespec64 ts;
7606
7607 if (get_timespec64(&ts, uts))
7608 return -EFAULT;
Jens Axboe7c834372022-02-21 05:49:30 -07007609 timeout = ktime_add_ns(timespec64_to_ktime(ts), ktime_get_ns());
Xiaoguang Wang44df58d2021-09-14 22:38:52 +08007610 }
7611
Jens Axboe2b188cc2019-01-07 10:46:33 -07007612 if (sig) {
Arnd Bergmann9e75ad52019-03-25 15:34:53 +01007613#ifdef CONFIG_COMPAT
7614 if (in_compat_syscall())
7615 ret = set_compat_user_sigmask((const compat_sigset_t __user *)sig,
Oleg Nesterovb7724342019-07-16 16:29:53 -07007616 sigsz);
Arnd Bergmann9e75ad52019-03-25 15:34:53 +01007617 else
7618#endif
Oleg Nesterovb7724342019-07-16 16:29:53 -07007619 ret = set_user_sigmask(sig, sigsz);
Arnd Bergmann9e75ad52019-03-25 15:34:53 +01007620
Jens Axboe2b188cc2019-01-07 10:46:33 -07007621 if (ret)
7622 return ret;
7623 }
7624
Pavel Begunkov902910992021-08-09 09:07:32 -06007625 init_waitqueue_func_entry(&iowq.wq, io_wake_function);
7626 iowq.wq.private = current;
7627 INIT_LIST_HEAD(&iowq.wq.entry);
7628 iowq.ctx = ctx;
Jens Axboebda52162019-09-24 13:47:15 -06007629 iowq.nr_timeouts = atomic_read(&ctx->cq_timeouts);
Jens Axboe5fd46172021-08-06 14:04:31 -06007630 iowq.cq_tail = READ_ONCE(ctx->rings->cq.head) + min_events;
Pavel Begunkov902910992021-08-09 09:07:32 -06007631
Dmitrii Dolgovc826bd72019-10-15 19:02:01 +02007632 trace_io_uring_cqring_wait(ctx, min_events);
Jens Axboebda52162019-09-24 13:47:15 -06007633 do {
Jens Axboeca0a2652021-03-04 17:15:48 -07007634 /* if we can't even flush overflow, don't wait for more */
Pavel Begunkov90f67362021-08-09 20:18:12 +01007635 if (!io_cqring_overflow_flush(ctx)) {
Jens Axboeca0a2652021-03-04 17:15:48 -07007636 ret = -EBUSY;
7637 break;
7638 }
Pavel Begunkov311997b2021-06-14 23:37:28 +01007639 prepare_to_wait_exclusive(&ctx->cq_wait, &iowq.wq,
Jens Axboebda52162019-09-24 13:47:15 -06007640 TASK_INTERRUPTIBLE);
Jens Axboe7c834372022-02-21 05:49:30 -07007641 ret = io_cqring_wait_schedule(ctx, &iowq, timeout);
Pavel Begunkov311997b2021-06-14 23:37:28 +01007642 finish_wait(&ctx->cq_wait, &iowq.wq);
Jens Axboeca0a2652021-03-04 17:15:48 -07007643 cond_resched();
Pavel Begunkoveeb60b92021-02-04 13:51:58 +00007644 } while (ret > 0);
Jens Axboebda52162019-09-24 13:47:15 -06007645
Jens Axboeb7db41c2020-07-04 08:55:50 -06007646 restore_saved_sigmask_unless(ret == -EINTR);
Jens Axboe2b188cc2019-01-07 10:46:33 -07007647
Hristo Venev75b28af2019-08-26 17:23:46 +00007648 return READ_ONCE(rings->cq.head) == READ_ONCE(rings->cq.tail) ? ret : 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07007649}
7650
Pavel Begunkov9123c8f2021-06-14 02:36:20 +01007651static void io_free_page_table(void **table, size_t size)
Pavel Begunkov846a4ef2021-04-01 15:44:03 +01007652{
Pavel Begunkov9123c8f2021-06-14 02:36:20 +01007653 unsigned i, nr_tables = DIV_ROUND_UP(size, PAGE_SIZE);
Pavel Begunkov846a4ef2021-04-01 15:44:03 +01007654
7655 for (i = 0; i < nr_tables; i++)
Pavel Begunkov9123c8f2021-06-14 02:36:20 +01007656 kfree(table[i]);
7657 kfree(table);
7658}
7659
7660static void **io_alloc_page_table(size_t size)
7661{
7662 unsigned i, nr_tables = DIV_ROUND_UP(size, PAGE_SIZE);
7663 size_t init_size = size;
7664 void **table;
7665
Pavel Begunkov0bea96f2021-08-20 10:36:36 +01007666 table = kcalloc(nr_tables, sizeof(*table), GFP_KERNEL_ACCOUNT);
Pavel Begunkov9123c8f2021-06-14 02:36:20 +01007667 if (!table)
7668 return NULL;
7669
7670 for (i = 0; i < nr_tables; i++) {
Pavel Begunkov27f6b312021-06-15 13:20:13 +01007671 unsigned int this_size = min_t(size_t, size, PAGE_SIZE);
Pavel Begunkov9123c8f2021-06-14 02:36:20 +01007672
Pavel Begunkov0bea96f2021-08-20 10:36:36 +01007673 table[i] = kzalloc(this_size, GFP_KERNEL_ACCOUNT);
Pavel Begunkov9123c8f2021-06-14 02:36:20 +01007674 if (!table[i]) {
7675 io_free_page_table(table, init_size);
7676 return NULL;
7677 }
7678 size -= this_size;
7679 }
7680 return table;
Pavel Begunkov846a4ef2021-04-01 15:44:03 +01007681}
7682
Pavel Begunkov28a9fe22021-04-01 15:43:47 +01007683static void io_rsrc_node_destroy(struct io_rsrc_node *ref_node)
7684{
7685 percpu_ref_exit(&ref_node->refs);
7686 kfree(ref_node);
7687}
7688
Pavel Begunkovb9bd2be2021-08-09 09:09:47 -06007689static void io_rsrc_node_ref_zero(struct percpu_ref *ref)
7690{
7691 struct io_rsrc_node *node = container_of(ref, struct io_rsrc_node, refs);
7692 struct io_ring_ctx *ctx = node->rsrc_data->ctx;
7693 unsigned long flags;
7694 bool first_add = false;
Dylan Yudaken82cc3382022-01-21 04:38:56 -08007695 unsigned long delay = HZ;
Pavel Begunkovb9bd2be2021-08-09 09:09:47 -06007696
7697 spin_lock_irqsave(&ctx->rsrc_ref_lock, flags);
7698 node->done = true;
7699
Dylan Yudaken82cc3382022-01-21 04:38:56 -08007700 /* if we are mid-quiesce then do not delay */
7701 if (node->rsrc_data->quiesce)
7702 delay = 0;
7703
Pavel Begunkovb9bd2be2021-08-09 09:09:47 -06007704 while (!list_empty(&ctx->rsrc_ref_list)) {
7705 node = list_first_entry(&ctx->rsrc_ref_list,
7706 struct io_rsrc_node, node);
7707 /* recycle ref nodes in order */
7708 if (!node->done)
7709 break;
7710 list_del(&node->node);
7711 first_add |= llist_add(&node->llist, &ctx->rsrc_put_llist);
7712 }
7713 spin_unlock_irqrestore(&ctx->rsrc_ref_lock, flags);
7714
7715 if (first_add)
Dylan Yudaken82cc3382022-01-21 04:38:56 -08007716 mod_delayed_work(system_wq, &ctx->rsrc_put_work, delay);
Pavel Begunkovb9bd2be2021-08-09 09:09:47 -06007717}
7718
7719static struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx)
7720{
7721 struct io_rsrc_node *ref_node;
7722
7723 ref_node = kzalloc(sizeof(*ref_node), GFP_KERNEL);
7724 if (!ref_node)
7725 return NULL;
7726
7727 if (percpu_ref_init(&ref_node->refs, io_rsrc_node_ref_zero,
7728 0, GFP_KERNEL)) {
7729 kfree(ref_node);
7730 return NULL;
7731 }
7732 INIT_LIST_HEAD(&ref_node->node);
7733 INIT_LIST_HEAD(&ref_node->rsrc_list);
7734 ref_node->done = false;
7735 return ref_node;
7736}
7737
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01007738static void io_rsrc_node_switch(struct io_ring_ctx *ctx,
7739 struct io_rsrc_data *data_to_kill)
Pavel Begunkov1642b442020-12-30 21:34:14 +00007740{
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01007741 WARN_ON_ONCE(!ctx->rsrc_backup_node);
7742 WARN_ON_ONCE(data_to_kill && !ctx->rsrc_node);
Pavel Begunkov82fbcfa2021-04-01 15:43:43 +01007743
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01007744 if (data_to_kill) {
7745 struct io_rsrc_node *rsrc_node = ctx->rsrc_node;
Pavel Begunkov82fbcfa2021-04-01 15:43:43 +01007746
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01007747 rsrc_node->rsrc_data = data_to_kill;
Jens Axboe4956b9e2021-08-09 07:49:41 -06007748 spin_lock_irq(&ctx->rsrc_ref_lock);
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01007749 list_add_tail(&rsrc_node->node, &ctx->rsrc_ref_list);
Jens Axboe4956b9e2021-08-09 07:49:41 -06007750 spin_unlock_irq(&ctx->rsrc_ref_lock);
Pavel Begunkov82fbcfa2021-04-01 15:43:43 +01007751
Pavel Begunkov3e942492021-04-11 01:46:34 +01007752 atomic_inc(&data_to_kill->refs);
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01007753 percpu_ref_kill(&rsrc_node->refs);
7754 ctx->rsrc_node = NULL;
7755 }
7756
7757 if (!ctx->rsrc_node) {
7758 ctx->rsrc_node = ctx->rsrc_backup_node;
7759 ctx->rsrc_backup_node = NULL;
7760 }
Pavel Begunkov1642b442020-12-30 21:34:14 +00007761}
7762
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01007763static int io_rsrc_node_switch_start(struct io_ring_ctx *ctx)
Pavel Begunkov8dd03af2021-03-19 17:22:36 +00007764{
7765 if (ctx->rsrc_backup_node)
7766 return 0;
Pavel Begunkovb895c9a2021-04-01 15:43:40 +01007767 ctx->rsrc_backup_node = io_rsrc_node_alloc(ctx);
Pavel Begunkov8dd03af2021-03-19 17:22:36 +00007768 return ctx->rsrc_backup_node ? 0 : -ENOMEM;
7769}
7770
Pavel Begunkov40ae0ff2021-04-01 15:43:44 +01007771static int io_rsrc_ref_quiesce(struct io_rsrc_data *data, struct io_ring_ctx *ctx)
Hao Xu8bad28d2021-02-19 17:19:36 +08007772{
7773 int ret;
Xiaoguang Wang05589552020-03-31 14:05:18 +08007774
Pavel Begunkov215c3902021-04-01 15:43:48 +01007775 /* As we may drop ->uring_lock, other task may have started quiesce */
Hao Xu8bad28d2021-02-19 17:19:36 +08007776 if (data->quiesce)
7777 return -ENXIO;
7778
7779 data->quiesce = true;
Pavel Begunkov1ffc5422020-12-30 21:34:15 +00007780 do {
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01007781 ret = io_rsrc_node_switch_start(ctx);
Pavel Begunkov8dd03af2021-03-19 17:22:36 +00007782 if (ret)
Pavel Begunkovf2303b12021-02-20 18:03:49 +00007783 break;
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01007784 io_rsrc_node_switch(ctx, data);
7785
Pavel Begunkov3e942492021-04-11 01:46:34 +01007786 /* kill initial ref, already quiesced if zero */
7787 if (atomic_dec_and_test(&data->refs))
7788 break;
Jens Axboec018db42021-08-09 08:15:50 -06007789 mutex_unlock(&ctx->uring_lock);
Hao Xu8bad28d2021-02-19 17:19:36 +08007790 flush_delayed_work(&ctx->rsrc_put_work);
Pavel Begunkov1ffc5422020-12-30 21:34:15 +00007791 ret = wait_for_completion_interruptible(&data->done);
Jens Axboec018db42021-08-09 08:15:50 -06007792 if (!ret) {
7793 mutex_lock(&ctx->uring_lock);
Dylan Yudaken0d773aa2022-02-22 08:17:51 -08007794 if (atomic_read(&data->refs) > 0) {
7795 /*
7796 * it has been revived by another thread while
7797 * we were unlocked
7798 */
7799 mutex_unlock(&ctx->uring_lock);
7800 } else {
7801 break;
7802 }
Jens Axboec018db42021-08-09 08:15:50 -06007803 }
Jens Axboe05f3fb32019-12-09 11:22:50 -07007804
Pavel Begunkov3e942492021-04-11 01:46:34 +01007805 atomic_inc(&data->refs);
7806 /* wait for all works potentially completing data->done */
7807 flush_delayed_work(&ctx->rsrc_put_work);
Jens Axboecb5e1b82021-02-25 07:37:35 -07007808 reinit_completion(&data->done);
Pavel Begunkov8dd03af2021-03-19 17:22:36 +00007809
Hao Xu8bad28d2021-02-19 17:19:36 +08007810 ret = io_run_task_work_sig();
7811 mutex_lock(&ctx->uring_lock);
Pavel Begunkovf2303b12021-02-20 18:03:49 +00007812 } while (ret >= 0);
Hao Xu8bad28d2021-02-19 17:19:36 +08007813 data->quiesce = false;
7814
Hao Xu8bad28d2021-02-19 17:19:36 +08007815 return ret;
Bijan Mottahedehd7954b22021-01-15 17:37:50 +00007816}
7817
Pavel Begunkov2d091d62021-06-14 02:36:21 +01007818static u64 *io_get_tag_slot(struct io_rsrc_data *data, unsigned int idx)
7819{
7820 unsigned int off = idx & IO_RSRC_TAG_TABLE_MASK;
7821 unsigned int table_idx = idx >> IO_RSRC_TAG_TABLE_SHIFT;
7822
7823 return &data->tags[table_idx][off];
7824}
7825
Pavel Begunkov44b31f22021-04-25 14:32:16 +01007826static void io_rsrc_data_free(struct io_rsrc_data *data)
7827{
Pavel Begunkov2d091d62021-06-14 02:36:21 +01007828 size_t size = data->nr * sizeof(data->tags[0][0]);
7829
7830 if (data->tags)
7831 io_free_page_table((void **)data->tags, size);
Pavel Begunkov44b31f22021-04-25 14:32:16 +01007832 kfree(data);
7833}
7834
Pavel Begunkovd878c812021-06-14 02:36:18 +01007835static int io_rsrc_data_alloc(struct io_ring_ctx *ctx, rsrc_put_fn *do_put,
7836 u64 __user *utags, unsigned nr,
7837 struct io_rsrc_data **pdata)
Bijan Mottahedeh1ad555c2021-01-15 17:37:51 +00007838{
Pavel Begunkovb895c9a2021-04-01 15:43:40 +01007839 struct io_rsrc_data *data;
Pavel Begunkov2d091d62021-06-14 02:36:21 +01007840 int ret = -ENOMEM;
Pavel Begunkovd878c812021-06-14 02:36:18 +01007841 unsigned i;
Bijan Mottahedeh1ad555c2021-01-15 17:37:51 +00007842
7843 data = kzalloc(sizeof(*data), GFP_KERNEL);
7844 if (!data)
Pavel Begunkovd878c812021-06-14 02:36:18 +01007845 return -ENOMEM;
Pavel Begunkov2d091d62021-06-14 02:36:21 +01007846 data->tags = (u64 **)io_alloc_page_table(nr * sizeof(data->tags[0][0]));
Pavel Begunkovb60c8dc2021-04-25 14:32:18 +01007847 if (!data->tags) {
7848 kfree(data);
Pavel Begunkovd878c812021-06-14 02:36:18 +01007849 return -ENOMEM;
7850 }
Pavel Begunkov2d091d62021-06-14 02:36:21 +01007851
7852 data->nr = nr;
7853 data->ctx = ctx;
7854 data->do_put = do_put;
Pavel Begunkovd878c812021-06-14 02:36:18 +01007855 if (utags) {
Pavel Begunkov2d091d62021-06-14 02:36:21 +01007856 ret = -EFAULT;
Pavel Begunkovd878c812021-06-14 02:36:18 +01007857 for (i = 0; i < nr; i++) {
Colin Ian Kingfdd1dc32021-06-15 14:00:11 +01007858 u64 *tag_slot = io_get_tag_slot(data, i);
7859
7860 if (copy_from_user(tag_slot, &utags[i],
7861 sizeof(*tag_slot)))
Pavel Begunkov2d091d62021-06-14 02:36:21 +01007862 goto fail;
Pavel Begunkovd878c812021-06-14 02:36:18 +01007863 }
Pavel Begunkovb60c8dc2021-04-25 14:32:18 +01007864 }
7865
Pavel Begunkov3e942492021-04-11 01:46:34 +01007866 atomic_set(&data->refs, 1);
Bijan Mottahedeh1ad555c2021-01-15 17:37:51 +00007867 init_completion(&data->done);
Pavel Begunkovd878c812021-06-14 02:36:18 +01007868 *pdata = data;
7869 return 0;
Pavel Begunkov2d091d62021-06-14 02:36:21 +01007870fail:
7871 io_rsrc_data_free(data);
7872 return ret;
Bijan Mottahedeh1ad555c2021-01-15 17:37:51 +00007873}
7874
Pavel Begunkov9123c8f2021-06-14 02:36:20 +01007875static bool io_alloc_file_tables(struct io_file_table *table, unsigned nr_files)
7876{
Pavel Begunkov0bea96f2021-08-20 10:36:36 +01007877 table->files = kvcalloc(nr_files, sizeof(table->files[0]),
7878 GFP_KERNEL_ACCOUNT);
Pavel Begunkov9123c8f2021-06-14 02:36:20 +01007879 return !!table->files;
7880}
7881
Pavel Begunkov042b0d82021-08-09 13:04:01 +01007882static void io_free_file_tables(struct io_file_table *table)
Pavel Begunkov9123c8f2021-06-14 02:36:20 +01007883{
Pavel Begunkov042b0d82021-08-09 13:04:01 +01007884 kvfree(table->files);
Pavel Begunkov9123c8f2021-06-14 02:36:20 +01007885 table->files = NULL;
7886}
7887
Jens Axboe2b188cc2019-01-07 10:46:33 -07007888static void __io_sqe_files_unregister(struct io_ring_ctx *ctx)
7889{
7890#if defined(CONFIG_UNIX)
7891 if (ctx->ring_sock) {
7892 struct sock *sock = ctx->ring_sock->sk;
7893 struct sk_buff *skb;
7894
7895 while ((skb = skb_dequeue(&sock->sk_receive_queue)) != NULL)
7896 kfree_skb(skb);
7897 }
7898#else
7899 int i;
7900
7901 for (i = 0; i < ctx->nr_user_files; i++) {
7902 struct file *file;
7903
7904 file = io_file_from_index(ctx, i);
7905 if (file)
7906 fput(file);
7907 }
7908#endif
Pavel Begunkov042b0d82021-08-09 13:04:01 +01007909 io_free_file_tables(&ctx->file_table);
Pavel Begunkov44b31f22021-04-25 14:32:16 +01007910 io_rsrc_data_free(ctx->file_data);
Pavel Begunkovfff4db72021-04-25 14:32:15 +01007911 ctx->file_data = NULL;
7912 ctx->nr_user_files = 0;
Bijan Mottahedeh1ad555c2021-01-15 17:37:51 +00007913}
7914
Bijan Mottahedehd7954b22021-01-15 17:37:50 +00007915static int io_sqe_files_unregister(struct io_ring_ctx *ctx)
7916{
Pavel Begunkovb1e7cad2022-06-13 06:32:44 +01007917 unsigned nr = ctx->nr_user_files;
Bijan Mottahedehd7954b22021-01-15 17:37:50 +00007918 int ret;
7919
Pavel Begunkov08480402021-04-13 02:58:38 +01007920 if (!ctx->file_data)
Bijan Mottahedehd7954b22021-01-15 17:37:50 +00007921 return -ENXIO;
Pavel Begunkovb1e7cad2022-06-13 06:32:44 +01007922
7923 /*
7924 * Quiesce may unlock ->uring_lock, and while it's not held
7925 * prevent new requests using the table.
7926 */
7927 ctx->nr_user_files = 0;
Pavel Begunkov08480402021-04-13 02:58:38 +01007928 ret = io_rsrc_ref_quiesce(ctx->file_data, ctx);
Pavel Begunkovb1e7cad2022-06-13 06:32:44 +01007929 ctx->nr_user_files = nr;
Pavel Begunkov08480402021-04-13 02:58:38 +01007930 if (!ret)
7931 __io_sqe_files_unregister(ctx);
7932 return ret;
Jens Axboe6b063142019-01-10 22:13:58 -07007933}
7934
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007935static void io_sq_thread_unpark(struct io_sq_data *sqd)
Pavel Begunkov09a6f4e2021-03-14 20:57:10 +00007936 __releases(&sqd->lock)
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007937{
Pavel Begunkov521d6a72021-03-11 23:29:38 +00007938 WARN_ON_ONCE(sqd->thread == current);
7939
Pavel Begunkov9e138a42021-03-14 20:57:12 +00007940 /*
7941 * Do the dance but not conditional clear_bit() because it'd race with
7942 * other threads incrementing park_pending and setting the bit.
7943 */
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007944 clear_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state);
Pavel Begunkov9e138a42021-03-14 20:57:12 +00007945 if (atomic_dec_return(&sqd->park_pending))
7946 set_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state);
Pavel Begunkov09a6f4e2021-03-14 20:57:10 +00007947 mutex_unlock(&sqd->lock);
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007948}
7949
Jens Axboe86e0d672021-03-05 08:44:39 -07007950static void io_sq_thread_park(struct io_sq_data *sqd)
Pavel Begunkov09a6f4e2021-03-14 20:57:10 +00007951 __acquires(&sqd->lock)
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007952{
Pavel Begunkov521d6a72021-03-11 23:29:38 +00007953 WARN_ON_ONCE(sqd->thread == current);
7954
Pavel Begunkov9e138a42021-03-14 20:57:12 +00007955 atomic_inc(&sqd->park_pending);
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007956 set_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state);
Pavel Begunkov09a6f4e2021-03-14 20:57:10 +00007957 mutex_lock(&sqd->lock);
Jens Axboe05962f92021-03-06 13:58:48 -07007958 if (sqd->thread)
Jens Axboe86e0d672021-03-05 08:44:39 -07007959 wake_up_process(sqd->thread);
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007960}
7961
7962static void io_sq_thread_stop(struct io_sq_data *sqd)
7963{
Pavel Begunkov521d6a72021-03-11 23:29:38 +00007964 WARN_ON_ONCE(sqd->thread == current);
Pavel Begunkov88885f62021-04-11 01:46:38 +01007965 WARN_ON_ONCE(test_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state));
Pavel Begunkov521d6a72021-03-11 23:29:38 +00007966
Jens Axboe05962f92021-03-06 13:58:48 -07007967 set_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state);
Pavel Begunkov88885f62021-04-11 01:46:38 +01007968 mutex_lock(&sqd->lock);
Jens Axboee8f98f242021-03-09 16:32:13 -07007969 if (sqd->thread)
7970 wake_up_process(sqd->thread);
Pavel Begunkov09a6f4e2021-03-14 20:57:10 +00007971 mutex_unlock(&sqd->lock);
Jens Axboe05962f92021-03-06 13:58:48 -07007972 wait_for_completion(&sqd->exited);
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007973}
7974
Jens Axboe534ca6d2020-09-02 13:52:19 -06007975static void io_put_sq_data(struct io_sq_data *sqd)
Jens Axboe6c271ce2019-01-10 11:22:30 -07007976{
Jens Axboe534ca6d2020-09-02 13:52:19 -06007977 if (refcount_dec_and_test(&sqd->refs)) {
Pavel Begunkov9e138a42021-03-14 20:57:12 +00007978 WARN_ON_ONCE(atomic_read(&sqd->park_pending));
7979
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007980 io_sq_thread_stop(sqd);
7981 kfree(sqd);
7982 }
7983}
7984
7985static void io_sq_thread_finish(struct io_ring_ctx *ctx)
7986{
7987 struct io_sq_data *sqd = ctx->sq_data;
7988
7989 if (sqd) {
Jens Axboe05962f92021-03-06 13:58:48 -07007990 io_sq_thread_park(sqd);
Pavel Begunkov521d6a72021-03-11 23:29:38 +00007991 list_del_init(&ctx->sqd_list);
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007992 io_sqd_update_thread_idle(sqd);
Jens Axboe05962f92021-03-06 13:58:48 -07007993 io_sq_thread_unpark(sqd);
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007994
7995 io_put_sq_data(sqd);
7996 ctx->sq_data = NULL;
Jens Axboe534ca6d2020-09-02 13:52:19 -06007997 }
7998}
7999
Jens Axboeaa061652020-09-02 14:50:27 -06008000static struct io_sq_data *io_attach_sq_data(struct io_uring_params *p)
8001{
8002 struct io_ring_ctx *ctx_attach;
8003 struct io_sq_data *sqd;
8004 struct fd f;
8005
8006 f = fdget(p->wq_fd);
8007 if (!f.file)
8008 return ERR_PTR(-ENXIO);
8009 if (f.file->f_op != &io_uring_fops) {
8010 fdput(f);
8011 return ERR_PTR(-EINVAL);
8012 }
8013
8014 ctx_attach = f.file->private_data;
8015 sqd = ctx_attach->sq_data;
8016 if (!sqd) {
8017 fdput(f);
8018 return ERR_PTR(-EINVAL);
8019 }
Jens Axboe5c2469e2021-03-11 10:17:56 -07008020 if (sqd->task_tgid != current->tgid) {
8021 fdput(f);
8022 return ERR_PTR(-EPERM);
8023 }
Jens Axboeaa061652020-09-02 14:50:27 -06008024
8025 refcount_inc(&sqd->refs);
8026 fdput(f);
8027 return sqd;
8028}
8029
Pavel Begunkov26984fb2021-03-11 23:29:37 +00008030static struct io_sq_data *io_get_sq_data(struct io_uring_params *p,
8031 bool *attached)
Jens Axboe534ca6d2020-09-02 13:52:19 -06008032{
8033 struct io_sq_data *sqd;
8034
Pavel Begunkov26984fb2021-03-11 23:29:37 +00008035 *attached = false;
Jens Axboe5c2469e2021-03-11 10:17:56 -07008036 if (p->flags & IORING_SETUP_ATTACH_WQ) {
8037 sqd = io_attach_sq_data(p);
Pavel Begunkov26984fb2021-03-11 23:29:37 +00008038 if (!IS_ERR(sqd)) {
8039 *attached = true;
Jens Axboe5c2469e2021-03-11 10:17:56 -07008040 return sqd;
Pavel Begunkov26984fb2021-03-11 23:29:37 +00008041 }
Jens Axboe5c2469e2021-03-11 10:17:56 -07008042 /* fall through for EPERM case, setup new sqd/task */
8043 if (PTR_ERR(sqd) != -EPERM)
8044 return sqd;
8045 }
Jens Axboeaa061652020-09-02 14:50:27 -06008046
Jens Axboe534ca6d2020-09-02 13:52:19 -06008047 sqd = kzalloc(sizeof(*sqd), GFP_KERNEL);
8048 if (!sqd)
8049 return ERR_PTR(-ENOMEM);
8050
Pavel Begunkov9e138a42021-03-14 20:57:12 +00008051 atomic_set(&sqd->park_pending, 0);
Jens Axboe534ca6d2020-09-02 13:52:19 -06008052 refcount_set(&sqd->refs, 1);
Jens Axboe69fb2132020-09-14 11:16:23 -06008053 INIT_LIST_HEAD(&sqd->ctx_list);
Pavel Begunkov09a6f4e2021-03-14 20:57:10 +00008054 mutex_init(&sqd->lock);
Jens Axboe534ca6d2020-09-02 13:52:19 -06008055 init_waitqueue_head(&sqd->wait);
Jens Axboe37d1e2e2021-02-17 21:03:43 -07008056 init_completion(&sqd->exited);
Jens Axboe534ca6d2020-09-02 13:52:19 -06008057 return sqd;
8058}
8059
Jens Axboe6b063142019-01-10 22:13:58 -07008060#if defined(CONFIG_UNIX)
Jens Axboe6b063142019-01-10 22:13:58 -07008061/*
8062 * Ensure the UNIX gc is aware of our file set, so we are certain that
8063 * the io_uring can be safely unregistered on process exit, even if we have
8064 * loops in the file referencing.
8065 */
8066static int __io_sqe_files_scm(struct io_ring_ctx *ctx, int nr, int offset)
8067{
8068 struct sock *sk = ctx->ring_sock->sk;
8069 struct scm_fp_list *fpl;
8070 struct sk_buff *skb;
Jens Axboe08a45172019-10-03 08:11:03 -06008071 int i, nr_files;
Jens Axboe6b063142019-01-10 22:13:58 -07008072
Jens Axboe6b063142019-01-10 22:13:58 -07008073 fpl = kzalloc(sizeof(*fpl), GFP_KERNEL);
8074 if (!fpl)
8075 return -ENOMEM;
8076
8077 skb = alloc_skb(0, GFP_KERNEL);
8078 if (!skb) {
8079 kfree(fpl);
8080 return -ENOMEM;
8081 }
8082
8083 skb->sk = sk;
Pavel Begunkov813d8fe2022-10-16 22:42:54 +01008084 skb->scm_io_uring = 1;
Jens Axboe6b063142019-01-10 22:13:58 -07008085
Jens Axboe08a45172019-10-03 08:11:03 -06008086 nr_files = 0;
Jens Axboe62e398b2021-02-21 16:19:37 -07008087 fpl->user = get_uid(current_user());
Jens Axboe6b063142019-01-10 22:13:58 -07008088 for (i = 0; i < nr; i++) {
Jens Axboe65e19f52019-10-26 07:20:21 -06008089 struct file *file = io_file_from_index(ctx, i + offset);
8090
8091 if (!file)
Jens Axboe08a45172019-10-03 08:11:03 -06008092 continue;
Jens Axboe65e19f52019-10-26 07:20:21 -06008093 fpl->fp[nr_files] = get_file(file);
Jens Axboe08a45172019-10-03 08:11:03 -06008094 unix_inflight(fpl->user, fpl->fp[nr_files]);
8095 nr_files++;
Jens Axboe6b063142019-01-10 22:13:58 -07008096 }
8097
Jens Axboe08a45172019-10-03 08:11:03 -06008098 if (nr_files) {
8099 fpl->max = SCM_MAX_FD;
8100 fpl->count = nr_files;
8101 UNIXCB(skb).fp = fpl;
Jens Axboe05f3fb32019-12-09 11:22:50 -07008102 skb->destructor = unix_destruct_scm;
Jens Axboe08a45172019-10-03 08:11:03 -06008103 refcount_add(skb->truesize, &sk->sk_wmem_alloc);
8104 skb_queue_head(&sk->sk_receive_queue, skb);
Jens Axboe6b063142019-01-10 22:13:58 -07008105
Pavel Begunkov285f5d72022-04-06 12:43:58 +01008106 for (i = 0; i < nr; i++) {
8107 struct file *file = io_file_from_index(ctx, i + offset);
8108
8109 if (file)
8110 fput(file);
8111 }
Jens Axboe08a45172019-10-03 08:11:03 -06008112 } else {
8113 kfree_skb(skb);
Pavel Begunkov0853bd62022-03-25 16:36:31 +00008114 free_uid(fpl->user);
Jens Axboe08a45172019-10-03 08:11:03 -06008115 kfree(fpl);
8116 }
Jens Axboe6b063142019-01-10 22:13:58 -07008117
8118 return 0;
8119}
8120
8121/*
8122 * If UNIX sockets are enabled, fd passing can cause a reference cycle which
8123 * causes regular reference counting to break down. We rely on the UNIX
8124 * garbage collection to take care of this problem for us.
8125 */
8126static int io_sqe_files_scm(struct io_ring_ctx *ctx)
8127{
8128 unsigned left, total;
8129 int ret = 0;
8130
8131 total = 0;
8132 left = ctx->nr_user_files;
8133 while (left) {
8134 unsigned this_files = min_t(unsigned, left, SCM_MAX_FD);
Jens Axboe6b063142019-01-10 22:13:58 -07008135
8136 ret = __io_sqe_files_scm(ctx, this_files, total);
8137 if (ret)
8138 break;
8139 left -= this_files;
8140 total += this_files;
8141 }
8142
8143 if (!ret)
8144 return 0;
8145
8146 while (total < ctx->nr_user_files) {
Jens Axboe65e19f52019-10-26 07:20:21 -06008147 struct file *file = io_file_from_index(ctx, total);
8148
8149 if (file)
8150 fput(file);
Jens Axboe6b063142019-01-10 22:13:58 -07008151 total++;
8152 }
8153
8154 return ret;
8155}
8156#else
8157static int io_sqe_files_scm(struct io_ring_ctx *ctx)
8158{
8159 return 0;
8160}
8161#endif
8162
Pavel Begunkov47e90392021-04-01 15:43:56 +01008163static void io_rsrc_file_put(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc)
Jens Axboec3a31e62019-10-03 13:59:56 -06008164{
Bijan Mottahedeh50238532021-01-15 17:37:45 +00008165 struct file *file = prsrc->file;
Jens Axboec3a31e62019-10-03 13:59:56 -06008166#if defined(CONFIG_UNIX)
Jens Axboec3a31e62019-10-03 13:59:56 -06008167 struct sock *sock = ctx->ring_sock->sk;
8168 struct sk_buff_head list, *head = &sock->sk_receive_queue;
8169 struct sk_buff *skb;
8170 int i;
8171
8172 __skb_queue_head_init(&list);
8173
8174 /*
8175 * Find the skb that holds this file in its SCM_RIGHTS. When found,
8176 * remove this entry and rearrange the file array.
8177 */
8178 skb = skb_dequeue(head);
8179 while (skb) {
8180 struct scm_fp_list *fp;
8181
8182 fp = UNIXCB(skb).fp;
8183 for (i = 0; i < fp->count; i++) {
8184 int left;
8185
8186 if (fp->fp[i] != file)
8187 continue;
8188
8189 unix_notinflight(fp->user, fp->fp[i]);
8190 left = fp->count - 1 - i;
8191 if (left) {
8192 memmove(&fp->fp[i], &fp->fp[i + 1],
8193 left * sizeof(struct file *));
8194 }
8195 fp->count--;
8196 if (!fp->count) {
8197 kfree_skb(skb);
8198 skb = NULL;
8199 } else {
8200 __skb_queue_tail(&list, skb);
8201 }
8202 fput(file);
8203 file = NULL;
8204 break;
8205 }
8206
8207 if (!file)
8208 break;
8209
8210 __skb_queue_tail(&list, skb);
8211
8212 skb = skb_dequeue(head);
8213 }
8214
8215 if (skb_peek(&list)) {
8216 spin_lock_irq(&head->lock);
8217 while ((skb = __skb_dequeue(&list)) != NULL)
8218 __skb_queue_tail(head, skb);
8219 spin_unlock_irq(&head->lock);
8220 }
8221#else
Jens Axboe05f3fb32019-12-09 11:22:50 -07008222 fput(file);
Jens Axboec3a31e62019-10-03 13:59:56 -06008223#endif
8224}
8225
Pavel Begunkovb895c9a2021-04-01 15:43:40 +01008226static void __io_rsrc_put_work(struct io_rsrc_node *ref_node)
Jens Axboe05f3fb32019-12-09 11:22:50 -07008227{
Pavel Begunkovb895c9a2021-04-01 15:43:40 +01008228 struct io_rsrc_data *rsrc_data = ref_node->rsrc_data;
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00008229 struct io_ring_ctx *ctx = rsrc_data->ctx;
8230 struct io_rsrc_put *prsrc, *tmp;
Xiaoguang Wang05589552020-03-31 14:05:18 +08008231
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00008232 list_for_each_entry_safe(prsrc, tmp, &ref_node->rsrc_list, list) {
8233 list_del(&prsrc->list);
Pavel Begunkovb60c8dc2021-04-25 14:32:18 +01008234
8235 if (prsrc->tag) {
8236 bool lock_ring = ctx->flags & IORING_SETUP_IOPOLL;
Pavel Begunkovb60c8dc2021-04-25 14:32:18 +01008237
8238 io_ring_submit_lock(ctx, lock_ring);
Jens Axboe79ebeae2021-08-10 15:18:27 -06008239 spin_lock(&ctx->completion_lock);
Pavel Begunkovb850d6d2022-08-29 14:30:13 +01008240 io_fill_cqe_aux(ctx, prsrc->tag, 0, 0);
Pavel Begunkovb60c8dc2021-04-25 14:32:18 +01008241 io_commit_cqring(ctx);
Jens Axboe79ebeae2021-08-10 15:18:27 -06008242 spin_unlock(&ctx->completion_lock);
Pavel Begunkovb60c8dc2021-04-25 14:32:18 +01008243 io_cqring_ev_posted(ctx);
8244 io_ring_submit_unlock(ctx, lock_ring);
8245 }
8246
Pavel Begunkov40ae0ff2021-04-01 15:43:44 +01008247 rsrc_data->do_put(ctx, prsrc);
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00008248 kfree(prsrc);
Jens Axboe05f3fb32019-12-09 11:22:50 -07008249 }
8250
Pavel Begunkov28a9fe22021-04-01 15:43:47 +01008251 io_rsrc_node_destroy(ref_node);
Pavel Begunkov3e942492021-04-11 01:46:34 +01008252 if (atomic_dec_and_test(&rsrc_data->refs))
8253 complete(&rsrc_data->done);
Jens Axboe05f3fb32019-12-09 11:22:50 -07008254}
8255
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00008256static void io_rsrc_put_work(struct work_struct *work)
Jens Axboe4a38aed22020-05-14 17:21:15 -06008257{
8258 struct io_ring_ctx *ctx;
8259 struct llist_node *node;
8260
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00008261 ctx = container_of(work, struct io_ring_ctx, rsrc_put_work.work);
8262 node = llist_del_all(&ctx->rsrc_put_llist);
Jens Axboe4a38aed22020-05-14 17:21:15 -06008263
8264 while (node) {
Pavel Begunkovb895c9a2021-04-01 15:43:40 +01008265 struct io_rsrc_node *ref_node;
Jens Axboe4a38aed22020-05-14 17:21:15 -06008266 struct llist_node *next = node->next;
8267
Pavel Begunkovb895c9a2021-04-01 15:43:40 +01008268 ref_node = llist_entry(node, struct io_rsrc_node, llist);
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00008269 __io_rsrc_put_work(ref_node);
Jens Axboe4a38aed22020-05-14 17:21:15 -06008270 node = next;
8271 }
8272}
8273
Jens Axboe05f3fb32019-12-09 11:22:50 -07008274static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg,
Pavel Begunkov792e3582021-04-25 14:32:21 +01008275 unsigned nr_args, u64 __user *tags)
Jens Axboe05f3fb32019-12-09 11:22:50 -07008276{
8277 __s32 __user *fds = (__s32 __user *) arg;
Jens Axboe05f3fb32019-12-09 11:22:50 -07008278 struct file *file;
Pavel Begunkovf3baed32021-04-01 15:43:42 +01008279 int fd, ret;
Pavel Begunkov846a4ef2021-04-01 15:44:03 +01008280 unsigned i;
Jens Axboe05f3fb32019-12-09 11:22:50 -07008281
8282 if (ctx->file_data)
8283 return -EBUSY;
8284 if (!nr_args)
8285 return -EINVAL;
8286 if (nr_args > IORING_MAX_FIXED_FILES)
8287 return -EMFILE;
Pavel Begunkov3a1b8a42021-08-20 10:36:35 +01008288 if (nr_args > rlimit(RLIMIT_NOFILE))
8289 return -EMFILE;
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01008290 ret = io_rsrc_node_switch_start(ctx);
Pavel Begunkovf3baed32021-04-01 15:43:42 +01008291 if (ret)
8292 return ret;
Pavel Begunkovd878c812021-06-14 02:36:18 +01008293 ret = io_rsrc_data_alloc(ctx, io_rsrc_file_put, tags, nr_args,
8294 &ctx->file_data);
8295 if (ret)
8296 return ret;
Jens Axboe05f3fb32019-12-09 11:22:50 -07008297
Pavel Begunkovf3baed32021-04-01 15:43:42 +01008298 ret = -ENOMEM;
Pavel Begunkovaeca2412021-04-11 01:46:37 +01008299 if (!io_alloc_file_tables(&ctx->file_table, nr_args))
Pavel Begunkov600cf3f2020-10-10 18:34:15 +01008300 goto out_free;
Jens Axboe05f3fb32019-12-09 11:22:50 -07008301
Jens Axboe05f3fb32019-12-09 11:22:50 -07008302 for (i = 0; i < nr_args; i++, ctx->nr_user_files++) {
Pavel Begunkovd878c812021-06-14 02:36:18 +01008303 if (copy_from_user(&fd, &fds[i], sizeof(fd))) {
Pavel Begunkov600cf3f2020-10-10 18:34:15 +01008304 ret = -EFAULT;
8305 goto out_fput;
Jens Axboe05f3fb32019-12-09 11:22:50 -07008306 }
Pavel Begunkov600cf3f2020-10-10 18:34:15 +01008307 /* allow sparse sets */
Pavel Begunkov792e3582021-04-25 14:32:21 +01008308 if (fd == -1) {
8309 ret = -EINVAL;
Pavel Begunkov2d091d62021-06-14 02:36:21 +01008310 if (unlikely(*io_get_tag_slot(ctx->file_data, i)))
Pavel Begunkov792e3582021-04-25 14:32:21 +01008311 goto out_fput;
Pavel Begunkov600cf3f2020-10-10 18:34:15 +01008312 continue;
Pavel Begunkov792e3582021-04-25 14:32:21 +01008313 }
Jens Axboe05f3fb32019-12-09 11:22:50 -07008314
Jens Axboe05f3fb32019-12-09 11:22:50 -07008315 file = fget(fd);
Jens Axboe05f3fb32019-12-09 11:22:50 -07008316 ret = -EBADF;
Pavel Begunkov792e3582021-04-25 14:32:21 +01008317 if (unlikely(!file))
Pavel Begunkov600cf3f2020-10-10 18:34:15 +01008318 goto out_fput;
Jens Axboe05f3fb32019-12-09 11:22:50 -07008319
8320 /*
8321 * Don't allow io_uring instances to be registered. If UNIX
8322 * isn't enabled, then this causes a reference cycle and this
8323 * instance can never get freed. If UNIX is enabled we'll
8324 * handle it just fine, but there's still no point in allowing
8325 * a ring fd as it doesn't support regular read/write anyway.
8326 */
8327 if (file->f_op == &io_uring_fops) {
8328 fput(file);
Pavel Begunkov600cf3f2020-10-10 18:34:15 +01008329 goto out_fput;
Jens Axboe05f3fb32019-12-09 11:22:50 -07008330 }
Pavel Begunkovaeca2412021-04-11 01:46:37 +01008331 io_fixed_file_set(io_fixed_file_slot(&ctx->file_table, i), file);
Jens Axboe05f3fb32019-12-09 11:22:50 -07008332 }
8333
Jens Axboe05f3fb32019-12-09 11:22:50 -07008334 ret = io_sqe_files_scm(ctx);
Xiaoguang Wang05589552020-03-31 14:05:18 +08008335 if (ret) {
Pavel Begunkov08480402021-04-13 02:58:38 +01008336 __io_sqe_files_unregister(ctx);
Xiaoguang Wang05589552020-03-31 14:05:18 +08008337 return ret;
8338 }
Jens Axboe05f3fb32019-12-09 11:22:50 -07008339
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01008340 io_rsrc_node_switch(ctx, NULL);
Jens Axboe05f3fb32019-12-09 11:22:50 -07008341 return ret;
Pavel Begunkov600cf3f2020-10-10 18:34:15 +01008342out_fput:
8343 for (i = 0; i < ctx->nr_user_files; i++) {
8344 file = io_file_from_index(ctx, i);
8345 if (file)
8346 fput(file);
8347 }
Pavel Begunkov042b0d82021-08-09 13:04:01 +01008348 io_free_file_tables(&ctx->file_table);
Pavel Begunkov600cf3f2020-10-10 18:34:15 +01008349 ctx->nr_user_files = 0;
Pavel Begunkov600cf3f2020-10-10 18:34:15 +01008350out_free:
Pavel Begunkov44b31f22021-04-25 14:32:16 +01008351 io_rsrc_data_free(ctx->file_data);
Jens Axboe55cbc252020-10-14 07:35:57 -06008352 ctx->file_data = NULL;
Jens Axboe05f3fb32019-12-09 11:22:50 -07008353 return ret;
8354}
8355
Jens Axboec3a31e62019-10-03 13:59:56 -06008356static int io_sqe_file_register(struct io_ring_ctx *ctx, struct file *file,
8357 int index)
8358{
8359#if defined(CONFIG_UNIX)
8360 struct sock *sock = ctx->ring_sock->sk;
8361 struct sk_buff_head *head = &sock->sk_receive_queue;
8362 struct sk_buff *skb;
8363
8364 /*
8365 * See if we can merge this file into an existing skb SCM_RIGHTS
8366 * file set. If there's no room, fall back to allocating a new skb
8367 * and filling it in.
8368 */
8369 spin_lock_irq(&head->lock);
8370 skb = skb_peek(head);
8371 if (skb) {
8372 struct scm_fp_list *fpl = UNIXCB(skb).fp;
8373
8374 if (fpl->count < SCM_MAX_FD) {
8375 __skb_unlink(skb, head);
8376 spin_unlock_irq(&head->lock);
8377 fpl->fp[fpl->count] = get_file(file);
8378 unix_inflight(fpl->user, fpl->fp[fpl->count]);
8379 fpl->count++;
8380 spin_lock_irq(&head->lock);
8381 __skb_queue_head(head, skb);
8382 } else {
8383 skb = NULL;
8384 }
8385 }
8386 spin_unlock_irq(&head->lock);
8387
8388 if (skb) {
8389 fput(file);
8390 return 0;
8391 }
8392
8393 return __io_sqe_files_scm(ctx, 1, index);
8394#else
8395 return 0;
8396#endif
8397}
8398
Pavel Begunkov9c7b0ba2021-09-14 16:12:52 +01008399static int io_queue_rsrc_removal(struct io_rsrc_data *data, unsigned idx,
8400 struct io_rsrc_node *node, void *rsrc)
8401{
Pavel Begunkov5218d5c2022-04-07 14:05:04 +01008402 u64 *tag_slot = io_get_tag_slot(data, idx);
Pavel Begunkov9c7b0ba2021-09-14 16:12:52 +01008403 struct io_rsrc_put *prsrc;
8404
8405 prsrc = kzalloc(sizeof(*prsrc), GFP_KERNEL);
8406 if (!prsrc)
8407 return -ENOMEM;
8408
Pavel Begunkov5218d5c2022-04-07 14:05:04 +01008409 prsrc->tag = *tag_slot;
8410 *tag_slot = 0;
Pavel Begunkov9c7b0ba2021-09-14 16:12:52 +01008411 prsrc->rsrc = rsrc;
8412 list_add(&prsrc->list, &node->rsrc_list);
8413 return 0;
8414}
8415
Pavel Begunkovb9445592021-08-25 12:25:45 +01008416static int io_install_fixed_file(struct io_kiocb *req, struct file *file,
8417 unsigned int issue_flags, u32 slot_index)
8418{
8419 struct io_ring_ctx *ctx = req->ctx;
8420 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
Pavel Begunkov9c7b0ba2021-09-14 16:12:52 +01008421 bool needs_switch = false;
Pavel Begunkovb9445592021-08-25 12:25:45 +01008422 struct io_fixed_file *file_slot;
8423 int ret = -EBADF;
8424
8425 io_ring_submit_lock(ctx, !force_nonblock);
8426 if (file->f_op == &io_uring_fops)
8427 goto err;
8428 ret = -ENXIO;
8429 if (!ctx->file_data)
8430 goto err;
8431 ret = -EINVAL;
8432 if (slot_index >= ctx->nr_user_files)
8433 goto err;
8434
8435 slot_index = array_index_nospec(slot_index, ctx->nr_user_files);
8436 file_slot = io_fixed_file_slot(&ctx->file_table, slot_index);
Pavel Begunkov9c7b0ba2021-09-14 16:12:52 +01008437
8438 if (file_slot->file_ptr) {
8439 struct file *old_file;
8440
8441 ret = io_rsrc_node_switch_start(ctx);
8442 if (ret)
8443 goto err;
8444
8445 old_file = (struct file *)(file_slot->file_ptr & FFS_MASK);
8446 ret = io_queue_rsrc_removal(ctx->file_data, slot_index,
8447 ctx->rsrc_node, old_file);
8448 if (ret)
8449 goto err;
8450 file_slot->file_ptr = 0;
8451 needs_switch = true;
8452 }
Pavel Begunkovb9445592021-08-25 12:25:45 +01008453
8454 *io_get_tag_slot(ctx->file_data, slot_index) = 0;
8455 io_fixed_file_set(file_slot, file);
8456 ret = io_sqe_file_register(ctx, file, slot_index);
8457 if (ret) {
8458 file_slot->file_ptr = 0;
8459 goto err;
8460 }
8461
8462 ret = 0;
8463err:
Pavel Begunkov9c7b0ba2021-09-14 16:12:52 +01008464 if (needs_switch)
8465 io_rsrc_node_switch(ctx, ctx->file_data);
Pavel Begunkovb9445592021-08-25 12:25:45 +01008466 io_ring_submit_unlock(ctx, !force_nonblock);
8467 if (ret)
8468 fput(file);
8469 return ret;
8470}
8471
Pavel Begunkov7df778b2021-09-24 20:04:29 +01008472static int io_close_fixed(struct io_kiocb *req, unsigned int issue_flags)
8473{
8474 unsigned int offset = req->close.file_slot - 1;
8475 struct io_ring_ctx *ctx = req->ctx;
8476 struct io_fixed_file *file_slot;
8477 struct file *file;
Pavel Begunkovb8ed0f72022-04-07 14:05:05 +01008478 int ret;
Pavel Begunkov7df778b2021-09-24 20:04:29 +01008479
8480 io_ring_submit_lock(ctx, !(issue_flags & IO_URING_F_NONBLOCK));
8481 ret = -ENXIO;
8482 if (unlikely(!ctx->file_data))
8483 goto out;
8484 ret = -EINVAL;
8485 if (offset >= ctx->nr_user_files)
8486 goto out;
8487 ret = io_rsrc_node_switch_start(ctx);
8488 if (ret)
8489 goto out;
8490
Pavel Begunkovb8ed0f72022-04-07 14:05:05 +01008491 offset = array_index_nospec(offset, ctx->nr_user_files);
8492 file_slot = io_fixed_file_slot(&ctx->file_table, offset);
Pavel Begunkov7df778b2021-09-24 20:04:29 +01008493 ret = -EBADF;
8494 if (!file_slot->file_ptr)
8495 goto out;
8496
8497 file = (struct file *)(file_slot->file_ptr & FFS_MASK);
8498 ret = io_queue_rsrc_removal(ctx->file_data, offset, ctx->rsrc_node, file);
8499 if (ret)
8500 goto out;
8501
8502 file_slot->file_ptr = 0;
8503 io_rsrc_node_switch(ctx, ctx->file_data);
8504 ret = 0;
8505out:
8506 io_ring_submit_unlock(ctx, !(issue_flags & IO_URING_F_NONBLOCK));
8507 return ret;
8508}
8509
Jens Axboe05f3fb32019-12-09 11:22:50 -07008510static int __io_sqe_files_update(struct io_ring_ctx *ctx,
Pavel Begunkovc3bdad02021-04-25 14:32:22 +01008511 struct io_uring_rsrc_update2 *up,
Jens Axboe05f3fb32019-12-09 11:22:50 -07008512 unsigned nr_args)
8513{
Pavel Begunkovc3bdad02021-04-25 14:32:22 +01008514 u64 __user *tags = u64_to_user_ptr(up->tags);
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +01008515 __s32 __user *fds = u64_to_user_ptr(up->data);
Pavel Begunkovb895c9a2021-04-01 15:43:40 +01008516 struct io_rsrc_data *data = ctx->file_data;
Pavel Begunkova04b0ac2021-04-01 15:44:04 +01008517 struct io_fixed_file *file_slot;
8518 struct file *file;
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +01008519 int fd, i, err = 0;
8520 unsigned int done;
Xiaoguang Wang05589552020-03-31 14:05:18 +08008521 bool needs_switch = false;
Jens Axboec3a31e62019-10-03 13:59:56 -06008522
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +01008523 if (!ctx->file_data)
8524 return -ENXIO;
8525 if (up->offset + nr_args > ctx->nr_user_files)
Jens Axboec3a31e62019-10-03 13:59:56 -06008526 return -EINVAL;
8527
Pavel Begunkov67973b92021-01-26 13:51:09 +00008528 for (done = 0; done < nr_args; done++) {
Pavel Begunkovc3bdad02021-04-25 14:32:22 +01008529 u64 tag = 0;
8530
8531 if ((tags && copy_from_user(&tag, &tags[done], sizeof(tag))) ||
8532 copy_from_user(&fd, &fds[done], sizeof(fd))) {
Jens Axboec3a31e62019-10-03 13:59:56 -06008533 err = -EFAULT;
8534 break;
8535 }
Pavel Begunkovc3bdad02021-04-25 14:32:22 +01008536 if ((fd == IORING_REGISTER_FILES_SKIP || fd == -1) && tag) {
8537 err = -EINVAL;
8538 break;
8539 }
noah4e0377a2021-01-26 15:23:28 -05008540 if (fd == IORING_REGISTER_FILES_SKIP)
8541 continue;
8542
Pavel Begunkov67973b92021-01-26 13:51:09 +00008543 i = array_index_nospec(up->offset + done, ctx->nr_user_files);
Pavel Begunkovaeca2412021-04-11 01:46:37 +01008544 file_slot = io_fixed_file_slot(&ctx->file_table, i);
Pavel Begunkovea64ec022021-02-04 13:52:07 +00008545
Pavel Begunkova04b0ac2021-04-01 15:44:04 +01008546 if (file_slot->file_ptr) {
8547 file = (struct file *)(file_slot->file_ptr & FFS_MASK);
Pavel Begunkovb8ed0f72022-04-07 14:05:05 +01008548 err = io_queue_rsrc_removal(data, i, ctx->rsrc_node, file);
Hillf Dantona5318d32020-03-23 17:47:15 +08008549 if (err)
8550 break;
Pavel Begunkova04b0ac2021-04-01 15:44:04 +01008551 file_slot->file_ptr = 0;
Xiaoguang Wang05589552020-03-31 14:05:18 +08008552 needs_switch = true;
Jens Axboec3a31e62019-10-03 13:59:56 -06008553 }
8554 if (fd != -1) {
Jens Axboec3a31e62019-10-03 13:59:56 -06008555 file = fget(fd);
8556 if (!file) {
8557 err = -EBADF;
8558 break;
8559 }
8560 /*
8561 * Don't allow io_uring instances to be registered. If
8562 * UNIX isn't enabled, then this causes a reference
8563 * cycle and this instance can never get freed. If UNIX
8564 * is enabled we'll handle it just fine, but there's
8565 * still no point in allowing a ring fd as it doesn't
8566 * support regular read/write anyway.
8567 */
8568 if (file->f_op == &io_uring_fops) {
8569 fput(file);
8570 err = -EBADF;
8571 break;
8572 }
Pavel Begunkov50c981b2022-04-06 12:43:57 +01008573 *io_get_tag_slot(data, i) = tag;
Pavel Begunkov9a321c92021-04-01 15:44:01 +01008574 io_fixed_file_set(file_slot, file);
Jens Axboec3a31e62019-10-03 13:59:56 -06008575 err = io_sqe_file_register(ctx, file, i);
Yang Yingliangf3bd9da2020-07-09 10:11:41 +00008576 if (err) {
Pavel Begunkova04b0ac2021-04-01 15:44:04 +01008577 file_slot->file_ptr = 0;
Yang Yingliangf3bd9da2020-07-09 10:11:41 +00008578 fput(file);
Jens Axboec3a31e62019-10-03 13:59:56 -06008579 break;
Yang Yingliangf3bd9da2020-07-09 10:11:41 +00008580 }
Jens Axboec3a31e62019-10-03 13:59:56 -06008581 }
Jens Axboe05f3fb32019-12-09 11:22:50 -07008582 }
8583
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01008584 if (needs_switch)
8585 io_rsrc_node_switch(ctx, data);
Jens Axboec3a31e62019-10-03 13:59:56 -06008586 return done ? done : err;
8587}
Xiaoguang Wang05589552020-03-31 14:05:18 +08008588
Jens Axboe685fe7f2021-03-08 09:37:51 -07008589static struct io_wq *io_init_wq_offload(struct io_ring_ctx *ctx,
8590 struct task_struct *task)
Pavel Begunkov24369c22020-01-28 03:15:48 +03008591{
Jens Axboee9418942021-02-19 12:33:30 -07008592 struct io_wq_hash *hash;
Pavel Begunkov24369c22020-01-28 03:15:48 +03008593 struct io_wq_data data;
Pavel Begunkov24369c22020-01-28 03:15:48 +03008594 unsigned int concurrency;
Pavel Begunkov24369c22020-01-28 03:15:48 +03008595
Yang Yingliang362a9e62021-07-20 16:38:05 +08008596 mutex_lock(&ctx->uring_lock);
Jens Axboee9418942021-02-19 12:33:30 -07008597 hash = ctx->hash_map;
8598 if (!hash) {
8599 hash = kzalloc(sizeof(*hash), GFP_KERNEL);
Yang Yingliang362a9e62021-07-20 16:38:05 +08008600 if (!hash) {
8601 mutex_unlock(&ctx->uring_lock);
Jens Axboee9418942021-02-19 12:33:30 -07008602 return ERR_PTR(-ENOMEM);
Yang Yingliang362a9e62021-07-20 16:38:05 +08008603 }
Jens Axboee9418942021-02-19 12:33:30 -07008604 refcount_set(&hash->refs, 1);
8605 init_waitqueue_head(&hash->wait);
8606 ctx->hash_map = hash;
8607 }
Yang Yingliang362a9e62021-07-20 16:38:05 +08008608 mutex_unlock(&ctx->uring_lock);
Jens Axboee9418942021-02-19 12:33:30 -07008609
8610 data.hash = hash;
Jens Axboe685fe7f2021-03-08 09:37:51 -07008611 data.task = task;
Pavel Begunkovebc11b62021-08-09 13:04:05 +01008612 data.free_work = io_wq_free_work;
Pavel Begunkovf5fa38c2020-06-08 21:08:20 +03008613 data.do_work = io_wq_submit_work;
Pavel Begunkov24369c22020-01-28 03:15:48 +03008614
Jens Axboed25e3a32021-02-16 11:41:41 -07008615 /* Do QD, or 4 * CPUS, whatever is smallest */
8616 concurrency = min(ctx->sq_entries, 4 * num_online_cpus());
Pavel Begunkov24369c22020-01-28 03:15:48 +03008617
Jens Axboe5aa75ed2021-02-16 12:56:50 -07008618 return io_wq_create(concurrency, &data);
Pavel Begunkov24369c22020-01-28 03:15:48 +03008619}
8620
Jens Axboe5aa75ed2021-02-16 12:56:50 -07008621static int io_uring_alloc_task_context(struct task_struct *task,
8622 struct io_ring_ctx *ctx)
Jens Axboe0f212202020-09-13 13:09:39 -06008623{
8624 struct io_uring_task *tctx;
Jens Axboed8a6df12020-10-15 16:24:45 -06008625 int ret;
Jens Axboe0f212202020-09-13 13:09:39 -06008626
Pavel Begunkov09899b12021-06-14 02:36:22 +01008627 tctx = kzalloc(sizeof(*tctx), GFP_KERNEL);
Jens Axboe0f212202020-09-13 13:09:39 -06008628 if (unlikely(!tctx))
8629 return -ENOMEM;
8630
Jens Axboed8a6df12020-10-15 16:24:45 -06008631 ret = percpu_counter_init(&tctx->inflight, 0, GFP_KERNEL);
8632 if (unlikely(ret)) {
8633 kfree(tctx);
8634 return ret;
8635 }
8636
Jens Axboe685fe7f2021-03-08 09:37:51 -07008637 tctx->io_wq = io_init_wq_offload(ctx, task);
Jens Axboe5aa75ed2021-02-16 12:56:50 -07008638 if (IS_ERR(tctx->io_wq)) {
8639 ret = PTR_ERR(tctx->io_wq);
8640 percpu_counter_destroy(&tctx->inflight);
8641 kfree(tctx);
8642 return ret;
8643 }
8644
Jens Axboe0f212202020-09-13 13:09:39 -06008645 xa_init(&tctx->xa);
8646 init_waitqueue_head(&tctx->wait);
Jens Axboefdaf0832020-10-30 09:37:30 -06008647 atomic_set(&tctx->in_idle, 0);
Pavel Begunkovb303fe22021-04-11 01:46:26 +01008648 atomic_set(&tctx->inflight_tracked, 0);
Jens Axboe0f212202020-09-13 13:09:39 -06008649 task->io_uring = tctx;
Jens Axboe7cbf1722021-02-10 00:03:20 +00008650 spin_lock_init(&tctx->task_lock);
8651 INIT_WQ_LIST(&tctx->task_list);
Jens Axboe7cbf1722021-02-10 00:03:20 +00008652 init_task_work(&tctx->task_work, tctx_task_work);
Jens Axboe0f212202020-09-13 13:09:39 -06008653 return 0;
8654}
8655
8656void __io_uring_free(struct task_struct *tsk)
8657{
8658 struct io_uring_task *tctx = tsk->io_uring;
8659
8660 WARN_ON_ONCE(!xa_empty(&tctx->xa));
Pavel Begunkovef8eaa42021-02-27 11:16:45 +00008661 WARN_ON_ONCE(tctx->io_wq);
Pavel Begunkov09899b12021-06-14 02:36:22 +01008662 WARN_ON_ONCE(tctx->cached_refs);
Pavel Begunkovef8eaa42021-02-27 11:16:45 +00008663
Jens Axboed8a6df12020-10-15 16:24:45 -06008664 percpu_counter_destroy(&tctx->inflight);
Jens Axboe0f212202020-09-13 13:09:39 -06008665 kfree(tctx);
8666 tsk->io_uring = NULL;
8667}
8668
Stefano Garzarella7e84e1c2020-08-27 16:58:31 +02008669static int io_sq_offload_create(struct io_ring_ctx *ctx,
8670 struct io_uring_params *p)
Jens Axboe2b188cc2019-01-07 10:46:33 -07008671{
8672 int ret;
8673
Jens Axboed25e3a32021-02-16 11:41:41 -07008674 /* Retain compatibility with failing for an invalid attach attempt */
8675 if ((ctx->flags & (IORING_SETUP_ATTACH_WQ | IORING_SETUP_SQPOLL)) ==
8676 IORING_SETUP_ATTACH_WQ) {
8677 struct fd f;
8678
8679 f = fdget(p->wq_fd);
8680 if (!f.file)
8681 return -ENXIO;
Jens Axboe0cc936f2021-07-22 17:08:07 -06008682 if (f.file->f_op != &io_uring_fops) {
8683 fdput(f);
Pavel Begunkovf2a48dd2021-04-20 12:03:33 +01008684 return -EINVAL;
Jens Axboe0cc936f2021-07-22 17:08:07 -06008685 }
8686 fdput(f);
Jens Axboed25e3a32021-02-16 11:41:41 -07008687 }
Jens Axboe6c271ce2019-01-10 11:22:30 -07008688 if (ctx->flags & IORING_SETUP_SQPOLL) {
Jens Axboe46fe18b2021-03-04 12:39:36 -07008689 struct task_struct *tsk;
Jens Axboe534ca6d2020-09-02 13:52:19 -06008690 struct io_sq_data *sqd;
Pavel Begunkov26984fb2021-03-11 23:29:37 +00008691 bool attached;
Jens Axboe534ca6d2020-09-02 13:52:19 -06008692
Pavel Begunkov26984fb2021-03-11 23:29:37 +00008693 sqd = io_get_sq_data(p, &attached);
Jens Axboe534ca6d2020-09-02 13:52:19 -06008694 if (IS_ERR(sqd)) {
8695 ret = PTR_ERR(sqd);
8696 goto err;
8697 }
Jens Axboe69fb2132020-09-14 11:16:23 -06008698
Stefan Metzmacher7c30f36a2021-03-07 11:54:28 +01008699 ctx->sq_creds = get_current_cred();
Jens Axboe534ca6d2020-09-02 13:52:19 -06008700 ctx->sq_data = sqd;
Jens Axboe917257d2019-04-13 09:28:55 -06008701 ctx->sq_thread_idle = msecs_to_jiffies(p->sq_thread_idle);
8702 if (!ctx->sq_thread_idle)
8703 ctx->sq_thread_idle = HZ;
8704
Pavel Begunkov78d7f6b2021-03-10 13:13:53 +00008705 io_sq_thread_park(sqd);
Pavel Begunkovde75a3d2021-03-18 11:54:35 +00008706 list_add(&ctx->sqd_list, &sqd->ctx_list);
8707 io_sqd_update_thread_idle(sqd);
Pavel Begunkov26984fb2021-03-11 23:29:37 +00008708 /* don't attach to a dying SQPOLL thread, would be racy */
Pavel Begunkovf2a48dd2021-04-20 12:03:33 +01008709 ret = (attached && !sqd->thread) ? -ENXIO : 0;
Pavel Begunkov78d7f6b2021-03-10 13:13:53 +00008710 io_sq_thread_unpark(sqd);
8711
Pavel Begunkovde75a3d2021-03-18 11:54:35 +00008712 if (ret < 0)
8713 goto err;
8714 if (attached)
Jens Axboe5aa75ed2021-02-16 12:56:50 -07008715 return 0;
Jens Axboeaa061652020-09-02 14:50:27 -06008716
Jens Axboe6c271ce2019-01-10 11:22:30 -07008717 if (p->flags & IORING_SETUP_SQ_AFF) {
Jens Axboe44a9bd12019-05-14 20:00:30 -06008718 int cpu = p->sq_thread_cpu;
Jens Axboe6c271ce2019-01-10 11:22:30 -07008719
Jens Axboe917257d2019-04-13 09:28:55 -06008720 ret = -EINVAL;
Pavel Begunkovf2a48dd2021-04-20 12:03:33 +01008721 if (cpu >= nr_cpu_ids || !cpu_online(cpu))
Jens Axboee8f98f242021-03-09 16:32:13 -07008722 goto err_sqpoll;
Jens Axboe37d1e2e2021-02-17 21:03:43 -07008723 sqd->sq_cpu = cpu;
Jens Axboe6c271ce2019-01-10 11:22:30 -07008724 } else {
Jens Axboe37d1e2e2021-02-17 21:03:43 -07008725 sqd->sq_cpu = -1;
Jens Axboe6c271ce2019-01-10 11:22:30 -07008726 }
Jens Axboe37d1e2e2021-02-17 21:03:43 -07008727
8728 sqd->task_pid = current->pid;
Jens Axboe5c2469e2021-03-11 10:17:56 -07008729 sqd->task_tgid = current->tgid;
Jens Axboe46fe18b2021-03-04 12:39:36 -07008730 tsk = create_io_thread(io_sq_thread, sqd, NUMA_NO_NODE);
8731 if (IS_ERR(tsk)) {
8732 ret = PTR_ERR(tsk);
Jens Axboee8f98f242021-03-09 16:32:13 -07008733 goto err_sqpoll;
Jens Axboe6c271ce2019-01-10 11:22:30 -07008734 }
Pavel Begunkov97a73a02021-03-08 17:30:54 +00008735
Jens Axboe46fe18b2021-03-04 12:39:36 -07008736 sqd->thread = tsk;
Pavel Begunkov97a73a02021-03-08 17:30:54 +00008737 ret = io_uring_alloc_task_context(tsk, ctx);
Jens Axboe46fe18b2021-03-04 12:39:36 -07008738 wake_up_new_task(tsk);
Jens Axboe0f212202020-09-13 13:09:39 -06008739 if (ret)
8740 goto err;
Jens Axboe6c271ce2019-01-10 11:22:30 -07008741 } else if (p->flags & IORING_SETUP_SQ_AFF) {
8742 /* Can't have SQ_AFF without SQPOLL */
8743 ret = -EINVAL;
8744 goto err;
8745 }
8746
Jens Axboe2b188cc2019-01-07 10:46:33 -07008747 return 0;
Pavel Begunkovf2a48dd2021-04-20 12:03:33 +01008748err_sqpoll:
8749 complete(&ctx->sq_data->exited);
Jens Axboe2b188cc2019-01-07 10:46:33 -07008750err:
Jens Axboe37d1e2e2021-02-17 21:03:43 -07008751 io_sq_thread_finish(ctx);
Jens Axboe2b188cc2019-01-07 10:46:33 -07008752 return ret;
8753}
8754
Bijan Mottahedeha087e2b2020-06-16 16:36:07 -07008755static inline void __io_unaccount_mem(struct user_struct *user,
8756 unsigned long nr_pages)
Jens Axboe2b188cc2019-01-07 10:46:33 -07008757{
8758 atomic_long_sub(nr_pages, &user->locked_vm);
8759}
8760
Bijan Mottahedeha087e2b2020-06-16 16:36:07 -07008761static inline int __io_account_mem(struct user_struct *user,
8762 unsigned long nr_pages)
Jens Axboe2b188cc2019-01-07 10:46:33 -07008763{
8764 unsigned long page_limit, cur_pages, new_pages;
8765
8766 /* Don't allow more pages than we can safely lock */
8767 page_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
8768
8769 do {
8770 cur_pages = atomic_long_read(&user->locked_vm);
8771 new_pages = cur_pages + nr_pages;
8772 if (new_pages > page_limit)
8773 return -ENOMEM;
8774 } while (atomic_long_cmpxchg(&user->locked_vm, cur_pages,
8775 new_pages) != cur_pages);
8776
8777 return 0;
8778}
8779
Jens Axboe26bfa89e2021-02-09 20:14:12 -07008780static void io_unaccount_mem(struct io_ring_ctx *ctx, unsigned long nr_pages)
Bijan Mottahedeha087e2b2020-06-16 16:36:07 -07008781{
Jens Axboe62e398b2021-02-21 16:19:37 -07008782 if (ctx->user)
Bijan Mottahedeha087e2b2020-06-16 16:36:07 -07008783 __io_unaccount_mem(ctx->user, nr_pages);
Bijan Mottahedeh30975822020-06-16 16:36:09 -07008784
Jens Axboe26bfa89e2021-02-09 20:14:12 -07008785 if (ctx->mm_account)
8786 atomic64_sub(nr_pages, &ctx->mm_account->pinned_vm);
Bijan Mottahedeha087e2b2020-06-16 16:36:07 -07008787}
8788
Jens Axboe26bfa89e2021-02-09 20:14:12 -07008789static int io_account_mem(struct io_ring_ctx *ctx, unsigned long nr_pages)
Bijan Mottahedeha087e2b2020-06-16 16:36:07 -07008790{
Bijan Mottahedeh30975822020-06-16 16:36:09 -07008791 int ret;
8792
Jens Axboe62e398b2021-02-21 16:19:37 -07008793 if (ctx->user) {
Bijan Mottahedeh30975822020-06-16 16:36:09 -07008794 ret = __io_account_mem(ctx->user, nr_pages);
8795 if (ret)
8796 return ret;
8797 }
8798
Jens Axboe26bfa89e2021-02-09 20:14:12 -07008799 if (ctx->mm_account)
8800 atomic64_add(nr_pages, &ctx->mm_account->pinned_vm);
Bijan Mottahedeha087e2b2020-06-16 16:36:07 -07008801
8802 return 0;
8803}
8804
Jens Axboe2b188cc2019-01-07 10:46:33 -07008805static void io_mem_free(void *ptr)
8806{
Mark Rutland52e04ef2019-04-30 17:30:21 +01008807 struct page *page;
Jens Axboe2b188cc2019-01-07 10:46:33 -07008808
Mark Rutland52e04ef2019-04-30 17:30:21 +01008809 if (!ptr)
8810 return;
8811
8812 page = virt_to_head_page(ptr);
Jens Axboe2b188cc2019-01-07 10:46:33 -07008813 if (put_page_testzero(page))
8814 free_compound_page(page);
8815}
8816
8817static void *io_mem_alloc(size_t size)
8818{
Shakeel Butt246dfbc2022-01-24 21:17:36 -08008819 gfp_t gfp = GFP_KERNEL_ACCOUNT | __GFP_ZERO | __GFP_NOWARN | __GFP_COMP;
Jens Axboe2b188cc2019-01-07 10:46:33 -07008820
Shakeel Butt246dfbc2022-01-24 21:17:36 -08008821 return (void *) __get_free_pages(gfp, get_order(size));
Jens Axboe2b188cc2019-01-07 10:46:33 -07008822}
8823
Hristo Venev75b28af2019-08-26 17:23:46 +00008824static unsigned long rings_size(unsigned sq_entries, unsigned cq_entries,
8825 size_t *sq_offset)
8826{
8827 struct io_rings *rings;
8828 size_t off, sq_array_size;
8829
8830 off = struct_size(rings, cqes, cq_entries);
8831 if (off == SIZE_MAX)
8832 return SIZE_MAX;
8833
8834#ifdef CONFIG_SMP
8835 off = ALIGN(off, SMP_CACHE_BYTES);
8836 if (off == 0)
8837 return SIZE_MAX;
8838#endif
8839
Dmitry Vyukovb36200f2020-07-11 11:31:11 +02008840 if (sq_offset)
8841 *sq_offset = off;
8842
Hristo Venev75b28af2019-08-26 17:23:46 +00008843 sq_array_size = array_size(sizeof(u32), sq_entries);
8844 if (sq_array_size == SIZE_MAX)
8845 return SIZE_MAX;
8846
8847 if (check_add_overflow(off, sq_array_size, &off))
8848 return SIZE_MAX;
8849
Hristo Venev75b28af2019-08-26 17:23:46 +00008850 return off;
8851}
8852
Pavel Begunkov41edf1a2021-04-25 14:32:23 +01008853static void io_buffer_unmap(struct io_ring_ctx *ctx, struct io_mapped_ubuf **slot)
Pavel Begunkov7f61a1e2021-04-11 01:46:35 +01008854{
Pavel Begunkov41edf1a2021-04-25 14:32:23 +01008855 struct io_mapped_ubuf *imu = *slot;
Pavel Begunkov7f61a1e2021-04-11 01:46:35 +01008856 unsigned int i;
8857
Pavel Begunkov62248432021-04-28 13:11:29 +01008858 if (imu != ctx->dummy_ubuf) {
8859 for (i = 0; i < imu->nr_bvecs; i++)
8860 unpin_user_page(imu->bvec[i].bv_page);
8861 if (imu->acct_pages)
8862 io_unaccount_mem(ctx, imu->acct_pages);
8863 kvfree(imu);
8864 }
Pavel Begunkov41edf1a2021-04-25 14:32:23 +01008865 *slot = NULL;
Pavel Begunkov7f61a1e2021-04-11 01:46:35 +01008866}
8867
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01008868static void io_rsrc_buf_put(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc)
8869{
Pavel Begunkov634d00d2021-04-25 14:32:26 +01008870 io_buffer_unmap(ctx, &prsrc->buf);
8871 prsrc->buf = NULL;
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01008872}
8873
8874static void __io_sqe_buffers_unregister(struct io_ring_ctx *ctx)
Jens Axboeedafcce2019-01-09 09:16:05 -07008875{
Pavel Begunkov7f61a1e2021-04-11 01:46:35 +01008876 unsigned int i;
Jens Axboeedafcce2019-01-09 09:16:05 -07008877
Pavel Begunkov7f61a1e2021-04-11 01:46:35 +01008878 for (i = 0; i < ctx->nr_user_bufs; i++)
8879 io_buffer_unmap(ctx, &ctx->user_bufs[i]);
Jens Axboeedafcce2019-01-09 09:16:05 -07008880 kfree(ctx->user_bufs);
Zqiangbb6659c2021-04-30 16:25:15 +08008881 io_rsrc_data_free(ctx->buf_data);
Jens Axboeedafcce2019-01-09 09:16:05 -07008882 ctx->user_bufs = NULL;
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01008883 ctx->buf_data = NULL;
Jens Axboeedafcce2019-01-09 09:16:05 -07008884 ctx->nr_user_bufs = 0;
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01008885}
8886
Jens Axboeedafcce2019-01-09 09:16:05 -07008887static int io_sqe_buffers_unregister(struct io_ring_ctx *ctx)
8888{
Pavel Begunkov91f5a602022-06-13 06:30:06 +01008889 unsigned nr = ctx->nr_user_bufs;
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01008890 int ret;
Jens Axboeedafcce2019-01-09 09:16:05 -07008891
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01008892 if (!ctx->buf_data)
Jens Axboeedafcce2019-01-09 09:16:05 -07008893 return -ENXIO;
8894
Pavel Begunkov91f5a602022-06-13 06:30:06 +01008895 /*
8896 * Quiesce may unlock ->uring_lock, and while it's not held
8897 * prevent new requests using the table.
8898 */
8899 ctx->nr_user_bufs = 0;
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01008900 ret = io_rsrc_ref_quiesce(ctx->buf_data, ctx);
Pavel Begunkov91f5a602022-06-13 06:30:06 +01008901 ctx->nr_user_bufs = nr;
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01008902 if (!ret)
8903 __io_sqe_buffers_unregister(ctx);
8904 return ret;
Jens Axboeedafcce2019-01-09 09:16:05 -07008905}
8906
8907static int io_copy_iov(struct io_ring_ctx *ctx, struct iovec *dst,
8908 void __user *arg, unsigned index)
8909{
8910 struct iovec __user *src;
8911
8912#ifdef CONFIG_COMPAT
8913 if (ctx->compat) {
8914 struct compat_iovec __user *ciovs;
8915 struct compat_iovec ciov;
8916
8917 ciovs = (struct compat_iovec __user *) arg;
8918 if (copy_from_user(&ciov, &ciovs[index], sizeof(ciov)))
8919 return -EFAULT;
8920
Jens Axboed55e5f52019-12-11 16:12:15 -07008921 dst->iov_base = u64_to_user_ptr((u64)ciov.iov_base);
Jens Axboeedafcce2019-01-09 09:16:05 -07008922 dst->iov_len = ciov.iov_len;
8923 return 0;
8924 }
8925#endif
8926 src = (struct iovec __user *) arg;
8927 if (copy_from_user(dst, &src[index], sizeof(*dst)))
8928 return -EFAULT;
8929 return 0;
8930}
8931
Jens Axboede293932020-09-17 16:19:16 -06008932/*
8933 * Not super efficient, but this is just a registration time. And we do cache
8934 * the last compound head, so generally we'll only do a full search if we don't
8935 * match that one.
8936 *
8937 * We check if the given compound head page has already been accounted, to
8938 * avoid double accounting it. This allows us to account the full size of the
8939 * page, not just the constituent pages of a huge page.
8940 */
8941static bool headpage_already_acct(struct io_ring_ctx *ctx, struct page **pages,
8942 int nr_pages, struct page *hpage)
8943{
8944 int i, j;
8945
8946 /* check current page array */
8947 for (i = 0; i < nr_pages; i++) {
8948 if (!PageCompound(pages[i]))
8949 continue;
8950 if (compound_head(pages[i]) == hpage)
8951 return true;
8952 }
8953
8954 /* check previously registered pages */
8955 for (i = 0; i < ctx->nr_user_bufs; i++) {
Pavel Begunkov41edf1a2021-04-25 14:32:23 +01008956 struct io_mapped_ubuf *imu = ctx->user_bufs[i];
Jens Axboede293932020-09-17 16:19:16 -06008957
8958 for (j = 0; j < imu->nr_bvecs; j++) {
8959 if (!PageCompound(imu->bvec[j].bv_page))
8960 continue;
8961 if (compound_head(imu->bvec[j].bv_page) == hpage)
8962 return true;
8963 }
8964 }
8965
8966 return false;
8967}
8968
8969static int io_buffer_account_pin(struct io_ring_ctx *ctx, struct page **pages,
8970 int nr_pages, struct io_mapped_ubuf *imu,
8971 struct page **last_hpage)
8972{
8973 int i, ret;
8974
Pavel Begunkov216e5832021-05-29 12:01:02 +01008975 imu->acct_pages = 0;
Jens Axboede293932020-09-17 16:19:16 -06008976 for (i = 0; i < nr_pages; i++) {
8977 if (!PageCompound(pages[i])) {
8978 imu->acct_pages++;
8979 } else {
8980 struct page *hpage;
8981
8982 hpage = compound_head(pages[i]);
8983 if (hpage == *last_hpage)
8984 continue;
8985 *last_hpage = hpage;
8986 if (headpage_already_acct(ctx, pages, i, hpage))
8987 continue;
8988 imu->acct_pages += page_size(hpage) >> PAGE_SHIFT;
8989 }
8990 }
8991
8992 if (!imu->acct_pages)
8993 return 0;
8994
Jens Axboe26bfa89e2021-02-09 20:14:12 -07008995 ret = io_account_mem(ctx, imu->acct_pages);
Jens Axboede293932020-09-17 16:19:16 -06008996 if (ret)
8997 imu->acct_pages = 0;
8998 return ret;
8999}
9000
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08009001static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov,
Pavel Begunkov41edf1a2021-04-25 14:32:23 +01009002 struct io_mapped_ubuf **pimu,
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08009003 struct page **last_hpage)
Jens Axboeedafcce2019-01-09 09:16:05 -07009004{
Pavel Begunkov41edf1a2021-04-25 14:32:23 +01009005 struct io_mapped_ubuf *imu = NULL;
Jens Axboeedafcce2019-01-09 09:16:05 -07009006 struct vm_area_struct **vmas = NULL;
9007 struct page **pages = NULL;
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08009008 unsigned long off, start, end, ubuf;
9009 size_t size;
9010 int ret, pret, nr_pages, i;
Jens Axboeedafcce2019-01-09 09:16:05 -07009011
Pavel Begunkov62248432021-04-28 13:11:29 +01009012 if (!iov->iov_base) {
9013 *pimu = ctx->dummy_ubuf;
9014 return 0;
9015 }
9016
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08009017 ubuf = (unsigned long) iov->iov_base;
9018 end = (ubuf + iov->iov_len + PAGE_SIZE - 1) >> PAGE_SHIFT;
9019 start = ubuf >> PAGE_SHIFT;
9020 nr_pages = end - start;
9021
Pavel Begunkov41edf1a2021-04-25 14:32:23 +01009022 *pimu = NULL;
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08009023 ret = -ENOMEM;
9024
9025 pages = kvmalloc_array(nr_pages, sizeof(struct page *), GFP_KERNEL);
9026 if (!pages)
9027 goto done;
9028
9029 vmas = kvmalloc_array(nr_pages, sizeof(struct vm_area_struct *),
9030 GFP_KERNEL);
9031 if (!vmas)
9032 goto done;
9033
Pavel Begunkov41edf1a2021-04-25 14:32:23 +01009034 imu = kvmalloc(struct_size(imu, bvec, nr_pages), GFP_KERNEL);
Pavel Begunkova2b41982021-04-26 00:16:31 +01009035 if (!imu)
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08009036 goto done;
9037
9038 ret = 0;
9039 mmap_read_lock(current->mm);
9040 pret = pin_user_pages(ubuf, nr_pages, FOLL_WRITE | FOLL_LONGTERM,
9041 pages, vmas);
9042 if (pret == nr_pages) {
9043 /* don't support file backed memory */
9044 for (i = 0; i < nr_pages; i++) {
9045 struct vm_area_struct *vma = vmas[i];
9046
Pavel Begunkov40dad762021-06-09 15:26:54 +01009047 if (vma_is_shmem(vma))
9048 continue;
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08009049 if (vma->vm_file &&
9050 !is_file_hugepages(vma->vm_file)) {
9051 ret = -EOPNOTSUPP;
9052 break;
9053 }
9054 }
9055 } else {
9056 ret = pret < 0 ? pret : -EFAULT;
9057 }
9058 mmap_read_unlock(current->mm);
9059 if (ret) {
9060 /*
9061 * if we did partial map, or found file backed vmas,
9062 * release any pages we did get
9063 */
9064 if (pret > 0)
9065 unpin_user_pages(pages, pret);
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08009066 goto done;
9067 }
9068
9069 ret = io_buffer_account_pin(ctx, pages, pret, imu, last_hpage);
9070 if (ret) {
9071 unpin_user_pages(pages, pret);
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08009072 goto done;
9073 }
9074
9075 off = ubuf & ~PAGE_MASK;
9076 size = iov->iov_len;
9077 for (i = 0; i < nr_pages; i++) {
9078 size_t vec_len;
9079
9080 vec_len = min_t(size_t, size, PAGE_SIZE - off);
9081 imu->bvec[i].bv_page = pages[i];
9082 imu->bvec[i].bv_len = vec_len;
9083 imu->bvec[i].bv_offset = off;
9084 off = 0;
9085 size -= vec_len;
9086 }
9087 /* store original address for later verification */
9088 imu->ubuf = ubuf;
Pavel Begunkov4751f532021-04-01 15:43:55 +01009089 imu->ubuf_end = ubuf + iov->iov_len;
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08009090 imu->nr_bvecs = nr_pages;
Pavel Begunkov41edf1a2021-04-25 14:32:23 +01009091 *pimu = imu;
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08009092 ret = 0;
9093done:
Pavel Begunkov41edf1a2021-04-25 14:32:23 +01009094 if (ret)
9095 kvfree(imu);
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08009096 kvfree(pages);
9097 kvfree(vmas);
9098 return ret;
9099}
9100
Bijan Mottahedeh2b358602021-01-06 12:39:11 -08009101static int io_buffers_map_alloc(struct io_ring_ctx *ctx, unsigned int nr_args)
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08009102{
Pavel Begunkov87094462021-04-11 01:46:36 +01009103 ctx->user_bufs = kcalloc(nr_args, sizeof(*ctx->user_bufs), GFP_KERNEL);
9104 return ctx->user_bufs ? 0 : -ENOMEM;
Bijan Mottahedeh2b358602021-01-06 12:39:11 -08009105}
9106
9107static int io_buffer_validate(struct iovec *iov)
9108{
Pavel Begunkov50e96982021-03-24 22:59:01 +00009109 unsigned long tmp, acct_len = iov->iov_len + (PAGE_SIZE - 1);
9110
Bijan Mottahedeh2b358602021-01-06 12:39:11 -08009111 /*
9112 * Don't impose further limits on the size and buffer
9113 * constraints here, we'll -EINVAL later when IO is
9114 * submitted if they are wrong.
9115 */
Pavel Begunkov62248432021-04-28 13:11:29 +01009116 if (!iov->iov_base)
9117 return iov->iov_len ? -EFAULT : 0;
9118 if (!iov->iov_len)
Bijan Mottahedeh2b358602021-01-06 12:39:11 -08009119 return -EFAULT;
9120
9121 /* arbitrary limit, but we need something */
9122 if (iov->iov_len > SZ_1G)
9123 return -EFAULT;
9124
Pavel Begunkov50e96982021-03-24 22:59:01 +00009125 if (check_add_overflow((unsigned long)iov->iov_base, acct_len, &tmp))
9126 return -EOVERFLOW;
9127
Bijan Mottahedeh2b358602021-01-06 12:39:11 -08009128 return 0;
9129}
9130
9131static int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg,
Pavel Begunkov634d00d2021-04-25 14:32:26 +01009132 unsigned int nr_args, u64 __user *tags)
Bijan Mottahedeh2b358602021-01-06 12:39:11 -08009133{
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01009134 struct page *last_hpage = NULL;
9135 struct io_rsrc_data *data;
Bijan Mottahedeh2b358602021-01-06 12:39:11 -08009136 int i, ret;
9137 struct iovec iov;
Bijan Mottahedeh2b358602021-01-06 12:39:11 -08009138
Pavel Begunkov87094462021-04-11 01:46:36 +01009139 if (ctx->user_bufs)
9140 return -EBUSY;
Pavel Begunkov489809e2021-05-14 12:06:44 +01009141 if (!nr_args || nr_args > IORING_MAX_REG_BUFFERS)
Pavel Begunkov87094462021-04-11 01:46:36 +01009142 return -EINVAL;
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01009143 ret = io_rsrc_node_switch_start(ctx);
Bijan Mottahedeh2b358602021-01-06 12:39:11 -08009144 if (ret)
9145 return ret;
Pavel Begunkovd878c812021-06-14 02:36:18 +01009146 ret = io_rsrc_data_alloc(ctx, io_rsrc_buf_put, tags, nr_args, &data);
9147 if (ret)
9148 return ret;
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01009149 ret = io_buffers_map_alloc(ctx, nr_args);
9150 if (ret) {
Zqiangbb6659c2021-04-30 16:25:15 +08009151 io_rsrc_data_free(data);
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01009152 return ret;
9153 }
Bijan Mottahedeh2b358602021-01-06 12:39:11 -08009154
Pavel Begunkov87094462021-04-11 01:46:36 +01009155 for (i = 0; i < nr_args; i++, ctx->nr_user_bufs++) {
Jens Axboeedafcce2019-01-09 09:16:05 -07009156 ret = io_copy_iov(ctx, &iov, arg, i);
9157 if (ret)
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08009158 break;
Bijan Mottahedeh2b358602021-01-06 12:39:11 -08009159 ret = io_buffer_validate(&iov);
9160 if (ret)
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08009161 break;
Pavel Begunkov2d091d62021-06-14 02:36:21 +01009162 if (!iov.iov_base && *io_get_tag_slot(data, i)) {
Colin Ian Kingcf3770e2021-04-29 11:46:02 +01009163 ret = -EINVAL;
9164 break;
9165 }
Jens Axboeedafcce2019-01-09 09:16:05 -07009166
Pavel Begunkov41edf1a2021-04-25 14:32:23 +01009167 ret = io_sqe_buffer_register(ctx, &iov, &ctx->user_bufs[i],
9168 &last_hpage);
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08009169 if (ret)
9170 break;
Jens Axboeedafcce2019-01-09 09:16:05 -07009171 }
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08009172
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01009173 WARN_ON_ONCE(ctx->buf_data);
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08009174
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01009175 ctx->buf_data = data;
9176 if (ret)
9177 __io_sqe_buffers_unregister(ctx);
9178 else
9179 io_rsrc_node_switch(ctx, NULL);
Jens Axboeedafcce2019-01-09 09:16:05 -07009180 return ret;
9181}
9182
Pavel Begunkov634d00d2021-04-25 14:32:26 +01009183static int __io_sqe_buffers_update(struct io_ring_ctx *ctx,
9184 struct io_uring_rsrc_update2 *up,
9185 unsigned int nr_args)
9186{
9187 u64 __user *tags = u64_to_user_ptr(up->tags);
9188 struct iovec iov, __user *iovs = u64_to_user_ptr(up->data);
Pavel Begunkov634d00d2021-04-25 14:32:26 +01009189 struct page *last_hpage = NULL;
9190 bool needs_switch = false;
9191 __u32 done;
9192 int i, err;
9193
9194 if (!ctx->buf_data)
9195 return -ENXIO;
9196 if (up->offset + nr_args > ctx->nr_user_bufs)
9197 return -EINVAL;
9198
9199 for (done = 0; done < nr_args; done++) {
Pavel Begunkov0b8c0e72021-04-26 15:17:38 +01009200 struct io_mapped_ubuf *imu;
9201 int offset = up->offset + done;
Pavel Begunkov634d00d2021-04-25 14:32:26 +01009202 u64 tag = 0;
9203
9204 err = io_copy_iov(ctx, &iov, iovs, done);
9205 if (err)
9206 break;
9207 if (tags && copy_from_user(&tag, &tags[done], sizeof(tag))) {
9208 err = -EFAULT;
9209 break;
9210 }
Pavel Begunkov0b8c0e72021-04-26 15:17:38 +01009211 err = io_buffer_validate(&iov);
9212 if (err)
9213 break;
Colin Ian Kingcf3770e2021-04-29 11:46:02 +01009214 if (!iov.iov_base && tag) {
9215 err = -EINVAL;
9216 break;
9217 }
Pavel Begunkov0b8c0e72021-04-26 15:17:38 +01009218 err = io_sqe_buffer_register(ctx, &iov, &imu, &last_hpage);
9219 if (err)
9220 break;
Pavel Begunkov634d00d2021-04-25 14:32:26 +01009221
Pavel Begunkov0b8c0e72021-04-26 15:17:38 +01009222 i = array_index_nospec(offset, ctx->nr_user_bufs);
Pavel Begunkov62248432021-04-28 13:11:29 +01009223 if (ctx->user_bufs[i] != ctx->dummy_ubuf) {
Pavel Begunkovb8ed0f72022-04-07 14:05:05 +01009224 err = io_queue_rsrc_removal(ctx->buf_data, i,
Pavel Begunkov0b8c0e72021-04-26 15:17:38 +01009225 ctx->rsrc_node, ctx->user_bufs[i]);
9226 if (unlikely(err)) {
9227 io_buffer_unmap(ctx, &imu);
Pavel Begunkov634d00d2021-04-25 14:32:26 +01009228 break;
Pavel Begunkov0b8c0e72021-04-26 15:17:38 +01009229 }
Pavel Begunkov634d00d2021-04-25 14:32:26 +01009230 ctx->user_bufs[i] = NULL;
9231 needs_switch = true;
9232 }
9233
Pavel Begunkov0b8c0e72021-04-26 15:17:38 +01009234 ctx->user_bufs[i] = imu;
Pavel Begunkov2d091d62021-06-14 02:36:21 +01009235 *io_get_tag_slot(ctx->buf_data, offset) = tag;
Pavel Begunkov634d00d2021-04-25 14:32:26 +01009236 }
9237
9238 if (needs_switch)
9239 io_rsrc_node_switch(ctx, ctx->buf_data);
9240 return done ? done : err;
9241}
9242
Jens Axboe9b402842019-04-11 11:45:41 -06009243static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg)
9244{
9245 __s32 __user *fds = arg;
9246 int fd;
9247
9248 if (ctx->cq_ev_fd)
9249 return -EBUSY;
9250
9251 if (copy_from_user(&fd, fds, sizeof(*fds)))
9252 return -EFAULT;
9253
9254 ctx->cq_ev_fd = eventfd_ctx_fdget(fd);
9255 if (IS_ERR(ctx->cq_ev_fd)) {
9256 int ret = PTR_ERR(ctx->cq_ev_fd);
Pavel Begunkovfe7e3252021-06-24 15:09:57 +01009257
Jens Axboe9b402842019-04-11 11:45:41 -06009258 ctx->cq_ev_fd = NULL;
9259 return ret;
9260 }
9261
9262 return 0;
9263}
9264
9265static int io_eventfd_unregister(struct io_ring_ctx *ctx)
9266{
9267 if (ctx->cq_ev_fd) {
9268 eventfd_ctx_put(ctx->cq_ev_fd);
9269 ctx->cq_ev_fd = NULL;
9270 return 0;
9271 }
9272
9273 return -ENXIO;
9274}
9275
Jens Axboe5a2e7452020-02-23 16:23:11 -07009276static void io_destroy_buffers(struct io_ring_ctx *ctx)
9277{
Jens Axboe9e15c3a2021-03-13 12:29:43 -07009278 struct io_buffer *buf;
9279 unsigned long index;
9280
Ye Bin2d447d32021-11-22 10:47:37 +08009281 xa_for_each(&ctx->io_buffers, index, buf)
Jens Axboe9e15c3a2021-03-13 12:29:43 -07009282 __io_remove_buffers(ctx, buf, index, -1U);
Jens Axboe5a2e7452020-02-23 16:23:11 -07009283}
9284
Pavel Begunkov72558342021-08-09 20:18:09 +01009285static void io_req_cache_free(struct list_head *list)
Jens Axboe1b4c3512021-02-10 00:03:19 +00009286{
Jens Axboe68e68ee2021-02-13 09:00:02 -07009287 struct io_kiocb *req, *nxt;
Jens Axboe1b4c3512021-02-10 00:03:19 +00009288
Pavel Begunkovbb943b82021-08-09 20:18:10 +01009289 list_for_each_entry_safe(req, nxt, list, inflight_entry) {
9290 list_del(&req->inflight_entry);
Jens Axboe1b4c3512021-02-10 00:03:19 +00009291 kmem_cache_free(req_cachep, req);
9292 }
9293}
9294
Jens Axboe4010fec2021-02-27 15:04:18 -07009295static void io_req_caches_free(struct io_ring_ctx *ctx)
Jens Axboe2b188cc2019-01-07 10:46:33 -07009296{
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01009297 struct io_submit_state *state = &ctx->submit_state;
Pavel Begunkovbf019da2021-02-10 00:03:17 +00009298
Jens Axboe9a4fdbd2021-02-13 09:09:44 -07009299 mutex_lock(&ctx->uring_lock);
9300
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01009301 if (state->free_reqs) {
9302 kmem_cache_free_bulk(req_cachep, state->free_reqs, state->reqs);
9303 state->free_reqs = 0;
Pavel Begunkov8e5c66c2021-02-22 11:45:55 +00009304 }
Jens Axboe9a4fdbd2021-02-13 09:09:44 -07009305
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01009306 io_flush_cached_locked_reqs(ctx, state);
9307 io_req_cache_free(&state->free_list);
Jens Axboe9a4fdbd2021-02-13 09:09:44 -07009308 mutex_unlock(&ctx->uring_lock);
9309}
9310
Pavel Begunkov43597aa2021-08-10 02:44:23 +01009311static void io_wait_rsrc_data(struct io_rsrc_data *data)
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01009312{
Pavel Begunkov43597aa2021-08-10 02:44:23 +01009313 if (data && !atomic_dec_and_test(&data->refs))
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01009314 wait_for_completion(&data->done);
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01009315}
9316
Jens Axboe2b188cc2019-01-07 10:46:33 -07009317static void io_ring_ctx_free(struct io_ring_ctx *ctx)
9318{
Jens Axboe37d1e2e2021-02-17 21:03:43 -07009319 io_sq_thread_finish(ctx);
Jens Axboe2aede0e2020-09-14 10:45:53 -06009320
Pavel Begunkov43597aa2021-08-10 02:44:23 +01009321 /* __io_rsrc_put_work() may need uring_lock to progress, wait w/o it */
9322 io_wait_rsrc_data(ctx->buf_data);
9323 io_wait_rsrc_data(ctx->file_data);
9324
Hao Xu8bad28d2021-02-19 17:19:36 +08009325 mutex_lock(&ctx->uring_lock);
Pavel Begunkov43597aa2021-08-10 02:44:23 +01009326 if (ctx->buf_data)
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01009327 __io_sqe_buffers_unregister(ctx);
Pavel Begunkov43597aa2021-08-10 02:44:23 +01009328 if (ctx->file_data)
Pavel Begunkov08480402021-04-13 02:58:38 +01009329 __io_sqe_files_unregister(ctx);
Pavel Begunkovc4ea0602021-04-01 15:43:58 +01009330 if (ctx->rings)
9331 __io_cqring_overflow_flush(ctx, true);
Hao Xu8bad28d2021-02-19 17:19:36 +08009332 mutex_unlock(&ctx->uring_lock);
Jens Axboe9b402842019-04-11 11:45:41 -06009333 io_eventfd_unregister(ctx);
Jens Axboe5a2e7452020-02-23 16:23:11 -07009334 io_destroy_buffers(ctx);
Pavel Begunkov07db2982021-04-20 12:03:32 +01009335 if (ctx->sq_creds)
9336 put_cred(ctx->sq_creds);
Jens Axboedef596e2019-01-09 08:59:42 -07009337
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01009338 /* there are no registered resources left, nobody uses it */
9339 if (ctx->rsrc_node)
9340 io_rsrc_node_destroy(ctx->rsrc_node);
Pavel Begunkov8dd03af2021-03-19 17:22:36 +00009341 if (ctx->rsrc_backup_node)
Pavel Begunkovb895c9a2021-04-01 15:43:40 +01009342 io_rsrc_node_destroy(ctx->rsrc_backup_node);
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01009343 flush_delayed_work(&ctx->rsrc_put_work);
9344
9345 WARN_ON_ONCE(!list_empty(&ctx->rsrc_ref_list));
9346 WARN_ON_ONCE(!llist_empty(&ctx->rsrc_put_llist));
Jens Axboe2b188cc2019-01-07 10:46:33 -07009347
9348#if defined(CONFIG_UNIX)
Eric Biggers355e8d22019-06-12 14:58:43 -07009349 if (ctx->ring_sock) {
9350 ctx->ring_sock->file = NULL; /* so that iput() is called */
Jens Axboe2b188cc2019-01-07 10:46:33 -07009351 sock_release(ctx->ring_sock);
Eric Biggers355e8d22019-06-12 14:58:43 -07009352 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07009353#endif
Pavel Begunkovef9dd632021-08-28 19:54:38 -06009354 WARN_ON_ONCE(!list_empty(&ctx->ltimeout_list));
Jens Axboe2b188cc2019-01-07 10:46:33 -07009355
Pavel Begunkovcd148d42022-10-16 22:42:55 +01009356 if (ctx->mm_account) {
9357 mmdrop(ctx->mm_account);
9358 ctx->mm_account = NULL;
9359 }
9360
Hristo Venev75b28af2019-08-26 17:23:46 +00009361 io_mem_free(ctx->rings);
Jens Axboe2b188cc2019-01-07 10:46:33 -07009362 io_mem_free(ctx->sq_sqes);
Jens Axboe2b188cc2019-01-07 10:46:33 -07009363
9364 percpu_ref_exit(&ctx->refs);
Jens Axboe2b188cc2019-01-07 10:46:33 -07009365 free_uid(ctx->user);
Jens Axboe4010fec2021-02-27 15:04:18 -07009366 io_req_caches_free(ctx);
Jens Axboee9418942021-02-19 12:33:30 -07009367 if (ctx->hash_map)
9368 io_wq_put_hash(ctx->hash_map);
Jens Axboe78076bb2019-12-04 19:56:40 -07009369 kfree(ctx->cancel_hash);
Pavel Begunkov62248432021-04-28 13:11:29 +01009370 kfree(ctx->dummy_ubuf);
Jens Axboe2b188cc2019-01-07 10:46:33 -07009371 kfree(ctx);
9372}
9373
9374static __poll_t io_uring_poll(struct file *file, poll_table *wait)
9375{
9376 struct io_ring_ctx *ctx = file->private_data;
9377 __poll_t mask = 0;
9378
Pavel Begunkov311997b2021-06-14 23:37:28 +01009379 poll_wait(file, &ctx->poll_wait, wait);
Stefan Bühler4f7067c2019-04-24 23:54:17 +02009380 /*
9381 * synchronizes with barrier from wq_has_sleeper call in
9382 * io_commit_cqring
9383 */
Jens Axboe2b188cc2019-01-07 10:46:33 -07009384 smp_rmb();
Jens Axboe90554202020-09-03 12:12:41 -06009385 if (!io_sqring_full(ctx))
Jens Axboe2b188cc2019-01-07 10:46:33 -07009386 mask |= EPOLLOUT | EPOLLWRNORM;
Hao Xued670c32021-02-05 16:34:21 +08009387
9388 /*
9389 * Don't flush cqring overflow list here, just do a simple check.
9390 * Otherwise there could possible be ABBA deadlock:
9391 * CPU0 CPU1
9392 * ---- ----
9393 * lock(&ctx->uring_lock);
9394 * lock(&ep->mtx);
9395 * lock(&ctx->uring_lock);
9396 * lock(&ep->mtx);
9397 *
9398 * Users may get EPOLLIN meanwhile seeing nothing in cqring, this
9399 * pushs them to do the flush.
9400 */
Pavel Begunkov5ed7a372021-06-14 23:37:27 +01009401 if (io_cqring_events(ctx) || test_bit(0, &ctx->check_cq_overflow))
Jens Axboe2b188cc2019-01-07 10:46:33 -07009402 mask |= EPOLLIN | EPOLLRDNORM;
9403
9404 return mask;
9405}
9406
Yejune Deng0bead8c2020-12-24 11:02:20 +08009407static int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id)
Jens Axboe071698e2020-01-28 10:04:42 -07009408{
Jens Axboe4379bf82021-02-15 13:40:22 -07009409 const struct cred *creds;
Jens Axboe071698e2020-01-28 10:04:42 -07009410
Matthew Wilcox (Oracle)61cf9372021-03-08 14:16:16 +00009411 creds = xa_erase(&ctx->personalities, id);
Jens Axboe4379bf82021-02-15 13:40:22 -07009412 if (creds) {
9413 put_cred(creds);
Yejune Deng0bead8c2020-12-24 11:02:20 +08009414 return 0;
Jens Axboe1e6fa522020-10-15 08:46:24 -06009415 }
Yejune Deng0bead8c2020-12-24 11:02:20 +08009416
9417 return -EINVAL;
9418}
9419
Pavel Begunkovd56d9382021-03-06 11:02:13 +00009420struct io_tctx_exit {
9421 struct callback_head task_work;
9422 struct completion completion;
Pavel Begunkovbaf186c2021-03-06 11:02:15 +00009423 struct io_ring_ctx *ctx;
Pavel Begunkovd56d9382021-03-06 11:02:13 +00009424};
9425
9426static void io_tctx_exit_cb(struct callback_head *cb)
9427{
9428 struct io_uring_task *tctx = current->io_uring;
9429 struct io_tctx_exit *work;
9430
9431 work = container_of(cb, struct io_tctx_exit, task_work);
9432 /*
9433 * When @in_idle, we're in cancellation and it's racy to remove the
9434 * node. It'll be removed by the end of cancellation, just ignore it.
9435 */
9436 if (!atomic_read(&tctx->in_idle))
Pavel Begunkoveef51da2021-06-14 02:36:15 +01009437 io_uring_del_tctx_node((unsigned long)work->ctx);
Pavel Begunkovd56d9382021-03-06 11:02:13 +00009438 complete(&work->completion);
9439}
9440
Pavel Begunkov28090c12021-04-25 23:34:45 +01009441static bool io_cancel_ctx_cb(struct io_wq_work *work, void *data)
9442{
9443 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
9444
9445 return req->ctx == data;
9446}
9447
Jens Axboe85faa7b2020-04-09 18:14:00 -06009448static void io_ring_exit_work(struct work_struct *work)
9449{
Pavel Begunkovd56d9382021-03-06 11:02:13 +00009450 struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx, exit_work);
Pavel Begunkovb5bb3a22021-03-06 11:02:16 +00009451 unsigned long timeout = jiffies + HZ * 60 * 5;
Pavel Begunkov58d3be22021-08-09 13:04:17 +01009452 unsigned long interval = HZ / 20;
Pavel Begunkovd56d9382021-03-06 11:02:13 +00009453 struct io_tctx_exit exit;
9454 struct io_tctx_node *node;
9455 int ret;
Jens Axboe85faa7b2020-04-09 18:14:00 -06009456
Jens Axboe56952e92020-06-17 15:00:04 -06009457 /*
9458 * If we're doing polled IO and end up having requests being
9459 * submitted async (out-of-line), then completions can come in while
9460 * we're waiting for refs to drop. We need to reap these manually,
9461 * as nobody else will be looking for them.
9462 */
Pavel Begunkovb2edc0a2020-07-07 16:36:22 +03009463 do {
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01009464 io_uring_try_cancel_requests(ctx, NULL, true);
Pavel Begunkov28090c12021-04-25 23:34:45 +01009465 if (ctx->sq_data) {
9466 struct io_sq_data *sqd = ctx->sq_data;
9467 struct task_struct *tsk;
9468
9469 io_sq_thread_park(sqd);
9470 tsk = sqd->thread;
9471 if (tsk && tsk->io_uring && tsk->io_uring->io_wq)
9472 io_wq_cancel_cb(tsk->io_uring->io_wq,
9473 io_cancel_ctx_cb, ctx, true);
9474 io_sq_thread_unpark(sqd);
9475 }
Pavel Begunkovb5bb3a22021-03-06 11:02:16 +00009476
Pavel Begunkov58d3be22021-08-09 13:04:17 +01009477 if (WARN_ON_ONCE(time_after(jiffies, timeout))) {
9478 /* there is little hope left, don't run it too often */
9479 interval = HZ * 60;
9480 }
9481 } while (!wait_for_completion_timeout(&ctx->ref_comp, interval));
Pavel Begunkovd56d9382021-03-06 11:02:13 +00009482
Pavel Begunkov7f006512021-04-14 13:38:34 +01009483 init_completion(&exit.completion);
9484 init_task_work(&exit.task_work, io_tctx_exit_cb);
9485 exit.ctx = ctx;
Pavel Begunkov89b50662021-04-01 15:43:50 +01009486 /*
9487 * Some may use context even when all refs and requests have been put,
9488 * and they are free to do so while still holding uring_lock or
Pavel Begunkov5b0a6ac2021-06-30 21:54:04 +01009489 * completion_lock, see io_req_task_submit(). Apart from other work,
Pavel Begunkov89b50662021-04-01 15:43:50 +01009490 * this lock/unlock section also waits them to finish.
9491 */
Pavel Begunkovd56d9382021-03-06 11:02:13 +00009492 mutex_lock(&ctx->uring_lock);
9493 while (!list_empty(&ctx->tctx_list)) {
Pavel Begunkovb5bb3a22021-03-06 11:02:16 +00009494 WARN_ON_ONCE(time_after(jiffies, timeout));
9495
Pavel Begunkovd56d9382021-03-06 11:02:13 +00009496 node = list_first_entry(&ctx->tctx_list, struct io_tctx_node,
9497 ctx_node);
Pavel Begunkov7f006512021-04-14 13:38:34 +01009498 /* don't spin on a single task if cancellation failed */
9499 list_rotate_left(&ctx->tctx_list);
Pavel Begunkovd56d9382021-03-06 11:02:13 +00009500 ret = task_work_add(node->task, &exit.task_work, TWA_SIGNAL);
9501 if (WARN_ON_ONCE(ret))
9502 continue;
9503 wake_up_process(node->task);
9504
9505 mutex_unlock(&ctx->uring_lock);
9506 wait_for_completion(&exit.completion);
Pavel Begunkovd56d9382021-03-06 11:02:13 +00009507 mutex_lock(&ctx->uring_lock);
9508 }
9509 mutex_unlock(&ctx->uring_lock);
Jens Axboe79ebeae2021-08-10 15:18:27 -06009510 spin_lock(&ctx->completion_lock);
9511 spin_unlock(&ctx->completion_lock);
Pavel Begunkovd56d9382021-03-06 11:02:13 +00009512
Jens Axboe85faa7b2020-04-09 18:14:00 -06009513 io_ring_ctx_free(ctx);
9514}
9515
Pavel Begunkov80c4cbd2021-03-25 18:32:43 +00009516/* Returns true if we found and killed one or more timeouts */
9517static bool io_kill_timeouts(struct io_ring_ctx *ctx, struct task_struct *tsk,
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01009518 bool cancel_all)
Pavel Begunkov80c4cbd2021-03-25 18:32:43 +00009519{
9520 struct io_kiocb *req, *tmp;
9521 int canceled = 0;
9522
Jens Axboe79ebeae2021-08-10 15:18:27 -06009523 spin_lock(&ctx->completion_lock);
9524 spin_lock_irq(&ctx->timeout_lock);
Pavel Begunkov80c4cbd2021-03-25 18:32:43 +00009525 list_for_each_entry_safe(req, tmp, &ctx->timeout_list, timeout.list) {
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01009526 if (io_match_task(req, tsk, cancel_all)) {
Pavel Begunkov80c4cbd2021-03-25 18:32:43 +00009527 io_kill_timeout(req, -ECANCELED);
9528 canceled++;
9529 }
9530 }
Jens Axboe79ebeae2021-08-10 15:18:27 -06009531 spin_unlock_irq(&ctx->timeout_lock);
Pavel Begunkov51520422021-03-29 11:39:29 +01009532 if (canceled != 0)
9533 io_commit_cqring(ctx);
Jens Axboe79ebeae2021-08-10 15:18:27 -06009534 spin_unlock(&ctx->completion_lock);
Pavel Begunkov80c4cbd2021-03-25 18:32:43 +00009535 if (canceled != 0)
9536 io_cqring_ev_posted(ctx);
9537 return canceled != 0;
9538}
9539
Jens Axboe2b188cc2019-01-07 10:46:33 -07009540static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx)
9541{
Matthew Wilcox (Oracle)61cf9372021-03-08 14:16:16 +00009542 unsigned long index;
9543 struct creds *creds;
9544
Jens Axboe2b188cc2019-01-07 10:46:33 -07009545 mutex_lock(&ctx->uring_lock);
9546 percpu_ref_kill(&ctx->refs);
Pavel Begunkov634578f2020-12-06 22:22:44 +00009547 if (ctx->rings)
Pavel Begunkov6c2450a2021-02-23 12:40:22 +00009548 __io_cqring_overflow_flush(ctx, true);
Matthew Wilcox (Oracle)61cf9372021-03-08 14:16:16 +00009549 xa_for_each(&ctx->personalities, index, creds)
9550 io_unregister_personality(ctx, index);
Jens Axboe2b188cc2019-01-07 10:46:33 -07009551 mutex_unlock(&ctx->uring_lock);
9552
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01009553 io_kill_timeouts(ctx, NULL, true);
9554 io_poll_remove_all(ctx, NULL, true);
Jens Axboe561fb042019-10-24 07:25:42 -06009555
Jens Axboe15dff282019-11-13 09:09:23 -07009556 /* if we failed setting up the ctx, we might not have any rings */
Pavel Begunkovb2edc0a2020-07-07 16:36:22 +03009557 io_iopoll_try_reap_events(ctx);
Jens Axboe309fc032020-07-10 09:13:34 -06009558
Jens Axboe85faa7b2020-04-09 18:14:00 -06009559 INIT_WORK(&ctx->exit_work, io_ring_exit_work);
Jens Axboefc666772020-08-19 11:10:51 -06009560 /*
9561 * Use system_unbound_wq to avoid spawning tons of event kworkers
9562 * if we're exiting a ton of rings at the same time. It just adds
9563 * noise and overhead, there's no discernable change in runtime
9564 * over using system_wq.
9565 */
9566 queue_work(system_unbound_wq, &ctx->exit_work);
Jens Axboe2b188cc2019-01-07 10:46:33 -07009567}
9568
9569static int io_uring_release(struct inode *inode, struct file *file)
9570{
9571 struct io_ring_ctx *ctx = file->private_data;
9572
9573 file->private_data = NULL;
9574 io_ring_ctx_wait_and_kill(ctx);
9575 return 0;
9576}
9577
Pavel Begunkovf6edbab2020-11-06 13:00:26 +00009578struct io_task_cancel {
9579 struct task_struct *task;
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01009580 bool all;
Pavel Begunkovf6edbab2020-11-06 13:00:26 +00009581};
Pavel Begunkov67c4d9e2020-06-15 10:24:05 +03009582
Pavel Begunkovf6edbab2020-11-06 13:00:26 +00009583static bool io_cancel_task_cb(struct io_wq_work *work, void *data)
Jens Axboeb711d4e2020-08-16 08:23:05 -07009584{
Pavel Begunkov9a472ef2020-11-05 22:31:37 +00009585 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
Pavel Begunkovf6edbab2020-11-06 13:00:26 +00009586 struct io_task_cancel *cancel = data;
Pavel Begunkov9a472ef2020-11-05 22:31:37 +00009587
Pavel Begunkov1c939a52021-11-26 14:38:15 +00009588 return io_match_task_safe(req, cancel->task, cancel->all);
Jens Axboeb711d4e2020-08-16 08:23:05 -07009589}
9590
Pavel Begunkove1915f72021-03-11 23:29:35 +00009591static bool io_cancel_defer_files(struct io_ring_ctx *ctx,
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01009592 struct task_struct *task, bool cancel_all)
Pavel Begunkovb7ddce32020-09-06 00:45:14 +03009593{
Pavel Begunkove1915f72021-03-11 23:29:35 +00009594 struct io_defer_entry *de;
Pavel Begunkovb7ddce32020-09-06 00:45:14 +03009595 LIST_HEAD(list);
9596
Jens Axboe79ebeae2021-08-10 15:18:27 -06009597 spin_lock(&ctx->completion_lock);
Pavel Begunkovb7ddce32020-09-06 00:45:14 +03009598 list_for_each_entry_reverse(de, &ctx->defer_list, list) {
Pavel Begunkov1c939a52021-11-26 14:38:15 +00009599 if (io_match_task_safe(de->req, task, cancel_all)) {
Pavel Begunkovb7ddce32020-09-06 00:45:14 +03009600 list_cut_position(&list, &ctx->defer_list, &de->list);
9601 break;
9602 }
9603 }
Jens Axboe79ebeae2021-08-10 15:18:27 -06009604 spin_unlock(&ctx->completion_lock);
Pavel Begunkove1915f72021-03-11 23:29:35 +00009605 if (list_empty(&list))
9606 return false;
Pavel Begunkovb7ddce32020-09-06 00:45:14 +03009607
9608 while (!list_empty(&list)) {
9609 de = list_first_entry(&list, struct io_defer_entry, list);
9610 list_del_init(&de->list);
Pavel Begunkovf41db2732021-02-28 22:35:12 +00009611 io_req_complete_failed(de->req, -ECANCELED);
Pavel Begunkovb7ddce32020-09-06 00:45:14 +03009612 kfree(de);
9613 }
Pavel Begunkove1915f72021-03-11 23:29:35 +00009614 return true;
Pavel Begunkovb7ddce32020-09-06 00:45:14 +03009615}
9616
Pavel Begunkov1b007642021-03-06 11:02:17 +00009617static bool io_uring_try_cancel_iowq(struct io_ring_ctx *ctx)
9618{
9619 struct io_tctx_node *node;
9620 enum io_wq_cancel cret;
9621 bool ret = false;
9622
9623 mutex_lock(&ctx->uring_lock);
9624 list_for_each_entry(node, &ctx->tctx_list, ctx_node) {
9625 struct io_uring_task *tctx = node->task->io_uring;
9626
9627 /*
9628 * io_wq will stay alive while we hold uring_lock, because it's
9629 * killed after ctx nodes, which requires to take the lock.
9630 */
9631 if (!tctx || !tctx->io_wq)
9632 continue;
9633 cret = io_wq_cancel_cb(tctx->io_wq, io_cancel_ctx_cb, ctx, true);
9634 ret |= (cret != IO_WQ_CANCEL_NOTFOUND);
9635 }
9636 mutex_unlock(&ctx->uring_lock);
9637
9638 return ret;
9639}
9640
Pavel Begunkov9936c7c2021-02-04 13:51:56 +00009641static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
9642 struct task_struct *task,
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01009643 bool cancel_all)
Pavel Begunkov9936c7c2021-02-04 13:51:56 +00009644{
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01009645 struct io_task_cancel cancel = { .task = task, .all = cancel_all, };
Pavel Begunkov1b007642021-03-06 11:02:17 +00009646 struct io_uring_task *tctx = task ? task->io_uring : NULL;
Pavel Begunkov9936c7c2021-02-04 13:51:56 +00009647
9648 while (1) {
9649 enum io_wq_cancel cret;
9650 bool ret = false;
9651
Pavel Begunkov1b007642021-03-06 11:02:17 +00009652 if (!task) {
9653 ret |= io_uring_try_cancel_iowq(ctx);
9654 } else if (tctx && tctx->io_wq) {
9655 /*
9656 * Cancels requests of all rings, not only @ctx, but
9657 * it's fine as the task is in exit/exec.
9658 */
Jens Axboe5aa75ed2021-02-16 12:56:50 -07009659 cret = io_wq_cancel_cb(tctx->io_wq, io_cancel_task_cb,
Pavel Begunkov9936c7c2021-02-04 13:51:56 +00009660 &cancel, true);
9661 ret |= (cret != IO_WQ_CANCEL_NOTFOUND);
9662 }
9663
9664 /* SQPOLL thread does its own polling */
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01009665 if ((!(ctx->flags & IORING_SETUP_SQPOLL) && cancel_all) ||
Jens Axboed052d1d2021-03-11 10:49:20 -07009666 (ctx->sq_data && ctx->sq_data->thread == current)) {
Pavel Begunkov9936c7c2021-02-04 13:51:56 +00009667 while (!list_empty_careful(&ctx->iopoll_list)) {
9668 io_iopoll_try_reap_events(ctx);
9669 ret = true;
9670 }
9671 }
9672
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01009673 ret |= io_cancel_defer_files(ctx, task, cancel_all);
9674 ret |= io_poll_remove_all(ctx, task, cancel_all);
9675 ret |= io_kill_timeouts(ctx, task, cancel_all);
Pavel Begunkove5dc4802021-06-26 21:40:46 +01009676 if (task)
9677 ret |= io_run_task_work();
Pavel Begunkov9936c7c2021-02-04 13:51:56 +00009678 if (!ret)
9679 break;
9680 cond_resched();
9681 }
9682}
9683
Pavel Begunkoveef51da2021-06-14 02:36:15 +01009684static int __io_uring_add_tctx_node(struct io_ring_ctx *ctx)
Jens Axboe0f212202020-09-13 13:09:39 -06009685{
Matthew Wilcox (Oracle)236434c2020-10-09 13:49:52 +01009686 struct io_uring_task *tctx = current->io_uring;
Pavel Begunkov13bf43f2021-03-06 11:02:12 +00009687 struct io_tctx_node *node;
Pavel Begunkova528b042020-12-21 18:34:04 +00009688 int ret;
Matthew Wilcox (Oracle)236434c2020-10-09 13:49:52 +01009689
9690 if (unlikely(!tctx)) {
Jens Axboe5aa75ed2021-02-16 12:56:50 -07009691 ret = io_uring_alloc_task_context(current, ctx);
Jens Axboe0f212202020-09-13 13:09:39 -06009692 if (unlikely(ret))
9693 return ret;
Pavel Begunkove139a1e2021-10-19 23:43:46 +01009694
Matthew Wilcox (Oracle)236434c2020-10-09 13:49:52 +01009695 tctx = current->io_uring;
Pavel Begunkove139a1e2021-10-19 23:43:46 +01009696 if (ctx->iowq_limits_set) {
9697 unsigned int limits[2] = { ctx->iowq_limits[0],
9698 ctx->iowq_limits[1], };
9699
9700 ret = io_wq_max_workers(tctx->io_wq, limits);
9701 if (ret)
9702 return ret;
9703 }
Jens Axboe0f212202020-09-13 13:09:39 -06009704 }
Pavel Begunkovcf27f3b2021-03-19 17:22:31 +00009705 if (!xa_load(&tctx->xa, (unsigned long)ctx)) {
9706 node = kmalloc(sizeof(*node), GFP_KERNEL);
9707 if (!node)
9708 return -ENOMEM;
9709 node->ctx = ctx;
9710 node->task = current;
Jens Axboe0f212202020-09-13 13:09:39 -06009711
Pavel Begunkovcf27f3b2021-03-19 17:22:31 +00009712 ret = xa_err(xa_store(&tctx->xa, (unsigned long)ctx,
9713 node, GFP_KERNEL));
9714 if (ret) {
9715 kfree(node);
9716 return ret;
Jens Axboe0f212202020-09-13 13:09:39 -06009717 }
Pavel Begunkovcf27f3b2021-03-19 17:22:31 +00009718
9719 mutex_lock(&ctx->uring_lock);
9720 list_add(&node->ctx_node, &ctx->tctx_list);
9721 mutex_unlock(&ctx->uring_lock);
Jens Axboe0f212202020-09-13 13:09:39 -06009722 }
Pavel Begunkovcf27f3b2021-03-19 17:22:31 +00009723 tctx->last = ctx;
Jens Axboe0f212202020-09-13 13:09:39 -06009724 return 0;
9725}
9726
9727/*
Pavel Begunkovcf27f3b2021-03-19 17:22:31 +00009728 * Note that this task has used io_uring. We use it for cancelation purposes.
9729 */
Pavel Begunkoveef51da2021-06-14 02:36:15 +01009730static inline int io_uring_add_tctx_node(struct io_ring_ctx *ctx)
Pavel Begunkovcf27f3b2021-03-19 17:22:31 +00009731{
9732 struct io_uring_task *tctx = current->io_uring;
9733
9734 if (likely(tctx && tctx->last == ctx))
9735 return 0;
Pavel Begunkoveef51da2021-06-14 02:36:15 +01009736 return __io_uring_add_tctx_node(ctx);
Pavel Begunkovcf27f3b2021-03-19 17:22:31 +00009737}
9738
9739/*
Jens Axboe0f212202020-09-13 13:09:39 -06009740 * Remove this io_uring_file -> task mapping.
9741 */
Pavel Begunkoveef51da2021-06-14 02:36:15 +01009742static void io_uring_del_tctx_node(unsigned long index)
Jens Axboe0f212202020-09-13 13:09:39 -06009743{
9744 struct io_uring_task *tctx = current->io_uring;
Pavel Begunkov13bf43f2021-03-06 11:02:12 +00009745 struct io_tctx_node *node;
Pavel Begunkov29412672021-03-06 11:02:11 +00009746
Pavel Begunkoveebd2e32021-03-06 11:02:14 +00009747 if (!tctx)
9748 return;
Pavel Begunkov13bf43f2021-03-06 11:02:12 +00009749 node = xa_erase(&tctx->xa, index);
9750 if (!node)
Pavel Begunkov29412672021-03-06 11:02:11 +00009751 return;
Jens Axboe0f212202020-09-13 13:09:39 -06009752
Pavel Begunkov13bf43f2021-03-06 11:02:12 +00009753 WARN_ON_ONCE(current != node->task);
9754 WARN_ON_ONCE(list_empty(&node->ctx_node));
9755
9756 mutex_lock(&node->ctx->uring_lock);
9757 list_del(&node->ctx_node);
9758 mutex_unlock(&node->ctx->uring_lock);
9759
Pavel Begunkovbaf186c2021-03-06 11:02:15 +00009760 if (tctx->last == node->ctx)
Jens Axboe0f212202020-09-13 13:09:39 -06009761 tctx->last = NULL;
Pavel Begunkov13bf43f2021-03-06 11:02:12 +00009762 kfree(node);
Jens Axboe0f212202020-09-13 13:09:39 -06009763}
9764
Pavel Begunkov8452d4a2021-02-27 11:16:46 +00009765static void io_uring_clean_tctx(struct io_uring_task *tctx)
Pavel Begunkovde7f1d92021-01-04 20:43:29 +00009766{
Pavel Begunkovba5ef6d2021-05-20 13:21:20 +01009767 struct io_wq *wq = tctx->io_wq;
Pavel Begunkov13bf43f2021-03-06 11:02:12 +00009768 struct io_tctx_node *node;
Pavel Begunkovde7f1d92021-01-04 20:43:29 +00009769 unsigned long index;
9770
Jens Axboe8bab4c02021-09-24 07:12:27 -06009771 xa_for_each(&tctx->xa, index, node) {
Pavel Begunkoveef51da2021-06-14 02:36:15 +01009772 io_uring_del_tctx_node(index);
Jens Axboe8bab4c02021-09-24 07:12:27 -06009773 cond_resched();
9774 }
Marco Elverb16ef422021-05-27 11:25:48 +02009775 if (wq) {
9776 /*
9777 * Must be after io_uring_del_task_file() (removes nodes under
9778 * uring_lock) to avoid race with io_uring_try_cancel_iowq().
9779 */
Pavel Begunkovba5ef6d2021-05-20 13:21:20 +01009780 io_wq_put_and_exit(wq);
Pavel Begunkovdadebc32021-08-23 13:30:44 +01009781 tctx->io_wq = NULL;
Marco Elverb16ef422021-05-27 11:25:48 +02009782 }
Pavel Begunkovde7f1d92021-01-04 20:43:29 +00009783}
9784
Pavel Begunkov3f48cf12021-04-11 01:46:27 +01009785static s64 tctx_inflight(struct io_uring_task *tctx, bool tracked)
Pavel Begunkov521d6a72021-03-11 23:29:38 +00009786{
Pavel Begunkov3f48cf12021-04-11 01:46:27 +01009787 if (tracked)
9788 return atomic_read(&tctx->inflight_tracked);
Pavel Begunkov521d6a72021-03-11 23:29:38 +00009789 return percpu_counter_sum(&tctx->inflight);
9790}
9791
Pavel Begunkov78cc6872021-06-14 02:36:23 +01009792/*
9793 * Find any io_uring ctx that this task has registered or done IO on, and cancel
Jens Axboe8e129762021-12-09 08:54:29 -07009794 * requests. @sqd should be not-null IFF it's an SQPOLL thread cancellation.
Pavel Begunkov78cc6872021-06-14 02:36:23 +01009795 */
9796static void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd)
Pavel Begunkov0e9ddb32021-02-07 22:34:26 +00009797{
Pavel Begunkov521d6a72021-03-11 23:29:38 +00009798 struct io_uring_task *tctx = current->io_uring;
Pavel Begunkov734551d2021-04-18 14:52:09 +01009799 struct io_ring_ctx *ctx;
Jens Axboefdaf0832020-10-30 09:37:30 -06009800 s64 inflight;
Pavel Begunkov0e9ddb32021-02-07 22:34:26 +00009801 DEFINE_WAIT(wait);
Jens Axboefdaf0832020-10-30 09:37:30 -06009802
Pavel Begunkov78cc6872021-06-14 02:36:23 +01009803 WARN_ON_ONCE(sqd && sqd->thread != current);
9804
Palash Oswal6d042ff2021-04-27 18:21:49 +05309805 if (!current->io_uring)
9806 return;
Pavel Begunkov17a91052021-05-23 15:48:39 +01009807 if (tctx->io_wq)
9808 io_wq_exit_start(tctx->io_wq);
9809
Jens Axboefdaf0832020-10-30 09:37:30 -06009810 atomic_inc(&tctx->in_idle);
Jens Axboed8a6df12020-10-15 16:24:45 -06009811 do {
Pavel Begunkove9dbe222021-08-09 13:04:20 +01009812 io_uring_drop_tctx_refs(current);
Jens Axboe0f212202020-09-13 13:09:39 -06009813 /* read completions before cancelations */
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01009814 inflight = tctx_inflight(tctx, !cancel_all);
Jens Axboed8a6df12020-10-15 16:24:45 -06009815 if (!inflight)
9816 break;
Jens Axboe0f212202020-09-13 13:09:39 -06009817
Pavel Begunkov78cc6872021-06-14 02:36:23 +01009818 if (!sqd) {
9819 struct io_tctx_node *node;
9820 unsigned long index;
9821
9822 xa_for_each(&tctx->xa, index, node) {
9823 /* sqpoll task will cancel all its requests */
9824 if (node->ctx->sq_data)
9825 continue;
9826 io_uring_try_cancel_requests(node->ctx, current,
9827 cancel_all);
9828 }
9829 } else {
9830 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list)
9831 io_uring_try_cancel_requests(ctx, current,
9832 cancel_all);
9833 }
9834
Jens Axboe8e129762021-12-09 08:54:29 -07009835 prepare_to_wait(&tctx->wait, &wait, TASK_INTERRUPTIBLE);
9836 io_run_task_work();
Pavel Begunkove9dbe222021-08-09 13:04:20 +01009837 io_uring_drop_tctx_refs(current);
Jens Axboe8e129762021-12-09 08:54:29 -07009838
Jens Axboe0f212202020-09-13 13:09:39 -06009839 /*
Pavel Begunkova1bb3cd2021-01-26 15:28:26 +00009840 * If we've seen completions, retry without waiting. This
9841 * avoids a race where a completion comes in before we did
9842 * prepare_to_wait().
Jens Axboe0f212202020-09-13 13:09:39 -06009843 */
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01009844 if (inflight == tctx_inflight(tctx, !cancel_all))
Pavel Begunkova1bb3cd2021-01-26 15:28:26 +00009845 schedule();
Pavel Begunkovf57555e2020-12-20 13:21:44 +00009846 finish_wait(&tctx->wait, &wait);
Jens Axboed8a6df12020-10-15 16:24:45 -06009847 } while (1);
Pavel Begunkovde7f1d92021-01-04 20:43:29 +00009848
Pavel Begunkov8452d4a2021-02-27 11:16:46 +00009849 io_uring_clean_tctx(tctx);
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01009850 if (cancel_all) {
Pavel Begunkovb168b1a2022-01-09 00:53:22 +00009851 /*
9852 * We shouldn't run task_works after cancel, so just leave
9853 * ->in_idle set for normal exit.
9854 */
9855 atomic_dec(&tctx->in_idle);
Pavel Begunkov3f48cf12021-04-11 01:46:27 +01009856 /* for exec all current's requests should be gone, kill tctx */
9857 __io_uring_free(current);
9858 }
Pavel Begunkov44e728b2020-06-15 10:24:04 +03009859}
9860
Hao Xuf552a272021-08-12 12:14:35 +08009861void __io_uring_cancel(bool cancel_all)
Pavel Begunkov78cc6872021-06-14 02:36:23 +01009862{
Hao Xuf552a272021-08-12 12:14:35 +08009863 io_uring_cancel_generic(cancel_all, NULL);
Pavel Begunkov78cc6872021-06-14 02:36:23 +01009864}
9865
Roman Penyaev6c5c2402019-11-28 12:53:22 +01009866static void *io_uring_validate_mmap_request(struct file *file,
9867 loff_t pgoff, size_t sz)
Jens Axboe2b188cc2019-01-07 10:46:33 -07009868{
Jens Axboe2b188cc2019-01-07 10:46:33 -07009869 struct io_ring_ctx *ctx = file->private_data;
Roman Penyaev6c5c2402019-11-28 12:53:22 +01009870 loff_t offset = pgoff << PAGE_SHIFT;
Jens Axboe2b188cc2019-01-07 10:46:33 -07009871 struct page *page;
9872 void *ptr;
9873
9874 switch (offset) {
9875 case IORING_OFF_SQ_RING:
Hristo Venev75b28af2019-08-26 17:23:46 +00009876 case IORING_OFF_CQ_RING:
9877 ptr = ctx->rings;
Jens Axboe2b188cc2019-01-07 10:46:33 -07009878 break;
9879 case IORING_OFF_SQES:
9880 ptr = ctx->sq_sqes;
9881 break;
Jens Axboe2b188cc2019-01-07 10:46:33 -07009882 default:
Roman Penyaev6c5c2402019-11-28 12:53:22 +01009883 return ERR_PTR(-EINVAL);
Jens Axboe2b188cc2019-01-07 10:46:33 -07009884 }
9885
9886 page = virt_to_head_page(ptr);
Matthew Wilcox (Oracle)a50b8542019-09-23 15:34:25 -07009887 if (sz > page_size(page))
Roman Penyaev6c5c2402019-11-28 12:53:22 +01009888 return ERR_PTR(-EINVAL);
9889
9890 return ptr;
9891}
9892
9893#ifdef CONFIG_MMU
9894
9895static int io_uring_mmap(struct file *file, struct vm_area_struct *vma)
9896{
9897 size_t sz = vma->vm_end - vma->vm_start;
9898 unsigned long pfn;
9899 void *ptr;
9900
9901 ptr = io_uring_validate_mmap_request(file, vma->vm_pgoff, sz);
9902 if (IS_ERR(ptr))
9903 return PTR_ERR(ptr);
Jens Axboe2b188cc2019-01-07 10:46:33 -07009904
9905 pfn = virt_to_phys(ptr) >> PAGE_SHIFT;
9906 return remap_pfn_range(vma, vma->vm_start, pfn, sz, vma->vm_page_prot);
9907}
9908
Roman Penyaev6c5c2402019-11-28 12:53:22 +01009909#else /* !CONFIG_MMU */
9910
9911static int io_uring_mmap(struct file *file, struct vm_area_struct *vma)
9912{
9913 return vma->vm_flags & (VM_SHARED | VM_MAYSHARE) ? 0 : -EINVAL;
9914}
9915
9916static unsigned int io_uring_nommu_mmap_capabilities(struct file *file)
9917{
9918 return NOMMU_MAP_DIRECT | NOMMU_MAP_READ | NOMMU_MAP_WRITE;
9919}
9920
9921static unsigned long io_uring_nommu_get_unmapped_area(struct file *file,
9922 unsigned long addr, unsigned long len,
9923 unsigned long pgoff, unsigned long flags)
9924{
9925 void *ptr;
9926
9927 ptr = io_uring_validate_mmap_request(file, pgoff, len);
9928 if (IS_ERR(ptr))
9929 return PTR_ERR(ptr);
9930
9931 return (unsigned long) ptr;
9932}
9933
9934#endif /* !CONFIG_MMU */
9935
Pavel Begunkovd9d05212021-01-08 20:57:25 +00009936static int io_sqpoll_wait_sq(struct io_ring_ctx *ctx)
Jens Axboe90554202020-09-03 12:12:41 -06009937{
9938 DEFINE_WAIT(wait);
9939
9940 do {
9941 if (!io_sqring_full(ctx))
9942 break;
Jens Axboe90554202020-09-03 12:12:41 -06009943 prepare_to_wait(&ctx->sqo_sq_wait, &wait, TASK_INTERRUPTIBLE);
9944
9945 if (!io_sqring_full(ctx))
9946 break;
Jens Axboe90554202020-09-03 12:12:41 -06009947 schedule();
9948 } while (!signal_pending(current));
9949
9950 finish_wait(&ctx->sqo_sq_wait, &wait);
Yang Li51993282021-03-09 14:30:41 +08009951 return 0;
Jens Axboe90554202020-09-03 12:12:41 -06009952}
9953
Hao Xuc73ebb62020-11-03 10:54:37 +08009954static int io_get_ext_arg(unsigned flags, const void __user *argp, size_t *argsz,
9955 struct __kernel_timespec __user **ts,
9956 const sigset_t __user **sig)
9957{
9958 struct io_uring_getevents_arg arg;
9959
9960 /*
9961 * If EXT_ARG isn't set, then we have no timespec and the argp pointer
9962 * is just a pointer to the sigset_t.
9963 */
9964 if (!(flags & IORING_ENTER_EXT_ARG)) {
9965 *sig = (const sigset_t __user *) argp;
9966 *ts = NULL;
9967 return 0;
9968 }
9969
9970 /*
9971 * EXT_ARG is set - ensure we agree on the size of it and copy in our
9972 * timespec and sigset_t pointers if good.
9973 */
9974 if (*argsz != sizeof(arg))
9975 return -EINVAL;
9976 if (copy_from_user(&arg, argp, sizeof(arg)))
9977 return -EFAULT;
Dylan Yudaken99475482022-04-12 09:30:42 -07009978 if (arg.pad)
9979 return -EINVAL;
Hao Xuc73ebb62020-11-03 10:54:37 +08009980 *sig = u64_to_user_ptr(arg.sigmask);
9981 *argsz = arg.sigmask_sz;
9982 *ts = u64_to_user_ptr(arg.ts);
9983 return 0;
9984}
9985
Jens Axboe2b188cc2019-01-07 10:46:33 -07009986SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
Hao Xuc73ebb62020-11-03 10:54:37 +08009987 u32, min_complete, u32, flags, const void __user *, argp,
9988 size_t, argsz)
Jens Axboe2b188cc2019-01-07 10:46:33 -07009989{
9990 struct io_ring_ctx *ctx;
Jens Axboe2b188cc2019-01-07 10:46:33 -07009991 int submitted = 0;
9992 struct fd f;
Pavel Begunkov33f993d2021-03-19 17:22:30 +00009993 long ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07009994
Jens Axboe4c6e2772020-07-01 11:29:10 -06009995 io_run_task_work();
Jens Axboeb41e9852020-02-17 09:52:41 -07009996
Pavel Begunkov33f993d2021-03-19 17:22:30 +00009997 if (unlikely(flags & ~(IORING_ENTER_GETEVENTS | IORING_ENTER_SQ_WAKEUP |
9998 IORING_ENTER_SQ_WAIT | IORING_ENTER_EXT_ARG)))
Jens Axboe2b188cc2019-01-07 10:46:33 -07009999 return -EINVAL;
10000
10001 f = fdget(fd);
Pavel Begunkov33f993d2021-03-19 17:22:30 +000010002 if (unlikely(!f.file))
Jens Axboe2b188cc2019-01-07 10:46:33 -070010003 return -EBADF;
10004
10005 ret = -EOPNOTSUPP;
Pavel Begunkov33f993d2021-03-19 17:22:30 +000010006 if (unlikely(f.file->f_op != &io_uring_fops))
Jens Axboe2b188cc2019-01-07 10:46:33 -070010007 goto out_fput;
10008
10009 ret = -ENXIO;
10010 ctx = f.file->private_data;
Pavel Begunkov33f993d2021-03-19 17:22:30 +000010011 if (unlikely(!percpu_ref_tryget(&ctx->refs)))
Jens Axboe2b188cc2019-01-07 10:46:33 -070010012 goto out_fput;
10013
Stefano Garzarella7e84e1c2020-08-27 16:58:31 +020010014 ret = -EBADFD;
Pavel Begunkov33f993d2021-03-19 17:22:30 +000010015 if (unlikely(ctx->flags & IORING_SETUP_R_DISABLED))
Stefano Garzarella7e84e1c2020-08-27 16:58:31 +020010016 goto out;
10017
Jens Axboe6c271ce2019-01-10 11:22:30 -070010018 /*
10019 * For SQ polling, the thread will do all submissions and completions.
10020 * Just return the requested submit count, and wake the thread if
10021 * we were asked to.
10022 */
Jens Axboeb2a9ead2019-09-12 14:19:16 -060010023 ret = 0;
Jens Axboe6c271ce2019-01-10 11:22:30 -070010024 if (ctx->flags & IORING_SETUP_SQPOLL) {
Pavel Begunkov90f67362021-08-09 20:18:12 +010010025 io_cqring_overflow_flush(ctx);
Pavel Begunkov89448c42020-12-17 00:24:39 +000010026
Jens Axboe21f96522021-08-14 09:04:40 -060010027 if (unlikely(ctx->sq_data->thread == NULL)) {
10028 ret = -EOWNERDEAD;
Stefan Metzmacher04147482021-03-07 11:54:29 +010010029 goto out;
Jens Axboe21f96522021-08-14 09:04:40 -060010030 }
Jens Axboe6c271ce2019-01-10 11:22:30 -070010031 if (flags & IORING_ENTER_SQ_WAKEUP)
Jens Axboe534ca6d2020-09-02 13:52:19 -060010032 wake_up(&ctx->sq_data->wait);
Pavel Begunkovd9d05212021-01-08 20:57:25 +000010033 if (flags & IORING_ENTER_SQ_WAIT) {
10034 ret = io_sqpoll_wait_sq(ctx);
10035 if (ret)
10036 goto out;
10037 }
Jens Axboe6c271ce2019-01-10 11:22:30 -070010038 submitted = to_submit;
Jens Axboeb2a9ead2019-09-12 14:19:16 -060010039 } else if (to_submit) {
Pavel Begunkoveef51da2021-06-14 02:36:15 +010010040 ret = io_uring_add_tctx_node(ctx);
Jens Axboe0f212202020-09-13 13:09:39 -060010041 if (unlikely(ret))
10042 goto out;
Jens Axboe2b188cc2019-01-07 10:46:33 -070010043 mutex_lock(&ctx->uring_lock);
Jens Axboe0f212202020-09-13 13:09:39 -060010044 submitted = io_submit_sqes(ctx, to_submit);
Jens Axboe2b188cc2019-01-07 10:46:33 -070010045 mutex_unlock(&ctx->uring_lock);
Pavel Begunkov7c504e652019-12-18 19:53:45 +030010046
10047 if (submitted != to_submit)
10048 goto out;
Jens Axboe2b188cc2019-01-07 10:46:33 -070010049 }
10050 if (flags & IORING_ENTER_GETEVENTS) {
Hao Xuc73ebb62020-11-03 10:54:37 +080010051 const sigset_t __user *sig;
10052 struct __kernel_timespec __user *ts;
10053
10054 ret = io_get_ext_arg(flags, argp, &argsz, &ts, &sig);
10055 if (unlikely(ret))
10056 goto out;
10057
Jens Axboe2b188cc2019-01-07 10:46:33 -070010058 min_complete = min(min_complete, ctx->cq_entries);
10059
Xiaoguang Wang32b22442020-03-11 09:26:09 +080010060 /*
10061 * When SETUP_IOPOLL and SETUP_SQPOLL are both enabled, user
10062 * space applications don't need to do io completion events
10063 * polling again, they can rely on io_sq_thread to do polling
10064 * work, which can reduce cpu usage and uring_lock contention.
10065 */
10066 if (ctx->flags & IORING_SETUP_IOPOLL &&
10067 !(ctx->flags & IORING_SETUP_SQPOLL)) {
Pavel Begunkov7668b922020-07-07 16:36:21 +030010068 ret = io_iopoll_check(ctx, min_complete);
Jens Axboedef596e2019-01-09 08:59:42 -070010069 } else {
Hao Xuc73ebb62020-11-03 10:54:37 +080010070 ret = io_cqring_wait(ctx, min_complete, sig, argsz, ts);
Jens Axboedef596e2019-01-09 08:59:42 -070010071 }
Jens Axboe2b188cc2019-01-07 10:46:33 -070010072 }
10073
Pavel Begunkov7c504e652019-12-18 19:53:45 +030010074out:
Pavel Begunkov6805b322019-10-08 02:18:42 +030010075 percpu_ref_put(&ctx->refs);
Jens Axboe2b188cc2019-01-07 10:46:33 -070010076out_fput:
10077 fdput(f);
10078 return submitted ? submitted : ret;
10079}
10080
Tobias Klauserbebdb652020-02-26 18:38:32 +010010081#ifdef CONFIG_PROC_FS
Matthew Wilcox (Oracle)61cf9372021-03-08 14:16:16 +000010082static int io_uring_show_cred(struct seq_file *m, unsigned int id,
10083 const struct cred *cred)
Jens Axboe87ce9552020-01-30 08:25:34 -070010084{
Jens Axboe87ce9552020-01-30 08:25:34 -070010085 struct user_namespace *uns = seq_user_ns(m);
10086 struct group_info *gi;
10087 kernel_cap_t cap;
10088 unsigned __capi;
10089 int g;
10090
10091 seq_printf(m, "%5d\n", id);
10092 seq_put_decimal_ull(m, "\tUid:\t", from_kuid_munged(uns, cred->uid));
10093 seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->euid));
10094 seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->suid));
10095 seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->fsuid));
10096 seq_put_decimal_ull(m, "\n\tGid:\t", from_kgid_munged(uns, cred->gid));
10097 seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->egid));
10098 seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->sgid));
10099 seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->fsgid));
10100 seq_puts(m, "\n\tGroups:\t");
10101 gi = cred->group_info;
10102 for (g = 0; g < gi->ngroups; g++) {
10103 seq_put_decimal_ull(m, g ? " " : "",
10104 from_kgid_munged(uns, gi->gid[g]));
10105 }
10106 seq_puts(m, "\n\tCapEff:\t");
10107 cap = cred->cap_effective;
10108 CAP_FOR_EACH_U32(__capi)
10109 seq_put_hex_ll(m, NULL, cap.cap[CAP_LAST_U32 - __capi], 8);
10110 seq_putc(m, '\n');
10111 return 0;
10112}
10113
10114static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m)
10115{
Joseph Qidbbe9c62020-09-29 09:01:22 -060010116 struct io_sq_data *sq = NULL;
Jens Axboefad8e0d2020-09-28 08:57:48 -060010117 bool has_lock;
Jens Axboe87ce9552020-01-30 08:25:34 -070010118 int i;
10119
Jens Axboefad8e0d2020-09-28 08:57:48 -060010120 /*
10121 * Avoid ABBA deadlock between the seq lock and the io_uring mutex,
10122 * since fdinfo case grabs it in the opposite direction of normal use
10123 * cases. If we fail to get the lock, we just don't iterate any
10124 * structures that could be going away outside the io_uring mutex.
10125 */
10126 has_lock = mutex_trylock(&ctx->uring_lock);
10127
Jens Axboe5f3f26f2021-02-25 10:17:46 -070010128 if (has_lock && (ctx->flags & IORING_SETUP_SQPOLL)) {
Joseph Qidbbe9c62020-09-29 09:01:22 -060010129 sq = ctx->sq_data;
Jens Axboe5f3f26f2021-02-25 10:17:46 -070010130 if (!sq->thread)
10131 sq = NULL;
10132 }
Joseph Qidbbe9c62020-09-29 09:01:22 -060010133
10134 seq_printf(m, "SqThread:\t%d\n", sq ? task_pid_nr(sq->thread) : -1);
10135 seq_printf(m, "SqThreadCpu:\t%d\n", sq ? task_cpu(sq->thread) : -1);
Jens Axboe87ce9552020-01-30 08:25:34 -070010136 seq_printf(m, "UserFiles:\t%u\n", ctx->nr_user_files);
Jens Axboefad8e0d2020-09-28 08:57:48 -060010137 for (i = 0; has_lock && i < ctx->nr_user_files; i++) {
Jens Axboe7b29f922021-03-12 08:30:14 -070010138 struct file *f = io_file_from_index(ctx, i);
Jens Axboe87ce9552020-01-30 08:25:34 -070010139
Jens Axboe87ce9552020-01-30 08:25:34 -070010140 if (f)
10141 seq_printf(m, "%5u: %s\n", i, file_dentry(f)->d_iname);
10142 else
10143 seq_printf(m, "%5u: <none>\n", i);
10144 }
10145 seq_printf(m, "UserBufs:\t%u\n", ctx->nr_user_bufs);
Jens Axboefad8e0d2020-09-28 08:57:48 -060010146 for (i = 0; has_lock && i < ctx->nr_user_bufs; i++) {
Pavel Begunkov41edf1a2021-04-25 14:32:23 +010010147 struct io_mapped_ubuf *buf = ctx->user_bufs[i];
Pavel Begunkov4751f532021-04-01 15:43:55 +010010148 unsigned int len = buf->ubuf_end - buf->ubuf;
Jens Axboe87ce9552020-01-30 08:25:34 -070010149
Pavel Begunkov4751f532021-04-01 15:43:55 +010010150 seq_printf(m, "%5u: 0x%llx/%u\n", i, buf->ubuf, len);
Jens Axboe87ce9552020-01-30 08:25:34 -070010151 }
Matthew Wilcox (Oracle)61cf9372021-03-08 14:16:16 +000010152 if (has_lock && !xa_empty(&ctx->personalities)) {
10153 unsigned long index;
10154 const struct cred *cred;
10155
Jens Axboe87ce9552020-01-30 08:25:34 -070010156 seq_printf(m, "Personalities:\n");
Matthew Wilcox (Oracle)61cf9372021-03-08 14:16:16 +000010157 xa_for_each(&ctx->personalities, index, cred)
10158 io_uring_show_cred(m, index, cred);
Jens Axboe87ce9552020-01-30 08:25:34 -070010159 }
Jens Axboed7718a92020-02-14 22:23:12 -070010160 seq_printf(m, "PollList:\n");
Jens Axboe79ebeae2021-08-10 15:18:27 -060010161 spin_lock(&ctx->completion_lock);
Jens Axboed7718a92020-02-14 22:23:12 -070010162 for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) {
10163 struct hlist_head *list = &ctx->cancel_hash[i];
10164 struct io_kiocb *req;
10165
10166 hlist_for_each_entry(req, list, hash_node)
10167 seq_printf(m, " op=%d, task_works=%d\n", req->opcode,
10168 req->task->task_works != NULL);
10169 }
Jens Axboe79ebeae2021-08-10 15:18:27 -060010170 spin_unlock(&ctx->completion_lock);
Jens Axboefad8e0d2020-09-28 08:57:48 -060010171 if (has_lock)
10172 mutex_unlock(&ctx->uring_lock);
Jens Axboe87ce9552020-01-30 08:25:34 -070010173}
10174
10175static void io_uring_show_fdinfo(struct seq_file *m, struct file *f)
10176{
10177 struct io_ring_ctx *ctx = f->private_data;
10178
10179 if (percpu_ref_tryget(&ctx->refs)) {
10180 __io_uring_show_fdinfo(ctx, m);
10181 percpu_ref_put(&ctx->refs);
10182 }
10183}
Tobias Klauserbebdb652020-02-26 18:38:32 +010010184#endif
Jens Axboe87ce9552020-01-30 08:25:34 -070010185
Jens Axboe2b188cc2019-01-07 10:46:33 -070010186static const struct file_operations io_uring_fops = {
10187 .release = io_uring_release,
10188 .mmap = io_uring_mmap,
Roman Penyaev6c5c2402019-11-28 12:53:22 +010010189#ifndef CONFIG_MMU
10190 .get_unmapped_area = io_uring_nommu_get_unmapped_area,
10191 .mmap_capabilities = io_uring_nommu_mmap_capabilities,
10192#endif
Jens Axboe2b188cc2019-01-07 10:46:33 -070010193 .poll = io_uring_poll,
Tobias Klauserbebdb652020-02-26 18:38:32 +010010194#ifdef CONFIG_PROC_FS
Jens Axboe87ce9552020-01-30 08:25:34 -070010195 .show_fdinfo = io_uring_show_fdinfo,
Tobias Klauserbebdb652020-02-26 18:38:32 +010010196#endif
Jens Axboe2b188cc2019-01-07 10:46:33 -070010197};
10198
10199static int io_allocate_scq_urings(struct io_ring_ctx *ctx,
10200 struct io_uring_params *p)
10201{
Hristo Venev75b28af2019-08-26 17:23:46 +000010202 struct io_rings *rings;
10203 size_t size, sq_array_offset;
Jens Axboe2b188cc2019-01-07 10:46:33 -070010204
Jens Axboebd740482020-08-05 12:58:23 -060010205 /* make sure these are sane, as we already accounted them */
10206 ctx->sq_entries = p->sq_entries;
10207 ctx->cq_entries = p->cq_entries;
10208
Hristo Venev75b28af2019-08-26 17:23:46 +000010209 size = rings_size(p->sq_entries, p->cq_entries, &sq_array_offset);
10210 if (size == SIZE_MAX)
10211 return -EOVERFLOW;
10212
10213 rings = io_mem_alloc(size);
10214 if (!rings)
Jens Axboe2b188cc2019-01-07 10:46:33 -070010215 return -ENOMEM;
10216
Hristo Venev75b28af2019-08-26 17:23:46 +000010217 ctx->rings = rings;
10218 ctx->sq_array = (u32 *)((char *)rings + sq_array_offset);
10219 rings->sq_ring_mask = p->sq_entries - 1;
10220 rings->cq_ring_mask = p->cq_entries - 1;
10221 rings->sq_ring_entries = p->sq_entries;
10222 rings->cq_ring_entries = p->cq_entries;
Jens Axboe2b188cc2019-01-07 10:46:33 -070010223
10224 size = array_size(sizeof(struct io_uring_sqe), p->sq_entries);
Jens Axboeeb065d32019-11-20 09:26:29 -070010225 if (size == SIZE_MAX) {
10226 io_mem_free(ctx->rings);
10227 ctx->rings = NULL;
Jens Axboe2b188cc2019-01-07 10:46:33 -070010228 return -EOVERFLOW;
Jens Axboeeb065d32019-11-20 09:26:29 -070010229 }
Jens Axboe2b188cc2019-01-07 10:46:33 -070010230
10231 ctx->sq_sqes = io_mem_alloc(size);
Jens Axboeeb065d32019-11-20 09:26:29 -070010232 if (!ctx->sq_sqes) {
10233 io_mem_free(ctx->rings);
10234 ctx->rings = NULL;
Jens Axboe2b188cc2019-01-07 10:46:33 -070010235 return -ENOMEM;
Jens Axboeeb065d32019-11-20 09:26:29 -070010236 }
Jens Axboe2b188cc2019-01-07 10:46:33 -070010237
Jens Axboe2b188cc2019-01-07 10:46:33 -070010238 return 0;
10239}
10240
Pavel Begunkov9faadcc2020-12-21 18:34:05 +000010241static int io_uring_install_fd(struct io_ring_ctx *ctx, struct file *file)
10242{
10243 int ret, fd;
10244
10245 fd = get_unused_fd_flags(O_RDWR | O_CLOEXEC);
10246 if (fd < 0)
10247 return fd;
10248
Pavel Begunkoveef51da2021-06-14 02:36:15 +010010249 ret = io_uring_add_tctx_node(ctx);
Pavel Begunkov9faadcc2020-12-21 18:34:05 +000010250 if (ret) {
10251 put_unused_fd(fd);
10252 return ret;
10253 }
10254 fd_install(fd, file);
10255 return fd;
10256}
10257
Jens Axboe2b188cc2019-01-07 10:46:33 -070010258/*
10259 * Allocate an anonymous fd, this is what constitutes the application
10260 * visible backing of an io_uring instance. The application mmaps this
10261 * fd to gain access to the SQ/CQ ring details. If UNIX sockets are enabled,
10262 * we have to tie this fd to a socket for file garbage collection purposes.
10263 */
Pavel Begunkov9faadcc2020-12-21 18:34:05 +000010264static struct file *io_uring_get_file(struct io_ring_ctx *ctx)
Jens Axboe2b188cc2019-01-07 10:46:33 -070010265{
10266 struct file *file;
Jens Axboe2b188cc2019-01-07 10:46:33 -070010267#if defined(CONFIG_UNIX)
Pavel Begunkov9faadcc2020-12-21 18:34:05 +000010268 int ret;
10269
Jens Axboe2b188cc2019-01-07 10:46:33 -070010270 ret = sock_create_kern(&init_net, PF_UNIX, SOCK_RAW, IPPROTO_IP,
10271 &ctx->ring_sock);
10272 if (ret)
Pavel Begunkov9faadcc2020-12-21 18:34:05 +000010273 return ERR_PTR(ret);
Jens Axboe2b188cc2019-01-07 10:46:33 -070010274#endif
10275
Jens Axboe2b188cc2019-01-07 10:46:33 -070010276 file = anon_inode_getfile("[io_uring]", &io_uring_fops, ctx,
10277 O_RDWR | O_CLOEXEC);
Pavel Begunkov9faadcc2020-12-21 18:34:05 +000010278#if defined(CONFIG_UNIX)
Jens Axboe2b188cc2019-01-07 10:46:33 -070010279 if (IS_ERR(file)) {
Pavel Begunkov9faadcc2020-12-21 18:34:05 +000010280 sock_release(ctx->ring_sock);
10281 ctx->ring_sock = NULL;
10282 } else {
10283 ctx->ring_sock->file = file;
Jens Axboe2b188cc2019-01-07 10:46:33 -070010284 }
Jens Axboe2b188cc2019-01-07 10:46:33 -070010285#endif
Pavel Begunkov9faadcc2020-12-21 18:34:05 +000010286 return file;
Jens Axboe2b188cc2019-01-07 10:46:33 -070010287}
10288
Xiaoguang Wang7f136572020-05-05 16:28:53 +080010289static int io_uring_create(unsigned entries, struct io_uring_params *p,
10290 struct io_uring_params __user *params)
Jens Axboe2b188cc2019-01-07 10:46:33 -070010291{
Jens Axboe2b188cc2019-01-07 10:46:33 -070010292 struct io_ring_ctx *ctx;
Pavel Begunkov9faadcc2020-12-21 18:34:05 +000010293 struct file *file;
Jens Axboe2b188cc2019-01-07 10:46:33 -070010294 int ret;
10295
Jens Axboe8110c1a2019-12-28 15:39:54 -070010296 if (!entries)
Jens Axboe2b188cc2019-01-07 10:46:33 -070010297 return -EINVAL;
Jens Axboe8110c1a2019-12-28 15:39:54 -070010298 if (entries > IORING_MAX_ENTRIES) {
10299 if (!(p->flags & IORING_SETUP_CLAMP))
10300 return -EINVAL;
10301 entries = IORING_MAX_ENTRIES;
10302 }
Jens Axboe2b188cc2019-01-07 10:46:33 -070010303
10304 /*
10305 * Use twice as many entries for the CQ ring. It's possible for the
10306 * application to drive a higher depth than the size of the SQ ring,
10307 * since the sqes are only used at submission time. This allows for
Jens Axboe33a107f2019-10-04 12:10:03 -060010308 * some flexibility in overcommitting a bit. If the application has
10309 * set IORING_SETUP_CQSIZE, it will have passed in the desired number
10310 * of CQ ring entries manually.
Jens Axboe2b188cc2019-01-07 10:46:33 -070010311 */
10312 p->sq_entries = roundup_pow_of_two(entries);
Jens Axboe33a107f2019-10-04 12:10:03 -060010313 if (p->flags & IORING_SETUP_CQSIZE) {
10314 /*
10315 * If IORING_SETUP_CQSIZE is set, we do the same roundup
10316 * to a power-of-two, if it isn't already. We do NOT impose
10317 * any cq vs sq ring sizing.
10318 */
Joseph Qieb2667b32020-11-24 15:03:03 +080010319 if (!p->cq_entries)
Jens Axboe33a107f2019-10-04 12:10:03 -060010320 return -EINVAL;
Jens Axboe8110c1a2019-12-28 15:39:54 -070010321 if (p->cq_entries > IORING_MAX_CQ_ENTRIES) {
10322 if (!(p->flags & IORING_SETUP_CLAMP))
10323 return -EINVAL;
10324 p->cq_entries = IORING_MAX_CQ_ENTRIES;
10325 }
Joseph Qieb2667b32020-11-24 15:03:03 +080010326 p->cq_entries = roundup_pow_of_two(p->cq_entries);
10327 if (p->cq_entries < p->sq_entries)
10328 return -EINVAL;
Jens Axboe33a107f2019-10-04 12:10:03 -060010329 } else {
10330 p->cq_entries = 2 * p->sq_entries;
10331 }
Jens Axboe2b188cc2019-01-07 10:46:33 -070010332
Jens Axboe2b188cc2019-01-07 10:46:33 -070010333 ctx = io_ring_ctx_alloc(p);
Jens Axboe62e398b2021-02-21 16:19:37 -070010334 if (!ctx)
Jens Axboe2b188cc2019-01-07 10:46:33 -070010335 return -ENOMEM;
Jens Axboe2b188cc2019-01-07 10:46:33 -070010336 ctx->compat = in_compat_syscall();
Jens Axboe62e398b2021-02-21 16:19:37 -070010337 if (!capable(CAP_IPC_LOCK))
10338 ctx->user = get_uid(current_user());
Jens Axboe2aede0e2020-09-14 10:45:53 -060010339
10340 /*
10341 * This is just grabbed for accounting purposes. When a process exits,
10342 * the mm is exited and dropped before the files, hence we need to hang
10343 * on to this mm purely for the purposes of being able to unaccount
10344 * memory (locked/pinned vm). It's not used for anything else.
10345 */
Jens Axboe6b7898e2020-08-25 07:58:00 -060010346 mmgrab(current->mm);
Jens Axboe2aede0e2020-09-14 10:45:53 -060010347 ctx->mm_account = current->mm;
Jens Axboe6b7898e2020-08-25 07:58:00 -060010348
Jens Axboe2b188cc2019-01-07 10:46:33 -070010349 ret = io_allocate_scq_urings(ctx, p);
10350 if (ret)
10351 goto err;
10352
Stefano Garzarella7e84e1c2020-08-27 16:58:31 +020010353 ret = io_sq_offload_create(ctx, p);
Jens Axboe2b188cc2019-01-07 10:46:33 -070010354 if (ret)
10355 goto err;
Pavel Begunkoveae071c2021-04-25 14:32:24 +010010356 /* always set a rsrc node */
Pavel Begunkov47b228c2021-04-29 11:46:48 +010010357 ret = io_rsrc_node_switch_start(ctx);
10358 if (ret)
10359 goto err;
Pavel Begunkoveae071c2021-04-25 14:32:24 +010010360 io_rsrc_node_switch(ctx, NULL);
Jens Axboe2b188cc2019-01-07 10:46:33 -070010361
Jens Axboe2b188cc2019-01-07 10:46:33 -070010362 memset(&p->sq_off, 0, sizeof(p->sq_off));
Hristo Venev75b28af2019-08-26 17:23:46 +000010363 p->sq_off.head = offsetof(struct io_rings, sq.head);
10364 p->sq_off.tail = offsetof(struct io_rings, sq.tail);
10365 p->sq_off.ring_mask = offsetof(struct io_rings, sq_ring_mask);
10366 p->sq_off.ring_entries = offsetof(struct io_rings, sq_ring_entries);
10367 p->sq_off.flags = offsetof(struct io_rings, sq_flags);
10368 p->sq_off.dropped = offsetof(struct io_rings, sq_dropped);
10369 p->sq_off.array = (char *)ctx->sq_array - (char *)ctx->rings;
Jens Axboe2b188cc2019-01-07 10:46:33 -070010370
10371 memset(&p->cq_off, 0, sizeof(p->cq_off));
Hristo Venev75b28af2019-08-26 17:23:46 +000010372 p->cq_off.head = offsetof(struct io_rings, cq.head);
10373 p->cq_off.tail = offsetof(struct io_rings, cq.tail);
10374 p->cq_off.ring_mask = offsetof(struct io_rings, cq_ring_mask);
10375 p->cq_off.ring_entries = offsetof(struct io_rings, cq_ring_entries);
10376 p->cq_off.overflow = offsetof(struct io_rings, cq_overflow);
10377 p->cq_off.cqes = offsetof(struct io_rings, cqes);
Stefano Garzarella0d9b5b32020-05-15 18:38:04 +020010378 p->cq_off.flags = offsetof(struct io_rings, cq_flags);
Jens Axboeac90f242019-09-06 10:26:21 -060010379
Xiaoguang Wang7f136572020-05-05 16:28:53 +080010380 p->features = IORING_FEAT_SINGLE_MMAP | IORING_FEAT_NODROP |
10381 IORING_FEAT_SUBMIT_STABLE | IORING_FEAT_RW_CUR_POS |
Jiufei Xue5769a352020-06-17 17:53:55 +080010382 IORING_FEAT_CUR_PERSONALITY | IORING_FEAT_FAST_POLL |
Hao Xuc73ebb62020-11-03 10:54:37 +080010383 IORING_FEAT_POLL_32BITS | IORING_FEAT_SQPOLL_NONFIXED |
Pavel Begunkov96905572021-06-10 16:37:38 +010010384 IORING_FEAT_EXT_ARG | IORING_FEAT_NATIVE_WORKERS |
10385 IORING_FEAT_RSRC_TAGS;
Xiaoguang Wang7f136572020-05-05 16:28:53 +080010386
10387 if (copy_to_user(params, p, sizeof(*p))) {
10388 ret = -EFAULT;
10389 goto err;
10390 }
Jens Axboed1719f72020-07-30 13:43:53 -060010391
Pavel Begunkov9faadcc2020-12-21 18:34:05 +000010392 file = io_uring_get_file(ctx);
10393 if (IS_ERR(file)) {
10394 ret = PTR_ERR(file);
10395 goto err;
10396 }
10397
Jens Axboed1719f72020-07-30 13:43:53 -060010398 /*
Jens Axboe044c1ab2019-10-28 09:15:33 -060010399 * Install ring fd as the very last thing, so we don't risk someone
10400 * having closed it before we finish setup
10401 */
Pavel Begunkov9faadcc2020-12-21 18:34:05 +000010402 ret = io_uring_install_fd(ctx, file);
10403 if (ret < 0) {
10404 /* fput will clean it up */
10405 fput(file);
10406 return ret;
10407 }
Jens Axboe044c1ab2019-10-28 09:15:33 -060010408
Dmitrii Dolgovc826bd72019-10-15 19:02:01 +020010409 trace_io_uring_create(ret, ctx, p->sq_entries, p->cq_entries, p->flags);
Jens Axboe2b188cc2019-01-07 10:46:33 -070010410 return ret;
10411err:
10412 io_ring_ctx_wait_and_kill(ctx);
10413 return ret;
10414}
10415
10416/*
10417 * Sets up an aio uring context, and returns the fd. Applications asks for a
10418 * ring size, we return the actual sq/cq ring sizes (among other things) in the
10419 * params structure passed in.
10420 */
10421static long io_uring_setup(u32 entries, struct io_uring_params __user *params)
10422{
10423 struct io_uring_params p;
Jens Axboe2b188cc2019-01-07 10:46:33 -070010424 int i;
10425
10426 if (copy_from_user(&p, params, sizeof(p)))
10427 return -EFAULT;
10428 for (i = 0; i < ARRAY_SIZE(p.resv); i++) {
10429 if (p.resv[i])
10430 return -EINVAL;
10431 }
10432
Jens Axboe6c271ce2019-01-10 11:22:30 -070010433 if (p.flags & ~(IORING_SETUP_IOPOLL | IORING_SETUP_SQPOLL |
Jens Axboe8110c1a2019-12-28 15:39:54 -070010434 IORING_SETUP_SQ_AFF | IORING_SETUP_CQSIZE |
Stefano Garzarella7e84e1c2020-08-27 16:58:31 +020010435 IORING_SETUP_CLAMP | IORING_SETUP_ATTACH_WQ |
10436 IORING_SETUP_R_DISABLED))
Jens Axboe2b188cc2019-01-07 10:46:33 -070010437 return -EINVAL;
10438
Xiaoguang Wang7f136572020-05-05 16:28:53 +080010439 return io_uring_create(entries, &p, params);
Jens Axboe2b188cc2019-01-07 10:46:33 -070010440}
10441
10442SYSCALL_DEFINE2(io_uring_setup, u32, entries,
10443 struct io_uring_params __user *, params)
10444{
10445 return io_uring_setup(entries, params);
10446}
10447
Jens Axboe66f4af92020-01-16 15:36:52 -070010448static int io_probe(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args)
10449{
10450 struct io_uring_probe *p;
10451 size_t size;
10452 int i, ret;
10453
10454 size = struct_size(p, ops, nr_args);
10455 if (size == SIZE_MAX)
10456 return -EOVERFLOW;
10457 p = kzalloc(size, GFP_KERNEL);
10458 if (!p)
10459 return -ENOMEM;
10460
10461 ret = -EFAULT;
10462 if (copy_from_user(p, arg, size))
10463 goto out;
10464 ret = -EINVAL;
10465 if (memchr_inv(p, 0, size))
10466 goto out;
10467
10468 p->last_op = IORING_OP_LAST - 1;
10469 if (nr_args > IORING_OP_LAST)
10470 nr_args = IORING_OP_LAST;
10471
10472 for (i = 0; i < nr_args; i++) {
10473 p->ops[i].op = i;
10474 if (!io_op_defs[i].not_supported)
10475 p->ops[i].flags = IO_URING_OP_SUPPORTED;
10476 }
10477 p->ops_len = i;
10478
10479 ret = 0;
10480 if (copy_to_user(arg, p, size))
10481 ret = -EFAULT;
10482out:
10483 kfree(p);
10484 return ret;
10485}
10486
Jens Axboe071698e2020-01-28 10:04:42 -070010487static int io_register_personality(struct io_ring_ctx *ctx)
10488{
Jens Axboe4379bf82021-02-15 13:40:22 -070010489 const struct cred *creds;
Matthew Wilcox (Oracle)61cf9372021-03-08 14:16:16 +000010490 u32 id;
Jens Axboe1e6fa522020-10-15 08:46:24 -060010491 int ret;
Jens Axboe071698e2020-01-28 10:04:42 -070010492
Jens Axboe4379bf82021-02-15 13:40:22 -070010493 creds = get_current_cred();
Jens Axboe1e6fa522020-10-15 08:46:24 -060010494
Matthew Wilcox (Oracle)61cf9372021-03-08 14:16:16 +000010495 ret = xa_alloc_cyclic(&ctx->personalities, &id, (void *)creds,
10496 XA_LIMIT(0, USHRT_MAX), &ctx->pers_next, GFP_KERNEL);
Jens Axboea30f8952021-08-20 14:53:59 -060010497 if (ret < 0) {
10498 put_cred(creds);
10499 return ret;
10500 }
10501 return id;
Jens Axboe071698e2020-01-28 10:04:42 -070010502}
10503
Stefano Garzarella21b55db2020-08-27 16:58:30 +020010504static int io_register_restrictions(struct io_ring_ctx *ctx, void __user *arg,
10505 unsigned int nr_args)
10506{
10507 struct io_uring_restriction *res;
10508 size_t size;
10509 int i, ret;
10510
Stefano Garzarella7e84e1c2020-08-27 16:58:31 +020010511 /* Restrictions allowed only if rings started disabled */
10512 if (!(ctx->flags & IORING_SETUP_R_DISABLED))
10513 return -EBADFD;
10514
Stefano Garzarella21b55db2020-08-27 16:58:30 +020010515 /* We allow only a single restrictions registration */
Stefano Garzarella7e84e1c2020-08-27 16:58:31 +020010516 if (ctx->restrictions.registered)
Stefano Garzarella21b55db2020-08-27 16:58:30 +020010517 return -EBUSY;
10518
10519 if (!arg || nr_args > IORING_MAX_RESTRICTIONS)
10520 return -EINVAL;
10521
10522 size = array_size(nr_args, sizeof(*res));
10523 if (size == SIZE_MAX)
10524 return -EOVERFLOW;
10525
10526 res = memdup_user(arg, size);
10527 if (IS_ERR(res))
10528 return PTR_ERR(res);
10529
10530 ret = 0;
10531
10532 for (i = 0; i < nr_args; i++) {
10533 switch (res[i].opcode) {
10534 case IORING_RESTRICTION_REGISTER_OP:
10535 if (res[i].register_op >= IORING_REGISTER_LAST) {
10536 ret = -EINVAL;
10537 goto out;
10538 }
10539
10540 __set_bit(res[i].register_op,
10541 ctx->restrictions.register_op);
10542 break;
10543 case IORING_RESTRICTION_SQE_OP:
10544 if (res[i].sqe_op >= IORING_OP_LAST) {
10545 ret = -EINVAL;
10546 goto out;
10547 }
10548
10549 __set_bit(res[i].sqe_op, ctx->restrictions.sqe_op);
10550 break;
10551 case IORING_RESTRICTION_SQE_FLAGS_ALLOWED:
10552 ctx->restrictions.sqe_flags_allowed = res[i].sqe_flags;
10553 break;
10554 case IORING_RESTRICTION_SQE_FLAGS_REQUIRED:
10555 ctx->restrictions.sqe_flags_required = res[i].sqe_flags;
10556 break;
10557 default:
10558 ret = -EINVAL;
10559 goto out;
10560 }
10561 }
10562
10563out:
10564 /* Reset all restrictions if an error happened */
10565 if (ret != 0)
10566 memset(&ctx->restrictions, 0, sizeof(ctx->restrictions));
10567 else
Stefano Garzarella7e84e1c2020-08-27 16:58:31 +020010568 ctx->restrictions.registered = true;
Stefano Garzarella21b55db2020-08-27 16:58:30 +020010569
10570 kfree(res);
10571 return ret;
10572}
10573
Stefano Garzarella7e84e1c2020-08-27 16:58:31 +020010574static int io_register_enable_rings(struct io_ring_ctx *ctx)
10575{
10576 if (!(ctx->flags & IORING_SETUP_R_DISABLED))
10577 return -EBADFD;
10578
10579 if (ctx->restrictions.registered)
10580 ctx->restricted = 1;
10581
Pavel Begunkov0298ef92021-03-08 13:20:57 +000010582 ctx->flags &= ~IORING_SETUP_R_DISABLED;
10583 if (ctx->sq_data && wq_has_sleeper(&ctx->sq_data->wait))
10584 wake_up(&ctx->sq_data->wait);
Stefano Garzarella7e84e1c2020-08-27 16:58:31 +020010585 return 0;
10586}
10587
Pavel Begunkovfdecb662021-04-25 14:32:20 +010010588static int __io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned type,
Pavel Begunkovc3bdad02021-04-25 14:32:22 +010010589 struct io_uring_rsrc_update2 *up,
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +010010590 unsigned nr_args)
10591{
10592 __u32 tmp;
10593 int err;
10594
10595 if (check_add_overflow(up->offset, nr_args, &tmp))
10596 return -EOVERFLOW;
10597 err = io_rsrc_node_switch_start(ctx);
10598 if (err)
10599 return err;
10600
Pavel Begunkovfdecb662021-04-25 14:32:20 +010010601 switch (type) {
10602 case IORING_RSRC_FILE:
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +010010603 return __io_sqe_files_update(ctx, up, nr_args);
Pavel Begunkov634d00d2021-04-25 14:32:26 +010010604 case IORING_RSRC_BUFFER:
10605 return __io_sqe_buffers_update(ctx, up, nr_args);
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +010010606 }
10607 return -EINVAL;
10608}
10609
Pavel Begunkovc3bdad02021-04-25 14:32:22 +010010610static int io_register_files_update(struct io_ring_ctx *ctx, void __user *arg,
10611 unsigned nr_args)
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +010010612{
Pavel Begunkovc3bdad02021-04-25 14:32:22 +010010613 struct io_uring_rsrc_update2 up;
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +010010614
10615 if (!nr_args)
10616 return -EINVAL;
Pavel Begunkovc3bdad02021-04-25 14:32:22 +010010617 memset(&up, 0, sizeof(up));
10618 if (copy_from_user(&up, arg, sizeof(struct io_uring_rsrc_update)))
10619 return -EFAULT;
Dylan Yudaken7a7c9f92022-04-12 09:30:40 -070010620 if (up.resv || up.resv2)
Dylan Yudaken22aa1592022-04-12 09:30:39 -070010621 return -EINVAL;
Pavel Begunkovc3bdad02021-04-25 14:32:22 +010010622 return __io_register_rsrc_update(ctx, IORING_RSRC_FILE, &up, nr_args);
10623}
10624
10625static int io_register_rsrc_update(struct io_ring_ctx *ctx, void __user *arg,
Pavel Begunkov992da012021-06-10 16:37:37 +010010626 unsigned size, unsigned type)
Pavel Begunkovc3bdad02021-04-25 14:32:22 +010010627{
10628 struct io_uring_rsrc_update2 up;
10629
10630 if (size != sizeof(up))
10631 return -EINVAL;
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +010010632 if (copy_from_user(&up, arg, sizeof(up)))
10633 return -EFAULT;
Dylan Yudaken7a7c9f92022-04-12 09:30:40 -070010634 if (!up.nr || up.resv || up.resv2)
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +010010635 return -EINVAL;
Pavel Begunkov992da012021-06-10 16:37:37 +010010636 return __io_register_rsrc_update(ctx, type, &up, up.nr);
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +010010637}
10638
Pavel Begunkov792e3582021-04-25 14:32:21 +010010639static int io_register_rsrc(struct io_ring_ctx *ctx, void __user *arg,
Pavel Begunkov992da012021-06-10 16:37:37 +010010640 unsigned int size, unsigned int type)
Pavel Begunkov792e3582021-04-25 14:32:21 +010010641{
10642 struct io_uring_rsrc_register rr;
10643
10644 /* keep it extendible */
10645 if (size != sizeof(rr))
10646 return -EINVAL;
10647
10648 memset(&rr, 0, sizeof(rr));
10649 if (copy_from_user(&rr, arg, size))
10650 return -EFAULT;
Pavel Begunkov992da012021-06-10 16:37:37 +010010651 if (!rr.nr || rr.resv || rr.resv2)
Pavel Begunkov792e3582021-04-25 14:32:21 +010010652 return -EINVAL;
10653
Pavel Begunkov992da012021-06-10 16:37:37 +010010654 switch (type) {
Pavel Begunkov792e3582021-04-25 14:32:21 +010010655 case IORING_RSRC_FILE:
10656 return io_sqe_files_register(ctx, u64_to_user_ptr(rr.data),
10657 rr.nr, u64_to_user_ptr(rr.tags));
Pavel Begunkov634d00d2021-04-25 14:32:26 +010010658 case IORING_RSRC_BUFFER:
10659 return io_sqe_buffers_register(ctx, u64_to_user_ptr(rr.data),
10660 rr.nr, u64_to_user_ptr(rr.tags));
Pavel Begunkov792e3582021-04-25 14:32:21 +010010661 }
10662 return -EINVAL;
10663}
10664
Jens Axboefe764212021-06-17 10:19:54 -060010665static int io_register_iowq_aff(struct io_ring_ctx *ctx, void __user *arg,
10666 unsigned len)
10667{
10668 struct io_uring_task *tctx = current->io_uring;
10669 cpumask_var_t new_mask;
10670 int ret;
10671
10672 if (!tctx || !tctx->io_wq)
10673 return -EINVAL;
10674
10675 if (!alloc_cpumask_var(&new_mask, GFP_KERNEL))
10676 return -ENOMEM;
10677
10678 cpumask_clear(new_mask);
10679 if (len > cpumask_size())
10680 len = cpumask_size();
10681
Eugene Syromiatnikovfe223dd2022-04-06 13:55:33 +020010682 if (in_compat_syscall()) {
10683 ret = compat_get_bitmap(cpumask_bits(new_mask),
10684 (const compat_ulong_t __user *)arg,
10685 len * 8 /* CHAR_BIT */);
10686 } else {
10687 ret = copy_from_user(new_mask, arg, len);
10688 }
10689
10690 if (ret) {
Jens Axboefe764212021-06-17 10:19:54 -060010691 free_cpumask_var(new_mask);
10692 return -EFAULT;
10693 }
10694
10695 ret = io_wq_cpu_affinity(tctx->io_wq, new_mask);
10696 free_cpumask_var(new_mask);
10697 return ret;
10698}
10699
10700static int io_unregister_iowq_aff(struct io_ring_ctx *ctx)
10701{
10702 struct io_uring_task *tctx = current->io_uring;
10703
10704 if (!tctx || !tctx->io_wq)
10705 return -EINVAL;
10706
10707 return io_wq_cpu_affinity(tctx->io_wq, NULL);
10708}
10709
Jens Axboe2e480052021-08-27 11:33:19 -060010710static int io_register_iowq_max_workers(struct io_ring_ctx *ctx,
10711 void __user *arg)
Pavel Begunkovb22fa622021-10-21 13:20:29 +010010712 __must_hold(&ctx->uring_lock)
Jens Axboe2e480052021-08-27 11:33:19 -060010713{
Pavel Begunkovb22fa622021-10-21 13:20:29 +010010714 struct io_tctx_node *node;
Jens Axboefa846932021-09-01 14:15:59 -060010715 struct io_uring_task *tctx = NULL;
10716 struct io_sq_data *sqd = NULL;
Jens Axboe2e480052021-08-27 11:33:19 -060010717 __u32 new_count[2];
10718 int i, ret;
10719
Jens Axboe2e480052021-08-27 11:33:19 -060010720 if (copy_from_user(new_count, arg, sizeof(new_count)))
10721 return -EFAULT;
10722 for (i = 0; i < ARRAY_SIZE(new_count); i++)
10723 if (new_count[i] > INT_MAX)
10724 return -EINVAL;
10725
Jens Axboefa846932021-09-01 14:15:59 -060010726 if (ctx->flags & IORING_SETUP_SQPOLL) {
10727 sqd = ctx->sq_data;
10728 if (sqd) {
Jens Axboe009ad9f2021-09-08 19:07:26 -060010729 /*
10730 * Observe the correct sqd->lock -> ctx->uring_lock
10731 * ordering. Fine to drop uring_lock here, we hold
10732 * a ref to the ctx.
10733 */
Jens Axboe41d3a6b2021-09-13 13:08:51 -060010734 refcount_inc(&sqd->refs);
Jens Axboe009ad9f2021-09-08 19:07:26 -060010735 mutex_unlock(&ctx->uring_lock);
Jens Axboefa846932021-09-01 14:15:59 -060010736 mutex_lock(&sqd->lock);
Jens Axboe009ad9f2021-09-08 19:07:26 -060010737 mutex_lock(&ctx->uring_lock);
Jens Axboe41d3a6b2021-09-13 13:08:51 -060010738 if (sqd->thread)
10739 tctx = sqd->thread->io_uring;
Jens Axboefa846932021-09-01 14:15:59 -060010740 }
10741 } else {
10742 tctx = current->io_uring;
10743 }
10744
Pavel Begunkove139a1e2021-10-19 23:43:46 +010010745 BUILD_BUG_ON(sizeof(new_count) != sizeof(ctx->iowq_limits));
Jens Axboefa846932021-09-01 14:15:59 -060010746
Pavel Begunkov4cac4872021-11-08 15:10:03 +000010747 for (i = 0; i < ARRAY_SIZE(new_count); i++)
10748 if (new_count[i])
10749 ctx->iowq_limits[i] = new_count[i];
Pavel Begunkove139a1e2021-10-19 23:43:46 +010010750 ctx->iowq_limits_set = true;
10751
10752 ret = -EINVAL;
10753 if (tctx && tctx->io_wq) {
10754 ret = io_wq_max_workers(tctx->io_wq, new_count);
10755 if (ret)
10756 goto err;
10757 } else {
10758 memset(new_count, 0, sizeof(new_count));
10759 }
Jens Axboefa846932021-09-01 14:15:59 -060010760
Jens Axboe41d3a6b2021-09-13 13:08:51 -060010761 if (sqd) {
Jens Axboefa846932021-09-01 14:15:59 -060010762 mutex_unlock(&sqd->lock);
Jens Axboe41d3a6b2021-09-13 13:08:51 -060010763 io_put_sq_data(sqd);
10764 }
Jens Axboe2e480052021-08-27 11:33:19 -060010765
10766 if (copy_to_user(arg, new_count, sizeof(new_count)))
10767 return -EFAULT;
10768
Pavel Begunkovb22fa622021-10-21 13:20:29 +010010769 /* that's it for SQPOLL, only the SQPOLL task creates requests */
10770 if (sqd)
10771 return 0;
10772
10773 /* now propagate the restriction to all registered users */
10774 list_for_each_entry(node, &ctx->tctx_list, ctx_node) {
10775 struct io_uring_task *tctx = node->task->io_uring;
10776
10777 if (WARN_ON_ONCE(!tctx->io_wq))
10778 continue;
10779
10780 for (i = 0; i < ARRAY_SIZE(new_count); i++)
10781 new_count[i] = ctx->iowq_limits[i];
10782 /* ignore errors, it always returns zero anyway */
10783 (void)io_wq_max_workers(tctx->io_wq, new_count);
10784 }
Jens Axboe2e480052021-08-27 11:33:19 -060010785 return 0;
Jens Axboefa846932021-09-01 14:15:59 -060010786err:
Jens Axboe41d3a6b2021-09-13 13:08:51 -060010787 if (sqd) {
Jens Axboefa846932021-09-01 14:15:59 -060010788 mutex_unlock(&sqd->lock);
Jens Axboe41d3a6b2021-09-13 13:08:51 -060010789 io_put_sq_data(sqd);
10790 }
Jens Axboefa846932021-09-01 14:15:59 -060010791 return ret;
Jens Axboe2e480052021-08-27 11:33:19 -060010792}
10793
Jens Axboe071698e2020-01-28 10:04:42 -070010794static bool io_register_op_must_quiesce(int op)
10795{
10796 switch (op) {
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +010010797 case IORING_REGISTER_BUFFERS:
10798 case IORING_UNREGISTER_BUFFERS:
Pavel Begunkovf4f7d212021-04-01 15:44:02 +010010799 case IORING_REGISTER_FILES:
Jens Axboe071698e2020-01-28 10:04:42 -070010800 case IORING_UNREGISTER_FILES:
10801 case IORING_REGISTER_FILES_UPDATE:
10802 case IORING_REGISTER_PROBE:
10803 case IORING_REGISTER_PERSONALITY:
10804 case IORING_UNREGISTER_PERSONALITY:
Pavel Begunkov992da012021-06-10 16:37:37 +010010805 case IORING_REGISTER_FILES2:
10806 case IORING_REGISTER_FILES_UPDATE2:
10807 case IORING_REGISTER_BUFFERS2:
10808 case IORING_REGISTER_BUFFERS_UPDATE:
Jens Axboefe764212021-06-17 10:19:54 -060010809 case IORING_REGISTER_IOWQ_AFF:
10810 case IORING_UNREGISTER_IOWQ_AFF:
Jens Axboe2e480052021-08-27 11:33:19 -060010811 case IORING_REGISTER_IOWQ_MAX_WORKERS:
Jens Axboe071698e2020-01-28 10:04:42 -070010812 return false;
10813 default:
10814 return true;
10815 }
10816}
10817
Pavel Begunkove73c5c72021-08-09 13:04:12 +010010818static int io_ctx_quiesce(struct io_ring_ctx *ctx)
10819{
10820 long ret;
10821
10822 percpu_ref_kill(&ctx->refs);
10823
10824 /*
10825 * Drop uring mutex before waiting for references to exit. If another
10826 * thread is currently inside io_uring_enter() it might need to grab the
10827 * uring_lock to make progress. If we hold it here across the drain
10828 * wait, then we can deadlock. It's safe to drop the mutex here, since
10829 * no new references will come in after we've killed the percpu ref.
10830 */
10831 mutex_unlock(&ctx->uring_lock);
10832 do {
10833 ret = wait_for_completion_interruptible(&ctx->ref_comp);
10834 if (!ret)
10835 break;
10836 ret = io_run_task_work_sig();
10837 } while (ret >= 0);
10838 mutex_lock(&ctx->uring_lock);
10839
10840 if (ret)
10841 io_refs_resurrect(&ctx->refs, &ctx->ref_comp);
10842 return ret;
10843}
10844
Jens Axboeedafcce2019-01-09 09:16:05 -070010845static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,
10846 void __user *arg, unsigned nr_args)
Jens Axboeb19062a2019-04-15 10:49:38 -060010847 __releases(ctx->uring_lock)
10848 __acquires(ctx->uring_lock)
Jens Axboeedafcce2019-01-09 09:16:05 -070010849{
10850 int ret;
10851
Jens Axboe35fa71a2019-04-22 10:23:23 -060010852 /*
10853 * We're inside the ring mutex, if the ref is already dying, then
10854 * someone else killed the ctx or is already going through
10855 * io_uring_register().
10856 */
10857 if (percpu_ref_is_dying(&ctx->refs))
10858 return -ENXIO;
10859
Pavel Begunkov75c40212021-04-15 13:07:40 +010010860 if (ctx->restricted) {
10861 if (opcode >= IORING_REGISTER_LAST)
10862 return -EINVAL;
10863 opcode = array_index_nospec(opcode, IORING_REGISTER_LAST);
10864 if (!test_bit(opcode, ctx->restrictions.register_op))
10865 return -EACCES;
10866 }
10867
Jens Axboe071698e2020-01-28 10:04:42 -070010868 if (io_register_op_must_quiesce(opcode)) {
Pavel Begunkove73c5c72021-08-09 13:04:12 +010010869 ret = io_ctx_quiesce(ctx);
10870 if (ret)
Pavel Begunkovf70865d2021-04-11 01:46:40 +010010871 return ret;
Jens Axboe05f3fb32019-12-09 11:22:50 -070010872 }
Jens Axboeedafcce2019-01-09 09:16:05 -070010873
10874 switch (opcode) {
10875 case IORING_REGISTER_BUFFERS:
Pavel Begunkov634d00d2021-04-25 14:32:26 +010010876 ret = io_sqe_buffers_register(ctx, arg, nr_args, NULL);
Jens Axboeedafcce2019-01-09 09:16:05 -070010877 break;
10878 case IORING_UNREGISTER_BUFFERS:
10879 ret = -EINVAL;
10880 if (arg || nr_args)
10881 break;
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -080010882 ret = io_sqe_buffers_unregister(ctx);
Jens Axboeedafcce2019-01-09 09:16:05 -070010883 break;
Jens Axboe6b063142019-01-10 22:13:58 -070010884 case IORING_REGISTER_FILES:
Pavel Begunkov792e3582021-04-25 14:32:21 +010010885 ret = io_sqe_files_register(ctx, arg, nr_args, NULL);
Jens Axboe6b063142019-01-10 22:13:58 -070010886 break;
10887 case IORING_UNREGISTER_FILES:
10888 ret = -EINVAL;
10889 if (arg || nr_args)
10890 break;
10891 ret = io_sqe_files_unregister(ctx);
10892 break;
Jens Axboec3a31e62019-10-03 13:59:56 -060010893 case IORING_REGISTER_FILES_UPDATE:
Pavel Begunkovc3bdad02021-04-25 14:32:22 +010010894 ret = io_register_files_update(ctx, arg, nr_args);
Jens Axboec3a31e62019-10-03 13:59:56 -060010895 break;
Jens Axboe9b402842019-04-11 11:45:41 -060010896 case IORING_REGISTER_EVENTFD:
Jens Axboef2842ab2020-01-08 11:04:00 -070010897 case IORING_REGISTER_EVENTFD_ASYNC:
Jens Axboe9b402842019-04-11 11:45:41 -060010898 ret = -EINVAL;
10899 if (nr_args != 1)
10900 break;
10901 ret = io_eventfd_register(ctx, arg);
Jens Axboef2842ab2020-01-08 11:04:00 -070010902 if (ret)
10903 break;
10904 if (opcode == IORING_REGISTER_EVENTFD_ASYNC)
10905 ctx->eventfd_async = 1;
10906 else
10907 ctx->eventfd_async = 0;
Jens Axboe9b402842019-04-11 11:45:41 -060010908 break;
10909 case IORING_UNREGISTER_EVENTFD:
10910 ret = -EINVAL;
10911 if (arg || nr_args)
10912 break;
10913 ret = io_eventfd_unregister(ctx);
10914 break;
Jens Axboe66f4af92020-01-16 15:36:52 -070010915 case IORING_REGISTER_PROBE:
10916 ret = -EINVAL;
10917 if (!arg || nr_args > 256)
10918 break;
10919 ret = io_probe(ctx, arg, nr_args);
10920 break;
Jens Axboe071698e2020-01-28 10:04:42 -070010921 case IORING_REGISTER_PERSONALITY:
10922 ret = -EINVAL;
10923 if (arg || nr_args)
10924 break;
10925 ret = io_register_personality(ctx);
10926 break;
10927 case IORING_UNREGISTER_PERSONALITY:
10928 ret = -EINVAL;
10929 if (arg)
10930 break;
10931 ret = io_unregister_personality(ctx, nr_args);
10932 break;
Stefano Garzarella7e84e1c2020-08-27 16:58:31 +020010933 case IORING_REGISTER_ENABLE_RINGS:
10934 ret = -EINVAL;
10935 if (arg || nr_args)
10936 break;
10937 ret = io_register_enable_rings(ctx);
10938 break;
Stefano Garzarella21b55db2020-08-27 16:58:30 +020010939 case IORING_REGISTER_RESTRICTIONS:
10940 ret = io_register_restrictions(ctx, arg, nr_args);
10941 break;
Pavel Begunkov992da012021-06-10 16:37:37 +010010942 case IORING_REGISTER_FILES2:
10943 ret = io_register_rsrc(ctx, arg, nr_args, IORING_RSRC_FILE);
Pavel Begunkov792e3582021-04-25 14:32:21 +010010944 break;
Pavel Begunkov992da012021-06-10 16:37:37 +010010945 case IORING_REGISTER_FILES_UPDATE2:
10946 ret = io_register_rsrc_update(ctx, arg, nr_args,
10947 IORING_RSRC_FILE);
10948 break;
10949 case IORING_REGISTER_BUFFERS2:
10950 ret = io_register_rsrc(ctx, arg, nr_args, IORING_RSRC_BUFFER);
10951 break;
10952 case IORING_REGISTER_BUFFERS_UPDATE:
10953 ret = io_register_rsrc_update(ctx, arg, nr_args,
10954 IORING_RSRC_BUFFER);
Pavel Begunkovc3bdad02021-04-25 14:32:22 +010010955 break;
Jens Axboefe764212021-06-17 10:19:54 -060010956 case IORING_REGISTER_IOWQ_AFF:
10957 ret = -EINVAL;
10958 if (!arg || !nr_args)
10959 break;
10960 ret = io_register_iowq_aff(ctx, arg, nr_args);
10961 break;
10962 case IORING_UNREGISTER_IOWQ_AFF:
10963 ret = -EINVAL;
10964 if (arg || nr_args)
10965 break;
10966 ret = io_unregister_iowq_aff(ctx);
10967 break;
Jens Axboe2e480052021-08-27 11:33:19 -060010968 case IORING_REGISTER_IOWQ_MAX_WORKERS:
10969 ret = -EINVAL;
10970 if (!arg || nr_args != 2)
10971 break;
10972 ret = io_register_iowq_max_workers(ctx, arg);
10973 break;
Jens Axboeedafcce2019-01-09 09:16:05 -070010974 default:
10975 ret = -EINVAL;
10976 break;
10977 }
10978
Jens Axboe071698e2020-01-28 10:04:42 -070010979 if (io_register_op_must_quiesce(opcode)) {
Jens Axboe05f3fb32019-12-09 11:22:50 -070010980 /* bring the ctx back to life */
Jens Axboe05f3fb32019-12-09 11:22:50 -070010981 percpu_ref_reinit(&ctx->refs);
Jens Axboe0f158b42020-05-14 17:18:39 -060010982 reinit_completion(&ctx->ref_comp);
Jens Axboe05f3fb32019-12-09 11:22:50 -070010983 }
Jens Axboeedafcce2019-01-09 09:16:05 -070010984 return ret;
10985}
10986
10987SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode,
10988 void __user *, arg, unsigned int, nr_args)
10989{
10990 struct io_ring_ctx *ctx;
10991 long ret = -EBADF;
10992 struct fd f;
10993
10994 f = fdget(fd);
10995 if (!f.file)
10996 return -EBADF;
10997
10998 ret = -EOPNOTSUPP;
10999 if (f.file->f_op != &io_uring_fops)
11000 goto out_fput;
11001
11002 ctx = f.file->private_data;
11003
Pavel Begunkovb6c23dd2021-02-20 15:17:18 +000011004 io_run_task_work();
11005
Jens Axboeedafcce2019-01-09 09:16:05 -070011006 mutex_lock(&ctx->uring_lock);
11007 ret = __io_uring_register(ctx, opcode, arg, nr_args);
11008 mutex_unlock(&ctx->uring_lock);
Dmitrii Dolgovc826bd72019-10-15 19:02:01 +020011009 trace_io_uring_register(ctx, opcode, ctx->nr_user_files, ctx->nr_user_bufs,
11010 ctx->cq_ev_fd != NULL, ret);
Jens Axboeedafcce2019-01-09 09:16:05 -070011011out_fput:
11012 fdput(f);
11013 return ret;
11014}
11015
Jens Axboe2b188cc2019-01-07 10:46:33 -070011016static int __init io_uring_init(void)
11017{
Stefan Metzmacherd7f62e82020-01-29 14:39:41 +010011018#define __BUILD_BUG_VERIFY_ELEMENT(stype, eoffset, etype, ename) do { \
11019 BUILD_BUG_ON(offsetof(stype, ename) != eoffset); \
11020 BUILD_BUG_ON(sizeof(etype) != sizeof_field(stype, ename)); \
11021} while (0)
11022
11023#define BUILD_BUG_SQE_ELEM(eoffset, etype, ename) \
11024 __BUILD_BUG_VERIFY_ELEMENT(struct io_uring_sqe, eoffset, etype, ename)
11025 BUILD_BUG_ON(sizeof(struct io_uring_sqe) != 64);
11026 BUILD_BUG_SQE_ELEM(0, __u8, opcode);
11027 BUILD_BUG_SQE_ELEM(1, __u8, flags);
11028 BUILD_BUG_SQE_ELEM(2, __u16, ioprio);
11029 BUILD_BUG_SQE_ELEM(4, __s32, fd);
11030 BUILD_BUG_SQE_ELEM(8, __u64, off);
11031 BUILD_BUG_SQE_ELEM(8, __u64, addr2);
11032 BUILD_BUG_SQE_ELEM(16, __u64, addr);
Pavel Begunkov7d67af22020-02-24 11:32:45 +030011033 BUILD_BUG_SQE_ELEM(16, __u64, splice_off_in);
Stefan Metzmacherd7f62e82020-01-29 14:39:41 +010011034 BUILD_BUG_SQE_ELEM(24, __u32, len);
11035 BUILD_BUG_SQE_ELEM(28, __kernel_rwf_t, rw_flags);
11036 BUILD_BUG_SQE_ELEM(28, /* compat */ int, rw_flags);
11037 BUILD_BUG_SQE_ELEM(28, /* compat */ __u32, rw_flags);
11038 BUILD_BUG_SQE_ELEM(28, __u32, fsync_flags);
Jiufei Xue5769a352020-06-17 17:53:55 +080011039 BUILD_BUG_SQE_ELEM(28, /* compat */ __u16, poll_events);
11040 BUILD_BUG_SQE_ELEM(28, __u32, poll32_events);
Stefan Metzmacherd7f62e82020-01-29 14:39:41 +010011041 BUILD_BUG_SQE_ELEM(28, __u32, sync_range_flags);
11042 BUILD_BUG_SQE_ELEM(28, __u32, msg_flags);
11043 BUILD_BUG_SQE_ELEM(28, __u32, timeout_flags);
11044 BUILD_BUG_SQE_ELEM(28, __u32, accept_flags);
11045 BUILD_BUG_SQE_ELEM(28, __u32, cancel_flags);
11046 BUILD_BUG_SQE_ELEM(28, __u32, open_flags);
11047 BUILD_BUG_SQE_ELEM(28, __u32, statx_flags);
11048 BUILD_BUG_SQE_ELEM(28, __u32, fadvise_advice);
Pavel Begunkov7d67af22020-02-24 11:32:45 +030011049 BUILD_BUG_SQE_ELEM(28, __u32, splice_flags);
Stefan Metzmacherd7f62e82020-01-29 14:39:41 +010011050 BUILD_BUG_SQE_ELEM(32, __u64, user_data);
11051 BUILD_BUG_SQE_ELEM(40, __u16, buf_index);
Pavel Begunkov16340ea2021-06-24 15:09:58 +010011052 BUILD_BUG_SQE_ELEM(40, __u16, buf_group);
Stefan Metzmacherd7f62e82020-01-29 14:39:41 +010011053 BUILD_BUG_SQE_ELEM(42, __u16, personality);
Pavel Begunkov7d67af22020-02-24 11:32:45 +030011054 BUILD_BUG_SQE_ELEM(44, __s32, splice_fd_in);
Pavel Begunkovb9445592021-08-25 12:25:45 +010011055 BUILD_BUG_SQE_ELEM(44, __u32, file_index);
Stefan Metzmacherd7f62e82020-01-29 14:39:41 +010011056
Pavel Begunkovb0d658ec2021-04-27 16:13:53 +010011057 BUILD_BUG_ON(sizeof(struct io_uring_files_update) !=
11058 sizeof(struct io_uring_rsrc_update));
11059 BUILD_BUG_ON(sizeof(struct io_uring_rsrc_update) >
11060 sizeof(struct io_uring_rsrc_update2));
Pavel Begunkov90499ad2021-08-25 20:51:40 +010011061
11062 /* ->buf_index is u16 */
11063 BUILD_BUG_ON(IORING_MAX_REG_BUFFERS >= (1u << 16));
11064
Pavel Begunkovb0d658ec2021-04-27 16:13:53 +010011065 /* should fit into one byte */
11066 BUILD_BUG_ON(SQE_VALID_FLAGS >= (1 << 8));
11067
Jens Axboed3656342019-12-18 09:50:26 -070011068 BUILD_BUG_ON(ARRAY_SIZE(io_op_defs) != IORING_OP_LAST);
Hao Xu32c2d332021-09-07 11:22:43 +080011069 BUILD_BUG_ON(__REQ_F_LAST_BIT > 8 * sizeof(int));
Pavel Begunkov16340ea2021-06-24 15:09:58 +010011070
Jens Axboe91f245d2021-02-09 13:48:50 -070011071 req_cachep = KMEM_CACHE(io_kiocb, SLAB_HWCACHE_ALIGN | SLAB_PANIC |
11072 SLAB_ACCOUNT);
Jens Axboe2b188cc2019-01-07 10:46:33 -070011073 return 0;
11074};
11075__initcall(io_uring_init);