blob: 6e19fe04a5d16f635c25cdfd1be1e197edb15016 [file] [log] [blame]
Jens Axboe2b188cc2019-01-07 10:46:33 -07001// SPDX-License-Identifier: GPL-2.0
2/*
3 * Shared application/kernel submission and completion ring pairs, for
4 * supporting fast/efficient IO.
5 *
6 * A note on the read/write ordering memory barriers that are matched between
Stefan Bühler1e84b972019-04-24 23:54:16 +02007 * the application and kernel side.
8 *
9 * After the application reads the CQ ring tail, it must use an
10 * appropriate smp_rmb() to pair with the smp_wmb() the kernel uses
11 * before writing the tail (using smp_load_acquire to read the tail will
12 * do). It also needs a smp_mb() before updating CQ head (ordering the
13 * entry load(s) with the head store), pairing with an implicit barrier
14 * through a control-dependency in io_get_cqring (smp_store_release to
15 * store head will do). Failure to do so could lead to reading invalid
16 * CQ entries.
17 *
18 * Likewise, the application must use an appropriate smp_wmb() before
19 * writing the SQ tail (ordering SQ entry stores with the tail store),
20 * which pairs with smp_load_acquire in io_get_sqring (smp_store_release
21 * to store the tail will do). And it needs a barrier ordering the SQ
22 * head load before writing new SQ entries (smp_load_acquire to read
23 * head will do).
24 *
25 * When using the SQ poll thread (IORING_SETUP_SQPOLL), the application
26 * needs to check the SQ flags for IORING_SQ_NEED_WAKEUP *after*
27 * updating the SQ tail; a full memory barrier smp_mb() is needed
28 * between.
Jens Axboe2b188cc2019-01-07 10:46:33 -070029 *
30 * Also see the examples in the liburing library:
31 *
32 * git://git.kernel.dk/liburing
33 *
34 * io_uring also uses READ/WRITE_ONCE() for _any_ store or load that happens
35 * from data shared between the kernel and application. This is done both
36 * for ordering purposes, but also to ensure that once a value is loaded from
37 * data that the application could potentially modify, it remains stable.
38 *
39 * Copyright (C) 2018-2019 Jens Axboe
Christoph Hellwigc992fe22019-01-11 09:43:02 -070040 * Copyright (c) 2018-2019 Christoph Hellwig
Jens Axboe2b188cc2019-01-07 10:46:33 -070041 */
42#include <linux/kernel.h>
43#include <linux/init.h>
44#include <linux/errno.h>
45#include <linux/syscalls.h>
46#include <linux/compat.h>
Jens Axboe52de1fe2020-02-27 10:15:42 -070047#include <net/compat.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070048#include <linux/refcount.h>
49#include <linux/uio.h>
Pavel Begunkov6b47ee62020-01-18 20:22:41 +030050#include <linux/bits.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070051
52#include <linux/sched/signal.h>
53#include <linux/fs.h>
54#include <linux/file.h>
55#include <linux/fdtable.h>
56#include <linux/mm.h>
57#include <linux/mman.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070058#include <linux/percpu.h>
59#include <linux/slab.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070060#include <linux/blkdev.h>
Jens Axboeedafcce2019-01-09 09:16:05 -070061#include <linux/bvec.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070062#include <linux/net.h>
63#include <net/sock.h>
64#include <net/af_unix.h>
Jens Axboe6b063142019-01-10 22:13:58 -070065#include <net/scm.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070066#include <linux/anon_inodes.h>
67#include <linux/sched/mm.h>
68#include <linux/uaccess.h>
69#include <linux/nospec.h>
Jens Axboeedafcce2019-01-09 09:16:05 -070070#include <linux/sizes.h>
71#include <linux/hugetlb.h>
Jens Axboeaa4c3962019-11-29 10:14:00 -070072#include <linux/highmem.h>
Jens Axboe15b71ab2019-12-11 11:20:36 -070073#include <linux/namei.h>
74#include <linux/fsnotify.h>
Jens Axboe4840e412019-12-25 22:03:45 -070075#include <linux/fadvise.h>
Jens Axboe3e4827b2020-01-08 15:18:09 -070076#include <linux/eventpoll.h>
Pavel Begunkov7d67af22020-02-24 11:32:45 +030077#include <linux/splice.h>
Jens Axboeb41e9852020-02-17 09:52:41 -070078#include <linux/task_work.h>
Jens Axboebcf5a062020-05-22 09:24:42 -060079#include <linux/pagemap.h>
Jens Axboe0f212202020-09-13 13:09:39 -060080#include <linux/io_uring.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070081
Dmitrii Dolgovc826bd72019-10-15 19:02:01 +020082#define CREATE_TRACE_POINTS
83#include <trace/events/io_uring.h>
84
Jens Axboe2b188cc2019-01-07 10:46:33 -070085#include <uapi/linux/io_uring.h>
86
87#include "internal.h"
Jens Axboe561fb042019-10-24 07:25:42 -060088#include "io-wq.h"
Jens Axboe2b188cc2019-01-07 10:46:33 -070089
Daniel Xu5277dea2019-09-14 14:23:45 -070090#define IORING_MAX_ENTRIES 32768
Jens Axboe33a107f2019-10-04 12:10:03 -060091#define IORING_MAX_CQ_ENTRIES (2 * IORING_MAX_ENTRIES)
Jens Axboe65e19f52019-10-26 07:20:21 -060092
93/*
94 * Shift of 9 is 512 entries, or exactly one page on 64-bit archs
95 */
96#define IORING_FILE_TABLE_SHIFT 9
97#define IORING_MAX_FILES_TABLE (1U << IORING_FILE_TABLE_SHIFT)
98#define IORING_FILE_TABLE_MASK (IORING_MAX_FILES_TABLE - 1)
99#define IORING_MAX_FIXED_FILES (64 * IORING_MAX_FILES_TABLE)
Stefano Garzarella21b55db2020-08-27 16:58:30 +0200100#define IORING_MAX_RESTRICTIONS (IORING_RESTRICTION_LAST + \
101 IORING_REGISTER_LAST + IORING_OP_LAST)
Jens Axboe2b188cc2019-01-07 10:46:33 -0700102
Pavel Begunkov489809e2021-05-14 12:06:44 +0100103#define IORING_MAX_REG_BUFFERS (1U << 14)
104
Pavel Begunkovb16fed662021-02-18 18:29:40 +0000105#define SQE_VALID_FLAGS (IOSQE_FIXED_FILE|IOSQE_IO_DRAIN|IOSQE_IO_LINK| \
106 IOSQE_IO_HARDLINK | IOSQE_ASYNC | \
107 IOSQE_BUFFER_SELECT)
108
Jens Axboe2b188cc2019-01-07 10:46:33 -0700109struct io_uring {
110 u32 head ____cacheline_aligned_in_smp;
111 u32 tail ____cacheline_aligned_in_smp;
112};
113
Stefan Bühler1e84b972019-04-24 23:54:16 +0200114/*
Hristo Venev75b28af2019-08-26 17:23:46 +0000115 * This data is shared with the application through the mmap at offsets
116 * IORING_OFF_SQ_RING and IORING_OFF_CQ_RING.
Stefan Bühler1e84b972019-04-24 23:54:16 +0200117 *
118 * The offsets to the member fields are published through struct
119 * io_sqring_offsets when calling io_uring_setup.
120 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000121struct io_rings {
Stefan Bühler1e84b972019-04-24 23:54:16 +0200122 /*
123 * Head and tail offsets into the ring; the offsets need to be
124 * masked to get valid indices.
125 *
Hristo Venev75b28af2019-08-26 17:23:46 +0000126 * The kernel controls head of the sq ring and the tail of the cq ring,
127 * and the application controls tail of the sq ring and the head of the
128 * cq ring.
Stefan Bühler1e84b972019-04-24 23:54:16 +0200129 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000130 struct io_uring sq, cq;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200131 /*
Hristo Venev75b28af2019-08-26 17:23:46 +0000132 * Bitmasks to apply to head and tail offsets (constant, equals
Stefan Bühler1e84b972019-04-24 23:54:16 +0200133 * ring_entries - 1)
134 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000135 u32 sq_ring_mask, cq_ring_mask;
136 /* Ring sizes (constant, power of 2) */
137 u32 sq_ring_entries, cq_ring_entries;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200138 /*
139 * Number of invalid entries dropped by the kernel due to
140 * invalid index stored in array
141 *
142 * Written by the kernel, shouldn't be modified by the
143 * application (i.e. get number of "new events" by comparing to
144 * cached value).
145 *
146 * After a new SQ head value was read by the application this
147 * counter includes all submissions that were dropped reaching
148 * the new SQ head (and possibly more).
149 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000150 u32 sq_dropped;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200151 /*
Stefano Garzarella0d9b5b32020-05-15 18:38:04 +0200152 * Runtime SQ flags
Stefan Bühler1e84b972019-04-24 23:54:16 +0200153 *
154 * Written by the kernel, shouldn't be modified by the
155 * application.
156 *
157 * The application needs a full memory barrier before checking
158 * for IORING_SQ_NEED_WAKEUP after updating the sq tail.
159 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000160 u32 sq_flags;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200161 /*
Stefano Garzarella0d9b5b32020-05-15 18:38:04 +0200162 * Runtime CQ flags
163 *
164 * Written by the application, shouldn't be modified by the
165 * kernel.
166 */
167 u32 cq_flags;
168 /*
Stefan Bühler1e84b972019-04-24 23:54:16 +0200169 * Number of completion events lost because the queue was full;
170 * this should be avoided by the application by making sure
LimingWu0b4295b2019-12-05 20:18:18 +0800171 * there are not more requests pending than there is space in
Stefan Bühler1e84b972019-04-24 23:54:16 +0200172 * the completion queue.
173 *
174 * Written by the kernel, shouldn't be modified by the
175 * application (i.e. get number of "new events" by comparing to
176 * cached value).
177 *
178 * As completion events come in out of order this counter is not
179 * ordered with any other data.
180 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000181 u32 cq_overflow;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200182 /*
183 * Ring buffer of completion events.
184 *
185 * The kernel writes completion events fresh every time they are
186 * produced, so the application is allowed to modify pending
187 * entries.
188 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000189 struct io_uring_cqe cqes[] ____cacheline_aligned_in_smp;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700190};
191
Pavel Begunkov45d189c2021-02-10 00:03:07 +0000192enum io_uring_cmd_flags {
193 IO_URING_F_NONBLOCK = 1,
Pavel Begunkov889fca72021-02-10 00:03:09 +0000194 IO_URING_F_COMPLETE_DEFER = 2,
Pavel Begunkov45d189c2021-02-10 00:03:07 +0000195};
196
Jens Axboeedafcce2019-01-09 09:16:05 -0700197struct io_mapped_ubuf {
198 u64 ubuf;
Pavel Begunkov4751f532021-04-01 15:43:55 +0100199 u64 ubuf_end;
Jens Axboeedafcce2019-01-09 09:16:05 -0700200 unsigned int nr_bvecs;
Jens Axboede293932020-09-17 16:19:16 -0600201 unsigned long acct_pages;
Pavel Begunkov41edf1a2021-04-25 14:32:23 +0100202 struct bio_vec bvec[];
Jens Axboeedafcce2019-01-09 09:16:05 -0700203};
204
Bijan Mottahedeh50238532021-01-15 17:37:45 +0000205struct io_ring_ctx;
206
Pavel Begunkov6c2450a2021-02-23 12:40:22 +0000207struct io_overflow_cqe {
208 struct io_uring_cqe cqe;
209 struct list_head list;
210};
211
Pavel Begunkova04b0ac2021-04-01 15:44:04 +0100212struct io_fixed_file {
213 /* file * with additional FFS_* flags */
214 unsigned long file_ptr;
215};
216
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +0000217struct io_rsrc_put {
218 struct list_head list;
Pavel Begunkovb60c8dc2021-04-25 14:32:18 +0100219 u64 tag;
Bijan Mottahedeh50238532021-01-15 17:37:45 +0000220 union {
221 void *rsrc;
222 struct file *file;
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +0100223 struct io_mapped_ubuf *buf;
Bijan Mottahedeh50238532021-01-15 17:37:45 +0000224 };
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +0000225};
226
Pavel Begunkovaeca2412021-04-11 01:46:37 +0100227struct io_file_table {
228 /* two level table */
229 struct io_fixed_file **files;
Jens Axboe31b51512019-01-18 22:56:34 -0700230};
231
Pavel Begunkovb895c9a2021-04-01 15:43:40 +0100232struct io_rsrc_node {
Xiaoguang Wang05589552020-03-31 14:05:18 +0800233 struct percpu_ref refs;
234 struct list_head node;
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +0000235 struct list_head rsrc_list;
Pavel Begunkovb895c9a2021-04-01 15:43:40 +0100236 struct io_rsrc_data *rsrc_data;
Jens Axboe4a38aed22020-05-14 17:21:15 -0600237 struct llist_node llist;
Pavel Begunkove2978222020-11-18 14:56:26 +0000238 bool done;
Xiaoguang Wang05589552020-03-31 14:05:18 +0800239};
240
Pavel Begunkov40ae0ff2021-04-01 15:43:44 +0100241typedef void (rsrc_put_fn)(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc);
242
Pavel Begunkovb895c9a2021-04-01 15:43:40 +0100243struct io_rsrc_data {
Jens Axboe05f3fb32019-12-09 11:22:50 -0700244 struct io_ring_ctx *ctx;
245
Pavel Begunkovb60c8dc2021-04-25 14:32:18 +0100246 u64 *tags;
Pavel Begunkov40ae0ff2021-04-01 15:43:44 +0100247 rsrc_put_fn *do_put;
Pavel Begunkov3e942492021-04-11 01:46:34 +0100248 atomic_t refs;
Jens Axboe05f3fb32019-12-09 11:22:50 -0700249 struct completion done;
Hao Xu8bad28d2021-02-19 17:19:36 +0800250 bool quiesce;
Jens Axboe05f3fb32019-12-09 11:22:50 -0700251};
252
Jens Axboe5a2e7452020-02-23 16:23:11 -0700253struct io_buffer {
254 struct list_head list;
255 __u64 addr;
Thadeu Lima de Souza Cascardod1f82802021-05-05 09:47:06 -0300256 __u32 len;
Jens Axboe5a2e7452020-02-23 16:23:11 -0700257 __u16 bid;
258};
259
Stefano Garzarella21b55db2020-08-27 16:58:30 +0200260struct io_restriction {
261 DECLARE_BITMAP(register_op, IORING_REGISTER_LAST);
262 DECLARE_BITMAP(sqe_op, IORING_OP_LAST);
263 u8 sqe_flags_allowed;
264 u8 sqe_flags_required;
Stefano Garzarella7e84e1c2020-08-27 16:58:31 +0200265 bool registered;
Stefano Garzarella21b55db2020-08-27 16:58:30 +0200266};
267
Jens Axboe37d1e2e2021-02-17 21:03:43 -0700268enum {
269 IO_SQ_THREAD_SHOULD_STOP = 0,
270 IO_SQ_THREAD_SHOULD_PARK,
271};
272
Jens Axboe534ca6d2020-09-02 13:52:19 -0600273struct io_sq_data {
274 refcount_t refs;
Pavel Begunkov9e138a42021-03-14 20:57:12 +0000275 atomic_t park_pending;
Pavel Begunkov09a6f4e2021-03-14 20:57:10 +0000276 struct mutex lock;
Jens Axboe69fb2132020-09-14 11:16:23 -0600277
278 /* ctx's that are using this sqd */
279 struct list_head ctx_list;
Jens Axboe69fb2132020-09-14 11:16:23 -0600280
Jens Axboe534ca6d2020-09-02 13:52:19 -0600281 struct task_struct *thread;
282 struct wait_queue_head wait;
Xiaoguang Wang08369242020-11-03 14:15:59 +0800283
284 unsigned sq_thread_idle;
Jens Axboe37d1e2e2021-02-17 21:03:43 -0700285 int sq_cpu;
286 pid_t task_pid;
Jens Axboe5c2469e2021-03-11 10:17:56 -0700287 pid_t task_tgid;
Jens Axboe37d1e2e2021-02-17 21:03:43 -0700288
289 unsigned long state;
Jens Axboe37d1e2e2021-02-17 21:03:43 -0700290 struct completion exited;
Jens Axboe534ca6d2020-09-02 13:52:19 -0600291};
292
Pavel Begunkov258b29a2021-02-10 00:03:10 +0000293#define IO_IOPOLL_BATCH 8
Pavel Begunkov6dd0be12021-02-10 00:03:13 +0000294#define IO_COMPL_BATCH 32
Pavel Begunkov6ff119a2021-02-10 00:03:18 +0000295#define IO_REQ_CACHE_SIZE 32
Pavel Begunkovbf019da2021-02-10 00:03:17 +0000296#define IO_REQ_ALLOC_BATCH 8
Pavel Begunkov258b29a2021-02-10 00:03:10 +0000297
298struct io_comp_state {
Pavel Begunkov6dd0be12021-02-10 00:03:13 +0000299 struct io_kiocb *reqs[IO_COMPL_BATCH];
Jens Axboe1b4c3512021-02-10 00:03:19 +0000300 unsigned int nr;
Jens Axboec7dae4b2021-02-09 19:53:37 -0700301 unsigned int locked_free_nr;
302 /* inline/task_work completion list, under ->uring_lock */
Jens Axboe1b4c3512021-02-10 00:03:19 +0000303 struct list_head free_list;
Jens Axboec7dae4b2021-02-09 19:53:37 -0700304 /* IRQ completion list, under ->completion_lock */
305 struct list_head locked_free_list;
Pavel Begunkov258b29a2021-02-10 00:03:10 +0000306};
307
Pavel Begunkova1ab7b32021-02-18 18:29:42 +0000308struct io_submit_link {
309 struct io_kiocb *head;
310 struct io_kiocb *last;
311};
312
Pavel Begunkov258b29a2021-02-10 00:03:10 +0000313struct io_submit_state {
314 struct blk_plug plug;
Pavel Begunkova1ab7b32021-02-18 18:29:42 +0000315 struct io_submit_link link;
Pavel Begunkov258b29a2021-02-10 00:03:10 +0000316
317 /*
318 * io_kiocb alloc cache
319 */
Pavel Begunkovbf019da2021-02-10 00:03:17 +0000320 void *reqs[IO_REQ_CACHE_SIZE];
Pavel Begunkov258b29a2021-02-10 00:03:10 +0000321 unsigned int free_reqs;
322
323 bool plug_started;
324
325 /*
326 * Batch completion logic
327 */
328 struct io_comp_state comp;
329
330 /*
331 * File reference cache
332 */
333 struct file *file;
334 unsigned int fd;
335 unsigned int file_refs;
336 unsigned int ios_left;
337};
338
Jens Axboe2b188cc2019-01-07 10:46:33 -0700339struct io_ring_ctx {
340 struct {
341 struct percpu_ref refs;
342 } ____cacheline_aligned_in_smp;
343
344 struct {
345 unsigned int flags;
Randy Dunlape1d85332020-02-05 20:57:10 -0800346 unsigned int compat: 1;
Randy Dunlape1d85332020-02-05 20:57:10 -0800347 unsigned int drain_next: 1;
348 unsigned int eventfd_async: 1;
Stefano Garzarella21b55db2020-08-27 16:58:30 +0200349 unsigned int restricted: 1;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700350
Hristo Venev75b28af2019-08-26 17:23:46 +0000351 /*
352 * Ring buffer of indices into array of io_uring_sqe, which is
353 * mmapped by the application using the IORING_OFF_SQES offset.
354 *
355 * This indirection could e.g. be used to assign fixed
356 * io_uring_sqe entries to operations and only submit them to
357 * the queue when needed.
358 *
359 * The kernel modifies neither the indices array nor the entries
360 * array.
361 */
362 u32 *sq_array;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700363 unsigned cached_sq_head;
364 unsigned sq_entries;
365 unsigned sq_mask;
Jens Axboe6c271ce2019-01-10 11:22:30 -0700366 unsigned sq_thread_idle;
Jens Axboe498ccd92019-10-25 10:04:25 -0600367 unsigned cached_sq_dropped;
Pavel Begunkov2c3bac6d2020-10-18 10:17:40 +0100368 unsigned cached_cq_overflow;
Jens Axboead3eb2c2019-12-18 17:12:20 -0700369 unsigned long sq_check_overflow;
Jens Axboede0617e2019-04-06 21:51:27 -0600370
371 struct list_head defer_list;
Jens Axboe5262f562019-09-17 12:26:57 -0600372 struct list_head timeout_list;
Jens Axboe1d7bb1d2019-11-06 11:31:17 -0700373 struct list_head cq_overflow_list;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700374
Jens Axboead3eb2c2019-12-18 17:12:20 -0700375 struct io_uring_sqe *sq_sqes;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700376 } ____cacheline_aligned_in_smp;
377
Jens Axboe3c1a2ea2021-02-11 10:48:03 -0700378 struct {
379 struct mutex uring_lock;
380 wait_queue_head_t wait;
381 } ____cacheline_aligned_in_smp;
382
383 struct io_submit_state submit_state;
384
Hristo Venev75b28af2019-08-26 17:23:46 +0000385 struct io_rings *rings;
386
Stefan Metzmacher7c30f36a2021-03-07 11:54:28 +0100387 const struct cred *sq_creds; /* cred used for __io_sq_thread() */
Jens Axboe534ca6d2020-09-02 13:52:19 -0600388 struct io_sq_data *sq_data; /* if using sq thread polling */
389
Jens Axboe90554202020-09-03 12:12:41 -0600390 struct wait_queue_head sqo_sq_wait;
Jens Axboe69fb2132020-09-14 11:16:23 -0600391 struct list_head sqd_list;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700392
Jens Axboe6b063142019-01-10 22:13:58 -0700393 /*
394 * If used, fixed file set. Writers must ensure that ->refs is dead,
395 * readers must ensure that ->refs is alive as long as the file* is
396 * used. Only updated through io_uring_register(2).
397 */
Pavel Begunkovb895c9a2021-04-01 15:43:40 +0100398 struct io_rsrc_data *file_data;
Pavel Begunkovaeca2412021-04-11 01:46:37 +0100399 struct io_file_table file_table;
Jens Axboe6b063142019-01-10 22:13:58 -0700400 unsigned nr_user_files;
401
Jens Axboeedafcce2019-01-09 09:16:05 -0700402 /* if used, fixed mapped user buffers */
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +0100403 struct io_rsrc_data *buf_data;
Jens Axboeedafcce2019-01-09 09:16:05 -0700404 unsigned nr_user_bufs;
Pavel Begunkov41edf1a2021-04-25 14:32:23 +0100405 struct io_mapped_ubuf **user_bufs;
Jens Axboeedafcce2019-01-09 09:16:05 -0700406
Jens Axboe9e15c3a2021-03-13 12:29:43 -0700407 struct xarray io_buffers;
Jens Axboe5a2e7452020-02-23 16:23:11 -0700408
Matthew Wilcox (Oracle)61cf9372021-03-08 14:16:16 +0000409 struct xarray personalities;
410 u32 pers_next;
Jens Axboe071698e2020-01-28 10:04:42 -0700411
Jens Axboe206aefd2019-11-07 18:27:42 -0700412 struct {
413 unsigned cached_cq_tail;
414 unsigned cq_entries;
415 unsigned cq_mask;
416 atomic_t cq_timeouts;
Marcelo Diop-Gonzalezf0105052021-01-15 11:54:40 -0500417 unsigned cq_last_tm_flush;
Hao Xu7b289c32021-04-13 15:20:39 +0800418 unsigned cq_extra;
Jens Axboead3eb2c2019-12-18 17:12:20 -0700419 unsigned long cq_check_overflow;
Jens Axboe206aefd2019-11-07 18:27:42 -0700420 struct wait_queue_head cq_wait;
421 struct fasync_struct *cq_fasync;
422 struct eventfd_ctx *cq_ev_fd;
423 } ____cacheline_aligned_in_smp;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700424
425 struct {
Jens Axboe2b188cc2019-01-07 10:46:33 -0700426 spinlock_t completion_lock;
Jens Axboee94f1412019-12-19 12:06:02 -0700427
Jens Axboedef596e2019-01-09 08:59:42 -0700428 /*
Pavel Begunkov540e32a2020-07-13 23:37:09 +0300429 * ->iopoll_list is protected by the ctx->uring_lock for
Jens Axboedef596e2019-01-09 08:59:42 -0700430 * io_uring instances that don't use IORING_SETUP_SQPOLL.
431 * For SQPOLL, only the single threaded io_sq_thread() will
432 * manipulate the list, hence no extra locking is needed there.
433 */
Pavel Begunkov540e32a2020-07-13 23:37:09 +0300434 struct list_head iopoll_list;
Jens Axboe78076bb2019-12-04 19:56:40 -0700435 struct hlist_head *cancel_hash;
436 unsigned cancel_hash_bits;
Jens Axboee94f1412019-12-19 12:06:02 -0700437 bool poll_multi_file;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700438 } ____cacheline_aligned_in_smp;
Jens Axboe85faa7b2020-04-09 18:14:00 -0600439
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +0000440 struct delayed_work rsrc_put_work;
441 struct llist_head rsrc_put_llist;
Bijan Mottahedehd67d2262021-01-15 17:37:46 +0000442 struct list_head rsrc_ref_list;
443 spinlock_t rsrc_ref_lock;
Pavel Begunkova7f0ed52021-04-01 15:43:46 +0100444 struct io_rsrc_node *rsrc_node;
Pavel Begunkovb895c9a2021-04-01 15:43:40 +0100445 struct io_rsrc_node *rsrc_backup_node;
Pavel Begunkov62248432021-04-28 13:11:29 +0100446 struct io_mapped_ubuf *dummy_ubuf;
Jens Axboe4a38aed22020-05-14 17:21:15 -0600447
Stefano Garzarella21b55db2020-08-27 16:58:30 +0200448 struct io_restriction restrictions;
Jens Axboe3c1a2ea2021-02-11 10:48:03 -0700449
450 /* Keep this last, we don't need it for the fast path */
Pavel Begunkovb986af72021-05-16 22:58:06 +0100451 struct {
452 #if defined(CONFIG_UNIX)
453 struct socket *ring_sock;
454 #endif
455 /* hashed buffered write serialization */
456 struct io_wq_hash *hash_map;
457
458 /* Only used for accounting purposes */
459 struct user_struct *user;
460 struct mm_struct *mm_account;
461
462 /* ctx exit and cancelation */
463 struct callback_head *exit_task_work;
464 struct work_struct exit_work;
465 struct list_head tctx_list;
466 struct completion ref_comp;
467 };
Jens Axboe2b188cc2019-01-07 10:46:33 -0700468};
469
Stefan Metzmacher53e043b2021-03-15 12:56:56 +0100470struct io_uring_task {
471 /* submission side */
472 struct xarray xa;
473 struct wait_queue_head wait;
Stefan Metzmacheree53fb22021-03-15 12:56:57 +0100474 const struct io_ring_ctx *last;
475 struct io_wq *io_wq;
Stefan Metzmacher53e043b2021-03-15 12:56:56 +0100476 struct percpu_counter inflight;
Pavel Begunkovb303fe22021-04-11 01:46:26 +0100477 atomic_t inflight_tracked;
Stefan Metzmacher53e043b2021-03-15 12:56:56 +0100478 atomic_t in_idle;
Stefan Metzmacher53e043b2021-03-15 12:56:56 +0100479
480 spinlock_t task_lock;
481 struct io_wq_work_list task_list;
482 unsigned long task_state;
483 struct callback_head task_work;
484};
485
Jens Axboe09bb8392019-03-13 12:39:28 -0600486/*
487 * First field must be the file pointer in all the
488 * iocb unions! See also 'struct kiocb' in <linux/fs.h>
489 */
Jens Axboe221c5eb2019-01-17 09:41:58 -0700490struct io_poll_iocb {
491 struct file *file;
Pavel Begunkov018043b2020-10-27 23:17:18 +0000492 struct wait_queue_head *head;
Jens Axboe221c5eb2019-01-17 09:41:58 -0700493 __poll_t events;
Jens Axboe8c838782019-03-12 15:48:16 -0600494 bool done;
Jens Axboe221c5eb2019-01-17 09:41:58 -0700495 bool canceled;
Jens Axboe392edb42019-12-09 17:52:20 -0700496 struct wait_queue_entry wait;
Jens Axboe221c5eb2019-01-17 09:41:58 -0700497};
498
Pavel Begunkov9d805892021-04-13 02:58:40 +0100499struct io_poll_update {
Pavel Begunkov018043b2020-10-27 23:17:18 +0000500 struct file *file;
Pavel Begunkov9d805892021-04-13 02:58:40 +0100501 u64 old_user_data;
502 u64 new_user_data;
503 __poll_t events;
Jens Axboeb69de282021-03-17 08:37:41 -0600504 bool update_events;
505 bool update_user_data;
Pavel Begunkov018043b2020-10-27 23:17:18 +0000506};
507
Jens Axboeb5dba592019-12-11 14:02:38 -0700508struct io_close {
509 struct file *file;
Jens Axboeb5dba592019-12-11 14:02:38 -0700510 int fd;
511};
512
Jens Axboead8a48a2019-11-15 08:49:11 -0700513struct io_timeout_data {
514 struct io_kiocb *req;
515 struct hrtimer timer;
516 struct timespec64 ts;
517 enum hrtimer_mode mode;
518};
519
Jens Axboe8ed8d3c2019-12-16 11:55:28 -0700520struct io_accept {
521 struct file *file;
522 struct sockaddr __user *addr;
523 int __user *addr_len;
524 int flags;
Jens Axboe09952e32020-03-19 20:16:56 -0600525 unsigned long nofile;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -0700526};
527
528struct io_sync {
529 struct file *file;
530 loff_t len;
531 loff_t off;
532 int flags;
Jens Axboed63d1b52019-12-10 10:38:56 -0700533 int mode;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -0700534};
535
Jens Axboefbf23842019-12-17 18:45:56 -0700536struct io_cancel {
537 struct file *file;
538 u64 addr;
539};
540
Jens Axboeb29472e2019-12-17 18:50:29 -0700541struct io_timeout {
542 struct file *file;
Pavel Begunkovbfe68a22020-05-30 14:54:18 +0300543 u32 off;
544 u32 target_seq;
Pavel Begunkov135fcde2020-07-13 23:37:12 +0300545 struct list_head list;
Pavel Begunkov90cd7e42020-10-27 23:25:36 +0000546 /* head of the link, used by linked timeouts only */
547 struct io_kiocb *head;
Jens Axboeb29472e2019-12-17 18:50:29 -0700548};
549
Pavel Begunkov0bdf7a22020-10-10 18:34:10 +0100550struct io_timeout_rem {
551 struct file *file;
552 u64 addr;
Pavel Begunkov9c8e11b2020-11-30 19:11:16 +0000553
554 /* timeout update */
555 struct timespec64 ts;
556 u32 flags;
Pavel Begunkov0bdf7a22020-10-10 18:34:10 +0100557};
558
Jens Axboe9adbd452019-12-20 08:45:55 -0700559struct io_rw {
560 /* NOTE: kiocb has the file as the first member, so don't do it here */
561 struct kiocb kiocb;
562 u64 addr;
563 u64 len;
564};
565
Jens Axboe3fbb51c2019-12-20 08:51:52 -0700566struct io_connect {
567 struct file *file;
568 struct sockaddr __user *addr;
569 int addr_len;
570};
571
Jens Axboee47293f2019-12-20 08:58:21 -0700572struct io_sr_msg {
573 struct file *file;
Jens Axboefddafac2020-01-04 20:19:44 -0700574 union {
Pavel Begunkov4af34172021-04-11 01:46:30 +0100575 struct compat_msghdr __user *umsg_compat;
576 struct user_msghdr __user *umsg;
577 void __user *buf;
Jens Axboefddafac2020-01-04 20:19:44 -0700578 };
Jens Axboee47293f2019-12-20 08:58:21 -0700579 int msg_flags;
Jens Axboebcda7ba2020-02-23 16:42:51 -0700580 int bgid;
Jens Axboefddafac2020-01-04 20:19:44 -0700581 size_t len;
Jens Axboebcda7ba2020-02-23 16:42:51 -0700582 struct io_buffer *kbuf;
Jens Axboee47293f2019-12-20 08:58:21 -0700583};
584
Jens Axboe15b71ab2019-12-11 11:20:36 -0700585struct io_open {
586 struct file *file;
587 int dfd;
Jens Axboe15b71ab2019-12-11 11:20:36 -0700588 struct filename *filename;
Jens Axboec12cedf2020-01-08 17:41:21 -0700589 struct open_how how;
Jens Axboe4022e7a2020-03-19 19:23:18 -0600590 unsigned long nofile;
Jens Axboe15b71ab2019-12-11 11:20:36 -0700591};
592
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +0000593struct io_rsrc_update {
Jens Axboe05f3fb32019-12-09 11:22:50 -0700594 struct file *file;
595 u64 arg;
596 u32 nr_args;
597 u32 offset;
598};
599
Jens Axboe4840e412019-12-25 22:03:45 -0700600struct io_fadvise {
601 struct file *file;
602 u64 offset;
603 u32 len;
604 u32 advice;
605};
606
Jens Axboec1ca7572019-12-25 22:18:28 -0700607struct io_madvise {
608 struct file *file;
609 u64 addr;
610 u32 len;
611 u32 advice;
612};
613
Jens Axboe3e4827b2020-01-08 15:18:09 -0700614struct io_epoll {
615 struct file *file;
616 int epfd;
617 int op;
618 int fd;
619 struct epoll_event event;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700620};
621
Pavel Begunkov7d67af22020-02-24 11:32:45 +0300622struct io_splice {
623 struct file *file_out;
624 struct file *file_in;
625 loff_t off_out;
626 loff_t off_in;
627 u64 len;
628 unsigned int flags;
629};
630
Jens Axboeddf0322d2020-02-23 16:41:33 -0700631struct io_provide_buf {
632 struct file *file;
633 __u64 addr;
Pavel Begunkov38134ad2021-04-15 13:07:39 +0100634 __u32 len;
Jens Axboeddf0322d2020-02-23 16:41:33 -0700635 __u32 bgid;
636 __u16 nbufs;
637 __u16 bid;
638};
639
Bijan Mottahedeh1d9e1282020-05-22 21:31:16 -0700640struct io_statx {
641 struct file *file;
642 int dfd;
643 unsigned int mask;
644 unsigned int flags;
Bijan Mottahedehe62753e2020-05-22 21:31:18 -0700645 const char __user *filename;
Bijan Mottahedeh1d9e1282020-05-22 21:31:16 -0700646 struct statx __user *buffer;
647};
648
Jens Axboe36f4fa62020-09-05 11:14:22 -0600649struct io_shutdown {
650 struct file *file;
651 int how;
652};
653
Jens Axboe80a261f2020-09-28 14:23:58 -0600654struct io_rename {
655 struct file *file;
656 int old_dfd;
657 int new_dfd;
658 struct filename *oldpath;
659 struct filename *newpath;
660 int flags;
661};
662
Jens Axboe14a11432020-09-28 14:27:37 -0600663struct io_unlink {
664 struct file *file;
665 int dfd;
666 int flags;
667 struct filename *filename;
668};
669
Pavel Begunkov3ca405e2020-07-13 23:37:08 +0300670struct io_completion {
671 struct file *file;
672 struct list_head list;
Pavel Begunkov8c3f9cd2021-02-28 22:35:15 +0000673 u32 cflags;
Pavel Begunkov3ca405e2020-07-13 23:37:08 +0300674};
675
Jens Axboef499a022019-12-02 16:28:46 -0700676struct io_async_connect {
677 struct sockaddr_storage address;
678};
679
Jens Axboe03b12302019-12-02 18:50:25 -0700680struct io_async_msghdr {
681 struct iovec fast_iov[UIO_FASTIOV];
Pavel Begunkov257e84a2021-02-05 00:58:00 +0000682 /* points to an allocated iov, if NULL we use fast_iov instead */
683 struct iovec *free_iov;
Jens Axboe03b12302019-12-02 18:50:25 -0700684 struct sockaddr __user *uaddr;
685 struct msghdr msg;
Jens Axboeb5379162020-02-09 11:29:15 -0700686 struct sockaddr_storage addr;
Jens Axboe03b12302019-12-02 18:50:25 -0700687};
688
Jens Axboef67676d2019-12-02 11:03:47 -0700689struct io_async_rw {
690 struct iovec fast_iov[UIO_FASTIOV];
Jens Axboeff6165b2020-08-13 09:47:43 -0600691 const struct iovec *free_iovec;
692 struct iov_iter iter;
Jens Axboe227c0c92020-08-13 11:51:40 -0600693 size_t bytes_done;
Jens Axboebcf5a062020-05-22 09:24:42 -0600694 struct wait_page_queue wpq;
Jens Axboef67676d2019-12-02 11:03:47 -0700695};
696
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300697enum {
698 REQ_F_FIXED_FILE_BIT = IOSQE_FIXED_FILE_BIT,
699 REQ_F_IO_DRAIN_BIT = IOSQE_IO_DRAIN_BIT,
700 REQ_F_LINK_BIT = IOSQE_IO_LINK_BIT,
701 REQ_F_HARDLINK_BIT = IOSQE_IO_HARDLINK_BIT,
702 REQ_F_FORCE_ASYNC_BIT = IOSQE_ASYNC_BIT,
Jens Axboebcda7ba2020-02-23 16:42:51 -0700703 REQ_F_BUFFER_SELECT_BIT = IOSQE_BUFFER_SELECT_BIT,
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300704
Pavel Begunkovdddca222021-04-27 16:13:52 +0100705 /* first byte is taken by user flags, shift it to not overlap */
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +0100706 REQ_F_FAIL_BIT = 8,
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300707 REQ_F_INFLIGHT_BIT,
708 REQ_F_CUR_POS_BIT,
709 REQ_F_NOWAIT_BIT,
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300710 REQ_F_LINK_TIMEOUT_BIT,
Pavel Begunkov99bc4c32020-02-07 22:04:45 +0300711 REQ_F_NEED_CLEANUP_BIT,
Jens Axboed7718a92020-02-14 22:23:12 -0700712 REQ_F_POLLED_BIT,
Jens Axboebcda7ba2020-02-23 16:42:51 -0700713 REQ_F_BUFFER_SELECTED_BIT,
Pavel Begunkov900fad42020-10-19 16:39:16 +0100714 REQ_F_LTIMEOUT_ACTIVE_BIT,
Pavel Begunkove342c802021-01-19 13:32:47 +0000715 REQ_F_COMPLETE_INLINE_BIT,
Jens Axboe230d50d2021-04-01 20:41:15 -0600716 REQ_F_REISSUE_BIT,
Pavel Begunkov8c130822021-03-22 01:58:32 +0000717 REQ_F_DONT_REISSUE_BIT,
Jens Axboe7b29f922021-03-12 08:30:14 -0700718 /* keep async read/write and isreg together and in order */
719 REQ_F_ASYNC_READ_BIT,
720 REQ_F_ASYNC_WRITE_BIT,
721 REQ_F_ISREG_BIT,
Jens Axboe84557872020-03-03 15:28:17 -0700722
723 /* not a real bit, just to check we're not overflowing the space */
724 __REQ_F_LAST_BIT,
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300725};
726
727enum {
728 /* ctx owns file */
729 REQ_F_FIXED_FILE = BIT(REQ_F_FIXED_FILE_BIT),
730 /* drain existing IO first */
731 REQ_F_IO_DRAIN = BIT(REQ_F_IO_DRAIN_BIT),
732 /* linked sqes */
733 REQ_F_LINK = BIT(REQ_F_LINK_BIT),
734 /* doesn't sever on completion < 0 */
735 REQ_F_HARDLINK = BIT(REQ_F_HARDLINK_BIT),
736 /* IOSQE_ASYNC */
737 REQ_F_FORCE_ASYNC = BIT(REQ_F_FORCE_ASYNC_BIT),
Jens Axboebcda7ba2020-02-23 16:42:51 -0700738 /* IOSQE_BUFFER_SELECT */
739 REQ_F_BUFFER_SELECT = BIT(REQ_F_BUFFER_SELECT_BIT),
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300740
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300741 /* fail rest of links */
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +0100742 REQ_F_FAIL = BIT(REQ_F_FAIL_BIT),
Pavel Begunkovb05a1bc2021-03-04 13:59:24 +0000743 /* on inflight list, should be cancelled and waited on exit reliably */
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300744 REQ_F_INFLIGHT = BIT(REQ_F_INFLIGHT_BIT),
745 /* read/write uses file position */
746 REQ_F_CUR_POS = BIT(REQ_F_CUR_POS_BIT),
747 /* must not punt to workers */
748 REQ_F_NOWAIT = BIT(REQ_F_NOWAIT_BIT),
Pavel Begunkov900fad42020-10-19 16:39:16 +0100749 /* has or had linked timeout */
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300750 REQ_F_LINK_TIMEOUT = BIT(REQ_F_LINK_TIMEOUT_BIT),
Pavel Begunkov99bc4c32020-02-07 22:04:45 +0300751 /* needs cleanup */
752 REQ_F_NEED_CLEANUP = BIT(REQ_F_NEED_CLEANUP_BIT),
Jens Axboed7718a92020-02-14 22:23:12 -0700753 /* already went through poll handler */
754 REQ_F_POLLED = BIT(REQ_F_POLLED_BIT),
Jens Axboebcda7ba2020-02-23 16:42:51 -0700755 /* buffer already selected */
756 REQ_F_BUFFER_SELECTED = BIT(REQ_F_BUFFER_SELECTED_BIT),
Pavel Begunkov900fad42020-10-19 16:39:16 +0100757 /* linked timeout is active, i.e. prepared by link's head */
758 REQ_F_LTIMEOUT_ACTIVE = BIT(REQ_F_LTIMEOUT_ACTIVE_BIT),
Pavel Begunkove342c802021-01-19 13:32:47 +0000759 /* completion is deferred through io_comp_state */
760 REQ_F_COMPLETE_INLINE = BIT(REQ_F_COMPLETE_INLINE_BIT),
Jens Axboe230d50d2021-04-01 20:41:15 -0600761 /* caller should reissue async */
762 REQ_F_REISSUE = BIT(REQ_F_REISSUE_BIT),
Pavel Begunkov8c130822021-03-22 01:58:32 +0000763 /* don't attempt request reissue, see io_rw_reissue() */
764 REQ_F_DONT_REISSUE = BIT(REQ_F_DONT_REISSUE_BIT),
Jens Axboe7b29f922021-03-12 08:30:14 -0700765 /* supports async reads */
766 REQ_F_ASYNC_READ = BIT(REQ_F_ASYNC_READ_BIT),
767 /* supports async writes */
768 REQ_F_ASYNC_WRITE = BIT(REQ_F_ASYNC_WRITE_BIT),
769 /* regular file */
770 REQ_F_ISREG = BIT(REQ_F_ISREG_BIT),
Jens Axboed7718a92020-02-14 22:23:12 -0700771};
772
773struct async_poll {
774 struct io_poll_iocb poll;
Jens Axboe807abcb2020-07-17 17:09:27 -0600775 struct io_poll_iocb *double_poll;
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300776};
777
Jens Axboe7cbf1722021-02-10 00:03:20 +0000778struct io_task_work {
779 struct io_wq_work_node node;
780 task_work_func_t func;
781};
782
Pavel Begunkov992da012021-06-10 16:37:37 +0100783enum {
784 IORING_RSRC_FILE = 0,
785 IORING_RSRC_BUFFER = 1,
786};
787
Jens Axboe09bb8392019-03-13 12:39:28 -0600788/*
789 * NOTE! Each of the iocb union members has the file pointer
790 * as the first entry in their struct definition. So you can
791 * access the file pointer through any of the sub-structs,
792 * or directly as just 'ki_filp' in this struct.
793 */
Jens Axboe2b188cc2019-01-07 10:46:33 -0700794struct io_kiocb {
Jens Axboe221c5eb2019-01-17 09:41:58 -0700795 union {
Jens Axboe09bb8392019-03-13 12:39:28 -0600796 struct file *file;
Jens Axboe9adbd452019-12-20 08:45:55 -0700797 struct io_rw rw;
Jens Axboe221c5eb2019-01-17 09:41:58 -0700798 struct io_poll_iocb poll;
Pavel Begunkov9d805892021-04-13 02:58:40 +0100799 struct io_poll_update poll_update;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -0700800 struct io_accept accept;
801 struct io_sync sync;
Jens Axboefbf23842019-12-17 18:45:56 -0700802 struct io_cancel cancel;
Jens Axboeb29472e2019-12-17 18:50:29 -0700803 struct io_timeout timeout;
Pavel Begunkov0bdf7a22020-10-10 18:34:10 +0100804 struct io_timeout_rem timeout_rem;
Jens Axboe3fbb51c2019-12-20 08:51:52 -0700805 struct io_connect connect;
Jens Axboee47293f2019-12-20 08:58:21 -0700806 struct io_sr_msg sr_msg;
Jens Axboe15b71ab2019-12-11 11:20:36 -0700807 struct io_open open;
Jens Axboeb5dba592019-12-11 14:02:38 -0700808 struct io_close close;
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +0000809 struct io_rsrc_update rsrc_update;
Jens Axboe4840e412019-12-25 22:03:45 -0700810 struct io_fadvise fadvise;
Jens Axboec1ca7572019-12-25 22:18:28 -0700811 struct io_madvise madvise;
Jens Axboe3e4827b2020-01-08 15:18:09 -0700812 struct io_epoll epoll;
Pavel Begunkov7d67af22020-02-24 11:32:45 +0300813 struct io_splice splice;
Jens Axboeddf0322d2020-02-23 16:41:33 -0700814 struct io_provide_buf pbuf;
Bijan Mottahedeh1d9e1282020-05-22 21:31:16 -0700815 struct io_statx statx;
Jens Axboe36f4fa62020-09-05 11:14:22 -0600816 struct io_shutdown shutdown;
Jens Axboe80a261f2020-09-28 14:23:58 -0600817 struct io_rename rename;
Jens Axboe14a11432020-09-28 14:27:37 -0600818 struct io_unlink unlink;
Pavel Begunkov3ca405e2020-07-13 23:37:08 +0300819 /* use only after cleaning per-op data, see io_clean_op() */
820 struct io_completion compl;
Jens Axboe221c5eb2019-01-17 09:41:58 -0700821 };
Jens Axboe2b188cc2019-01-07 10:46:33 -0700822
Jens Axboee8c2bc12020-08-15 18:44:09 -0700823 /* opcode allocated if it needs to store data for async defer */
824 void *async_data;
Jens Axboed625c6e2019-12-17 19:53:05 -0700825 u8 opcode;
Xiaoguang Wang65a65432020-06-11 23:39:36 +0800826 /* polled IO has completed */
827 u8 iopoll_completed;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700828
Bijan Mottahedeh4f4eeba2020-05-19 14:52:49 -0700829 u16 buf_index;
Pavel Begunkov9cf7c102020-07-13 23:37:15 +0300830 u32 result;
Bijan Mottahedeh4f4eeba2020-05-19 14:52:49 -0700831
Pavel Begunkov010e8e62020-07-30 18:43:45 +0300832 struct io_ring_ctx *ctx;
833 unsigned int flags;
Jens Axboeabc54d62021-02-24 13:32:30 -0700834 atomic_t refs;
Pavel Begunkov010e8e62020-07-30 18:43:45 +0300835 struct task_struct *task;
836 u64 user_data;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700837
Pavel Begunkovf2f87372020-10-27 23:25:37 +0000838 struct io_kiocb *link;
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +0000839 struct percpu_ref *fixed_rsrc_refs;
Jens Axboed7718a92020-02-14 22:23:12 -0700840
Pavel Begunkovb303fe22021-04-11 01:46:26 +0100841 /* used with ctx->iopoll_list with reads/writes */
Pavel Begunkov010e8e62020-07-30 18:43:45 +0300842 struct list_head inflight_entry;
Jens Axboe7cbf1722021-02-10 00:03:20 +0000843 union {
844 struct io_task_work io_task_work;
845 struct callback_head task_work;
846 };
Pavel Begunkov010e8e62020-07-30 18:43:45 +0300847 /* for polled requests, i.e. IORING_OP_POLL_ADD and async armed poll */
848 struct hlist_node hash_node;
849 struct async_poll *apoll;
850 struct io_wq_work work;
Pavel Begunkoveae071c2021-04-25 14:32:24 +0100851 /* store used ubuf, so we can prevent reloading */
852 struct io_mapped_ubuf *imu;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700853};
854
Pavel Begunkov13bf43f2021-03-06 11:02:12 +0000855struct io_tctx_node {
856 struct list_head ctx_node;
857 struct task_struct *task;
Pavel Begunkov13bf43f2021-03-06 11:02:12 +0000858 struct io_ring_ctx *ctx;
859};
860
Pavel Begunkov27dc8332020-07-13 23:37:14 +0300861struct io_defer_entry {
862 struct list_head list;
863 struct io_kiocb *req;
Pavel Begunkov9cf7c102020-07-13 23:37:15 +0300864 u32 seq;
Pavel Begunkov27dc8332020-07-13 23:37:14 +0300865};
866
Jens Axboed3656342019-12-18 09:50:26 -0700867struct io_op_def {
Jens Axboed3656342019-12-18 09:50:26 -0700868 /* needs req->file assigned */
869 unsigned needs_file : 1;
Jens Axboed3656342019-12-18 09:50:26 -0700870 /* hash wq insertion if file is a regular file */
871 unsigned hash_reg_file : 1;
872 /* unbound wq insertion if file is a non-regular file */
873 unsigned unbound_nonreg_file : 1;
Jens Axboe66f4af92020-01-16 15:36:52 -0700874 /* opcode is not supported by this kernel */
875 unsigned not_supported : 1;
Jens Axboe8a727582020-02-20 09:59:44 -0700876 /* set if opcode supports polled "wait" */
877 unsigned pollin : 1;
878 unsigned pollout : 1;
Jens Axboebcda7ba2020-02-23 16:42:51 -0700879 /* op supports buffer selection */
880 unsigned buffer_select : 1;
Pavel Begunkov26f05052021-02-28 22:35:18 +0000881 /* do prep async if is going to be punted */
882 unsigned needs_async_setup : 1;
Jens Axboe27926b62020-10-28 09:33:23 -0600883 /* should block plug */
884 unsigned plug : 1;
Jens Axboee8c2bc12020-08-15 18:44:09 -0700885 /* size of async data needed, if any */
886 unsigned short async_size;
Jens Axboed3656342019-12-18 09:50:26 -0700887};
888
Jens Axboe09186822020-10-13 15:01:40 -0600889static const struct io_op_def io_op_defs[] = {
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300890 [IORING_OP_NOP] = {},
891 [IORING_OP_READV] = {
Jens Axboed3656342019-12-18 09:50:26 -0700892 .needs_file = 1,
893 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -0700894 .pollin = 1,
Jens Axboe4d954c22020-02-27 07:31:19 -0700895 .buffer_select = 1,
Pavel Begunkov26f05052021-02-28 22:35:18 +0000896 .needs_async_setup = 1,
Jens Axboe27926b62020-10-28 09:33:23 -0600897 .plug = 1,
Jens Axboee8c2bc12020-08-15 18:44:09 -0700898 .async_size = sizeof(struct io_async_rw),
Jens Axboed3656342019-12-18 09:50:26 -0700899 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300900 [IORING_OP_WRITEV] = {
Jens Axboed3656342019-12-18 09:50:26 -0700901 .needs_file = 1,
902 .hash_reg_file = 1,
903 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -0700904 .pollout = 1,
Pavel Begunkov26f05052021-02-28 22:35:18 +0000905 .needs_async_setup = 1,
Jens Axboe27926b62020-10-28 09:33:23 -0600906 .plug = 1,
Jens Axboee8c2bc12020-08-15 18:44:09 -0700907 .async_size = sizeof(struct io_async_rw),
Jens Axboed3656342019-12-18 09:50:26 -0700908 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300909 [IORING_OP_FSYNC] = {
Jens Axboed3656342019-12-18 09:50:26 -0700910 .needs_file = 1,
911 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300912 [IORING_OP_READ_FIXED] = {
Jens Axboed3656342019-12-18 09:50:26 -0700913 .needs_file = 1,
914 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -0700915 .pollin = 1,
Jens Axboe27926b62020-10-28 09:33:23 -0600916 .plug = 1,
Jens Axboee8c2bc12020-08-15 18:44:09 -0700917 .async_size = sizeof(struct io_async_rw),
Jens Axboed3656342019-12-18 09:50:26 -0700918 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300919 [IORING_OP_WRITE_FIXED] = {
Jens Axboed3656342019-12-18 09:50:26 -0700920 .needs_file = 1,
921 .hash_reg_file = 1,
922 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -0700923 .pollout = 1,
Jens Axboe27926b62020-10-28 09:33:23 -0600924 .plug = 1,
Jens Axboee8c2bc12020-08-15 18:44:09 -0700925 .async_size = sizeof(struct io_async_rw),
Jens Axboed3656342019-12-18 09:50:26 -0700926 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300927 [IORING_OP_POLL_ADD] = {
Jens Axboed3656342019-12-18 09:50:26 -0700928 .needs_file = 1,
929 .unbound_nonreg_file = 1,
930 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300931 [IORING_OP_POLL_REMOVE] = {},
932 [IORING_OP_SYNC_FILE_RANGE] = {
Jens Axboed3656342019-12-18 09:50:26 -0700933 .needs_file = 1,
934 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300935 [IORING_OP_SENDMSG] = {
Jens Axboed3656342019-12-18 09:50:26 -0700936 .needs_file = 1,
937 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -0700938 .pollout = 1,
Pavel Begunkov26f05052021-02-28 22:35:18 +0000939 .needs_async_setup = 1,
Jens Axboee8c2bc12020-08-15 18:44:09 -0700940 .async_size = sizeof(struct io_async_msghdr),
Jens Axboed3656342019-12-18 09:50:26 -0700941 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300942 [IORING_OP_RECVMSG] = {
Jens Axboed3656342019-12-18 09:50:26 -0700943 .needs_file = 1,
944 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -0700945 .pollin = 1,
Jens Axboe52de1fe2020-02-27 10:15:42 -0700946 .buffer_select = 1,
Pavel Begunkov26f05052021-02-28 22:35:18 +0000947 .needs_async_setup = 1,
Jens Axboee8c2bc12020-08-15 18:44:09 -0700948 .async_size = sizeof(struct io_async_msghdr),
Jens Axboed3656342019-12-18 09:50:26 -0700949 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300950 [IORING_OP_TIMEOUT] = {
Jens Axboee8c2bc12020-08-15 18:44:09 -0700951 .async_size = sizeof(struct io_timeout_data),
Jens Axboed3656342019-12-18 09:50:26 -0700952 },
Pavel Begunkov9c8e11b2020-11-30 19:11:16 +0000953 [IORING_OP_TIMEOUT_REMOVE] = {
954 /* used by timeout updates' prep() */
Pavel Begunkov9c8e11b2020-11-30 19:11:16 +0000955 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300956 [IORING_OP_ACCEPT] = {
Jens Axboed3656342019-12-18 09:50:26 -0700957 .needs_file = 1,
958 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -0700959 .pollin = 1,
Jens Axboed3656342019-12-18 09:50:26 -0700960 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300961 [IORING_OP_ASYNC_CANCEL] = {},
962 [IORING_OP_LINK_TIMEOUT] = {
Jens Axboee8c2bc12020-08-15 18:44:09 -0700963 .async_size = sizeof(struct io_timeout_data),
Jens Axboed3656342019-12-18 09:50:26 -0700964 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300965 [IORING_OP_CONNECT] = {
Jens Axboed3656342019-12-18 09:50:26 -0700966 .needs_file = 1,
967 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -0700968 .pollout = 1,
Pavel Begunkov26f05052021-02-28 22:35:18 +0000969 .needs_async_setup = 1,
Jens Axboee8c2bc12020-08-15 18:44:09 -0700970 .async_size = sizeof(struct io_async_connect),
Jens Axboed3656342019-12-18 09:50:26 -0700971 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300972 [IORING_OP_FALLOCATE] = {
Jens Axboed3656342019-12-18 09:50:26 -0700973 .needs_file = 1,
974 },
Jens Axboe44526be2021-02-15 13:32:18 -0700975 [IORING_OP_OPENAT] = {},
976 [IORING_OP_CLOSE] = {},
977 [IORING_OP_FILES_UPDATE] = {},
978 [IORING_OP_STATX] = {},
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300979 [IORING_OP_READ] = {
Jens Axboe3a6820f2019-12-22 15:19:35 -0700980 .needs_file = 1,
981 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -0700982 .pollin = 1,
Jens Axboebcda7ba2020-02-23 16:42:51 -0700983 .buffer_select = 1,
Jens Axboe27926b62020-10-28 09:33:23 -0600984 .plug = 1,
Jens Axboee8c2bc12020-08-15 18:44:09 -0700985 .async_size = sizeof(struct io_async_rw),
Jens Axboe3a6820f2019-12-22 15:19:35 -0700986 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300987 [IORING_OP_WRITE] = {
Jens Axboe3a6820f2019-12-22 15:19:35 -0700988 .needs_file = 1,
989 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -0700990 .pollout = 1,
Jens Axboe27926b62020-10-28 09:33:23 -0600991 .plug = 1,
Jens Axboee8c2bc12020-08-15 18:44:09 -0700992 .async_size = sizeof(struct io_async_rw),
Jens Axboe3a6820f2019-12-22 15:19:35 -0700993 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300994 [IORING_OP_FADVISE] = {
Jens Axboe4840e412019-12-25 22:03:45 -0700995 .needs_file = 1,
996 },
Jens Axboe44526be2021-02-15 13:32:18 -0700997 [IORING_OP_MADVISE] = {},
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300998 [IORING_OP_SEND] = {
Jens Axboefddafac2020-01-04 20:19:44 -0700999 .needs_file = 1,
1000 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -07001001 .pollout = 1,
Jens Axboefddafac2020-01-04 20:19:44 -07001002 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +03001003 [IORING_OP_RECV] = {
Jens Axboefddafac2020-01-04 20:19:44 -07001004 .needs_file = 1,
1005 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -07001006 .pollin = 1,
Jens Axboebcda7ba2020-02-23 16:42:51 -07001007 .buffer_select = 1,
Jens Axboefddafac2020-01-04 20:19:44 -07001008 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +03001009 [IORING_OP_OPENAT2] = {
Jens Axboecebdb982020-01-08 17:59:24 -07001010 },
Jens Axboe3e4827b2020-01-08 15:18:09 -07001011 [IORING_OP_EPOLL_CTL] = {
1012 .unbound_nonreg_file = 1,
Jens Axboe3e4827b2020-01-08 15:18:09 -07001013 },
Pavel Begunkov7d67af22020-02-24 11:32:45 +03001014 [IORING_OP_SPLICE] = {
1015 .needs_file = 1,
1016 .hash_reg_file = 1,
1017 .unbound_nonreg_file = 1,
Jens Axboeddf0322d2020-02-23 16:41:33 -07001018 },
1019 [IORING_OP_PROVIDE_BUFFERS] = {},
Jens Axboe067524e2020-03-02 16:32:28 -07001020 [IORING_OP_REMOVE_BUFFERS] = {},
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03001021 [IORING_OP_TEE] = {
1022 .needs_file = 1,
1023 .hash_reg_file = 1,
1024 .unbound_nonreg_file = 1,
1025 },
Jens Axboe36f4fa62020-09-05 11:14:22 -06001026 [IORING_OP_SHUTDOWN] = {
1027 .needs_file = 1,
1028 },
Jens Axboe44526be2021-02-15 13:32:18 -07001029 [IORING_OP_RENAMEAT] = {},
1030 [IORING_OP_UNLINKAT] = {},
Jens Axboed3656342019-12-18 09:50:26 -07001031};
1032
Pavel Begunkov7a612352021-03-09 00:37:59 +00001033static bool io_disarm_next(struct io_kiocb *req);
Pavel Begunkovd56d9382021-03-06 11:02:13 +00001034static void io_uring_del_task_file(unsigned long index);
Pavel Begunkov9936c7c2021-02-04 13:51:56 +00001035static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
1036 struct task_struct *task,
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01001037 bool cancel_all);
Pavel Begunkov734551d2021-04-18 14:52:09 +01001038static void io_uring_cancel_sqpoll(struct io_sq_data *sqd);
Pavel Begunkovb895c9a2021-04-01 15:43:40 +01001039static struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx);
Pavel Begunkov1ffc5422020-12-30 21:34:15 +00001040
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01001041static bool io_cqring_fill_event(struct io_ring_ctx *ctx, u64 user_data,
1042 long res, unsigned int cflags);
Jackie Liuec9c02a2019-11-08 23:50:36 +08001043static void io_put_req(struct io_kiocb *req);
Pavel Begunkov216578e2020-10-13 09:44:00 +01001044static void io_put_req_deferred(struct io_kiocb *req, int nr);
Jens Axboec7dae4b2021-02-09 19:53:37 -07001045static void io_dismantle_req(struct io_kiocb *req);
1046static void io_put_task(struct task_struct *task, int nr);
Jens Axboe94ae5e72019-11-14 19:39:52 -07001047static struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req);
1048static void io_queue_linked_timeout(struct io_kiocb *req);
Pavel Begunkovfdecb662021-04-25 14:32:20 +01001049static int __io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned type,
Pavel Begunkovc3bdad02021-04-25 14:32:22 +01001050 struct io_uring_rsrc_update2 *up,
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +01001051 unsigned nr_args);
Pavel Begunkov68fb8972021-03-19 17:22:41 +00001052static void io_clean_op(struct io_kiocb *req);
Pavel Begunkov8371adf2020-10-10 18:34:08 +01001053static struct file *io_file_get(struct io_submit_state *state,
1054 struct io_kiocb *req, int fd, bool fixed);
Pavel Begunkovc5eef2b2021-02-10 00:03:22 +00001055static void __io_queue_sqe(struct io_kiocb *req);
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00001056static void io_rsrc_put_work(struct work_struct *work);
Jens Axboede0617e2019-04-06 21:51:27 -06001057
Pavel Begunkov907d1df2021-01-26 23:35:10 +00001058static void io_req_task_queue(struct io_kiocb *req);
Jens Axboe65453d12021-02-10 00:03:21 +00001059static void io_submit_flush_completions(struct io_comp_state *cs,
1060 struct io_ring_ctx *ctx);
Jens Axboe50826202021-02-23 09:02:26 -07001061static bool io_poll_remove_waitqs(struct io_kiocb *req);
Pavel Begunkov179ae0d2021-02-28 22:35:20 +00001062static int io_req_prep_async(struct io_kiocb *req);
Jens Axboe9a56a232019-01-09 09:06:50 -07001063
Jens Axboe2b188cc2019-01-07 10:46:33 -07001064static struct kmem_cache *req_cachep;
1065
Jens Axboe09186822020-10-13 15:01:40 -06001066static const struct file_operations io_uring_fops;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001067
1068struct sock *io_uring_get_socket(struct file *file)
1069{
1070#if defined(CONFIG_UNIX)
1071 if (file->f_op == &io_uring_fops) {
1072 struct io_ring_ctx *ctx = file->private_data;
1073
1074 return ctx->ring_sock->sk;
1075 }
1076#endif
1077 return NULL;
1078}
1079EXPORT_SYMBOL(io_uring_get_socket);
1080
Pavel Begunkovf2f87372020-10-27 23:25:37 +00001081#define io_for_each_link(pos, head) \
1082 for (pos = (head); pos; pos = pos->link)
1083
Pavel Begunkovb895c9a2021-04-01 15:43:40 +01001084static inline void io_req_set_rsrc_node(struct io_kiocb *req)
Jens Axboec40f6372020-06-25 15:39:59 -06001085{
Pavel Begunkov36f72fe2020-11-18 19:57:26 +00001086 struct io_ring_ctx *ctx = req->ctx;
1087
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00001088 if (!req->fixed_rsrc_refs) {
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01001089 req->fixed_rsrc_refs = &ctx->rsrc_node->refs;
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00001090 percpu_ref_get(req->fixed_rsrc_refs);
Pavel Begunkov36f72fe2020-11-18 19:57:26 +00001091 }
1092}
1093
Pavel Begunkovf70865d2021-04-11 01:46:40 +01001094static void io_refs_resurrect(struct percpu_ref *ref, struct completion *compl)
1095{
1096 bool got = percpu_ref_tryget(ref);
1097
1098 /* already at zero, wait for ->release() */
1099 if (!got)
1100 wait_for_completion(compl);
1101 percpu_ref_resurrect(ref);
1102 if (got)
1103 percpu_ref_put(ref);
1104}
1105
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01001106static bool io_match_task(struct io_kiocb *head, struct task_struct *task,
1107 bool cancel_all)
Pavel Begunkov08d23632020-11-06 13:00:22 +00001108{
1109 struct io_kiocb *req;
1110
Pavel Begunkov68207682021-03-22 01:58:25 +00001111 if (task && head->task != task)
Pavel Begunkov08d23632020-11-06 13:00:22 +00001112 return false;
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01001113 if (cancel_all)
Pavel Begunkov08d23632020-11-06 13:00:22 +00001114 return true;
1115
1116 io_for_each_link(req, head) {
Pavel Begunkovb05a1bc2021-03-04 13:59:24 +00001117 if (req->flags & REQ_F_INFLIGHT)
Jens Axboe02a13672021-01-23 15:49:31 -07001118 return true;
Pavel Begunkov08d23632020-11-06 13:00:22 +00001119 }
1120 return false;
1121}
1122
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01001123static inline void req_set_fail(struct io_kiocb *req)
Jens Axboec40f6372020-06-25 15:39:59 -06001124{
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01001125 req->flags |= REQ_F_FAIL;
Jens Axboec40f6372020-06-25 15:39:59 -06001126}
Jens Axboe4a38aed22020-05-14 17:21:15 -06001127
Jens Axboe2b188cc2019-01-07 10:46:33 -07001128static void io_ring_ctx_ref_free(struct percpu_ref *ref)
1129{
1130 struct io_ring_ctx *ctx = container_of(ref, struct io_ring_ctx, refs);
1131
Jens Axboe0f158b42020-05-14 17:18:39 -06001132 complete(&ctx->ref_comp);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001133}
1134
Pavel Begunkov8eb7e2d2020-06-29 13:13:02 +03001135static inline bool io_is_timeout_noseq(struct io_kiocb *req)
1136{
1137 return !req->timeout.off;
1138}
1139
Jens Axboe2b188cc2019-01-07 10:46:33 -07001140static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
1141{
1142 struct io_ring_ctx *ctx;
Jens Axboe78076bb2019-12-04 19:56:40 -07001143 int hash_bits;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001144
1145 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
1146 if (!ctx)
1147 return NULL;
1148
Jens Axboe78076bb2019-12-04 19:56:40 -07001149 /*
1150 * Use 5 bits less than the max cq entries, that should give us around
1151 * 32 entries per hash list if totally full and uniformly spread.
1152 */
1153 hash_bits = ilog2(p->cq_entries);
1154 hash_bits -= 5;
1155 if (hash_bits <= 0)
1156 hash_bits = 1;
1157 ctx->cancel_hash_bits = hash_bits;
1158 ctx->cancel_hash = kmalloc((1U << hash_bits) * sizeof(struct hlist_head),
1159 GFP_KERNEL);
1160 if (!ctx->cancel_hash)
1161 goto err;
1162 __hash_init(ctx->cancel_hash, 1U << hash_bits);
1163
Pavel Begunkov62248432021-04-28 13:11:29 +01001164 ctx->dummy_ubuf = kzalloc(sizeof(*ctx->dummy_ubuf), GFP_KERNEL);
1165 if (!ctx->dummy_ubuf)
1166 goto err;
1167 /* set invalid range, so io_import_fixed() fails meeting it */
1168 ctx->dummy_ubuf->ubuf = -1UL;
1169
Roman Gushchin21482892019-05-07 10:01:48 -07001170 if (percpu_ref_init(&ctx->refs, io_ring_ctx_ref_free,
Jens Axboe206aefd2019-11-07 18:27:42 -07001171 PERCPU_REF_ALLOW_REINIT, GFP_KERNEL))
1172 goto err;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001173
1174 ctx->flags = p->flags;
Jens Axboe90554202020-09-03 12:12:41 -06001175 init_waitqueue_head(&ctx->sqo_sq_wait);
Jens Axboe69fb2132020-09-14 11:16:23 -06001176 INIT_LIST_HEAD(&ctx->sqd_list);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001177 init_waitqueue_head(&ctx->cq_wait);
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001178 INIT_LIST_HEAD(&ctx->cq_overflow_list);
Jens Axboe0f158b42020-05-14 17:18:39 -06001179 init_completion(&ctx->ref_comp);
Jens Axboe9e15c3a2021-03-13 12:29:43 -07001180 xa_init_flags(&ctx->io_buffers, XA_FLAGS_ALLOC1);
Matthew Wilcox (Oracle)61cf9372021-03-08 14:16:16 +00001181 xa_init_flags(&ctx->personalities, XA_FLAGS_ALLOC1);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001182 mutex_init(&ctx->uring_lock);
1183 init_waitqueue_head(&ctx->wait);
1184 spin_lock_init(&ctx->completion_lock);
Pavel Begunkov540e32a2020-07-13 23:37:09 +03001185 INIT_LIST_HEAD(&ctx->iopoll_list);
Jens Axboede0617e2019-04-06 21:51:27 -06001186 INIT_LIST_HEAD(&ctx->defer_list);
Jens Axboe5262f562019-09-17 12:26:57 -06001187 INIT_LIST_HEAD(&ctx->timeout_list);
Bijan Mottahedehd67d2262021-01-15 17:37:46 +00001188 spin_lock_init(&ctx->rsrc_ref_lock);
1189 INIT_LIST_HEAD(&ctx->rsrc_ref_list);
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00001190 INIT_DELAYED_WORK(&ctx->rsrc_put_work, io_rsrc_put_work);
1191 init_llist_head(&ctx->rsrc_put_llist);
Pavel Begunkov13bf43f2021-03-06 11:02:12 +00001192 INIT_LIST_HEAD(&ctx->tctx_list);
Jens Axboe1b4c3512021-02-10 00:03:19 +00001193 INIT_LIST_HEAD(&ctx->submit_state.comp.free_list);
Jens Axboec7dae4b2021-02-09 19:53:37 -07001194 INIT_LIST_HEAD(&ctx->submit_state.comp.locked_free_list);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001195 return ctx;
Jens Axboe206aefd2019-11-07 18:27:42 -07001196err:
Pavel Begunkov62248432021-04-28 13:11:29 +01001197 kfree(ctx->dummy_ubuf);
Jens Axboe78076bb2019-12-04 19:56:40 -07001198 kfree(ctx->cancel_hash);
Jens Axboe206aefd2019-11-07 18:27:42 -07001199 kfree(ctx);
1200 return NULL;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001201}
1202
Pavel Begunkov9cf7c102020-07-13 23:37:15 +03001203static bool req_need_defer(struct io_kiocb *req, u32 seq)
Jens Axboede0617e2019-04-06 21:51:27 -06001204{
Jens Axboe2bc99302020-07-09 09:43:27 -06001205 if (unlikely(req->flags & REQ_F_IO_DRAIN)) {
1206 struct io_ring_ctx *ctx = req->ctx;
Jackie Liua197f662019-11-08 08:09:12 -07001207
Hao Xu7b289c32021-04-13 15:20:39 +08001208 return seq + ctx->cq_extra != ctx->cached_cq_tail
Pavel Begunkov2c3bac6d2020-10-18 10:17:40 +01001209 + READ_ONCE(ctx->cached_cq_overflow);
Jens Axboe2bc99302020-07-09 09:43:27 -06001210 }
Jens Axboe7adf4ea2019-10-10 21:42:58 -06001211
Bob Liu9d858b22019-11-13 18:06:25 +08001212 return false;
Jens Axboe7adf4ea2019-10-10 21:42:58 -06001213}
1214
Pavel Begunkovce3d5aa2021-02-01 18:59:55 +00001215static void io_req_track_inflight(struct io_kiocb *req)
1216{
Pavel Begunkovce3d5aa2021-02-01 18:59:55 +00001217 if (!(req->flags & REQ_F_INFLIGHT)) {
Pavel Begunkovce3d5aa2021-02-01 18:59:55 +00001218 req->flags |= REQ_F_INFLIGHT;
Pavel Begunkovb303fe22021-04-11 01:46:26 +01001219 atomic_inc(&current->io_uring->inflight_tracked);
Pavel Begunkovce3d5aa2021-02-01 18:59:55 +00001220 }
1221}
1222
Pavel Begunkovcbdcb432020-06-29 19:18:43 +03001223static void io_prep_async_work(struct io_kiocb *req)
Jens Axboe561fb042019-10-24 07:25:42 -06001224{
Jens Axboed3656342019-12-18 09:50:26 -07001225 const struct io_op_def *def = &io_op_defs[req->opcode];
Pavel Begunkov23329512020-10-10 18:34:06 +01001226 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe54a91f32019-09-10 09:15:04 -06001227
Jens Axboe003e8dc2021-03-06 09:22:27 -07001228 if (!req->work.creds)
1229 req->work.creds = get_current_cred();
1230
Pavel Begunkove1d675d2021-03-22 01:58:29 +00001231 req->work.list.next = NULL;
1232 req->work.flags = 0;
Pavel Begunkovfeaadc42020-10-22 16:47:16 +01001233 if (req->flags & REQ_F_FORCE_ASYNC)
1234 req->work.flags |= IO_WQ_WORK_CONCURRENT;
1235
Jens Axboed3656342019-12-18 09:50:26 -07001236 if (req->flags & REQ_F_ISREG) {
Pavel Begunkov23329512020-10-10 18:34:06 +01001237 if (def->hash_reg_file || (ctx->flags & IORING_SETUP_IOPOLL))
Pavel Begunkov8766dd52020-03-14 00:31:04 +03001238 io_wq_hash_work(&req->work, file_inode(req->file));
Jens Axboe4b982bd2021-04-01 08:38:34 -06001239 } else if (!req->file || !S_ISBLK(file_inode(req->file)->i_mode)) {
Jens Axboed3656342019-12-18 09:50:26 -07001240 if (def->unbound_nonreg_file)
Jens Axboe3529d8c2019-12-19 18:24:38 -07001241 req->work.flags |= IO_WQ_WORK_UNBOUND;
Jens Axboe54a91f32019-09-10 09:15:04 -06001242 }
Pavel Begunkove1d675d2021-03-22 01:58:29 +00001243
1244 switch (req->opcode) {
1245 case IORING_OP_SPLICE:
1246 case IORING_OP_TEE:
Pavel Begunkove1d675d2021-03-22 01:58:29 +00001247 if (!S_ISREG(file_inode(req->splice.file_in)->i_mode))
1248 req->work.flags |= IO_WQ_WORK_UNBOUND;
1249 break;
1250 }
Jens Axboe561fb042019-10-24 07:25:42 -06001251}
1252
Pavel Begunkovcbdcb432020-06-29 19:18:43 +03001253static void io_prep_async_link(struct io_kiocb *req)
1254{
1255 struct io_kiocb *cur;
1256
Pavel Begunkovf2f87372020-10-27 23:25:37 +00001257 io_for_each_link(cur, req)
1258 io_prep_async_work(cur);
Pavel Begunkovcbdcb432020-06-29 19:18:43 +03001259}
1260
Pavel Begunkovebf93662021-03-01 18:20:47 +00001261static void io_queue_async_work(struct io_kiocb *req)
Jens Axboe561fb042019-10-24 07:25:42 -06001262{
Jackie Liua197f662019-11-08 08:09:12 -07001263 struct io_ring_ctx *ctx = req->ctx;
Pavel Begunkovcbdcb432020-06-29 19:18:43 +03001264 struct io_kiocb *link = io_prep_linked_timeout(req);
Jens Axboe5aa75ed2021-02-16 12:56:50 -07001265 struct io_uring_task *tctx = req->task->io_uring;
Jens Axboe561fb042019-10-24 07:25:42 -06001266
Jens Axboe3bfe6102021-02-16 14:15:30 -07001267 BUG_ON(!tctx);
1268 BUG_ON(!tctx->io_wq);
Jens Axboe561fb042019-10-24 07:25:42 -06001269
Pavel Begunkovcbdcb432020-06-29 19:18:43 +03001270 /* init ->work of the whole link before punting */
1271 io_prep_async_link(req);
Pavel Begunkovd07f1e8a2021-03-22 01:45:58 +00001272 trace_io_uring_queue_async_work(ctx, io_wq_is_hashed(&req->work), req,
1273 &req->work, req->flags);
Pavel Begunkovebf93662021-03-01 18:20:47 +00001274 io_wq_enqueue(tctx->io_wq, &req->work);
Jens Axboe7271ef32020-08-10 09:55:22 -06001275 if (link)
1276 io_queue_linked_timeout(link);
Pavel Begunkovcbdcb432020-06-29 19:18:43 +03001277}
1278
Pavel Begunkov1ee41602021-03-25 18:32:42 +00001279static void io_kill_timeout(struct io_kiocb *req, int status)
Pavel Begunkov8c855882021-04-13 02:58:41 +01001280 __must_hold(&req->ctx->completion_lock)
Jens Axboe5262f562019-09-17 12:26:57 -06001281{
Jens Axboee8c2bc12020-08-15 18:44:09 -07001282 struct io_timeout_data *io = req->async_data;
Jens Axboe5262f562019-09-17 12:26:57 -06001283
Pavel Begunkovfd9c7bc2021-04-13 02:58:42 +01001284 if (hrtimer_try_to_cancel(&io->timer) != -1) {
Pavel Begunkov01cec8c2020-07-30 18:43:50 +03001285 atomic_set(&req->ctx->cq_timeouts,
1286 atomic_read(&req->ctx->cq_timeouts) + 1);
Pavel Begunkov135fcde2020-07-13 23:37:12 +03001287 list_del_init(&req->timeout.list);
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01001288 io_cqring_fill_event(req->ctx, req->user_data, status, 0);
Pavel Begunkov216578e2020-10-13 09:44:00 +01001289 io_put_req_deferred(req, 1);
Jens Axboe5262f562019-09-17 12:26:57 -06001290 }
1291}
1292
Pavel Begunkov04518942020-05-26 20:34:05 +03001293static void __io_queue_deferred(struct io_ring_ctx *ctx)
1294{
1295 do {
Pavel Begunkov27dc8332020-07-13 23:37:14 +03001296 struct io_defer_entry *de = list_first_entry(&ctx->defer_list,
1297 struct io_defer_entry, list);
Pavel Begunkov04518942020-05-26 20:34:05 +03001298
Pavel Begunkov9cf7c102020-07-13 23:37:15 +03001299 if (req_need_defer(de->req, de->seq))
Pavel Begunkov04518942020-05-26 20:34:05 +03001300 break;
Pavel Begunkov27dc8332020-07-13 23:37:14 +03001301 list_del_init(&de->list);
Pavel Begunkov907d1df2021-01-26 23:35:10 +00001302 io_req_task_queue(de->req);
Pavel Begunkov27dc8332020-07-13 23:37:14 +03001303 kfree(de);
Pavel Begunkov04518942020-05-26 20:34:05 +03001304 } while (!list_empty(&ctx->defer_list));
1305}
1306
Pavel Begunkov360428f2020-05-30 14:54:17 +03001307static void io_flush_timeouts(struct io_ring_ctx *ctx)
1308{
Marcelo Diop-Gonzalezf0105052021-01-15 11:54:40 -05001309 u32 seq;
1310
1311 if (list_empty(&ctx->timeout_list))
1312 return;
1313
1314 seq = ctx->cached_cq_tail - atomic_read(&ctx->cq_timeouts);
1315
1316 do {
1317 u32 events_needed, events_got;
Pavel Begunkov360428f2020-05-30 14:54:17 +03001318 struct io_kiocb *req = list_first_entry(&ctx->timeout_list,
Pavel Begunkov135fcde2020-07-13 23:37:12 +03001319 struct io_kiocb, timeout.list);
Pavel Begunkov360428f2020-05-30 14:54:17 +03001320
Pavel Begunkov8eb7e2d2020-06-29 13:13:02 +03001321 if (io_is_timeout_noseq(req))
Pavel Begunkov360428f2020-05-30 14:54:17 +03001322 break;
Marcelo Diop-Gonzalezf0105052021-01-15 11:54:40 -05001323
1324 /*
1325 * Since seq can easily wrap around over time, subtract
1326 * the last seq at which timeouts were flushed before comparing.
1327 * Assuming not more than 2^31-1 events have happened since,
1328 * these subtractions won't have wrapped, so we can check if
1329 * target is in [last_seq, current_seq] by comparing the two.
1330 */
1331 events_needed = req->timeout.target_seq - ctx->cq_last_tm_flush;
1332 events_got = seq - ctx->cq_last_tm_flush;
1333 if (events_got < events_needed)
Pavel Begunkov360428f2020-05-30 14:54:17 +03001334 break;
Pavel Begunkovbfe68a22020-05-30 14:54:18 +03001335
Pavel Begunkov135fcde2020-07-13 23:37:12 +03001336 list_del_init(&req->timeout.list);
Pavel Begunkov1ee41602021-03-25 18:32:42 +00001337 io_kill_timeout(req, 0);
Marcelo Diop-Gonzalezf0105052021-01-15 11:54:40 -05001338 } while (!list_empty(&ctx->timeout_list));
1339
1340 ctx->cq_last_tm_flush = seq;
Pavel Begunkov360428f2020-05-30 14:54:17 +03001341}
1342
Jens Axboede0617e2019-04-06 21:51:27 -06001343static void io_commit_cqring(struct io_ring_ctx *ctx)
1344{
Pavel Begunkov360428f2020-05-30 14:54:17 +03001345 io_flush_timeouts(ctx);
Pavel Begunkovec30e042021-01-19 13:32:38 +00001346
1347 /* order cqe stores with ring update */
1348 smp_store_release(&ctx->rings->cq.tail, ctx->cached_cq_tail);
Jens Axboede0617e2019-04-06 21:51:27 -06001349
Pavel Begunkov04518942020-05-26 20:34:05 +03001350 if (unlikely(!list_empty(&ctx->defer_list)))
1351 __io_queue_deferred(ctx);
Jens Axboede0617e2019-04-06 21:51:27 -06001352}
1353
Jens Axboe90554202020-09-03 12:12:41 -06001354static inline bool io_sqring_full(struct io_ring_ctx *ctx)
1355{
1356 struct io_rings *r = ctx->rings;
1357
1358 return READ_ONCE(r->sq.tail) - ctx->cached_sq_head == r->sq_ring_entries;
1359}
1360
Pavel Begunkov888aae22021-01-19 13:32:39 +00001361static inline unsigned int __io_cqring_events(struct io_ring_ctx *ctx)
1362{
1363 return ctx->cached_cq_tail - READ_ONCE(ctx->rings->cq.head);
1364}
1365
Pavel Begunkov8d133262021-04-11 01:46:33 +01001366static inline struct io_uring_cqe *io_get_cqring(struct io_ring_ctx *ctx)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001367{
Hristo Venev75b28af2019-08-26 17:23:46 +00001368 struct io_rings *rings = ctx->rings;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001369 unsigned tail;
1370
Stefan Bühler115e12e2019-04-24 23:54:18 +02001371 /*
1372 * writes to the cq entry need to come after reading head; the
1373 * control dependency is enough as we're using WRITE_ONCE to
1374 * fill the cq entry
1375 */
Pavel Begunkov888aae22021-01-19 13:32:39 +00001376 if (__io_cqring_events(ctx) == rings->cq_ring_entries)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001377 return NULL;
1378
Pavel Begunkov888aae22021-01-19 13:32:39 +00001379 tail = ctx->cached_cq_tail++;
Hristo Venev75b28af2019-08-26 17:23:46 +00001380 return &rings->cqes[tail & ctx->cq_mask];
Jens Axboe2b188cc2019-01-07 10:46:33 -07001381}
1382
Jens Axboef2842ab2020-01-08 11:04:00 -07001383static inline bool io_should_trigger_evfd(struct io_ring_ctx *ctx)
1384{
Pavel Begunkov44c769d2021-04-11 01:46:31 +01001385 if (likely(!ctx->cq_ev_fd))
Jens Axboef0b493e2020-02-01 21:30:11 -07001386 return false;
Stefano Garzarella7e55a192020-05-15 18:38:05 +02001387 if (READ_ONCE(ctx->rings->cq_flags) & IORING_CQ_EVENTFD_DISABLED)
1388 return false;
Pavel Begunkov44c769d2021-04-11 01:46:31 +01001389 return !ctx->eventfd_async || io_wq_current_is_worker();
Jens Axboef2842ab2020-01-08 11:04:00 -07001390}
1391
Jens Axboeb41e9852020-02-17 09:52:41 -07001392static void io_cqring_ev_posted(struct io_ring_ctx *ctx)
Jens Axboe8c838782019-03-12 15:48:16 -06001393{
Pavel Begunkovb1445e52021-01-07 03:15:43 +00001394 /* see waitqueue_active() comment */
1395 smp_mb();
1396
Jens Axboe8c838782019-03-12 15:48:16 -06001397 if (waitqueue_active(&ctx->wait))
1398 wake_up(&ctx->wait);
Jens Axboe534ca6d2020-09-02 13:52:19 -06001399 if (ctx->sq_data && waitqueue_active(&ctx->sq_data->wait))
1400 wake_up(&ctx->sq_data->wait);
Jens Axboeb41e9852020-02-17 09:52:41 -07001401 if (io_should_trigger_evfd(ctx))
Jens Axboe9b402842019-04-11 11:45:41 -06001402 eventfd_signal(ctx->cq_ev_fd, 1);
Pavel Begunkovb1445e52021-01-07 03:15:43 +00001403 if (waitqueue_active(&ctx->cq_wait)) {
Pavel Begunkov4aa84f22021-01-07 03:15:42 +00001404 wake_up_interruptible(&ctx->cq_wait);
1405 kill_fasync(&ctx->cq_fasync, SIGIO, POLL_IN);
1406 }
Jens Axboe8c838782019-03-12 15:48:16 -06001407}
1408
Pavel Begunkov80c18e42021-01-07 03:15:41 +00001409static void io_cqring_ev_posted_iopoll(struct io_ring_ctx *ctx)
1410{
Pavel Begunkovb1445e52021-01-07 03:15:43 +00001411 /* see waitqueue_active() comment */
1412 smp_mb();
1413
Pavel Begunkov80c18e42021-01-07 03:15:41 +00001414 if (ctx->flags & IORING_SETUP_SQPOLL) {
1415 if (waitqueue_active(&ctx->wait))
1416 wake_up(&ctx->wait);
1417 }
1418 if (io_should_trigger_evfd(ctx))
1419 eventfd_signal(ctx->cq_ev_fd, 1);
Pavel Begunkovb1445e52021-01-07 03:15:43 +00001420 if (waitqueue_active(&ctx->cq_wait)) {
Pavel Begunkov4aa84f22021-01-07 03:15:42 +00001421 wake_up_interruptible(&ctx->cq_wait);
1422 kill_fasync(&ctx->cq_fasync, SIGIO, POLL_IN);
1423 }
Pavel Begunkov80c18e42021-01-07 03:15:41 +00001424}
1425
Jens Axboec4a2ed72019-11-21 21:01:26 -07001426/* Returns true if there are no backlogged entries after the flush */
Pavel Begunkov6c2450a2021-02-23 12:40:22 +00001427static bool __io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001428{
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001429 struct io_rings *rings = ctx->rings;
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001430 unsigned long flags;
Jens Axboeb18032b2021-01-24 16:58:56 -07001431 bool all_flushed, posted;
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001432
Pavel Begunkove23de152020-12-17 00:24:37 +00001433 if (!force && __io_cqring_events(ctx) == rings->cq_ring_entries)
1434 return false;
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001435
Jens Axboeb18032b2021-01-24 16:58:56 -07001436 posted = false;
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001437 spin_lock_irqsave(&ctx->completion_lock, flags);
Pavel Begunkov6c2450a2021-02-23 12:40:22 +00001438 while (!list_empty(&ctx->cq_overflow_list)) {
1439 struct io_uring_cqe *cqe = io_get_cqring(ctx);
1440 struct io_overflow_cqe *ocqe;
Jens Axboee6c8aa92020-09-28 13:10:13 -06001441
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001442 if (!cqe && !force)
1443 break;
Pavel Begunkov6c2450a2021-02-23 12:40:22 +00001444 ocqe = list_first_entry(&ctx->cq_overflow_list,
1445 struct io_overflow_cqe, list);
1446 if (cqe)
1447 memcpy(cqe, &ocqe->cqe, sizeof(*cqe));
1448 else
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001449 WRITE_ONCE(ctx->rings->cq_overflow,
Pavel Begunkov6c2450a2021-02-23 12:40:22 +00001450 ++ctx->cached_cq_overflow);
Jens Axboeb18032b2021-01-24 16:58:56 -07001451 posted = true;
Pavel Begunkov6c2450a2021-02-23 12:40:22 +00001452 list_del(&ocqe->list);
1453 kfree(ocqe);
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001454 }
1455
Pavel Begunkov09e88402020-12-17 00:24:38 +00001456 all_flushed = list_empty(&ctx->cq_overflow_list);
1457 if (all_flushed) {
1458 clear_bit(0, &ctx->sq_check_overflow);
1459 clear_bit(0, &ctx->cq_check_overflow);
1460 ctx->rings->sq_flags &= ~IORING_SQ_CQ_OVERFLOW;
1461 }
Pavel Begunkov46930142020-07-30 18:43:49 +03001462
Jens Axboeb18032b2021-01-24 16:58:56 -07001463 if (posted)
1464 io_commit_cqring(ctx);
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001465 spin_unlock_irqrestore(&ctx->completion_lock, flags);
Jens Axboeb18032b2021-01-24 16:58:56 -07001466 if (posted)
1467 io_cqring_ev_posted(ctx);
Pavel Begunkov09e88402020-12-17 00:24:38 +00001468 return all_flushed;
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001469}
1470
Pavel Begunkov6c2450a2021-02-23 12:40:22 +00001471static bool io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force)
Pavel Begunkov6c503152021-01-04 20:36:36 +00001472{
Jens Axboeca0a2652021-03-04 17:15:48 -07001473 bool ret = true;
1474
Pavel Begunkov6c503152021-01-04 20:36:36 +00001475 if (test_bit(0, &ctx->cq_check_overflow)) {
1476 /* iopoll syncs against uring_lock, not completion_lock */
1477 if (ctx->flags & IORING_SETUP_IOPOLL)
1478 mutex_lock(&ctx->uring_lock);
Pavel Begunkov6c2450a2021-02-23 12:40:22 +00001479 ret = __io_cqring_overflow_flush(ctx, force);
Pavel Begunkov6c503152021-01-04 20:36:36 +00001480 if (ctx->flags & IORING_SETUP_IOPOLL)
1481 mutex_unlock(&ctx->uring_lock);
1482 }
Jens Axboeca0a2652021-03-04 17:15:48 -07001483
1484 return ret;
Pavel Begunkov6c503152021-01-04 20:36:36 +00001485}
1486
Jens Axboeabc54d62021-02-24 13:32:30 -07001487/*
1488 * Shamelessly stolen from the mm implementation of page reference checking,
1489 * see commit f958d7b528b1 for details.
1490 */
1491#define req_ref_zero_or_close_to_overflow(req) \
1492 ((unsigned int) atomic_read(&(req->refs)) + 127u <= 127u)
1493
Jens Axboede9b4cc2021-02-24 13:28:27 -07001494static inline bool req_ref_inc_not_zero(struct io_kiocb *req)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001495{
Jens Axboeabc54d62021-02-24 13:32:30 -07001496 return atomic_inc_not_zero(&req->refs);
Jens Axboede9b4cc2021-02-24 13:28:27 -07001497}
1498
1499static inline bool req_ref_sub_and_test(struct io_kiocb *req, int refs)
1500{
Jens Axboeabc54d62021-02-24 13:32:30 -07001501 WARN_ON_ONCE(req_ref_zero_or_close_to_overflow(req));
1502 return atomic_sub_and_test(refs, &req->refs);
Jens Axboede9b4cc2021-02-24 13:28:27 -07001503}
1504
1505static inline bool req_ref_put_and_test(struct io_kiocb *req)
1506{
Jens Axboeabc54d62021-02-24 13:32:30 -07001507 WARN_ON_ONCE(req_ref_zero_or_close_to_overflow(req));
1508 return atomic_dec_and_test(&req->refs);
Jens Axboede9b4cc2021-02-24 13:28:27 -07001509}
1510
1511static inline void req_ref_put(struct io_kiocb *req)
1512{
Jens Axboeabc54d62021-02-24 13:32:30 -07001513 WARN_ON_ONCE(req_ref_put_and_test(req));
Jens Axboede9b4cc2021-02-24 13:28:27 -07001514}
1515
1516static inline void req_ref_get(struct io_kiocb *req)
1517{
Jens Axboeabc54d62021-02-24 13:32:30 -07001518 WARN_ON_ONCE(req_ref_zero_or_close_to_overflow(req));
1519 atomic_inc(&req->refs);
Jens Axboede9b4cc2021-02-24 13:28:27 -07001520}
1521
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01001522static bool io_cqring_event_overflow(struct io_ring_ctx *ctx, u64 user_data,
1523 long res, unsigned int cflags)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001524{
Pavel Begunkovcce4b8b2021-04-13 02:58:44 +01001525 struct io_overflow_cqe *ocqe;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001526
Pavel Begunkovcce4b8b2021-04-13 02:58:44 +01001527 ocqe = kmalloc(sizeof(*ocqe), GFP_ATOMIC | __GFP_ACCOUNT);
1528 if (!ocqe) {
1529 /*
1530 * If we're in ring overflow flush mode, or in task cancel mode,
1531 * or cannot allocate an overflow entry, then we need to drop it
1532 * on the floor.
1533 */
1534 WRITE_ONCE(ctx->rings->cq_overflow, ++ctx->cached_cq_overflow);
1535 return false;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001536 }
Pavel Begunkovcce4b8b2021-04-13 02:58:44 +01001537 if (list_empty(&ctx->cq_overflow_list)) {
1538 set_bit(0, &ctx->sq_check_overflow);
1539 set_bit(0, &ctx->cq_check_overflow);
1540 ctx->rings->sq_flags |= IORING_SQ_CQ_OVERFLOW;
1541 }
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01001542 ocqe->cqe.user_data = user_data;
Pavel Begunkovcce4b8b2021-04-13 02:58:44 +01001543 ocqe->cqe.res = res;
1544 ocqe->cqe.flags = cflags;
1545 list_add_tail(&ocqe->list, &ctx->cq_overflow_list);
1546 return true;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001547}
1548
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01001549static inline bool __io_cqring_fill_event(struct io_ring_ctx *ctx, u64 user_data,
1550 long res, unsigned int cflags)
Pavel Begunkov8d133262021-04-11 01:46:33 +01001551{
Jens Axboe2b188cc2019-01-07 10:46:33 -07001552 struct io_uring_cqe *cqe;
1553
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01001554 trace_io_uring_complete(ctx, user_data, res, cflags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001555
1556 /*
1557 * If we can't get a cq entry, userspace overflowed the
1558 * submission (by quite a lot). Increment the overflow count in
1559 * the ring.
1560 */
1561 cqe = io_get_cqring(ctx);
1562 if (likely(cqe)) {
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01001563 WRITE_ONCE(cqe->user_data, user_data);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001564 WRITE_ONCE(cqe->res, res);
1565 WRITE_ONCE(cqe->flags, cflags);
Pavel Begunkov8d133262021-04-11 01:46:33 +01001566 return true;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001567 }
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01001568 return io_cqring_event_overflow(ctx, user_data, res, cflags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001569}
1570
Pavel Begunkov8d133262021-04-11 01:46:33 +01001571/* not as hot to bloat with inlining */
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01001572static noinline bool io_cqring_fill_event(struct io_ring_ctx *ctx, u64 user_data,
1573 long res, unsigned int cflags)
Jens Axboebcda7ba2020-02-23 16:42:51 -07001574{
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01001575 return __io_cqring_fill_event(ctx, user_data, res, cflags);
Jens Axboebcda7ba2020-02-23 16:42:51 -07001576}
1577
Pavel Begunkov7a612352021-03-09 00:37:59 +00001578static void io_req_complete_post(struct io_kiocb *req, long res,
1579 unsigned int cflags)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001580{
Jens Axboe78e19bb2019-11-06 15:21:34 -07001581 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001582 unsigned long flags;
1583
1584 spin_lock_irqsave(&ctx->completion_lock, flags);
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01001585 __io_cqring_fill_event(ctx, req->user_data, res, cflags);
Jens Axboec7dae4b2021-02-09 19:53:37 -07001586 /*
1587 * If we're the last reference to this request, add to our locked
1588 * free_list cache.
1589 */
Jens Axboede9b4cc2021-02-24 13:28:27 -07001590 if (req_ref_put_and_test(req)) {
Jens Axboec7dae4b2021-02-09 19:53:37 -07001591 struct io_comp_state *cs = &ctx->submit_state.comp;
1592
Pavel Begunkov7a612352021-03-09 00:37:59 +00001593 if (req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) {
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01001594 if (req->flags & (REQ_F_LINK_TIMEOUT | REQ_F_FAIL))
Pavel Begunkov7a612352021-03-09 00:37:59 +00001595 io_disarm_next(req);
1596 if (req->link) {
1597 io_req_task_queue(req->link);
1598 req->link = NULL;
1599 }
1600 }
Jens Axboec7dae4b2021-02-09 19:53:37 -07001601 io_dismantle_req(req);
1602 io_put_task(req->task, 1);
1603 list_add(&req->compl.list, &cs->locked_free_list);
1604 cs->locked_free_nr++;
Pavel Begunkov180f8292021-03-14 20:57:09 +00001605 } else {
1606 if (!percpu_ref_tryget(&ctx->refs))
1607 req = NULL;
1608 }
Pavel Begunkov7a612352021-03-09 00:37:59 +00001609 io_commit_cqring(ctx);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001610 spin_unlock_irqrestore(&ctx->completion_lock, flags);
Pavel Begunkov7a612352021-03-09 00:37:59 +00001611
Pavel Begunkov180f8292021-03-14 20:57:09 +00001612 if (req) {
1613 io_cqring_ev_posted(ctx);
Jens Axboec7dae4b2021-02-09 19:53:37 -07001614 percpu_ref_put(&ctx->refs);
Pavel Begunkov180f8292021-03-14 20:57:09 +00001615 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07001616}
1617
Jens Axboe4e3d9ff2021-04-15 17:44:34 -06001618static inline bool io_req_needs_clean(struct io_kiocb *req)
1619{
Jens Axboe75652a302021-04-15 09:52:40 -06001620 return req->flags & (REQ_F_BUFFER_SELECTED | REQ_F_NEED_CLEANUP |
Pavel Begunkov3a0a6902021-04-20 12:03:31 +01001621 REQ_F_POLLED | REQ_F_INFLIGHT);
Jens Axboe4e3d9ff2021-04-15 17:44:34 -06001622}
1623
Pavel Begunkova38d68d2021-01-19 13:32:45 +00001624static void io_req_complete_state(struct io_kiocb *req, long res,
Pavel Begunkov889fca72021-02-10 00:03:09 +00001625 unsigned int cflags)
Jens Axboebcda7ba2020-02-23 16:42:51 -07001626{
Jens Axboe4e3d9ff2021-04-15 17:44:34 -06001627 if (io_req_needs_clean(req))
Pavel Begunkov68fb8972021-03-19 17:22:41 +00001628 io_clean_op(req);
Pavel Begunkova38d68d2021-01-19 13:32:45 +00001629 req->result = res;
1630 req->compl.cflags = cflags;
Pavel Begunkove342c802021-01-19 13:32:47 +00001631 req->flags |= REQ_F_COMPLETE_INLINE;
Jens Axboee1e16092020-06-22 09:17:17 -06001632}
Jens Axboe2b188cc2019-01-07 10:46:33 -07001633
Pavel Begunkov889fca72021-02-10 00:03:09 +00001634static inline void __io_req_complete(struct io_kiocb *req, unsigned issue_flags,
1635 long res, unsigned cflags)
Pavel Begunkova38d68d2021-01-19 13:32:45 +00001636{
Pavel Begunkov889fca72021-02-10 00:03:09 +00001637 if (issue_flags & IO_URING_F_COMPLETE_DEFER)
1638 io_req_complete_state(req, res, cflags);
Pavel Begunkova38d68d2021-01-19 13:32:45 +00001639 else
Jens Axboec7dae4b2021-02-09 19:53:37 -07001640 io_req_complete_post(req, res, cflags);
Pavel Begunkova38d68d2021-01-19 13:32:45 +00001641}
Jens Axboebcda7ba2020-02-23 16:42:51 -07001642
Pavel Begunkova38d68d2021-01-19 13:32:45 +00001643static inline void io_req_complete(struct io_kiocb *req, long res)
Jens Axboee1e16092020-06-22 09:17:17 -06001644{
Pavel Begunkov889fca72021-02-10 00:03:09 +00001645 __io_req_complete(req, 0, res, 0);
Jens Axboebcda7ba2020-02-23 16:42:51 -07001646}
1647
Pavel Begunkovf41db2732021-02-28 22:35:12 +00001648static void io_req_complete_failed(struct io_kiocb *req, long res)
1649{
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01001650 req_set_fail(req);
Pavel Begunkovf41db2732021-02-28 22:35:12 +00001651 io_put_req(req);
1652 io_req_complete_post(req, res, 0);
1653}
1654
Pavel Begunkovdac7a092021-03-19 17:22:39 +00001655static void io_flush_cached_locked_reqs(struct io_ring_ctx *ctx,
1656 struct io_comp_state *cs)
1657{
1658 spin_lock_irq(&ctx->completion_lock);
1659 list_splice_init(&cs->locked_free_list, &cs->free_list);
1660 cs->locked_free_nr = 0;
1661 spin_unlock_irq(&ctx->completion_lock);
1662}
1663
Pavel Begunkovdd78f492021-03-19 17:22:35 +00001664/* Returns true IFF there are requests in the cache */
Jens Axboec7dae4b2021-02-09 19:53:37 -07001665static bool io_flush_cached_reqs(struct io_ring_ctx *ctx)
Pavel Begunkove5d1bc02021-02-10 00:03:23 +00001666{
Jens Axboec7dae4b2021-02-09 19:53:37 -07001667 struct io_submit_state *state = &ctx->submit_state;
1668 struct io_comp_state *cs = &state->comp;
Pavel Begunkovdd78f492021-03-19 17:22:35 +00001669 int nr;
Pavel Begunkove5d1bc02021-02-10 00:03:23 +00001670
Jens Axboec7dae4b2021-02-09 19:53:37 -07001671 /*
1672 * If we have more than a batch's worth of requests in our IRQ side
1673 * locked cache, grab the lock and move them over to our submission
1674 * side cache.
1675 */
Pavel Begunkovdac7a092021-03-19 17:22:39 +00001676 if (READ_ONCE(cs->locked_free_nr) > IO_COMPL_BATCH)
1677 io_flush_cached_locked_reqs(ctx, cs);
Jens Axboec7dae4b2021-02-09 19:53:37 -07001678
Pavel Begunkovdd78f492021-03-19 17:22:35 +00001679 nr = state->free_reqs;
Jens Axboec7dae4b2021-02-09 19:53:37 -07001680 while (!list_empty(&cs->free_list)) {
Pavel Begunkovdd78f492021-03-19 17:22:35 +00001681 struct io_kiocb *req = list_first_entry(&cs->free_list,
1682 struct io_kiocb, compl.list);
1683
Jens Axboe2b188cc2019-01-07 10:46:33 -07001684 list_del(&req->compl.list);
Pavel Begunkovdd78f492021-03-19 17:22:35 +00001685 state->reqs[nr++] = req;
1686 if (nr == ARRAY_SIZE(state->reqs))
Pavel Begunkove5d1bc02021-02-10 00:03:23 +00001687 break;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001688 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07001689
Pavel Begunkovdd78f492021-03-19 17:22:35 +00001690 state->free_reqs = nr;
1691 return nr != 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001692}
1693
Pavel Begunkov258b29a2021-02-10 00:03:10 +00001694static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001695{
Pavel Begunkov258b29a2021-02-10 00:03:10 +00001696 struct io_submit_state *state = &ctx->submit_state;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001697
Pavel Begunkovbf019da2021-02-10 00:03:17 +00001698 BUILD_BUG_ON(IO_REQ_ALLOC_BATCH > ARRAY_SIZE(state->reqs));
Jens Axboe2b188cc2019-01-07 10:46:33 -07001699
Pavel Begunkovf6b6c7d2020-06-21 13:09:53 +03001700 if (!state->free_reqs) {
Pavel Begunkov291b2822020-09-30 22:57:01 +03001701 gfp_t gfp = GFP_KERNEL | __GFP_NOWARN;
Jens Axboe2579f912019-01-09 09:10:43 -07001702 int ret;
1703
Jens Axboec7dae4b2021-02-09 19:53:37 -07001704 if (io_flush_cached_reqs(ctx))
Pavel Begunkove5d1bc02021-02-10 00:03:23 +00001705 goto got_req;
1706
Pavel Begunkovbf019da2021-02-10 00:03:17 +00001707 ret = kmem_cache_alloc_bulk(req_cachep, gfp, IO_REQ_ALLOC_BATCH,
1708 state->reqs);
Jens Axboefd6fab22019-03-14 16:30:06 -06001709
1710 /*
1711 * Bulk alloc is all-or-nothing. If we fail to get a batch,
1712 * retry single alloc to be on the safe side.
1713 */
1714 if (unlikely(ret <= 0)) {
1715 state->reqs[0] = kmem_cache_alloc(req_cachep, gfp);
1716 if (!state->reqs[0])
Pavel Begunkov3893f392021-02-10 00:03:15 +00001717 return NULL;
Jens Axboefd6fab22019-03-14 16:30:06 -06001718 ret = 1;
1719 }
Pavel Begunkov291b2822020-09-30 22:57:01 +03001720 state->free_reqs = ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001721 }
Pavel Begunkove5d1bc02021-02-10 00:03:23 +00001722got_req:
Pavel Begunkov291b2822020-09-30 22:57:01 +03001723 state->free_reqs--;
1724 return state->reqs[state->free_reqs];
Jens Axboe2b188cc2019-01-07 10:46:33 -07001725}
1726
Pavel Begunkove1d767f2021-03-19 17:22:43 +00001727static inline void io_put_file(struct file *file)
Pavel Begunkov8da11c12020-02-24 11:32:44 +03001728{
Pavel Begunkove1d767f2021-03-19 17:22:43 +00001729 if (file)
Pavel Begunkov8da11c12020-02-24 11:32:44 +03001730 fput(file);
1731}
1732
Pavel Begunkov4edf20f2020-10-13 09:43:59 +01001733static void io_dismantle_req(struct io_kiocb *req)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001734{
Pavel Begunkov094bae42021-03-19 17:22:42 +00001735 unsigned int flags = req->flags;
Pavel Begunkov929a3af2020-02-19 00:19:09 +03001736
Pavel Begunkov3a0a6902021-04-20 12:03:31 +01001737 if (io_req_needs_clean(req))
1738 io_clean_op(req);
Pavel Begunkove1d767f2021-03-19 17:22:43 +00001739 if (!(flags & REQ_F_FIXED_FILE))
1740 io_put_file(req->file);
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00001741 if (req->fixed_rsrc_refs)
1742 percpu_ref_put(req->fixed_rsrc_refs);
Pavel Begunkov094bae42021-03-19 17:22:42 +00001743 if (req->async_data)
1744 kfree(req->async_data);
Jens Axboe003e8dc2021-03-06 09:22:27 -07001745 if (req->work.creds) {
1746 put_cred(req->work.creds);
1747 req->work.creds = NULL;
1748 }
Pavel Begunkove6543a82020-06-28 12:52:30 +03001749}
Pavel Begunkov2b85edf2019-12-28 14:13:03 +03001750
Pavel Begunkovb23fcf42021-03-01 18:20:48 +00001751/* must to be called somewhat shortly after putting a request */
Pavel Begunkov7c660732021-01-25 11:42:21 +00001752static inline void io_put_task(struct task_struct *task, int nr)
1753{
1754 struct io_uring_task *tctx = task->io_uring;
1755
1756 percpu_counter_sub(&tctx->inflight, nr);
1757 if (unlikely(atomic_read(&tctx->in_idle)))
1758 wake_up(&tctx->wait);
1759 put_task_struct_many(task, nr);
1760}
1761
Pavel Begunkov216578e2020-10-13 09:44:00 +01001762static void __io_free_req(struct io_kiocb *req)
Pavel Begunkove6543a82020-06-28 12:52:30 +03001763{
Jens Axboe51a4cc12020-08-10 10:55:56 -06001764 struct io_ring_ctx *ctx = req->ctx;
Pavel Begunkovecfc5172020-06-29 13:13:03 +03001765
Pavel Begunkov216578e2020-10-13 09:44:00 +01001766 io_dismantle_req(req);
Pavel Begunkov7c660732021-01-25 11:42:21 +00001767 io_put_task(req->task, 1);
Pavel Begunkove6543a82020-06-28 12:52:30 +03001768
Pavel Begunkov3893f392021-02-10 00:03:15 +00001769 kmem_cache_free(req_cachep, req);
Pavel Begunkovecfc5172020-06-29 13:13:03 +03001770 percpu_ref_put(&ctx->refs);
Jens Axboee65ef562019-03-12 10:16:44 -06001771}
1772
Pavel Begunkovf2f87372020-10-27 23:25:37 +00001773static inline void io_remove_next_linked(struct io_kiocb *req)
1774{
1775 struct io_kiocb *nxt = req->link;
1776
1777 req->link = nxt->link;
1778 nxt->link = NULL;
1779}
1780
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00001781static bool io_kill_linked_timeout(struct io_kiocb *req)
1782 __must_hold(&req->ctx->completion_lock)
Jens Axboe9e645e112019-05-10 16:07:28 -06001783{
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00001784 struct io_kiocb *link = req->link;
Pavel Begunkovf2f87372020-10-27 23:25:37 +00001785
Pavel Begunkov900fad42020-10-19 16:39:16 +01001786 /*
1787 * Can happen if a linked timeout fired and link had been like
1788 * req -> link t-out -> link t-out [-> ...]
1789 */
Pavel Begunkovc9abd7a2020-10-22 16:43:11 +01001790 if (link && (link->flags & REQ_F_LTIMEOUT_ACTIVE)) {
1791 struct io_timeout_data *io = link->async_data;
Pavel Begunkov7c86ffe2020-06-29 13:12:59 +03001792
Pavel Begunkovf2f87372020-10-27 23:25:37 +00001793 io_remove_next_linked(req);
Pavel Begunkov90cd7e42020-10-27 23:25:36 +00001794 link->timeout.head = NULL;
Pavel Begunkovfd9c7bc2021-04-13 02:58:42 +01001795 if (hrtimer_try_to_cancel(&io->timer) != -1) {
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01001796 io_cqring_fill_event(link->ctx, link->user_data,
1797 -ECANCELED, 0);
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00001798 io_put_req_deferred(link, 1);
Pavel Begunkovd4729fb2021-03-22 01:58:24 +00001799 return true;
Pavel Begunkovc9abd7a2020-10-22 16:43:11 +01001800 }
1801 }
Pavel Begunkovd4729fb2021-03-22 01:58:24 +00001802 return false;
Pavel Begunkov7c86ffe2020-06-29 13:12:59 +03001803}
1804
Pavel Begunkovd148ca42020-10-18 10:17:39 +01001805static void io_fail_links(struct io_kiocb *req)
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00001806 __must_hold(&req->ctx->completion_lock)
Jens Axboe9e645e112019-05-10 16:07:28 -06001807{
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00001808 struct io_kiocb *nxt, *link = req->link;
Jens Axboe9e645e112019-05-10 16:07:28 -06001809
Pavel Begunkovf2f87372020-10-27 23:25:37 +00001810 req->link = NULL;
Pavel Begunkovf2f87372020-10-27 23:25:37 +00001811 while (link) {
1812 nxt = link->link;
1813 link->link = NULL;
1814
Dmitrii Dolgovc826bd72019-10-15 19:02:01 +02001815 trace_io_uring_fail_link(req, link);
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01001816 io_cqring_fill_event(link->ctx, link->user_data, -ECANCELED, 0);
Jens Axboe1575f212021-02-27 15:20:49 -07001817 io_put_req_deferred(link, 2);
Pavel Begunkovf2f87372020-10-27 23:25:37 +00001818 link = nxt;
Jens Axboe9e645e112019-05-10 16:07:28 -06001819 }
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00001820}
Pavel Begunkov7c86ffe2020-06-29 13:12:59 +03001821
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00001822static bool io_disarm_next(struct io_kiocb *req)
1823 __must_hold(&req->ctx->completion_lock)
1824{
1825 bool posted = false;
1826
1827 if (likely(req->flags & REQ_F_LINK_TIMEOUT))
1828 posted = io_kill_linked_timeout(req);
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01001829 if (unlikely((req->flags & REQ_F_FAIL) &&
Pavel Begunkove4335ed2021-04-11 01:46:39 +01001830 !(req->flags & REQ_F_HARDLINK))) {
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00001831 posted |= (req->link != NULL);
1832 io_fail_links(req);
1833 }
1834 return posted;
Jens Axboe9e645e112019-05-10 16:07:28 -06001835}
1836
Pavel Begunkov3fa5e0f2020-06-30 15:20:43 +03001837static struct io_kiocb *__io_req_find_next(struct io_kiocb *req)
Jens Axboe9e645e112019-05-10 16:07:28 -06001838{
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00001839 struct io_kiocb *nxt;
Jens Axboe2665abf2019-11-05 12:40:47 -07001840
Jens Axboe9e645e112019-05-10 16:07:28 -06001841 /*
1842 * If LINK is set, we have dependent requests in this chain. If we
1843 * didn't fail this request, queue the first one up, moving any other
1844 * dependencies to the next request. In case of failure, fail the rest
1845 * of the chain.
1846 */
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01001847 if (req->flags & (REQ_F_LINK_TIMEOUT | REQ_F_FAIL)) {
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00001848 struct io_ring_ctx *ctx = req->ctx;
1849 unsigned long flags;
1850 bool posted;
Pavel Begunkovf2f87372020-10-27 23:25:37 +00001851
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00001852 spin_lock_irqsave(&ctx->completion_lock, flags);
1853 posted = io_disarm_next(req);
1854 if (posted)
1855 io_commit_cqring(req->ctx);
1856 spin_unlock_irqrestore(&ctx->completion_lock, flags);
1857 if (posted)
1858 io_cqring_ev_posted(ctx);
Pavel Begunkovf2f87372020-10-27 23:25:37 +00001859 }
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00001860 nxt = req->link;
1861 req->link = NULL;
1862 return nxt;
Jens Axboe4d7dd462019-11-20 13:03:52 -07001863}
Jens Axboe2665abf2019-11-05 12:40:47 -07001864
Pavel Begunkovf2f87372020-10-27 23:25:37 +00001865static inline struct io_kiocb *io_req_find_next(struct io_kiocb *req)
Pavel Begunkov3fa5e0f2020-06-30 15:20:43 +03001866{
Pavel Begunkovcdbff982021-02-12 18:41:16 +00001867 if (likely(!(req->flags & (REQ_F_LINK|REQ_F_HARDLINK))))
Pavel Begunkov3fa5e0f2020-06-30 15:20:43 +03001868 return NULL;
1869 return __io_req_find_next(req);
1870}
1871
Pavel Begunkov2c323952021-02-28 22:04:53 +00001872static void ctx_flush_and_put(struct io_ring_ctx *ctx)
1873{
1874 if (!ctx)
1875 return;
1876 if (ctx->submit_state.comp.nr) {
1877 mutex_lock(&ctx->uring_lock);
1878 io_submit_flush_completions(&ctx->submit_state.comp, ctx);
1879 mutex_unlock(&ctx->uring_lock);
1880 }
1881 percpu_ref_put(&ctx->refs);
1882}
1883
Jens Axboe7cbf1722021-02-10 00:03:20 +00001884static bool __tctx_task_work(struct io_uring_task *tctx)
1885{
Jens Axboe65453d12021-02-10 00:03:21 +00001886 struct io_ring_ctx *ctx = NULL;
Jens Axboe7cbf1722021-02-10 00:03:20 +00001887 struct io_wq_work_list list;
1888 struct io_wq_work_node *node;
1889
1890 if (wq_list_empty(&tctx->task_list))
1891 return false;
1892
Jens Axboe0b81e802021-02-16 10:33:53 -07001893 spin_lock_irq(&tctx->task_lock);
Jens Axboe7cbf1722021-02-10 00:03:20 +00001894 list = tctx->task_list;
1895 INIT_WQ_LIST(&tctx->task_list);
Jens Axboe0b81e802021-02-16 10:33:53 -07001896 spin_unlock_irq(&tctx->task_lock);
Jens Axboe7cbf1722021-02-10 00:03:20 +00001897
1898 node = list.first;
1899 while (node) {
1900 struct io_wq_work_node *next = node->next;
1901 struct io_kiocb *req;
1902
1903 req = container_of(node, struct io_kiocb, io_task_work.node);
Pavel Begunkov2c323952021-02-28 22:04:53 +00001904 if (req->ctx != ctx) {
1905 ctx_flush_and_put(ctx);
1906 ctx = req->ctx;
1907 percpu_ref_get(&ctx->refs);
1908 }
1909
Jens Axboe7cbf1722021-02-10 00:03:20 +00001910 req->task_work.func(&req->task_work);
1911 node = next;
Jens Axboe65453d12021-02-10 00:03:21 +00001912 }
1913
Pavel Begunkov2c323952021-02-28 22:04:53 +00001914 ctx_flush_and_put(ctx);
Jens Axboe7cbf1722021-02-10 00:03:20 +00001915 return list.first != NULL;
1916}
1917
1918static void tctx_task_work(struct callback_head *cb)
1919{
1920 struct io_uring_task *tctx = container_of(cb, struct io_uring_task, task_work);
1921
Jens Axboe1d5f3602021-02-26 14:54:16 -07001922 clear_bit(0, &tctx->task_state);
1923
Jens Axboe7cbf1722021-02-10 00:03:20 +00001924 while (__tctx_task_work(tctx))
1925 cond_resched();
Jens Axboe7cbf1722021-02-10 00:03:20 +00001926}
1927
Pavel Begunkovc15b79d2021-03-19 17:22:44 +00001928static int io_req_task_work_add(struct io_kiocb *req)
Jens Axboe7cbf1722021-02-10 00:03:20 +00001929{
Pavel Begunkovc15b79d2021-03-19 17:22:44 +00001930 struct task_struct *tsk = req->task;
Jens Axboe7cbf1722021-02-10 00:03:20 +00001931 struct io_uring_task *tctx = tsk->io_uring;
Pavel Begunkovc15b79d2021-03-19 17:22:44 +00001932 enum task_work_notify_mode notify;
Jens Axboe7cbf1722021-02-10 00:03:20 +00001933 struct io_wq_work_node *node, *prev;
Jens Axboe0b81e802021-02-16 10:33:53 -07001934 unsigned long flags;
Pavel Begunkovc15b79d2021-03-19 17:22:44 +00001935 int ret = 0;
1936
1937 if (unlikely(tsk->flags & PF_EXITING))
1938 return -ESRCH;
Jens Axboe7cbf1722021-02-10 00:03:20 +00001939
1940 WARN_ON_ONCE(!tctx);
1941
Jens Axboe0b81e802021-02-16 10:33:53 -07001942 spin_lock_irqsave(&tctx->task_lock, flags);
Jens Axboe7cbf1722021-02-10 00:03:20 +00001943 wq_list_add_tail(&req->io_task_work.node, &tctx->task_list);
Jens Axboe0b81e802021-02-16 10:33:53 -07001944 spin_unlock_irqrestore(&tctx->task_lock, flags);
Jens Axboe7cbf1722021-02-10 00:03:20 +00001945
1946 /* task_work already pending, we're done */
1947 if (test_bit(0, &tctx->task_state) ||
1948 test_and_set_bit(0, &tctx->task_state))
1949 return 0;
1950
Pavel Begunkovc15b79d2021-03-19 17:22:44 +00001951 /*
1952 * SQPOLL kernel thread doesn't need notification, just a wakeup. For
1953 * all other cases, use TWA_SIGNAL unconditionally to ensure we're
1954 * processing task_work. There's no reliable way to tell if TWA_RESUME
1955 * will do the job.
1956 */
1957 notify = (req->ctx->flags & IORING_SETUP_SQPOLL) ? TWA_NONE : TWA_SIGNAL;
1958
1959 if (!task_work_add(tsk, &tctx->task_work, notify)) {
1960 wake_up_process(tsk);
Jens Axboe7cbf1722021-02-10 00:03:20 +00001961 return 0;
Pavel Begunkovc15b79d2021-03-19 17:22:44 +00001962 }
Jens Axboe7cbf1722021-02-10 00:03:20 +00001963
1964 /*
1965 * Slow path - we failed, find and delete work. if the work is not
1966 * in the list, it got run and we're fine.
1967 */
Jens Axboe0b81e802021-02-16 10:33:53 -07001968 spin_lock_irqsave(&tctx->task_lock, flags);
Jens Axboe7cbf1722021-02-10 00:03:20 +00001969 wq_list_for_each(node, prev, &tctx->task_list) {
1970 if (&req->io_task_work.node == node) {
1971 wq_list_del(&tctx->task_list, node, prev);
1972 ret = 1;
1973 break;
1974 }
1975 }
Jens Axboe0b81e802021-02-16 10:33:53 -07001976 spin_unlock_irqrestore(&tctx->task_lock, flags);
Jens Axboe7cbf1722021-02-10 00:03:20 +00001977 clear_bit(0, &tctx->task_state);
1978 return ret;
1979}
1980
Pavel Begunkov9b465712021-03-15 14:23:07 +00001981static bool io_run_task_work_head(struct callback_head **work_head)
1982{
1983 struct callback_head *work, *next;
1984 bool executed = false;
1985
1986 do {
1987 work = xchg(work_head, NULL);
1988 if (!work)
1989 break;
1990
1991 do {
1992 next = work->next;
1993 work->func(work);
1994 work = next;
1995 cond_resched();
1996 } while (work);
1997 executed = true;
1998 } while (1);
1999
2000 return executed;
2001}
2002
2003static void io_task_work_add_head(struct callback_head **work_head,
2004 struct callback_head *task_work)
2005{
2006 struct callback_head *head;
2007
2008 do {
2009 head = READ_ONCE(*work_head);
2010 task_work->next = head;
2011 } while (cmpxchg(work_head, head, task_work) != head);
2012}
2013
Pavel Begunkoveab30c42021-01-19 13:32:42 +00002014static void io_req_task_work_add_fallback(struct io_kiocb *req,
Jens Axboe7cbf1722021-02-10 00:03:20 +00002015 task_work_func_t cb)
Pavel Begunkoveab30c42021-01-19 13:32:42 +00002016{
Pavel Begunkoveab30c42021-01-19 13:32:42 +00002017 init_task_work(&req->task_work, cb);
Pavel Begunkov9b465712021-03-15 14:23:07 +00002018 io_task_work_add_head(&req->ctx->exit_task_work, &req->task_work);
Pavel Begunkoveab30c42021-01-19 13:32:42 +00002019}
2020
Jens Axboec40f6372020-06-25 15:39:59 -06002021static void io_req_task_cancel(struct callback_head *cb)
2022{
2023 struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
Jens Axboe87ceb6a2020-09-14 08:20:12 -06002024 struct io_ring_ctx *ctx = req->ctx;
Jens Axboec40f6372020-06-25 15:39:59 -06002025
Pavel Begunkove83acd72021-02-28 22:35:09 +00002026 /* ctx is guaranteed to stay alive while we hold uring_lock */
Pavel Begunkov792bb6e2021-02-18 22:32:51 +00002027 mutex_lock(&ctx->uring_lock);
Pavel Begunkov25935532021-03-19 17:22:40 +00002028 io_req_complete_failed(req, req->result);
Pavel Begunkov792bb6e2021-02-18 22:32:51 +00002029 mutex_unlock(&ctx->uring_lock);
Jens Axboec40f6372020-06-25 15:39:59 -06002030}
2031
2032static void __io_req_task_submit(struct io_kiocb *req)
2033{
2034 struct io_ring_ctx *ctx = req->ctx;
2035
Pavel Begunkov04fc6c82021-02-12 03:23:54 +00002036 /* ctx stays valid until unlock, even if we drop all ours ctx->refs */
Pavel Begunkov81b6d052021-01-04 20:36:35 +00002037 mutex_lock(&ctx->uring_lock);
Pavel Begunkov70aacfe2021-03-01 13:02:15 +00002038 if (!(current->flags & PF_EXITING) && !current->in_execve)
Pavel Begunkovc5eef2b2021-02-10 00:03:22 +00002039 __io_queue_sqe(req);
Pavel Begunkov81b6d052021-01-04 20:36:35 +00002040 else
Pavel Begunkov25935532021-03-19 17:22:40 +00002041 io_req_complete_failed(req, -EFAULT);
Pavel Begunkov81b6d052021-01-04 20:36:35 +00002042 mutex_unlock(&ctx->uring_lock);
Jens Axboe9e645e112019-05-10 16:07:28 -06002043}
2044
Jens Axboec40f6372020-06-25 15:39:59 -06002045static void io_req_task_submit(struct callback_head *cb)
2046{
2047 struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
2048
2049 __io_req_task_submit(req);
2050}
2051
Pavel Begunkova3df76982021-02-18 22:32:52 +00002052static void io_req_task_queue_fail(struct io_kiocb *req, int ret)
2053{
Pavel Begunkova3df76982021-02-18 22:32:52 +00002054 req->result = ret;
2055 req->task_work.func = io_req_task_cancel;
2056
2057 if (unlikely(io_req_task_work_add(req)))
2058 io_req_task_work_add_fallback(req, io_req_task_cancel);
2059}
2060
Pavel Begunkov2c4b8eb2021-02-28 22:35:10 +00002061static void io_req_task_queue(struct io_kiocb *req)
2062{
2063 req->task_work.func = io_req_task_submit;
2064
2065 if (unlikely(io_req_task_work_add(req)))
2066 io_req_task_queue_fail(req, -ECANCELED);
2067}
2068
Pavel Begunkovf2f87372020-10-27 23:25:37 +00002069static inline void io_queue_next(struct io_kiocb *req)
Jackie Liuc69f8db2019-11-09 11:00:08 +08002070{
Pavel Begunkov9b5f7bd2020-06-29 13:13:00 +03002071 struct io_kiocb *nxt = io_req_find_next(req);
Pavel Begunkov944e58b2019-11-21 23:21:01 +03002072
Pavel Begunkov906a8c32020-06-27 14:04:55 +03002073 if (nxt)
2074 io_req_task_queue(nxt);
Jackie Liuc69f8db2019-11-09 11:00:08 +08002075}
2076
Jens Axboe9e645e112019-05-10 16:07:28 -06002077static void io_free_req(struct io_kiocb *req)
2078{
Pavel Begunkovc3524382020-06-28 12:52:32 +03002079 io_queue_next(req);
Jens Axboe9e645e112019-05-10 16:07:28 -06002080 __io_free_req(req);
Jens Axboee65ef562019-03-12 10:16:44 -06002081}
2082
Pavel Begunkov2d6500d2020-06-28 12:52:33 +03002083struct req_batch {
Pavel Begunkov5af1d132020-07-18 11:32:52 +03002084 struct task_struct *task;
2085 int task_refs;
Jens Axboe1b4c3512021-02-10 00:03:19 +00002086 int ctx_refs;
Pavel Begunkov2d6500d2020-06-28 12:52:33 +03002087};
2088
Pavel Begunkov5af1d132020-07-18 11:32:52 +03002089static inline void io_init_req_batch(struct req_batch *rb)
Pavel Begunkov7a743e22020-03-03 21:33:13 +03002090{
Pavel Begunkov5af1d132020-07-18 11:32:52 +03002091 rb->task_refs = 0;
Pavel Begunkov9ae72462021-02-10 00:03:16 +00002092 rb->ctx_refs = 0;
Pavel Begunkov5af1d132020-07-18 11:32:52 +03002093 rb->task = NULL;
2094}
Pavel Begunkov8766dd52020-03-14 00:31:04 +03002095
Pavel Begunkov2d6500d2020-06-28 12:52:33 +03002096static void io_req_free_batch_finish(struct io_ring_ctx *ctx,
2097 struct req_batch *rb)
2098{
Pavel Begunkov6e833d52021-02-11 18:28:20 +00002099 if (rb->task)
Pavel Begunkov7c660732021-01-25 11:42:21 +00002100 io_put_task(rb->task, rb->task_refs);
Pavel Begunkov9ae72462021-02-10 00:03:16 +00002101 if (rb->ctx_refs)
2102 percpu_ref_put_many(&ctx->refs, rb->ctx_refs);
Pavel Begunkov2d6500d2020-06-28 12:52:33 +03002103}
2104
Pavel Begunkov6ff119a2021-02-10 00:03:18 +00002105static void io_req_free_batch(struct req_batch *rb, struct io_kiocb *req,
2106 struct io_submit_state *state)
Pavel Begunkov2d6500d2020-06-28 12:52:33 +03002107{
Pavel Begunkovf2f87372020-10-27 23:25:37 +00002108 io_queue_next(req);
Pavel Begunkov96670652021-03-19 17:22:32 +00002109 io_dismantle_req(req);
Pavel Begunkov2d6500d2020-06-28 12:52:33 +03002110
Jens Axboee3bc8e92020-09-24 08:45:57 -06002111 if (req->task != rb->task) {
Pavel Begunkov7c660732021-01-25 11:42:21 +00002112 if (rb->task)
2113 io_put_task(rb->task, rb->task_refs);
Jens Axboee3bc8e92020-09-24 08:45:57 -06002114 rb->task = req->task;
2115 rb->task_refs = 0;
Pavel Begunkov5af1d132020-07-18 11:32:52 +03002116 }
Jens Axboee3bc8e92020-09-24 08:45:57 -06002117 rb->task_refs++;
Pavel Begunkov9ae72462021-02-10 00:03:16 +00002118 rb->ctx_refs++;
Pavel Begunkov5af1d132020-07-18 11:32:52 +03002119
Pavel Begunkovbd759042021-02-12 03:23:50 +00002120 if (state->free_reqs != ARRAY_SIZE(state->reqs))
Pavel Begunkov6ff119a2021-02-10 00:03:18 +00002121 state->reqs[state->free_reqs++] = req;
Pavel Begunkovbd759042021-02-12 03:23:50 +00002122 else
2123 list_add(&req->compl.list, &state->comp.free_list);
Pavel Begunkov7a743e22020-03-03 21:33:13 +03002124}
2125
Pavel Begunkov905c1722021-02-10 00:03:14 +00002126static void io_submit_flush_completions(struct io_comp_state *cs,
2127 struct io_ring_ctx *ctx)
2128{
2129 int i, nr = cs->nr;
2130 struct io_kiocb *req;
2131 struct req_batch rb;
2132
2133 io_init_req_batch(&rb);
2134 spin_lock_irq(&ctx->completion_lock);
2135 for (i = 0; i < nr; i++) {
2136 req = cs->reqs[i];
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01002137 __io_cqring_fill_event(ctx, req->user_data, req->result,
2138 req->compl.cflags);
Pavel Begunkov905c1722021-02-10 00:03:14 +00002139 }
2140 io_commit_cqring(ctx);
2141 spin_unlock_irq(&ctx->completion_lock);
2142
2143 io_cqring_ev_posted(ctx);
2144 for (i = 0; i < nr; i++) {
2145 req = cs->reqs[i];
2146
2147 /* submission and completion refs */
Jens Axboede9b4cc2021-02-24 13:28:27 -07002148 if (req_ref_sub_and_test(req, 2))
Pavel Begunkov6ff119a2021-02-10 00:03:18 +00002149 io_req_free_batch(&rb, req, &ctx->submit_state);
Pavel Begunkov905c1722021-02-10 00:03:14 +00002150 }
2151
2152 io_req_free_batch_finish(ctx, &rb);
2153 cs->nr = 0;
Jens Axboee65ef562019-03-12 10:16:44 -06002154}
2155
Jens Axboeba816ad2019-09-28 11:36:45 -06002156/*
2157 * Drop reference to request, return next in chain (if there is one) if this
2158 * was the last reference to this request.
2159 */
Pavel Begunkov0d850352021-03-19 17:22:37 +00002160static inline struct io_kiocb *io_put_req_find_next(struct io_kiocb *req)
Jens Axboee65ef562019-03-12 10:16:44 -06002161{
Pavel Begunkov9b5f7bd2020-06-29 13:13:00 +03002162 struct io_kiocb *nxt = NULL;
2163
Jens Axboede9b4cc2021-02-24 13:28:27 -07002164 if (req_ref_put_and_test(req)) {
Pavel Begunkov9b5f7bd2020-06-29 13:13:00 +03002165 nxt = io_req_find_next(req);
Jens Axboe4d7dd462019-11-20 13:03:52 -07002166 __io_free_req(req);
Jens Axboe2a44f462020-02-25 13:25:41 -07002167 }
Pavel Begunkov9b5f7bd2020-06-29 13:13:00 +03002168 return nxt;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002169}
2170
Pavel Begunkov0d850352021-03-19 17:22:37 +00002171static inline void io_put_req(struct io_kiocb *req)
Jens Axboe2b188cc2019-01-07 10:46:33 -07002172{
Jens Axboede9b4cc2021-02-24 13:28:27 -07002173 if (req_ref_put_and_test(req))
Jens Axboedef596e2019-01-09 08:59:42 -07002174 io_free_req(req);
2175}
2176
Pavel Begunkov216578e2020-10-13 09:44:00 +01002177static void io_put_req_deferred_cb(struct callback_head *cb)
2178{
2179 struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
2180
2181 io_free_req(req);
2182}
2183
2184static void io_free_req_deferred(struct io_kiocb *req)
2185{
Jens Axboe7cbf1722021-02-10 00:03:20 +00002186 req->task_work.func = io_put_req_deferred_cb;
Pavel Begunkova05432f2021-03-19 17:22:38 +00002187 if (unlikely(io_req_task_work_add(req)))
Pavel Begunkoveab30c42021-01-19 13:32:42 +00002188 io_req_task_work_add_fallback(req, io_put_req_deferred_cb);
Pavel Begunkov216578e2020-10-13 09:44:00 +01002189}
2190
2191static inline void io_put_req_deferred(struct io_kiocb *req, int refs)
2192{
Jens Axboede9b4cc2021-02-24 13:28:27 -07002193 if (req_ref_sub_and_test(req, refs))
Pavel Begunkov216578e2020-10-13 09:44:00 +01002194 io_free_req_deferred(req);
2195}
2196
Pavel Begunkov6c503152021-01-04 20:36:36 +00002197static unsigned io_cqring_events(struct io_ring_ctx *ctx)
Jens Axboea3a0e432019-08-20 11:03:11 -06002198{
2199 /* See comment at the top of this file */
2200 smp_rmb();
Pavel Begunkove23de152020-12-17 00:24:37 +00002201 return __io_cqring_events(ctx);
Jens Axboea3a0e432019-08-20 11:03:11 -06002202}
2203
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03002204static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx)
2205{
2206 struct io_rings *rings = ctx->rings;
2207
2208 /* make sure SQ entry isn't read before tail */
2209 return smp_load_acquire(&rings->sq.tail) - ctx->cached_sq_head;
2210}
2211
Pavel Begunkov8ff069b2020-07-16 23:28:04 +03002212static unsigned int io_put_kbuf(struct io_kiocb *req, struct io_buffer *kbuf)
Jens Axboee94f1412019-12-19 12:06:02 -07002213{
Pavel Begunkov8ff069b2020-07-16 23:28:04 +03002214 unsigned int cflags;
Jens Axboee94f1412019-12-19 12:06:02 -07002215
Jens Axboebcda7ba2020-02-23 16:42:51 -07002216 cflags = kbuf->bid << IORING_CQE_BUFFER_SHIFT;
2217 cflags |= IORING_CQE_F_BUFFER;
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03002218 req->flags &= ~REQ_F_BUFFER_SELECTED;
Jens Axboebcda7ba2020-02-23 16:42:51 -07002219 kfree(kbuf);
2220 return cflags;
2221}
2222
Pavel Begunkov8ff069b2020-07-16 23:28:04 +03002223static inline unsigned int io_put_rw_kbuf(struct io_kiocb *req)
2224{
2225 struct io_buffer *kbuf;
2226
2227 kbuf = (struct io_buffer *) (unsigned long) req->rw.addr;
2228 return io_put_kbuf(req, kbuf);
2229}
2230
Jens Axboe4c6e2772020-07-01 11:29:10 -06002231static inline bool io_run_task_work(void)
2232{
Jens Axboe6200b0a2020-09-13 14:38:30 -06002233 /*
2234 * Not safe to run on exiting task, and the task_work handling will
2235 * not add work to such a task.
2236 */
2237 if (unlikely(current->flags & PF_EXITING))
2238 return false;
Jens Axboe4c6e2772020-07-01 11:29:10 -06002239 if (current->task_works) {
2240 __set_current_state(TASK_RUNNING);
2241 task_work_run();
2242 return true;
2243 }
2244
2245 return false;
2246}
2247
Jens Axboedef596e2019-01-09 08:59:42 -07002248/*
2249 * Find and free completed poll iocbs
2250 */
2251static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events,
2252 struct list_head *done)
2253{
Jens Axboe8237e042019-12-28 10:48:22 -07002254 struct req_batch rb;
Jens Axboedef596e2019-01-09 08:59:42 -07002255 struct io_kiocb *req;
Xiaoguang Wangbbde0172020-06-16 02:06:38 +08002256
2257 /* order with ->result store in io_complete_rw_iopoll() */
2258 smp_rmb();
Jens Axboedef596e2019-01-09 08:59:42 -07002259
Pavel Begunkov5af1d132020-07-18 11:32:52 +03002260 io_init_req_batch(&rb);
Jens Axboedef596e2019-01-09 08:59:42 -07002261 while (!list_empty(done)) {
Jens Axboebcda7ba2020-02-23 16:42:51 -07002262 int cflags = 0;
2263
Pavel Begunkovd21ffe72020-07-13 23:37:10 +03002264 req = list_first_entry(done, struct io_kiocb, inflight_entry);
Pavel Begunkovd21ffe72020-07-13 23:37:10 +03002265 list_del(&req->inflight_entry);
Jens Axboedef596e2019-01-09 08:59:42 -07002266
Pavel Begunkov8c130822021-03-22 01:58:32 +00002267 if (READ_ONCE(req->result) == -EAGAIN &&
2268 !(req->flags & REQ_F_DONT_REISSUE)) {
Pavel Begunkovf1613402021-02-11 18:28:21 +00002269 req->iopoll_completed = 0;
Pavel Begunkov8c130822021-03-22 01:58:32 +00002270 req_ref_get(req);
2271 io_queue_async_work(req);
2272 continue;
Pavel Begunkovf1613402021-02-11 18:28:21 +00002273 }
2274
Jens Axboebcda7ba2020-02-23 16:42:51 -07002275 if (req->flags & REQ_F_BUFFER_SELECTED)
Pavel Begunkov8ff069b2020-07-16 23:28:04 +03002276 cflags = io_put_rw_kbuf(req);
Jens Axboebcda7ba2020-02-23 16:42:51 -07002277
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01002278 __io_cqring_fill_event(ctx, req->user_data, req->result, cflags);
Jens Axboedef596e2019-01-09 08:59:42 -07002279 (*nr_events)++;
2280
Jens Axboede9b4cc2021-02-24 13:28:27 -07002281 if (req_ref_put_and_test(req))
Pavel Begunkov6ff119a2021-02-10 00:03:18 +00002282 io_req_free_batch(&rb, req, &ctx->submit_state);
Jens Axboedef596e2019-01-09 08:59:42 -07002283 }
Jens Axboedef596e2019-01-09 08:59:42 -07002284
Jens Axboe09bb8392019-03-13 12:39:28 -06002285 io_commit_cqring(ctx);
Pavel Begunkov80c18e42021-01-07 03:15:41 +00002286 io_cqring_ev_posted_iopoll(ctx);
Pavel Begunkov2d6500d2020-06-28 12:52:33 +03002287 io_req_free_batch_finish(ctx, &rb);
Bijan Mottahedeh581f9812020-04-03 13:51:33 -07002288}
2289
Jens Axboedef596e2019-01-09 08:59:42 -07002290static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events,
2291 long min)
2292{
2293 struct io_kiocb *req, *tmp;
2294 LIST_HEAD(done);
2295 bool spin;
2296 int ret;
2297
2298 /*
2299 * Only spin for completions if we don't have multiple devices hanging
2300 * off our complete list, and we're under the requested amount.
2301 */
2302 spin = !ctx->poll_multi_file && *nr_events < min;
2303
2304 ret = 0;
Pavel Begunkovd21ffe72020-07-13 23:37:10 +03002305 list_for_each_entry_safe(req, tmp, &ctx->iopoll_list, inflight_entry) {
Jens Axboe9adbd452019-12-20 08:45:55 -07002306 struct kiocb *kiocb = &req->rw.kiocb;
Jens Axboedef596e2019-01-09 08:59:42 -07002307
2308 /*
Bijan Mottahedeh581f9812020-04-03 13:51:33 -07002309 * Move completed and retryable entries to our local lists.
2310 * If we find a request that requires polling, break out
2311 * and complete those lists first, if we have entries there.
Jens Axboedef596e2019-01-09 08:59:42 -07002312 */
Xiaoguang Wang65a65432020-06-11 23:39:36 +08002313 if (READ_ONCE(req->iopoll_completed)) {
Pavel Begunkovd21ffe72020-07-13 23:37:10 +03002314 list_move_tail(&req->inflight_entry, &done);
Jens Axboedef596e2019-01-09 08:59:42 -07002315 continue;
2316 }
2317 if (!list_empty(&done))
2318 break;
2319
2320 ret = kiocb->ki_filp->f_op->iopoll(kiocb, spin);
2321 if (ret < 0)
2322 break;
2323
Pavel Begunkov3aadc232020-07-06 17:59:29 +03002324 /* iopoll may have completed current req */
2325 if (READ_ONCE(req->iopoll_completed))
Pavel Begunkovd21ffe72020-07-13 23:37:10 +03002326 list_move_tail(&req->inflight_entry, &done);
Pavel Begunkov3aadc232020-07-06 17:59:29 +03002327
Jens Axboedef596e2019-01-09 08:59:42 -07002328 if (ret && spin)
2329 spin = false;
2330 ret = 0;
2331 }
2332
2333 if (!list_empty(&done))
2334 io_iopoll_complete(ctx, nr_events, &done);
2335
2336 return ret;
2337}
2338
2339/*
Jens Axboedef596e2019-01-09 08:59:42 -07002340 * We can't just wait for polled events to come to us, we have to actively
2341 * find and complete them.
2342 */
Pavel Begunkovb2edc0a2020-07-07 16:36:22 +03002343static void io_iopoll_try_reap_events(struct io_ring_ctx *ctx)
Jens Axboedef596e2019-01-09 08:59:42 -07002344{
2345 if (!(ctx->flags & IORING_SETUP_IOPOLL))
2346 return;
2347
2348 mutex_lock(&ctx->uring_lock);
Pavel Begunkov540e32a2020-07-13 23:37:09 +03002349 while (!list_empty(&ctx->iopoll_list)) {
Jens Axboedef596e2019-01-09 08:59:42 -07002350 unsigned int nr_events = 0;
2351
Pavel Begunkovb2edc0a2020-07-07 16:36:22 +03002352 io_do_iopoll(ctx, &nr_events, 0);
Jens Axboe08f54392019-08-21 22:19:11 -06002353
Pavel Begunkovb2edc0a2020-07-07 16:36:22 +03002354 /* let it sleep and repeat later if can't complete a request */
2355 if (nr_events == 0)
2356 break;
Jens Axboe08f54392019-08-21 22:19:11 -06002357 /*
2358 * Ensure we allow local-to-the-cpu processing to take place,
2359 * in this case we need to ensure that we reap all events.
Pavel Begunkov3fcee5a2020-07-06 17:59:31 +03002360 * Also let task_work, etc. to progress by releasing the mutex
Jens Axboe08f54392019-08-21 22:19:11 -06002361 */
Pavel Begunkov3fcee5a2020-07-06 17:59:31 +03002362 if (need_resched()) {
2363 mutex_unlock(&ctx->uring_lock);
2364 cond_resched();
2365 mutex_lock(&ctx->uring_lock);
2366 }
Jens Axboedef596e2019-01-09 08:59:42 -07002367 }
2368 mutex_unlock(&ctx->uring_lock);
2369}
2370
Pavel Begunkov7668b922020-07-07 16:36:21 +03002371static int io_iopoll_check(struct io_ring_ctx *ctx, long min)
Jens Axboedef596e2019-01-09 08:59:42 -07002372{
Pavel Begunkov7668b922020-07-07 16:36:21 +03002373 unsigned int nr_events = 0;
Pavel Begunkove9979b32021-04-13 02:58:45 +01002374 int ret = 0;
Jens Axboedef596e2019-01-09 08:59:42 -07002375
Xiaoguang Wangc7849be2020-02-22 14:46:05 +08002376 /*
2377 * We disallow the app entering submit/complete with polling, but we
2378 * still need to lock the ring to prevent racing with polled issue
2379 * that got punted to a workqueue.
2380 */
2381 mutex_lock(&ctx->uring_lock);
Pavel Begunkovf39c8a52021-04-13 02:58:46 +01002382 /*
2383 * Don't enter poll loop if we already have events pending.
2384 * If we do, we can potentially be spinning for commands that
2385 * already triggered a CQE (eg in error).
2386 */
2387 if (test_bit(0, &ctx->cq_check_overflow))
2388 __io_cqring_overflow_flush(ctx, false);
2389 if (io_cqring_events(ctx))
2390 goto out;
Jens Axboedef596e2019-01-09 08:59:42 -07002391 do {
Jens Axboe500f9fb2019-08-19 12:15:59 -06002392 /*
2393 * If a submit got punted to a workqueue, we can have the
2394 * application entering polling for a command before it gets
2395 * issued. That app will hold the uring_lock for the duration
2396 * of the poll right here, so we need to take a breather every
2397 * now and then to ensure that the issue has a chance to add
2398 * the poll to the issued list. Otherwise we can spin here
2399 * forever, while the workqueue is stuck trying to acquire the
2400 * very same mutex.
2401 */
Pavel Begunkove9979b32021-04-13 02:58:45 +01002402 if (list_empty(&ctx->iopoll_list)) {
Jens Axboe500f9fb2019-08-19 12:15:59 -06002403 mutex_unlock(&ctx->uring_lock);
Jens Axboe4c6e2772020-07-01 11:29:10 -06002404 io_run_task_work();
Jens Axboe500f9fb2019-08-19 12:15:59 -06002405 mutex_lock(&ctx->uring_lock);
Pavel Begunkove9979b32021-04-13 02:58:45 +01002406
2407 if (list_empty(&ctx->iopoll_list))
2408 break;
Jens Axboe500f9fb2019-08-19 12:15:59 -06002409 }
Pavel Begunkovf39c8a52021-04-13 02:58:46 +01002410 ret = io_do_iopoll(ctx, &nr_events, min);
2411 } while (!ret && nr_events < min && !need_resched());
2412out:
Jens Axboe500f9fb2019-08-19 12:15:59 -06002413 mutex_unlock(&ctx->uring_lock);
Jens Axboedef596e2019-01-09 08:59:42 -07002414 return ret;
2415}
2416
Jens Axboe491381ce2019-10-17 09:20:46 -06002417static void kiocb_end_write(struct io_kiocb *req)
Jens Axboe2b188cc2019-01-07 10:46:33 -07002418{
Jens Axboe491381ce2019-10-17 09:20:46 -06002419 /*
2420 * Tell lockdep we inherited freeze protection from submission
2421 * thread.
2422 */
2423 if (req->flags & REQ_F_ISREG) {
Pavel Begunkov1c986792021-03-22 01:58:31 +00002424 struct super_block *sb = file_inode(req->file)->i_sb;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002425
Pavel Begunkov1c986792021-03-22 01:58:31 +00002426 __sb_writers_acquired(sb, SB_FREEZE_WRITE);
2427 sb_end_write(sb);
Jens Axboe2b188cc2019-01-07 10:46:33 -07002428 }
2429}
2430
Jens Axboeb63534c2020-06-04 11:28:00 -06002431#ifdef CONFIG_BLOCK
Pavel Begunkovdc2a6e92021-01-19 13:32:35 +00002432static bool io_resubmit_prep(struct io_kiocb *req)
Jens Axboeb63534c2020-06-04 11:28:00 -06002433{
Pavel Begunkovab454432021-03-22 01:58:33 +00002434 struct io_async_rw *rw = req->async_data;
Jens Axboeb63534c2020-06-04 11:28:00 -06002435
Pavel Begunkovab454432021-03-22 01:58:33 +00002436 if (!rw)
2437 return !io_req_prep_async(req);
2438 /* may have left rw->iter inconsistent on -EIOCBQUEUED */
2439 iov_iter_revert(&rw->iter, req->result - iov_iter_count(&rw->iter));
2440 return true;
Jens Axboeb63534c2020-06-04 11:28:00 -06002441}
Jens Axboeb63534c2020-06-04 11:28:00 -06002442
Jens Axboe3e6a0d32021-03-01 13:56:00 -07002443static bool io_rw_should_reissue(struct io_kiocb *req)
Jens Axboeb63534c2020-06-04 11:28:00 -06002444{
Jens Axboe355afae2020-09-02 09:30:31 -06002445 umode_t mode = file_inode(req->file)->i_mode;
Jens Axboe3e6a0d32021-03-01 13:56:00 -07002446 struct io_ring_ctx *ctx = req->ctx;
Jens Axboeb63534c2020-06-04 11:28:00 -06002447
Jens Axboe355afae2020-09-02 09:30:31 -06002448 if (!S_ISBLK(mode) && !S_ISREG(mode))
2449 return false;
Jens Axboe3e6a0d32021-03-01 13:56:00 -07002450 if ((req->flags & REQ_F_NOWAIT) || (io_wq_current_is_worker() &&
2451 !(ctx->flags & IORING_SETUP_IOPOLL)))
Jens Axboeb63534c2020-06-04 11:28:00 -06002452 return false;
Jens Axboe7c977a52021-02-23 19:17:35 -07002453 /*
2454 * If ref is dying, we might be running poll reap from the exit work.
2455 * Don't attempt to reissue from that path, just let it fail with
2456 * -EAGAIN.
2457 */
Jens Axboe3e6a0d32021-03-01 13:56:00 -07002458 if (percpu_ref_is_dying(&ctx->refs))
2459 return false;
2460 return true;
2461}
Jens Axboee82ad482021-04-02 19:45:34 -06002462#else
Jens Axboea1ff1e32021-04-12 06:40:02 -06002463static bool io_resubmit_prep(struct io_kiocb *req)
2464{
2465 return false;
2466}
Jens Axboee82ad482021-04-02 19:45:34 -06002467static bool io_rw_should_reissue(struct io_kiocb *req)
2468{
2469 return false;
2470}
Jens Axboe3e6a0d32021-03-01 13:56:00 -07002471#endif
2472
Jens Axboea1d7c392020-06-22 11:09:46 -06002473static void __io_complete_rw(struct io_kiocb *req, long res, long res2,
Pavel Begunkov889fca72021-02-10 00:03:09 +00002474 unsigned int issue_flags)
Jens Axboea1d7c392020-06-22 11:09:46 -06002475{
Pavel Begunkov2f8e45f2021-02-11 18:28:23 +00002476 int cflags = 0;
2477
Pavel Begunkovb65c1282021-03-22 01:45:59 +00002478 if (req->rw.kiocb.ki_flags & IOCB_WRITE)
2479 kiocb_end_write(req);
Pavel Begunkov9532b992021-03-22 01:58:34 +00002480 if (res != req->result) {
2481 if ((res == -EAGAIN || res == -EOPNOTSUPP) &&
2482 io_rw_should_reissue(req)) {
2483 req->flags |= REQ_F_REISSUE;
2484 return;
2485 }
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01002486 req_set_fail(req);
Pavel Begunkov9532b992021-03-22 01:58:34 +00002487 }
Pavel Begunkov2f8e45f2021-02-11 18:28:23 +00002488 if (req->flags & REQ_F_BUFFER_SELECTED)
2489 cflags = io_put_rw_kbuf(req);
2490 __io_req_complete(req, issue_flags, res, cflags);
Jens Axboeba816ad2019-09-28 11:36:45 -06002491}
2492
2493static void io_complete_rw(struct kiocb *kiocb, long res, long res2)
2494{
Jens Axboe9adbd452019-12-20 08:45:55 -07002495 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
Jens Axboeba816ad2019-09-28 11:36:45 -06002496
Pavel Begunkov889fca72021-02-10 00:03:09 +00002497 __io_complete_rw(req, res, res2, 0);
Jens Axboe2b188cc2019-01-07 10:46:33 -07002498}
2499
Jens Axboedef596e2019-01-09 08:59:42 -07002500static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2)
2501{
Jens Axboe9adbd452019-12-20 08:45:55 -07002502 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
Jens Axboedef596e2019-01-09 08:59:42 -07002503
Jens Axboe491381ce2019-10-17 09:20:46 -06002504 if (kiocb->ki_flags & IOCB_WRITE)
2505 kiocb_end_write(req);
Pavel Begunkov9532b992021-03-22 01:58:34 +00002506 if (unlikely(res != req->result)) {
Jens Axboea1ff1e32021-04-12 06:40:02 -06002507 if (!(res == -EAGAIN && io_rw_should_reissue(req) &&
2508 io_resubmit_prep(req))) {
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01002509 req_set_fail(req);
Pavel Begunkov9532b992021-03-22 01:58:34 +00002510 req->flags |= REQ_F_DONT_REISSUE;
2511 }
Pavel Begunkov8c130822021-03-22 01:58:32 +00002512 }
Xiaoguang Wangbbde0172020-06-16 02:06:38 +08002513
2514 WRITE_ONCE(req->result, res);
Jens Axboeb9b0e0d2021-02-23 08:18:36 -07002515 /* order with io_iopoll_complete() checking ->result */
Pavel Begunkovcd664b02020-06-25 12:37:10 +03002516 smp_wmb();
2517 WRITE_ONCE(req->iopoll_completed, 1);
Jens Axboedef596e2019-01-09 08:59:42 -07002518}
2519
2520/*
2521 * After the iocb has been issued, it's safe to be found on the poll list.
2522 * Adding the kiocb to the list AFTER submission ensures that we don't
Pavel Begunkovf39c8a52021-04-13 02:58:46 +01002523 * find it from a io_do_iopoll() thread before the issuer is done
Jens Axboedef596e2019-01-09 08:59:42 -07002524 * accessing the kiocb cookie.
2525 */
Xiaoguang Wang2e9dbe92020-11-13 00:44:08 +08002526static void io_iopoll_req_issued(struct io_kiocb *req, bool in_async)
Jens Axboedef596e2019-01-09 08:59:42 -07002527{
2528 struct io_ring_ctx *ctx = req->ctx;
2529
2530 /*
2531 * Track whether we have multiple files in our lists. This will impact
2532 * how we do polling eventually, not spinning if we're on potentially
2533 * different devices.
2534 */
Pavel Begunkov540e32a2020-07-13 23:37:09 +03002535 if (list_empty(&ctx->iopoll_list)) {
Jens Axboedef596e2019-01-09 08:59:42 -07002536 ctx->poll_multi_file = false;
2537 } else if (!ctx->poll_multi_file) {
2538 struct io_kiocb *list_req;
2539
Pavel Begunkov540e32a2020-07-13 23:37:09 +03002540 list_req = list_first_entry(&ctx->iopoll_list, struct io_kiocb,
Pavel Begunkovd21ffe72020-07-13 23:37:10 +03002541 inflight_entry);
Jens Axboe9adbd452019-12-20 08:45:55 -07002542 if (list_req->file != req->file)
Jens Axboedef596e2019-01-09 08:59:42 -07002543 ctx->poll_multi_file = true;
2544 }
2545
2546 /*
2547 * For fast devices, IO may have already completed. If it has, add
2548 * it to the front so we find it first.
2549 */
Xiaoguang Wang65a65432020-06-11 23:39:36 +08002550 if (READ_ONCE(req->iopoll_completed))
Pavel Begunkovd21ffe72020-07-13 23:37:10 +03002551 list_add(&req->inflight_entry, &ctx->iopoll_list);
Jens Axboedef596e2019-01-09 08:59:42 -07002552 else
Pavel Begunkovd21ffe72020-07-13 23:37:10 +03002553 list_add_tail(&req->inflight_entry, &ctx->iopoll_list);
Xiaoguang Wangbdcd3ea2020-02-25 22:12:08 +08002554
Xiaoguang Wang2e9dbe92020-11-13 00:44:08 +08002555 /*
2556 * If IORING_SETUP_SQPOLL is enabled, sqes are either handled in sq thread
2557 * task context or in io worker task context. If current task context is
2558 * sq thread, we don't need to check whether should wake up sq thread.
2559 */
2560 if (in_async && (ctx->flags & IORING_SETUP_SQPOLL) &&
Jens Axboe534ca6d2020-09-02 13:52:19 -06002561 wq_has_sleeper(&ctx->sq_data->wait))
2562 wake_up(&ctx->sq_data->wait);
Jens Axboedef596e2019-01-09 08:59:42 -07002563}
2564
Pavel Begunkov9f13c352020-05-17 14:13:41 +03002565static inline void io_state_file_put(struct io_submit_state *state)
2566{
Pavel Begunkov02b23a92021-01-19 13:32:41 +00002567 if (state->file_refs) {
2568 fput_many(state->file, state->file_refs);
2569 state->file_refs = 0;
2570 }
Jens Axboe9a56a232019-01-09 09:06:50 -07002571}
2572
2573/*
2574 * Get as many references to a file as we have IOs left in this submission,
2575 * assuming most submissions are for one file, or at least that each file
2576 * has more than one submission.
2577 */
Pavel Begunkov8da11c12020-02-24 11:32:44 +03002578static struct file *__io_file_get(struct io_submit_state *state, int fd)
Jens Axboe9a56a232019-01-09 09:06:50 -07002579{
2580 if (!state)
2581 return fget(fd);
2582
Pavel Begunkov6e1271e2020-11-20 15:50:50 +00002583 if (state->file_refs) {
Jens Axboe9a56a232019-01-09 09:06:50 -07002584 if (state->fd == fd) {
Pavel Begunkov6e1271e2020-11-20 15:50:50 +00002585 state->file_refs--;
Jens Axboe9a56a232019-01-09 09:06:50 -07002586 return state->file;
2587 }
Pavel Begunkov02b23a92021-01-19 13:32:41 +00002588 io_state_file_put(state);
Jens Axboe9a56a232019-01-09 09:06:50 -07002589 }
2590 state->file = fget_many(fd, state->ios_left);
Pavel Begunkov6e1271e2020-11-20 15:50:50 +00002591 if (unlikely(!state->file))
Jens Axboe9a56a232019-01-09 09:06:50 -07002592 return NULL;
2593
2594 state->fd = fd;
Pavel Begunkov6e1271e2020-11-20 15:50:50 +00002595 state->file_refs = state->ios_left - 1;
Jens Axboe9a56a232019-01-09 09:06:50 -07002596 return state->file;
2597}
2598
Jens Axboe4503b762020-06-01 10:00:27 -06002599static bool io_bdev_nowait(struct block_device *bdev)
2600{
Jeffle Xu9ba0d0c2020-10-19 16:59:42 +08002601 return !bdev || blk_queue_nowait(bdev_get_queue(bdev));
Jens Axboe4503b762020-06-01 10:00:27 -06002602}
2603
Jens Axboe2b188cc2019-01-07 10:46:33 -07002604/*
2605 * If we tracked the file through the SCM inflight mechanism, we could support
2606 * any file. For now, just ensure that anything potentially problematic is done
2607 * inline.
2608 */
Jens Axboe7b29f922021-03-12 08:30:14 -07002609static bool __io_file_supports_async(struct file *file, int rw)
Jens Axboe2b188cc2019-01-07 10:46:33 -07002610{
2611 umode_t mode = file_inode(file)->i_mode;
2612
Jens Axboe4503b762020-06-01 10:00:27 -06002613 if (S_ISBLK(mode)) {
Christoph Hellwig4e7b5672020-11-23 13:38:40 +01002614 if (IS_ENABLED(CONFIG_BLOCK) &&
2615 io_bdev_nowait(I_BDEV(file->f_mapping->host)))
Jens Axboe4503b762020-06-01 10:00:27 -06002616 return true;
2617 return false;
2618 }
2619 if (S_ISCHR(mode) || S_ISSOCK(mode))
Jens Axboe2b188cc2019-01-07 10:46:33 -07002620 return true;
Jens Axboe4503b762020-06-01 10:00:27 -06002621 if (S_ISREG(mode)) {
Christoph Hellwig4e7b5672020-11-23 13:38:40 +01002622 if (IS_ENABLED(CONFIG_BLOCK) &&
2623 io_bdev_nowait(file->f_inode->i_sb->s_bdev) &&
Jens Axboe4503b762020-06-01 10:00:27 -06002624 file->f_op != &io_uring_fops)
2625 return true;
2626 return false;
2627 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07002628
Jens Axboec5b85622020-06-09 19:23:05 -06002629 /* any ->read/write should understand O_NONBLOCK */
2630 if (file->f_flags & O_NONBLOCK)
2631 return true;
2632
Jens Axboeaf197f52020-04-28 13:15:06 -06002633 if (!(file->f_mode & FMODE_NOWAIT))
2634 return false;
2635
2636 if (rw == READ)
2637 return file->f_op->read_iter != NULL;
2638
2639 return file->f_op->write_iter != NULL;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002640}
2641
Jens Axboe7b29f922021-03-12 08:30:14 -07002642static bool io_file_supports_async(struct io_kiocb *req, int rw)
2643{
2644 if (rw == READ && (req->flags & REQ_F_ASYNC_READ))
2645 return true;
2646 else if (rw == WRITE && (req->flags & REQ_F_ASYNC_WRITE))
2647 return true;
2648
2649 return __io_file_supports_async(req->file, rw);
2650}
2651
Pavel Begunkova88fc402020-09-30 22:57:53 +03002652static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboe2b188cc2019-01-07 10:46:33 -07002653{
Jens Axboedef596e2019-01-09 08:59:42 -07002654 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe9adbd452019-12-20 08:45:55 -07002655 struct kiocb *kiocb = &req->rw.kiocb;
Pavel Begunkov75c668c2021-02-04 13:52:05 +00002656 struct file *file = req->file;
Jens Axboe09bb8392019-03-13 12:39:28 -06002657 unsigned ioprio;
2658 int ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002659
Jens Axboe7b29f922021-03-12 08:30:14 -07002660 if (!(req->flags & REQ_F_ISREG) && S_ISREG(file_inode(file)->i_mode))
Jens Axboe491381ce2019-10-17 09:20:46 -06002661 req->flags |= REQ_F_ISREG;
2662
Jens Axboe2b188cc2019-01-07 10:46:33 -07002663 kiocb->ki_pos = READ_ONCE(sqe->off);
Pavel Begunkov75c668c2021-02-04 13:52:05 +00002664 if (kiocb->ki_pos == -1 && !(file->f_mode & FMODE_STREAM)) {
Jens Axboeba042912019-12-25 16:33:42 -07002665 req->flags |= REQ_F_CUR_POS;
Pavel Begunkov75c668c2021-02-04 13:52:05 +00002666 kiocb->ki_pos = file->f_pos;
Jens Axboeba042912019-12-25 16:33:42 -07002667 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07002668 kiocb->ki_hint = ki_hint_validate(file_write_hint(kiocb->ki_filp));
Pavel Begunkov3e577dc2020-02-01 03:58:42 +03002669 kiocb->ki_flags = iocb_flags(kiocb->ki_filp);
2670 ret = kiocb_set_rw_flags(kiocb, READ_ONCE(sqe->rw_flags));
2671 if (unlikely(ret))
2672 return ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002673
Pavel Begunkov75c668c2021-02-04 13:52:05 +00002674 /* don't allow async punt for O_NONBLOCK or RWF_NOWAIT */
2675 if ((kiocb->ki_flags & IOCB_NOWAIT) || (file->f_flags & O_NONBLOCK))
2676 req->flags |= REQ_F_NOWAIT;
2677
Jens Axboe2b188cc2019-01-07 10:46:33 -07002678 ioprio = READ_ONCE(sqe->ioprio);
2679 if (ioprio) {
2680 ret = ioprio_check_cap(ioprio);
2681 if (ret)
Jens Axboe09bb8392019-03-13 12:39:28 -06002682 return ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002683
2684 kiocb->ki_ioprio = ioprio;
2685 } else
2686 kiocb->ki_ioprio = get_current_ioprio();
2687
Jens Axboedef596e2019-01-09 08:59:42 -07002688 if (ctx->flags & IORING_SETUP_IOPOLL) {
Jens Axboedef596e2019-01-09 08:59:42 -07002689 if (!(kiocb->ki_flags & IOCB_DIRECT) ||
2690 !kiocb->ki_filp->f_op->iopoll)
Jens Axboe09bb8392019-03-13 12:39:28 -06002691 return -EOPNOTSUPP;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002692
Jens Axboedef596e2019-01-09 08:59:42 -07002693 kiocb->ki_flags |= IOCB_HIPRI;
2694 kiocb->ki_complete = io_complete_rw_iopoll;
Xiaoguang Wang65a65432020-06-11 23:39:36 +08002695 req->iopoll_completed = 0;
Jens Axboedef596e2019-01-09 08:59:42 -07002696 } else {
Jens Axboe09bb8392019-03-13 12:39:28 -06002697 if (kiocb->ki_flags & IOCB_HIPRI)
2698 return -EINVAL;
Jens Axboedef596e2019-01-09 08:59:42 -07002699 kiocb->ki_complete = io_complete_rw;
2700 }
Jens Axboe9adbd452019-12-20 08:45:55 -07002701
Pavel Begunkoveae071c2021-04-25 14:32:24 +01002702 if (req->opcode == IORING_OP_READ_FIXED ||
2703 req->opcode == IORING_OP_WRITE_FIXED) {
2704 req->imu = NULL;
2705 io_req_set_rsrc_node(req);
2706 }
2707
Jens Axboe3529d8c2019-12-19 18:24:38 -07002708 req->rw.addr = READ_ONCE(sqe->addr);
2709 req->rw.len = READ_ONCE(sqe->len);
Bijan Mottahedeh4f4eeba2020-05-19 14:52:49 -07002710 req->buf_index = READ_ONCE(sqe->buf_index);
Jens Axboe2b188cc2019-01-07 10:46:33 -07002711 return 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002712}
2713
2714static inline void io_rw_done(struct kiocb *kiocb, ssize_t ret)
2715{
2716 switch (ret) {
2717 case -EIOCBQUEUED:
2718 break;
2719 case -ERESTARTSYS:
2720 case -ERESTARTNOINTR:
2721 case -ERESTARTNOHAND:
2722 case -ERESTART_RESTARTBLOCK:
2723 /*
2724 * We can't just restart the syscall, since previously
2725 * submitted sqes may already be in progress. Just fail this
2726 * IO with EINTR.
2727 */
2728 ret = -EINTR;
Gustavo A. R. Silvadf561f662020-08-23 17:36:59 -05002729 fallthrough;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002730 default:
2731 kiocb->ki_complete(kiocb, ret, 0);
2732 }
2733}
2734
Jens Axboea1d7c392020-06-22 11:09:46 -06002735static void kiocb_done(struct kiocb *kiocb, ssize_t ret,
Pavel Begunkov889fca72021-02-10 00:03:09 +00002736 unsigned int issue_flags)
Jens Axboeba816ad2019-09-28 11:36:45 -06002737{
Jens Axboeba042912019-12-25 16:33:42 -07002738 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
Jens Axboee8c2bc12020-08-15 18:44:09 -07002739 struct io_async_rw *io = req->async_data;
Pavel Begunkov97284632021-04-08 19:28:03 +01002740 bool check_reissue = kiocb->ki_complete == io_complete_rw;
Jens Axboeba042912019-12-25 16:33:42 -07002741
Jens Axboe227c0c92020-08-13 11:51:40 -06002742 /* add previously done IO, if any */
Jens Axboee8c2bc12020-08-15 18:44:09 -07002743 if (io && io->bytes_done > 0) {
Jens Axboe227c0c92020-08-13 11:51:40 -06002744 if (ret < 0)
Jens Axboee8c2bc12020-08-15 18:44:09 -07002745 ret = io->bytes_done;
Jens Axboe227c0c92020-08-13 11:51:40 -06002746 else
Jens Axboee8c2bc12020-08-15 18:44:09 -07002747 ret += io->bytes_done;
Jens Axboe227c0c92020-08-13 11:51:40 -06002748 }
2749
Jens Axboeba042912019-12-25 16:33:42 -07002750 if (req->flags & REQ_F_CUR_POS)
2751 req->file->f_pos = kiocb->ki_pos;
Pavel Begunkovbcaec082020-02-24 11:30:18 +03002752 if (ret >= 0 && kiocb->ki_complete == io_complete_rw)
Pavel Begunkov889fca72021-02-10 00:03:09 +00002753 __io_complete_rw(req, ret, 0, issue_flags);
Jens Axboeba816ad2019-09-28 11:36:45 -06002754 else
2755 io_rw_done(kiocb, ret);
Pavel Begunkov97284632021-04-08 19:28:03 +01002756
2757 if (check_reissue && req->flags & REQ_F_REISSUE) {
2758 req->flags &= ~REQ_F_REISSUE;
Jens Axboea7be7c22021-04-15 11:31:14 -06002759 if (io_resubmit_prep(req)) {
Pavel Begunkov8c130822021-03-22 01:58:32 +00002760 req_ref_get(req);
2761 io_queue_async_work(req);
2762 } else {
Pavel Begunkov97284632021-04-08 19:28:03 +01002763 int cflags = 0;
2764
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01002765 req_set_fail(req);
Pavel Begunkov97284632021-04-08 19:28:03 +01002766 if (req->flags & REQ_F_BUFFER_SELECTED)
2767 cflags = io_put_rw_kbuf(req);
2768 __io_req_complete(req, issue_flags, ret, cflags);
2769 }
2770 }
Jens Axboeba816ad2019-09-28 11:36:45 -06002771}
2772
Pavel Begunkoveae071c2021-04-25 14:32:24 +01002773static int __io_import_fixed(struct io_kiocb *req, int rw, struct iov_iter *iter,
2774 struct io_mapped_ubuf *imu)
Jens Axboeedafcce2019-01-09 09:16:05 -07002775{
Jens Axboe9adbd452019-12-20 08:45:55 -07002776 size_t len = req->rw.len;
Pavel Begunkov75769e32021-04-01 15:43:54 +01002777 u64 buf_end, buf_addr = req->rw.addr;
Jens Axboeedafcce2019-01-09 09:16:05 -07002778 size_t offset;
Jens Axboeedafcce2019-01-09 09:16:05 -07002779
Pavel Begunkov75769e32021-04-01 15:43:54 +01002780 if (unlikely(check_add_overflow(buf_addr, (u64)len, &buf_end)))
Jens Axboeedafcce2019-01-09 09:16:05 -07002781 return -EFAULT;
2782 /* not inside the mapped region */
Pavel Begunkov4751f532021-04-01 15:43:55 +01002783 if (unlikely(buf_addr < imu->ubuf || buf_end > imu->ubuf_end))
Jens Axboeedafcce2019-01-09 09:16:05 -07002784 return -EFAULT;
2785
2786 /*
2787 * May not be a start of buffer, set size appropriately
2788 * and advance us to the beginning.
2789 */
2790 offset = buf_addr - imu->ubuf;
2791 iov_iter_bvec(iter, rw, imu->bvec, imu->nr_bvecs, offset + len);
Jens Axboebd11b3a2019-07-20 08:37:31 -06002792
2793 if (offset) {
2794 /*
2795 * Don't use iov_iter_advance() here, as it's really slow for
2796 * using the latter parts of a big fixed buffer - it iterates
2797 * over each segment manually. We can cheat a bit here, because
2798 * we know that:
2799 *
2800 * 1) it's a BVEC iter, we set it up
2801 * 2) all bvecs are PAGE_SIZE in size, except potentially the
2802 * first and last bvec
2803 *
2804 * So just find our index, and adjust the iterator afterwards.
2805 * If the offset is within the first bvec (or the whole first
2806 * bvec, just use iov_iter_advance(). This makes it easier
2807 * since we can just skip the first segment, which may not
2808 * be PAGE_SIZE aligned.
2809 */
2810 const struct bio_vec *bvec = imu->bvec;
2811
2812 if (offset <= bvec->bv_len) {
2813 iov_iter_advance(iter, offset);
2814 } else {
2815 unsigned long seg_skip;
2816
2817 /* skip first vec */
2818 offset -= bvec->bv_len;
2819 seg_skip = 1 + (offset >> PAGE_SHIFT);
2820
2821 iter->bvec = bvec + seg_skip;
2822 iter->nr_segs -= seg_skip;
Aleix Roca Nonell99c79f62019-08-15 14:03:22 +02002823 iter->count -= bvec->bv_len + offset;
Jens Axboebd11b3a2019-07-20 08:37:31 -06002824 iter->iov_offset = offset & ~PAGE_MASK;
Jens Axboebd11b3a2019-07-20 08:37:31 -06002825 }
2826 }
2827
Pavel Begunkov847595d2021-02-04 13:52:06 +00002828 return 0;
Jens Axboeedafcce2019-01-09 09:16:05 -07002829}
2830
Pavel Begunkoveae071c2021-04-25 14:32:24 +01002831static int io_import_fixed(struct io_kiocb *req, int rw, struct iov_iter *iter)
2832{
2833 struct io_ring_ctx *ctx = req->ctx;
2834 struct io_mapped_ubuf *imu = req->imu;
2835 u16 index, buf_index = req->buf_index;
2836
2837 if (likely(!imu)) {
2838 if (unlikely(buf_index >= ctx->nr_user_bufs))
2839 return -EFAULT;
2840 index = array_index_nospec(buf_index, ctx->nr_user_bufs);
2841 imu = READ_ONCE(ctx->user_bufs[index]);
2842 req->imu = imu;
2843 }
2844 return __io_import_fixed(req, rw, iter, imu);
2845}
2846
Jens Axboebcda7ba2020-02-23 16:42:51 -07002847static void io_ring_submit_unlock(struct io_ring_ctx *ctx, bool needs_lock)
2848{
2849 if (needs_lock)
2850 mutex_unlock(&ctx->uring_lock);
2851}
2852
2853static void io_ring_submit_lock(struct io_ring_ctx *ctx, bool needs_lock)
2854{
2855 /*
2856 * "Normal" inline submissions always hold the uring_lock, since we
2857 * grab it from the system call. Same is true for the SQPOLL offload.
2858 * The only exception is when we've detached the request and issue it
2859 * from an async worker thread, grab the lock for that case.
2860 */
2861 if (needs_lock)
2862 mutex_lock(&ctx->uring_lock);
2863}
2864
2865static struct io_buffer *io_buffer_select(struct io_kiocb *req, size_t *len,
2866 int bgid, struct io_buffer *kbuf,
2867 bool needs_lock)
2868{
2869 struct io_buffer *head;
2870
2871 if (req->flags & REQ_F_BUFFER_SELECTED)
2872 return kbuf;
2873
2874 io_ring_submit_lock(req->ctx, needs_lock);
2875
2876 lockdep_assert_held(&req->ctx->uring_lock);
2877
Jens Axboe9e15c3a2021-03-13 12:29:43 -07002878 head = xa_load(&req->ctx->io_buffers, bgid);
Jens Axboebcda7ba2020-02-23 16:42:51 -07002879 if (head) {
2880 if (!list_empty(&head->list)) {
2881 kbuf = list_last_entry(&head->list, struct io_buffer,
2882 list);
2883 list_del(&kbuf->list);
2884 } else {
2885 kbuf = head;
Jens Axboe9e15c3a2021-03-13 12:29:43 -07002886 xa_erase(&req->ctx->io_buffers, bgid);
Jens Axboebcda7ba2020-02-23 16:42:51 -07002887 }
2888 if (*len > kbuf->len)
2889 *len = kbuf->len;
2890 } else {
2891 kbuf = ERR_PTR(-ENOBUFS);
2892 }
2893
2894 io_ring_submit_unlock(req->ctx, needs_lock);
2895
2896 return kbuf;
2897}
2898
Jens Axboe4d954c22020-02-27 07:31:19 -07002899static void __user *io_rw_buffer_select(struct io_kiocb *req, size_t *len,
2900 bool needs_lock)
2901{
2902 struct io_buffer *kbuf;
Bijan Mottahedeh4f4eeba2020-05-19 14:52:49 -07002903 u16 bgid;
Jens Axboe4d954c22020-02-27 07:31:19 -07002904
2905 kbuf = (struct io_buffer *) (unsigned long) req->rw.addr;
Bijan Mottahedeh4f4eeba2020-05-19 14:52:49 -07002906 bgid = req->buf_index;
Jens Axboe4d954c22020-02-27 07:31:19 -07002907 kbuf = io_buffer_select(req, len, bgid, kbuf, needs_lock);
2908 if (IS_ERR(kbuf))
2909 return kbuf;
2910 req->rw.addr = (u64) (unsigned long) kbuf;
2911 req->flags |= REQ_F_BUFFER_SELECTED;
2912 return u64_to_user_ptr(kbuf->addr);
2913}
2914
2915#ifdef CONFIG_COMPAT
2916static ssize_t io_compat_import(struct io_kiocb *req, struct iovec *iov,
2917 bool needs_lock)
2918{
2919 struct compat_iovec __user *uiov;
2920 compat_ssize_t clen;
2921 void __user *buf;
2922 ssize_t len;
2923
2924 uiov = u64_to_user_ptr(req->rw.addr);
2925 if (!access_ok(uiov, sizeof(*uiov)))
2926 return -EFAULT;
2927 if (__get_user(clen, &uiov->iov_len))
2928 return -EFAULT;
2929 if (clen < 0)
2930 return -EINVAL;
2931
2932 len = clen;
2933 buf = io_rw_buffer_select(req, &len, needs_lock);
2934 if (IS_ERR(buf))
2935 return PTR_ERR(buf);
2936 iov[0].iov_base = buf;
2937 iov[0].iov_len = (compat_size_t) len;
2938 return 0;
2939}
2940#endif
2941
2942static ssize_t __io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov,
2943 bool needs_lock)
2944{
2945 struct iovec __user *uiov = u64_to_user_ptr(req->rw.addr);
2946 void __user *buf;
2947 ssize_t len;
2948
2949 if (copy_from_user(iov, uiov, sizeof(*uiov)))
2950 return -EFAULT;
2951
2952 len = iov[0].iov_len;
2953 if (len < 0)
2954 return -EINVAL;
2955 buf = io_rw_buffer_select(req, &len, needs_lock);
2956 if (IS_ERR(buf))
2957 return PTR_ERR(buf);
2958 iov[0].iov_base = buf;
2959 iov[0].iov_len = len;
2960 return 0;
2961}
2962
2963static ssize_t io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov,
2964 bool needs_lock)
2965{
Jens Axboedddb3e22020-06-04 11:27:01 -06002966 if (req->flags & REQ_F_BUFFER_SELECTED) {
2967 struct io_buffer *kbuf;
2968
2969 kbuf = (struct io_buffer *) (unsigned long) req->rw.addr;
2970 iov[0].iov_base = u64_to_user_ptr(kbuf->addr);
2971 iov[0].iov_len = kbuf->len;
Jens Axboe4d954c22020-02-27 07:31:19 -07002972 return 0;
Jens Axboedddb3e22020-06-04 11:27:01 -06002973 }
Pavel Begunkovdd201662020-12-19 03:15:43 +00002974 if (req->rw.len != 1)
Jens Axboe4d954c22020-02-27 07:31:19 -07002975 return -EINVAL;
2976
2977#ifdef CONFIG_COMPAT
2978 if (req->ctx->compat)
2979 return io_compat_import(req, iov, needs_lock);
2980#endif
2981
2982 return __io_iov_buffer_select(req, iov, needs_lock);
2983}
2984
Pavel Begunkov847595d2021-02-04 13:52:06 +00002985static int io_import_iovec(int rw, struct io_kiocb *req, struct iovec **iovec,
2986 struct iov_iter *iter, bool needs_lock)
Jens Axboe2b188cc2019-01-07 10:46:33 -07002987{
Jens Axboe9adbd452019-12-20 08:45:55 -07002988 void __user *buf = u64_to_user_ptr(req->rw.addr);
2989 size_t sqe_len = req->rw.len;
Pavel Begunkov847595d2021-02-04 13:52:06 +00002990 u8 opcode = req->opcode;
Jens Axboe4d954c22020-02-27 07:31:19 -07002991 ssize_t ret;
Jens Axboeedafcce2019-01-09 09:16:05 -07002992
Pavel Begunkov7d009162019-11-25 23:14:40 +03002993 if (opcode == IORING_OP_READ_FIXED || opcode == IORING_OP_WRITE_FIXED) {
Jens Axboeedafcce2019-01-09 09:16:05 -07002994 *iovec = NULL;
Jens Axboe9adbd452019-12-20 08:45:55 -07002995 return io_import_fixed(req, rw, iter);
Jens Axboeedafcce2019-01-09 09:16:05 -07002996 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07002997
Jens Axboebcda7ba2020-02-23 16:42:51 -07002998 /* buffer index only valid with fixed read/write, or buffer select */
Bijan Mottahedeh4f4eeba2020-05-19 14:52:49 -07002999 if (req->buf_index && !(req->flags & REQ_F_BUFFER_SELECT))
Jens Axboe9adbd452019-12-20 08:45:55 -07003000 return -EINVAL;
3001
Jens Axboe3a6820f2019-12-22 15:19:35 -07003002 if (opcode == IORING_OP_READ || opcode == IORING_OP_WRITE) {
Jens Axboebcda7ba2020-02-23 16:42:51 -07003003 if (req->flags & REQ_F_BUFFER_SELECT) {
Jens Axboe4d954c22020-02-27 07:31:19 -07003004 buf = io_rw_buffer_select(req, &sqe_len, needs_lock);
Pavel Begunkov867a23e2020-08-20 11:34:39 +03003005 if (IS_ERR(buf))
Jens Axboe4d954c22020-02-27 07:31:19 -07003006 return PTR_ERR(buf);
Jens Axboe3f9d6442020-03-11 12:27:04 -06003007 req->rw.len = sqe_len;
Jens Axboebcda7ba2020-02-23 16:42:51 -07003008 }
3009
Jens Axboe3a6820f2019-12-22 15:19:35 -07003010 ret = import_single_range(rw, buf, sqe_len, *iovec, iter);
3011 *iovec = NULL;
David Laight10fc72e2020-11-07 13:16:25 +00003012 return ret;
Jens Axboe3a6820f2019-12-22 15:19:35 -07003013 }
3014
Jens Axboe4d954c22020-02-27 07:31:19 -07003015 if (req->flags & REQ_F_BUFFER_SELECT) {
3016 ret = io_iov_buffer_select(req, *iovec, needs_lock);
Pavel Begunkov847595d2021-02-04 13:52:06 +00003017 if (!ret)
3018 iov_iter_init(iter, rw, *iovec, 1, (*iovec)->iov_len);
Jens Axboe4d954c22020-02-27 07:31:19 -07003019 *iovec = NULL;
3020 return ret;
3021 }
3022
Christoph Hellwig89cd35c2020-09-25 06:51:41 +02003023 return __import_iovec(rw, buf, sqe_len, UIO_FASTIOV, iovec, iter,
3024 req->ctx->compat);
Jens Axboe2b188cc2019-01-07 10:46:33 -07003025}
3026
Jens Axboe0fef9482020-08-26 10:36:20 -06003027static inline loff_t *io_kiocb_ppos(struct kiocb *kiocb)
3028{
Pavel Begunkov5b09e372020-09-30 22:57:15 +03003029 return (kiocb->ki_filp->f_mode & FMODE_STREAM) ? NULL : &kiocb->ki_pos;
Jens Axboe0fef9482020-08-26 10:36:20 -06003030}
3031
Jens Axboe32960612019-09-23 11:05:34 -06003032/*
3033 * For files that don't have ->read_iter() and ->write_iter(), handle them
3034 * by looping over ->read() or ->write() manually.
3035 */
Jens Axboe4017eb92020-10-22 14:14:12 -06003036static ssize_t loop_rw_iter(int rw, struct io_kiocb *req, struct iov_iter *iter)
Jens Axboe32960612019-09-23 11:05:34 -06003037{
Jens Axboe4017eb92020-10-22 14:14:12 -06003038 struct kiocb *kiocb = &req->rw.kiocb;
3039 struct file *file = req->file;
Jens Axboe32960612019-09-23 11:05:34 -06003040 ssize_t ret = 0;
3041
3042 /*
3043 * Don't support polled IO through this interface, and we can't
3044 * support non-blocking either. For the latter, this just causes
3045 * the kiocb to be handled from an async context.
3046 */
3047 if (kiocb->ki_flags & IOCB_HIPRI)
3048 return -EOPNOTSUPP;
3049 if (kiocb->ki_flags & IOCB_NOWAIT)
3050 return -EAGAIN;
3051
3052 while (iov_iter_count(iter)) {
Pavel Begunkov311ae9e2019-11-24 11:58:24 +03003053 struct iovec iovec;
Jens Axboe32960612019-09-23 11:05:34 -06003054 ssize_t nr;
3055
Pavel Begunkov311ae9e2019-11-24 11:58:24 +03003056 if (!iov_iter_is_bvec(iter)) {
3057 iovec = iov_iter_iovec(iter);
3058 } else {
Jens Axboe4017eb92020-10-22 14:14:12 -06003059 iovec.iov_base = u64_to_user_ptr(req->rw.addr);
3060 iovec.iov_len = req->rw.len;
Pavel Begunkov311ae9e2019-11-24 11:58:24 +03003061 }
3062
Jens Axboe32960612019-09-23 11:05:34 -06003063 if (rw == READ) {
3064 nr = file->f_op->read(file, iovec.iov_base,
Jens Axboe0fef9482020-08-26 10:36:20 -06003065 iovec.iov_len, io_kiocb_ppos(kiocb));
Jens Axboe32960612019-09-23 11:05:34 -06003066 } else {
3067 nr = file->f_op->write(file, iovec.iov_base,
Jens Axboe0fef9482020-08-26 10:36:20 -06003068 iovec.iov_len, io_kiocb_ppos(kiocb));
Jens Axboe32960612019-09-23 11:05:34 -06003069 }
3070
3071 if (nr < 0) {
3072 if (!ret)
3073 ret = nr;
3074 break;
3075 }
3076 ret += nr;
3077 if (nr != iovec.iov_len)
3078 break;
Jens Axboe4017eb92020-10-22 14:14:12 -06003079 req->rw.len -= nr;
3080 req->rw.addr += nr;
Jens Axboe32960612019-09-23 11:05:34 -06003081 iov_iter_advance(iter, nr);
3082 }
3083
3084 return ret;
3085}
3086
Jens Axboeff6165b2020-08-13 09:47:43 -06003087static void io_req_map_rw(struct io_kiocb *req, const struct iovec *iovec,
3088 const struct iovec *fast_iov, struct iov_iter *iter)
Jens Axboef67676d2019-12-02 11:03:47 -07003089{
Jens Axboee8c2bc12020-08-15 18:44:09 -07003090 struct io_async_rw *rw = req->async_data;
Pavel Begunkovb64e3442020-07-13 22:59:18 +03003091
Jens Axboeff6165b2020-08-13 09:47:43 -06003092 memcpy(&rw->iter, iter, sizeof(*iter));
Pavel Begunkovafb87652020-09-06 00:45:46 +03003093 rw->free_iovec = iovec;
Jens Axboe227c0c92020-08-13 11:51:40 -06003094 rw->bytes_done = 0;
Jens Axboeff6165b2020-08-13 09:47:43 -06003095 /* can only be fixed buffers, no need to do anything */
Pavel Begunkov9c3a2052020-11-23 23:20:27 +00003096 if (iov_iter_is_bvec(iter))
Jens Axboeff6165b2020-08-13 09:47:43 -06003097 return;
Pavel Begunkovb64e3442020-07-13 22:59:18 +03003098 if (!iovec) {
Jens Axboeff6165b2020-08-13 09:47:43 -06003099 unsigned iov_off = 0;
3100
3101 rw->iter.iov = rw->fast_iov;
3102 if (iter->iov != fast_iov) {
3103 iov_off = iter->iov - fast_iov;
3104 rw->iter.iov += iov_off;
3105 }
3106 if (rw->fast_iov != fast_iov)
3107 memcpy(rw->fast_iov + iov_off, fast_iov + iov_off,
Xiaoguang Wang45097da2020-04-08 22:29:58 +08003108 sizeof(struct iovec) * iter->nr_segs);
Pavel Begunkov99bc4c32020-02-07 22:04:45 +03003109 } else {
3110 req->flags |= REQ_F_NEED_CLEANUP;
Jens Axboef67676d2019-12-02 11:03:47 -07003111 }
3112}
3113
Pavel Begunkov6cb78682021-02-28 22:35:17 +00003114static inline int io_alloc_async_data(struct io_kiocb *req)
Xiaoguang Wang3d9932a2020-03-27 15:36:52 +08003115{
Jens Axboee8c2bc12020-08-15 18:44:09 -07003116 WARN_ON_ONCE(!io_op_defs[req->opcode].async_size);
3117 req->async_data = kmalloc(io_op_defs[req->opcode].async_size, GFP_KERNEL);
3118 return req->async_data == NULL;
Xiaoguang Wang3d9932a2020-03-27 15:36:52 +08003119}
3120
Jens Axboeff6165b2020-08-13 09:47:43 -06003121static int io_setup_async_rw(struct io_kiocb *req, const struct iovec *iovec,
3122 const struct iovec *fast_iov,
Jens Axboe227c0c92020-08-13 11:51:40 -06003123 struct iov_iter *iter, bool force)
Jens Axboeb7bb4f72019-12-15 22:13:43 -07003124{
Pavel Begunkov26f05052021-02-28 22:35:18 +00003125 if (!force && !io_op_defs[req->opcode].needs_async_setup)
Jens Axboe74566df2020-01-13 19:23:24 -07003126 return 0;
Jens Axboee8c2bc12020-08-15 18:44:09 -07003127 if (!req->async_data) {
Pavel Begunkov6cb78682021-02-28 22:35:17 +00003128 if (io_alloc_async_data(req)) {
Pavel Begunkov6bf985d2021-02-04 13:52:01 +00003129 kfree(iovec);
Jens Axboe5d204bc2020-01-31 12:06:52 -07003130 return -ENOMEM;
Pavel Begunkov6bf985d2021-02-04 13:52:01 +00003131 }
Jens Axboeb7bb4f72019-12-15 22:13:43 -07003132
Jens Axboeff6165b2020-08-13 09:47:43 -06003133 io_req_map_rw(req, iovec, fast_iov, iter);
Jens Axboe5d204bc2020-01-31 12:06:52 -07003134 }
Jens Axboeb7bb4f72019-12-15 22:13:43 -07003135 return 0;
Jens Axboef67676d2019-12-02 11:03:47 -07003136}
3137
Pavel Begunkov73debe62020-09-30 22:57:54 +03003138static inline int io_rw_prep_async(struct io_kiocb *req, int rw)
Pavel Begunkovc3e330a2020-07-13 22:59:19 +03003139{
Jens Axboee8c2bc12020-08-15 18:44:09 -07003140 struct io_async_rw *iorw = req->async_data;
Pavel Begunkovf4bff102020-09-06 00:45:45 +03003141 struct iovec *iov = iorw->fast_iov;
Pavel Begunkov847595d2021-02-04 13:52:06 +00003142 int ret;
Pavel Begunkovc3e330a2020-07-13 22:59:19 +03003143
Pavel Begunkov2846c482020-11-07 13:16:27 +00003144 ret = io_import_iovec(rw, req, &iov, &iorw->iter, false);
Pavel Begunkovc3e330a2020-07-13 22:59:19 +03003145 if (unlikely(ret < 0))
3146 return ret;
3147
Pavel Begunkovab0b1962020-09-06 00:45:47 +03003148 iorw->bytes_done = 0;
3149 iorw->free_iovec = iov;
3150 if (iov)
3151 req->flags |= REQ_F_NEED_CLEANUP;
Pavel Begunkovc3e330a2020-07-13 22:59:19 +03003152 return 0;
3153}
3154
Pavel Begunkov73debe62020-09-30 22:57:54 +03003155static int io_read_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboef67676d2019-12-02 11:03:47 -07003156{
Jens Axboe3529d8c2019-12-19 18:24:38 -07003157 if (unlikely(!(req->file->f_mode & FMODE_READ)))
3158 return -EBADF;
Pavel Begunkov93642ef2021-02-18 18:29:44 +00003159 return io_prep_rw(req, sqe);
Jens Axboef67676d2019-12-02 11:03:47 -07003160}
3161
Jens Axboec1dd91d2020-08-03 16:43:59 -06003162/*
3163 * This is our waitqueue callback handler, registered through lock_page_async()
3164 * when we initially tried to do the IO with the iocb armed our waitqueue.
3165 * This gets called when the page is unlocked, and we generally expect that to
3166 * happen when the page IO is completed and the page is now uptodate. This will
3167 * queue a task_work based retry of the operation, attempting to copy the data
3168 * again. If the latter fails because the page was NOT uptodate, then we will
3169 * do a thread based blocking retry of the operation. That's the unexpected
3170 * slow path.
3171 */
Jens Axboebcf5a062020-05-22 09:24:42 -06003172static int io_async_buf_func(struct wait_queue_entry *wait, unsigned mode,
3173 int sync, void *arg)
3174{
3175 struct wait_page_queue *wpq;
3176 struct io_kiocb *req = wait->private;
Jens Axboebcf5a062020-05-22 09:24:42 -06003177 struct wait_page_key *key = arg;
Jens Axboebcf5a062020-05-22 09:24:42 -06003178
3179 wpq = container_of(wait, struct wait_page_queue, wait);
3180
Linus Torvaldscdc8fcb2020-08-03 13:01:22 -07003181 if (!wake_page_match(wpq, key))
3182 return 0;
3183
Hao Xuc8d317a2020-09-29 20:00:45 +08003184 req->rw.kiocb.ki_flags &= ~IOCB_WAITQ;
Jens Axboebcf5a062020-05-22 09:24:42 -06003185 list_del_init(&wait->entry);
3186
Jens Axboebcf5a062020-05-22 09:24:42 -06003187 /* submit ref gets dropped, acquire a new one */
Jens Axboede9b4cc2021-02-24 13:28:27 -07003188 req_ref_get(req);
Pavel Begunkov921b9052021-02-12 03:23:53 +00003189 io_req_task_queue(req);
Jens Axboebcf5a062020-05-22 09:24:42 -06003190 return 1;
3191}
3192
Jens Axboec1dd91d2020-08-03 16:43:59 -06003193/*
3194 * This controls whether a given IO request should be armed for async page
3195 * based retry. If we return false here, the request is handed to the async
3196 * worker threads for retry. If we're doing buffered reads on a regular file,
3197 * we prepare a private wait_page_queue entry and retry the operation. This
3198 * will either succeed because the page is now uptodate and unlocked, or it
3199 * will register a callback when the page is unlocked at IO completion. Through
3200 * that callback, io_uring uses task_work to setup a retry of the operation.
3201 * That retry will attempt the buffered read again. The retry will generally
3202 * succeed, or in rare cases where it fails, we then fall back to using the
3203 * async worker threads for a blocking retry.
3204 */
Jens Axboe227c0c92020-08-13 11:51:40 -06003205static bool io_rw_should_retry(struct io_kiocb *req)
Jens Axboebcf5a062020-05-22 09:24:42 -06003206{
Jens Axboee8c2bc12020-08-15 18:44:09 -07003207 struct io_async_rw *rw = req->async_data;
3208 struct wait_page_queue *wait = &rw->wpq;
Jens Axboebcf5a062020-05-22 09:24:42 -06003209 struct kiocb *kiocb = &req->rw.kiocb;
Jens Axboebcf5a062020-05-22 09:24:42 -06003210
3211 /* never retry for NOWAIT, we just complete with -EAGAIN */
3212 if (req->flags & REQ_F_NOWAIT)
3213 return false;
3214
Jens Axboe227c0c92020-08-13 11:51:40 -06003215 /* Only for buffered IO */
Jens Axboe3b2a4432020-08-16 10:58:43 -07003216 if (kiocb->ki_flags & (IOCB_DIRECT | IOCB_HIPRI))
Jens Axboebcf5a062020-05-22 09:24:42 -06003217 return false;
Jens Axboe3b2a4432020-08-16 10:58:43 -07003218
Jens Axboebcf5a062020-05-22 09:24:42 -06003219 /*
3220 * just use poll if we can, and don't attempt if the fs doesn't
3221 * support callback based unlocks
3222 */
3223 if (file_can_poll(req->file) || !(req->file->f_mode & FMODE_BUF_RASYNC))
3224 return false;
3225
Jens Axboe3b2a4432020-08-16 10:58:43 -07003226 wait->wait.func = io_async_buf_func;
3227 wait->wait.private = req;
3228 wait->wait.flags = 0;
3229 INIT_LIST_HEAD(&wait->wait.entry);
3230 kiocb->ki_flags |= IOCB_WAITQ;
Hao Xuc8d317a2020-09-29 20:00:45 +08003231 kiocb->ki_flags &= ~IOCB_NOWAIT;
Jens Axboe3b2a4432020-08-16 10:58:43 -07003232 kiocb->ki_waitq = wait;
Jens Axboe3b2a4432020-08-16 10:58:43 -07003233 return true;
Jens Axboebcf5a062020-05-22 09:24:42 -06003234}
3235
3236static int io_iter_do_read(struct io_kiocb *req, struct iov_iter *iter)
3237{
3238 if (req->file->f_op->read_iter)
3239 return call_read_iter(req->file, &req->rw.kiocb, iter);
Guoyu Huang2dd21112020-08-05 03:53:50 -07003240 else if (req->file->f_op->read)
Jens Axboe4017eb92020-10-22 14:14:12 -06003241 return loop_rw_iter(READ, req, iter);
Guoyu Huang2dd21112020-08-05 03:53:50 -07003242 else
3243 return -EINVAL;
Jens Axboebcf5a062020-05-22 09:24:42 -06003244}
3245
Pavel Begunkov889fca72021-02-10 00:03:09 +00003246static int io_read(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe2b188cc2019-01-07 10:46:33 -07003247{
3248 struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
Jens Axboe9adbd452019-12-20 08:45:55 -07003249 struct kiocb *kiocb = &req->rw.kiocb;
Jens Axboeff6165b2020-08-13 09:47:43 -06003250 struct iov_iter __iter, *iter = &__iter;
Jens Axboee8c2bc12020-08-15 18:44:09 -07003251 struct io_async_rw *rw = req->async_data;
Jens Axboe227c0c92020-08-13 11:51:40 -06003252 ssize_t io_size, ret, ret2;
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003253 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003254
Pavel Begunkov2846c482020-11-07 13:16:27 +00003255 if (rw) {
Jens Axboee8c2bc12020-08-15 18:44:09 -07003256 iter = &rw->iter;
Pavel Begunkov2846c482020-11-07 13:16:27 +00003257 iovec = NULL;
3258 } else {
3259 ret = io_import_iovec(READ, req, &iovec, iter, !force_nonblock);
3260 if (ret < 0)
3261 return ret;
3262 }
Pavel Begunkov632546c2020-11-07 13:16:26 +00003263 io_size = iov_iter_count(iter);
Pavel Begunkovfa15baf2020-08-01 13:50:02 +03003264 req->result = io_size;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003265
Jens Axboefd6c2e42019-12-18 12:19:41 -07003266 /* Ensure we clear previously set non-block flag */
3267 if (!force_nonblock)
Jens Axboe29de5f62020-02-20 09:56:08 -07003268 kiocb->ki_flags &= ~IOCB_NOWAIT;
Pavel Begunkova88fc402020-09-30 22:57:53 +03003269 else
3270 kiocb->ki_flags |= IOCB_NOWAIT;
3271
Pavel Begunkov24c74672020-06-21 13:09:51 +03003272 /* If the file doesn't support async, just async punt */
Jens Axboe7b29f922021-03-12 08:30:14 -07003273 if (force_nonblock && !io_file_supports_async(req, READ)) {
Pavel Begunkov6713e7a2021-02-04 13:51:59 +00003274 ret = io_setup_async_rw(req, iovec, inline_vecs, iter, true);
Pavel Begunkov6bf985d2021-02-04 13:52:01 +00003275 return ret ?: -EAGAIN;
Pavel Begunkov6713e7a2021-02-04 13:51:59 +00003276 }
Jens Axboe9e645e112019-05-10 16:07:28 -06003277
Pavel Begunkov632546c2020-11-07 13:16:26 +00003278 ret = rw_verify_area(READ, req->file, io_kiocb_ppos(kiocb), io_size);
Pavel Begunkov5ea5dd42021-02-04 13:52:03 +00003279 if (unlikely(ret)) {
3280 kfree(iovec);
3281 return ret;
3282 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07003283
Jens Axboe227c0c92020-08-13 11:51:40 -06003284 ret = io_iter_do_read(req, iter);
Jens Axboe32960612019-09-23 11:05:34 -06003285
Jens Axboe230d50d2021-04-01 20:41:15 -06003286 if (ret == -EAGAIN || (req->flags & REQ_F_REISSUE)) {
Pavel Begunkov6ad7f232021-04-08 01:54:39 +01003287 req->flags &= ~REQ_F_REISSUE;
Jens Axboeeefdf302020-08-27 16:40:19 -06003288 /* IOPOLL retry should happen for io-wq threads */
3289 if (!force_nonblock && !(req->ctx->flags & IORING_SETUP_IOPOLL))
Jens Axboef91daf52020-08-15 15:58:42 -07003290 goto done;
Pavel Begunkov75c668c2021-02-04 13:52:05 +00003291 /* no retry on NONBLOCK nor RWF_NOWAIT */
3292 if (req->flags & REQ_F_NOWAIT)
Jens Axboe355afae2020-09-02 09:30:31 -06003293 goto done;
Jens Axboe84216312020-08-24 11:45:26 -06003294 /* some cases will consume bytes even on error returns */
Pavel Begunkov632546c2020-11-07 13:16:26 +00003295 iov_iter_revert(iter, io_size - iov_iter_count(iter));
Jens Axboef38c7e32020-09-25 15:23:43 -06003296 ret = 0;
Jens Axboe230d50d2021-04-01 20:41:15 -06003297 } else if (ret == -EIOCBQUEUED) {
3298 goto out_free;
Pavel Begunkov7335e3b2021-02-04 13:52:02 +00003299 } else if (ret <= 0 || ret == io_size || !force_nonblock ||
Pavel Begunkov75c668c2021-02-04 13:52:05 +00003300 (req->flags & REQ_F_NOWAIT) || !(req->flags & REQ_F_ISREG)) {
Pavel Begunkov7335e3b2021-02-04 13:52:02 +00003301 /* read all, failed, already did sync or don't want to retry */
Jens Axboe00d23d52020-08-25 12:59:22 -06003302 goto done;
Jens Axboe227c0c92020-08-13 11:51:40 -06003303 }
3304
Jens Axboe227c0c92020-08-13 11:51:40 -06003305 ret2 = io_setup_async_rw(req, iovec, inline_vecs, iter, true);
Pavel Begunkov6bf985d2021-02-04 13:52:01 +00003306 if (ret2)
3307 return ret2;
3308
Pavel Begunkovfe1cdd52021-02-17 21:02:36 +00003309 iovec = NULL;
Jens Axboee8c2bc12020-08-15 18:44:09 -07003310 rw = req->async_data;
Jens Axboe227c0c92020-08-13 11:51:40 -06003311 /* now use our persistent iterator, if we aren't already */
Jens Axboee8c2bc12020-08-15 18:44:09 -07003312 iter = &rw->iter;
Jens Axboe227c0c92020-08-13 11:51:40 -06003313
Pavel Begunkovb23df912021-02-04 13:52:04 +00003314 do {
3315 io_size -= ret;
3316 rw->bytes_done += ret;
3317 /* if we can retry, do so with the callbacks armed */
3318 if (!io_rw_should_retry(req)) {
3319 kiocb->ki_flags &= ~IOCB_WAITQ;
3320 return -EAGAIN;
3321 }
3322
3323 /*
3324 * Now retry read with the IOCB_WAITQ parts set in the iocb. If
3325 * we get -EIOCBQUEUED, then we'll get a notification when the
3326 * desired page gets unlocked. We can also get a partial read
3327 * here, and if we do, then just retry at the new offset.
3328 */
3329 ret = io_iter_do_read(req, iter);
3330 if (ret == -EIOCBQUEUED)
3331 return 0;
Jens Axboe227c0c92020-08-13 11:51:40 -06003332 /* we got some bytes, but not all. retry. */
Jens Axboeb5b0ecb2021-03-04 21:02:58 -07003333 kiocb->ki_flags &= ~IOCB_WAITQ;
Pavel Begunkovb23df912021-02-04 13:52:04 +00003334 } while (ret > 0 && ret < io_size);
Jens Axboe227c0c92020-08-13 11:51:40 -06003335done:
Pavel Begunkov889fca72021-02-10 00:03:09 +00003336 kiocb_done(kiocb, ret, issue_flags);
Pavel Begunkovfe1cdd52021-02-17 21:02:36 +00003337out_free:
3338 /* it's faster to check here then delegate to kfree */
3339 if (iovec)
3340 kfree(iovec);
Pavel Begunkov5ea5dd42021-02-04 13:52:03 +00003341 return 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003342}
3343
Pavel Begunkov73debe62020-09-30 22:57:54 +03003344static int io_write_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboef67676d2019-12-02 11:03:47 -07003345{
Jens Axboe3529d8c2019-12-19 18:24:38 -07003346 if (unlikely(!(req->file->f_mode & FMODE_WRITE)))
3347 return -EBADF;
Pavel Begunkov93642ef2021-02-18 18:29:44 +00003348 return io_prep_rw(req, sqe);
Jens Axboef67676d2019-12-02 11:03:47 -07003349}
3350
Pavel Begunkov889fca72021-02-10 00:03:09 +00003351static int io_write(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe2b188cc2019-01-07 10:46:33 -07003352{
3353 struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
Jens Axboe9adbd452019-12-20 08:45:55 -07003354 struct kiocb *kiocb = &req->rw.kiocb;
Jens Axboeff6165b2020-08-13 09:47:43 -06003355 struct iov_iter __iter, *iter = &__iter;
Jens Axboee8c2bc12020-08-15 18:44:09 -07003356 struct io_async_rw *rw = req->async_data;
Pavel Begunkovfa15baf2020-08-01 13:50:02 +03003357 ssize_t ret, ret2, io_size;
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003358 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003359
Pavel Begunkov2846c482020-11-07 13:16:27 +00003360 if (rw) {
Jens Axboee8c2bc12020-08-15 18:44:09 -07003361 iter = &rw->iter;
Pavel Begunkov2846c482020-11-07 13:16:27 +00003362 iovec = NULL;
3363 } else {
3364 ret = io_import_iovec(WRITE, req, &iovec, iter, !force_nonblock);
3365 if (ret < 0)
3366 return ret;
3367 }
Pavel Begunkov632546c2020-11-07 13:16:26 +00003368 io_size = iov_iter_count(iter);
Pavel Begunkovfa15baf2020-08-01 13:50:02 +03003369 req->result = io_size;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003370
Jens Axboefd6c2e42019-12-18 12:19:41 -07003371 /* Ensure we clear previously set non-block flag */
3372 if (!force_nonblock)
Pavel Begunkova88fc402020-09-30 22:57:53 +03003373 kiocb->ki_flags &= ~IOCB_NOWAIT;
3374 else
3375 kiocb->ki_flags |= IOCB_NOWAIT;
Jens Axboefd6c2e42019-12-18 12:19:41 -07003376
Pavel Begunkov24c74672020-06-21 13:09:51 +03003377 /* If the file doesn't support async, just async punt */
Jens Axboe7b29f922021-03-12 08:30:14 -07003378 if (force_nonblock && !io_file_supports_async(req, WRITE))
Jens Axboef67676d2019-12-02 11:03:47 -07003379 goto copy_iov;
Jens Axboef67676d2019-12-02 11:03:47 -07003380
Jens Axboe10d59342019-12-09 20:16:22 -07003381 /* file path doesn't support NOWAIT for non-direct_IO */
3382 if (force_nonblock && !(kiocb->ki_flags & IOCB_DIRECT) &&
3383 (req->flags & REQ_F_ISREG))
Jens Axboef67676d2019-12-02 11:03:47 -07003384 goto copy_iov;
Jens Axboe9e645e112019-05-10 16:07:28 -06003385
Pavel Begunkov632546c2020-11-07 13:16:26 +00003386 ret = rw_verify_area(WRITE, req->file, io_kiocb_ppos(kiocb), io_size);
Pavel Begunkovfa15baf2020-08-01 13:50:02 +03003387 if (unlikely(ret))
3388 goto out_free;
Roman Penyaev9bf79332019-03-25 20:09:24 +01003389
Pavel Begunkovfa15baf2020-08-01 13:50:02 +03003390 /*
3391 * Open-code file_start_write here to grab freeze protection,
3392 * which will be released by another thread in
3393 * io_complete_rw(). Fool lockdep by telling it the lock got
3394 * released so that it doesn't complain about the held lock when
3395 * we return to userspace.
3396 */
3397 if (req->flags & REQ_F_ISREG) {
Darrick J. Wong8a3c84b2020-11-10 16:50:21 -08003398 sb_start_write(file_inode(req->file)->i_sb);
Pavel Begunkovfa15baf2020-08-01 13:50:02 +03003399 __sb_writers_release(file_inode(req->file)->i_sb,
3400 SB_FREEZE_WRITE);
3401 }
3402 kiocb->ki_flags |= IOCB_WRITE;
Roman Penyaev9bf79332019-03-25 20:09:24 +01003403
Pavel Begunkovfa15baf2020-08-01 13:50:02 +03003404 if (req->file->f_op->write_iter)
Jens Axboeff6165b2020-08-13 09:47:43 -06003405 ret2 = call_write_iter(req->file, kiocb, iter);
Guoyu Huang2dd21112020-08-05 03:53:50 -07003406 else if (req->file->f_op->write)
Jens Axboe4017eb92020-10-22 14:14:12 -06003407 ret2 = loop_rw_iter(WRITE, req, iter);
Guoyu Huang2dd21112020-08-05 03:53:50 -07003408 else
3409 ret2 = -EINVAL;
Jens Axboe4ed734b2020-03-20 11:23:41 -06003410
Pavel Begunkov6ad7f232021-04-08 01:54:39 +01003411 if (req->flags & REQ_F_REISSUE) {
3412 req->flags &= ~REQ_F_REISSUE;
Jens Axboe230d50d2021-04-01 20:41:15 -06003413 ret2 = -EAGAIN;
Pavel Begunkov6ad7f232021-04-08 01:54:39 +01003414 }
Jens Axboe230d50d2021-04-01 20:41:15 -06003415
Pavel Begunkovfa15baf2020-08-01 13:50:02 +03003416 /*
3417 * Raw bdev writes will return -EOPNOTSUPP for IOCB_NOWAIT. Just
3418 * retry them without IOCB_NOWAIT.
3419 */
3420 if (ret2 == -EOPNOTSUPP && (kiocb->ki_flags & IOCB_NOWAIT))
3421 ret2 = -EAGAIN;
Pavel Begunkov75c668c2021-02-04 13:52:05 +00003422 /* no retry on NONBLOCK nor RWF_NOWAIT */
3423 if (ret2 == -EAGAIN && (req->flags & REQ_F_NOWAIT))
Jens Axboe355afae2020-09-02 09:30:31 -06003424 goto done;
Pavel Begunkovfa15baf2020-08-01 13:50:02 +03003425 if (!force_nonblock || ret2 != -EAGAIN) {
Jens Axboeeefdf302020-08-27 16:40:19 -06003426 /* IOPOLL retry should happen for io-wq threads */
3427 if ((req->ctx->flags & IORING_SETUP_IOPOLL) && ret2 == -EAGAIN)
3428 goto copy_iov;
Jens Axboe355afae2020-09-02 09:30:31 -06003429done:
Pavel Begunkov889fca72021-02-10 00:03:09 +00003430 kiocb_done(kiocb, ret2, issue_flags);
Pavel Begunkovfa15baf2020-08-01 13:50:02 +03003431 } else {
Jens Axboef67676d2019-12-02 11:03:47 -07003432copy_iov:
Jens Axboe84216312020-08-24 11:45:26 -06003433 /* some cases will consume bytes even on error returns */
Pavel Begunkov632546c2020-11-07 13:16:26 +00003434 iov_iter_revert(iter, io_size - iov_iter_count(iter));
Jens Axboe227c0c92020-08-13 11:51:40 -06003435 ret = io_setup_async_rw(req, iovec, inline_vecs, iter, false);
Pavel Begunkov6bf985d2021-02-04 13:52:01 +00003436 return ret ?: -EAGAIN;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003437 }
Jens Axboe31b51512019-01-18 22:56:34 -07003438out_free:
Pavel Begunkovf261c162020-08-20 11:34:10 +03003439 /* it's reportedly faster than delegating the null check to kfree() */
Pavel Begunkov252917c2020-07-13 22:59:20 +03003440 if (iovec)
Xiaoguang Wang6f2cc162020-06-18 15:01:56 +08003441 kfree(iovec);
Jens Axboe2b188cc2019-01-07 10:46:33 -07003442 return ret;
3443}
3444
Jens Axboe80a261f2020-09-28 14:23:58 -06003445static int io_renameat_prep(struct io_kiocb *req,
3446 const struct io_uring_sqe *sqe)
3447{
3448 struct io_rename *ren = &req->rename;
3449 const char __user *oldf, *newf;
3450
3451 if (unlikely(req->flags & REQ_F_FIXED_FILE))
3452 return -EBADF;
3453
3454 ren->old_dfd = READ_ONCE(sqe->fd);
3455 oldf = u64_to_user_ptr(READ_ONCE(sqe->addr));
3456 newf = u64_to_user_ptr(READ_ONCE(sqe->addr2));
3457 ren->new_dfd = READ_ONCE(sqe->len);
3458 ren->flags = READ_ONCE(sqe->rename_flags);
3459
3460 ren->oldpath = getname(oldf);
3461 if (IS_ERR(ren->oldpath))
3462 return PTR_ERR(ren->oldpath);
3463
3464 ren->newpath = getname(newf);
3465 if (IS_ERR(ren->newpath)) {
3466 putname(ren->oldpath);
3467 return PTR_ERR(ren->newpath);
3468 }
3469
3470 req->flags |= REQ_F_NEED_CLEANUP;
3471 return 0;
3472}
3473
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003474static int io_renameat(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe80a261f2020-09-28 14:23:58 -06003475{
3476 struct io_rename *ren = &req->rename;
3477 int ret;
3478
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003479 if (issue_flags & IO_URING_F_NONBLOCK)
Jens Axboe80a261f2020-09-28 14:23:58 -06003480 return -EAGAIN;
3481
3482 ret = do_renameat2(ren->old_dfd, ren->oldpath, ren->new_dfd,
3483 ren->newpath, ren->flags);
3484
3485 req->flags &= ~REQ_F_NEED_CLEANUP;
3486 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01003487 req_set_fail(req);
Jens Axboe80a261f2020-09-28 14:23:58 -06003488 io_req_complete(req, ret);
3489 return 0;
3490}
3491
Jens Axboe14a11432020-09-28 14:27:37 -06003492static int io_unlinkat_prep(struct io_kiocb *req,
3493 const struct io_uring_sqe *sqe)
3494{
3495 struct io_unlink *un = &req->unlink;
3496 const char __user *fname;
3497
3498 if (unlikely(req->flags & REQ_F_FIXED_FILE))
3499 return -EBADF;
3500
3501 un->dfd = READ_ONCE(sqe->fd);
3502
3503 un->flags = READ_ONCE(sqe->unlink_flags);
3504 if (un->flags & ~AT_REMOVEDIR)
3505 return -EINVAL;
3506
3507 fname = u64_to_user_ptr(READ_ONCE(sqe->addr));
3508 un->filename = getname(fname);
3509 if (IS_ERR(un->filename))
3510 return PTR_ERR(un->filename);
3511
3512 req->flags |= REQ_F_NEED_CLEANUP;
3513 return 0;
3514}
3515
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003516static int io_unlinkat(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe14a11432020-09-28 14:27:37 -06003517{
3518 struct io_unlink *un = &req->unlink;
3519 int ret;
3520
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003521 if (issue_flags & IO_URING_F_NONBLOCK)
Jens Axboe14a11432020-09-28 14:27:37 -06003522 return -EAGAIN;
3523
3524 if (un->flags & AT_REMOVEDIR)
3525 ret = do_rmdir(un->dfd, un->filename);
3526 else
3527 ret = do_unlinkat(un->dfd, un->filename);
3528
3529 req->flags &= ~REQ_F_NEED_CLEANUP;
3530 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01003531 req_set_fail(req);
Jens Axboe14a11432020-09-28 14:27:37 -06003532 io_req_complete(req, ret);
3533 return 0;
3534}
3535
Jens Axboe36f4fa62020-09-05 11:14:22 -06003536static int io_shutdown_prep(struct io_kiocb *req,
3537 const struct io_uring_sqe *sqe)
3538{
3539#if defined(CONFIG_NET)
3540 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3541 return -EINVAL;
3542 if (sqe->ioprio || sqe->off || sqe->addr || sqe->rw_flags ||
3543 sqe->buf_index)
3544 return -EINVAL;
3545
3546 req->shutdown.how = READ_ONCE(sqe->len);
3547 return 0;
3548#else
3549 return -EOPNOTSUPP;
3550#endif
3551}
3552
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003553static int io_shutdown(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe36f4fa62020-09-05 11:14:22 -06003554{
3555#if defined(CONFIG_NET)
3556 struct socket *sock;
3557 int ret;
3558
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003559 if (issue_flags & IO_URING_F_NONBLOCK)
Jens Axboe36f4fa62020-09-05 11:14:22 -06003560 return -EAGAIN;
3561
Linus Torvalds48aba792020-12-16 12:44:05 -08003562 sock = sock_from_file(req->file);
Jens Axboe36f4fa62020-09-05 11:14:22 -06003563 if (unlikely(!sock))
Linus Torvalds48aba792020-12-16 12:44:05 -08003564 return -ENOTSOCK;
Jens Axboe36f4fa62020-09-05 11:14:22 -06003565
3566 ret = __sys_shutdown_sock(sock, req->shutdown.how);
Jens Axboea1464682020-12-14 20:57:27 -07003567 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01003568 req_set_fail(req);
Jens Axboe36f4fa62020-09-05 11:14:22 -06003569 io_req_complete(req, ret);
3570 return 0;
3571#else
3572 return -EOPNOTSUPP;
3573#endif
3574}
3575
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03003576static int __io_splice_prep(struct io_kiocb *req,
3577 const struct io_uring_sqe *sqe)
Pavel Begunkov7d67af22020-02-24 11:32:45 +03003578{
3579 struct io_splice* sp = &req->splice;
3580 unsigned int valid_flags = SPLICE_F_FD_IN_FIXED | SPLICE_F_ALL;
Pavel Begunkov7d67af22020-02-24 11:32:45 +03003581
Pavel Begunkov3232dd02020-06-03 18:03:22 +03003582 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3583 return -EINVAL;
Pavel Begunkov7d67af22020-02-24 11:32:45 +03003584
3585 sp->file_in = NULL;
Pavel Begunkov7d67af22020-02-24 11:32:45 +03003586 sp->len = READ_ONCE(sqe->len);
3587 sp->flags = READ_ONCE(sqe->splice_flags);
3588
3589 if (unlikely(sp->flags & ~valid_flags))
3590 return -EINVAL;
3591
Pavel Begunkov8371adf2020-10-10 18:34:08 +01003592 sp->file_in = io_file_get(NULL, req, READ_ONCE(sqe->splice_fd_in),
3593 (sp->flags & SPLICE_F_FD_IN_FIXED));
3594 if (!sp->file_in)
3595 return -EBADF;
Pavel Begunkov7d67af22020-02-24 11:32:45 +03003596 req->flags |= REQ_F_NEED_CLEANUP;
Pavel Begunkov7d67af22020-02-24 11:32:45 +03003597 return 0;
3598}
3599
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03003600static int io_tee_prep(struct io_kiocb *req,
3601 const struct io_uring_sqe *sqe)
3602{
3603 if (READ_ONCE(sqe->splice_off_in) || READ_ONCE(sqe->off))
3604 return -EINVAL;
3605 return __io_splice_prep(req, sqe);
3606}
3607
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003608static int io_tee(struct io_kiocb *req, unsigned int issue_flags)
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03003609{
3610 struct io_splice *sp = &req->splice;
3611 struct file *in = sp->file_in;
3612 struct file *out = sp->file_out;
3613 unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED;
3614 long ret = 0;
3615
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003616 if (issue_flags & IO_URING_F_NONBLOCK)
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03003617 return -EAGAIN;
3618 if (sp->len)
3619 ret = do_tee(in, out, sp->len, flags);
3620
Pavel Begunkove1d767f2021-03-19 17:22:43 +00003621 if (!(sp->flags & SPLICE_F_FD_IN_FIXED))
3622 io_put_file(in);
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03003623 req->flags &= ~REQ_F_NEED_CLEANUP;
3624
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03003625 if (ret != sp->len)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01003626 req_set_fail(req);
Jens Axboee1e16092020-06-22 09:17:17 -06003627 io_req_complete(req, ret);
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03003628 return 0;
3629}
3630
3631static int io_splice_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
3632{
3633 struct io_splice* sp = &req->splice;
3634
3635 sp->off_in = READ_ONCE(sqe->splice_off_in);
3636 sp->off_out = READ_ONCE(sqe->off);
3637 return __io_splice_prep(req, sqe);
3638}
3639
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003640static int io_splice(struct io_kiocb *req, unsigned int issue_flags)
Pavel Begunkov7d67af22020-02-24 11:32:45 +03003641{
3642 struct io_splice *sp = &req->splice;
3643 struct file *in = sp->file_in;
3644 struct file *out = sp->file_out;
3645 unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED;
3646 loff_t *poff_in, *poff_out;
Pavel Begunkovc9687422020-05-04 23:00:54 +03003647 long ret = 0;
Pavel Begunkov7d67af22020-02-24 11:32:45 +03003648
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003649 if (issue_flags & IO_URING_F_NONBLOCK)
Pavel Begunkov2fb3e822020-05-01 17:09:38 +03003650 return -EAGAIN;
Pavel Begunkov7d67af22020-02-24 11:32:45 +03003651
3652 poff_in = (sp->off_in == -1) ? NULL : &sp->off_in;
3653 poff_out = (sp->off_out == -1) ? NULL : &sp->off_out;
Pavel Begunkovc9687422020-05-04 23:00:54 +03003654
Jens Axboe948a7742020-05-17 14:21:38 -06003655 if (sp->len)
Pavel Begunkovc9687422020-05-04 23:00:54 +03003656 ret = do_splice(in, poff_in, out, poff_out, sp->len, flags);
Pavel Begunkov7d67af22020-02-24 11:32:45 +03003657
Pavel Begunkove1d767f2021-03-19 17:22:43 +00003658 if (!(sp->flags & SPLICE_F_FD_IN_FIXED))
3659 io_put_file(in);
Pavel Begunkov7d67af22020-02-24 11:32:45 +03003660 req->flags &= ~REQ_F_NEED_CLEANUP;
3661
Pavel Begunkov7d67af22020-02-24 11:32:45 +03003662 if (ret != sp->len)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01003663 req_set_fail(req);
Jens Axboee1e16092020-06-22 09:17:17 -06003664 io_req_complete(req, ret);
Pavel Begunkov7d67af22020-02-24 11:32:45 +03003665 return 0;
3666}
3667
Jens Axboe2b188cc2019-01-07 10:46:33 -07003668/*
3669 * IORING_OP_NOP just posts a completion event, nothing else.
3670 */
Pavel Begunkov889fca72021-02-10 00:03:09 +00003671static int io_nop(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe2b188cc2019-01-07 10:46:33 -07003672{
3673 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003674
Jens Axboedef596e2019-01-09 08:59:42 -07003675 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
3676 return -EINVAL;
3677
Pavel Begunkov889fca72021-02-10 00:03:09 +00003678 __io_req_complete(req, issue_flags, 0, 0);
Jens Axboe2b188cc2019-01-07 10:46:33 -07003679 return 0;
3680}
3681
Pavel Begunkov1155c762021-02-18 18:29:38 +00003682static int io_fsync_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Christoph Hellwigc992fe22019-01-11 09:43:02 -07003683{
Jens Axboe6b063142019-01-10 22:13:58 -07003684 struct io_ring_ctx *ctx = req->ctx;
Christoph Hellwigc992fe22019-01-11 09:43:02 -07003685
Jens Axboe09bb8392019-03-13 12:39:28 -06003686 if (!req->file)
3687 return -EBADF;
Christoph Hellwigc992fe22019-01-11 09:43:02 -07003688
Jens Axboe6b063142019-01-10 22:13:58 -07003689 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
Jens Axboedef596e2019-01-09 08:59:42 -07003690 return -EINVAL;
Jens Axboeedafcce2019-01-09 09:16:05 -07003691 if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index))
Christoph Hellwigc992fe22019-01-11 09:43:02 -07003692 return -EINVAL;
3693
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07003694 req->sync.flags = READ_ONCE(sqe->fsync_flags);
3695 if (unlikely(req->sync.flags & ~IORING_FSYNC_DATASYNC))
3696 return -EINVAL;
3697
3698 req->sync.off = READ_ONCE(sqe->off);
3699 req->sync.len = READ_ONCE(sqe->len);
Christoph Hellwigc992fe22019-01-11 09:43:02 -07003700 return 0;
3701}
3702
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003703static int io_fsync(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe78912932020-01-14 22:09:06 -07003704{
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07003705 loff_t end = req->sync.off + req->sync.len;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07003706 int ret;
3707
Pavel Begunkovac45abc2020-06-08 21:08:18 +03003708 /* fsync always requires a blocking context */
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003709 if (issue_flags & IO_URING_F_NONBLOCK)
Pavel Begunkovac45abc2020-06-08 21:08:18 +03003710 return -EAGAIN;
3711
Jens Axboe9adbd452019-12-20 08:45:55 -07003712 ret = vfs_fsync_range(req->file, req->sync.off,
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07003713 end > 0 ? end : LLONG_MAX,
3714 req->sync.flags & IORING_FSYNC_DATASYNC);
3715 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01003716 req_set_fail(req);
Jens Axboee1e16092020-06-22 09:17:17 -06003717 io_req_complete(req, ret);
Christoph Hellwigc992fe22019-01-11 09:43:02 -07003718 return 0;
3719}
3720
Jens Axboed63d1b52019-12-10 10:38:56 -07003721static int io_fallocate_prep(struct io_kiocb *req,
3722 const struct io_uring_sqe *sqe)
3723{
3724 if (sqe->ioprio || sqe->buf_index || sqe->rw_flags)
3725 return -EINVAL;
Pavel Begunkov3232dd02020-06-03 18:03:22 +03003726 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3727 return -EINVAL;
Jens Axboed63d1b52019-12-10 10:38:56 -07003728
3729 req->sync.off = READ_ONCE(sqe->off);
3730 req->sync.len = READ_ONCE(sqe->addr);
3731 req->sync.mode = READ_ONCE(sqe->len);
3732 return 0;
3733}
3734
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003735static int io_fallocate(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboed63d1b52019-12-10 10:38:56 -07003736{
Pavel Begunkovac45abc2020-06-08 21:08:18 +03003737 int ret;
Jens Axboed63d1b52019-12-10 10:38:56 -07003738
Pavel Begunkovac45abc2020-06-08 21:08:18 +03003739 /* fallocate always requiring blocking context */
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003740 if (issue_flags & IO_URING_F_NONBLOCK)
Pavel Begunkovac45abc2020-06-08 21:08:18 +03003741 return -EAGAIN;
Pavel Begunkovac45abc2020-06-08 21:08:18 +03003742 ret = vfs_fallocate(req->file, req->sync.mode, req->sync.off,
3743 req->sync.len);
Pavel Begunkovac45abc2020-06-08 21:08:18 +03003744 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01003745 req_set_fail(req);
Jens Axboee1e16092020-06-22 09:17:17 -06003746 io_req_complete(req, ret);
Jens Axboed63d1b52019-12-10 10:38:56 -07003747 return 0;
3748}
3749
Pavel Begunkovec65fea2020-06-03 18:03:24 +03003750static int __io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboe15b71ab2019-12-11 11:20:36 -07003751{
Jens Axboef8748882020-01-08 17:47:02 -07003752 const char __user *fname;
Jens Axboe15b71ab2019-12-11 11:20:36 -07003753 int ret;
3754
Pavel Begunkovec65fea2020-06-03 18:03:24 +03003755 if (unlikely(sqe->ioprio || sqe->buf_index))
Jens Axboe15b71ab2019-12-11 11:20:36 -07003756 return -EINVAL;
Pavel Begunkovec65fea2020-06-03 18:03:24 +03003757 if (unlikely(req->flags & REQ_F_FIXED_FILE))
Jens Axboecf3040c2020-02-06 21:31:40 -07003758 return -EBADF;
Jens Axboe15b71ab2019-12-11 11:20:36 -07003759
Pavel Begunkovec65fea2020-06-03 18:03:24 +03003760 /* open.how should be already initialised */
3761 if (!(req->open.how.flags & O_PATH) && force_o_largefile())
Jens Axboe08a1d26eb2020-04-08 09:20:54 -06003762 req->open.how.flags |= O_LARGEFILE;
Jens Axboe15b71ab2019-12-11 11:20:36 -07003763
Pavel Begunkov25e72d12020-06-03 18:03:23 +03003764 req->open.dfd = READ_ONCE(sqe->fd);
3765 fname = u64_to_user_ptr(READ_ONCE(sqe->addr));
Jens Axboef8748882020-01-08 17:47:02 -07003766 req->open.filename = getname(fname);
Jens Axboe15b71ab2019-12-11 11:20:36 -07003767 if (IS_ERR(req->open.filename)) {
3768 ret = PTR_ERR(req->open.filename);
3769 req->open.filename = NULL;
3770 return ret;
3771 }
Jens Axboe4022e7a2020-03-19 19:23:18 -06003772 req->open.nofile = rlimit(RLIMIT_NOFILE);
Pavel Begunkov8fef80b2020-02-07 23:59:53 +03003773 req->flags |= REQ_F_NEED_CLEANUP;
Jens Axboe15b71ab2019-12-11 11:20:36 -07003774 return 0;
3775}
3776
Pavel Begunkovec65fea2020-06-03 18:03:24 +03003777static int io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
3778{
3779 u64 flags, mode;
3780
Jens Axboe14587a462020-09-05 11:36:08 -06003781 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
Jens Axboe4eb8dde2020-09-18 19:36:24 -06003782 return -EINVAL;
Pavel Begunkovec65fea2020-06-03 18:03:24 +03003783 mode = READ_ONCE(sqe->len);
3784 flags = READ_ONCE(sqe->open_flags);
3785 req->open.how = build_open_how(flags, mode);
3786 return __io_openat_prep(req, sqe);
3787}
3788
Jens Axboecebdb982020-01-08 17:59:24 -07003789static int io_openat2_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
3790{
3791 struct open_how __user *how;
Jens Axboecebdb982020-01-08 17:59:24 -07003792 size_t len;
3793 int ret;
3794
Jens Axboe14587a462020-09-05 11:36:08 -06003795 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
Jens Axboe4eb8dde2020-09-18 19:36:24 -06003796 return -EINVAL;
Jens Axboecebdb982020-01-08 17:59:24 -07003797 how = u64_to_user_ptr(READ_ONCE(sqe->addr2));
3798 len = READ_ONCE(sqe->len);
Jens Axboecebdb982020-01-08 17:59:24 -07003799 if (len < OPEN_HOW_SIZE_VER0)
3800 return -EINVAL;
3801
3802 ret = copy_struct_from_user(&req->open.how, sizeof(req->open.how), how,
3803 len);
3804 if (ret)
3805 return ret;
3806
Pavel Begunkovec65fea2020-06-03 18:03:24 +03003807 return __io_openat_prep(req, sqe);
Jens Axboecebdb982020-01-08 17:59:24 -07003808}
3809
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003810static int io_openat2(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe15b71ab2019-12-11 11:20:36 -07003811{
3812 struct open_flags op;
Jens Axboe15b71ab2019-12-11 11:20:36 -07003813 struct file *file;
Jens Axboe3a81fd02020-12-10 12:25:36 -07003814 bool nonblock_set;
3815 bool resolve_nonblock;
Jens Axboe15b71ab2019-12-11 11:20:36 -07003816 int ret;
3817
Jens Axboecebdb982020-01-08 17:59:24 -07003818 ret = build_open_flags(&req->open.how, &op);
Jens Axboe15b71ab2019-12-11 11:20:36 -07003819 if (ret)
3820 goto err;
Jens Axboe3a81fd02020-12-10 12:25:36 -07003821 nonblock_set = op.open_flag & O_NONBLOCK;
3822 resolve_nonblock = req->open.how.resolve & RESOLVE_CACHED;
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003823 if (issue_flags & IO_URING_F_NONBLOCK) {
Jens Axboe3a81fd02020-12-10 12:25:36 -07003824 /*
3825 * Don't bother trying for O_TRUNC, O_CREAT, or O_TMPFILE open,
3826 * it'll always -EAGAIN
3827 */
3828 if (req->open.how.flags & (O_TRUNC | O_CREAT | O_TMPFILE))
3829 return -EAGAIN;
3830 op.lookup_flags |= LOOKUP_CACHED;
3831 op.open_flag |= O_NONBLOCK;
3832 }
Jens Axboe15b71ab2019-12-11 11:20:36 -07003833
Jens Axboe4022e7a2020-03-19 19:23:18 -06003834 ret = __get_unused_fd_flags(req->open.how.flags, req->open.nofile);
Jens Axboe15b71ab2019-12-11 11:20:36 -07003835 if (ret < 0)
3836 goto err;
3837
3838 file = do_filp_open(req->open.dfd, req->open.filename, &op);
Jens Axboe3a81fd02020-12-10 12:25:36 -07003839 /* only retry if RESOLVE_CACHED wasn't already set by application */
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003840 if ((!resolve_nonblock && (issue_flags & IO_URING_F_NONBLOCK)) &&
3841 file == ERR_PTR(-EAGAIN)) {
Jens Axboe3a81fd02020-12-10 12:25:36 -07003842 /*
3843 * We could hang on to this 'fd', but seems like marginal
3844 * gain for something that is now known to be a slower path.
3845 * So just put it, and we'll get a new one when we retry.
3846 */
3847 put_unused_fd(ret);
3848 return -EAGAIN;
3849 }
3850
Jens Axboe15b71ab2019-12-11 11:20:36 -07003851 if (IS_ERR(file)) {
3852 put_unused_fd(ret);
3853 ret = PTR_ERR(file);
3854 } else {
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003855 if ((issue_flags & IO_URING_F_NONBLOCK) && !nonblock_set)
Jens Axboe3a81fd02020-12-10 12:25:36 -07003856 file->f_flags &= ~O_NONBLOCK;
Jens Axboe15b71ab2019-12-11 11:20:36 -07003857 fsnotify_open(file);
3858 fd_install(ret, file);
3859 }
3860err:
3861 putname(req->open.filename);
Pavel Begunkov8fef80b2020-02-07 23:59:53 +03003862 req->flags &= ~REQ_F_NEED_CLEANUP;
Jens Axboe15b71ab2019-12-11 11:20:36 -07003863 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01003864 req_set_fail(req);
Pavel Begunkov0bdf3392021-04-11 01:46:29 +01003865 __io_req_complete(req, issue_flags, ret, 0);
Jens Axboe15b71ab2019-12-11 11:20:36 -07003866 return 0;
3867}
3868
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003869static int io_openat(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboecebdb982020-01-08 17:59:24 -07003870{
Pavel Begunkove45cff52021-02-28 22:35:14 +00003871 return io_openat2(req, issue_flags);
Jens Axboecebdb982020-01-08 17:59:24 -07003872}
3873
Jens Axboe067524e2020-03-02 16:32:28 -07003874static int io_remove_buffers_prep(struct io_kiocb *req,
3875 const struct io_uring_sqe *sqe)
3876{
3877 struct io_provide_buf *p = &req->pbuf;
3878 u64 tmp;
3879
3880 if (sqe->ioprio || sqe->rw_flags || sqe->addr || sqe->len || sqe->off)
3881 return -EINVAL;
3882
3883 tmp = READ_ONCE(sqe->fd);
3884 if (!tmp || tmp > USHRT_MAX)
3885 return -EINVAL;
3886
3887 memset(p, 0, sizeof(*p));
3888 p->nbufs = tmp;
3889 p->bgid = READ_ONCE(sqe->buf_group);
3890 return 0;
3891}
3892
3893static int __io_remove_buffers(struct io_ring_ctx *ctx, struct io_buffer *buf,
3894 int bgid, unsigned nbufs)
3895{
3896 unsigned i = 0;
3897
3898 /* shouldn't happen */
3899 if (!nbufs)
3900 return 0;
3901
3902 /* the head kbuf is the list itself */
3903 while (!list_empty(&buf->list)) {
3904 struct io_buffer *nxt;
3905
3906 nxt = list_first_entry(&buf->list, struct io_buffer, list);
3907 list_del(&nxt->list);
3908 kfree(nxt);
3909 if (++i == nbufs)
3910 return i;
3911 }
3912 i++;
3913 kfree(buf);
Jens Axboe9e15c3a2021-03-13 12:29:43 -07003914 xa_erase(&ctx->io_buffers, bgid);
Jens Axboe067524e2020-03-02 16:32:28 -07003915
3916 return i;
3917}
3918
Pavel Begunkov889fca72021-02-10 00:03:09 +00003919static int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe067524e2020-03-02 16:32:28 -07003920{
3921 struct io_provide_buf *p = &req->pbuf;
3922 struct io_ring_ctx *ctx = req->ctx;
3923 struct io_buffer *head;
3924 int ret = 0;
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003925 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
Jens Axboe067524e2020-03-02 16:32:28 -07003926
3927 io_ring_submit_lock(ctx, !force_nonblock);
3928
3929 lockdep_assert_held(&ctx->uring_lock);
3930
3931 ret = -ENOENT;
Jens Axboe9e15c3a2021-03-13 12:29:43 -07003932 head = xa_load(&ctx->io_buffers, p->bgid);
Jens Axboe067524e2020-03-02 16:32:28 -07003933 if (head)
3934 ret = __io_remove_buffers(ctx, head, p->bgid, p->nbufs);
Jens Axboe067524e2020-03-02 16:32:28 -07003935 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01003936 req_set_fail(req);
Pavel Begunkov31bff9a2020-12-06 22:22:43 +00003937
Pavel Begunkov9fb8cb42021-02-28 22:35:13 +00003938 /* complete before unlock, IOPOLL may need the lock */
3939 __io_req_complete(req, issue_flags, ret, 0);
3940 io_ring_submit_unlock(ctx, !force_nonblock);
Jens Axboe067524e2020-03-02 16:32:28 -07003941 return 0;
3942}
3943
Jens Axboeddf0322d2020-02-23 16:41:33 -07003944static int io_provide_buffers_prep(struct io_kiocb *req,
3945 const struct io_uring_sqe *sqe)
3946{
Pavel Begunkov38134ad2021-04-15 13:07:39 +01003947 unsigned long size, tmp_check;
Jens Axboeddf0322d2020-02-23 16:41:33 -07003948 struct io_provide_buf *p = &req->pbuf;
3949 u64 tmp;
3950
3951 if (sqe->ioprio || sqe->rw_flags)
3952 return -EINVAL;
3953
3954 tmp = READ_ONCE(sqe->fd);
3955 if (!tmp || tmp > USHRT_MAX)
3956 return -E2BIG;
3957 p->nbufs = tmp;
3958 p->addr = READ_ONCE(sqe->addr);
3959 p->len = READ_ONCE(sqe->len);
3960
Pavel Begunkov38134ad2021-04-15 13:07:39 +01003961 if (check_mul_overflow((unsigned long)p->len, (unsigned long)p->nbufs,
3962 &size))
3963 return -EOVERFLOW;
3964 if (check_add_overflow((unsigned long)p->addr, size, &tmp_check))
3965 return -EOVERFLOW;
3966
Pavel Begunkovd81269f2021-03-19 10:21:19 +00003967 size = (unsigned long)p->len * p->nbufs;
3968 if (!access_ok(u64_to_user_ptr(p->addr), size))
Jens Axboeddf0322d2020-02-23 16:41:33 -07003969 return -EFAULT;
3970
3971 p->bgid = READ_ONCE(sqe->buf_group);
3972 tmp = READ_ONCE(sqe->off);
3973 if (tmp > USHRT_MAX)
3974 return -E2BIG;
3975 p->bid = tmp;
3976 return 0;
3977}
3978
3979static int io_add_buffers(struct io_provide_buf *pbuf, struct io_buffer **head)
3980{
3981 struct io_buffer *buf;
3982 u64 addr = pbuf->addr;
3983 int i, bid = pbuf->bid;
3984
3985 for (i = 0; i < pbuf->nbufs; i++) {
3986 buf = kmalloc(sizeof(*buf), GFP_KERNEL);
3987 if (!buf)
3988 break;
3989
3990 buf->addr = addr;
Thadeu Lima de Souza Cascardod1f82802021-05-05 09:47:06 -03003991 buf->len = min_t(__u32, pbuf->len, MAX_RW_COUNT);
Jens Axboeddf0322d2020-02-23 16:41:33 -07003992 buf->bid = bid;
3993 addr += pbuf->len;
3994 bid++;
3995 if (!*head) {
3996 INIT_LIST_HEAD(&buf->list);
3997 *head = buf;
3998 } else {
3999 list_add_tail(&buf->list, &(*head)->list);
4000 }
4001 }
4002
4003 return i ? i : -ENOMEM;
4004}
4005
Pavel Begunkov889fca72021-02-10 00:03:09 +00004006static int io_provide_buffers(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboeddf0322d2020-02-23 16:41:33 -07004007{
4008 struct io_provide_buf *p = &req->pbuf;
4009 struct io_ring_ctx *ctx = req->ctx;
4010 struct io_buffer *head, *list;
4011 int ret = 0;
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004012 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
Jens Axboeddf0322d2020-02-23 16:41:33 -07004013
4014 io_ring_submit_lock(ctx, !force_nonblock);
4015
4016 lockdep_assert_held(&ctx->uring_lock);
4017
Jens Axboe9e15c3a2021-03-13 12:29:43 -07004018 list = head = xa_load(&ctx->io_buffers, p->bgid);
Jens Axboeddf0322d2020-02-23 16:41:33 -07004019
4020 ret = io_add_buffers(p, &head);
Jens Axboe9e15c3a2021-03-13 12:29:43 -07004021 if (ret >= 0 && !list) {
4022 ret = xa_insert(&ctx->io_buffers, p->bgid, head, GFP_KERNEL);
4023 if (ret < 0)
Jens Axboe067524e2020-03-02 16:32:28 -07004024 __io_remove_buffers(ctx, head, p->bgid, -1U);
Jens Axboeddf0322d2020-02-23 16:41:33 -07004025 }
Jens Axboeddf0322d2020-02-23 16:41:33 -07004026 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004027 req_set_fail(req);
Pavel Begunkov9fb8cb42021-02-28 22:35:13 +00004028 /* complete before unlock, IOPOLL may need the lock */
4029 __io_req_complete(req, issue_flags, ret, 0);
4030 io_ring_submit_unlock(ctx, !force_nonblock);
Jens Axboeddf0322d2020-02-23 16:41:33 -07004031 return 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07004032}
4033
Jens Axboe3e4827b2020-01-08 15:18:09 -07004034static int io_epoll_ctl_prep(struct io_kiocb *req,
4035 const struct io_uring_sqe *sqe)
4036{
4037#if defined(CONFIG_EPOLL)
4038 if (sqe->ioprio || sqe->buf_index)
4039 return -EINVAL;
Pavel Begunkov2d74d042021-05-14 12:05:46 +01004040 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
Pavel Begunkov3232dd02020-06-03 18:03:22 +03004041 return -EINVAL;
Jens Axboe3e4827b2020-01-08 15:18:09 -07004042
4043 req->epoll.epfd = READ_ONCE(sqe->fd);
4044 req->epoll.op = READ_ONCE(sqe->len);
4045 req->epoll.fd = READ_ONCE(sqe->off);
4046
4047 if (ep_op_has_event(req->epoll.op)) {
4048 struct epoll_event __user *ev;
4049
4050 ev = u64_to_user_ptr(READ_ONCE(sqe->addr));
4051 if (copy_from_user(&req->epoll.event, ev, sizeof(*ev)))
4052 return -EFAULT;
4053 }
4054
4055 return 0;
4056#else
4057 return -EOPNOTSUPP;
4058#endif
4059}
4060
Pavel Begunkov889fca72021-02-10 00:03:09 +00004061static int io_epoll_ctl(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe3e4827b2020-01-08 15:18:09 -07004062{
4063#if defined(CONFIG_EPOLL)
4064 struct io_epoll *ie = &req->epoll;
4065 int ret;
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004066 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
Jens Axboe3e4827b2020-01-08 15:18:09 -07004067
4068 ret = do_epoll_ctl(ie->epfd, ie->op, ie->fd, &ie->event, force_nonblock);
4069 if (force_nonblock && ret == -EAGAIN)
4070 return -EAGAIN;
4071
4072 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004073 req_set_fail(req);
Pavel Begunkov889fca72021-02-10 00:03:09 +00004074 __io_req_complete(req, issue_flags, ret, 0);
Jens Axboe3e4827b2020-01-08 15:18:09 -07004075 return 0;
4076#else
4077 return -EOPNOTSUPP;
4078#endif
4079}
4080
Jens Axboec1ca7572019-12-25 22:18:28 -07004081static int io_madvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4082{
4083#if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU)
4084 if (sqe->ioprio || sqe->buf_index || sqe->off)
4085 return -EINVAL;
Pavel Begunkov3232dd02020-06-03 18:03:22 +03004086 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4087 return -EINVAL;
Jens Axboec1ca7572019-12-25 22:18:28 -07004088
4089 req->madvise.addr = READ_ONCE(sqe->addr);
4090 req->madvise.len = READ_ONCE(sqe->len);
4091 req->madvise.advice = READ_ONCE(sqe->fadvise_advice);
4092 return 0;
4093#else
4094 return -EOPNOTSUPP;
4095#endif
4096}
4097
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004098static int io_madvise(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboec1ca7572019-12-25 22:18:28 -07004099{
4100#if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU)
4101 struct io_madvise *ma = &req->madvise;
4102 int ret;
4103
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004104 if (issue_flags & IO_URING_F_NONBLOCK)
Jens Axboec1ca7572019-12-25 22:18:28 -07004105 return -EAGAIN;
4106
Minchan Kim0726b012020-10-17 16:14:50 -07004107 ret = do_madvise(current->mm, ma->addr, ma->len, ma->advice);
Jens Axboec1ca7572019-12-25 22:18:28 -07004108 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004109 req_set_fail(req);
Jens Axboee1e16092020-06-22 09:17:17 -06004110 io_req_complete(req, ret);
Jens Axboec1ca7572019-12-25 22:18:28 -07004111 return 0;
4112#else
4113 return -EOPNOTSUPP;
4114#endif
4115}
4116
Jens Axboe4840e412019-12-25 22:03:45 -07004117static int io_fadvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4118{
4119 if (sqe->ioprio || sqe->buf_index || sqe->addr)
4120 return -EINVAL;
Pavel Begunkov3232dd02020-06-03 18:03:22 +03004121 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4122 return -EINVAL;
Jens Axboe4840e412019-12-25 22:03:45 -07004123
4124 req->fadvise.offset = READ_ONCE(sqe->off);
4125 req->fadvise.len = READ_ONCE(sqe->len);
4126 req->fadvise.advice = READ_ONCE(sqe->fadvise_advice);
4127 return 0;
4128}
4129
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004130static int io_fadvise(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe4840e412019-12-25 22:03:45 -07004131{
4132 struct io_fadvise *fa = &req->fadvise;
4133 int ret;
4134
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004135 if (issue_flags & IO_URING_F_NONBLOCK) {
Jens Axboe3e694262020-02-01 09:22:49 -07004136 switch (fa->advice) {
4137 case POSIX_FADV_NORMAL:
4138 case POSIX_FADV_RANDOM:
4139 case POSIX_FADV_SEQUENTIAL:
4140 break;
4141 default:
4142 return -EAGAIN;
4143 }
4144 }
Jens Axboe4840e412019-12-25 22:03:45 -07004145
4146 ret = vfs_fadvise(req->file, fa->offset, fa->len, fa->advice);
4147 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004148 req_set_fail(req);
Pavel Begunkov0bdf3392021-04-11 01:46:29 +01004149 __io_req_complete(req, issue_flags, ret, 0);
Jens Axboe4840e412019-12-25 22:03:45 -07004150 return 0;
4151}
4152
Jens Axboeeddc7ef2019-12-13 21:18:10 -07004153static int io_statx_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4154{
Pavel Begunkov2d74d042021-05-14 12:05:46 +01004155 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
Pavel Begunkov3232dd02020-06-03 18:03:22 +03004156 return -EINVAL;
Jens Axboeeddc7ef2019-12-13 21:18:10 -07004157 if (sqe->ioprio || sqe->buf_index)
4158 return -EINVAL;
Pavel Begunkov9c280f92020-04-08 08:58:46 +03004159 if (req->flags & REQ_F_FIXED_FILE)
Jens Axboecf3040c2020-02-06 21:31:40 -07004160 return -EBADF;
Jens Axboeeddc7ef2019-12-13 21:18:10 -07004161
Bijan Mottahedeh1d9e1282020-05-22 21:31:16 -07004162 req->statx.dfd = READ_ONCE(sqe->fd);
4163 req->statx.mask = READ_ONCE(sqe->len);
Bijan Mottahedehe62753e2020-05-22 21:31:18 -07004164 req->statx.filename = u64_to_user_ptr(READ_ONCE(sqe->addr));
Bijan Mottahedeh1d9e1282020-05-22 21:31:16 -07004165 req->statx.buffer = u64_to_user_ptr(READ_ONCE(sqe->addr2));
4166 req->statx.flags = READ_ONCE(sqe->statx_flags);
Jens Axboeeddc7ef2019-12-13 21:18:10 -07004167
Jens Axboeeddc7ef2019-12-13 21:18:10 -07004168 return 0;
4169}
4170
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004171static int io_statx(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboeeddc7ef2019-12-13 21:18:10 -07004172{
Bijan Mottahedeh1d9e1282020-05-22 21:31:16 -07004173 struct io_statx *ctx = &req->statx;
Jens Axboeeddc7ef2019-12-13 21:18:10 -07004174 int ret;
4175
Pavel Begunkov59d70012021-03-22 01:58:30 +00004176 if (issue_flags & IO_URING_F_NONBLOCK)
Jens Axboeeddc7ef2019-12-13 21:18:10 -07004177 return -EAGAIN;
4178
Bijan Mottahedehe62753e2020-05-22 21:31:18 -07004179 ret = do_statx(ctx->dfd, ctx->filename, ctx->flags, ctx->mask,
4180 ctx->buffer);
Jens Axboeeddc7ef2019-12-13 21:18:10 -07004181
Jens Axboeeddc7ef2019-12-13 21:18:10 -07004182 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004183 req_set_fail(req);
Jens Axboee1e16092020-06-22 09:17:17 -06004184 io_req_complete(req, ret);
Jens Axboeeddc7ef2019-12-13 21:18:10 -07004185 return 0;
4186}
4187
Jens Axboeb5dba592019-12-11 14:02:38 -07004188static int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4189{
Jens Axboe14587a462020-09-05 11:36:08 -06004190 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
Pavel Begunkov3232dd02020-06-03 18:03:22 +03004191 return -EINVAL;
Jens Axboeb5dba592019-12-11 14:02:38 -07004192 if (sqe->ioprio || sqe->off || sqe->addr || sqe->len ||
4193 sqe->rw_flags || sqe->buf_index)
4194 return -EINVAL;
Pavel Begunkov9c280f92020-04-08 08:58:46 +03004195 if (req->flags & REQ_F_FIXED_FILE)
Jens Axboecf3040c2020-02-06 21:31:40 -07004196 return -EBADF;
Jens Axboeb5dba592019-12-11 14:02:38 -07004197
4198 req->close.fd = READ_ONCE(sqe->fd);
Jens Axboeb5dba592019-12-11 14:02:38 -07004199 return 0;
4200}
4201
Pavel Begunkov889fca72021-02-10 00:03:09 +00004202static int io_close(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboeb5dba592019-12-11 14:02:38 -07004203{
Jens Axboe9eac1902021-01-19 15:50:37 -07004204 struct files_struct *files = current->files;
Pavel Begunkov3af73b22020-06-08 21:08:17 +03004205 struct io_close *close = &req->close;
Jens Axboe9eac1902021-01-19 15:50:37 -07004206 struct fdtable *fdt;
Pavel Begunkova1fde922021-04-11 01:46:28 +01004207 struct file *file = NULL;
4208 int ret = -EBADF;
Jens Axboeb5dba592019-12-11 14:02:38 -07004209
Jens Axboe9eac1902021-01-19 15:50:37 -07004210 spin_lock(&files->file_lock);
4211 fdt = files_fdtable(files);
4212 if (close->fd >= fdt->max_fds) {
4213 spin_unlock(&files->file_lock);
4214 goto err;
4215 }
4216 file = fdt->fd[close->fd];
Pavel Begunkova1fde922021-04-11 01:46:28 +01004217 if (!file || file->f_op == &io_uring_fops) {
Jens Axboe9eac1902021-01-19 15:50:37 -07004218 spin_unlock(&files->file_lock);
4219 file = NULL;
4220 goto err;
Pavel Begunkov3af73b22020-06-08 21:08:17 +03004221 }
Jens Axboeb5dba592019-12-11 14:02:38 -07004222
4223 /* if the file has a flush method, be safe and punt to async */
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004224 if (file->f_op->flush && (issue_flags & IO_URING_F_NONBLOCK)) {
Jens Axboe9eac1902021-01-19 15:50:37 -07004225 spin_unlock(&files->file_lock);
Pavel Begunkov0bf0eef2020-05-26 20:34:06 +03004226 return -EAGAIN;
Pavel Begunkova2100672020-03-02 23:45:16 +03004227 }
Jens Axboeb5dba592019-12-11 14:02:38 -07004228
Jens Axboe9eac1902021-01-19 15:50:37 -07004229 ret = __close_fd_get_file(close->fd, &file);
4230 spin_unlock(&files->file_lock);
4231 if (ret < 0) {
4232 if (ret == -ENOENT)
4233 ret = -EBADF;
4234 goto err;
4235 }
4236
Pavel Begunkov3af73b22020-06-08 21:08:17 +03004237 /* No ->flush() or already async, safely close from here */
Jens Axboe9eac1902021-01-19 15:50:37 -07004238 ret = filp_close(file, current->files);
4239err:
Pavel Begunkov3af73b22020-06-08 21:08:17 +03004240 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004241 req_set_fail(req);
Jens Axboe9eac1902021-01-19 15:50:37 -07004242 if (file)
4243 fput(file);
Pavel Begunkov889fca72021-02-10 00:03:09 +00004244 __io_req_complete(req, issue_flags, ret, 0);
Jens Axboe1a417f42020-01-31 17:16:48 -07004245 return 0;
Jens Axboeb5dba592019-12-11 14:02:38 -07004246}
4247
Pavel Begunkov1155c762021-02-18 18:29:38 +00004248static int io_sfr_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboe5d17b4a2019-04-09 14:56:44 -06004249{
4250 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe5d17b4a2019-04-09 14:56:44 -06004251
Jens Axboe5d17b4a2019-04-09 14:56:44 -06004252 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
4253 return -EINVAL;
4254 if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index))
4255 return -EINVAL;
4256
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004257 req->sync.off = READ_ONCE(sqe->off);
4258 req->sync.len = READ_ONCE(sqe->len);
4259 req->sync.flags = READ_ONCE(sqe->sync_range_flags);
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004260 return 0;
4261}
4262
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004263static int io_sync_file_range(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004264{
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004265 int ret;
4266
Pavel Begunkovac45abc2020-06-08 21:08:18 +03004267 /* sync_file_range always requires a blocking context */
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004268 if (issue_flags & IO_URING_F_NONBLOCK)
Pavel Begunkovac45abc2020-06-08 21:08:18 +03004269 return -EAGAIN;
4270
Jens Axboe9adbd452019-12-20 08:45:55 -07004271 ret = sync_file_range(req->file, req->sync.off, req->sync.len,
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004272 req->sync.flags);
4273 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004274 req_set_fail(req);
Jens Axboee1e16092020-06-22 09:17:17 -06004275 io_req_complete(req, ret);
Jens Axboe5d17b4a2019-04-09 14:56:44 -06004276 return 0;
4277}
4278
YueHaibing469956e2020-03-04 15:53:52 +08004279#if defined(CONFIG_NET)
Pavel Begunkov02d27d82020-02-28 10:36:36 +03004280static int io_setup_async_msg(struct io_kiocb *req,
4281 struct io_async_msghdr *kmsg)
4282{
Jens Axboee8c2bc12020-08-15 18:44:09 -07004283 struct io_async_msghdr *async_msg = req->async_data;
4284
4285 if (async_msg)
Pavel Begunkov02d27d82020-02-28 10:36:36 +03004286 return -EAGAIN;
Jens Axboee8c2bc12020-08-15 18:44:09 -07004287 if (io_alloc_async_data(req)) {
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004288 kfree(kmsg->free_iov);
Pavel Begunkov02d27d82020-02-28 10:36:36 +03004289 return -ENOMEM;
4290 }
Jens Axboee8c2bc12020-08-15 18:44:09 -07004291 async_msg = req->async_data;
Pavel Begunkov02d27d82020-02-28 10:36:36 +03004292 req->flags |= REQ_F_NEED_CLEANUP;
Jens Axboee8c2bc12020-08-15 18:44:09 -07004293 memcpy(async_msg, kmsg, sizeof(*kmsg));
Pavel Begunkov2a780802021-02-05 00:57:58 +00004294 async_msg->msg.msg_name = &async_msg->addr;
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004295 /* if were using fast_iov, set it to the new one */
4296 if (!async_msg->free_iov)
4297 async_msg->msg.msg_iter.iov = async_msg->fast_iov;
4298
Pavel Begunkov02d27d82020-02-28 10:36:36 +03004299 return -EAGAIN;
4300}
4301
Pavel Begunkov2ae523e2020-07-12 20:41:06 +03004302static int io_sendmsg_copy_hdr(struct io_kiocb *req,
4303 struct io_async_msghdr *iomsg)
4304{
Pavel Begunkov2ae523e2020-07-12 20:41:06 +03004305 iomsg->msg.msg_name = &iomsg->addr;
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004306 iomsg->free_iov = iomsg->fast_iov;
Pavel Begunkov2ae523e2020-07-12 20:41:06 +03004307 return sendmsg_copy_msghdr(&iomsg->msg, req->sr_msg.umsg,
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004308 req->sr_msg.msg_flags, &iomsg->free_iov);
Pavel Begunkov2ae523e2020-07-12 20:41:06 +03004309}
4310
Pavel Begunkov93642ef2021-02-18 18:29:44 +00004311static int io_sendmsg_prep_async(struct io_kiocb *req)
4312{
4313 int ret;
4314
Pavel Begunkov93642ef2021-02-18 18:29:44 +00004315 ret = io_sendmsg_copy_hdr(req, req->async_data);
4316 if (!ret)
4317 req->flags |= REQ_F_NEED_CLEANUP;
4318 return ret;
4319}
4320
Jens Axboe3529d8c2019-12-19 18:24:38 -07004321static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboeaa1fa282019-04-19 13:38:09 -06004322{
Jens Axboee47293f2019-12-20 08:58:21 -07004323 struct io_sr_msg *sr = &req->sr_msg;
Jens Axboe03b12302019-12-02 18:50:25 -07004324
Pavel Begunkovd2b6f482020-06-03 18:03:25 +03004325 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4326 return -EINVAL;
4327
Pavel Begunkov270a5942020-07-12 20:41:04 +03004328 sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
Jens Axboefddafac2020-01-04 20:19:44 -07004329 sr->len = READ_ONCE(sqe->len);
Pavel Begunkov04411802021-04-01 15:44:00 +01004330 sr->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL;
4331 if (sr->msg_flags & MSG_DONTWAIT)
4332 req->flags |= REQ_F_NOWAIT;
Jens Axboe3529d8c2019-12-19 18:24:38 -07004333
Jens Axboed8768362020-02-27 14:17:49 -07004334#ifdef CONFIG_COMPAT
4335 if (req->ctx->compat)
4336 sr->msg_flags |= MSG_CMSG_COMPAT;
4337#endif
Pavel Begunkov93642ef2021-02-18 18:29:44 +00004338 return 0;
Jens Axboe03b12302019-12-02 18:50:25 -07004339}
4340
Pavel Begunkov889fca72021-02-10 00:03:09 +00004341static int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe03b12302019-12-02 18:50:25 -07004342{
Pavel Begunkov6b754c82020-07-16 23:28:00 +03004343 struct io_async_msghdr iomsg, *kmsg;
Jens Axboe03b12302019-12-02 18:50:25 -07004344 struct socket *sock;
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004345 unsigned flags;
Stefan Metzmacher00312752021-03-20 20:33:36 +01004346 int min_ret = 0;
Jens Axboe03b12302019-12-02 18:50:25 -07004347 int ret;
4348
Florent Revestdba4a922020-12-04 12:36:04 +01004349 sock = sock_from_file(req->file);
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004350 if (unlikely(!sock))
Florent Revestdba4a922020-12-04 12:36:04 +01004351 return -ENOTSOCK;
Jens Axboe03b12302019-12-02 18:50:25 -07004352
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004353 kmsg = req->async_data;
4354 if (!kmsg) {
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004355 ret = io_sendmsg_copy_hdr(req, &iomsg);
Jens Axboefddafac2020-01-04 20:19:44 -07004356 if (ret)
4357 return ret;
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004358 kmsg = &iomsg;
Jens Axboefddafac2020-01-04 20:19:44 -07004359 }
4360
Pavel Begunkov04411802021-04-01 15:44:00 +01004361 flags = req->sr_msg.msg_flags;
4362 if (issue_flags & IO_URING_F_NONBLOCK)
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004363 flags |= MSG_DONTWAIT;
Stefan Metzmacher00312752021-03-20 20:33:36 +01004364 if (flags & MSG_WAITALL)
4365 min_ret = iov_iter_count(&kmsg->msg.msg_iter);
4366
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004367 ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags);
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004368 if ((issue_flags & IO_URING_F_NONBLOCK) && ret == -EAGAIN)
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004369 return io_setup_async_msg(req, kmsg);
4370 if (ret == -ERESTARTSYS)
4371 ret = -EINTR;
4372
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004373 /* fast path, check for non-NULL to avoid function call */
4374 if (kmsg->free_iov)
4375 kfree(kmsg->free_iov);
Jens Axboe03b12302019-12-02 18:50:25 -07004376 req->flags &= ~REQ_F_NEED_CLEANUP;
Stefan Metzmacher00312752021-03-20 20:33:36 +01004377 if (ret < min_ret)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004378 req_set_fail(req);
Pavel Begunkov889fca72021-02-10 00:03:09 +00004379 __io_req_complete(req, issue_flags, ret, 0);
Jens Axboefddafac2020-01-04 20:19:44 -07004380 return 0;
Jens Axboefddafac2020-01-04 20:19:44 -07004381}
4382
Pavel Begunkov889fca72021-02-10 00:03:09 +00004383static int io_send(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe03b12302019-12-02 18:50:25 -07004384{
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004385 struct io_sr_msg *sr = &req->sr_msg;
4386 struct msghdr msg;
4387 struct iovec iov;
Jens Axboe03b12302019-12-02 18:50:25 -07004388 struct socket *sock;
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004389 unsigned flags;
Stefan Metzmacher00312752021-03-20 20:33:36 +01004390 int min_ret = 0;
Jens Axboe03b12302019-12-02 18:50:25 -07004391 int ret;
4392
Florent Revestdba4a922020-12-04 12:36:04 +01004393 sock = sock_from_file(req->file);
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004394 if (unlikely(!sock))
Florent Revestdba4a922020-12-04 12:36:04 +01004395 return -ENOTSOCK;
Jens Axboe03b12302019-12-02 18:50:25 -07004396
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004397 ret = import_single_range(WRITE, sr->buf, sr->len, &iov, &msg.msg_iter);
4398 if (unlikely(ret))
Zheng Bin14db8412020-09-09 20:12:37 +08004399 return ret;
Jens Axboe03b12302019-12-02 18:50:25 -07004400
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004401 msg.msg_name = NULL;
4402 msg.msg_control = NULL;
4403 msg.msg_controllen = 0;
4404 msg.msg_namelen = 0;
Jens Axboe03b12302019-12-02 18:50:25 -07004405
Pavel Begunkov04411802021-04-01 15:44:00 +01004406 flags = req->sr_msg.msg_flags;
4407 if (issue_flags & IO_URING_F_NONBLOCK)
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004408 flags |= MSG_DONTWAIT;
Stefan Metzmacher00312752021-03-20 20:33:36 +01004409 if (flags & MSG_WAITALL)
4410 min_ret = iov_iter_count(&msg.msg_iter);
4411
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004412 msg.msg_flags = flags;
4413 ret = sock_sendmsg(sock, &msg);
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004414 if ((issue_flags & IO_URING_F_NONBLOCK) && ret == -EAGAIN)
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004415 return -EAGAIN;
4416 if (ret == -ERESTARTSYS)
4417 ret = -EINTR;
Jens Axboe03b12302019-12-02 18:50:25 -07004418
Stefan Metzmacher00312752021-03-20 20:33:36 +01004419 if (ret < min_ret)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004420 req_set_fail(req);
Pavel Begunkov889fca72021-02-10 00:03:09 +00004421 __io_req_complete(req, issue_flags, ret, 0);
Jens Axboe03b12302019-12-02 18:50:25 -07004422 return 0;
Jens Axboe03b12302019-12-02 18:50:25 -07004423}
4424
Pavel Begunkov1400e692020-07-12 20:41:05 +03004425static int __io_recvmsg_copy_hdr(struct io_kiocb *req,
4426 struct io_async_msghdr *iomsg)
Jens Axboe52de1fe2020-02-27 10:15:42 -07004427{
4428 struct io_sr_msg *sr = &req->sr_msg;
4429 struct iovec __user *uiov;
4430 size_t iov_len;
4431 int ret;
4432
Pavel Begunkov1400e692020-07-12 20:41:05 +03004433 ret = __copy_msghdr_from_user(&iomsg->msg, sr->umsg,
4434 &iomsg->uaddr, &uiov, &iov_len);
Jens Axboe52de1fe2020-02-27 10:15:42 -07004435 if (ret)
4436 return ret;
4437
4438 if (req->flags & REQ_F_BUFFER_SELECT) {
4439 if (iov_len > 1)
4440 return -EINVAL;
Pavel Begunkov5476dfe2021-02-05 00:57:59 +00004441 if (copy_from_user(iomsg->fast_iov, uiov, sizeof(*uiov)))
Jens Axboe52de1fe2020-02-27 10:15:42 -07004442 return -EFAULT;
Pavel Begunkov5476dfe2021-02-05 00:57:59 +00004443 sr->len = iomsg->fast_iov[0].iov_len;
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004444 iomsg->free_iov = NULL;
Jens Axboe52de1fe2020-02-27 10:15:42 -07004445 } else {
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004446 iomsg->free_iov = iomsg->fast_iov;
Christoph Hellwig89cd35c2020-09-25 06:51:41 +02004447 ret = __import_iovec(READ, uiov, iov_len, UIO_FASTIOV,
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004448 &iomsg->free_iov, &iomsg->msg.msg_iter,
Christoph Hellwig89cd35c2020-09-25 06:51:41 +02004449 false);
Jens Axboe52de1fe2020-02-27 10:15:42 -07004450 if (ret > 0)
4451 ret = 0;
4452 }
4453
4454 return ret;
4455}
4456
4457#ifdef CONFIG_COMPAT
4458static int __io_compat_recvmsg_copy_hdr(struct io_kiocb *req,
Pavel Begunkov1400e692020-07-12 20:41:05 +03004459 struct io_async_msghdr *iomsg)
Jens Axboe52de1fe2020-02-27 10:15:42 -07004460{
Jens Axboe52de1fe2020-02-27 10:15:42 -07004461 struct io_sr_msg *sr = &req->sr_msg;
4462 struct compat_iovec __user *uiov;
4463 compat_uptr_t ptr;
4464 compat_size_t len;
4465 int ret;
4466
Pavel Begunkov4af34172021-04-11 01:46:30 +01004467 ret = __get_compat_msghdr(&iomsg->msg, sr->umsg_compat, &iomsg->uaddr,
4468 &ptr, &len);
Jens Axboe52de1fe2020-02-27 10:15:42 -07004469 if (ret)
4470 return ret;
4471
4472 uiov = compat_ptr(ptr);
4473 if (req->flags & REQ_F_BUFFER_SELECT) {
4474 compat_ssize_t clen;
4475
4476 if (len > 1)
4477 return -EINVAL;
4478 if (!access_ok(uiov, sizeof(*uiov)))
4479 return -EFAULT;
4480 if (__get_user(clen, &uiov->iov_len))
4481 return -EFAULT;
4482 if (clen < 0)
4483 return -EINVAL;
Pavel Begunkov2d280bc2020-11-29 18:33:32 +00004484 sr->len = clen;
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004485 iomsg->free_iov = NULL;
Jens Axboe52de1fe2020-02-27 10:15:42 -07004486 } else {
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004487 iomsg->free_iov = iomsg->fast_iov;
Christoph Hellwig89cd35c2020-09-25 06:51:41 +02004488 ret = __import_iovec(READ, (struct iovec __user *)uiov, len,
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004489 UIO_FASTIOV, &iomsg->free_iov,
Christoph Hellwig89cd35c2020-09-25 06:51:41 +02004490 &iomsg->msg.msg_iter, true);
Jens Axboe52de1fe2020-02-27 10:15:42 -07004491 if (ret < 0)
4492 return ret;
4493 }
4494
4495 return 0;
4496}
Jens Axboe03b12302019-12-02 18:50:25 -07004497#endif
Jens Axboe52de1fe2020-02-27 10:15:42 -07004498
Pavel Begunkov1400e692020-07-12 20:41:05 +03004499static int io_recvmsg_copy_hdr(struct io_kiocb *req,
4500 struct io_async_msghdr *iomsg)
Jens Axboe52de1fe2020-02-27 10:15:42 -07004501{
Pavel Begunkov1400e692020-07-12 20:41:05 +03004502 iomsg->msg.msg_name = &iomsg->addr;
Jens Axboe52de1fe2020-02-27 10:15:42 -07004503
4504#ifdef CONFIG_COMPAT
4505 if (req->ctx->compat)
Pavel Begunkov1400e692020-07-12 20:41:05 +03004506 return __io_compat_recvmsg_copy_hdr(req, iomsg);
Jens Axboe52de1fe2020-02-27 10:15:42 -07004507#endif
4508
Pavel Begunkov1400e692020-07-12 20:41:05 +03004509 return __io_recvmsg_copy_hdr(req, iomsg);
Jens Axboe52de1fe2020-02-27 10:15:42 -07004510}
4511
Jens Axboebcda7ba2020-02-23 16:42:51 -07004512static struct io_buffer *io_recv_buffer_select(struct io_kiocb *req,
Pavel Begunkov7fbb1b52020-07-16 23:28:05 +03004513 bool needs_lock)
Jens Axboebcda7ba2020-02-23 16:42:51 -07004514{
4515 struct io_sr_msg *sr = &req->sr_msg;
4516 struct io_buffer *kbuf;
4517
Jens Axboebcda7ba2020-02-23 16:42:51 -07004518 kbuf = io_buffer_select(req, &sr->len, sr->bgid, sr->kbuf, needs_lock);
4519 if (IS_ERR(kbuf))
4520 return kbuf;
4521
4522 sr->kbuf = kbuf;
4523 req->flags |= REQ_F_BUFFER_SELECTED;
Jens Axboebcda7ba2020-02-23 16:42:51 -07004524 return kbuf;
Jens Axboe03b12302019-12-02 18:50:25 -07004525}
4526
Pavel Begunkov7fbb1b52020-07-16 23:28:05 +03004527static inline unsigned int io_put_recv_kbuf(struct io_kiocb *req)
4528{
4529 return io_put_kbuf(req, req->sr_msg.kbuf);
4530}
4531
Pavel Begunkov93642ef2021-02-18 18:29:44 +00004532static int io_recvmsg_prep_async(struct io_kiocb *req)
Jens Axboe03b12302019-12-02 18:50:25 -07004533{
Pavel Begunkov99bc4c32020-02-07 22:04:45 +03004534 int ret;
Jens Axboe06b76d42019-12-19 14:44:26 -07004535
Pavel Begunkov93642ef2021-02-18 18:29:44 +00004536 ret = io_recvmsg_copy_hdr(req, req->async_data);
4537 if (!ret)
4538 req->flags |= REQ_F_NEED_CLEANUP;
4539 return ret;
4540}
4541
4542static int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4543{
4544 struct io_sr_msg *sr = &req->sr_msg;
4545
Pavel Begunkovd2b6f482020-06-03 18:03:25 +03004546 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4547 return -EINVAL;
4548
Pavel Begunkov270a5942020-07-12 20:41:04 +03004549 sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
Jens Axboe0b7b21e2020-01-31 08:34:59 -07004550 sr->len = READ_ONCE(sqe->len);
Jens Axboebcda7ba2020-02-23 16:42:51 -07004551 sr->bgid = READ_ONCE(sqe->buf_group);
Pavel Begunkov04411802021-04-01 15:44:00 +01004552 sr->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL;
4553 if (sr->msg_flags & MSG_DONTWAIT)
4554 req->flags |= REQ_F_NOWAIT;
Jens Axboe3529d8c2019-12-19 18:24:38 -07004555
Jens Axboed8768362020-02-27 14:17:49 -07004556#ifdef CONFIG_COMPAT
4557 if (req->ctx->compat)
4558 sr->msg_flags |= MSG_CMSG_COMPAT;
4559#endif
Pavel Begunkov93642ef2021-02-18 18:29:44 +00004560 return 0;
Jens Axboe03b12302019-12-02 18:50:25 -07004561}
4562
Pavel Begunkov889fca72021-02-10 00:03:09 +00004563static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe03b12302019-12-02 18:50:25 -07004564{
Pavel Begunkov6b754c82020-07-16 23:28:00 +03004565 struct io_async_msghdr iomsg, *kmsg;
Jens Axboe0fa03c62019-04-19 13:34:07 -06004566 struct socket *sock;
Pavel Begunkov7fbb1b52020-07-16 23:28:05 +03004567 struct io_buffer *kbuf;
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004568 unsigned flags;
Stefan Metzmacher00312752021-03-20 20:33:36 +01004569 int min_ret = 0;
Jens Axboe52de1fe2020-02-27 10:15:42 -07004570 int ret, cflags = 0;
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004571 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
Jens Axboe0fa03c62019-04-19 13:34:07 -06004572
Florent Revestdba4a922020-12-04 12:36:04 +01004573 sock = sock_from_file(req->file);
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004574 if (unlikely(!sock))
Florent Revestdba4a922020-12-04 12:36:04 +01004575 return -ENOTSOCK;
Jens Axboe0fa03c62019-04-19 13:34:07 -06004576
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004577 kmsg = req->async_data;
4578 if (!kmsg) {
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004579 ret = io_recvmsg_copy_hdr(req, &iomsg);
4580 if (ret)
Pavel Begunkov681fda82020-07-15 22:20:45 +03004581 return ret;
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004582 kmsg = &iomsg;
Jens Axboe0fa03c62019-04-19 13:34:07 -06004583 }
4584
Pavel Begunkovbc02ef32020-07-16 23:28:03 +03004585 if (req->flags & REQ_F_BUFFER_SELECT) {
Pavel Begunkov7fbb1b52020-07-16 23:28:05 +03004586 kbuf = io_recv_buffer_select(req, !force_nonblock);
Pavel Begunkovbc02ef32020-07-16 23:28:03 +03004587 if (IS_ERR(kbuf))
4588 return PTR_ERR(kbuf);
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004589 kmsg->fast_iov[0].iov_base = u64_to_user_ptr(kbuf->addr);
Pavel Begunkov5476dfe2021-02-05 00:57:59 +00004590 kmsg->fast_iov[0].iov_len = req->sr_msg.len;
4591 iov_iter_init(&kmsg->msg.msg_iter, READ, kmsg->fast_iov,
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004592 1, req->sr_msg.len);
4593 }
4594
Pavel Begunkov04411802021-04-01 15:44:00 +01004595 flags = req->sr_msg.msg_flags;
4596 if (force_nonblock)
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004597 flags |= MSG_DONTWAIT;
Stefan Metzmacher00312752021-03-20 20:33:36 +01004598 if (flags & MSG_WAITALL)
4599 min_ret = iov_iter_count(&kmsg->msg.msg_iter);
4600
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004601 ret = __sys_recvmsg_sock(sock, &kmsg->msg, req->sr_msg.umsg,
4602 kmsg->uaddr, flags);
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03004603 if (force_nonblock && ret == -EAGAIN)
4604 return io_setup_async_msg(req, kmsg);
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004605 if (ret == -ERESTARTSYS)
4606 ret = -EINTR;
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03004607
Pavel Begunkov7fbb1b52020-07-16 23:28:05 +03004608 if (req->flags & REQ_F_BUFFER_SELECTED)
4609 cflags = io_put_recv_kbuf(req);
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004610 /* fast path, check for non-NULL to avoid function call */
4611 if (kmsg->free_iov)
4612 kfree(kmsg->free_iov);
Pavel Begunkov99bc4c32020-02-07 22:04:45 +03004613 req->flags &= ~REQ_F_NEED_CLEANUP;
Stefan Metzmacher00312752021-03-20 20:33:36 +01004614 if (ret < min_ret || ((flags & MSG_WAITALL) && (kmsg->msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))))
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004615 req_set_fail(req);
Pavel Begunkov889fca72021-02-10 00:03:09 +00004616 __io_req_complete(req, issue_flags, ret, cflags);
Jens Axboe0fa03c62019-04-19 13:34:07 -06004617 return 0;
Jens Axboe0fa03c62019-04-19 13:34:07 -06004618}
4619
Pavel Begunkov889fca72021-02-10 00:03:09 +00004620static int io_recv(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboefddafac2020-01-04 20:19:44 -07004621{
Pavel Begunkov6b754c82020-07-16 23:28:00 +03004622 struct io_buffer *kbuf;
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004623 struct io_sr_msg *sr = &req->sr_msg;
4624 struct msghdr msg;
4625 void __user *buf = sr->buf;
Jens Axboefddafac2020-01-04 20:19:44 -07004626 struct socket *sock;
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004627 struct iovec iov;
4628 unsigned flags;
Stefan Metzmacher00312752021-03-20 20:33:36 +01004629 int min_ret = 0;
Jens Axboebcda7ba2020-02-23 16:42:51 -07004630 int ret, cflags = 0;
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004631 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
Jens Axboefddafac2020-01-04 20:19:44 -07004632
Florent Revestdba4a922020-12-04 12:36:04 +01004633 sock = sock_from_file(req->file);
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004634 if (unlikely(!sock))
Florent Revestdba4a922020-12-04 12:36:04 +01004635 return -ENOTSOCK;
Jens Axboefddafac2020-01-04 20:19:44 -07004636
Pavel Begunkovbc02ef32020-07-16 23:28:03 +03004637 if (req->flags & REQ_F_BUFFER_SELECT) {
Pavel Begunkov7fbb1b52020-07-16 23:28:05 +03004638 kbuf = io_recv_buffer_select(req, !force_nonblock);
Jens Axboebcda7ba2020-02-23 16:42:51 -07004639 if (IS_ERR(kbuf))
4640 return PTR_ERR(kbuf);
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004641 buf = u64_to_user_ptr(kbuf->addr);
Jens Axboefddafac2020-01-04 20:19:44 -07004642 }
4643
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004644 ret = import_single_range(READ, buf, sr->len, &iov, &msg.msg_iter);
Pavel Begunkov14c32ee2020-07-16 23:28:01 +03004645 if (unlikely(ret))
4646 goto out_free;
Jens Axboefddafac2020-01-04 20:19:44 -07004647
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004648 msg.msg_name = NULL;
4649 msg.msg_control = NULL;
4650 msg.msg_controllen = 0;
4651 msg.msg_namelen = 0;
4652 msg.msg_iocb = NULL;
4653 msg.msg_flags = 0;
4654
Pavel Begunkov04411802021-04-01 15:44:00 +01004655 flags = req->sr_msg.msg_flags;
4656 if (force_nonblock)
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004657 flags |= MSG_DONTWAIT;
Stefan Metzmacher00312752021-03-20 20:33:36 +01004658 if (flags & MSG_WAITALL)
4659 min_ret = iov_iter_count(&msg.msg_iter);
4660
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004661 ret = sock_recvmsg(sock, &msg, flags);
4662 if (force_nonblock && ret == -EAGAIN)
4663 return -EAGAIN;
4664 if (ret == -ERESTARTSYS)
4665 ret = -EINTR;
Pavel Begunkov14c32ee2020-07-16 23:28:01 +03004666out_free:
Pavel Begunkov7fbb1b52020-07-16 23:28:05 +03004667 if (req->flags & REQ_F_BUFFER_SELECTED)
4668 cflags = io_put_recv_kbuf(req);
Stefan Metzmacher00312752021-03-20 20:33:36 +01004669 if (ret < min_ret || ((flags & MSG_WAITALL) && (msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))))
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004670 req_set_fail(req);
Pavel Begunkov889fca72021-02-10 00:03:09 +00004671 __io_req_complete(req, issue_flags, ret, cflags);
Jens Axboefddafac2020-01-04 20:19:44 -07004672 return 0;
Jens Axboefddafac2020-01-04 20:19:44 -07004673}
4674
Jens Axboe3529d8c2019-12-19 18:24:38 -07004675static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboe17f2fe32019-10-17 14:42:58 -06004676{
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004677 struct io_accept *accept = &req->accept;
4678
Jens Axboe14587a462020-09-05 11:36:08 -06004679 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
Jens Axboe17f2fe32019-10-17 14:42:58 -06004680 return -EINVAL;
Hrvoje Zeba8042d6c2019-11-25 14:40:22 -05004681 if (sqe->ioprio || sqe->len || sqe->buf_index)
Jens Axboe17f2fe32019-10-17 14:42:58 -06004682 return -EINVAL;
4683
Jens Axboed55e5f52019-12-11 16:12:15 -07004684 accept->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));
4685 accept->addr_len = u64_to_user_ptr(READ_ONCE(sqe->addr2));
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004686 accept->flags = READ_ONCE(sqe->accept_flags);
Jens Axboe09952e32020-03-19 20:16:56 -06004687 accept->nofile = rlimit(RLIMIT_NOFILE);
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004688 return 0;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004689}
Jens Axboe17f2fe32019-10-17 14:42:58 -06004690
Pavel Begunkov889fca72021-02-10 00:03:09 +00004691static int io_accept(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004692{
4693 struct io_accept *accept = &req->accept;
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004694 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
Pavel Begunkovac45abc2020-06-08 21:08:18 +03004695 unsigned int file_flags = force_nonblock ? O_NONBLOCK : 0;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004696 int ret;
4697
Jiufei Xuee697dee2020-06-10 13:41:59 +08004698 if (req->file->f_flags & O_NONBLOCK)
4699 req->flags |= REQ_F_NOWAIT;
4700
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004701 ret = __sys_accept4_file(req->file, file_flags, accept->addr,
Jens Axboe09952e32020-03-19 20:16:56 -06004702 accept->addr_len, accept->flags,
4703 accept->nofile);
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004704 if (ret == -EAGAIN && force_nonblock)
Jens Axboe17f2fe32019-10-17 14:42:58 -06004705 return -EAGAIN;
Pavel Begunkovac45abc2020-06-08 21:08:18 +03004706 if (ret < 0) {
4707 if (ret == -ERESTARTSYS)
4708 ret = -EINTR;
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004709 req_set_fail(req);
Pavel Begunkovac45abc2020-06-08 21:08:18 +03004710 }
Pavel Begunkov889fca72021-02-10 00:03:09 +00004711 __io_req_complete(req, issue_flags, ret, 0);
Jens Axboe17f2fe32019-10-17 14:42:58 -06004712 return 0;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004713}
4714
Pavel Begunkov93642ef2021-02-18 18:29:44 +00004715static int io_connect_prep_async(struct io_kiocb *req)
4716{
4717 struct io_async_connect *io = req->async_data;
4718 struct io_connect *conn = &req->connect;
4719
4720 return move_addr_to_kernel(conn->addr, conn->addr_len, &io->address);
4721}
4722
Jens Axboe3529d8c2019-12-19 18:24:38 -07004723static int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboef499a022019-12-02 16:28:46 -07004724{
Jens Axboe3529d8c2019-12-19 18:24:38 -07004725 struct io_connect *conn = &req->connect;
Jens Axboef499a022019-12-02 16:28:46 -07004726
Jens Axboe14587a462020-09-05 11:36:08 -06004727 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
Jens Axboe3fbb51c2019-12-20 08:51:52 -07004728 return -EINVAL;
4729 if (sqe->ioprio || sqe->len || sqe->buf_index || sqe->rw_flags)
4730 return -EINVAL;
4731
Jens Axboe3529d8c2019-12-19 18:24:38 -07004732 conn->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));
4733 conn->addr_len = READ_ONCE(sqe->addr2);
Pavel Begunkov93642ef2021-02-18 18:29:44 +00004734 return 0;
Jens Axboef499a022019-12-02 16:28:46 -07004735}
4736
Pavel Begunkov889fca72021-02-10 00:03:09 +00004737static int io_connect(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboef8e85cf2019-11-23 14:24:24 -07004738{
Jens Axboee8c2bc12020-08-15 18:44:09 -07004739 struct io_async_connect __io, *io;
Jens Axboef8e85cf2019-11-23 14:24:24 -07004740 unsigned file_flags;
Jens Axboe3fbb51c2019-12-20 08:51:52 -07004741 int ret;
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004742 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
Jens Axboef8e85cf2019-11-23 14:24:24 -07004743
Jens Axboee8c2bc12020-08-15 18:44:09 -07004744 if (req->async_data) {
4745 io = req->async_data;
Jens Axboef499a022019-12-02 16:28:46 -07004746 } else {
Jens Axboe3529d8c2019-12-19 18:24:38 -07004747 ret = move_addr_to_kernel(req->connect.addr,
4748 req->connect.addr_len,
Jens Axboee8c2bc12020-08-15 18:44:09 -07004749 &__io.address);
Jens Axboef499a022019-12-02 16:28:46 -07004750 if (ret)
4751 goto out;
4752 io = &__io;
4753 }
4754
Jens Axboe3fbb51c2019-12-20 08:51:52 -07004755 file_flags = force_nonblock ? O_NONBLOCK : 0;
4756
Jens Axboee8c2bc12020-08-15 18:44:09 -07004757 ret = __sys_connect_file(req->file, &io->address,
Jens Axboe3fbb51c2019-12-20 08:51:52 -07004758 req->connect.addr_len, file_flags);
Jens Axboe87f80d62019-12-03 11:23:54 -07004759 if ((ret == -EAGAIN || ret == -EINPROGRESS) && force_nonblock) {
Jens Axboee8c2bc12020-08-15 18:44:09 -07004760 if (req->async_data)
Jens Axboeb7bb4f72019-12-15 22:13:43 -07004761 return -EAGAIN;
Jens Axboee8c2bc12020-08-15 18:44:09 -07004762 if (io_alloc_async_data(req)) {
Jens Axboef499a022019-12-02 16:28:46 -07004763 ret = -ENOMEM;
4764 goto out;
4765 }
Jens Axboee8c2bc12020-08-15 18:44:09 -07004766 memcpy(req->async_data, &__io, sizeof(__io));
Jens Axboef8e85cf2019-11-23 14:24:24 -07004767 return -EAGAIN;
Jens Axboef499a022019-12-02 16:28:46 -07004768 }
Jens Axboef8e85cf2019-11-23 14:24:24 -07004769 if (ret == -ERESTARTSYS)
4770 ret = -EINTR;
Jens Axboef499a022019-12-02 16:28:46 -07004771out:
Jens Axboe4e88d6e2019-12-07 20:59:47 -07004772 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004773 req_set_fail(req);
Pavel Begunkov889fca72021-02-10 00:03:09 +00004774 __io_req_complete(req, issue_flags, ret, 0);
Jens Axboef8e85cf2019-11-23 14:24:24 -07004775 return 0;
Jens Axboef8e85cf2019-11-23 14:24:24 -07004776}
YueHaibing469956e2020-03-04 15:53:52 +08004777#else /* !CONFIG_NET */
Jens Axboe99a10082021-02-19 09:35:19 -07004778#define IO_NETOP_FN(op) \
4779static int io_##op(struct io_kiocb *req, unsigned int issue_flags) \
4780{ \
4781 return -EOPNOTSUPP; \
Jens Axboef8e85cf2019-11-23 14:24:24 -07004782}
4783
Jens Axboe99a10082021-02-19 09:35:19 -07004784#define IO_NETOP_PREP(op) \
4785IO_NETOP_FN(op) \
4786static int io_##op##_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) \
4787{ \
4788 return -EOPNOTSUPP; \
4789} \
4790
4791#define IO_NETOP_PREP_ASYNC(op) \
4792IO_NETOP_PREP(op) \
4793static int io_##op##_prep_async(struct io_kiocb *req) \
4794{ \
4795 return -EOPNOTSUPP; \
YueHaibing469956e2020-03-04 15:53:52 +08004796}
4797
Jens Axboe99a10082021-02-19 09:35:19 -07004798IO_NETOP_PREP_ASYNC(sendmsg);
4799IO_NETOP_PREP_ASYNC(recvmsg);
4800IO_NETOP_PREP_ASYNC(connect);
4801IO_NETOP_PREP(accept);
4802IO_NETOP_FN(send);
4803IO_NETOP_FN(recv);
YueHaibing469956e2020-03-04 15:53:52 +08004804#endif /* CONFIG_NET */
Jens Axboe17f2fe32019-10-17 14:42:58 -06004805
Jens Axboed7718a92020-02-14 22:23:12 -07004806struct io_poll_table {
4807 struct poll_table_struct pt;
4808 struct io_kiocb *req;
4809 int error;
4810};
4811
Jens Axboed7718a92020-02-14 22:23:12 -07004812static int __io_async_wake(struct io_kiocb *req, struct io_poll_iocb *poll,
4813 __poll_t mask, task_work_func_t func)
4814{
Jens Axboeaa96bf82020-04-03 11:26:26 -06004815 int ret;
Jens Axboed7718a92020-02-14 22:23:12 -07004816
4817 /* for instances that support it check for an event match first: */
4818 if (mask && !(mask & poll->events))
4819 return 0;
4820
4821 trace_io_uring_task_add(req->ctx, req->opcode, req->user_data, mask);
4822
4823 list_del_init(&poll->wait.entry);
4824
Jens Axboed7718a92020-02-14 22:23:12 -07004825 req->result = mask;
Jens Axboe7cbf1722021-02-10 00:03:20 +00004826 req->task_work.func = func;
Jens Axboe6d816e02020-08-11 08:04:14 -06004827
Jens Axboed7718a92020-02-14 22:23:12 -07004828 /*
Jens Axboee3aabf92020-05-18 11:04:17 -06004829 * If this fails, then the task is exiting. When a task exits, the
4830 * work gets canceled, so just cancel this request as well instead
4831 * of executing it. We can't safely execute it anyway, as we may not
4832 * have the needed state needed for it anyway.
Jens Axboed7718a92020-02-14 22:23:12 -07004833 */
Jens Axboe355fb9e2020-10-22 20:19:35 -06004834 ret = io_req_task_work_add(req);
Jens Axboeaa96bf82020-04-03 11:26:26 -06004835 if (unlikely(ret)) {
Jens Axboee3aabf92020-05-18 11:04:17 -06004836 WRITE_ONCE(poll->canceled, true);
Pavel Begunkoveab30c42021-01-19 13:32:42 +00004837 io_req_task_work_add_fallback(req, func);
Jens Axboeaa96bf82020-04-03 11:26:26 -06004838 }
Jens Axboed7718a92020-02-14 22:23:12 -07004839 return 1;
4840}
4841
Jens Axboe74ce6ce2020-04-13 11:09:12 -06004842static bool io_poll_rewait(struct io_kiocb *req, struct io_poll_iocb *poll)
4843 __acquires(&req->ctx->completion_lock)
4844{
4845 struct io_ring_ctx *ctx = req->ctx;
4846
4847 if (!req->result && !READ_ONCE(poll->canceled)) {
4848 struct poll_table_struct pt = { ._key = poll->events };
4849
4850 req->result = vfs_poll(req->file, &pt) & poll->events;
4851 }
4852
4853 spin_lock_irq(&ctx->completion_lock);
4854 if (!req->result && !READ_ONCE(poll->canceled)) {
4855 add_wait_queue(poll->head, &poll->wait);
4856 return true;
4857 }
4858
4859 return false;
4860}
4861
Jens Axboed4e7cd32020-08-15 11:44:50 -07004862static struct io_poll_iocb *io_poll_get_double(struct io_kiocb *req)
Jens Axboe18bceab2020-05-15 11:56:54 -06004863{
Jens Axboee8c2bc12020-08-15 18:44:09 -07004864 /* pure poll stashes this in ->async_data, poll driven retry elsewhere */
Jens Axboed4e7cd32020-08-15 11:44:50 -07004865 if (req->opcode == IORING_OP_POLL_ADD)
Jens Axboee8c2bc12020-08-15 18:44:09 -07004866 return req->async_data;
Jens Axboed4e7cd32020-08-15 11:44:50 -07004867 return req->apoll->double_poll;
4868}
4869
4870static struct io_poll_iocb *io_poll_get_single(struct io_kiocb *req)
4871{
4872 if (req->opcode == IORING_OP_POLL_ADD)
4873 return &req->poll;
4874 return &req->apoll->poll;
4875}
4876
4877static void io_poll_remove_double(struct io_kiocb *req)
Pavel Begunkove07785b2021-04-01 15:43:57 +01004878 __must_hold(&req->ctx->completion_lock)
Jens Axboed4e7cd32020-08-15 11:44:50 -07004879{
4880 struct io_poll_iocb *poll = io_poll_get_double(req);
Jens Axboe18bceab2020-05-15 11:56:54 -06004881
4882 lockdep_assert_held(&req->ctx->completion_lock);
4883
4884 if (poll && poll->head) {
4885 struct wait_queue_head *head = poll->head;
4886
4887 spin_lock(&head->lock);
4888 list_del_init(&poll->wait.entry);
4889 if (poll->wait.private)
Jens Axboede9b4cc2021-02-24 13:28:27 -07004890 req_ref_put(req);
Jens Axboe18bceab2020-05-15 11:56:54 -06004891 poll->head = NULL;
4892 spin_unlock(&head->lock);
4893 }
4894}
4895
Pavel Begunkove27414b2021-04-09 09:13:20 +01004896static bool io_poll_complete(struct io_kiocb *req, __poll_t mask)
Pavel Begunkove07785b2021-04-01 15:43:57 +01004897 __must_hold(&req->ctx->completion_lock)
Jens Axboe18bceab2020-05-15 11:56:54 -06004898{
4899 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe88e41cf2021-02-22 22:08:01 -07004900 unsigned flags = IORING_CQE_F_MORE;
Pavel Begunkove27414b2021-04-09 09:13:20 +01004901 int error;
Jens Axboe18bceab2020-05-15 11:56:54 -06004902
Pavel Begunkove27414b2021-04-09 09:13:20 +01004903 if (READ_ONCE(req->poll.canceled)) {
Jens Axboe45ab03b2021-02-23 08:19:33 -07004904 error = -ECANCELED;
Jens Axboe88e41cf2021-02-22 22:08:01 -07004905 req->poll.events |= EPOLLONESHOT;
Pavel Begunkove27414b2021-04-09 09:13:20 +01004906 } else {
Jens Axboe50826202021-02-23 09:02:26 -07004907 error = mangle_poll(mask);
Pavel Begunkove27414b2021-04-09 09:13:20 +01004908 }
Jens Axboeb69de282021-03-17 08:37:41 -06004909 if (req->poll.events & EPOLLONESHOT)
4910 flags = 0;
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01004911 if (!io_cqring_fill_event(ctx, req->user_data, error, flags)) {
Jens Axboe50826202021-02-23 09:02:26 -07004912 io_poll_remove_waitqs(req);
Jens Axboe88e41cf2021-02-22 22:08:01 -07004913 req->poll.done = true;
4914 flags = 0;
4915 }
Hao Xu7b289c32021-04-13 15:20:39 +08004916 if (flags & IORING_CQE_F_MORE)
4917 ctx->cq_extra++;
4918
Jens Axboe18bceab2020-05-15 11:56:54 -06004919 io_commit_cqring(ctx);
Jens Axboe88e41cf2021-02-22 22:08:01 -07004920 return !(flags & IORING_CQE_F_MORE);
Jens Axboe18bceab2020-05-15 11:56:54 -06004921}
4922
Jens Axboe18bceab2020-05-15 11:56:54 -06004923static void io_poll_task_func(struct callback_head *cb)
4924{
4925 struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
Jens Axboe6d816e02020-08-11 08:04:14 -06004926 struct io_ring_ctx *ctx = req->ctx;
Pavel Begunkovdd221f462020-10-18 10:17:42 +01004927 struct io_kiocb *nxt;
Jens Axboe18bceab2020-05-15 11:56:54 -06004928
Pavel Begunkovdd221f462020-10-18 10:17:42 +01004929 if (io_poll_rewait(req, &req->poll)) {
4930 spin_unlock_irq(&ctx->completion_lock);
4931 } else {
Pavel Begunkovf40b9642021-04-09 09:13:19 +01004932 bool done;
Jens Axboe88e41cf2021-02-22 22:08:01 -07004933
Pavel Begunkove27414b2021-04-09 09:13:20 +01004934 done = io_poll_complete(req, req->result);
Jens Axboe88e41cf2021-02-22 22:08:01 -07004935 if (done) {
4936 hash_del(&req->hash_node);
Pavel Begunkovf40b9642021-04-09 09:13:19 +01004937 } else {
Jens Axboe88e41cf2021-02-22 22:08:01 -07004938 req->result = 0;
4939 add_wait_queue(req->poll.head, &req->poll.wait);
4940 }
Pavel Begunkovdd221f462020-10-18 10:17:42 +01004941 spin_unlock_irq(&ctx->completion_lock);
Pavel Begunkovdd221f462020-10-18 10:17:42 +01004942 io_cqring_ev_posted(ctx);
Pavel Begunkovdd221f462020-10-18 10:17:42 +01004943
Jens Axboe88e41cf2021-02-22 22:08:01 -07004944 if (done) {
4945 nxt = io_put_req_find_next(req);
4946 if (nxt)
4947 __io_req_task_submit(nxt);
4948 }
Pavel Begunkovea1164e2020-06-30 15:20:41 +03004949 }
Jens Axboe18bceab2020-05-15 11:56:54 -06004950}
4951
4952static int io_poll_double_wake(struct wait_queue_entry *wait, unsigned mode,
4953 int sync, void *key)
4954{
4955 struct io_kiocb *req = wait->private;
Jens Axboed4e7cd32020-08-15 11:44:50 -07004956 struct io_poll_iocb *poll = io_poll_get_single(req);
Jens Axboe18bceab2020-05-15 11:56:54 -06004957 __poll_t mask = key_to_poll(key);
4958
4959 /* for instances that support it check for an event match first: */
4960 if (mask && !(mask & poll->events))
4961 return 0;
Jens Axboe88e41cf2021-02-22 22:08:01 -07004962 if (!(poll->events & EPOLLONESHOT))
4963 return poll->wait.func(&poll->wait, mode, sync, key);
Jens Axboe18bceab2020-05-15 11:56:54 -06004964
Jens Axboe8706e042020-09-28 08:38:54 -06004965 list_del_init(&wait->entry);
4966
Jens Axboe807abcb2020-07-17 17:09:27 -06004967 if (poll && poll->head) {
Jens Axboe18bceab2020-05-15 11:56:54 -06004968 bool done;
4969
Jens Axboe807abcb2020-07-17 17:09:27 -06004970 spin_lock(&poll->head->lock);
4971 done = list_empty(&poll->wait.entry);
Jens Axboe18bceab2020-05-15 11:56:54 -06004972 if (!done)
Jens Axboe807abcb2020-07-17 17:09:27 -06004973 list_del_init(&poll->wait.entry);
Jens Axboed4e7cd32020-08-15 11:44:50 -07004974 /* make sure double remove sees this as being gone */
4975 wait->private = NULL;
Jens Axboe807abcb2020-07-17 17:09:27 -06004976 spin_unlock(&poll->head->lock);
Jens Axboec8b5e262020-10-25 13:53:26 -06004977 if (!done) {
4978 /* use wait func handler, so it matches the rq type */
4979 poll->wait.func(&poll->wait, mode, sync, key);
4980 }
Jens Axboe18bceab2020-05-15 11:56:54 -06004981 }
Jens Axboede9b4cc2021-02-24 13:28:27 -07004982 req_ref_put(req);
Jens Axboe18bceab2020-05-15 11:56:54 -06004983 return 1;
4984}
4985
4986static void io_init_poll_iocb(struct io_poll_iocb *poll, __poll_t events,
4987 wait_queue_func_t wake_func)
4988{
4989 poll->head = NULL;
4990 poll->done = false;
4991 poll->canceled = false;
Jens Axboe464dca62021-03-19 14:06:24 -06004992#define IO_POLL_UNMASK (EPOLLERR|EPOLLHUP|EPOLLNVAL|EPOLLRDHUP)
4993 /* mask in events that we always want/need */
4994 poll->events = events | IO_POLL_UNMASK;
Jens Axboe18bceab2020-05-15 11:56:54 -06004995 INIT_LIST_HEAD(&poll->wait.entry);
4996 init_waitqueue_func_entry(&poll->wait, wake_func);
4997}
4998
4999static void __io_queue_proc(struct io_poll_iocb *poll, struct io_poll_table *pt,
Jens Axboe807abcb2020-07-17 17:09:27 -06005000 struct wait_queue_head *head,
5001 struct io_poll_iocb **poll_ptr)
Jens Axboe18bceab2020-05-15 11:56:54 -06005002{
5003 struct io_kiocb *req = pt->req;
5004
5005 /*
5006 * If poll->head is already set, it's because the file being polled
5007 * uses multiple waitqueues for poll handling (eg one for read, one
5008 * for write). Setup a separate io_poll_iocb if this happens.
5009 */
5010 if (unlikely(poll->head)) {
Pavel Begunkov58852d42020-10-16 20:55:56 +01005011 struct io_poll_iocb *poll_one = poll;
5012
Jens Axboe18bceab2020-05-15 11:56:54 -06005013 /* already have a 2nd entry, fail a third attempt */
Jens Axboe807abcb2020-07-17 17:09:27 -06005014 if (*poll_ptr) {
Jens Axboe18bceab2020-05-15 11:56:54 -06005015 pt->error = -EINVAL;
5016 return;
5017 }
Jens Axboeea6a693d2021-04-15 09:47:13 -06005018 /*
5019 * Can't handle multishot for double wait for now, turn it
5020 * into one-shot mode.
5021 */
Pavel Begunkov7a274722021-05-17 12:43:34 +01005022 if (!(poll_one->events & EPOLLONESHOT))
5023 poll_one->events |= EPOLLONESHOT;
Jens Axboe1c3b3e62021-02-28 16:07:30 -07005024 /* double add on the same waitqueue head, ignore */
Pavel Begunkov7a274722021-05-17 12:43:34 +01005025 if (poll_one->head == head)
Jens Axboe1c3b3e62021-02-28 16:07:30 -07005026 return;
Jens Axboe18bceab2020-05-15 11:56:54 -06005027 poll = kmalloc(sizeof(*poll), GFP_ATOMIC);
5028 if (!poll) {
5029 pt->error = -ENOMEM;
5030 return;
5031 }
Pavel Begunkov58852d42020-10-16 20:55:56 +01005032 io_init_poll_iocb(poll, poll_one->events, io_poll_double_wake);
Jens Axboede9b4cc2021-02-24 13:28:27 -07005033 req_ref_get(req);
Jens Axboe18bceab2020-05-15 11:56:54 -06005034 poll->wait.private = req;
Jens Axboe807abcb2020-07-17 17:09:27 -06005035 *poll_ptr = poll;
Jens Axboe18bceab2020-05-15 11:56:54 -06005036 }
5037
5038 pt->error = 0;
5039 poll->head = head;
Jiufei Xuea31eb4a2020-06-17 17:53:56 +08005040
5041 if (poll->events & EPOLLEXCLUSIVE)
5042 add_wait_queue_exclusive(head, &poll->wait);
5043 else
5044 add_wait_queue(head, &poll->wait);
Jens Axboe18bceab2020-05-15 11:56:54 -06005045}
5046
5047static void io_async_queue_proc(struct file *file, struct wait_queue_head *head,
5048 struct poll_table_struct *p)
5049{
5050 struct io_poll_table *pt = container_of(p, struct io_poll_table, pt);
Jens Axboe807abcb2020-07-17 17:09:27 -06005051 struct async_poll *apoll = pt->req->apoll;
Jens Axboe18bceab2020-05-15 11:56:54 -06005052
Jens Axboe807abcb2020-07-17 17:09:27 -06005053 __io_queue_proc(&apoll->poll, pt, head, &apoll->double_poll);
Jens Axboe18bceab2020-05-15 11:56:54 -06005054}
5055
Jens Axboed7718a92020-02-14 22:23:12 -07005056static void io_async_task_func(struct callback_head *cb)
5057{
5058 struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
5059 struct async_poll *apoll = req->apoll;
5060 struct io_ring_ctx *ctx = req->ctx;
5061
5062 trace_io_uring_task_run(req->ctx, req->opcode, req->user_data);
5063
Jens Axboe74ce6ce2020-04-13 11:09:12 -06005064 if (io_poll_rewait(req, &apoll->poll)) {
Jens Axboed7718a92020-02-14 22:23:12 -07005065 spin_unlock_irq(&ctx->completion_lock);
Jens Axboe74ce6ce2020-04-13 11:09:12 -06005066 return;
Jens Axboed7718a92020-02-14 22:23:12 -07005067 }
5068
Pavel Begunkov0ea13b42021-04-09 09:13:21 +01005069 hash_del(&req->hash_node);
Jens Axboed4e7cd32020-08-15 11:44:50 -07005070 io_poll_remove_double(req);
Jens Axboe74ce6ce2020-04-13 11:09:12 -06005071 spin_unlock_irq(&ctx->completion_lock);
5072
Pavel Begunkov0be0b0e2020-06-30 15:20:42 +03005073 if (!READ_ONCE(apoll->poll.canceled))
5074 __io_req_task_submit(req);
5075 else
Pavel Begunkov25935532021-03-19 17:22:40 +00005076 io_req_complete_failed(req, -ECANCELED);
Jens Axboed7718a92020-02-14 22:23:12 -07005077}
5078
5079static int io_async_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
5080 void *key)
5081{
5082 struct io_kiocb *req = wait->private;
5083 struct io_poll_iocb *poll = &req->apoll->poll;
5084
5085 trace_io_uring_poll_wake(req->ctx, req->opcode, req->user_data,
5086 key_to_poll(key));
5087
5088 return __io_async_wake(req, poll, key_to_poll(key), io_async_task_func);
5089}
5090
5091static void io_poll_req_insert(struct io_kiocb *req)
5092{
5093 struct io_ring_ctx *ctx = req->ctx;
5094 struct hlist_head *list;
5095
5096 list = &ctx->cancel_hash[hash_long(req->user_data, ctx->cancel_hash_bits)];
5097 hlist_add_head(&req->hash_node, list);
5098}
5099
5100static __poll_t __io_arm_poll_handler(struct io_kiocb *req,
5101 struct io_poll_iocb *poll,
5102 struct io_poll_table *ipt, __poll_t mask,
5103 wait_queue_func_t wake_func)
5104 __acquires(&ctx->completion_lock)
5105{
5106 struct io_ring_ctx *ctx = req->ctx;
5107 bool cancel = false;
5108
Pavel Begunkov4d52f332020-10-18 10:17:43 +01005109 INIT_HLIST_NODE(&req->hash_node);
Jens Axboe18bceab2020-05-15 11:56:54 -06005110 io_init_poll_iocb(poll, mask, wake_func);
Pavel Begunkovb90cd192020-06-21 13:09:52 +03005111 poll->file = req->file;
Jens Axboe18bceab2020-05-15 11:56:54 -06005112 poll->wait.private = req;
Jens Axboed7718a92020-02-14 22:23:12 -07005113
5114 ipt->pt._key = mask;
5115 ipt->req = req;
5116 ipt->error = -EINVAL;
5117
Jens Axboed7718a92020-02-14 22:23:12 -07005118 mask = vfs_poll(req->file, &ipt->pt) & poll->events;
5119
5120 spin_lock_irq(&ctx->completion_lock);
5121 if (likely(poll->head)) {
5122 spin_lock(&poll->head->lock);
5123 if (unlikely(list_empty(&poll->wait.entry))) {
5124 if (ipt->error)
5125 cancel = true;
5126 ipt->error = 0;
5127 mask = 0;
5128 }
Jens Axboe88e41cf2021-02-22 22:08:01 -07005129 if ((mask && (poll->events & EPOLLONESHOT)) || ipt->error)
Jens Axboed7718a92020-02-14 22:23:12 -07005130 list_del_init(&poll->wait.entry);
5131 else if (cancel)
5132 WRITE_ONCE(poll->canceled, true);
5133 else if (!poll->done) /* actually waiting for an event */
5134 io_poll_req_insert(req);
5135 spin_unlock(&poll->head->lock);
5136 }
5137
5138 return mask;
5139}
5140
5141static bool io_arm_poll_handler(struct io_kiocb *req)
5142{
5143 const struct io_op_def *def = &io_op_defs[req->opcode];
5144 struct io_ring_ctx *ctx = req->ctx;
5145 struct async_poll *apoll;
5146 struct io_poll_table ipt;
5147 __poll_t mask, ret;
Jens Axboe9dab14b2020-08-25 12:27:50 -06005148 int rw;
Jens Axboed7718a92020-02-14 22:23:12 -07005149
5150 if (!req->file || !file_can_poll(req->file))
5151 return false;
Pavel Begunkov24c74672020-06-21 13:09:51 +03005152 if (req->flags & REQ_F_POLLED)
Jens Axboed7718a92020-02-14 22:23:12 -07005153 return false;
Jens Axboe9dab14b2020-08-25 12:27:50 -06005154 if (def->pollin)
5155 rw = READ;
5156 else if (def->pollout)
5157 rw = WRITE;
5158 else
5159 return false;
5160 /* if we can't nonblock try, then no point in arming a poll handler */
Jens Axboe7b29f922021-03-12 08:30:14 -07005161 if (!io_file_supports_async(req, rw))
Jens Axboed7718a92020-02-14 22:23:12 -07005162 return false;
5163
5164 apoll = kmalloc(sizeof(*apoll), GFP_ATOMIC);
5165 if (unlikely(!apoll))
5166 return false;
Jens Axboe807abcb2020-07-17 17:09:27 -06005167 apoll->double_poll = NULL;
Jens Axboed7718a92020-02-14 22:23:12 -07005168
5169 req->flags |= REQ_F_POLLED;
Jens Axboed7718a92020-02-14 22:23:12 -07005170 req->apoll = apoll;
Jens Axboed7718a92020-02-14 22:23:12 -07005171
Jens Axboe88e41cf2021-02-22 22:08:01 -07005172 mask = EPOLLONESHOT;
Jens Axboed7718a92020-02-14 22:23:12 -07005173 if (def->pollin)
Nathan Chancellor8755d972020-03-02 16:01:19 -07005174 mask |= POLLIN | POLLRDNORM;
Jens Axboed7718a92020-02-14 22:23:12 -07005175 if (def->pollout)
5176 mask |= POLLOUT | POLLWRNORM;
Luke Hsiao901341b2020-08-21 21:41:05 -07005177
5178 /* If reading from MSG_ERRQUEUE using recvmsg, ignore POLLIN */
5179 if ((req->opcode == IORING_OP_RECVMSG) &&
5180 (req->sr_msg.msg_flags & MSG_ERRQUEUE))
5181 mask &= ~POLLIN;
5182
Jens Axboed7718a92020-02-14 22:23:12 -07005183 mask |= POLLERR | POLLPRI;
5184
5185 ipt.pt._qproc = io_async_queue_proc;
5186
5187 ret = __io_arm_poll_handler(req, &apoll->poll, &ipt, mask,
5188 io_async_wake);
Jens Axboea36da652020-08-11 09:50:19 -06005189 if (ret || ipt.error) {
Jens Axboed4e7cd32020-08-15 11:44:50 -07005190 io_poll_remove_double(req);
Jens Axboed7718a92020-02-14 22:23:12 -07005191 spin_unlock_irq(&ctx->completion_lock);
Jens Axboed7718a92020-02-14 22:23:12 -07005192 return false;
5193 }
5194 spin_unlock_irq(&ctx->completion_lock);
5195 trace_io_uring_poll_arm(ctx, req->opcode, req->user_data, mask,
5196 apoll->poll.events);
5197 return true;
5198}
5199
5200static bool __io_poll_remove_one(struct io_kiocb *req,
Jens Axboeb2e720a2021-03-31 09:03:03 -06005201 struct io_poll_iocb *poll, bool do_cancel)
Pavel Begunkove07785b2021-04-01 15:43:57 +01005202 __must_hold(&req->ctx->completion_lock)
Jens Axboed7718a92020-02-14 22:23:12 -07005203{
Jens Axboeb41e9852020-02-17 09:52:41 -07005204 bool do_complete = false;
Jens Axboe221c5eb2019-01-17 09:41:58 -07005205
Jens Axboe50826202021-02-23 09:02:26 -07005206 if (!poll->head)
5207 return false;
Jens Axboe221c5eb2019-01-17 09:41:58 -07005208 spin_lock(&poll->head->lock);
Jens Axboeb2e720a2021-03-31 09:03:03 -06005209 if (do_cancel)
5210 WRITE_ONCE(poll->canceled, true);
Jens Axboe392edb42019-12-09 17:52:20 -07005211 if (!list_empty(&poll->wait.entry)) {
5212 list_del_init(&poll->wait.entry);
Jens Axboeb41e9852020-02-17 09:52:41 -07005213 do_complete = true;
Jens Axboe221c5eb2019-01-17 09:41:58 -07005214 }
5215 spin_unlock(&poll->head->lock);
Jens Axboe3bfa5bc2020-05-17 13:54:12 -06005216 hash_del(&req->hash_node);
Jens Axboed7718a92020-02-14 22:23:12 -07005217 return do_complete;
5218}
5219
Jens Axboeb2c3f7e2021-02-23 08:58:04 -07005220static bool io_poll_remove_waitqs(struct io_kiocb *req)
Pavel Begunkove07785b2021-04-01 15:43:57 +01005221 __must_hold(&req->ctx->completion_lock)
Jens Axboed7718a92020-02-14 22:23:12 -07005222{
5223 bool do_complete;
5224
Jens Axboed4e7cd32020-08-15 11:44:50 -07005225 io_poll_remove_double(req);
Pavel Begunkove31001a2021-04-13 02:58:43 +01005226 do_complete = __io_poll_remove_one(req, io_poll_get_single(req), true);
Jens Axboed4e7cd32020-08-15 11:44:50 -07005227
Pavel Begunkove31001a2021-04-13 02:58:43 +01005228 if (req->opcode != IORING_OP_POLL_ADD && do_complete) {
Jens Axboed7718a92020-02-14 22:23:12 -07005229 /* non-poll requests have submit ref still */
Pavel Begunkove31001a2021-04-13 02:58:43 +01005230 req_ref_put(req);
Xiaoguang Wangb1f573b2020-04-12 14:50:54 +08005231 }
Jens Axboeb2c3f7e2021-02-23 08:58:04 -07005232 return do_complete;
5233}
Xiaoguang Wangb1f573b2020-04-12 14:50:54 +08005234
Jens Axboeb2c3f7e2021-02-23 08:58:04 -07005235static bool io_poll_remove_one(struct io_kiocb *req)
Pavel Begunkove07785b2021-04-01 15:43:57 +01005236 __must_hold(&req->ctx->completion_lock)
Jens Axboeb2c3f7e2021-02-23 08:58:04 -07005237{
5238 bool do_complete;
5239
5240 do_complete = io_poll_remove_waitqs(req);
Jens Axboeb41e9852020-02-17 09:52:41 -07005241 if (do_complete) {
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01005242 io_cqring_fill_event(req->ctx, req->user_data, -ECANCELED, 0);
Jens Axboeb41e9852020-02-17 09:52:41 -07005243 io_commit_cqring(req->ctx);
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01005244 req_set_fail(req);
Pavel Begunkov216578e2020-10-13 09:44:00 +01005245 io_put_req_deferred(req, 1);
Jens Axboeb41e9852020-02-17 09:52:41 -07005246 }
5247
5248 return do_complete;
Jens Axboe221c5eb2019-01-17 09:41:58 -07005249}
5250
Jens Axboe76e1b642020-09-26 15:05:03 -06005251/*
5252 * Returns true if we found and killed one or more poll requests
5253 */
Pavel Begunkov6b819282020-11-06 13:00:25 +00005254static bool io_poll_remove_all(struct io_ring_ctx *ctx, struct task_struct *tsk,
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01005255 bool cancel_all)
Jens Axboe221c5eb2019-01-17 09:41:58 -07005256{
Jens Axboe78076bb2019-12-04 19:56:40 -07005257 struct hlist_node *tmp;
Jens Axboe221c5eb2019-01-17 09:41:58 -07005258 struct io_kiocb *req;
Jens Axboe8e2e1fa2020-04-13 17:05:14 -06005259 int posted = 0, i;
Jens Axboe221c5eb2019-01-17 09:41:58 -07005260
5261 spin_lock_irq(&ctx->completion_lock);
Jens Axboe78076bb2019-12-04 19:56:40 -07005262 for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) {
5263 struct hlist_head *list;
5264
5265 list = &ctx->cancel_hash[i];
Jens Axboef3606e32020-09-22 08:18:24 -06005266 hlist_for_each_entry_safe(req, tmp, list, hash_node) {
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01005267 if (io_match_task(req, tsk, cancel_all))
Jens Axboef3606e32020-09-22 08:18:24 -06005268 posted += io_poll_remove_one(req);
5269 }
Jens Axboe221c5eb2019-01-17 09:41:58 -07005270 }
5271 spin_unlock_irq(&ctx->completion_lock);
Jens Axboeb41e9852020-02-17 09:52:41 -07005272
Jens Axboe8e2e1fa2020-04-13 17:05:14 -06005273 if (posted)
5274 io_cqring_ev_posted(ctx);
Jens Axboe76e1b642020-09-26 15:05:03 -06005275
5276 return posted != 0;
Jens Axboe221c5eb2019-01-17 09:41:58 -07005277}
5278
Pavel Begunkov9ba5fac2021-04-14 13:38:35 +01005279static struct io_kiocb *io_poll_find(struct io_ring_ctx *ctx, __u64 sqe_addr,
5280 bool poll_only)
Pavel Begunkove07785b2021-04-01 15:43:57 +01005281 __must_hold(&ctx->completion_lock)
Jens Axboe47f46762019-11-09 17:43:02 -07005282{
Jens Axboe78076bb2019-12-04 19:56:40 -07005283 struct hlist_head *list;
Jens Axboe47f46762019-11-09 17:43:02 -07005284 struct io_kiocb *req;
5285
Jens Axboe78076bb2019-12-04 19:56:40 -07005286 list = &ctx->cancel_hash[hash_long(sqe_addr, ctx->cancel_hash_bits)];
5287 hlist_for_each_entry(req, list, hash_node) {
Jens Axboeb41e9852020-02-17 09:52:41 -07005288 if (sqe_addr != req->user_data)
5289 continue;
Pavel Begunkov9ba5fac2021-04-14 13:38:35 +01005290 if (poll_only && req->opcode != IORING_OP_POLL_ADD)
5291 continue;
Jens Axboeb2cb8052021-03-17 08:17:19 -06005292 return req;
Jens Axboe47f46762019-11-09 17:43:02 -07005293 }
Jens Axboeb2cb8052021-03-17 08:17:19 -06005294 return NULL;
Jens Axboe47f46762019-11-09 17:43:02 -07005295}
5296
Pavel Begunkov9ba5fac2021-04-14 13:38:35 +01005297static int io_poll_cancel(struct io_ring_ctx *ctx, __u64 sqe_addr,
5298 bool poll_only)
Pavel Begunkove07785b2021-04-01 15:43:57 +01005299 __must_hold(&ctx->completion_lock)
Jens Axboeb2cb8052021-03-17 08:17:19 -06005300{
5301 struct io_kiocb *req;
5302
Pavel Begunkov9ba5fac2021-04-14 13:38:35 +01005303 req = io_poll_find(ctx, sqe_addr, poll_only);
Jens Axboeb2cb8052021-03-17 08:17:19 -06005304 if (!req)
5305 return -ENOENT;
5306 if (io_poll_remove_one(req))
5307 return 0;
5308
5309 return -EALREADY;
Jens Axboe221c5eb2019-01-17 09:41:58 -07005310}
5311
Pavel Begunkov9096af32021-04-14 13:38:36 +01005312static __poll_t io_poll_parse_events(const struct io_uring_sqe *sqe,
5313 unsigned int flags)
5314{
5315 u32 events;
5316
5317 events = READ_ONCE(sqe->poll32_events);
5318#ifdef __BIG_ENDIAN
5319 events = swahw32(events);
5320#endif
5321 if (!(flags & IORING_POLL_ADD_MULTI))
5322 events |= EPOLLONESHOT;
5323 return demangle_poll(events) | (events & (EPOLLEXCLUSIVE|EPOLLONESHOT));
5324}
5325
Pavel Begunkovc5de0032021-04-14 13:38:37 +01005326static int io_poll_update_prep(struct io_kiocb *req,
Jens Axboe3529d8c2019-12-19 18:24:38 -07005327 const struct io_uring_sqe *sqe)
Jens Axboe221c5eb2019-01-17 09:41:58 -07005328{
Pavel Begunkovc5de0032021-04-14 13:38:37 +01005329 struct io_poll_update *upd = &req->poll_update;
5330 u32 flags;
5331
Jens Axboe221c5eb2019-01-17 09:41:58 -07005332 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
5333 return -EINVAL;
Pavel Begunkovc5de0032021-04-14 13:38:37 +01005334 if (sqe->ioprio || sqe->buf_index)
5335 return -EINVAL;
5336 flags = READ_ONCE(sqe->len);
5337 if (flags & ~(IORING_POLL_UPDATE_EVENTS | IORING_POLL_UPDATE_USER_DATA |
5338 IORING_POLL_ADD_MULTI))
5339 return -EINVAL;
5340 /* meaningless without update */
5341 if (flags == IORING_POLL_ADD_MULTI)
Jens Axboe221c5eb2019-01-17 09:41:58 -07005342 return -EINVAL;
5343
Pavel Begunkovc5de0032021-04-14 13:38:37 +01005344 upd->old_user_data = READ_ONCE(sqe->addr);
5345 upd->update_events = flags & IORING_POLL_UPDATE_EVENTS;
5346 upd->update_user_data = flags & IORING_POLL_UPDATE_USER_DATA;
Jens Axboe0969e782019-12-17 18:40:57 -07005347
Pavel Begunkovc5de0032021-04-14 13:38:37 +01005348 upd->new_user_data = READ_ONCE(sqe->off);
5349 if (!upd->update_user_data && upd->new_user_data)
5350 return -EINVAL;
5351 if (upd->update_events)
5352 upd->events = io_poll_parse_events(sqe, flags);
5353 else if (sqe->poll32_events)
5354 return -EINVAL;
Jens Axboe0969e782019-12-17 18:40:57 -07005355
Jens Axboe221c5eb2019-01-17 09:41:58 -07005356 return 0;
5357}
5358
Jens Axboe221c5eb2019-01-17 09:41:58 -07005359static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
5360 void *key)
5361{
Jens Axboec2f2eb72020-02-10 09:07:05 -07005362 struct io_kiocb *req = wait->private;
5363 struct io_poll_iocb *poll = &req->poll;
Jens Axboe221c5eb2019-01-17 09:41:58 -07005364
Jens Axboed7718a92020-02-14 22:23:12 -07005365 return __io_async_wake(req, poll, key_to_poll(key), io_poll_task_func);
Jens Axboe221c5eb2019-01-17 09:41:58 -07005366}
5367
Jens Axboe221c5eb2019-01-17 09:41:58 -07005368static void io_poll_queue_proc(struct file *file, struct wait_queue_head *head,
5369 struct poll_table_struct *p)
5370{
5371 struct io_poll_table *pt = container_of(p, struct io_poll_table, pt);
5372
Jens Axboee8c2bc12020-08-15 18:44:09 -07005373 __io_queue_proc(&pt->req->poll, pt, head, (struct io_poll_iocb **) &pt->req->async_data);
Jens Axboeeac406c2019-11-14 12:09:58 -07005374}
5375
Jens Axboe3529d8c2019-12-19 18:24:38 -07005376static int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboe221c5eb2019-01-17 09:41:58 -07005377{
5378 struct io_poll_iocb *poll = &req->poll;
Pavel Begunkovc5de0032021-04-14 13:38:37 +01005379 u32 flags;
Jens Axboe221c5eb2019-01-17 09:41:58 -07005380
5381 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
5382 return -EINVAL;
Pavel Begunkovc5de0032021-04-14 13:38:37 +01005383 if (sqe->ioprio || sqe->buf_index || sqe->off || sqe->addr)
Jens Axboe88e41cf2021-02-22 22:08:01 -07005384 return -EINVAL;
5385 flags = READ_ONCE(sqe->len);
Pavel Begunkovc5de0032021-04-14 13:38:37 +01005386 if (flags & ~IORING_POLL_ADD_MULTI)
Jens Axboe221c5eb2019-01-17 09:41:58 -07005387 return -EINVAL;
5388
Pavel Begunkovc5de0032021-04-14 13:38:37 +01005389 poll->events = io_poll_parse_events(sqe, flags);
Jens Axboe0969e782019-12-17 18:40:57 -07005390 return 0;
5391}
5392
Pavel Begunkov61e98202021-02-10 00:03:08 +00005393static int io_poll_add(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe0969e782019-12-17 18:40:57 -07005394{
5395 struct io_poll_iocb *poll = &req->poll;
5396 struct io_ring_ctx *ctx = req->ctx;
5397 struct io_poll_table ipt;
Jens Axboe0969e782019-12-17 18:40:57 -07005398 __poll_t mask;
Jens Axboe0969e782019-12-17 18:40:57 -07005399
Jens Axboed7718a92020-02-14 22:23:12 -07005400 ipt.pt._qproc = io_poll_queue_proc;
Jens Axboe36703242019-07-25 10:20:18 -06005401
Jens Axboed7718a92020-02-14 22:23:12 -07005402 mask = __io_arm_poll_handler(req, &req->poll, &ipt, poll->events,
5403 io_poll_wake);
Jens Axboe221c5eb2019-01-17 09:41:58 -07005404
Jens Axboe8c838782019-03-12 15:48:16 -06005405 if (mask) { /* no async, we'd stolen it */
Jens Axboe8c838782019-03-12 15:48:16 -06005406 ipt.error = 0;
Pavel Begunkove27414b2021-04-09 09:13:20 +01005407 io_poll_complete(req, mask);
Jens Axboe8c838782019-03-12 15:48:16 -06005408 }
Jens Axboe221c5eb2019-01-17 09:41:58 -07005409 spin_unlock_irq(&ctx->completion_lock);
5410
Jens Axboe8c838782019-03-12 15:48:16 -06005411 if (mask) {
5412 io_cqring_ev_posted(ctx);
Jens Axboe88e41cf2021-02-22 22:08:01 -07005413 if (poll->events & EPOLLONESHOT)
5414 io_put_req(req);
Jens Axboe221c5eb2019-01-17 09:41:58 -07005415 }
Jens Axboe8c838782019-03-12 15:48:16 -06005416 return ipt.error;
Jens Axboe221c5eb2019-01-17 09:41:58 -07005417}
5418
Pavel Begunkovc5de0032021-04-14 13:38:37 +01005419static int io_poll_update(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboeb69de282021-03-17 08:37:41 -06005420{
5421 struct io_ring_ctx *ctx = req->ctx;
5422 struct io_kiocb *preq;
Jens Axboecb3b200e2021-04-06 09:49:31 -06005423 bool completing;
Jens Axboeb69de282021-03-17 08:37:41 -06005424 int ret;
5425
5426 spin_lock_irq(&ctx->completion_lock);
Pavel Begunkov9ba5fac2021-04-14 13:38:35 +01005427 preq = io_poll_find(ctx, req->poll_update.old_user_data, true);
Jens Axboeb69de282021-03-17 08:37:41 -06005428 if (!preq) {
5429 ret = -ENOENT;
5430 goto err;
Jens Axboeb69de282021-03-17 08:37:41 -06005431 }
Jens Axboecb3b200e2021-04-06 09:49:31 -06005432
Pavel Begunkovc5de0032021-04-14 13:38:37 +01005433 if (!req->poll_update.update_events && !req->poll_update.update_user_data) {
5434 completing = true;
5435 ret = io_poll_remove_one(preq) ? 0 : -EALREADY;
5436 goto err;
5437 }
5438
Jens Axboecb3b200e2021-04-06 09:49:31 -06005439 /*
5440 * Don't allow racy completion with singleshot, as we cannot safely
5441 * update those. For multishot, if we're racing with completion, just
5442 * let completion re-add it.
5443 */
5444 completing = !__io_poll_remove_one(preq, &preq->poll, false);
5445 if (completing && (preq->poll.events & EPOLLONESHOT)) {
5446 ret = -EALREADY;
5447 goto err;
Jens Axboeb69de282021-03-17 08:37:41 -06005448 }
5449 /* we now have a detached poll request. reissue. */
5450 ret = 0;
5451err:
Jens Axboeb69de282021-03-17 08:37:41 -06005452 if (ret < 0) {
Jens Axboecb3b200e2021-04-06 09:49:31 -06005453 spin_unlock_irq(&ctx->completion_lock);
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01005454 req_set_fail(req);
Jens Axboeb69de282021-03-17 08:37:41 -06005455 io_req_complete(req, ret);
5456 return 0;
5457 }
5458 /* only mask one event flags, keep behavior flags */
Pavel Begunkov9d805892021-04-13 02:58:40 +01005459 if (req->poll_update.update_events) {
Jens Axboeb69de282021-03-17 08:37:41 -06005460 preq->poll.events &= ~0xffff;
Pavel Begunkov9d805892021-04-13 02:58:40 +01005461 preq->poll.events |= req->poll_update.events & 0xffff;
Jens Axboeb69de282021-03-17 08:37:41 -06005462 preq->poll.events |= IO_POLL_UNMASK;
5463 }
Pavel Begunkov9d805892021-04-13 02:58:40 +01005464 if (req->poll_update.update_user_data)
5465 preq->user_data = req->poll_update.new_user_data;
Jens Axboecb3b200e2021-04-06 09:49:31 -06005466 spin_unlock_irq(&ctx->completion_lock);
5467
Jens Axboeb69de282021-03-17 08:37:41 -06005468 /* complete update request, we're done with it */
5469 io_req_complete(req, ret);
5470
Jens Axboecb3b200e2021-04-06 09:49:31 -06005471 if (!completing) {
Pavel Begunkovc5de0032021-04-14 13:38:37 +01005472 ret = io_poll_add(preq, issue_flags);
Jens Axboecb3b200e2021-04-06 09:49:31 -06005473 if (ret < 0) {
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01005474 req_set_fail(preq);
Jens Axboecb3b200e2021-04-06 09:49:31 -06005475 io_req_complete(preq, ret);
5476 }
Jens Axboeb69de282021-03-17 08:37:41 -06005477 }
5478 return 0;
5479}
5480
Jens Axboe5262f562019-09-17 12:26:57 -06005481static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer)
5482{
Jens Axboead8a48a2019-11-15 08:49:11 -07005483 struct io_timeout_data *data = container_of(timer,
5484 struct io_timeout_data, timer);
5485 struct io_kiocb *req = data->req;
5486 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe5262f562019-09-17 12:26:57 -06005487 unsigned long flags;
5488
Jens Axboe5262f562019-09-17 12:26:57 -06005489 spin_lock_irqsave(&ctx->completion_lock, flags);
Pavel Begunkova71976f2020-10-10 18:34:11 +01005490 list_del_init(&req->timeout.list);
Pavel Begunkov01cec8c2020-07-30 18:43:50 +03005491 atomic_set(&req->ctx->cq_timeouts,
5492 atomic_read(&req->ctx->cq_timeouts) + 1);
5493
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01005494 io_cqring_fill_event(ctx, req->user_data, -ETIME, 0);
Jens Axboe5262f562019-09-17 12:26:57 -06005495 io_commit_cqring(ctx);
5496 spin_unlock_irqrestore(&ctx->completion_lock, flags);
5497
5498 io_cqring_ev_posted(ctx);
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01005499 req_set_fail(req);
Jens Axboe5262f562019-09-17 12:26:57 -06005500 io_put_req(req);
5501 return HRTIMER_NORESTART;
5502}
5503
Pavel Begunkovfbd15842020-11-30 19:11:15 +00005504static struct io_kiocb *io_timeout_extract(struct io_ring_ctx *ctx,
5505 __u64 user_data)
Pavel Begunkove07785b2021-04-01 15:43:57 +01005506 __must_hold(&ctx->completion_lock)
Jens Axboe47f46762019-11-09 17:43:02 -07005507{
Pavel Begunkovfbd15842020-11-30 19:11:15 +00005508 struct io_timeout_data *io;
Jens Axboef254ac02020-08-12 17:33:30 -06005509 struct io_kiocb *req;
Pavel Begunkovfd9c7bc2021-04-13 02:58:42 +01005510 bool found = false;
Jens Axboef254ac02020-08-12 17:33:30 -06005511
5512 list_for_each_entry(req, &ctx->timeout_list, timeout.list) {
Pavel Begunkovfd9c7bc2021-04-13 02:58:42 +01005513 found = user_data == req->user_data;
5514 if (found)
Jens Axboef254ac02020-08-12 17:33:30 -06005515 break;
Jens Axboef254ac02020-08-12 17:33:30 -06005516 }
Pavel Begunkovfd9c7bc2021-04-13 02:58:42 +01005517 if (!found)
5518 return ERR_PTR(-ENOENT);
Jens Axboef254ac02020-08-12 17:33:30 -06005519
Pavel Begunkovfbd15842020-11-30 19:11:15 +00005520 io = req->async_data;
Pavel Begunkovfd9c7bc2021-04-13 02:58:42 +01005521 if (hrtimer_try_to_cancel(&io->timer) == -1)
Pavel Begunkovfbd15842020-11-30 19:11:15 +00005522 return ERR_PTR(-EALREADY);
5523 list_del_init(&req->timeout.list);
5524 return req;
5525}
5526
5527static int io_timeout_cancel(struct io_ring_ctx *ctx, __u64 user_data)
Pavel Begunkove07785b2021-04-01 15:43:57 +01005528 __must_hold(&ctx->completion_lock)
Pavel Begunkovfbd15842020-11-30 19:11:15 +00005529{
5530 struct io_kiocb *req = io_timeout_extract(ctx, user_data);
5531
5532 if (IS_ERR(req))
5533 return PTR_ERR(req);
5534
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01005535 req_set_fail(req);
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01005536 io_cqring_fill_event(ctx, req->user_data, -ECANCELED, 0);
Pavel Begunkovfbd15842020-11-30 19:11:15 +00005537 io_put_req_deferred(req, 1);
5538 return 0;
Jens Axboef254ac02020-08-12 17:33:30 -06005539}
5540
Pavel Begunkov9c8e11b2020-11-30 19:11:16 +00005541static int io_timeout_update(struct io_ring_ctx *ctx, __u64 user_data,
5542 struct timespec64 *ts, enum hrtimer_mode mode)
Pavel Begunkove07785b2021-04-01 15:43:57 +01005543 __must_hold(&ctx->completion_lock)
Pavel Begunkov9c8e11b2020-11-30 19:11:16 +00005544{
5545 struct io_kiocb *req = io_timeout_extract(ctx, user_data);
5546 struct io_timeout_data *data;
5547
5548 if (IS_ERR(req))
5549 return PTR_ERR(req);
5550
5551 req->timeout.off = 0; /* noseq */
5552 data = req->async_data;
5553 list_add_tail(&req->timeout.list, &ctx->timeout_list);
5554 hrtimer_init(&data->timer, CLOCK_MONOTONIC, mode);
5555 data->timer.function = io_timeout_fn;
5556 hrtimer_start(&data->timer, timespec64_to_ktime(*ts), mode);
5557 return 0;
Jens Axboe47f46762019-11-09 17:43:02 -07005558}
5559
Jens Axboe3529d8c2019-12-19 18:24:38 -07005560static int io_timeout_remove_prep(struct io_kiocb *req,
5561 const struct io_uring_sqe *sqe)
Jens Axboeb29472e2019-12-17 18:50:29 -07005562{
Pavel Begunkov9c8e11b2020-11-30 19:11:16 +00005563 struct io_timeout_rem *tr = &req->timeout_rem;
5564
Jens Axboeb29472e2019-12-17 18:50:29 -07005565 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
5566 return -EINVAL;
Daniele Albano61710e42020-07-18 14:15:16 -06005567 if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT)))
5568 return -EINVAL;
Pavel Begunkov9c8e11b2020-11-30 19:11:16 +00005569 if (sqe->ioprio || sqe->buf_index || sqe->len)
Jens Axboeb29472e2019-12-17 18:50:29 -07005570 return -EINVAL;
5571
Pavel Begunkov9c8e11b2020-11-30 19:11:16 +00005572 tr->addr = READ_ONCE(sqe->addr);
5573 tr->flags = READ_ONCE(sqe->timeout_flags);
5574 if (tr->flags & IORING_TIMEOUT_UPDATE) {
5575 if (tr->flags & ~(IORING_TIMEOUT_UPDATE|IORING_TIMEOUT_ABS))
5576 return -EINVAL;
5577 if (get_timespec64(&tr->ts, u64_to_user_ptr(sqe->addr2)))
5578 return -EFAULT;
5579 } else if (tr->flags) {
5580 /* timeout removal doesn't support flags */
5581 return -EINVAL;
5582 }
5583
Jens Axboeb29472e2019-12-17 18:50:29 -07005584 return 0;
5585}
5586
Pavel Begunkov8662dae2021-01-19 13:32:44 +00005587static inline enum hrtimer_mode io_translate_timeout_mode(unsigned int flags)
5588{
5589 return (flags & IORING_TIMEOUT_ABS) ? HRTIMER_MODE_ABS
5590 : HRTIMER_MODE_REL;
5591}
5592
Jens Axboe11365042019-10-16 09:08:32 -06005593/*
5594 * Remove or update an existing timeout command
5595 */
Pavel Begunkov61e98202021-02-10 00:03:08 +00005596static int io_timeout_remove(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe11365042019-10-16 09:08:32 -06005597{
Pavel Begunkov9c8e11b2020-11-30 19:11:16 +00005598 struct io_timeout_rem *tr = &req->timeout_rem;
Jens Axboe11365042019-10-16 09:08:32 -06005599 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe47f46762019-11-09 17:43:02 -07005600 int ret;
Jens Axboe11365042019-10-16 09:08:32 -06005601
Jens Axboe11365042019-10-16 09:08:32 -06005602 spin_lock_irq(&ctx->completion_lock);
Pavel Begunkov8662dae2021-01-19 13:32:44 +00005603 if (!(req->timeout_rem.flags & IORING_TIMEOUT_UPDATE))
Pavel Begunkov9c8e11b2020-11-30 19:11:16 +00005604 ret = io_timeout_cancel(ctx, tr->addr);
Pavel Begunkov8662dae2021-01-19 13:32:44 +00005605 else
5606 ret = io_timeout_update(ctx, tr->addr, &tr->ts,
5607 io_translate_timeout_mode(tr->flags));
Jens Axboe11365042019-10-16 09:08:32 -06005608
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01005609 io_cqring_fill_event(ctx, req->user_data, ret, 0);
Jens Axboe11365042019-10-16 09:08:32 -06005610 io_commit_cqring(ctx);
5611 spin_unlock_irq(&ctx->completion_lock);
Jens Axboe5262f562019-09-17 12:26:57 -06005612 io_cqring_ev_posted(ctx);
Jens Axboe4e88d6e2019-12-07 20:59:47 -07005613 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01005614 req_set_fail(req);
Jackie Liuec9c02a2019-11-08 23:50:36 +08005615 io_put_req(req);
Jens Axboe11365042019-10-16 09:08:32 -06005616 return 0;
Jens Axboe5262f562019-09-17 12:26:57 -06005617}
5618
Jens Axboe3529d8c2019-12-19 18:24:38 -07005619static int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe,
Jens Axboe2d283902019-12-04 11:08:05 -07005620 bool is_timeout_link)
Jens Axboe5262f562019-09-17 12:26:57 -06005621{
Jens Axboead8a48a2019-11-15 08:49:11 -07005622 struct io_timeout_data *data;
Jens Axboea41525a2019-10-15 16:48:15 -06005623 unsigned flags;
Pavel Begunkov56080b02020-05-26 20:34:04 +03005624 u32 off = READ_ONCE(sqe->off);
Jens Axboe5262f562019-09-17 12:26:57 -06005625
Jens Axboead8a48a2019-11-15 08:49:11 -07005626 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
Jens Axboe5262f562019-09-17 12:26:57 -06005627 return -EINVAL;
Jens Axboead8a48a2019-11-15 08:49:11 -07005628 if (sqe->ioprio || sqe->buf_index || sqe->len != 1)
Jens Axboea41525a2019-10-15 16:48:15 -06005629 return -EINVAL;
Pavel Begunkov56080b02020-05-26 20:34:04 +03005630 if (off && is_timeout_link)
Jens Axboe2d283902019-12-04 11:08:05 -07005631 return -EINVAL;
Jens Axboea41525a2019-10-15 16:48:15 -06005632 flags = READ_ONCE(sqe->timeout_flags);
5633 if (flags & ~IORING_TIMEOUT_ABS)
Jens Axboe5262f562019-09-17 12:26:57 -06005634 return -EINVAL;
Arnd Bergmannbdf20072019-10-01 09:53:29 -06005635
Pavel Begunkovbfe68a22020-05-30 14:54:18 +03005636 req->timeout.off = off;
Jens Axboe26a61672019-12-20 09:02:01 -07005637
Jens Axboee8c2bc12020-08-15 18:44:09 -07005638 if (!req->async_data && io_alloc_async_data(req))
Jens Axboe26a61672019-12-20 09:02:01 -07005639 return -ENOMEM;
5640
Jens Axboee8c2bc12020-08-15 18:44:09 -07005641 data = req->async_data;
Jens Axboead8a48a2019-11-15 08:49:11 -07005642 data->req = req;
Jens Axboead8a48a2019-11-15 08:49:11 -07005643
5644 if (get_timespec64(&data->ts, u64_to_user_ptr(sqe->addr)))
Jens Axboe5262f562019-09-17 12:26:57 -06005645 return -EFAULT;
5646
Pavel Begunkov8662dae2021-01-19 13:32:44 +00005647 data->mode = io_translate_timeout_mode(flags);
Jens Axboead8a48a2019-11-15 08:49:11 -07005648 hrtimer_init(&data->timer, CLOCK_MONOTONIC, data->mode);
Pavel Begunkov2482b582021-03-25 18:32:44 +00005649 if (is_timeout_link)
5650 io_req_track_inflight(req);
Jens Axboead8a48a2019-11-15 08:49:11 -07005651 return 0;
5652}
5653
Pavel Begunkov61e98202021-02-10 00:03:08 +00005654static int io_timeout(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboead8a48a2019-11-15 08:49:11 -07005655{
Jens Axboead8a48a2019-11-15 08:49:11 -07005656 struct io_ring_ctx *ctx = req->ctx;
Jens Axboee8c2bc12020-08-15 18:44:09 -07005657 struct io_timeout_data *data = req->async_data;
Jens Axboead8a48a2019-11-15 08:49:11 -07005658 struct list_head *entry;
Pavel Begunkovbfe68a22020-05-30 14:54:18 +03005659 u32 tail, off = req->timeout.off;
Jens Axboead8a48a2019-11-15 08:49:11 -07005660
Pavel Begunkov733f5c92020-05-26 20:34:03 +03005661 spin_lock_irq(&ctx->completion_lock);
Jens Axboe93bd25b2019-11-11 23:34:31 -07005662
Jens Axboe5262f562019-09-17 12:26:57 -06005663 /*
5664 * sqe->off holds how many events that need to occur for this
Jens Axboe93bd25b2019-11-11 23:34:31 -07005665 * timeout event to be satisfied. If it isn't set, then this is
5666 * a pure timeout request, sequence isn't used.
Jens Axboe5262f562019-09-17 12:26:57 -06005667 */
Pavel Begunkov8eb7e2d2020-06-29 13:13:02 +03005668 if (io_is_timeout_noseq(req)) {
Jens Axboe93bd25b2019-11-11 23:34:31 -07005669 entry = ctx->timeout_list.prev;
5670 goto add;
5671 }
Jens Axboe5262f562019-09-17 12:26:57 -06005672
Pavel Begunkovbfe68a22020-05-30 14:54:18 +03005673 tail = ctx->cached_cq_tail - atomic_read(&ctx->cq_timeouts);
5674 req->timeout.target_seq = tail + off;
Jens Axboe5262f562019-09-17 12:26:57 -06005675
Marcelo Diop-Gonzalezf0105052021-01-15 11:54:40 -05005676 /* Update the last seq here in case io_flush_timeouts() hasn't.
5677 * This is safe because ->completion_lock is held, and submissions
5678 * and completions are never mixed in the same ->completion_lock section.
5679 */
5680 ctx->cq_last_tm_flush = tail;
5681
Jens Axboe5262f562019-09-17 12:26:57 -06005682 /*
5683 * Insertion sort, ensuring the first entry in the list is always
5684 * the one we need first.
5685 */
Jens Axboe5262f562019-09-17 12:26:57 -06005686 list_for_each_prev(entry, &ctx->timeout_list) {
Pavel Begunkov135fcde2020-07-13 23:37:12 +03005687 struct io_kiocb *nxt = list_entry(entry, struct io_kiocb,
5688 timeout.list);
Jens Axboe5262f562019-09-17 12:26:57 -06005689
Pavel Begunkov8eb7e2d2020-06-29 13:13:02 +03005690 if (io_is_timeout_noseq(nxt))
Jens Axboe93bd25b2019-11-11 23:34:31 -07005691 continue;
Pavel Begunkovbfe68a22020-05-30 14:54:18 +03005692 /* nxt.seq is behind @tail, otherwise would've been completed */
5693 if (off >= nxt->timeout.target_seq - tail)
Jens Axboe5262f562019-09-17 12:26:57 -06005694 break;
5695 }
Jens Axboe93bd25b2019-11-11 23:34:31 -07005696add:
Pavel Begunkov135fcde2020-07-13 23:37:12 +03005697 list_add(&req->timeout.list, entry);
Jens Axboead8a48a2019-11-15 08:49:11 -07005698 data->timer.function = io_timeout_fn;
5699 hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), data->mode);
Jens Axboe842f9612019-10-29 12:34:10 -06005700 spin_unlock_irq(&ctx->completion_lock);
Jens Axboe5262f562019-09-17 12:26:57 -06005701 return 0;
5702}
5703
Pavel Begunkovf458dd842021-03-08 12:14:14 +00005704struct io_cancel_data {
5705 struct io_ring_ctx *ctx;
5706 u64 user_data;
5707};
5708
Jens Axboe62755e32019-10-28 21:49:21 -06005709static bool io_cancel_cb(struct io_wq_work *work, void *data)
Jens Axboede0617e2019-04-06 21:51:27 -06005710{
Jens Axboe62755e32019-10-28 21:49:21 -06005711 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
Pavel Begunkovf458dd842021-03-08 12:14:14 +00005712 struct io_cancel_data *cd = data;
Jens Axboede0617e2019-04-06 21:51:27 -06005713
Pavel Begunkovf458dd842021-03-08 12:14:14 +00005714 return req->ctx == cd->ctx && req->user_data == cd->user_data;
Jens Axboe62755e32019-10-28 21:49:21 -06005715}
5716
Pavel Begunkovf458dd842021-03-08 12:14:14 +00005717static int io_async_cancel_one(struct io_uring_task *tctx, u64 user_data,
5718 struct io_ring_ctx *ctx)
Jens Axboe62755e32019-10-28 21:49:21 -06005719{
Pavel Begunkovf458dd842021-03-08 12:14:14 +00005720 struct io_cancel_data data = { .ctx = ctx, .user_data = user_data, };
Jens Axboe62755e32019-10-28 21:49:21 -06005721 enum io_wq_cancel cancel_ret;
Jens Axboe62755e32019-10-28 21:49:21 -06005722 int ret = 0;
5723
Pavel Begunkovf458dd842021-03-08 12:14:14 +00005724 if (!tctx || !tctx->io_wq)
Jens Axboe5aa75ed2021-02-16 12:56:50 -07005725 return -ENOENT;
5726
Pavel Begunkovf458dd842021-03-08 12:14:14 +00005727 cancel_ret = io_wq_cancel_cb(tctx->io_wq, io_cancel_cb, &data, false);
Jens Axboe62755e32019-10-28 21:49:21 -06005728 switch (cancel_ret) {
5729 case IO_WQ_CANCEL_OK:
5730 ret = 0;
5731 break;
5732 case IO_WQ_CANCEL_RUNNING:
5733 ret = -EALREADY;
5734 break;
5735 case IO_WQ_CANCEL_NOTFOUND:
5736 ret = -ENOENT;
5737 break;
5738 }
5739
Jens Axboee977d6d2019-11-05 12:39:45 -07005740 return ret;
5741}
5742
Jens Axboe47f46762019-11-09 17:43:02 -07005743static void io_async_find_and_cancel(struct io_ring_ctx *ctx,
5744 struct io_kiocb *req, __u64 sqe_addr,
Pavel Begunkov014db002020-03-03 21:33:12 +03005745 int success_ret)
Jens Axboe47f46762019-11-09 17:43:02 -07005746{
5747 unsigned long flags;
5748 int ret;
5749
Pavel Begunkovf458dd842021-03-08 12:14:14 +00005750 ret = io_async_cancel_one(req->task->io_uring, sqe_addr, ctx);
Jens Axboe47f46762019-11-09 17:43:02 -07005751 spin_lock_irqsave(&ctx->completion_lock, flags);
Pavel Begunkovdf9727a2021-04-01 15:43:59 +01005752 if (ret != -ENOENT)
5753 goto done;
Jens Axboe47f46762019-11-09 17:43:02 -07005754 ret = io_timeout_cancel(ctx, sqe_addr);
5755 if (ret != -ENOENT)
5756 goto done;
Pavel Begunkov9ba5fac2021-04-14 13:38:35 +01005757 ret = io_poll_cancel(ctx, sqe_addr, false);
Jens Axboe47f46762019-11-09 17:43:02 -07005758done:
Jens Axboeb0dd8a42019-11-18 12:14:54 -07005759 if (!ret)
5760 ret = success_ret;
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01005761 io_cqring_fill_event(ctx, req->user_data, ret, 0);
Jens Axboe47f46762019-11-09 17:43:02 -07005762 io_commit_cqring(ctx);
5763 spin_unlock_irqrestore(&ctx->completion_lock, flags);
5764 io_cqring_ev_posted(ctx);
5765
Jens Axboe4e88d6e2019-12-07 20:59:47 -07005766 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01005767 req_set_fail(req);
Jens Axboe47f46762019-11-09 17:43:02 -07005768}
5769
Jens Axboe3529d8c2019-12-19 18:24:38 -07005770static int io_async_cancel_prep(struct io_kiocb *req,
5771 const struct io_uring_sqe *sqe)
Jens Axboee977d6d2019-11-05 12:39:45 -07005772{
Jens Axboefbf23842019-12-17 18:45:56 -07005773 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
Jens Axboee977d6d2019-11-05 12:39:45 -07005774 return -EINVAL;
Daniele Albano61710e42020-07-18 14:15:16 -06005775 if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT)))
5776 return -EINVAL;
5777 if (sqe->ioprio || sqe->off || sqe->len || sqe->cancel_flags)
Jens Axboee977d6d2019-11-05 12:39:45 -07005778 return -EINVAL;
5779
Jens Axboefbf23842019-12-17 18:45:56 -07005780 req->cancel.addr = READ_ONCE(sqe->addr);
5781 return 0;
5782}
5783
Pavel Begunkov61e98202021-02-10 00:03:08 +00005784static int io_async_cancel(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboefbf23842019-12-17 18:45:56 -07005785{
5786 struct io_ring_ctx *ctx = req->ctx;
Pavel Begunkov58f99372021-03-12 16:25:55 +00005787 u64 sqe_addr = req->cancel.addr;
5788 struct io_tctx_node *node;
5789 int ret;
Jens Axboefbf23842019-12-17 18:45:56 -07005790
Pavel Begunkov58f99372021-03-12 16:25:55 +00005791 /* tasks should wait for their io-wq threads, so safe w/o sync */
5792 ret = io_async_cancel_one(req->task->io_uring, sqe_addr, ctx);
5793 spin_lock_irq(&ctx->completion_lock);
5794 if (ret != -ENOENT)
5795 goto done;
5796 ret = io_timeout_cancel(ctx, sqe_addr);
5797 if (ret != -ENOENT)
5798 goto done;
Pavel Begunkov9ba5fac2021-04-14 13:38:35 +01005799 ret = io_poll_cancel(ctx, sqe_addr, false);
Pavel Begunkov58f99372021-03-12 16:25:55 +00005800 if (ret != -ENOENT)
5801 goto done;
5802 spin_unlock_irq(&ctx->completion_lock);
5803
5804 /* slow path, try all io-wq's */
5805 io_ring_submit_lock(ctx, !(issue_flags & IO_URING_F_NONBLOCK));
5806 ret = -ENOENT;
5807 list_for_each_entry(node, &ctx->tctx_list, ctx_node) {
5808 struct io_uring_task *tctx = node->task->io_uring;
5809
Pavel Begunkov58f99372021-03-12 16:25:55 +00005810 ret = io_async_cancel_one(tctx, req->cancel.addr, ctx);
5811 if (ret != -ENOENT)
5812 break;
5813 }
5814 io_ring_submit_unlock(ctx, !(issue_flags & IO_URING_F_NONBLOCK));
5815
5816 spin_lock_irq(&ctx->completion_lock);
5817done:
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01005818 io_cqring_fill_event(ctx, req->user_data, ret, 0);
Pavel Begunkov58f99372021-03-12 16:25:55 +00005819 io_commit_cqring(ctx);
5820 spin_unlock_irq(&ctx->completion_lock);
5821 io_cqring_ev_posted(ctx);
5822
5823 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01005824 req_set_fail(req);
Pavel Begunkov58f99372021-03-12 16:25:55 +00005825 io_put_req(req);
Jens Axboe62755e32019-10-28 21:49:21 -06005826 return 0;
5827}
5828
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00005829static int io_rsrc_update_prep(struct io_kiocb *req,
Jens Axboe05f3fb32019-12-09 11:22:50 -07005830 const struct io_uring_sqe *sqe)
5831{
Daniele Albano61710e42020-07-18 14:15:16 -06005832 if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT)))
5833 return -EINVAL;
5834 if (sqe->ioprio || sqe->rw_flags)
Jens Axboe05f3fb32019-12-09 11:22:50 -07005835 return -EINVAL;
5836
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00005837 req->rsrc_update.offset = READ_ONCE(sqe->off);
5838 req->rsrc_update.nr_args = READ_ONCE(sqe->len);
5839 if (!req->rsrc_update.nr_args)
Jens Axboe05f3fb32019-12-09 11:22:50 -07005840 return -EINVAL;
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00005841 req->rsrc_update.arg = READ_ONCE(sqe->addr);
Jens Axboe05f3fb32019-12-09 11:22:50 -07005842 return 0;
5843}
5844
Pavel Begunkov889fca72021-02-10 00:03:09 +00005845static int io_files_update(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe05f3fb32019-12-09 11:22:50 -07005846{
5847 struct io_ring_ctx *ctx = req->ctx;
Pavel Begunkovc3bdad02021-04-25 14:32:22 +01005848 struct io_uring_rsrc_update2 up;
Jens Axboe05f3fb32019-12-09 11:22:50 -07005849 int ret;
5850
Pavel Begunkov45d189c2021-02-10 00:03:07 +00005851 if (issue_flags & IO_URING_F_NONBLOCK)
Jens Axboe05f3fb32019-12-09 11:22:50 -07005852 return -EAGAIN;
Jens Axboe05f3fb32019-12-09 11:22:50 -07005853
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00005854 up.offset = req->rsrc_update.offset;
5855 up.data = req->rsrc_update.arg;
Pavel Begunkovc3bdad02021-04-25 14:32:22 +01005856 up.nr = 0;
5857 up.tags = 0;
Colin Ian King615cee42021-04-26 10:47:35 +01005858 up.resv = 0;
Jens Axboe05f3fb32019-12-09 11:22:50 -07005859
5860 mutex_lock(&ctx->uring_lock);
Pavel Begunkovfdecb662021-04-25 14:32:20 +01005861 ret = __io_register_rsrc_update(ctx, IORING_RSRC_FILE,
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +01005862 &up, req->rsrc_update.nr_args);
Jens Axboe05f3fb32019-12-09 11:22:50 -07005863 mutex_unlock(&ctx->uring_lock);
5864
5865 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01005866 req_set_fail(req);
Pavel Begunkov889fca72021-02-10 00:03:09 +00005867 __io_req_complete(req, issue_flags, ret, 0);
Jens Axboe05f3fb32019-12-09 11:22:50 -07005868 return 0;
5869}
5870
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005871static int io_req_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboef67676d2019-12-02 11:03:47 -07005872{
Jens Axboed625c6e2019-12-17 19:53:05 -07005873 switch (req->opcode) {
Jens Axboee7815732019-12-17 19:45:06 -07005874 case IORING_OP_NOP:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005875 return 0;
Jens Axboef67676d2019-12-02 11:03:47 -07005876 case IORING_OP_READV:
5877 case IORING_OP_READ_FIXED:
Jens Axboe3a6820f2019-12-22 15:19:35 -07005878 case IORING_OP_READ:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005879 return io_read_prep(req, sqe);
Jens Axboef67676d2019-12-02 11:03:47 -07005880 case IORING_OP_WRITEV:
5881 case IORING_OP_WRITE_FIXED:
Jens Axboe3a6820f2019-12-22 15:19:35 -07005882 case IORING_OP_WRITE:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005883 return io_write_prep(req, sqe);
Jens Axboe0969e782019-12-17 18:40:57 -07005884 case IORING_OP_POLL_ADD:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005885 return io_poll_add_prep(req, sqe);
Jens Axboe0969e782019-12-17 18:40:57 -07005886 case IORING_OP_POLL_REMOVE:
Pavel Begunkovc5de0032021-04-14 13:38:37 +01005887 return io_poll_update_prep(req, sqe);
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07005888 case IORING_OP_FSYNC:
Pavel Begunkov1155c762021-02-18 18:29:38 +00005889 return io_fsync_prep(req, sqe);
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07005890 case IORING_OP_SYNC_FILE_RANGE:
Pavel Begunkov1155c762021-02-18 18:29:38 +00005891 return io_sfr_prep(req, sqe);
Jens Axboe03b12302019-12-02 18:50:25 -07005892 case IORING_OP_SENDMSG:
Jens Axboefddafac2020-01-04 20:19:44 -07005893 case IORING_OP_SEND:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005894 return io_sendmsg_prep(req, sqe);
Jens Axboe03b12302019-12-02 18:50:25 -07005895 case IORING_OP_RECVMSG:
Jens Axboefddafac2020-01-04 20:19:44 -07005896 case IORING_OP_RECV:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005897 return io_recvmsg_prep(req, sqe);
Jens Axboef499a022019-12-02 16:28:46 -07005898 case IORING_OP_CONNECT:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005899 return io_connect_prep(req, sqe);
Jens Axboe2d283902019-12-04 11:08:05 -07005900 case IORING_OP_TIMEOUT:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005901 return io_timeout_prep(req, sqe, false);
Jens Axboeb29472e2019-12-17 18:50:29 -07005902 case IORING_OP_TIMEOUT_REMOVE:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005903 return io_timeout_remove_prep(req, sqe);
Jens Axboefbf23842019-12-17 18:45:56 -07005904 case IORING_OP_ASYNC_CANCEL:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005905 return io_async_cancel_prep(req, sqe);
Jens Axboe2d283902019-12-04 11:08:05 -07005906 case IORING_OP_LINK_TIMEOUT:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005907 return io_timeout_prep(req, sqe, true);
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07005908 case IORING_OP_ACCEPT:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005909 return io_accept_prep(req, sqe);
Jens Axboed63d1b52019-12-10 10:38:56 -07005910 case IORING_OP_FALLOCATE:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005911 return io_fallocate_prep(req, sqe);
Jens Axboe15b71ab2019-12-11 11:20:36 -07005912 case IORING_OP_OPENAT:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005913 return io_openat_prep(req, sqe);
Jens Axboeb5dba592019-12-11 14:02:38 -07005914 case IORING_OP_CLOSE:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005915 return io_close_prep(req, sqe);
Jens Axboe05f3fb32019-12-09 11:22:50 -07005916 case IORING_OP_FILES_UPDATE:
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00005917 return io_rsrc_update_prep(req, sqe);
Jens Axboeeddc7ef2019-12-13 21:18:10 -07005918 case IORING_OP_STATX:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005919 return io_statx_prep(req, sqe);
Jens Axboe4840e412019-12-25 22:03:45 -07005920 case IORING_OP_FADVISE:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005921 return io_fadvise_prep(req, sqe);
Jens Axboec1ca7572019-12-25 22:18:28 -07005922 case IORING_OP_MADVISE:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005923 return io_madvise_prep(req, sqe);
Jens Axboecebdb982020-01-08 17:59:24 -07005924 case IORING_OP_OPENAT2:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005925 return io_openat2_prep(req, sqe);
Jens Axboe3e4827b2020-01-08 15:18:09 -07005926 case IORING_OP_EPOLL_CTL:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005927 return io_epoll_ctl_prep(req, sqe);
Pavel Begunkov7d67af22020-02-24 11:32:45 +03005928 case IORING_OP_SPLICE:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005929 return io_splice_prep(req, sqe);
Jens Axboeddf0322d2020-02-23 16:41:33 -07005930 case IORING_OP_PROVIDE_BUFFERS:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005931 return io_provide_buffers_prep(req, sqe);
Jens Axboe067524e2020-03-02 16:32:28 -07005932 case IORING_OP_REMOVE_BUFFERS:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005933 return io_remove_buffers_prep(req, sqe);
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03005934 case IORING_OP_TEE:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005935 return io_tee_prep(req, sqe);
Jens Axboe36f4fa62020-09-05 11:14:22 -06005936 case IORING_OP_SHUTDOWN:
5937 return io_shutdown_prep(req, sqe);
Jens Axboe80a261f2020-09-28 14:23:58 -06005938 case IORING_OP_RENAMEAT:
5939 return io_renameat_prep(req, sqe);
Jens Axboe14a11432020-09-28 14:27:37 -06005940 case IORING_OP_UNLINKAT:
5941 return io_unlinkat_prep(req, sqe);
Jens Axboef67676d2019-12-02 11:03:47 -07005942 }
5943
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005944 printk_once(KERN_WARNING "io_uring: unhandled opcode %d\n",
5945 req->opcode);
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01005946 return -EINVAL;
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005947}
5948
Pavel Begunkov93642ef2021-02-18 18:29:44 +00005949static int io_req_prep_async(struct io_kiocb *req)
Jens Axboedef596e2019-01-09 08:59:42 -07005950{
Pavel Begunkovb7e298d2021-02-28 22:35:19 +00005951 if (!io_op_defs[req->opcode].needs_async_setup)
5952 return 0;
5953 if (WARN_ON_ONCE(req->async_data))
5954 return -EFAULT;
5955 if (io_alloc_async_data(req))
5956 return -EAGAIN;
5957
Pavel Begunkov93642ef2021-02-18 18:29:44 +00005958 switch (req->opcode) {
5959 case IORING_OP_READV:
Pavel Begunkov93642ef2021-02-18 18:29:44 +00005960 return io_rw_prep_async(req, READ);
5961 case IORING_OP_WRITEV:
Pavel Begunkov93642ef2021-02-18 18:29:44 +00005962 return io_rw_prep_async(req, WRITE);
5963 case IORING_OP_SENDMSG:
Pavel Begunkov93642ef2021-02-18 18:29:44 +00005964 return io_sendmsg_prep_async(req);
5965 case IORING_OP_RECVMSG:
Pavel Begunkov93642ef2021-02-18 18:29:44 +00005966 return io_recvmsg_prep_async(req);
5967 case IORING_OP_CONNECT:
5968 return io_connect_prep_async(req);
5969 }
Pavel Begunkovb7e298d2021-02-28 22:35:19 +00005970 printk_once(KERN_WARNING "io_uring: prep_async() bad opcode %d\n",
5971 req->opcode);
5972 return -EFAULT;
Jens Axboedef596e2019-01-09 08:59:42 -07005973}
5974
Pavel Begunkov9cf7c102020-07-13 23:37:15 +03005975static u32 io_get_sequence(struct io_kiocb *req)
5976{
5977 struct io_kiocb *pos;
5978 struct io_ring_ctx *ctx = req->ctx;
Pavel Begunkovf2f87372020-10-27 23:25:37 +00005979 u32 total_submitted, nr_reqs = 0;
Pavel Begunkov9cf7c102020-07-13 23:37:15 +03005980
Pavel Begunkovf2f87372020-10-27 23:25:37 +00005981 io_for_each_link(pos, req)
5982 nr_reqs++;
Pavel Begunkov9cf7c102020-07-13 23:37:15 +03005983
5984 total_submitted = ctx->cached_sq_head - ctx->cached_sq_dropped;
5985 return total_submitted - nr_reqs;
5986}
5987
Pavel Begunkovbe7053b2021-02-18 18:29:45 +00005988static int io_req_defer(struct io_kiocb *req)
Jens Axboedef596e2019-01-09 08:59:42 -07005989{
5990 struct io_ring_ctx *ctx = req->ctx;
Pavel Begunkov27dc8332020-07-13 23:37:14 +03005991 struct io_defer_entry *de;
Jens Axboedef596e2019-01-09 08:59:42 -07005992 int ret;
Pavel Begunkov9cf7c102020-07-13 23:37:15 +03005993 u32 seq;
Jens Axboedef596e2019-01-09 08:59:42 -07005994
5995 /* Still need defer if there is pending req in defer list. */
Pavel Begunkov9cf7c102020-07-13 23:37:15 +03005996 if (likely(list_empty_careful(&ctx->defer_list) &&
5997 !(req->flags & REQ_F_IO_DRAIN)))
5998 return 0;
5999
6000 seq = io_get_sequence(req);
6001 /* Still a chance to pass the sequence check */
6002 if (!req_need_defer(req, seq) && list_empty_careful(&ctx->defer_list))
Jens Axboedef596e2019-01-09 08:59:42 -07006003 return 0;
6004
Pavel Begunkovb7e298d2021-02-28 22:35:19 +00006005 ret = io_req_prep_async(req);
Pavel Begunkovbe7053b2021-02-18 18:29:45 +00006006 if (ret)
6007 return ret;
Pavel Begunkovcbdcb432020-06-29 19:18:43 +03006008 io_prep_async_link(req);
Pavel Begunkov27dc8332020-07-13 23:37:14 +03006009 de = kmalloc(sizeof(*de), GFP_KERNEL);
6010 if (!de)
6011 return -ENOMEM;
Jens Axboe31b51512019-01-18 22:56:34 -07006012
6013 spin_lock_irq(&ctx->completion_lock);
Pavel Begunkov9cf7c102020-07-13 23:37:15 +03006014 if (!req_need_defer(req, seq) && list_empty(&ctx->defer_list)) {
Jens Axboe31b51512019-01-18 22:56:34 -07006015 spin_unlock_irq(&ctx->completion_lock);
Pavel Begunkov27dc8332020-07-13 23:37:14 +03006016 kfree(de);
Pavel Begunkovae348172020-07-23 20:25:20 +03006017 io_queue_async_work(req);
6018 return -EIOCBQUEUED;
Jens Axboe31b51512019-01-18 22:56:34 -07006019 }
6020
6021 trace_io_uring_defer(ctx, req, req->user_data);
Pavel Begunkov27dc8332020-07-13 23:37:14 +03006022 de->req = req;
Pavel Begunkov9cf7c102020-07-13 23:37:15 +03006023 de->seq = seq;
Pavel Begunkov27dc8332020-07-13 23:37:14 +03006024 list_add_tail(&de->list, &ctx->defer_list);
Jens Axboe31b51512019-01-18 22:56:34 -07006025 spin_unlock_irq(&ctx->completion_lock);
6026 return -EIOCBQUEUED;
6027}
6028
Pavel Begunkov68fb8972021-03-19 17:22:41 +00006029static void io_clean_op(struct io_kiocb *req)
Pavel Begunkov99bc4c32020-02-07 22:04:45 +03006030{
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03006031 if (req->flags & REQ_F_BUFFER_SELECTED) {
6032 switch (req->opcode) {
6033 case IORING_OP_READV:
6034 case IORING_OP_READ_FIXED:
6035 case IORING_OP_READ:
Jens Axboebcda7ba2020-02-23 16:42:51 -07006036 kfree((void *)(unsigned long)req->rw.addr);
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03006037 break;
6038 case IORING_OP_RECVMSG:
6039 case IORING_OP_RECV:
Jens Axboe52de1fe2020-02-27 10:15:42 -07006040 kfree(req->sr_msg.kbuf);
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03006041 break;
6042 }
6043 req->flags &= ~REQ_F_BUFFER_SELECTED;
Pavel Begunkov99bc4c32020-02-07 22:04:45 +03006044 }
6045
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03006046 if (req->flags & REQ_F_NEED_CLEANUP) {
6047 switch (req->opcode) {
6048 case IORING_OP_READV:
6049 case IORING_OP_READ_FIXED:
6050 case IORING_OP_READ:
6051 case IORING_OP_WRITEV:
6052 case IORING_OP_WRITE_FIXED:
Jens Axboee8c2bc12020-08-15 18:44:09 -07006053 case IORING_OP_WRITE: {
6054 struct io_async_rw *io = req->async_data;
6055 if (io->free_iovec)
6056 kfree(io->free_iovec);
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03006057 break;
Jens Axboee8c2bc12020-08-15 18:44:09 -07006058 }
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03006059 case IORING_OP_RECVMSG:
Jens Axboee8c2bc12020-08-15 18:44:09 -07006060 case IORING_OP_SENDMSG: {
6061 struct io_async_msghdr *io = req->async_data;
Pavel Begunkov257e84a2021-02-05 00:58:00 +00006062
6063 kfree(io->free_iov);
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03006064 break;
Jens Axboee8c2bc12020-08-15 18:44:09 -07006065 }
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03006066 case IORING_OP_SPLICE:
6067 case IORING_OP_TEE:
Pavel Begunkove1d767f2021-03-19 17:22:43 +00006068 if (!(req->splice.flags & SPLICE_F_FD_IN_FIXED))
6069 io_put_file(req->splice.file_in);
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03006070 break;
Jens Axboef3cd48502020-09-24 14:55:54 -06006071 case IORING_OP_OPENAT:
6072 case IORING_OP_OPENAT2:
6073 if (req->open.filename)
6074 putname(req->open.filename);
6075 break;
Jens Axboe80a261f2020-09-28 14:23:58 -06006076 case IORING_OP_RENAMEAT:
6077 putname(req->rename.oldpath);
6078 putname(req->rename.newpath);
6079 break;
Jens Axboe14a11432020-09-28 14:27:37 -06006080 case IORING_OP_UNLINKAT:
6081 putname(req->unlink.filename);
6082 break;
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03006083 }
6084 req->flags &= ~REQ_F_NEED_CLEANUP;
6085 }
Jens Axboe75652a302021-04-15 09:52:40 -06006086 if ((req->flags & REQ_F_POLLED) && req->apoll) {
6087 kfree(req->apoll->double_poll);
6088 kfree(req->apoll);
6089 req->apoll = NULL;
6090 }
Pavel Begunkov3a0a6902021-04-20 12:03:31 +01006091 if (req->flags & REQ_F_INFLIGHT) {
6092 struct io_uring_task *tctx = req->task->io_uring;
6093
6094 atomic_dec(&tctx->inflight_tracked);
6095 req->flags &= ~REQ_F_INFLIGHT;
6096 }
Pavel Begunkov99bc4c32020-02-07 22:04:45 +03006097}
6098
Pavel Begunkov889fca72021-02-10 00:03:09 +00006099static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboeedafcce2019-01-09 09:16:05 -07006100{
Jens Axboeedafcce2019-01-09 09:16:05 -07006101 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe5730b272021-02-27 15:57:30 -07006102 const struct cred *creds = NULL;
Jens Axboed625c6e2019-12-17 19:53:05 -07006103 int ret;
Jens Axboeedafcce2019-01-09 09:16:05 -07006104
Jens Axboe003e8dc2021-03-06 09:22:27 -07006105 if (req->work.creds && req->work.creds != current_cred())
6106 creds = override_creds(req->work.creds);
Jens Axboe5730b272021-02-27 15:57:30 -07006107
Jens Axboed625c6e2019-12-17 19:53:05 -07006108 switch (req->opcode) {
Jens Axboe2b188cc2019-01-07 10:46:33 -07006109 case IORING_OP_NOP:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006110 ret = io_nop(req, issue_flags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006111 break;
Jens Axboe2b188cc2019-01-07 10:46:33 -07006112 case IORING_OP_READV:
Jens Axboe3529d8c2019-12-19 18:24:38 -07006113 case IORING_OP_READ_FIXED:
Jens Axboe3a6820f2019-12-22 15:19:35 -07006114 case IORING_OP_READ:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006115 ret = io_read(req, issue_flags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006116 break;
6117 case IORING_OP_WRITEV:
Jens Axboe2b188cc2019-01-07 10:46:33 -07006118 case IORING_OP_WRITE_FIXED:
Jens Axboe3a6820f2019-12-22 15:19:35 -07006119 case IORING_OP_WRITE:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006120 ret = io_write(req, issue_flags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006121 break;
6122 case IORING_OP_FSYNC:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006123 ret = io_fsync(req, issue_flags);
Jackie Liuba5290c2019-10-09 09:19:59 +08006124 break;
Jens Axboe2b188cc2019-01-07 10:46:33 -07006125 case IORING_OP_POLL_ADD:
Pavel Begunkov61e98202021-02-10 00:03:08 +00006126 ret = io_poll_add(req, issue_flags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006127 break;
6128 case IORING_OP_POLL_REMOVE:
Pavel Begunkovc5de0032021-04-14 13:38:37 +01006129 ret = io_poll_update(req, issue_flags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006130 break;
Jens Axboeb76da702019-11-20 13:05:32 -07006131 case IORING_OP_SYNC_FILE_RANGE:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006132 ret = io_sync_file_range(req, issue_flags);
Jens Axboeb76da702019-11-20 13:05:32 -07006133 break;
6134 case IORING_OP_SENDMSG:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006135 ret = io_sendmsg(req, issue_flags);
Pavel Begunkov062d04d2020-10-10 18:34:12 +01006136 break;
Jens Axboefddafac2020-01-04 20:19:44 -07006137 case IORING_OP_SEND:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006138 ret = io_send(req, issue_flags);
Jens Axboeb76da702019-11-20 13:05:32 -07006139 break;
6140 case IORING_OP_RECVMSG:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006141 ret = io_recvmsg(req, issue_flags);
Pavel Begunkov062d04d2020-10-10 18:34:12 +01006142 break;
Jens Axboefddafac2020-01-04 20:19:44 -07006143 case IORING_OP_RECV:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006144 ret = io_recv(req, issue_flags);
Jens Axboeb76da702019-11-20 13:05:32 -07006145 break;
Jens Axboe561fb042019-10-24 07:25:42 -06006146 case IORING_OP_TIMEOUT:
Pavel Begunkov61e98202021-02-10 00:03:08 +00006147 ret = io_timeout(req, issue_flags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006148 break;
6149 case IORING_OP_TIMEOUT_REMOVE:
Pavel Begunkov61e98202021-02-10 00:03:08 +00006150 ret = io_timeout_remove(req, issue_flags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006151 break;
6152 case IORING_OP_ACCEPT:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006153 ret = io_accept(req, issue_flags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006154 break;
6155 case IORING_OP_CONNECT:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006156 ret = io_connect(req, issue_flags);
Jens Axboe31b51512019-01-18 22:56:34 -07006157 break;
6158 case IORING_OP_ASYNC_CANCEL:
Pavel Begunkov61e98202021-02-10 00:03:08 +00006159 ret = io_async_cancel(req, issue_flags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006160 break;
Jens Axboed63d1b52019-12-10 10:38:56 -07006161 case IORING_OP_FALLOCATE:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006162 ret = io_fallocate(req, issue_flags);
Jens Axboed63d1b52019-12-10 10:38:56 -07006163 break;
Jens Axboe15b71ab2019-12-11 11:20:36 -07006164 case IORING_OP_OPENAT:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006165 ret = io_openat(req, issue_flags);
Jens Axboe15b71ab2019-12-11 11:20:36 -07006166 break;
Jens Axboeb5dba592019-12-11 14:02:38 -07006167 case IORING_OP_CLOSE:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006168 ret = io_close(req, issue_flags);
Jens Axboeb5dba592019-12-11 14:02:38 -07006169 break;
Jens Axboe05f3fb32019-12-09 11:22:50 -07006170 case IORING_OP_FILES_UPDATE:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006171 ret = io_files_update(req, issue_flags);
Jens Axboe05f3fb32019-12-09 11:22:50 -07006172 break;
Jens Axboeeddc7ef2019-12-13 21:18:10 -07006173 case IORING_OP_STATX:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006174 ret = io_statx(req, issue_flags);
Jens Axboeeddc7ef2019-12-13 21:18:10 -07006175 break;
Jens Axboe4840e412019-12-25 22:03:45 -07006176 case IORING_OP_FADVISE:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006177 ret = io_fadvise(req, issue_flags);
Jens Axboe4840e412019-12-25 22:03:45 -07006178 break;
Jens Axboec1ca7572019-12-25 22:18:28 -07006179 case IORING_OP_MADVISE:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006180 ret = io_madvise(req, issue_flags);
Jens Axboec1ca7572019-12-25 22:18:28 -07006181 break;
Jens Axboecebdb982020-01-08 17:59:24 -07006182 case IORING_OP_OPENAT2:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006183 ret = io_openat2(req, issue_flags);
Jens Axboecebdb982020-01-08 17:59:24 -07006184 break;
Jens Axboe3e4827b2020-01-08 15:18:09 -07006185 case IORING_OP_EPOLL_CTL:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006186 ret = io_epoll_ctl(req, issue_flags);
Jens Axboe3e4827b2020-01-08 15:18:09 -07006187 break;
Pavel Begunkov7d67af22020-02-24 11:32:45 +03006188 case IORING_OP_SPLICE:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006189 ret = io_splice(req, issue_flags);
Pavel Begunkov7d67af22020-02-24 11:32:45 +03006190 break;
Jens Axboeddf0322d2020-02-23 16:41:33 -07006191 case IORING_OP_PROVIDE_BUFFERS:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006192 ret = io_provide_buffers(req, issue_flags);
Jens Axboeddf0322d2020-02-23 16:41:33 -07006193 break;
Jens Axboe067524e2020-03-02 16:32:28 -07006194 case IORING_OP_REMOVE_BUFFERS:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006195 ret = io_remove_buffers(req, issue_flags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006196 break;
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03006197 case IORING_OP_TEE:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006198 ret = io_tee(req, issue_flags);
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03006199 break;
Jens Axboe36f4fa62020-09-05 11:14:22 -06006200 case IORING_OP_SHUTDOWN:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006201 ret = io_shutdown(req, issue_flags);
Jens Axboe36f4fa62020-09-05 11:14:22 -06006202 break;
Jens Axboe80a261f2020-09-28 14:23:58 -06006203 case IORING_OP_RENAMEAT:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006204 ret = io_renameat(req, issue_flags);
Jens Axboe80a261f2020-09-28 14:23:58 -06006205 break;
Jens Axboe14a11432020-09-28 14:27:37 -06006206 case IORING_OP_UNLINKAT:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006207 ret = io_unlinkat(req, issue_flags);
Jens Axboe14a11432020-09-28 14:27:37 -06006208 break;
Jens Axboe2b188cc2019-01-07 10:46:33 -07006209 default:
6210 ret = -EINVAL;
6211 break;
6212 }
Jens Axboe31b51512019-01-18 22:56:34 -07006213
Jens Axboe5730b272021-02-27 15:57:30 -07006214 if (creds)
6215 revert_creds(creds);
6216
Jens Axboe2b188cc2019-01-07 10:46:33 -07006217 if (ret)
6218 return ret;
6219
Jens Axboeb5325762020-05-19 21:20:27 -06006220 /* If the op doesn't have a file, we're not polling for it */
6221 if ((ctx->flags & IORING_SETUP_IOPOLL) && req->file) {
Jens Axboe11ba8202020-01-15 21:51:17 -07006222 const bool in_async = io_wq_current_is_worker();
6223
Jens Axboe11ba8202020-01-15 21:51:17 -07006224 /* workqueue context doesn't hold uring_lock, grab it now */
6225 if (in_async)
6226 mutex_lock(&ctx->uring_lock);
6227
Xiaoguang Wang2e9dbe92020-11-13 00:44:08 +08006228 io_iopoll_req_issued(req, in_async);
Jens Axboe11ba8202020-01-15 21:51:17 -07006229
6230 if (in_async)
6231 mutex_unlock(&ctx->uring_lock);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006232 }
6233
6234 return 0;
6235}
6236
Pavel Begunkov5280f7e2021-02-04 13:52:08 +00006237static void io_wq_submit_work(struct io_wq_work *work)
Pavel Begunkovd4c81f32020-06-08 21:08:19 +03006238{
Jens Axboe2b188cc2019-01-07 10:46:33 -07006239 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
Pavel Begunkov6df1db62020-07-03 22:15:06 +03006240 struct io_kiocb *timeout;
Jens Axboe561fb042019-10-24 07:25:42 -06006241 int ret = 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07006242
Pavel Begunkov6df1db62020-07-03 22:15:06 +03006243 timeout = io_prep_linked_timeout(req);
6244 if (timeout)
6245 io_queue_linked_timeout(timeout);
Pavel Begunkovd4c81f32020-06-08 21:08:19 +03006246
Jens Axboe4014d942021-01-19 15:53:54 -07006247 if (work->flags & IO_WQ_WORK_CANCEL)
Jens Axboe561fb042019-10-24 07:25:42 -06006248 ret = -ECANCELED;
Jens Axboe31b51512019-01-18 22:56:34 -07006249
Jens Axboe561fb042019-10-24 07:25:42 -06006250 if (!ret) {
Jens Axboe561fb042019-10-24 07:25:42 -06006251 do {
Pavel Begunkov889fca72021-02-10 00:03:09 +00006252 ret = io_issue_sqe(req, 0);
Jens Axboe561fb042019-10-24 07:25:42 -06006253 /*
6254 * We can get EAGAIN for polled IO even though we're
6255 * forcing a sync submission from here, since we can't
6256 * wait for request slots on the block side.
6257 */
6258 if (ret != -EAGAIN)
6259 break;
6260 cond_resched();
6261 } while (1);
6262 }
Jens Axboe31b51512019-01-18 22:56:34 -07006263
Pavel Begunkova3df76982021-02-18 22:32:52 +00006264 /* avoid locking problems by failing it from a clean context */
Jens Axboe561fb042019-10-24 07:25:42 -06006265 if (ret) {
Pavel Begunkova3df76982021-02-18 22:32:52 +00006266 /* io-wq is going to take one down */
Jens Axboede9b4cc2021-02-24 13:28:27 -07006267 req_ref_get(req);
Pavel Begunkova3df76982021-02-18 22:32:52 +00006268 io_req_task_queue_fail(req, ret);
Jens Axboeedafcce2019-01-09 09:16:05 -07006269 }
Jens Axboe31b51512019-01-18 22:56:34 -07006270}
Jens Axboe2b188cc2019-01-07 10:46:33 -07006271
Jens Axboe7b29f922021-03-12 08:30:14 -07006272#define FFS_ASYNC_READ 0x1UL
6273#define FFS_ASYNC_WRITE 0x2UL
6274#ifdef CONFIG_64BIT
6275#define FFS_ISREG 0x4UL
6276#else
6277#define FFS_ISREG 0x0UL
6278#endif
6279#define FFS_MASK ~(FFS_ASYNC_READ|FFS_ASYNC_WRITE|FFS_ISREG)
6280
Pavel Begunkovaeca2412021-04-11 01:46:37 +01006281static inline struct io_fixed_file *io_fixed_file_slot(struct io_file_table *table,
Pavel Begunkova04b0ac2021-04-01 15:44:04 +01006282 unsigned i)
Jens Axboe09bb8392019-03-13 12:39:28 -06006283{
Pavel Begunkovaeca2412021-04-11 01:46:37 +01006284 struct io_fixed_file *table_l2;
Jens Axboe65e19f52019-10-26 07:20:21 -06006285
Pavel Begunkovaeca2412021-04-11 01:46:37 +01006286 table_l2 = table->files[i >> IORING_FILE_TABLE_SHIFT];
6287 return &table_l2[i & IORING_FILE_TABLE_MASK];
Pavel Begunkovdafecf12021-02-28 22:35:11 +00006288}
6289
Jens Axboe09bb8392019-03-13 12:39:28 -06006290static inline struct file *io_file_from_index(struct io_ring_ctx *ctx,
6291 int index)
6292{
Pavel Begunkovaeca2412021-04-11 01:46:37 +01006293 struct io_fixed_file *slot = io_fixed_file_slot(&ctx->file_table, index);
Jens Axboe65e19f52019-10-26 07:20:21 -06006294
Pavel Begunkova04b0ac2021-04-01 15:44:04 +01006295 return (struct file *) (slot->file_ptr & FFS_MASK);
Jens Axboe65e19f52019-10-26 07:20:21 -06006296}
6297
Pavel Begunkova04b0ac2021-04-01 15:44:04 +01006298static void io_fixed_file_set(struct io_fixed_file *file_slot, struct file *file)
Pavel Begunkov9a321c92021-04-01 15:44:01 +01006299{
6300 unsigned long file_ptr = (unsigned long) file;
6301
6302 if (__io_file_supports_async(file, READ))
6303 file_ptr |= FFS_ASYNC_READ;
6304 if (__io_file_supports_async(file, WRITE))
6305 file_ptr |= FFS_ASYNC_WRITE;
6306 if (S_ISREG(file_inode(file)->i_mode))
6307 file_ptr |= FFS_ISREG;
Pavel Begunkova04b0ac2021-04-01 15:44:04 +01006308 file_slot->file_ptr = file_ptr;
Jens Axboe09bb8392019-03-13 12:39:28 -06006309}
6310
Pavel Begunkov8371adf2020-10-10 18:34:08 +01006311static struct file *io_file_get(struct io_submit_state *state,
6312 struct io_kiocb *req, int fd, bool fixed)
Pavel Begunkov8da11c12020-02-24 11:32:44 +03006313{
6314 struct io_ring_ctx *ctx = req->ctx;
6315 struct file *file;
6316
6317 if (fixed) {
Jens Axboe7b29f922021-03-12 08:30:14 -07006318 unsigned long file_ptr;
6319
Pavel Begunkov479f5172020-10-10 18:34:07 +01006320 if (unlikely((unsigned int)fd >= ctx->nr_user_files))
Pavel Begunkov8371adf2020-10-10 18:34:08 +01006321 return NULL;
Pavel Begunkov8da11c12020-02-24 11:32:44 +03006322 fd = array_index_nospec(fd, ctx->nr_user_files);
Pavel Begunkovaeca2412021-04-11 01:46:37 +01006323 file_ptr = io_fixed_file_slot(&ctx->file_table, fd)->file_ptr;
Jens Axboe7b29f922021-03-12 08:30:14 -07006324 file = (struct file *) (file_ptr & FFS_MASK);
6325 file_ptr &= ~FFS_MASK;
6326 /* mask in overlapping REQ_F and FFS bits */
6327 req->flags |= (file_ptr << REQ_F_ASYNC_READ_BIT);
Pavel Begunkovb895c9a2021-04-01 15:43:40 +01006328 io_req_set_rsrc_node(req);
Pavel Begunkov8da11c12020-02-24 11:32:44 +03006329 } else {
6330 trace_io_uring_file_get(ctx, fd);
6331 file = __io_file_get(state, fd);
Jens Axboed44f5542021-03-12 08:27:05 -07006332
6333 /* we don't allow fixed io_uring files */
6334 if (file && unlikely(file->f_op == &io_uring_fops))
6335 io_req_track_inflight(req);
Pavel Begunkov8da11c12020-02-24 11:32:44 +03006336 }
6337
Pavel Begunkov8371adf2020-10-10 18:34:08 +01006338 return file;
Pavel Begunkov8da11c12020-02-24 11:32:44 +03006339}
6340
Jens Axboe2665abf2019-11-05 12:40:47 -07006341static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer)
6342{
Jens Axboead8a48a2019-11-15 08:49:11 -07006343 struct io_timeout_data *data = container_of(timer,
6344 struct io_timeout_data, timer);
Pavel Begunkov90cd7e42020-10-27 23:25:36 +00006345 struct io_kiocb *prev, *req = data->req;
Jens Axboe2665abf2019-11-05 12:40:47 -07006346 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe2665abf2019-11-05 12:40:47 -07006347 unsigned long flags;
Jens Axboe2665abf2019-11-05 12:40:47 -07006348
6349 spin_lock_irqsave(&ctx->completion_lock, flags);
Pavel Begunkov90cd7e42020-10-27 23:25:36 +00006350 prev = req->timeout.head;
6351 req->timeout.head = NULL;
Jens Axboe2665abf2019-11-05 12:40:47 -07006352
6353 /*
6354 * We don't expect the list to be empty, that will only happen if we
6355 * race with the completion of the linked work.
6356 */
Pavel Begunkov447c19f2021-05-14 12:02:50 +01006357 if (prev) {
Pavel Begunkovf2f87372020-10-27 23:25:37 +00006358 io_remove_next_linked(prev);
Pavel Begunkov447c19f2021-05-14 12:02:50 +01006359 if (!req_ref_inc_not_zero(prev))
6360 prev = NULL;
6361 }
Jens Axboe2665abf2019-11-05 12:40:47 -07006362 spin_unlock_irqrestore(&ctx->completion_lock, flags);
6363
6364 if (prev) {
Pavel Begunkov014db002020-03-03 21:33:12 +03006365 io_async_find_and_cancel(ctx, req, prev->user_data, -ETIME);
Pavel Begunkov9ae1f8d2021-02-01 18:59:51 +00006366 io_put_req_deferred(prev, 1);
Pavel Begunkova2982322021-05-07 21:06:38 +01006367 io_put_req_deferred(req, 1);
Jens Axboe47f46762019-11-09 17:43:02 -07006368 } else {
Pavel Begunkov9ae1f8d2021-02-01 18:59:51 +00006369 io_req_complete_post(req, -ETIME, 0);
Jens Axboe2665abf2019-11-05 12:40:47 -07006370 }
Jens Axboe2665abf2019-11-05 12:40:47 -07006371 return HRTIMER_NORESTART;
6372}
6373
Pavel Begunkovde968c12021-03-19 17:22:33 +00006374static void io_queue_linked_timeout(struct io_kiocb *req)
Jens Axboe2665abf2019-11-05 12:40:47 -07006375{
Pavel Begunkovde968c12021-03-19 17:22:33 +00006376 struct io_ring_ctx *ctx = req->ctx;
6377
6378 spin_lock_irq(&ctx->completion_lock);
Jens Axboe76a46e02019-11-10 23:34:16 -07006379 /*
Pavel Begunkovf2f87372020-10-27 23:25:37 +00006380 * If the back reference is NULL, then our linked request finished
6381 * before we got a chance to setup the timer
Jens Axboe76a46e02019-11-10 23:34:16 -07006382 */
Pavel Begunkov90cd7e42020-10-27 23:25:36 +00006383 if (req->timeout.head) {
Jens Axboee8c2bc12020-08-15 18:44:09 -07006384 struct io_timeout_data *data = req->async_data;
Jens Axboe94ae5e72019-11-14 19:39:52 -07006385
Jens Axboead8a48a2019-11-15 08:49:11 -07006386 data->timer.function = io_link_timeout_fn;
6387 hrtimer_start(&data->timer, timespec64_to_ktime(data->ts),
6388 data->mode);
Jens Axboe2665abf2019-11-05 12:40:47 -07006389 }
Jens Axboe76a46e02019-11-10 23:34:16 -07006390 spin_unlock_irq(&ctx->completion_lock);
Jens Axboe2665abf2019-11-05 12:40:47 -07006391 /* drop submission reference */
Jens Axboe76a46e02019-11-10 23:34:16 -07006392 io_put_req(req);
Jens Axboe2665abf2019-11-05 12:40:47 -07006393}
6394
Jens Axboead8a48a2019-11-15 08:49:11 -07006395static struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req)
Jens Axboe2665abf2019-11-05 12:40:47 -07006396{
Pavel Begunkovf2f87372020-10-27 23:25:37 +00006397 struct io_kiocb *nxt = req->link;
Jens Axboe2b188cc2019-01-07 10:46:33 -07006398
Pavel Begunkovf2f87372020-10-27 23:25:37 +00006399 if (!nxt || (req->flags & REQ_F_LINK_TIMEOUT) ||
6400 nxt->opcode != IORING_OP_LINK_TIMEOUT)
Jens Axboed7718a92020-02-14 22:23:12 -07006401 return NULL;
Jens Axboe2665abf2019-11-05 12:40:47 -07006402
Pavel Begunkov90cd7e42020-10-27 23:25:36 +00006403 nxt->timeout.head = req;
Pavel Begunkov900fad42020-10-19 16:39:16 +01006404 nxt->flags |= REQ_F_LTIMEOUT_ACTIVE;
Jens Axboe76a46e02019-11-10 23:34:16 -07006405 req->flags |= REQ_F_LINK_TIMEOUT;
Jens Axboe76a46e02019-11-10 23:34:16 -07006406 return nxt;
Jens Axboe2665abf2019-11-05 12:40:47 -07006407}
6408
Pavel Begunkovc5eef2b2021-02-10 00:03:22 +00006409static void __io_queue_sqe(struct io_kiocb *req)
Jens Axboe2b188cc2019-01-07 10:46:33 -07006410{
Pavel Begunkovd3d72982021-02-12 03:23:51 +00006411 struct io_kiocb *linked_timeout = io_prep_linked_timeout(req);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006412 int ret;
6413
Pavel Begunkovc5eef2b2021-02-10 00:03:22 +00006414 ret = io_issue_sqe(req, IO_URING_F_NONBLOCK|IO_URING_F_COMPLETE_DEFER);
Jens Axboe491381ce2019-10-17 09:20:46 -06006415
6416 /*
6417 * We async punt it if the file wasn't marked NOWAIT, or if the file
6418 * doesn't support non-blocking read/write attempts
6419 */
Pavel Begunkov18400382021-03-19 17:22:34 +00006420 if (likely(!ret)) {
Pavel Begunkov0d63c142020-10-22 16:47:18 +01006421 /* drop submission reference */
Pavel Begunkove342c802021-01-19 13:32:47 +00006422 if (req->flags & REQ_F_COMPLETE_INLINE) {
Pavel Begunkovc5eef2b2021-02-10 00:03:22 +00006423 struct io_ring_ctx *ctx = req->ctx;
6424 struct io_comp_state *cs = &ctx->submit_state.comp;
Jens Axboee65ef562019-03-12 10:16:44 -06006425
Pavel Begunkov6dd0be12021-02-10 00:03:13 +00006426 cs->reqs[cs->nr++] = req;
Pavel Begunkovd3d72982021-02-12 03:23:51 +00006427 if (cs->nr == ARRAY_SIZE(cs->reqs))
Pavel Begunkovc5eef2b2021-02-10 00:03:22 +00006428 io_submit_flush_completions(cs, ctx);
Pavel Begunkov9affd662021-01-19 13:32:46 +00006429 } else {
Pavel Begunkovd3d72982021-02-12 03:23:51 +00006430 io_put_req(req);
Pavel Begunkov0d63c142020-10-22 16:47:18 +01006431 }
Pavel Begunkov18400382021-03-19 17:22:34 +00006432 } else if (ret == -EAGAIN && !(req->flags & REQ_F_NOWAIT)) {
6433 if (!io_arm_poll_handler(req)) {
6434 /*
6435 * Queued up for async execution, worker will release
6436 * submit reference when the iocb is actually submitted.
6437 */
6438 io_queue_async_work(req);
6439 }
Pavel Begunkov0d63c142020-10-22 16:47:18 +01006440 } else {
Pavel Begunkovf41db2732021-02-28 22:35:12 +00006441 io_req_complete_failed(req, ret);
Jens Axboe9e645e112019-05-10 16:07:28 -06006442 }
Pavel Begunkovd3d72982021-02-12 03:23:51 +00006443 if (linked_timeout)
6444 io_queue_linked_timeout(linked_timeout);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006445}
6446
Pavel Begunkovbe7053b2021-02-18 18:29:45 +00006447static void io_queue_sqe(struct io_kiocb *req)
Jackie Liu4fe2c962019-09-09 20:50:40 +08006448{
6449 int ret;
6450
Pavel Begunkovbe7053b2021-02-18 18:29:45 +00006451 ret = io_req_defer(req);
Jackie Liu4fe2c962019-09-09 20:50:40 +08006452 if (ret) {
6453 if (ret != -EIOCBQUEUED) {
Pavel Begunkov11185912020-01-22 23:09:35 +03006454fail_req:
Pavel Begunkovf41db2732021-02-28 22:35:12 +00006455 io_req_complete_failed(req, ret);
Jackie Liu4fe2c962019-09-09 20:50:40 +08006456 }
Pavel Begunkov25508782019-12-30 21:24:47 +03006457 } else if (req->flags & REQ_F_FORCE_ASYNC) {
Pavel Begunkovb7e298d2021-02-28 22:35:19 +00006458 ret = io_req_prep_async(req);
Pavel Begunkovbe7053b2021-02-18 18:29:45 +00006459 if (unlikely(ret))
6460 goto fail_req;
Jens Axboece35a472019-12-17 08:04:44 -07006461 io_queue_async_work(req);
6462 } else {
Pavel Begunkovc5eef2b2021-02-10 00:03:22 +00006463 __io_queue_sqe(req);
Jens Axboece35a472019-12-17 08:04:44 -07006464 }
Jackie Liu4fe2c962019-09-09 20:50:40 +08006465}
6466
Stefano Garzarella21b55db2020-08-27 16:58:30 +02006467/*
6468 * Check SQE restrictions (opcode and flags).
6469 *
6470 * Returns 'true' if SQE is allowed, 'false' otherwise.
6471 */
6472static inline bool io_check_restriction(struct io_ring_ctx *ctx,
6473 struct io_kiocb *req,
6474 unsigned int sqe_flags)
6475{
6476 if (!ctx->restricted)
6477 return true;
6478
6479 if (!test_bit(req->opcode, ctx->restrictions.sqe_op))
6480 return false;
6481
6482 if ((sqe_flags & ctx->restrictions.sqe_flags_required) !=
6483 ctx->restrictions.sqe_flags_required)
6484 return false;
6485
6486 if (sqe_flags & ~(ctx->restrictions.sqe_flags_allowed |
6487 ctx->restrictions.sqe_flags_required))
6488 return false;
6489
6490 return true;
6491}
6492
Pavel Begunkovef4ff582020-04-12 02:05:05 +03006493static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,
Pavel Begunkov258b29a2021-02-10 00:03:10 +00006494 const struct io_uring_sqe *sqe)
Pavel Begunkov0553b8b2020-04-08 08:58:45 +03006495{
Pavel Begunkov258b29a2021-02-10 00:03:10 +00006496 struct io_submit_state *state;
Pavel Begunkovef4ff582020-04-12 02:05:05 +03006497 unsigned int sqe_flags;
Jens Axboe003e8dc2021-03-06 09:22:27 -07006498 int personality, ret = 0;
Pavel Begunkovef4ff582020-04-12 02:05:05 +03006499
Pavel Begunkov0553b8b2020-04-08 08:58:45 +03006500 req->opcode = READ_ONCE(sqe->opcode);
Pavel Begunkov5be9ad12021-02-12 18:41:17 +00006501 /* same numerical values with corresponding REQ_F_*, safe to copy */
6502 req->flags = sqe_flags = READ_ONCE(sqe->flags);
Pavel Begunkov0553b8b2020-04-08 08:58:45 +03006503 req->user_data = READ_ONCE(sqe->user_data);
Jens Axboee8c2bc12020-08-15 18:44:09 -07006504 req->async_data = NULL;
Pavel Begunkov0553b8b2020-04-08 08:58:45 +03006505 req->file = NULL;
6506 req->ctx = ctx;
Pavel Begunkovf2f87372020-10-27 23:25:37 +00006507 req->link = NULL;
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00006508 req->fixed_rsrc_refs = NULL;
Pavel Begunkov0553b8b2020-04-08 08:58:45 +03006509 /* one is dropped after submission, the other at completion */
Jens Axboeabc54d62021-02-24 13:32:30 -07006510 atomic_set(&req->refs, 2);
Pavel Begunkov4dd28242020-06-15 10:33:13 +03006511 req->task = current;
Pavel Begunkov0553b8b2020-04-08 08:58:45 +03006512 req->result = 0;
Jens Axboe93e68e02021-03-09 07:02:21 -07006513 req->work.creds = NULL;
Pavel Begunkovef4ff582020-04-12 02:05:05 +03006514
Pavel Begunkov5be9ad12021-02-12 18:41:17 +00006515 /* enforce forwards compatibility on users */
Pavel Begunkovdddca222021-04-27 16:13:52 +01006516 if (unlikely(sqe_flags & ~SQE_VALID_FLAGS))
Pavel Begunkov5be9ad12021-02-12 18:41:17 +00006517 return -EINVAL;
Pavel Begunkovef4ff582020-04-12 02:05:05 +03006518 if (unlikely(req->opcode >= IORING_OP_LAST))
6519 return -EINVAL;
Stefano Garzarella21b55db2020-08-27 16:58:30 +02006520 if (unlikely(!io_check_restriction(ctx, req, sqe_flags)))
6521 return -EACCES;
6522
Pavel Begunkovef4ff582020-04-12 02:05:05 +03006523 if ((sqe_flags & IOSQE_BUFFER_SELECT) &&
6524 !io_op_defs[req->opcode].buffer_select)
6525 return -EOPNOTSUPP;
6526
Jens Axboe003e8dc2021-03-06 09:22:27 -07006527 personality = READ_ONCE(sqe->personality);
6528 if (personality) {
Matthew Wilcox (Oracle)61cf9372021-03-08 14:16:16 +00006529 req->work.creds = xa_load(&ctx->personalities, personality);
Jens Axboe003e8dc2021-03-06 09:22:27 -07006530 if (!req->work.creds)
6531 return -EINVAL;
6532 get_cred(req->work.creds);
Jens Axboe003e8dc2021-03-06 09:22:27 -07006533 }
Pavel Begunkov258b29a2021-02-10 00:03:10 +00006534 state = &ctx->submit_state;
Pavel Begunkovef4ff582020-04-12 02:05:05 +03006535
Jens Axboe27926b62020-10-28 09:33:23 -06006536 /*
6537 * Plug now if we have more than 1 IO left after this, and the target
6538 * is potentially a read/write to block based storage.
6539 */
6540 if (!state->plug_started && state->ios_left > 1 &&
6541 io_op_defs[req->opcode].plug) {
6542 blk_start_plug(&state->plug);
6543 state->plug_started = true;
6544 }
Jens Axboe63ff8222020-05-07 14:56:15 -06006545
Pavel Begunkovbd5bbda2020-11-20 15:50:51 +00006546 if (io_op_defs[req->opcode].needs_file) {
6547 bool fixed = req->flags & REQ_F_FIXED_FILE;
Jens Axboe63ff8222020-05-07 14:56:15 -06006548
Pavel Begunkovbd5bbda2020-11-20 15:50:51 +00006549 req->file = io_file_get(state, req, READ_ONCE(sqe->fd), fixed);
Pavel Begunkovba13e232021-02-01 18:59:52 +00006550 if (unlikely(!req->file))
Pavel Begunkovbd5bbda2020-11-20 15:50:51 +00006551 ret = -EBADF;
6552 }
6553
Pavel Begunkov71b547c2020-10-10 18:34:09 +01006554 state->ios_left--;
6555 return ret;
Pavel Begunkov0553b8b2020-04-08 08:58:45 +03006556}
6557
Pavel Begunkova6b8cadc2021-02-18 18:29:41 +00006558static int io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,
Pavel Begunkova1ab7b32021-02-18 18:29:42 +00006559 const struct io_uring_sqe *sqe)
Jens Axboe6c271ce2019-01-10 11:22:30 -07006560{
Pavel Begunkova1ab7b32021-02-18 18:29:42 +00006561 struct io_submit_link *link = &ctx->submit_state.link;
Jens Axboe6c271ce2019-01-10 11:22:30 -07006562 int ret;
6563
Pavel Begunkova6b8cadc2021-02-18 18:29:41 +00006564 ret = io_init_req(ctx, req, sqe);
6565 if (unlikely(ret)) {
6566fail_req:
Pavel Begunkovde59bc12021-02-18 18:29:47 +00006567 if (link->head) {
6568 /* fail even hard links since we don't submit */
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01006569 req_set_fail(link->head);
Pavel Begunkovf41db2732021-02-28 22:35:12 +00006570 io_req_complete_failed(link->head, -ECANCELED);
Pavel Begunkovde59bc12021-02-18 18:29:47 +00006571 link->head = NULL;
6572 }
Pavel Begunkovf41db2732021-02-28 22:35:12 +00006573 io_req_complete_failed(req, ret);
Pavel Begunkova6b8cadc2021-02-18 18:29:41 +00006574 return ret;
6575 }
Pavel Begunkovbe7053b2021-02-18 18:29:45 +00006576 ret = io_req_prep(req, sqe);
6577 if (unlikely(ret))
6578 goto fail_req;
Pavel Begunkova6b8cadc2021-02-18 18:29:41 +00006579
Pavel Begunkovbe7053b2021-02-18 18:29:45 +00006580 /* don't need @sqe from now on */
Pavel Begunkova6b8cadc2021-02-18 18:29:41 +00006581 trace_io_uring_submit_sqe(ctx, req->opcode, req->user_data,
6582 true, ctx->flags & IORING_SETUP_SQPOLL);
6583
Jens Axboe6c271ce2019-01-10 11:22:30 -07006584 /*
6585 * If we already have a head request, queue this one for async
6586 * submittal once the head completes. If we don't have a head but
6587 * IOSQE_IO_LINK is set in the sqe, start a new head. This one will be
6588 * submitted sync once the chain is complete. If none of those
6589 * conditions are true (normal request), then just queue it.
6590 */
6591 if (link->head) {
6592 struct io_kiocb *head = link->head;
6593
6594 /*
6595 * Taking sequential execution of a link, draining both sides
6596 * of the link also fullfils IOSQE_IO_DRAIN semantics for all
6597 * requests in the link. So, it drains the head and the
6598 * next after the link request. The last one is done via
6599 * drain_next flag to persist the effect across calls.
6600 */
6601 if (req->flags & REQ_F_IO_DRAIN) {
6602 head->flags |= REQ_F_IO_DRAIN;
6603 ctx->drain_next = 1;
6604 }
Pavel Begunkovb7e298d2021-02-28 22:35:19 +00006605 ret = io_req_prep_async(req);
Pavel Begunkovcf109602021-02-18 18:29:43 +00006606 if (unlikely(ret))
Pavel Begunkova6b8cadc2021-02-18 18:29:41 +00006607 goto fail_req;
Jens Axboe6c271ce2019-01-10 11:22:30 -07006608 trace_io_uring_link(ctx, req, head);
6609 link->last->link = req;
6610 link->last = req;
6611
6612 /* last request of a link, enqueue the link */
6613 if (!(req->flags & (REQ_F_LINK | REQ_F_HARDLINK))) {
Pavel Begunkovde59bc12021-02-18 18:29:47 +00006614 io_queue_sqe(head);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006615 link->head = NULL;
6616 }
Jackie Liu4fe2c962019-09-09 20:50:40 +08006617 } else {
6618 if (unlikely(ctx->drain_next)) {
6619 req->flags |= REQ_F_IO_DRAIN;
6620 ctx->drain_next = 0;
6621 }
6622 if (req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) {
Jackie Liu4fe2c962019-09-09 20:50:40 +08006623 link->head = req;
6624 link->last = req;
6625 } else {
Pavel Begunkovbe7053b2021-02-18 18:29:45 +00006626 io_queue_sqe(req);
Jackie Liu4fe2c962019-09-09 20:50:40 +08006627 }
6628 }
6629
6630 return 0;
6631}
6632
6633/*
6634 * Batched submission is done, ensure local IO is flushed out.
6635 */
6636static void io_submit_state_end(struct io_submit_state *state,
6637 struct io_ring_ctx *ctx)
Pavel Begunkov1b4a51b2019-11-21 11:54:28 +03006638{
Pavel Begunkova1ab7b32021-02-18 18:29:42 +00006639 if (state->link.head)
Pavel Begunkovde59bc12021-02-18 18:29:47 +00006640 io_queue_sqe(state->link.head);
Jens Axboe3529d8c2019-12-19 18:24:38 -07006641 if (state->comp.nr)
Jens Axboe9e645e112019-05-10 16:07:28 -06006642 io_submit_flush_completions(&state->comp, ctx);
Jackie Liua197f662019-11-08 08:09:12 -07006643 if (state->plug_started)
Pavel Begunkov32fe5252019-12-17 22:26:58 +03006644 blk_finish_plug(&state->plug);
Jens Axboe75c6a032020-01-28 10:15:23 -07006645 io_state_file_put(state);
Jens Axboe9e645e112019-05-10 16:07:28 -06006646}
Pavel Begunkov32fe5252019-12-17 22:26:58 +03006647
Jens Axboe9e645e112019-05-10 16:07:28 -06006648/*
6649 * Start submission side cache.
Pavel Begunkov32fe5252019-12-17 22:26:58 +03006650 */
Jens Axboe9e645e112019-05-10 16:07:28 -06006651static void io_submit_state_start(struct io_submit_state *state,
Pavel Begunkov196be952019-11-07 01:41:06 +03006652 unsigned int max_ios)
Jens Axboe9e645e112019-05-10 16:07:28 -06006653{
6654 state->plug_started = false;
Jens Axboebcda7ba2020-02-23 16:42:51 -07006655 state->ios_left = max_ios;
Pavel Begunkova1ab7b32021-02-18 18:29:42 +00006656 /* set only head, no need to init link_last in advance */
6657 state->link.head = NULL;
Jens Axboe75c6a032020-01-28 10:15:23 -07006658}
6659
Jens Axboe193155c2020-02-22 23:22:19 -07006660static void io_commit_sqring(struct io_ring_ctx *ctx)
6661{
Jens Axboe75c6a032020-01-28 10:15:23 -07006662 struct io_rings *rings = ctx->rings;
6663
6664 /*
Jens Axboe193155c2020-02-22 23:22:19 -07006665 * Ensure any loads from the SQEs are done at this point,
Jens Axboe75c6a032020-01-28 10:15:23 -07006666 * since once we write the new head, the application could
6667 * write new data to them.
Pavel Begunkov6b47ee62020-01-18 20:22:41 +03006668 */
Pavel Begunkov8da11c12020-02-24 11:32:44 +03006669 smp_store_release(&rings->sq.head, ctx->cached_sq_head);
Jens Axboebcda7ba2020-02-23 16:42:51 -07006670}
6671
Jens Axboe9e645e112019-05-10 16:07:28 -06006672/*
Jens Axboe3529d8c2019-12-19 18:24:38 -07006673 * Fetch an sqe, if one is available. Note that sqe_ptr will point to memory
Jens Axboe9e645e112019-05-10 16:07:28 -06006674 * that is mapped by userspace. This means that care needs to be taken to
6675 * ensure that reads are stable, as we cannot rely on userspace always
Jens Axboe78e19bb2019-11-06 15:21:34 -07006676 * being a good citizen. If members of the sqe are validated and then later
6677 * used, it's important that those reads are done through READ_ONCE() to
Pavel Begunkov2e6e1fd2019-12-05 16:15:45 +03006678 * prevent a re-load down the line.
Jens Axboe9e645e112019-05-10 16:07:28 -06006679 */
6680static const struct io_uring_sqe *io_get_sqe(struct io_ring_ctx *ctx)
Jens Axboe9e645e112019-05-10 16:07:28 -06006681{
6682 u32 *sq_array = ctx->sq_array;
6683 unsigned head;
6684
6685 /*
6686 * The cached sq head (or cq tail) serves two purposes:
6687 *
6688 * 1) allows us to batch the cost of updating the user visible
Pavel Begunkov9d763772019-12-17 02:22:07 +03006689 * head updates.
Jens Axboe9e645e112019-05-10 16:07:28 -06006690 * 2) allows the kernel side to track the head on its own, even
Pavel Begunkov8cdf2192020-01-25 00:40:24 +03006691 * though the application is the one updating it.
6692 */
6693 head = READ_ONCE(sq_array[ctx->cached_sq_head++ & ctx->sq_mask]);
6694 if (likely(head < ctx->sq_entries))
6695 return &ctx->sq_sqes[head];
6696
6697 /* drop invalid entries */
Pavel Begunkov711be032020-01-17 03:57:59 +03006698 ctx->cached_sq_dropped++;
6699 WRITE_ONCE(ctx->rings->sq_dropped, ctx->cached_sq_dropped);
6700 return NULL;
6701}
Jens Axboeb7bb4f72019-12-15 22:13:43 -07006702
Jens Axboe0f212202020-09-13 13:09:39 -06006703static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr)
Jens Axboe6c271ce2019-01-10 11:22:30 -07006704{
Pavel Begunkov46c4e162021-02-18 18:29:37 +00006705 int submitted = 0;
Jens Axboe6c271ce2019-01-10 11:22:30 -07006706
Pavel Begunkovee7d46d2019-12-30 21:24:45 +03006707 /* make sure SQ entry isn't read before tail */
6708 nr = min3(nr, ctx->sq_entries, io_sqring_entries(ctx));
Pavel Begunkov9ef4f122019-12-30 21:24:44 +03006709
Pavel Begunkov2b85edf2019-12-28 14:13:03 +03006710 if (!percpu_ref_tryget_many(&ctx->refs, nr))
6711 return -EAGAIN;
Jens Axboe6c271ce2019-01-10 11:22:30 -07006712
Jens Axboed8a6df12020-10-15 16:24:45 -06006713 percpu_counter_add(&current->io_uring->inflight, nr);
Jens Axboefaf7b512020-10-07 12:48:53 -06006714 refcount_add(nr, &current->usage);
Pavel Begunkovba88ff12021-02-10 00:03:11 +00006715 io_submit_state_start(&ctx->submit_state, nr);
Pavel Begunkovb14cca02020-01-17 04:45:59 +03006716
Pavel Begunkov46c4e162021-02-18 18:29:37 +00006717 while (submitted < nr) {
Jens Axboe3529d8c2019-12-19 18:24:38 -07006718 const struct io_uring_sqe *sqe;
Pavel Begunkov196be952019-11-07 01:41:06 +03006719 struct io_kiocb *req;
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03006720
Pavel Begunkov258b29a2021-02-10 00:03:10 +00006721 req = io_alloc_req(ctx);
Pavel Begunkov196be952019-11-07 01:41:06 +03006722 if (unlikely(!req)) {
6723 if (!submitted)
6724 submitted = -EAGAIN;
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03006725 break;
Jens Axboe9e645e112019-05-10 16:07:28 -06006726 }
Pavel Begunkov4fccfcb2021-02-12 11:55:17 +00006727 sqe = io_get_sqe(ctx);
6728 if (unlikely(!sqe)) {
6729 kmem_cache_free(req_cachep, req);
6730 break;
6731 }
Jens Axboed3656342019-12-18 09:50:26 -07006732 /* will complete beyond this point, count as submitted */
6733 submitted++;
Pavel Begunkova1ab7b32021-02-18 18:29:42 +00006734 if (io_submit_sqe(ctx, req, sqe))
Jens Axboed3656342019-12-18 09:50:26 -07006735 break;
Jens Axboe6c271ce2019-01-10 11:22:30 -07006736 }
6737
Pavel Begunkov9466f432020-01-25 22:34:01 +03006738 if (unlikely(submitted != nr)) {
6739 int ref_used = (submitted == -EAGAIN) ? 0 : submitted;
Jens Axboed8a6df12020-10-15 16:24:45 -06006740 struct io_uring_task *tctx = current->io_uring;
6741 int unused = nr - ref_used;
Pavel Begunkov9466f432020-01-25 22:34:01 +03006742
Jens Axboed8a6df12020-10-15 16:24:45 -06006743 percpu_ref_put_many(&ctx->refs, unused);
6744 percpu_counter_sub(&tctx->inflight, unused);
6745 put_task_struct_many(current, unused);
Pavel Begunkov9466f432020-01-25 22:34:01 +03006746 }
Jens Axboe6c271ce2019-01-10 11:22:30 -07006747
Pavel Begunkova1ab7b32021-02-18 18:29:42 +00006748 io_submit_state_end(&ctx->submit_state, ctx);
Pavel Begunkovae9428c2019-11-06 00:22:14 +03006749 /* Commit SQ ring head once we've consumed and submitted all SQEs */
6750 io_commit_sqring(ctx);
6751
Jens Axboe6c271ce2019-01-10 11:22:30 -07006752 return submitted;
6753}
6754
Pavel Begunkove4b6d902021-05-16 22:58:00 +01006755static inline bool io_sqd_events_pending(struct io_sq_data *sqd)
6756{
6757 return READ_ONCE(sqd->state);
6758}
6759
Xiaoguang Wang23b36282020-07-23 20:57:24 +08006760static inline void io_ring_set_wakeup_flag(struct io_ring_ctx *ctx)
6761{
6762 /* Tell userspace we may need a wakeup call */
6763 spin_lock_irq(&ctx->completion_lock);
6764 ctx->rings->sq_flags |= IORING_SQ_NEED_WAKEUP;
6765 spin_unlock_irq(&ctx->completion_lock);
6766}
6767
6768static inline void io_ring_clear_wakeup_flag(struct io_ring_ctx *ctx)
6769{
6770 spin_lock_irq(&ctx->completion_lock);
6771 ctx->rings->sq_flags &= ~IORING_SQ_NEED_WAKEUP;
6772 spin_unlock_irq(&ctx->completion_lock);
6773}
6774
Xiaoguang Wang08369242020-11-03 14:15:59 +08006775static int __io_sq_thread(struct io_ring_ctx *ctx, bool cap_entries)
Jens Axboe6c271ce2019-01-10 11:22:30 -07006776{
Jens Axboec8d1ba52020-09-14 11:07:26 -06006777 unsigned int to_submit;
Xiaoguang Wangbdcd3ea2020-02-25 22:12:08 +08006778 int ret = 0;
Jens Axboe6c271ce2019-01-10 11:22:30 -07006779
Jens Axboec8d1ba52020-09-14 11:07:26 -06006780 to_submit = io_sqring_entries(ctx);
Jens Axboee95eee22020-09-08 09:11:32 -06006781 /* if we're handling multiple rings, cap submit size for fairness */
6782 if (cap_entries && to_submit > 8)
6783 to_submit = 8;
6784
Xiaoguang Wang906a3c62020-11-12 14:56:00 +08006785 if (!list_empty(&ctx->iopoll_list) || to_submit) {
6786 unsigned nr_events = 0;
6787
Xiaoguang Wang08369242020-11-03 14:15:59 +08006788 mutex_lock(&ctx->uring_lock);
Xiaoguang Wang906a3c62020-11-12 14:56:00 +08006789 if (!list_empty(&ctx->iopoll_list))
6790 io_do_iopoll(ctx, &nr_events, 0);
6791
Pavel Begunkov3b763ba2021-04-18 14:52:08 +01006792 /*
6793 * Don't submit if refs are dying, good for io_uring_register(),
6794 * but also it is relied upon by io_ring_exit_work()
6795 */
Pavel Begunkov0298ef92021-03-08 13:20:57 +00006796 if (to_submit && likely(!percpu_ref_is_dying(&ctx->refs)) &&
6797 !(ctx->flags & IORING_SETUP_R_DISABLED))
Xiaoguang Wang08369242020-11-03 14:15:59 +08006798 ret = io_submit_sqes(ctx, to_submit);
6799 mutex_unlock(&ctx->uring_lock);
Jens Axboe90554202020-09-03 12:12:41 -06006800
Pavel Begunkovacfb3812021-05-16 22:58:03 +01006801 if (to_submit && wq_has_sleeper(&ctx->sqo_sq_wait))
6802 wake_up(&ctx->sqo_sq_wait);
6803 }
Jens Axboe90554202020-09-03 12:12:41 -06006804
Xiaoguang Wang08369242020-11-03 14:15:59 +08006805 return ret;
6806}
6807
6808static void io_sqd_update_thread_idle(struct io_sq_data *sqd)
6809{
6810 struct io_ring_ctx *ctx;
6811 unsigned sq_thread_idle = 0;
6812
Pavel Begunkovc9dca272021-03-10 13:13:55 +00006813 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list)
6814 sq_thread_idle = max(sq_thread_idle, ctx->sq_thread_idle);
Xiaoguang Wang08369242020-11-03 14:15:59 +08006815 sqd->sq_thread_idle = sq_thread_idle;
Jens Axboec8d1ba52020-09-14 11:07:26 -06006816}
6817
Pavel Begunkove4b6d902021-05-16 22:58:00 +01006818static bool io_sqd_handle_event(struct io_sq_data *sqd)
6819{
6820 bool did_sig = false;
6821 struct ksignal ksig;
6822
6823 if (test_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state) ||
6824 signal_pending(current)) {
6825 mutex_unlock(&sqd->lock);
6826 if (signal_pending(current))
6827 did_sig = get_signal(&ksig);
6828 cond_resched();
6829 mutex_lock(&sqd->lock);
6830 }
6831 io_run_task_work();
Pavel Begunkove4b6d902021-05-16 22:58:00 +01006832 return did_sig || test_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state);
6833}
6834
Jens Axboe6c271ce2019-01-10 11:22:30 -07006835static int io_sq_thread(void *data)
6836{
Jens Axboe69fb2132020-09-14 11:16:23 -06006837 struct io_sq_data *sqd = data;
6838 struct io_ring_ctx *ctx;
Xiaoguang Wanga0d92052020-11-12 14:55:59 +08006839 unsigned long timeout = 0;
Jens Axboe37d1e2e2021-02-17 21:03:43 -07006840 char buf[TASK_COMM_LEN];
Xiaoguang Wang08369242020-11-03 14:15:59 +08006841 DEFINE_WAIT(wait);
Jens Axboe6c271ce2019-01-10 11:22:30 -07006842
Pavel Begunkov696ee882021-04-01 09:55:04 +01006843 snprintf(buf, sizeof(buf), "iou-sqp-%d", sqd->task_pid);
Jens Axboe37d1e2e2021-02-17 21:03:43 -07006844 set_task_comm(current, buf);
Jens Axboe28cea78a2020-09-14 10:51:17 -06006845
Jens Axboe37d1e2e2021-02-17 21:03:43 -07006846 if (sqd->sq_cpu != -1)
6847 set_cpus_allowed_ptr(current, cpumask_of(sqd->sq_cpu));
6848 else
6849 set_cpus_allowed_ptr(current, cpu_online_mask);
6850 current->flags |= PF_NO_SETAFFINITY;
6851
Pavel Begunkov09a6f4e2021-03-14 20:57:10 +00006852 mutex_lock(&sqd->lock);
Pavel Begunkove4b6d902021-05-16 22:58:00 +01006853 while (1) {
Xiaoguang Wang08369242020-11-03 14:15:59 +08006854 int ret;
6855 bool cap_entries, sqt_spin, needs_sched;
Jens Axboec1edbf52019-11-10 16:56:04 -07006856
Pavel Begunkove4b6d902021-05-16 22:58:00 +01006857 if (io_sqd_events_pending(sqd) || signal_pending(current)) {
6858 if (io_sqd_handle_event(sqd))
Pavel Begunkovc7d95612021-04-13 11:43:00 +01006859 break;
Xiaoguang Wang08369242020-11-03 14:15:59 +08006860 timeout = jiffies + sqd->sq_thread_idle;
Pavel Begunkov7d41e852021-03-10 13:13:54 +00006861 continue;
Xiaoguang Wang08369242020-11-03 14:15:59 +08006862 }
Pavel Begunkove4b6d902021-05-16 22:58:00 +01006863
Xiaoguang Wang08369242020-11-03 14:15:59 +08006864 sqt_spin = false;
Jens Axboee95eee22020-09-08 09:11:32 -06006865 cap_entries = !list_is_singular(&sqd->ctx_list);
Jens Axboe69fb2132020-09-14 11:16:23 -06006866 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) {
Stefan Metzmacher7c30f36a2021-03-07 11:54:28 +01006867 const struct cred *creds = NULL;
6868
6869 if (ctx->sq_creds != current_cred())
6870 creds = override_creds(ctx->sq_creds);
Xiaoguang Wang08369242020-11-03 14:15:59 +08006871 ret = __io_sq_thread(ctx, cap_entries);
Stefan Metzmacher7c30f36a2021-03-07 11:54:28 +01006872 if (creds)
6873 revert_creds(creds);
Xiaoguang Wang08369242020-11-03 14:15:59 +08006874 if (!sqt_spin && (ret > 0 || !list_empty(&ctx->iopoll_list)))
6875 sqt_spin = true;
Jens Axboe6c271ce2019-01-10 11:22:30 -07006876 }
6877
Xiaoguang Wang08369242020-11-03 14:15:59 +08006878 if (sqt_spin || !time_after(jiffies, timeout)) {
Jens Axboec8d1ba52020-09-14 11:07:26 -06006879 io_run_task_work();
6880 cond_resched();
Xiaoguang Wang08369242020-11-03 14:15:59 +08006881 if (sqt_spin)
6882 timeout = jiffies + sqd->sq_thread_idle;
6883 continue;
6884 }
6885
Xiaoguang Wang08369242020-11-03 14:15:59 +08006886 prepare_to_wait(&sqd->wait, &wait, TASK_INTERRUPTIBLE);
Pavel Begunkove4b6d902021-05-16 22:58:00 +01006887 if (!io_sqd_events_pending(sqd)) {
Hao Xu724cb4f2021-04-21 23:19:11 +08006888 needs_sched = true;
6889 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) {
Pavel Begunkovaaa9f0f2021-05-16 22:58:01 +01006890 io_ring_set_wakeup_flag(ctx);
6891
Hao Xu724cb4f2021-04-21 23:19:11 +08006892 if ((ctx->flags & IORING_SETUP_IOPOLL) &&
6893 !list_empty_careful(&ctx->iopoll_list)) {
6894 needs_sched = false;
6895 break;
6896 }
6897 if (io_sqring_entries(ctx)) {
6898 needs_sched = false;
6899 break;
6900 }
6901 }
6902
6903 if (needs_sched) {
6904 mutex_unlock(&sqd->lock);
6905 schedule();
6906 mutex_lock(&sqd->lock);
6907 }
Jens Axboe69fb2132020-09-14 11:16:23 -06006908 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list)
6909 io_ring_clear_wakeup_flag(ctx);
Jens Axboe6c271ce2019-01-10 11:22:30 -07006910 }
Xiaoguang Wang08369242020-11-03 14:15:59 +08006911
6912 finish_wait(&sqd->wait, &wait);
6913 timeout = jiffies + sqd->sq_thread_idle;
Jens Axboe6c271ce2019-01-10 11:22:30 -07006914 }
6915
Pavel Begunkov734551d2021-04-18 14:52:09 +01006916 io_uring_cancel_sqpoll(sqd);
Jens Axboe37d1e2e2021-02-17 21:03:43 -07006917 sqd->thread = NULL;
Jens Axboe05962f92021-03-06 13:58:48 -07006918 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list)
Jens Axboe5f3f26f2021-02-25 10:17:46 -07006919 io_ring_set_wakeup_flag(ctx);
Pavel Begunkov521d6a72021-03-11 23:29:38 +00006920 io_run_task_work();
Pavel Begunkov734551d2021-04-18 14:52:09 +01006921 mutex_unlock(&sqd->lock);
6922
Jens Axboe37d1e2e2021-02-17 21:03:43 -07006923 complete(&sqd->exited);
6924 do_exit(0);
Jens Axboe6c271ce2019-01-10 11:22:30 -07006925}
6926
Jens Axboebda52162019-09-24 13:47:15 -06006927struct io_wait_queue {
6928 struct wait_queue_entry wq;
6929 struct io_ring_ctx *ctx;
6930 unsigned to_wait;
6931 unsigned nr_timeouts;
6932};
6933
Pavel Begunkov6c503152021-01-04 20:36:36 +00006934static inline bool io_should_wake(struct io_wait_queue *iowq)
Jens Axboebda52162019-09-24 13:47:15 -06006935{
6936 struct io_ring_ctx *ctx = iowq->ctx;
6937
6938 /*
Brian Gianforcarod195a662019-12-13 03:09:50 -08006939 * Wake up if we have enough events, or if a timeout occurred since we
Jens Axboebda52162019-09-24 13:47:15 -06006940 * started waiting. For timeouts, we always want to return to userspace,
6941 * regardless of event count.
6942 */
Pavel Begunkov6c503152021-01-04 20:36:36 +00006943 return io_cqring_events(ctx) >= iowq->to_wait ||
Jens Axboebda52162019-09-24 13:47:15 -06006944 atomic_read(&ctx->cq_timeouts) != iowq->nr_timeouts;
6945}
6946
6947static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode,
6948 int wake_flags, void *key)
6949{
6950 struct io_wait_queue *iowq = container_of(curr, struct io_wait_queue,
6951 wq);
6952
Pavel Begunkov6c503152021-01-04 20:36:36 +00006953 /*
6954 * Cannot safely flush overflowed CQEs from here, ensure we wake up
6955 * the task, and the next invocation will do it.
6956 */
6957 if (io_should_wake(iowq) || test_bit(0, &iowq->ctx->cq_check_overflow))
6958 return autoremove_wake_function(curr, mode, wake_flags, key);
6959 return -1;
Jens Axboebda52162019-09-24 13:47:15 -06006960}
6961
Jens Axboeaf9c1a42020-09-24 13:32:18 -06006962static int io_run_task_work_sig(void)
6963{
6964 if (io_run_task_work())
6965 return 1;
6966 if (!signal_pending(current))
6967 return 0;
Jens Axboe0b8cfa92021-03-21 14:16:08 -06006968 if (test_thread_flag(TIF_NOTIFY_SIGNAL))
Jens Axboe792ee0f62020-10-22 20:17:18 -06006969 return -ERESTARTSYS;
Jens Axboeaf9c1a42020-09-24 13:32:18 -06006970 return -EINTR;
6971}
6972
Pavel Begunkoveeb60b92021-02-04 13:51:58 +00006973/* when returns >0, the caller should retry */
6974static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx,
6975 struct io_wait_queue *iowq,
6976 signed long *timeout)
6977{
6978 int ret;
6979
6980 /* make sure we run task_work before checking for signals */
6981 ret = io_run_task_work_sig();
6982 if (ret || io_should_wake(iowq))
6983 return ret;
6984 /* let the caller flush overflows, retry */
6985 if (test_bit(0, &ctx->cq_check_overflow))
6986 return 1;
6987
6988 *timeout = schedule_timeout(*timeout);
6989 return !*timeout ? -ETIME : 1;
6990}
6991
Jens Axboe2b188cc2019-01-07 10:46:33 -07006992/*
6993 * Wait until events become available, if we don't already have some. The
6994 * application must reap them itself, as they reside on the shared cq ring.
6995 */
6996static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events,
Hao Xuc73ebb62020-11-03 10:54:37 +08006997 const sigset_t __user *sig, size_t sigsz,
6998 struct __kernel_timespec __user *uts)
Jens Axboe2b188cc2019-01-07 10:46:33 -07006999{
Jens Axboebda52162019-09-24 13:47:15 -06007000 struct io_wait_queue iowq = {
7001 .wq = {
7002 .private = current,
7003 .func = io_wake_function,
7004 .entry = LIST_HEAD_INIT(iowq.wq.entry),
7005 },
7006 .ctx = ctx,
7007 .to_wait = min_events,
7008 };
Hristo Venev75b28af2019-08-26 17:23:46 +00007009 struct io_rings *rings = ctx->rings;
Pavel Begunkovc1d5a222021-02-04 13:51:57 +00007010 signed long timeout = MAX_SCHEDULE_TIMEOUT;
7011 int ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07007012
Jens Axboeb41e9852020-02-17 09:52:41 -07007013 do {
Pavel Begunkov6c2450a2021-02-23 12:40:22 +00007014 io_cqring_overflow_flush(ctx, false);
Pavel Begunkov6c503152021-01-04 20:36:36 +00007015 if (io_cqring_events(ctx) >= min_events)
Jens Axboeb41e9852020-02-17 09:52:41 -07007016 return 0;
Jens Axboe4c6e2772020-07-01 11:29:10 -06007017 if (!io_run_task_work())
Jens Axboeb41e9852020-02-17 09:52:41 -07007018 break;
Jens Axboeb41e9852020-02-17 09:52:41 -07007019 } while (1);
Jens Axboe2b188cc2019-01-07 10:46:33 -07007020
7021 if (sig) {
Arnd Bergmann9e75ad52019-03-25 15:34:53 +01007022#ifdef CONFIG_COMPAT
7023 if (in_compat_syscall())
7024 ret = set_compat_user_sigmask((const compat_sigset_t __user *)sig,
Oleg Nesterovb7724342019-07-16 16:29:53 -07007025 sigsz);
Arnd Bergmann9e75ad52019-03-25 15:34:53 +01007026 else
7027#endif
Oleg Nesterovb7724342019-07-16 16:29:53 -07007028 ret = set_user_sigmask(sig, sigsz);
Arnd Bergmann9e75ad52019-03-25 15:34:53 +01007029
Jens Axboe2b188cc2019-01-07 10:46:33 -07007030 if (ret)
7031 return ret;
7032 }
7033
Hao Xuc73ebb62020-11-03 10:54:37 +08007034 if (uts) {
Pavel Begunkovc1d5a222021-02-04 13:51:57 +00007035 struct timespec64 ts;
7036
Hao Xuc73ebb62020-11-03 10:54:37 +08007037 if (get_timespec64(&ts, uts))
7038 return -EFAULT;
7039 timeout = timespec64_to_jiffies(&ts);
7040 }
7041
Jens Axboebda52162019-09-24 13:47:15 -06007042 iowq.nr_timeouts = atomic_read(&ctx->cq_timeouts);
Dmitrii Dolgovc826bd72019-10-15 19:02:01 +02007043 trace_io_uring_cqring_wait(ctx, min_events);
Jens Axboebda52162019-09-24 13:47:15 -06007044 do {
Jens Axboeca0a2652021-03-04 17:15:48 -07007045 /* if we can't even flush overflow, don't wait for more */
Pavel Begunkov6c2450a2021-02-23 12:40:22 +00007046 if (!io_cqring_overflow_flush(ctx, false)) {
Jens Axboeca0a2652021-03-04 17:15:48 -07007047 ret = -EBUSY;
7048 break;
7049 }
Jens Axboebda52162019-09-24 13:47:15 -06007050 prepare_to_wait_exclusive(&ctx->wait, &iowq.wq,
7051 TASK_INTERRUPTIBLE);
Pavel Begunkoveeb60b92021-02-04 13:51:58 +00007052 ret = io_cqring_wait_schedule(ctx, &iowq, &timeout);
7053 finish_wait(&ctx->wait, &iowq.wq);
Jens Axboeca0a2652021-03-04 17:15:48 -07007054 cond_resched();
Pavel Begunkoveeb60b92021-02-04 13:51:58 +00007055 } while (ret > 0);
Jens Axboebda52162019-09-24 13:47:15 -06007056
Jens Axboeb7db41c2020-07-04 08:55:50 -06007057 restore_saved_sigmask_unless(ret == -EINTR);
Jens Axboe2b188cc2019-01-07 10:46:33 -07007058
Hristo Venev75b28af2019-08-26 17:23:46 +00007059 return READ_ONCE(rings->cq.head) == READ_ONCE(rings->cq.tail) ? ret : 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07007060}
7061
Pavel Begunkovaeca2412021-04-11 01:46:37 +01007062static void io_free_file_tables(struct io_file_table *table, unsigned nr_files)
Pavel Begunkov846a4ef2021-04-01 15:44:03 +01007063{
7064 unsigned i, nr_tables = DIV_ROUND_UP(nr_files, IORING_MAX_FILES_TABLE);
7065
7066 for (i = 0; i < nr_tables; i++)
Pavel Begunkovaeca2412021-04-11 01:46:37 +01007067 kfree(table->files[i]);
7068 kfree(table->files);
7069 table->files = NULL;
Pavel Begunkov846a4ef2021-04-01 15:44:03 +01007070}
7071
Bijan Mottahedeh2a63b2d2021-01-15 17:37:47 +00007072static inline void io_rsrc_ref_lock(struct io_ring_ctx *ctx)
Pavel Begunkov1642b442020-12-30 21:34:14 +00007073{
Bijan Mottahedeh2a63b2d2021-01-15 17:37:47 +00007074 spin_lock_bh(&ctx->rsrc_ref_lock);
Pavel Begunkov1642b442020-12-30 21:34:14 +00007075}
7076
Bijan Mottahedeh2a63b2d2021-01-15 17:37:47 +00007077static inline void io_rsrc_ref_unlock(struct io_ring_ctx *ctx)
Jens Axboe6b063142019-01-10 22:13:58 -07007078{
Bijan Mottahedeh2a63b2d2021-01-15 17:37:47 +00007079 spin_unlock_bh(&ctx->rsrc_ref_lock);
7080}
7081
Pavel Begunkov28a9fe22021-04-01 15:43:47 +01007082static void io_rsrc_node_destroy(struct io_rsrc_node *ref_node)
7083{
7084 percpu_ref_exit(&ref_node->refs);
7085 kfree(ref_node);
7086}
7087
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01007088static void io_rsrc_node_switch(struct io_ring_ctx *ctx,
7089 struct io_rsrc_data *data_to_kill)
Jens Axboe6b063142019-01-10 22:13:58 -07007090{
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01007091 WARN_ON_ONCE(!ctx->rsrc_backup_node);
7092 WARN_ON_ONCE(data_to_kill && !ctx->rsrc_node);
Pavel Begunkov82fbcfa2021-04-01 15:43:43 +01007093
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01007094 if (data_to_kill) {
7095 struct io_rsrc_node *rsrc_node = ctx->rsrc_node;
Pavel Begunkov82fbcfa2021-04-01 15:43:43 +01007096
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01007097 rsrc_node->rsrc_data = data_to_kill;
7098 io_rsrc_ref_lock(ctx);
7099 list_add_tail(&rsrc_node->node, &ctx->rsrc_ref_list);
7100 io_rsrc_ref_unlock(ctx);
Pavel Begunkov82fbcfa2021-04-01 15:43:43 +01007101
Pavel Begunkov3e942492021-04-11 01:46:34 +01007102 atomic_inc(&data_to_kill->refs);
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01007103 percpu_ref_kill(&rsrc_node->refs);
7104 ctx->rsrc_node = NULL;
7105 }
7106
7107 if (!ctx->rsrc_node) {
7108 ctx->rsrc_node = ctx->rsrc_backup_node;
7109 ctx->rsrc_backup_node = NULL;
7110 }
Jens Axboe6b063142019-01-10 22:13:58 -07007111}
7112
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01007113static int io_rsrc_node_switch_start(struct io_ring_ctx *ctx)
Pavel Begunkov8dd03af2021-03-19 17:22:36 +00007114{
7115 if (ctx->rsrc_backup_node)
7116 return 0;
Pavel Begunkovb895c9a2021-04-01 15:43:40 +01007117 ctx->rsrc_backup_node = io_rsrc_node_alloc(ctx);
Pavel Begunkov8dd03af2021-03-19 17:22:36 +00007118 return ctx->rsrc_backup_node ? 0 : -ENOMEM;
7119}
7120
Pavel Begunkov40ae0ff2021-04-01 15:43:44 +01007121static int io_rsrc_ref_quiesce(struct io_rsrc_data *data, struct io_ring_ctx *ctx)
Hao Xu8bad28d2021-02-19 17:19:36 +08007122{
7123 int ret;
Xiaoguang Wang05589552020-03-31 14:05:18 +08007124
Pavel Begunkov215c3902021-04-01 15:43:48 +01007125 /* As we may drop ->uring_lock, other task may have started quiesce */
Hao Xu8bad28d2021-02-19 17:19:36 +08007126 if (data->quiesce)
7127 return -ENXIO;
7128
7129 data->quiesce = true;
Pavel Begunkov1ffc5422020-12-30 21:34:15 +00007130 do {
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01007131 ret = io_rsrc_node_switch_start(ctx);
Pavel Begunkov8dd03af2021-03-19 17:22:36 +00007132 if (ret)
Pavel Begunkovf2303b12021-02-20 18:03:49 +00007133 break;
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01007134 io_rsrc_node_switch(ctx, data);
7135
Pavel Begunkov3e942492021-04-11 01:46:34 +01007136 /* kill initial ref, already quiesced if zero */
7137 if (atomic_dec_and_test(&data->refs))
7138 break;
Hao Xu8bad28d2021-02-19 17:19:36 +08007139 flush_delayed_work(&ctx->rsrc_put_work);
Pavel Begunkov1ffc5422020-12-30 21:34:15 +00007140 ret = wait_for_completion_interruptible(&data->done);
7141 if (!ret)
7142 break;
Jens Axboe05f3fb32019-12-09 11:22:50 -07007143
Pavel Begunkov3e942492021-04-11 01:46:34 +01007144 atomic_inc(&data->refs);
7145 /* wait for all works potentially completing data->done */
7146 flush_delayed_work(&ctx->rsrc_put_work);
Jens Axboecb5e1b82021-02-25 07:37:35 -07007147 reinit_completion(&data->done);
Pavel Begunkov8dd03af2021-03-19 17:22:36 +00007148
Hao Xu8bad28d2021-02-19 17:19:36 +08007149 mutex_unlock(&ctx->uring_lock);
7150 ret = io_run_task_work_sig();
7151 mutex_lock(&ctx->uring_lock);
Pavel Begunkovf2303b12021-02-20 18:03:49 +00007152 } while (ret >= 0);
Hao Xu8bad28d2021-02-19 17:19:36 +08007153 data->quiesce = false;
7154
Hao Xu8bad28d2021-02-19 17:19:36 +08007155 return ret;
Bijan Mottahedehd7954b22021-01-15 17:37:50 +00007156}
7157
Pavel Begunkov44b31f22021-04-25 14:32:16 +01007158static void io_rsrc_data_free(struct io_rsrc_data *data)
7159{
Pavel Begunkovb60c8dc2021-04-25 14:32:18 +01007160 kvfree(data->tags);
Pavel Begunkov44b31f22021-04-25 14:32:16 +01007161 kfree(data);
7162}
7163
Pavel Begunkov40ae0ff2021-04-01 15:43:44 +01007164static struct io_rsrc_data *io_rsrc_data_alloc(struct io_ring_ctx *ctx,
Pavel Begunkovb60c8dc2021-04-25 14:32:18 +01007165 rsrc_put_fn *do_put,
7166 unsigned nr)
Bijan Mottahedeh1ad555c2021-01-15 17:37:51 +00007167{
Pavel Begunkovb895c9a2021-04-01 15:43:40 +01007168 struct io_rsrc_data *data;
Bijan Mottahedeh1ad555c2021-01-15 17:37:51 +00007169
7170 data = kzalloc(sizeof(*data), GFP_KERNEL);
7171 if (!data)
7172 return NULL;
7173
Pavel Begunkovb60c8dc2021-04-25 14:32:18 +01007174 data->tags = kvcalloc(nr, sizeof(*data->tags), GFP_KERNEL);
7175 if (!data->tags) {
7176 kfree(data);
7177 return NULL;
7178 }
7179
Pavel Begunkov3e942492021-04-11 01:46:34 +01007180 atomic_set(&data->refs, 1);
Bijan Mottahedeh1ad555c2021-01-15 17:37:51 +00007181 data->ctx = ctx;
Pavel Begunkov40ae0ff2021-04-01 15:43:44 +01007182 data->do_put = do_put;
Bijan Mottahedeh1ad555c2021-01-15 17:37:51 +00007183 init_completion(&data->done);
7184 return data;
7185}
7186
Roman Penyaev2bbcd6d2019-05-16 10:53:57 +02007187static void __io_sqe_files_unregister(struct io_ring_ctx *ctx)
7188{
Jens Axboe06058632019-04-13 09:26:03 -06007189#if defined(CONFIG_UNIX)
Jens Axboe6c271ce2019-01-10 11:22:30 -07007190 if (ctx->ring_sock) {
7191 struct sock *sock = ctx->ring_sock->sk;
7192 struct sk_buff *skb;
7193
7194 while ((skb = skb_dequeue(&sock->sk_receive_queue)) != NULL)
Jens Axboe6b063142019-01-10 22:13:58 -07007195 kfree_skb(skb);
7196 }
Jens Axboe6c271ce2019-01-10 11:22:30 -07007197#else
7198 int i;
Jens Axboe6b063142019-01-10 22:13:58 -07007199
7200 for (i = 0; i < ctx->nr_user_files; i++) {
7201 struct file *file;
7202
7203 file = io_file_from_index(ctx, i);
7204 if (file)
7205 fput(file);
7206 }
7207#endif
Pavel Begunkovfff4db72021-04-25 14:32:15 +01007208 io_free_file_tables(&ctx->file_table, ctx->nr_user_files);
Pavel Begunkov44b31f22021-04-25 14:32:16 +01007209 io_rsrc_data_free(ctx->file_data);
Pavel Begunkovfff4db72021-04-25 14:32:15 +01007210 ctx->file_data = NULL;
7211 ctx->nr_user_files = 0;
Bijan Mottahedeh1ad555c2021-01-15 17:37:51 +00007212}
7213
Bijan Mottahedehd7954b22021-01-15 17:37:50 +00007214static int io_sqe_files_unregister(struct io_ring_ctx *ctx)
7215{
Bijan Mottahedehd7954b22021-01-15 17:37:50 +00007216 int ret;
7217
Pavel Begunkov08480402021-04-13 02:58:38 +01007218 if (!ctx->file_data)
Bijan Mottahedehd7954b22021-01-15 17:37:50 +00007219 return -ENXIO;
Pavel Begunkov08480402021-04-13 02:58:38 +01007220 ret = io_rsrc_ref_quiesce(ctx->file_data, ctx);
7221 if (!ret)
7222 __io_sqe_files_unregister(ctx);
7223 return ret;
Jens Axboe6c271ce2019-01-10 11:22:30 -07007224}
7225
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007226static void io_sq_thread_unpark(struct io_sq_data *sqd)
Pavel Begunkov09a6f4e2021-03-14 20:57:10 +00007227 __releases(&sqd->lock)
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007228{
Pavel Begunkov521d6a72021-03-11 23:29:38 +00007229 WARN_ON_ONCE(sqd->thread == current);
7230
Pavel Begunkov9e138a42021-03-14 20:57:12 +00007231 /*
7232 * Do the dance but not conditional clear_bit() because it'd race with
7233 * other threads incrementing park_pending and setting the bit.
7234 */
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007235 clear_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state);
Pavel Begunkov9e138a42021-03-14 20:57:12 +00007236 if (atomic_dec_return(&sqd->park_pending))
7237 set_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state);
Pavel Begunkov09a6f4e2021-03-14 20:57:10 +00007238 mutex_unlock(&sqd->lock);
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007239}
7240
Jens Axboe86e0d672021-03-05 08:44:39 -07007241static void io_sq_thread_park(struct io_sq_data *sqd)
Pavel Begunkov09a6f4e2021-03-14 20:57:10 +00007242 __acquires(&sqd->lock)
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007243{
Pavel Begunkov521d6a72021-03-11 23:29:38 +00007244 WARN_ON_ONCE(sqd->thread == current);
7245
Pavel Begunkov9e138a42021-03-14 20:57:12 +00007246 atomic_inc(&sqd->park_pending);
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007247 set_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state);
Pavel Begunkov09a6f4e2021-03-14 20:57:10 +00007248 mutex_lock(&sqd->lock);
Jens Axboe05962f92021-03-06 13:58:48 -07007249 if (sqd->thread)
Jens Axboe86e0d672021-03-05 08:44:39 -07007250 wake_up_process(sqd->thread);
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007251}
7252
7253static void io_sq_thread_stop(struct io_sq_data *sqd)
7254{
Pavel Begunkov521d6a72021-03-11 23:29:38 +00007255 WARN_ON_ONCE(sqd->thread == current);
Pavel Begunkov88885f62021-04-11 01:46:38 +01007256 WARN_ON_ONCE(test_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state));
Pavel Begunkov521d6a72021-03-11 23:29:38 +00007257
Jens Axboe05962f92021-03-06 13:58:48 -07007258 set_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state);
Pavel Begunkov88885f62021-04-11 01:46:38 +01007259 mutex_lock(&sqd->lock);
Jens Axboee8f98f242021-03-09 16:32:13 -07007260 if (sqd->thread)
7261 wake_up_process(sqd->thread);
Pavel Begunkov09a6f4e2021-03-14 20:57:10 +00007262 mutex_unlock(&sqd->lock);
Jens Axboe05962f92021-03-06 13:58:48 -07007263 wait_for_completion(&sqd->exited);
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007264}
7265
Jens Axboe534ca6d2020-09-02 13:52:19 -06007266static void io_put_sq_data(struct io_sq_data *sqd)
Jens Axboe6c271ce2019-01-10 11:22:30 -07007267{
Jens Axboe534ca6d2020-09-02 13:52:19 -06007268 if (refcount_dec_and_test(&sqd->refs)) {
Pavel Begunkov9e138a42021-03-14 20:57:12 +00007269 WARN_ON_ONCE(atomic_read(&sqd->park_pending));
7270
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007271 io_sq_thread_stop(sqd);
7272 kfree(sqd);
7273 }
7274}
7275
7276static void io_sq_thread_finish(struct io_ring_ctx *ctx)
7277{
7278 struct io_sq_data *sqd = ctx->sq_data;
7279
7280 if (sqd) {
Jens Axboe05962f92021-03-06 13:58:48 -07007281 io_sq_thread_park(sqd);
Pavel Begunkov521d6a72021-03-11 23:29:38 +00007282 list_del_init(&ctx->sqd_list);
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007283 io_sqd_update_thread_idle(sqd);
Jens Axboe05962f92021-03-06 13:58:48 -07007284 io_sq_thread_unpark(sqd);
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007285
7286 io_put_sq_data(sqd);
7287 ctx->sq_data = NULL;
Jens Axboe534ca6d2020-09-02 13:52:19 -06007288 }
7289}
7290
Jens Axboeaa061652020-09-02 14:50:27 -06007291static struct io_sq_data *io_attach_sq_data(struct io_uring_params *p)
7292{
7293 struct io_ring_ctx *ctx_attach;
7294 struct io_sq_data *sqd;
7295 struct fd f;
7296
7297 f = fdget(p->wq_fd);
7298 if (!f.file)
7299 return ERR_PTR(-ENXIO);
7300 if (f.file->f_op != &io_uring_fops) {
7301 fdput(f);
7302 return ERR_PTR(-EINVAL);
7303 }
7304
7305 ctx_attach = f.file->private_data;
7306 sqd = ctx_attach->sq_data;
7307 if (!sqd) {
7308 fdput(f);
7309 return ERR_PTR(-EINVAL);
7310 }
Jens Axboe5c2469e2021-03-11 10:17:56 -07007311 if (sqd->task_tgid != current->tgid) {
7312 fdput(f);
7313 return ERR_PTR(-EPERM);
7314 }
Jens Axboeaa061652020-09-02 14:50:27 -06007315
7316 refcount_inc(&sqd->refs);
7317 fdput(f);
7318 return sqd;
7319}
7320
Pavel Begunkov26984fb2021-03-11 23:29:37 +00007321static struct io_sq_data *io_get_sq_data(struct io_uring_params *p,
7322 bool *attached)
Jens Axboe534ca6d2020-09-02 13:52:19 -06007323{
7324 struct io_sq_data *sqd;
7325
Pavel Begunkov26984fb2021-03-11 23:29:37 +00007326 *attached = false;
Jens Axboe5c2469e2021-03-11 10:17:56 -07007327 if (p->flags & IORING_SETUP_ATTACH_WQ) {
7328 sqd = io_attach_sq_data(p);
Pavel Begunkov26984fb2021-03-11 23:29:37 +00007329 if (!IS_ERR(sqd)) {
7330 *attached = true;
Jens Axboe5c2469e2021-03-11 10:17:56 -07007331 return sqd;
Pavel Begunkov26984fb2021-03-11 23:29:37 +00007332 }
Jens Axboe5c2469e2021-03-11 10:17:56 -07007333 /* fall through for EPERM case, setup new sqd/task */
7334 if (PTR_ERR(sqd) != -EPERM)
7335 return sqd;
7336 }
Jens Axboeaa061652020-09-02 14:50:27 -06007337
Jens Axboe534ca6d2020-09-02 13:52:19 -06007338 sqd = kzalloc(sizeof(*sqd), GFP_KERNEL);
7339 if (!sqd)
7340 return ERR_PTR(-ENOMEM);
7341
Pavel Begunkov9e138a42021-03-14 20:57:12 +00007342 atomic_set(&sqd->park_pending, 0);
Jens Axboe534ca6d2020-09-02 13:52:19 -06007343 refcount_set(&sqd->refs, 1);
Jens Axboe69fb2132020-09-14 11:16:23 -06007344 INIT_LIST_HEAD(&sqd->ctx_list);
Pavel Begunkov09a6f4e2021-03-14 20:57:10 +00007345 mutex_init(&sqd->lock);
Jens Axboe534ca6d2020-09-02 13:52:19 -06007346 init_waitqueue_head(&sqd->wait);
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007347 init_completion(&sqd->exited);
Jens Axboe534ca6d2020-09-02 13:52:19 -06007348 return sqd;
7349}
7350
Jens Axboe6b063142019-01-10 22:13:58 -07007351#if defined(CONFIG_UNIX)
Jens Axboe6b063142019-01-10 22:13:58 -07007352/*
7353 * Ensure the UNIX gc is aware of our file set, so we are certain that
7354 * the io_uring can be safely unregistered on process exit, even if we have
7355 * loops in the file referencing.
7356 */
7357static int __io_sqe_files_scm(struct io_ring_ctx *ctx, int nr, int offset)
7358{
7359 struct sock *sk = ctx->ring_sock->sk;
7360 struct scm_fp_list *fpl;
7361 struct sk_buff *skb;
Jens Axboe08a45172019-10-03 08:11:03 -06007362 int i, nr_files;
Jens Axboe6b063142019-01-10 22:13:58 -07007363
Jens Axboe6b063142019-01-10 22:13:58 -07007364 fpl = kzalloc(sizeof(*fpl), GFP_KERNEL);
7365 if (!fpl)
7366 return -ENOMEM;
7367
7368 skb = alloc_skb(0, GFP_KERNEL);
7369 if (!skb) {
7370 kfree(fpl);
7371 return -ENOMEM;
7372 }
7373
7374 skb->sk = sk;
Jens Axboe6b063142019-01-10 22:13:58 -07007375
Jens Axboe08a45172019-10-03 08:11:03 -06007376 nr_files = 0;
Jens Axboe62e398b2021-02-21 16:19:37 -07007377 fpl->user = get_uid(current_user());
Jens Axboe6b063142019-01-10 22:13:58 -07007378 for (i = 0; i < nr; i++) {
Jens Axboe65e19f52019-10-26 07:20:21 -06007379 struct file *file = io_file_from_index(ctx, i + offset);
7380
7381 if (!file)
Jens Axboe08a45172019-10-03 08:11:03 -06007382 continue;
Jens Axboe65e19f52019-10-26 07:20:21 -06007383 fpl->fp[nr_files] = get_file(file);
Jens Axboe08a45172019-10-03 08:11:03 -06007384 unix_inflight(fpl->user, fpl->fp[nr_files]);
7385 nr_files++;
Jens Axboe6b063142019-01-10 22:13:58 -07007386 }
7387
Jens Axboe08a45172019-10-03 08:11:03 -06007388 if (nr_files) {
7389 fpl->max = SCM_MAX_FD;
7390 fpl->count = nr_files;
7391 UNIXCB(skb).fp = fpl;
Jens Axboe05f3fb32019-12-09 11:22:50 -07007392 skb->destructor = unix_destruct_scm;
Jens Axboe08a45172019-10-03 08:11:03 -06007393 refcount_add(skb->truesize, &sk->sk_wmem_alloc);
7394 skb_queue_head(&sk->sk_receive_queue, skb);
Jens Axboe6b063142019-01-10 22:13:58 -07007395
Jens Axboe08a45172019-10-03 08:11:03 -06007396 for (i = 0; i < nr_files; i++)
7397 fput(fpl->fp[i]);
7398 } else {
7399 kfree_skb(skb);
7400 kfree(fpl);
7401 }
Jens Axboe6b063142019-01-10 22:13:58 -07007402
7403 return 0;
7404}
7405
7406/*
7407 * If UNIX sockets are enabled, fd passing can cause a reference cycle which
7408 * causes regular reference counting to break down. We rely on the UNIX
7409 * garbage collection to take care of this problem for us.
7410 */
7411static int io_sqe_files_scm(struct io_ring_ctx *ctx)
7412{
7413 unsigned left, total;
7414 int ret = 0;
7415
7416 total = 0;
7417 left = ctx->nr_user_files;
7418 while (left) {
7419 unsigned this_files = min_t(unsigned, left, SCM_MAX_FD);
7420
7421 ret = __io_sqe_files_scm(ctx, this_files, total);
7422 if (ret)
7423 break;
7424 left -= this_files;
7425 total += this_files;
7426 }
7427
7428 if (!ret)
7429 return 0;
7430
7431 while (total < ctx->nr_user_files) {
Jens Axboe65e19f52019-10-26 07:20:21 -06007432 struct file *file = io_file_from_index(ctx, total);
7433
7434 if (file)
7435 fput(file);
Jens Axboe6b063142019-01-10 22:13:58 -07007436 total++;
7437 }
7438
7439 return ret;
7440}
7441#else
7442static int io_sqe_files_scm(struct io_ring_ctx *ctx)
7443{
7444 return 0;
7445}
7446#endif
7447
Pavel Begunkovaeca2412021-04-11 01:46:37 +01007448static bool io_alloc_file_tables(struct io_file_table *table, unsigned nr_files)
Jens Axboe65e19f52019-10-26 07:20:21 -06007449{
Pavel Begunkov846a4ef2021-04-01 15:44:03 +01007450 unsigned i, nr_tables = DIV_ROUND_UP(nr_files, IORING_MAX_FILES_TABLE);
7451
Pavel Begunkovaeca2412021-04-11 01:46:37 +01007452 table->files = kcalloc(nr_tables, sizeof(*table->files), GFP_KERNEL);
7453 if (!table->files)
Pavel Begunkov846a4ef2021-04-01 15:44:03 +01007454 return false;
Jens Axboe65e19f52019-10-26 07:20:21 -06007455
7456 for (i = 0; i < nr_tables; i++) {
Pavel Begunkov846a4ef2021-04-01 15:44:03 +01007457 unsigned int this_files = min(nr_files, IORING_MAX_FILES_TABLE);
Jens Axboe65e19f52019-10-26 07:20:21 -06007458
Pavel Begunkovaeca2412021-04-11 01:46:37 +01007459 table->files[i] = kcalloc(this_files, sizeof(*table->files[i]),
Jens Axboe65e19f52019-10-26 07:20:21 -06007460 GFP_KERNEL);
Pavel Begunkovaeca2412021-04-11 01:46:37 +01007461 if (!table->files[i])
Jens Axboe65e19f52019-10-26 07:20:21 -06007462 break;
7463 nr_files -= this_files;
7464 }
7465
7466 if (i == nr_tables)
Pavel Begunkov846a4ef2021-04-01 15:44:03 +01007467 return true;
Jens Axboe65e19f52019-10-26 07:20:21 -06007468
Pavel Begunkovaeca2412021-04-11 01:46:37 +01007469 io_free_file_tables(table, nr_tables * IORING_MAX_FILES_TABLE);
Pavel Begunkov846a4ef2021-04-01 15:44:03 +01007470 return false;
Jens Axboe65e19f52019-10-26 07:20:21 -06007471}
7472
Pavel Begunkov47e90392021-04-01 15:43:56 +01007473static void io_rsrc_file_put(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc)
Jens Axboec3a31e62019-10-03 13:59:56 -06007474{
Bijan Mottahedeh50238532021-01-15 17:37:45 +00007475 struct file *file = prsrc->file;
Jens Axboec3a31e62019-10-03 13:59:56 -06007476#if defined(CONFIG_UNIX)
Jens Axboec3a31e62019-10-03 13:59:56 -06007477 struct sock *sock = ctx->ring_sock->sk;
7478 struct sk_buff_head list, *head = &sock->sk_receive_queue;
7479 struct sk_buff *skb;
7480 int i;
7481
7482 __skb_queue_head_init(&list);
7483
7484 /*
7485 * Find the skb that holds this file in its SCM_RIGHTS. When found,
7486 * remove this entry and rearrange the file array.
7487 */
7488 skb = skb_dequeue(head);
7489 while (skb) {
7490 struct scm_fp_list *fp;
7491
7492 fp = UNIXCB(skb).fp;
7493 for (i = 0; i < fp->count; i++) {
7494 int left;
7495
7496 if (fp->fp[i] != file)
7497 continue;
7498
7499 unix_notinflight(fp->user, fp->fp[i]);
7500 left = fp->count - 1 - i;
7501 if (left) {
7502 memmove(&fp->fp[i], &fp->fp[i + 1],
7503 left * sizeof(struct file *));
7504 }
7505 fp->count--;
7506 if (!fp->count) {
7507 kfree_skb(skb);
7508 skb = NULL;
7509 } else {
7510 __skb_queue_tail(&list, skb);
7511 }
7512 fput(file);
7513 file = NULL;
7514 break;
7515 }
7516
7517 if (!file)
7518 break;
7519
7520 __skb_queue_tail(&list, skb);
7521
7522 skb = skb_dequeue(head);
7523 }
7524
7525 if (skb_peek(&list)) {
7526 spin_lock_irq(&head->lock);
7527 while ((skb = __skb_dequeue(&list)) != NULL)
7528 __skb_queue_tail(head, skb);
7529 spin_unlock_irq(&head->lock);
7530 }
7531#else
Jens Axboe05f3fb32019-12-09 11:22:50 -07007532 fput(file);
Jens Axboec3a31e62019-10-03 13:59:56 -06007533#endif
7534}
7535
Pavel Begunkovb895c9a2021-04-01 15:43:40 +01007536static void __io_rsrc_put_work(struct io_rsrc_node *ref_node)
Jens Axboe05f3fb32019-12-09 11:22:50 -07007537{
Pavel Begunkovb895c9a2021-04-01 15:43:40 +01007538 struct io_rsrc_data *rsrc_data = ref_node->rsrc_data;
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00007539 struct io_ring_ctx *ctx = rsrc_data->ctx;
7540 struct io_rsrc_put *prsrc, *tmp;
Xiaoguang Wang05589552020-03-31 14:05:18 +08007541
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00007542 list_for_each_entry_safe(prsrc, tmp, &ref_node->rsrc_list, list) {
7543 list_del(&prsrc->list);
Pavel Begunkovb60c8dc2021-04-25 14:32:18 +01007544
7545 if (prsrc->tag) {
7546 bool lock_ring = ctx->flags & IORING_SETUP_IOPOLL;
7547 unsigned long flags;
7548
7549 io_ring_submit_lock(ctx, lock_ring);
7550 spin_lock_irqsave(&ctx->completion_lock, flags);
7551 io_cqring_fill_event(ctx, prsrc->tag, 0, 0);
Pavel Begunkov2840f712021-04-27 16:13:51 +01007552 ctx->cq_extra++;
Pavel Begunkovb60c8dc2021-04-25 14:32:18 +01007553 io_commit_cqring(ctx);
7554 spin_unlock_irqrestore(&ctx->completion_lock, flags);
7555 io_cqring_ev_posted(ctx);
7556 io_ring_submit_unlock(ctx, lock_ring);
7557 }
7558
Pavel Begunkov40ae0ff2021-04-01 15:43:44 +01007559 rsrc_data->do_put(ctx, prsrc);
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00007560 kfree(prsrc);
Jens Axboe05f3fb32019-12-09 11:22:50 -07007561 }
7562
Pavel Begunkov28a9fe22021-04-01 15:43:47 +01007563 io_rsrc_node_destroy(ref_node);
Pavel Begunkov3e942492021-04-11 01:46:34 +01007564 if (atomic_dec_and_test(&rsrc_data->refs))
7565 complete(&rsrc_data->done);
Jens Axboe05f3fb32019-12-09 11:22:50 -07007566}
7567
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00007568static void io_rsrc_put_work(struct work_struct *work)
Jens Axboe4a38aed22020-05-14 17:21:15 -06007569{
7570 struct io_ring_ctx *ctx;
7571 struct llist_node *node;
7572
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00007573 ctx = container_of(work, struct io_ring_ctx, rsrc_put_work.work);
7574 node = llist_del_all(&ctx->rsrc_put_llist);
Jens Axboe4a38aed22020-05-14 17:21:15 -06007575
7576 while (node) {
Pavel Begunkovb895c9a2021-04-01 15:43:40 +01007577 struct io_rsrc_node *ref_node;
Jens Axboe4a38aed22020-05-14 17:21:15 -06007578 struct llist_node *next = node->next;
7579
Pavel Begunkovb895c9a2021-04-01 15:43:40 +01007580 ref_node = llist_entry(node, struct io_rsrc_node, llist);
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00007581 __io_rsrc_put_work(ref_node);
Jens Axboe4a38aed22020-05-14 17:21:15 -06007582 node = next;
7583 }
7584}
7585
Bijan Mottahedeh00835dc2021-01-15 17:37:52 +00007586static void io_rsrc_node_ref_zero(struct percpu_ref *ref)
Jens Axboe05f3fb32019-12-09 11:22:50 -07007587{
Pavel Begunkovb895c9a2021-04-01 15:43:40 +01007588 struct io_rsrc_node *node = container_of(ref, struct io_rsrc_node, refs);
Pavel Begunkov3e942492021-04-11 01:46:34 +01007589 struct io_ring_ctx *ctx = node->rsrc_data->ctx;
Pavel Begunkove2978222020-11-18 14:56:26 +00007590 bool first_add = false;
Jens Axboe05f3fb32019-12-09 11:22:50 -07007591
Bijan Mottahedeh2a63b2d2021-01-15 17:37:47 +00007592 io_rsrc_ref_lock(ctx);
Pavel Begunkovb895c9a2021-04-01 15:43:40 +01007593 node->done = true;
Pavel Begunkove2978222020-11-18 14:56:26 +00007594
Bijan Mottahedehd67d2262021-01-15 17:37:46 +00007595 while (!list_empty(&ctx->rsrc_ref_list)) {
Pavel Begunkovb895c9a2021-04-01 15:43:40 +01007596 node = list_first_entry(&ctx->rsrc_ref_list,
7597 struct io_rsrc_node, node);
Pavel Begunkove2978222020-11-18 14:56:26 +00007598 /* recycle ref nodes in order */
Pavel Begunkovb895c9a2021-04-01 15:43:40 +01007599 if (!node->done)
Pavel Begunkove2978222020-11-18 14:56:26 +00007600 break;
Pavel Begunkovb895c9a2021-04-01 15:43:40 +01007601 list_del(&node->node);
7602 first_add |= llist_add(&node->llist, &ctx->rsrc_put_llist);
Pavel Begunkove2978222020-11-18 14:56:26 +00007603 }
Bijan Mottahedeh2a63b2d2021-01-15 17:37:47 +00007604 io_rsrc_ref_unlock(ctx);
Pavel Begunkove2978222020-11-18 14:56:26 +00007605
Pavel Begunkov3e942492021-04-11 01:46:34 +01007606 if (first_add)
7607 mod_delayed_work(system_wq, &ctx->rsrc_put_work, HZ);
Xiaoguang Wang05589552020-03-31 14:05:18 +08007608}
7609
Pavel Begunkovb895c9a2021-04-01 15:43:40 +01007610static struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx)
Xiaoguang Wang05589552020-03-31 14:05:18 +08007611{
Pavel Begunkovb895c9a2021-04-01 15:43:40 +01007612 struct io_rsrc_node *ref_node;
Xiaoguang Wang05589552020-03-31 14:05:18 +08007613
7614 ref_node = kzalloc(sizeof(*ref_node), GFP_KERNEL);
7615 if (!ref_node)
Matthew Wilcox (Oracle)3e2224c2021-01-06 16:09:26 +00007616 return NULL;
Xiaoguang Wang05589552020-03-31 14:05:18 +08007617
Bijan Mottahedeh00835dc2021-01-15 17:37:52 +00007618 if (percpu_ref_init(&ref_node->refs, io_rsrc_node_ref_zero,
Xiaoguang Wang05589552020-03-31 14:05:18 +08007619 0, GFP_KERNEL)) {
7620 kfree(ref_node);
Matthew Wilcox (Oracle)3e2224c2021-01-06 16:09:26 +00007621 return NULL;
Xiaoguang Wang05589552020-03-31 14:05:18 +08007622 }
7623 INIT_LIST_HEAD(&ref_node->node);
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00007624 INIT_LIST_HEAD(&ref_node->rsrc_list);
Pavel Begunkove2978222020-11-18 14:56:26 +00007625 ref_node->done = false;
Xiaoguang Wang05589552020-03-31 14:05:18 +08007626 return ref_node;
Xiaoguang Wang05589552020-03-31 14:05:18 +08007627}
7628
Jens Axboe05f3fb32019-12-09 11:22:50 -07007629static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg,
Pavel Begunkov792e3582021-04-25 14:32:21 +01007630 unsigned nr_args, u64 __user *tags)
Jens Axboe05f3fb32019-12-09 11:22:50 -07007631{
7632 __s32 __user *fds = (__s32 __user *) arg;
Jens Axboe05f3fb32019-12-09 11:22:50 -07007633 struct file *file;
Pavel Begunkovf3baed32021-04-01 15:43:42 +01007634 int fd, ret;
Pavel Begunkov846a4ef2021-04-01 15:44:03 +01007635 unsigned i;
Pavel Begunkovb895c9a2021-04-01 15:43:40 +01007636 struct io_rsrc_data *file_data;
Jens Axboe05f3fb32019-12-09 11:22:50 -07007637
7638 if (ctx->file_data)
7639 return -EBUSY;
7640 if (!nr_args)
7641 return -EINVAL;
7642 if (nr_args > IORING_MAX_FIXED_FILES)
7643 return -EMFILE;
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01007644 ret = io_rsrc_node_switch_start(ctx);
Pavel Begunkovf3baed32021-04-01 15:43:42 +01007645 if (ret)
7646 return ret;
Jens Axboe05f3fb32019-12-09 11:22:50 -07007647
Pavel Begunkovb60c8dc2021-04-25 14:32:18 +01007648 file_data = io_rsrc_data_alloc(ctx, io_rsrc_file_put, nr_args);
Pavel Begunkov5398ae62020-10-10 18:34:14 +01007649 if (!file_data)
Jens Axboe05f3fb32019-12-09 11:22:50 -07007650 return -ENOMEM;
Dan Carpenter13770a72021-02-01 15:23:42 +03007651 ctx->file_data = file_data;
Pavel Begunkovf3baed32021-04-01 15:43:42 +01007652 ret = -ENOMEM;
Pavel Begunkovaeca2412021-04-11 01:46:37 +01007653 if (!io_alloc_file_tables(&ctx->file_table, nr_args))
Pavel Begunkov600cf3f2020-10-10 18:34:15 +01007654 goto out_free;
Jens Axboe05f3fb32019-12-09 11:22:50 -07007655
Jens Axboe05f3fb32019-12-09 11:22:50 -07007656 for (i = 0; i < nr_args; i++, ctx->nr_user_files++) {
Pavel Begunkov792e3582021-04-25 14:32:21 +01007657 u64 tag = 0;
7658
7659 if ((tags && copy_from_user(&tag, &tags[i], sizeof(tag))) ||
7660 copy_from_user(&fd, &fds[i], sizeof(fd))) {
Pavel Begunkov600cf3f2020-10-10 18:34:15 +01007661 ret = -EFAULT;
7662 goto out_fput;
Jens Axboe05f3fb32019-12-09 11:22:50 -07007663 }
Pavel Begunkov600cf3f2020-10-10 18:34:15 +01007664 /* allow sparse sets */
Pavel Begunkov792e3582021-04-25 14:32:21 +01007665 if (fd == -1) {
7666 ret = -EINVAL;
7667 if (unlikely(tag))
7668 goto out_fput;
Pavel Begunkov600cf3f2020-10-10 18:34:15 +01007669 continue;
Pavel Begunkov792e3582021-04-25 14:32:21 +01007670 }
Jens Axboe05f3fb32019-12-09 11:22:50 -07007671
Jens Axboe05f3fb32019-12-09 11:22:50 -07007672 file = fget(fd);
Jens Axboe05f3fb32019-12-09 11:22:50 -07007673 ret = -EBADF;
Pavel Begunkov792e3582021-04-25 14:32:21 +01007674 if (unlikely(!file))
Pavel Begunkov600cf3f2020-10-10 18:34:15 +01007675 goto out_fput;
Jens Axboe05f3fb32019-12-09 11:22:50 -07007676
7677 /*
7678 * Don't allow io_uring instances to be registered. If UNIX
7679 * isn't enabled, then this causes a reference cycle and this
7680 * instance can never get freed. If UNIX is enabled we'll
7681 * handle it just fine, but there's still no point in allowing
7682 * a ring fd as it doesn't support regular read/write anyway.
7683 */
7684 if (file->f_op == &io_uring_fops) {
7685 fput(file);
Pavel Begunkov600cf3f2020-10-10 18:34:15 +01007686 goto out_fput;
Jens Axboe05f3fb32019-12-09 11:22:50 -07007687 }
Pavel Begunkov792e3582021-04-25 14:32:21 +01007688 ctx->file_data->tags[i] = tag;
Pavel Begunkovaeca2412021-04-11 01:46:37 +01007689 io_fixed_file_set(io_fixed_file_slot(&ctx->file_table, i), file);
Jens Axboe05f3fb32019-12-09 11:22:50 -07007690 }
7691
Jens Axboe05f3fb32019-12-09 11:22:50 -07007692 ret = io_sqe_files_scm(ctx);
Xiaoguang Wang05589552020-03-31 14:05:18 +08007693 if (ret) {
Pavel Begunkov08480402021-04-13 02:58:38 +01007694 __io_sqe_files_unregister(ctx);
Xiaoguang Wang05589552020-03-31 14:05:18 +08007695 return ret;
7696 }
Jens Axboe05f3fb32019-12-09 11:22:50 -07007697
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01007698 io_rsrc_node_switch(ctx, NULL);
Jens Axboe05f3fb32019-12-09 11:22:50 -07007699 return ret;
Pavel Begunkov600cf3f2020-10-10 18:34:15 +01007700out_fput:
7701 for (i = 0; i < ctx->nr_user_files; i++) {
7702 file = io_file_from_index(ctx, i);
7703 if (file)
7704 fput(file);
7705 }
Pavel Begunkovaeca2412021-04-11 01:46:37 +01007706 io_free_file_tables(&ctx->file_table, nr_args);
Pavel Begunkov600cf3f2020-10-10 18:34:15 +01007707 ctx->nr_user_files = 0;
Pavel Begunkov600cf3f2020-10-10 18:34:15 +01007708out_free:
Pavel Begunkov44b31f22021-04-25 14:32:16 +01007709 io_rsrc_data_free(ctx->file_data);
Jens Axboe55cbc252020-10-14 07:35:57 -06007710 ctx->file_data = NULL;
Jens Axboe05f3fb32019-12-09 11:22:50 -07007711 return ret;
7712}
7713
Jens Axboec3a31e62019-10-03 13:59:56 -06007714static int io_sqe_file_register(struct io_ring_ctx *ctx, struct file *file,
7715 int index)
7716{
7717#if defined(CONFIG_UNIX)
7718 struct sock *sock = ctx->ring_sock->sk;
7719 struct sk_buff_head *head = &sock->sk_receive_queue;
7720 struct sk_buff *skb;
7721
7722 /*
7723 * See if we can merge this file into an existing skb SCM_RIGHTS
7724 * file set. If there's no room, fall back to allocating a new skb
7725 * and filling it in.
7726 */
7727 spin_lock_irq(&head->lock);
7728 skb = skb_peek(head);
7729 if (skb) {
7730 struct scm_fp_list *fpl = UNIXCB(skb).fp;
7731
7732 if (fpl->count < SCM_MAX_FD) {
7733 __skb_unlink(skb, head);
7734 spin_unlock_irq(&head->lock);
7735 fpl->fp[fpl->count] = get_file(file);
7736 unix_inflight(fpl->user, fpl->fp[fpl->count]);
7737 fpl->count++;
7738 spin_lock_irq(&head->lock);
7739 __skb_queue_head(head, skb);
7740 } else {
7741 skb = NULL;
7742 }
7743 }
7744 spin_unlock_irq(&head->lock);
7745
7746 if (skb) {
7747 fput(file);
7748 return 0;
7749 }
7750
7751 return __io_sqe_files_scm(ctx, 1, index);
7752#else
7753 return 0;
7754#endif
7755}
7756
Pavel Begunkovb60c8dc2021-04-25 14:32:18 +01007757static int io_queue_rsrc_removal(struct io_rsrc_data *data, unsigned idx,
Pavel Begunkove7c78372021-04-01 15:43:45 +01007758 struct io_rsrc_node *node, void *rsrc)
Jens Axboe05f3fb32019-12-09 11:22:50 -07007759{
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00007760 struct io_rsrc_put *prsrc;
Jens Axboe05f3fb32019-12-09 11:22:50 -07007761
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00007762 prsrc = kzalloc(sizeof(*prsrc), GFP_KERNEL);
7763 if (!prsrc)
Hillf Dantona5318d32020-03-23 17:47:15 +08007764 return -ENOMEM;
Jens Axboe05f3fb32019-12-09 11:22:50 -07007765
Pavel Begunkovb60c8dc2021-04-25 14:32:18 +01007766 prsrc->tag = data->tags[idx];
Bijan Mottahedeh50238532021-01-15 17:37:45 +00007767 prsrc->rsrc = rsrc;
Pavel Begunkove7c78372021-04-01 15:43:45 +01007768 list_add(&prsrc->list, &node->rsrc_list);
Hillf Dantona5318d32020-03-23 17:47:15 +08007769 return 0;
Jens Axboe05f3fb32019-12-09 11:22:50 -07007770}
7771
7772static int __io_sqe_files_update(struct io_ring_ctx *ctx,
Pavel Begunkovc3bdad02021-04-25 14:32:22 +01007773 struct io_uring_rsrc_update2 *up,
Jens Axboe05f3fb32019-12-09 11:22:50 -07007774 unsigned nr_args)
7775{
Pavel Begunkovc3bdad02021-04-25 14:32:22 +01007776 u64 __user *tags = u64_to_user_ptr(up->tags);
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +01007777 __s32 __user *fds = u64_to_user_ptr(up->data);
Pavel Begunkovb895c9a2021-04-01 15:43:40 +01007778 struct io_rsrc_data *data = ctx->file_data;
Pavel Begunkova04b0ac2021-04-01 15:44:04 +01007779 struct io_fixed_file *file_slot;
7780 struct file *file;
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +01007781 int fd, i, err = 0;
7782 unsigned int done;
Xiaoguang Wang05589552020-03-31 14:05:18 +08007783 bool needs_switch = false;
Jens Axboec3a31e62019-10-03 13:59:56 -06007784
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +01007785 if (!ctx->file_data)
7786 return -ENXIO;
7787 if (up->offset + nr_args > ctx->nr_user_files)
Jens Axboec3a31e62019-10-03 13:59:56 -06007788 return -EINVAL;
7789
Pavel Begunkov67973b92021-01-26 13:51:09 +00007790 for (done = 0; done < nr_args; done++) {
Pavel Begunkovc3bdad02021-04-25 14:32:22 +01007791 u64 tag = 0;
7792
7793 if ((tags && copy_from_user(&tag, &tags[done], sizeof(tag))) ||
7794 copy_from_user(&fd, &fds[done], sizeof(fd))) {
Jens Axboec3a31e62019-10-03 13:59:56 -06007795 err = -EFAULT;
7796 break;
7797 }
Pavel Begunkovc3bdad02021-04-25 14:32:22 +01007798 if ((fd == IORING_REGISTER_FILES_SKIP || fd == -1) && tag) {
7799 err = -EINVAL;
7800 break;
7801 }
noah4e0377a2021-01-26 15:23:28 -05007802 if (fd == IORING_REGISTER_FILES_SKIP)
7803 continue;
7804
Pavel Begunkov67973b92021-01-26 13:51:09 +00007805 i = array_index_nospec(up->offset + done, ctx->nr_user_files);
Pavel Begunkovaeca2412021-04-11 01:46:37 +01007806 file_slot = io_fixed_file_slot(&ctx->file_table, i);
Pavel Begunkovea64ec022021-02-04 13:52:07 +00007807
Pavel Begunkova04b0ac2021-04-01 15:44:04 +01007808 if (file_slot->file_ptr) {
7809 file = (struct file *)(file_slot->file_ptr & FFS_MASK);
Pavel Begunkovb60c8dc2021-04-25 14:32:18 +01007810 err = io_queue_rsrc_removal(data, up->offset + done,
7811 ctx->rsrc_node, file);
Hillf Dantona5318d32020-03-23 17:47:15 +08007812 if (err)
7813 break;
Pavel Begunkova04b0ac2021-04-01 15:44:04 +01007814 file_slot->file_ptr = 0;
Xiaoguang Wang05589552020-03-31 14:05:18 +08007815 needs_switch = true;
Jens Axboec3a31e62019-10-03 13:59:56 -06007816 }
7817 if (fd != -1) {
Jens Axboec3a31e62019-10-03 13:59:56 -06007818 file = fget(fd);
7819 if (!file) {
7820 err = -EBADF;
7821 break;
7822 }
7823 /*
7824 * Don't allow io_uring instances to be registered. If
7825 * UNIX isn't enabled, then this causes a reference
7826 * cycle and this instance can never get freed. If UNIX
7827 * is enabled we'll handle it just fine, but there's
7828 * still no point in allowing a ring fd as it doesn't
7829 * support regular read/write anyway.
7830 */
7831 if (file->f_op == &io_uring_fops) {
7832 fput(file);
7833 err = -EBADF;
7834 break;
7835 }
Pavel Begunkovc3bdad02021-04-25 14:32:22 +01007836 data->tags[up->offset + done] = tag;
Pavel Begunkov9a321c92021-04-01 15:44:01 +01007837 io_fixed_file_set(file_slot, file);
Jens Axboec3a31e62019-10-03 13:59:56 -06007838 err = io_sqe_file_register(ctx, file, i);
Yang Yingliangf3bd9da2020-07-09 10:11:41 +00007839 if (err) {
Pavel Begunkova04b0ac2021-04-01 15:44:04 +01007840 file_slot->file_ptr = 0;
Yang Yingliangf3bd9da2020-07-09 10:11:41 +00007841 fput(file);
Jens Axboec3a31e62019-10-03 13:59:56 -06007842 break;
Yang Yingliangf3bd9da2020-07-09 10:11:41 +00007843 }
Jens Axboec3a31e62019-10-03 13:59:56 -06007844 }
Jens Axboe05f3fb32019-12-09 11:22:50 -07007845 }
7846
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01007847 if (needs_switch)
7848 io_rsrc_node_switch(ctx, data);
Jens Axboec3a31e62019-10-03 13:59:56 -06007849 return done ? done : err;
7850}
Xiaoguang Wang05589552020-03-31 14:05:18 +08007851
Pavel Begunkov5280f7e2021-02-04 13:52:08 +00007852static struct io_wq_work *io_free_work(struct io_wq_work *work)
Jens Axboe7d723062019-11-12 22:31:31 -07007853{
7854 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
7855
Pavel Begunkov5280f7e2021-02-04 13:52:08 +00007856 req = io_put_req_find_next(req);
7857 return req ? &req->work : NULL;
Jens Axboe7d723062019-11-12 22:31:31 -07007858}
7859
Jens Axboe685fe7f2021-03-08 09:37:51 -07007860static struct io_wq *io_init_wq_offload(struct io_ring_ctx *ctx,
7861 struct task_struct *task)
Pavel Begunkov24369c22020-01-28 03:15:48 +03007862{
Jens Axboee9418942021-02-19 12:33:30 -07007863 struct io_wq_hash *hash;
Pavel Begunkov24369c22020-01-28 03:15:48 +03007864 struct io_wq_data data;
Pavel Begunkov24369c22020-01-28 03:15:48 +03007865 unsigned int concurrency;
Pavel Begunkov24369c22020-01-28 03:15:48 +03007866
Jens Axboee9418942021-02-19 12:33:30 -07007867 hash = ctx->hash_map;
7868 if (!hash) {
7869 hash = kzalloc(sizeof(*hash), GFP_KERNEL);
7870 if (!hash)
7871 return ERR_PTR(-ENOMEM);
7872 refcount_set(&hash->refs, 1);
7873 init_waitqueue_head(&hash->wait);
7874 ctx->hash_map = hash;
7875 }
7876
7877 data.hash = hash;
Jens Axboe685fe7f2021-03-08 09:37:51 -07007878 data.task = task;
Pavel Begunkove9fd9392020-03-04 16:14:12 +03007879 data.free_work = io_free_work;
Pavel Begunkovf5fa38c2020-06-08 21:08:20 +03007880 data.do_work = io_wq_submit_work;
Pavel Begunkov24369c22020-01-28 03:15:48 +03007881
Jens Axboed25e3a32021-02-16 11:41:41 -07007882 /* Do QD, or 4 * CPUS, whatever is smallest */
7883 concurrency = min(ctx->sq_entries, 4 * num_online_cpus());
Pavel Begunkov24369c22020-01-28 03:15:48 +03007884
Jens Axboe5aa75ed2021-02-16 12:56:50 -07007885 return io_wq_create(concurrency, &data);
Pavel Begunkov24369c22020-01-28 03:15:48 +03007886}
7887
Jens Axboe5aa75ed2021-02-16 12:56:50 -07007888static int io_uring_alloc_task_context(struct task_struct *task,
7889 struct io_ring_ctx *ctx)
Jens Axboe0f212202020-09-13 13:09:39 -06007890{
7891 struct io_uring_task *tctx;
Jens Axboed8a6df12020-10-15 16:24:45 -06007892 int ret;
Jens Axboe0f212202020-09-13 13:09:39 -06007893
7894 tctx = kmalloc(sizeof(*tctx), GFP_KERNEL);
7895 if (unlikely(!tctx))
7896 return -ENOMEM;
7897
Jens Axboed8a6df12020-10-15 16:24:45 -06007898 ret = percpu_counter_init(&tctx->inflight, 0, GFP_KERNEL);
7899 if (unlikely(ret)) {
7900 kfree(tctx);
7901 return ret;
7902 }
7903
Jens Axboe685fe7f2021-03-08 09:37:51 -07007904 tctx->io_wq = io_init_wq_offload(ctx, task);
Jens Axboe5aa75ed2021-02-16 12:56:50 -07007905 if (IS_ERR(tctx->io_wq)) {
7906 ret = PTR_ERR(tctx->io_wq);
7907 percpu_counter_destroy(&tctx->inflight);
7908 kfree(tctx);
7909 return ret;
7910 }
7911
Jens Axboe0f212202020-09-13 13:09:39 -06007912 xa_init(&tctx->xa);
7913 init_waitqueue_head(&tctx->wait);
7914 tctx->last = NULL;
Jens Axboefdaf0832020-10-30 09:37:30 -06007915 atomic_set(&tctx->in_idle, 0);
Pavel Begunkovb303fe22021-04-11 01:46:26 +01007916 atomic_set(&tctx->inflight_tracked, 0);
Jens Axboe0f212202020-09-13 13:09:39 -06007917 task->io_uring = tctx;
Jens Axboe7cbf1722021-02-10 00:03:20 +00007918 spin_lock_init(&tctx->task_lock);
7919 INIT_WQ_LIST(&tctx->task_list);
7920 tctx->task_state = 0;
7921 init_task_work(&tctx->task_work, tctx_task_work);
Jens Axboe0f212202020-09-13 13:09:39 -06007922 return 0;
7923}
7924
7925void __io_uring_free(struct task_struct *tsk)
7926{
7927 struct io_uring_task *tctx = tsk->io_uring;
7928
7929 WARN_ON_ONCE(!xa_empty(&tctx->xa));
Pavel Begunkovef8eaa42021-02-27 11:16:45 +00007930 WARN_ON_ONCE(tctx->io_wq);
7931
Jens Axboed8a6df12020-10-15 16:24:45 -06007932 percpu_counter_destroy(&tctx->inflight);
Jens Axboe0f212202020-09-13 13:09:39 -06007933 kfree(tctx);
7934 tsk->io_uring = NULL;
7935}
7936
Stefano Garzarella7e84e1c2020-08-27 16:58:31 +02007937static int io_sq_offload_create(struct io_ring_ctx *ctx,
7938 struct io_uring_params *p)
Jens Axboe6b063142019-01-10 22:13:58 -07007939{
7940 int ret;
7941
Jens Axboed25e3a32021-02-16 11:41:41 -07007942 /* Retain compatibility with failing for an invalid attach attempt */
7943 if ((ctx->flags & (IORING_SETUP_ATTACH_WQ | IORING_SETUP_SQPOLL)) ==
7944 IORING_SETUP_ATTACH_WQ) {
7945 struct fd f;
7946
7947 f = fdget(p->wq_fd);
7948 if (!f.file)
7949 return -ENXIO;
Jens Axboed25e3a32021-02-16 11:41:41 -07007950 fdput(f);
Pavel Begunkovf2a48dd2021-04-20 12:03:33 +01007951 if (f.file->f_op != &io_uring_fops)
7952 return -EINVAL;
Jens Axboed25e3a32021-02-16 11:41:41 -07007953 }
Jens Axboe6b063142019-01-10 22:13:58 -07007954 if (ctx->flags & IORING_SETUP_SQPOLL) {
Jens Axboe46fe18b2021-03-04 12:39:36 -07007955 struct task_struct *tsk;
Jens Axboe534ca6d2020-09-02 13:52:19 -06007956 struct io_sq_data *sqd;
Pavel Begunkov26984fb2021-03-11 23:29:37 +00007957 bool attached;
Jens Axboe534ca6d2020-09-02 13:52:19 -06007958
Pavel Begunkov26984fb2021-03-11 23:29:37 +00007959 sqd = io_get_sq_data(p, &attached);
Jens Axboe534ca6d2020-09-02 13:52:19 -06007960 if (IS_ERR(sqd)) {
7961 ret = PTR_ERR(sqd);
7962 goto err;
7963 }
Jens Axboe69fb2132020-09-14 11:16:23 -06007964
Stefan Metzmacher7c30f36a2021-03-07 11:54:28 +01007965 ctx->sq_creds = get_current_cred();
Jens Axboe534ca6d2020-09-02 13:52:19 -06007966 ctx->sq_data = sqd;
Jens Axboe6b063142019-01-10 22:13:58 -07007967 ctx->sq_thread_idle = msecs_to_jiffies(p->sq_thread_idle);
7968 if (!ctx->sq_thread_idle)
7969 ctx->sq_thread_idle = HZ;
7970
Pavel Begunkov78d7f6b2021-03-10 13:13:53 +00007971 io_sq_thread_park(sqd);
Pavel Begunkovde75a3d2021-03-18 11:54:35 +00007972 list_add(&ctx->sqd_list, &sqd->ctx_list);
7973 io_sqd_update_thread_idle(sqd);
Pavel Begunkov26984fb2021-03-11 23:29:37 +00007974 /* don't attach to a dying SQPOLL thread, would be racy */
Pavel Begunkovf2a48dd2021-04-20 12:03:33 +01007975 ret = (attached && !sqd->thread) ? -ENXIO : 0;
Pavel Begunkov78d7f6b2021-03-10 13:13:53 +00007976 io_sq_thread_unpark(sqd);
7977
Pavel Begunkovde75a3d2021-03-18 11:54:35 +00007978 if (ret < 0)
7979 goto err;
7980 if (attached)
Jens Axboe5aa75ed2021-02-16 12:56:50 -07007981 return 0;
Jens Axboeaa061652020-09-02 14:50:27 -06007982
Jens Axboe6b063142019-01-10 22:13:58 -07007983 if (p->flags & IORING_SETUP_SQ_AFF) {
7984 int cpu = p->sq_thread_cpu;
7985
7986 ret = -EINVAL;
Pavel Begunkovf2a48dd2021-04-20 12:03:33 +01007987 if (cpu >= nr_cpu_ids || !cpu_online(cpu))
Jens Axboee8f98f242021-03-09 16:32:13 -07007988 goto err_sqpoll;
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007989 sqd->sq_cpu = cpu;
Jens Axboe6c271ce2019-01-10 11:22:30 -07007990 } else {
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007991 sqd->sq_cpu = -1;
Jens Axboe6c271ce2019-01-10 11:22:30 -07007992 }
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007993
7994 sqd->task_pid = current->pid;
Jens Axboe5c2469e2021-03-11 10:17:56 -07007995 sqd->task_tgid = current->tgid;
Jens Axboe46fe18b2021-03-04 12:39:36 -07007996 tsk = create_io_thread(io_sq_thread, sqd, NUMA_NO_NODE);
7997 if (IS_ERR(tsk)) {
7998 ret = PTR_ERR(tsk);
Jens Axboee8f98f242021-03-09 16:32:13 -07007999 goto err_sqpoll;
Jens Axboe6c271ce2019-01-10 11:22:30 -07008000 }
Pavel Begunkov97a73a02021-03-08 17:30:54 +00008001
Jens Axboe46fe18b2021-03-04 12:39:36 -07008002 sqd->thread = tsk;
Pavel Begunkov97a73a02021-03-08 17:30:54 +00008003 ret = io_uring_alloc_task_context(tsk, ctx);
Jens Axboe46fe18b2021-03-04 12:39:36 -07008004 wake_up_new_task(tsk);
Jens Axboe0f212202020-09-13 13:09:39 -06008005 if (ret)
8006 goto err;
Jens Axboe6c271ce2019-01-10 11:22:30 -07008007 } else if (p->flags & IORING_SETUP_SQ_AFF) {
8008 /* Can't have SQ_AFF without SQPOLL */
8009 ret = -EINVAL;
8010 goto err;
8011 }
8012
Jens Axboe2b188cc2019-01-07 10:46:33 -07008013 return 0;
Pavel Begunkovf2a48dd2021-04-20 12:03:33 +01008014err_sqpoll:
8015 complete(&ctx->sq_data->exited);
Jens Axboe2b188cc2019-01-07 10:46:33 -07008016err:
Jens Axboe37d1e2e2021-02-17 21:03:43 -07008017 io_sq_thread_finish(ctx);
Jens Axboe2b188cc2019-01-07 10:46:33 -07008018 return ret;
8019}
8020
Bijan Mottahedeha087e2b2020-06-16 16:36:07 -07008021static inline void __io_unaccount_mem(struct user_struct *user,
8022 unsigned long nr_pages)
Jens Axboe2b188cc2019-01-07 10:46:33 -07008023{
8024 atomic_long_sub(nr_pages, &user->locked_vm);
8025}
8026
Bijan Mottahedeha087e2b2020-06-16 16:36:07 -07008027static inline int __io_account_mem(struct user_struct *user,
8028 unsigned long nr_pages)
Jens Axboe2b188cc2019-01-07 10:46:33 -07008029{
8030 unsigned long page_limit, cur_pages, new_pages;
8031
8032 /* Don't allow more pages than we can safely lock */
8033 page_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
8034
8035 do {
8036 cur_pages = atomic_long_read(&user->locked_vm);
8037 new_pages = cur_pages + nr_pages;
8038 if (new_pages > page_limit)
8039 return -ENOMEM;
8040 } while (atomic_long_cmpxchg(&user->locked_vm, cur_pages,
8041 new_pages) != cur_pages);
8042
8043 return 0;
8044}
8045
Jens Axboe26bfa89e2021-02-09 20:14:12 -07008046static void io_unaccount_mem(struct io_ring_ctx *ctx, unsigned long nr_pages)
Bijan Mottahedeha087e2b2020-06-16 16:36:07 -07008047{
Jens Axboe62e398b2021-02-21 16:19:37 -07008048 if (ctx->user)
Bijan Mottahedeha087e2b2020-06-16 16:36:07 -07008049 __io_unaccount_mem(ctx->user, nr_pages);
Bijan Mottahedeh30975822020-06-16 16:36:09 -07008050
Jens Axboe26bfa89e2021-02-09 20:14:12 -07008051 if (ctx->mm_account)
8052 atomic64_sub(nr_pages, &ctx->mm_account->pinned_vm);
Bijan Mottahedeha087e2b2020-06-16 16:36:07 -07008053}
8054
Jens Axboe26bfa89e2021-02-09 20:14:12 -07008055static int io_account_mem(struct io_ring_ctx *ctx, unsigned long nr_pages)
Bijan Mottahedeha087e2b2020-06-16 16:36:07 -07008056{
Bijan Mottahedeh30975822020-06-16 16:36:09 -07008057 int ret;
8058
Jens Axboe62e398b2021-02-21 16:19:37 -07008059 if (ctx->user) {
Bijan Mottahedeh30975822020-06-16 16:36:09 -07008060 ret = __io_account_mem(ctx->user, nr_pages);
8061 if (ret)
8062 return ret;
8063 }
8064
Jens Axboe26bfa89e2021-02-09 20:14:12 -07008065 if (ctx->mm_account)
8066 atomic64_add(nr_pages, &ctx->mm_account->pinned_vm);
Bijan Mottahedeha087e2b2020-06-16 16:36:07 -07008067
8068 return 0;
8069}
8070
Jens Axboe2b188cc2019-01-07 10:46:33 -07008071static void io_mem_free(void *ptr)
8072{
Mark Rutland52e04ef2019-04-30 17:30:21 +01008073 struct page *page;
Jens Axboe2b188cc2019-01-07 10:46:33 -07008074
Mark Rutland52e04ef2019-04-30 17:30:21 +01008075 if (!ptr)
8076 return;
8077
8078 page = virt_to_head_page(ptr);
Jens Axboe2b188cc2019-01-07 10:46:33 -07008079 if (put_page_testzero(page))
8080 free_compound_page(page);
8081}
8082
8083static void *io_mem_alloc(size_t size)
8084{
8085 gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN | __GFP_COMP |
Jens Axboe26bfa89e2021-02-09 20:14:12 -07008086 __GFP_NORETRY | __GFP_ACCOUNT;
Jens Axboe2b188cc2019-01-07 10:46:33 -07008087
8088 return (void *) __get_free_pages(gfp_flags, get_order(size));
8089}
8090
Hristo Venev75b28af2019-08-26 17:23:46 +00008091static unsigned long rings_size(unsigned sq_entries, unsigned cq_entries,
8092 size_t *sq_offset)
8093{
8094 struct io_rings *rings;
8095 size_t off, sq_array_size;
8096
8097 off = struct_size(rings, cqes, cq_entries);
8098 if (off == SIZE_MAX)
8099 return SIZE_MAX;
8100
8101#ifdef CONFIG_SMP
8102 off = ALIGN(off, SMP_CACHE_BYTES);
8103 if (off == 0)
8104 return SIZE_MAX;
8105#endif
8106
Dmitry Vyukovb36200f2020-07-11 11:31:11 +02008107 if (sq_offset)
8108 *sq_offset = off;
8109
Hristo Venev75b28af2019-08-26 17:23:46 +00008110 sq_array_size = array_size(sizeof(u32), sq_entries);
8111 if (sq_array_size == SIZE_MAX)
8112 return SIZE_MAX;
8113
8114 if (check_add_overflow(off, sq_array_size, &off))
8115 return SIZE_MAX;
8116
Hristo Venev75b28af2019-08-26 17:23:46 +00008117 return off;
8118}
8119
Pavel Begunkov41edf1a2021-04-25 14:32:23 +01008120static void io_buffer_unmap(struct io_ring_ctx *ctx, struct io_mapped_ubuf **slot)
Pavel Begunkov7f61a1e2021-04-11 01:46:35 +01008121{
Pavel Begunkov41edf1a2021-04-25 14:32:23 +01008122 struct io_mapped_ubuf *imu = *slot;
Pavel Begunkov7f61a1e2021-04-11 01:46:35 +01008123 unsigned int i;
8124
Pavel Begunkov62248432021-04-28 13:11:29 +01008125 if (imu != ctx->dummy_ubuf) {
8126 for (i = 0; i < imu->nr_bvecs; i++)
8127 unpin_user_page(imu->bvec[i].bv_page);
8128 if (imu->acct_pages)
8129 io_unaccount_mem(ctx, imu->acct_pages);
8130 kvfree(imu);
8131 }
Pavel Begunkov41edf1a2021-04-25 14:32:23 +01008132 *slot = NULL;
Pavel Begunkov7f61a1e2021-04-11 01:46:35 +01008133}
8134
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01008135static void io_rsrc_buf_put(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc)
8136{
Pavel Begunkov634d00d2021-04-25 14:32:26 +01008137 io_buffer_unmap(ctx, &prsrc->buf);
8138 prsrc->buf = NULL;
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01008139}
8140
8141static void __io_sqe_buffers_unregister(struct io_ring_ctx *ctx)
Jens Axboeedafcce2019-01-09 09:16:05 -07008142{
Pavel Begunkov7f61a1e2021-04-11 01:46:35 +01008143 unsigned int i;
Jens Axboeedafcce2019-01-09 09:16:05 -07008144
Pavel Begunkov7f61a1e2021-04-11 01:46:35 +01008145 for (i = 0; i < ctx->nr_user_bufs; i++)
8146 io_buffer_unmap(ctx, &ctx->user_bufs[i]);
Jens Axboeedafcce2019-01-09 09:16:05 -07008147 kfree(ctx->user_bufs);
Zqiangbb6659c2021-04-30 16:25:15 +08008148 io_rsrc_data_free(ctx->buf_data);
Jens Axboeedafcce2019-01-09 09:16:05 -07008149 ctx->user_bufs = NULL;
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01008150 ctx->buf_data = NULL;
Jens Axboeedafcce2019-01-09 09:16:05 -07008151 ctx->nr_user_bufs = 0;
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01008152}
8153
Jens Axboeedafcce2019-01-09 09:16:05 -07008154static int io_sqe_buffers_unregister(struct io_ring_ctx *ctx)
8155{
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01008156 int ret;
Jens Axboeedafcce2019-01-09 09:16:05 -07008157
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01008158 if (!ctx->buf_data)
Jens Axboeedafcce2019-01-09 09:16:05 -07008159 return -ENXIO;
8160
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01008161 ret = io_rsrc_ref_quiesce(ctx->buf_data, ctx);
8162 if (!ret)
8163 __io_sqe_buffers_unregister(ctx);
8164 return ret;
Jens Axboeedafcce2019-01-09 09:16:05 -07008165}
8166
8167static int io_copy_iov(struct io_ring_ctx *ctx, struct iovec *dst,
8168 void __user *arg, unsigned index)
8169{
8170 struct iovec __user *src;
8171
8172#ifdef CONFIG_COMPAT
8173 if (ctx->compat) {
8174 struct compat_iovec __user *ciovs;
8175 struct compat_iovec ciov;
8176
8177 ciovs = (struct compat_iovec __user *) arg;
8178 if (copy_from_user(&ciov, &ciovs[index], sizeof(ciov)))
8179 return -EFAULT;
8180
Jens Axboed55e5f52019-12-11 16:12:15 -07008181 dst->iov_base = u64_to_user_ptr((u64)ciov.iov_base);
Jens Axboeedafcce2019-01-09 09:16:05 -07008182 dst->iov_len = ciov.iov_len;
8183 return 0;
8184 }
8185#endif
8186 src = (struct iovec __user *) arg;
8187 if (copy_from_user(dst, &src[index], sizeof(*dst)))
8188 return -EFAULT;
8189 return 0;
8190}
8191
Jens Axboede293932020-09-17 16:19:16 -06008192/*
8193 * Not super efficient, but this is just a registration time. And we do cache
8194 * the last compound head, so generally we'll only do a full search if we don't
8195 * match that one.
8196 *
8197 * We check if the given compound head page has already been accounted, to
8198 * avoid double accounting it. This allows us to account the full size of the
8199 * page, not just the constituent pages of a huge page.
8200 */
8201static bool headpage_already_acct(struct io_ring_ctx *ctx, struct page **pages,
8202 int nr_pages, struct page *hpage)
8203{
8204 int i, j;
8205
8206 /* check current page array */
8207 for (i = 0; i < nr_pages; i++) {
8208 if (!PageCompound(pages[i]))
8209 continue;
8210 if (compound_head(pages[i]) == hpage)
8211 return true;
8212 }
8213
8214 /* check previously registered pages */
8215 for (i = 0; i < ctx->nr_user_bufs; i++) {
Pavel Begunkov41edf1a2021-04-25 14:32:23 +01008216 struct io_mapped_ubuf *imu = ctx->user_bufs[i];
Jens Axboede293932020-09-17 16:19:16 -06008217
8218 for (j = 0; j < imu->nr_bvecs; j++) {
8219 if (!PageCompound(imu->bvec[j].bv_page))
8220 continue;
8221 if (compound_head(imu->bvec[j].bv_page) == hpage)
8222 return true;
8223 }
8224 }
8225
8226 return false;
8227}
8228
8229static int io_buffer_account_pin(struct io_ring_ctx *ctx, struct page **pages,
8230 int nr_pages, struct io_mapped_ubuf *imu,
8231 struct page **last_hpage)
8232{
8233 int i, ret;
8234
Pavel Begunkov216e5832021-05-29 12:01:02 +01008235 imu->acct_pages = 0;
Jens Axboede293932020-09-17 16:19:16 -06008236 for (i = 0; i < nr_pages; i++) {
8237 if (!PageCompound(pages[i])) {
8238 imu->acct_pages++;
8239 } else {
8240 struct page *hpage;
8241
8242 hpage = compound_head(pages[i]);
8243 if (hpage == *last_hpage)
8244 continue;
8245 *last_hpage = hpage;
8246 if (headpage_already_acct(ctx, pages, i, hpage))
8247 continue;
8248 imu->acct_pages += page_size(hpage) >> PAGE_SHIFT;
8249 }
8250 }
8251
8252 if (!imu->acct_pages)
8253 return 0;
8254
Jens Axboe26bfa89e2021-02-09 20:14:12 -07008255 ret = io_account_mem(ctx, imu->acct_pages);
Jens Axboede293932020-09-17 16:19:16 -06008256 if (ret)
8257 imu->acct_pages = 0;
8258 return ret;
8259}
8260
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08008261static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov,
Pavel Begunkov41edf1a2021-04-25 14:32:23 +01008262 struct io_mapped_ubuf **pimu,
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08008263 struct page **last_hpage)
Jens Axboeedafcce2019-01-09 09:16:05 -07008264{
Pavel Begunkov41edf1a2021-04-25 14:32:23 +01008265 struct io_mapped_ubuf *imu = NULL;
Jens Axboeedafcce2019-01-09 09:16:05 -07008266 struct vm_area_struct **vmas = NULL;
8267 struct page **pages = NULL;
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08008268 unsigned long off, start, end, ubuf;
8269 size_t size;
8270 int ret, pret, nr_pages, i;
Jens Axboeedafcce2019-01-09 09:16:05 -07008271
Pavel Begunkov62248432021-04-28 13:11:29 +01008272 if (!iov->iov_base) {
8273 *pimu = ctx->dummy_ubuf;
8274 return 0;
8275 }
8276
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08008277 ubuf = (unsigned long) iov->iov_base;
8278 end = (ubuf + iov->iov_len + PAGE_SIZE - 1) >> PAGE_SHIFT;
8279 start = ubuf >> PAGE_SHIFT;
8280 nr_pages = end - start;
8281
Pavel Begunkov41edf1a2021-04-25 14:32:23 +01008282 *pimu = NULL;
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08008283 ret = -ENOMEM;
8284
8285 pages = kvmalloc_array(nr_pages, sizeof(struct page *), GFP_KERNEL);
8286 if (!pages)
8287 goto done;
8288
8289 vmas = kvmalloc_array(nr_pages, sizeof(struct vm_area_struct *),
8290 GFP_KERNEL);
8291 if (!vmas)
8292 goto done;
8293
Pavel Begunkov41edf1a2021-04-25 14:32:23 +01008294 imu = kvmalloc(struct_size(imu, bvec, nr_pages), GFP_KERNEL);
Pavel Begunkova2b41982021-04-26 00:16:31 +01008295 if (!imu)
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08008296 goto done;
8297
8298 ret = 0;
8299 mmap_read_lock(current->mm);
8300 pret = pin_user_pages(ubuf, nr_pages, FOLL_WRITE | FOLL_LONGTERM,
8301 pages, vmas);
8302 if (pret == nr_pages) {
8303 /* don't support file backed memory */
8304 for (i = 0; i < nr_pages; i++) {
8305 struct vm_area_struct *vma = vmas[i];
8306
8307 if (vma->vm_file &&
8308 !is_file_hugepages(vma->vm_file)) {
8309 ret = -EOPNOTSUPP;
8310 break;
8311 }
8312 }
8313 } else {
8314 ret = pret < 0 ? pret : -EFAULT;
8315 }
8316 mmap_read_unlock(current->mm);
8317 if (ret) {
8318 /*
8319 * if we did partial map, or found file backed vmas,
8320 * release any pages we did get
8321 */
8322 if (pret > 0)
8323 unpin_user_pages(pages, pret);
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08008324 goto done;
8325 }
8326
8327 ret = io_buffer_account_pin(ctx, pages, pret, imu, last_hpage);
8328 if (ret) {
8329 unpin_user_pages(pages, pret);
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08008330 goto done;
8331 }
8332
8333 off = ubuf & ~PAGE_MASK;
8334 size = iov->iov_len;
8335 for (i = 0; i < nr_pages; i++) {
8336 size_t vec_len;
8337
8338 vec_len = min_t(size_t, size, PAGE_SIZE - off);
8339 imu->bvec[i].bv_page = pages[i];
8340 imu->bvec[i].bv_len = vec_len;
8341 imu->bvec[i].bv_offset = off;
8342 off = 0;
8343 size -= vec_len;
8344 }
8345 /* store original address for later verification */
8346 imu->ubuf = ubuf;
Pavel Begunkov4751f532021-04-01 15:43:55 +01008347 imu->ubuf_end = ubuf + iov->iov_len;
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08008348 imu->nr_bvecs = nr_pages;
Pavel Begunkov41edf1a2021-04-25 14:32:23 +01008349 *pimu = imu;
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08008350 ret = 0;
8351done:
Pavel Begunkov41edf1a2021-04-25 14:32:23 +01008352 if (ret)
8353 kvfree(imu);
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08008354 kvfree(pages);
8355 kvfree(vmas);
8356 return ret;
8357}
8358
Bijan Mottahedeh2b358602021-01-06 12:39:11 -08008359static int io_buffers_map_alloc(struct io_ring_ctx *ctx, unsigned int nr_args)
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08008360{
Pavel Begunkov87094462021-04-11 01:46:36 +01008361 ctx->user_bufs = kcalloc(nr_args, sizeof(*ctx->user_bufs), GFP_KERNEL);
8362 return ctx->user_bufs ? 0 : -ENOMEM;
Bijan Mottahedeh2b358602021-01-06 12:39:11 -08008363}
8364
8365static int io_buffer_validate(struct iovec *iov)
8366{
Pavel Begunkov50e96982021-03-24 22:59:01 +00008367 unsigned long tmp, acct_len = iov->iov_len + (PAGE_SIZE - 1);
8368
Bijan Mottahedeh2b358602021-01-06 12:39:11 -08008369 /*
8370 * Don't impose further limits on the size and buffer
8371 * constraints here, we'll -EINVAL later when IO is
8372 * submitted if they are wrong.
8373 */
Pavel Begunkov62248432021-04-28 13:11:29 +01008374 if (!iov->iov_base)
8375 return iov->iov_len ? -EFAULT : 0;
8376 if (!iov->iov_len)
Bijan Mottahedeh2b358602021-01-06 12:39:11 -08008377 return -EFAULT;
8378
8379 /* arbitrary limit, but we need something */
8380 if (iov->iov_len > SZ_1G)
8381 return -EFAULT;
8382
Pavel Begunkov50e96982021-03-24 22:59:01 +00008383 if (check_add_overflow((unsigned long)iov->iov_base, acct_len, &tmp))
8384 return -EOVERFLOW;
8385
Bijan Mottahedeh2b358602021-01-06 12:39:11 -08008386 return 0;
8387}
8388
8389static int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg,
Pavel Begunkov634d00d2021-04-25 14:32:26 +01008390 unsigned int nr_args, u64 __user *tags)
Bijan Mottahedeh2b358602021-01-06 12:39:11 -08008391{
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01008392 struct page *last_hpage = NULL;
8393 struct io_rsrc_data *data;
Bijan Mottahedeh2b358602021-01-06 12:39:11 -08008394 int i, ret;
8395 struct iovec iov;
Bijan Mottahedeh2b358602021-01-06 12:39:11 -08008396
Pavel Begunkov87094462021-04-11 01:46:36 +01008397 if (ctx->user_bufs)
8398 return -EBUSY;
Pavel Begunkov489809e2021-05-14 12:06:44 +01008399 if (!nr_args || nr_args > IORING_MAX_REG_BUFFERS)
Pavel Begunkov87094462021-04-11 01:46:36 +01008400 return -EINVAL;
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01008401 ret = io_rsrc_node_switch_start(ctx);
Bijan Mottahedeh2b358602021-01-06 12:39:11 -08008402 if (ret)
8403 return ret;
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01008404 data = io_rsrc_data_alloc(ctx, io_rsrc_buf_put, nr_args);
8405 if (!data)
8406 return -ENOMEM;
8407 ret = io_buffers_map_alloc(ctx, nr_args);
8408 if (ret) {
Zqiangbb6659c2021-04-30 16:25:15 +08008409 io_rsrc_data_free(data);
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01008410 return ret;
8411 }
Bijan Mottahedeh2b358602021-01-06 12:39:11 -08008412
Pavel Begunkov87094462021-04-11 01:46:36 +01008413 for (i = 0; i < nr_args; i++, ctx->nr_user_bufs++) {
Pavel Begunkov634d00d2021-04-25 14:32:26 +01008414 u64 tag = 0;
Jens Axboeedafcce2019-01-09 09:16:05 -07008415
Pavel Begunkov634d00d2021-04-25 14:32:26 +01008416 if (tags && copy_from_user(&tag, &tags[i], sizeof(tag))) {
8417 ret = -EFAULT;
8418 break;
8419 }
Jens Axboeedafcce2019-01-09 09:16:05 -07008420 ret = io_copy_iov(ctx, &iov, arg, i);
8421 if (ret)
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08008422 break;
Bijan Mottahedeh2b358602021-01-06 12:39:11 -08008423 ret = io_buffer_validate(&iov);
8424 if (ret)
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08008425 break;
Colin Ian Kingcf3770e2021-04-29 11:46:02 +01008426 if (!iov.iov_base && tag) {
8427 ret = -EINVAL;
8428 break;
8429 }
Jens Axboeedafcce2019-01-09 09:16:05 -07008430
Pavel Begunkov41edf1a2021-04-25 14:32:23 +01008431 ret = io_sqe_buffer_register(ctx, &iov, &ctx->user_bufs[i],
8432 &last_hpage);
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08008433 if (ret)
8434 break;
Pavel Begunkov634d00d2021-04-25 14:32:26 +01008435 data->tags[i] = tag;
Jens Axboeedafcce2019-01-09 09:16:05 -07008436 }
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08008437
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01008438 WARN_ON_ONCE(ctx->buf_data);
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08008439
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01008440 ctx->buf_data = data;
8441 if (ret)
8442 __io_sqe_buffers_unregister(ctx);
8443 else
8444 io_rsrc_node_switch(ctx, NULL);
Jens Axboeedafcce2019-01-09 09:16:05 -07008445 return ret;
8446}
8447
Pavel Begunkov634d00d2021-04-25 14:32:26 +01008448static int __io_sqe_buffers_update(struct io_ring_ctx *ctx,
8449 struct io_uring_rsrc_update2 *up,
8450 unsigned int nr_args)
8451{
8452 u64 __user *tags = u64_to_user_ptr(up->tags);
8453 struct iovec iov, __user *iovs = u64_to_user_ptr(up->data);
Pavel Begunkov634d00d2021-04-25 14:32:26 +01008454 struct page *last_hpage = NULL;
8455 bool needs_switch = false;
8456 __u32 done;
8457 int i, err;
8458
8459 if (!ctx->buf_data)
8460 return -ENXIO;
8461 if (up->offset + nr_args > ctx->nr_user_bufs)
8462 return -EINVAL;
8463
8464 for (done = 0; done < nr_args; done++) {
Pavel Begunkov0b8c0e72021-04-26 15:17:38 +01008465 struct io_mapped_ubuf *imu;
8466 int offset = up->offset + done;
Pavel Begunkov634d00d2021-04-25 14:32:26 +01008467 u64 tag = 0;
8468
8469 err = io_copy_iov(ctx, &iov, iovs, done);
8470 if (err)
8471 break;
8472 if (tags && copy_from_user(&tag, &tags[done], sizeof(tag))) {
8473 err = -EFAULT;
8474 break;
8475 }
Pavel Begunkov0b8c0e72021-04-26 15:17:38 +01008476 err = io_buffer_validate(&iov);
8477 if (err)
8478 break;
Colin Ian Kingcf3770e2021-04-29 11:46:02 +01008479 if (!iov.iov_base && tag) {
8480 err = -EINVAL;
8481 break;
8482 }
Pavel Begunkov0b8c0e72021-04-26 15:17:38 +01008483 err = io_sqe_buffer_register(ctx, &iov, &imu, &last_hpage);
8484 if (err)
8485 break;
Pavel Begunkov634d00d2021-04-25 14:32:26 +01008486
Pavel Begunkov0b8c0e72021-04-26 15:17:38 +01008487 i = array_index_nospec(offset, ctx->nr_user_bufs);
Pavel Begunkov62248432021-04-28 13:11:29 +01008488 if (ctx->user_bufs[i] != ctx->dummy_ubuf) {
Pavel Begunkov0b8c0e72021-04-26 15:17:38 +01008489 err = io_queue_rsrc_removal(ctx->buf_data, offset,
8490 ctx->rsrc_node, ctx->user_bufs[i]);
8491 if (unlikely(err)) {
8492 io_buffer_unmap(ctx, &imu);
Pavel Begunkov634d00d2021-04-25 14:32:26 +01008493 break;
Pavel Begunkov0b8c0e72021-04-26 15:17:38 +01008494 }
Pavel Begunkov634d00d2021-04-25 14:32:26 +01008495 ctx->user_bufs[i] = NULL;
8496 needs_switch = true;
8497 }
8498
Pavel Begunkov0b8c0e72021-04-26 15:17:38 +01008499 ctx->user_bufs[i] = imu;
8500 ctx->buf_data->tags[offset] = tag;
Pavel Begunkov634d00d2021-04-25 14:32:26 +01008501 }
8502
8503 if (needs_switch)
8504 io_rsrc_node_switch(ctx, ctx->buf_data);
8505 return done ? done : err;
8506}
8507
Jens Axboe9b402842019-04-11 11:45:41 -06008508static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg)
8509{
8510 __s32 __user *fds = arg;
8511 int fd;
8512
8513 if (ctx->cq_ev_fd)
8514 return -EBUSY;
8515
8516 if (copy_from_user(&fd, fds, sizeof(*fds)))
8517 return -EFAULT;
8518
8519 ctx->cq_ev_fd = eventfd_ctx_fdget(fd);
8520 if (IS_ERR(ctx->cq_ev_fd)) {
8521 int ret = PTR_ERR(ctx->cq_ev_fd);
8522 ctx->cq_ev_fd = NULL;
8523 return ret;
8524 }
8525
8526 return 0;
8527}
8528
8529static int io_eventfd_unregister(struct io_ring_ctx *ctx)
8530{
8531 if (ctx->cq_ev_fd) {
8532 eventfd_ctx_put(ctx->cq_ev_fd);
8533 ctx->cq_ev_fd = NULL;
8534 return 0;
8535 }
8536
8537 return -ENXIO;
8538}
8539
Jens Axboe5a2e7452020-02-23 16:23:11 -07008540static void io_destroy_buffers(struct io_ring_ctx *ctx)
8541{
Jens Axboe9e15c3a2021-03-13 12:29:43 -07008542 struct io_buffer *buf;
8543 unsigned long index;
8544
8545 xa_for_each(&ctx->io_buffers, index, buf)
8546 __io_remove_buffers(ctx, buf, index, -1U);
Jens Axboe5a2e7452020-02-23 16:23:11 -07008547}
8548
Jens Axboe68e68ee2021-02-13 09:00:02 -07008549static void io_req_cache_free(struct list_head *list, struct task_struct *tsk)
Jens Axboe1b4c3512021-02-10 00:03:19 +00008550{
Jens Axboe68e68ee2021-02-13 09:00:02 -07008551 struct io_kiocb *req, *nxt;
Jens Axboe1b4c3512021-02-10 00:03:19 +00008552
Jens Axboe68e68ee2021-02-13 09:00:02 -07008553 list_for_each_entry_safe(req, nxt, list, compl.list) {
8554 if (tsk && req->task != tsk)
8555 continue;
Jens Axboe1b4c3512021-02-10 00:03:19 +00008556 list_del(&req->compl.list);
8557 kmem_cache_free(req_cachep, req);
8558 }
8559}
8560
Jens Axboe4010fec2021-02-27 15:04:18 -07008561static void io_req_caches_free(struct io_ring_ctx *ctx)
Jens Axboe2b188cc2019-01-07 10:46:33 -07008562{
Pavel Begunkovbf019da2021-02-10 00:03:17 +00008563 struct io_submit_state *submit_state = &ctx->submit_state;
Pavel Begunkove5547d22021-02-23 22:17:20 +00008564 struct io_comp_state *cs = &ctx->submit_state.comp;
Pavel Begunkovbf019da2021-02-10 00:03:17 +00008565
Jens Axboe9a4fdbd2021-02-13 09:09:44 -07008566 mutex_lock(&ctx->uring_lock);
8567
Pavel Begunkov8e5c66c2021-02-22 11:45:55 +00008568 if (submit_state->free_reqs) {
Jens Axboe9a4fdbd2021-02-13 09:09:44 -07008569 kmem_cache_free_bulk(req_cachep, submit_state->free_reqs,
8570 submit_state->reqs);
Pavel Begunkov8e5c66c2021-02-22 11:45:55 +00008571 submit_state->free_reqs = 0;
8572 }
Jens Axboe9a4fdbd2021-02-13 09:09:44 -07008573
Pavel Begunkovdac7a092021-03-19 17:22:39 +00008574 io_flush_cached_locked_reqs(ctx, cs);
Pavel Begunkove5547d22021-02-23 22:17:20 +00008575 io_req_cache_free(&cs->free_list, NULL);
Jens Axboe9a4fdbd2021-02-13 09:09:44 -07008576 mutex_unlock(&ctx->uring_lock);
8577}
8578
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01008579static bool io_wait_rsrc_data(struct io_rsrc_data *data)
8580{
8581 if (!data)
8582 return false;
8583 if (!atomic_dec_and_test(&data->refs))
8584 wait_for_completion(&data->done);
8585 return true;
8586}
8587
Jens Axboe2b188cc2019-01-07 10:46:33 -07008588static void io_ring_ctx_free(struct io_ring_ctx *ctx)
8589{
Jens Axboe37d1e2e2021-02-17 21:03:43 -07008590 io_sq_thread_finish(ctx);
Jens Axboe2aede0e2020-09-14 10:45:53 -06008591
Jens Axboe37d1e2e2021-02-17 21:03:43 -07008592 if (ctx->mm_account) {
Jens Axboe2aede0e2020-09-14 10:45:53 -06008593 mmdrop(ctx->mm_account);
8594 ctx->mm_account = NULL;
Bijan Mottahedeh30975822020-06-16 16:36:09 -07008595 }
Jens Axboedef596e2019-01-09 08:59:42 -07008596
Hao Xu8bad28d2021-02-19 17:19:36 +08008597 mutex_lock(&ctx->uring_lock);
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01008598 if (io_wait_rsrc_data(ctx->buf_data))
8599 __io_sqe_buffers_unregister(ctx);
8600 if (io_wait_rsrc_data(ctx->file_data))
Pavel Begunkov08480402021-04-13 02:58:38 +01008601 __io_sqe_files_unregister(ctx);
Pavel Begunkovc4ea0602021-04-01 15:43:58 +01008602 if (ctx->rings)
8603 __io_cqring_overflow_flush(ctx, true);
Hao Xu8bad28d2021-02-19 17:19:36 +08008604 mutex_unlock(&ctx->uring_lock);
Jens Axboe9b402842019-04-11 11:45:41 -06008605 io_eventfd_unregister(ctx);
Jens Axboe5a2e7452020-02-23 16:23:11 -07008606 io_destroy_buffers(ctx);
Pavel Begunkov07db2982021-04-20 12:03:32 +01008607 if (ctx->sq_creds)
8608 put_cred(ctx->sq_creds);
Jens Axboedef596e2019-01-09 08:59:42 -07008609
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01008610 /* there are no registered resources left, nobody uses it */
8611 if (ctx->rsrc_node)
8612 io_rsrc_node_destroy(ctx->rsrc_node);
Pavel Begunkov8dd03af2021-03-19 17:22:36 +00008613 if (ctx->rsrc_backup_node)
Pavel Begunkovb895c9a2021-04-01 15:43:40 +01008614 io_rsrc_node_destroy(ctx->rsrc_backup_node);
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01008615 flush_delayed_work(&ctx->rsrc_put_work);
8616
8617 WARN_ON_ONCE(!list_empty(&ctx->rsrc_ref_list));
8618 WARN_ON_ONCE(!llist_empty(&ctx->rsrc_put_llist));
Jens Axboe2b188cc2019-01-07 10:46:33 -07008619
8620#if defined(CONFIG_UNIX)
Eric Biggers355e8d22019-06-12 14:58:43 -07008621 if (ctx->ring_sock) {
8622 ctx->ring_sock->file = NULL; /* so that iput() is called */
Jens Axboe2b188cc2019-01-07 10:46:33 -07008623 sock_release(ctx->ring_sock);
Eric Biggers355e8d22019-06-12 14:58:43 -07008624 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07008625#endif
8626
Hristo Venev75b28af2019-08-26 17:23:46 +00008627 io_mem_free(ctx->rings);
Jens Axboe2b188cc2019-01-07 10:46:33 -07008628 io_mem_free(ctx->sq_sqes);
Jens Axboe2b188cc2019-01-07 10:46:33 -07008629
8630 percpu_ref_exit(&ctx->refs);
Jens Axboe2b188cc2019-01-07 10:46:33 -07008631 free_uid(ctx->user);
Jens Axboe4010fec2021-02-27 15:04:18 -07008632 io_req_caches_free(ctx);
Jens Axboee9418942021-02-19 12:33:30 -07008633 if (ctx->hash_map)
8634 io_wq_put_hash(ctx->hash_map);
Jens Axboe78076bb2019-12-04 19:56:40 -07008635 kfree(ctx->cancel_hash);
Pavel Begunkov62248432021-04-28 13:11:29 +01008636 kfree(ctx->dummy_ubuf);
Jens Axboe2b188cc2019-01-07 10:46:33 -07008637 kfree(ctx);
8638}
8639
8640static __poll_t io_uring_poll(struct file *file, poll_table *wait)
8641{
8642 struct io_ring_ctx *ctx = file->private_data;
8643 __poll_t mask = 0;
8644
8645 poll_wait(file, &ctx->cq_wait, wait);
Stefan Bühler4f7067c2019-04-24 23:54:17 +02008646 /*
8647 * synchronizes with barrier from wq_has_sleeper call in
8648 * io_commit_cqring
8649 */
Jens Axboe2b188cc2019-01-07 10:46:33 -07008650 smp_rmb();
Jens Axboe90554202020-09-03 12:12:41 -06008651 if (!io_sqring_full(ctx))
Jens Axboe2b188cc2019-01-07 10:46:33 -07008652 mask |= EPOLLOUT | EPOLLWRNORM;
Hao Xued670c32021-02-05 16:34:21 +08008653
8654 /*
8655 * Don't flush cqring overflow list here, just do a simple check.
8656 * Otherwise there could possible be ABBA deadlock:
8657 * CPU0 CPU1
8658 * ---- ----
8659 * lock(&ctx->uring_lock);
8660 * lock(&ep->mtx);
8661 * lock(&ctx->uring_lock);
8662 * lock(&ep->mtx);
8663 *
8664 * Users may get EPOLLIN meanwhile seeing nothing in cqring, this
8665 * pushs them to do the flush.
8666 */
8667 if (io_cqring_events(ctx) || test_bit(0, &ctx->cq_check_overflow))
Jens Axboe2b188cc2019-01-07 10:46:33 -07008668 mask |= EPOLLIN | EPOLLRDNORM;
8669
8670 return mask;
8671}
8672
8673static int io_uring_fasync(int fd, struct file *file, int on)
8674{
8675 struct io_ring_ctx *ctx = file->private_data;
8676
8677 return fasync_helper(fd, file, on, &ctx->cq_fasync);
8678}
8679
Yejune Deng0bead8c2020-12-24 11:02:20 +08008680static int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id)
Jens Axboe071698e2020-01-28 10:04:42 -07008681{
Jens Axboe4379bf82021-02-15 13:40:22 -07008682 const struct cred *creds;
Jens Axboe071698e2020-01-28 10:04:42 -07008683
Matthew Wilcox (Oracle)61cf9372021-03-08 14:16:16 +00008684 creds = xa_erase(&ctx->personalities, id);
Jens Axboe4379bf82021-02-15 13:40:22 -07008685 if (creds) {
8686 put_cred(creds);
Yejune Deng0bead8c2020-12-24 11:02:20 +08008687 return 0;
Jens Axboe1e6fa522020-10-15 08:46:24 -06008688 }
Yejune Deng0bead8c2020-12-24 11:02:20 +08008689
8690 return -EINVAL;
8691}
8692
Pavel Begunkov9b465712021-03-15 14:23:07 +00008693static inline bool io_run_ctx_fallback(struct io_ring_ctx *ctx)
Jens Axboe7c25c0d2021-02-16 07:17:00 -07008694{
Pavel Begunkov9b465712021-03-15 14:23:07 +00008695 return io_run_task_work_head(&ctx->exit_task_work);
Jens Axboe7c25c0d2021-02-16 07:17:00 -07008696}
8697
Pavel Begunkovd56d9382021-03-06 11:02:13 +00008698struct io_tctx_exit {
8699 struct callback_head task_work;
8700 struct completion completion;
Pavel Begunkovbaf186c2021-03-06 11:02:15 +00008701 struct io_ring_ctx *ctx;
Pavel Begunkovd56d9382021-03-06 11:02:13 +00008702};
8703
8704static void io_tctx_exit_cb(struct callback_head *cb)
8705{
8706 struct io_uring_task *tctx = current->io_uring;
8707 struct io_tctx_exit *work;
8708
8709 work = container_of(cb, struct io_tctx_exit, task_work);
8710 /*
8711 * When @in_idle, we're in cancellation and it's racy to remove the
8712 * node. It'll be removed by the end of cancellation, just ignore it.
8713 */
8714 if (!atomic_read(&tctx->in_idle))
Pavel Begunkovbaf186c2021-03-06 11:02:15 +00008715 io_uring_del_task_file((unsigned long)work->ctx);
Pavel Begunkovd56d9382021-03-06 11:02:13 +00008716 complete(&work->completion);
8717}
8718
Pavel Begunkov28090c12021-04-25 23:34:45 +01008719static bool io_cancel_ctx_cb(struct io_wq_work *work, void *data)
8720{
8721 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
8722
8723 return req->ctx == data;
8724}
8725
Jens Axboe85faa7b2020-04-09 18:14:00 -06008726static void io_ring_exit_work(struct work_struct *work)
8727{
Pavel Begunkovd56d9382021-03-06 11:02:13 +00008728 struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx, exit_work);
Pavel Begunkovb5bb3a22021-03-06 11:02:16 +00008729 unsigned long timeout = jiffies + HZ * 60 * 5;
Pavel Begunkovd56d9382021-03-06 11:02:13 +00008730 struct io_tctx_exit exit;
8731 struct io_tctx_node *node;
8732 int ret;
Jens Axboe85faa7b2020-04-09 18:14:00 -06008733
Jens Axboe56952e92020-06-17 15:00:04 -06008734 /*
8735 * If we're doing polled IO and end up having requests being
8736 * submitted async (out-of-line), then completions can come in while
8737 * we're waiting for refs to drop. We need to reap these manually,
8738 * as nobody else will be looking for them.
8739 */
Pavel Begunkovb2edc0a2020-07-07 16:36:22 +03008740 do {
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01008741 io_uring_try_cancel_requests(ctx, NULL, true);
Pavel Begunkov28090c12021-04-25 23:34:45 +01008742 if (ctx->sq_data) {
8743 struct io_sq_data *sqd = ctx->sq_data;
8744 struct task_struct *tsk;
8745
8746 io_sq_thread_park(sqd);
8747 tsk = sqd->thread;
8748 if (tsk && tsk->io_uring && tsk->io_uring->io_wq)
8749 io_wq_cancel_cb(tsk->io_uring->io_wq,
8750 io_cancel_ctx_cb, ctx, true);
8751 io_sq_thread_unpark(sqd);
8752 }
Pavel Begunkovb5bb3a22021-03-06 11:02:16 +00008753
8754 WARN_ON_ONCE(time_after(jiffies, timeout));
Pavel Begunkovb2edc0a2020-07-07 16:36:22 +03008755 } while (!wait_for_completion_timeout(&ctx->ref_comp, HZ/20));
Pavel Begunkovd56d9382021-03-06 11:02:13 +00008756
Pavel Begunkov7f006512021-04-14 13:38:34 +01008757 init_completion(&exit.completion);
8758 init_task_work(&exit.task_work, io_tctx_exit_cb);
8759 exit.ctx = ctx;
Pavel Begunkov89b50662021-04-01 15:43:50 +01008760 /*
8761 * Some may use context even when all refs and requests have been put,
8762 * and they are free to do so while still holding uring_lock or
8763 * completion_lock, see __io_req_task_submit(). Apart from other work,
8764 * this lock/unlock section also waits them to finish.
8765 */
Pavel Begunkovd56d9382021-03-06 11:02:13 +00008766 mutex_lock(&ctx->uring_lock);
8767 while (!list_empty(&ctx->tctx_list)) {
Pavel Begunkovb5bb3a22021-03-06 11:02:16 +00008768 WARN_ON_ONCE(time_after(jiffies, timeout));
8769
Pavel Begunkovd56d9382021-03-06 11:02:13 +00008770 node = list_first_entry(&ctx->tctx_list, struct io_tctx_node,
8771 ctx_node);
Pavel Begunkov7f006512021-04-14 13:38:34 +01008772 /* don't spin on a single task if cancellation failed */
8773 list_rotate_left(&ctx->tctx_list);
Pavel Begunkovd56d9382021-03-06 11:02:13 +00008774 ret = task_work_add(node->task, &exit.task_work, TWA_SIGNAL);
8775 if (WARN_ON_ONCE(ret))
8776 continue;
8777 wake_up_process(node->task);
8778
8779 mutex_unlock(&ctx->uring_lock);
8780 wait_for_completion(&exit.completion);
Pavel Begunkovd56d9382021-03-06 11:02:13 +00008781 mutex_lock(&ctx->uring_lock);
8782 }
8783 mutex_unlock(&ctx->uring_lock);
Pavel Begunkov89b50662021-04-01 15:43:50 +01008784 spin_lock_irq(&ctx->completion_lock);
8785 spin_unlock_irq(&ctx->completion_lock);
Pavel Begunkovd56d9382021-03-06 11:02:13 +00008786
Jens Axboe85faa7b2020-04-09 18:14:00 -06008787 io_ring_ctx_free(ctx);
8788}
8789
Pavel Begunkov80c4cbd2021-03-25 18:32:43 +00008790/* Returns true if we found and killed one or more timeouts */
8791static bool io_kill_timeouts(struct io_ring_ctx *ctx, struct task_struct *tsk,
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01008792 bool cancel_all)
Pavel Begunkov80c4cbd2021-03-25 18:32:43 +00008793{
8794 struct io_kiocb *req, *tmp;
8795 int canceled = 0;
8796
8797 spin_lock_irq(&ctx->completion_lock);
8798 list_for_each_entry_safe(req, tmp, &ctx->timeout_list, timeout.list) {
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01008799 if (io_match_task(req, tsk, cancel_all)) {
Pavel Begunkov80c4cbd2021-03-25 18:32:43 +00008800 io_kill_timeout(req, -ECANCELED);
8801 canceled++;
8802 }
8803 }
Pavel Begunkov51520422021-03-29 11:39:29 +01008804 if (canceled != 0)
8805 io_commit_cqring(ctx);
Pavel Begunkov80c4cbd2021-03-25 18:32:43 +00008806 spin_unlock_irq(&ctx->completion_lock);
Pavel Begunkov80c4cbd2021-03-25 18:32:43 +00008807 if (canceled != 0)
8808 io_cqring_ev_posted(ctx);
8809 return canceled != 0;
8810}
8811
Jens Axboe2b188cc2019-01-07 10:46:33 -07008812static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx)
8813{
Matthew Wilcox (Oracle)61cf9372021-03-08 14:16:16 +00008814 unsigned long index;
8815 struct creds *creds;
8816
Jens Axboe2b188cc2019-01-07 10:46:33 -07008817 mutex_lock(&ctx->uring_lock);
8818 percpu_ref_kill(&ctx->refs);
Pavel Begunkov634578f2020-12-06 22:22:44 +00008819 if (ctx->rings)
Pavel Begunkov6c2450a2021-02-23 12:40:22 +00008820 __io_cqring_overflow_flush(ctx, true);
Matthew Wilcox (Oracle)61cf9372021-03-08 14:16:16 +00008821 xa_for_each(&ctx->personalities, index, creds)
8822 io_unregister_personality(ctx, index);
Jens Axboe2b188cc2019-01-07 10:46:33 -07008823 mutex_unlock(&ctx->uring_lock);
8824
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01008825 io_kill_timeouts(ctx, NULL, true);
8826 io_poll_remove_all(ctx, NULL, true);
Jens Axboe561fb042019-10-24 07:25:42 -06008827
Jens Axboe15dff282019-11-13 09:09:23 -07008828 /* if we failed setting up the ctx, we might not have any rings */
Pavel Begunkovb2edc0a2020-07-07 16:36:22 +03008829 io_iopoll_try_reap_events(ctx);
Jens Axboe309fc032020-07-10 09:13:34 -06008830
Jens Axboe85faa7b2020-04-09 18:14:00 -06008831 INIT_WORK(&ctx->exit_work, io_ring_exit_work);
Jens Axboefc666772020-08-19 11:10:51 -06008832 /*
8833 * Use system_unbound_wq to avoid spawning tons of event kworkers
8834 * if we're exiting a ton of rings at the same time. It just adds
8835 * noise and overhead, there's no discernable change in runtime
8836 * over using system_wq.
8837 */
8838 queue_work(system_unbound_wq, &ctx->exit_work);
Jens Axboe2b188cc2019-01-07 10:46:33 -07008839}
8840
8841static int io_uring_release(struct inode *inode, struct file *file)
8842{
8843 struct io_ring_ctx *ctx = file->private_data;
8844
8845 file->private_data = NULL;
8846 io_ring_ctx_wait_and_kill(ctx);
8847 return 0;
8848}
8849
Pavel Begunkovf6edbab2020-11-06 13:00:26 +00008850struct io_task_cancel {
8851 struct task_struct *task;
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01008852 bool all;
Pavel Begunkovf6edbab2020-11-06 13:00:26 +00008853};
Pavel Begunkov67c4d9e2020-06-15 10:24:05 +03008854
Pavel Begunkovf6edbab2020-11-06 13:00:26 +00008855static bool io_cancel_task_cb(struct io_wq_work *work, void *data)
Jens Axboeb711d4e2020-08-16 08:23:05 -07008856{
Pavel Begunkov9a472ef2020-11-05 22:31:37 +00008857 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
Pavel Begunkovf6edbab2020-11-06 13:00:26 +00008858 struct io_task_cancel *cancel = data;
Pavel Begunkov9a472ef2020-11-05 22:31:37 +00008859 bool ret;
8860
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01008861 if (!cancel->all && (req->flags & REQ_F_LINK_TIMEOUT)) {
Pavel Begunkov9a472ef2020-11-05 22:31:37 +00008862 unsigned long flags;
8863 struct io_ring_ctx *ctx = req->ctx;
8864
8865 /* protect against races with linked timeouts */
8866 spin_lock_irqsave(&ctx->completion_lock, flags);
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01008867 ret = io_match_task(req, cancel->task, cancel->all);
Pavel Begunkov9a472ef2020-11-05 22:31:37 +00008868 spin_unlock_irqrestore(&ctx->completion_lock, flags);
8869 } else {
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01008870 ret = io_match_task(req, cancel->task, cancel->all);
Pavel Begunkov9a472ef2020-11-05 22:31:37 +00008871 }
8872 return ret;
Jens Axboeb711d4e2020-08-16 08:23:05 -07008873}
8874
Pavel Begunkove1915f72021-03-11 23:29:35 +00008875static bool io_cancel_defer_files(struct io_ring_ctx *ctx,
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01008876 struct task_struct *task, bool cancel_all)
Pavel Begunkovb7ddce32020-09-06 00:45:14 +03008877{
Pavel Begunkove1915f72021-03-11 23:29:35 +00008878 struct io_defer_entry *de;
Pavel Begunkovb7ddce32020-09-06 00:45:14 +03008879 LIST_HEAD(list);
8880
8881 spin_lock_irq(&ctx->completion_lock);
8882 list_for_each_entry_reverse(de, &ctx->defer_list, list) {
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01008883 if (io_match_task(de->req, task, cancel_all)) {
Pavel Begunkovb7ddce32020-09-06 00:45:14 +03008884 list_cut_position(&list, &ctx->defer_list, &de->list);
8885 break;
8886 }
8887 }
8888 spin_unlock_irq(&ctx->completion_lock);
Pavel Begunkove1915f72021-03-11 23:29:35 +00008889 if (list_empty(&list))
8890 return false;
Pavel Begunkovb7ddce32020-09-06 00:45:14 +03008891
8892 while (!list_empty(&list)) {
8893 de = list_first_entry(&list, struct io_defer_entry, list);
8894 list_del_init(&de->list);
Pavel Begunkovf41db2732021-02-28 22:35:12 +00008895 io_req_complete_failed(de->req, -ECANCELED);
Pavel Begunkovb7ddce32020-09-06 00:45:14 +03008896 kfree(de);
8897 }
Pavel Begunkove1915f72021-03-11 23:29:35 +00008898 return true;
Pavel Begunkovb7ddce32020-09-06 00:45:14 +03008899}
8900
Pavel Begunkov1b007642021-03-06 11:02:17 +00008901static bool io_uring_try_cancel_iowq(struct io_ring_ctx *ctx)
8902{
8903 struct io_tctx_node *node;
8904 enum io_wq_cancel cret;
8905 bool ret = false;
8906
8907 mutex_lock(&ctx->uring_lock);
8908 list_for_each_entry(node, &ctx->tctx_list, ctx_node) {
8909 struct io_uring_task *tctx = node->task->io_uring;
8910
8911 /*
8912 * io_wq will stay alive while we hold uring_lock, because it's
8913 * killed after ctx nodes, which requires to take the lock.
8914 */
8915 if (!tctx || !tctx->io_wq)
8916 continue;
8917 cret = io_wq_cancel_cb(tctx->io_wq, io_cancel_ctx_cb, ctx, true);
8918 ret |= (cret != IO_WQ_CANCEL_NOTFOUND);
8919 }
8920 mutex_unlock(&ctx->uring_lock);
8921
8922 return ret;
8923}
8924
Pavel Begunkov9936c7c2021-02-04 13:51:56 +00008925static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
8926 struct task_struct *task,
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01008927 bool cancel_all)
Pavel Begunkov9936c7c2021-02-04 13:51:56 +00008928{
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01008929 struct io_task_cancel cancel = { .task = task, .all = cancel_all, };
Pavel Begunkov1b007642021-03-06 11:02:17 +00008930 struct io_uring_task *tctx = task ? task->io_uring : NULL;
Pavel Begunkov9936c7c2021-02-04 13:51:56 +00008931
8932 while (1) {
8933 enum io_wq_cancel cret;
8934 bool ret = false;
8935
Pavel Begunkov1b007642021-03-06 11:02:17 +00008936 if (!task) {
8937 ret |= io_uring_try_cancel_iowq(ctx);
8938 } else if (tctx && tctx->io_wq) {
8939 /*
8940 * Cancels requests of all rings, not only @ctx, but
8941 * it's fine as the task is in exit/exec.
8942 */
Jens Axboe5aa75ed2021-02-16 12:56:50 -07008943 cret = io_wq_cancel_cb(tctx->io_wq, io_cancel_task_cb,
Pavel Begunkov9936c7c2021-02-04 13:51:56 +00008944 &cancel, true);
8945 ret |= (cret != IO_WQ_CANCEL_NOTFOUND);
8946 }
8947
8948 /* SQPOLL thread does its own polling */
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01008949 if ((!(ctx->flags & IORING_SETUP_SQPOLL) && cancel_all) ||
Jens Axboed052d1d2021-03-11 10:49:20 -07008950 (ctx->sq_data && ctx->sq_data->thread == current)) {
Pavel Begunkov9936c7c2021-02-04 13:51:56 +00008951 while (!list_empty_careful(&ctx->iopoll_list)) {
8952 io_iopoll_try_reap_events(ctx);
8953 ret = true;
8954 }
8955 }
8956
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01008957 ret |= io_cancel_defer_files(ctx, task, cancel_all);
8958 ret |= io_poll_remove_all(ctx, task, cancel_all);
8959 ret |= io_kill_timeouts(ctx, task, cancel_all);
Pavel Begunkov9936c7c2021-02-04 13:51:56 +00008960 ret |= io_run_task_work();
Pavel Begunkovba50a032021-02-26 15:47:56 +00008961 ret |= io_run_ctx_fallback(ctx);
Pavel Begunkov9936c7c2021-02-04 13:51:56 +00008962 if (!ret)
8963 break;
8964 cond_resched();
8965 }
8966}
8967
Pavel Begunkovcf27f3b2021-03-19 17:22:31 +00008968static int __io_uring_add_task_file(struct io_ring_ctx *ctx)
Jens Axboe0f212202020-09-13 13:09:39 -06008969{
Matthew Wilcox (Oracle)236434c2020-10-09 13:49:52 +01008970 struct io_uring_task *tctx = current->io_uring;
Pavel Begunkov13bf43f2021-03-06 11:02:12 +00008971 struct io_tctx_node *node;
Pavel Begunkova528b042020-12-21 18:34:04 +00008972 int ret;
Matthew Wilcox (Oracle)236434c2020-10-09 13:49:52 +01008973
8974 if (unlikely(!tctx)) {
Jens Axboe5aa75ed2021-02-16 12:56:50 -07008975 ret = io_uring_alloc_task_context(current, ctx);
Jens Axboe0f212202020-09-13 13:09:39 -06008976 if (unlikely(ret))
8977 return ret;
Matthew Wilcox (Oracle)236434c2020-10-09 13:49:52 +01008978 tctx = current->io_uring;
Jens Axboe0f212202020-09-13 13:09:39 -06008979 }
Pavel Begunkovcf27f3b2021-03-19 17:22:31 +00008980 if (!xa_load(&tctx->xa, (unsigned long)ctx)) {
8981 node = kmalloc(sizeof(*node), GFP_KERNEL);
8982 if (!node)
8983 return -ENOMEM;
8984 node->ctx = ctx;
8985 node->task = current;
Jens Axboe0f212202020-09-13 13:09:39 -06008986
Pavel Begunkovcf27f3b2021-03-19 17:22:31 +00008987 ret = xa_err(xa_store(&tctx->xa, (unsigned long)ctx,
8988 node, GFP_KERNEL));
8989 if (ret) {
8990 kfree(node);
8991 return ret;
Jens Axboe0f212202020-09-13 13:09:39 -06008992 }
Pavel Begunkovcf27f3b2021-03-19 17:22:31 +00008993
8994 mutex_lock(&ctx->uring_lock);
8995 list_add(&node->ctx_node, &ctx->tctx_list);
8996 mutex_unlock(&ctx->uring_lock);
Jens Axboe0f212202020-09-13 13:09:39 -06008997 }
Pavel Begunkovcf27f3b2021-03-19 17:22:31 +00008998 tctx->last = ctx;
Jens Axboe0f212202020-09-13 13:09:39 -06008999 return 0;
9000}
9001
9002/*
Pavel Begunkovcf27f3b2021-03-19 17:22:31 +00009003 * Note that this task has used io_uring. We use it for cancelation purposes.
9004 */
9005static inline int io_uring_add_task_file(struct io_ring_ctx *ctx)
9006{
9007 struct io_uring_task *tctx = current->io_uring;
9008
9009 if (likely(tctx && tctx->last == ctx))
9010 return 0;
9011 return __io_uring_add_task_file(ctx);
9012}
9013
9014/*
Jens Axboe0f212202020-09-13 13:09:39 -06009015 * Remove this io_uring_file -> task mapping.
9016 */
Pavel Begunkov29412672021-03-06 11:02:11 +00009017static void io_uring_del_task_file(unsigned long index)
Jens Axboe0f212202020-09-13 13:09:39 -06009018{
9019 struct io_uring_task *tctx = current->io_uring;
Pavel Begunkov13bf43f2021-03-06 11:02:12 +00009020 struct io_tctx_node *node;
Pavel Begunkov29412672021-03-06 11:02:11 +00009021
Pavel Begunkoveebd2e32021-03-06 11:02:14 +00009022 if (!tctx)
9023 return;
Pavel Begunkov13bf43f2021-03-06 11:02:12 +00009024 node = xa_erase(&tctx->xa, index);
9025 if (!node)
Pavel Begunkov29412672021-03-06 11:02:11 +00009026 return;
Jens Axboe0f212202020-09-13 13:09:39 -06009027
Pavel Begunkov13bf43f2021-03-06 11:02:12 +00009028 WARN_ON_ONCE(current != node->task);
9029 WARN_ON_ONCE(list_empty(&node->ctx_node));
9030
9031 mutex_lock(&node->ctx->uring_lock);
9032 list_del(&node->ctx_node);
9033 mutex_unlock(&node->ctx->uring_lock);
9034
Pavel Begunkovbaf186c2021-03-06 11:02:15 +00009035 if (tctx->last == node->ctx)
Jens Axboe0f212202020-09-13 13:09:39 -06009036 tctx->last = NULL;
Pavel Begunkov13bf43f2021-03-06 11:02:12 +00009037 kfree(node);
Jens Axboe0f212202020-09-13 13:09:39 -06009038}
9039
Pavel Begunkov8452d4a2021-02-27 11:16:46 +00009040static void io_uring_clean_tctx(struct io_uring_task *tctx)
Pavel Begunkovde7f1d92021-01-04 20:43:29 +00009041{
Pavel Begunkovba5ef6d2021-05-20 13:21:20 +01009042 struct io_wq *wq = tctx->io_wq;
Pavel Begunkov13bf43f2021-03-06 11:02:12 +00009043 struct io_tctx_node *node;
Pavel Begunkovde7f1d92021-01-04 20:43:29 +00009044 unsigned long index;
9045
Pavel Begunkov13bf43f2021-03-06 11:02:12 +00009046 xa_for_each(&tctx->xa, index, node)
Pavel Begunkov29412672021-03-06 11:02:11 +00009047 io_uring_del_task_file(index);
Marco Elverb16ef422021-05-27 11:25:48 +02009048 if (wq) {
9049 /*
9050 * Must be after io_uring_del_task_file() (removes nodes under
9051 * uring_lock) to avoid race with io_uring_try_cancel_iowq().
9052 */
9053 tctx->io_wq = NULL;
Pavel Begunkovba5ef6d2021-05-20 13:21:20 +01009054 io_wq_put_and_exit(wq);
Marco Elverb16ef422021-05-27 11:25:48 +02009055 }
Pavel Begunkovde7f1d92021-01-04 20:43:29 +00009056}
9057
Pavel Begunkov3f48cf12021-04-11 01:46:27 +01009058static s64 tctx_inflight(struct io_uring_task *tctx, bool tracked)
Pavel Begunkov521d6a72021-03-11 23:29:38 +00009059{
Pavel Begunkov3f48cf12021-04-11 01:46:27 +01009060 if (tracked)
9061 return atomic_read(&tctx->inflight_tracked);
Pavel Begunkov521d6a72021-03-11 23:29:38 +00009062 return percpu_counter_sum(&tctx->inflight);
9063}
9064
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01009065static void io_uring_try_cancel(bool cancel_all)
Jens Axboe0f212202020-09-13 13:09:39 -06009066{
9067 struct io_uring_task *tctx = current->io_uring;
Pavel Begunkov13bf43f2021-03-06 11:02:12 +00009068 struct io_tctx_node *node;
Matthew Wilcox (Oracle)ce765372020-10-09 13:49:51 +01009069 unsigned long index;
Jens Axboe0f212202020-09-13 13:09:39 -06009070
Pavel Begunkov521d6a72021-03-11 23:29:38 +00009071 xa_for_each(&tctx->xa, index, node) {
9072 struct io_ring_ctx *ctx = node->ctx;
9073
Pavel Begunkov9f59a9d2021-04-25 23:34:46 +01009074 /* sqpoll task will cancel all its requests */
9075 if (!ctx->sq_data)
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01009076 io_uring_try_cancel_requests(ctx, current, cancel_all);
Pavel Begunkov521d6a72021-03-11 23:29:38 +00009077 }
Jens Axboefdaf0832020-10-30 09:37:30 -06009078}
9079
Pavel Begunkov521d6a72021-03-11 23:29:38 +00009080/* should only be called by SQPOLL task */
Pavel Begunkov734551d2021-04-18 14:52:09 +01009081static void io_uring_cancel_sqpoll(struct io_sq_data *sqd)
Pavel Begunkov0e9ddb32021-02-07 22:34:26 +00009082{
Pavel Begunkov521d6a72021-03-11 23:29:38 +00009083 struct io_uring_task *tctx = current->io_uring;
Pavel Begunkov734551d2021-04-18 14:52:09 +01009084 struct io_ring_ctx *ctx;
Jens Axboefdaf0832020-10-30 09:37:30 -06009085 s64 inflight;
Pavel Begunkov0e9ddb32021-02-07 22:34:26 +00009086 DEFINE_WAIT(wait);
Jens Axboefdaf0832020-10-30 09:37:30 -06009087
Palash Oswal6d042ff2021-04-27 18:21:49 +05309088 if (!current->io_uring)
9089 return;
Pavel Begunkov17a91052021-05-23 15:48:39 +01009090 if (tctx->io_wq)
9091 io_wq_exit_start(tctx->io_wq);
9092
Pavel Begunkov734551d2021-04-18 14:52:09 +01009093 WARN_ON_ONCE(!sqd || sqd->thread != current);
Pavel Begunkov521d6a72021-03-11 23:29:38 +00009094
Pavel Begunkov0e9ddb32021-02-07 22:34:26 +00009095 atomic_inc(&tctx->in_idle);
9096 do {
9097 /* read completions before cancelations */
Pavel Begunkov3f48cf12021-04-11 01:46:27 +01009098 inflight = tctx_inflight(tctx, false);
Pavel Begunkov0e9ddb32021-02-07 22:34:26 +00009099 if (!inflight)
9100 break;
Pavel Begunkov734551d2021-04-18 14:52:09 +01009101 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list)
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01009102 io_uring_try_cancel_requests(ctx, current, true);
Jens Axboefdaf0832020-10-30 09:37:30 -06009103
Pavel Begunkov0e9ddb32021-02-07 22:34:26 +00009104 prepare_to_wait(&tctx->wait, &wait, TASK_UNINTERRUPTIBLE);
9105 /*
9106 * If we've seen completions, retry without waiting. This
9107 * avoids a race where a completion comes in before we did
9108 * prepare_to_wait().
9109 */
Pavel Begunkov3f48cf12021-04-11 01:46:27 +01009110 if (inflight == tctx_inflight(tctx, false))
Pavel Begunkov0e9ddb32021-02-07 22:34:26 +00009111 schedule();
9112 finish_wait(&tctx->wait, &wait);
9113 } while (1);
9114 atomic_dec(&tctx->in_idle);
Jens Axboe0f212202020-09-13 13:09:39 -06009115}
9116
Jens Axboe0f212202020-09-13 13:09:39 -06009117/*
9118 * Find any io_uring fd that this task has registered or done IO on, and cancel
9119 * requests.
9120 */
Pavel Begunkov3f48cf12021-04-11 01:46:27 +01009121void __io_uring_cancel(struct files_struct *files)
Jens Axboe0f212202020-09-13 13:09:39 -06009122{
9123 struct io_uring_task *tctx = current->io_uring;
9124 DEFINE_WAIT(wait);
Jens Axboed8a6df12020-10-15 16:24:45 -06009125 s64 inflight;
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01009126 bool cancel_all = !files;
Jens Axboe0f212202020-09-13 13:09:39 -06009127
Pavel Begunkov17a91052021-05-23 15:48:39 +01009128 if (tctx->io_wq)
9129 io_wq_exit_start(tctx->io_wq);
9130
Jens Axboe0f212202020-09-13 13:09:39 -06009131 /* make sure overflow events are dropped */
Jens Axboefdaf0832020-10-30 09:37:30 -06009132 atomic_inc(&tctx->in_idle);
Jens Axboed8a6df12020-10-15 16:24:45 -06009133 do {
Jens Axboe0f212202020-09-13 13:09:39 -06009134 /* read completions before cancelations */
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01009135 inflight = tctx_inflight(tctx, !cancel_all);
Jens Axboed8a6df12020-10-15 16:24:45 -06009136 if (!inflight)
9137 break;
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01009138 io_uring_try_cancel(cancel_all);
Jens Axboe0f212202020-09-13 13:09:39 -06009139 prepare_to_wait(&tctx->wait, &wait, TASK_UNINTERRUPTIBLE);
9140
9141 /*
Pavel Begunkova1bb3cd2021-01-26 15:28:26 +00009142 * If we've seen completions, retry without waiting. This
9143 * avoids a race where a completion comes in before we did
9144 * prepare_to_wait().
Jens Axboe0f212202020-09-13 13:09:39 -06009145 */
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01009146 if (inflight == tctx_inflight(tctx, !cancel_all))
Pavel Begunkova1bb3cd2021-01-26 15:28:26 +00009147 schedule();
Pavel Begunkovf57555e2020-12-20 13:21:44 +00009148 finish_wait(&tctx->wait, &wait);
Jens Axboed8a6df12020-10-15 16:24:45 -06009149 } while (1);
Jens Axboefdaf0832020-10-30 09:37:30 -06009150 atomic_dec(&tctx->in_idle);
Pavel Begunkovde7f1d92021-01-04 20:43:29 +00009151
Pavel Begunkov8452d4a2021-02-27 11:16:46 +00009152 io_uring_clean_tctx(tctx);
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01009153 if (cancel_all) {
Pavel Begunkov3f48cf12021-04-11 01:46:27 +01009154 /* for exec all current's requests should be gone, kill tctx */
9155 __io_uring_free(current);
9156 }
Pavel Begunkov44e728b2020-06-15 10:24:04 +03009157}
9158
Roman Penyaev6c5c2402019-11-28 12:53:22 +01009159static void *io_uring_validate_mmap_request(struct file *file,
9160 loff_t pgoff, size_t sz)
Jens Axboe2b188cc2019-01-07 10:46:33 -07009161{
Jens Axboe2b188cc2019-01-07 10:46:33 -07009162 struct io_ring_ctx *ctx = file->private_data;
Roman Penyaev6c5c2402019-11-28 12:53:22 +01009163 loff_t offset = pgoff << PAGE_SHIFT;
Jens Axboe2b188cc2019-01-07 10:46:33 -07009164 struct page *page;
9165 void *ptr;
9166
9167 switch (offset) {
9168 case IORING_OFF_SQ_RING:
Hristo Venev75b28af2019-08-26 17:23:46 +00009169 case IORING_OFF_CQ_RING:
9170 ptr = ctx->rings;
Jens Axboe2b188cc2019-01-07 10:46:33 -07009171 break;
9172 case IORING_OFF_SQES:
9173 ptr = ctx->sq_sqes;
9174 break;
Jens Axboe2b188cc2019-01-07 10:46:33 -07009175 default:
Roman Penyaev6c5c2402019-11-28 12:53:22 +01009176 return ERR_PTR(-EINVAL);
Jens Axboe2b188cc2019-01-07 10:46:33 -07009177 }
9178
9179 page = virt_to_head_page(ptr);
Matthew Wilcox (Oracle)a50b8542019-09-23 15:34:25 -07009180 if (sz > page_size(page))
Roman Penyaev6c5c2402019-11-28 12:53:22 +01009181 return ERR_PTR(-EINVAL);
9182
9183 return ptr;
9184}
9185
9186#ifdef CONFIG_MMU
9187
9188static int io_uring_mmap(struct file *file, struct vm_area_struct *vma)
9189{
9190 size_t sz = vma->vm_end - vma->vm_start;
9191 unsigned long pfn;
9192 void *ptr;
9193
9194 ptr = io_uring_validate_mmap_request(file, vma->vm_pgoff, sz);
9195 if (IS_ERR(ptr))
9196 return PTR_ERR(ptr);
Jens Axboe2b188cc2019-01-07 10:46:33 -07009197
9198 pfn = virt_to_phys(ptr) >> PAGE_SHIFT;
9199 return remap_pfn_range(vma, vma->vm_start, pfn, sz, vma->vm_page_prot);
9200}
9201
Roman Penyaev6c5c2402019-11-28 12:53:22 +01009202#else /* !CONFIG_MMU */
9203
9204static int io_uring_mmap(struct file *file, struct vm_area_struct *vma)
9205{
9206 return vma->vm_flags & (VM_SHARED | VM_MAYSHARE) ? 0 : -EINVAL;
9207}
9208
9209static unsigned int io_uring_nommu_mmap_capabilities(struct file *file)
9210{
9211 return NOMMU_MAP_DIRECT | NOMMU_MAP_READ | NOMMU_MAP_WRITE;
9212}
9213
9214static unsigned long io_uring_nommu_get_unmapped_area(struct file *file,
9215 unsigned long addr, unsigned long len,
9216 unsigned long pgoff, unsigned long flags)
9217{
9218 void *ptr;
9219
9220 ptr = io_uring_validate_mmap_request(file, pgoff, len);
9221 if (IS_ERR(ptr))
9222 return PTR_ERR(ptr);
9223
9224 return (unsigned long) ptr;
9225}
9226
9227#endif /* !CONFIG_MMU */
9228
Pavel Begunkovd9d05212021-01-08 20:57:25 +00009229static int io_sqpoll_wait_sq(struct io_ring_ctx *ctx)
Jens Axboe90554202020-09-03 12:12:41 -06009230{
9231 DEFINE_WAIT(wait);
9232
9233 do {
9234 if (!io_sqring_full(ctx))
9235 break;
Jens Axboe90554202020-09-03 12:12:41 -06009236 prepare_to_wait(&ctx->sqo_sq_wait, &wait, TASK_INTERRUPTIBLE);
9237
9238 if (!io_sqring_full(ctx))
9239 break;
Jens Axboe90554202020-09-03 12:12:41 -06009240 schedule();
9241 } while (!signal_pending(current));
9242
9243 finish_wait(&ctx->sqo_sq_wait, &wait);
Yang Li51993282021-03-09 14:30:41 +08009244 return 0;
Jens Axboe90554202020-09-03 12:12:41 -06009245}
9246
Hao Xuc73ebb62020-11-03 10:54:37 +08009247static int io_get_ext_arg(unsigned flags, const void __user *argp, size_t *argsz,
9248 struct __kernel_timespec __user **ts,
9249 const sigset_t __user **sig)
9250{
9251 struct io_uring_getevents_arg arg;
9252
9253 /*
9254 * If EXT_ARG isn't set, then we have no timespec and the argp pointer
9255 * is just a pointer to the sigset_t.
9256 */
9257 if (!(flags & IORING_ENTER_EXT_ARG)) {
9258 *sig = (const sigset_t __user *) argp;
9259 *ts = NULL;
9260 return 0;
9261 }
9262
9263 /*
9264 * EXT_ARG is set - ensure we agree on the size of it and copy in our
9265 * timespec and sigset_t pointers if good.
9266 */
9267 if (*argsz != sizeof(arg))
9268 return -EINVAL;
9269 if (copy_from_user(&arg, argp, sizeof(arg)))
9270 return -EFAULT;
9271 *sig = u64_to_user_ptr(arg.sigmask);
9272 *argsz = arg.sigmask_sz;
9273 *ts = u64_to_user_ptr(arg.ts);
9274 return 0;
9275}
9276
Jens Axboe2b188cc2019-01-07 10:46:33 -07009277SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
Hao Xuc73ebb62020-11-03 10:54:37 +08009278 u32, min_complete, u32, flags, const void __user *, argp,
9279 size_t, argsz)
Jens Axboe2b188cc2019-01-07 10:46:33 -07009280{
9281 struct io_ring_ctx *ctx;
Jens Axboe2b188cc2019-01-07 10:46:33 -07009282 int submitted = 0;
9283 struct fd f;
Pavel Begunkov33f993d2021-03-19 17:22:30 +00009284 long ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07009285
Jens Axboe4c6e2772020-07-01 11:29:10 -06009286 io_run_task_work();
Jens Axboeb41e9852020-02-17 09:52:41 -07009287
Pavel Begunkov33f993d2021-03-19 17:22:30 +00009288 if (unlikely(flags & ~(IORING_ENTER_GETEVENTS | IORING_ENTER_SQ_WAKEUP |
9289 IORING_ENTER_SQ_WAIT | IORING_ENTER_EXT_ARG)))
Jens Axboe2b188cc2019-01-07 10:46:33 -07009290 return -EINVAL;
9291
9292 f = fdget(fd);
Pavel Begunkov33f993d2021-03-19 17:22:30 +00009293 if (unlikely(!f.file))
Jens Axboe2b188cc2019-01-07 10:46:33 -07009294 return -EBADF;
9295
9296 ret = -EOPNOTSUPP;
Pavel Begunkov33f993d2021-03-19 17:22:30 +00009297 if (unlikely(f.file->f_op != &io_uring_fops))
Jens Axboe2b188cc2019-01-07 10:46:33 -07009298 goto out_fput;
9299
9300 ret = -ENXIO;
9301 ctx = f.file->private_data;
Pavel Begunkov33f993d2021-03-19 17:22:30 +00009302 if (unlikely(!percpu_ref_tryget(&ctx->refs)))
Jens Axboe2b188cc2019-01-07 10:46:33 -07009303 goto out_fput;
9304
Stefano Garzarella7e84e1c2020-08-27 16:58:31 +02009305 ret = -EBADFD;
Pavel Begunkov33f993d2021-03-19 17:22:30 +00009306 if (unlikely(ctx->flags & IORING_SETUP_R_DISABLED))
Stefano Garzarella7e84e1c2020-08-27 16:58:31 +02009307 goto out;
9308
Jens Axboe6c271ce2019-01-10 11:22:30 -07009309 /*
9310 * For SQ polling, the thread will do all submissions and completions.
9311 * Just return the requested submit count, and wake the thread if
9312 * we were asked to.
9313 */
Jens Axboeb2a9ead2019-09-12 14:19:16 -06009314 ret = 0;
Jens Axboe6c271ce2019-01-10 11:22:30 -07009315 if (ctx->flags & IORING_SETUP_SQPOLL) {
Pavel Begunkov6c2450a2021-02-23 12:40:22 +00009316 io_cqring_overflow_flush(ctx, false);
Pavel Begunkov89448c42020-12-17 00:24:39 +00009317
Pavel Begunkovd9d05212021-01-08 20:57:25 +00009318 ret = -EOWNERDEAD;
Stefan Metzmacher04147482021-03-07 11:54:29 +01009319 if (unlikely(ctx->sq_data->thread == NULL)) {
9320 goto out;
9321 }
Jens Axboe6c271ce2019-01-10 11:22:30 -07009322 if (flags & IORING_ENTER_SQ_WAKEUP)
Jens Axboe534ca6d2020-09-02 13:52:19 -06009323 wake_up(&ctx->sq_data->wait);
Pavel Begunkovd9d05212021-01-08 20:57:25 +00009324 if (flags & IORING_ENTER_SQ_WAIT) {
9325 ret = io_sqpoll_wait_sq(ctx);
9326 if (ret)
9327 goto out;
9328 }
Jens Axboe6c271ce2019-01-10 11:22:30 -07009329 submitted = to_submit;
Jens Axboeb2a9ead2019-09-12 14:19:16 -06009330 } else if (to_submit) {
Pavel Begunkovbaf186c2021-03-06 11:02:15 +00009331 ret = io_uring_add_task_file(ctx);
Jens Axboe0f212202020-09-13 13:09:39 -06009332 if (unlikely(ret))
9333 goto out;
Jens Axboe2b188cc2019-01-07 10:46:33 -07009334 mutex_lock(&ctx->uring_lock);
Jens Axboe0f212202020-09-13 13:09:39 -06009335 submitted = io_submit_sqes(ctx, to_submit);
Jens Axboe2b188cc2019-01-07 10:46:33 -07009336 mutex_unlock(&ctx->uring_lock);
Pavel Begunkov7c504e652019-12-18 19:53:45 +03009337
9338 if (submitted != to_submit)
9339 goto out;
Jens Axboe2b188cc2019-01-07 10:46:33 -07009340 }
9341 if (flags & IORING_ENTER_GETEVENTS) {
Hao Xuc73ebb62020-11-03 10:54:37 +08009342 const sigset_t __user *sig;
9343 struct __kernel_timespec __user *ts;
9344
9345 ret = io_get_ext_arg(flags, argp, &argsz, &ts, &sig);
9346 if (unlikely(ret))
9347 goto out;
9348
Jens Axboe2b188cc2019-01-07 10:46:33 -07009349 min_complete = min(min_complete, ctx->cq_entries);
9350
Xiaoguang Wang32b22442020-03-11 09:26:09 +08009351 /*
9352 * When SETUP_IOPOLL and SETUP_SQPOLL are both enabled, user
9353 * space applications don't need to do io completion events
9354 * polling again, they can rely on io_sq_thread to do polling
9355 * work, which can reduce cpu usage and uring_lock contention.
9356 */
9357 if (ctx->flags & IORING_SETUP_IOPOLL &&
9358 !(ctx->flags & IORING_SETUP_SQPOLL)) {
Pavel Begunkov7668b922020-07-07 16:36:21 +03009359 ret = io_iopoll_check(ctx, min_complete);
Jens Axboedef596e2019-01-09 08:59:42 -07009360 } else {
Hao Xuc73ebb62020-11-03 10:54:37 +08009361 ret = io_cqring_wait(ctx, min_complete, sig, argsz, ts);
Jens Axboedef596e2019-01-09 08:59:42 -07009362 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07009363 }
9364
Pavel Begunkov7c504e652019-12-18 19:53:45 +03009365out:
Pavel Begunkov6805b322019-10-08 02:18:42 +03009366 percpu_ref_put(&ctx->refs);
Jens Axboe2b188cc2019-01-07 10:46:33 -07009367out_fput:
9368 fdput(f);
9369 return submitted ? submitted : ret;
9370}
9371
Tobias Klauserbebdb652020-02-26 18:38:32 +01009372#ifdef CONFIG_PROC_FS
Matthew Wilcox (Oracle)61cf9372021-03-08 14:16:16 +00009373static int io_uring_show_cred(struct seq_file *m, unsigned int id,
9374 const struct cred *cred)
Jens Axboe87ce9552020-01-30 08:25:34 -07009375{
Jens Axboe87ce9552020-01-30 08:25:34 -07009376 struct user_namespace *uns = seq_user_ns(m);
9377 struct group_info *gi;
9378 kernel_cap_t cap;
9379 unsigned __capi;
9380 int g;
9381
9382 seq_printf(m, "%5d\n", id);
9383 seq_put_decimal_ull(m, "\tUid:\t", from_kuid_munged(uns, cred->uid));
9384 seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->euid));
9385 seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->suid));
9386 seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->fsuid));
9387 seq_put_decimal_ull(m, "\n\tGid:\t", from_kgid_munged(uns, cred->gid));
9388 seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->egid));
9389 seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->sgid));
9390 seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->fsgid));
9391 seq_puts(m, "\n\tGroups:\t");
9392 gi = cred->group_info;
9393 for (g = 0; g < gi->ngroups; g++) {
9394 seq_put_decimal_ull(m, g ? " " : "",
9395 from_kgid_munged(uns, gi->gid[g]));
9396 }
9397 seq_puts(m, "\n\tCapEff:\t");
9398 cap = cred->cap_effective;
9399 CAP_FOR_EACH_U32(__capi)
9400 seq_put_hex_ll(m, NULL, cap.cap[CAP_LAST_U32 - __capi], 8);
9401 seq_putc(m, '\n');
9402 return 0;
9403}
9404
9405static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m)
9406{
Joseph Qidbbe9c62020-09-29 09:01:22 -06009407 struct io_sq_data *sq = NULL;
Jens Axboefad8e0d2020-09-28 08:57:48 -06009408 bool has_lock;
Jens Axboe87ce9552020-01-30 08:25:34 -07009409 int i;
9410
Jens Axboefad8e0d2020-09-28 08:57:48 -06009411 /*
9412 * Avoid ABBA deadlock between the seq lock and the io_uring mutex,
9413 * since fdinfo case grabs it in the opposite direction of normal use
9414 * cases. If we fail to get the lock, we just don't iterate any
9415 * structures that could be going away outside the io_uring mutex.
9416 */
9417 has_lock = mutex_trylock(&ctx->uring_lock);
9418
Jens Axboe5f3f26f2021-02-25 10:17:46 -07009419 if (has_lock && (ctx->flags & IORING_SETUP_SQPOLL)) {
Joseph Qidbbe9c62020-09-29 09:01:22 -06009420 sq = ctx->sq_data;
Jens Axboe5f3f26f2021-02-25 10:17:46 -07009421 if (!sq->thread)
9422 sq = NULL;
9423 }
Joseph Qidbbe9c62020-09-29 09:01:22 -06009424
9425 seq_printf(m, "SqThread:\t%d\n", sq ? task_pid_nr(sq->thread) : -1);
9426 seq_printf(m, "SqThreadCpu:\t%d\n", sq ? task_cpu(sq->thread) : -1);
Jens Axboe87ce9552020-01-30 08:25:34 -07009427 seq_printf(m, "UserFiles:\t%u\n", ctx->nr_user_files);
Jens Axboefad8e0d2020-09-28 08:57:48 -06009428 for (i = 0; has_lock && i < ctx->nr_user_files; i++) {
Jens Axboe7b29f922021-03-12 08:30:14 -07009429 struct file *f = io_file_from_index(ctx, i);
Jens Axboe87ce9552020-01-30 08:25:34 -07009430
Jens Axboe87ce9552020-01-30 08:25:34 -07009431 if (f)
9432 seq_printf(m, "%5u: %s\n", i, file_dentry(f)->d_iname);
9433 else
9434 seq_printf(m, "%5u: <none>\n", i);
9435 }
9436 seq_printf(m, "UserBufs:\t%u\n", ctx->nr_user_bufs);
Jens Axboefad8e0d2020-09-28 08:57:48 -06009437 for (i = 0; has_lock && i < ctx->nr_user_bufs; i++) {
Pavel Begunkov41edf1a2021-04-25 14:32:23 +01009438 struct io_mapped_ubuf *buf = ctx->user_bufs[i];
Pavel Begunkov4751f532021-04-01 15:43:55 +01009439 unsigned int len = buf->ubuf_end - buf->ubuf;
Jens Axboe87ce9552020-01-30 08:25:34 -07009440
Pavel Begunkov4751f532021-04-01 15:43:55 +01009441 seq_printf(m, "%5u: 0x%llx/%u\n", i, buf->ubuf, len);
Jens Axboe87ce9552020-01-30 08:25:34 -07009442 }
Matthew Wilcox (Oracle)61cf9372021-03-08 14:16:16 +00009443 if (has_lock && !xa_empty(&ctx->personalities)) {
9444 unsigned long index;
9445 const struct cred *cred;
9446
Jens Axboe87ce9552020-01-30 08:25:34 -07009447 seq_printf(m, "Personalities:\n");
Matthew Wilcox (Oracle)61cf9372021-03-08 14:16:16 +00009448 xa_for_each(&ctx->personalities, index, cred)
9449 io_uring_show_cred(m, index, cred);
Jens Axboe87ce9552020-01-30 08:25:34 -07009450 }
Jens Axboed7718a92020-02-14 22:23:12 -07009451 seq_printf(m, "PollList:\n");
9452 spin_lock_irq(&ctx->completion_lock);
9453 for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) {
9454 struct hlist_head *list = &ctx->cancel_hash[i];
9455 struct io_kiocb *req;
9456
9457 hlist_for_each_entry(req, list, hash_node)
9458 seq_printf(m, " op=%d, task_works=%d\n", req->opcode,
9459 req->task->task_works != NULL);
9460 }
9461 spin_unlock_irq(&ctx->completion_lock);
Jens Axboefad8e0d2020-09-28 08:57:48 -06009462 if (has_lock)
9463 mutex_unlock(&ctx->uring_lock);
Jens Axboe87ce9552020-01-30 08:25:34 -07009464}
9465
9466static void io_uring_show_fdinfo(struct seq_file *m, struct file *f)
9467{
9468 struct io_ring_ctx *ctx = f->private_data;
9469
9470 if (percpu_ref_tryget(&ctx->refs)) {
9471 __io_uring_show_fdinfo(ctx, m);
9472 percpu_ref_put(&ctx->refs);
9473 }
9474}
Tobias Klauserbebdb652020-02-26 18:38:32 +01009475#endif
Jens Axboe87ce9552020-01-30 08:25:34 -07009476
Jens Axboe2b188cc2019-01-07 10:46:33 -07009477static const struct file_operations io_uring_fops = {
9478 .release = io_uring_release,
9479 .mmap = io_uring_mmap,
Roman Penyaev6c5c2402019-11-28 12:53:22 +01009480#ifndef CONFIG_MMU
9481 .get_unmapped_area = io_uring_nommu_get_unmapped_area,
9482 .mmap_capabilities = io_uring_nommu_mmap_capabilities,
9483#endif
Jens Axboe2b188cc2019-01-07 10:46:33 -07009484 .poll = io_uring_poll,
9485 .fasync = io_uring_fasync,
Tobias Klauserbebdb652020-02-26 18:38:32 +01009486#ifdef CONFIG_PROC_FS
Jens Axboe87ce9552020-01-30 08:25:34 -07009487 .show_fdinfo = io_uring_show_fdinfo,
Tobias Klauserbebdb652020-02-26 18:38:32 +01009488#endif
Jens Axboe2b188cc2019-01-07 10:46:33 -07009489};
9490
9491static int io_allocate_scq_urings(struct io_ring_ctx *ctx,
9492 struct io_uring_params *p)
9493{
Hristo Venev75b28af2019-08-26 17:23:46 +00009494 struct io_rings *rings;
9495 size_t size, sq_array_offset;
Jens Axboe2b188cc2019-01-07 10:46:33 -07009496
Jens Axboebd740482020-08-05 12:58:23 -06009497 /* make sure these are sane, as we already accounted them */
9498 ctx->sq_entries = p->sq_entries;
9499 ctx->cq_entries = p->cq_entries;
9500
Hristo Venev75b28af2019-08-26 17:23:46 +00009501 size = rings_size(p->sq_entries, p->cq_entries, &sq_array_offset);
9502 if (size == SIZE_MAX)
9503 return -EOVERFLOW;
9504
9505 rings = io_mem_alloc(size);
9506 if (!rings)
Jens Axboe2b188cc2019-01-07 10:46:33 -07009507 return -ENOMEM;
9508
Hristo Venev75b28af2019-08-26 17:23:46 +00009509 ctx->rings = rings;
9510 ctx->sq_array = (u32 *)((char *)rings + sq_array_offset);
9511 rings->sq_ring_mask = p->sq_entries - 1;
9512 rings->cq_ring_mask = p->cq_entries - 1;
9513 rings->sq_ring_entries = p->sq_entries;
9514 rings->cq_ring_entries = p->cq_entries;
9515 ctx->sq_mask = rings->sq_ring_mask;
9516 ctx->cq_mask = rings->cq_ring_mask;
Jens Axboe2b188cc2019-01-07 10:46:33 -07009517
9518 size = array_size(sizeof(struct io_uring_sqe), p->sq_entries);
Jens Axboeeb065d32019-11-20 09:26:29 -07009519 if (size == SIZE_MAX) {
9520 io_mem_free(ctx->rings);
9521 ctx->rings = NULL;
Jens Axboe2b188cc2019-01-07 10:46:33 -07009522 return -EOVERFLOW;
Jens Axboeeb065d32019-11-20 09:26:29 -07009523 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07009524
9525 ctx->sq_sqes = io_mem_alloc(size);
Jens Axboeeb065d32019-11-20 09:26:29 -07009526 if (!ctx->sq_sqes) {
9527 io_mem_free(ctx->rings);
9528 ctx->rings = NULL;
Jens Axboe2b188cc2019-01-07 10:46:33 -07009529 return -ENOMEM;
Jens Axboeeb065d32019-11-20 09:26:29 -07009530 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07009531
Jens Axboe2b188cc2019-01-07 10:46:33 -07009532 return 0;
9533}
9534
Pavel Begunkov9faadcc2020-12-21 18:34:05 +00009535static int io_uring_install_fd(struct io_ring_ctx *ctx, struct file *file)
9536{
9537 int ret, fd;
9538
9539 fd = get_unused_fd_flags(O_RDWR | O_CLOEXEC);
9540 if (fd < 0)
9541 return fd;
9542
Pavel Begunkovbaf186c2021-03-06 11:02:15 +00009543 ret = io_uring_add_task_file(ctx);
Pavel Begunkov9faadcc2020-12-21 18:34:05 +00009544 if (ret) {
9545 put_unused_fd(fd);
9546 return ret;
9547 }
9548 fd_install(fd, file);
9549 return fd;
9550}
9551
Jens Axboe2b188cc2019-01-07 10:46:33 -07009552/*
9553 * Allocate an anonymous fd, this is what constitutes the application
9554 * visible backing of an io_uring instance. The application mmaps this
9555 * fd to gain access to the SQ/CQ ring details. If UNIX sockets are enabled,
9556 * we have to tie this fd to a socket for file garbage collection purposes.
9557 */
Pavel Begunkov9faadcc2020-12-21 18:34:05 +00009558static struct file *io_uring_get_file(struct io_ring_ctx *ctx)
Jens Axboe2b188cc2019-01-07 10:46:33 -07009559{
9560 struct file *file;
Jens Axboe2b188cc2019-01-07 10:46:33 -07009561#if defined(CONFIG_UNIX)
Pavel Begunkov9faadcc2020-12-21 18:34:05 +00009562 int ret;
9563
Jens Axboe2b188cc2019-01-07 10:46:33 -07009564 ret = sock_create_kern(&init_net, PF_UNIX, SOCK_RAW, IPPROTO_IP,
9565 &ctx->ring_sock);
9566 if (ret)
Pavel Begunkov9faadcc2020-12-21 18:34:05 +00009567 return ERR_PTR(ret);
Jens Axboe2b188cc2019-01-07 10:46:33 -07009568#endif
9569
Jens Axboe2b188cc2019-01-07 10:46:33 -07009570 file = anon_inode_getfile("[io_uring]", &io_uring_fops, ctx,
9571 O_RDWR | O_CLOEXEC);
Pavel Begunkov9faadcc2020-12-21 18:34:05 +00009572#if defined(CONFIG_UNIX)
Jens Axboe2b188cc2019-01-07 10:46:33 -07009573 if (IS_ERR(file)) {
Pavel Begunkov9faadcc2020-12-21 18:34:05 +00009574 sock_release(ctx->ring_sock);
9575 ctx->ring_sock = NULL;
9576 } else {
9577 ctx->ring_sock->file = file;
Jens Axboe2b188cc2019-01-07 10:46:33 -07009578 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07009579#endif
Pavel Begunkov9faadcc2020-12-21 18:34:05 +00009580 return file;
Jens Axboe2b188cc2019-01-07 10:46:33 -07009581}
9582
Xiaoguang Wang7f136572020-05-05 16:28:53 +08009583static int io_uring_create(unsigned entries, struct io_uring_params *p,
9584 struct io_uring_params __user *params)
Jens Axboe2b188cc2019-01-07 10:46:33 -07009585{
Jens Axboe2b188cc2019-01-07 10:46:33 -07009586 struct io_ring_ctx *ctx;
Pavel Begunkov9faadcc2020-12-21 18:34:05 +00009587 struct file *file;
Jens Axboe2b188cc2019-01-07 10:46:33 -07009588 int ret;
9589
Jens Axboe8110c1a2019-12-28 15:39:54 -07009590 if (!entries)
Jens Axboe2b188cc2019-01-07 10:46:33 -07009591 return -EINVAL;
Jens Axboe8110c1a2019-12-28 15:39:54 -07009592 if (entries > IORING_MAX_ENTRIES) {
9593 if (!(p->flags & IORING_SETUP_CLAMP))
9594 return -EINVAL;
9595 entries = IORING_MAX_ENTRIES;
9596 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07009597
9598 /*
9599 * Use twice as many entries for the CQ ring. It's possible for the
9600 * application to drive a higher depth than the size of the SQ ring,
9601 * since the sqes are only used at submission time. This allows for
Jens Axboe33a107f2019-10-04 12:10:03 -06009602 * some flexibility in overcommitting a bit. If the application has
9603 * set IORING_SETUP_CQSIZE, it will have passed in the desired number
9604 * of CQ ring entries manually.
Jens Axboe2b188cc2019-01-07 10:46:33 -07009605 */
9606 p->sq_entries = roundup_pow_of_two(entries);
Jens Axboe33a107f2019-10-04 12:10:03 -06009607 if (p->flags & IORING_SETUP_CQSIZE) {
9608 /*
9609 * If IORING_SETUP_CQSIZE is set, we do the same roundup
9610 * to a power-of-two, if it isn't already. We do NOT impose
9611 * any cq vs sq ring sizing.
9612 */
Joseph Qieb2667b32020-11-24 15:03:03 +08009613 if (!p->cq_entries)
Jens Axboe33a107f2019-10-04 12:10:03 -06009614 return -EINVAL;
Jens Axboe8110c1a2019-12-28 15:39:54 -07009615 if (p->cq_entries > IORING_MAX_CQ_ENTRIES) {
9616 if (!(p->flags & IORING_SETUP_CLAMP))
9617 return -EINVAL;
9618 p->cq_entries = IORING_MAX_CQ_ENTRIES;
9619 }
Joseph Qieb2667b32020-11-24 15:03:03 +08009620 p->cq_entries = roundup_pow_of_two(p->cq_entries);
9621 if (p->cq_entries < p->sq_entries)
9622 return -EINVAL;
Jens Axboe33a107f2019-10-04 12:10:03 -06009623 } else {
9624 p->cq_entries = 2 * p->sq_entries;
9625 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07009626
Jens Axboe2b188cc2019-01-07 10:46:33 -07009627 ctx = io_ring_ctx_alloc(p);
Jens Axboe62e398b2021-02-21 16:19:37 -07009628 if (!ctx)
Jens Axboe2b188cc2019-01-07 10:46:33 -07009629 return -ENOMEM;
Jens Axboe2b188cc2019-01-07 10:46:33 -07009630 ctx->compat = in_compat_syscall();
Jens Axboe62e398b2021-02-21 16:19:37 -07009631 if (!capable(CAP_IPC_LOCK))
9632 ctx->user = get_uid(current_user());
Jens Axboe2aede0e2020-09-14 10:45:53 -06009633
9634 /*
9635 * This is just grabbed for accounting purposes. When a process exits,
9636 * the mm is exited and dropped before the files, hence we need to hang
9637 * on to this mm purely for the purposes of being able to unaccount
9638 * memory (locked/pinned vm). It's not used for anything else.
9639 */
Jens Axboe6b7898e2020-08-25 07:58:00 -06009640 mmgrab(current->mm);
Jens Axboe2aede0e2020-09-14 10:45:53 -06009641 ctx->mm_account = current->mm;
Jens Axboe6b7898e2020-08-25 07:58:00 -06009642
Jens Axboe2b188cc2019-01-07 10:46:33 -07009643 ret = io_allocate_scq_urings(ctx, p);
9644 if (ret)
9645 goto err;
9646
Stefano Garzarella7e84e1c2020-08-27 16:58:31 +02009647 ret = io_sq_offload_create(ctx, p);
Jens Axboe2b188cc2019-01-07 10:46:33 -07009648 if (ret)
9649 goto err;
Pavel Begunkoveae071c2021-04-25 14:32:24 +01009650 /* always set a rsrc node */
Pavel Begunkov47b228c2021-04-29 11:46:48 +01009651 ret = io_rsrc_node_switch_start(ctx);
9652 if (ret)
9653 goto err;
Pavel Begunkoveae071c2021-04-25 14:32:24 +01009654 io_rsrc_node_switch(ctx, NULL);
Jens Axboe2b188cc2019-01-07 10:46:33 -07009655
Jens Axboe2b188cc2019-01-07 10:46:33 -07009656 memset(&p->sq_off, 0, sizeof(p->sq_off));
Hristo Venev75b28af2019-08-26 17:23:46 +00009657 p->sq_off.head = offsetof(struct io_rings, sq.head);
9658 p->sq_off.tail = offsetof(struct io_rings, sq.tail);
9659 p->sq_off.ring_mask = offsetof(struct io_rings, sq_ring_mask);
9660 p->sq_off.ring_entries = offsetof(struct io_rings, sq_ring_entries);
9661 p->sq_off.flags = offsetof(struct io_rings, sq_flags);
9662 p->sq_off.dropped = offsetof(struct io_rings, sq_dropped);
9663 p->sq_off.array = (char *)ctx->sq_array - (char *)ctx->rings;
Jens Axboe2b188cc2019-01-07 10:46:33 -07009664
9665 memset(&p->cq_off, 0, sizeof(p->cq_off));
Hristo Venev75b28af2019-08-26 17:23:46 +00009666 p->cq_off.head = offsetof(struct io_rings, cq.head);
9667 p->cq_off.tail = offsetof(struct io_rings, cq.tail);
9668 p->cq_off.ring_mask = offsetof(struct io_rings, cq_ring_mask);
9669 p->cq_off.ring_entries = offsetof(struct io_rings, cq_ring_entries);
9670 p->cq_off.overflow = offsetof(struct io_rings, cq_overflow);
9671 p->cq_off.cqes = offsetof(struct io_rings, cqes);
Stefano Garzarella0d9b5b32020-05-15 18:38:04 +02009672 p->cq_off.flags = offsetof(struct io_rings, cq_flags);
Jens Axboeac90f242019-09-06 10:26:21 -06009673
Xiaoguang Wang7f136572020-05-05 16:28:53 +08009674 p->features = IORING_FEAT_SINGLE_MMAP | IORING_FEAT_NODROP |
9675 IORING_FEAT_SUBMIT_STABLE | IORING_FEAT_RW_CUR_POS |
Jiufei Xue5769a352020-06-17 17:53:55 +08009676 IORING_FEAT_CUR_PERSONALITY | IORING_FEAT_FAST_POLL |
Hao Xuc73ebb62020-11-03 10:54:37 +08009677 IORING_FEAT_POLL_32BITS | IORING_FEAT_SQPOLL_NONFIXED |
Pavel Begunkov96905572021-06-10 16:37:38 +01009678 IORING_FEAT_EXT_ARG | IORING_FEAT_NATIVE_WORKERS |
9679 IORING_FEAT_RSRC_TAGS;
Xiaoguang Wang7f136572020-05-05 16:28:53 +08009680
9681 if (copy_to_user(params, p, sizeof(*p))) {
9682 ret = -EFAULT;
9683 goto err;
9684 }
Jens Axboed1719f72020-07-30 13:43:53 -06009685
Pavel Begunkov9faadcc2020-12-21 18:34:05 +00009686 file = io_uring_get_file(ctx);
9687 if (IS_ERR(file)) {
9688 ret = PTR_ERR(file);
9689 goto err;
9690 }
9691
Jens Axboed1719f72020-07-30 13:43:53 -06009692 /*
Jens Axboe044c1ab2019-10-28 09:15:33 -06009693 * Install ring fd as the very last thing, so we don't risk someone
9694 * having closed it before we finish setup
9695 */
Pavel Begunkov9faadcc2020-12-21 18:34:05 +00009696 ret = io_uring_install_fd(ctx, file);
9697 if (ret < 0) {
9698 /* fput will clean it up */
9699 fput(file);
9700 return ret;
9701 }
Jens Axboe044c1ab2019-10-28 09:15:33 -06009702
Dmitrii Dolgovc826bd72019-10-15 19:02:01 +02009703 trace_io_uring_create(ret, ctx, p->sq_entries, p->cq_entries, p->flags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07009704 return ret;
9705err:
9706 io_ring_ctx_wait_and_kill(ctx);
9707 return ret;
9708}
9709
9710/*
9711 * Sets up an aio uring context, and returns the fd. Applications asks for a
9712 * ring size, we return the actual sq/cq ring sizes (among other things) in the
9713 * params structure passed in.
9714 */
9715static long io_uring_setup(u32 entries, struct io_uring_params __user *params)
9716{
9717 struct io_uring_params p;
Jens Axboe2b188cc2019-01-07 10:46:33 -07009718 int i;
9719
9720 if (copy_from_user(&p, params, sizeof(p)))
9721 return -EFAULT;
9722 for (i = 0; i < ARRAY_SIZE(p.resv); i++) {
9723 if (p.resv[i])
9724 return -EINVAL;
9725 }
9726
Jens Axboe6c271ce2019-01-10 11:22:30 -07009727 if (p.flags & ~(IORING_SETUP_IOPOLL | IORING_SETUP_SQPOLL |
Jens Axboe8110c1a2019-12-28 15:39:54 -07009728 IORING_SETUP_SQ_AFF | IORING_SETUP_CQSIZE |
Stefano Garzarella7e84e1c2020-08-27 16:58:31 +02009729 IORING_SETUP_CLAMP | IORING_SETUP_ATTACH_WQ |
9730 IORING_SETUP_R_DISABLED))
Jens Axboe2b188cc2019-01-07 10:46:33 -07009731 return -EINVAL;
9732
Xiaoguang Wang7f136572020-05-05 16:28:53 +08009733 return io_uring_create(entries, &p, params);
Jens Axboe2b188cc2019-01-07 10:46:33 -07009734}
9735
9736SYSCALL_DEFINE2(io_uring_setup, u32, entries,
9737 struct io_uring_params __user *, params)
9738{
9739 return io_uring_setup(entries, params);
9740}
9741
Jens Axboe66f4af92020-01-16 15:36:52 -07009742static int io_probe(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args)
9743{
9744 struct io_uring_probe *p;
9745 size_t size;
9746 int i, ret;
9747
9748 size = struct_size(p, ops, nr_args);
9749 if (size == SIZE_MAX)
9750 return -EOVERFLOW;
9751 p = kzalloc(size, GFP_KERNEL);
9752 if (!p)
9753 return -ENOMEM;
9754
9755 ret = -EFAULT;
9756 if (copy_from_user(p, arg, size))
9757 goto out;
9758 ret = -EINVAL;
9759 if (memchr_inv(p, 0, size))
9760 goto out;
9761
9762 p->last_op = IORING_OP_LAST - 1;
9763 if (nr_args > IORING_OP_LAST)
9764 nr_args = IORING_OP_LAST;
9765
9766 for (i = 0; i < nr_args; i++) {
9767 p->ops[i].op = i;
9768 if (!io_op_defs[i].not_supported)
9769 p->ops[i].flags = IO_URING_OP_SUPPORTED;
9770 }
9771 p->ops_len = i;
9772
9773 ret = 0;
9774 if (copy_to_user(arg, p, size))
9775 ret = -EFAULT;
9776out:
9777 kfree(p);
9778 return ret;
9779}
9780
Jens Axboe071698e2020-01-28 10:04:42 -07009781static int io_register_personality(struct io_ring_ctx *ctx)
9782{
Jens Axboe4379bf82021-02-15 13:40:22 -07009783 const struct cred *creds;
Matthew Wilcox (Oracle)61cf9372021-03-08 14:16:16 +00009784 u32 id;
Jens Axboe1e6fa522020-10-15 08:46:24 -06009785 int ret;
Jens Axboe071698e2020-01-28 10:04:42 -07009786
Jens Axboe4379bf82021-02-15 13:40:22 -07009787 creds = get_current_cred();
Jens Axboe1e6fa522020-10-15 08:46:24 -06009788
Matthew Wilcox (Oracle)61cf9372021-03-08 14:16:16 +00009789 ret = xa_alloc_cyclic(&ctx->personalities, &id, (void *)creds,
9790 XA_LIMIT(0, USHRT_MAX), &ctx->pers_next, GFP_KERNEL);
9791 if (!ret)
9792 return id;
9793 put_cred(creds);
Jens Axboe1e6fa522020-10-15 08:46:24 -06009794 return ret;
Jens Axboe071698e2020-01-28 10:04:42 -07009795}
9796
Stefano Garzarella21b55db2020-08-27 16:58:30 +02009797static int io_register_restrictions(struct io_ring_ctx *ctx, void __user *arg,
9798 unsigned int nr_args)
9799{
9800 struct io_uring_restriction *res;
9801 size_t size;
9802 int i, ret;
9803
Stefano Garzarella7e84e1c2020-08-27 16:58:31 +02009804 /* Restrictions allowed only if rings started disabled */
9805 if (!(ctx->flags & IORING_SETUP_R_DISABLED))
9806 return -EBADFD;
9807
Stefano Garzarella21b55db2020-08-27 16:58:30 +02009808 /* We allow only a single restrictions registration */
Stefano Garzarella7e84e1c2020-08-27 16:58:31 +02009809 if (ctx->restrictions.registered)
Stefano Garzarella21b55db2020-08-27 16:58:30 +02009810 return -EBUSY;
9811
9812 if (!arg || nr_args > IORING_MAX_RESTRICTIONS)
9813 return -EINVAL;
9814
9815 size = array_size(nr_args, sizeof(*res));
9816 if (size == SIZE_MAX)
9817 return -EOVERFLOW;
9818
9819 res = memdup_user(arg, size);
9820 if (IS_ERR(res))
9821 return PTR_ERR(res);
9822
9823 ret = 0;
9824
9825 for (i = 0; i < nr_args; i++) {
9826 switch (res[i].opcode) {
9827 case IORING_RESTRICTION_REGISTER_OP:
9828 if (res[i].register_op >= IORING_REGISTER_LAST) {
9829 ret = -EINVAL;
9830 goto out;
9831 }
9832
9833 __set_bit(res[i].register_op,
9834 ctx->restrictions.register_op);
9835 break;
9836 case IORING_RESTRICTION_SQE_OP:
9837 if (res[i].sqe_op >= IORING_OP_LAST) {
9838 ret = -EINVAL;
9839 goto out;
9840 }
9841
9842 __set_bit(res[i].sqe_op, ctx->restrictions.sqe_op);
9843 break;
9844 case IORING_RESTRICTION_SQE_FLAGS_ALLOWED:
9845 ctx->restrictions.sqe_flags_allowed = res[i].sqe_flags;
9846 break;
9847 case IORING_RESTRICTION_SQE_FLAGS_REQUIRED:
9848 ctx->restrictions.sqe_flags_required = res[i].sqe_flags;
9849 break;
9850 default:
9851 ret = -EINVAL;
9852 goto out;
9853 }
9854 }
9855
9856out:
9857 /* Reset all restrictions if an error happened */
9858 if (ret != 0)
9859 memset(&ctx->restrictions, 0, sizeof(ctx->restrictions));
9860 else
Stefano Garzarella7e84e1c2020-08-27 16:58:31 +02009861 ctx->restrictions.registered = true;
Stefano Garzarella21b55db2020-08-27 16:58:30 +02009862
9863 kfree(res);
9864 return ret;
9865}
9866
Stefano Garzarella7e84e1c2020-08-27 16:58:31 +02009867static int io_register_enable_rings(struct io_ring_ctx *ctx)
9868{
9869 if (!(ctx->flags & IORING_SETUP_R_DISABLED))
9870 return -EBADFD;
9871
9872 if (ctx->restrictions.registered)
9873 ctx->restricted = 1;
9874
Pavel Begunkov0298ef92021-03-08 13:20:57 +00009875 ctx->flags &= ~IORING_SETUP_R_DISABLED;
9876 if (ctx->sq_data && wq_has_sleeper(&ctx->sq_data->wait))
9877 wake_up(&ctx->sq_data->wait);
Stefano Garzarella7e84e1c2020-08-27 16:58:31 +02009878 return 0;
9879}
9880
Pavel Begunkovfdecb662021-04-25 14:32:20 +01009881static int __io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned type,
Pavel Begunkovc3bdad02021-04-25 14:32:22 +01009882 struct io_uring_rsrc_update2 *up,
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +01009883 unsigned nr_args)
9884{
9885 __u32 tmp;
9886 int err;
9887
Pavel Begunkovc3bdad02021-04-25 14:32:22 +01009888 if (up->resv)
9889 return -EINVAL;
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +01009890 if (check_add_overflow(up->offset, nr_args, &tmp))
9891 return -EOVERFLOW;
9892 err = io_rsrc_node_switch_start(ctx);
9893 if (err)
9894 return err;
9895
Pavel Begunkovfdecb662021-04-25 14:32:20 +01009896 switch (type) {
9897 case IORING_RSRC_FILE:
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +01009898 return __io_sqe_files_update(ctx, up, nr_args);
Pavel Begunkov634d00d2021-04-25 14:32:26 +01009899 case IORING_RSRC_BUFFER:
9900 return __io_sqe_buffers_update(ctx, up, nr_args);
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +01009901 }
9902 return -EINVAL;
9903}
9904
Pavel Begunkovc3bdad02021-04-25 14:32:22 +01009905static int io_register_files_update(struct io_ring_ctx *ctx, void __user *arg,
9906 unsigned nr_args)
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +01009907{
Pavel Begunkovc3bdad02021-04-25 14:32:22 +01009908 struct io_uring_rsrc_update2 up;
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +01009909
9910 if (!nr_args)
9911 return -EINVAL;
Pavel Begunkovc3bdad02021-04-25 14:32:22 +01009912 memset(&up, 0, sizeof(up));
9913 if (copy_from_user(&up, arg, sizeof(struct io_uring_rsrc_update)))
9914 return -EFAULT;
9915 return __io_register_rsrc_update(ctx, IORING_RSRC_FILE, &up, nr_args);
9916}
9917
9918static int io_register_rsrc_update(struct io_ring_ctx *ctx, void __user *arg,
Pavel Begunkov992da012021-06-10 16:37:37 +01009919 unsigned size, unsigned type)
Pavel Begunkovc3bdad02021-04-25 14:32:22 +01009920{
9921 struct io_uring_rsrc_update2 up;
9922
9923 if (size != sizeof(up))
9924 return -EINVAL;
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +01009925 if (copy_from_user(&up, arg, sizeof(up)))
9926 return -EFAULT;
Pavel Begunkov992da012021-06-10 16:37:37 +01009927 if (!up.nr || up.resv)
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +01009928 return -EINVAL;
Pavel Begunkov992da012021-06-10 16:37:37 +01009929 return __io_register_rsrc_update(ctx, type, &up, up.nr);
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +01009930}
9931
Pavel Begunkov792e3582021-04-25 14:32:21 +01009932static int io_register_rsrc(struct io_ring_ctx *ctx, void __user *arg,
Pavel Begunkov992da012021-06-10 16:37:37 +01009933 unsigned int size, unsigned int type)
Pavel Begunkov792e3582021-04-25 14:32:21 +01009934{
9935 struct io_uring_rsrc_register rr;
9936
9937 /* keep it extendible */
9938 if (size != sizeof(rr))
9939 return -EINVAL;
9940
9941 memset(&rr, 0, sizeof(rr));
9942 if (copy_from_user(&rr, arg, size))
9943 return -EFAULT;
Pavel Begunkov992da012021-06-10 16:37:37 +01009944 if (!rr.nr || rr.resv || rr.resv2)
Pavel Begunkov792e3582021-04-25 14:32:21 +01009945 return -EINVAL;
9946
Pavel Begunkov992da012021-06-10 16:37:37 +01009947 switch (type) {
Pavel Begunkov792e3582021-04-25 14:32:21 +01009948 case IORING_RSRC_FILE:
9949 return io_sqe_files_register(ctx, u64_to_user_ptr(rr.data),
9950 rr.nr, u64_to_user_ptr(rr.tags));
Pavel Begunkov634d00d2021-04-25 14:32:26 +01009951 case IORING_RSRC_BUFFER:
9952 return io_sqe_buffers_register(ctx, u64_to_user_ptr(rr.data),
9953 rr.nr, u64_to_user_ptr(rr.tags));
Pavel Begunkov792e3582021-04-25 14:32:21 +01009954 }
9955 return -EINVAL;
9956}
9957
Jens Axboe071698e2020-01-28 10:04:42 -07009958static bool io_register_op_must_quiesce(int op)
9959{
9960 switch (op) {
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01009961 case IORING_REGISTER_BUFFERS:
9962 case IORING_UNREGISTER_BUFFERS:
Pavel Begunkovf4f7d212021-04-01 15:44:02 +01009963 case IORING_REGISTER_FILES:
Jens Axboe071698e2020-01-28 10:04:42 -07009964 case IORING_UNREGISTER_FILES:
9965 case IORING_REGISTER_FILES_UPDATE:
9966 case IORING_REGISTER_PROBE:
9967 case IORING_REGISTER_PERSONALITY:
9968 case IORING_UNREGISTER_PERSONALITY:
Pavel Begunkov992da012021-06-10 16:37:37 +01009969 case IORING_REGISTER_FILES2:
9970 case IORING_REGISTER_FILES_UPDATE2:
9971 case IORING_REGISTER_BUFFERS2:
9972 case IORING_REGISTER_BUFFERS_UPDATE:
Jens Axboe071698e2020-01-28 10:04:42 -07009973 return false;
9974 default:
9975 return true;
9976 }
9977}
9978
Jens Axboeedafcce2019-01-09 09:16:05 -07009979static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,
9980 void __user *arg, unsigned nr_args)
Jens Axboeb19062a2019-04-15 10:49:38 -06009981 __releases(ctx->uring_lock)
9982 __acquires(ctx->uring_lock)
Jens Axboeedafcce2019-01-09 09:16:05 -07009983{
9984 int ret;
9985
Jens Axboe35fa71a2019-04-22 10:23:23 -06009986 /*
9987 * We're inside the ring mutex, if the ref is already dying, then
9988 * someone else killed the ctx or is already going through
9989 * io_uring_register().
9990 */
9991 if (percpu_ref_is_dying(&ctx->refs))
9992 return -ENXIO;
9993
Pavel Begunkov75c40212021-04-15 13:07:40 +01009994 if (ctx->restricted) {
9995 if (opcode >= IORING_REGISTER_LAST)
9996 return -EINVAL;
9997 opcode = array_index_nospec(opcode, IORING_REGISTER_LAST);
9998 if (!test_bit(opcode, ctx->restrictions.register_op))
9999 return -EACCES;
10000 }
10001
Jens Axboe071698e2020-01-28 10:04:42 -070010002 if (io_register_op_must_quiesce(opcode)) {
Jens Axboe05f3fb32019-12-09 11:22:50 -070010003 percpu_ref_kill(&ctx->refs);
Jens Axboeb19062a2019-04-15 10:49:38 -060010004
Jens Axboe05f3fb32019-12-09 11:22:50 -070010005 /*
10006 * Drop uring mutex before waiting for references to exit. If
10007 * another thread is currently inside io_uring_enter() it might
10008 * need to grab the uring_lock to make progress. If we hold it
10009 * here across the drain wait, then we can deadlock. It's safe
10010 * to drop the mutex here, since no new references will come in
10011 * after we've killed the percpu ref.
10012 */
10013 mutex_unlock(&ctx->uring_lock);
Jens Axboeaf9c1a42020-09-24 13:32:18 -060010014 do {
10015 ret = wait_for_completion_interruptible(&ctx->ref_comp);
10016 if (!ret)
10017 break;
Jens Axboeed6930c2020-10-08 19:09:46 -060010018 ret = io_run_task_work_sig();
10019 if (ret < 0)
10020 break;
Jens Axboeaf9c1a42020-09-24 13:32:18 -060010021 } while (1);
Jens Axboe05f3fb32019-12-09 11:22:50 -070010022 mutex_lock(&ctx->uring_lock);
Jens Axboeaf9c1a42020-09-24 13:32:18 -060010023
Jens Axboec1503682020-01-08 08:26:07 -070010024 if (ret) {
Pavel Begunkovf70865d2021-04-11 01:46:40 +010010025 io_refs_resurrect(&ctx->refs, &ctx->ref_comp);
10026 return ret;
Jens Axboec1503682020-01-08 08:26:07 -070010027 }
Jens Axboe05f3fb32019-12-09 11:22:50 -070010028 }
Jens Axboeedafcce2019-01-09 09:16:05 -070010029
10030 switch (opcode) {
10031 case IORING_REGISTER_BUFFERS:
Pavel Begunkov634d00d2021-04-25 14:32:26 +010010032 ret = io_sqe_buffers_register(ctx, arg, nr_args, NULL);
Jens Axboeedafcce2019-01-09 09:16:05 -070010033 break;
10034 case IORING_UNREGISTER_BUFFERS:
10035 ret = -EINVAL;
10036 if (arg || nr_args)
10037 break;
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -080010038 ret = io_sqe_buffers_unregister(ctx);
Jens Axboeedafcce2019-01-09 09:16:05 -070010039 break;
Jens Axboe6b063142019-01-10 22:13:58 -070010040 case IORING_REGISTER_FILES:
Pavel Begunkov792e3582021-04-25 14:32:21 +010010041 ret = io_sqe_files_register(ctx, arg, nr_args, NULL);
Jens Axboe6b063142019-01-10 22:13:58 -070010042 break;
10043 case IORING_UNREGISTER_FILES:
10044 ret = -EINVAL;
10045 if (arg || nr_args)
10046 break;
10047 ret = io_sqe_files_unregister(ctx);
10048 break;
Jens Axboec3a31e62019-10-03 13:59:56 -060010049 case IORING_REGISTER_FILES_UPDATE:
Pavel Begunkovc3bdad02021-04-25 14:32:22 +010010050 ret = io_register_files_update(ctx, arg, nr_args);
Jens Axboec3a31e62019-10-03 13:59:56 -060010051 break;
Jens Axboe9b402842019-04-11 11:45:41 -060010052 case IORING_REGISTER_EVENTFD:
Jens Axboef2842ab2020-01-08 11:04:00 -070010053 case IORING_REGISTER_EVENTFD_ASYNC:
Jens Axboe9b402842019-04-11 11:45:41 -060010054 ret = -EINVAL;
10055 if (nr_args != 1)
10056 break;
10057 ret = io_eventfd_register(ctx, arg);
Jens Axboef2842ab2020-01-08 11:04:00 -070010058 if (ret)
10059 break;
10060 if (opcode == IORING_REGISTER_EVENTFD_ASYNC)
10061 ctx->eventfd_async = 1;
10062 else
10063 ctx->eventfd_async = 0;
Jens Axboe9b402842019-04-11 11:45:41 -060010064 break;
10065 case IORING_UNREGISTER_EVENTFD:
10066 ret = -EINVAL;
10067 if (arg || nr_args)
10068 break;
10069 ret = io_eventfd_unregister(ctx);
10070 break;
Jens Axboe66f4af92020-01-16 15:36:52 -070010071 case IORING_REGISTER_PROBE:
10072 ret = -EINVAL;
10073 if (!arg || nr_args > 256)
10074 break;
10075 ret = io_probe(ctx, arg, nr_args);
10076 break;
Jens Axboe071698e2020-01-28 10:04:42 -070010077 case IORING_REGISTER_PERSONALITY:
10078 ret = -EINVAL;
10079 if (arg || nr_args)
10080 break;
10081 ret = io_register_personality(ctx);
10082 break;
10083 case IORING_UNREGISTER_PERSONALITY:
10084 ret = -EINVAL;
10085 if (arg)
10086 break;
10087 ret = io_unregister_personality(ctx, nr_args);
10088 break;
Stefano Garzarella7e84e1c2020-08-27 16:58:31 +020010089 case IORING_REGISTER_ENABLE_RINGS:
10090 ret = -EINVAL;
10091 if (arg || nr_args)
10092 break;
10093 ret = io_register_enable_rings(ctx);
10094 break;
Stefano Garzarella21b55db2020-08-27 16:58:30 +020010095 case IORING_REGISTER_RESTRICTIONS:
10096 ret = io_register_restrictions(ctx, arg, nr_args);
10097 break;
Pavel Begunkov992da012021-06-10 16:37:37 +010010098 case IORING_REGISTER_FILES2:
10099 ret = io_register_rsrc(ctx, arg, nr_args, IORING_RSRC_FILE);
Pavel Begunkov792e3582021-04-25 14:32:21 +010010100 break;
Pavel Begunkov992da012021-06-10 16:37:37 +010010101 case IORING_REGISTER_FILES_UPDATE2:
10102 ret = io_register_rsrc_update(ctx, arg, nr_args,
10103 IORING_RSRC_FILE);
10104 break;
10105 case IORING_REGISTER_BUFFERS2:
10106 ret = io_register_rsrc(ctx, arg, nr_args, IORING_RSRC_BUFFER);
10107 break;
10108 case IORING_REGISTER_BUFFERS_UPDATE:
10109 ret = io_register_rsrc_update(ctx, arg, nr_args,
10110 IORING_RSRC_BUFFER);
Pavel Begunkovc3bdad02021-04-25 14:32:22 +010010111 break;
Jens Axboeedafcce2019-01-09 09:16:05 -070010112 default:
10113 ret = -EINVAL;
10114 break;
10115 }
10116
Jens Axboe071698e2020-01-28 10:04:42 -070010117 if (io_register_op_must_quiesce(opcode)) {
Jens Axboe05f3fb32019-12-09 11:22:50 -070010118 /* bring the ctx back to life */
Jens Axboe05f3fb32019-12-09 11:22:50 -070010119 percpu_ref_reinit(&ctx->refs);
Jens Axboe0f158b42020-05-14 17:18:39 -060010120 reinit_completion(&ctx->ref_comp);
Jens Axboe05f3fb32019-12-09 11:22:50 -070010121 }
Jens Axboeedafcce2019-01-09 09:16:05 -070010122 return ret;
10123}
10124
10125SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode,
10126 void __user *, arg, unsigned int, nr_args)
10127{
10128 struct io_ring_ctx *ctx;
10129 long ret = -EBADF;
10130 struct fd f;
10131
10132 f = fdget(fd);
10133 if (!f.file)
10134 return -EBADF;
10135
10136 ret = -EOPNOTSUPP;
10137 if (f.file->f_op != &io_uring_fops)
10138 goto out_fput;
10139
10140 ctx = f.file->private_data;
10141
Pavel Begunkovb6c23dd2021-02-20 15:17:18 +000010142 io_run_task_work();
10143
Jens Axboeedafcce2019-01-09 09:16:05 -070010144 mutex_lock(&ctx->uring_lock);
10145 ret = __io_uring_register(ctx, opcode, arg, nr_args);
10146 mutex_unlock(&ctx->uring_lock);
Dmitrii Dolgovc826bd72019-10-15 19:02:01 +020010147 trace_io_uring_register(ctx, opcode, ctx->nr_user_files, ctx->nr_user_bufs,
10148 ctx->cq_ev_fd != NULL, ret);
Jens Axboeedafcce2019-01-09 09:16:05 -070010149out_fput:
10150 fdput(f);
10151 return ret;
10152}
10153
Jens Axboe2b188cc2019-01-07 10:46:33 -070010154static int __init io_uring_init(void)
10155{
Stefan Metzmacherd7f62e82020-01-29 14:39:41 +010010156#define __BUILD_BUG_VERIFY_ELEMENT(stype, eoffset, etype, ename) do { \
10157 BUILD_BUG_ON(offsetof(stype, ename) != eoffset); \
10158 BUILD_BUG_ON(sizeof(etype) != sizeof_field(stype, ename)); \
10159} while (0)
10160
10161#define BUILD_BUG_SQE_ELEM(eoffset, etype, ename) \
10162 __BUILD_BUG_VERIFY_ELEMENT(struct io_uring_sqe, eoffset, etype, ename)
10163 BUILD_BUG_ON(sizeof(struct io_uring_sqe) != 64);
10164 BUILD_BUG_SQE_ELEM(0, __u8, opcode);
10165 BUILD_BUG_SQE_ELEM(1, __u8, flags);
10166 BUILD_BUG_SQE_ELEM(2, __u16, ioprio);
10167 BUILD_BUG_SQE_ELEM(4, __s32, fd);
10168 BUILD_BUG_SQE_ELEM(8, __u64, off);
10169 BUILD_BUG_SQE_ELEM(8, __u64, addr2);
10170 BUILD_BUG_SQE_ELEM(16, __u64, addr);
Pavel Begunkov7d67af22020-02-24 11:32:45 +030010171 BUILD_BUG_SQE_ELEM(16, __u64, splice_off_in);
Stefan Metzmacherd7f62e82020-01-29 14:39:41 +010010172 BUILD_BUG_SQE_ELEM(24, __u32, len);
10173 BUILD_BUG_SQE_ELEM(28, __kernel_rwf_t, rw_flags);
10174 BUILD_BUG_SQE_ELEM(28, /* compat */ int, rw_flags);
10175 BUILD_BUG_SQE_ELEM(28, /* compat */ __u32, rw_flags);
10176 BUILD_BUG_SQE_ELEM(28, __u32, fsync_flags);
Jiufei Xue5769a352020-06-17 17:53:55 +080010177 BUILD_BUG_SQE_ELEM(28, /* compat */ __u16, poll_events);
10178 BUILD_BUG_SQE_ELEM(28, __u32, poll32_events);
Stefan Metzmacherd7f62e82020-01-29 14:39:41 +010010179 BUILD_BUG_SQE_ELEM(28, __u32, sync_range_flags);
10180 BUILD_BUG_SQE_ELEM(28, __u32, msg_flags);
10181 BUILD_BUG_SQE_ELEM(28, __u32, timeout_flags);
10182 BUILD_BUG_SQE_ELEM(28, __u32, accept_flags);
10183 BUILD_BUG_SQE_ELEM(28, __u32, cancel_flags);
10184 BUILD_BUG_SQE_ELEM(28, __u32, open_flags);
10185 BUILD_BUG_SQE_ELEM(28, __u32, statx_flags);
10186 BUILD_BUG_SQE_ELEM(28, __u32, fadvise_advice);
Pavel Begunkov7d67af22020-02-24 11:32:45 +030010187 BUILD_BUG_SQE_ELEM(28, __u32, splice_flags);
Stefan Metzmacherd7f62e82020-01-29 14:39:41 +010010188 BUILD_BUG_SQE_ELEM(32, __u64, user_data);
10189 BUILD_BUG_SQE_ELEM(40, __u16, buf_index);
10190 BUILD_BUG_SQE_ELEM(42, __u16, personality);
Pavel Begunkov7d67af22020-02-24 11:32:45 +030010191 BUILD_BUG_SQE_ELEM(44, __s32, splice_fd_in);
Stefan Metzmacherd7f62e82020-01-29 14:39:41 +010010192
Pavel Begunkovb0d658ec2021-04-27 16:13:53 +010010193 BUILD_BUG_ON(sizeof(struct io_uring_files_update) !=
10194 sizeof(struct io_uring_rsrc_update));
10195 BUILD_BUG_ON(sizeof(struct io_uring_rsrc_update) >
10196 sizeof(struct io_uring_rsrc_update2));
10197 /* should fit into one byte */
10198 BUILD_BUG_ON(SQE_VALID_FLAGS >= (1 << 8));
10199
Jens Axboed3656342019-12-18 09:50:26 -070010200 BUILD_BUG_ON(ARRAY_SIZE(io_op_defs) != IORING_OP_LAST);
Jens Axboe84557872020-03-03 15:28:17 -070010201 BUILD_BUG_ON(__REQ_F_LAST_BIT >= 8 * sizeof(int));
Jens Axboe91f245d2021-02-09 13:48:50 -070010202 req_cachep = KMEM_CACHE(io_kiocb, SLAB_HWCACHE_ALIGN | SLAB_PANIC |
10203 SLAB_ACCOUNT);
Jens Axboe2b188cc2019-01-07 10:46:33 -070010204 return 0;
10205};
10206__initcall(io_uring_init);