blob: 1a3ca6577a10186d270023a029cb2df603d79f6e [file] [log] [blame]
Jens Axboe2b188cc2019-01-07 10:46:33 -07001// SPDX-License-Identifier: GPL-2.0
2/*
3 * Shared application/kernel submission and completion ring pairs, for
4 * supporting fast/efficient IO.
5 *
6 * A note on the read/write ordering memory barriers that are matched between
Stefan Bühler1e84b972019-04-24 23:54:16 +02007 * the application and kernel side.
8 *
9 * After the application reads the CQ ring tail, it must use an
10 * appropriate smp_rmb() to pair with the smp_wmb() the kernel uses
11 * before writing the tail (using smp_load_acquire to read the tail will
12 * do). It also needs a smp_mb() before updating CQ head (ordering the
13 * entry load(s) with the head store), pairing with an implicit barrier
14 * through a control-dependency in io_get_cqring (smp_store_release to
15 * store head will do). Failure to do so could lead to reading invalid
16 * CQ entries.
17 *
18 * Likewise, the application must use an appropriate smp_wmb() before
19 * writing the SQ tail (ordering SQ entry stores with the tail store),
20 * which pairs with smp_load_acquire in io_get_sqring (smp_store_release
21 * to store the tail will do). And it needs a barrier ordering the SQ
22 * head load before writing new SQ entries (smp_load_acquire to read
23 * head will do).
24 *
25 * When using the SQ poll thread (IORING_SETUP_SQPOLL), the application
26 * needs to check the SQ flags for IORING_SQ_NEED_WAKEUP *after*
27 * updating the SQ tail; a full memory barrier smp_mb() is needed
28 * between.
Jens Axboe2b188cc2019-01-07 10:46:33 -070029 *
30 * Also see the examples in the liburing library:
31 *
32 * git://git.kernel.dk/liburing
33 *
34 * io_uring also uses READ/WRITE_ONCE() for _any_ store or load that happens
35 * from data shared between the kernel and application. This is done both
36 * for ordering purposes, but also to ensure that once a value is loaded from
37 * data that the application could potentially modify, it remains stable.
38 *
39 * Copyright (C) 2018-2019 Jens Axboe
Christoph Hellwigc992fe22019-01-11 09:43:02 -070040 * Copyright (c) 2018-2019 Christoph Hellwig
Jens Axboe2b188cc2019-01-07 10:46:33 -070041 */
42#include <linux/kernel.h>
43#include <linux/init.h>
44#include <linux/errno.h>
45#include <linux/syscalls.h>
46#include <linux/compat.h>
47#include <linux/refcount.h>
48#include <linux/uio.h>
Pavel Begunkov6b47ee62020-01-18 20:22:41 +030049#include <linux/bits.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070050
51#include <linux/sched/signal.h>
52#include <linux/fs.h>
53#include <linux/file.h>
54#include <linux/fdtable.h>
55#include <linux/mm.h>
56#include <linux/mman.h>
57#include <linux/mmu_context.h>
58#include <linux/percpu.h>
59#include <linux/slab.h>
Jens Axboe6c271ce2019-01-10 11:22:30 -070060#include <linux/kthread.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070061#include <linux/blkdev.h>
Jens Axboeedafcce2019-01-09 09:16:05 -070062#include <linux/bvec.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070063#include <linux/net.h>
64#include <net/sock.h>
65#include <net/af_unix.h>
Jens Axboe6b063142019-01-10 22:13:58 -070066#include <net/scm.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070067#include <linux/anon_inodes.h>
68#include <linux/sched/mm.h>
69#include <linux/uaccess.h>
70#include <linux/nospec.h>
Jens Axboeedafcce2019-01-09 09:16:05 -070071#include <linux/sizes.h>
72#include <linux/hugetlb.h>
Jens Axboeaa4c3962019-11-29 10:14:00 -070073#include <linux/highmem.h>
Jens Axboe15b71ab2019-12-11 11:20:36 -070074#include <linux/namei.h>
75#include <linux/fsnotify.h>
Jens Axboe4840e412019-12-25 22:03:45 -070076#include <linux/fadvise.h>
Jens Axboe3e4827b2020-01-08 15:18:09 -070077#include <linux/eventpoll.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070078
Dmitrii Dolgovc826bd72019-10-15 19:02:01 +020079#define CREATE_TRACE_POINTS
80#include <trace/events/io_uring.h>
81
Jens Axboe2b188cc2019-01-07 10:46:33 -070082#include <uapi/linux/io_uring.h>
83
84#include "internal.h"
Jens Axboe561fb042019-10-24 07:25:42 -060085#include "io-wq.h"
Jens Axboe2b188cc2019-01-07 10:46:33 -070086
Daniel Xu5277dea2019-09-14 14:23:45 -070087#define IORING_MAX_ENTRIES 32768
Jens Axboe33a107f2019-10-04 12:10:03 -060088#define IORING_MAX_CQ_ENTRIES (2 * IORING_MAX_ENTRIES)
Jens Axboe65e19f52019-10-26 07:20:21 -060089
90/*
91 * Shift of 9 is 512 entries, or exactly one page on 64-bit archs
92 */
93#define IORING_FILE_TABLE_SHIFT 9
94#define IORING_MAX_FILES_TABLE (1U << IORING_FILE_TABLE_SHIFT)
95#define IORING_FILE_TABLE_MASK (IORING_MAX_FILES_TABLE - 1)
96#define IORING_MAX_FIXED_FILES (64 * IORING_MAX_FILES_TABLE)
Jens Axboe2b188cc2019-01-07 10:46:33 -070097
98struct io_uring {
99 u32 head ____cacheline_aligned_in_smp;
100 u32 tail ____cacheline_aligned_in_smp;
101};
102
Stefan Bühler1e84b972019-04-24 23:54:16 +0200103/*
Hristo Venev75b28af2019-08-26 17:23:46 +0000104 * This data is shared with the application through the mmap at offsets
105 * IORING_OFF_SQ_RING and IORING_OFF_CQ_RING.
Stefan Bühler1e84b972019-04-24 23:54:16 +0200106 *
107 * The offsets to the member fields are published through struct
108 * io_sqring_offsets when calling io_uring_setup.
109 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000110struct io_rings {
Stefan Bühler1e84b972019-04-24 23:54:16 +0200111 /*
112 * Head and tail offsets into the ring; the offsets need to be
113 * masked to get valid indices.
114 *
Hristo Venev75b28af2019-08-26 17:23:46 +0000115 * The kernel controls head of the sq ring and the tail of the cq ring,
116 * and the application controls tail of the sq ring and the head of the
117 * cq ring.
Stefan Bühler1e84b972019-04-24 23:54:16 +0200118 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000119 struct io_uring sq, cq;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200120 /*
Hristo Venev75b28af2019-08-26 17:23:46 +0000121 * Bitmasks to apply to head and tail offsets (constant, equals
Stefan Bühler1e84b972019-04-24 23:54:16 +0200122 * ring_entries - 1)
123 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000124 u32 sq_ring_mask, cq_ring_mask;
125 /* Ring sizes (constant, power of 2) */
126 u32 sq_ring_entries, cq_ring_entries;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200127 /*
128 * Number of invalid entries dropped by the kernel due to
129 * invalid index stored in array
130 *
131 * Written by the kernel, shouldn't be modified by the
132 * application (i.e. get number of "new events" by comparing to
133 * cached value).
134 *
135 * After a new SQ head value was read by the application this
136 * counter includes all submissions that were dropped reaching
137 * the new SQ head (and possibly more).
138 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000139 u32 sq_dropped;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200140 /*
141 * Runtime flags
142 *
143 * Written by the kernel, shouldn't be modified by the
144 * application.
145 *
146 * The application needs a full memory barrier before checking
147 * for IORING_SQ_NEED_WAKEUP after updating the sq tail.
148 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000149 u32 sq_flags;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200150 /*
151 * Number of completion events lost because the queue was full;
152 * this should be avoided by the application by making sure
LimingWu0b4295b2019-12-05 20:18:18 +0800153 * there are not more requests pending than there is space in
Stefan Bühler1e84b972019-04-24 23:54:16 +0200154 * the completion queue.
155 *
156 * Written by the kernel, shouldn't be modified by the
157 * application (i.e. get number of "new events" by comparing to
158 * cached value).
159 *
160 * As completion events come in out of order this counter is not
161 * ordered with any other data.
162 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000163 u32 cq_overflow;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200164 /*
165 * Ring buffer of completion events.
166 *
167 * The kernel writes completion events fresh every time they are
168 * produced, so the application is allowed to modify pending
169 * entries.
170 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000171 struct io_uring_cqe cqes[] ____cacheline_aligned_in_smp;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700172};
173
Jens Axboeedafcce2019-01-09 09:16:05 -0700174struct io_mapped_ubuf {
175 u64 ubuf;
176 size_t len;
177 struct bio_vec *bvec;
178 unsigned int nr_bvecs;
179};
180
Jens Axboe65e19f52019-10-26 07:20:21 -0600181struct fixed_file_table {
182 struct file **files;
Jens Axboe31b51512019-01-18 22:56:34 -0700183};
184
Jens Axboe05f3fb32019-12-09 11:22:50 -0700185enum {
186 FFD_F_ATOMIC,
187};
188
189struct fixed_file_data {
190 struct fixed_file_table *table;
191 struct io_ring_ctx *ctx;
192
193 struct percpu_ref refs;
194 struct llist_head put_llist;
195 unsigned long state;
196 struct work_struct ref_work;
197 struct completion done;
198};
199
Jens Axboe2b188cc2019-01-07 10:46:33 -0700200struct io_ring_ctx {
201 struct {
202 struct percpu_ref refs;
203 } ____cacheline_aligned_in_smp;
204
205 struct {
206 unsigned int flags;
Randy Dunlape1d85332020-02-05 20:57:10 -0800207 unsigned int compat: 1;
208 unsigned int account_mem: 1;
209 unsigned int cq_overflow_flushed: 1;
210 unsigned int drain_next: 1;
211 unsigned int eventfd_async: 1;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700212
Hristo Venev75b28af2019-08-26 17:23:46 +0000213 /*
214 * Ring buffer of indices into array of io_uring_sqe, which is
215 * mmapped by the application using the IORING_OFF_SQES offset.
216 *
217 * This indirection could e.g. be used to assign fixed
218 * io_uring_sqe entries to operations and only submit them to
219 * the queue when needed.
220 *
221 * The kernel modifies neither the indices array nor the entries
222 * array.
223 */
224 u32 *sq_array;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700225 unsigned cached_sq_head;
226 unsigned sq_entries;
227 unsigned sq_mask;
Jens Axboe6c271ce2019-01-10 11:22:30 -0700228 unsigned sq_thread_idle;
Jens Axboe498ccd92019-10-25 10:04:25 -0600229 unsigned cached_sq_dropped;
Jens Axboe206aefd2019-11-07 18:27:42 -0700230 atomic_t cached_cq_overflow;
Jens Axboead3eb2c2019-12-18 17:12:20 -0700231 unsigned long sq_check_overflow;
Jens Axboede0617e2019-04-06 21:51:27 -0600232
233 struct list_head defer_list;
Jens Axboe5262f562019-09-17 12:26:57 -0600234 struct list_head timeout_list;
Jens Axboe1d7bb1d2019-11-06 11:31:17 -0700235 struct list_head cq_overflow_list;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700236
Jens Axboefcb323c2019-10-24 12:39:47 -0600237 wait_queue_head_t inflight_wait;
Jens Axboead3eb2c2019-12-18 17:12:20 -0700238 struct io_uring_sqe *sq_sqes;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700239 } ____cacheline_aligned_in_smp;
240
Hristo Venev75b28af2019-08-26 17:23:46 +0000241 struct io_rings *rings;
242
Jens Axboe2b188cc2019-01-07 10:46:33 -0700243 /* IO offload */
Jens Axboe561fb042019-10-24 07:25:42 -0600244 struct io_wq *io_wq;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700245 struct task_struct *sqo_thread; /* if using sq thread polling */
246 struct mm_struct *sqo_mm;
247 wait_queue_head_t sqo_wait;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700248
Jens Axboe6b063142019-01-10 22:13:58 -0700249 /*
250 * If used, fixed file set. Writers must ensure that ->refs is dead,
251 * readers must ensure that ->refs is alive as long as the file* is
252 * used. Only updated through io_uring_register(2).
253 */
Jens Axboe05f3fb32019-12-09 11:22:50 -0700254 struct fixed_file_data *file_data;
Jens Axboe6b063142019-01-10 22:13:58 -0700255 unsigned nr_user_files;
Pavel Begunkovb14cca02020-01-17 04:45:59 +0300256 int ring_fd;
257 struct file *ring_file;
Jens Axboe6b063142019-01-10 22:13:58 -0700258
Jens Axboeedafcce2019-01-09 09:16:05 -0700259 /* if used, fixed mapped user buffers */
260 unsigned nr_user_bufs;
261 struct io_mapped_ubuf *user_bufs;
262
Jens Axboe2b188cc2019-01-07 10:46:33 -0700263 struct user_struct *user;
264
Jens Axboe0b8c0ec2019-12-02 08:50:00 -0700265 const struct cred *creds;
Jens Axboe181e4482019-11-25 08:52:30 -0700266
Jens Axboe206aefd2019-11-07 18:27:42 -0700267 /* 0 is for ctx quiesce/reinit/free, 1 is for sqo_thread started */
268 struct completion *completions;
269
Jens Axboe0ddf92e2019-11-08 08:52:53 -0700270 /* if all else fails... */
271 struct io_kiocb *fallback_req;
272
Jens Axboe206aefd2019-11-07 18:27:42 -0700273#if defined(CONFIG_UNIX)
274 struct socket *ring_sock;
275#endif
276
Jens Axboe071698e2020-01-28 10:04:42 -0700277 struct idr personality_idr;
278
Jens Axboe206aefd2019-11-07 18:27:42 -0700279 struct {
280 unsigned cached_cq_tail;
281 unsigned cq_entries;
282 unsigned cq_mask;
283 atomic_t cq_timeouts;
Jens Axboead3eb2c2019-12-18 17:12:20 -0700284 unsigned long cq_check_overflow;
Jens Axboe206aefd2019-11-07 18:27:42 -0700285 struct wait_queue_head cq_wait;
286 struct fasync_struct *cq_fasync;
287 struct eventfd_ctx *cq_ev_fd;
288 } ____cacheline_aligned_in_smp;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700289
290 struct {
291 struct mutex uring_lock;
292 wait_queue_head_t wait;
293 } ____cacheline_aligned_in_smp;
294
295 struct {
296 spinlock_t completion_lock;
Jens Axboee94f1412019-12-19 12:06:02 -0700297 struct llist_head poll_llist;
298
Jens Axboedef596e2019-01-09 08:59:42 -0700299 /*
300 * ->poll_list is protected by the ctx->uring_lock for
301 * io_uring instances that don't use IORING_SETUP_SQPOLL.
302 * For SQPOLL, only the single threaded io_sq_thread() will
303 * manipulate the list, hence no extra locking is needed there.
304 */
305 struct list_head poll_list;
Jens Axboe78076bb2019-12-04 19:56:40 -0700306 struct hlist_head *cancel_hash;
307 unsigned cancel_hash_bits;
Jens Axboee94f1412019-12-19 12:06:02 -0700308 bool poll_multi_file;
Jens Axboefcb323c2019-10-24 12:39:47 -0600309
310 spinlock_t inflight_lock;
311 struct list_head inflight_list;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700312 } ____cacheline_aligned_in_smp;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700313};
314
Jens Axboe09bb8392019-03-13 12:39:28 -0600315/*
316 * First field must be the file pointer in all the
317 * iocb unions! See also 'struct kiocb' in <linux/fs.h>
318 */
Jens Axboe221c5eb2019-01-17 09:41:58 -0700319struct io_poll_iocb {
320 struct file *file;
Jens Axboe0969e782019-12-17 18:40:57 -0700321 union {
322 struct wait_queue_head *head;
323 u64 addr;
324 };
Jens Axboe221c5eb2019-01-17 09:41:58 -0700325 __poll_t events;
Jens Axboe8c838782019-03-12 15:48:16 -0600326 bool done;
Jens Axboe221c5eb2019-01-17 09:41:58 -0700327 bool canceled;
Jens Axboe392edb42019-12-09 17:52:20 -0700328 struct wait_queue_entry wait;
Jens Axboe221c5eb2019-01-17 09:41:58 -0700329};
330
Jens Axboeb5dba592019-12-11 14:02:38 -0700331struct io_close {
332 struct file *file;
333 struct file *put_file;
334 int fd;
335};
336
Jens Axboead8a48a2019-11-15 08:49:11 -0700337struct io_timeout_data {
338 struct io_kiocb *req;
339 struct hrtimer timer;
340 struct timespec64 ts;
341 enum hrtimer_mode mode;
Pavel Begunkovcc42e0a2019-11-25 23:14:38 +0300342 u32 seq_offset;
Jens Axboead8a48a2019-11-15 08:49:11 -0700343};
344
Jens Axboe8ed8d3c2019-12-16 11:55:28 -0700345struct io_accept {
346 struct file *file;
347 struct sockaddr __user *addr;
348 int __user *addr_len;
349 int flags;
350};
351
352struct io_sync {
353 struct file *file;
354 loff_t len;
355 loff_t off;
356 int flags;
Jens Axboed63d1b52019-12-10 10:38:56 -0700357 int mode;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -0700358};
359
Jens Axboefbf23842019-12-17 18:45:56 -0700360struct io_cancel {
361 struct file *file;
362 u64 addr;
363};
364
Jens Axboeb29472e2019-12-17 18:50:29 -0700365struct io_timeout {
366 struct file *file;
367 u64 addr;
368 int flags;
Jens Axboe26a61672019-12-20 09:02:01 -0700369 unsigned count;
Jens Axboeb29472e2019-12-17 18:50:29 -0700370};
371
Jens Axboe9adbd452019-12-20 08:45:55 -0700372struct io_rw {
373 /* NOTE: kiocb has the file as the first member, so don't do it here */
374 struct kiocb kiocb;
375 u64 addr;
376 u64 len;
377};
378
Jens Axboe3fbb51c2019-12-20 08:51:52 -0700379struct io_connect {
380 struct file *file;
381 struct sockaddr __user *addr;
382 int addr_len;
383};
384
Jens Axboee47293f2019-12-20 08:58:21 -0700385struct io_sr_msg {
386 struct file *file;
Jens Axboefddafac2020-01-04 20:19:44 -0700387 union {
388 struct user_msghdr __user *msg;
389 void __user *buf;
390 };
Jens Axboee47293f2019-12-20 08:58:21 -0700391 int msg_flags;
Jens Axboefddafac2020-01-04 20:19:44 -0700392 size_t len;
Jens Axboee47293f2019-12-20 08:58:21 -0700393};
394
Jens Axboe15b71ab2019-12-11 11:20:36 -0700395struct io_open {
396 struct file *file;
397 int dfd;
Jens Axboeeddc7ef2019-12-13 21:18:10 -0700398 union {
Jens Axboeeddc7ef2019-12-13 21:18:10 -0700399 unsigned mask;
400 };
Jens Axboe15b71ab2019-12-11 11:20:36 -0700401 struct filename *filename;
Jens Axboeeddc7ef2019-12-13 21:18:10 -0700402 struct statx __user *buffer;
Jens Axboec12cedf2020-01-08 17:41:21 -0700403 struct open_how how;
Jens Axboe15b71ab2019-12-11 11:20:36 -0700404};
405
Jens Axboe05f3fb32019-12-09 11:22:50 -0700406struct io_files_update {
407 struct file *file;
408 u64 arg;
409 u32 nr_args;
410 u32 offset;
411};
412
Jens Axboe4840e412019-12-25 22:03:45 -0700413struct io_fadvise {
414 struct file *file;
415 u64 offset;
416 u32 len;
417 u32 advice;
418};
419
Jens Axboec1ca7572019-12-25 22:18:28 -0700420struct io_madvise {
421 struct file *file;
422 u64 addr;
423 u32 len;
424 u32 advice;
425};
426
Jens Axboe3e4827b2020-01-08 15:18:09 -0700427struct io_epoll {
428 struct file *file;
429 int epfd;
430 int op;
431 int fd;
432 struct epoll_event event;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700433};
434
Jens Axboef499a022019-12-02 16:28:46 -0700435struct io_async_connect {
436 struct sockaddr_storage address;
437};
438
Jens Axboe03b12302019-12-02 18:50:25 -0700439struct io_async_msghdr {
440 struct iovec fast_iov[UIO_FASTIOV];
441 struct iovec *iov;
442 struct sockaddr __user *uaddr;
443 struct msghdr msg;
444};
445
Jens Axboef67676d2019-12-02 11:03:47 -0700446struct io_async_rw {
447 struct iovec fast_iov[UIO_FASTIOV];
448 struct iovec *iov;
449 ssize_t nr_segs;
450 ssize_t size;
451};
452
Jens Axboe1a6b74f2019-12-02 10:33:15 -0700453struct io_async_ctx {
Jens Axboef67676d2019-12-02 11:03:47 -0700454 union {
455 struct io_async_rw rw;
Jens Axboe03b12302019-12-02 18:50:25 -0700456 struct io_async_msghdr msg;
Jens Axboef499a022019-12-02 16:28:46 -0700457 struct io_async_connect connect;
Jens Axboe2d283902019-12-04 11:08:05 -0700458 struct io_timeout_data timeout;
Jens Axboef67676d2019-12-02 11:03:47 -0700459 };
Jens Axboe1a6b74f2019-12-02 10:33:15 -0700460};
461
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300462enum {
463 REQ_F_FIXED_FILE_BIT = IOSQE_FIXED_FILE_BIT,
464 REQ_F_IO_DRAIN_BIT = IOSQE_IO_DRAIN_BIT,
465 REQ_F_LINK_BIT = IOSQE_IO_LINK_BIT,
466 REQ_F_HARDLINK_BIT = IOSQE_IO_HARDLINK_BIT,
467 REQ_F_FORCE_ASYNC_BIT = IOSQE_ASYNC_BIT,
468
469 REQ_F_LINK_NEXT_BIT,
470 REQ_F_FAIL_LINK_BIT,
471 REQ_F_INFLIGHT_BIT,
472 REQ_F_CUR_POS_BIT,
473 REQ_F_NOWAIT_BIT,
474 REQ_F_IOPOLL_COMPLETED_BIT,
475 REQ_F_LINK_TIMEOUT_BIT,
476 REQ_F_TIMEOUT_BIT,
477 REQ_F_ISREG_BIT,
478 REQ_F_MUST_PUNT_BIT,
479 REQ_F_TIMEOUT_NOSEQ_BIT,
480 REQ_F_COMP_LOCKED_BIT,
Pavel Begunkov99bc4c32020-02-07 22:04:45 +0300481 REQ_F_NEED_CLEANUP_BIT,
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300482};
483
484enum {
485 /* ctx owns file */
486 REQ_F_FIXED_FILE = BIT(REQ_F_FIXED_FILE_BIT),
487 /* drain existing IO first */
488 REQ_F_IO_DRAIN = BIT(REQ_F_IO_DRAIN_BIT),
489 /* linked sqes */
490 REQ_F_LINK = BIT(REQ_F_LINK_BIT),
491 /* doesn't sever on completion < 0 */
492 REQ_F_HARDLINK = BIT(REQ_F_HARDLINK_BIT),
493 /* IOSQE_ASYNC */
494 REQ_F_FORCE_ASYNC = BIT(REQ_F_FORCE_ASYNC_BIT),
495
496 /* already grabbed next link */
497 REQ_F_LINK_NEXT = BIT(REQ_F_LINK_NEXT_BIT),
498 /* fail rest of links */
499 REQ_F_FAIL_LINK = BIT(REQ_F_FAIL_LINK_BIT),
500 /* on inflight list */
501 REQ_F_INFLIGHT = BIT(REQ_F_INFLIGHT_BIT),
502 /* read/write uses file position */
503 REQ_F_CUR_POS = BIT(REQ_F_CUR_POS_BIT),
504 /* must not punt to workers */
505 REQ_F_NOWAIT = BIT(REQ_F_NOWAIT_BIT),
506 /* polled IO has completed */
507 REQ_F_IOPOLL_COMPLETED = BIT(REQ_F_IOPOLL_COMPLETED_BIT),
508 /* has linked timeout */
509 REQ_F_LINK_TIMEOUT = BIT(REQ_F_LINK_TIMEOUT_BIT),
510 /* timeout request */
511 REQ_F_TIMEOUT = BIT(REQ_F_TIMEOUT_BIT),
512 /* regular file */
513 REQ_F_ISREG = BIT(REQ_F_ISREG_BIT),
514 /* must be punted even for NONBLOCK */
515 REQ_F_MUST_PUNT = BIT(REQ_F_MUST_PUNT_BIT),
516 /* no timeout sequence */
517 REQ_F_TIMEOUT_NOSEQ = BIT(REQ_F_TIMEOUT_NOSEQ_BIT),
518 /* completion under lock */
519 REQ_F_COMP_LOCKED = BIT(REQ_F_COMP_LOCKED_BIT),
Pavel Begunkov99bc4c32020-02-07 22:04:45 +0300520 /* needs cleanup */
521 REQ_F_NEED_CLEANUP = BIT(REQ_F_NEED_CLEANUP_BIT),
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300522};
523
Jens Axboe09bb8392019-03-13 12:39:28 -0600524/*
525 * NOTE! Each of the iocb union members has the file pointer
526 * as the first entry in their struct definition. So you can
527 * access the file pointer through any of the sub-structs,
528 * or directly as just 'ki_filp' in this struct.
529 */
Jens Axboe2b188cc2019-01-07 10:46:33 -0700530struct io_kiocb {
Jens Axboe221c5eb2019-01-17 09:41:58 -0700531 union {
Jens Axboe09bb8392019-03-13 12:39:28 -0600532 struct file *file;
Jens Axboe9adbd452019-12-20 08:45:55 -0700533 struct io_rw rw;
Jens Axboe221c5eb2019-01-17 09:41:58 -0700534 struct io_poll_iocb poll;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -0700535 struct io_accept accept;
536 struct io_sync sync;
Jens Axboefbf23842019-12-17 18:45:56 -0700537 struct io_cancel cancel;
Jens Axboeb29472e2019-12-17 18:50:29 -0700538 struct io_timeout timeout;
Jens Axboe3fbb51c2019-12-20 08:51:52 -0700539 struct io_connect connect;
Jens Axboee47293f2019-12-20 08:58:21 -0700540 struct io_sr_msg sr_msg;
Jens Axboe15b71ab2019-12-11 11:20:36 -0700541 struct io_open open;
Jens Axboeb5dba592019-12-11 14:02:38 -0700542 struct io_close close;
Jens Axboe05f3fb32019-12-09 11:22:50 -0700543 struct io_files_update files_update;
Jens Axboe4840e412019-12-25 22:03:45 -0700544 struct io_fadvise fadvise;
Jens Axboec1ca7572019-12-25 22:18:28 -0700545 struct io_madvise madvise;
Jens Axboe3e4827b2020-01-08 15:18:09 -0700546 struct io_epoll epoll;
Jens Axboe221c5eb2019-01-17 09:41:58 -0700547 };
Jens Axboe2b188cc2019-01-07 10:46:33 -0700548
Jens Axboe1a6b74f2019-12-02 10:33:15 -0700549 struct io_async_ctx *io;
Pavel Begunkovb14cca02020-01-17 04:45:59 +0300550 /*
551 * llist_node is only used for poll deferred completions
552 */
553 struct llist_node llist_node;
Pavel Begunkovcf6fd4b2019-11-25 23:14:39 +0300554 bool in_async;
555 bool needs_fixed_file;
Jens Axboed625c6e2019-12-17 19:53:05 -0700556 u8 opcode;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700557
558 struct io_ring_ctx *ctx;
Jens Axboeeac406c2019-11-14 12:09:58 -0700559 union {
560 struct list_head list;
Jens Axboe78076bb2019-12-04 19:56:40 -0700561 struct hlist_node hash_node;
Jens Axboeeac406c2019-11-14 12:09:58 -0700562 };
Jens Axboe9e645e112019-05-10 16:07:28 -0600563 struct list_head link_list;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700564 unsigned int flags;
Jens Axboec16361c2019-01-17 08:39:48 -0700565 refcount_t refs;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700566 u64 user_data;
Jens Axboe9e645e112019-05-10 16:07:28 -0600567 u32 result;
Jens Axboede0617e2019-04-06 21:51:27 -0600568 u32 sequence;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700569
Jens Axboefcb323c2019-10-24 12:39:47 -0600570 struct list_head inflight_entry;
571
Jens Axboe561fb042019-10-24 07:25:42 -0600572 struct io_wq_work work;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700573};
574
575#define IO_PLUG_THRESHOLD 2
Jens Axboedef596e2019-01-09 08:59:42 -0700576#define IO_IOPOLL_BATCH 8
Jens Axboe2b188cc2019-01-07 10:46:33 -0700577
Jens Axboe9a56a232019-01-09 09:06:50 -0700578struct io_submit_state {
579 struct blk_plug plug;
580
581 /*
Jens Axboe2579f912019-01-09 09:10:43 -0700582 * io_kiocb alloc cache
583 */
584 void *reqs[IO_IOPOLL_BATCH];
Pavel Begunkov6c8a3132020-02-01 03:58:00 +0300585 unsigned int free_reqs;
Jens Axboe2579f912019-01-09 09:10:43 -0700586
587 /*
Jens Axboe9a56a232019-01-09 09:06:50 -0700588 * File reference cache
589 */
590 struct file *file;
591 unsigned int fd;
592 unsigned int has_refs;
593 unsigned int used_refs;
594 unsigned int ios_left;
595};
596
Jens Axboed3656342019-12-18 09:50:26 -0700597struct io_op_def {
598 /* needs req->io allocated for deferral/async */
599 unsigned async_ctx : 1;
600 /* needs current->mm setup, does mm access */
601 unsigned needs_mm : 1;
602 /* needs req->file assigned */
603 unsigned needs_file : 1;
604 /* needs req->file assigned IFF fd is >= 0 */
605 unsigned fd_non_neg : 1;
606 /* hash wq insertion if file is a regular file */
607 unsigned hash_reg_file : 1;
608 /* unbound wq insertion if file is a non-regular file */
609 unsigned unbound_nonreg_file : 1;
Jens Axboe66f4af92020-01-16 15:36:52 -0700610 /* opcode is not supported by this kernel */
611 unsigned not_supported : 1;
Jens Axboef86cd202020-01-29 13:46:44 -0700612 /* needs file table */
613 unsigned file_table : 1;
Jens Axboed3656342019-12-18 09:50:26 -0700614};
615
616static const struct io_op_def io_op_defs[] = {
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300617 [IORING_OP_NOP] = {},
618 [IORING_OP_READV] = {
Jens Axboed3656342019-12-18 09:50:26 -0700619 .async_ctx = 1,
620 .needs_mm = 1,
621 .needs_file = 1,
622 .unbound_nonreg_file = 1,
623 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300624 [IORING_OP_WRITEV] = {
Jens Axboed3656342019-12-18 09:50:26 -0700625 .async_ctx = 1,
626 .needs_mm = 1,
627 .needs_file = 1,
628 .hash_reg_file = 1,
629 .unbound_nonreg_file = 1,
630 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300631 [IORING_OP_FSYNC] = {
Jens Axboed3656342019-12-18 09:50:26 -0700632 .needs_file = 1,
633 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300634 [IORING_OP_READ_FIXED] = {
Jens Axboed3656342019-12-18 09:50:26 -0700635 .needs_file = 1,
636 .unbound_nonreg_file = 1,
637 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300638 [IORING_OP_WRITE_FIXED] = {
Jens Axboed3656342019-12-18 09:50:26 -0700639 .needs_file = 1,
640 .hash_reg_file = 1,
641 .unbound_nonreg_file = 1,
642 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300643 [IORING_OP_POLL_ADD] = {
Jens Axboed3656342019-12-18 09:50:26 -0700644 .needs_file = 1,
645 .unbound_nonreg_file = 1,
646 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300647 [IORING_OP_POLL_REMOVE] = {},
648 [IORING_OP_SYNC_FILE_RANGE] = {
Jens Axboed3656342019-12-18 09:50:26 -0700649 .needs_file = 1,
650 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300651 [IORING_OP_SENDMSG] = {
Jens Axboed3656342019-12-18 09:50:26 -0700652 .async_ctx = 1,
653 .needs_mm = 1,
654 .needs_file = 1,
655 .unbound_nonreg_file = 1,
656 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300657 [IORING_OP_RECVMSG] = {
Jens Axboed3656342019-12-18 09:50:26 -0700658 .async_ctx = 1,
659 .needs_mm = 1,
660 .needs_file = 1,
661 .unbound_nonreg_file = 1,
662 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300663 [IORING_OP_TIMEOUT] = {
Jens Axboed3656342019-12-18 09:50:26 -0700664 .async_ctx = 1,
665 .needs_mm = 1,
666 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300667 [IORING_OP_TIMEOUT_REMOVE] = {},
668 [IORING_OP_ACCEPT] = {
Jens Axboed3656342019-12-18 09:50:26 -0700669 .needs_mm = 1,
670 .needs_file = 1,
671 .unbound_nonreg_file = 1,
Jens Axboef86cd202020-01-29 13:46:44 -0700672 .file_table = 1,
Jens Axboed3656342019-12-18 09:50:26 -0700673 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300674 [IORING_OP_ASYNC_CANCEL] = {},
675 [IORING_OP_LINK_TIMEOUT] = {
Jens Axboed3656342019-12-18 09:50:26 -0700676 .async_ctx = 1,
677 .needs_mm = 1,
678 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300679 [IORING_OP_CONNECT] = {
Jens Axboed3656342019-12-18 09:50:26 -0700680 .async_ctx = 1,
681 .needs_mm = 1,
682 .needs_file = 1,
683 .unbound_nonreg_file = 1,
684 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300685 [IORING_OP_FALLOCATE] = {
Jens Axboed3656342019-12-18 09:50:26 -0700686 .needs_file = 1,
687 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300688 [IORING_OP_OPENAT] = {
Jens Axboed3656342019-12-18 09:50:26 -0700689 .needs_file = 1,
690 .fd_non_neg = 1,
Jens Axboef86cd202020-01-29 13:46:44 -0700691 .file_table = 1,
Jens Axboed3656342019-12-18 09:50:26 -0700692 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300693 [IORING_OP_CLOSE] = {
Jens Axboed3656342019-12-18 09:50:26 -0700694 .needs_file = 1,
Jens Axboef86cd202020-01-29 13:46:44 -0700695 .file_table = 1,
Jens Axboed3656342019-12-18 09:50:26 -0700696 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300697 [IORING_OP_FILES_UPDATE] = {
Jens Axboed3656342019-12-18 09:50:26 -0700698 .needs_mm = 1,
Jens Axboef86cd202020-01-29 13:46:44 -0700699 .file_table = 1,
Jens Axboed3656342019-12-18 09:50:26 -0700700 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300701 [IORING_OP_STATX] = {
Jens Axboed3656342019-12-18 09:50:26 -0700702 .needs_mm = 1,
703 .needs_file = 1,
704 .fd_non_neg = 1,
705 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300706 [IORING_OP_READ] = {
Jens Axboe3a6820f2019-12-22 15:19:35 -0700707 .needs_mm = 1,
708 .needs_file = 1,
709 .unbound_nonreg_file = 1,
710 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300711 [IORING_OP_WRITE] = {
Jens Axboe3a6820f2019-12-22 15:19:35 -0700712 .needs_mm = 1,
713 .needs_file = 1,
714 .unbound_nonreg_file = 1,
715 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300716 [IORING_OP_FADVISE] = {
Jens Axboe4840e412019-12-25 22:03:45 -0700717 .needs_file = 1,
718 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300719 [IORING_OP_MADVISE] = {
Jens Axboec1ca7572019-12-25 22:18:28 -0700720 .needs_mm = 1,
721 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300722 [IORING_OP_SEND] = {
Jens Axboefddafac2020-01-04 20:19:44 -0700723 .needs_mm = 1,
724 .needs_file = 1,
725 .unbound_nonreg_file = 1,
726 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300727 [IORING_OP_RECV] = {
Jens Axboefddafac2020-01-04 20:19:44 -0700728 .needs_mm = 1,
729 .needs_file = 1,
730 .unbound_nonreg_file = 1,
731 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300732 [IORING_OP_OPENAT2] = {
Jens Axboecebdb982020-01-08 17:59:24 -0700733 .needs_file = 1,
734 .fd_non_neg = 1,
Jens Axboef86cd202020-01-29 13:46:44 -0700735 .file_table = 1,
Jens Axboecebdb982020-01-08 17:59:24 -0700736 },
Jens Axboe3e4827b2020-01-08 15:18:09 -0700737 [IORING_OP_EPOLL_CTL] = {
738 .unbound_nonreg_file = 1,
739 .file_table = 1,
740 },
Jens Axboed3656342019-12-18 09:50:26 -0700741};
742
Jens Axboe561fb042019-10-24 07:25:42 -0600743static void io_wq_submit_work(struct io_wq_work **workptr);
Jens Axboe78e19bb2019-11-06 15:21:34 -0700744static void io_cqring_fill_event(struct io_kiocb *req, long res);
Jackie Liuec9c02a2019-11-08 23:50:36 +0800745static void io_put_req(struct io_kiocb *req);
Jens Axboe978db572019-11-14 22:39:04 -0700746static void __io_double_put_req(struct io_kiocb *req);
Jens Axboe94ae5e72019-11-14 19:39:52 -0700747static struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req);
748static void io_queue_linked_timeout(struct io_kiocb *req);
Jens Axboe05f3fb32019-12-09 11:22:50 -0700749static int __io_sqe_files_update(struct io_ring_ctx *ctx,
750 struct io_uring_files_update *ip,
751 unsigned nr_args);
Jens Axboef86cd202020-01-29 13:46:44 -0700752static int io_grab_files(struct io_kiocb *req);
Jens Axboe2faf8522020-02-04 19:54:55 -0700753static void io_ring_file_ref_flush(struct fixed_file_data *data);
Pavel Begunkov99bc4c32020-02-07 22:04:45 +0300754static void io_cleanup_req(struct io_kiocb *req);
Jens Axboede0617e2019-04-06 21:51:27 -0600755
Jens Axboe2b188cc2019-01-07 10:46:33 -0700756static struct kmem_cache *req_cachep;
757
758static const struct file_operations io_uring_fops;
759
760struct sock *io_uring_get_socket(struct file *file)
761{
762#if defined(CONFIG_UNIX)
763 if (file->f_op == &io_uring_fops) {
764 struct io_ring_ctx *ctx = file->private_data;
765
766 return ctx->ring_sock->sk;
767 }
768#endif
769 return NULL;
770}
771EXPORT_SYMBOL(io_uring_get_socket);
772
773static void io_ring_ctx_ref_free(struct percpu_ref *ref)
774{
775 struct io_ring_ctx *ctx = container_of(ref, struct io_ring_ctx, refs);
776
Jens Axboe206aefd2019-11-07 18:27:42 -0700777 complete(&ctx->completions[0]);
Jens Axboe2b188cc2019-01-07 10:46:33 -0700778}
779
780static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
781{
782 struct io_ring_ctx *ctx;
Jens Axboe78076bb2019-12-04 19:56:40 -0700783 int hash_bits;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700784
785 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
786 if (!ctx)
787 return NULL;
788
Jens Axboe0ddf92e2019-11-08 08:52:53 -0700789 ctx->fallback_req = kmem_cache_alloc(req_cachep, GFP_KERNEL);
790 if (!ctx->fallback_req)
791 goto err;
792
Jens Axboe206aefd2019-11-07 18:27:42 -0700793 ctx->completions = kmalloc(2 * sizeof(struct completion), GFP_KERNEL);
794 if (!ctx->completions)
795 goto err;
796
Jens Axboe78076bb2019-12-04 19:56:40 -0700797 /*
798 * Use 5 bits less than the max cq entries, that should give us around
799 * 32 entries per hash list if totally full and uniformly spread.
800 */
801 hash_bits = ilog2(p->cq_entries);
802 hash_bits -= 5;
803 if (hash_bits <= 0)
804 hash_bits = 1;
805 ctx->cancel_hash_bits = hash_bits;
806 ctx->cancel_hash = kmalloc((1U << hash_bits) * sizeof(struct hlist_head),
807 GFP_KERNEL);
808 if (!ctx->cancel_hash)
809 goto err;
810 __hash_init(ctx->cancel_hash, 1U << hash_bits);
811
Roman Gushchin21482892019-05-07 10:01:48 -0700812 if (percpu_ref_init(&ctx->refs, io_ring_ctx_ref_free,
Jens Axboe206aefd2019-11-07 18:27:42 -0700813 PERCPU_REF_ALLOW_REINIT, GFP_KERNEL))
814 goto err;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700815
816 ctx->flags = p->flags;
817 init_waitqueue_head(&ctx->cq_wait);
Jens Axboe1d7bb1d2019-11-06 11:31:17 -0700818 INIT_LIST_HEAD(&ctx->cq_overflow_list);
Jens Axboe206aefd2019-11-07 18:27:42 -0700819 init_completion(&ctx->completions[0]);
820 init_completion(&ctx->completions[1]);
Jens Axboe071698e2020-01-28 10:04:42 -0700821 idr_init(&ctx->personality_idr);
Jens Axboe2b188cc2019-01-07 10:46:33 -0700822 mutex_init(&ctx->uring_lock);
823 init_waitqueue_head(&ctx->wait);
824 spin_lock_init(&ctx->completion_lock);
Jens Axboee94f1412019-12-19 12:06:02 -0700825 init_llist_head(&ctx->poll_llist);
Jens Axboedef596e2019-01-09 08:59:42 -0700826 INIT_LIST_HEAD(&ctx->poll_list);
Jens Axboede0617e2019-04-06 21:51:27 -0600827 INIT_LIST_HEAD(&ctx->defer_list);
Jens Axboe5262f562019-09-17 12:26:57 -0600828 INIT_LIST_HEAD(&ctx->timeout_list);
Jens Axboefcb323c2019-10-24 12:39:47 -0600829 init_waitqueue_head(&ctx->inflight_wait);
830 spin_lock_init(&ctx->inflight_lock);
831 INIT_LIST_HEAD(&ctx->inflight_list);
Jens Axboe2b188cc2019-01-07 10:46:33 -0700832 return ctx;
Jens Axboe206aefd2019-11-07 18:27:42 -0700833err:
Jens Axboe0ddf92e2019-11-08 08:52:53 -0700834 if (ctx->fallback_req)
835 kmem_cache_free(req_cachep, ctx->fallback_req);
Jens Axboe206aefd2019-11-07 18:27:42 -0700836 kfree(ctx->completions);
Jens Axboe78076bb2019-12-04 19:56:40 -0700837 kfree(ctx->cancel_hash);
Jens Axboe206aefd2019-11-07 18:27:42 -0700838 kfree(ctx);
839 return NULL;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700840}
841
Bob Liu9d858b22019-11-13 18:06:25 +0800842static inline bool __req_need_defer(struct io_kiocb *req)
Jens Axboede0617e2019-04-06 21:51:27 -0600843{
Jackie Liua197f662019-11-08 08:09:12 -0700844 struct io_ring_ctx *ctx = req->ctx;
845
Jens Axboe498ccd92019-10-25 10:04:25 -0600846 return req->sequence != ctx->cached_cq_tail + ctx->cached_sq_dropped
847 + atomic_read(&ctx->cached_cq_overflow);
Jens Axboede0617e2019-04-06 21:51:27 -0600848}
849
Bob Liu9d858b22019-11-13 18:06:25 +0800850static inline bool req_need_defer(struct io_kiocb *req)
Jens Axboe7adf4ea2019-10-10 21:42:58 -0600851{
Pavel Begunkov87987892020-01-18 01:22:30 +0300852 if (unlikely(req->flags & REQ_F_IO_DRAIN))
Bob Liu9d858b22019-11-13 18:06:25 +0800853 return __req_need_defer(req);
Jens Axboe7adf4ea2019-10-10 21:42:58 -0600854
Bob Liu9d858b22019-11-13 18:06:25 +0800855 return false;
Jens Axboe7adf4ea2019-10-10 21:42:58 -0600856}
857
858static struct io_kiocb *io_get_deferred_req(struct io_ring_ctx *ctx)
Jens Axboede0617e2019-04-06 21:51:27 -0600859{
860 struct io_kiocb *req;
861
Jens Axboe7adf4ea2019-10-10 21:42:58 -0600862 req = list_first_entry_or_null(&ctx->defer_list, struct io_kiocb, list);
Bob Liu9d858b22019-11-13 18:06:25 +0800863 if (req && !req_need_defer(req)) {
Jens Axboede0617e2019-04-06 21:51:27 -0600864 list_del_init(&req->list);
865 return req;
866 }
867
868 return NULL;
869}
870
Jens Axboe5262f562019-09-17 12:26:57 -0600871static struct io_kiocb *io_get_timeout_req(struct io_ring_ctx *ctx)
872{
Jens Axboe7adf4ea2019-10-10 21:42:58 -0600873 struct io_kiocb *req;
874
875 req = list_first_entry_or_null(&ctx->timeout_list, struct io_kiocb, list);
Jens Axboe93bd25b2019-11-11 23:34:31 -0700876 if (req) {
877 if (req->flags & REQ_F_TIMEOUT_NOSEQ)
878 return NULL;
Linus Torvaldsfb4b3d32019-11-25 10:40:27 -0800879 if (!__req_need_defer(req)) {
Jens Axboe93bd25b2019-11-11 23:34:31 -0700880 list_del_init(&req->list);
881 return req;
882 }
Jens Axboe7adf4ea2019-10-10 21:42:58 -0600883 }
884
885 return NULL;
Jens Axboe5262f562019-09-17 12:26:57 -0600886}
887
Jens Axboede0617e2019-04-06 21:51:27 -0600888static void __io_commit_cqring(struct io_ring_ctx *ctx)
Jens Axboe2b188cc2019-01-07 10:46:33 -0700889{
Hristo Venev75b28af2019-08-26 17:23:46 +0000890 struct io_rings *rings = ctx->rings;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700891
Pavel Begunkov07910152020-01-17 03:52:46 +0300892 /* order cqe stores with ring update */
893 smp_store_release(&rings->cq.tail, ctx->cached_cq_tail);
Jens Axboe2b188cc2019-01-07 10:46:33 -0700894
Pavel Begunkov07910152020-01-17 03:52:46 +0300895 if (wq_has_sleeper(&ctx->cq_wait)) {
896 wake_up_interruptible(&ctx->cq_wait);
897 kill_fasync(&ctx->cq_fasync, SIGIO, POLL_IN);
Jens Axboe2b188cc2019-01-07 10:46:33 -0700898 }
899}
900
Jens Axboecccf0ee2020-01-27 16:34:48 -0700901static inline void io_req_work_grab_env(struct io_kiocb *req,
902 const struct io_op_def *def)
Jens Axboe18d9be12019-09-10 09:13:05 -0600903{
Jens Axboecccf0ee2020-01-27 16:34:48 -0700904 if (!req->work.mm && def->needs_mm) {
905 mmgrab(current->mm);
906 req->work.mm = current->mm;
907 }
908 if (!req->work.creds)
909 req->work.creds = get_current_cred();
910}
911
912static inline void io_req_work_drop_env(struct io_kiocb *req)
913{
914 if (req->work.mm) {
915 mmdrop(req->work.mm);
916 req->work.mm = NULL;
917 }
918 if (req->work.creds) {
919 put_cred(req->work.creds);
920 req->work.creds = NULL;
921 }
Jens Axboe561fb042019-10-24 07:25:42 -0600922}
923
Jens Axboe94ae5e72019-11-14 19:39:52 -0700924static inline bool io_prep_async_work(struct io_kiocb *req,
925 struct io_kiocb **link)
Jens Axboe561fb042019-10-24 07:25:42 -0600926{
Jens Axboed3656342019-12-18 09:50:26 -0700927 const struct io_op_def *def = &io_op_defs[req->opcode];
Jens Axboe561fb042019-10-24 07:25:42 -0600928 bool do_hashed = false;
Jens Axboe54a91f32019-09-10 09:15:04 -0600929
Jens Axboed3656342019-12-18 09:50:26 -0700930 if (req->flags & REQ_F_ISREG) {
931 if (def->hash_reg_file)
Jens Axboe3529d8c2019-12-19 18:24:38 -0700932 do_hashed = true;
Jens Axboed3656342019-12-18 09:50:26 -0700933 } else {
934 if (def->unbound_nonreg_file)
Jens Axboe3529d8c2019-12-19 18:24:38 -0700935 req->work.flags |= IO_WQ_WORK_UNBOUND;
Jens Axboe54a91f32019-09-10 09:15:04 -0600936 }
Jens Axboecccf0ee2020-01-27 16:34:48 -0700937
938 io_req_work_grab_env(req, def);
Jens Axboe54a91f32019-09-10 09:15:04 -0600939
Jens Axboe94ae5e72019-11-14 19:39:52 -0700940 *link = io_prep_linked_timeout(req);
Jens Axboe561fb042019-10-24 07:25:42 -0600941 return do_hashed;
942}
943
Jackie Liua197f662019-11-08 08:09:12 -0700944static inline void io_queue_async_work(struct io_kiocb *req)
Jens Axboe561fb042019-10-24 07:25:42 -0600945{
Jackie Liua197f662019-11-08 08:09:12 -0700946 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe94ae5e72019-11-14 19:39:52 -0700947 struct io_kiocb *link;
948 bool do_hashed;
949
950 do_hashed = io_prep_async_work(req, &link);
Jens Axboe561fb042019-10-24 07:25:42 -0600951
952 trace_io_uring_queue_async_work(ctx, do_hashed, req, &req->work,
953 req->flags);
954 if (!do_hashed) {
955 io_wq_enqueue(ctx->io_wq, &req->work);
956 } else {
957 io_wq_enqueue_hashed(ctx->io_wq, &req->work,
958 file_inode(req->file));
959 }
Jens Axboe94ae5e72019-11-14 19:39:52 -0700960
961 if (link)
962 io_queue_linked_timeout(link);
Jens Axboe18d9be12019-09-10 09:13:05 -0600963}
964
Jens Axboe5262f562019-09-17 12:26:57 -0600965static void io_kill_timeout(struct io_kiocb *req)
966{
967 int ret;
968
Jens Axboe2d283902019-12-04 11:08:05 -0700969 ret = hrtimer_try_to_cancel(&req->io->timeout.timer);
Jens Axboe5262f562019-09-17 12:26:57 -0600970 if (ret != -1) {
971 atomic_inc(&req->ctx->cq_timeouts);
Jens Axboe842f9612019-10-29 12:34:10 -0600972 list_del_init(&req->list);
Jens Axboe78e19bb2019-11-06 15:21:34 -0700973 io_cqring_fill_event(req, 0);
Jackie Liuec9c02a2019-11-08 23:50:36 +0800974 io_put_req(req);
Jens Axboe5262f562019-09-17 12:26:57 -0600975 }
976}
977
978static void io_kill_timeouts(struct io_ring_ctx *ctx)
979{
980 struct io_kiocb *req, *tmp;
981
982 spin_lock_irq(&ctx->completion_lock);
983 list_for_each_entry_safe(req, tmp, &ctx->timeout_list, list)
984 io_kill_timeout(req);
985 spin_unlock_irq(&ctx->completion_lock);
986}
987
Jens Axboede0617e2019-04-06 21:51:27 -0600988static void io_commit_cqring(struct io_ring_ctx *ctx)
989{
990 struct io_kiocb *req;
991
Jens Axboe5262f562019-09-17 12:26:57 -0600992 while ((req = io_get_timeout_req(ctx)) != NULL)
993 io_kill_timeout(req);
994
Jens Axboede0617e2019-04-06 21:51:27 -0600995 __io_commit_cqring(ctx);
996
Pavel Begunkov87987892020-01-18 01:22:30 +0300997 while ((req = io_get_deferred_req(ctx)) != NULL)
Jackie Liua197f662019-11-08 08:09:12 -0700998 io_queue_async_work(req);
Jens Axboede0617e2019-04-06 21:51:27 -0600999}
1000
Jens Axboe2b188cc2019-01-07 10:46:33 -07001001static struct io_uring_cqe *io_get_cqring(struct io_ring_ctx *ctx)
1002{
Hristo Venev75b28af2019-08-26 17:23:46 +00001003 struct io_rings *rings = ctx->rings;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001004 unsigned tail;
1005
1006 tail = ctx->cached_cq_tail;
Stefan Bühler115e12e2019-04-24 23:54:18 +02001007 /*
1008 * writes to the cq entry need to come after reading head; the
1009 * control dependency is enough as we're using WRITE_ONCE to
1010 * fill the cq entry
1011 */
Hristo Venev75b28af2019-08-26 17:23:46 +00001012 if (tail - READ_ONCE(rings->cq.head) == rings->cq_ring_entries)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001013 return NULL;
1014
1015 ctx->cached_cq_tail++;
Hristo Venev75b28af2019-08-26 17:23:46 +00001016 return &rings->cqes[tail & ctx->cq_mask];
Jens Axboe2b188cc2019-01-07 10:46:33 -07001017}
1018
Jens Axboef2842ab2020-01-08 11:04:00 -07001019static inline bool io_should_trigger_evfd(struct io_ring_ctx *ctx)
1020{
Jens Axboef0b493e2020-02-01 21:30:11 -07001021 if (!ctx->cq_ev_fd)
1022 return false;
Jens Axboef2842ab2020-01-08 11:04:00 -07001023 if (!ctx->eventfd_async)
1024 return true;
1025 return io_wq_current_is_worker() || in_interrupt();
1026}
1027
Jens Axboef0b493e2020-02-01 21:30:11 -07001028static void __io_cqring_ev_posted(struct io_ring_ctx *ctx, bool trigger_ev)
Jens Axboe8c838782019-03-12 15:48:16 -06001029{
1030 if (waitqueue_active(&ctx->wait))
1031 wake_up(&ctx->wait);
1032 if (waitqueue_active(&ctx->sqo_wait))
1033 wake_up(&ctx->sqo_wait);
Jens Axboef0b493e2020-02-01 21:30:11 -07001034 if (trigger_ev)
Jens Axboe9b402842019-04-11 11:45:41 -06001035 eventfd_signal(ctx->cq_ev_fd, 1);
Jens Axboe8c838782019-03-12 15:48:16 -06001036}
1037
Jens Axboef0b493e2020-02-01 21:30:11 -07001038static void io_cqring_ev_posted(struct io_ring_ctx *ctx)
1039{
1040 __io_cqring_ev_posted(ctx, io_should_trigger_evfd(ctx));
1041}
1042
Jens Axboec4a2ed72019-11-21 21:01:26 -07001043/* Returns true if there are no backlogged entries after the flush */
1044static bool io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001045{
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001046 struct io_rings *rings = ctx->rings;
1047 struct io_uring_cqe *cqe;
1048 struct io_kiocb *req;
1049 unsigned long flags;
1050 LIST_HEAD(list);
1051
1052 if (!force) {
1053 if (list_empty_careful(&ctx->cq_overflow_list))
Jens Axboec4a2ed72019-11-21 21:01:26 -07001054 return true;
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001055 if ((ctx->cached_cq_tail - READ_ONCE(rings->cq.head) ==
1056 rings->cq_ring_entries))
Jens Axboec4a2ed72019-11-21 21:01:26 -07001057 return false;
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001058 }
1059
1060 spin_lock_irqsave(&ctx->completion_lock, flags);
1061
1062 /* if force is set, the ring is going away. always drop after that */
1063 if (force)
Jens Axboe69b3e542020-01-08 11:01:46 -07001064 ctx->cq_overflow_flushed = 1;
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001065
Jens Axboec4a2ed72019-11-21 21:01:26 -07001066 cqe = NULL;
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001067 while (!list_empty(&ctx->cq_overflow_list)) {
1068 cqe = io_get_cqring(ctx);
1069 if (!cqe && !force)
1070 break;
1071
1072 req = list_first_entry(&ctx->cq_overflow_list, struct io_kiocb,
1073 list);
1074 list_move(&req->list, &list);
1075 if (cqe) {
1076 WRITE_ONCE(cqe->user_data, req->user_data);
1077 WRITE_ONCE(cqe->res, req->result);
1078 WRITE_ONCE(cqe->flags, 0);
1079 } else {
1080 WRITE_ONCE(ctx->rings->cq_overflow,
1081 atomic_inc_return(&ctx->cached_cq_overflow));
1082 }
1083 }
1084
1085 io_commit_cqring(ctx);
Jens Axboead3eb2c2019-12-18 17:12:20 -07001086 if (cqe) {
1087 clear_bit(0, &ctx->sq_check_overflow);
1088 clear_bit(0, &ctx->cq_check_overflow);
1089 }
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001090 spin_unlock_irqrestore(&ctx->completion_lock, flags);
1091 io_cqring_ev_posted(ctx);
1092
1093 while (!list_empty(&list)) {
1094 req = list_first_entry(&list, struct io_kiocb, list);
1095 list_del(&req->list);
Jackie Liuec9c02a2019-11-08 23:50:36 +08001096 io_put_req(req);
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001097 }
Jens Axboec4a2ed72019-11-21 21:01:26 -07001098
1099 return cqe != NULL;
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001100}
1101
Jens Axboe78e19bb2019-11-06 15:21:34 -07001102static void io_cqring_fill_event(struct io_kiocb *req, long res)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001103{
Jens Axboe78e19bb2019-11-06 15:21:34 -07001104 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001105 struct io_uring_cqe *cqe;
1106
Jens Axboe78e19bb2019-11-06 15:21:34 -07001107 trace_io_uring_complete(ctx, req->user_data, res);
Jens Axboe51c3ff62019-11-03 06:52:50 -07001108
Jens Axboe2b188cc2019-01-07 10:46:33 -07001109 /*
1110 * If we can't get a cq entry, userspace overflowed the
1111 * submission (by quite a lot). Increment the overflow count in
1112 * the ring.
1113 */
1114 cqe = io_get_cqring(ctx);
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001115 if (likely(cqe)) {
Jens Axboe78e19bb2019-11-06 15:21:34 -07001116 WRITE_ONCE(cqe->user_data, req->user_data);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001117 WRITE_ONCE(cqe->res, res);
1118 WRITE_ONCE(cqe->flags, 0);
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001119 } else if (ctx->cq_overflow_flushed) {
Jens Axboe2b188cc2019-01-07 10:46:33 -07001120 WRITE_ONCE(ctx->rings->cq_overflow,
1121 atomic_inc_return(&ctx->cached_cq_overflow));
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001122 } else {
Jens Axboead3eb2c2019-12-18 17:12:20 -07001123 if (list_empty(&ctx->cq_overflow_list)) {
1124 set_bit(0, &ctx->sq_check_overflow);
1125 set_bit(0, &ctx->cq_check_overflow);
1126 }
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001127 refcount_inc(&req->refs);
1128 req->result = res;
1129 list_add_tail(&req->list, &ctx->cq_overflow_list);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001130 }
1131}
1132
Jens Axboe78e19bb2019-11-06 15:21:34 -07001133static void io_cqring_add_event(struct io_kiocb *req, long res)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001134{
Jens Axboe78e19bb2019-11-06 15:21:34 -07001135 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001136 unsigned long flags;
1137
1138 spin_lock_irqsave(&ctx->completion_lock, flags);
Jens Axboe78e19bb2019-11-06 15:21:34 -07001139 io_cqring_fill_event(req, res);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001140 io_commit_cqring(ctx);
1141 spin_unlock_irqrestore(&ctx->completion_lock, flags);
1142
Jens Axboe8c838782019-03-12 15:48:16 -06001143 io_cqring_ev_posted(ctx);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001144}
1145
Jens Axboe0ddf92e2019-11-08 08:52:53 -07001146static inline bool io_is_fallback_req(struct io_kiocb *req)
1147{
1148 return req == (struct io_kiocb *)
1149 ((unsigned long) req->ctx->fallback_req & ~1UL);
1150}
1151
1152static struct io_kiocb *io_get_fallback_req(struct io_ring_ctx *ctx)
1153{
1154 struct io_kiocb *req;
1155
1156 req = ctx->fallback_req;
1157 if (!test_and_set_bit_lock(0, (unsigned long *) ctx->fallback_req))
1158 return req;
1159
1160 return NULL;
1161}
1162
Jens Axboe2579f912019-01-09 09:10:43 -07001163static struct io_kiocb *io_get_req(struct io_ring_ctx *ctx,
1164 struct io_submit_state *state)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001165{
Jens Axboefd6fab22019-03-14 16:30:06 -06001166 gfp_t gfp = GFP_KERNEL | __GFP_NOWARN;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001167 struct io_kiocb *req;
1168
Jens Axboe2579f912019-01-09 09:10:43 -07001169 if (!state) {
Jens Axboefd6fab22019-03-14 16:30:06 -06001170 req = kmem_cache_alloc(req_cachep, gfp);
Jens Axboe2579f912019-01-09 09:10:43 -07001171 if (unlikely(!req))
Jens Axboe0ddf92e2019-11-08 08:52:53 -07001172 goto fallback;
Jens Axboe2579f912019-01-09 09:10:43 -07001173 } else if (!state->free_reqs) {
1174 size_t sz;
1175 int ret;
1176
1177 sz = min_t(size_t, state->ios_left, ARRAY_SIZE(state->reqs));
Jens Axboefd6fab22019-03-14 16:30:06 -06001178 ret = kmem_cache_alloc_bulk(req_cachep, gfp, sz, state->reqs);
1179
1180 /*
1181 * Bulk alloc is all-or-nothing. If we fail to get a batch,
1182 * retry single alloc to be on the safe side.
1183 */
1184 if (unlikely(ret <= 0)) {
1185 state->reqs[0] = kmem_cache_alloc(req_cachep, gfp);
1186 if (!state->reqs[0])
Jens Axboe0ddf92e2019-11-08 08:52:53 -07001187 goto fallback;
Jens Axboefd6fab22019-03-14 16:30:06 -06001188 ret = 1;
1189 }
Jens Axboe2579f912019-01-09 09:10:43 -07001190 state->free_reqs = ret - 1;
Pavel Begunkov6c8a3132020-02-01 03:58:00 +03001191 req = state->reqs[ret - 1];
Jens Axboe2579f912019-01-09 09:10:43 -07001192 } else {
Jens Axboe2579f912019-01-09 09:10:43 -07001193 state->free_reqs--;
Pavel Begunkov6c8a3132020-02-01 03:58:00 +03001194 req = state->reqs[state->free_reqs];
Jens Axboe2b188cc2019-01-07 10:46:33 -07001195 }
1196
Jens Axboe0ddf92e2019-11-08 08:52:53 -07001197got_it:
Jens Axboe1a6b74f2019-12-02 10:33:15 -07001198 req->io = NULL;
Jens Axboe60c112b2019-06-21 10:20:18 -06001199 req->file = NULL;
Jens Axboe2579f912019-01-09 09:10:43 -07001200 req->ctx = ctx;
1201 req->flags = 0;
Jens Axboee65ef562019-03-12 10:16:44 -06001202 /* one is dropped after submission, the other at completion */
1203 refcount_set(&req->refs, 2);
Jens Axboe9e645e112019-05-10 16:07:28 -06001204 req->result = 0;
Jens Axboe561fb042019-10-24 07:25:42 -06001205 INIT_IO_WORK(&req->work, io_wq_submit_work);
Jens Axboe2579f912019-01-09 09:10:43 -07001206 return req;
Jens Axboe0ddf92e2019-11-08 08:52:53 -07001207fallback:
1208 req = io_get_fallback_req(ctx);
1209 if (req)
1210 goto got_it;
Pavel Begunkov6805b322019-10-08 02:18:42 +03001211 percpu_ref_put(&ctx->refs);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001212 return NULL;
1213}
1214
Pavel Begunkov2b85edf2019-12-28 14:13:03 +03001215static void __io_req_do_free(struct io_kiocb *req)
Jens Axboedef596e2019-01-09 08:59:42 -07001216{
Pavel Begunkov2b85edf2019-12-28 14:13:03 +03001217 if (likely(!io_is_fallback_req(req)))
1218 kmem_cache_free(req_cachep, req);
1219 else
1220 clear_bit_unlock(0, (unsigned long *) req->ctx->fallback_req);
1221}
1222
Jens Axboec6ca97b302019-12-28 12:11:08 -07001223static void __io_req_aux_free(struct io_kiocb *req)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001224{
Jens Axboefcb323c2019-10-24 12:39:47 -06001225 struct io_ring_ctx *ctx = req->ctx;
1226
YueHaibing96fd84d2020-01-07 22:22:44 +08001227 kfree(req->io);
Jens Axboe05f3fb32019-12-09 11:22:50 -07001228 if (req->file) {
1229 if (req->flags & REQ_F_FIXED_FILE)
1230 percpu_ref_put(&ctx->file_data->refs);
1231 else
1232 fput(req->file);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001233 }
Jens Axboecccf0ee2020-01-27 16:34:48 -07001234
1235 io_req_work_drop_env(req);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001236}
1237
1238static void __io_free_req(struct io_kiocb *req)
1239{
Jens Axboec6ca97b302019-12-28 12:11:08 -07001240 __io_req_aux_free(req);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001241
Pavel Begunkov99bc4c32020-02-07 22:04:45 +03001242 if (req->flags & REQ_F_NEED_CLEANUP)
1243 io_cleanup_req(req);
1244
Jens Axboefcb323c2019-10-24 12:39:47 -06001245 if (req->flags & REQ_F_INFLIGHT) {
Jens Axboec6ca97b302019-12-28 12:11:08 -07001246 struct io_ring_ctx *ctx = req->ctx;
Jens Axboefcb323c2019-10-24 12:39:47 -06001247 unsigned long flags;
1248
1249 spin_lock_irqsave(&ctx->inflight_lock, flags);
1250 list_del(&req->inflight_entry);
1251 if (waitqueue_active(&ctx->inflight_wait))
1252 wake_up(&ctx->inflight_wait);
1253 spin_unlock_irqrestore(&ctx->inflight_lock, flags);
1254 }
Pavel Begunkov2b85edf2019-12-28 14:13:03 +03001255
1256 percpu_ref_put(&req->ctx->refs);
1257 __io_req_do_free(req);
Jens Axboee65ef562019-03-12 10:16:44 -06001258}
1259
Jens Axboec6ca97b302019-12-28 12:11:08 -07001260struct req_batch {
1261 void *reqs[IO_IOPOLL_BATCH];
1262 int to_free;
1263 int need_iter;
1264};
1265
1266static void io_free_req_many(struct io_ring_ctx *ctx, struct req_batch *rb)
1267{
Jens Axboe10fef4b2020-01-09 07:52:28 -07001268 int fixed_refs = rb->to_free;
1269
Jens Axboec6ca97b302019-12-28 12:11:08 -07001270 if (!rb->to_free)
1271 return;
1272 if (rb->need_iter) {
1273 int i, inflight = 0;
1274 unsigned long flags;
1275
Jens Axboe10fef4b2020-01-09 07:52:28 -07001276 fixed_refs = 0;
Jens Axboec6ca97b302019-12-28 12:11:08 -07001277 for (i = 0; i < rb->to_free; i++) {
1278 struct io_kiocb *req = rb->reqs[i];
1279
Jens Axboe10fef4b2020-01-09 07:52:28 -07001280 if (req->flags & REQ_F_FIXED_FILE) {
Jens Axboec6ca97b302019-12-28 12:11:08 -07001281 req->file = NULL;
Jens Axboe10fef4b2020-01-09 07:52:28 -07001282 fixed_refs++;
1283 }
Jens Axboec6ca97b302019-12-28 12:11:08 -07001284 if (req->flags & REQ_F_INFLIGHT)
1285 inflight++;
Jens Axboec6ca97b302019-12-28 12:11:08 -07001286 __io_req_aux_free(req);
1287 }
1288 if (!inflight)
1289 goto do_free;
1290
1291 spin_lock_irqsave(&ctx->inflight_lock, flags);
1292 for (i = 0; i < rb->to_free; i++) {
1293 struct io_kiocb *req = rb->reqs[i];
1294
Jens Axboe10fef4b2020-01-09 07:52:28 -07001295 if (req->flags & REQ_F_INFLIGHT) {
Jens Axboec6ca97b302019-12-28 12:11:08 -07001296 list_del(&req->inflight_entry);
1297 if (!--inflight)
1298 break;
1299 }
1300 }
1301 spin_unlock_irqrestore(&ctx->inflight_lock, flags);
1302
1303 if (waitqueue_active(&ctx->inflight_wait))
1304 wake_up(&ctx->inflight_wait);
1305 }
1306do_free:
1307 kmem_cache_free_bulk(req_cachep, rb->to_free, rb->reqs);
Jens Axboe10fef4b2020-01-09 07:52:28 -07001308 if (fixed_refs)
1309 percpu_ref_put_many(&ctx->file_data->refs, fixed_refs);
Jens Axboec6ca97b302019-12-28 12:11:08 -07001310 percpu_ref_put_many(&ctx->refs, rb->to_free);
Jens Axboec6ca97b302019-12-28 12:11:08 -07001311 rb->to_free = rb->need_iter = 0;
Jens Axboee65ef562019-03-12 10:16:44 -06001312}
1313
Jackie Liua197f662019-11-08 08:09:12 -07001314static bool io_link_cancel_timeout(struct io_kiocb *req)
Jens Axboe9e645e112019-05-10 16:07:28 -06001315{
Jackie Liua197f662019-11-08 08:09:12 -07001316 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe2665abf2019-11-05 12:40:47 -07001317 int ret;
1318
Jens Axboe2d283902019-12-04 11:08:05 -07001319 ret = hrtimer_try_to_cancel(&req->io->timeout.timer);
Jens Axboe2665abf2019-11-05 12:40:47 -07001320 if (ret != -1) {
Jens Axboe78e19bb2019-11-06 15:21:34 -07001321 io_cqring_fill_event(req, -ECANCELED);
Jens Axboe2665abf2019-11-05 12:40:47 -07001322 io_commit_cqring(ctx);
1323 req->flags &= ~REQ_F_LINK;
Jackie Liuec9c02a2019-11-08 23:50:36 +08001324 io_put_req(req);
Jens Axboe2665abf2019-11-05 12:40:47 -07001325 return true;
1326 }
1327
1328 return false;
1329}
1330
Jens Axboeba816ad2019-09-28 11:36:45 -06001331static void io_req_link_next(struct io_kiocb *req, struct io_kiocb **nxtptr)
Jens Axboe9e645e112019-05-10 16:07:28 -06001332{
Jens Axboe2665abf2019-11-05 12:40:47 -07001333 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe2665abf2019-11-05 12:40:47 -07001334 bool wake_ev = false;
Jens Axboe9e645e112019-05-10 16:07:28 -06001335
Jens Axboe4d7dd462019-11-20 13:03:52 -07001336 /* Already got next link */
1337 if (req->flags & REQ_F_LINK_NEXT)
1338 return;
1339
Jens Axboe9e645e112019-05-10 16:07:28 -06001340 /*
1341 * The list should never be empty when we are called here. But could
1342 * potentially happen if the chain is messed up, check to be on the
1343 * safe side.
1344 */
Pavel Begunkov44932332019-12-05 16:16:35 +03001345 while (!list_empty(&req->link_list)) {
1346 struct io_kiocb *nxt = list_first_entry(&req->link_list,
1347 struct io_kiocb, link_list);
Jens Axboe94ae5e72019-11-14 19:39:52 -07001348
Pavel Begunkov44932332019-12-05 16:16:35 +03001349 if (unlikely((req->flags & REQ_F_LINK_TIMEOUT) &&
1350 (nxt->flags & REQ_F_TIMEOUT))) {
1351 list_del_init(&nxt->link_list);
Jens Axboe94ae5e72019-11-14 19:39:52 -07001352 wake_ev |= io_link_cancel_timeout(nxt);
Jens Axboe94ae5e72019-11-14 19:39:52 -07001353 req->flags &= ~REQ_F_LINK_TIMEOUT;
1354 continue;
1355 }
Jens Axboe9e645e112019-05-10 16:07:28 -06001356
Pavel Begunkov44932332019-12-05 16:16:35 +03001357 list_del_init(&req->link_list);
1358 if (!list_empty(&nxt->link_list))
1359 nxt->flags |= REQ_F_LINK;
Pavel Begunkovb18fdf72019-11-21 23:21:02 +03001360 *nxtptr = nxt;
Jens Axboe94ae5e72019-11-14 19:39:52 -07001361 break;
Jens Axboe9e645e112019-05-10 16:07:28 -06001362 }
Jens Axboe2665abf2019-11-05 12:40:47 -07001363
Jens Axboe4d7dd462019-11-20 13:03:52 -07001364 req->flags |= REQ_F_LINK_NEXT;
Jens Axboe2665abf2019-11-05 12:40:47 -07001365 if (wake_ev)
1366 io_cqring_ev_posted(ctx);
Jens Axboe9e645e112019-05-10 16:07:28 -06001367}
1368
1369/*
1370 * Called if REQ_F_LINK is set, and we fail the head request
1371 */
1372static void io_fail_links(struct io_kiocb *req)
1373{
Jens Axboe2665abf2019-11-05 12:40:47 -07001374 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe2665abf2019-11-05 12:40:47 -07001375 unsigned long flags;
1376
1377 spin_lock_irqsave(&ctx->completion_lock, flags);
Jens Axboe9e645e112019-05-10 16:07:28 -06001378
1379 while (!list_empty(&req->link_list)) {
Pavel Begunkov44932332019-12-05 16:16:35 +03001380 struct io_kiocb *link = list_first_entry(&req->link_list,
1381 struct io_kiocb, link_list);
Jens Axboe9e645e112019-05-10 16:07:28 -06001382
Pavel Begunkov44932332019-12-05 16:16:35 +03001383 list_del_init(&link->link_list);
Dmitrii Dolgovc826bd72019-10-15 19:02:01 +02001384 trace_io_uring_fail_link(req, link);
Jens Axboe2665abf2019-11-05 12:40:47 -07001385
1386 if ((req->flags & REQ_F_LINK_TIMEOUT) &&
Jens Axboed625c6e2019-12-17 19:53:05 -07001387 link->opcode == IORING_OP_LINK_TIMEOUT) {
Jackie Liua197f662019-11-08 08:09:12 -07001388 io_link_cancel_timeout(link);
Jens Axboe2665abf2019-11-05 12:40:47 -07001389 } else {
Jens Axboe78e19bb2019-11-06 15:21:34 -07001390 io_cqring_fill_event(link, -ECANCELED);
Jens Axboe978db572019-11-14 22:39:04 -07001391 __io_double_put_req(link);
Jens Axboe2665abf2019-11-05 12:40:47 -07001392 }
Jens Axboe5d960722019-11-19 15:31:28 -07001393 req->flags &= ~REQ_F_LINK_TIMEOUT;
Jens Axboe9e645e112019-05-10 16:07:28 -06001394 }
Jens Axboe2665abf2019-11-05 12:40:47 -07001395
1396 io_commit_cqring(ctx);
1397 spin_unlock_irqrestore(&ctx->completion_lock, flags);
1398 io_cqring_ev_posted(ctx);
Jens Axboe9e645e112019-05-10 16:07:28 -06001399}
1400
Jens Axboe4d7dd462019-11-20 13:03:52 -07001401static void io_req_find_next(struct io_kiocb *req, struct io_kiocb **nxt)
Jens Axboe9e645e112019-05-10 16:07:28 -06001402{
Jens Axboe4d7dd462019-11-20 13:03:52 -07001403 if (likely(!(req->flags & REQ_F_LINK)))
Jens Axboe2665abf2019-11-05 12:40:47 -07001404 return;
Jens Axboe2665abf2019-11-05 12:40:47 -07001405
Jens Axboe9e645e112019-05-10 16:07:28 -06001406 /*
1407 * If LINK is set, we have dependent requests in this chain. If we
1408 * didn't fail this request, queue the first one up, moving any other
1409 * dependencies to the next request. In case of failure, fail the rest
1410 * of the chain.
1411 */
Jens Axboe2665abf2019-11-05 12:40:47 -07001412 if (req->flags & REQ_F_FAIL_LINK) {
1413 io_fail_links(req);
Jens Axboe7c9e7f02019-11-12 08:15:53 -07001414 } else if ((req->flags & (REQ_F_LINK_TIMEOUT | REQ_F_COMP_LOCKED)) ==
1415 REQ_F_LINK_TIMEOUT) {
Jens Axboe2665abf2019-11-05 12:40:47 -07001416 struct io_ring_ctx *ctx = req->ctx;
1417 unsigned long flags;
1418
1419 /*
1420 * If this is a timeout link, we could be racing with the
1421 * timeout timer. Grab the completion lock for this case to
Jens Axboe7c9e7f02019-11-12 08:15:53 -07001422 * protect against that.
Jens Axboe2665abf2019-11-05 12:40:47 -07001423 */
1424 spin_lock_irqsave(&ctx->completion_lock, flags);
1425 io_req_link_next(req, nxt);
1426 spin_unlock_irqrestore(&ctx->completion_lock, flags);
1427 } else {
1428 io_req_link_next(req, nxt);
Jens Axboe9e645e112019-05-10 16:07:28 -06001429 }
Jens Axboe4d7dd462019-11-20 13:03:52 -07001430}
Jens Axboe9e645e112019-05-10 16:07:28 -06001431
Jackie Liuc69f8db2019-11-09 11:00:08 +08001432static void io_free_req(struct io_kiocb *req)
1433{
Pavel Begunkov944e58b2019-11-21 23:21:01 +03001434 struct io_kiocb *nxt = NULL;
1435
1436 io_req_find_next(req, &nxt);
Pavel Begunkov70cf9f32019-11-21 23:21:00 +03001437 __io_free_req(req);
Pavel Begunkov944e58b2019-11-21 23:21:01 +03001438
1439 if (nxt)
1440 io_queue_async_work(nxt);
Jackie Liuc69f8db2019-11-09 11:00:08 +08001441}
1442
Jens Axboeba816ad2019-09-28 11:36:45 -06001443/*
1444 * Drop reference to request, return next in chain (if there is one) if this
1445 * was the last reference to this request.
1446 */
Pavel Begunkovf9bd67f2019-11-21 23:21:03 +03001447__attribute__((nonnull))
Jackie Liuec9c02a2019-11-08 23:50:36 +08001448static void io_put_req_find_next(struct io_kiocb *req, struct io_kiocb **nxtptr)
Jens Axboee65ef562019-03-12 10:16:44 -06001449{
Pavel Begunkovf9bd67f2019-11-21 23:21:03 +03001450 io_req_find_next(req, nxtptr);
Jens Axboe4d7dd462019-11-20 13:03:52 -07001451
Jens Axboee65ef562019-03-12 10:16:44 -06001452 if (refcount_dec_and_test(&req->refs))
Jens Axboe4d7dd462019-11-20 13:03:52 -07001453 __io_free_req(req);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001454}
1455
Jens Axboe2b188cc2019-01-07 10:46:33 -07001456static void io_put_req(struct io_kiocb *req)
1457{
Jens Axboedef596e2019-01-09 08:59:42 -07001458 if (refcount_dec_and_test(&req->refs))
1459 io_free_req(req);
1460}
1461
Jens Axboe978db572019-11-14 22:39:04 -07001462/*
1463 * Must only be used if we don't need to care about links, usually from
1464 * within the completion handling itself.
1465 */
1466static void __io_double_put_req(struct io_kiocb *req)
Jens Axboea3a0e432019-08-20 11:03:11 -06001467{
Jens Axboe78e19bb2019-11-06 15:21:34 -07001468 /* drop both submit and complete references */
1469 if (refcount_sub_and_test(2, &req->refs))
1470 __io_free_req(req);
1471}
1472
Jens Axboe978db572019-11-14 22:39:04 -07001473static void io_double_put_req(struct io_kiocb *req)
1474{
1475 /* drop both submit and complete references */
1476 if (refcount_sub_and_test(2, &req->refs))
1477 io_free_req(req);
1478}
1479
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001480static unsigned io_cqring_events(struct io_ring_ctx *ctx, bool noflush)
Jens Axboea3a0e432019-08-20 11:03:11 -06001481{
Jens Axboe84f97dc2019-11-06 11:27:53 -07001482 struct io_rings *rings = ctx->rings;
1483
Jens Axboead3eb2c2019-12-18 17:12:20 -07001484 if (test_bit(0, &ctx->cq_check_overflow)) {
1485 /*
1486 * noflush == true is from the waitqueue handler, just ensure
1487 * we wake up the task, and the next invocation will flush the
1488 * entries. We cannot safely to it from here.
1489 */
1490 if (noflush && !list_empty(&ctx->cq_overflow_list))
1491 return -1U;
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001492
Jens Axboead3eb2c2019-12-18 17:12:20 -07001493 io_cqring_overflow_flush(ctx, false);
1494 }
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001495
Jens Axboea3a0e432019-08-20 11:03:11 -06001496 /* See comment at the top of this file */
1497 smp_rmb();
Jens Axboead3eb2c2019-12-18 17:12:20 -07001498 return ctx->cached_cq_tail - READ_ONCE(rings->cq.head);
Jens Axboea3a0e432019-08-20 11:03:11 -06001499}
1500
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03001501static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx)
1502{
1503 struct io_rings *rings = ctx->rings;
1504
1505 /* make sure SQ entry isn't read before tail */
1506 return smp_load_acquire(&rings->sq.tail) - ctx->cached_sq_head;
1507}
1508
Jens Axboe8237e042019-12-28 10:48:22 -07001509static inline bool io_req_multi_free(struct req_batch *rb, struct io_kiocb *req)
Jens Axboee94f1412019-12-19 12:06:02 -07001510{
Jens Axboec6ca97b302019-12-28 12:11:08 -07001511 if ((req->flags & REQ_F_LINK) || io_is_fallback_req(req))
1512 return false;
Jens Axboee94f1412019-12-19 12:06:02 -07001513
Jens Axboec6ca97b302019-12-28 12:11:08 -07001514 if (!(req->flags & REQ_F_FIXED_FILE) || req->io)
1515 rb->need_iter++;
1516
1517 rb->reqs[rb->to_free++] = req;
1518 if (unlikely(rb->to_free == ARRAY_SIZE(rb->reqs)))
1519 io_free_req_many(req->ctx, rb);
1520 return true;
Jens Axboee94f1412019-12-19 12:06:02 -07001521}
1522
Jens Axboedef596e2019-01-09 08:59:42 -07001523/*
1524 * Find and free completed poll iocbs
1525 */
1526static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events,
1527 struct list_head *done)
1528{
Jens Axboe8237e042019-12-28 10:48:22 -07001529 struct req_batch rb;
Jens Axboedef596e2019-01-09 08:59:42 -07001530 struct io_kiocb *req;
Jens Axboedef596e2019-01-09 08:59:42 -07001531
Jens Axboec6ca97b302019-12-28 12:11:08 -07001532 rb.to_free = rb.need_iter = 0;
Jens Axboedef596e2019-01-09 08:59:42 -07001533 while (!list_empty(done)) {
1534 req = list_first_entry(done, struct io_kiocb, list);
1535 list_del(&req->list);
1536
Jens Axboe78e19bb2019-11-06 15:21:34 -07001537 io_cqring_fill_event(req, req->result);
Jens Axboedef596e2019-01-09 08:59:42 -07001538 (*nr_events)++;
1539
Jens Axboe8237e042019-12-28 10:48:22 -07001540 if (refcount_dec_and_test(&req->refs) &&
1541 !io_req_multi_free(&rb, req))
1542 io_free_req(req);
Jens Axboedef596e2019-01-09 08:59:42 -07001543 }
Jens Axboedef596e2019-01-09 08:59:42 -07001544
Jens Axboe09bb8392019-03-13 12:39:28 -06001545 io_commit_cqring(ctx);
Jens Axboe8237e042019-12-28 10:48:22 -07001546 io_free_req_many(ctx, &rb);
Jens Axboedef596e2019-01-09 08:59:42 -07001547}
1548
1549static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events,
1550 long min)
1551{
1552 struct io_kiocb *req, *tmp;
1553 LIST_HEAD(done);
1554 bool spin;
1555 int ret;
1556
1557 /*
1558 * Only spin for completions if we don't have multiple devices hanging
1559 * off our complete list, and we're under the requested amount.
1560 */
1561 spin = !ctx->poll_multi_file && *nr_events < min;
1562
1563 ret = 0;
1564 list_for_each_entry_safe(req, tmp, &ctx->poll_list, list) {
Jens Axboe9adbd452019-12-20 08:45:55 -07001565 struct kiocb *kiocb = &req->rw.kiocb;
Jens Axboedef596e2019-01-09 08:59:42 -07001566
1567 /*
1568 * Move completed entries to our local list. If we find a
1569 * request that requires polling, break out and complete
1570 * the done list first, if we have entries there.
1571 */
1572 if (req->flags & REQ_F_IOPOLL_COMPLETED) {
1573 list_move_tail(&req->list, &done);
1574 continue;
1575 }
1576 if (!list_empty(&done))
1577 break;
1578
1579 ret = kiocb->ki_filp->f_op->iopoll(kiocb, spin);
1580 if (ret < 0)
1581 break;
1582
1583 if (ret && spin)
1584 spin = false;
1585 ret = 0;
1586 }
1587
1588 if (!list_empty(&done))
1589 io_iopoll_complete(ctx, nr_events, &done);
1590
1591 return ret;
1592}
1593
1594/*
Brian Gianforcarod195a662019-12-13 03:09:50 -08001595 * Poll for a minimum of 'min' events. Note that if min == 0 we consider that a
Jens Axboedef596e2019-01-09 08:59:42 -07001596 * non-spinning poll check - we'll still enter the driver poll loop, but only
1597 * as a non-spinning completion check.
1598 */
1599static int io_iopoll_getevents(struct io_ring_ctx *ctx, unsigned int *nr_events,
1600 long min)
1601{
Jens Axboe08f54392019-08-21 22:19:11 -06001602 while (!list_empty(&ctx->poll_list) && !need_resched()) {
Jens Axboedef596e2019-01-09 08:59:42 -07001603 int ret;
1604
1605 ret = io_do_iopoll(ctx, nr_events, min);
1606 if (ret < 0)
1607 return ret;
1608 if (!min || *nr_events >= min)
1609 return 0;
1610 }
1611
1612 return 1;
1613}
1614
1615/*
1616 * We can't just wait for polled events to come to us, we have to actively
1617 * find and complete them.
1618 */
1619static void io_iopoll_reap_events(struct io_ring_ctx *ctx)
1620{
1621 if (!(ctx->flags & IORING_SETUP_IOPOLL))
1622 return;
1623
1624 mutex_lock(&ctx->uring_lock);
1625 while (!list_empty(&ctx->poll_list)) {
1626 unsigned int nr_events = 0;
1627
1628 io_iopoll_getevents(ctx, &nr_events, 1);
Jens Axboe08f54392019-08-21 22:19:11 -06001629
1630 /*
1631 * Ensure we allow local-to-the-cpu processing to take place,
1632 * in this case we need to ensure that we reap all events.
1633 */
1634 cond_resched();
Jens Axboedef596e2019-01-09 08:59:42 -07001635 }
1636 mutex_unlock(&ctx->uring_lock);
1637}
1638
Jens Axboe2b2ed972019-10-25 10:06:15 -06001639static int __io_iopoll_check(struct io_ring_ctx *ctx, unsigned *nr_events,
1640 long min)
Jens Axboedef596e2019-01-09 08:59:42 -07001641{
Jens Axboe2b2ed972019-10-25 10:06:15 -06001642 int iters = 0, ret = 0;
Jens Axboedef596e2019-01-09 08:59:42 -07001643
1644 do {
1645 int tmin = 0;
1646
Jens Axboe500f9fb2019-08-19 12:15:59 -06001647 /*
Jens Axboea3a0e432019-08-20 11:03:11 -06001648 * Don't enter poll loop if we already have events pending.
1649 * If we do, we can potentially be spinning for commands that
1650 * already triggered a CQE (eg in error).
1651 */
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001652 if (io_cqring_events(ctx, false))
Jens Axboea3a0e432019-08-20 11:03:11 -06001653 break;
1654
1655 /*
Jens Axboe500f9fb2019-08-19 12:15:59 -06001656 * If a submit got punted to a workqueue, we can have the
1657 * application entering polling for a command before it gets
1658 * issued. That app will hold the uring_lock for the duration
1659 * of the poll right here, so we need to take a breather every
1660 * now and then to ensure that the issue has a chance to add
1661 * the poll to the issued list. Otherwise we can spin here
1662 * forever, while the workqueue is stuck trying to acquire the
1663 * very same mutex.
1664 */
1665 if (!(++iters & 7)) {
1666 mutex_unlock(&ctx->uring_lock);
1667 mutex_lock(&ctx->uring_lock);
1668 }
1669
Jens Axboedef596e2019-01-09 08:59:42 -07001670 if (*nr_events < min)
1671 tmin = min - *nr_events;
1672
1673 ret = io_iopoll_getevents(ctx, nr_events, tmin);
1674 if (ret <= 0)
1675 break;
1676 ret = 0;
1677 } while (min && !*nr_events && !need_resched());
1678
Jens Axboe2b2ed972019-10-25 10:06:15 -06001679 return ret;
1680}
1681
1682static int io_iopoll_check(struct io_ring_ctx *ctx, unsigned *nr_events,
1683 long min)
1684{
1685 int ret;
1686
1687 /*
1688 * We disallow the app entering submit/complete with polling, but we
1689 * still need to lock the ring to prevent racing with polled issue
1690 * that got punted to a workqueue.
1691 */
1692 mutex_lock(&ctx->uring_lock);
1693 ret = __io_iopoll_check(ctx, nr_events, min);
Jens Axboe500f9fb2019-08-19 12:15:59 -06001694 mutex_unlock(&ctx->uring_lock);
Jens Axboedef596e2019-01-09 08:59:42 -07001695 return ret;
1696}
1697
Jens Axboe491381ce2019-10-17 09:20:46 -06001698static void kiocb_end_write(struct io_kiocb *req)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001699{
Jens Axboe491381ce2019-10-17 09:20:46 -06001700 /*
1701 * Tell lockdep we inherited freeze protection from submission
1702 * thread.
1703 */
1704 if (req->flags & REQ_F_ISREG) {
1705 struct inode *inode = file_inode(req->file);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001706
Jens Axboe491381ce2019-10-17 09:20:46 -06001707 __sb_writers_acquired(inode->i_sb, SB_FREEZE_WRITE);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001708 }
Jens Axboe491381ce2019-10-17 09:20:46 -06001709 file_end_write(req->file);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001710}
1711
Jens Axboe4e88d6e2019-12-07 20:59:47 -07001712static inline void req_set_fail_links(struct io_kiocb *req)
1713{
1714 if ((req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) == REQ_F_LINK)
1715 req->flags |= REQ_F_FAIL_LINK;
1716}
1717
Jens Axboeba816ad2019-09-28 11:36:45 -06001718static void io_complete_rw_common(struct kiocb *kiocb, long res)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001719{
Jens Axboe9adbd452019-12-20 08:45:55 -07001720 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001721
Jens Axboe491381ce2019-10-17 09:20:46 -06001722 if (kiocb->ki_flags & IOCB_WRITE)
1723 kiocb_end_write(req);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001724
Jens Axboe4e88d6e2019-12-07 20:59:47 -07001725 if (res != req->result)
1726 req_set_fail_links(req);
Jens Axboe78e19bb2019-11-06 15:21:34 -07001727 io_cqring_add_event(req, res);
Jens Axboeba816ad2019-09-28 11:36:45 -06001728}
1729
1730static void io_complete_rw(struct kiocb *kiocb, long res, long res2)
1731{
Jens Axboe9adbd452019-12-20 08:45:55 -07001732 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
Jens Axboeba816ad2019-09-28 11:36:45 -06001733
1734 io_complete_rw_common(kiocb, res);
Jens Axboee65ef562019-03-12 10:16:44 -06001735 io_put_req(req);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001736}
1737
Jens Axboeba816ad2019-09-28 11:36:45 -06001738static struct io_kiocb *__io_complete_rw(struct kiocb *kiocb, long res)
1739{
Jens Axboe9adbd452019-12-20 08:45:55 -07001740 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
Jackie Liuec9c02a2019-11-08 23:50:36 +08001741 struct io_kiocb *nxt = NULL;
Jens Axboeba816ad2019-09-28 11:36:45 -06001742
1743 io_complete_rw_common(kiocb, res);
Jackie Liuec9c02a2019-11-08 23:50:36 +08001744 io_put_req_find_next(req, &nxt);
1745
1746 return nxt;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001747}
1748
Jens Axboedef596e2019-01-09 08:59:42 -07001749static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2)
1750{
Jens Axboe9adbd452019-12-20 08:45:55 -07001751 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
Jens Axboedef596e2019-01-09 08:59:42 -07001752
Jens Axboe491381ce2019-10-17 09:20:46 -06001753 if (kiocb->ki_flags & IOCB_WRITE)
1754 kiocb_end_write(req);
Jens Axboedef596e2019-01-09 08:59:42 -07001755
Jens Axboe4e88d6e2019-12-07 20:59:47 -07001756 if (res != req->result)
1757 req_set_fail_links(req);
Jens Axboe9e645e112019-05-10 16:07:28 -06001758 req->result = res;
Jens Axboedef596e2019-01-09 08:59:42 -07001759 if (res != -EAGAIN)
1760 req->flags |= REQ_F_IOPOLL_COMPLETED;
1761}
1762
1763/*
1764 * After the iocb has been issued, it's safe to be found on the poll list.
1765 * Adding the kiocb to the list AFTER submission ensures that we don't
1766 * find it from a io_iopoll_getevents() thread before the issuer is done
1767 * accessing the kiocb cookie.
1768 */
1769static void io_iopoll_req_issued(struct io_kiocb *req)
1770{
1771 struct io_ring_ctx *ctx = req->ctx;
1772
1773 /*
1774 * Track whether we have multiple files in our lists. This will impact
1775 * how we do polling eventually, not spinning if we're on potentially
1776 * different devices.
1777 */
1778 if (list_empty(&ctx->poll_list)) {
1779 ctx->poll_multi_file = false;
1780 } else if (!ctx->poll_multi_file) {
1781 struct io_kiocb *list_req;
1782
1783 list_req = list_first_entry(&ctx->poll_list, struct io_kiocb,
1784 list);
Jens Axboe9adbd452019-12-20 08:45:55 -07001785 if (list_req->file != req->file)
Jens Axboedef596e2019-01-09 08:59:42 -07001786 ctx->poll_multi_file = true;
1787 }
1788
1789 /*
1790 * For fast devices, IO may have already completed. If it has, add
1791 * it to the front so we find it first.
1792 */
1793 if (req->flags & REQ_F_IOPOLL_COMPLETED)
1794 list_add(&req->list, &ctx->poll_list);
1795 else
1796 list_add_tail(&req->list, &ctx->poll_list);
1797}
1798
Jens Axboe3d6770f2019-04-13 11:50:54 -06001799static void io_file_put(struct io_submit_state *state)
Jens Axboe9a56a232019-01-09 09:06:50 -07001800{
Jens Axboe3d6770f2019-04-13 11:50:54 -06001801 if (state->file) {
Jens Axboe9a56a232019-01-09 09:06:50 -07001802 int diff = state->has_refs - state->used_refs;
1803
1804 if (diff)
1805 fput_many(state->file, diff);
1806 state->file = NULL;
1807 }
1808}
1809
1810/*
1811 * Get as many references to a file as we have IOs left in this submission,
1812 * assuming most submissions are for one file, or at least that each file
1813 * has more than one submission.
1814 */
1815static struct file *io_file_get(struct io_submit_state *state, int fd)
1816{
1817 if (!state)
1818 return fget(fd);
1819
1820 if (state->file) {
1821 if (state->fd == fd) {
1822 state->used_refs++;
1823 state->ios_left--;
1824 return state->file;
1825 }
Jens Axboe3d6770f2019-04-13 11:50:54 -06001826 io_file_put(state);
Jens Axboe9a56a232019-01-09 09:06:50 -07001827 }
1828 state->file = fget_many(fd, state->ios_left);
1829 if (!state->file)
1830 return NULL;
1831
1832 state->fd = fd;
1833 state->has_refs = state->ios_left;
1834 state->used_refs = 1;
1835 state->ios_left--;
1836 return state->file;
1837}
1838
Jens Axboe2b188cc2019-01-07 10:46:33 -07001839/*
1840 * If we tracked the file through the SCM inflight mechanism, we could support
1841 * any file. For now, just ensure that anything potentially problematic is done
1842 * inline.
1843 */
1844static bool io_file_supports_async(struct file *file)
1845{
1846 umode_t mode = file_inode(file)->i_mode;
1847
Jens Axboe10d59342019-12-09 20:16:22 -07001848 if (S_ISBLK(mode) || S_ISCHR(mode) || S_ISSOCK(mode))
Jens Axboe2b188cc2019-01-07 10:46:33 -07001849 return true;
1850 if (S_ISREG(mode) && file->f_op != &io_uring_fops)
1851 return true;
1852
1853 return false;
1854}
1855
Jens Axboe3529d8c2019-12-19 18:24:38 -07001856static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,
1857 bool force_nonblock)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001858{
Jens Axboedef596e2019-01-09 08:59:42 -07001859 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe9adbd452019-12-20 08:45:55 -07001860 struct kiocb *kiocb = &req->rw.kiocb;
Jens Axboe09bb8392019-03-13 12:39:28 -06001861 unsigned ioprio;
1862 int ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001863
Jens Axboe491381ce2019-10-17 09:20:46 -06001864 if (S_ISREG(file_inode(req->file)->i_mode))
1865 req->flags |= REQ_F_ISREG;
1866
Jens Axboe2b188cc2019-01-07 10:46:33 -07001867 kiocb->ki_pos = READ_ONCE(sqe->off);
Jens Axboeba042912019-12-25 16:33:42 -07001868 if (kiocb->ki_pos == -1 && !(req->file->f_mode & FMODE_STREAM)) {
1869 req->flags |= REQ_F_CUR_POS;
1870 kiocb->ki_pos = req->file->f_pos;
1871 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07001872 kiocb->ki_hint = ki_hint_validate(file_write_hint(kiocb->ki_filp));
Pavel Begunkov3e577dc2020-02-01 03:58:42 +03001873 kiocb->ki_flags = iocb_flags(kiocb->ki_filp);
1874 ret = kiocb_set_rw_flags(kiocb, READ_ONCE(sqe->rw_flags));
1875 if (unlikely(ret))
1876 return ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001877
1878 ioprio = READ_ONCE(sqe->ioprio);
1879 if (ioprio) {
1880 ret = ioprio_check_cap(ioprio);
1881 if (ret)
Jens Axboe09bb8392019-03-13 12:39:28 -06001882 return ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001883
1884 kiocb->ki_ioprio = ioprio;
1885 } else
1886 kiocb->ki_ioprio = get_current_ioprio();
1887
Stefan Bühler8449eed2019-04-27 20:34:19 +02001888 /* don't allow async punt if RWF_NOWAIT was requested */
Jens Axboe491381ce2019-10-17 09:20:46 -06001889 if ((kiocb->ki_flags & IOCB_NOWAIT) ||
1890 (req->file->f_flags & O_NONBLOCK))
Stefan Bühler8449eed2019-04-27 20:34:19 +02001891 req->flags |= REQ_F_NOWAIT;
1892
1893 if (force_nonblock)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001894 kiocb->ki_flags |= IOCB_NOWAIT;
Stefan Bühler8449eed2019-04-27 20:34:19 +02001895
Jens Axboedef596e2019-01-09 08:59:42 -07001896 if (ctx->flags & IORING_SETUP_IOPOLL) {
Jens Axboedef596e2019-01-09 08:59:42 -07001897 if (!(kiocb->ki_flags & IOCB_DIRECT) ||
1898 !kiocb->ki_filp->f_op->iopoll)
Jens Axboe09bb8392019-03-13 12:39:28 -06001899 return -EOPNOTSUPP;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001900
Jens Axboedef596e2019-01-09 08:59:42 -07001901 kiocb->ki_flags |= IOCB_HIPRI;
1902 kiocb->ki_complete = io_complete_rw_iopoll;
Jens Axboe6873e0b2019-10-30 13:53:09 -06001903 req->result = 0;
Jens Axboedef596e2019-01-09 08:59:42 -07001904 } else {
Jens Axboe09bb8392019-03-13 12:39:28 -06001905 if (kiocb->ki_flags & IOCB_HIPRI)
1906 return -EINVAL;
Jens Axboedef596e2019-01-09 08:59:42 -07001907 kiocb->ki_complete = io_complete_rw;
1908 }
Jens Axboe9adbd452019-12-20 08:45:55 -07001909
Jens Axboe3529d8c2019-12-19 18:24:38 -07001910 req->rw.addr = READ_ONCE(sqe->addr);
1911 req->rw.len = READ_ONCE(sqe->len);
Jens Axboe9adbd452019-12-20 08:45:55 -07001912 /* we own ->private, reuse it for the buffer index */
1913 req->rw.kiocb.private = (void *) (unsigned long)
Jens Axboe3529d8c2019-12-19 18:24:38 -07001914 READ_ONCE(sqe->buf_index);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001915 return 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001916}
1917
1918static inline void io_rw_done(struct kiocb *kiocb, ssize_t ret)
1919{
1920 switch (ret) {
1921 case -EIOCBQUEUED:
1922 break;
1923 case -ERESTARTSYS:
1924 case -ERESTARTNOINTR:
1925 case -ERESTARTNOHAND:
1926 case -ERESTART_RESTARTBLOCK:
1927 /*
1928 * We can't just restart the syscall, since previously
1929 * submitted sqes may already be in progress. Just fail this
1930 * IO with EINTR.
1931 */
1932 ret = -EINTR;
1933 /* fall through */
1934 default:
1935 kiocb->ki_complete(kiocb, ret, 0);
1936 }
1937}
1938
Jens Axboeba816ad2019-09-28 11:36:45 -06001939static void kiocb_done(struct kiocb *kiocb, ssize_t ret, struct io_kiocb **nxt,
1940 bool in_async)
1941{
Jens Axboeba042912019-12-25 16:33:42 -07001942 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
1943
1944 if (req->flags & REQ_F_CUR_POS)
1945 req->file->f_pos = kiocb->ki_pos;
Pavel Begunkovf9bd67f2019-11-21 23:21:03 +03001946 if (in_async && ret >= 0 && kiocb->ki_complete == io_complete_rw)
Jens Axboeba816ad2019-09-28 11:36:45 -06001947 *nxt = __io_complete_rw(kiocb, ret);
1948 else
1949 io_rw_done(kiocb, ret);
1950}
1951
Jens Axboe9adbd452019-12-20 08:45:55 -07001952static ssize_t io_import_fixed(struct io_kiocb *req, int rw,
Pavel Begunkov7d009162019-11-25 23:14:40 +03001953 struct iov_iter *iter)
Jens Axboeedafcce2019-01-09 09:16:05 -07001954{
Jens Axboe9adbd452019-12-20 08:45:55 -07001955 struct io_ring_ctx *ctx = req->ctx;
1956 size_t len = req->rw.len;
Jens Axboeedafcce2019-01-09 09:16:05 -07001957 struct io_mapped_ubuf *imu;
1958 unsigned index, buf_index;
1959 size_t offset;
1960 u64 buf_addr;
1961
1962 /* attempt to use fixed buffers without having provided iovecs */
1963 if (unlikely(!ctx->user_bufs))
1964 return -EFAULT;
1965
Jens Axboe9adbd452019-12-20 08:45:55 -07001966 buf_index = (unsigned long) req->rw.kiocb.private;
Jens Axboeedafcce2019-01-09 09:16:05 -07001967 if (unlikely(buf_index >= ctx->nr_user_bufs))
1968 return -EFAULT;
1969
1970 index = array_index_nospec(buf_index, ctx->nr_user_bufs);
1971 imu = &ctx->user_bufs[index];
Jens Axboe9adbd452019-12-20 08:45:55 -07001972 buf_addr = req->rw.addr;
Jens Axboeedafcce2019-01-09 09:16:05 -07001973
1974 /* overflow */
1975 if (buf_addr + len < buf_addr)
1976 return -EFAULT;
1977 /* not inside the mapped region */
1978 if (buf_addr < imu->ubuf || buf_addr + len > imu->ubuf + imu->len)
1979 return -EFAULT;
1980
1981 /*
1982 * May not be a start of buffer, set size appropriately
1983 * and advance us to the beginning.
1984 */
1985 offset = buf_addr - imu->ubuf;
1986 iov_iter_bvec(iter, rw, imu->bvec, imu->nr_bvecs, offset + len);
Jens Axboebd11b3a2019-07-20 08:37:31 -06001987
1988 if (offset) {
1989 /*
1990 * Don't use iov_iter_advance() here, as it's really slow for
1991 * using the latter parts of a big fixed buffer - it iterates
1992 * over each segment manually. We can cheat a bit here, because
1993 * we know that:
1994 *
1995 * 1) it's a BVEC iter, we set it up
1996 * 2) all bvecs are PAGE_SIZE in size, except potentially the
1997 * first and last bvec
1998 *
1999 * So just find our index, and adjust the iterator afterwards.
2000 * If the offset is within the first bvec (or the whole first
2001 * bvec, just use iov_iter_advance(). This makes it easier
2002 * since we can just skip the first segment, which may not
2003 * be PAGE_SIZE aligned.
2004 */
2005 const struct bio_vec *bvec = imu->bvec;
2006
2007 if (offset <= bvec->bv_len) {
2008 iov_iter_advance(iter, offset);
2009 } else {
2010 unsigned long seg_skip;
2011
2012 /* skip first vec */
2013 offset -= bvec->bv_len;
2014 seg_skip = 1 + (offset >> PAGE_SHIFT);
2015
2016 iter->bvec = bvec + seg_skip;
2017 iter->nr_segs -= seg_skip;
Aleix Roca Nonell99c79f62019-08-15 14:03:22 +02002018 iter->count -= bvec->bv_len + offset;
Jens Axboebd11b3a2019-07-20 08:37:31 -06002019 iter->iov_offset = offset & ~PAGE_MASK;
Jens Axboebd11b3a2019-07-20 08:37:31 -06002020 }
2021 }
2022
Jens Axboe5e559562019-11-13 16:12:46 -07002023 return len;
Jens Axboeedafcce2019-01-09 09:16:05 -07002024}
2025
Pavel Begunkovcf6fd4b2019-11-25 23:14:39 +03002026static ssize_t io_import_iovec(int rw, struct io_kiocb *req,
2027 struct iovec **iovec, struct iov_iter *iter)
Jens Axboe2b188cc2019-01-07 10:46:33 -07002028{
Jens Axboe9adbd452019-12-20 08:45:55 -07002029 void __user *buf = u64_to_user_ptr(req->rw.addr);
2030 size_t sqe_len = req->rw.len;
Jens Axboeedafcce2019-01-09 09:16:05 -07002031 u8 opcode;
2032
Jens Axboed625c6e2019-12-17 19:53:05 -07002033 opcode = req->opcode;
Pavel Begunkov7d009162019-11-25 23:14:40 +03002034 if (opcode == IORING_OP_READ_FIXED || opcode == IORING_OP_WRITE_FIXED) {
Jens Axboeedafcce2019-01-09 09:16:05 -07002035 *iovec = NULL;
Jens Axboe9adbd452019-12-20 08:45:55 -07002036 return io_import_fixed(req, rw, iter);
Jens Axboeedafcce2019-01-09 09:16:05 -07002037 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07002038
Jens Axboe9adbd452019-12-20 08:45:55 -07002039 /* buffer index only valid with fixed read/write */
2040 if (req->rw.kiocb.private)
2041 return -EINVAL;
2042
Jens Axboe3a6820f2019-12-22 15:19:35 -07002043 if (opcode == IORING_OP_READ || opcode == IORING_OP_WRITE) {
2044 ssize_t ret;
2045 ret = import_single_range(rw, buf, sqe_len, *iovec, iter);
2046 *iovec = NULL;
2047 return ret;
2048 }
2049
Jens Axboef67676d2019-12-02 11:03:47 -07002050 if (req->io) {
2051 struct io_async_rw *iorw = &req->io->rw;
2052
2053 *iovec = iorw->iov;
2054 iov_iter_init(iter, rw, *iovec, iorw->nr_segs, iorw->size);
2055 if (iorw->iov == iorw->fast_iov)
2056 *iovec = NULL;
2057 return iorw->size;
2058 }
2059
Jens Axboe2b188cc2019-01-07 10:46:33 -07002060#ifdef CONFIG_COMPAT
Pavel Begunkovcf6fd4b2019-11-25 23:14:39 +03002061 if (req->ctx->compat)
Jens Axboe2b188cc2019-01-07 10:46:33 -07002062 return compat_import_iovec(rw, buf, sqe_len, UIO_FASTIOV,
2063 iovec, iter);
2064#endif
2065
2066 return import_iovec(rw, buf, sqe_len, UIO_FASTIOV, iovec, iter);
2067}
2068
Jens Axboe32960612019-09-23 11:05:34 -06002069/*
2070 * For files that don't have ->read_iter() and ->write_iter(), handle them
2071 * by looping over ->read() or ->write() manually.
2072 */
2073static ssize_t loop_rw_iter(int rw, struct file *file, struct kiocb *kiocb,
2074 struct iov_iter *iter)
2075{
2076 ssize_t ret = 0;
2077
2078 /*
2079 * Don't support polled IO through this interface, and we can't
2080 * support non-blocking either. For the latter, this just causes
2081 * the kiocb to be handled from an async context.
2082 */
2083 if (kiocb->ki_flags & IOCB_HIPRI)
2084 return -EOPNOTSUPP;
2085 if (kiocb->ki_flags & IOCB_NOWAIT)
2086 return -EAGAIN;
2087
2088 while (iov_iter_count(iter)) {
Pavel Begunkov311ae9e2019-11-24 11:58:24 +03002089 struct iovec iovec;
Jens Axboe32960612019-09-23 11:05:34 -06002090 ssize_t nr;
2091
Pavel Begunkov311ae9e2019-11-24 11:58:24 +03002092 if (!iov_iter_is_bvec(iter)) {
2093 iovec = iov_iter_iovec(iter);
2094 } else {
2095 /* fixed buffers import bvec */
2096 iovec.iov_base = kmap(iter->bvec->bv_page)
2097 + iter->iov_offset;
2098 iovec.iov_len = min(iter->count,
2099 iter->bvec->bv_len - iter->iov_offset);
2100 }
2101
Jens Axboe32960612019-09-23 11:05:34 -06002102 if (rw == READ) {
2103 nr = file->f_op->read(file, iovec.iov_base,
2104 iovec.iov_len, &kiocb->ki_pos);
2105 } else {
2106 nr = file->f_op->write(file, iovec.iov_base,
2107 iovec.iov_len, &kiocb->ki_pos);
2108 }
2109
Pavel Begunkov311ae9e2019-11-24 11:58:24 +03002110 if (iov_iter_is_bvec(iter))
2111 kunmap(iter->bvec->bv_page);
2112
Jens Axboe32960612019-09-23 11:05:34 -06002113 if (nr < 0) {
2114 if (!ret)
2115 ret = nr;
2116 break;
2117 }
2118 ret += nr;
2119 if (nr != iovec.iov_len)
2120 break;
2121 iov_iter_advance(iter, nr);
2122 }
2123
2124 return ret;
2125}
2126
Jens Axboeb7bb4f72019-12-15 22:13:43 -07002127static void io_req_map_rw(struct io_kiocb *req, ssize_t io_size,
Jens Axboef67676d2019-12-02 11:03:47 -07002128 struct iovec *iovec, struct iovec *fast_iov,
2129 struct iov_iter *iter)
2130{
2131 req->io->rw.nr_segs = iter->nr_segs;
2132 req->io->rw.size = io_size;
2133 req->io->rw.iov = iovec;
2134 if (!req->io->rw.iov) {
2135 req->io->rw.iov = req->io->rw.fast_iov;
2136 memcpy(req->io->rw.iov, fast_iov,
2137 sizeof(struct iovec) * iter->nr_segs);
Pavel Begunkov99bc4c32020-02-07 22:04:45 +03002138 } else {
2139 req->flags |= REQ_F_NEED_CLEANUP;
Jens Axboef67676d2019-12-02 11:03:47 -07002140 }
2141}
2142
Jens Axboeb7bb4f72019-12-15 22:13:43 -07002143static int io_alloc_async_ctx(struct io_kiocb *req)
Jens Axboef67676d2019-12-02 11:03:47 -07002144{
Jens Axboed3656342019-12-18 09:50:26 -07002145 if (!io_op_defs[req->opcode].async_ctx)
2146 return 0;
Jens Axboef67676d2019-12-02 11:03:47 -07002147 req->io = kmalloc(sizeof(*req->io), GFP_KERNEL);
Jens Axboe06b76d42019-12-19 14:44:26 -07002148 return req->io == NULL;
Jens Axboeb7bb4f72019-12-15 22:13:43 -07002149}
2150
Jens Axboeb7bb4f72019-12-15 22:13:43 -07002151static int io_setup_async_rw(struct io_kiocb *req, ssize_t io_size,
2152 struct iovec *iovec, struct iovec *fast_iov,
2153 struct iov_iter *iter)
2154{
Jens Axboe980ad262020-01-24 23:08:54 -07002155 if (!io_op_defs[req->opcode].async_ctx)
Jens Axboe74566df2020-01-13 19:23:24 -07002156 return 0;
Jens Axboe5d204bc2020-01-31 12:06:52 -07002157 if (!req->io) {
2158 if (io_alloc_async_ctx(req))
2159 return -ENOMEM;
Jens Axboeb7bb4f72019-12-15 22:13:43 -07002160
Jens Axboe5d204bc2020-01-31 12:06:52 -07002161 io_req_map_rw(req, io_size, iovec, fast_iov, iter);
2162 }
Jens Axboeb7bb4f72019-12-15 22:13:43 -07002163 return 0;
Jens Axboef67676d2019-12-02 11:03:47 -07002164}
2165
Jens Axboe3529d8c2019-12-19 18:24:38 -07002166static int io_read_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe,
2167 bool force_nonblock)
Jens Axboef67676d2019-12-02 11:03:47 -07002168{
Jens Axboe3529d8c2019-12-19 18:24:38 -07002169 struct io_async_ctx *io;
2170 struct iov_iter iter;
Jens Axboef67676d2019-12-02 11:03:47 -07002171 ssize_t ret;
2172
Jens Axboe3529d8c2019-12-19 18:24:38 -07002173 ret = io_prep_rw(req, sqe, force_nonblock);
2174 if (ret)
2175 return ret;
Jens Axboef67676d2019-12-02 11:03:47 -07002176
Jens Axboe3529d8c2019-12-19 18:24:38 -07002177 if (unlikely(!(req->file->f_mode & FMODE_READ)))
2178 return -EBADF;
Jens Axboef67676d2019-12-02 11:03:47 -07002179
Jens Axboe3529d8c2019-12-19 18:24:38 -07002180 if (!req->io)
2181 return 0;
2182
2183 io = req->io;
2184 io->rw.iov = io->rw.fast_iov;
2185 req->io = NULL;
2186 ret = io_import_iovec(READ, req, &io->rw.iov, &iter);
2187 req->io = io;
2188 if (ret < 0)
2189 return ret;
2190
2191 io_req_map_rw(req, ret, io->rw.iov, io->rw.fast_iov, &iter);
2192 return 0;
Jens Axboef67676d2019-12-02 11:03:47 -07002193}
2194
Pavel Begunkov267bc902019-11-07 01:41:08 +03002195static int io_read(struct io_kiocb *req, struct io_kiocb **nxt,
Jens Axboe8358e3a2019-04-23 08:17:58 -06002196 bool force_nonblock)
Jens Axboe2b188cc2019-01-07 10:46:33 -07002197{
2198 struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
Jens Axboe9adbd452019-12-20 08:45:55 -07002199 struct kiocb *kiocb = &req->rw.kiocb;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002200 struct iov_iter iter;
Jens Axboe31b51512019-01-18 22:56:34 -07002201 size_t iov_count;
Jens Axboef67676d2019-12-02 11:03:47 -07002202 ssize_t io_size, ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002203
Jens Axboe3529d8c2019-12-19 18:24:38 -07002204 ret = io_import_iovec(READ, req, &iovec, &iter);
Jens Axboe06b76d42019-12-19 14:44:26 -07002205 if (ret < 0)
2206 return ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002207
Jens Axboefd6c2e42019-12-18 12:19:41 -07002208 /* Ensure we clear previously set non-block flag */
2209 if (!force_nonblock)
Jens Axboe9adbd452019-12-20 08:45:55 -07002210 req->rw.kiocb.ki_flags &= ~IOCB_NOWAIT;
Jens Axboefd6c2e42019-12-18 12:19:41 -07002211
Bijan Mottahedeh797f3f52020-01-15 18:37:45 -08002212 req->result = 0;
Jens Axboef67676d2019-12-02 11:03:47 -07002213 io_size = ret;
Jens Axboe9e645e112019-05-10 16:07:28 -06002214 if (req->flags & REQ_F_LINK)
Jens Axboef67676d2019-12-02 11:03:47 -07002215 req->result = io_size;
2216
2217 /*
2218 * If the file doesn't support async, mark it as REQ_F_MUST_PUNT so
2219 * we know to async punt it even if it was opened O_NONBLOCK
2220 */
Jens Axboe9adbd452019-12-20 08:45:55 -07002221 if (force_nonblock && !io_file_supports_async(req->file)) {
Jens Axboef67676d2019-12-02 11:03:47 -07002222 req->flags |= REQ_F_MUST_PUNT;
2223 goto copy_iov;
2224 }
Jens Axboe9e645e112019-05-10 16:07:28 -06002225
Jens Axboe31b51512019-01-18 22:56:34 -07002226 iov_count = iov_iter_count(&iter);
Jens Axboe9adbd452019-12-20 08:45:55 -07002227 ret = rw_verify_area(READ, req->file, &kiocb->ki_pos, iov_count);
Jens Axboe2b188cc2019-01-07 10:46:33 -07002228 if (!ret) {
2229 ssize_t ret2;
2230
Jens Axboe9adbd452019-12-20 08:45:55 -07002231 if (req->file->f_op->read_iter)
2232 ret2 = call_read_iter(req->file, kiocb, &iter);
Jens Axboe32960612019-09-23 11:05:34 -06002233 else
Jens Axboe9adbd452019-12-20 08:45:55 -07002234 ret2 = loop_rw_iter(READ, req->file, kiocb, &iter);
Jens Axboe32960612019-09-23 11:05:34 -06002235
Jens Axboe9d93a3f2019-05-15 13:53:07 -06002236 /* Catch -EAGAIN return for forced non-blocking submission */
Jens Axboef67676d2019-12-02 11:03:47 -07002237 if (!force_nonblock || ret2 != -EAGAIN) {
Pavel Begunkovcf6fd4b2019-11-25 23:14:39 +03002238 kiocb_done(kiocb, ret2, nxt, req->in_async);
Jens Axboef67676d2019-12-02 11:03:47 -07002239 } else {
2240copy_iov:
Jens Axboeb7bb4f72019-12-15 22:13:43 -07002241 ret = io_setup_async_rw(req, io_size, iovec,
Jens Axboef67676d2019-12-02 11:03:47 -07002242 inline_vecs, &iter);
2243 if (ret)
2244 goto out_free;
2245 return -EAGAIN;
2246 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07002247 }
Jens Axboef67676d2019-12-02 11:03:47 -07002248out_free:
Pavel Begunkov1e950812020-02-06 19:51:16 +03002249 kfree(iovec);
Pavel Begunkov99bc4c32020-02-07 22:04:45 +03002250 req->flags &= ~REQ_F_NEED_CLEANUP;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002251 return ret;
2252}
2253
Jens Axboe3529d8c2019-12-19 18:24:38 -07002254static int io_write_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe,
2255 bool force_nonblock)
Jens Axboef67676d2019-12-02 11:03:47 -07002256{
Jens Axboe3529d8c2019-12-19 18:24:38 -07002257 struct io_async_ctx *io;
2258 struct iov_iter iter;
Jens Axboef67676d2019-12-02 11:03:47 -07002259 ssize_t ret;
2260
Jens Axboe3529d8c2019-12-19 18:24:38 -07002261 ret = io_prep_rw(req, sqe, force_nonblock);
2262 if (ret)
2263 return ret;
Jens Axboef67676d2019-12-02 11:03:47 -07002264
Jens Axboe3529d8c2019-12-19 18:24:38 -07002265 if (unlikely(!(req->file->f_mode & FMODE_WRITE)))
2266 return -EBADF;
Jens Axboef67676d2019-12-02 11:03:47 -07002267
Jens Axboe3529d8c2019-12-19 18:24:38 -07002268 if (!req->io)
2269 return 0;
2270
2271 io = req->io;
2272 io->rw.iov = io->rw.fast_iov;
2273 req->io = NULL;
2274 ret = io_import_iovec(WRITE, req, &io->rw.iov, &iter);
2275 req->io = io;
2276 if (ret < 0)
2277 return ret;
2278
2279 io_req_map_rw(req, ret, io->rw.iov, io->rw.fast_iov, &iter);
2280 return 0;
Jens Axboef67676d2019-12-02 11:03:47 -07002281}
2282
Pavel Begunkov267bc902019-11-07 01:41:08 +03002283static int io_write(struct io_kiocb *req, struct io_kiocb **nxt,
Jens Axboe8358e3a2019-04-23 08:17:58 -06002284 bool force_nonblock)
Jens Axboe2b188cc2019-01-07 10:46:33 -07002285{
2286 struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
Jens Axboe9adbd452019-12-20 08:45:55 -07002287 struct kiocb *kiocb = &req->rw.kiocb;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002288 struct iov_iter iter;
Jens Axboe31b51512019-01-18 22:56:34 -07002289 size_t iov_count;
Jens Axboef67676d2019-12-02 11:03:47 -07002290 ssize_t ret, io_size;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002291
Jens Axboe3529d8c2019-12-19 18:24:38 -07002292 ret = io_import_iovec(WRITE, req, &iovec, &iter);
Jens Axboe06b76d42019-12-19 14:44:26 -07002293 if (ret < 0)
2294 return ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002295
Jens Axboefd6c2e42019-12-18 12:19:41 -07002296 /* Ensure we clear previously set non-block flag */
2297 if (!force_nonblock)
Jens Axboe9adbd452019-12-20 08:45:55 -07002298 req->rw.kiocb.ki_flags &= ~IOCB_NOWAIT;
Jens Axboefd6c2e42019-12-18 12:19:41 -07002299
Bijan Mottahedeh797f3f52020-01-15 18:37:45 -08002300 req->result = 0;
Jens Axboef67676d2019-12-02 11:03:47 -07002301 io_size = ret;
Jens Axboe9e645e112019-05-10 16:07:28 -06002302 if (req->flags & REQ_F_LINK)
Jens Axboef67676d2019-12-02 11:03:47 -07002303 req->result = io_size;
2304
2305 /*
2306 * If the file doesn't support async, mark it as REQ_F_MUST_PUNT so
2307 * we know to async punt it even if it was opened O_NONBLOCK
2308 */
2309 if (force_nonblock && !io_file_supports_async(req->file)) {
2310 req->flags |= REQ_F_MUST_PUNT;
2311 goto copy_iov;
2312 }
2313
Jens Axboe10d59342019-12-09 20:16:22 -07002314 /* file path doesn't support NOWAIT for non-direct_IO */
2315 if (force_nonblock && !(kiocb->ki_flags & IOCB_DIRECT) &&
2316 (req->flags & REQ_F_ISREG))
Jens Axboef67676d2019-12-02 11:03:47 -07002317 goto copy_iov;
Jens Axboe9e645e112019-05-10 16:07:28 -06002318
Jens Axboe31b51512019-01-18 22:56:34 -07002319 iov_count = iov_iter_count(&iter);
Jens Axboe9adbd452019-12-20 08:45:55 -07002320 ret = rw_verify_area(WRITE, req->file, &kiocb->ki_pos, iov_count);
Jens Axboe2b188cc2019-01-07 10:46:33 -07002321 if (!ret) {
Roman Penyaev9bf79332019-03-25 20:09:24 +01002322 ssize_t ret2;
2323
Jens Axboe2b188cc2019-01-07 10:46:33 -07002324 /*
2325 * Open-code file_start_write here to grab freeze protection,
2326 * which will be released by another thread in
2327 * io_complete_rw(). Fool lockdep by telling it the lock got
2328 * released so that it doesn't complain about the held lock when
2329 * we return to userspace.
2330 */
Jens Axboe491381ce2019-10-17 09:20:46 -06002331 if (req->flags & REQ_F_ISREG) {
Jens Axboe9adbd452019-12-20 08:45:55 -07002332 __sb_start_write(file_inode(req->file)->i_sb,
Jens Axboe2b188cc2019-01-07 10:46:33 -07002333 SB_FREEZE_WRITE, true);
Jens Axboe9adbd452019-12-20 08:45:55 -07002334 __sb_writers_release(file_inode(req->file)->i_sb,
Jens Axboe2b188cc2019-01-07 10:46:33 -07002335 SB_FREEZE_WRITE);
2336 }
2337 kiocb->ki_flags |= IOCB_WRITE;
Roman Penyaev9bf79332019-03-25 20:09:24 +01002338
Jens Axboe9adbd452019-12-20 08:45:55 -07002339 if (req->file->f_op->write_iter)
2340 ret2 = call_write_iter(req->file, kiocb, &iter);
Jens Axboe32960612019-09-23 11:05:34 -06002341 else
Jens Axboe9adbd452019-12-20 08:45:55 -07002342 ret2 = loop_rw_iter(WRITE, req->file, kiocb, &iter);
Jens Axboefaac9962020-02-07 15:45:22 -07002343 /*
2344 * Raw bdev writes will -EOPNOTSUPP for IOCB_NOWAIT. Just
2345 * retry them without IOCB_NOWAIT.
2346 */
2347 if (ret2 == -EOPNOTSUPP && (kiocb->ki_flags & IOCB_NOWAIT))
2348 ret2 = -EAGAIN;
Jens Axboef67676d2019-12-02 11:03:47 -07002349 if (!force_nonblock || ret2 != -EAGAIN) {
Pavel Begunkovcf6fd4b2019-11-25 23:14:39 +03002350 kiocb_done(kiocb, ret2, nxt, req->in_async);
Jens Axboef67676d2019-12-02 11:03:47 -07002351 } else {
2352copy_iov:
Jens Axboeb7bb4f72019-12-15 22:13:43 -07002353 ret = io_setup_async_rw(req, io_size, iovec,
Jens Axboef67676d2019-12-02 11:03:47 -07002354 inline_vecs, &iter);
2355 if (ret)
2356 goto out_free;
2357 return -EAGAIN;
2358 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07002359 }
Jens Axboe31b51512019-01-18 22:56:34 -07002360out_free:
Pavel Begunkov99bc4c32020-02-07 22:04:45 +03002361 req->flags &= ~REQ_F_NEED_CLEANUP;
Pavel Begunkov1e950812020-02-06 19:51:16 +03002362 kfree(iovec);
Jens Axboe2b188cc2019-01-07 10:46:33 -07002363 return ret;
2364}
2365
2366/*
2367 * IORING_OP_NOP just posts a completion event, nothing else.
2368 */
Jens Axboe78e19bb2019-11-06 15:21:34 -07002369static int io_nop(struct io_kiocb *req)
Jens Axboe2b188cc2019-01-07 10:46:33 -07002370{
2371 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002372
Jens Axboedef596e2019-01-09 08:59:42 -07002373 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
2374 return -EINVAL;
2375
Jens Axboe78e19bb2019-11-06 15:21:34 -07002376 io_cqring_add_event(req, 0);
Jens Axboee65ef562019-03-12 10:16:44 -06002377 io_put_req(req);
Jens Axboe2b188cc2019-01-07 10:46:33 -07002378 return 0;
2379}
2380
Jens Axboe3529d8c2019-12-19 18:24:38 -07002381static int io_prep_fsync(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Christoph Hellwigc992fe22019-01-11 09:43:02 -07002382{
Jens Axboe6b063142019-01-10 22:13:58 -07002383 struct io_ring_ctx *ctx = req->ctx;
Christoph Hellwigc992fe22019-01-11 09:43:02 -07002384
Jens Axboe09bb8392019-03-13 12:39:28 -06002385 if (!req->file)
2386 return -EBADF;
Christoph Hellwigc992fe22019-01-11 09:43:02 -07002387
Jens Axboe6b063142019-01-10 22:13:58 -07002388 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
Jens Axboedef596e2019-01-09 08:59:42 -07002389 return -EINVAL;
Jens Axboeedafcce2019-01-09 09:16:05 -07002390 if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index))
Christoph Hellwigc992fe22019-01-11 09:43:02 -07002391 return -EINVAL;
2392
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07002393 req->sync.flags = READ_ONCE(sqe->fsync_flags);
2394 if (unlikely(req->sync.flags & ~IORING_FSYNC_DATASYNC))
2395 return -EINVAL;
2396
2397 req->sync.off = READ_ONCE(sqe->off);
2398 req->sync.len = READ_ONCE(sqe->len);
Christoph Hellwigc992fe22019-01-11 09:43:02 -07002399 return 0;
2400}
2401
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07002402static bool io_req_cancelled(struct io_kiocb *req)
2403{
2404 if (req->work.flags & IO_WQ_WORK_CANCEL) {
2405 req_set_fail_links(req);
2406 io_cqring_add_event(req, -ECANCELED);
2407 io_put_req(req);
2408 return true;
2409 }
2410
2411 return false;
2412}
2413
Jens Axboe78912932020-01-14 22:09:06 -07002414static void io_link_work_cb(struct io_wq_work **workptr)
2415{
2416 struct io_wq_work *work = *workptr;
2417 struct io_kiocb *link = work->data;
2418
2419 io_queue_linked_timeout(link);
2420 work->func = io_wq_submit_work;
2421}
2422
2423static void io_wq_assign_next(struct io_wq_work **workptr, struct io_kiocb *nxt)
2424{
2425 struct io_kiocb *link;
2426
2427 io_prep_async_work(nxt, &link);
2428 *workptr = &nxt->work;
2429 if (link) {
2430 nxt->work.flags |= IO_WQ_WORK_CB;
2431 nxt->work.func = io_link_work_cb;
2432 nxt->work.data = link;
2433 }
2434}
2435
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07002436static void io_fsync_finish(struct io_wq_work **workptr)
2437{
2438 struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);
2439 loff_t end = req->sync.off + req->sync.len;
2440 struct io_kiocb *nxt = NULL;
2441 int ret;
2442
2443 if (io_req_cancelled(req))
2444 return;
2445
Jens Axboe9adbd452019-12-20 08:45:55 -07002446 ret = vfs_fsync_range(req->file, req->sync.off,
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07002447 end > 0 ? end : LLONG_MAX,
2448 req->sync.flags & IORING_FSYNC_DATASYNC);
2449 if (ret < 0)
2450 req_set_fail_links(req);
2451 io_cqring_add_event(req, ret);
2452 io_put_req_find_next(req, &nxt);
2453 if (nxt)
Jens Axboe78912932020-01-14 22:09:06 -07002454 io_wq_assign_next(workptr, nxt);
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07002455}
2456
Jens Axboefc4df992019-12-10 14:38:45 -07002457static int io_fsync(struct io_kiocb *req, struct io_kiocb **nxt,
2458 bool force_nonblock)
Christoph Hellwigc992fe22019-01-11 09:43:02 -07002459{
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07002460 struct io_wq_work *work, *old_work;
Christoph Hellwigc992fe22019-01-11 09:43:02 -07002461
2462 /* fsync always requires a blocking context */
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07002463 if (force_nonblock) {
2464 io_put_req(req);
2465 req->work.func = io_fsync_finish;
Christoph Hellwigc992fe22019-01-11 09:43:02 -07002466 return -EAGAIN;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07002467 }
Christoph Hellwigc992fe22019-01-11 09:43:02 -07002468
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07002469 work = old_work = &req->work;
2470 io_fsync_finish(&work);
2471 if (work && work != old_work)
2472 *nxt = container_of(work, struct io_kiocb, work);
Christoph Hellwigc992fe22019-01-11 09:43:02 -07002473 return 0;
2474}
2475
Jens Axboed63d1b52019-12-10 10:38:56 -07002476static void io_fallocate_finish(struct io_wq_work **workptr)
2477{
2478 struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);
2479 struct io_kiocb *nxt = NULL;
2480 int ret;
2481
2482 ret = vfs_fallocate(req->file, req->sync.mode, req->sync.off,
2483 req->sync.len);
2484 if (ret < 0)
2485 req_set_fail_links(req);
2486 io_cqring_add_event(req, ret);
2487 io_put_req_find_next(req, &nxt);
2488 if (nxt)
2489 io_wq_assign_next(workptr, nxt);
2490}
2491
2492static int io_fallocate_prep(struct io_kiocb *req,
2493 const struct io_uring_sqe *sqe)
2494{
2495 if (sqe->ioprio || sqe->buf_index || sqe->rw_flags)
2496 return -EINVAL;
2497
2498 req->sync.off = READ_ONCE(sqe->off);
2499 req->sync.len = READ_ONCE(sqe->addr);
2500 req->sync.mode = READ_ONCE(sqe->len);
2501 return 0;
2502}
2503
2504static int io_fallocate(struct io_kiocb *req, struct io_kiocb **nxt,
2505 bool force_nonblock)
2506{
2507 struct io_wq_work *work, *old_work;
2508
2509 /* fallocate always requiring blocking context */
2510 if (force_nonblock) {
2511 io_put_req(req);
2512 req->work.func = io_fallocate_finish;
2513 return -EAGAIN;
2514 }
2515
2516 work = old_work = &req->work;
2517 io_fallocate_finish(&work);
2518 if (work && work != old_work)
2519 *nxt = container_of(work, struct io_kiocb, work);
2520
2521 return 0;
2522}
2523
Jens Axboe15b71ab2019-12-11 11:20:36 -07002524static int io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
2525{
Jens Axboef8748882020-01-08 17:47:02 -07002526 const char __user *fname;
Jens Axboe15b71ab2019-12-11 11:20:36 -07002527 int ret;
2528
2529 if (sqe->ioprio || sqe->buf_index)
2530 return -EINVAL;
Jens Axboecf3040c2020-02-06 21:31:40 -07002531 if (sqe->flags & IOSQE_FIXED_FILE)
2532 return -EBADF;
Jens Axboe15b71ab2019-12-11 11:20:36 -07002533
2534 req->open.dfd = READ_ONCE(sqe->fd);
Jens Axboec12cedf2020-01-08 17:41:21 -07002535 req->open.how.mode = READ_ONCE(sqe->len);
Jens Axboef8748882020-01-08 17:47:02 -07002536 fname = u64_to_user_ptr(READ_ONCE(sqe->addr));
Jens Axboec12cedf2020-01-08 17:41:21 -07002537 req->open.how.flags = READ_ONCE(sqe->open_flags);
Jens Axboe15b71ab2019-12-11 11:20:36 -07002538
Jens Axboef8748882020-01-08 17:47:02 -07002539 req->open.filename = getname(fname);
Jens Axboe15b71ab2019-12-11 11:20:36 -07002540 if (IS_ERR(req->open.filename)) {
2541 ret = PTR_ERR(req->open.filename);
2542 req->open.filename = NULL;
2543 return ret;
2544 }
2545
Pavel Begunkov8fef80b2020-02-07 23:59:53 +03002546 req->flags |= REQ_F_NEED_CLEANUP;
Jens Axboe15b71ab2019-12-11 11:20:36 -07002547 return 0;
2548}
2549
Jens Axboecebdb982020-01-08 17:59:24 -07002550static int io_openat2_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
2551{
2552 struct open_how __user *how;
2553 const char __user *fname;
2554 size_t len;
2555 int ret;
2556
2557 if (sqe->ioprio || sqe->buf_index)
2558 return -EINVAL;
Jens Axboecf3040c2020-02-06 21:31:40 -07002559 if (sqe->flags & IOSQE_FIXED_FILE)
2560 return -EBADF;
Jens Axboecebdb982020-01-08 17:59:24 -07002561
2562 req->open.dfd = READ_ONCE(sqe->fd);
2563 fname = u64_to_user_ptr(READ_ONCE(sqe->addr));
2564 how = u64_to_user_ptr(READ_ONCE(sqe->addr2));
2565 len = READ_ONCE(sqe->len);
2566
2567 if (len < OPEN_HOW_SIZE_VER0)
2568 return -EINVAL;
2569
2570 ret = copy_struct_from_user(&req->open.how, sizeof(req->open.how), how,
2571 len);
2572 if (ret)
2573 return ret;
2574
2575 if (!(req->open.how.flags & O_PATH) && force_o_largefile())
2576 req->open.how.flags |= O_LARGEFILE;
2577
2578 req->open.filename = getname(fname);
2579 if (IS_ERR(req->open.filename)) {
2580 ret = PTR_ERR(req->open.filename);
2581 req->open.filename = NULL;
2582 return ret;
2583 }
2584
Pavel Begunkov8fef80b2020-02-07 23:59:53 +03002585 req->flags |= REQ_F_NEED_CLEANUP;
Jens Axboecebdb982020-01-08 17:59:24 -07002586 return 0;
2587}
2588
2589static int io_openat2(struct io_kiocb *req, struct io_kiocb **nxt,
2590 bool force_nonblock)
Jens Axboe15b71ab2019-12-11 11:20:36 -07002591{
2592 struct open_flags op;
Jens Axboe15b71ab2019-12-11 11:20:36 -07002593 struct file *file;
2594 int ret;
2595
Jens Axboef86cd202020-01-29 13:46:44 -07002596 if (force_nonblock)
Jens Axboe15b71ab2019-12-11 11:20:36 -07002597 return -EAGAIN;
Jens Axboe15b71ab2019-12-11 11:20:36 -07002598
Jens Axboecebdb982020-01-08 17:59:24 -07002599 ret = build_open_flags(&req->open.how, &op);
Jens Axboe15b71ab2019-12-11 11:20:36 -07002600 if (ret)
2601 goto err;
2602
Jens Axboecebdb982020-01-08 17:59:24 -07002603 ret = get_unused_fd_flags(req->open.how.flags);
Jens Axboe15b71ab2019-12-11 11:20:36 -07002604 if (ret < 0)
2605 goto err;
2606
2607 file = do_filp_open(req->open.dfd, req->open.filename, &op);
2608 if (IS_ERR(file)) {
2609 put_unused_fd(ret);
2610 ret = PTR_ERR(file);
2611 } else {
2612 fsnotify_open(file);
2613 fd_install(ret, file);
2614 }
2615err:
2616 putname(req->open.filename);
Pavel Begunkov8fef80b2020-02-07 23:59:53 +03002617 req->flags &= ~REQ_F_NEED_CLEANUP;
Jens Axboe15b71ab2019-12-11 11:20:36 -07002618 if (ret < 0)
2619 req_set_fail_links(req);
2620 io_cqring_add_event(req, ret);
2621 io_put_req_find_next(req, nxt);
2622 return 0;
2623}
2624
Jens Axboecebdb982020-01-08 17:59:24 -07002625static int io_openat(struct io_kiocb *req, struct io_kiocb **nxt,
2626 bool force_nonblock)
2627{
2628 req->open.how = build_open_how(req->open.how.flags, req->open.how.mode);
2629 return io_openat2(req, nxt, force_nonblock);
2630}
2631
Jens Axboe3e4827b2020-01-08 15:18:09 -07002632static int io_epoll_ctl_prep(struct io_kiocb *req,
2633 const struct io_uring_sqe *sqe)
2634{
2635#if defined(CONFIG_EPOLL)
2636 if (sqe->ioprio || sqe->buf_index)
2637 return -EINVAL;
2638
2639 req->epoll.epfd = READ_ONCE(sqe->fd);
2640 req->epoll.op = READ_ONCE(sqe->len);
2641 req->epoll.fd = READ_ONCE(sqe->off);
2642
2643 if (ep_op_has_event(req->epoll.op)) {
2644 struct epoll_event __user *ev;
2645
2646 ev = u64_to_user_ptr(READ_ONCE(sqe->addr));
2647 if (copy_from_user(&req->epoll.event, ev, sizeof(*ev)))
2648 return -EFAULT;
2649 }
2650
2651 return 0;
2652#else
2653 return -EOPNOTSUPP;
2654#endif
2655}
2656
2657static int io_epoll_ctl(struct io_kiocb *req, struct io_kiocb **nxt,
2658 bool force_nonblock)
2659{
2660#if defined(CONFIG_EPOLL)
2661 struct io_epoll *ie = &req->epoll;
2662 int ret;
2663
2664 ret = do_epoll_ctl(ie->epfd, ie->op, ie->fd, &ie->event, force_nonblock);
2665 if (force_nonblock && ret == -EAGAIN)
2666 return -EAGAIN;
2667
2668 if (ret < 0)
2669 req_set_fail_links(req);
2670 io_cqring_add_event(req, ret);
2671 io_put_req_find_next(req, nxt);
2672 return 0;
2673#else
2674 return -EOPNOTSUPP;
2675#endif
2676}
2677
Jens Axboec1ca7572019-12-25 22:18:28 -07002678static int io_madvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
2679{
2680#if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU)
2681 if (sqe->ioprio || sqe->buf_index || sqe->off)
2682 return -EINVAL;
2683
2684 req->madvise.addr = READ_ONCE(sqe->addr);
2685 req->madvise.len = READ_ONCE(sqe->len);
2686 req->madvise.advice = READ_ONCE(sqe->fadvise_advice);
2687 return 0;
2688#else
2689 return -EOPNOTSUPP;
2690#endif
2691}
2692
2693static int io_madvise(struct io_kiocb *req, struct io_kiocb **nxt,
2694 bool force_nonblock)
2695{
2696#if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU)
2697 struct io_madvise *ma = &req->madvise;
2698 int ret;
2699
2700 if (force_nonblock)
2701 return -EAGAIN;
2702
2703 ret = do_madvise(ma->addr, ma->len, ma->advice);
2704 if (ret < 0)
2705 req_set_fail_links(req);
2706 io_cqring_add_event(req, ret);
2707 io_put_req_find_next(req, nxt);
2708 return 0;
2709#else
2710 return -EOPNOTSUPP;
2711#endif
2712}
2713
Jens Axboe4840e412019-12-25 22:03:45 -07002714static int io_fadvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
2715{
2716 if (sqe->ioprio || sqe->buf_index || sqe->addr)
2717 return -EINVAL;
2718
2719 req->fadvise.offset = READ_ONCE(sqe->off);
2720 req->fadvise.len = READ_ONCE(sqe->len);
2721 req->fadvise.advice = READ_ONCE(sqe->fadvise_advice);
2722 return 0;
2723}
2724
2725static int io_fadvise(struct io_kiocb *req, struct io_kiocb **nxt,
2726 bool force_nonblock)
2727{
2728 struct io_fadvise *fa = &req->fadvise;
2729 int ret;
2730
Jens Axboe3e694262020-02-01 09:22:49 -07002731 if (force_nonblock) {
2732 switch (fa->advice) {
2733 case POSIX_FADV_NORMAL:
2734 case POSIX_FADV_RANDOM:
2735 case POSIX_FADV_SEQUENTIAL:
2736 break;
2737 default:
2738 return -EAGAIN;
2739 }
2740 }
Jens Axboe4840e412019-12-25 22:03:45 -07002741
2742 ret = vfs_fadvise(req->file, fa->offset, fa->len, fa->advice);
2743 if (ret < 0)
2744 req_set_fail_links(req);
2745 io_cqring_add_event(req, ret);
2746 io_put_req_find_next(req, nxt);
2747 return 0;
2748}
2749
Jens Axboeeddc7ef2019-12-13 21:18:10 -07002750static int io_statx_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
2751{
Jens Axboef8748882020-01-08 17:47:02 -07002752 const char __user *fname;
Jens Axboeeddc7ef2019-12-13 21:18:10 -07002753 unsigned lookup_flags;
2754 int ret;
2755
2756 if (sqe->ioprio || sqe->buf_index)
2757 return -EINVAL;
Jens Axboecf3040c2020-02-06 21:31:40 -07002758 if (sqe->flags & IOSQE_FIXED_FILE)
2759 return -EBADF;
Jens Axboeeddc7ef2019-12-13 21:18:10 -07002760
2761 req->open.dfd = READ_ONCE(sqe->fd);
2762 req->open.mask = READ_ONCE(sqe->len);
Jens Axboef8748882020-01-08 17:47:02 -07002763 fname = u64_to_user_ptr(READ_ONCE(sqe->addr));
Jens Axboeeddc7ef2019-12-13 21:18:10 -07002764 req->open.buffer = u64_to_user_ptr(READ_ONCE(sqe->addr2));
Jens Axboec12cedf2020-01-08 17:41:21 -07002765 req->open.how.flags = READ_ONCE(sqe->statx_flags);
Jens Axboeeddc7ef2019-12-13 21:18:10 -07002766
Jens Axboec12cedf2020-01-08 17:41:21 -07002767 if (vfs_stat_set_lookup_flags(&lookup_flags, req->open.how.flags))
Jens Axboeeddc7ef2019-12-13 21:18:10 -07002768 return -EINVAL;
2769
Jens Axboef8748882020-01-08 17:47:02 -07002770 req->open.filename = getname_flags(fname, lookup_flags, NULL);
Jens Axboeeddc7ef2019-12-13 21:18:10 -07002771 if (IS_ERR(req->open.filename)) {
2772 ret = PTR_ERR(req->open.filename);
2773 req->open.filename = NULL;
2774 return ret;
2775 }
2776
Pavel Begunkov8fef80b2020-02-07 23:59:53 +03002777 req->flags |= REQ_F_NEED_CLEANUP;
Jens Axboeeddc7ef2019-12-13 21:18:10 -07002778 return 0;
2779}
2780
2781static int io_statx(struct io_kiocb *req, struct io_kiocb **nxt,
2782 bool force_nonblock)
2783{
2784 struct io_open *ctx = &req->open;
2785 unsigned lookup_flags;
2786 struct path path;
2787 struct kstat stat;
2788 int ret;
2789
2790 if (force_nonblock)
2791 return -EAGAIN;
2792
Jens Axboec12cedf2020-01-08 17:41:21 -07002793 if (vfs_stat_set_lookup_flags(&lookup_flags, ctx->how.flags))
Jens Axboeeddc7ef2019-12-13 21:18:10 -07002794 return -EINVAL;
2795
2796retry:
2797 /* filename_lookup() drops it, keep a reference */
2798 ctx->filename->refcnt++;
2799
2800 ret = filename_lookup(ctx->dfd, ctx->filename, lookup_flags, &path,
2801 NULL);
2802 if (ret)
2803 goto err;
2804
Jens Axboec12cedf2020-01-08 17:41:21 -07002805 ret = vfs_getattr(&path, &stat, ctx->mask, ctx->how.flags);
Jens Axboeeddc7ef2019-12-13 21:18:10 -07002806 path_put(&path);
2807 if (retry_estale(ret, lookup_flags)) {
2808 lookup_flags |= LOOKUP_REVAL;
2809 goto retry;
2810 }
2811 if (!ret)
2812 ret = cp_statx(&stat, ctx->buffer);
2813err:
2814 putname(ctx->filename);
Pavel Begunkov8fef80b2020-02-07 23:59:53 +03002815 req->flags &= ~REQ_F_NEED_CLEANUP;
Jens Axboeeddc7ef2019-12-13 21:18:10 -07002816 if (ret < 0)
2817 req_set_fail_links(req);
2818 io_cqring_add_event(req, ret);
2819 io_put_req_find_next(req, nxt);
2820 return 0;
2821}
2822
Jens Axboeb5dba592019-12-11 14:02:38 -07002823static int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
2824{
2825 /*
2826 * If we queue this for async, it must not be cancellable. That would
2827 * leave the 'file' in an undeterminate state.
2828 */
2829 req->work.flags |= IO_WQ_WORK_NO_CANCEL;
2830
2831 if (sqe->ioprio || sqe->off || sqe->addr || sqe->len ||
2832 sqe->rw_flags || sqe->buf_index)
2833 return -EINVAL;
2834 if (sqe->flags & IOSQE_FIXED_FILE)
Jens Axboecf3040c2020-02-06 21:31:40 -07002835 return -EBADF;
Jens Axboeb5dba592019-12-11 14:02:38 -07002836
2837 req->close.fd = READ_ONCE(sqe->fd);
2838 if (req->file->f_op == &io_uring_fops ||
Pavel Begunkovb14cca02020-01-17 04:45:59 +03002839 req->close.fd == req->ctx->ring_fd)
Jens Axboeb5dba592019-12-11 14:02:38 -07002840 return -EBADF;
2841
2842 return 0;
2843}
2844
2845static void io_close_finish(struct io_wq_work **workptr)
2846{
2847 struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);
2848 struct io_kiocb *nxt = NULL;
2849
2850 /* Invoked with files, we need to do the close */
2851 if (req->work.files) {
2852 int ret;
2853
2854 ret = filp_close(req->close.put_file, req->work.files);
Jens Axboe1a417f42020-01-31 17:16:48 -07002855 if (ret < 0)
Jens Axboeb5dba592019-12-11 14:02:38 -07002856 req_set_fail_links(req);
Jens Axboeb5dba592019-12-11 14:02:38 -07002857 io_cqring_add_event(req, ret);
2858 }
2859
2860 fput(req->close.put_file);
2861
Jens Axboeb5dba592019-12-11 14:02:38 -07002862 io_put_req_find_next(req, &nxt);
2863 if (nxt)
2864 io_wq_assign_next(workptr, nxt);
2865}
2866
2867static int io_close(struct io_kiocb *req, struct io_kiocb **nxt,
2868 bool force_nonblock)
2869{
2870 int ret;
2871
2872 req->close.put_file = NULL;
2873 ret = __close_fd_get_file(req->close.fd, &req->close.put_file);
2874 if (ret < 0)
2875 return ret;
2876
2877 /* if the file has a flush method, be safe and punt to async */
Jens Axboef86cd202020-01-29 13:46:44 -07002878 if (req->close.put_file->f_op->flush && !io_wq_current_is_worker())
Jens Axboeb5dba592019-12-11 14:02:38 -07002879 goto eagain;
Jens Axboeb5dba592019-12-11 14:02:38 -07002880
2881 /*
2882 * No ->flush(), safely close from here and just punt the
2883 * fput() to async context.
2884 */
2885 ret = filp_close(req->close.put_file, current->files);
2886
2887 if (ret < 0)
2888 req_set_fail_links(req);
2889 io_cqring_add_event(req, ret);
2890
2891 if (io_wq_current_is_worker()) {
2892 struct io_wq_work *old_work, *work;
2893
2894 old_work = work = &req->work;
2895 io_close_finish(&work);
2896 if (work && work != old_work)
2897 *nxt = container_of(work, struct io_kiocb, work);
2898 return 0;
2899 }
2900
2901eagain:
2902 req->work.func = io_close_finish;
Jens Axboe1a417f42020-01-31 17:16:48 -07002903 /*
2904 * Do manual async queue here to avoid grabbing files - we don't
2905 * need the files, and it'll cause io_close_finish() to close
2906 * the file again and cause a double CQE entry for this request
2907 */
2908 io_queue_async_work(req);
2909 return 0;
Jens Axboeb5dba592019-12-11 14:02:38 -07002910}
2911
Jens Axboe3529d8c2019-12-19 18:24:38 -07002912static int io_prep_sfr(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboe5d17b4a2019-04-09 14:56:44 -06002913{
2914 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe5d17b4a2019-04-09 14:56:44 -06002915
2916 if (!req->file)
2917 return -EBADF;
Jens Axboe5d17b4a2019-04-09 14:56:44 -06002918
2919 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
2920 return -EINVAL;
2921 if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index))
2922 return -EINVAL;
2923
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07002924 req->sync.off = READ_ONCE(sqe->off);
2925 req->sync.len = READ_ONCE(sqe->len);
2926 req->sync.flags = READ_ONCE(sqe->sync_range_flags);
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07002927 return 0;
2928}
2929
2930static void io_sync_file_range_finish(struct io_wq_work **workptr)
2931{
2932 struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);
2933 struct io_kiocb *nxt = NULL;
2934 int ret;
2935
2936 if (io_req_cancelled(req))
2937 return;
2938
Jens Axboe9adbd452019-12-20 08:45:55 -07002939 ret = sync_file_range(req->file, req->sync.off, req->sync.len,
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07002940 req->sync.flags);
2941 if (ret < 0)
2942 req_set_fail_links(req);
2943 io_cqring_add_event(req, ret);
2944 io_put_req_find_next(req, &nxt);
2945 if (nxt)
Jens Axboe78912932020-01-14 22:09:06 -07002946 io_wq_assign_next(workptr, nxt);
Jens Axboe5d17b4a2019-04-09 14:56:44 -06002947}
2948
Jens Axboefc4df992019-12-10 14:38:45 -07002949static int io_sync_file_range(struct io_kiocb *req, struct io_kiocb **nxt,
Jens Axboe5d17b4a2019-04-09 14:56:44 -06002950 bool force_nonblock)
2951{
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07002952 struct io_wq_work *work, *old_work;
Jens Axboe5d17b4a2019-04-09 14:56:44 -06002953
2954 /* sync_file_range always requires a blocking context */
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07002955 if (force_nonblock) {
2956 io_put_req(req);
2957 req->work.func = io_sync_file_range_finish;
Jens Axboe5d17b4a2019-04-09 14:56:44 -06002958 return -EAGAIN;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07002959 }
Jens Axboe5d17b4a2019-04-09 14:56:44 -06002960
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07002961 work = old_work = &req->work;
2962 io_sync_file_range_finish(&work);
2963 if (work && work != old_work)
2964 *nxt = container_of(work, struct io_kiocb, work);
Jens Axboe5d17b4a2019-04-09 14:56:44 -06002965 return 0;
2966}
2967
Jens Axboe3529d8c2019-12-19 18:24:38 -07002968static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboeaa1fa282019-04-19 13:38:09 -06002969{
Jens Axboe03b12302019-12-02 18:50:25 -07002970#if defined(CONFIG_NET)
Jens Axboee47293f2019-12-20 08:58:21 -07002971 struct io_sr_msg *sr = &req->sr_msg;
Jens Axboe3529d8c2019-12-19 18:24:38 -07002972 struct io_async_ctx *io = req->io;
Pavel Begunkov99bc4c32020-02-07 22:04:45 +03002973 int ret;
Jens Axboe03b12302019-12-02 18:50:25 -07002974
Jens Axboee47293f2019-12-20 08:58:21 -07002975 sr->msg_flags = READ_ONCE(sqe->msg_flags);
2976 sr->msg = u64_to_user_ptr(READ_ONCE(sqe->addr));
Jens Axboefddafac2020-01-04 20:19:44 -07002977 sr->len = READ_ONCE(sqe->len);
Jens Axboe3529d8c2019-12-19 18:24:38 -07002978
Jens Axboefddafac2020-01-04 20:19:44 -07002979 if (!io || req->opcode == IORING_OP_SEND)
Jens Axboe3529d8c2019-12-19 18:24:38 -07002980 return 0;
2981
Jens Axboed9688562019-12-09 19:35:20 -07002982 io->msg.iov = io->msg.fast_iov;
Pavel Begunkov99bc4c32020-02-07 22:04:45 +03002983 ret = sendmsg_copy_msghdr(&io->msg.msg, sr->msg, sr->msg_flags,
Jens Axboee47293f2019-12-20 08:58:21 -07002984 &io->msg.iov);
Pavel Begunkov99bc4c32020-02-07 22:04:45 +03002985 if (!ret)
2986 req->flags |= REQ_F_NEED_CLEANUP;
2987 return ret;
Jens Axboe03b12302019-12-02 18:50:25 -07002988#else
Jens Axboee47293f2019-12-20 08:58:21 -07002989 return -EOPNOTSUPP;
Jens Axboe03b12302019-12-02 18:50:25 -07002990#endif
2991}
2992
Jens Axboefc4df992019-12-10 14:38:45 -07002993static int io_sendmsg(struct io_kiocb *req, struct io_kiocb **nxt,
2994 bool force_nonblock)
Jens Axboe03b12302019-12-02 18:50:25 -07002995{
2996#if defined(CONFIG_NET)
Jens Axboe0b416c32019-12-15 10:57:46 -07002997 struct io_async_msghdr *kmsg = NULL;
Jens Axboe03b12302019-12-02 18:50:25 -07002998 struct socket *sock;
2999 int ret;
3000
3001 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3002 return -EINVAL;
3003
3004 sock = sock_from_file(req->file, &ret);
3005 if (sock) {
Jens Axboeb7bb4f72019-12-15 22:13:43 -07003006 struct io_async_ctx io;
Jens Axboe03b12302019-12-02 18:50:25 -07003007 struct sockaddr_storage addr;
Jens Axboe03b12302019-12-02 18:50:25 -07003008 unsigned flags;
3009
Jens Axboe03b12302019-12-02 18:50:25 -07003010 if (req->io) {
Jens Axboe0b416c32019-12-15 10:57:46 -07003011 kmsg = &req->io->msg;
3012 kmsg->msg.msg_name = &addr;
3013 /* if iov is set, it's allocated already */
3014 if (!kmsg->iov)
3015 kmsg->iov = kmsg->fast_iov;
3016 kmsg->msg.msg_iter.iov = kmsg->iov;
Jens Axboe03b12302019-12-02 18:50:25 -07003017 } else {
Jens Axboe3529d8c2019-12-19 18:24:38 -07003018 struct io_sr_msg *sr = &req->sr_msg;
3019
Jens Axboe0b416c32019-12-15 10:57:46 -07003020 kmsg = &io.msg;
3021 kmsg->msg.msg_name = &addr;
Jens Axboe3529d8c2019-12-19 18:24:38 -07003022
3023 io.msg.iov = io.msg.fast_iov;
3024 ret = sendmsg_copy_msghdr(&io.msg.msg, sr->msg,
3025 sr->msg_flags, &io.msg.iov);
Jens Axboe03b12302019-12-02 18:50:25 -07003026 if (ret)
Jens Axboe3529d8c2019-12-19 18:24:38 -07003027 return ret;
Jens Axboe03b12302019-12-02 18:50:25 -07003028 }
3029
Jens Axboee47293f2019-12-20 08:58:21 -07003030 flags = req->sr_msg.msg_flags;
3031 if (flags & MSG_DONTWAIT)
3032 req->flags |= REQ_F_NOWAIT;
3033 else if (force_nonblock)
3034 flags |= MSG_DONTWAIT;
3035
Jens Axboe0b416c32019-12-15 10:57:46 -07003036 ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags);
Jens Axboe03b12302019-12-02 18:50:25 -07003037 if (force_nonblock && ret == -EAGAIN) {
Jens Axboeb7bb4f72019-12-15 22:13:43 -07003038 if (req->io)
3039 return -EAGAIN;
Pavel Begunkov1e950812020-02-06 19:51:16 +03003040 if (io_alloc_async_ctx(req)) {
3041 if (kmsg && kmsg->iov != kmsg->fast_iov)
3042 kfree(kmsg->iov);
Jens Axboeb7bb4f72019-12-15 22:13:43 -07003043 return -ENOMEM;
Pavel Begunkov1e950812020-02-06 19:51:16 +03003044 }
Pavel Begunkov99bc4c32020-02-07 22:04:45 +03003045 req->flags |= REQ_F_NEED_CLEANUP;
Jens Axboeb7bb4f72019-12-15 22:13:43 -07003046 memcpy(&req->io->msg, &io.msg, sizeof(io.msg));
Jens Axboe0b416c32019-12-15 10:57:46 -07003047 return -EAGAIN;
Jens Axboe03b12302019-12-02 18:50:25 -07003048 }
3049 if (ret == -ERESTARTSYS)
3050 ret = -EINTR;
3051 }
3052
Pavel Begunkov1e950812020-02-06 19:51:16 +03003053 if (kmsg && kmsg->iov != kmsg->fast_iov)
Jens Axboe0b416c32019-12-15 10:57:46 -07003054 kfree(kmsg->iov);
Pavel Begunkov99bc4c32020-02-07 22:04:45 +03003055 req->flags &= ~REQ_F_NEED_CLEANUP;
Jens Axboe03b12302019-12-02 18:50:25 -07003056 io_cqring_add_event(req, ret);
Jens Axboe4e88d6e2019-12-07 20:59:47 -07003057 if (ret < 0)
3058 req_set_fail_links(req);
Jens Axboe03b12302019-12-02 18:50:25 -07003059 io_put_req_find_next(req, nxt);
3060 return 0;
3061#else
3062 return -EOPNOTSUPP;
3063#endif
3064}
3065
Jens Axboefddafac2020-01-04 20:19:44 -07003066static int io_send(struct io_kiocb *req, struct io_kiocb **nxt,
3067 bool force_nonblock)
3068{
3069#if defined(CONFIG_NET)
3070 struct socket *sock;
3071 int ret;
3072
3073 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3074 return -EINVAL;
3075
3076 sock = sock_from_file(req->file, &ret);
3077 if (sock) {
3078 struct io_sr_msg *sr = &req->sr_msg;
3079 struct msghdr msg;
3080 struct iovec iov;
3081 unsigned flags;
3082
3083 ret = import_single_range(WRITE, sr->buf, sr->len, &iov,
3084 &msg.msg_iter);
3085 if (ret)
3086 return ret;
3087
3088 msg.msg_name = NULL;
3089 msg.msg_control = NULL;
3090 msg.msg_controllen = 0;
3091 msg.msg_namelen = 0;
3092
3093 flags = req->sr_msg.msg_flags;
3094 if (flags & MSG_DONTWAIT)
3095 req->flags |= REQ_F_NOWAIT;
3096 else if (force_nonblock)
3097 flags |= MSG_DONTWAIT;
3098
Jens Axboe0b7b21e2020-01-31 08:34:59 -07003099 msg.msg_flags = flags;
3100 ret = sock_sendmsg(sock, &msg);
Jens Axboefddafac2020-01-04 20:19:44 -07003101 if (force_nonblock && ret == -EAGAIN)
3102 return -EAGAIN;
3103 if (ret == -ERESTARTSYS)
3104 ret = -EINTR;
3105 }
3106
3107 io_cqring_add_event(req, ret);
3108 if (ret < 0)
3109 req_set_fail_links(req);
3110 io_put_req_find_next(req, nxt);
3111 return 0;
3112#else
3113 return -EOPNOTSUPP;
3114#endif
3115}
3116
Jens Axboe3529d8c2019-12-19 18:24:38 -07003117static int io_recvmsg_prep(struct io_kiocb *req,
3118 const struct io_uring_sqe *sqe)
Jens Axboe03b12302019-12-02 18:50:25 -07003119{
3120#if defined(CONFIG_NET)
Jens Axboee47293f2019-12-20 08:58:21 -07003121 struct io_sr_msg *sr = &req->sr_msg;
Jens Axboe3529d8c2019-12-19 18:24:38 -07003122 struct io_async_ctx *io = req->io;
Pavel Begunkov99bc4c32020-02-07 22:04:45 +03003123 int ret;
Jens Axboe06b76d42019-12-19 14:44:26 -07003124
Jens Axboe3529d8c2019-12-19 18:24:38 -07003125 sr->msg_flags = READ_ONCE(sqe->msg_flags);
3126 sr->msg = u64_to_user_ptr(READ_ONCE(sqe->addr));
Jens Axboe0b7b21e2020-01-31 08:34:59 -07003127 sr->len = READ_ONCE(sqe->len);
Jens Axboe3529d8c2019-12-19 18:24:38 -07003128
Jens Axboefddafac2020-01-04 20:19:44 -07003129 if (!io || req->opcode == IORING_OP_RECV)
Jens Axboe06b76d42019-12-19 14:44:26 -07003130 return 0;
Jens Axboe03b12302019-12-02 18:50:25 -07003131
Jens Axboed9688562019-12-09 19:35:20 -07003132 io->msg.iov = io->msg.fast_iov;
Pavel Begunkov99bc4c32020-02-07 22:04:45 +03003133 ret = recvmsg_copy_msghdr(&io->msg.msg, sr->msg, sr->msg_flags,
Jens Axboee47293f2019-12-20 08:58:21 -07003134 &io->msg.uaddr, &io->msg.iov);
Pavel Begunkov99bc4c32020-02-07 22:04:45 +03003135 if (!ret)
3136 req->flags |= REQ_F_NEED_CLEANUP;
3137 return ret;
Jens Axboe03b12302019-12-02 18:50:25 -07003138#else
Jens Axboee47293f2019-12-20 08:58:21 -07003139 return -EOPNOTSUPP;
Jens Axboe03b12302019-12-02 18:50:25 -07003140#endif
3141}
3142
Jens Axboefc4df992019-12-10 14:38:45 -07003143static int io_recvmsg(struct io_kiocb *req, struct io_kiocb **nxt,
3144 bool force_nonblock)
Jens Axboe03b12302019-12-02 18:50:25 -07003145{
3146#if defined(CONFIG_NET)
Jens Axboe0b416c32019-12-15 10:57:46 -07003147 struct io_async_msghdr *kmsg = NULL;
Jens Axboe0fa03c62019-04-19 13:34:07 -06003148 struct socket *sock;
3149 int ret;
3150
3151 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3152 return -EINVAL;
3153
3154 sock = sock_from_file(req->file, &ret);
3155 if (sock) {
Jens Axboeb7bb4f72019-12-15 22:13:43 -07003156 struct io_async_ctx io;
Jens Axboe03b12302019-12-02 18:50:25 -07003157 struct sockaddr_storage addr;
Jens Axboe0fa03c62019-04-19 13:34:07 -06003158 unsigned flags;
3159
Jens Axboe03b12302019-12-02 18:50:25 -07003160 if (req->io) {
Jens Axboe0b416c32019-12-15 10:57:46 -07003161 kmsg = &req->io->msg;
3162 kmsg->msg.msg_name = &addr;
3163 /* if iov is set, it's allocated already */
3164 if (!kmsg->iov)
3165 kmsg->iov = kmsg->fast_iov;
3166 kmsg->msg.msg_iter.iov = kmsg->iov;
Jens Axboe03b12302019-12-02 18:50:25 -07003167 } else {
Jens Axboe3529d8c2019-12-19 18:24:38 -07003168 struct io_sr_msg *sr = &req->sr_msg;
3169
Jens Axboe0b416c32019-12-15 10:57:46 -07003170 kmsg = &io.msg;
3171 kmsg->msg.msg_name = &addr;
Jens Axboe3529d8c2019-12-19 18:24:38 -07003172
3173 io.msg.iov = io.msg.fast_iov;
3174 ret = recvmsg_copy_msghdr(&io.msg.msg, sr->msg,
3175 sr->msg_flags, &io.msg.uaddr,
3176 &io.msg.iov);
Jens Axboe03b12302019-12-02 18:50:25 -07003177 if (ret)
Jens Axboe3529d8c2019-12-19 18:24:38 -07003178 return ret;
Jens Axboe03b12302019-12-02 18:50:25 -07003179 }
Jens Axboe0fa03c62019-04-19 13:34:07 -06003180
Jens Axboee47293f2019-12-20 08:58:21 -07003181 flags = req->sr_msg.msg_flags;
3182 if (flags & MSG_DONTWAIT)
3183 req->flags |= REQ_F_NOWAIT;
3184 else if (force_nonblock)
3185 flags |= MSG_DONTWAIT;
3186
3187 ret = __sys_recvmsg_sock(sock, &kmsg->msg, req->sr_msg.msg,
3188 kmsg->uaddr, flags);
Jens Axboe03b12302019-12-02 18:50:25 -07003189 if (force_nonblock && ret == -EAGAIN) {
Jens Axboeb7bb4f72019-12-15 22:13:43 -07003190 if (req->io)
3191 return -EAGAIN;
Pavel Begunkov1e950812020-02-06 19:51:16 +03003192 if (io_alloc_async_ctx(req)) {
3193 if (kmsg && kmsg->iov != kmsg->fast_iov)
3194 kfree(kmsg->iov);
Jens Axboeb7bb4f72019-12-15 22:13:43 -07003195 return -ENOMEM;
Pavel Begunkov1e950812020-02-06 19:51:16 +03003196 }
Jens Axboeb7bb4f72019-12-15 22:13:43 -07003197 memcpy(&req->io->msg, &io.msg, sizeof(io.msg));
Pavel Begunkov99bc4c32020-02-07 22:04:45 +03003198 req->flags |= REQ_F_NEED_CLEANUP;
Jens Axboe0b416c32019-12-15 10:57:46 -07003199 return -EAGAIN;
Jens Axboe03b12302019-12-02 18:50:25 -07003200 }
Jens Axboe441cdbd2019-12-02 18:49:10 -07003201 if (ret == -ERESTARTSYS)
3202 ret = -EINTR;
Jens Axboe0fa03c62019-04-19 13:34:07 -06003203 }
3204
Pavel Begunkov1e950812020-02-06 19:51:16 +03003205 if (kmsg && kmsg->iov != kmsg->fast_iov)
Jens Axboe0b416c32019-12-15 10:57:46 -07003206 kfree(kmsg->iov);
Pavel Begunkov99bc4c32020-02-07 22:04:45 +03003207 req->flags &= ~REQ_F_NEED_CLEANUP;
Jens Axboe78e19bb2019-11-06 15:21:34 -07003208 io_cqring_add_event(req, ret);
Jens Axboe4e88d6e2019-12-07 20:59:47 -07003209 if (ret < 0)
3210 req_set_fail_links(req);
Jackie Liuec9c02a2019-11-08 23:50:36 +08003211 io_put_req_find_next(req, nxt);
Jens Axboe0fa03c62019-04-19 13:34:07 -06003212 return 0;
3213#else
3214 return -EOPNOTSUPP;
3215#endif
3216}
3217
Jens Axboefddafac2020-01-04 20:19:44 -07003218static int io_recv(struct io_kiocb *req, struct io_kiocb **nxt,
3219 bool force_nonblock)
3220{
3221#if defined(CONFIG_NET)
3222 struct socket *sock;
3223 int ret;
3224
3225 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3226 return -EINVAL;
3227
3228 sock = sock_from_file(req->file, &ret);
3229 if (sock) {
3230 struct io_sr_msg *sr = &req->sr_msg;
3231 struct msghdr msg;
3232 struct iovec iov;
3233 unsigned flags;
3234
3235 ret = import_single_range(READ, sr->buf, sr->len, &iov,
3236 &msg.msg_iter);
3237 if (ret)
3238 return ret;
3239
3240 msg.msg_name = NULL;
3241 msg.msg_control = NULL;
3242 msg.msg_controllen = 0;
3243 msg.msg_namelen = 0;
3244 msg.msg_iocb = NULL;
3245 msg.msg_flags = 0;
3246
3247 flags = req->sr_msg.msg_flags;
3248 if (flags & MSG_DONTWAIT)
3249 req->flags |= REQ_F_NOWAIT;
3250 else if (force_nonblock)
3251 flags |= MSG_DONTWAIT;
3252
Jens Axboe0b7b21e2020-01-31 08:34:59 -07003253 ret = sock_recvmsg(sock, &msg, flags);
Jens Axboefddafac2020-01-04 20:19:44 -07003254 if (force_nonblock && ret == -EAGAIN)
3255 return -EAGAIN;
3256 if (ret == -ERESTARTSYS)
3257 ret = -EINTR;
3258 }
3259
3260 io_cqring_add_event(req, ret);
3261 if (ret < 0)
3262 req_set_fail_links(req);
3263 io_put_req_find_next(req, nxt);
3264 return 0;
3265#else
3266 return -EOPNOTSUPP;
3267#endif
3268}
3269
3270
Jens Axboe3529d8c2019-12-19 18:24:38 -07003271static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboe17f2fe32019-10-17 14:42:58 -06003272{
3273#if defined(CONFIG_NET)
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07003274 struct io_accept *accept = &req->accept;
3275
Jens Axboe17f2fe32019-10-17 14:42:58 -06003276 if (unlikely(req->ctx->flags & (IORING_SETUP_IOPOLL|IORING_SETUP_SQPOLL)))
3277 return -EINVAL;
Hrvoje Zeba8042d6c2019-11-25 14:40:22 -05003278 if (sqe->ioprio || sqe->len || sqe->buf_index)
Jens Axboe17f2fe32019-10-17 14:42:58 -06003279 return -EINVAL;
3280
Jens Axboed55e5f52019-12-11 16:12:15 -07003281 accept->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));
3282 accept->addr_len = u64_to_user_ptr(READ_ONCE(sqe->addr2));
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07003283 accept->flags = READ_ONCE(sqe->accept_flags);
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07003284 return 0;
3285#else
3286 return -EOPNOTSUPP;
3287#endif
3288}
Jens Axboe17f2fe32019-10-17 14:42:58 -06003289
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07003290#if defined(CONFIG_NET)
3291static int __io_accept(struct io_kiocb *req, struct io_kiocb **nxt,
3292 bool force_nonblock)
3293{
3294 struct io_accept *accept = &req->accept;
3295 unsigned file_flags;
3296 int ret;
3297
3298 file_flags = force_nonblock ? O_NONBLOCK : 0;
3299 ret = __sys_accept4_file(req->file, file_flags, accept->addr,
3300 accept->addr_len, accept->flags);
3301 if (ret == -EAGAIN && force_nonblock)
Jens Axboe17f2fe32019-10-17 14:42:58 -06003302 return -EAGAIN;
Jens Axboe8e3cca12019-11-09 19:52:33 -07003303 if (ret == -ERESTARTSYS)
3304 ret = -EINTR;
Jens Axboe4e88d6e2019-12-07 20:59:47 -07003305 if (ret < 0)
3306 req_set_fail_links(req);
Jens Axboe78e19bb2019-11-06 15:21:34 -07003307 io_cqring_add_event(req, ret);
Jackie Liuec9c02a2019-11-08 23:50:36 +08003308 io_put_req_find_next(req, nxt);
Jens Axboe17f2fe32019-10-17 14:42:58 -06003309 return 0;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07003310}
3311
3312static void io_accept_finish(struct io_wq_work **workptr)
3313{
3314 struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);
3315 struct io_kiocb *nxt = NULL;
3316
3317 if (io_req_cancelled(req))
3318 return;
3319 __io_accept(req, &nxt, false);
3320 if (nxt)
Jens Axboe78912932020-01-14 22:09:06 -07003321 io_wq_assign_next(workptr, nxt);
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07003322}
3323#endif
3324
3325static int io_accept(struct io_kiocb *req, struct io_kiocb **nxt,
3326 bool force_nonblock)
3327{
3328#if defined(CONFIG_NET)
3329 int ret;
3330
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07003331 ret = __io_accept(req, nxt, force_nonblock);
3332 if (ret == -EAGAIN && force_nonblock) {
3333 req->work.func = io_accept_finish;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07003334 io_put_req(req);
3335 return -EAGAIN;
3336 }
3337 return 0;
Jens Axboe17f2fe32019-10-17 14:42:58 -06003338#else
3339 return -EOPNOTSUPP;
3340#endif
3341}
3342
Jens Axboe3529d8c2019-12-19 18:24:38 -07003343static int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboef499a022019-12-02 16:28:46 -07003344{
3345#if defined(CONFIG_NET)
Jens Axboe3529d8c2019-12-19 18:24:38 -07003346 struct io_connect *conn = &req->connect;
3347 struct io_async_ctx *io = req->io;
Jens Axboef499a022019-12-02 16:28:46 -07003348
Jens Axboe3fbb51c2019-12-20 08:51:52 -07003349 if (unlikely(req->ctx->flags & (IORING_SETUP_IOPOLL|IORING_SETUP_SQPOLL)))
3350 return -EINVAL;
3351 if (sqe->ioprio || sqe->len || sqe->buf_index || sqe->rw_flags)
3352 return -EINVAL;
3353
Jens Axboe3529d8c2019-12-19 18:24:38 -07003354 conn->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));
3355 conn->addr_len = READ_ONCE(sqe->addr2);
3356
3357 if (!io)
3358 return 0;
3359
3360 return move_addr_to_kernel(conn->addr, conn->addr_len,
Jens Axboe3fbb51c2019-12-20 08:51:52 -07003361 &io->connect.address);
Jens Axboef499a022019-12-02 16:28:46 -07003362#else
Jens Axboe3fbb51c2019-12-20 08:51:52 -07003363 return -EOPNOTSUPP;
Jens Axboef499a022019-12-02 16:28:46 -07003364#endif
3365}
3366
Jens Axboefc4df992019-12-10 14:38:45 -07003367static int io_connect(struct io_kiocb *req, struct io_kiocb **nxt,
3368 bool force_nonblock)
Jens Axboef8e85cf2019-11-23 14:24:24 -07003369{
3370#if defined(CONFIG_NET)
Jens Axboef499a022019-12-02 16:28:46 -07003371 struct io_async_ctx __io, *io;
Jens Axboef8e85cf2019-11-23 14:24:24 -07003372 unsigned file_flags;
Jens Axboe3fbb51c2019-12-20 08:51:52 -07003373 int ret;
Jens Axboef8e85cf2019-11-23 14:24:24 -07003374
Jens Axboef499a022019-12-02 16:28:46 -07003375 if (req->io) {
3376 io = req->io;
3377 } else {
Jens Axboe3529d8c2019-12-19 18:24:38 -07003378 ret = move_addr_to_kernel(req->connect.addr,
3379 req->connect.addr_len,
3380 &__io.connect.address);
Jens Axboef499a022019-12-02 16:28:46 -07003381 if (ret)
3382 goto out;
3383 io = &__io;
3384 }
3385
Jens Axboe3fbb51c2019-12-20 08:51:52 -07003386 file_flags = force_nonblock ? O_NONBLOCK : 0;
3387
3388 ret = __sys_connect_file(req->file, &io->connect.address,
3389 req->connect.addr_len, file_flags);
Jens Axboe87f80d62019-12-03 11:23:54 -07003390 if ((ret == -EAGAIN || ret == -EINPROGRESS) && force_nonblock) {
Jens Axboeb7bb4f72019-12-15 22:13:43 -07003391 if (req->io)
3392 return -EAGAIN;
3393 if (io_alloc_async_ctx(req)) {
Jens Axboef499a022019-12-02 16:28:46 -07003394 ret = -ENOMEM;
3395 goto out;
3396 }
Jens Axboeb7bb4f72019-12-15 22:13:43 -07003397 memcpy(&req->io->connect, &__io.connect, sizeof(__io.connect));
Jens Axboef8e85cf2019-11-23 14:24:24 -07003398 return -EAGAIN;
Jens Axboef499a022019-12-02 16:28:46 -07003399 }
Jens Axboef8e85cf2019-11-23 14:24:24 -07003400 if (ret == -ERESTARTSYS)
3401 ret = -EINTR;
Jens Axboef499a022019-12-02 16:28:46 -07003402out:
Jens Axboe4e88d6e2019-12-07 20:59:47 -07003403 if (ret < 0)
3404 req_set_fail_links(req);
Jens Axboef8e85cf2019-11-23 14:24:24 -07003405 io_cqring_add_event(req, ret);
3406 io_put_req_find_next(req, nxt);
3407 return 0;
3408#else
3409 return -EOPNOTSUPP;
3410#endif
3411}
3412
Jens Axboe221c5eb2019-01-17 09:41:58 -07003413static void io_poll_remove_one(struct io_kiocb *req)
3414{
3415 struct io_poll_iocb *poll = &req->poll;
3416
3417 spin_lock(&poll->head->lock);
3418 WRITE_ONCE(poll->canceled, true);
Jens Axboe392edb42019-12-09 17:52:20 -07003419 if (!list_empty(&poll->wait.entry)) {
3420 list_del_init(&poll->wait.entry);
Jackie Liua197f662019-11-08 08:09:12 -07003421 io_queue_async_work(req);
Jens Axboe221c5eb2019-01-17 09:41:58 -07003422 }
3423 spin_unlock(&poll->head->lock);
Jens Axboe78076bb2019-12-04 19:56:40 -07003424 hash_del(&req->hash_node);
Jens Axboe221c5eb2019-01-17 09:41:58 -07003425}
3426
3427static void io_poll_remove_all(struct io_ring_ctx *ctx)
3428{
Jens Axboe78076bb2019-12-04 19:56:40 -07003429 struct hlist_node *tmp;
Jens Axboe221c5eb2019-01-17 09:41:58 -07003430 struct io_kiocb *req;
Jens Axboe78076bb2019-12-04 19:56:40 -07003431 int i;
Jens Axboe221c5eb2019-01-17 09:41:58 -07003432
3433 spin_lock_irq(&ctx->completion_lock);
Jens Axboe78076bb2019-12-04 19:56:40 -07003434 for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) {
3435 struct hlist_head *list;
3436
3437 list = &ctx->cancel_hash[i];
3438 hlist_for_each_entry_safe(req, tmp, list, hash_node)
3439 io_poll_remove_one(req);
Jens Axboe221c5eb2019-01-17 09:41:58 -07003440 }
3441 spin_unlock_irq(&ctx->completion_lock);
3442}
3443
Jens Axboe47f46762019-11-09 17:43:02 -07003444static int io_poll_cancel(struct io_ring_ctx *ctx, __u64 sqe_addr)
3445{
Jens Axboe78076bb2019-12-04 19:56:40 -07003446 struct hlist_head *list;
Jens Axboe47f46762019-11-09 17:43:02 -07003447 struct io_kiocb *req;
3448
Jens Axboe78076bb2019-12-04 19:56:40 -07003449 list = &ctx->cancel_hash[hash_long(sqe_addr, ctx->cancel_hash_bits)];
3450 hlist_for_each_entry(req, list, hash_node) {
3451 if (sqe_addr == req->user_data) {
Jens Axboeeac406c2019-11-14 12:09:58 -07003452 io_poll_remove_one(req);
3453 return 0;
3454 }
Jens Axboe47f46762019-11-09 17:43:02 -07003455 }
3456
3457 return -ENOENT;
3458}
3459
Jens Axboe3529d8c2019-12-19 18:24:38 -07003460static int io_poll_remove_prep(struct io_kiocb *req,
3461 const struct io_uring_sqe *sqe)
Jens Axboe221c5eb2019-01-17 09:41:58 -07003462{
Jens Axboe221c5eb2019-01-17 09:41:58 -07003463 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3464 return -EINVAL;
3465 if (sqe->ioprio || sqe->off || sqe->len || sqe->buf_index ||
3466 sqe->poll_events)
3467 return -EINVAL;
3468
Jens Axboe0969e782019-12-17 18:40:57 -07003469 req->poll.addr = READ_ONCE(sqe->addr);
Jens Axboe0969e782019-12-17 18:40:57 -07003470 return 0;
3471}
3472
3473/*
3474 * Find a running poll command that matches one specified in sqe->addr,
3475 * and remove it if found.
3476 */
3477static int io_poll_remove(struct io_kiocb *req)
3478{
3479 struct io_ring_ctx *ctx = req->ctx;
3480 u64 addr;
3481 int ret;
3482
Jens Axboe0969e782019-12-17 18:40:57 -07003483 addr = req->poll.addr;
Jens Axboe221c5eb2019-01-17 09:41:58 -07003484 spin_lock_irq(&ctx->completion_lock);
Jens Axboe0969e782019-12-17 18:40:57 -07003485 ret = io_poll_cancel(ctx, addr);
Jens Axboe221c5eb2019-01-17 09:41:58 -07003486 spin_unlock_irq(&ctx->completion_lock);
3487
Jens Axboe78e19bb2019-11-06 15:21:34 -07003488 io_cqring_add_event(req, ret);
Jens Axboe4e88d6e2019-12-07 20:59:47 -07003489 if (ret < 0)
3490 req_set_fail_links(req);
Jens Axboee65ef562019-03-12 10:16:44 -06003491 io_put_req(req);
Jens Axboe221c5eb2019-01-17 09:41:58 -07003492 return 0;
3493}
3494
Jens Axboeb0dd8a42019-11-18 12:14:54 -07003495static void io_poll_complete(struct io_kiocb *req, __poll_t mask, int error)
Jens Axboe221c5eb2019-01-17 09:41:58 -07003496{
Jackie Liua197f662019-11-08 08:09:12 -07003497 struct io_ring_ctx *ctx = req->ctx;
3498
Jens Axboe8c838782019-03-12 15:48:16 -06003499 req->poll.done = true;
Jens Axboeb0dd8a42019-11-18 12:14:54 -07003500 if (error)
3501 io_cqring_fill_event(req, error);
3502 else
3503 io_cqring_fill_event(req, mangle_poll(mask));
Jens Axboe8c838782019-03-12 15:48:16 -06003504 io_commit_cqring(ctx);
Jens Axboe221c5eb2019-01-17 09:41:58 -07003505}
3506
Jens Axboe561fb042019-10-24 07:25:42 -06003507static void io_poll_complete_work(struct io_wq_work **workptr)
Jens Axboe221c5eb2019-01-17 09:41:58 -07003508{
Jens Axboe561fb042019-10-24 07:25:42 -06003509 struct io_wq_work *work = *workptr;
Jens Axboe221c5eb2019-01-17 09:41:58 -07003510 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
3511 struct io_poll_iocb *poll = &req->poll;
3512 struct poll_table_struct pt = { ._key = poll->events };
3513 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe89723d02019-11-05 15:32:58 -07003514 struct io_kiocb *nxt = NULL;
Jens Axboe221c5eb2019-01-17 09:41:58 -07003515 __poll_t mask = 0;
Jens Axboeb0dd8a42019-11-18 12:14:54 -07003516 int ret = 0;
Jens Axboe221c5eb2019-01-17 09:41:58 -07003517
Jens Axboeb0dd8a42019-11-18 12:14:54 -07003518 if (work->flags & IO_WQ_WORK_CANCEL) {
Jens Axboe561fb042019-10-24 07:25:42 -06003519 WRITE_ONCE(poll->canceled, true);
Jens Axboeb0dd8a42019-11-18 12:14:54 -07003520 ret = -ECANCELED;
3521 } else if (READ_ONCE(poll->canceled)) {
3522 ret = -ECANCELED;
3523 }
Jens Axboe561fb042019-10-24 07:25:42 -06003524
Jens Axboeb0dd8a42019-11-18 12:14:54 -07003525 if (ret != -ECANCELED)
Jens Axboe221c5eb2019-01-17 09:41:58 -07003526 mask = vfs_poll(poll->file, &pt) & poll->events;
3527
3528 /*
3529 * Note that ->ki_cancel callers also delete iocb from active_reqs after
3530 * calling ->ki_cancel. We need the ctx_lock roundtrip here to
3531 * synchronize with them. In the cancellation case the list_del_init
3532 * itself is not actually needed, but harmless so we keep it in to
3533 * avoid further branches in the fast path.
3534 */
3535 spin_lock_irq(&ctx->completion_lock);
Jens Axboeb0dd8a42019-11-18 12:14:54 -07003536 if (!mask && ret != -ECANCELED) {
Jens Axboe392edb42019-12-09 17:52:20 -07003537 add_wait_queue(poll->head, &poll->wait);
Jens Axboe221c5eb2019-01-17 09:41:58 -07003538 spin_unlock_irq(&ctx->completion_lock);
3539 return;
3540 }
Jens Axboe78076bb2019-12-04 19:56:40 -07003541 hash_del(&req->hash_node);
Jens Axboeb0dd8a42019-11-18 12:14:54 -07003542 io_poll_complete(req, mask, ret);
Jens Axboe221c5eb2019-01-17 09:41:58 -07003543 spin_unlock_irq(&ctx->completion_lock);
3544
Jens Axboe8c838782019-03-12 15:48:16 -06003545 io_cqring_ev_posted(ctx);
Jens Axboe89723d02019-11-05 15:32:58 -07003546
Jens Axboe4e88d6e2019-12-07 20:59:47 -07003547 if (ret < 0)
3548 req_set_fail_links(req);
Jackie Liuec9c02a2019-11-08 23:50:36 +08003549 io_put_req_find_next(req, &nxt);
Jens Axboe89723d02019-11-05 15:32:58 -07003550 if (nxt)
Jens Axboe78912932020-01-14 22:09:06 -07003551 io_wq_assign_next(workptr, nxt);
Jens Axboe221c5eb2019-01-17 09:41:58 -07003552}
3553
Jens Axboee94f1412019-12-19 12:06:02 -07003554static void __io_poll_flush(struct io_ring_ctx *ctx, struct llist_node *nodes)
3555{
Jens Axboee94f1412019-12-19 12:06:02 -07003556 struct io_kiocb *req, *tmp;
Jens Axboe8237e042019-12-28 10:48:22 -07003557 struct req_batch rb;
Jens Axboee94f1412019-12-19 12:06:02 -07003558
Jens Axboec6ca97b302019-12-28 12:11:08 -07003559 rb.to_free = rb.need_iter = 0;
Jens Axboee94f1412019-12-19 12:06:02 -07003560 spin_lock_irq(&ctx->completion_lock);
3561 llist_for_each_entry_safe(req, tmp, nodes, llist_node) {
3562 hash_del(&req->hash_node);
3563 io_poll_complete(req, req->result, 0);
3564
Jens Axboe8237e042019-12-28 10:48:22 -07003565 if (refcount_dec_and_test(&req->refs) &&
3566 !io_req_multi_free(&rb, req)) {
3567 req->flags |= REQ_F_COMP_LOCKED;
3568 io_free_req(req);
Jens Axboee94f1412019-12-19 12:06:02 -07003569 }
3570 }
3571 spin_unlock_irq(&ctx->completion_lock);
3572
3573 io_cqring_ev_posted(ctx);
Jens Axboe8237e042019-12-28 10:48:22 -07003574 io_free_req_many(ctx, &rb);
Jens Axboee94f1412019-12-19 12:06:02 -07003575}
3576
3577static void io_poll_flush(struct io_wq_work **workptr)
3578{
3579 struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);
3580 struct llist_node *nodes;
3581
3582 nodes = llist_del_all(&req->ctx->poll_llist);
3583 if (nodes)
3584 __io_poll_flush(req->ctx, nodes);
3585}
3586
Jens Axboef0b493e2020-02-01 21:30:11 -07003587static void io_poll_trigger_evfd(struct io_wq_work **workptr)
3588{
3589 struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);
3590
3591 eventfd_signal(req->ctx->cq_ev_fd, 1);
3592 io_put_req(req);
3593}
3594
Jens Axboe221c5eb2019-01-17 09:41:58 -07003595static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
3596 void *key)
3597{
Jens Axboee9444752019-11-26 15:02:04 -07003598 struct io_poll_iocb *poll = wait->private;
Jens Axboe221c5eb2019-01-17 09:41:58 -07003599 struct io_kiocb *req = container_of(poll, struct io_kiocb, poll);
3600 struct io_ring_ctx *ctx = req->ctx;
3601 __poll_t mask = key_to_poll(key);
Jens Axboe221c5eb2019-01-17 09:41:58 -07003602
3603 /* for instances that support it check for an event match first: */
Jens Axboe8c838782019-03-12 15:48:16 -06003604 if (mask && !(mask & poll->events))
3605 return 0;
Jens Axboe221c5eb2019-01-17 09:41:58 -07003606
Jens Axboe392edb42019-12-09 17:52:20 -07003607 list_del_init(&poll->wait.entry);
Jens Axboe8c838782019-03-12 15:48:16 -06003608
Jens Axboe7c9e7f02019-11-12 08:15:53 -07003609 /*
3610 * Run completion inline if we can. We're using trylock here because
3611 * we are violating the completion_lock -> poll wq lock ordering.
3612 * If we have a link timeout we're going to need the completion_lock
3613 * for finalizing the request, mark us as having grabbed that already.
3614 */
Jens Axboee94f1412019-12-19 12:06:02 -07003615 if (mask) {
3616 unsigned long flags;
Jens Axboe8c838782019-03-12 15:48:16 -06003617
Jens Axboee94f1412019-12-19 12:06:02 -07003618 if (llist_empty(&ctx->poll_llist) &&
3619 spin_trylock_irqsave(&ctx->completion_lock, flags)) {
Jens Axboef0b493e2020-02-01 21:30:11 -07003620 bool trigger_ev;
3621
Jens Axboee94f1412019-12-19 12:06:02 -07003622 hash_del(&req->hash_node);
3623 io_poll_complete(req, mask, 0);
Jens Axboee94f1412019-12-19 12:06:02 -07003624
Jens Axboef0b493e2020-02-01 21:30:11 -07003625 trigger_ev = io_should_trigger_evfd(ctx);
3626 if (trigger_ev && eventfd_signal_count()) {
3627 trigger_ev = false;
3628 req->work.func = io_poll_trigger_evfd;
3629 } else {
3630 req->flags |= REQ_F_COMP_LOCKED;
3631 io_put_req(req);
3632 req = NULL;
3633 }
3634 spin_unlock_irqrestore(&ctx->completion_lock, flags);
3635 __io_cqring_ev_posted(ctx, trigger_ev);
Jens Axboee94f1412019-12-19 12:06:02 -07003636 } else {
3637 req->result = mask;
3638 req->llist_node.next = NULL;
3639 /* if the list wasn't empty, we're done */
3640 if (!llist_add(&req->llist_node, &ctx->poll_llist))
3641 req = NULL;
3642 else
3643 req->work.func = io_poll_flush;
3644 }
Jens Axboe8c838782019-03-12 15:48:16 -06003645 }
Jens Axboee94f1412019-12-19 12:06:02 -07003646 if (req)
3647 io_queue_async_work(req);
Jens Axboe8c838782019-03-12 15:48:16 -06003648
Jens Axboe221c5eb2019-01-17 09:41:58 -07003649 return 1;
3650}
3651
3652struct io_poll_table {
3653 struct poll_table_struct pt;
3654 struct io_kiocb *req;
3655 int error;
3656};
3657
3658static void io_poll_queue_proc(struct file *file, struct wait_queue_head *head,
3659 struct poll_table_struct *p)
3660{
3661 struct io_poll_table *pt = container_of(p, struct io_poll_table, pt);
3662
3663 if (unlikely(pt->req->poll.head)) {
3664 pt->error = -EINVAL;
3665 return;
3666 }
3667
3668 pt->error = 0;
3669 pt->req->poll.head = head;
Jens Axboe392edb42019-12-09 17:52:20 -07003670 add_wait_queue(head, &pt->req->poll.wait);
Jens Axboe221c5eb2019-01-17 09:41:58 -07003671}
3672
Jens Axboeeac406c2019-11-14 12:09:58 -07003673static void io_poll_req_insert(struct io_kiocb *req)
3674{
3675 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe78076bb2019-12-04 19:56:40 -07003676 struct hlist_head *list;
Jens Axboeeac406c2019-11-14 12:09:58 -07003677
Jens Axboe78076bb2019-12-04 19:56:40 -07003678 list = &ctx->cancel_hash[hash_long(req->user_data, ctx->cancel_hash_bits)];
3679 hlist_add_head(&req->hash_node, list);
Jens Axboeeac406c2019-11-14 12:09:58 -07003680}
3681
Jens Axboe3529d8c2019-12-19 18:24:38 -07003682static int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboe221c5eb2019-01-17 09:41:58 -07003683{
3684 struct io_poll_iocb *poll = &req->poll;
Jens Axboe221c5eb2019-01-17 09:41:58 -07003685 u16 events;
Jens Axboe221c5eb2019-01-17 09:41:58 -07003686
3687 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3688 return -EINVAL;
3689 if (sqe->addr || sqe->ioprio || sqe->off || sqe->len || sqe->buf_index)
3690 return -EINVAL;
Jens Axboe09bb8392019-03-13 12:39:28 -06003691 if (!poll->file)
3692 return -EBADF;
Jens Axboe221c5eb2019-01-17 09:41:58 -07003693
Jens Axboe221c5eb2019-01-17 09:41:58 -07003694 events = READ_ONCE(sqe->poll_events);
3695 poll->events = demangle_poll(events) | EPOLLERR | EPOLLHUP;
Jens Axboe0969e782019-12-17 18:40:57 -07003696 return 0;
3697}
3698
3699static int io_poll_add(struct io_kiocb *req, struct io_kiocb **nxt)
3700{
3701 struct io_poll_iocb *poll = &req->poll;
3702 struct io_ring_ctx *ctx = req->ctx;
3703 struct io_poll_table ipt;
3704 bool cancel = false;
3705 __poll_t mask;
Jens Axboe0969e782019-12-17 18:40:57 -07003706
3707 INIT_IO_WORK(&req->work, io_poll_complete_work);
Jens Axboe78076bb2019-12-04 19:56:40 -07003708 INIT_HLIST_NODE(&req->hash_node);
Jens Axboe221c5eb2019-01-17 09:41:58 -07003709
Jens Axboe221c5eb2019-01-17 09:41:58 -07003710 poll->head = NULL;
Jens Axboe8c838782019-03-12 15:48:16 -06003711 poll->done = false;
Jens Axboe221c5eb2019-01-17 09:41:58 -07003712 poll->canceled = false;
3713
3714 ipt.pt._qproc = io_poll_queue_proc;
3715 ipt.pt._key = poll->events;
3716 ipt.req = req;
3717 ipt.error = -EINVAL; /* same as no support for IOCB_CMD_POLL */
3718
3719 /* initialized the list so that we can do list_empty checks */
Jens Axboe392edb42019-12-09 17:52:20 -07003720 INIT_LIST_HEAD(&poll->wait.entry);
3721 init_waitqueue_func_entry(&poll->wait, io_poll_wake);
3722 poll->wait.private = poll;
Jens Axboe221c5eb2019-01-17 09:41:58 -07003723
Jens Axboe36703242019-07-25 10:20:18 -06003724 INIT_LIST_HEAD(&req->list);
3725
Jens Axboe221c5eb2019-01-17 09:41:58 -07003726 mask = vfs_poll(poll->file, &ipt.pt) & poll->events;
Jens Axboe221c5eb2019-01-17 09:41:58 -07003727
3728 spin_lock_irq(&ctx->completion_lock);
Jens Axboe8c838782019-03-12 15:48:16 -06003729 if (likely(poll->head)) {
3730 spin_lock(&poll->head->lock);
Jens Axboe392edb42019-12-09 17:52:20 -07003731 if (unlikely(list_empty(&poll->wait.entry))) {
Jens Axboe8c838782019-03-12 15:48:16 -06003732 if (ipt.error)
3733 cancel = true;
3734 ipt.error = 0;
3735 mask = 0;
3736 }
3737 if (mask || ipt.error)
Jens Axboe392edb42019-12-09 17:52:20 -07003738 list_del_init(&poll->wait.entry);
Jens Axboe8c838782019-03-12 15:48:16 -06003739 else if (cancel)
3740 WRITE_ONCE(poll->canceled, true);
3741 else if (!poll->done) /* actually waiting for an event */
Jens Axboeeac406c2019-11-14 12:09:58 -07003742 io_poll_req_insert(req);
Jens Axboe8c838782019-03-12 15:48:16 -06003743 spin_unlock(&poll->head->lock);
Jens Axboe221c5eb2019-01-17 09:41:58 -07003744 }
Jens Axboe8c838782019-03-12 15:48:16 -06003745 if (mask) { /* no async, we'd stolen it */
Jens Axboe8c838782019-03-12 15:48:16 -06003746 ipt.error = 0;
Jens Axboeb0dd8a42019-11-18 12:14:54 -07003747 io_poll_complete(req, mask, 0);
Jens Axboe8c838782019-03-12 15:48:16 -06003748 }
Jens Axboe221c5eb2019-01-17 09:41:58 -07003749 spin_unlock_irq(&ctx->completion_lock);
3750
Jens Axboe8c838782019-03-12 15:48:16 -06003751 if (mask) {
3752 io_cqring_ev_posted(ctx);
Jackie Liuec9c02a2019-11-08 23:50:36 +08003753 io_put_req_find_next(req, nxt);
Jens Axboe221c5eb2019-01-17 09:41:58 -07003754 }
Jens Axboe8c838782019-03-12 15:48:16 -06003755 return ipt.error;
Jens Axboe221c5eb2019-01-17 09:41:58 -07003756}
3757
Jens Axboe5262f562019-09-17 12:26:57 -06003758static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer)
3759{
Jens Axboead8a48a2019-11-15 08:49:11 -07003760 struct io_timeout_data *data = container_of(timer,
3761 struct io_timeout_data, timer);
3762 struct io_kiocb *req = data->req;
3763 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe5262f562019-09-17 12:26:57 -06003764 unsigned long flags;
3765
Jens Axboe5262f562019-09-17 12:26:57 -06003766 atomic_inc(&ctx->cq_timeouts);
3767
3768 spin_lock_irqsave(&ctx->completion_lock, flags);
zhangyi (F)ef036812019-10-23 15:10:08 +08003769 /*
Jens Axboe11365042019-10-16 09:08:32 -06003770 * We could be racing with timeout deletion. If the list is empty,
3771 * then timeout lookup already found it and will be handling it.
zhangyi (F)ef036812019-10-23 15:10:08 +08003772 */
Jens Axboe842f9612019-10-29 12:34:10 -06003773 if (!list_empty(&req->list)) {
Jens Axboe11365042019-10-16 09:08:32 -06003774 struct io_kiocb *prev;
Jens Axboe5262f562019-09-17 12:26:57 -06003775
Jens Axboe11365042019-10-16 09:08:32 -06003776 /*
3777 * Adjust the reqs sequence before the current one because it
Brian Gianforcarod195a662019-12-13 03:09:50 -08003778 * will consume a slot in the cq_ring and the cq_tail
Jens Axboe11365042019-10-16 09:08:32 -06003779 * pointer will be increased, otherwise other timeout reqs may
3780 * return in advance without waiting for enough wait_nr.
3781 */
3782 prev = req;
3783 list_for_each_entry_continue_reverse(prev, &ctx->timeout_list, list)
3784 prev->sequence++;
Jens Axboe11365042019-10-16 09:08:32 -06003785 list_del_init(&req->list);
Jens Axboe11365042019-10-16 09:08:32 -06003786 }
Jens Axboe842f9612019-10-29 12:34:10 -06003787
Jens Axboe78e19bb2019-11-06 15:21:34 -07003788 io_cqring_fill_event(req, -ETIME);
Jens Axboe5262f562019-09-17 12:26:57 -06003789 io_commit_cqring(ctx);
3790 spin_unlock_irqrestore(&ctx->completion_lock, flags);
3791
3792 io_cqring_ev_posted(ctx);
Jens Axboe4e88d6e2019-12-07 20:59:47 -07003793 req_set_fail_links(req);
Jens Axboe5262f562019-09-17 12:26:57 -06003794 io_put_req(req);
3795 return HRTIMER_NORESTART;
3796}
3797
Jens Axboe47f46762019-11-09 17:43:02 -07003798static int io_timeout_cancel(struct io_ring_ctx *ctx, __u64 user_data)
3799{
3800 struct io_kiocb *req;
3801 int ret = -ENOENT;
3802
3803 list_for_each_entry(req, &ctx->timeout_list, list) {
3804 if (user_data == req->user_data) {
3805 list_del_init(&req->list);
3806 ret = 0;
3807 break;
3808 }
3809 }
3810
3811 if (ret == -ENOENT)
3812 return ret;
3813
Jens Axboe2d283902019-12-04 11:08:05 -07003814 ret = hrtimer_try_to_cancel(&req->io->timeout.timer);
Jens Axboe47f46762019-11-09 17:43:02 -07003815 if (ret == -1)
3816 return -EALREADY;
3817
Jens Axboe4e88d6e2019-12-07 20:59:47 -07003818 req_set_fail_links(req);
Jens Axboe47f46762019-11-09 17:43:02 -07003819 io_cqring_fill_event(req, -ECANCELED);
3820 io_put_req(req);
3821 return 0;
3822}
3823
Jens Axboe3529d8c2019-12-19 18:24:38 -07003824static int io_timeout_remove_prep(struct io_kiocb *req,
3825 const struct io_uring_sqe *sqe)
Jens Axboeb29472e2019-12-17 18:50:29 -07003826{
Jens Axboeb29472e2019-12-17 18:50:29 -07003827 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3828 return -EINVAL;
3829 if (sqe->flags || sqe->ioprio || sqe->buf_index || sqe->len)
3830 return -EINVAL;
3831
3832 req->timeout.addr = READ_ONCE(sqe->addr);
3833 req->timeout.flags = READ_ONCE(sqe->timeout_flags);
3834 if (req->timeout.flags)
3835 return -EINVAL;
3836
Jens Axboeb29472e2019-12-17 18:50:29 -07003837 return 0;
3838}
3839
Jens Axboe11365042019-10-16 09:08:32 -06003840/*
3841 * Remove or update an existing timeout command
3842 */
Jens Axboefc4df992019-12-10 14:38:45 -07003843static int io_timeout_remove(struct io_kiocb *req)
Jens Axboe11365042019-10-16 09:08:32 -06003844{
3845 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe47f46762019-11-09 17:43:02 -07003846 int ret;
Jens Axboe11365042019-10-16 09:08:32 -06003847
Jens Axboe11365042019-10-16 09:08:32 -06003848 spin_lock_irq(&ctx->completion_lock);
Jens Axboeb29472e2019-12-17 18:50:29 -07003849 ret = io_timeout_cancel(ctx, req->timeout.addr);
Jens Axboe11365042019-10-16 09:08:32 -06003850
Jens Axboe47f46762019-11-09 17:43:02 -07003851 io_cqring_fill_event(req, ret);
Jens Axboe11365042019-10-16 09:08:32 -06003852 io_commit_cqring(ctx);
3853 spin_unlock_irq(&ctx->completion_lock);
Jens Axboe5262f562019-09-17 12:26:57 -06003854 io_cqring_ev_posted(ctx);
Jens Axboe4e88d6e2019-12-07 20:59:47 -07003855 if (ret < 0)
3856 req_set_fail_links(req);
Jackie Liuec9c02a2019-11-08 23:50:36 +08003857 io_put_req(req);
Jens Axboe11365042019-10-16 09:08:32 -06003858 return 0;
Jens Axboe5262f562019-09-17 12:26:57 -06003859}
3860
Jens Axboe3529d8c2019-12-19 18:24:38 -07003861static int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe,
Jens Axboe2d283902019-12-04 11:08:05 -07003862 bool is_timeout_link)
Jens Axboe5262f562019-09-17 12:26:57 -06003863{
Jens Axboead8a48a2019-11-15 08:49:11 -07003864 struct io_timeout_data *data;
Jens Axboea41525a2019-10-15 16:48:15 -06003865 unsigned flags;
Jens Axboe5262f562019-09-17 12:26:57 -06003866
Jens Axboead8a48a2019-11-15 08:49:11 -07003867 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
Jens Axboe5262f562019-09-17 12:26:57 -06003868 return -EINVAL;
Jens Axboead8a48a2019-11-15 08:49:11 -07003869 if (sqe->ioprio || sqe->buf_index || sqe->len != 1)
Jens Axboea41525a2019-10-15 16:48:15 -06003870 return -EINVAL;
Jens Axboe2d283902019-12-04 11:08:05 -07003871 if (sqe->off && is_timeout_link)
3872 return -EINVAL;
Jens Axboea41525a2019-10-15 16:48:15 -06003873 flags = READ_ONCE(sqe->timeout_flags);
3874 if (flags & ~IORING_TIMEOUT_ABS)
Jens Axboe5262f562019-09-17 12:26:57 -06003875 return -EINVAL;
Arnd Bergmannbdf20072019-10-01 09:53:29 -06003876
Jens Axboe26a61672019-12-20 09:02:01 -07003877 req->timeout.count = READ_ONCE(sqe->off);
3878
Jens Axboe3529d8c2019-12-19 18:24:38 -07003879 if (!req->io && io_alloc_async_ctx(req))
Jens Axboe26a61672019-12-20 09:02:01 -07003880 return -ENOMEM;
3881
3882 data = &req->io->timeout;
Jens Axboead8a48a2019-11-15 08:49:11 -07003883 data->req = req;
Jens Axboead8a48a2019-11-15 08:49:11 -07003884 req->flags |= REQ_F_TIMEOUT;
3885
3886 if (get_timespec64(&data->ts, u64_to_user_ptr(sqe->addr)))
Jens Axboe5262f562019-09-17 12:26:57 -06003887 return -EFAULT;
3888
Jens Axboe11365042019-10-16 09:08:32 -06003889 if (flags & IORING_TIMEOUT_ABS)
Jens Axboead8a48a2019-11-15 08:49:11 -07003890 data->mode = HRTIMER_MODE_ABS;
Jens Axboe11365042019-10-16 09:08:32 -06003891 else
Jens Axboead8a48a2019-11-15 08:49:11 -07003892 data->mode = HRTIMER_MODE_REL;
Jens Axboe11365042019-10-16 09:08:32 -06003893
Jens Axboead8a48a2019-11-15 08:49:11 -07003894 hrtimer_init(&data->timer, CLOCK_MONOTONIC, data->mode);
3895 return 0;
3896}
3897
Jens Axboefc4df992019-12-10 14:38:45 -07003898static int io_timeout(struct io_kiocb *req)
Jens Axboead8a48a2019-11-15 08:49:11 -07003899{
3900 unsigned count;
3901 struct io_ring_ctx *ctx = req->ctx;
3902 struct io_timeout_data *data;
3903 struct list_head *entry;
3904 unsigned span = 0;
Jens Axboead8a48a2019-11-15 08:49:11 -07003905
Jens Axboe2d283902019-12-04 11:08:05 -07003906 data = &req->io->timeout;
Jens Axboe93bd25b2019-11-11 23:34:31 -07003907
Jens Axboe5262f562019-09-17 12:26:57 -06003908 /*
3909 * sqe->off holds how many events that need to occur for this
Jens Axboe93bd25b2019-11-11 23:34:31 -07003910 * timeout event to be satisfied. If it isn't set, then this is
3911 * a pure timeout request, sequence isn't used.
Jens Axboe5262f562019-09-17 12:26:57 -06003912 */
Jens Axboe26a61672019-12-20 09:02:01 -07003913 count = req->timeout.count;
Jens Axboe93bd25b2019-11-11 23:34:31 -07003914 if (!count) {
3915 req->flags |= REQ_F_TIMEOUT_NOSEQ;
3916 spin_lock_irq(&ctx->completion_lock);
3917 entry = ctx->timeout_list.prev;
3918 goto add;
3919 }
Jens Axboe5262f562019-09-17 12:26:57 -06003920
3921 req->sequence = ctx->cached_sq_head + count - 1;
Jens Axboe2d283902019-12-04 11:08:05 -07003922 data->seq_offset = count;
Jens Axboe5262f562019-09-17 12:26:57 -06003923
3924 /*
3925 * Insertion sort, ensuring the first entry in the list is always
3926 * the one we need first.
3927 */
Jens Axboe5262f562019-09-17 12:26:57 -06003928 spin_lock_irq(&ctx->completion_lock);
3929 list_for_each_prev(entry, &ctx->timeout_list) {
3930 struct io_kiocb *nxt = list_entry(entry, struct io_kiocb, list);
yangerkun5da0fb12019-10-15 21:59:29 +08003931 unsigned nxt_sq_head;
3932 long long tmp, tmp_nxt;
Jens Axboe2d283902019-12-04 11:08:05 -07003933 u32 nxt_offset = nxt->io->timeout.seq_offset;
Jens Axboe5262f562019-09-17 12:26:57 -06003934
Jens Axboe93bd25b2019-11-11 23:34:31 -07003935 if (nxt->flags & REQ_F_TIMEOUT_NOSEQ)
3936 continue;
3937
yangerkun5da0fb12019-10-15 21:59:29 +08003938 /*
3939 * Since cached_sq_head + count - 1 can overflow, use type long
3940 * long to store it.
3941 */
3942 tmp = (long long)ctx->cached_sq_head + count - 1;
Pavel Begunkovcc42e0a2019-11-25 23:14:38 +03003943 nxt_sq_head = nxt->sequence - nxt_offset + 1;
3944 tmp_nxt = (long long)nxt_sq_head + nxt_offset - 1;
yangerkun5da0fb12019-10-15 21:59:29 +08003945
3946 /*
3947 * cached_sq_head may overflow, and it will never overflow twice
3948 * once there is some timeout req still be valid.
3949 */
3950 if (ctx->cached_sq_head < nxt_sq_head)
yangerkun8b07a652019-10-17 12:12:35 +08003951 tmp += UINT_MAX;
yangerkun5da0fb12019-10-15 21:59:29 +08003952
zhangyi (F)a1f58ba2019-10-23 15:10:09 +08003953 if (tmp > tmp_nxt)
Jens Axboe5262f562019-09-17 12:26:57 -06003954 break;
zhangyi (F)a1f58ba2019-10-23 15:10:09 +08003955
3956 /*
3957 * Sequence of reqs after the insert one and itself should
3958 * be adjusted because each timeout req consumes a slot.
3959 */
3960 span++;
3961 nxt->sequence++;
Jens Axboe5262f562019-09-17 12:26:57 -06003962 }
zhangyi (F)a1f58ba2019-10-23 15:10:09 +08003963 req->sequence -= span;
Jens Axboe93bd25b2019-11-11 23:34:31 -07003964add:
Jens Axboe5262f562019-09-17 12:26:57 -06003965 list_add(&req->list, entry);
Jens Axboead8a48a2019-11-15 08:49:11 -07003966 data->timer.function = io_timeout_fn;
3967 hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), data->mode);
Jens Axboe842f9612019-10-29 12:34:10 -06003968 spin_unlock_irq(&ctx->completion_lock);
Jens Axboe5262f562019-09-17 12:26:57 -06003969 return 0;
3970}
3971
Jens Axboe62755e32019-10-28 21:49:21 -06003972static bool io_cancel_cb(struct io_wq_work *work, void *data)
Jens Axboede0617e2019-04-06 21:51:27 -06003973{
Jens Axboe62755e32019-10-28 21:49:21 -06003974 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
Jens Axboede0617e2019-04-06 21:51:27 -06003975
Jens Axboe62755e32019-10-28 21:49:21 -06003976 return req->user_data == (unsigned long) data;
3977}
3978
Jens Axboee977d6d2019-11-05 12:39:45 -07003979static int io_async_cancel_one(struct io_ring_ctx *ctx, void *sqe_addr)
Jens Axboe62755e32019-10-28 21:49:21 -06003980{
Jens Axboe62755e32019-10-28 21:49:21 -06003981 enum io_wq_cancel cancel_ret;
Jens Axboe62755e32019-10-28 21:49:21 -06003982 int ret = 0;
3983
Jens Axboe62755e32019-10-28 21:49:21 -06003984 cancel_ret = io_wq_cancel_cb(ctx->io_wq, io_cancel_cb, sqe_addr);
3985 switch (cancel_ret) {
3986 case IO_WQ_CANCEL_OK:
3987 ret = 0;
3988 break;
3989 case IO_WQ_CANCEL_RUNNING:
3990 ret = -EALREADY;
3991 break;
3992 case IO_WQ_CANCEL_NOTFOUND:
3993 ret = -ENOENT;
3994 break;
3995 }
3996
Jens Axboee977d6d2019-11-05 12:39:45 -07003997 return ret;
3998}
3999
Jens Axboe47f46762019-11-09 17:43:02 -07004000static void io_async_find_and_cancel(struct io_ring_ctx *ctx,
4001 struct io_kiocb *req, __u64 sqe_addr,
Jens Axboeb0dd8a42019-11-18 12:14:54 -07004002 struct io_kiocb **nxt, int success_ret)
Jens Axboe47f46762019-11-09 17:43:02 -07004003{
4004 unsigned long flags;
4005 int ret;
4006
4007 ret = io_async_cancel_one(ctx, (void *) (unsigned long) sqe_addr);
4008 if (ret != -ENOENT) {
4009 spin_lock_irqsave(&ctx->completion_lock, flags);
4010 goto done;
4011 }
4012
4013 spin_lock_irqsave(&ctx->completion_lock, flags);
4014 ret = io_timeout_cancel(ctx, sqe_addr);
4015 if (ret != -ENOENT)
4016 goto done;
4017 ret = io_poll_cancel(ctx, sqe_addr);
4018done:
Jens Axboeb0dd8a42019-11-18 12:14:54 -07004019 if (!ret)
4020 ret = success_ret;
Jens Axboe47f46762019-11-09 17:43:02 -07004021 io_cqring_fill_event(req, ret);
4022 io_commit_cqring(ctx);
4023 spin_unlock_irqrestore(&ctx->completion_lock, flags);
4024 io_cqring_ev_posted(ctx);
4025
Jens Axboe4e88d6e2019-12-07 20:59:47 -07004026 if (ret < 0)
4027 req_set_fail_links(req);
Jens Axboe47f46762019-11-09 17:43:02 -07004028 io_put_req_find_next(req, nxt);
4029}
4030
Jens Axboe3529d8c2019-12-19 18:24:38 -07004031static int io_async_cancel_prep(struct io_kiocb *req,
4032 const struct io_uring_sqe *sqe)
Jens Axboee977d6d2019-11-05 12:39:45 -07004033{
Jens Axboefbf23842019-12-17 18:45:56 -07004034 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
Jens Axboee977d6d2019-11-05 12:39:45 -07004035 return -EINVAL;
4036 if (sqe->flags || sqe->ioprio || sqe->off || sqe->len ||
4037 sqe->cancel_flags)
4038 return -EINVAL;
4039
Jens Axboefbf23842019-12-17 18:45:56 -07004040 req->cancel.addr = READ_ONCE(sqe->addr);
4041 return 0;
4042}
4043
4044static int io_async_cancel(struct io_kiocb *req, struct io_kiocb **nxt)
4045{
4046 struct io_ring_ctx *ctx = req->ctx;
Jens Axboefbf23842019-12-17 18:45:56 -07004047
4048 io_async_find_and_cancel(ctx, req, req->cancel.addr, nxt, 0);
Jens Axboe62755e32019-10-28 21:49:21 -06004049 return 0;
4050}
4051
Jens Axboe05f3fb32019-12-09 11:22:50 -07004052static int io_files_update_prep(struct io_kiocb *req,
4053 const struct io_uring_sqe *sqe)
4054{
4055 if (sqe->flags || sqe->ioprio || sqe->rw_flags)
4056 return -EINVAL;
4057
4058 req->files_update.offset = READ_ONCE(sqe->off);
4059 req->files_update.nr_args = READ_ONCE(sqe->len);
4060 if (!req->files_update.nr_args)
4061 return -EINVAL;
4062 req->files_update.arg = READ_ONCE(sqe->addr);
4063 return 0;
4064}
4065
4066static int io_files_update(struct io_kiocb *req, bool force_nonblock)
4067{
4068 struct io_ring_ctx *ctx = req->ctx;
4069 struct io_uring_files_update up;
4070 int ret;
4071
Jens Axboef86cd202020-01-29 13:46:44 -07004072 if (force_nonblock)
Jens Axboe05f3fb32019-12-09 11:22:50 -07004073 return -EAGAIN;
Jens Axboe05f3fb32019-12-09 11:22:50 -07004074
4075 up.offset = req->files_update.offset;
4076 up.fds = req->files_update.arg;
4077
4078 mutex_lock(&ctx->uring_lock);
4079 ret = __io_sqe_files_update(ctx, &up, req->files_update.nr_args);
4080 mutex_unlock(&ctx->uring_lock);
4081
4082 if (ret < 0)
4083 req_set_fail_links(req);
4084 io_cqring_add_event(req, ret);
4085 io_put_req(req);
4086 return 0;
4087}
4088
Jens Axboe3529d8c2019-12-19 18:24:38 -07004089static int io_req_defer_prep(struct io_kiocb *req,
4090 const struct io_uring_sqe *sqe)
Jens Axboef67676d2019-12-02 11:03:47 -07004091{
Jens Axboee7815732019-12-17 19:45:06 -07004092 ssize_t ret = 0;
Jens Axboef67676d2019-12-02 11:03:47 -07004093
Jens Axboef86cd202020-01-29 13:46:44 -07004094 if (io_op_defs[req->opcode].file_table) {
4095 ret = io_grab_files(req);
4096 if (unlikely(ret))
4097 return ret;
4098 }
4099
Jens Axboecccf0ee2020-01-27 16:34:48 -07004100 io_req_work_grab_env(req, &io_op_defs[req->opcode]);
4101
Jens Axboed625c6e2019-12-17 19:53:05 -07004102 switch (req->opcode) {
Jens Axboee7815732019-12-17 19:45:06 -07004103 case IORING_OP_NOP:
4104 break;
Jens Axboef67676d2019-12-02 11:03:47 -07004105 case IORING_OP_READV:
4106 case IORING_OP_READ_FIXED:
Jens Axboe3a6820f2019-12-22 15:19:35 -07004107 case IORING_OP_READ:
Jens Axboe3529d8c2019-12-19 18:24:38 -07004108 ret = io_read_prep(req, sqe, true);
Jens Axboef67676d2019-12-02 11:03:47 -07004109 break;
4110 case IORING_OP_WRITEV:
4111 case IORING_OP_WRITE_FIXED:
Jens Axboe3a6820f2019-12-22 15:19:35 -07004112 case IORING_OP_WRITE:
Jens Axboe3529d8c2019-12-19 18:24:38 -07004113 ret = io_write_prep(req, sqe, true);
Jens Axboef67676d2019-12-02 11:03:47 -07004114 break;
Jens Axboe0969e782019-12-17 18:40:57 -07004115 case IORING_OP_POLL_ADD:
Jens Axboe3529d8c2019-12-19 18:24:38 -07004116 ret = io_poll_add_prep(req, sqe);
Jens Axboe0969e782019-12-17 18:40:57 -07004117 break;
4118 case IORING_OP_POLL_REMOVE:
Jens Axboe3529d8c2019-12-19 18:24:38 -07004119 ret = io_poll_remove_prep(req, sqe);
Jens Axboe0969e782019-12-17 18:40:57 -07004120 break;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004121 case IORING_OP_FSYNC:
Jens Axboe3529d8c2019-12-19 18:24:38 -07004122 ret = io_prep_fsync(req, sqe);
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004123 break;
4124 case IORING_OP_SYNC_FILE_RANGE:
Jens Axboe3529d8c2019-12-19 18:24:38 -07004125 ret = io_prep_sfr(req, sqe);
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004126 break;
Jens Axboe03b12302019-12-02 18:50:25 -07004127 case IORING_OP_SENDMSG:
Jens Axboefddafac2020-01-04 20:19:44 -07004128 case IORING_OP_SEND:
Jens Axboe3529d8c2019-12-19 18:24:38 -07004129 ret = io_sendmsg_prep(req, sqe);
Jens Axboe03b12302019-12-02 18:50:25 -07004130 break;
4131 case IORING_OP_RECVMSG:
Jens Axboefddafac2020-01-04 20:19:44 -07004132 case IORING_OP_RECV:
Jens Axboe3529d8c2019-12-19 18:24:38 -07004133 ret = io_recvmsg_prep(req, sqe);
Jens Axboe03b12302019-12-02 18:50:25 -07004134 break;
Jens Axboef499a022019-12-02 16:28:46 -07004135 case IORING_OP_CONNECT:
Jens Axboe3529d8c2019-12-19 18:24:38 -07004136 ret = io_connect_prep(req, sqe);
Jens Axboef499a022019-12-02 16:28:46 -07004137 break;
Jens Axboe2d283902019-12-04 11:08:05 -07004138 case IORING_OP_TIMEOUT:
Jens Axboe3529d8c2019-12-19 18:24:38 -07004139 ret = io_timeout_prep(req, sqe, false);
Jens Axboeb7bb4f72019-12-15 22:13:43 -07004140 break;
Jens Axboeb29472e2019-12-17 18:50:29 -07004141 case IORING_OP_TIMEOUT_REMOVE:
Jens Axboe3529d8c2019-12-19 18:24:38 -07004142 ret = io_timeout_remove_prep(req, sqe);
Jens Axboeb29472e2019-12-17 18:50:29 -07004143 break;
Jens Axboefbf23842019-12-17 18:45:56 -07004144 case IORING_OP_ASYNC_CANCEL:
Jens Axboe3529d8c2019-12-19 18:24:38 -07004145 ret = io_async_cancel_prep(req, sqe);
Jens Axboefbf23842019-12-17 18:45:56 -07004146 break;
Jens Axboe2d283902019-12-04 11:08:05 -07004147 case IORING_OP_LINK_TIMEOUT:
Jens Axboe3529d8c2019-12-19 18:24:38 -07004148 ret = io_timeout_prep(req, sqe, true);
Jens Axboeb7bb4f72019-12-15 22:13:43 -07004149 break;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004150 case IORING_OP_ACCEPT:
Jens Axboe3529d8c2019-12-19 18:24:38 -07004151 ret = io_accept_prep(req, sqe);
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004152 break;
Jens Axboed63d1b52019-12-10 10:38:56 -07004153 case IORING_OP_FALLOCATE:
4154 ret = io_fallocate_prep(req, sqe);
4155 break;
Jens Axboe15b71ab2019-12-11 11:20:36 -07004156 case IORING_OP_OPENAT:
4157 ret = io_openat_prep(req, sqe);
4158 break;
Jens Axboeb5dba592019-12-11 14:02:38 -07004159 case IORING_OP_CLOSE:
4160 ret = io_close_prep(req, sqe);
4161 break;
Jens Axboe05f3fb32019-12-09 11:22:50 -07004162 case IORING_OP_FILES_UPDATE:
4163 ret = io_files_update_prep(req, sqe);
4164 break;
Jens Axboeeddc7ef2019-12-13 21:18:10 -07004165 case IORING_OP_STATX:
4166 ret = io_statx_prep(req, sqe);
4167 break;
Jens Axboe4840e412019-12-25 22:03:45 -07004168 case IORING_OP_FADVISE:
4169 ret = io_fadvise_prep(req, sqe);
4170 break;
Jens Axboec1ca7572019-12-25 22:18:28 -07004171 case IORING_OP_MADVISE:
4172 ret = io_madvise_prep(req, sqe);
4173 break;
Jens Axboecebdb982020-01-08 17:59:24 -07004174 case IORING_OP_OPENAT2:
4175 ret = io_openat2_prep(req, sqe);
4176 break;
Jens Axboe3e4827b2020-01-08 15:18:09 -07004177 case IORING_OP_EPOLL_CTL:
4178 ret = io_epoll_ctl_prep(req, sqe);
4179 break;
Jens Axboef67676d2019-12-02 11:03:47 -07004180 default:
Jens Axboee7815732019-12-17 19:45:06 -07004181 printk_once(KERN_WARNING "io_uring: unhandled opcode %d\n",
4182 req->opcode);
4183 ret = -EINVAL;
Jens Axboeb7bb4f72019-12-15 22:13:43 -07004184 break;
Jens Axboef67676d2019-12-02 11:03:47 -07004185 }
4186
Jens Axboeb7bb4f72019-12-15 22:13:43 -07004187 return ret;
Jens Axboef67676d2019-12-02 11:03:47 -07004188}
4189
Jens Axboe3529d8c2019-12-19 18:24:38 -07004190static int io_req_defer(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboede0617e2019-04-06 21:51:27 -06004191{
Jackie Liua197f662019-11-08 08:09:12 -07004192 struct io_ring_ctx *ctx = req->ctx;
Jens Axboef67676d2019-12-02 11:03:47 -07004193 int ret;
Jens Axboede0617e2019-04-06 21:51:27 -06004194
Bob Liu9d858b22019-11-13 18:06:25 +08004195 /* Still need defer if there is pending req in defer list. */
4196 if (!req_need_defer(req) && list_empty(&ctx->defer_list))
Jens Axboede0617e2019-04-06 21:51:27 -06004197 return 0;
4198
Jens Axboe3529d8c2019-12-19 18:24:38 -07004199 if (!req->io && io_alloc_async_ctx(req))
Jens Axboede0617e2019-04-06 21:51:27 -06004200 return -EAGAIN;
4201
Jens Axboe3529d8c2019-12-19 18:24:38 -07004202 ret = io_req_defer_prep(req, sqe);
Jens Axboeb7bb4f72019-12-15 22:13:43 -07004203 if (ret < 0)
Jens Axboe2d283902019-12-04 11:08:05 -07004204 return ret;
Jens Axboe2d283902019-12-04 11:08:05 -07004205
Jens Axboede0617e2019-04-06 21:51:27 -06004206 spin_lock_irq(&ctx->completion_lock);
Bob Liu9d858b22019-11-13 18:06:25 +08004207 if (!req_need_defer(req) && list_empty(&ctx->defer_list)) {
Jens Axboede0617e2019-04-06 21:51:27 -06004208 spin_unlock_irq(&ctx->completion_lock);
Jens Axboede0617e2019-04-06 21:51:27 -06004209 return 0;
4210 }
4211
Jens Axboe915967f2019-11-21 09:01:20 -07004212 trace_io_uring_defer(ctx, req, req->user_data);
Jens Axboede0617e2019-04-06 21:51:27 -06004213 list_add_tail(&req->list, &ctx->defer_list);
4214 spin_unlock_irq(&ctx->completion_lock);
4215 return -EIOCBQUEUED;
4216}
4217
Pavel Begunkov99bc4c32020-02-07 22:04:45 +03004218static void io_cleanup_req(struct io_kiocb *req)
4219{
4220 struct io_async_ctx *io = req->io;
4221
4222 switch (req->opcode) {
4223 case IORING_OP_READV:
4224 case IORING_OP_READ_FIXED:
4225 case IORING_OP_READ:
4226 case IORING_OP_WRITEV:
4227 case IORING_OP_WRITE_FIXED:
4228 case IORING_OP_WRITE:
4229 if (io->rw.iov != io->rw.fast_iov)
4230 kfree(io->rw.iov);
4231 break;
4232 case IORING_OP_SENDMSG:
4233 case IORING_OP_RECVMSG:
4234 if (io->msg.iov != io->msg.fast_iov)
4235 kfree(io->msg.iov);
4236 break;
Pavel Begunkov8fef80b2020-02-07 23:59:53 +03004237 case IORING_OP_OPENAT:
4238 case IORING_OP_OPENAT2:
4239 case IORING_OP_STATX:
4240 putname(req->open.filename);
4241 break;
Pavel Begunkov99bc4c32020-02-07 22:04:45 +03004242 }
4243
4244 req->flags &= ~REQ_F_NEED_CLEANUP;
4245}
4246
Jens Axboe3529d8c2019-12-19 18:24:38 -07004247static int io_issue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
4248 struct io_kiocb **nxt, bool force_nonblock)
Jens Axboe2b188cc2019-01-07 10:46:33 -07004249{
Jackie Liua197f662019-11-08 08:09:12 -07004250 struct io_ring_ctx *ctx = req->ctx;
Jens Axboed625c6e2019-12-17 19:53:05 -07004251 int ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07004252
Jens Axboed625c6e2019-12-17 19:53:05 -07004253 switch (req->opcode) {
Jens Axboe2b188cc2019-01-07 10:46:33 -07004254 case IORING_OP_NOP:
Jens Axboe78e19bb2019-11-06 15:21:34 -07004255 ret = io_nop(req);
Jens Axboe2b188cc2019-01-07 10:46:33 -07004256 break;
4257 case IORING_OP_READV:
Jens Axboe3529d8c2019-12-19 18:24:38 -07004258 case IORING_OP_READ_FIXED:
Jens Axboe3a6820f2019-12-22 15:19:35 -07004259 case IORING_OP_READ:
Jens Axboe3529d8c2019-12-19 18:24:38 -07004260 if (sqe) {
4261 ret = io_read_prep(req, sqe, force_nonblock);
4262 if (ret < 0)
4263 break;
4264 }
Pavel Begunkov267bc902019-11-07 01:41:08 +03004265 ret = io_read(req, nxt, force_nonblock);
Jens Axboe2b188cc2019-01-07 10:46:33 -07004266 break;
4267 case IORING_OP_WRITEV:
Jens Axboeedafcce2019-01-09 09:16:05 -07004268 case IORING_OP_WRITE_FIXED:
Jens Axboe3a6820f2019-12-22 15:19:35 -07004269 case IORING_OP_WRITE:
Jens Axboe3529d8c2019-12-19 18:24:38 -07004270 if (sqe) {
4271 ret = io_write_prep(req, sqe, force_nonblock);
4272 if (ret < 0)
4273 break;
4274 }
Pavel Begunkov267bc902019-11-07 01:41:08 +03004275 ret = io_write(req, nxt, force_nonblock);
Jens Axboe2b188cc2019-01-07 10:46:33 -07004276 break;
Christoph Hellwigc992fe22019-01-11 09:43:02 -07004277 case IORING_OP_FSYNC:
Jens Axboe3529d8c2019-12-19 18:24:38 -07004278 if (sqe) {
4279 ret = io_prep_fsync(req, sqe);
4280 if (ret < 0)
4281 break;
4282 }
Jens Axboefc4df992019-12-10 14:38:45 -07004283 ret = io_fsync(req, nxt, force_nonblock);
Christoph Hellwigc992fe22019-01-11 09:43:02 -07004284 break;
Jens Axboe221c5eb2019-01-17 09:41:58 -07004285 case IORING_OP_POLL_ADD:
Jens Axboe3529d8c2019-12-19 18:24:38 -07004286 if (sqe) {
4287 ret = io_poll_add_prep(req, sqe);
4288 if (ret)
4289 break;
4290 }
Jens Axboefc4df992019-12-10 14:38:45 -07004291 ret = io_poll_add(req, nxt);
Jens Axboe221c5eb2019-01-17 09:41:58 -07004292 break;
4293 case IORING_OP_POLL_REMOVE:
Jens Axboe3529d8c2019-12-19 18:24:38 -07004294 if (sqe) {
4295 ret = io_poll_remove_prep(req, sqe);
4296 if (ret < 0)
4297 break;
4298 }
Jens Axboefc4df992019-12-10 14:38:45 -07004299 ret = io_poll_remove(req);
Jens Axboe221c5eb2019-01-17 09:41:58 -07004300 break;
Jens Axboe5d17b4a2019-04-09 14:56:44 -06004301 case IORING_OP_SYNC_FILE_RANGE:
Jens Axboe3529d8c2019-12-19 18:24:38 -07004302 if (sqe) {
4303 ret = io_prep_sfr(req, sqe);
4304 if (ret < 0)
4305 break;
4306 }
Jens Axboefc4df992019-12-10 14:38:45 -07004307 ret = io_sync_file_range(req, nxt, force_nonblock);
Jens Axboe5d17b4a2019-04-09 14:56:44 -06004308 break;
Jens Axboe0fa03c62019-04-19 13:34:07 -06004309 case IORING_OP_SENDMSG:
Jens Axboefddafac2020-01-04 20:19:44 -07004310 case IORING_OP_SEND:
Jens Axboe3529d8c2019-12-19 18:24:38 -07004311 if (sqe) {
4312 ret = io_sendmsg_prep(req, sqe);
4313 if (ret < 0)
4314 break;
4315 }
Jens Axboefddafac2020-01-04 20:19:44 -07004316 if (req->opcode == IORING_OP_SENDMSG)
4317 ret = io_sendmsg(req, nxt, force_nonblock);
4318 else
4319 ret = io_send(req, nxt, force_nonblock);
Jens Axboe0fa03c62019-04-19 13:34:07 -06004320 break;
Jens Axboeaa1fa282019-04-19 13:38:09 -06004321 case IORING_OP_RECVMSG:
Jens Axboefddafac2020-01-04 20:19:44 -07004322 case IORING_OP_RECV:
Jens Axboe3529d8c2019-12-19 18:24:38 -07004323 if (sqe) {
4324 ret = io_recvmsg_prep(req, sqe);
4325 if (ret)
4326 break;
4327 }
Jens Axboefddafac2020-01-04 20:19:44 -07004328 if (req->opcode == IORING_OP_RECVMSG)
4329 ret = io_recvmsg(req, nxt, force_nonblock);
4330 else
4331 ret = io_recv(req, nxt, force_nonblock);
Jens Axboeaa1fa282019-04-19 13:38:09 -06004332 break;
Jens Axboe5262f562019-09-17 12:26:57 -06004333 case IORING_OP_TIMEOUT:
Jens Axboe3529d8c2019-12-19 18:24:38 -07004334 if (sqe) {
4335 ret = io_timeout_prep(req, sqe, false);
4336 if (ret)
4337 break;
4338 }
Jens Axboefc4df992019-12-10 14:38:45 -07004339 ret = io_timeout(req);
Jens Axboe5262f562019-09-17 12:26:57 -06004340 break;
Jens Axboe11365042019-10-16 09:08:32 -06004341 case IORING_OP_TIMEOUT_REMOVE:
Jens Axboe3529d8c2019-12-19 18:24:38 -07004342 if (sqe) {
4343 ret = io_timeout_remove_prep(req, sqe);
4344 if (ret)
4345 break;
4346 }
Jens Axboefc4df992019-12-10 14:38:45 -07004347 ret = io_timeout_remove(req);
Jens Axboe11365042019-10-16 09:08:32 -06004348 break;
Jens Axboe17f2fe32019-10-17 14:42:58 -06004349 case IORING_OP_ACCEPT:
Jens Axboe3529d8c2019-12-19 18:24:38 -07004350 if (sqe) {
4351 ret = io_accept_prep(req, sqe);
4352 if (ret)
4353 break;
4354 }
Jens Axboefc4df992019-12-10 14:38:45 -07004355 ret = io_accept(req, nxt, force_nonblock);
Jens Axboe17f2fe32019-10-17 14:42:58 -06004356 break;
Jens Axboef8e85cf2019-11-23 14:24:24 -07004357 case IORING_OP_CONNECT:
Jens Axboe3529d8c2019-12-19 18:24:38 -07004358 if (sqe) {
4359 ret = io_connect_prep(req, sqe);
4360 if (ret)
4361 break;
4362 }
Jens Axboefc4df992019-12-10 14:38:45 -07004363 ret = io_connect(req, nxt, force_nonblock);
Jens Axboef8e85cf2019-11-23 14:24:24 -07004364 break;
Jens Axboe62755e32019-10-28 21:49:21 -06004365 case IORING_OP_ASYNC_CANCEL:
Jens Axboe3529d8c2019-12-19 18:24:38 -07004366 if (sqe) {
4367 ret = io_async_cancel_prep(req, sqe);
4368 if (ret)
4369 break;
4370 }
Jens Axboefc4df992019-12-10 14:38:45 -07004371 ret = io_async_cancel(req, nxt);
Jens Axboe62755e32019-10-28 21:49:21 -06004372 break;
Jens Axboed63d1b52019-12-10 10:38:56 -07004373 case IORING_OP_FALLOCATE:
4374 if (sqe) {
4375 ret = io_fallocate_prep(req, sqe);
4376 if (ret)
4377 break;
4378 }
4379 ret = io_fallocate(req, nxt, force_nonblock);
4380 break;
Jens Axboe15b71ab2019-12-11 11:20:36 -07004381 case IORING_OP_OPENAT:
4382 if (sqe) {
4383 ret = io_openat_prep(req, sqe);
4384 if (ret)
4385 break;
4386 }
4387 ret = io_openat(req, nxt, force_nonblock);
4388 break;
Jens Axboeb5dba592019-12-11 14:02:38 -07004389 case IORING_OP_CLOSE:
4390 if (sqe) {
4391 ret = io_close_prep(req, sqe);
4392 if (ret)
4393 break;
4394 }
4395 ret = io_close(req, nxt, force_nonblock);
4396 break;
Jens Axboe05f3fb32019-12-09 11:22:50 -07004397 case IORING_OP_FILES_UPDATE:
4398 if (sqe) {
4399 ret = io_files_update_prep(req, sqe);
4400 if (ret)
4401 break;
4402 }
4403 ret = io_files_update(req, force_nonblock);
4404 break;
Jens Axboeeddc7ef2019-12-13 21:18:10 -07004405 case IORING_OP_STATX:
4406 if (sqe) {
4407 ret = io_statx_prep(req, sqe);
4408 if (ret)
4409 break;
4410 }
4411 ret = io_statx(req, nxt, force_nonblock);
4412 break;
Jens Axboe4840e412019-12-25 22:03:45 -07004413 case IORING_OP_FADVISE:
4414 if (sqe) {
4415 ret = io_fadvise_prep(req, sqe);
4416 if (ret)
4417 break;
4418 }
4419 ret = io_fadvise(req, nxt, force_nonblock);
4420 break;
Jens Axboec1ca7572019-12-25 22:18:28 -07004421 case IORING_OP_MADVISE:
4422 if (sqe) {
4423 ret = io_madvise_prep(req, sqe);
4424 if (ret)
4425 break;
4426 }
4427 ret = io_madvise(req, nxt, force_nonblock);
4428 break;
Jens Axboecebdb982020-01-08 17:59:24 -07004429 case IORING_OP_OPENAT2:
4430 if (sqe) {
4431 ret = io_openat2_prep(req, sqe);
4432 if (ret)
4433 break;
4434 }
4435 ret = io_openat2(req, nxt, force_nonblock);
4436 break;
Jens Axboe3e4827b2020-01-08 15:18:09 -07004437 case IORING_OP_EPOLL_CTL:
4438 if (sqe) {
4439 ret = io_epoll_ctl_prep(req, sqe);
4440 if (ret)
4441 break;
4442 }
4443 ret = io_epoll_ctl(req, nxt, force_nonblock);
4444 break;
Jens Axboe2b188cc2019-01-07 10:46:33 -07004445 default:
4446 ret = -EINVAL;
4447 break;
4448 }
4449
Jens Axboedef596e2019-01-09 08:59:42 -07004450 if (ret)
4451 return ret;
4452
4453 if (ctx->flags & IORING_SETUP_IOPOLL) {
Jens Axboe11ba8202020-01-15 21:51:17 -07004454 const bool in_async = io_wq_current_is_worker();
4455
Jens Axboe9e645e112019-05-10 16:07:28 -06004456 if (req->result == -EAGAIN)
Jens Axboedef596e2019-01-09 08:59:42 -07004457 return -EAGAIN;
4458
Jens Axboe11ba8202020-01-15 21:51:17 -07004459 /* workqueue context doesn't hold uring_lock, grab it now */
4460 if (in_async)
4461 mutex_lock(&ctx->uring_lock);
4462
Jens Axboedef596e2019-01-09 08:59:42 -07004463 io_iopoll_req_issued(req);
Jens Axboe11ba8202020-01-15 21:51:17 -07004464
4465 if (in_async)
4466 mutex_unlock(&ctx->uring_lock);
Jens Axboedef596e2019-01-09 08:59:42 -07004467 }
4468
4469 return 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07004470}
4471
Jens Axboe561fb042019-10-24 07:25:42 -06004472static void io_wq_submit_work(struct io_wq_work **workptr)
Jens Axboe31b51512019-01-18 22:56:34 -07004473{
Jens Axboe561fb042019-10-24 07:25:42 -06004474 struct io_wq_work *work = *workptr;
Jens Axboe2b188cc2019-01-07 10:46:33 -07004475 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
Jens Axboe561fb042019-10-24 07:25:42 -06004476 struct io_kiocb *nxt = NULL;
4477 int ret = 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07004478
Jens Axboe0c9d5cc2019-12-11 19:29:43 -07004479 /* if NO_CANCEL is set, we must still run the work */
4480 if ((work->flags & (IO_WQ_WORK_CANCEL|IO_WQ_WORK_NO_CANCEL)) ==
4481 IO_WQ_WORK_CANCEL) {
Jens Axboe561fb042019-10-24 07:25:42 -06004482 ret = -ECANCELED;
Jens Axboe0c9d5cc2019-12-11 19:29:43 -07004483 }
Jens Axboe31b51512019-01-18 22:56:34 -07004484
Jens Axboe561fb042019-10-24 07:25:42 -06004485 if (!ret) {
Pavel Begunkovcf6fd4b2019-11-25 23:14:39 +03004486 req->in_async = true;
Jens Axboe561fb042019-10-24 07:25:42 -06004487 do {
Jens Axboe3529d8c2019-12-19 18:24:38 -07004488 ret = io_issue_sqe(req, NULL, &nxt, false);
Jens Axboe561fb042019-10-24 07:25:42 -06004489 /*
4490 * We can get EAGAIN for polled IO even though we're
4491 * forcing a sync submission from here, since we can't
4492 * wait for request slots on the block side.
4493 */
4494 if (ret != -EAGAIN)
4495 break;
4496 cond_resched();
4497 } while (1);
4498 }
Jens Axboe31b51512019-01-18 22:56:34 -07004499
Jens Axboe561fb042019-10-24 07:25:42 -06004500 /* drop submission reference */
Jackie Liuec9c02a2019-11-08 23:50:36 +08004501 io_put_req(req);
Jens Axboe817869d2019-04-30 14:44:05 -06004502
Jens Axboe561fb042019-10-24 07:25:42 -06004503 if (ret) {
Jens Axboe4e88d6e2019-12-07 20:59:47 -07004504 req_set_fail_links(req);
Jens Axboe78e19bb2019-11-06 15:21:34 -07004505 io_cqring_add_event(req, ret);
Jens Axboe817869d2019-04-30 14:44:05 -06004506 io_put_req(req);
Jens Axboeedafcce2019-01-09 09:16:05 -07004507 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07004508
Jens Axboe561fb042019-10-24 07:25:42 -06004509 /* if a dependent link is ready, pass it back */
Jens Axboe78912932020-01-14 22:09:06 -07004510 if (!ret && nxt)
4511 io_wq_assign_next(workptr, nxt);
Jens Axboe31b51512019-01-18 22:56:34 -07004512}
Jens Axboe2b188cc2019-01-07 10:46:33 -07004513
Jens Axboe15b71ab2019-12-11 11:20:36 -07004514static int io_req_needs_file(struct io_kiocb *req, int fd)
Jens Axboe9e3aa612019-12-11 15:55:43 -07004515{
Jens Axboed3656342019-12-18 09:50:26 -07004516 if (!io_op_defs[req->opcode].needs_file)
Jens Axboe9e3aa612019-12-11 15:55:43 -07004517 return 0;
Jens Axboed3656342019-12-18 09:50:26 -07004518 if (fd == -1 && io_op_defs[req->opcode].fd_non_neg)
4519 return 0;
4520 return 1;
Jens Axboe09bb8392019-03-13 12:39:28 -06004521}
4522
Jens Axboe65e19f52019-10-26 07:20:21 -06004523static inline struct file *io_file_from_index(struct io_ring_ctx *ctx,
4524 int index)
Jens Axboe09bb8392019-03-13 12:39:28 -06004525{
Jens Axboe65e19f52019-10-26 07:20:21 -06004526 struct fixed_file_table *table;
4527
Jens Axboe05f3fb32019-12-09 11:22:50 -07004528 table = &ctx->file_data->table[index >> IORING_FILE_TABLE_SHIFT];
4529 return table->files[index & IORING_FILE_TABLE_MASK];;
Jens Axboe65e19f52019-10-26 07:20:21 -06004530}
4531
Jens Axboe3529d8c2019-12-19 18:24:38 -07004532static int io_req_set_file(struct io_submit_state *state, struct io_kiocb *req,
4533 const struct io_uring_sqe *sqe)
Jens Axboe09bb8392019-03-13 12:39:28 -06004534{
Jackie Liua197f662019-11-08 08:09:12 -07004535 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe09bb8392019-03-13 12:39:28 -06004536 unsigned flags;
Jens Axboed3656342019-12-18 09:50:26 -07004537 int fd;
Jens Axboe09bb8392019-03-13 12:39:28 -06004538
Jens Axboe3529d8c2019-12-19 18:24:38 -07004539 flags = READ_ONCE(sqe->flags);
4540 fd = READ_ONCE(sqe->fd);
Jens Axboe09bb8392019-03-13 12:39:28 -06004541
Jens Axboed3656342019-12-18 09:50:26 -07004542 if (!io_req_needs_file(req, fd))
4543 return 0;
Jens Axboe09bb8392019-03-13 12:39:28 -06004544
4545 if (flags & IOSQE_FIXED_FILE) {
Jens Axboe05f3fb32019-12-09 11:22:50 -07004546 if (unlikely(!ctx->file_data ||
Jens Axboe09bb8392019-03-13 12:39:28 -06004547 (unsigned) fd >= ctx->nr_user_files))
4548 return -EBADF;
Jens Axboeb7620122019-10-26 07:22:55 -06004549 fd = array_index_nospec(fd, ctx->nr_user_files);
Jens Axboe65e19f52019-10-26 07:20:21 -06004550 req->file = io_file_from_index(ctx, fd);
4551 if (!req->file)
Jens Axboe08a45172019-10-03 08:11:03 -06004552 return -EBADF;
Jens Axboe09bb8392019-03-13 12:39:28 -06004553 req->flags |= REQ_F_FIXED_FILE;
Jens Axboe05f3fb32019-12-09 11:22:50 -07004554 percpu_ref_get(&ctx->file_data->refs);
Jens Axboe09bb8392019-03-13 12:39:28 -06004555 } else {
Pavel Begunkovcf6fd4b2019-11-25 23:14:39 +03004556 if (req->needs_fixed_file)
Jens Axboe09bb8392019-03-13 12:39:28 -06004557 return -EBADF;
Dmitrii Dolgovc826bd72019-10-15 19:02:01 +02004558 trace_io_uring_file_get(ctx, fd);
Jens Axboe09bb8392019-03-13 12:39:28 -06004559 req->file = io_file_get(state, fd);
4560 if (unlikely(!req->file))
4561 return -EBADF;
4562 }
4563
4564 return 0;
4565}
4566
Jackie Liua197f662019-11-08 08:09:12 -07004567static int io_grab_files(struct io_kiocb *req)
Jens Axboe2b188cc2019-01-07 10:46:33 -07004568{
Jens Axboefcb323c2019-10-24 12:39:47 -06004569 int ret = -EBADF;
Jackie Liua197f662019-11-08 08:09:12 -07004570 struct io_ring_ctx *ctx = req->ctx;
Jens Axboefcb323c2019-10-24 12:39:47 -06004571
Jens Axboef86cd202020-01-29 13:46:44 -07004572 if (req->work.files)
4573 return 0;
Pavel Begunkovb14cca02020-01-17 04:45:59 +03004574 if (!ctx->ring_file)
Jens Axboeb5dba592019-12-11 14:02:38 -07004575 return -EBADF;
4576
Jens Axboefcb323c2019-10-24 12:39:47 -06004577 rcu_read_lock();
4578 spin_lock_irq(&ctx->inflight_lock);
4579 /*
4580 * We use the f_ops->flush() handler to ensure that we can flush
4581 * out work accessing these files if the fd is closed. Check if
4582 * the fd has changed since we started down this path, and disallow
4583 * this operation if it has.
4584 */
Pavel Begunkovb14cca02020-01-17 04:45:59 +03004585 if (fcheck(ctx->ring_fd) == ctx->ring_file) {
Jens Axboefcb323c2019-10-24 12:39:47 -06004586 list_add(&req->inflight_entry, &ctx->inflight_list);
4587 req->flags |= REQ_F_INFLIGHT;
4588 req->work.files = current->files;
4589 ret = 0;
4590 }
4591 spin_unlock_irq(&ctx->inflight_lock);
4592 rcu_read_unlock();
4593
4594 return ret;
4595}
4596
Jens Axboe2665abf2019-11-05 12:40:47 -07004597static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer)
4598{
Jens Axboead8a48a2019-11-15 08:49:11 -07004599 struct io_timeout_data *data = container_of(timer,
4600 struct io_timeout_data, timer);
4601 struct io_kiocb *req = data->req;
Jens Axboe2665abf2019-11-05 12:40:47 -07004602 struct io_ring_ctx *ctx = req->ctx;
4603 struct io_kiocb *prev = NULL;
4604 unsigned long flags;
Jens Axboe2665abf2019-11-05 12:40:47 -07004605
4606 spin_lock_irqsave(&ctx->completion_lock, flags);
4607
4608 /*
4609 * We don't expect the list to be empty, that will only happen if we
4610 * race with the completion of the linked work.
4611 */
Pavel Begunkov44932332019-12-05 16:16:35 +03004612 if (!list_empty(&req->link_list)) {
4613 prev = list_entry(req->link_list.prev, struct io_kiocb,
4614 link_list);
Jens Axboe5d960722019-11-19 15:31:28 -07004615 if (refcount_inc_not_zero(&prev->refs)) {
Pavel Begunkov44932332019-12-05 16:16:35 +03004616 list_del_init(&req->link_list);
Jens Axboe5d960722019-11-19 15:31:28 -07004617 prev->flags &= ~REQ_F_LINK_TIMEOUT;
4618 } else
Jens Axboe76a46e02019-11-10 23:34:16 -07004619 prev = NULL;
Jens Axboe2665abf2019-11-05 12:40:47 -07004620 }
4621
4622 spin_unlock_irqrestore(&ctx->completion_lock, flags);
4623
4624 if (prev) {
Jens Axboe4e88d6e2019-12-07 20:59:47 -07004625 req_set_fail_links(prev);
Jens Axboeb0dd8a42019-11-18 12:14:54 -07004626 io_async_find_and_cancel(ctx, req, prev->user_data, NULL,
4627 -ETIME);
Jens Axboe76a46e02019-11-10 23:34:16 -07004628 io_put_req(prev);
Jens Axboe47f46762019-11-09 17:43:02 -07004629 } else {
4630 io_cqring_add_event(req, -ETIME);
4631 io_put_req(req);
Jens Axboe2665abf2019-11-05 12:40:47 -07004632 }
Jens Axboe2665abf2019-11-05 12:40:47 -07004633 return HRTIMER_NORESTART;
4634}
4635
Jens Axboead8a48a2019-11-15 08:49:11 -07004636static void io_queue_linked_timeout(struct io_kiocb *req)
Jens Axboe2665abf2019-11-05 12:40:47 -07004637{
Jens Axboe76a46e02019-11-10 23:34:16 -07004638 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe2665abf2019-11-05 12:40:47 -07004639
Jens Axboe76a46e02019-11-10 23:34:16 -07004640 /*
4641 * If the list is now empty, then our linked request finished before
4642 * we got a chance to setup the timer
4643 */
4644 spin_lock_irq(&ctx->completion_lock);
Pavel Begunkov44932332019-12-05 16:16:35 +03004645 if (!list_empty(&req->link_list)) {
Jens Axboe2d283902019-12-04 11:08:05 -07004646 struct io_timeout_data *data = &req->io->timeout;
Jens Axboe94ae5e72019-11-14 19:39:52 -07004647
Jens Axboead8a48a2019-11-15 08:49:11 -07004648 data->timer.function = io_link_timeout_fn;
4649 hrtimer_start(&data->timer, timespec64_to_ktime(data->ts),
4650 data->mode);
Jens Axboe2665abf2019-11-05 12:40:47 -07004651 }
Jens Axboe76a46e02019-11-10 23:34:16 -07004652 spin_unlock_irq(&ctx->completion_lock);
Jens Axboe2665abf2019-11-05 12:40:47 -07004653
Jens Axboe2665abf2019-11-05 12:40:47 -07004654 /* drop submission reference */
Jens Axboe76a46e02019-11-10 23:34:16 -07004655 io_put_req(req);
Jens Axboe2665abf2019-11-05 12:40:47 -07004656}
4657
Jens Axboead8a48a2019-11-15 08:49:11 -07004658static struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req)
Jens Axboe2665abf2019-11-05 12:40:47 -07004659{
4660 struct io_kiocb *nxt;
Jens Axboe2b188cc2019-01-07 10:46:33 -07004661
Jens Axboe2665abf2019-11-05 12:40:47 -07004662 if (!(req->flags & REQ_F_LINK))
4663 return NULL;
4664
Pavel Begunkov44932332019-12-05 16:16:35 +03004665 nxt = list_first_entry_or_null(&req->link_list, struct io_kiocb,
4666 link_list);
Jens Axboed625c6e2019-12-17 19:53:05 -07004667 if (!nxt || nxt->opcode != IORING_OP_LINK_TIMEOUT)
Jens Axboe76a46e02019-11-10 23:34:16 -07004668 return NULL;
Jens Axboe2665abf2019-11-05 12:40:47 -07004669
Jens Axboe76a46e02019-11-10 23:34:16 -07004670 req->flags |= REQ_F_LINK_TIMEOUT;
Jens Axboe76a46e02019-11-10 23:34:16 -07004671 return nxt;
Jens Axboe2665abf2019-11-05 12:40:47 -07004672}
4673
Jens Axboe3529d8c2019-12-19 18:24:38 -07004674static void __io_queue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboe2b188cc2019-01-07 10:46:33 -07004675{
Jens Axboe4a0a7a12019-12-09 20:01:01 -07004676 struct io_kiocb *linked_timeout;
Pavel Begunkovf9bd67f2019-11-21 23:21:03 +03004677 struct io_kiocb *nxt = NULL;
Jens Axboe2b188cc2019-01-07 10:46:33 -07004678 int ret;
4679
Jens Axboe4a0a7a12019-12-09 20:01:01 -07004680again:
4681 linked_timeout = io_prep_linked_timeout(req);
4682
Jens Axboe3529d8c2019-12-19 18:24:38 -07004683 ret = io_issue_sqe(req, sqe, &nxt, true);
Jens Axboe491381ce2019-10-17 09:20:46 -06004684
4685 /*
4686 * We async punt it if the file wasn't marked NOWAIT, or if the file
4687 * doesn't support non-blocking read/write attempts
4688 */
4689 if (ret == -EAGAIN && (!(req->flags & REQ_F_NOWAIT) ||
4690 (req->flags & REQ_F_MUST_PUNT))) {
Pavel Begunkov86a761f2020-01-22 23:09:36 +03004691punt:
Jens Axboef86cd202020-01-29 13:46:44 -07004692 if (io_op_defs[req->opcode].file_table) {
Pavel Begunkovbbad27b2019-11-19 23:32:47 +03004693 ret = io_grab_files(req);
4694 if (ret)
4695 goto err;
Jens Axboe2b188cc2019-01-07 10:46:33 -07004696 }
Pavel Begunkovbbad27b2019-11-19 23:32:47 +03004697
4698 /*
4699 * Queued up for async execution, worker will release
4700 * submit reference when the iocb is actually submitted.
4701 */
4702 io_queue_async_work(req);
Jens Axboe4a0a7a12019-12-09 20:01:01 -07004703 goto done_req;
Jens Axboe2b188cc2019-01-07 10:46:33 -07004704 }
Jens Axboee65ef562019-03-12 10:16:44 -06004705
Jens Axboefcb323c2019-10-24 12:39:47 -06004706err:
Jens Axboee65ef562019-03-12 10:16:44 -06004707 /* drop submission reference */
4708 io_put_req(req);
4709
Pavel Begunkovf9bd67f2019-11-21 23:21:03 +03004710 if (linked_timeout) {
Jens Axboe76a46e02019-11-10 23:34:16 -07004711 if (!ret)
Pavel Begunkovf9bd67f2019-11-21 23:21:03 +03004712 io_queue_linked_timeout(linked_timeout);
Jens Axboe76a46e02019-11-10 23:34:16 -07004713 else
Pavel Begunkovf9bd67f2019-11-21 23:21:03 +03004714 io_put_req(linked_timeout);
Jens Axboe76a46e02019-11-10 23:34:16 -07004715 }
4716
Jens Axboee65ef562019-03-12 10:16:44 -06004717 /* and drop final reference, if we failed */
Jens Axboe9e645e112019-05-10 16:07:28 -06004718 if (ret) {
Jens Axboe78e19bb2019-11-06 15:21:34 -07004719 io_cqring_add_event(req, ret);
Jens Axboe4e88d6e2019-12-07 20:59:47 -07004720 req_set_fail_links(req);
Jens Axboee65ef562019-03-12 10:16:44 -06004721 io_put_req(req);
Jens Axboe9e645e112019-05-10 16:07:28 -06004722 }
Jens Axboe4a0a7a12019-12-09 20:01:01 -07004723done_req:
4724 if (nxt) {
4725 req = nxt;
4726 nxt = NULL;
Pavel Begunkov86a761f2020-01-22 23:09:36 +03004727
4728 if (req->flags & REQ_F_FORCE_ASYNC)
4729 goto punt;
Jens Axboe4a0a7a12019-12-09 20:01:01 -07004730 goto again;
4731 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07004732}
4733
Jens Axboe3529d8c2019-12-19 18:24:38 -07004734static void io_queue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jackie Liu4fe2c962019-09-09 20:50:40 +08004735{
4736 int ret;
4737
Jens Axboe3529d8c2019-12-19 18:24:38 -07004738 ret = io_req_defer(req, sqe);
Jackie Liu4fe2c962019-09-09 20:50:40 +08004739 if (ret) {
4740 if (ret != -EIOCBQUEUED) {
Pavel Begunkov11185912020-01-22 23:09:35 +03004741fail_req:
Jens Axboe78e19bb2019-11-06 15:21:34 -07004742 io_cqring_add_event(req, ret);
Jens Axboe4e88d6e2019-12-07 20:59:47 -07004743 req_set_fail_links(req);
Jens Axboe78e19bb2019-11-06 15:21:34 -07004744 io_double_put_req(req);
Jackie Liu4fe2c962019-09-09 20:50:40 +08004745 }
Pavel Begunkov25508782019-12-30 21:24:47 +03004746 } else if (req->flags & REQ_F_FORCE_ASYNC) {
Pavel Begunkov11185912020-01-22 23:09:35 +03004747 ret = io_req_defer_prep(req, sqe);
4748 if (unlikely(ret < 0))
4749 goto fail_req;
Jens Axboece35a472019-12-17 08:04:44 -07004750 /*
4751 * Never try inline submit of IOSQE_ASYNC is set, go straight
4752 * to async execution.
4753 */
4754 req->work.flags |= IO_WQ_WORK_CONCURRENT;
4755 io_queue_async_work(req);
4756 } else {
Jens Axboe3529d8c2019-12-19 18:24:38 -07004757 __io_queue_sqe(req, sqe);
Jens Axboece35a472019-12-17 08:04:44 -07004758 }
Jackie Liu4fe2c962019-09-09 20:50:40 +08004759}
4760
Pavel Begunkov1b4a51b2019-11-21 11:54:28 +03004761static inline void io_queue_link_head(struct io_kiocb *req)
Jackie Liu4fe2c962019-09-09 20:50:40 +08004762{
Jens Axboe94ae5e72019-11-14 19:39:52 -07004763 if (unlikely(req->flags & REQ_F_FAIL_LINK)) {
Pavel Begunkov1b4a51b2019-11-21 11:54:28 +03004764 io_cqring_add_event(req, -ECANCELED);
4765 io_double_put_req(req);
4766 } else
Jens Axboe3529d8c2019-12-19 18:24:38 -07004767 io_queue_sqe(req, NULL);
Jackie Liu4fe2c962019-09-09 20:50:40 +08004768}
4769
Jens Axboe4e88d6e2019-12-07 20:59:47 -07004770#define SQE_VALID_FLAGS (IOSQE_FIXED_FILE|IOSQE_IO_DRAIN|IOSQE_IO_LINK| \
Jens Axboece35a472019-12-17 08:04:44 -07004771 IOSQE_IO_HARDLINK | IOSQE_ASYNC)
Jens Axboe9e645e112019-05-10 16:07:28 -06004772
Jens Axboe3529d8c2019-12-19 18:24:38 -07004773static bool io_submit_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
4774 struct io_submit_state *state, struct io_kiocb **link)
Jens Axboe9e645e112019-05-10 16:07:28 -06004775{
Jens Axboe75c6a032020-01-28 10:15:23 -07004776 const struct cred *old_creds = NULL;
Jackie Liua197f662019-11-08 08:09:12 -07004777 struct io_ring_ctx *ctx = req->ctx;
Pavel Begunkov32fe5252019-12-17 22:26:58 +03004778 unsigned int sqe_flags;
Jens Axboe75c6a032020-01-28 10:15:23 -07004779 int ret, id;
Jens Axboe9e645e112019-05-10 16:07:28 -06004780
Pavel Begunkov32fe5252019-12-17 22:26:58 +03004781 sqe_flags = READ_ONCE(sqe->flags);
Jens Axboe9e645e112019-05-10 16:07:28 -06004782
4783 /* enforce forwards compatibility on users */
Pavel Begunkov32fe5252019-12-17 22:26:58 +03004784 if (unlikely(sqe_flags & ~SQE_VALID_FLAGS)) {
Jens Axboe9e645e112019-05-10 16:07:28 -06004785 ret = -EINVAL;
Pavel Begunkov196be952019-11-07 01:41:06 +03004786 goto err_req;
Jens Axboe9e645e112019-05-10 16:07:28 -06004787 }
4788
Jens Axboe75c6a032020-01-28 10:15:23 -07004789 id = READ_ONCE(sqe->personality);
4790 if (id) {
4791 const struct cred *personality_creds;
4792
4793 personality_creds = idr_find(&ctx->personality_idr, id);
4794 if (unlikely(!personality_creds)) {
4795 ret = -EINVAL;
4796 goto err_req;
4797 }
4798 old_creds = override_creds(personality_creds);
4799 }
4800
Pavel Begunkov6b47ee62020-01-18 20:22:41 +03004801 /* same numerical values with corresponding REQ_F_*, safe to copy */
4802 req->flags |= sqe_flags & (IOSQE_IO_DRAIN|IOSQE_IO_HARDLINK|
4803 IOSQE_ASYNC);
Jens Axboe9e645e112019-05-10 16:07:28 -06004804
Jens Axboe3529d8c2019-12-19 18:24:38 -07004805 ret = io_req_set_file(state, req, sqe);
Jens Axboe9e645e112019-05-10 16:07:28 -06004806 if (unlikely(ret)) {
4807err_req:
Jens Axboe78e19bb2019-11-06 15:21:34 -07004808 io_cqring_add_event(req, ret);
4809 io_double_put_req(req);
Jens Axboe75c6a032020-01-28 10:15:23 -07004810 if (old_creds)
4811 revert_creds(old_creds);
Pavel Begunkov2e6e1fd2019-12-05 16:15:45 +03004812 return false;
Jens Axboe9e645e112019-05-10 16:07:28 -06004813 }
4814
Jens Axboe9e645e112019-05-10 16:07:28 -06004815 /*
4816 * If we already have a head request, queue this one for async
4817 * submittal once the head completes. If we don't have a head but
4818 * IOSQE_IO_LINK is set in the sqe, start a new head. This one will be
4819 * submitted sync once the chain is complete. If none of those
4820 * conditions are true (normal request), then just queue it.
4821 */
4822 if (*link) {
Pavel Begunkov9d763772019-12-17 02:22:07 +03004823 struct io_kiocb *head = *link;
Jens Axboe9e645e112019-05-10 16:07:28 -06004824
Pavel Begunkov8cdf2192020-01-25 00:40:24 +03004825 /*
4826 * Taking sequential execution of a link, draining both sides
4827 * of the link also fullfils IOSQE_IO_DRAIN semantics for all
4828 * requests in the link. So, it drains the head and the
4829 * next after the link request. The last one is done via
4830 * drain_next flag to persist the effect across calls.
4831 */
Pavel Begunkov711be032020-01-17 03:57:59 +03004832 if (sqe_flags & IOSQE_IO_DRAIN) {
4833 head->flags |= REQ_F_IO_DRAIN;
4834 ctx->drain_next = 1;
4835 }
Jens Axboeb7bb4f72019-12-15 22:13:43 -07004836 if (io_alloc_async_ctx(req)) {
Jens Axboe9e645e112019-05-10 16:07:28 -06004837 ret = -EAGAIN;
4838 goto err_req;
4839 }
4840
Jens Axboe3529d8c2019-12-19 18:24:38 -07004841 ret = io_req_defer_prep(req, sqe);
Jens Axboe2d283902019-12-04 11:08:05 -07004842 if (ret) {
Jens Axboe4e88d6e2019-12-07 20:59:47 -07004843 /* fail even hard links since we don't submit */
Pavel Begunkov9d763772019-12-17 02:22:07 +03004844 head->flags |= REQ_F_FAIL_LINK;
Jens Axboef67676d2019-12-02 11:03:47 -07004845 goto err_req;
Jens Axboe2d283902019-12-04 11:08:05 -07004846 }
Pavel Begunkov9d763772019-12-17 02:22:07 +03004847 trace_io_uring_link(ctx, req, head);
4848 list_add_tail(&req->link_list, &head->link_list);
Jens Axboe9e645e112019-05-10 16:07:28 -06004849
Pavel Begunkov32fe5252019-12-17 22:26:58 +03004850 /* last request of a link, enqueue the link */
4851 if (!(sqe_flags & (IOSQE_IO_LINK|IOSQE_IO_HARDLINK))) {
4852 io_queue_link_head(head);
4853 *link = NULL;
4854 }
Jens Axboe9e645e112019-05-10 16:07:28 -06004855 } else {
Pavel Begunkov711be032020-01-17 03:57:59 +03004856 if (unlikely(ctx->drain_next)) {
4857 req->flags |= REQ_F_IO_DRAIN;
4858 req->ctx->drain_next = 0;
4859 }
4860 if (sqe_flags & (IOSQE_IO_LINK|IOSQE_IO_HARDLINK)) {
4861 req->flags |= REQ_F_LINK;
Pavel Begunkov711be032020-01-17 03:57:59 +03004862 INIT_LIST_HEAD(&req->link_list);
4863 ret = io_req_defer_prep(req, sqe);
4864 if (ret)
4865 req->flags |= REQ_F_FAIL_LINK;
4866 *link = req;
4867 } else {
4868 io_queue_sqe(req, sqe);
4869 }
Jens Axboe9e645e112019-05-10 16:07:28 -06004870 }
Pavel Begunkov2e6e1fd2019-12-05 16:15:45 +03004871
Jens Axboe75c6a032020-01-28 10:15:23 -07004872 if (old_creds)
4873 revert_creds(old_creds);
Pavel Begunkov2e6e1fd2019-12-05 16:15:45 +03004874 return true;
Jens Axboe9e645e112019-05-10 16:07:28 -06004875}
4876
Jens Axboe9a56a232019-01-09 09:06:50 -07004877/*
4878 * Batched submission is done, ensure local IO is flushed out.
4879 */
4880static void io_submit_state_end(struct io_submit_state *state)
4881{
4882 blk_finish_plug(&state->plug);
Jens Axboe3d6770f2019-04-13 11:50:54 -06004883 io_file_put(state);
Jens Axboe2579f912019-01-09 09:10:43 -07004884 if (state->free_reqs)
Pavel Begunkov6c8a3132020-02-01 03:58:00 +03004885 kmem_cache_free_bulk(req_cachep, state->free_reqs, state->reqs);
Jens Axboe9a56a232019-01-09 09:06:50 -07004886}
4887
4888/*
4889 * Start submission side cache.
4890 */
4891static void io_submit_state_start(struct io_submit_state *state,
Jackie Liu22efde52019-12-02 17:14:52 +08004892 unsigned int max_ios)
Jens Axboe9a56a232019-01-09 09:06:50 -07004893{
4894 blk_start_plug(&state->plug);
Jens Axboe2579f912019-01-09 09:10:43 -07004895 state->free_reqs = 0;
Jens Axboe9a56a232019-01-09 09:06:50 -07004896 state->file = NULL;
4897 state->ios_left = max_ios;
4898}
4899
Jens Axboe2b188cc2019-01-07 10:46:33 -07004900static void io_commit_sqring(struct io_ring_ctx *ctx)
4901{
Hristo Venev75b28af2019-08-26 17:23:46 +00004902 struct io_rings *rings = ctx->rings;
Jens Axboe2b188cc2019-01-07 10:46:33 -07004903
Pavel Begunkovcaf582c2019-12-30 21:24:46 +03004904 /*
4905 * Ensure any loads from the SQEs are done at this point,
4906 * since once we write the new head, the application could
4907 * write new data to them.
4908 */
4909 smp_store_release(&rings->sq.head, ctx->cached_sq_head);
Jens Axboe2b188cc2019-01-07 10:46:33 -07004910}
4911
4912/*
Jens Axboe3529d8c2019-12-19 18:24:38 -07004913 * Fetch an sqe, if one is available. Note that sqe_ptr will point to memory
Jens Axboe2b188cc2019-01-07 10:46:33 -07004914 * that is mapped by userspace. This means that care needs to be taken to
4915 * ensure that reads are stable, as we cannot rely on userspace always
4916 * being a good citizen. If members of the sqe are validated and then later
4917 * used, it's important that those reads are done through READ_ONCE() to
4918 * prevent a re-load down the line.
4919 */
Jens Axboe3529d8c2019-12-19 18:24:38 -07004920static bool io_get_sqring(struct io_ring_ctx *ctx, struct io_kiocb *req,
4921 const struct io_uring_sqe **sqe_ptr)
Jens Axboe2b188cc2019-01-07 10:46:33 -07004922{
Hristo Venev75b28af2019-08-26 17:23:46 +00004923 u32 *sq_array = ctx->sq_array;
Jens Axboe2b188cc2019-01-07 10:46:33 -07004924 unsigned head;
4925
4926 /*
4927 * The cached sq head (or cq tail) serves two purposes:
4928 *
4929 * 1) allows us to batch the cost of updating the user visible
4930 * head updates.
4931 * 2) allows the kernel side to track the head on its own, even
4932 * though the application is the one updating it.
4933 */
Pavel Begunkovee7d46d2019-12-30 21:24:45 +03004934 head = READ_ONCE(sq_array[ctx->cached_sq_head & ctx->sq_mask]);
Pavel Begunkov9835d6f2019-11-21 21:24:56 +03004935 if (likely(head < ctx->sq_entries)) {
Pavel Begunkovcf6fd4b2019-11-25 23:14:39 +03004936 /*
4937 * All io need record the previous position, if LINK vs DARIN,
4938 * it can be used to mark the position of the first IO in the
4939 * link list.
4940 */
4941 req->sequence = ctx->cached_sq_head;
Jens Axboe3529d8c2019-12-19 18:24:38 -07004942 *sqe_ptr = &ctx->sq_sqes[head];
4943 req->opcode = READ_ONCE((*sqe_ptr)->opcode);
4944 req->user_data = READ_ONCE((*sqe_ptr)->user_data);
Jens Axboe2b188cc2019-01-07 10:46:33 -07004945 ctx->cached_sq_head++;
4946 return true;
4947 }
4948
4949 /* drop invalid entries */
4950 ctx->cached_sq_head++;
Jens Axboe498ccd92019-10-25 10:04:25 -06004951 ctx->cached_sq_dropped++;
Pavel Begunkovee7d46d2019-12-30 21:24:45 +03004952 WRITE_ONCE(ctx->rings->sq_dropped, ctx->cached_sq_dropped);
Jens Axboe2b188cc2019-01-07 10:46:33 -07004953 return false;
4954}
4955
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03004956static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr,
Pavel Begunkovae9428c2019-11-06 00:22:14 +03004957 struct file *ring_file, int ring_fd,
4958 struct mm_struct **mm, bool async)
Jens Axboe6c271ce2019-01-10 11:22:30 -07004959{
4960 struct io_submit_state state, *statep = NULL;
Jens Axboe9e645e112019-05-10 16:07:28 -06004961 struct io_kiocb *link = NULL;
Jens Axboe9e645e112019-05-10 16:07:28 -06004962 int i, submitted = 0;
Pavel Begunkov95a1b3ff2019-10-27 23:15:41 +03004963 bool mm_fault = false;
Jens Axboe6c271ce2019-01-10 11:22:30 -07004964
Jens Axboec4a2ed72019-11-21 21:01:26 -07004965 /* if we have a backlog and couldn't flush it all, return BUSY */
Jens Axboead3eb2c2019-12-18 17:12:20 -07004966 if (test_bit(0, &ctx->sq_check_overflow)) {
4967 if (!list_empty(&ctx->cq_overflow_list) &&
4968 !io_cqring_overflow_flush(ctx, false))
4969 return -EBUSY;
4970 }
Jens Axboe6c271ce2019-01-10 11:22:30 -07004971
Pavel Begunkovee7d46d2019-12-30 21:24:45 +03004972 /* make sure SQ entry isn't read before tail */
4973 nr = min3(nr, ctx->sq_entries, io_sqring_entries(ctx));
Pavel Begunkov9ef4f122019-12-30 21:24:44 +03004974
Pavel Begunkov2b85edf2019-12-28 14:13:03 +03004975 if (!percpu_ref_tryget_many(&ctx->refs, nr))
4976 return -EAGAIN;
Jens Axboe6c271ce2019-01-10 11:22:30 -07004977
4978 if (nr > IO_PLUG_THRESHOLD) {
Jackie Liu22efde52019-12-02 17:14:52 +08004979 io_submit_state_start(&state, nr);
Jens Axboe6c271ce2019-01-10 11:22:30 -07004980 statep = &state;
4981 }
4982
Pavel Begunkovb14cca02020-01-17 04:45:59 +03004983 ctx->ring_fd = ring_fd;
4984 ctx->ring_file = ring_file;
4985
Jens Axboe6c271ce2019-01-10 11:22:30 -07004986 for (i = 0; i < nr; i++) {
Jens Axboe3529d8c2019-12-19 18:24:38 -07004987 const struct io_uring_sqe *sqe;
Pavel Begunkov196be952019-11-07 01:41:06 +03004988 struct io_kiocb *req;
Pavel Begunkov1cb1edb2020-02-06 21:16:09 +03004989 int err;
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03004990
Pavel Begunkov196be952019-11-07 01:41:06 +03004991 req = io_get_req(ctx, statep);
4992 if (unlikely(!req)) {
4993 if (!submitted)
4994 submitted = -EAGAIN;
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03004995 break;
Jens Axboe9e645e112019-05-10 16:07:28 -06004996 }
Jens Axboe3529d8c2019-12-19 18:24:38 -07004997 if (!io_get_sqring(ctx, req, &sqe)) {
Pavel Begunkov2b85edf2019-12-28 14:13:03 +03004998 __io_req_do_free(req);
Pavel Begunkov196be952019-11-07 01:41:06 +03004999 break;
5000 }
Jens Axboe9e645e112019-05-10 16:07:28 -06005001
Jens Axboed3656342019-12-18 09:50:26 -07005002 /* will complete beyond this point, count as submitted */
5003 submitted++;
5004
5005 if (unlikely(req->opcode >= IORING_OP_LAST)) {
Pavel Begunkov1cb1edb2020-02-06 21:16:09 +03005006 err = -EINVAL;
5007fail_req:
5008 io_cqring_add_event(req, err);
Jens Axboed3656342019-12-18 09:50:26 -07005009 io_double_put_req(req);
5010 break;
5011 }
5012
5013 if (io_op_defs[req->opcode].needs_mm && !*mm) {
Pavel Begunkov95a1b3ff2019-10-27 23:15:41 +03005014 mm_fault = mm_fault || !mmget_not_zero(ctx->sqo_mm);
Pavel Begunkov1cb1edb2020-02-06 21:16:09 +03005015 if (unlikely(mm_fault)) {
5016 err = -EFAULT;
5017 goto fail_req;
Pavel Begunkov95a1b3ff2019-10-27 23:15:41 +03005018 }
Pavel Begunkov1cb1edb2020-02-06 21:16:09 +03005019 use_mm(ctx->sqo_mm);
5020 *mm = ctx->sqo_mm;
Pavel Begunkov95a1b3ff2019-10-27 23:15:41 +03005021 }
5022
Pavel Begunkovcf6fd4b2019-11-25 23:14:39 +03005023 req->in_async = async;
5024 req->needs_fixed_file = async;
Jens Axboe354420f2020-01-08 18:55:15 -07005025 trace_io_uring_submit_sqe(ctx, req->opcode, req->user_data,
5026 true, async);
Jens Axboe3529d8c2019-12-19 18:24:38 -07005027 if (!io_submit_sqe(req, sqe, statep, &link))
Pavel Begunkov2e6e1fd2019-12-05 16:15:45 +03005028 break;
Jens Axboe6c271ce2019-01-10 11:22:30 -07005029 }
5030
Pavel Begunkov9466f432020-01-25 22:34:01 +03005031 if (unlikely(submitted != nr)) {
5032 int ref_used = (submitted == -EAGAIN) ? 0 : submitted;
5033
5034 percpu_ref_put_many(&ctx->refs, nr - ref_used);
5035 }
Jens Axboe9e645e112019-05-10 16:07:28 -06005036 if (link)
Pavel Begunkov1b4a51b2019-11-21 11:54:28 +03005037 io_queue_link_head(link);
Jens Axboe6c271ce2019-01-10 11:22:30 -07005038 if (statep)
5039 io_submit_state_end(&state);
5040
Pavel Begunkovae9428c2019-11-06 00:22:14 +03005041 /* Commit SQ ring head once we've consumed and submitted all SQEs */
5042 io_commit_sqring(ctx);
5043
Jens Axboe6c271ce2019-01-10 11:22:30 -07005044 return submitted;
5045}
5046
5047static int io_sq_thread(void *data)
5048{
Jens Axboe6c271ce2019-01-10 11:22:30 -07005049 struct io_ring_ctx *ctx = data;
5050 struct mm_struct *cur_mm = NULL;
Jens Axboe181e4482019-11-25 08:52:30 -07005051 const struct cred *old_cred;
Jens Axboe6c271ce2019-01-10 11:22:30 -07005052 mm_segment_t old_fs;
5053 DEFINE_WAIT(wait);
5054 unsigned inflight;
5055 unsigned long timeout;
Jens Axboec1edbf52019-11-10 16:56:04 -07005056 int ret;
Jens Axboe6c271ce2019-01-10 11:22:30 -07005057
Jens Axboe206aefd2019-11-07 18:27:42 -07005058 complete(&ctx->completions[1]);
Jackie Liua4c0b3d2019-07-08 13:41:12 +08005059
Jens Axboe6c271ce2019-01-10 11:22:30 -07005060 old_fs = get_fs();
5061 set_fs(USER_DS);
Jens Axboe181e4482019-11-25 08:52:30 -07005062 old_cred = override_creds(ctx->creds);
Jens Axboe6c271ce2019-01-10 11:22:30 -07005063
Jens Axboec1edbf52019-11-10 16:56:04 -07005064 ret = timeout = inflight = 0;
Roman Penyaev2bbcd6d2019-05-16 10:53:57 +02005065 while (!kthread_should_park()) {
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03005066 unsigned int to_submit;
Jens Axboe6c271ce2019-01-10 11:22:30 -07005067
5068 if (inflight) {
5069 unsigned nr_events = 0;
5070
5071 if (ctx->flags & IORING_SETUP_IOPOLL) {
Jens Axboe2b2ed972019-10-25 10:06:15 -06005072 /*
5073 * inflight is the count of the maximum possible
5074 * entries we submitted, but it can be smaller
5075 * if we dropped some of them. If we don't have
5076 * poll entries available, then we know that we
5077 * have nothing left to poll for. Reset the
5078 * inflight count to zero in that case.
5079 */
5080 mutex_lock(&ctx->uring_lock);
5081 if (!list_empty(&ctx->poll_list))
5082 __io_iopoll_check(ctx, &nr_events, 0);
5083 else
5084 inflight = 0;
5085 mutex_unlock(&ctx->uring_lock);
Jens Axboe6c271ce2019-01-10 11:22:30 -07005086 } else {
5087 /*
5088 * Normal IO, just pretend everything completed.
5089 * We don't have to poll completions for that.
5090 */
5091 nr_events = inflight;
5092 }
5093
5094 inflight -= nr_events;
5095 if (!inflight)
5096 timeout = jiffies + ctx->sq_thread_idle;
5097 }
5098
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03005099 to_submit = io_sqring_entries(ctx);
Jens Axboec1edbf52019-11-10 16:56:04 -07005100
5101 /*
5102 * If submit got -EBUSY, flag us as needing the application
5103 * to enter the kernel to reap and flush events.
5104 */
5105 if (!to_submit || ret == -EBUSY) {
Jens Axboe6c271ce2019-01-10 11:22:30 -07005106 /*
5107 * We're polling. If we're within the defined idle
5108 * period, then let us spin without work before going
Jens Axboec1edbf52019-11-10 16:56:04 -07005109 * to sleep. The exception is if we got EBUSY doing
5110 * more IO, we should wait for the application to
5111 * reap events and wake us up.
Jens Axboe6c271ce2019-01-10 11:22:30 -07005112 */
Jens Axboec1edbf52019-11-10 16:56:04 -07005113 if (inflight ||
Jens Axboedf069d82020-02-04 16:48:34 -07005114 (!time_after(jiffies, timeout) && ret != -EBUSY &&
5115 !percpu_ref_is_dying(&ctx->refs))) {
Jens Axboe9831a902019-09-19 09:48:55 -06005116 cond_resched();
Jens Axboe6c271ce2019-01-10 11:22:30 -07005117 continue;
5118 }
5119
5120 /*
5121 * Drop cur_mm before scheduling, we can't hold it for
5122 * long periods (or over schedule()). Do this before
5123 * adding ourselves to the waitqueue, as the unuse/drop
5124 * may sleep.
5125 */
5126 if (cur_mm) {
5127 unuse_mm(cur_mm);
5128 mmput(cur_mm);
5129 cur_mm = NULL;
5130 }
5131
5132 prepare_to_wait(&ctx->sqo_wait, &wait,
5133 TASK_INTERRUPTIBLE);
5134
5135 /* Tell userspace we may need a wakeup call */
Hristo Venev75b28af2019-08-26 17:23:46 +00005136 ctx->rings->sq_flags |= IORING_SQ_NEED_WAKEUP;
Stefan Bühler0d7bae62019-04-19 11:57:45 +02005137 /* make sure to read SQ tail after writing flags */
5138 smp_mb();
Jens Axboe6c271ce2019-01-10 11:22:30 -07005139
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03005140 to_submit = io_sqring_entries(ctx);
Jens Axboec1edbf52019-11-10 16:56:04 -07005141 if (!to_submit || ret == -EBUSY) {
Roman Penyaev2bbcd6d2019-05-16 10:53:57 +02005142 if (kthread_should_park()) {
Jens Axboe6c271ce2019-01-10 11:22:30 -07005143 finish_wait(&ctx->sqo_wait, &wait);
5144 break;
5145 }
5146 if (signal_pending(current))
5147 flush_signals(current);
5148 schedule();
5149 finish_wait(&ctx->sqo_wait, &wait);
5150
Hristo Venev75b28af2019-08-26 17:23:46 +00005151 ctx->rings->sq_flags &= ~IORING_SQ_NEED_WAKEUP;
Jens Axboe6c271ce2019-01-10 11:22:30 -07005152 continue;
5153 }
5154 finish_wait(&ctx->sqo_wait, &wait);
5155
Hristo Venev75b28af2019-08-26 17:23:46 +00005156 ctx->rings->sq_flags &= ~IORING_SQ_NEED_WAKEUP;
Jens Axboe6c271ce2019-01-10 11:22:30 -07005157 }
5158
Jens Axboe8a4955f2019-12-09 14:52:35 -07005159 mutex_lock(&ctx->uring_lock);
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07005160 ret = io_submit_sqes(ctx, to_submit, NULL, -1, &cur_mm, true);
Jens Axboe8a4955f2019-12-09 14:52:35 -07005161 mutex_unlock(&ctx->uring_lock);
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07005162 if (ret > 0)
5163 inflight += ret;
Jens Axboe6c271ce2019-01-10 11:22:30 -07005164 }
5165
5166 set_fs(old_fs);
5167 if (cur_mm) {
5168 unuse_mm(cur_mm);
5169 mmput(cur_mm);
5170 }
Jens Axboe181e4482019-11-25 08:52:30 -07005171 revert_creds(old_cred);
Jens Axboe06058632019-04-13 09:26:03 -06005172
Roman Penyaev2bbcd6d2019-05-16 10:53:57 +02005173 kthread_parkme();
Jens Axboe06058632019-04-13 09:26:03 -06005174
Jens Axboe6c271ce2019-01-10 11:22:30 -07005175 return 0;
5176}
5177
Jens Axboebda52162019-09-24 13:47:15 -06005178struct io_wait_queue {
5179 struct wait_queue_entry wq;
5180 struct io_ring_ctx *ctx;
5181 unsigned to_wait;
5182 unsigned nr_timeouts;
5183};
5184
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07005185static inline bool io_should_wake(struct io_wait_queue *iowq, bool noflush)
Jens Axboebda52162019-09-24 13:47:15 -06005186{
5187 struct io_ring_ctx *ctx = iowq->ctx;
5188
5189 /*
Brian Gianforcarod195a662019-12-13 03:09:50 -08005190 * Wake up if we have enough events, or if a timeout occurred since we
Jens Axboebda52162019-09-24 13:47:15 -06005191 * started waiting. For timeouts, we always want to return to userspace,
5192 * regardless of event count.
5193 */
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07005194 return io_cqring_events(ctx, noflush) >= iowq->to_wait ||
Jens Axboebda52162019-09-24 13:47:15 -06005195 atomic_read(&ctx->cq_timeouts) != iowq->nr_timeouts;
5196}
5197
5198static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode,
5199 int wake_flags, void *key)
5200{
5201 struct io_wait_queue *iowq = container_of(curr, struct io_wait_queue,
5202 wq);
5203
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07005204 /* use noflush == true, as we can't safely rely on locking context */
5205 if (!io_should_wake(iowq, true))
Jens Axboebda52162019-09-24 13:47:15 -06005206 return -1;
5207
5208 return autoremove_wake_function(curr, mode, wake_flags, key);
5209}
5210
Jens Axboe2b188cc2019-01-07 10:46:33 -07005211/*
5212 * Wait until events become available, if we don't already have some. The
5213 * application must reap them itself, as they reside on the shared cq ring.
5214 */
5215static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events,
5216 const sigset_t __user *sig, size_t sigsz)
5217{
Jens Axboebda52162019-09-24 13:47:15 -06005218 struct io_wait_queue iowq = {
5219 .wq = {
5220 .private = current,
5221 .func = io_wake_function,
5222 .entry = LIST_HEAD_INIT(iowq.wq.entry),
5223 },
5224 .ctx = ctx,
5225 .to_wait = min_events,
5226 };
Hristo Venev75b28af2019-08-26 17:23:46 +00005227 struct io_rings *rings = ctx->rings;
Jackie Liue9ffa5c2019-10-29 11:16:42 +08005228 int ret = 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07005229
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07005230 if (io_cqring_events(ctx, false) >= min_events)
Jens Axboe2b188cc2019-01-07 10:46:33 -07005231 return 0;
5232
5233 if (sig) {
Arnd Bergmann9e75ad52019-03-25 15:34:53 +01005234#ifdef CONFIG_COMPAT
5235 if (in_compat_syscall())
5236 ret = set_compat_user_sigmask((const compat_sigset_t __user *)sig,
Oleg Nesterovb7724342019-07-16 16:29:53 -07005237 sigsz);
Arnd Bergmann9e75ad52019-03-25 15:34:53 +01005238 else
5239#endif
Oleg Nesterovb7724342019-07-16 16:29:53 -07005240 ret = set_user_sigmask(sig, sigsz);
Arnd Bergmann9e75ad52019-03-25 15:34:53 +01005241
Jens Axboe2b188cc2019-01-07 10:46:33 -07005242 if (ret)
5243 return ret;
5244 }
5245
Jens Axboebda52162019-09-24 13:47:15 -06005246 iowq.nr_timeouts = atomic_read(&ctx->cq_timeouts);
Dmitrii Dolgovc826bd72019-10-15 19:02:01 +02005247 trace_io_uring_cqring_wait(ctx, min_events);
Jens Axboebda52162019-09-24 13:47:15 -06005248 do {
5249 prepare_to_wait_exclusive(&ctx->wait, &iowq.wq,
5250 TASK_INTERRUPTIBLE);
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07005251 if (io_should_wake(&iowq, false))
Jens Axboebda52162019-09-24 13:47:15 -06005252 break;
5253 schedule();
5254 if (signal_pending(current)) {
Jackie Liue9ffa5c2019-10-29 11:16:42 +08005255 ret = -EINTR;
Jens Axboebda52162019-09-24 13:47:15 -06005256 break;
5257 }
5258 } while (1);
5259 finish_wait(&ctx->wait, &iowq.wq);
5260
Jackie Liue9ffa5c2019-10-29 11:16:42 +08005261 restore_saved_sigmask_unless(ret == -EINTR);
Jens Axboe2b188cc2019-01-07 10:46:33 -07005262
Hristo Venev75b28af2019-08-26 17:23:46 +00005263 return READ_ONCE(rings->cq.head) == READ_ONCE(rings->cq.tail) ? ret : 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07005264}
5265
Jens Axboe6b063142019-01-10 22:13:58 -07005266static void __io_sqe_files_unregister(struct io_ring_ctx *ctx)
5267{
5268#if defined(CONFIG_UNIX)
5269 if (ctx->ring_sock) {
5270 struct sock *sock = ctx->ring_sock->sk;
5271 struct sk_buff *skb;
5272
5273 while ((skb = skb_dequeue(&sock->sk_receive_queue)) != NULL)
5274 kfree_skb(skb);
5275 }
5276#else
5277 int i;
5278
Jens Axboe65e19f52019-10-26 07:20:21 -06005279 for (i = 0; i < ctx->nr_user_files; i++) {
5280 struct file *file;
5281
5282 file = io_file_from_index(ctx, i);
5283 if (file)
5284 fput(file);
5285 }
Jens Axboe6b063142019-01-10 22:13:58 -07005286#endif
5287}
5288
Jens Axboe05f3fb32019-12-09 11:22:50 -07005289static void io_file_ref_kill(struct percpu_ref *ref)
5290{
5291 struct fixed_file_data *data;
5292
5293 data = container_of(ref, struct fixed_file_data, refs);
5294 complete(&data->done);
5295}
5296
Jens Axboe6b063142019-01-10 22:13:58 -07005297static int io_sqe_files_unregister(struct io_ring_ctx *ctx)
5298{
Jens Axboe05f3fb32019-12-09 11:22:50 -07005299 struct fixed_file_data *data = ctx->file_data;
Jens Axboe65e19f52019-10-26 07:20:21 -06005300 unsigned nr_tables, i;
5301
Jens Axboe05f3fb32019-12-09 11:22:50 -07005302 if (!data)
Jens Axboe6b063142019-01-10 22:13:58 -07005303 return -ENXIO;
5304
Jens Axboe05f3fb32019-12-09 11:22:50 -07005305 percpu_ref_kill_and_confirm(&data->refs, io_file_ref_kill);
Jens Axboee46a7952020-01-17 11:15:34 -07005306 flush_work(&data->ref_work);
Jens Axboe2faf8522020-02-04 19:54:55 -07005307 wait_for_completion(&data->done);
5308 io_ring_file_ref_flush(data);
Jens Axboe05f3fb32019-12-09 11:22:50 -07005309 percpu_ref_exit(&data->refs);
5310
Jens Axboe6b063142019-01-10 22:13:58 -07005311 __io_sqe_files_unregister(ctx);
Jens Axboe65e19f52019-10-26 07:20:21 -06005312 nr_tables = DIV_ROUND_UP(ctx->nr_user_files, IORING_MAX_FILES_TABLE);
5313 for (i = 0; i < nr_tables; i++)
Jens Axboe05f3fb32019-12-09 11:22:50 -07005314 kfree(data->table[i].files);
5315 kfree(data->table);
5316 kfree(data);
5317 ctx->file_data = NULL;
Jens Axboe6b063142019-01-10 22:13:58 -07005318 ctx->nr_user_files = 0;
5319 return 0;
5320}
5321
Jens Axboe6c271ce2019-01-10 11:22:30 -07005322static void io_sq_thread_stop(struct io_ring_ctx *ctx)
5323{
5324 if (ctx->sqo_thread) {
Jens Axboe206aefd2019-11-07 18:27:42 -07005325 wait_for_completion(&ctx->completions[1]);
Roman Penyaev2bbcd6d2019-05-16 10:53:57 +02005326 /*
5327 * The park is a bit of a work-around, without it we get
5328 * warning spews on shutdown with SQPOLL set and affinity
5329 * set to a single CPU.
5330 */
Jens Axboe06058632019-04-13 09:26:03 -06005331 kthread_park(ctx->sqo_thread);
Jens Axboe6c271ce2019-01-10 11:22:30 -07005332 kthread_stop(ctx->sqo_thread);
5333 ctx->sqo_thread = NULL;
5334 }
5335}
5336
Jens Axboe6b063142019-01-10 22:13:58 -07005337static void io_finish_async(struct io_ring_ctx *ctx)
5338{
Jens Axboe6c271ce2019-01-10 11:22:30 -07005339 io_sq_thread_stop(ctx);
5340
Jens Axboe561fb042019-10-24 07:25:42 -06005341 if (ctx->io_wq) {
5342 io_wq_destroy(ctx->io_wq);
5343 ctx->io_wq = NULL;
Jens Axboe6b063142019-01-10 22:13:58 -07005344 }
5345}
5346
5347#if defined(CONFIG_UNIX)
Jens Axboe6b063142019-01-10 22:13:58 -07005348/*
5349 * Ensure the UNIX gc is aware of our file set, so we are certain that
5350 * the io_uring can be safely unregistered on process exit, even if we have
5351 * loops in the file referencing.
5352 */
5353static int __io_sqe_files_scm(struct io_ring_ctx *ctx, int nr, int offset)
5354{
5355 struct sock *sk = ctx->ring_sock->sk;
5356 struct scm_fp_list *fpl;
5357 struct sk_buff *skb;
Jens Axboe08a45172019-10-03 08:11:03 -06005358 int i, nr_files;
Jens Axboe6b063142019-01-10 22:13:58 -07005359
5360 if (!capable(CAP_SYS_RESOURCE) && !capable(CAP_SYS_ADMIN)) {
5361 unsigned long inflight = ctx->user->unix_inflight + nr;
5362
5363 if (inflight > task_rlimit(current, RLIMIT_NOFILE))
5364 return -EMFILE;
5365 }
5366
5367 fpl = kzalloc(sizeof(*fpl), GFP_KERNEL);
5368 if (!fpl)
5369 return -ENOMEM;
5370
5371 skb = alloc_skb(0, GFP_KERNEL);
5372 if (!skb) {
5373 kfree(fpl);
5374 return -ENOMEM;
5375 }
5376
5377 skb->sk = sk;
Jens Axboe6b063142019-01-10 22:13:58 -07005378
Jens Axboe08a45172019-10-03 08:11:03 -06005379 nr_files = 0;
Jens Axboe6b063142019-01-10 22:13:58 -07005380 fpl->user = get_uid(ctx->user);
5381 for (i = 0; i < nr; i++) {
Jens Axboe65e19f52019-10-26 07:20:21 -06005382 struct file *file = io_file_from_index(ctx, i + offset);
5383
5384 if (!file)
Jens Axboe08a45172019-10-03 08:11:03 -06005385 continue;
Jens Axboe65e19f52019-10-26 07:20:21 -06005386 fpl->fp[nr_files] = get_file(file);
Jens Axboe08a45172019-10-03 08:11:03 -06005387 unix_inflight(fpl->user, fpl->fp[nr_files]);
5388 nr_files++;
Jens Axboe6b063142019-01-10 22:13:58 -07005389 }
5390
Jens Axboe08a45172019-10-03 08:11:03 -06005391 if (nr_files) {
5392 fpl->max = SCM_MAX_FD;
5393 fpl->count = nr_files;
5394 UNIXCB(skb).fp = fpl;
Jens Axboe05f3fb32019-12-09 11:22:50 -07005395 skb->destructor = unix_destruct_scm;
Jens Axboe08a45172019-10-03 08:11:03 -06005396 refcount_add(skb->truesize, &sk->sk_wmem_alloc);
5397 skb_queue_head(&sk->sk_receive_queue, skb);
Jens Axboe6b063142019-01-10 22:13:58 -07005398
Jens Axboe08a45172019-10-03 08:11:03 -06005399 for (i = 0; i < nr_files; i++)
5400 fput(fpl->fp[i]);
5401 } else {
5402 kfree_skb(skb);
5403 kfree(fpl);
5404 }
Jens Axboe6b063142019-01-10 22:13:58 -07005405
5406 return 0;
5407}
5408
5409/*
5410 * If UNIX sockets are enabled, fd passing can cause a reference cycle which
5411 * causes regular reference counting to break down. We rely on the UNIX
5412 * garbage collection to take care of this problem for us.
5413 */
5414static int io_sqe_files_scm(struct io_ring_ctx *ctx)
5415{
5416 unsigned left, total;
5417 int ret = 0;
5418
5419 total = 0;
5420 left = ctx->nr_user_files;
5421 while (left) {
5422 unsigned this_files = min_t(unsigned, left, SCM_MAX_FD);
Jens Axboe6b063142019-01-10 22:13:58 -07005423
5424 ret = __io_sqe_files_scm(ctx, this_files, total);
5425 if (ret)
5426 break;
5427 left -= this_files;
5428 total += this_files;
5429 }
5430
5431 if (!ret)
5432 return 0;
5433
5434 while (total < ctx->nr_user_files) {
Jens Axboe65e19f52019-10-26 07:20:21 -06005435 struct file *file = io_file_from_index(ctx, total);
5436
5437 if (file)
5438 fput(file);
Jens Axboe6b063142019-01-10 22:13:58 -07005439 total++;
5440 }
5441
5442 return ret;
5443}
5444#else
5445static int io_sqe_files_scm(struct io_ring_ctx *ctx)
5446{
5447 return 0;
5448}
5449#endif
5450
Jens Axboe65e19f52019-10-26 07:20:21 -06005451static int io_sqe_alloc_file_tables(struct io_ring_ctx *ctx, unsigned nr_tables,
5452 unsigned nr_files)
5453{
5454 int i;
5455
5456 for (i = 0; i < nr_tables; i++) {
Jens Axboe05f3fb32019-12-09 11:22:50 -07005457 struct fixed_file_table *table = &ctx->file_data->table[i];
Jens Axboe65e19f52019-10-26 07:20:21 -06005458 unsigned this_files;
5459
5460 this_files = min(nr_files, IORING_MAX_FILES_TABLE);
5461 table->files = kcalloc(this_files, sizeof(struct file *),
5462 GFP_KERNEL);
5463 if (!table->files)
5464 break;
5465 nr_files -= this_files;
5466 }
5467
5468 if (i == nr_tables)
5469 return 0;
5470
5471 for (i = 0; i < nr_tables; i++) {
Jens Axboe05f3fb32019-12-09 11:22:50 -07005472 struct fixed_file_table *table = &ctx->file_data->table[i];
Jens Axboe65e19f52019-10-26 07:20:21 -06005473 kfree(table->files);
5474 }
5475 return 1;
5476}
5477
Jens Axboe05f3fb32019-12-09 11:22:50 -07005478static void io_ring_file_put(struct io_ring_ctx *ctx, struct file *file)
Jens Axboec3a31e62019-10-03 13:59:56 -06005479{
5480#if defined(CONFIG_UNIX)
Jens Axboec3a31e62019-10-03 13:59:56 -06005481 struct sock *sock = ctx->ring_sock->sk;
5482 struct sk_buff_head list, *head = &sock->sk_receive_queue;
5483 struct sk_buff *skb;
5484 int i;
5485
5486 __skb_queue_head_init(&list);
5487
5488 /*
5489 * Find the skb that holds this file in its SCM_RIGHTS. When found,
5490 * remove this entry and rearrange the file array.
5491 */
5492 skb = skb_dequeue(head);
5493 while (skb) {
5494 struct scm_fp_list *fp;
5495
5496 fp = UNIXCB(skb).fp;
5497 for (i = 0; i < fp->count; i++) {
5498 int left;
5499
5500 if (fp->fp[i] != file)
5501 continue;
5502
5503 unix_notinflight(fp->user, fp->fp[i]);
5504 left = fp->count - 1 - i;
5505 if (left) {
5506 memmove(&fp->fp[i], &fp->fp[i + 1],
5507 left * sizeof(struct file *));
5508 }
5509 fp->count--;
5510 if (!fp->count) {
5511 kfree_skb(skb);
5512 skb = NULL;
5513 } else {
5514 __skb_queue_tail(&list, skb);
5515 }
5516 fput(file);
5517 file = NULL;
5518 break;
5519 }
5520
5521 if (!file)
5522 break;
5523
5524 __skb_queue_tail(&list, skb);
5525
5526 skb = skb_dequeue(head);
5527 }
5528
5529 if (skb_peek(&list)) {
5530 spin_lock_irq(&head->lock);
5531 while ((skb = __skb_dequeue(&list)) != NULL)
5532 __skb_queue_tail(head, skb);
5533 spin_unlock_irq(&head->lock);
5534 }
5535#else
Jens Axboe05f3fb32019-12-09 11:22:50 -07005536 fput(file);
Jens Axboec3a31e62019-10-03 13:59:56 -06005537#endif
5538}
5539
Jens Axboe05f3fb32019-12-09 11:22:50 -07005540struct io_file_put {
5541 struct llist_node llist;
5542 struct file *file;
5543 struct completion *done;
5544};
5545
Jens Axboe2faf8522020-02-04 19:54:55 -07005546static void io_ring_file_ref_flush(struct fixed_file_data *data)
Jens Axboe05f3fb32019-12-09 11:22:50 -07005547{
5548 struct io_file_put *pfile, *tmp;
Jens Axboe05f3fb32019-12-09 11:22:50 -07005549 struct llist_node *node;
5550
Jens Axboe05f3fb32019-12-09 11:22:50 -07005551 while ((node = llist_del_all(&data->put_llist)) != NULL) {
5552 llist_for_each_entry_safe(pfile, tmp, node, llist) {
5553 io_ring_file_put(data->ctx, pfile->file);
5554 if (pfile->done)
5555 complete(pfile->done);
5556 else
5557 kfree(pfile);
5558 }
5559 }
Jens Axboe2faf8522020-02-04 19:54:55 -07005560}
Jens Axboe05f3fb32019-12-09 11:22:50 -07005561
Jens Axboe2faf8522020-02-04 19:54:55 -07005562static void io_ring_file_ref_switch(struct work_struct *work)
5563{
5564 struct fixed_file_data *data;
5565
5566 data = container_of(work, struct fixed_file_data, ref_work);
5567 io_ring_file_ref_flush(data);
Jens Axboe05f3fb32019-12-09 11:22:50 -07005568 percpu_ref_get(&data->refs);
5569 percpu_ref_switch_to_percpu(&data->refs);
5570}
5571
5572static void io_file_data_ref_zero(struct percpu_ref *ref)
5573{
5574 struct fixed_file_data *data;
5575
5576 data = container_of(ref, struct fixed_file_data, refs);
5577
Jens Axboe2faf8522020-02-04 19:54:55 -07005578 /*
5579 * We can't safely switch from inside this context, punt to wq. If
5580 * the table ref is going away, the table is being unregistered.
5581 * Don't queue up the async work for that case, the caller will
5582 * handle it.
5583 */
5584 if (!percpu_ref_is_dying(&data->refs))
5585 queue_work(system_wq, &data->ref_work);
Jens Axboe05f3fb32019-12-09 11:22:50 -07005586}
5587
5588static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg,
5589 unsigned nr_args)
5590{
5591 __s32 __user *fds = (__s32 __user *) arg;
5592 unsigned nr_tables;
5593 struct file *file;
5594 int fd, ret = 0;
5595 unsigned i;
5596
5597 if (ctx->file_data)
5598 return -EBUSY;
5599 if (!nr_args)
5600 return -EINVAL;
5601 if (nr_args > IORING_MAX_FIXED_FILES)
5602 return -EMFILE;
5603
5604 ctx->file_data = kzalloc(sizeof(*ctx->file_data), GFP_KERNEL);
5605 if (!ctx->file_data)
5606 return -ENOMEM;
5607 ctx->file_data->ctx = ctx;
5608 init_completion(&ctx->file_data->done);
5609
5610 nr_tables = DIV_ROUND_UP(nr_args, IORING_MAX_FILES_TABLE);
5611 ctx->file_data->table = kcalloc(nr_tables,
5612 sizeof(struct fixed_file_table),
5613 GFP_KERNEL);
5614 if (!ctx->file_data->table) {
5615 kfree(ctx->file_data);
5616 ctx->file_data = NULL;
5617 return -ENOMEM;
5618 }
5619
5620 if (percpu_ref_init(&ctx->file_data->refs, io_file_data_ref_zero,
5621 PERCPU_REF_ALLOW_REINIT, GFP_KERNEL)) {
5622 kfree(ctx->file_data->table);
5623 kfree(ctx->file_data);
5624 ctx->file_data = NULL;
5625 return -ENOMEM;
5626 }
5627 ctx->file_data->put_llist.first = NULL;
5628 INIT_WORK(&ctx->file_data->ref_work, io_ring_file_ref_switch);
5629
5630 if (io_sqe_alloc_file_tables(ctx, nr_tables, nr_args)) {
5631 percpu_ref_exit(&ctx->file_data->refs);
5632 kfree(ctx->file_data->table);
5633 kfree(ctx->file_data);
5634 ctx->file_data = NULL;
5635 return -ENOMEM;
5636 }
5637
5638 for (i = 0; i < nr_args; i++, ctx->nr_user_files++) {
5639 struct fixed_file_table *table;
5640 unsigned index;
5641
5642 ret = -EFAULT;
5643 if (copy_from_user(&fd, &fds[i], sizeof(fd)))
5644 break;
5645 /* allow sparse sets */
5646 if (fd == -1) {
5647 ret = 0;
5648 continue;
5649 }
5650
5651 table = &ctx->file_data->table[i >> IORING_FILE_TABLE_SHIFT];
5652 index = i & IORING_FILE_TABLE_MASK;
5653 file = fget(fd);
5654
5655 ret = -EBADF;
5656 if (!file)
5657 break;
5658
5659 /*
5660 * Don't allow io_uring instances to be registered. If UNIX
5661 * isn't enabled, then this causes a reference cycle and this
5662 * instance can never get freed. If UNIX is enabled we'll
5663 * handle it just fine, but there's still no point in allowing
5664 * a ring fd as it doesn't support regular read/write anyway.
5665 */
5666 if (file->f_op == &io_uring_fops) {
5667 fput(file);
5668 break;
5669 }
5670 ret = 0;
5671 table->files[index] = file;
5672 }
5673
5674 if (ret) {
5675 for (i = 0; i < ctx->nr_user_files; i++) {
5676 file = io_file_from_index(ctx, i);
5677 if (file)
5678 fput(file);
5679 }
5680 for (i = 0; i < nr_tables; i++)
5681 kfree(ctx->file_data->table[i].files);
5682
5683 kfree(ctx->file_data->table);
5684 kfree(ctx->file_data);
5685 ctx->file_data = NULL;
5686 ctx->nr_user_files = 0;
5687 return ret;
5688 }
5689
5690 ret = io_sqe_files_scm(ctx);
5691 if (ret)
5692 io_sqe_files_unregister(ctx);
5693
5694 return ret;
5695}
5696
Jens Axboec3a31e62019-10-03 13:59:56 -06005697static int io_sqe_file_register(struct io_ring_ctx *ctx, struct file *file,
5698 int index)
5699{
5700#if defined(CONFIG_UNIX)
5701 struct sock *sock = ctx->ring_sock->sk;
5702 struct sk_buff_head *head = &sock->sk_receive_queue;
5703 struct sk_buff *skb;
5704
5705 /*
5706 * See if we can merge this file into an existing skb SCM_RIGHTS
5707 * file set. If there's no room, fall back to allocating a new skb
5708 * and filling it in.
5709 */
5710 spin_lock_irq(&head->lock);
5711 skb = skb_peek(head);
5712 if (skb) {
5713 struct scm_fp_list *fpl = UNIXCB(skb).fp;
5714
5715 if (fpl->count < SCM_MAX_FD) {
5716 __skb_unlink(skb, head);
5717 spin_unlock_irq(&head->lock);
5718 fpl->fp[fpl->count] = get_file(file);
5719 unix_inflight(fpl->user, fpl->fp[fpl->count]);
5720 fpl->count++;
5721 spin_lock_irq(&head->lock);
5722 __skb_queue_head(head, skb);
5723 } else {
5724 skb = NULL;
5725 }
5726 }
5727 spin_unlock_irq(&head->lock);
5728
5729 if (skb) {
5730 fput(file);
5731 return 0;
5732 }
5733
5734 return __io_sqe_files_scm(ctx, 1, index);
5735#else
5736 return 0;
5737#endif
5738}
5739
Jens Axboe05f3fb32019-12-09 11:22:50 -07005740static void io_atomic_switch(struct percpu_ref *ref)
Jens Axboec3a31e62019-10-03 13:59:56 -06005741{
Jens Axboe05f3fb32019-12-09 11:22:50 -07005742 struct fixed_file_data *data;
5743
5744 data = container_of(ref, struct fixed_file_data, refs);
5745 clear_bit(FFD_F_ATOMIC, &data->state);
5746}
5747
5748static bool io_queue_file_removal(struct fixed_file_data *data,
5749 struct file *file)
5750{
5751 struct io_file_put *pfile, pfile_stack;
5752 DECLARE_COMPLETION_ONSTACK(done);
5753
5754 /*
5755 * If we fail allocating the struct we need for doing async reomval
5756 * of this file, just punt to sync and wait for it.
5757 */
5758 pfile = kzalloc(sizeof(*pfile), GFP_KERNEL);
5759 if (!pfile) {
5760 pfile = &pfile_stack;
5761 pfile->done = &done;
5762 }
5763
5764 pfile->file = file;
5765 llist_add(&pfile->llist, &data->put_llist);
5766
5767 if (pfile == &pfile_stack) {
5768 if (!test_and_set_bit(FFD_F_ATOMIC, &data->state)) {
5769 percpu_ref_put(&data->refs);
5770 percpu_ref_switch_to_atomic(&data->refs,
5771 io_atomic_switch);
5772 }
5773 wait_for_completion(&done);
5774 flush_work(&data->ref_work);
5775 return false;
5776 }
5777
5778 return true;
5779}
5780
5781static int __io_sqe_files_update(struct io_ring_ctx *ctx,
5782 struct io_uring_files_update *up,
5783 unsigned nr_args)
5784{
5785 struct fixed_file_data *data = ctx->file_data;
5786 bool ref_switch = false;
5787 struct file *file;
Jens Axboec3a31e62019-10-03 13:59:56 -06005788 __s32 __user *fds;
5789 int fd, i, err;
5790 __u32 done;
5791
Jens Axboe05f3fb32019-12-09 11:22:50 -07005792 if (check_add_overflow(up->offset, nr_args, &done))
Jens Axboec3a31e62019-10-03 13:59:56 -06005793 return -EOVERFLOW;
5794 if (done > ctx->nr_user_files)
5795 return -EINVAL;
5796
5797 done = 0;
Jens Axboe05f3fb32019-12-09 11:22:50 -07005798 fds = u64_to_user_ptr(up->fds);
Jens Axboec3a31e62019-10-03 13:59:56 -06005799 while (nr_args) {
Jens Axboe65e19f52019-10-26 07:20:21 -06005800 struct fixed_file_table *table;
5801 unsigned index;
5802
Jens Axboec3a31e62019-10-03 13:59:56 -06005803 err = 0;
5804 if (copy_from_user(&fd, &fds[done], sizeof(fd))) {
5805 err = -EFAULT;
5806 break;
5807 }
Jens Axboe05f3fb32019-12-09 11:22:50 -07005808 i = array_index_nospec(up->offset, ctx->nr_user_files);
5809 table = &ctx->file_data->table[i >> IORING_FILE_TABLE_SHIFT];
Jens Axboe65e19f52019-10-26 07:20:21 -06005810 index = i & IORING_FILE_TABLE_MASK;
5811 if (table->files[index]) {
Jens Axboe05f3fb32019-12-09 11:22:50 -07005812 file = io_file_from_index(ctx, index);
Jens Axboe65e19f52019-10-26 07:20:21 -06005813 table->files[index] = NULL;
Jens Axboe05f3fb32019-12-09 11:22:50 -07005814 if (io_queue_file_removal(data, file))
5815 ref_switch = true;
Jens Axboec3a31e62019-10-03 13:59:56 -06005816 }
5817 if (fd != -1) {
Jens Axboec3a31e62019-10-03 13:59:56 -06005818 file = fget(fd);
5819 if (!file) {
5820 err = -EBADF;
5821 break;
5822 }
5823 /*
5824 * Don't allow io_uring instances to be registered. If
5825 * UNIX isn't enabled, then this causes a reference
5826 * cycle and this instance can never get freed. If UNIX
5827 * is enabled we'll handle it just fine, but there's
5828 * still no point in allowing a ring fd as it doesn't
5829 * support regular read/write anyway.
5830 */
5831 if (file->f_op == &io_uring_fops) {
5832 fput(file);
5833 err = -EBADF;
5834 break;
5835 }
Jens Axboe65e19f52019-10-26 07:20:21 -06005836 table->files[index] = file;
Jens Axboec3a31e62019-10-03 13:59:56 -06005837 err = io_sqe_file_register(ctx, file, i);
5838 if (err)
5839 break;
5840 }
5841 nr_args--;
5842 done++;
Jens Axboe05f3fb32019-12-09 11:22:50 -07005843 up->offset++;
5844 }
5845
5846 if (ref_switch && !test_and_set_bit(FFD_F_ATOMIC, &data->state)) {
5847 percpu_ref_put(&data->refs);
5848 percpu_ref_switch_to_atomic(&data->refs, io_atomic_switch);
Jens Axboec3a31e62019-10-03 13:59:56 -06005849 }
5850
5851 return done ? done : err;
5852}
Jens Axboe05f3fb32019-12-09 11:22:50 -07005853static int io_sqe_files_update(struct io_ring_ctx *ctx, void __user *arg,
5854 unsigned nr_args)
5855{
5856 struct io_uring_files_update up;
5857
5858 if (!ctx->file_data)
5859 return -ENXIO;
5860 if (!nr_args)
5861 return -EINVAL;
5862 if (copy_from_user(&up, arg, sizeof(up)))
5863 return -EFAULT;
5864 if (up.resv)
5865 return -EINVAL;
5866
5867 return __io_sqe_files_update(ctx, &up, nr_args);
5868}
Jens Axboec3a31e62019-10-03 13:59:56 -06005869
Jens Axboe7d723062019-11-12 22:31:31 -07005870static void io_put_work(struct io_wq_work *work)
5871{
5872 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
5873
5874 io_put_req(req);
5875}
5876
5877static void io_get_work(struct io_wq_work *work)
5878{
5879 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
5880
5881 refcount_inc(&req->refs);
5882}
5883
Pavel Begunkov24369c22020-01-28 03:15:48 +03005884static int io_init_wq_offload(struct io_ring_ctx *ctx,
5885 struct io_uring_params *p)
5886{
5887 struct io_wq_data data;
5888 struct fd f;
5889 struct io_ring_ctx *ctx_attach;
5890 unsigned int concurrency;
5891 int ret = 0;
5892
5893 data.user = ctx->user;
5894 data.get_work = io_get_work;
5895 data.put_work = io_put_work;
5896
5897 if (!(p->flags & IORING_SETUP_ATTACH_WQ)) {
5898 /* Do QD, or 4 * CPUS, whatever is smallest */
5899 concurrency = min(ctx->sq_entries, 4 * num_online_cpus());
5900
5901 ctx->io_wq = io_wq_create(concurrency, &data);
5902 if (IS_ERR(ctx->io_wq)) {
5903 ret = PTR_ERR(ctx->io_wq);
5904 ctx->io_wq = NULL;
5905 }
5906 return ret;
5907 }
5908
5909 f = fdget(p->wq_fd);
5910 if (!f.file)
5911 return -EBADF;
5912
5913 if (f.file->f_op != &io_uring_fops) {
5914 ret = -EINVAL;
5915 goto out_fput;
5916 }
5917
5918 ctx_attach = f.file->private_data;
5919 /* @io_wq is protected by holding the fd */
5920 if (!io_wq_get(ctx_attach->io_wq, &data)) {
5921 ret = -EINVAL;
5922 goto out_fput;
5923 }
5924
5925 ctx->io_wq = ctx_attach->io_wq;
5926out_fput:
5927 fdput(f);
5928 return ret;
5929}
5930
Jens Axboe6c271ce2019-01-10 11:22:30 -07005931static int io_sq_offload_start(struct io_ring_ctx *ctx,
5932 struct io_uring_params *p)
Jens Axboe2b188cc2019-01-07 10:46:33 -07005933{
5934 int ret;
5935
Jens Axboe6c271ce2019-01-10 11:22:30 -07005936 init_waitqueue_head(&ctx->sqo_wait);
Jens Axboe2b188cc2019-01-07 10:46:33 -07005937 mmgrab(current->mm);
5938 ctx->sqo_mm = current->mm;
5939
Jens Axboe6c271ce2019-01-10 11:22:30 -07005940 if (ctx->flags & IORING_SETUP_SQPOLL) {
Jens Axboe3ec482d2019-04-08 10:51:01 -06005941 ret = -EPERM;
5942 if (!capable(CAP_SYS_ADMIN))
5943 goto err;
5944
Jens Axboe917257d2019-04-13 09:28:55 -06005945 ctx->sq_thread_idle = msecs_to_jiffies(p->sq_thread_idle);
5946 if (!ctx->sq_thread_idle)
5947 ctx->sq_thread_idle = HZ;
5948
Jens Axboe6c271ce2019-01-10 11:22:30 -07005949 if (p->flags & IORING_SETUP_SQ_AFF) {
Jens Axboe44a9bd12019-05-14 20:00:30 -06005950 int cpu = p->sq_thread_cpu;
Jens Axboe6c271ce2019-01-10 11:22:30 -07005951
Jens Axboe917257d2019-04-13 09:28:55 -06005952 ret = -EINVAL;
Jens Axboe44a9bd12019-05-14 20:00:30 -06005953 if (cpu >= nr_cpu_ids)
5954 goto err;
Shenghui Wang7889f442019-05-07 16:03:19 +08005955 if (!cpu_online(cpu))
Jens Axboe917257d2019-04-13 09:28:55 -06005956 goto err;
5957
Jens Axboe6c271ce2019-01-10 11:22:30 -07005958 ctx->sqo_thread = kthread_create_on_cpu(io_sq_thread,
5959 ctx, cpu,
5960 "io_uring-sq");
5961 } else {
5962 ctx->sqo_thread = kthread_create(io_sq_thread, ctx,
5963 "io_uring-sq");
5964 }
5965 if (IS_ERR(ctx->sqo_thread)) {
5966 ret = PTR_ERR(ctx->sqo_thread);
5967 ctx->sqo_thread = NULL;
5968 goto err;
5969 }
5970 wake_up_process(ctx->sqo_thread);
5971 } else if (p->flags & IORING_SETUP_SQ_AFF) {
5972 /* Can't have SQ_AFF without SQPOLL */
5973 ret = -EINVAL;
5974 goto err;
5975 }
5976
Pavel Begunkov24369c22020-01-28 03:15:48 +03005977 ret = io_init_wq_offload(ctx, p);
5978 if (ret)
Jens Axboe2b188cc2019-01-07 10:46:33 -07005979 goto err;
Jens Axboe2b188cc2019-01-07 10:46:33 -07005980
5981 return 0;
5982err:
Jens Axboe54a91f32019-09-10 09:15:04 -06005983 io_finish_async(ctx);
Jens Axboe2b188cc2019-01-07 10:46:33 -07005984 mmdrop(ctx->sqo_mm);
5985 ctx->sqo_mm = NULL;
5986 return ret;
5987}
5988
5989static void io_unaccount_mem(struct user_struct *user, unsigned long nr_pages)
5990{
5991 atomic_long_sub(nr_pages, &user->locked_vm);
5992}
5993
5994static int io_account_mem(struct user_struct *user, unsigned long nr_pages)
5995{
5996 unsigned long page_limit, cur_pages, new_pages;
5997
5998 /* Don't allow more pages than we can safely lock */
5999 page_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
6000
6001 do {
6002 cur_pages = atomic_long_read(&user->locked_vm);
6003 new_pages = cur_pages + nr_pages;
6004 if (new_pages > page_limit)
6005 return -ENOMEM;
6006 } while (atomic_long_cmpxchg(&user->locked_vm, cur_pages,
6007 new_pages) != cur_pages);
6008
6009 return 0;
6010}
6011
6012static void io_mem_free(void *ptr)
6013{
Mark Rutland52e04ef2019-04-30 17:30:21 +01006014 struct page *page;
Jens Axboe2b188cc2019-01-07 10:46:33 -07006015
Mark Rutland52e04ef2019-04-30 17:30:21 +01006016 if (!ptr)
6017 return;
6018
6019 page = virt_to_head_page(ptr);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006020 if (put_page_testzero(page))
6021 free_compound_page(page);
6022}
6023
6024static void *io_mem_alloc(size_t size)
6025{
6026 gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN | __GFP_COMP |
6027 __GFP_NORETRY;
6028
6029 return (void *) __get_free_pages(gfp_flags, get_order(size));
6030}
6031
Hristo Venev75b28af2019-08-26 17:23:46 +00006032static unsigned long rings_size(unsigned sq_entries, unsigned cq_entries,
6033 size_t *sq_offset)
6034{
6035 struct io_rings *rings;
6036 size_t off, sq_array_size;
6037
6038 off = struct_size(rings, cqes, cq_entries);
6039 if (off == SIZE_MAX)
6040 return SIZE_MAX;
6041
6042#ifdef CONFIG_SMP
6043 off = ALIGN(off, SMP_CACHE_BYTES);
6044 if (off == 0)
6045 return SIZE_MAX;
6046#endif
6047
6048 sq_array_size = array_size(sizeof(u32), sq_entries);
6049 if (sq_array_size == SIZE_MAX)
6050 return SIZE_MAX;
6051
6052 if (check_add_overflow(off, sq_array_size, &off))
6053 return SIZE_MAX;
6054
6055 if (sq_offset)
6056 *sq_offset = off;
6057
6058 return off;
6059}
6060
Jens Axboe2b188cc2019-01-07 10:46:33 -07006061static unsigned long ring_pages(unsigned sq_entries, unsigned cq_entries)
6062{
Hristo Venev75b28af2019-08-26 17:23:46 +00006063 size_t pages;
Jens Axboe2b188cc2019-01-07 10:46:33 -07006064
Hristo Venev75b28af2019-08-26 17:23:46 +00006065 pages = (size_t)1 << get_order(
6066 rings_size(sq_entries, cq_entries, NULL));
6067 pages += (size_t)1 << get_order(
6068 array_size(sizeof(struct io_uring_sqe), sq_entries));
Jens Axboe2b188cc2019-01-07 10:46:33 -07006069
Hristo Venev75b28af2019-08-26 17:23:46 +00006070 return pages;
Jens Axboe2b188cc2019-01-07 10:46:33 -07006071}
6072
Jens Axboeedafcce2019-01-09 09:16:05 -07006073static int io_sqe_buffer_unregister(struct io_ring_ctx *ctx)
6074{
6075 int i, j;
6076
6077 if (!ctx->user_bufs)
6078 return -ENXIO;
6079
6080 for (i = 0; i < ctx->nr_user_bufs; i++) {
6081 struct io_mapped_ubuf *imu = &ctx->user_bufs[i];
6082
6083 for (j = 0; j < imu->nr_bvecs; j++)
John Hubbardf1f6a7d2020-01-30 22:13:35 -08006084 unpin_user_page(imu->bvec[j].bv_page);
Jens Axboeedafcce2019-01-09 09:16:05 -07006085
6086 if (ctx->account_mem)
6087 io_unaccount_mem(ctx->user, imu->nr_bvecs);
Mark Rutlandd4ef6472019-05-01 16:59:16 +01006088 kvfree(imu->bvec);
Jens Axboeedafcce2019-01-09 09:16:05 -07006089 imu->nr_bvecs = 0;
6090 }
6091
6092 kfree(ctx->user_bufs);
6093 ctx->user_bufs = NULL;
6094 ctx->nr_user_bufs = 0;
6095 return 0;
6096}
6097
6098static int io_copy_iov(struct io_ring_ctx *ctx, struct iovec *dst,
6099 void __user *arg, unsigned index)
6100{
6101 struct iovec __user *src;
6102
6103#ifdef CONFIG_COMPAT
6104 if (ctx->compat) {
6105 struct compat_iovec __user *ciovs;
6106 struct compat_iovec ciov;
6107
6108 ciovs = (struct compat_iovec __user *) arg;
6109 if (copy_from_user(&ciov, &ciovs[index], sizeof(ciov)))
6110 return -EFAULT;
6111
Jens Axboed55e5f52019-12-11 16:12:15 -07006112 dst->iov_base = u64_to_user_ptr((u64)ciov.iov_base);
Jens Axboeedafcce2019-01-09 09:16:05 -07006113 dst->iov_len = ciov.iov_len;
6114 return 0;
6115 }
6116#endif
6117 src = (struct iovec __user *) arg;
6118 if (copy_from_user(dst, &src[index], sizeof(*dst)))
6119 return -EFAULT;
6120 return 0;
6121}
6122
6123static int io_sqe_buffer_register(struct io_ring_ctx *ctx, void __user *arg,
6124 unsigned nr_args)
6125{
6126 struct vm_area_struct **vmas = NULL;
6127 struct page **pages = NULL;
6128 int i, j, got_pages = 0;
6129 int ret = -EINVAL;
6130
6131 if (ctx->user_bufs)
6132 return -EBUSY;
6133 if (!nr_args || nr_args > UIO_MAXIOV)
6134 return -EINVAL;
6135
6136 ctx->user_bufs = kcalloc(nr_args, sizeof(struct io_mapped_ubuf),
6137 GFP_KERNEL);
6138 if (!ctx->user_bufs)
6139 return -ENOMEM;
6140
6141 for (i = 0; i < nr_args; i++) {
6142 struct io_mapped_ubuf *imu = &ctx->user_bufs[i];
6143 unsigned long off, start, end, ubuf;
6144 int pret, nr_pages;
6145 struct iovec iov;
6146 size_t size;
6147
6148 ret = io_copy_iov(ctx, &iov, arg, i);
6149 if (ret)
Pavel Begunkova2786822019-05-26 12:35:47 +03006150 goto err;
Jens Axboeedafcce2019-01-09 09:16:05 -07006151
6152 /*
6153 * Don't impose further limits on the size and buffer
6154 * constraints here, we'll -EINVAL later when IO is
6155 * submitted if they are wrong.
6156 */
6157 ret = -EFAULT;
6158 if (!iov.iov_base || !iov.iov_len)
6159 goto err;
6160
6161 /* arbitrary limit, but we need something */
6162 if (iov.iov_len > SZ_1G)
6163 goto err;
6164
6165 ubuf = (unsigned long) iov.iov_base;
6166 end = (ubuf + iov.iov_len + PAGE_SIZE - 1) >> PAGE_SHIFT;
6167 start = ubuf >> PAGE_SHIFT;
6168 nr_pages = end - start;
6169
6170 if (ctx->account_mem) {
6171 ret = io_account_mem(ctx->user, nr_pages);
6172 if (ret)
6173 goto err;
6174 }
6175
6176 ret = 0;
6177 if (!pages || nr_pages > got_pages) {
6178 kfree(vmas);
6179 kfree(pages);
Mark Rutlandd4ef6472019-05-01 16:59:16 +01006180 pages = kvmalloc_array(nr_pages, sizeof(struct page *),
Jens Axboeedafcce2019-01-09 09:16:05 -07006181 GFP_KERNEL);
Mark Rutlandd4ef6472019-05-01 16:59:16 +01006182 vmas = kvmalloc_array(nr_pages,
Jens Axboeedafcce2019-01-09 09:16:05 -07006183 sizeof(struct vm_area_struct *),
6184 GFP_KERNEL);
6185 if (!pages || !vmas) {
6186 ret = -ENOMEM;
6187 if (ctx->account_mem)
6188 io_unaccount_mem(ctx->user, nr_pages);
6189 goto err;
6190 }
6191 got_pages = nr_pages;
6192 }
6193
Mark Rutlandd4ef6472019-05-01 16:59:16 +01006194 imu->bvec = kvmalloc_array(nr_pages, sizeof(struct bio_vec),
Jens Axboeedafcce2019-01-09 09:16:05 -07006195 GFP_KERNEL);
6196 ret = -ENOMEM;
6197 if (!imu->bvec) {
6198 if (ctx->account_mem)
6199 io_unaccount_mem(ctx->user, nr_pages);
6200 goto err;
6201 }
6202
6203 ret = 0;
6204 down_read(&current->mm->mmap_sem);
John Hubbard2113b052020-01-30 22:13:13 -08006205 pret = pin_user_pages(ubuf, nr_pages,
Ira Weiny932f4a62019-05-13 17:17:03 -07006206 FOLL_WRITE | FOLL_LONGTERM,
6207 pages, vmas);
Jens Axboeedafcce2019-01-09 09:16:05 -07006208 if (pret == nr_pages) {
6209 /* don't support file backed memory */
6210 for (j = 0; j < nr_pages; j++) {
6211 struct vm_area_struct *vma = vmas[j];
6212
6213 if (vma->vm_file &&
6214 !is_file_hugepages(vma->vm_file)) {
6215 ret = -EOPNOTSUPP;
6216 break;
6217 }
6218 }
6219 } else {
6220 ret = pret < 0 ? pret : -EFAULT;
6221 }
6222 up_read(&current->mm->mmap_sem);
6223 if (ret) {
6224 /*
6225 * if we did partial map, or found file backed vmas,
6226 * release any pages we did get
6227 */
John Hubbard27c4d3a2019-08-04 19:32:06 -07006228 if (pret > 0)
John Hubbardf1f6a7d2020-01-30 22:13:35 -08006229 unpin_user_pages(pages, pret);
Jens Axboeedafcce2019-01-09 09:16:05 -07006230 if (ctx->account_mem)
6231 io_unaccount_mem(ctx->user, nr_pages);
Mark Rutlandd4ef6472019-05-01 16:59:16 +01006232 kvfree(imu->bvec);
Jens Axboeedafcce2019-01-09 09:16:05 -07006233 goto err;
6234 }
6235
6236 off = ubuf & ~PAGE_MASK;
6237 size = iov.iov_len;
6238 for (j = 0; j < nr_pages; j++) {
6239 size_t vec_len;
6240
6241 vec_len = min_t(size_t, size, PAGE_SIZE - off);
6242 imu->bvec[j].bv_page = pages[j];
6243 imu->bvec[j].bv_len = vec_len;
6244 imu->bvec[j].bv_offset = off;
6245 off = 0;
6246 size -= vec_len;
6247 }
6248 /* store original address for later verification */
6249 imu->ubuf = ubuf;
6250 imu->len = iov.iov_len;
6251 imu->nr_bvecs = nr_pages;
6252
6253 ctx->nr_user_bufs++;
6254 }
Mark Rutlandd4ef6472019-05-01 16:59:16 +01006255 kvfree(pages);
6256 kvfree(vmas);
Jens Axboeedafcce2019-01-09 09:16:05 -07006257 return 0;
6258err:
Mark Rutlandd4ef6472019-05-01 16:59:16 +01006259 kvfree(pages);
6260 kvfree(vmas);
Jens Axboeedafcce2019-01-09 09:16:05 -07006261 io_sqe_buffer_unregister(ctx);
6262 return ret;
6263}
6264
Jens Axboe9b402842019-04-11 11:45:41 -06006265static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg)
6266{
6267 __s32 __user *fds = arg;
6268 int fd;
6269
6270 if (ctx->cq_ev_fd)
6271 return -EBUSY;
6272
6273 if (copy_from_user(&fd, fds, sizeof(*fds)))
6274 return -EFAULT;
6275
6276 ctx->cq_ev_fd = eventfd_ctx_fdget(fd);
6277 if (IS_ERR(ctx->cq_ev_fd)) {
6278 int ret = PTR_ERR(ctx->cq_ev_fd);
6279 ctx->cq_ev_fd = NULL;
6280 return ret;
6281 }
6282
6283 return 0;
6284}
6285
6286static int io_eventfd_unregister(struct io_ring_ctx *ctx)
6287{
6288 if (ctx->cq_ev_fd) {
6289 eventfd_ctx_put(ctx->cq_ev_fd);
6290 ctx->cq_ev_fd = NULL;
6291 return 0;
6292 }
6293
6294 return -ENXIO;
6295}
6296
Jens Axboe2b188cc2019-01-07 10:46:33 -07006297static void io_ring_ctx_free(struct io_ring_ctx *ctx)
6298{
Jens Axboe6b063142019-01-10 22:13:58 -07006299 io_finish_async(ctx);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006300 if (ctx->sqo_mm)
6301 mmdrop(ctx->sqo_mm);
Jens Axboedef596e2019-01-09 08:59:42 -07006302
6303 io_iopoll_reap_events(ctx);
Jens Axboeedafcce2019-01-09 09:16:05 -07006304 io_sqe_buffer_unregister(ctx);
Jens Axboe6b063142019-01-10 22:13:58 -07006305 io_sqe_files_unregister(ctx);
Jens Axboe9b402842019-04-11 11:45:41 -06006306 io_eventfd_unregister(ctx);
Jens Axboedef596e2019-01-09 08:59:42 -07006307
Jens Axboe2b188cc2019-01-07 10:46:33 -07006308#if defined(CONFIG_UNIX)
Eric Biggers355e8d22019-06-12 14:58:43 -07006309 if (ctx->ring_sock) {
6310 ctx->ring_sock->file = NULL; /* so that iput() is called */
Jens Axboe2b188cc2019-01-07 10:46:33 -07006311 sock_release(ctx->ring_sock);
Eric Biggers355e8d22019-06-12 14:58:43 -07006312 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07006313#endif
6314
Hristo Venev75b28af2019-08-26 17:23:46 +00006315 io_mem_free(ctx->rings);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006316 io_mem_free(ctx->sq_sqes);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006317
6318 percpu_ref_exit(&ctx->refs);
6319 if (ctx->account_mem)
6320 io_unaccount_mem(ctx->user,
6321 ring_pages(ctx->sq_entries, ctx->cq_entries));
6322 free_uid(ctx->user);
Jens Axboe181e4482019-11-25 08:52:30 -07006323 put_cred(ctx->creds);
Jens Axboe206aefd2019-11-07 18:27:42 -07006324 kfree(ctx->completions);
Jens Axboe78076bb2019-12-04 19:56:40 -07006325 kfree(ctx->cancel_hash);
Jens Axboe0ddf92e2019-11-08 08:52:53 -07006326 kmem_cache_free(req_cachep, ctx->fallback_req);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006327 kfree(ctx);
6328}
6329
6330static __poll_t io_uring_poll(struct file *file, poll_table *wait)
6331{
6332 struct io_ring_ctx *ctx = file->private_data;
6333 __poll_t mask = 0;
6334
6335 poll_wait(file, &ctx->cq_wait, wait);
Stefan Bühler4f7067c2019-04-24 23:54:17 +02006336 /*
6337 * synchronizes with barrier from wq_has_sleeper call in
6338 * io_commit_cqring
6339 */
Jens Axboe2b188cc2019-01-07 10:46:33 -07006340 smp_rmb();
Hristo Venev75b28af2019-08-26 17:23:46 +00006341 if (READ_ONCE(ctx->rings->sq.tail) - ctx->cached_sq_head !=
6342 ctx->rings->sq_ring_entries)
Jens Axboe2b188cc2019-01-07 10:46:33 -07006343 mask |= EPOLLOUT | EPOLLWRNORM;
Stefano Garzarella63e5d812020-02-07 13:18:28 +01006344 if (io_cqring_events(ctx, false))
Jens Axboe2b188cc2019-01-07 10:46:33 -07006345 mask |= EPOLLIN | EPOLLRDNORM;
6346
6347 return mask;
6348}
6349
6350static int io_uring_fasync(int fd, struct file *file, int on)
6351{
6352 struct io_ring_ctx *ctx = file->private_data;
6353
6354 return fasync_helper(fd, file, on, &ctx->cq_fasync);
6355}
6356
Jens Axboe071698e2020-01-28 10:04:42 -07006357static int io_remove_personalities(int id, void *p, void *data)
6358{
6359 struct io_ring_ctx *ctx = data;
6360 const struct cred *cred;
6361
6362 cred = idr_remove(&ctx->personality_idr, id);
6363 if (cred)
6364 put_cred(cred);
6365 return 0;
6366}
6367
Jens Axboe2b188cc2019-01-07 10:46:33 -07006368static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx)
6369{
6370 mutex_lock(&ctx->uring_lock);
6371 percpu_ref_kill(&ctx->refs);
6372 mutex_unlock(&ctx->uring_lock);
6373
Jens Axboedf069d82020-02-04 16:48:34 -07006374 /*
6375 * Wait for sq thread to idle, if we have one. It won't spin on new
6376 * work after we've killed the ctx ref above. This is important to do
6377 * before we cancel existing commands, as the thread could otherwise
6378 * be queueing new work post that. If that's work we need to cancel,
6379 * it could cause shutdown to hang.
6380 */
6381 while (ctx->sqo_thread && !wq_has_sleeper(&ctx->sqo_wait))
6382 cpu_relax();
6383
Jens Axboe5262f562019-09-17 12:26:57 -06006384 io_kill_timeouts(ctx);
Jens Axboe221c5eb2019-01-17 09:41:58 -07006385 io_poll_remove_all(ctx);
Jens Axboe561fb042019-10-24 07:25:42 -06006386
6387 if (ctx->io_wq)
6388 io_wq_cancel_all(ctx->io_wq);
6389
Jens Axboedef596e2019-01-09 08:59:42 -07006390 io_iopoll_reap_events(ctx);
Jens Axboe15dff282019-11-13 09:09:23 -07006391 /* if we failed setting up the ctx, we might not have any rings */
6392 if (ctx->rings)
6393 io_cqring_overflow_flush(ctx, true);
Jens Axboe071698e2020-01-28 10:04:42 -07006394 idr_for_each(&ctx->personality_idr, io_remove_personalities, ctx);
Jens Axboe206aefd2019-11-07 18:27:42 -07006395 wait_for_completion(&ctx->completions[0]);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006396 io_ring_ctx_free(ctx);
6397}
6398
6399static int io_uring_release(struct inode *inode, struct file *file)
6400{
6401 struct io_ring_ctx *ctx = file->private_data;
6402
6403 file->private_data = NULL;
6404 io_ring_ctx_wait_and_kill(ctx);
6405 return 0;
6406}
6407
Jens Axboefcb323c2019-10-24 12:39:47 -06006408static void io_uring_cancel_files(struct io_ring_ctx *ctx,
6409 struct files_struct *files)
6410{
6411 struct io_kiocb *req;
6412 DEFINE_WAIT(wait);
6413
6414 while (!list_empty_careful(&ctx->inflight_list)) {
Jens Axboe768134d2019-11-10 20:30:53 -07006415 struct io_kiocb *cancel_req = NULL;
Jens Axboefcb323c2019-10-24 12:39:47 -06006416
6417 spin_lock_irq(&ctx->inflight_lock);
6418 list_for_each_entry(req, &ctx->inflight_list, inflight_entry) {
Jens Axboe768134d2019-11-10 20:30:53 -07006419 if (req->work.files != files)
6420 continue;
6421 /* req is being completed, ignore */
6422 if (!refcount_inc_not_zero(&req->refs))
6423 continue;
6424 cancel_req = req;
6425 break;
Jens Axboefcb323c2019-10-24 12:39:47 -06006426 }
Jens Axboe768134d2019-11-10 20:30:53 -07006427 if (cancel_req)
Jens Axboefcb323c2019-10-24 12:39:47 -06006428 prepare_to_wait(&ctx->inflight_wait, &wait,
Jens Axboe768134d2019-11-10 20:30:53 -07006429 TASK_UNINTERRUPTIBLE);
Jens Axboefcb323c2019-10-24 12:39:47 -06006430 spin_unlock_irq(&ctx->inflight_lock);
6431
Jens Axboe768134d2019-11-10 20:30:53 -07006432 /* We need to keep going until we don't find a matching req */
6433 if (!cancel_req)
Jens Axboefcb323c2019-10-24 12:39:47 -06006434 break;
Bob Liu2f6d9b92019-11-13 18:06:24 +08006435
6436 io_wq_cancel_work(ctx->io_wq, &cancel_req->work);
6437 io_put_req(cancel_req);
Jens Axboefcb323c2019-10-24 12:39:47 -06006438 schedule();
6439 }
Jens Axboe768134d2019-11-10 20:30:53 -07006440 finish_wait(&ctx->inflight_wait, &wait);
Jens Axboefcb323c2019-10-24 12:39:47 -06006441}
6442
6443static int io_uring_flush(struct file *file, void *data)
6444{
6445 struct io_ring_ctx *ctx = file->private_data;
6446
6447 io_uring_cancel_files(ctx, data);
Jens Axboefcb323c2019-10-24 12:39:47 -06006448 return 0;
6449}
6450
Roman Penyaev6c5c2402019-11-28 12:53:22 +01006451static void *io_uring_validate_mmap_request(struct file *file,
6452 loff_t pgoff, size_t sz)
Jens Axboe2b188cc2019-01-07 10:46:33 -07006453{
Jens Axboe2b188cc2019-01-07 10:46:33 -07006454 struct io_ring_ctx *ctx = file->private_data;
Roman Penyaev6c5c2402019-11-28 12:53:22 +01006455 loff_t offset = pgoff << PAGE_SHIFT;
Jens Axboe2b188cc2019-01-07 10:46:33 -07006456 struct page *page;
6457 void *ptr;
6458
6459 switch (offset) {
6460 case IORING_OFF_SQ_RING:
Hristo Venev75b28af2019-08-26 17:23:46 +00006461 case IORING_OFF_CQ_RING:
6462 ptr = ctx->rings;
Jens Axboe2b188cc2019-01-07 10:46:33 -07006463 break;
6464 case IORING_OFF_SQES:
6465 ptr = ctx->sq_sqes;
6466 break;
Jens Axboe2b188cc2019-01-07 10:46:33 -07006467 default:
Roman Penyaev6c5c2402019-11-28 12:53:22 +01006468 return ERR_PTR(-EINVAL);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006469 }
6470
6471 page = virt_to_head_page(ptr);
Matthew Wilcox (Oracle)a50b8542019-09-23 15:34:25 -07006472 if (sz > page_size(page))
Roman Penyaev6c5c2402019-11-28 12:53:22 +01006473 return ERR_PTR(-EINVAL);
6474
6475 return ptr;
6476}
6477
6478#ifdef CONFIG_MMU
6479
6480static int io_uring_mmap(struct file *file, struct vm_area_struct *vma)
6481{
6482 size_t sz = vma->vm_end - vma->vm_start;
6483 unsigned long pfn;
6484 void *ptr;
6485
6486 ptr = io_uring_validate_mmap_request(file, vma->vm_pgoff, sz);
6487 if (IS_ERR(ptr))
6488 return PTR_ERR(ptr);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006489
6490 pfn = virt_to_phys(ptr) >> PAGE_SHIFT;
6491 return remap_pfn_range(vma, vma->vm_start, pfn, sz, vma->vm_page_prot);
6492}
6493
Roman Penyaev6c5c2402019-11-28 12:53:22 +01006494#else /* !CONFIG_MMU */
6495
6496static int io_uring_mmap(struct file *file, struct vm_area_struct *vma)
6497{
6498 return vma->vm_flags & (VM_SHARED | VM_MAYSHARE) ? 0 : -EINVAL;
6499}
6500
6501static unsigned int io_uring_nommu_mmap_capabilities(struct file *file)
6502{
6503 return NOMMU_MAP_DIRECT | NOMMU_MAP_READ | NOMMU_MAP_WRITE;
6504}
6505
6506static unsigned long io_uring_nommu_get_unmapped_area(struct file *file,
6507 unsigned long addr, unsigned long len,
6508 unsigned long pgoff, unsigned long flags)
6509{
6510 void *ptr;
6511
6512 ptr = io_uring_validate_mmap_request(file, pgoff, len);
6513 if (IS_ERR(ptr))
6514 return PTR_ERR(ptr);
6515
6516 return (unsigned long) ptr;
6517}
6518
6519#endif /* !CONFIG_MMU */
6520
Jens Axboe2b188cc2019-01-07 10:46:33 -07006521SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
6522 u32, min_complete, u32, flags, const sigset_t __user *, sig,
6523 size_t, sigsz)
6524{
6525 struct io_ring_ctx *ctx;
6526 long ret = -EBADF;
6527 int submitted = 0;
6528 struct fd f;
6529
Jens Axboe6c271ce2019-01-10 11:22:30 -07006530 if (flags & ~(IORING_ENTER_GETEVENTS | IORING_ENTER_SQ_WAKEUP))
Jens Axboe2b188cc2019-01-07 10:46:33 -07006531 return -EINVAL;
6532
6533 f = fdget(fd);
6534 if (!f.file)
6535 return -EBADF;
6536
6537 ret = -EOPNOTSUPP;
6538 if (f.file->f_op != &io_uring_fops)
6539 goto out_fput;
6540
6541 ret = -ENXIO;
6542 ctx = f.file->private_data;
6543 if (!percpu_ref_tryget(&ctx->refs))
6544 goto out_fput;
6545
Jens Axboe6c271ce2019-01-10 11:22:30 -07006546 /*
6547 * For SQ polling, the thread will do all submissions and completions.
6548 * Just return the requested submit count, and wake the thread if
6549 * we were asked to.
6550 */
Jens Axboeb2a9ead2019-09-12 14:19:16 -06006551 ret = 0;
Jens Axboe6c271ce2019-01-10 11:22:30 -07006552 if (ctx->flags & IORING_SETUP_SQPOLL) {
Jens Axboec1edbf52019-11-10 16:56:04 -07006553 if (!list_empty_careful(&ctx->cq_overflow_list))
6554 io_cqring_overflow_flush(ctx, false);
Jens Axboe6c271ce2019-01-10 11:22:30 -07006555 if (flags & IORING_ENTER_SQ_WAKEUP)
6556 wake_up(&ctx->sqo_wait);
6557 submitted = to_submit;
Jens Axboeb2a9ead2019-09-12 14:19:16 -06006558 } else if (to_submit) {
Pavel Begunkovae9428c2019-11-06 00:22:14 +03006559 struct mm_struct *cur_mm;
Jens Axboe2b188cc2019-01-07 10:46:33 -07006560
6561 mutex_lock(&ctx->uring_lock);
Pavel Begunkovae9428c2019-11-06 00:22:14 +03006562 /* already have mm, so io_submit_sqes() won't try to grab it */
6563 cur_mm = ctx->sqo_mm;
6564 submitted = io_submit_sqes(ctx, to_submit, f.file, fd,
6565 &cur_mm, false);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006566 mutex_unlock(&ctx->uring_lock);
Pavel Begunkov7c504e652019-12-18 19:53:45 +03006567
6568 if (submitted != to_submit)
6569 goto out;
Jens Axboe2b188cc2019-01-07 10:46:33 -07006570 }
6571 if (flags & IORING_ENTER_GETEVENTS) {
Jens Axboedef596e2019-01-09 08:59:42 -07006572 unsigned nr_events = 0;
6573
Jens Axboe2b188cc2019-01-07 10:46:33 -07006574 min_complete = min(min_complete, ctx->cq_entries);
6575
Jens Axboedef596e2019-01-09 08:59:42 -07006576 if (ctx->flags & IORING_SETUP_IOPOLL) {
Jens Axboedef596e2019-01-09 08:59:42 -07006577 ret = io_iopoll_check(ctx, &nr_events, min_complete);
Jens Axboedef596e2019-01-09 08:59:42 -07006578 } else {
6579 ret = io_cqring_wait(ctx, min_complete, sig, sigsz);
6580 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07006581 }
6582
Pavel Begunkov7c504e652019-12-18 19:53:45 +03006583out:
Pavel Begunkov6805b322019-10-08 02:18:42 +03006584 percpu_ref_put(&ctx->refs);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006585out_fput:
6586 fdput(f);
6587 return submitted ? submitted : ret;
6588}
6589
Jens Axboe87ce9552020-01-30 08:25:34 -07006590static int io_uring_show_cred(int id, void *p, void *data)
6591{
6592 const struct cred *cred = p;
6593 struct seq_file *m = data;
6594 struct user_namespace *uns = seq_user_ns(m);
6595 struct group_info *gi;
6596 kernel_cap_t cap;
6597 unsigned __capi;
6598 int g;
6599
6600 seq_printf(m, "%5d\n", id);
6601 seq_put_decimal_ull(m, "\tUid:\t", from_kuid_munged(uns, cred->uid));
6602 seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->euid));
6603 seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->suid));
6604 seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->fsuid));
6605 seq_put_decimal_ull(m, "\n\tGid:\t", from_kgid_munged(uns, cred->gid));
6606 seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->egid));
6607 seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->sgid));
6608 seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->fsgid));
6609 seq_puts(m, "\n\tGroups:\t");
6610 gi = cred->group_info;
6611 for (g = 0; g < gi->ngroups; g++) {
6612 seq_put_decimal_ull(m, g ? " " : "",
6613 from_kgid_munged(uns, gi->gid[g]));
6614 }
6615 seq_puts(m, "\n\tCapEff:\t");
6616 cap = cred->cap_effective;
6617 CAP_FOR_EACH_U32(__capi)
6618 seq_put_hex_ll(m, NULL, cap.cap[CAP_LAST_U32 - __capi], 8);
6619 seq_putc(m, '\n');
6620 return 0;
6621}
6622
6623static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m)
6624{
6625 int i;
6626
6627 mutex_lock(&ctx->uring_lock);
6628 seq_printf(m, "UserFiles:\t%u\n", ctx->nr_user_files);
6629 for (i = 0; i < ctx->nr_user_files; i++) {
6630 struct fixed_file_table *table;
6631 struct file *f;
6632
6633 table = &ctx->file_data->table[i >> IORING_FILE_TABLE_SHIFT];
6634 f = table->files[i & IORING_FILE_TABLE_MASK];
6635 if (f)
6636 seq_printf(m, "%5u: %s\n", i, file_dentry(f)->d_iname);
6637 else
6638 seq_printf(m, "%5u: <none>\n", i);
6639 }
6640 seq_printf(m, "UserBufs:\t%u\n", ctx->nr_user_bufs);
6641 for (i = 0; i < ctx->nr_user_bufs; i++) {
6642 struct io_mapped_ubuf *buf = &ctx->user_bufs[i];
6643
6644 seq_printf(m, "%5u: 0x%llx/%u\n", i, buf->ubuf,
6645 (unsigned int) buf->len);
6646 }
6647 if (!idr_is_empty(&ctx->personality_idr)) {
6648 seq_printf(m, "Personalities:\n");
6649 idr_for_each(&ctx->personality_idr, io_uring_show_cred, m);
6650 }
6651 mutex_unlock(&ctx->uring_lock);
6652}
6653
6654static void io_uring_show_fdinfo(struct seq_file *m, struct file *f)
6655{
6656 struct io_ring_ctx *ctx = f->private_data;
6657
6658 if (percpu_ref_tryget(&ctx->refs)) {
6659 __io_uring_show_fdinfo(ctx, m);
6660 percpu_ref_put(&ctx->refs);
6661 }
6662}
6663
Jens Axboe2b188cc2019-01-07 10:46:33 -07006664static const struct file_operations io_uring_fops = {
6665 .release = io_uring_release,
Jens Axboefcb323c2019-10-24 12:39:47 -06006666 .flush = io_uring_flush,
Jens Axboe2b188cc2019-01-07 10:46:33 -07006667 .mmap = io_uring_mmap,
Roman Penyaev6c5c2402019-11-28 12:53:22 +01006668#ifndef CONFIG_MMU
6669 .get_unmapped_area = io_uring_nommu_get_unmapped_area,
6670 .mmap_capabilities = io_uring_nommu_mmap_capabilities,
6671#endif
Jens Axboe2b188cc2019-01-07 10:46:33 -07006672 .poll = io_uring_poll,
6673 .fasync = io_uring_fasync,
Jens Axboe87ce9552020-01-30 08:25:34 -07006674 .show_fdinfo = io_uring_show_fdinfo,
Jens Axboe2b188cc2019-01-07 10:46:33 -07006675};
6676
6677static int io_allocate_scq_urings(struct io_ring_ctx *ctx,
6678 struct io_uring_params *p)
6679{
Hristo Venev75b28af2019-08-26 17:23:46 +00006680 struct io_rings *rings;
6681 size_t size, sq_array_offset;
Jens Axboe2b188cc2019-01-07 10:46:33 -07006682
Hristo Venev75b28af2019-08-26 17:23:46 +00006683 size = rings_size(p->sq_entries, p->cq_entries, &sq_array_offset);
6684 if (size == SIZE_MAX)
6685 return -EOVERFLOW;
6686
6687 rings = io_mem_alloc(size);
6688 if (!rings)
Jens Axboe2b188cc2019-01-07 10:46:33 -07006689 return -ENOMEM;
6690
Hristo Venev75b28af2019-08-26 17:23:46 +00006691 ctx->rings = rings;
6692 ctx->sq_array = (u32 *)((char *)rings + sq_array_offset);
6693 rings->sq_ring_mask = p->sq_entries - 1;
6694 rings->cq_ring_mask = p->cq_entries - 1;
6695 rings->sq_ring_entries = p->sq_entries;
6696 rings->cq_ring_entries = p->cq_entries;
6697 ctx->sq_mask = rings->sq_ring_mask;
6698 ctx->cq_mask = rings->cq_ring_mask;
6699 ctx->sq_entries = rings->sq_ring_entries;
6700 ctx->cq_entries = rings->cq_ring_entries;
Jens Axboe2b188cc2019-01-07 10:46:33 -07006701
6702 size = array_size(sizeof(struct io_uring_sqe), p->sq_entries);
Jens Axboeeb065d32019-11-20 09:26:29 -07006703 if (size == SIZE_MAX) {
6704 io_mem_free(ctx->rings);
6705 ctx->rings = NULL;
Jens Axboe2b188cc2019-01-07 10:46:33 -07006706 return -EOVERFLOW;
Jens Axboeeb065d32019-11-20 09:26:29 -07006707 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07006708
6709 ctx->sq_sqes = io_mem_alloc(size);
Jens Axboeeb065d32019-11-20 09:26:29 -07006710 if (!ctx->sq_sqes) {
6711 io_mem_free(ctx->rings);
6712 ctx->rings = NULL;
Jens Axboe2b188cc2019-01-07 10:46:33 -07006713 return -ENOMEM;
Jens Axboeeb065d32019-11-20 09:26:29 -07006714 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07006715
Jens Axboe2b188cc2019-01-07 10:46:33 -07006716 return 0;
6717}
6718
6719/*
6720 * Allocate an anonymous fd, this is what constitutes the application
6721 * visible backing of an io_uring instance. The application mmaps this
6722 * fd to gain access to the SQ/CQ ring details. If UNIX sockets are enabled,
6723 * we have to tie this fd to a socket for file garbage collection purposes.
6724 */
6725static int io_uring_get_fd(struct io_ring_ctx *ctx)
6726{
6727 struct file *file;
6728 int ret;
6729
6730#if defined(CONFIG_UNIX)
6731 ret = sock_create_kern(&init_net, PF_UNIX, SOCK_RAW, IPPROTO_IP,
6732 &ctx->ring_sock);
6733 if (ret)
6734 return ret;
6735#endif
6736
6737 ret = get_unused_fd_flags(O_RDWR | O_CLOEXEC);
6738 if (ret < 0)
6739 goto err;
6740
6741 file = anon_inode_getfile("[io_uring]", &io_uring_fops, ctx,
6742 O_RDWR | O_CLOEXEC);
6743 if (IS_ERR(file)) {
6744 put_unused_fd(ret);
6745 ret = PTR_ERR(file);
6746 goto err;
6747 }
6748
6749#if defined(CONFIG_UNIX)
6750 ctx->ring_sock->file = file;
6751#endif
6752 fd_install(ret, file);
6753 return ret;
6754err:
6755#if defined(CONFIG_UNIX)
6756 sock_release(ctx->ring_sock);
6757 ctx->ring_sock = NULL;
6758#endif
6759 return ret;
6760}
6761
6762static int io_uring_create(unsigned entries, struct io_uring_params *p)
6763{
6764 struct user_struct *user = NULL;
6765 struct io_ring_ctx *ctx;
6766 bool account_mem;
6767 int ret;
6768
Jens Axboe8110c1a2019-12-28 15:39:54 -07006769 if (!entries)
Jens Axboe2b188cc2019-01-07 10:46:33 -07006770 return -EINVAL;
Jens Axboe8110c1a2019-12-28 15:39:54 -07006771 if (entries > IORING_MAX_ENTRIES) {
6772 if (!(p->flags & IORING_SETUP_CLAMP))
6773 return -EINVAL;
6774 entries = IORING_MAX_ENTRIES;
6775 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07006776
6777 /*
6778 * Use twice as many entries for the CQ ring. It's possible for the
6779 * application to drive a higher depth than the size of the SQ ring,
6780 * since the sqes are only used at submission time. This allows for
Jens Axboe33a107f2019-10-04 12:10:03 -06006781 * some flexibility in overcommitting a bit. If the application has
6782 * set IORING_SETUP_CQSIZE, it will have passed in the desired number
6783 * of CQ ring entries manually.
Jens Axboe2b188cc2019-01-07 10:46:33 -07006784 */
6785 p->sq_entries = roundup_pow_of_two(entries);
Jens Axboe33a107f2019-10-04 12:10:03 -06006786 if (p->flags & IORING_SETUP_CQSIZE) {
6787 /*
6788 * If IORING_SETUP_CQSIZE is set, we do the same roundup
6789 * to a power-of-two, if it isn't already. We do NOT impose
6790 * any cq vs sq ring sizing.
6791 */
Jens Axboe8110c1a2019-12-28 15:39:54 -07006792 if (p->cq_entries < p->sq_entries)
Jens Axboe33a107f2019-10-04 12:10:03 -06006793 return -EINVAL;
Jens Axboe8110c1a2019-12-28 15:39:54 -07006794 if (p->cq_entries > IORING_MAX_CQ_ENTRIES) {
6795 if (!(p->flags & IORING_SETUP_CLAMP))
6796 return -EINVAL;
6797 p->cq_entries = IORING_MAX_CQ_ENTRIES;
6798 }
Jens Axboe33a107f2019-10-04 12:10:03 -06006799 p->cq_entries = roundup_pow_of_two(p->cq_entries);
6800 } else {
6801 p->cq_entries = 2 * p->sq_entries;
6802 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07006803
6804 user = get_uid(current_user());
6805 account_mem = !capable(CAP_IPC_LOCK);
6806
6807 if (account_mem) {
6808 ret = io_account_mem(user,
6809 ring_pages(p->sq_entries, p->cq_entries));
6810 if (ret) {
6811 free_uid(user);
6812 return ret;
6813 }
6814 }
6815
6816 ctx = io_ring_ctx_alloc(p);
6817 if (!ctx) {
6818 if (account_mem)
6819 io_unaccount_mem(user, ring_pages(p->sq_entries,
6820 p->cq_entries));
6821 free_uid(user);
6822 return -ENOMEM;
6823 }
6824 ctx->compat = in_compat_syscall();
6825 ctx->account_mem = account_mem;
6826 ctx->user = user;
Jens Axboe0b8c0ec2019-12-02 08:50:00 -07006827 ctx->creds = get_current_cred();
Jens Axboe2b188cc2019-01-07 10:46:33 -07006828
6829 ret = io_allocate_scq_urings(ctx, p);
6830 if (ret)
6831 goto err;
6832
Jens Axboe6c271ce2019-01-10 11:22:30 -07006833 ret = io_sq_offload_start(ctx, p);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006834 if (ret)
6835 goto err;
6836
Jens Axboe2b188cc2019-01-07 10:46:33 -07006837 memset(&p->sq_off, 0, sizeof(p->sq_off));
Hristo Venev75b28af2019-08-26 17:23:46 +00006838 p->sq_off.head = offsetof(struct io_rings, sq.head);
6839 p->sq_off.tail = offsetof(struct io_rings, sq.tail);
6840 p->sq_off.ring_mask = offsetof(struct io_rings, sq_ring_mask);
6841 p->sq_off.ring_entries = offsetof(struct io_rings, sq_ring_entries);
6842 p->sq_off.flags = offsetof(struct io_rings, sq_flags);
6843 p->sq_off.dropped = offsetof(struct io_rings, sq_dropped);
6844 p->sq_off.array = (char *)ctx->sq_array - (char *)ctx->rings;
Jens Axboe2b188cc2019-01-07 10:46:33 -07006845
6846 memset(&p->cq_off, 0, sizeof(p->cq_off));
Hristo Venev75b28af2019-08-26 17:23:46 +00006847 p->cq_off.head = offsetof(struct io_rings, cq.head);
6848 p->cq_off.tail = offsetof(struct io_rings, cq.tail);
6849 p->cq_off.ring_mask = offsetof(struct io_rings, cq_ring_mask);
6850 p->cq_off.ring_entries = offsetof(struct io_rings, cq_ring_entries);
6851 p->cq_off.overflow = offsetof(struct io_rings, cq_overflow);
6852 p->cq_off.cqes = offsetof(struct io_rings, cqes);
Jens Axboeac90f242019-09-06 10:26:21 -06006853
Jens Axboe044c1ab2019-10-28 09:15:33 -06006854 /*
6855 * Install ring fd as the very last thing, so we don't risk someone
6856 * having closed it before we finish setup
6857 */
6858 ret = io_uring_get_fd(ctx);
6859 if (ret < 0)
6860 goto err;
6861
Jens Axboeda8c9692019-12-02 18:51:26 -07006862 p->features = IORING_FEAT_SINGLE_MMAP | IORING_FEAT_NODROP |
Jens Axboecccf0ee2020-01-27 16:34:48 -07006863 IORING_FEAT_SUBMIT_STABLE | IORING_FEAT_RW_CUR_POS |
6864 IORING_FEAT_CUR_PERSONALITY;
Dmitrii Dolgovc826bd72019-10-15 19:02:01 +02006865 trace_io_uring_create(ret, ctx, p->sq_entries, p->cq_entries, p->flags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006866 return ret;
6867err:
6868 io_ring_ctx_wait_and_kill(ctx);
6869 return ret;
6870}
6871
6872/*
6873 * Sets up an aio uring context, and returns the fd. Applications asks for a
6874 * ring size, we return the actual sq/cq ring sizes (among other things) in the
6875 * params structure passed in.
6876 */
6877static long io_uring_setup(u32 entries, struct io_uring_params __user *params)
6878{
6879 struct io_uring_params p;
6880 long ret;
6881 int i;
6882
6883 if (copy_from_user(&p, params, sizeof(p)))
6884 return -EFAULT;
6885 for (i = 0; i < ARRAY_SIZE(p.resv); i++) {
6886 if (p.resv[i])
6887 return -EINVAL;
6888 }
6889
Jens Axboe6c271ce2019-01-10 11:22:30 -07006890 if (p.flags & ~(IORING_SETUP_IOPOLL | IORING_SETUP_SQPOLL |
Jens Axboe8110c1a2019-12-28 15:39:54 -07006891 IORING_SETUP_SQ_AFF | IORING_SETUP_CQSIZE |
Pavel Begunkov24369c22020-01-28 03:15:48 +03006892 IORING_SETUP_CLAMP | IORING_SETUP_ATTACH_WQ))
Jens Axboe2b188cc2019-01-07 10:46:33 -07006893 return -EINVAL;
6894
6895 ret = io_uring_create(entries, &p);
6896 if (ret < 0)
6897 return ret;
6898
6899 if (copy_to_user(params, &p, sizeof(p)))
6900 return -EFAULT;
6901
6902 return ret;
6903}
6904
6905SYSCALL_DEFINE2(io_uring_setup, u32, entries,
6906 struct io_uring_params __user *, params)
6907{
6908 return io_uring_setup(entries, params);
6909}
6910
Jens Axboe66f4af92020-01-16 15:36:52 -07006911static int io_probe(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args)
6912{
6913 struct io_uring_probe *p;
6914 size_t size;
6915 int i, ret;
6916
6917 size = struct_size(p, ops, nr_args);
6918 if (size == SIZE_MAX)
6919 return -EOVERFLOW;
6920 p = kzalloc(size, GFP_KERNEL);
6921 if (!p)
6922 return -ENOMEM;
6923
6924 ret = -EFAULT;
6925 if (copy_from_user(p, arg, size))
6926 goto out;
6927 ret = -EINVAL;
6928 if (memchr_inv(p, 0, size))
6929 goto out;
6930
6931 p->last_op = IORING_OP_LAST - 1;
6932 if (nr_args > IORING_OP_LAST)
6933 nr_args = IORING_OP_LAST;
6934
6935 for (i = 0; i < nr_args; i++) {
6936 p->ops[i].op = i;
6937 if (!io_op_defs[i].not_supported)
6938 p->ops[i].flags = IO_URING_OP_SUPPORTED;
6939 }
6940 p->ops_len = i;
6941
6942 ret = 0;
6943 if (copy_to_user(arg, p, size))
6944 ret = -EFAULT;
6945out:
6946 kfree(p);
6947 return ret;
6948}
6949
Jens Axboe071698e2020-01-28 10:04:42 -07006950static int io_register_personality(struct io_ring_ctx *ctx)
6951{
6952 const struct cred *creds = get_current_cred();
6953 int id;
6954
6955 id = idr_alloc_cyclic(&ctx->personality_idr, (void *) creds, 1,
6956 USHRT_MAX, GFP_KERNEL);
6957 if (id < 0)
6958 put_cred(creds);
6959 return id;
6960}
6961
6962static int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id)
6963{
6964 const struct cred *old_creds;
6965
6966 old_creds = idr_remove(&ctx->personality_idr, id);
6967 if (old_creds) {
6968 put_cred(old_creds);
6969 return 0;
6970 }
6971
6972 return -EINVAL;
6973}
6974
6975static bool io_register_op_must_quiesce(int op)
6976{
6977 switch (op) {
6978 case IORING_UNREGISTER_FILES:
6979 case IORING_REGISTER_FILES_UPDATE:
6980 case IORING_REGISTER_PROBE:
6981 case IORING_REGISTER_PERSONALITY:
6982 case IORING_UNREGISTER_PERSONALITY:
6983 return false;
6984 default:
6985 return true;
6986 }
6987}
6988
Jens Axboeedafcce2019-01-09 09:16:05 -07006989static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,
6990 void __user *arg, unsigned nr_args)
Jens Axboeb19062a2019-04-15 10:49:38 -06006991 __releases(ctx->uring_lock)
6992 __acquires(ctx->uring_lock)
Jens Axboeedafcce2019-01-09 09:16:05 -07006993{
6994 int ret;
6995
Jens Axboe35fa71a2019-04-22 10:23:23 -06006996 /*
6997 * We're inside the ring mutex, if the ref is already dying, then
6998 * someone else killed the ctx or is already going through
6999 * io_uring_register().
7000 */
7001 if (percpu_ref_is_dying(&ctx->refs))
7002 return -ENXIO;
7003
Jens Axboe071698e2020-01-28 10:04:42 -07007004 if (io_register_op_must_quiesce(opcode)) {
Jens Axboe05f3fb32019-12-09 11:22:50 -07007005 percpu_ref_kill(&ctx->refs);
Jens Axboeb19062a2019-04-15 10:49:38 -06007006
Jens Axboe05f3fb32019-12-09 11:22:50 -07007007 /*
7008 * Drop uring mutex before waiting for references to exit. If
7009 * another thread is currently inside io_uring_enter() it might
7010 * need to grab the uring_lock to make progress. If we hold it
7011 * here across the drain wait, then we can deadlock. It's safe
7012 * to drop the mutex here, since no new references will come in
7013 * after we've killed the percpu ref.
7014 */
7015 mutex_unlock(&ctx->uring_lock);
Jens Axboec1503682020-01-08 08:26:07 -07007016 ret = wait_for_completion_interruptible(&ctx->completions[0]);
Jens Axboe05f3fb32019-12-09 11:22:50 -07007017 mutex_lock(&ctx->uring_lock);
Jens Axboec1503682020-01-08 08:26:07 -07007018 if (ret) {
7019 percpu_ref_resurrect(&ctx->refs);
7020 ret = -EINTR;
7021 goto out;
7022 }
Jens Axboe05f3fb32019-12-09 11:22:50 -07007023 }
Jens Axboeedafcce2019-01-09 09:16:05 -07007024
7025 switch (opcode) {
7026 case IORING_REGISTER_BUFFERS:
7027 ret = io_sqe_buffer_register(ctx, arg, nr_args);
7028 break;
7029 case IORING_UNREGISTER_BUFFERS:
7030 ret = -EINVAL;
7031 if (arg || nr_args)
7032 break;
7033 ret = io_sqe_buffer_unregister(ctx);
7034 break;
Jens Axboe6b063142019-01-10 22:13:58 -07007035 case IORING_REGISTER_FILES:
7036 ret = io_sqe_files_register(ctx, arg, nr_args);
7037 break;
7038 case IORING_UNREGISTER_FILES:
7039 ret = -EINVAL;
7040 if (arg || nr_args)
7041 break;
7042 ret = io_sqe_files_unregister(ctx);
7043 break;
Jens Axboec3a31e62019-10-03 13:59:56 -06007044 case IORING_REGISTER_FILES_UPDATE:
7045 ret = io_sqe_files_update(ctx, arg, nr_args);
7046 break;
Jens Axboe9b402842019-04-11 11:45:41 -06007047 case IORING_REGISTER_EVENTFD:
Jens Axboef2842ab2020-01-08 11:04:00 -07007048 case IORING_REGISTER_EVENTFD_ASYNC:
Jens Axboe9b402842019-04-11 11:45:41 -06007049 ret = -EINVAL;
7050 if (nr_args != 1)
7051 break;
7052 ret = io_eventfd_register(ctx, arg);
Jens Axboef2842ab2020-01-08 11:04:00 -07007053 if (ret)
7054 break;
7055 if (opcode == IORING_REGISTER_EVENTFD_ASYNC)
7056 ctx->eventfd_async = 1;
7057 else
7058 ctx->eventfd_async = 0;
Jens Axboe9b402842019-04-11 11:45:41 -06007059 break;
7060 case IORING_UNREGISTER_EVENTFD:
7061 ret = -EINVAL;
7062 if (arg || nr_args)
7063 break;
7064 ret = io_eventfd_unregister(ctx);
7065 break;
Jens Axboe66f4af92020-01-16 15:36:52 -07007066 case IORING_REGISTER_PROBE:
7067 ret = -EINVAL;
7068 if (!arg || nr_args > 256)
7069 break;
7070 ret = io_probe(ctx, arg, nr_args);
7071 break;
Jens Axboe071698e2020-01-28 10:04:42 -07007072 case IORING_REGISTER_PERSONALITY:
7073 ret = -EINVAL;
7074 if (arg || nr_args)
7075 break;
7076 ret = io_register_personality(ctx);
7077 break;
7078 case IORING_UNREGISTER_PERSONALITY:
7079 ret = -EINVAL;
7080 if (arg)
7081 break;
7082 ret = io_unregister_personality(ctx, nr_args);
7083 break;
Jens Axboeedafcce2019-01-09 09:16:05 -07007084 default:
7085 ret = -EINVAL;
7086 break;
7087 }
7088
Jens Axboe071698e2020-01-28 10:04:42 -07007089 if (io_register_op_must_quiesce(opcode)) {
Jens Axboe05f3fb32019-12-09 11:22:50 -07007090 /* bring the ctx back to life */
Jens Axboe05f3fb32019-12-09 11:22:50 -07007091 percpu_ref_reinit(&ctx->refs);
Jens Axboec1503682020-01-08 08:26:07 -07007092out:
7093 reinit_completion(&ctx->completions[0]);
Jens Axboe05f3fb32019-12-09 11:22:50 -07007094 }
Jens Axboeedafcce2019-01-09 09:16:05 -07007095 return ret;
7096}
7097
7098SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode,
7099 void __user *, arg, unsigned int, nr_args)
7100{
7101 struct io_ring_ctx *ctx;
7102 long ret = -EBADF;
7103 struct fd f;
7104
7105 f = fdget(fd);
7106 if (!f.file)
7107 return -EBADF;
7108
7109 ret = -EOPNOTSUPP;
7110 if (f.file->f_op != &io_uring_fops)
7111 goto out_fput;
7112
7113 ctx = f.file->private_data;
7114
7115 mutex_lock(&ctx->uring_lock);
7116 ret = __io_uring_register(ctx, opcode, arg, nr_args);
7117 mutex_unlock(&ctx->uring_lock);
Dmitrii Dolgovc826bd72019-10-15 19:02:01 +02007118 trace_io_uring_register(ctx, opcode, ctx->nr_user_files, ctx->nr_user_bufs,
7119 ctx->cq_ev_fd != NULL, ret);
Jens Axboeedafcce2019-01-09 09:16:05 -07007120out_fput:
7121 fdput(f);
7122 return ret;
7123}
7124
Jens Axboe2b188cc2019-01-07 10:46:33 -07007125static int __init io_uring_init(void)
7126{
Stefan Metzmacherd7f62e82020-01-29 14:39:41 +01007127#define __BUILD_BUG_VERIFY_ELEMENT(stype, eoffset, etype, ename) do { \
7128 BUILD_BUG_ON(offsetof(stype, ename) != eoffset); \
7129 BUILD_BUG_ON(sizeof(etype) != sizeof_field(stype, ename)); \
7130} while (0)
7131
7132#define BUILD_BUG_SQE_ELEM(eoffset, etype, ename) \
7133 __BUILD_BUG_VERIFY_ELEMENT(struct io_uring_sqe, eoffset, etype, ename)
7134 BUILD_BUG_ON(sizeof(struct io_uring_sqe) != 64);
7135 BUILD_BUG_SQE_ELEM(0, __u8, opcode);
7136 BUILD_BUG_SQE_ELEM(1, __u8, flags);
7137 BUILD_BUG_SQE_ELEM(2, __u16, ioprio);
7138 BUILD_BUG_SQE_ELEM(4, __s32, fd);
7139 BUILD_BUG_SQE_ELEM(8, __u64, off);
7140 BUILD_BUG_SQE_ELEM(8, __u64, addr2);
7141 BUILD_BUG_SQE_ELEM(16, __u64, addr);
7142 BUILD_BUG_SQE_ELEM(24, __u32, len);
7143 BUILD_BUG_SQE_ELEM(28, __kernel_rwf_t, rw_flags);
7144 BUILD_BUG_SQE_ELEM(28, /* compat */ int, rw_flags);
7145 BUILD_BUG_SQE_ELEM(28, /* compat */ __u32, rw_flags);
7146 BUILD_BUG_SQE_ELEM(28, __u32, fsync_flags);
7147 BUILD_BUG_SQE_ELEM(28, __u16, poll_events);
7148 BUILD_BUG_SQE_ELEM(28, __u32, sync_range_flags);
7149 BUILD_BUG_SQE_ELEM(28, __u32, msg_flags);
7150 BUILD_BUG_SQE_ELEM(28, __u32, timeout_flags);
7151 BUILD_BUG_SQE_ELEM(28, __u32, accept_flags);
7152 BUILD_BUG_SQE_ELEM(28, __u32, cancel_flags);
7153 BUILD_BUG_SQE_ELEM(28, __u32, open_flags);
7154 BUILD_BUG_SQE_ELEM(28, __u32, statx_flags);
7155 BUILD_BUG_SQE_ELEM(28, __u32, fadvise_advice);
7156 BUILD_BUG_SQE_ELEM(32, __u64, user_data);
7157 BUILD_BUG_SQE_ELEM(40, __u16, buf_index);
7158 BUILD_BUG_SQE_ELEM(42, __u16, personality);
7159
Jens Axboed3656342019-12-18 09:50:26 -07007160 BUILD_BUG_ON(ARRAY_SIZE(io_op_defs) != IORING_OP_LAST);
Jens Axboe2b188cc2019-01-07 10:46:33 -07007161 req_cachep = KMEM_CACHE(io_kiocb, SLAB_HWCACHE_ALIGN | SLAB_PANIC);
7162 return 0;
7163};
7164__initcall(io_uring_init);