blob: cd07df2afe616ca46280dd88bd00b3163736a89d [file] [log] [blame]
Jens Axboe2b188cc2019-01-07 10:46:33 -07001// SPDX-License-Identifier: GPL-2.0
2/*
3 * Shared application/kernel submission and completion ring pairs, for
4 * supporting fast/efficient IO.
5 *
6 * A note on the read/write ordering memory barriers that are matched between
Stefan Bühler1e84b972019-04-24 23:54:16 +02007 * the application and kernel side.
8 *
9 * After the application reads the CQ ring tail, it must use an
10 * appropriate smp_rmb() to pair with the smp_wmb() the kernel uses
11 * before writing the tail (using smp_load_acquire to read the tail will
12 * do). It also needs a smp_mb() before updating CQ head (ordering the
13 * entry load(s) with the head store), pairing with an implicit barrier
14 * through a control-dependency in io_get_cqring (smp_store_release to
15 * store head will do). Failure to do so could lead to reading invalid
16 * CQ entries.
17 *
18 * Likewise, the application must use an appropriate smp_wmb() before
19 * writing the SQ tail (ordering SQ entry stores with the tail store),
20 * which pairs with smp_load_acquire in io_get_sqring (smp_store_release
21 * to store the tail will do). And it needs a barrier ordering the SQ
22 * head load before writing new SQ entries (smp_load_acquire to read
23 * head will do).
24 *
25 * When using the SQ poll thread (IORING_SETUP_SQPOLL), the application
26 * needs to check the SQ flags for IORING_SQ_NEED_WAKEUP *after*
27 * updating the SQ tail; a full memory barrier smp_mb() is needed
28 * between.
Jens Axboe2b188cc2019-01-07 10:46:33 -070029 *
30 * Also see the examples in the liburing library:
31 *
32 * git://git.kernel.dk/liburing
33 *
34 * io_uring also uses READ/WRITE_ONCE() for _any_ store or load that happens
35 * from data shared between the kernel and application. This is done both
36 * for ordering purposes, but also to ensure that once a value is loaded from
37 * data that the application could potentially modify, it remains stable.
38 *
39 * Copyright (C) 2018-2019 Jens Axboe
Christoph Hellwigc992fe22019-01-11 09:43:02 -070040 * Copyright (c) 2018-2019 Christoph Hellwig
Jens Axboe2b188cc2019-01-07 10:46:33 -070041 */
42#include <linux/kernel.h>
43#include <linux/init.h>
44#include <linux/errno.h>
45#include <linux/syscalls.h>
46#include <linux/compat.h>
47#include <linux/refcount.h>
48#include <linux/uio.h>
Pavel Begunkov6b47ee62020-01-18 20:22:41 +030049#include <linux/bits.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070050
51#include <linux/sched/signal.h>
52#include <linux/fs.h>
53#include <linux/file.h>
54#include <linux/fdtable.h>
55#include <linux/mm.h>
56#include <linux/mman.h>
57#include <linux/mmu_context.h>
58#include <linux/percpu.h>
59#include <linux/slab.h>
Jens Axboe6c271ce2019-01-10 11:22:30 -070060#include <linux/kthread.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070061#include <linux/blkdev.h>
Jens Axboeedafcce2019-01-09 09:16:05 -070062#include <linux/bvec.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070063#include <linux/net.h>
64#include <net/sock.h>
65#include <net/af_unix.h>
Jens Axboe6b063142019-01-10 22:13:58 -070066#include <net/scm.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070067#include <linux/anon_inodes.h>
68#include <linux/sched/mm.h>
69#include <linux/uaccess.h>
70#include <linux/nospec.h>
Jens Axboeedafcce2019-01-09 09:16:05 -070071#include <linux/sizes.h>
72#include <linux/hugetlb.h>
Jens Axboeaa4c3962019-11-29 10:14:00 -070073#include <linux/highmem.h>
Jens Axboe15b71ab2019-12-11 11:20:36 -070074#include <linux/namei.h>
75#include <linux/fsnotify.h>
Jens Axboe4840e412019-12-25 22:03:45 -070076#include <linux/fadvise.h>
Jens Axboe3e4827b2020-01-08 15:18:09 -070077#include <linux/eventpoll.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070078
Dmitrii Dolgovc826bd72019-10-15 19:02:01 +020079#define CREATE_TRACE_POINTS
80#include <trace/events/io_uring.h>
81
Jens Axboe2b188cc2019-01-07 10:46:33 -070082#include <uapi/linux/io_uring.h>
83
84#include "internal.h"
Jens Axboe561fb042019-10-24 07:25:42 -060085#include "io-wq.h"
Jens Axboe2b188cc2019-01-07 10:46:33 -070086
Daniel Xu5277dea2019-09-14 14:23:45 -070087#define IORING_MAX_ENTRIES 32768
Jens Axboe33a107f2019-10-04 12:10:03 -060088#define IORING_MAX_CQ_ENTRIES (2 * IORING_MAX_ENTRIES)
Jens Axboe65e19f52019-10-26 07:20:21 -060089
90/*
91 * Shift of 9 is 512 entries, or exactly one page on 64-bit archs
92 */
93#define IORING_FILE_TABLE_SHIFT 9
94#define IORING_MAX_FILES_TABLE (1U << IORING_FILE_TABLE_SHIFT)
95#define IORING_FILE_TABLE_MASK (IORING_MAX_FILES_TABLE - 1)
96#define IORING_MAX_FIXED_FILES (64 * IORING_MAX_FILES_TABLE)
Jens Axboe2b188cc2019-01-07 10:46:33 -070097
98struct io_uring {
99 u32 head ____cacheline_aligned_in_smp;
100 u32 tail ____cacheline_aligned_in_smp;
101};
102
Stefan Bühler1e84b972019-04-24 23:54:16 +0200103/*
Hristo Venev75b28af2019-08-26 17:23:46 +0000104 * This data is shared with the application through the mmap at offsets
105 * IORING_OFF_SQ_RING and IORING_OFF_CQ_RING.
Stefan Bühler1e84b972019-04-24 23:54:16 +0200106 *
107 * The offsets to the member fields are published through struct
108 * io_sqring_offsets when calling io_uring_setup.
109 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000110struct io_rings {
Stefan Bühler1e84b972019-04-24 23:54:16 +0200111 /*
112 * Head and tail offsets into the ring; the offsets need to be
113 * masked to get valid indices.
114 *
Hristo Venev75b28af2019-08-26 17:23:46 +0000115 * The kernel controls head of the sq ring and the tail of the cq ring,
116 * and the application controls tail of the sq ring and the head of the
117 * cq ring.
Stefan Bühler1e84b972019-04-24 23:54:16 +0200118 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000119 struct io_uring sq, cq;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200120 /*
Hristo Venev75b28af2019-08-26 17:23:46 +0000121 * Bitmasks to apply to head and tail offsets (constant, equals
Stefan Bühler1e84b972019-04-24 23:54:16 +0200122 * ring_entries - 1)
123 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000124 u32 sq_ring_mask, cq_ring_mask;
125 /* Ring sizes (constant, power of 2) */
126 u32 sq_ring_entries, cq_ring_entries;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200127 /*
128 * Number of invalid entries dropped by the kernel due to
129 * invalid index stored in array
130 *
131 * Written by the kernel, shouldn't be modified by the
132 * application (i.e. get number of "new events" by comparing to
133 * cached value).
134 *
135 * After a new SQ head value was read by the application this
136 * counter includes all submissions that were dropped reaching
137 * the new SQ head (and possibly more).
138 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000139 u32 sq_dropped;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200140 /*
141 * Runtime flags
142 *
143 * Written by the kernel, shouldn't be modified by the
144 * application.
145 *
146 * The application needs a full memory barrier before checking
147 * for IORING_SQ_NEED_WAKEUP after updating the sq tail.
148 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000149 u32 sq_flags;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200150 /*
151 * Number of completion events lost because the queue was full;
152 * this should be avoided by the application by making sure
LimingWu0b4295b2019-12-05 20:18:18 +0800153 * there are not more requests pending than there is space in
Stefan Bühler1e84b972019-04-24 23:54:16 +0200154 * the completion queue.
155 *
156 * Written by the kernel, shouldn't be modified by the
157 * application (i.e. get number of "new events" by comparing to
158 * cached value).
159 *
160 * As completion events come in out of order this counter is not
161 * ordered with any other data.
162 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000163 u32 cq_overflow;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200164 /*
165 * Ring buffer of completion events.
166 *
167 * The kernel writes completion events fresh every time they are
168 * produced, so the application is allowed to modify pending
169 * entries.
170 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000171 struct io_uring_cqe cqes[] ____cacheline_aligned_in_smp;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700172};
173
Jens Axboeedafcce2019-01-09 09:16:05 -0700174struct io_mapped_ubuf {
175 u64 ubuf;
176 size_t len;
177 struct bio_vec *bvec;
178 unsigned int nr_bvecs;
179};
180
Jens Axboe65e19f52019-10-26 07:20:21 -0600181struct fixed_file_table {
182 struct file **files;
Jens Axboe31b51512019-01-18 22:56:34 -0700183};
184
Jens Axboe05f3fb32019-12-09 11:22:50 -0700185enum {
186 FFD_F_ATOMIC,
187};
188
189struct fixed_file_data {
190 struct fixed_file_table *table;
191 struct io_ring_ctx *ctx;
192
193 struct percpu_ref refs;
194 struct llist_head put_llist;
195 unsigned long state;
196 struct work_struct ref_work;
197 struct completion done;
198};
199
Jens Axboe2b188cc2019-01-07 10:46:33 -0700200struct io_ring_ctx {
201 struct {
202 struct percpu_ref refs;
203 } ____cacheline_aligned_in_smp;
204
205 struct {
206 unsigned int flags;
Jens Axboe69b3e542020-01-08 11:01:46 -0700207 int compat: 1;
208 int account_mem: 1;
209 int cq_overflow_flushed: 1;
210 int drain_next: 1;
Jens Axboef2842ab2020-01-08 11:04:00 -0700211 int eventfd_async: 1;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700212
Hristo Venev75b28af2019-08-26 17:23:46 +0000213 /*
214 * Ring buffer of indices into array of io_uring_sqe, which is
215 * mmapped by the application using the IORING_OFF_SQES offset.
216 *
217 * This indirection could e.g. be used to assign fixed
218 * io_uring_sqe entries to operations and only submit them to
219 * the queue when needed.
220 *
221 * The kernel modifies neither the indices array nor the entries
222 * array.
223 */
224 u32 *sq_array;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700225 unsigned cached_sq_head;
226 unsigned sq_entries;
227 unsigned sq_mask;
Jens Axboe6c271ce2019-01-10 11:22:30 -0700228 unsigned sq_thread_idle;
Jens Axboe498ccd92019-10-25 10:04:25 -0600229 unsigned cached_sq_dropped;
Jens Axboe206aefd2019-11-07 18:27:42 -0700230 atomic_t cached_cq_overflow;
Jens Axboead3eb2c2019-12-18 17:12:20 -0700231 unsigned long sq_check_overflow;
Jens Axboede0617e2019-04-06 21:51:27 -0600232
233 struct list_head defer_list;
Jens Axboe5262f562019-09-17 12:26:57 -0600234 struct list_head timeout_list;
Jens Axboe1d7bb1d2019-11-06 11:31:17 -0700235 struct list_head cq_overflow_list;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700236
Jens Axboefcb323c2019-10-24 12:39:47 -0600237 wait_queue_head_t inflight_wait;
Jens Axboead3eb2c2019-12-18 17:12:20 -0700238 struct io_uring_sqe *sq_sqes;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700239 } ____cacheline_aligned_in_smp;
240
Hristo Venev75b28af2019-08-26 17:23:46 +0000241 struct io_rings *rings;
242
Jens Axboe2b188cc2019-01-07 10:46:33 -0700243 /* IO offload */
Jens Axboe561fb042019-10-24 07:25:42 -0600244 struct io_wq *io_wq;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700245 struct task_struct *sqo_thread; /* if using sq thread polling */
246 struct mm_struct *sqo_mm;
247 wait_queue_head_t sqo_wait;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700248
Jens Axboe6b063142019-01-10 22:13:58 -0700249 /*
250 * If used, fixed file set. Writers must ensure that ->refs is dead,
251 * readers must ensure that ->refs is alive as long as the file* is
252 * used. Only updated through io_uring_register(2).
253 */
Jens Axboe05f3fb32019-12-09 11:22:50 -0700254 struct fixed_file_data *file_data;
Jens Axboe6b063142019-01-10 22:13:58 -0700255 unsigned nr_user_files;
Pavel Begunkovb14cca02020-01-17 04:45:59 +0300256 int ring_fd;
257 struct file *ring_file;
Jens Axboe6b063142019-01-10 22:13:58 -0700258
Jens Axboeedafcce2019-01-09 09:16:05 -0700259 /* if used, fixed mapped user buffers */
260 unsigned nr_user_bufs;
261 struct io_mapped_ubuf *user_bufs;
262
Jens Axboe2b188cc2019-01-07 10:46:33 -0700263 struct user_struct *user;
264
Jens Axboe0b8c0ec2019-12-02 08:50:00 -0700265 const struct cred *creds;
Jens Axboe181e4482019-11-25 08:52:30 -0700266
Jens Axboe206aefd2019-11-07 18:27:42 -0700267 /* 0 is for ctx quiesce/reinit/free, 1 is for sqo_thread started */
268 struct completion *completions;
269
Jens Axboe0ddf92e2019-11-08 08:52:53 -0700270 /* if all else fails... */
271 struct io_kiocb *fallback_req;
272
Jens Axboe206aefd2019-11-07 18:27:42 -0700273#if defined(CONFIG_UNIX)
274 struct socket *ring_sock;
275#endif
276
Jens Axboe071698e2020-01-28 10:04:42 -0700277 struct idr personality_idr;
278
Jens Axboe206aefd2019-11-07 18:27:42 -0700279 struct {
280 unsigned cached_cq_tail;
281 unsigned cq_entries;
282 unsigned cq_mask;
283 atomic_t cq_timeouts;
Jens Axboead3eb2c2019-12-18 17:12:20 -0700284 unsigned long cq_check_overflow;
Jens Axboe206aefd2019-11-07 18:27:42 -0700285 struct wait_queue_head cq_wait;
286 struct fasync_struct *cq_fasync;
287 struct eventfd_ctx *cq_ev_fd;
288 } ____cacheline_aligned_in_smp;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700289
290 struct {
291 struct mutex uring_lock;
292 wait_queue_head_t wait;
293 } ____cacheline_aligned_in_smp;
294
295 struct {
296 spinlock_t completion_lock;
Jens Axboee94f1412019-12-19 12:06:02 -0700297 struct llist_head poll_llist;
298
Jens Axboedef596e2019-01-09 08:59:42 -0700299 /*
300 * ->poll_list is protected by the ctx->uring_lock for
301 * io_uring instances that don't use IORING_SETUP_SQPOLL.
302 * For SQPOLL, only the single threaded io_sq_thread() will
303 * manipulate the list, hence no extra locking is needed there.
304 */
305 struct list_head poll_list;
Jens Axboe78076bb2019-12-04 19:56:40 -0700306 struct hlist_head *cancel_hash;
307 unsigned cancel_hash_bits;
Jens Axboee94f1412019-12-19 12:06:02 -0700308 bool poll_multi_file;
Jens Axboefcb323c2019-10-24 12:39:47 -0600309
310 spinlock_t inflight_lock;
311 struct list_head inflight_list;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700312 } ____cacheline_aligned_in_smp;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700313};
314
Jens Axboe09bb8392019-03-13 12:39:28 -0600315/*
316 * First field must be the file pointer in all the
317 * iocb unions! See also 'struct kiocb' in <linux/fs.h>
318 */
Jens Axboe221c5eb2019-01-17 09:41:58 -0700319struct io_poll_iocb {
320 struct file *file;
Jens Axboe0969e782019-12-17 18:40:57 -0700321 union {
322 struct wait_queue_head *head;
323 u64 addr;
324 };
Jens Axboe221c5eb2019-01-17 09:41:58 -0700325 __poll_t events;
Jens Axboe8c838782019-03-12 15:48:16 -0600326 bool done;
Jens Axboe221c5eb2019-01-17 09:41:58 -0700327 bool canceled;
Jens Axboe392edb42019-12-09 17:52:20 -0700328 struct wait_queue_entry wait;
Jens Axboe221c5eb2019-01-17 09:41:58 -0700329};
330
Jens Axboeb5dba592019-12-11 14:02:38 -0700331struct io_close {
332 struct file *file;
333 struct file *put_file;
334 int fd;
335};
336
Jens Axboead8a48a2019-11-15 08:49:11 -0700337struct io_timeout_data {
338 struct io_kiocb *req;
339 struct hrtimer timer;
340 struct timespec64 ts;
341 enum hrtimer_mode mode;
Pavel Begunkovcc42e0a2019-11-25 23:14:38 +0300342 u32 seq_offset;
Jens Axboead8a48a2019-11-15 08:49:11 -0700343};
344
Jens Axboe8ed8d3c2019-12-16 11:55:28 -0700345struct io_accept {
346 struct file *file;
347 struct sockaddr __user *addr;
348 int __user *addr_len;
349 int flags;
350};
351
352struct io_sync {
353 struct file *file;
354 loff_t len;
355 loff_t off;
356 int flags;
Jens Axboed63d1b52019-12-10 10:38:56 -0700357 int mode;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -0700358};
359
Jens Axboefbf23842019-12-17 18:45:56 -0700360struct io_cancel {
361 struct file *file;
362 u64 addr;
363};
364
Jens Axboeb29472e2019-12-17 18:50:29 -0700365struct io_timeout {
366 struct file *file;
367 u64 addr;
368 int flags;
Jens Axboe26a61672019-12-20 09:02:01 -0700369 unsigned count;
Jens Axboeb29472e2019-12-17 18:50:29 -0700370};
371
Jens Axboe9adbd452019-12-20 08:45:55 -0700372struct io_rw {
373 /* NOTE: kiocb has the file as the first member, so don't do it here */
374 struct kiocb kiocb;
375 u64 addr;
376 u64 len;
377};
378
Jens Axboe3fbb51c2019-12-20 08:51:52 -0700379struct io_connect {
380 struct file *file;
381 struct sockaddr __user *addr;
382 int addr_len;
383};
384
Jens Axboee47293f2019-12-20 08:58:21 -0700385struct io_sr_msg {
386 struct file *file;
Jens Axboefddafac2020-01-04 20:19:44 -0700387 union {
388 struct user_msghdr __user *msg;
389 void __user *buf;
390 };
Jens Axboee47293f2019-12-20 08:58:21 -0700391 int msg_flags;
Jens Axboefddafac2020-01-04 20:19:44 -0700392 size_t len;
Jens Axboee47293f2019-12-20 08:58:21 -0700393};
394
Jens Axboe15b71ab2019-12-11 11:20:36 -0700395struct io_open {
396 struct file *file;
397 int dfd;
Jens Axboeeddc7ef2019-12-13 21:18:10 -0700398 union {
Jens Axboeeddc7ef2019-12-13 21:18:10 -0700399 unsigned mask;
400 };
Jens Axboe15b71ab2019-12-11 11:20:36 -0700401 struct filename *filename;
Jens Axboeeddc7ef2019-12-13 21:18:10 -0700402 struct statx __user *buffer;
Jens Axboec12cedf2020-01-08 17:41:21 -0700403 struct open_how how;
Jens Axboe15b71ab2019-12-11 11:20:36 -0700404};
405
Jens Axboe05f3fb32019-12-09 11:22:50 -0700406struct io_files_update {
407 struct file *file;
408 u64 arg;
409 u32 nr_args;
410 u32 offset;
411};
412
Jens Axboe4840e412019-12-25 22:03:45 -0700413struct io_fadvise {
414 struct file *file;
415 u64 offset;
416 u32 len;
417 u32 advice;
418};
419
Jens Axboec1ca7572019-12-25 22:18:28 -0700420struct io_madvise {
421 struct file *file;
422 u64 addr;
423 u32 len;
424 u32 advice;
425};
426
Jens Axboe3e4827b2020-01-08 15:18:09 -0700427struct io_epoll {
428 struct file *file;
429 int epfd;
430 int op;
431 int fd;
432 struct epoll_event event;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700433};
434
Jens Axboef499a022019-12-02 16:28:46 -0700435struct io_async_connect {
436 struct sockaddr_storage address;
437};
438
Jens Axboe03b12302019-12-02 18:50:25 -0700439struct io_async_msghdr {
440 struct iovec fast_iov[UIO_FASTIOV];
441 struct iovec *iov;
442 struct sockaddr __user *uaddr;
443 struct msghdr msg;
444};
445
Jens Axboef67676d2019-12-02 11:03:47 -0700446struct io_async_rw {
447 struct iovec fast_iov[UIO_FASTIOV];
448 struct iovec *iov;
449 ssize_t nr_segs;
450 ssize_t size;
451};
452
Jens Axboe15b71ab2019-12-11 11:20:36 -0700453struct io_async_open {
454 struct filename *filename;
455};
456
Jens Axboe1a6b74f2019-12-02 10:33:15 -0700457struct io_async_ctx {
Jens Axboef67676d2019-12-02 11:03:47 -0700458 union {
459 struct io_async_rw rw;
Jens Axboe03b12302019-12-02 18:50:25 -0700460 struct io_async_msghdr msg;
Jens Axboef499a022019-12-02 16:28:46 -0700461 struct io_async_connect connect;
Jens Axboe2d283902019-12-04 11:08:05 -0700462 struct io_timeout_data timeout;
Jens Axboe15b71ab2019-12-11 11:20:36 -0700463 struct io_async_open open;
Jens Axboef67676d2019-12-02 11:03:47 -0700464 };
Jens Axboe1a6b74f2019-12-02 10:33:15 -0700465};
466
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300467enum {
468 REQ_F_FIXED_FILE_BIT = IOSQE_FIXED_FILE_BIT,
469 REQ_F_IO_DRAIN_BIT = IOSQE_IO_DRAIN_BIT,
470 REQ_F_LINK_BIT = IOSQE_IO_LINK_BIT,
471 REQ_F_HARDLINK_BIT = IOSQE_IO_HARDLINK_BIT,
472 REQ_F_FORCE_ASYNC_BIT = IOSQE_ASYNC_BIT,
473
474 REQ_F_LINK_NEXT_BIT,
475 REQ_F_FAIL_LINK_BIT,
476 REQ_F_INFLIGHT_BIT,
477 REQ_F_CUR_POS_BIT,
478 REQ_F_NOWAIT_BIT,
479 REQ_F_IOPOLL_COMPLETED_BIT,
480 REQ_F_LINK_TIMEOUT_BIT,
481 REQ_F_TIMEOUT_BIT,
482 REQ_F_ISREG_BIT,
483 REQ_F_MUST_PUNT_BIT,
484 REQ_F_TIMEOUT_NOSEQ_BIT,
485 REQ_F_COMP_LOCKED_BIT,
486};
487
488enum {
489 /* ctx owns file */
490 REQ_F_FIXED_FILE = BIT(REQ_F_FIXED_FILE_BIT),
491 /* drain existing IO first */
492 REQ_F_IO_DRAIN = BIT(REQ_F_IO_DRAIN_BIT),
493 /* linked sqes */
494 REQ_F_LINK = BIT(REQ_F_LINK_BIT),
495 /* doesn't sever on completion < 0 */
496 REQ_F_HARDLINK = BIT(REQ_F_HARDLINK_BIT),
497 /* IOSQE_ASYNC */
498 REQ_F_FORCE_ASYNC = BIT(REQ_F_FORCE_ASYNC_BIT),
499
500 /* already grabbed next link */
501 REQ_F_LINK_NEXT = BIT(REQ_F_LINK_NEXT_BIT),
502 /* fail rest of links */
503 REQ_F_FAIL_LINK = BIT(REQ_F_FAIL_LINK_BIT),
504 /* on inflight list */
505 REQ_F_INFLIGHT = BIT(REQ_F_INFLIGHT_BIT),
506 /* read/write uses file position */
507 REQ_F_CUR_POS = BIT(REQ_F_CUR_POS_BIT),
508 /* must not punt to workers */
509 REQ_F_NOWAIT = BIT(REQ_F_NOWAIT_BIT),
510 /* polled IO has completed */
511 REQ_F_IOPOLL_COMPLETED = BIT(REQ_F_IOPOLL_COMPLETED_BIT),
512 /* has linked timeout */
513 REQ_F_LINK_TIMEOUT = BIT(REQ_F_LINK_TIMEOUT_BIT),
514 /* timeout request */
515 REQ_F_TIMEOUT = BIT(REQ_F_TIMEOUT_BIT),
516 /* regular file */
517 REQ_F_ISREG = BIT(REQ_F_ISREG_BIT),
518 /* must be punted even for NONBLOCK */
519 REQ_F_MUST_PUNT = BIT(REQ_F_MUST_PUNT_BIT),
520 /* no timeout sequence */
521 REQ_F_TIMEOUT_NOSEQ = BIT(REQ_F_TIMEOUT_NOSEQ_BIT),
522 /* completion under lock */
523 REQ_F_COMP_LOCKED = BIT(REQ_F_COMP_LOCKED_BIT),
524};
525
Jens Axboe09bb8392019-03-13 12:39:28 -0600526/*
527 * NOTE! Each of the iocb union members has the file pointer
528 * as the first entry in their struct definition. So you can
529 * access the file pointer through any of the sub-structs,
530 * or directly as just 'ki_filp' in this struct.
531 */
Jens Axboe2b188cc2019-01-07 10:46:33 -0700532struct io_kiocb {
Jens Axboe221c5eb2019-01-17 09:41:58 -0700533 union {
Jens Axboe09bb8392019-03-13 12:39:28 -0600534 struct file *file;
Jens Axboe9adbd452019-12-20 08:45:55 -0700535 struct io_rw rw;
Jens Axboe221c5eb2019-01-17 09:41:58 -0700536 struct io_poll_iocb poll;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -0700537 struct io_accept accept;
538 struct io_sync sync;
Jens Axboefbf23842019-12-17 18:45:56 -0700539 struct io_cancel cancel;
Jens Axboeb29472e2019-12-17 18:50:29 -0700540 struct io_timeout timeout;
Jens Axboe3fbb51c2019-12-20 08:51:52 -0700541 struct io_connect connect;
Jens Axboee47293f2019-12-20 08:58:21 -0700542 struct io_sr_msg sr_msg;
Jens Axboe15b71ab2019-12-11 11:20:36 -0700543 struct io_open open;
Jens Axboeb5dba592019-12-11 14:02:38 -0700544 struct io_close close;
Jens Axboe05f3fb32019-12-09 11:22:50 -0700545 struct io_files_update files_update;
Jens Axboe4840e412019-12-25 22:03:45 -0700546 struct io_fadvise fadvise;
Jens Axboec1ca7572019-12-25 22:18:28 -0700547 struct io_madvise madvise;
Jens Axboe3e4827b2020-01-08 15:18:09 -0700548 struct io_epoll epoll;
Jens Axboe221c5eb2019-01-17 09:41:58 -0700549 };
Jens Axboe2b188cc2019-01-07 10:46:33 -0700550
Jens Axboe1a6b74f2019-12-02 10:33:15 -0700551 struct io_async_ctx *io;
Pavel Begunkovb14cca02020-01-17 04:45:59 +0300552 /*
553 * llist_node is only used for poll deferred completions
554 */
555 struct llist_node llist_node;
Pavel Begunkovcf6fd4b2019-11-25 23:14:39 +0300556 bool has_user;
557 bool in_async;
558 bool needs_fixed_file;
Jens Axboed625c6e2019-12-17 19:53:05 -0700559 u8 opcode;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700560
561 struct io_ring_ctx *ctx;
Jens Axboeeac406c2019-11-14 12:09:58 -0700562 union {
563 struct list_head list;
Jens Axboe78076bb2019-12-04 19:56:40 -0700564 struct hlist_node hash_node;
Jens Axboeeac406c2019-11-14 12:09:58 -0700565 };
Jens Axboe9e645e112019-05-10 16:07:28 -0600566 struct list_head link_list;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700567 unsigned int flags;
Jens Axboec16361c2019-01-17 08:39:48 -0700568 refcount_t refs;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700569 u64 user_data;
Jens Axboe9e645e112019-05-10 16:07:28 -0600570 u32 result;
Jens Axboede0617e2019-04-06 21:51:27 -0600571 u32 sequence;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700572
Jens Axboefcb323c2019-10-24 12:39:47 -0600573 struct list_head inflight_entry;
574
Jens Axboe561fb042019-10-24 07:25:42 -0600575 struct io_wq_work work;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700576};
577
578#define IO_PLUG_THRESHOLD 2
Jens Axboedef596e2019-01-09 08:59:42 -0700579#define IO_IOPOLL_BATCH 8
Jens Axboe2b188cc2019-01-07 10:46:33 -0700580
Jens Axboe9a56a232019-01-09 09:06:50 -0700581struct io_submit_state {
582 struct blk_plug plug;
583
584 /*
Jens Axboe2579f912019-01-09 09:10:43 -0700585 * io_kiocb alloc cache
586 */
587 void *reqs[IO_IOPOLL_BATCH];
588 unsigned int free_reqs;
589 unsigned int cur_req;
590
591 /*
Jens Axboe9a56a232019-01-09 09:06:50 -0700592 * File reference cache
593 */
594 struct file *file;
595 unsigned int fd;
596 unsigned int has_refs;
597 unsigned int used_refs;
598 unsigned int ios_left;
599};
600
Jens Axboed3656342019-12-18 09:50:26 -0700601struct io_op_def {
602 /* needs req->io allocated for deferral/async */
603 unsigned async_ctx : 1;
604 /* needs current->mm setup, does mm access */
605 unsigned needs_mm : 1;
606 /* needs req->file assigned */
607 unsigned needs_file : 1;
608 /* needs req->file assigned IFF fd is >= 0 */
609 unsigned fd_non_neg : 1;
610 /* hash wq insertion if file is a regular file */
611 unsigned hash_reg_file : 1;
612 /* unbound wq insertion if file is a non-regular file */
613 unsigned unbound_nonreg_file : 1;
Jens Axboe66f4af92020-01-16 15:36:52 -0700614 /* opcode is not supported by this kernel */
615 unsigned not_supported : 1;
Jens Axboef86cd202020-01-29 13:46:44 -0700616 /* needs file table */
617 unsigned file_table : 1;
Jens Axboed3656342019-12-18 09:50:26 -0700618};
619
620static const struct io_op_def io_op_defs[] = {
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300621 [IORING_OP_NOP] = {},
622 [IORING_OP_READV] = {
Jens Axboed3656342019-12-18 09:50:26 -0700623 .async_ctx = 1,
624 .needs_mm = 1,
625 .needs_file = 1,
626 .unbound_nonreg_file = 1,
627 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300628 [IORING_OP_WRITEV] = {
Jens Axboed3656342019-12-18 09:50:26 -0700629 .async_ctx = 1,
630 .needs_mm = 1,
631 .needs_file = 1,
632 .hash_reg_file = 1,
633 .unbound_nonreg_file = 1,
634 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300635 [IORING_OP_FSYNC] = {
Jens Axboed3656342019-12-18 09:50:26 -0700636 .needs_file = 1,
637 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300638 [IORING_OP_READ_FIXED] = {
Jens Axboed3656342019-12-18 09:50:26 -0700639 .needs_file = 1,
640 .unbound_nonreg_file = 1,
641 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300642 [IORING_OP_WRITE_FIXED] = {
Jens Axboed3656342019-12-18 09:50:26 -0700643 .needs_file = 1,
644 .hash_reg_file = 1,
645 .unbound_nonreg_file = 1,
646 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300647 [IORING_OP_POLL_ADD] = {
Jens Axboed3656342019-12-18 09:50:26 -0700648 .needs_file = 1,
649 .unbound_nonreg_file = 1,
650 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300651 [IORING_OP_POLL_REMOVE] = {},
652 [IORING_OP_SYNC_FILE_RANGE] = {
Jens Axboed3656342019-12-18 09:50:26 -0700653 .needs_file = 1,
654 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300655 [IORING_OP_SENDMSG] = {
Jens Axboed3656342019-12-18 09:50:26 -0700656 .async_ctx = 1,
657 .needs_mm = 1,
658 .needs_file = 1,
659 .unbound_nonreg_file = 1,
660 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300661 [IORING_OP_RECVMSG] = {
Jens Axboed3656342019-12-18 09:50:26 -0700662 .async_ctx = 1,
663 .needs_mm = 1,
664 .needs_file = 1,
665 .unbound_nonreg_file = 1,
666 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300667 [IORING_OP_TIMEOUT] = {
Jens Axboed3656342019-12-18 09:50:26 -0700668 .async_ctx = 1,
669 .needs_mm = 1,
670 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300671 [IORING_OP_TIMEOUT_REMOVE] = {},
672 [IORING_OP_ACCEPT] = {
Jens Axboed3656342019-12-18 09:50:26 -0700673 .needs_mm = 1,
674 .needs_file = 1,
675 .unbound_nonreg_file = 1,
Jens Axboef86cd202020-01-29 13:46:44 -0700676 .file_table = 1,
Jens Axboed3656342019-12-18 09:50:26 -0700677 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300678 [IORING_OP_ASYNC_CANCEL] = {},
679 [IORING_OP_LINK_TIMEOUT] = {
Jens Axboed3656342019-12-18 09:50:26 -0700680 .async_ctx = 1,
681 .needs_mm = 1,
682 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300683 [IORING_OP_CONNECT] = {
Jens Axboed3656342019-12-18 09:50:26 -0700684 .async_ctx = 1,
685 .needs_mm = 1,
686 .needs_file = 1,
687 .unbound_nonreg_file = 1,
688 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300689 [IORING_OP_FALLOCATE] = {
Jens Axboed3656342019-12-18 09:50:26 -0700690 .needs_file = 1,
691 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300692 [IORING_OP_OPENAT] = {
Jens Axboed3656342019-12-18 09:50:26 -0700693 .needs_file = 1,
694 .fd_non_neg = 1,
Jens Axboef86cd202020-01-29 13:46:44 -0700695 .file_table = 1,
Jens Axboed3656342019-12-18 09:50:26 -0700696 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300697 [IORING_OP_CLOSE] = {
Jens Axboed3656342019-12-18 09:50:26 -0700698 .needs_file = 1,
Jens Axboef86cd202020-01-29 13:46:44 -0700699 .file_table = 1,
Jens Axboed3656342019-12-18 09:50:26 -0700700 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300701 [IORING_OP_FILES_UPDATE] = {
Jens Axboed3656342019-12-18 09:50:26 -0700702 .needs_mm = 1,
Jens Axboef86cd202020-01-29 13:46:44 -0700703 .file_table = 1,
Jens Axboed3656342019-12-18 09:50:26 -0700704 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300705 [IORING_OP_STATX] = {
Jens Axboed3656342019-12-18 09:50:26 -0700706 .needs_mm = 1,
707 .needs_file = 1,
708 .fd_non_neg = 1,
709 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300710 [IORING_OP_READ] = {
Jens Axboe3a6820f2019-12-22 15:19:35 -0700711 .needs_mm = 1,
712 .needs_file = 1,
713 .unbound_nonreg_file = 1,
714 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300715 [IORING_OP_WRITE] = {
Jens Axboe3a6820f2019-12-22 15:19:35 -0700716 .needs_mm = 1,
717 .needs_file = 1,
718 .unbound_nonreg_file = 1,
719 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300720 [IORING_OP_FADVISE] = {
Jens Axboe4840e412019-12-25 22:03:45 -0700721 .needs_file = 1,
722 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300723 [IORING_OP_MADVISE] = {
Jens Axboec1ca7572019-12-25 22:18:28 -0700724 .needs_mm = 1,
725 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300726 [IORING_OP_SEND] = {
Jens Axboefddafac2020-01-04 20:19:44 -0700727 .needs_mm = 1,
728 .needs_file = 1,
729 .unbound_nonreg_file = 1,
730 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300731 [IORING_OP_RECV] = {
Jens Axboefddafac2020-01-04 20:19:44 -0700732 .needs_mm = 1,
733 .needs_file = 1,
734 .unbound_nonreg_file = 1,
735 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300736 [IORING_OP_OPENAT2] = {
Jens Axboecebdb982020-01-08 17:59:24 -0700737 .needs_file = 1,
738 .fd_non_neg = 1,
Jens Axboef86cd202020-01-29 13:46:44 -0700739 .file_table = 1,
Jens Axboecebdb982020-01-08 17:59:24 -0700740 },
Jens Axboe3e4827b2020-01-08 15:18:09 -0700741 [IORING_OP_EPOLL_CTL] = {
742 .unbound_nonreg_file = 1,
743 .file_table = 1,
744 },
Jens Axboed3656342019-12-18 09:50:26 -0700745};
746
Jens Axboe561fb042019-10-24 07:25:42 -0600747static void io_wq_submit_work(struct io_wq_work **workptr);
Jens Axboe78e19bb2019-11-06 15:21:34 -0700748static void io_cqring_fill_event(struct io_kiocb *req, long res);
Jackie Liuec9c02a2019-11-08 23:50:36 +0800749static void io_put_req(struct io_kiocb *req);
Jens Axboe978db572019-11-14 22:39:04 -0700750static void __io_double_put_req(struct io_kiocb *req);
Jens Axboe94ae5e72019-11-14 19:39:52 -0700751static struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req);
752static void io_queue_linked_timeout(struct io_kiocb *req);
Jens Axboe05f3fb32019-12-09 11:22:50 -0700753static int __io_sqe_files_update(struct io_ring_ctx *ctx,
754 struct io_uring_files_update *ip,
755 unsigned nr_args);
Jens Axboef86cd202020-01-29 13:46:44 -0700756static int io_grab_files(struct io_kiocb *req);
Jens Axboede0617e2019-04-06 21:51:27 -0600757
Jens Axboe2b188cc2019-01-07 10:46:33 -0700758static struct kmem_cache *req_cachep;
759
760static const struct file_operations io_uring_fops;
761
762struct sock *io_uring_get_socket(struct file *file)
763{
764#if defined(CONFIG_UNIX)
765 if (file->f_op == &io_uring_fops) {
766 struct io_ring_ctx *ctx = file->private_data;
767
768 return ctx->ring_sock->sk;
769 }
770#endif
771 return NULL;
772}
773EXPORT_SYMBOL(io_uring_get_socket);
774
775static void io_ring_ctx_ref_free(struct percpu_ref *ref)
776{
777 struct io_ring_ctx *ctx = container_of(ref, struct io_ring_ctx, refs);
778
Jens Axboe206aefd2019-11-07 18:27:42 -0700779 complete(&ctx->completions[0]);
Jens Axboe2b188cc2019-01-07 10:46:33 -0700780}
781
782static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
783{
784 struct io_ring_ctx *ctx;
Jens Axboe78076bb2019-12-04 19:56:40 -0700785 int hash_bits;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700786
787 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
788 if (!ctx)
789 return NULL;
790
Jens Axboe0ddf92e2019-11-08 08:52:53 -0700791 ctx->fallback_req = kmem_cache_alloc(req_cachep, GFP_KERNEL);
792 if (!ctx->fallback_req)
793 goto err;
794
Jens Axboe206aefd2019-11-07 18:27:42 -0700795 ctx->completions = kmalloc(2 * sizeof(struct completion), GFP_KERNEL);
796 if (!ctx->completions)
797 goto err;
798
Jens Axboe78076bb2019-12-04 19:56:40 -0700799 /*
800 * Use 5 bits less than the max cq entries, that should give us around
801 * 32 entries per hash list if totally full and uniformly spread.
802 */
803 hash_bits = ilog2(p->cq_entries);
804 hash_bits -= 5;
805 if (hash_bits <= 0)
806 hash_bits = 1;
807 ctx->cancel_hash_bits = hash_bits;
808 ctx->cancel_hash = kmalloc((1U << hash_bits) * sizeof(struct hlist_head),
809 GFP_KERNEL);
810 if (!ctx->cancel_hash)
811 goto err;
812 __hash_init(ctx->cancel_hash, 1U << hash_bits);
813
Roman Gushchin21482892019-05-07 10:01:48 -0700814 if (percpu_ref_init(&ctx->refs, io_ring_ctx_ref_free,
Jens Axboe206aefd2019-11-07 18:27:42 -0700815 PERCPU_REF_ALLOW_REINIT, GFP_KERNEL))
816 goto err;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700817
818 ctx->flags = p->flags;
819 init_waitqueue_head(&ctx->cq_wait);
Jens Axboe1d7bb1d2019-11-06 11:31:17 -0700820 INIT_LIST_HEAD(&ctx->cq_overflow_list);
Jens Axboe206aefd2019-11-07 18:27:42 -0700821 init_completion(&ctx->completions[0]);
822 init_completion(&ctx->completions[1]);
Jens Axboe071698e2020-01-28 10:04:42 -0700823 idr_init(&ctx->personality_idr);
Jens Axboe2b188cc2019-01-07 10:46:33 -0700824 mutex_init(&ctx->uring_lock);
825 init_waitqueue_head(&ctx->wait);
826 spin_lock_init(&ctx->completion_lock);
Jens Axboee94f1412019-12-19 12:06:02 -0700827 init_llist_head(&ctx->poll_llist);
Jens Axboedef596e2019-01-09 08:59:42 -0700828 INIT_LIST_HEAD(&ctx->poll_list);
Jens Axboede0617e2019-04-06 21:51:27 -0600829 INIT_LIST_HEAD(&ctx->defer_list);
Jens Axboe5262f562019-09-17 12:26:57 -0600830 INIT_LIST_HEAD(&ctx->timeout_list);
Jens Axboefcb323c2019-10-24 12:39:47 -0600831 init_waitqueue_head(&ctx->inflight_wait);
832 spin_lock_init(&ctx->inflight_lock);
833 INIT_LIST_HEAD(&ctx->inflight_list);
Jens Axboe2b188cc2019-01-07 10:46:33 -0700834 return ctx;
Jens Axboe206aefd2019-11-07 18:27:42 -0700835err:
Jens Axboe0ddf92e2019-11-08 08:52:53 -0700836 if (ctx->fallback_req)
837 kmem_cache_free(req_cachep, ctx->fallback_req);
Jens Axboe206aefd2019-11-07 18:27:42 -0700838 kfree(ctx->completions);
Jens Axboe78076bb2019-12-04 19:56:40 -0700839 kfree(ctx->cancel_hash);
Jens Axboe206aefd2019-11-07 18:27:42 -0700840 kfree(ctx);
841 return NULL;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700842}
843
Bob Liu9d858b22019-11-13 18:06:25 +0800844static inline bool __req_need_defer(struct io_kiocb *req)
Jens Axboede0617e2019-04-06 21:51:27 -0600845{
Jackie Liua197f662019-11-08 08:09:12 -0700846 struct io_ring_ctx *ctx = req->ctx;
847
Jens Axboe498ccd92019-10-25 10:04:25 -0600848 return req->sequence != ctx->cached_cq_tail + ctx->cached_sq_dropped
849 + atomic_read(&ctx->cached_cq_overflow);
Jens Axboede0617e2019-04-06 21:51:27 -0600850}
851
Bob Liu9d858b22019-11-13 18:06:25 +0800852static inline bool req_need_defer(struct io_kiocb *req)
Jens Axboe7adf4ea2019-10-10 21:42:58 -0600853{
Pavel Begunkov87987892020-01-18 01:22:30 +0300854 if (unlikely(req->flags & REQ_F_IO_DRAIN))
Bob Liu9d858b22019-11-13 18:06:25 +0800855 return __req_need_defer(req);
Jens Axboe7adf4ea2019-10-10 21:42:58 -0600856
Bob Liu9d858b22019-11-13 18:06:25 +0800857 return false;
Jens Axboe7adf4ea2019-10-10 21:42:58 -0600858}
859
860static struct io_kiocb *io_get_deferred_req(struct io_ring_ctx *ctx)
Jens Axboede0617e2019-04-06 21:51:27 -0600861{
862 struct io_kiocb *req;
863
Jens Axboe7adf4ea2019-10-10 21:42:58 -0600864 req = list_first_entry_or_null(&ctx->defer_list, struct io_kiocb, list);
Bob Liu9d858b22019-11-13 18:06:25 +0800865 if (req && !req_need_defer(req)) {
Jens Axboede0617e2019-04-06 21:51:27 -0600866 list_del_init(&req->list);
867 return req;
868 }
869
870 return NULL;
871}
872
Jens Axboe5262f562019-09-17 12:26:57 -0600873static struct io_kiocb *io_get_timeout_req(struct io_ring_ctx *ctx)
874{
Jens Axboe7adf4ea2019-10-10 21:42:58 -0600875 struct io_kiocb *req;
876
877 req = list_first_entry_or_null(&ctx->timeout_list, struct io_kiocb, list);
Jens Axboe93bd25b2019-11-11 23:34:31 -0700878 if (req) {
879 if (req->flags & REQ_F_TIMEOUT_NOSEQ)
880 return NULL;
Linus Torvaldsfb4b3d32019-11-25 10:40:27 -0800881 if (!__req_need_defer(req)) {
Jens Axboe93bd25b2019-11-11 23:34:31 -0700882 list_del_init(&req->list);
883 return req;
884 }
Jens Axboe7adf4ea2019-10-10 21:42:58 -0600885 }
886
887 return NULL;
Jens Axboe5262f562019-09-17 12:26:57 -0600888}
889
Jens Axboede0617e2019-04-06 21:51:27 -0600890static void __io_commit_cqring(struct io_ring_ctx *ctx)
Jens Axboe2b188cc2019-01-07 10:46:33 -0700891{
Hristo Venev75b28af2019-08-26 17:23:46 +0000892 struct io_rings *rings = ctx->rings;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700893
Pavel Begunkov07910152020-01-17 03:52:46 +0300894 /* order cqe stores with ring update */
895 smp_store_release(&rings->cq.tail, ctx->cached_cq_tail);
Jens Axboe2b188cc2019-01-07 10:46:33 -0700896
Pavel Begunkov07910152020-01-17 03:52:46 +0300897 if (wq_has_sleeper(&ctx->cq_wait)) {
898 wake_up_interruptible(&ctx->cq_wait);
899 kill_fasync(&ctx->cq_fasync, SIGIO, POLL_IN);
Jens Axboe2b188cc2019-01-07 10:46:33 -0700900 }
901}
902
Jens Axboecccf0ee2020-01-27 16:34:48 -0700903static inline void io_req_work_grab_env(struct io_kiocb *req,
904 const struct io_op_def *def)
Jens Axboe18d9be12019-09-10 09:13:05 -0600905{
Jens Axboecccf0ee2020-01-27 16:34:48 -0700906 if (!req->work.mm && def->needs_mm) {
907 mmgrab(current->mm);
908 req->work.mm = current->mm;
909 }
910 if (!req->work.creds)
911 req->work.creds = get_current_cred();
912}
913
914static inline void io_req_work_drop_env(struct io_kiocb *req)
915{
916 if (req->work.mm) {
917 mmdrop(req->work.mm);
918 req->work.mm = NULL;
919 }
920 if (req->work.creds) {
921 put_cred(req->work.creds);
922 req->work.creds = NULL;
923 }
Jens Axboe561fb042019-10-24 07:25:42 -0600924}
925
Jens Axboe94ae5e72019-11-14 19:39:52 -0700926static inline bool io_prep_async_work(struct io_kiocb *req,
927 struct io_kiocb **link)
Jens Axboe561fb042019-10-24 07:25:42 -0600928{
Jens Axboed3656342019-12-18 09:50:26 -0700929 const struct io_op_def *def = &io_op_defs[req->opcode];
Jens Axboe561fb042019-10-24 07:25:42 -0600930 bool do_hashed = false;
Jens Axboe54a91f32019-09-10 09:15:04 -0600931
Jens Axboed3656342019-12-18 09:50:26 -0700932 if (req->flags & REQ_F_ISREG) {
933 if (def->hash_reg_file)
Jens Axboe3529d8c2019-12-19 18:24:38 -0700934 do_hashed = true;
Jens Axboed3656342019-12-18 09:50:26 -0700935 } else {
936 if (def->unbound_nonreg_file)
Jens Axboe3529d8c2019-12-19 18:24:38 -0700937 req->work.flags |= IO_WQ_WORK_UNBOUND;
Jens Axboe54a91f32019-09-10 09:15:04 -0600938 }
Jens Axboecccf0ee2020-01-27 16:34:48 -0700939
940 io_req_work_grab_env(req, def);
Jens Axboe54a91f32019-09-10 09:15:04 -0600941
Jens Axboe94ae5e72019-11-14 19:39:52 -0700942 *link = io_prep_linked_timeout(req);
Jens Axboe561fb042019-10-24 07:25:42 -0600943 return do_hashed;
944}
945
Jackie Liua197f662019-11-08 08:09:12 -0700946static inline void io_queue_async_work(struct io_kiocb *req)
Jens Axboe561fb042019-10-24 07:25:42 -0600947{
Jackie Liua197f662019-11-08 08:09:12 -0700948 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe94ae5e72019-11-14 19:39:52 -0700949 struct io_kiocb *link;
950 bool do_hashed;
951
952 do_hashed = io_prep_async_work(req, &link);
Jens Axboe561fb042019-10-24 07:25:42 -0600953
954 trace_io_uring_queue_async_work(ctx, do_hashed, req, &req->work,
955 req->flags);
956 if (!do_hashed) {
957 io_wq_enqueue(ctx->io_wq, &req->work);
958 } else {
959 io_wq_enqueue_hashed(ctx->io_wq, &req->work,
960 file_inode(req->file));
961 }
Jens Axboe94ae5e72019-11-14 19:39:52 -0700962
963 if (link)
964 io_queue_linked_timeout(link);
Jens Axboe18d9be12019-09-10 09:13:05 -0600965}
966
Jens Axboe5262f562019-09-17 12:26:57 -0600967static void io_kill_timeout(struct io_kiocb *req)
968{
969 int ret;
970
Jens Axboe2d283902019-12-04 11:08:05 -0700971 ret = hrtimer_try_to_cancel(&req->io->timeout.timer);
Jens Axboe5262f562019-09-17 12:26:57 -0600972 if (ret != -1) {
973 atomic_inc(&req->ctx->cq_timeouts);
Jens Axboe842f9612019-10-29 12:34:10 -0600974 list_del_init(&req->list);
Jens Axboe78e19bb2019-11-06 15:21:34 -0700975 io_cqring_fill_event(req, 0);
Jackie Liuec9c02a2019-11-08 23:50:36 +0800976 io_put_req(req);
Jens Axboe5262f562019-09-17 12:26:57 -0600977 }
978}
979
980static void io_kill_timeouts(struct io_ring_ctx *ctx)
981{
982 struct io_kiocb *req, *tmp;
983
984 spin_lock_irq(&ctx->completion_lock);
985 list_for_each_entry_safe(req, tmp, &ctx->timeout_list, list)
986 io_kill_timeout(req);
987 spin_unlock_irq(&ctx->completion_lock);
988}
989
Jens Axboede0617e2019-04-06 21:51:27 -0600990static void io_commit_cqring(struct io_ring_ctx *ctx)
991{
992 struct io_kiocb *req;
993
Jens Axboe5262f562019-09-17 12:26:57 -0600994 while ((req = io_get_timeout_req(ctx)) != NULL)
995 io_kill_timeout(req);
996
Jens Axboede0617e2019-04-06 21:51:27 -0600997 __io_commit_cqring(ctx);
998
Pavel Begunkov87987892020-01-18 01:22:30 +0300999 while ((req = io_get_deferred_req(ctx)) != NULL)
Jackie Liua197f662019-11-08 08:09:12 -07001000 io_queue_async_work(req);
Jens Axboede0617e2019-04-06 21:51:27 -06001001}
1002
Jens Axboe2b188cc2019-01-07 10:46:33 -07001003static struct io_uring_cqe *io_get_cqring(struct io_ring_ctx *ctx)
1004{
Hristo Venev75b28af2019-08-26 17:23:46 +00001005 struct io_rings *rings = ctx->rings;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001006 unsigned tail;
1007
1008 tail = ctx->cached_cq_tail;
Stefan Bühler115e12e2019-04-24 23:54:18 +02001009 /*
1010 * writes to the cq entry need to come after reading head; the
1011 * control dependency is enough as we're using WRITE_ONCE to
1012 * fill the cq entry
1013 */
Hristo Venev75b28af2019-08-26 17:23:46 +00001014 if (tail - READ_ONCE(rings->cq.head) == rings->cq_ring_entries)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001015 return NULL;
1016
1017 ctx->cached_cq_tail++;
Hristo Venev75b28af2019-08-26 17:23:46 +00001018 return &rings->cqes[tail & ctx->cq_mask];
Jens Axboe2b188cc2019-01-07 10:46:33 -07001019}
1020
Jens Axboef2842ab2020-01-08 11:04:00 -07001021static inline bool io_should_trigger_evfd(struct io_ring_ctx *ctx)
1022{
Jens Axboef0b493e2020-02-01 21:30:11 -07001023 if (!ctx->cq_ev_fd)
1024 return false;
Jens Axboef2842ab2020-01-08 11:04:00 -07001025 if (!ctx->eventfd_async)
1026 return true;
1027 return io_wq_current_is_worker() || in_interrupt();
1028}
1029
Jens Axboef0b493e2020-02-01 21:30:11 -07001030static void __io_cqring_ev_posted(struct io_ring_ctx *ctx, bool trigger_ev)
Jens Axboe8c838782019-03-12 15:48:16 -06001031{
1032 if (waitqueue_active(&ctx->wait))
1033 wake_up(&ctx->wait);
1034 if (waitqueue_active(&ctx->sqo_wait))
1035 wake_up(&ctx->sqo_wait);
Jens Axboef0b493e2020-02-01 21:30:11 -07001036 if (trigger_ev)
Jens Axboe9b402842019-04-11 11:45:41 -06001037 eventfd_signal(ctx->cq_ev_fd, 1);
Jens Axboe8c838782019-03-12 15:48:16 -06001038}
1039
Jens Axboef0b493e2020-02-01 21:30:11 -07001040static void io_cqring_ev_posted(struct io_ring_ctx *ctx)
1041{
1042 __io_cqring_ev_posted(ctx, io_should_trigger_evfd(ctx));
1043}
1044
Jens Axboec4a2ed72019-11-21 21:01:26 -07001045/* Returns true if there are no backlogged entries after the flush */
1046static bool io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001047{
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001048 struct io_rings *rings = ctx->rings;
1049 struct io_uring_cqe *cqe;
1050 struct io_kiocb *req;
1051 unsigned long flags;
1052 LIST_HEAD(list);
1053
1054 if (!force) {
1055 if (list_empty_careful(&ctx->cq_overflow_list))
Jens Axboec4a2ed72019-11-21 21:01:26 -07001056 return true;
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001057 if ((ctx->cached_cq_tail - READ_ONCE(rings->cq.head) ==
1058 rings->cq_ring_entries))
Jens Axboec4a2ed72019-11-21 21:01:26 -07001059 return false;
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001060 }
1061
1062 spin_lock_irqsave(&ctx->completion_lock, flags);
1063
1064 /* if force is set, the ring is going away. always drop after that */
1065 if (force)
Jens Axboe69b3e542020-01-08 11:01:46 -07001066 ctx->cq_overflow_flushed = 1;
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001067
Jens Axboec4a2ed72019-11-21 21:01:26 -07001068 cqe = NULL;
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001069 while (!list_empty(&ctx->cq_overflow_list)) {
1070 cqe = io_get_cqring(ctx);
1071 if (!cqe && !force)
1072 break;
1073
1074 req = list_first_entry(&ctx->cq_overflow_list, struct io_kiocb,
1075 list);
1076 list_move(&req->list, &list);
1077 if (cqe) {
1078 WRITE_ONCE(cqe->user_data, req->user_data);
1079 WRITE_ONCE(cqe->res, req->result);
1080 WRITE_ONCE(cqe->flags, 0);
1081 } else {
1082 WRITE_ONCE(ctx->rings->cq_overflow,
1083 atomic_inc_return(&ctx->cached_cq_overflow));
1084 }
1085 }
1086
1087 io_commit_cqring(ctx);
Jens Axboead3eb2c2019-12-18 17:12:20 -07001088 if (cqe) {
1089 clear_bit(0, &ctx->sq_check_overflow);
1090 clear_bit(0, &ctx->cq_check_overflow);
1091 }
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001092 spin_unlock_irqrestore(&ctx->completion_lock, flags);
1093 io_cqring_ev_posted(ctx);
1094
1095 while (!list_empty(&list)) {
1096 req = list_first_entry(&list, struct io_kiocb, list);
1097 list_del(&req->list);
Jackie Liuec9c02a2019-11-08 23:50:36 +08001098 io_put_req(req);
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001099 }
Jens Axboec4a2ed72019-11-21 21:01:26 -07001100
1101 return cqe != NULL;
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001102}
1103
Jens Axboe78e19bb2019-11-06 15:21:34 -07001104static void io_cqring_fill_event(struct io_kiocb *req, long res)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001105{
Jens Axboe78e19bb2019-11-06 15:21:34 -07001106 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001107 struct io_uring_cqe *cqe;
1108
Jens Axboe78e19bb2019-11-06 15:21:34 -07001109 trace_io_uring_complete(ctx, req->user_data, res);
Jens Axboe51c3ff62019-11-03 06:52:50 -07001110
Jens Axboe2b188cc2019-01-07 10:46:33 -07001111 /*
1112 * If we can't get a cq entry, userspace overflowed the
1113 * submission (by quite a lot). Increment the overflow count in
1114 * the ring.
1115 */
1116 cqe = io_get_cqring(ctx);
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001117 if (likely(cqe)) {
Jens Axboe78e19bb2019-11-06 15:21:34 -07001118 WRITE_ONCE(cqe->user_data, req->user_data);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001119 WRITE_ONCE(cqe->res, res);
1120 WRITE_ONCE(cqe->flags, 0);
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001121 } else if (ctx->cq_overflow_flushed) {
Jens Axboe2b188cc2019-01-07 10:46:33 -07001122 WRITE_ONCE(ctx->rings->cq_overflow,
1123 atomic_inc_return(&ctx->cached_cq_overflow));
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001124 } else {
Jens Axboead3eb2c2019-12-18 17:12:20 -07001125 if (list_empty(&ctx->cq_overflow_list)) {
1126 set_bit(0, &ctx->sq_check_overflow);
1127 set_bit(0, &ctx->cq_check_overflow);
1128 }
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001129 refcount_inc(&req->refs);
1130 req->result = res;
1131 list_add_tail(&req->list, &ctx->cq_overflow_list);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001132 }
1133}
1134
Jens Axboe78e19bb2019-11-06 15:21:34 -07001135static void io_cqring_add_event(struct io_kiocb *req, long res)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001136{
Jens Axboe78e19bb2019-11-06 15:21:34 -07001137 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001138 unsigned long flags;
1139
1140 spin_lock_irqsave(&ctx->completion_lock, flags);
Jens Axboe78e19bb2019-11-06 15:21:34 -07001141 io_cqring_fill_event(req, res);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001142 io_commit_cqring(ctx);
1143 spin_unlock_irqrestore(&ctx->completion_lock, flags);
1144
Jens Axboe8c838782019-03-12 15:48:16 -06001145 io_cqring_ev_posted(ctx);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001146}
1147
Jens Axboe0ddf92e2019-11-08 08:52:53 -07001148static inline bool io_is_fallback_req(struct io_kiocb *req)
1149{
1150 return req == (struct io_kiocb *)
1151 ((unsigned long) req->ctx->fallback_req & ~1UL);
1152}
1153
1154static struct io_kiocb *io_get_fallback_req(struct io_ring_ctx *ctx)
1155{
1156 struct io_kiocb *req;
1157
1158 req = ctx->fallback_req;
1159 if (!test_and_set_bit_lock(0, (unsigned long *) ctx->fallback_req))
1160 return req;
1161
1162 return NULL;
1163}
1164
Jens Axboe2579f912019-01-09 09:10:43 -07001165static struct io_kiocb *io_get_req(struct io_ring_ctx *ctx,
1166 struct io_submit_state *state)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001167{
Jens Axboefd6fab22019-03-14 16:30:06 -06001168 gfp_t gfp = GFP_KERNEL | __GFP_NOWARN;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001169 struct io_kiocb *req;
1170
Jens Axboe2579f912019-01-09 09:10:43 -07001171 if (!state) {
Jens Axboefd6fab22019-03-14 16:30:06 -06001172 req = kmem_cache_alloc(req_cachep, gfp);
Jens Axboe2579f912019-01-09 09:10:43 -07001173 if (unlikely(!req))
Jens Axboe0ddf92e2019-11-08 08:52:53 -07001174 goto fallback;
Jens Axboe2579f912019-01-09 09:10:43 -07001175 } else if (!state->free_reqs) {
1176 size_t sz;
1177 int ret;
1178
1179 sz = min_t(size_t, state->ios_left, ARRAY_SIZE(state->reqs));
Jens Axboefd6fab22019-03-14 16:30:06 -06001180 ret = kmem_cache_alloc_bulk(req_cachep, gfp, sz, state->reqs);
1181
1182 /*
1183 * Bulk alloc is all-or-nothing. If we fail to get a batch,
1184 * retry single alloc to be on the safe side.
1185 */
1186 if (unlikely(ret <= 0)) {
1187 state->reqs[0] = kmem_cache_alloc(req_cachep, gfp);
1188 if (!state->reqs[0])
Jens Axboe0ddf92e2019-11-08 08:52:53 -07001189 goto fallback;
Jens Axboefd6fab22019-03-14 16:30:06 -06001190 ret = 1;
1191 }
Jens Axboe2579f912019-01-09 09:10:43 -07001192 state->free_reqs = ret - 1;
1193 state->cur_req = 1;
1194 req = state->reqs[0];
1195 } else {
1196 req = state->reqs[state->cur_req];
1197 state->free_reqs--;
1198 state->cur_req++;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001199 }
1200
Jens Axboe0ddf92e2019-11-08 08:52:53 -07001201got_it:
Jens Axboe1a6b74f2019-12-02 10:33:15 -07001202 req->io = NULL;
Jens Axboe60c112b2019-06-21 10:20:18 -06001203 req->file = NULL;
Jens Axboe2579f912019-01-09 09:10:43 -07001204 req->ctx = ctx;
1205 req->flags = 0;
Jens Axboee65ef562019-03-12 10:16:44 -06001206 /* one is dropped after submission, the other at completion */
1207 refcount_set(&req->refs, 2);
Jens Axboe9e645e112019-05-10 16:07:28 -06001208 req->result = 0;
Jens Axboe561fb042019-10-24 07:25:42 -06001209 INIT_IO_WORK(&req->work, io_wq_submit_work);
Jens Axboe2579f912019-01-09 09:10:43 -07001210 return req;
Jens Axboe0ddf92e2019-11-08 08:52:53 -07001211fallback:
1212 req = io_get_fallback_req(ctx);
1213 if (req)
1214 goto got_it;
Pavel Begunkov6805b322019-10-08 02:18:42 +03001215 percpu_ref_put(&ctx->refs);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001216 return NULL;
1217}
1218
Pavel Begunkov2b85edf2019-12-28 14:13:03 +03001219static void __io_req_do_free(struct io_kiocb *req)
Jens Axboedef596e2019-01-09 08:59:42 -07001220{
Pavel Begunkov2b85edf2019-12-28 14:13:03 +03001221 if (likely(!io_is_fallback_req(req)))
1222 kmem_cache_free(req_cachep, req);
1223 else
1224 clear_bit_unlock(0, (unsigned long *) req->ctx->fallback_req);
1225}
1226
Jens Axboec6ca97b302019-12-28 12:11:08 -07001227static void __io_req_aux_free(struct io_kiocb *req)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001228{
Jens Axboefcb323c2019-10-24 12:39:47 -06001229 struct io_ring_ctx *ctx = req->ctx;
1230
YueHaibing96fd84d2020-01-07 22:22:44 +08001231 kfree(req->io);
Jens Axboe05f3fb32019-12-09 11:22:50 -07001232 if (req->file) {
1233 if (req->flags & REQ_F_FIXED_FILE)
1234 percpu_ref_put(&ctx->file_data->refs);
1235 else
1236 fput(req->file);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001237 }
Jens Axboecccf0ee2020-01-27 16:34:48 -07001238
1239 io_req_work_drop_env(req);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001240}
1241
1242static void __io_free_req(struct io_kiocb *req)
1243{
Jens Axboec6ca97b302019-12-28 12:11:08 -07001244 __io_req_aux_free(req);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001245
Jens Axboefcb323c2019-10-24 12:39:47 -06001246 if (req->flags & REQ_F_INFLIGHT) {
Jens Axboec6ca97b302019-12-28 12:11:08 -07001247 struct io_ring_ctx *ctx = req->ctx;
Jens Axboefcb323c2019-10-24 12:39:47 -06001248 unsigned long flags;
1249
1250 spin_lock_irqsave(&ctx->inflight_lock, flags);
1251 list_del(&req->inflight_entry);
1252 if (waitqueue_active(&ctx->inflight_wait))
1253 wake_up(&ctx->inflight_wait);
1254 spin_unlock_irqrestore(&ctx->inflight_lock, flags);
1255 }
Pavel Begunkov2b85edf2019-12-28 14:13:03 +03001256
1257 percpu_ref_put(&req->ctx->refs);
1258 __io_req_do_free(req);
Jens Axboee65ef562019-03-12 10:16:44 -06001259}
1260
Jens Axboec6ca97b302019-12-28 12:11:08 -07001261struct req_batch {
1262 void *reqs[IO_IOPOLL_BATCH];
1263 int to_free;
1264 int need_iter;
1265};
1266
1267static void io_free_req_many(struct io_ring_ctx *ctx, struct req_batch *rb)
1268{
Jens Axboe10fef4b2020-01-09 07:52:28 -07001269 int fixed_refs = rb->to_free;
1270
Jens Axboec6ca97b302019-12-28 12:11:08 -07001271 if (!rb->to_free)
1272 return;
1273 if (rb->need_iter) {
1274 int i, inflight = 0;
1275 unsigned long flags;
1276
Jens Axboe10fef4b2020-01-09 07:52:28 -07001277 fixed_refs = 0;
Jens Axboec6ca97b302019-12-28 12:11:08 -07001278 for (i = 0; i < rb->to_free; i++) {
1279 struct io_kiocb *req = rb->reqs[i];
1280
Jens Axboe10fef4b2020-01-09 07:52:28 -07001281 if (req->flags & REQ_F_FIXED_FILE) {
Jens Axboec6ca97b302019-12-28 12:11:08 -07001282 req->file = NULL;
Jens Axboe10fef4b2020-01-09 07:52:28 -07001283 fixed_refs++;
1284 }
Jens Axboec6ca97b302019-12-28 12:11:08 -07001285 if (req->flags & REQ_F_INFLIGHT)
1286 inflight++;
Jens Axboec6ca97b302019-12-28 12:11:08 -07001287 __io_req_aux_free(req);
1288 }
1289 if (!inflight)
1290 goto do_free;
1291
1292 spin_lock_irqsave(&ctx->inflight_lock, flags);
1293 for (i = 0; i < rb->to_free; i++) {
1294 struct io_kiocb *req = rb->reqs[i];
1295
Jens Axboe10fef4b2020-01-09 07:52:28 -07001296 if (req->flags & REQ_F_INFLIGHT) {
Jens Axboec6ca97b302019-12-28 12:11:08 -07001297 list_del(&req->inflight_entry);
1298 if (!--inflight)
1299 break;
1300 }
1301 }
1302 spin_unlock_irqrestore(&ctx->inflight_lock, flags);
1303
1304 if (waitqueue_active(&ctx->inflight_wait))
1305 wake_up(&ctx->inflight_wait);
1306 }
1307do_free:
1308 kmem_cache_free_bulk(req_cachep, rb->to_free, rb->reqs);
Jens Axboe10fef4b2020-01-09 07:52:28 -07001309 if (fixed_refs)
1310 percpu_ref_put_many(&ctx->file_data->refs, fixed_refs);
Jens Axboec6ca97b302019-12-28 12:11:08 -07001311 percpu_ref_put_many(&ctx->refs, rb->to_free);
Jens Axboec6ca97b302019-12-28 12:11:08 -07001312 rb->to_free = rb->need_iter = 0;
Jens Axboee65ef562019-03-12 10:16:44 -06001313}
1314
Jackie Liua197f662019-11-08 08:09:12 -07001315static bool io_link_cancel_timeout(struct io_kiocb *req)
Jens Axboe9e645e112019-05-10 16:07:28 -06001316{
Jackie Liua197f662019-11-08 08:09:12 -07001317 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe2665abf2019-11-05 12:40:47 -07001318 int ret;
1319
Jens Axboe2d283902019-12-04 11:08:05 -07001320 ret = hrtimer_try_to_cancel(&req->io->timeout.timer);
Jens Axboe2665abf2019-11-05 12:40:47 -07001321 if (ret != -1) {
Jens Axboe78e19bb2019-11-06 15:21:34 -07001322 io_cqring_fill_event(req, -ECANCELED);
Jens Axboe2665abf2019-11-05 12:40:47 -07001323 io_commit_cqring(ctx);
1324 req->flags &= ~REQ_F_LINK;
Jackie Liuec9c02a2019-11-08 23:50:36 +08001325 io_put_req(req);
Jens Axboe2665abf2019-11-05 12:40:47 -07001326 return true;
1327 }
1328
1329 return false;
1330}
1331
Jens Axboeba816ad2019-09-28 11:36:45 -06001332static void io_req_link_next(struct io_kiocb *req, struct io_kiocb **nxtptr)
Jens Axboe9e645e112019-05-10 16:07:28 -06001333{
Jens Axboe2665abf2019-11-05 12:40:47 -07001334 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe2665abf2019-11-05 12:40:47 -07001335 bool wake_ev = false;
Jens Axboe9e645e112019-05-10 16:07:28 -06001336
Jens Axboe4d7dd462019-11-20 13:03:52 -07001337 /* Already got next link */
1338 if (req->flags & REQ_F_LINK_NEXT)
1339 return;
1340
Jens Axboe9e645e112019-05-10 16:07:28 -06001341 /*
1342 * The list should never be empty when we are called here. But could
1343 * potentially happen if the chain is messed up, check to be on the
1344 * safe side.
1345 */
Pavel Begunkov44932332019-12-05 16:16:35 +03001346 while (!list_empty(&req->link_list)) {
1347 struct io_kiocb *nxt = list_first_entry(&req->link_list,
1348 struct io_kiocb, link_list);
Jens Axboe94ae5e72019-11-14 19:39:52 -07001349
Pavel Begunkov44932332019-12-05 16:16:35 +03001350 if (unlikely((req->flags & REQ_F_LINK_TIMEOUT) &&
1351 (nxt->flags & REQ_F_TIMEOUT))) {
1352 list_del_init(&nxt->link_list);
Jens Axboe94ae5e72019-11-14 19:39:52 -07001353 wake_ev |= io_link_cancel_timeout(nxt);
Jens Axboe94ae5e72019-11-14 19:39:52 -07001354 req->flags &= ~REQ_F_LINK_TIMEOUT;
1355 continue;
1356 }
Jens Axboe9e645e112019-05-10 16:07:28 -06001357
Pavel Begunkov44932332019-12-05 16:16:35 +03001358 list_del_init(&req->link_list);
1359 if (!list_empty(&nxt->link_list))
1360 nxt->flags |= REQ_F_LINK;
Pavel Begunkovb18fdf72019-11-21 23:21:02 +03001361 *nxtptr = nxt;
Jens Axboe94ae5e72019-11-14 19:39:52 -07001362 break;
Jens Axboe9e645e112019-05-10 16:07:28 -06001363 }
Jens Axboe2665abf2019-11-05 12:40:47 -07001364
Jens Axboe4d7dd462019-11-20 13:03:52 -07001365 req->flags |= REQ_F_LINK_NEXT;
Jens Axboe2665abf2019-11-05 12:40:47 -07001366 if (wake_ev)
1367 io_cqring_ev_posted(ctx);
Jens Axboe9e645e112019-05-10 16:07:28 -06001368}
1369
1370/*
1371 * Called if REQ_F_LINK is set, and we fail the head request
1372 */
1373static void io_fail_links(struct io_kiocb *req)
1374{
Jens Axboe2665abf2019-11-05 12:40:47 -07001375 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe2665abf2019-11-05 12:40:47 -07001376 unsigned long flags;
1377
1378 spin_lock_irqsave(&ctx->completion_lock, flags);
Jens Axboe9e645e112019-05-10 16:07:28 -06001379
1380 while (!list_empty(&req->link_list)) {
Pavel Begunkov44932332019-12-05 16:16:35 +03001381 struct io_kiocb *link = list_first_entry(&req->link_list,
1382 struct io_kiocb, link_list);
Jens Axboe9e645e112019-05-10 16:07:28 -06001383
Pavel Begunkov44932332019-12-05 16:16:35 +03001384 list_del_init(&link->link_list);
Dmitrii Dolgovc826bd72019-10-15 19:02:01 +02001385 trace_io_uring_fail_link(req, link);
Jens Axboe2665abf2019-11-05 12:40:47 -07001386
1387 if ((req->flags & REQ_F_LINK_TIMEOUT) &&
Jens Axboed625c6e2019-12-17 19:53:05 -07001388 link->opcode == IORING_OP_LINK_TIMEOUT) {
Jackie Liua197f662019-11-08 08:09:12 -07001389 io_link_cancel_timeout(link);
Jens Axboe2665abf2019-11-05 12:40:47 -07001390 } else {
Jens Axboe78e19bb2019-11-06 15:21:34 -07001391 io_cqring_fill_event(link, -ECANCELED);
Jens Axboe978db572019-11-14 22:39:04 -07001392 __io_double_put_req(link);
Jens Axboe2665abf2019-11-05 12:40:47 -07001393 }
Jens Axboe5d960722019-11-19 15:31:28 -07001394 req->flags &= ~REQ_F_LINK_TIMEOUT;
Jens Axboe9e645e112019-05-10 16:07:28 -06001395 }
Jens Axboe2665abf2019-11-05 12:40:47 -07001396
1397 io_commit_cqring(ctx);
1398 spin_unlock_irqrestore(&ctx->completion_lock, flags);
1399 io_cqring_ev_posted(ctx);
Jens Axboe9e645e112019-05-10 16:07:28 -06001400}
1401
Jens Axboe4d7dd462019-11-20 13:03:52 -07001402static void io_req_find_next(struct io_kiocb *req, struct io_kiocb **nxt)
Jens Axboe9e645e112019-05-10 16:07:28 -06001403{
Jens Axboe4d7dd462019-11-20 13:03:52 -07001404 if (likely(!(req->flags & REQ_F_LINK)))
Jens Axboe2665abf2019-11-05 12:40:47 -07001405 return;
Jens Axboe2665abf2019-11-05 12:40:47 -07001406
Jens Axboe9e645e112019-05-10 16:07:28 -06001407 /*
1408 * If LINK is set, we have dependent requests in this chain. If we
1409 * didn't fail this request, queue the first one up, moving any other
1410 * dependencies to the next request. In case of failure, fail the rest
1411 * of the chain.
1412 */
Jens Axboe2665abf2019-11-05 12:40:47 -07001413 if (req->flags & REQ_F_FAIL_LINK) {
1414 io_fail_links(req);
Jens Axboe7c9e7f02019-11-12 08:15:53 -07001415 } else if ((req->flags & (REQ_F_LINK_TIMEOUT | REQ_F_COMP_LOCKED)) ==
1416 REQ_F_LINK_TIMEOUT) {
Jens Axboe2665abf2019-11-05 12:40:47 -07001417 struct io_ring_ctx *ctx = req->ctx;
1418 unsigned long flags;
1419
1420 /*
1421 * If this is a timeout link, we could be racing with the
1422 * timeout timer. Grab the completion lock for this case to
Jens Axboe7c9e7f02019-11-12 08:15:53 -07001423 * protect against that.
Jens Axboe2665abf2019-11-05 12:40:47 -07001424 */
1425 spin_lock_irqsave(&ctx->completion_lock, flags);
1426 io_req_link_next(req, nxt);
1427 spin_unlock_irqrestore(&ctx->completion_lock, flags);
1428 } else {
1429 io_req_link_next(req, nxt);
Jens Axboe9e645e112019-05-10 16:07:28 -06001430 }
Jens Axboe4d7dd462019-11-20 13:03:52 -07001431}
Jens Axboe9e645e112019-05-10 16:07:28 -06001432
Jackie Liuc69f8db2019-11-09 11:00:08 +08001433static void io_free_req(struct io_kiocb *req)
1434{
Pavel Begunkov944e58b2019-11-21 23:21:01 +03001435 struct io_kiocb *nxt = NULL;
1436
1437 io_req_find_next(req, &nxt);
Pavel Begunkov70cf9f32019-11-21 23:21:00 +03001438 __io_free_req(req);
Pavel Begunkov944e58b2019-11-21 23:21:01 +03001439
1440 if (nxt)
1441 io_queue_async_work(nxt);
Jackie Liuc69f8db2019-11-09 11:00:08 +08001442}
1443
Jens Axboeba816ad2019-09-28 11:36:45 -06001444/*
1445 * Drop reference to request, return next in chain (if there is one) if this
1446 * was the last reference to this request.
1447 */
Pavel Begunkovf9bd67f2019-11-21 23:21:03 +03001448__attribute__((nonnull))
Jackie Liuec9c02a2019-11-08 23:50:36 +08001449static void io_put_req_find_next(struct io_kiocb *req, struct io_kiocb **nxtptr)
Jens Axboee65ef562019-03-12 10:16:44 -06001450{
Pavel Begunkovf9bd67f2019-11-21 23:21:03 +03001451 io_req_find_next(req, nxtptr);
Jens Axboe4d7dd462019-11-20 13:03:52 -07001452
Jens Axboee65ef562019-03-12 10:16:44 -06001453 if (refcount_dec_and_test(&req->refs))
Jens Axboe4d7dd462019-11-20 13:03:52 -07001454 __io_free_req(req);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001455}
1456
Jens Axboe2b188cc2019-01-07 10:46:33 -07001457static void io_put_req(struct io_kiocb *req)
1458{
Jens Axboedef596e2019-01-09 08:59:42 -07001459 if (refcount_dec_and_test(&req->refs))
1460 io_free_req(req);
1461}
1462
Jens Axboe978db572019-11-14 22:39:04 -07001463/*
1464 * Must only be used if we don't need to care about links, usually from
1465 * within the completion handling itself.
1466 */
1467static void __io_double_put_req(struct io_kiocb *req)
Jens Axboea3a0e432019-08-20 11:03:11 -06001468{
Jens Axboe78e19bb2019-11-06 15:21:34 -07001469 /* drop both submit and complete references */
1470 if (refcount_sub_and_test(2, &req->refs))
1471 __io_free_req(req);
1472}
1473
Jens Axboe978db572019-11-14 22:39:04 -07001474static void io_double_put_req(struct io_kiocb *req)
1475{
1476 /* drop both submit and complete references */
1477 if (refcount_sub_and_test(2, &req->refs))
1478 io_free_req(req);
1479}
1480
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001481static unsigned io_cqring_events(struct io_ring_ctx *ctx, bool noflush)
Jens Axboea3a0e432019-08-20 11:03:11 -06001482{
Jens Axboe84f97dc2019-11-06 11:27:53 -07001483 struct io_rings *rings = ctx->rings;
1484
Jens Axboead3eb2c2019-12-18 17:12:20 -07001485 if (test_bit(0, &ctx->cq_check_overflow)) {
1486 /*
1487 * noflush == true is from the waitqueue handler, just ensure
1488 * we wake up the task, and the next invocation will flush the
1489 * entries. We cannot safely to it from here.
1490 */
1491 if (noflush && !list_empty(&ctx->cq_overflow_list))
1492 return -1U;
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001493
Jens Axboead3eb2c2019-12-18 17:12:20 -07001494 io_cqring_overflow_flush(ctx, false);
1495 }
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001496
Jens Axboea3a0e432019-08-20 11:03:11 -06001497 /* See comment at the top of this file */
1498 smp_rmb();
Jens Axboead3eb2c2019-12-18 17:12:20 -07001499 return ctx->cached_cq_tail - READ_ONCE(rings->cq.head);
Jens Axboea3a0e432019-08-20 11:03:11 -06001500}
1501
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03001502static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx)
1503{
1504 struct io_rings *rings = ctx->rings;
1505
1506 /* make sure SQ entry isn't read before tail */
1507 return smp_load_acquire(&rings->sq.tail) - ctx->cached_sq_head;
1508}
1509
Jens Axboe8237e042019-12-28 10:48:22 -07001510static inline bool io_req_multi_free(struct req_batch *rb, struct io_kiocb *req)
Jens Axboee94f1412019-12-19 12:06:02 -07001511{
Jens Axboec6ca97b302019-12-28 12:11:08 -07001512 if ((req->flags & REQ_F_LINK) || io_is_fallback_req(req))
1513 return false;
Jens Axboee94f1412019-12-19 12:06:02 -07001514
Jens Axboec6ca97b302019-12-28 12:11:08 -07001515 if (!(req->flags & REQ_F_FIXED_FILE) || req->io)
1516 rb->need_iter++;
1517
1518 rb->reqs[rb->to_free++] = req;
1519 if (unlikely(rb->to_free == ARRAY_SIZE(rb->reqs)))
1520 io_free_req_many(req->ctx, rb);
1521 return true;
Jens Axboee94f1412019-12-19 12:06:02 -07001522}
1523
Jens Axboedef596e2019-01-09 08:59:42 -07001524/*
1525 * Find and free completed poll iocbs
1526 */
1527static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events,
1528 struct list_head *done)
1529{
Jens Axboe8237e042019-12-28 10:48:22 -07001530 struct req_batch rb;
Jens Axboedef596e2019-01-09 08:59:42 -07001531 struct io_kiocb *req;
Jens Axboedef596e2019-01-09 08:59:42 -07001532
Jens Axboec6ca97b302019-12-28 12:11:08 -07001533 rb.to_free = rb.need_iter = 0;
Jens Axboedef596e2019-01-09 08:59:42 -07001534 while (!list_empty(done)) {
1535 req = list_first_entry(done, struct io_kiocb, list);
1536 list_del(&req->list);
1537
Jens Axboe78e19bb2019-11-06 15:21:34 -07001538 io_cqring_fill_event(req, req->result);
Jens Axboedef596e2019-01-09 08:59:42 -07001539 (*nr_events)++;
1540
Jens Axboe8237e042019-12-28 10:48:22 -07001541 if (refcount_dec_and_test(&req->refs) &&
1542 !io_req_multi_free(&rb, req))
1543 io_free_req(req);
Jens Axboedef596e2019-01-09 08:59:42 -07001544 }
Jens Axboedef596e2019-01-09 08:59:42 -07001545
Jens Axboe09bb8392019-03-13 12:39:28 -06001546 io_commit_cqring(ctx);
Jens Axboe8237e042019-12-28 10:48:22 -07001547 io_free_req_many(ctx, &rb);
Jens Axboedef596e2019-01-09 08:59:42 -07001548}
1549
1550static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events,
1551 long min)
1552{
1553 struct io_kiocb *req, *tmp;
1554 LIST_HEAD(done);
1555 bool spin;
1556 int ret;
1557
1558 /*
1559 * Only spin for completions if we don't have multiple devices hanging
1560 * off our complete list, and we're under the requested amount.
1561 */
1562 spin = !ctx->poll_multi_file && *nr_events < min;
1563
1564 ret = 0;
1565 list_for_each_entry_safe(req, tmp, &ctx->poll_list, list) {
Jens Axboe9adbd452019-12-20 08:45:55 -07001566 struct kiocb *kiocb = &req->rw.kiocb;
Jens Axboedef596e2019-01-09 08:59:42 -07001567
1568 /*
1569 * Move completed entries to our local list. If we find a
1570 * request that requires polling, break out and complete
1571 * the done list first, if we have entries there.
1572 */
1573 if (req->flags & REQ_F_IOPOLL_COMPLETED) {
1574 list_move_tail(&req->list, &done);
1575 continue;
1576 }
1577 if (!list_empty(&done))
1578 break;
1579
1580 ret = kiocb->ki_filp->f_op->iopoll(kiocb, spin);
1581 if (ret < 0)
1582 break;
1583
1584 if (ret && spin)
1585 spin = false;
1586 ret = 0;
1587 }
1588
1589 if (!list_empty(&done))
1590 io_iopoll_complete(ctx, nr_events, &done);
1591
1592 return ret;
1593}
1594
1595/*
Brian Gianforcarod195a662019-12-13 03:09:50 -08001596 * Poll for a minimum of 'min' events. Note that if min == 0 we consider that a
Jens Axboedef596e2019-01-09 08:59:42 -07001597 * non-spinning poll check - we'll still enter the driver poll loop, but only
1598 * as a non-spinning completion check.
1599 */
1600static int io_iopoll_getevents(struct io_ring_ctx *ctx, unsigned int *nr_events,
1601 long min)
1602{
Jens Axboe08f54392019-08-21 22:19:11 -06001603 while (!list_empty(&ctx->poll_list) && !need_resched()) {
Jens Axboedef596e2019-01-09 08:59:42 -07001604 int ret;
1605
1606 ret = io_do_iopoll(ctx, nr_events, min);
1607 if (ret < 0)
1608 return ret;
1609 if (!min || *nr_events >= min)
1610 return 0;
1611 }
1612
1613 return 1;
1614}
1615
1616/*
1617 * We can't just wait for polled events to come to us, we have to actively
1618 * find and complete them.
1619 */
1620static void io_iopoll_reap_events(struct io_ring_ctx *ctx)
1621{
1622 if (!(ctx->flags & IORING_SETUP_IOPOLL))
1623 return;
1624
1625 mutex_lock(&ctx->uring_lock);
1626 while (!list_empty(&ctx->poll_list)) {
1627 unsigned int nr_events = 0;
1628
1629 io_iopoll_getevents(ctx, &nr_events, 1);
Jens Axboe08f54392019-08-21 22:19:11 -06001630
1631 /*
1632 * Ensure we allow local-to-the-cpu processing to take place,
1633 * in this case we need to ensure that we reap all events.
1634 */
1635 cond_resched();
Jens Axboedef596e2019-01-09 08:59:42 -07001636 }
1637 mutex_unlock(&ctx->uring_lock);
1638}
1639
Jens Axboe2b2ed972019-10-25 10:06:15 -06001640static int __io_iopoll_check(struct io_ring_ctx *ctx, unsigned *nr_events,
1641 long min)
Jens Axboedef596e2019-01-09 08:59:42 -07001642{
Jens Axboe2b2ed972019-10-25 10:06:15 -06001643 int iters = 0, ret = 0;
Jens Axboedef596e2019-01-09 08:59:42 -07001644
1645 do {
1646 int tmin = 0;
1647
Jens Axboe500f9fb2019-08-19 12:15:59 -06001648 /*
Jens Axboea3a0e432019-08-20 11:03:11 -06001649 * Don't enter poll loop if we already have events pending.
1650 * If we do, we can potentially be spinning for commands that
1651 * already triggered a CQE (eg in error).
1652 */
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001653 if (io_cqring_events(ctx, false))
Jens Axboea3a0e432019-08-20 11:03:11 -06001654 break;
1655
1656 /*
Jens Axboe500f9fb2019-08-19 12:15:59 -06001657 * If a submit got punted to a workqueue, we can have the
1658 * application entering polling for a command before it gets
1659 * issued. That app will hold the uring_lock for the duration
1660 * of the poll right here, so we need to take a breather every
1661 * now and then to ensure that the issue has a chance to add
1662 * the poll to the issued list. Otherwise we can spin here
1663 * forever, while the workqueue is stuck trying to acquire the
1664 * very same mutex.
1665 */
1666 if (!(++iters & 7)) {
1667 mutex_unlock(&ctx->uring_lock);
1668 mutex_lock(&ctx->uring_lock);
1669 }
1670
Jens Axboedef596e2019-01-09 08:59:42 -07001671 if (*nr_events < min)
1672 tmin = min - *nr_events;
1673
1674 ret = io_iopoll_getevents(ctx, nr_events, tmin);
1675 if (ret <= 0)
1676 break;
1677 ret = 0;
1678 } while (min && !*nr_events && !need_resched());
1679
Jens Axboe2b2ed972019-10-25 10:06:15 -06001680 return ret;
1681}
1682
1683static int io_iopoll_check(struct io_ring_ctx *ctx, unsigned *nr_events,
1684 long min)
1685{
1686 int ret;
1687
1688 /*
1689 * We disallow the app entering submit/complete with polling, but we
1690 * still need to lock the ring to prevent racing with polled issue
1691 * that got punted to a workqueue.
1692 */
1693 mutex_lock(&ctx->uring_lock);
1694 ret = __io_iopoll_check(ctx, nr_events, min);
Jens Axboe500f9fb2019-08-19 12:15:59 -06001695 mutex_unlock(&ctx->uring_lock);
Jens Axboedef596e2019-01-09 08:59:42 -07001696 return ret;
1697}
1698
Jens Axboe491381ce2019-10-17 09:20:46 -06001699static void kiocb_end_write(struct io_kiocb *req)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001700{
Jens Axboe491381ce2019-10-17 09:20:46 -06001701 /*
1702 * Tell lockdep we inherited freeze protection from submission
1703 * thread.
1704 */
1705 if (req->flags & REQ_F_ISREG) {
1706 struct inode *inode = file_inode(req->file);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001707
Jens Axboe491381ce2019-10-17 09:20:46 -06001708 __sb_writers_acquired(inode->i_sb, SB_FREEZE_WRITE);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001709 }
Jens Axboe491381ce2019-10-17 09:20:46 -06001710 file_end_write(req->file);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001711}
1712
Jens Axboe4e88d6e2019-12-07 20:59:47 -07001713static inline void req_set_fail_links(struct io_kiocb *req)
1714{
1715 if ((req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) == REQ_F_LINK)
1716 req->flags |= REQ_F_FAIL_LINK;
1717}
1718
Jens Axboeba816ad2019-09-28 11:36:45 -06001719static void io_complete_rw_common(struct kiocb *kiocb, long res)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001720{
Jens Axboe9adbd452019-12-20 08:45:55 -07001721 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001722
Jens Axboe491381ce2019-10-17 09:20:46 -06001723 if (kiocb->ki_flags & IOCB_WRITE)
1724 kiocb_end_write(req);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001725
Jens Axboe4e88d6e2019-12-07 20:59:47 -07001726 if (res != req->result)
1727 req_set_fail_links(req);
Jens Axboe78e19bb2019-11-06 15:21:34 -07001728 io_cqring_add_event(req, res);
Jens Axboeba816ad2019-09-28 11:36:45 -06001729}
1730
1731static void io_complete_rw(struct kiocb *kiocb, long res, long res2)
1732{
Jens Axboe9adbd452019-12-20 08:45:55 -07001733 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
Jens Axboeba816ad2019-09-28 11:36:45 -06001734
1735 io_complete_rw_common(kiocb, res);
Jens Axboee65ef562019-03-12 10:16:44 -06001736 io_put_req(req);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001737}
1738
Jens Axboeba816ad2019-09-28 11:36:45 -06001739static struct io_kiocb *__io_complete_rw(struct kiocb *kiocb, long res)
1740{
Jens Axboe9adbd452019-12-20 08:45:55 -07001741 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
Jackie Liuec9c02a2019-11-08 23:50:36 +08001742 struct io_kiocb *nxt = NULL;
Jens Axboeba816ad2019-09-28 11:36:45 -06001743
1744 io_complete_rw_common(kiocb, res);
Jackie Liuec9c02a2019-11-08 23:50:36 +08001745 io_put_req_find_next(req, &nxt);
1746
1747 return nxt;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001748}
1749
Jens Axboedef596e2019-01-09 08:59:42 -07001750static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2)
1751{
Jens Axboe9adbd452019-12-20 08:45:55 -07001752 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
Jens Axboedef596e2019-01-09 08:59:42 -07001753
Jens Axboe491381ce2019-10-17 09:20:46 -06001754 if (kiocb->ki_flags & IOCB_WRITE)
1755 kiocb_end_write(req);
Jens Axboedef596e2019-01-09 08:59:42 -07001756
Jens Axboe4e88d6e2019-12-07 20:59:47 -07001757 if (res != req->result)
1758 req_set_fail_links(req);
Jens Axboe9e645e112019-05-10 16:07:28 -06001759 req->result = res;
Jens Axboedef596e2019-01-09 08:59:42 -07001760 if (res != -EAGAIN)
1761 req->flags |= REQ_F_IOPOLL_COMPLETED;
1762}
1763
1764/*
1765 * After the iocb has been issued, it's safe to be found on the poll list.
1766 * Adding the kiocb to the list AFTER submission ensures that we don't
1767 * find it from a io_iopoll_getevents() thread before the issuer is done
1768 * accessing the kiocb cookie.
1769 */
1770static void io_iopoll_req_issued(struct io_kiocb *req)
1771{
1772 struct io_ring_ctx *ctx = req->ctx;
1773
1774 /*
1775 * Track whether we have multiple files in our lists. This will impact
1776 * how we do polling eventually, not spinning if we're on potentially
1777 * different devices.
1778 */
1779 if (list_empty(&ctx->poll_list)) {
1780 ctx->poll_multi_file = false;
1781 } else if (!ctx->poll_multi_file) {
1782 struct io_kiocb *list_req;
1783
1784 list_req = list_first_entry(&ctx->poll_list, struct io_kiocb,
1785 list);
Jens Axboe9adbd452019-12-20 08:45:55 -07001786 if (list_req->file != req->file)
Jens Axboedef596e2019-01-09 08:59:42 -07001787 ctx->poll_multi_file = true;
1788 }
1789
1790 /*
1791 * For fast devices, IO may have already completed. If it has, add
1792 * it to the front so we find it first.
1793 */
1794 if (req->flags & REQ_F_IOPOLL_COMPLETED)
1795 list_add(&req->list, &ctx->poll_list);
1796 else
1797 list_add_tail(&req->list, &ctx->poll_list);
1798}
1799
Jens Axboe3d6770f2019-04-13 11:50:54 -06001800static void io_file_put(struct io_submit_state *state)
Jens Axboe9a56a232019-01-09 09:06:50 -07001801{
Jens Axboe3d6770f2019-04-13 11:50:54 -06001802 if (state->file) {
Jens Axboe9a56a232019-01-09 09:06:50 -07001803 int diff = state->has_refs - state->used_refs;
1804
1805 if (diff)
1806 fput_many(state->file, diff);
1807 state->file = NULL;
1808 }
1809}
1810
1811/*
1812 * Get as many references to a file as we have IOs left in this submission,
1813 * assuming most submissions are for one file, or at least that each file
1814 * has more than one submission.
1815 */
1816static struct file *io_file_get(struct io_submit_state *state, int fd)
1817{
1818 if (!state)
1819 return fget(fd);
1820
1821 if (state->file) {
1822 if (state->fd == fd) {
1823 state->used_refs++;
1824 state->ios_left--;
1825 return state->file;
1826 }
Jens Axboe3d6770f2019-04-13 11:50:54 -06001827 io_file_put(state);
Jens Axboe9a56a232019-01-09 09:06:50 -07001828 }
1829 state->file = fget_many(fd, state->ios_left);
1830 if (!state->file)
1831 return NULL;
1832
1833 state->fd = fd;
1834 state->has_refs = state->ios_left;
1835 state->used_refs = 1;
1836 state->ios_left--;
1837 return state->file;
1838}
1839
Jens Axboe2b188cc2019-01-07 10:46:33 -07001840/*
1841 * If we tracked the file through the SCM inflight mechanism, we could support
1842 * any file. For now, just ensure that anything potentially problematic is done
1843 * inline.
1844 */
1845static bool io_file_supports_async(struct file *file)
1846{
1847 umode_t mode = file_inode(file)->i_mode;
1848
Jens Axboe10d59342019-12-09 20:16:22 -07001849 if (S_ISBLK(mode) || S_ISCHR(mode) || S_ISSOCK(mode))
Jens Axboe2b188cc2019-01-07 10:46:33 -07001850 return true;
1851 if (S_ISREG(mode) && file->f_op != &io_uring_fops)
1852 return true;
1853
1854 return false;
1855}
1856
Jens Axboe3529d8c2019-12-19 18:24:38 -07001857static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,
1858 bool force_nonblock)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001859{
Jens Axboedef596e2019-01-09 08:59:42 -07001860 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe9adbd452019-12-20 08:45:55 -07001861 struct kiocb *kiocb = &req->rw.kiocb;
Jens Axboe09bb8392019-03-13 12:39:28 -06001862 unsigned ioprio;
1863 int ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001864
Jens Axboe09bb8392019-03-13 12:39:28 -06001865 if (!req->file)
1866 return -EBADF;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001867
Jens Axboe491381ce2019-10-17 09:20:46 -06001868 if (S_ISREG(file_inode(req->file)->i_mode))
1869 req->flags |= REQ_F_ISREG;
1870
Jens Axboe2b188cc2019-01-07 10:46:33 -07001871 kiocb->ki_pos = READ_ONCE(sqe->off);
Jens Axboeba042912019-12-25 16:33:42 -07001872 if (kiocb->ki_pos == -1 && !(req->file->f_mode & FMODE_STREAM)) {
1873 req->flags |= REQ_F_CUR_POS;
1874 kiocb->ki_pos = req->file->f_pos;
1875 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07001876 kiocb->ki_flags = iocb_flags(kiocb->ki_filp);
1877 kiocb->ki_hint = ki_hint_validate(file_write_hint(kiocb->ki_filp));
1878
1879 ioprio = READ_ONCE(sqe->ioprio);
1880 if (ioprio) {
1881 ret = ioprio_check_cap(ioprio);
1882 if (ret)
Jens Axboe09bb8392019-03-13 12:39:28 -06001883 return ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001884
1885 kiocb->ki_ioprio = ioprio;
1886 } else
1887 kiocb->ki_ioprio = get_current_ioprio();
1888
1889 ret = kiocb_set_rw_flags(kiocb, READ_ONCE(sqe->rw_flags));
1890 if (unlikely(ret))
Jens Axboe09bb8392019-03-13 12:39:28 -06001891 return ret;
Stefan Bühler8449eed2019-04-27 20:34:19 +02001892
1893 /* don't allow async punt if RWF_NOWAIT was requested */
Jens Axboe491381ce2019-10-17 09:20:46 -06001894 if ((kiocb->ki_flags & IOCB_NOWAIT) ||
1895 (req->file->f_flags & O_NONBLOCK))
Stefan Bühler8449eed2019-04-27 20:34:19 +02001896 req->flags |= REQ_F_NOWAIT;
1897
1898 if (force_nonblock)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001899 kiocb->ki_flags |= IOCB_NOWAIT;
Stefan Bühler8449eed2019-04-27 20:34:19 +02001900
Jens Axboedef596e2019-01-09 08:59:42 -07001901 if (ctx->flags & IORING_SETUP_IOPOLL) {
Jens Axboedef596e2019-01-09 08:59:42 -07001902 if (!(kiocb->ki_flags & IOCB_DIRECT) ||
1903 !kiocb->ki_filp->f_op->iopoll)
Jens Axboe09bb8392019-03-13 12:39:28 -06001904 return -EOPNOTSUPP;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001905
Jens Axboedef596e2019-01-09 08:59:42 -07001906 kiocb->ki_flags |= IOCB_HIPRI;
1907 kiocb->ki_complete = io_complete_rw_iopoll;
Jens Axboe6873e0b2019-10-30 13:53:09 -06001908 req->result = 0;
Jens Axboedef596e2019-01-09 08:59:42 -07001909 } else {
Jens Axboe09bb8392019-03-13 12:39:28 -06001910 if (kiocb->ki_flags & IOCB_HIPRI)
1911 return -EINVAL;
Jens Axboedef596e2019-01-09 08:59:42 -07001912 kiocb->ki_complete = io_complete_rw;
1913 }
Jens Axboe9adbd452019-12-20 08:45:55 -07001914
Jens Axboe3529d8c2019-12-19 18:24:38 -07001915 req->rw.addr = READ_ONCE(sqe->addr);
1916 req->rw.len = READ_ONCE(sqe->len);
Jens Axboe9adbd452019-12-20 08:45:55 -07001917 /* we own ->private, reuse it for the buffer index */
1918 req->rw.kiocb.private = (void *) (unsigned long)
Jens Axboe3529d8c2019-12-19 18:24:38 -07001919 READ_ONCE(sqe->buf_index);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001920 return 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001921}
1922
1923static inline void io_rw_done(struct kiocb *kiocb, ssize_t ret)
1924{
1925 switch (ret) {
1926 case -EIOCBQUEUED:
1927 break;
1928 case -ERESTARTSYS:
1929 case -ERESTARTNOINTR:
1930 case -ERESTARTNOHAND:
1931 case -ERESTART_RESTARTBLOCK:
1932 /*
1933 * We can't just restart the syscall, since previously
1934 * submitted sqes may already be in progress. Just fail this
1935 * IO with EINTR.
1936 */
1937 ret = -EINTR;
1938 /* fall through */
1939 default:
1940 kiocb->ki_complete(kiocb, ret, 0);
1941 }
1942}
1943
Jens Axboeba816ad2019-09-28 11:36:45 -06001944static void kiocb_done(struct kiocb *kiocb, ssize_t ret, struct io_kiocb **nxt,
1945 bool in_async)
1946{
Jens Axboeba042912019-12-25 16:33:42 -07001947 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
1948
1949 if (req->flags & REQ_F_CUR_POS)
1950 req->file->f_pos = kiocb->ki_pos;
Pavel Begunkovf9bd67f2019-11-21 23:21:03 +03001951 if (in_async && ret >= 0 && kiocb->ki_complete == io_complete_rw)
Jens Axboeba816ad2019-09-28 11:36:45 -06001952 *nxt = __io_complete_rw(kiocb, ret);
1953 else
1954 io_rw_done(kiocb, ret);
1955}
1956
Jens Axboe9adbd452019-12-20 08:45:55 -07001957static ssize_t io_import_fixed(struct io_kiocb *req, int rw,
Pavel Begunkov7d009162019-11-25 23:14:40 +03001958 struct iov_iter *iter)
Jens Axboeedafcce2019-01-09 09:16:05 -07001959{
Jens Axboe9adbd452019-12-20 08:45:55 -07001960 struct io_ring_ctx *ctx = req->ctx;
1961 size_t len = req->rw.len;
Jens Axboeedafcce2019-01-09 09:16:05 -07001962 struct io_mapped_ubuf *imu;
1963 unsigned index, buf_index;
1964 size_t offset;
1965 u64 buf_addr;
1966
1967 /* attempt to use fixed buffers without having provided iovecs */
1968 if (unlikely(!ctx->user_bufs))
1969 return -EFAULT;
1970
Jens Axboe9adbd452019-12-20 08:45:55 -07001971 buf_index = (unsigned long) req->rw.kiocb.private;
Jens Axboeedafcce2019-01-09 09:16:05 -07001972 if (unlikely(buf_index >= ctx->nr_user_bufs))
1973 return -EFAULT;
1974
1975 index = array_index_nospec(buf_index, ctx->nr_user_bufs);
1976 imu = &ctx->user_bufs[index];
Jens Axboe9adbd452019-12-20 08:45:55 -07001977 buf_addr = req->rw.addr;
Jens Axboeedafcce2019-01-09 09:16:05 -07001978
1979 /* overflow */
1980 if (buf_addr + len < buf_addr)
1981 return -EFAULT;
1982 /* not inside the mapped region */
1983 if (buf_addr < imu->ubuf || buf_addr + len > imu->ubuf + imu->len)
1984 return -EFAULT;
1985
1986 /*
1987 * May not be a start of buffer, set size appropriately
1988 * and advance us to the beginning.
1989 */
1990 offset = buf_addr - imu->ubuf;
1991 iov_iter_bvec(iter, rw, imu->bvec, imu->nr_bvecs, offset + len);
Jens Axboebd11b3a2019-07-20 08:37:31 -06001992
1993 if (offset) {
1994 /*
1995 * Don't use iov_iter_advance() here, as it's really slow for
1996 * using the latter parts of a big fixed buffer - it iterates
1997 * over each segment manually. We can cheat a bit here, because
1998 * we know that:
1999 *
2000 * 1) it's a BVEC iter, we set it up
2001 * 2) all bvecs are PAGE_SIZE in size, except potentially the
2002 * first and last bvec
2003 *
2004 * So just find our index, and adjust the iterator afterwards.
2005 * If the offset is within the first bvec (or the whole first
2006 * bvec, just use iov_iter_advance(). This makes it easier
2007 * since we can just skip the first segment, which may not
2008 * be PAGE_SIZE aligned.
2009 */
2010 const struct bio_vec *bvec = imu->bvec;
2011
2012 if (offset <= bvec->bv_len) {
2013 iov_iter_advance(iter, offset);
2014 } else {
2015 unsigned long seg_skip;
2016
2017 /* skip first vec */
2018 offset -= bvec->bv_len;
2019 seg_skip = 1 + (offset >> PAGE_SHIFT);
2020
2021 iter->bvec = bvec + seg_skip;
2022 iter->nr_segs -= seg_skip;
Aleix Roca Nonell99c79f62019-08-15 14:03:22 +02002023 iter->count -= bvec->bv_len + offset;
Jens Axboebd11b3a2019-07-20 08:37:31 -06002024 iter->iov_offset = offset & ~PAGE_MASK;
Jens Axboebd11b3a2019-07-20 08:37:31 -06002025 }
2026 }
2027
Jens Axboe5e559562019-11-13 16:12:46 -07002028 return len;
Jens Axboeedafcce2019-01-09 09:16:05 -07002029}
2030
Pavel Begunkovcf6fd4b2019-11-25 23:14:39 +03002031static ssize_t io_import_iovec(int rw, struct io_kiocb *req,
2032 struct iovec **iovec, struct iov_iter *iter)
Jens Axboe2b188cc2019-01-07 10:46:33 -07002033{
Jens Axboe9adbd452019-12-20 08:45:55 -07002034 void __user *buf = u64_to_user_ptr(req->rw.addr);
2035 size_t sqe_len = req->rw.len;
Jens Axboeedafcce2019-01-09 09:16:05 -07002036 u8 opcode;
2037
Jens Axboed625c6e2019-12-17 19:53:05 -07002038 opcode = req->opcode;
Pavel Begunkov7d009162019-11-25 23:14:40 +03002039 if (opcode == IORING_OP_READ_FIXED || opcode == IORING_OP_WRITE_FIXED) {
Jens Axboeedafcce2019-01-09 09:16:05 -07002040 *iovec = NULL;
Jens Axboe9adbd452019-12-20 08:45:55 -07002041 return io_import_fixed(req, rw, iter);
Jens Axboeedafcce2019-01-09 09:16:05 -07002042 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07002043
Jens Axboe9adbd452019-12-20 08:45:55 -07002044 /* buffer index only valid with fixed read/write */
2045 if (req->rw.kiocb.private)
2046 return -EINVAL;
2047
Jens Axboe3a6820f2019-12-22 15:19:35 -07002048 if (opcode == IORING_OP_READ || opcode == IORING_OP_WRITE) {
2049 ssize_t ret;
2050 ret = import_single_range(rw, buf, sqe_len, *iovec, iter);
2051 *iovec = NULL;
2052 return ret;
2053 }
2054
Jens Axboef67676d2019-12-02 11:03:47 -07002055 if (req->io) {
2056 struct io_async_rw *iorw = &req->io->rw;
2057
2058 *iovec = iorw->iov;
2059 iov_iter_init(iter, rw, *iovec, iorw->nr_segs, iorw->size);
2060 if (iorw->iov == iorw->fast_iov)
2061 *iovec = NULL;
2062 return iorw->size;
2063 }
2064
Pavel Begunkovcf6fd4b2019-11-25 23:14:39 +03002065 if (!req->has_user)
Jens Axboe2b188cc2019-01-07 10:46:33 -07002066 return -EFAULT;
2067
2068#ifdef CONFIG_COMPAT
Pavel Begunkovcf6fd4b2019-11-25 23:14:39 +03002069 if (req->ctx->compat)
Jens Axboe2b188cc2019-01-07 10:46:33 -07002070 return compat_import_iovec(rw, buf, sqe_len, UIO_FASTIOV,
2071 iovec, iter);
2072#endif
2073
2074 return import_iovec(rw, buf, sqe_len, UIO_FASTIOV, iovec, iter);
2075}
2076
Jens Axboe32960612019-09-23 11:05:34 -06002077/*
2078 * For files that don't have ->read_iter() and ->write_iter(), handle them
2079 * by looping over ->read() or ->write() manually.
2080 */
2081static ssize_t loop_rw_iter(int rw, struct file *file, struct kiocb *kiocb,
2082 struct iov_iter *iter)
2083{
2084 ssize_t ret = 0;
2085
2086 /*
2087 * Don't support polled IO through this interface, and we can't
2088 * support non-blocking either. For the latter, this just causes
2089 * the kiocb to be handled from an async context.
2090 */
2091 if (kiocb->ki_flags & IOCB_HIPRI)
2092 return -EOPNOTSUPP;
2093 if (kiocb->ki_flags & IOCB_NOWAIT)
2094 return -EAGAIN;
2095
2096 while (iov_iter_count(iter)) {
Pavel Begunkov311ae9e2019-11-24 11:58:24 +03002097 struct iovec iovec;
Jens Axboe32960612019-09-23 11:05:34 -06002098 ssize_t nr;
2099
Pavel Begunkov311ae9e2019-11-24 11:58:24 +03002100 if (!iov_iter_is_bvec(iter)) {
2101 iovec = iov_iter_iovec(iter);
2102 } else {
2103 /* fixed buffers import bvec */
2104 iovec.iov_base = kmap(iter->bvec->bv_page)
2105 + iter->iov_offset;
2106 iovec.iov_len = min(iter->count,
2107 iter->bvec->bv_len - iter->iov_offset);
2108 }
2109
Jens Axboe32960612019-09-23 11:05:34 -06002110 if (rw == READ) {
2111 nr = file->f_op->read(file, iovec.iov_base,
2112 iovec.iov_len, &kiocb->ki_pos);
2113 } else {
2114 nr = file->f_op->write(file, iovec.iov_base,
2115 iovec.iov_len, &kiocb->ki_pos);
2116 }
2117
Pavel Begunkov311ae9e2019-11-24 11:58:24 +03002118 if (iov_iter_is_bvec(iter))
2119 kunmap(iter->bvec->bv_page);
2120
Jens Axboe32960612019-09-23 11:05:34 -06002121 if (nr < 0) {
2122 if (!ret)
2123 ret = nr;
2124 break;
2125 }
2126 ret += nr;
2127 if (nr != iovec.iov_len)
2128 break;
2129 iov_iter_advance(iter, nr);
2130 }
2131
2132 return ret;
2133}
2134
Jens Axboeb7bb4f72019-12-15 22:13:43 -07002135static void io_req_map_rw(struct io_kiocb *req, ssize_t io_size,
Jens Axboef67676d2019-12-02 11:03:47 -07002136 struct iovec *iovec, struct iovec *fast_iov,
2137 struct iov_iter *iter)
2138{
2139 req->io->rw.nr_segs = iter->nr_segs;
2140 req->io->rw.size = io_size;
2141 req->io->rw.iov = iovec;
2142 if (!req->io->rw.iov) {
2143 req->io->rw.iov = req->io->rw.fast_iov;
2144 memcpy(req->io->rw.iov, fast_iov,
2145 sizeof(struct iovec) * iter->nr_segs);
2146 }
2147}
2148
Jens Axboeb7bb4f72019-12-15 22:13:43 -07002149static int io_alloc_async_ctx(struct io_kiocb *req)
Jens Axboef67676d2019-12-02 11:03:47 -07002150{
Jens Axboed3656342019-12-18 09:50:26 -07002151 if (!io_op_defs[req->opcode].async_ctx)
2152 return 0;
Jens Axboef67676d2019-12-02 11:03:47 -07002153 req->io = kmalloc(sizeof(*req->io), GFP_KERNEL);
Jens Axboe06b76d42019-12-19 14:44:26 -07002154 return req->io == NULL;
Jens Axboeb7bb4f72019-12-15 22:13:43 -07002155}
2156
2157static void io_rw_async(struct io_wq_work **workptr)
2158{
2159 struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);
2160 struct iovec *iov = NULL;
2161
2162 if (req->io->rw.iov != req->io->rw.fast_iov)
2163 iov = req->io->rw.iov;
2164 io_wq_submit_work(workptr);
2165 kfree(iov);
2166}
2167
2168static int io_setup_async_rw(struct io_kiocb *req, ssize_t io_size,
2169 struct iovec *iovec, struct iovec *fast_iov,
2170 struct iov_iter *iter)
2171{
Jens Axboe980ad262020-01-24 23:08:54 -07002172 if (!io_op_defs[req->opcode].async_ctx)
Jens Axboe74566df2020-01-13 19:23:24 -07002173 return 0;
Jens Axboeb7bb4f72019-12-15 22:13:43 -07002174 if (!req->io && io_alloc_async_ctx(req))
2175 return -ENOMEM;
2176
2177 io_req_map_rw(req, io_size, iovec, fast_iov, iter);
2178 req->work.func = io_rw_async;
2179 return 0;
Jens Axboef67676d2019-12-02 11:03:47 -07002180}
2181
Jens Axboe3529d8c2019-12-19 18:24:38 -07002182static int io_read_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe,
2183 bool force_nonblock)
Jens Axboef67676d2019-12-02 11:03:47 -07002184{
Jens Axboe3529d8c2019-12-19 18:24:38 -07002185 struct io_async_ctx *io;
2186 struct iov_iter iter;
Jens Axboef67676d2019-12-02 11:03:47 -07002187 ssize_t ret;
2188
Jens Axboe3529d8c2019-12-19 18:24:38 -07002189 ret = io_prep_rw(req, sqe, force_nonblock);
2190 if (ret)
2191 return ret;
Jens Axboef67676d2019-12-02 11:03:47 -07002192
Jens Axboe3529d8c2019-12-19 18:24:38 -07002193 if (unlikely(!(req->file->f_mode & FMODE_READ)))
2194 return -EBADF;
Jens Axboef67676d2019-12-02 11:03:47 -07002195
Jens Axboe3529d8c2019-12-19 18:24:38 -07002196 if (!req->io)
2197 return 0;
2198
2199 io = req->io;
2200 io->rw.iov = io->rw.fast_iov;
2201 req->io = NULL;
2202 ret = io_import_iovec(READ, req, &io->rw.iov, &iter);
2203 req->io = io;
2204 if (ret < 0)
2205 return ret;
2206
2207 io_req_map_rw(req, ret, io->rw.iov, io->rw.fast_iov, &iter);
2208 return 0;
Jens Axboef67676d2019-12-02 11:03:47 -07002209}
2210
Pavel Begunkov267bc902019-11-07 01:41:08 +03002211static int io_read(struct io_kiocb *req, struct io_kiocb **nxt,
Jens Axboe8358e3a2019-04-23 08:17:58 -06002212 bool force_nonblock)
Jens Axboe2b188cc2019-01-07 10:46:33 -07002213{
2214 struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
Jens Axboe9adbd452019-12-20 08:45:55 -07002215 struct kiocb *kiocb = &req->rw.kiocb;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002216 struct iov_iter iter;
Jens Axboe31b51512019-01-18 22:56:34 -07002217 size_t iov_count;
Jens Axboef67676d2019-12-02 11:03:47 -07002218 ssize_t io_size, ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002219
Jens Axboe3529d8c2019-12-19 18:24:38 -07002220 ret = io_import_iovec(READ, req, &iovec, &iter);
Jens Axboe06b76d42019-12-19 14:44:26 -07002221 if (ret < 0)
2222 return ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002223
Jens Axboefd6c2e42019-12-18 12:19:41 -07002224 /* Ensure we clear previously set non-block flag */
2225 if (!force_nonblock)
Jens Axboe9adbd452019-12-20 08:45:55 -07002226 req->rw.kiocb.ki_flags &= ~IOCB_NOWAIT;
Jens Axboefd6c2e42019-12-18 12:19:41 -07002227
Bijan Mottahedeh797f3f52020-01-15 18:37:45 -08002228 req->result = 0;
Jens Axboef67676d2019-12-02 11:03:47 -07002229 io_size = ret;
Jens Axboe9e645e112019-05-10 16:07:28 -06002230 if (req->flags & REQ_F_LINK)
Jens Axboef67676d2019-12-02 11:03:47 -07002231 req->result = io_size;
2232
2233 /*
2234 * If the file doesn't support async, mark it as REQ_F_MUST_PUNT so
2235 * we know to async punt it even if it was opened O_NONBLOCK
2236 */
Jens Axboe9adbd452019-12-20 08:45:55 -07002237 if (force_nonblock && !io_file_supports_async(req->file)) {
Jens Axboef67676d2019-12-02 11:03:47 -07002238 req->flags |= REQ_F_MUST_PUNT;
2239 goto copy_iov;
2240 }
Jens Axboe9e645e112019-05-10 16:07:28 -06002241
Jens Axboe31b51512019-01-18 22:56:34 -07002242 iov_count = iov_iter_count(&iter);
Jens Axboe9adbd452019-12-20 08:45:55 -07002243 ret = rw_verify_area(READ, req->file, &kiocb->ki_pos, iov_count);
Jens Axboe2b188cc2019-01-07 10:46:33 -07002244 if (!ret) {
2245 ssize_t ret2;
2246
Jens Axboe9adbd452019-12-20 08:45:55 -07002247 if (req->file->f_op->read_iter)
2248 ret2 = call_read_iter(req->file, kiocb, &iter);
Jens Axboe32960612019-09-23 11:05:34 -06002249 else
Jens Axboe9adbd452019-12-20 08:45:55 -07002250 ret2 = loop_rw_iter(READ, req->file, kiocb, &iter);
Jens Axboe32960612019-09-23 11:05:34 -06002251
Jens Axboe9d93a3f2019-05-15 13:53:07 -06002252 /* Catch -EAGAIN return for forced non-blocking submission */
Jens Axboef67676d2019-12-02 11:03:47 -07002253 if (!force_nonblock || ret2 != -EAGAIN) {
Pavel Begunkovcf6fd4b2019-11-25 23:14:39 +03002254 kiocb_done(kiocb, ret2, nxt, req->in_async);
Jens Axboef67676d2019-12-02 11:03:47 -07002255 } else {
2256copy_iov:
Jens Axboeb7bb4f72019-12-15 22:13:43 -07002257 ret = io_setup_async_rw(req, io_size, iovec,
Jens Axboef67676d2019-12-02 11:03:47 -07002258 inline_vecs, &iter);
2259 if (ret)
2260 goto out_free;
2261 return -EAGAIN;
2262 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07002263 }
Jens Axboef67676d2019-12-02 11:03:47 -07002264out_free:
Jens Axboeb7bb4f72019-12-15 22:13:43 -07002265 if (!io_wq_current_is_worker())
2266 kfree(iovec);
Jens Axboe2b188cc2019-01-07 10:46:33 -07002267 return ret;
2268}
2269
Jens Axboe3529d8c2019-12-19 18:24:38 -07002270static int io_write_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe,
2271 bool force_nonblock)
Jens Axboef67676d2019-12-02 11:03:47 -07002272{
Jens Axboe3529d8c2019-12-19 18:24:38 -07002273 struct io_async_ctx *io;
2274 struct iov_iter iter;
Jens Axboef67676d2019-12-02 11:03:47 -07002275 ssize_t ret;
2276
Jens Axboe3529d8c2019-12-19 18:24:38 -07002277 ret = io_prep_rw(req, sqe, force_nonblock);
2278 if (ret)
2279 return ret;
Jens Axboef67676d2019-12-02 11:03:47 -07002280
Jens Axboe3529d8c2019-12-19 18:24:38 -07002281 if (unlikely(!(req->file->f_mode & FMODE_WRITE)))
2282 return -EBADF;
Jens Axboef67676d2019-12-02 11:03:47 -07002283
Jens Axboe3529d8c2019-12-19 18:24:38 -07002284 if (!req->io)
2285 return 0;
2286
2287 io = req->io;
2288 io->rw.iov = io->rw.fast_iov;
2289 req->io = NULL;
2290 ret = io_import_iovec(WRITE, req, &io->rw.iov, &iter);
2291 req->io = io;
2292 if (ret < 0)
2293 return ret;
2294
2295 io_req_map_rw(req, ret, io->rw.iov, io->rw.fast_iov, &iter);
2296 return 0;
Jens Axboef67676d2019-12-02 11:03:47 -07002297}
2298
Pavel Begunkov267bc902019-11-07 01:41:08 +03002299static int io_write(struct io_kiocb *req, struct io_kiocb **nxt,
Jens Axboe8358e3a2019-04-23 08:17:58 -06002300 bool force_nonblock)
Jens Axboe2b188cc2019-01-07 10:46:33 -07002301{
2302 struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
Jens Axboe9adbd452019-12-20 08:45:55 -07002303 struct kiocb *kiocb = &req->rw.kiocb;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002304 struct iov_iter iter;
Jens Axboe31b51512019-01-18 22:56:34 -07002305 size_t iov_count;
Jens Axboef67676d2019-12-02 11:03:47 -07002306 ssize_t ret, io_size;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002307
Jens Axboe3529d8c2019-12-19 18:24:38 -07002308 ret = io_import_iovec(WRITE, req, &iovec, &iter);
Jens Axboe06b76d42019-12-19 14:44:26 -07002309 if (ret < 0)
2310 return ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002311
Jens Axboefd6c2e42019-12-18 12:19:41 -07002312 /* Ensure we clear previously set non-block flag */
2313 if (!force_nonblock)
Jens Axboe9adbd452019-12-20 08:45:55 -07002314 req->rw.kiocb.ki_flags &= ~IOCB_NOWAIT;
Jens Axboefd6c2e42019-12-18 12:19:41 -07002315
Bijan Mottahedeh797f3f52020-01-15 18:37:45 -08002316 req->result = 0;
Jens Axboef67676d2019-12-02 11:03:47 -07002317 io_size = ret;
Jens Axboe9e645e112019-05-10 16:07:28 -06002318 if (req->flags & REQ_F_LINK)
Jens Axboef67676d2019-12-02 11:03:47 -07002319 req->result = io_size;
2320
2321 /*
2322 * If the file doesn't support async, mark it as REQ_F_MUST_PUNT so
2323 * we know to async punt it even if it was opened O_NONBLOCK
2324 */
2325 if (force_nonblock && !io_file_supports_async(req->file)) {
2326 req->flags |= REQ_F_MUST_PUNT;
2327 goto copy_iov;
2328 }
2329
Jens Axboe10d59342019-12-09 20:16:22 -07002330 /* file path doesn't support NOWAIT for non-direct_IO */
2331 if (force_nonblock && !(kiocb->ki_flags & IOCB_DIRECT) &&
2332 (req->flags & REQ_F_ISREG))
Jens Axboef67676d2019-12-02 11:03:47 -07002333 goto copy_iov;
Jens Axboe9e645e112019-05-10 16:07:28 -06002334
Jens Axboe31b51512019-01-18 22:56:34 -07002335 iov_count = iov_iter_count(&iter);
Jens Axboe9adbd452019-12-20 08:45:55 -07002336 ret = rw_verify_area(WRITE, req->file, &kiocb->ki_pos, iov_count);
Jens Axboe2b188cc2019-01-07 10:46:33 -07002337 if (!ret) {
Roman Penyaev9bf79332019-03-25 20:09:24 +01002338 ssize_t ret2;
2339
Jens Axboe2b188cc2019-01-07 10:46:33 -07002340 /*
2341 * Open-code file_start_write here to grab freeze protection,
2342 * which will be released by another thread in
2343 * io_complete_rw(). Fool lockdep by telling it the lock got
2344 * released so that it doesn't complain about the held lock when
2345 * we return to userspace.
2346 */
Jens Axboe491381ce2019-10-17 09:20:46 -06002347 if (req->flags & REQ_F_ISREG) {
Jens Axboe9adbd452019-12-20 08:45:55 -07002348 __sb_start_write(file_inode(req->file)->i_sb,
Jens Axboe2b188cc2019-01-07 10:46:33 -07002349 SB_FREEZE_WRITE, true);
Jens Axboe9adbd452019-12-20 08:45:55 -07002350 __sb_writers_release(file_inode(req->file)->i_sb,
Jens Axboe2b188cc2019-01-07 10:46:33 -07002351 SB_FREEZE_WRITE);
2352 }
2353 kiocb->ki_flags |= IOCB_WRITE;
Roman Penyaev9bf79332019-03-25 20:09:24 +01002354
Jens Axboe9adbd452019-12-20 08:45:55 -07002355 if (req->file->f_op->write_iter)
2356 ret2 = call_write_iter(req->file, kiocb, &iter);
Jens Axboe32960612019-09-23 11:05:34 -06002357 else
Jens Axboe9adbd452019-12-20 08:45:55 -07002358 ret2 = loop_rw_iter(WRITE, req->file, kiocb, &iter);
Jens Axboef67676d2019-12-02 11:03:47 -07002359 if (!force_nonblock || ret2 != -EAGAIN) {
Pavel Begunkovcf6fd4b2019-11-25 23:14:39 +03002360 kiocb_done(kiocb, ret2, nxt, req->in_async);
Jens Axboef67676d2019-12-02 11:03:47 -07002361 } else {
2362copy_iov:
Jens Axboeb7bb4f72019-12-15 22:13:43 -07002363 ret = io_setup_async_rw(req, io_size, iovec,
Jens Axboef67676d2019-12-02 11:03:47 -07002364 inline_vecs, &iter);
2365 if (ret)
2366 goto out_free;
2367 return -EAGAIN;
2368 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07002369 }
Jens Axboe31b51512019-01-18 22:56:34 -07002370out_free:
Jens Axboeb7bb4f72019-12-15 22:13:43 -07002371 if (!io_wq_current_is_worker())
2372 kfree(iovec);
Jens Axboe2b188cc2019-01-07 10:46:33 -07002373 return ret;
2374}
2375
2376/*
2377 * IORING_OP_NOP just posts a completion event, nothing else.
2378 */
Jens Axboe78e19bb2019-11-06 15:21:34 -07002379static int io_nop(struct io_kiocb *req)
Jens Axboe2b188cc2019-01-07 10:46:33 -07002380{
2381 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002382
Jens Axboedef596e2019-01-09 08:59:42 -07002383 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
2384 return -EINVAL;
2385
Jens Axboe78e19bb2019-11-06 15:21:34 -07002386 io_cqring_add_event(req, 0);
Jens Axboee65ef562019-03-12 10:16:44 -06002387 io_put_req(req);
Jens Axboe2b188cc2019-01-07 10:46:33 -07002388 return 0;
2389}
2390
Jens Axboe3529d8c2019-12-19 18:24:38 -07002391static int io_prep_fsync(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Christoph Hellwigc992fe22019-01-11 09:43:02 -07002392{
Jens Axboe6b063142019-01-10 22:13:58 -07002393 struct io_ring_ctx *ctx = req->ctx;
Christoph Hellwigc992fe22019-01-11 09:43:02 -07002394
Jens Axboe09bb8392019-03-13 12:39:28 -06002395 if (!req->file)
2396 return -EBADF;
Christoph Hellwigc992fe22019-01-11 09:43:02 -07002397
Jens Axboe6b063142019-01-10 22:13:58 -07002398 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
Jens Axboedef596e2019-01-09 08:59:42 -07002399 return -EINVAL;
Jens Axboeedafcce2019-01-09 09:16:05 -07002400 if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index))
Christoph Hellwigc992fe22019-01-11 09:43:02 -07002401 return -EINVAL;
2402
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07002403 req->sync.flags = READ_ONCE(sqe->fsync_flags);
2404 if (unlikely(req->sync.flags & ~IORING_FSYNC_DATASYNC))
2405 return -EINVAL;
2406
2407 req->sync.off = READ_ONCE(sqe->off);
2408 req->sync.len = READ_ONCE(sqe->len);
Christoph Hellwigc992fe22019-01-11 09:43:02 -07002409 return 0;
2410}
2411
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07002412static bool io_req_cancelled(struct io_kiocb *req)
2413{
2414 if (req->work.flags & IO_WQ_WORK_CANCEL) {
2415 req_set_fail_links(req);
2416 io_cqring_add_event(req, -ECANCELED);
2417 io_put_req(req);
2418 return true;
2419 }
2420
2421 return false;
2422}
2423
Jens Axboe78912932020-01-14 22:09:06 -07002424static void io_link_work_cb(struct io_wq_work **workptr)
2425{
2426 struct io_wq_work *work = *workptr;
2427 struct io_kiocb *link = work->data;
2428
2429 io_queue_linked_timeout(link);
2430 work->func = io_wq_submit_work;
2431}
2432
2433static void io_wq_assign_next(struct io_wq_work **workptr, struct io_kiocb *nxt)
2434{
2435 struct io_kiocb *link;
2436
2437 io_prep_async_work(nxt, &link);
2438 *workptr = &nxt->work;
2439 if (link) {
2440 nxt->work.flags |= IO_WQ_WORK_CB;
2441 nxt->work.func = io_link_work_cb;
2442 nxt->work.data = link;
2443 }
2444}
2445
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07002446static void io_fsync_finish(struct io_wq_work **workptr)
2447{
2448 struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);
2449 loff_t end = req->sync.off + req->sync.len;
2450 struct io_kiocb *nxt = NULL;
2451 int ret;
2452
2453 if (io_req_cancelled(req))
2454 return;
2455
Jens Axboe9adbd452019-12-20 08:45:55 -07002456 ret = vfs_fsync_range(req->file, req->sync.off,
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07002457 end > 0 ? end : LLONG_MAX,
2458 req->sync.flags & IORING_FSYNC_DATASYNC);
2459 if (ret < 0)
2460 req_set_fail_links(req);
2461 io_cqring_add_event(req, ret);
2462 io_put_req_find_next(req, &nxt);
2463 if (nxt)
Jens Axboe78912932020-01-14 22:09:06 -07002464 io_wq_assign_next(workptr, nxt);
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07002465}
2466
Jens Axboefc4df992019-12-10 14:38:45 -07002467static int io_fsync(struct io_kiocb *req, struct io_kiocb **nxt,
2468 bool force_nonblock)
Christoph Hellwigc992fe22019-01-11 09:43:02 -07002469{
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07002470 struct io_wq_work *work, *old_work;
Christoph Hellwigc992fe22019-01-11 09:43:02 -07002471
2472 /* fsync always requires a blocking context */
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07002473 if (force_nonblock) {
2474 io_put_req(req);
2475 req->work.func = io_fsync_finish;
Christoph Hellwigc992fe22019-01-11 09:43:02 -07002476 return -EAGAIN;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07002477 }
Christoph Hellwigc992fe22019-01-11 09:43:02 -07002478
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07002479 work = old_work = &req->work;
2480 io_fsync_finish(&work);
2481 if (work && work != old_work)
2482 *nxt = container_of(work, struct io_kiocb, work);
Christoph Hellwigc992fe22019-01-11 09:43:02 -07002483 return 0;
2484}
2485
Jens Axboed63d1b52019-12-10 10:38:56 -07002486static void io_fallocate_finish(struct io_wq_work **workptr)
2487{
2488 struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);
2489 struct io_kiocb *nxt = NULL;
2490 int ret;
2491
2492 ret = vfs_fallocate(req->file, req->sync.mode, req->sync.off,
2493 req->sync.len);
2494 if (ret < 0)
2495 req_set_fail_links(req);
2496 io_cqring_add_event(req, ret);
2497 io_put_req_find_next(req, &nxt);
2498 if (nxt)
2499 io_wq_assign_next(workptr, nxt);
2500}
2501
2502static int io_fallocate_prep(struct io_kiocb *req,
2503 const struct io_uring_sqe *sqe)
2504{
2505 if (sqe->ioprio || sqe->buf_index || sqe->rw_flags)
2506 return -EINVAL;
2507
2508 req->sync.off = READ_ONCE(sqe->off);
2509 req->sync.len = READ_ONCE(sqe->addr);
2510 req->sync.mode = READ_ONCE(sqe->len);
2511 return 0;
2512}
2513
2514static int io_fallocate(struct io_kiocb *req, struct io_kiocb **nxt,
2515 bool force_nonblock)
2516{
2517 struct io_wq_work *work, *old_work;
2518
2519 /* fallocate always requiring blocking context */
2520 if (force_nonblock) {
2521 io_put_req(req);
2522 req->work.func = io_fallocate_finish;
2523 return -EAGAIN;
2524 }
2525
2526 work = old_work = &req->work;
2527 io_fallocate_finish(&work);
2528 if (work && work != old_work)
2529 *nxt = container_of(work, struct io_kiocb, work);
2530
2531 return 0;
2532}
2533
Jens Axboe15b71ab2019-12-11 11:20:36 -07002534static int io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
2535{
Jens Axboef8748882020-01-08 17:47:02 -07002536 const char __user *fname;
Jens Axboe15b71ab2019-12-11 11:20:36 -07002537 int ret;
2538
2539 if (sqe->ioprio || sqe->buf_index)
2540 return -EINVAL;
2541
2542 req->open.dfd = READ_ONCE(sqe->fd);
Jens Axboec12cedf2020-01-08 17:41:21 -07002543 req->open.how.mode = READ_ONCE(sqe->len);
Jens Axboef8748882020-01-08 17:47:02 -07002544 fname = u64_to_user_ptr(READ_ONCE(sqe->addr));
Jens Axboec12cedf2020-01-08 17:41:21 -07002545 req->open.how.flags = READ_ONCE(sqe->open_flags);
Jens Axboe15b71ab2019-12-11 11:20:36 -07002546
Jens Axboef8748882020-01-08 17:47:02 -07002547 req->open.filename = getname(fname);
Jens Axboe15b71ab2019-12-11 11:20:36 -07002548 if (IS_ERR(req->open.filename)) {
2549 ret = PTR_ERR(req->open.filename);
2550 req->open.filename = NULL;
2551 return ret;
2552 }
2553
2554 return 0;
2555}
2556
Jens Axboecebdb982020-01-08 17:59:24 -07002557static int io_openat2_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
2558{
2559 struct open_how __user *how;
2560 const char __user *fname;
2561 size_t len;
2562 int ret;
2563
2564 if (sqe->ioprio || sqe->buf_index)
2565 return -EINVAL;
2566
2567 req->open.dfd = READ_ONCE(sqe->fd);
2568 fname = u64_to_user_ptr(READ_ONCE(sqe->addr));
2569 how = u64_to_user_ptr(READ_ONCE(sqe->addr2));
2570 len = READ_ONCE(sqe->len);
2571
2572 if (len < OPEN_HOW_SIZE_VER0)
2573 return -EINVAL;
2574
2575 ret = copy_struct_from_user(&req->open.how, sizeof(req->open.how), how,
2576 len);
2577 if (ret)
2578 return ret;
2579
2580 if (!(req->open.how.flags & O_PATH) && force_o_largefile())
2581 req->open.how.flags |= O_LARGEFILE;
2582
2583 req->open.filename = getname(fname);
2584 if (IS_ERR(req->open.filename)) {
2585 ret = PTR_ERR(req->open.filename);
2586 req->open.filename = NULL;
2587 return ret;
2588 }
2589
2590 return 0;
2591}
2592
2593static int io_openat2(struct io_kiocb *req, struct io_kiocb **nxt,
2594 bool force_nonblock)
Jens Axboe15b71ab2019-12-11 11:20:36 -07002595{
2596 struct open_flags op;
Jens Axboe15b71ab2019-12-11 11:20:36 -07002597 struct file *file;
2598 int ret;
2599
Jens Axboef86cd202020-01-29 13:46:44 -07002600 if (force_nonblock)
Jens Axboe15b71ab2019-12-11 11:20:36 -07002601 return -EAGAIN;
Jens Axboe15b71ab2019-12-11 11:20:36 -07002602
Jens Axboecebdb982020-01-08 17:59:24 -07002603 ret = build_open_flags(&req->open.how, &op);
Jens Axboe15b71ab2019-12-11 11:20:36 -07002604 if (ret)
2605 goto err;
2606
Jens Axboecebdb982020-01-08 17:59:24 -07002607 ret = get_unused_fd_flags(req->open.how.flags);
Jens Axboe15b71ab2019-12-11 11:20:36 -07002608 if (ret < 0)
2609 goto err;
2610
2611 file = do_filp_open(req->open.dfd, req->open.filename, &op);
2612 if (IS_ERR(file)) {
2613 put_unused_fd(ret);
2614 ret = PTR_ERR(file);
2615 } else {
2616 fsnotify_open(file);
2617 fd_install(ret, file);
2618 }
2619err:
2620 putname(req->open.filename);
2621 if (ret < 0)
2622 req_set_fail_links(req);
2623 io_cqring_add_event(req, ret);
2624 io_put_req_find_next(req, nxt);
2625 return 0;
2626}
2627
Jens Axboecebdb982020-01-08 17:59:24 -07002628static int io_openat(struct io_kiocb *req, struct io_kiocb **nxt,
2629 bool force_nonblock)
2630{
2631 req->open.how = build_open_how(req->open.how.flags, req->open.how.mode);
2632 return io_openat2(req, nxt, force_nonblock);
2633}
2634
Jens Axboe3e4827b2020-01-08 15:18:09 -07002635static int io_epoll_ctl_prep(struct io_kiocb *req,
2636 const struct io_uring_sqe *sqe)
2637{
2638#if defined(CONFIG_EPOLL)
2639 if (sqe->ioprio || sqe->buf_index)
2640 return -EINVAL;
2641
2642 req->epoll.epfd = READ_ONCE(sqe->fd);
2643 req->epoll.op = READ_ONCE(sqe->len);
2644 req->epoll.fd = READ_ONCE(sqe->off);
2645
2646 if (ep_op_has_event(req->epoll.op)) {
2647 struct epoll_event __user *ev;
2648
2649 ev = u64_to_user_ptr(READ_ONCE(sqe->addr));
2650 if (copy_from_user(&req->epoll.event, ev, sizeof(*ev)))
2651 return -EFAULT;
2652 }
2653
2654 return 0;
2655#else
2656 return -EOPNOTSUPP;
2657#endif
2658}
2659
2660static int io_epoll_ctl(struct io_kiocb *req, struct io_kiocb **nxt,
2661 bool force_nonblock)
2662{
2663#if defined(CONFIG_EPOLL)
2664 struct io_epoll *ie = &req->epoll;
2665 int ret;
2666
2667 ret = do_epoll_ctl(ie->epfd, ie->op, ie->fd, &ie->event, force_nonblock);
2668 if (force_nonblock && ret == -EAGAIN)
2669 return -EAGAIN;
2670
2671 if (ret < 0)
2672 req_set_fail_links(req);
2673 io_cqring_add_event(req, ret);
2674 io_put_req_find_next(req, nxt);
2675 return 0;
2676#else
2677 return -EOPNOTSUPP;
2678#endif
2679}
2680
Jens Axboec1ca7572019-12-25 22:18:28 -07002681static int io_madvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
2682{
2683#if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU)
2684 if (sqe->ioprio || sqe->buf_index || sqe->off)
2685 return -EINVAL;
2686
2687 req->madvise.addr = READ_ONCE(sqe->addr);
2688 req->madvise.len = READ_ONCE(sqe->len);
2689 req->madvise.advice = READ_ONCE(sqe->fadvise_advice);
2690 return 0;
2691#else
2692 return -EOPNOTSUPP;
2693#endif
2694}
2695
2696static int io_madvise(struct io_kiocb *req, struct io_kiocb **nxt,
2697 bool force_nonblock)
2698{
2699#if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU)
2700 struct io_madvise *ma = &req->madvise;
2701 int ret;
2702
2703 if (force_nonblock)
2704 return -EAGAIN;
2705
2706 ret = do_madvise(ma->addr, ma->len, ma->advice);
2707 if (ret < 0)
2708 req_set_fail_links(req);
2709 io_cqring_add_event(req, ret);
2710 io_put_req_find_next(req, nxt);
2711 return 0;
2712#else
2713 return -EOPNOTSUPP;
2714#endif
2715}
2716
Jens Axboe4840e412019-12-25 22:03:45 -07002717static int io_fadvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
2718{
2719 if (sqe->ioprio || sqe->buf_index || sqe->addr)
2720 return -EINVAL;
2721
2722 req->fadvise.offset = READ_ONCE(sqe->off);
2723 req->fadvise.len = READ_ONCE(sqe->len);
2724 req->fadvise.advice = READ_ONCE(sqe->fadvise_advice);
2725 return 0;
2726}
2727
2728static int io_fadvise(struct io_kiocb *req, struct io_kiocb **nxt,
2729 bool force_nonblock)
2730{
2731 struct io_fadvise *fa = &req->fadvise;
2732 int ret;
2733
2734 /* DONTNEED may block, others _should_ not */
2735 if (fa->advice == POSIX_FADV_DONTNEED && force_nonblock)
2736 return -EAGAIN;
2737
2738 ret = vfs_fadvise(req->file, fa->offset, fa->len, fa->advice);
2739 if (ret < 0)
2740 req_set_fail_links(req);
2741 io_cqring_add_event(req, ret);
2742 io_put_req_find_next(req, nxt);
2743 return 0;
2744}
2745
Jens Axboeeddc7ef2019-12-13 21:18:10 -07002746static int io_statx_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
2747{
Jens Axboef8748882020-01-08 17:47:02 -07002748 const char __user *fname;
Jens Axboeeddc7ef2019-12-13 21:18:10 -07002749 unsigned lookup_flags;
2750 int ret;
2751
2752 if (sqe->ioprio || sqe->buf_index)
2753 return -EINVAL;
2754
2755 req->open.dfd = READ_ONCE(sqe->fd);
2756 req->open.mask = READ_ONCE(sqe->len);
Jens Axboef8748882020-01-08 17:47:02 -07002757 fname = u64_to_user_ptr(READ_ONCE(sqe->addr));
Jens Axboeeddc7ef2019-12-13 21:18:10 -07002758 req->open.buffer = u64_to_user_ptr(READ_ONCE(sqe->addr2));
Jens Axboec12cedf2020-01-08 17:41:21 -07002759 req->open.how.flags = READ_ONCE(sqe->statx_flags);
Jens Axboeeddc7ef2019-12-13 21:18:10 -07002760
Jens Axboec12cedf2020-01-08 17:41:21 -07002761 if (vfs_stat_set_lookup_flags(&lookup_flags, req->open.how.flags))
Jens Axboeeddc7ef2019-12-13 21:18:10 -07002762 return -EINVAL;
2763
Jens Axboef8748882020-01-08 17:47:02 -07002764 req->open.filename = getname_flags(fname, lookup_flags, NULL);
Jens Axboeeddc7ef2019-12-13 21:18:10 -07002765 if (IS_ERR(req->open.filename)) {
2766 ret = PTR_ERR(req->open.filename);
2767 req->open.filename = NULL;
2768 return ret;
2769 }
2770
2771 return 0;
2772}
2773
2774static int io_statx(struct io_kiocb *req, struct io_kiocb **nxt,
2775 bool force_nonblock)
2776{
2777 struct io_open *ctx = &req->open;
2778 unsigned lookup_flags;
2779 struct path path;
2780 struct kstat stat;
2781 int ret;
2782
2783 if (force_nonblock)
2784 return -EAGAIN;
2785
Jens Axboec12cedf2020-01-08 17:41:21 -07002786 if (vfs_stat_set_lookup_flags(&lookup_flags, ctx->how.flags))
Jens Axboeeddc7ef2019-12-13 21:18:10 -07002787 return -EINVAL;
2788
2789retry:
2790 /* filename_lookup() drops it, keep a reference */
2791 ctx->filename->refcnt++;
2792
2793 ret = filename_lookup(ctx->dfd, ctx->filename, lookup_flags, &path,
2794 NULL);
2795 if (ret)
2796 goto err;
2797
Jens Axboec12cedf2020-01-08 17:41:21 -07002798 ret = vfs_getattr(&path, &stat, ctx->mask, ctx->how.flags);
Jens Axboeeddc7ef2019-12-13 21:18:10 -07002799 path_put(&path);
2800 if (retry_estale(ret, lookup_flags)) {
2801 lookup_flags |= LOOKUP_REVAL;
2802 goto retry;
2803 }
2804 if (!ret)
2805 ret = cp_statx(&stat, ctx->buffer);
2806err:
2807 putname(ctx->filename);
2808 if (ret < 0)
2809 req_set_fail_links(req);
2810 io_cqring_add_event(req, ret);
2811 io_put_req_find_next(req, nxt);
2812 return 0;
2813}
2814
Jens Axboeb5dba592019-12-11 14:02:38 -07002815static int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
2816{
2817 /*
2818 * If we queue this for async, it must not be cancellable. That would
2819 * leave the 'file' in an undeterminate state.
2820 */
2821 req->work.flags |= IO_WQ_WORK_NO_CANCEL;
2822
2823 if (sqe->ioprio || sqe->off || sqe->addr || sqe->len ||
2824 sqe->rw_flags || sqe->buf_index)
2825 return -EINVAL;
2826 if (sqe->flags & IOSQE_FIXED_FILE)
2827 return -EINVAL;
2828
2829 req->close.fd = READ_ONCE(sqe->fd);
2830 if (req->file->f_op == &io_uring_fops ||
Pavel Begunkovb14cca02020-01-17 04:45:59 +03002831 req->close.fd == req->ctx->ring_fd)
Jens Axboeb5dba592019-12-11 14:02:38 -07002832 return -EBADF;
2833
2834 return 0;
2835}
2836
2837static void io_close_finish(struct io_wq_work **workptr)
2838{
2839 struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);
2840 struct io_kiocb *nxt = NULL;
2841
2842 /* Invoked with files, we need to do the close */
2843 if (req->work.files) {
2844 int ret;
2845
2846 ret = filp_close(req->close.put_file, req->work.files);
2847 if (ret < 0) {
2848 req_set_fail_links(req);
2849 }
2850 io_cqring_add_event(req, ret);
2851 }
2852
2853 fput(req->close.put_file);
2854
2855 /* we bypassed the re-issue, drop the submission reference */
2856 io_put_req(req);
2857 io_put_req_find_next(req, &nxt);
2858 if (nxt)
2859 io_wq_assign_next(workptr, nxt);
2860}
2861
2862static int io_close(struct io_kiocb *req, struct io_kiocb **nxt,
2863 bool force_nonblock)
2864{
2865 int ret;
2866
2867 req->close.put_file = NULL;
2868 ret = __close_fd_get_file(req->close.fd, &req->close.put_file);
2869 if (ret < 0)
2870 return ret;
2871
2872 /* if the file has a flush method, be safe and punt to async */
Jens Axboef86cd202020-01-29 13:46:44 -07002873 if (req->close.put_file->f_op->flush && !io_wq_current_is_worker())
Jens Axboeb5dba592019-12-11 14:02:38 -07002874 goto eagain;
Jens Axboeb5dba592019-12-11 14:02:38 -07002875
2876 /*
2877 * No ->flush(), safely close from here and just punt the
2878 * fput() to async context.
2879 */
2880 ret = filp_close(req->close.put_file, current->files);
2881
2882 if (ret < 0)
2883 req_set_fail_links(req);
2884 io_cqring_add_event(req, ret);
2885
2886 if (io_wq_current_is_worker()) {
2887 struct io_wq_work *old_work, *work;
2888
2889 old_work = work = &req->work;
2890 io_close_finish(&work);
2891 if (work && work != old_work)
2892 *nxt = container_of(work, struct io_kiocb, work);
2893 return 0;
2894 }
2895
2896eagain:
2897 req->work.func = io_close_finish;
2898 return -EAGAIN;
2899}
2900
Jens Axboe3529d8c2019-12-19 18:24:38 -07002901static int io_prep_sfr(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboe5d17b4a2019-04-09 14:56:44 -06002902{
2903 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe5d17b4a2019-04-09 14:56:44 -06002904
2905 if (!req->file)
2906 return -EBADF;
Jens Axboe5d17b4a2019-04-09 14:56:44 -06002907
2908 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
2909 return -EINVAL;
2910 if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index))
2911 return -EINVAL;
2912
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07002913 req->sync.off = READ_ONCE(sqe->off);
2914 req->sync.len = READ_ONCE(sqe->len);
2915 req->sync.flags = READ_ONCE(sqe->sync_range_flags);
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07002916 return 0;
2917}
2918
2919static void io_sync_file_range_finish(struct io_wq_work **workptr)
2920{
2921 struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);
2922 struct io_kiocb *nxt = NULL;
2923 int ret;
2924
2925 if (io_req_cancelled(req))
2926 return;
2927
Jens Axboe9adbd452019-12-20 08:45:55 -07002928 ret = sync_file_range(req->file, req->sync.off, req->sync.len,
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07002929 req->sync.flags);
2930 if (ret < 0)
2931 req_set_fail_links(req);
2932 io_cqring_add_event(req, ret);
2933 io_put_req_find_next(req, &nxt);
2934 if (nxt)
Jens Axboe78912932020-01-14 22:09:06 -07002935 io_wq_assign_next(workptr, nxt);
Jens Axboe5d17b4a2019-04-09 14:56:44 -06002936}
2937
Jens Axboefc4df992019-12-10 14:38:45 -07002938static int io_sync_file_range(struct io_kiocb *req, struct io_kiocb **nxt,
Jens Axboe5d17b4a2019-04-09 14:56:44 -06002939 bool force_nonblock)
2940{
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07002941 struct io_wq_work *work, *old_work;
Jens Axboe5d17b4a2019-04-09 14:56:44 -06002942
2943 /* sync_file_range always requires a blocking context */
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07002944 if (force_nonblock) {
2945 io_put_req(req);
2946 req->work.func = io_sync_file_range_finish;
Jens Axboe5d17b4a2019-04-09 14:56:44 -06002947 return -EAGAIN;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07002948 }
Jens Axboe5d17b4a2019-04-09 14:56:44 -06002949
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07002950 work = old_work = &req->work;
2951 io_sync_file_range_finish(&work);
2952 if (work && work != old_work)
2953 *nxt = container_of(work, struct io_kiocb, work);
Jens Axboe5d17b4a2019-04-09 14:56:44 -06002954 return 0;
2955}
2956
Jens Axboeb7bb4f72019-12-15 22:13:43 -07002957#if defined(CONFIG_NET)
2958static void io_sendrecv_async(struct io_wq_work **workptr)
2959{
2960 struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);
2961 struct iovec *iov = NULL;
2962
2963 if (req->io->rw.iov != req->io->rw.fast_iov)
2964 iov = req->io->msg.iov;
2965 io_wq_submit_work(workptr);
2966 kfree(iov);
2967}
2968#endif
2969
Jens Axboe3529d8c2019-12-19 18:24:38 -07002970static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboeaa1fa282019-04-19 13:38:09 -06002971{
Jens Axboe03b12302019-12-02 18:50:25 -07002972#if defined(CONFIG_NET)
Jens Axboee47293f2019-12-20 08:58:21 -07002973 struct io_sr_msg *sr = &req->sr_msg;
Jens Axboe3529d8c2019-12-19 18:24:38 -07002974 struct io_async_ctx *io = req->io;
Jens Axboe03b12302019-12-02 18:50:25 -07002975
Jens Axboee47293f2019-12-20 08:58:21 -07002976 sr->msg_flags = READ_ONCE(sqe->msg_flags);
2977 sr->msg = u64_to_user_ptr(READ_ONCE(sqe->addr));
Jens Axboefddafac2020-01-04 20:19:44 -07002978 sr->len = READ_ONCE(sqe->len);
Jens Axboe3529d8c2019-12-19 18:24:38 -07002979
Jens Axboefddafac2020-01-04 20:19:44 -07002980 if (!io || req->opcode == IORING_OP_SEND)
Jens Axboe3529d8c2019-12-19 18:24:38 -07002981 return 0;
2982
Jens Axboed9688562019-12-09 19:35:20 -07002983 io->msg.iov = io->msg.fast_iov;
Jens Axboe3529d8c2019-12-19 18:24:38 -07002984 return sendmsg_copy_msghdr(&io->msg.msg, sr->msg, sr->msg_flags,
Jens Axboee47293f2019-12-20 08:58:21 -07002985 &io->msg.iov);
Jens Axboe03b12302019-12-02 18:50:25 -07002986#else
Jens Axboee47293f2019-12-20 08:58:21 -07002987 return -EOPNOTSUPP;
Jens Axboe03b12302019-12-02 18:50:25 -07002988#endif
2989}
2990
Jens Axboefc4df992019-12-10 14:38:45 -07002991static int io_sendmsg(struct io_kiocb *req, struct io_kiocb **nxt,
2992 bool force_nonblock)
Jens Axboe03b12302019-12-02 18:50:25 -07002993{
2994#if defined(CONFIG_NET)
Jens Axboe0b416c32019-12-15 10:57:46 -07002995 struct io_async_msghdr *kmsg = NULL;
Jens Axboe03b12302019-12-02 18:50:25 -07002996 struct socket *sock;
2997 int ret;
2998
2999 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3000 return -EINVAL;
3001
3002 sock = sock_from_file(req->file, &ret);
3003 if (sock) {
Jens Axboeb7bb4f72019-12-15 22:13:43 -07003004 struct io_async_ctx io;
Jens Axboe03b12302019-12-02 18:50:25 -07003005 struct sockaddr_storage addr;
Jens Axboe03b12302019-12-02 18:50:25 -07003006 unsigned flags;
3007
Jens Axboe03b12302019-12-02 18:50:25 -07003008 if (req->io) {
Jens Axboe0b416c32019-12-15 10:57:46 -07003009 kmsg = &req->io->msg;
3010 kmsg->msg.msg_name = &addr;
3011 /* if iov is set, it's allocated already */
3012 if (!kmsg->iov)
3013 kmsg->iov = kmsg->fast_iov;
3014 kmsg->msg.msg_iter.iov = kmsg->iov;
Jens Axboe03b12302019-12-02 18:50:25 -07003015 } else {
Jens Axboe3529d8c2019-12-19 18:24:38 -07003016 struct io_sr_msg *sr = &req->sr_msg;
3017
Jens Axboe0b416c32019-12-15 10:57:46 -07003018 kmsg = &io.msg;
3019 kmsg->msg.msg_name = &addr;
Jens Axboe3529d8c2019-12-19 18:24:38 -07003020
3021 io.msg.iov = io.msg.fast_iov;
3022 ret = sendmsg_copy_msghdr(&io.msg.msg, sr->msg,
3023 sr->msg_flags, &io.msg.iov);
Jens Axboe03b12302019-12-02 18:50:25 -07003024 if (ret)
Jens Axboe3529d8c2019-12-19 18:24:38 -07003025 return ret;
Jens Axboe03b12302019-12-02 18:50:25 -07003026 }
3027
Jens Axboee47293f2019-12-20 08:58:21 -07003028 flags = req->sr_msg.msg_flags;
3029 if (flags & MSG_DONTWAIT)
3030 req->flags |= REQ_F_NOWAIT;
3031 else if (force_nonblock)
3032 flags |= MSG_DONTWAIT;
3033
Jens Axboe0b416c32019-12-15 10:57:46 -07003034 ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags);
Jens Axboe03b12302019-12-02 18:50:25 -07003035 if (force_nonblock && ret == -EAGAIN) {
Jens Axboeb7bb4f72019-12-15 22:13:43 -07003036 if (req->io)
3037 return -EAGAIN;
3038 if (io_alloc_async_ctx(req))
3039 return -ENOMEM;
3040 memcpy(&req->io->msg, &io.msg, sizeof(io.msg));
3041 req->work.func = io_sendrecv_async;
Jens Axboe0b416c32019-12-15 10:57:46 -07003042 return -EAGAIN;
Jens Axboe03b12302019-12-02 18:50:25 -07003043 }
3044 if (ret == -ERESTARTSYS)
3045 ret = -EINTR;
3046 }
3047
Jens Axboeb7bb4f72019-12-15 22:13:43 -07003048 if (!io_wq_current_is_worker() && kmsg && kmsg->iov != kmsg->fast_iov)
Jens Axboe0b416c32019-12-15 10:57:46 -07003049 kfree(kmsg->iov);
Jens Axboe03b12302019-12-02 18:50:25 -07003050 io_cqring_add_event(req, ret);
Jens Axboe4e88d6e2019-12-07 20:59:47 -07003051 if (ret < 0)
3052 req_set_fail_links(req);
Jens Axboe03b12302019-12-02 18:50:25 -07003053 io_put_req_find_next(req, nxt);
3054 return 0;
3055#else
3056 return -EOPNOTSUPP;
3057#endif
3058}
3059
Jens Axboefddafac2020-01-04 20:19:44 -07003060static int io_send(struct io_kiocb *req, struct io_kiocb **nxt,
3061 bool force_nonblock)
3062{
3063#if defined(CONFIG_NET)
3064 struct socket *sock;
3065 int ret;
3066
3067 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3068 return -EINVAL;
3069
3070 sock = sock_from_file(req->file, &ret);
3071 if (sock) {
3072 struct io_sr_msg *sr = &req->sr_msg;
3073 struct msghdr msg;
3074 struct iovec iov;
3075 unsigned flags;
3076
3077 ret = import_single_range(WRITE, sr->buf, sr->len, &iov,
3078 &msg.msg_iter);
3079 if (ret)
3080 return ret;
3081
3082 msg.msg_name = NULL;
3083 msg.msg_control = NULL;
3084 msg.msg_controllen = 0;
3085 msg.msg_namelen = 0;
3086
3087 flags = req->sr_msg.msg_flags;
3088 if (flags & MSG_DONTWAIT)
3089 req->flags |= REQ_F_NOWAIT;
3090 else if (force_nonblock)
3091 flags |= MSG_DONTWAIT;
3092
Jens Axboe0b7b21e2020-01-31 08:34:59 -07003093 msg.msg_flags = flags;
3094 ret = sock_sendmsg(sock, &msg);
Jens Axboefddafac2020-01-04 20:19:44 -07003095 if (force_nonblock && ret == -EAGAIN)
3096 return -EAGAIN;
3097 if (ret == -ERESTARTSYS)
3098 ret = -EINTR;
3099 }
3100
3101 io_cqring_add_event(req, ret);
3102 if (ret < 0)
3103 req_set_fail_links(req);
3104 io_put_req_find_next(req, nxt);
3105 return 0;
3106#else
3107 return -EOPNOTSUPP;
3108#endif
3109}
3110
Jens Axboe3529d8c2019-12-19 18:24:38 -07003111static int io_recvmsg_prep(struct io_kiocb *req,
3112 const struct io_uring_sqe *sqe)
Jens Axboe03b12302019-12-02 18:50:25 -07003113{
3114#if defined(CONFIG_NET)
Jens Axboee47293f2019-12-20 08:58:21 -07003115 struct io_sr_msg *sr = &req->sr_msg;
Jens Axboe3529d8c2019-12-19 18:24:38 -07003116 struct io_async_ctx *io = req->io;
Jens Axboe06b76d42019-12-19 14:44:26 -07003117
Jens Axboe3529d8c2019-12-19 18:24:38 -07003118 sr->msg_flags = READ_ONCE(sqe->msg_flags);
3119 sr->msg = u64_to_user_ptr(READ_ONCE(sqe->addr));
Jens Axboe0b7b21e2020-01-31 08:34:59 -07003120 sr->len = READ_ONCE(sqe->len);
Jens Axboe3529d8c2019-12-19 18:24:38 -07003121
Jens Axboefddafac2020-01-04 20:19:44 -07003122 if (!io || req->opcode == IORING_OP_RECV)
Jens Axboe06b76d42019-12-19 14:44:26 -07003123 return 0;
Jens Axboe03b12302019-12-02 18:50:25 -07003124
Jens Axboed9688562019-12-09 19:35:20 -07003125 io->msg.iov = io->msg.fast_iov;
Jens Axboe3529d8c2019-12-19 18:24:38 -07003126 return recvmsg_copy_msghdr(&io->msg.msg, sr->msg, sr->msg_flags,
Jens Axboee47293f2019-12-20 08:58:21 -07003127 &io->msg.uaddr, &io->msg.iov);
Jens Axboe03b12302019-12-02 18:50:25 -07003128#else
Jens Axboee47293f2019-12-20 08:58:21 -07003129 return -EOPNOTSUPP;
Jens Axboe03b12302019-12-02 18:50:25 -07003130#endif
3131}
3132
Jens Axboefc4df992019-12-10 14:38:45 -07003133static int io_recvmsg(struct io_kiocb *req, struct io_kiocb **nxt,
3134 bool force_nonblock)
Jens Axboe03b12302019-12-02 18:50:25 -07003135{
3136#if defined(CONFIG_NET)
Jens Axboe0b416c32019-12-15 10:57:46 -07003137 struct io_async_msghdr *kmsg = NULL;
Jens Axboe0fa03c62019-04-19 13:34:07 -06003138 struct socket *sock;
3139 int ret;
3140
3141 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3142 return -EINVAL;
3143
3144 sock = sock_from_file(req->file, &ret);
3145 if (sock) {
Jens Axboeb7bb4f72019-12-15 22:13:43 -07003146 struct io_async_ctx io;
Jens Axboe03b12302019-12-02 18:50:25 -07003147 struct sockaddr_storage addr;
Jens Axboe0fa03c62019-04-19 13:34:07 -06003148 unsigned flags;
3149
Jens Axboe03b12302019-12-02 18:50:25 -07003150 if (req->io) {
Jens Axboe0b416c32019-12-15 10:57:46 -07003151 kmsg = &req->io->msg;
3152 kmsg->msg.msg_name = &addr;
3153 /* if iov is set, it's allocated already */
3154 if (!kmsg->iov)
3155 kmsg->iov = kmsg->fast_iov;
3156 kmsg->msg.msg_iter.iov = kmsg->iov;
Jens Axboe03b12302019-12-02 18:50:25 -07003157 } else {
Jens Axboe3529d8c2019-12-19 18:24:38 -07003158 struct io_sr_msg *sr = &req->sr_msg;
3159
Jens Axboe0b416c32019-12-15 10:57:46 -07003160 kmsg = &io.msg;
3161 kmsg->msg.msg_name = &addr;
Jens Axboe3529d8c2019-12-19 18:24:38 -07003162
3163 io.msg.iov = io.msg.fast_iov;
3164 ret = recvmsg_copy_msghdr(&io.msg.msg, sr->msg,
3165 sr->msg_flags, &io.msg.uaddr,
3166 &io.msg.iov);
Jens Axboe03b12302019-12-02 18:50:25 -07003167 if (ret)
Jens Axboe3529d8c2019-12-19 18:24:38 -07003168 return ret;
Jens Axboe03b12302019-12-02 18:50:25 -07003169 }
Jens Axboe0fa03c62019-04-19 13:34:07 -06003170
Jens Axboee47293f2019-12-20 08:58:21 -07003171 flags = req->sr_msg.msg_flags;
3172 if (flags & MSG_DONTWAIT)
3173 req->flags |= REQ_F_NOWAIT;
3174 else if (force_nonblock)
3175 flags |= MSG_DONTWAIT;
3176
3177 ret = __sys_recvmsg_sock(sock, &kmsg->msg, req->sr_msg.msg,
3178 kmsg->uaddr, flags);
Jens Axboe03b12302019-12-02 18:50:25 -07003179 if (force_nonblock && ret == -EAGAIN) {
Jens Axboeb7bb4f72019-12-15 22:13:43 -07003180 if (req->io)
3181 return -EAGAIN;
3182 if (io_alloc_async_ctx(req))
3183 return -ENOMEM;
3184 memcpy(&req->io->msg, &io.msg, sizeof(io.msg));
3185 req->work.func = io_sendrecv_async;
Jens Axboe0b416c32019-12-15 10:57:46 -07003186 return -EAGAIN;
Jens Axboe03b12302019-12-02 18:50:25 -07003187 }
Jens Axboe441cdbd2019-12-02 18:49:10 -07003188 if (ret == -ERESTARTSYS)
3189 ret = -EINTR;
Jens Axboe0fa03c62019-04-19 13:34:07 -06003190 }
3191
Jens Axboeb7bb4f72019-12-15 22:13:43 -07003192 if (!io_wq_current_is_worker() && kmsg && kmsg->iov != kmsg->fast_iov)
Jens Axboe0b416c32019-12-15 10:57:46 -07003193 kfree(kmsg->iov);
Jens Axboe78e19bb2019-11-06 15:21:34 -07003194 io_cqring_add_event(req, ret);
Jens Axboe4e88d6e2019-12-07 20:59:47 -07003195 if (ret < 0)
3196 req_set_fail_links(req);
Jackie Liuec9c02a2019-11-08 23:50:36 +08003197 io_put_req_find_next(req, nxt);
Jens Axboe0fa03c62019-04-19 13:34:07 -06003198 return 0;
3199#else
3200 return -EOPNOTSUPP;
3201#endif
3202}
3203
Jens Axboefddafac2020-01-04 20:19:44 -07003204static int io_recv(struct io_kiocb *req, struct io_kiocb **nxt,
3205 bool force_nonblock)
3206{
3207#if defined(CONFIG_NET)
3208 struct socket *sock;
3209 int ret;
3210
3211 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3212 return -EINVAL;
3213
3214 sock = sock_from_file(req->file, &ret);
3215 if (sock) {
3216 struct io_sr_msg *sr = &req->sr_msg;
3217 struct msghdr msg;
3218 struct iovec iov;
3219 unsigned flags;
3220
3221 ret = import_single_range(READ, sr->buf, sr->len, &iov,
3222 &msg.msg_iter);
3223 if (ret)
3224 return ret;
3225
3226 msg.msg_name = NULL;
3227 msg.msg_control = NULL;
3228 msg.msg_controllen = 0;
3229 msg.msg_namelen = 0;
3230 msg.msg_iocb = NULL;
3231 msg.msg_flags = 0;
3232
3233 flags = req->sr_msg.msg_flags;
3234 if (flags & MSG_DONTWAIT)
3235 req->flags |= REQ_F_NOWAIT;
3236 else if (force_nonblock)
3237 flags |= MSG_DONTWAIT;
3238
Jens Axboe0b7b21e2020-01-31 08:34:59 -07003239 ret = sock_recvmsg(sock, &msg, flags);
Jens Axboefddafac2020-01-04 20:19:44 -07003240 if (force_nonblock && ret == -EAGAIN)
3241 return -EAGAIN;
3242 if (ret == -ERESTARTSYS)
3243 ret = -EINTR;
3244 }
3245
3246 io_cqring_add_event(req, ret);
3247 if (ret < 0)
3248 req_set_fail_links(req);
3249 io_put_req_find_next(req, nxt);
3250 return 0;
3251#else
3252 return -EOPNOTSUPP;
3253#endif
3254}
3255
3256
Jens Axboe3529d8c2019-12-19 18:24:38 -07003257static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboe17f2fe32019-10-17 14:42:58 -06003258{
3259#if defined(CONFIG_NET)
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07003260 struct io_accept *accept = &req->accept;
3261
Jens Axboe17f2fe32019-10-17 14:42:58 -06003262 if (unlikely(req->ctx->flags & (IORING_SETUP_IOPOLL|IORING_SETUP_SQPOLL)))
3263 return -EINVAL;
Hrvoje Zeba8042d6c2019-11-25 14:40:22 -05003264 if (sqe->ioprio || sqe->len || sqe->buf_index)
Jens Axboe17f2fe32019-10-17 14:42:58 -06003265 return -EINVAL;
3266
Jens Axboed55e5f52019-12-11 16:12:15 -07003267 accept->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));
3268 accept->addr_len = u64_to_user_ptr(READ_ONCE(sqe->addr2));
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07003269 accept->flags = READ_ONCE(sqe->accept_flags);
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07003270 return 0;
3271#else
3272 return -EOPNOTSUPP;
3273#endif
3274}
Jens Axboe17f2fe32019-10-17 14:42:58 -06003275
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07003276#if defined(CONFIG_NET)
3277static int __io_accept(struct io_kiocb *req, struct io_kiocb **nxt,
3278 bool force_nonblock)
3279{
3280 struct io_accept *accept = &req->accept;
3281 unsigned file_flags;
3282 int ret;
3283
3284 file_flags = force_nonblock ? O_NONBLOCK : 0;
3285 ret = __sys_accept4_file(req->file, file_flags, accept->addr,
3286 accept->addr_len, accept->flags);
3287 if (ret == -EAGAIN && force_nonblock)
Jens Axboe17f2fe32019-10-17 14:42:58 -06003288 return -EAGAIN;
Jens Axboe8e3cca12019-11-09 19:52:33 -07003289 if (ret == -ERESTARTSYS)
3290 ret = -EINTR;
Jens Axboe4e88d6e2019-12-07 20:59:47 -07003291 if (ret < 0)
3292 req_set_fail_links(req);
Jens Axboe78e19bb2019-11-06 15:21:34 -07003293 io_cqring_add_event(req, ret);
Jackie Liuec9c02a2019-11-08 23:50:36 +08003294 io_put_req_find_next(req, nxt);
Jens Axboe17f2fe32019-10-17 14:42:58 -06003295 return 0;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07003296}
3297
3298static void io_accept_finish(struct io_wq_work **workptr)
3299{
3300 struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);
3301 struct io_kiocb *nxt = NULL;
3302
3303 if (io_req_cancelled(req))
3304 return;
3305 __io_accept(req, &nxt, false);
3306 if (nxt)
Jens Axboe78912932020-01-14 22:09:06 -07003307 io_wq_assign_next(workptr, nxt);
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07003308}
3309#endif
3310
3311static int io_accept(struct io_kiocb *req, struct io_kiocb **nxt,
3312 bool force_nonblock)
3313{
3314#if defined(CONFIG_NET)
3315 int ret;
3316
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07003317 ret = __io_accept(req, nxt, force_nonblock);
3318 if (ret == -EAGAIN && force_nonblock) {
3319 req->work.func = io_accept_finish;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07003320 io_put_req(req);
3321 return -EAGAIN;
3322 }
3323 return 0;
Jens Axboe17f2fe32019-10-17 14:42:58 -06003324#else
3325 return -EOPNOTSUPP;
3326#endif
3327}
3328
Jens Axboe3529d8c2019-12-19 18:24:38 -07003329static int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboef499a022019-12-02 16:28:46 -07003330{
3331#if defined(CONFIG_NET)
Jens Axboe3529d8c2019-12-19 18:24:38 -07003332 struct io_connect *conn = &req->connect;
3333 struct io_async_ctx *io = req->io;
Jens Axboef499a022019-12-02 16:28:46 -07003334
Jens Axboe3fbb51c2019-12-20 08:51:52 -07003335 if (unlikely(req->ctx->flags & (IORING_SETUP_IOPOLL|IORING_SETUP_SQPOLL)))
3336 return -EINVAL;
3337 if (sqe->ioprio || sqe->len || sqe->buf_index || sqe->rw_flags)
3338 return -EINVAL;
3339
Jens Axboe3529d8c2019-12-19 18:24:38 -07003340 conn->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));
3341 conn->addr_len = READ_ONCE(sqe->addr2);
3342
3343 if (!io)
3344 return 0;
3345
3346 return move_addr_to_kernel(conn->addr, conn->addr_len,
Jens Axboe3fbb51c2019-12-20 08:51:52 -07003347 &io->connect.address);
Jens Axboef499a022019-12-02 16:28:46 -07003348#else
Jens Axboe3fbb51c2019-12-20 08:51:52 -07003349 return -EOPNOTSUPP;
Jens Axboef499a022019-12-02 16:28:46 -07003350#endif
3351}
3352
Jens Axboefc4df992019-12-10 14:38:45 -07003353static int io_connect(struct io_kiocb *req, struct io_kiocb **nxt,
3354 bool force_nonblock)
Jens Axboef8e85cf2019-11-23 14:24:24 -07003355{
3356#if defined(CONFIG_NET)
Jens Axboef499a022019-12-02 16:28:46 -07003357 struct io_async_ctx __io, *io;
Jens Axboef8e85cf2019-11-23 14:24:24 -07003358 unsigned file_flags;
Jens Axboe3fbb51c2019-12-20 08:51:52 -07003359 int ret;
Jens Axboef8e85cf2019-11-23 14:24:24 -07003360
Jens Axboef499a022019-12-02 16:28:46 -07003361 if (req->io) {
3362 io = req->io;
3363 } else {
Jens Axboe3529d8c2019-12-19 18:24:38 -07003364 ret = move_addr_to_kernel(req->connect.addr,
3365 req->connect.addr_len,
3366 &__io.connect.address);
Jens Axboef499a022019-12-02 16:28:46 -07003367 if (ret)
3368 goto out;
3369 io = &__io;
3370 }
3371
Jens Axboe3fbb51c2019-12-20 08:51:52 -07003372 file_flags = force_nonblock ? O_NONBLOCK : 0;
3373
3374 ret = __sys_connect_file(req->file, &io->connect.address,
3375 req->connect.addr_len, file_flags);
Jens Axboe87f80d62019-12-03 11:23:54 -07003376 if ((ret == -EAGAIN || ret == -EINPROGRESS) && force_nonblock) {
Jens Axboeb7bb4f72019-12-15 22:13:43 -07003377 if (req->io)
3378 return -EAGAIN;
3379 if (io_alloc_async_ctx(req)) {
Jens Axboef499a022019-12-02 16:28:46 -07003380 ret = -ENOMEM;
3381 goto out;
3382 }
Jens Axboeb7bb4f72019-12-15 22:13:43 -07003383 memcpy(&req->io->connect, &__io.connect, sizeof(__io.connect));
Jens Axboef8e85cf2019-11-23 14:24:24 -07003384 return -EAGAIN;
Jens Axboef499a022019-12-02 16:28:46 -07003385 }
Jens Axboef8e85cf2019-11-23 14:24:24 -07003386 if (ret == -ERESTARTSYS)
3387 ret = -EINTR;
Jens Axboef499a022019-12-02 16:28:46 -07003388out:
Jens Axboe4e88d6e2019-12-07 20:59:47 -07003389 if (ret < 0)
3390 req_set_fail_links(req);
Jens Axboef8e85cf2019-11-23 14:24:24 -07003391 io_cqring_add_event(req, ret);
3392 io_put_req_find_next(req, nxt);
3393 return 0;
3394#else
3395 return -EOPNOTSUPP;
3396#endif
3397}
3398
Jens Axboe221c5eb2019-01-17 09:41:58 -07003399static void io_poll_remove_one(struct io_kiocb *req)
3400{
3401 struct io_poll_iocb *poll = &req->poll;
3402
3403 spin_lock(&poll->head->lock);
3404 WRITE_ONCE(poll->canceled, true);
Jens Axboe392edb42019-12-09 17:52:20 -07003405 if (!list_empty(&poll->wait.entry)) {
3406 list_del_init(&poll->wait.entry);
Jackie Liua197f662019-11-08 08:09:12 -07003407 io_queue_async_work(req);
Jens Axboe221c5eb2019-01-17 09:41:58 -07003408 }
3409 spin_unlock(&poll->head->lock);
Jens Axboe78076bb2019-12-04 19:56:40 -07003410 hash_del(&req->hash_node);
Jens Axboe221c5eb2019-01-17 09:41:58 -07003411}
3412
3413static void io_poll_remove_all(struct io_ring_ctx *ctx)
3414{
Jens Axboe78076bb2019-12-04 19:56:40 -07003415 struct hlist_node *tmp;
Jens Axboe221c5eb2019-01-17 09:41:58 -07003416 struct io_kiocb *req;
Jens Axboe78076bb2019-12-04 19:56:40 -07003417 int i;
Jens Axboe221c5eb2019-01-17 09:41:58 -07003418
3419 spin_lock_irq(&ctx->completion_lock);
Jens Axboe78076bb2019-12-04 19:56:40 -07003420 for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) {
3421 struct hlist_head *list;
3422
3423 list = &ctx->cancel_hash[i];
3424 hlist_for_each_entry_safe(req, tmp, list, hash_node)
3425 io_poll_remove_one(req);
Jens Axboe221c5eb2019-01-17 09:41:58 -07003426 }
3427 spin_unlock_irq(&ctx->completion_lock);
3428}
3429
Jens Axboe47f46762019-11-09 17:43:02 -07003430static int io_poll_cancel(struct io_ring_ctx *ctx, __u64 sqe_addr)
3431{
Jens Axboe78076bb2019-12-04 19:56:40 -07003432 struct hlist_head *list;
Jens Axboe47f46762019-11-09 17:43:02 -07003433 struct io_kiocb *req;
3434
Jens Axboe78076bb2019-12-04 19:56:40 -07003435 list = &ctx->cancel_hash[hash_long(sqe_addr, ctx->cancel_hash_bits)];
3436 hlist_for_each_entry(req, list, hash_node) {
3437 if (sqe_addr == req->user_data) {
Jens Axboeeac406c2019-11-14 12:09:58 -07003438 io_poll_remove_one(req);
3439 return 0;
3440 }
Jens Axboe47f46762019-11-09 17:43:02 -07003441 }
3442
3443 return -ENOENT;
3444}
3445
Jens Axboe3529d8c2019-12-19 18:24:38 -07003446static int io_poll_remove_prep(struct io_kiocb *req,
3447 const struct io_uring_sqe *sqe)
Jens Axboe221c5eb2019-01-17 09:41:58 -07003448{
Jens Axboe221c5eb2019-01-17 09:41:58 -07003449 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3450 return -EINVAL;
3451 if (sqe->ioprio || sqe->off || sqe->len || sqe->buf_index ||
3452 sqe->poll_events)
3453 return -EINVAL;
3454
Jens Axboe0969e782019-12-17 18:40:57 -07003455 req->poll.addr = READ_ONCE(sqe->addr);
Jens Axboe0969e782019-12-17 18:40:57 -07003456 return 0;
3457}
3458
3459/*
3460 * Find a running poll command that matches one specified in sqe->addr,
3461 * and remove it if found.
3462 */
3463static int io_poll_remove(struct io_kiocb *req)
3464{
3465 struct io_ring_ctx *ctx = req->ctx;
3466 u64 addr;
3467 int ret;
3468
Jens Axboe0969e782019-12-17 18:40:57 -07003469 addr = req->poll.addr;
Jens Axboe221c5eb2019-01-17 09:41:58 -07003470 spin_lock_irq(&ctx->completion_lock);
Jens Axboe0969e782019-12-17 18:40:57 -07003471 ret = io_poll_cancel(ctx, addr);
Jens Axboe221c5eb2019-01-17 09:41:58 -07003472 spin_unlock_irq(&ctx->completion_lock);
3473
Jens Axboe78e19bb2019-11-06 15:21:34 -07003474 io_cqring_add_event(req, ret);
Jens Axboe4e88d6e2019-12-07 20:59:47 -07003475 if (ret < 0)
3476 req_set_fail_links(req);
Jens Axboee65ef562019-03-12 10:16:44 -06003477 io_put_req(req);
Jens Axboe221c5eb2019-01-17 09:41:58 -07003478 return 0;
3479}
3480
Jens Axboeb0dd8a42019-11-18 12:14:54 -07003481static void io_poll_complete(struct io_kiocb *req, __poll_t mask, int error)
Jens Axboe221c5eb2019-01-17 09:41:58 -07003482{
Jackie Liua197f662019-11-08 08:09:12 -07003483 struct io_ring_ctx *ctx = req->ctx;
3484
Jens Axboe8c838782019-03-12 15:48:16 -06003485 req->poll.done = true;
Jens Axboeb0dd8a42019-11-18 12:14:54 -07003486 if (error)
3487 io_cqring_fill_event(req, error);
3488 else
3489 io_cqring_fill_event(req, mangle_poll(mask));
Jens Axboe8c838782019-03-12 15:48:16 -06003490 io_commit_cqring(ctx);
Jens Axboe221c5eb2019-01-17 09:41:58 -07003491}
3492
Jens Axboe561fb042019-10-24 07:25:42 -06003493static void io_poll_complete_work(struct io_wq_work **workptr)
Jens Axboe221c5eb2019-01-17 09:41:58 -07003494{
Jens Axboe561fb042019-10-24 07:25:42 -06003495 struct io_wq_work *work = *workptr;
Jens Axboe221c5eb2019-01-17 09:41:58 -07003496 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
3497 struct io_poll_iocb *poll = &req->poll;
3498 struct poll_table_struct pt = { ._key = poll->events };
3499 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe89723d02019-11-05 15:32:58 -07003500 struct io_kiocb *nxt = NULL;
Jens Axboe221c5eb2019-01-17 09:41:58 -07003501 __poll_t mask = 0;
Jens Axboeb0dd8a42019-11-18 12:14:54 -07003502 int ret = 0;
Jens Axboe221c5eb2019-01-17 09:41:58 -07003503
Jens Axboeb0dd8a42019-11-18 12:14:54 -07003504 if (work->flags & IO_WQ_WORK_CANCEL) {
Jens Axboe561fb042019-10-24 07:25:42 -06003505 WRITE_ONCE(poll->canceled, true);
Jens Axboeb0dd8a42019-11-18 12:14:54 -07003506 ret = -ECANCELED;
3507 } else if (READ_ONCE(poll->canceled)) {
3508 ret = -ECANCELED;
3509 }
Jens Axboe561fb042019-10-24 07:25:42 -06003510
Jens Axboeb0dd8a42019-11-18 12:14:54 -07003511 if (ret != -ECANCELED)
Jens Axboe221c5eb2019-01-17 09:41:58 -07003512 mask = vfs_poll(poll->file, &pt) & poll->events;
3513
3514 /*
3515 * Note that ->ki_cancel callers also delete iocb from active_reqs after
3516 * calling ->ki_cancel. We need the ctx_lock roundtrip here to
3517 * synchronize with them. In the cancellation case the list_del_init
3518 * itself is not actually needed, but harmless so we keep it in to
3519 * avoid further branches in the fast path.
3520 */
3521 spin_lock_irq(&ctx->completion_lock);
Jens Axboeb0dd8a42019-11-18 12:14:54 -07003522 if (!mask && ret != -ECANCELED) {
Jens Axboe392edb42019-12-09 17:52:20 -07003523 add_wait_queue(poll->head, &poll->wait);
Jens Axboe221c5eb2019-01-17 09:41:58 -07003524 spin_unlock_irq(&ctx->completion_lock);
3525 return;
3526 }
Jens Axboe78076bb2019-12-04 19:56:40 -07003527 hash_del(&req->hash_node);
Jens Axboeb0dd8a42019-11-18 12:14:54 -07003528 io_poll_complete(req, mask, ret);
Jens Axboe221c5eb2019-01-17 09:41:58 -07003529 spin_unlock_irq(&ctx->completion_lock);
3530
Jens Axboe8c838782019-03-12 15:48:16 -06003531 io_cqring_ev_posted(ctx);
Jens Axboe89723d02019-11-05 15:32:58 -07003532
Jens Axboe4e88d6e2019-12-07 20:59:47 -07003533 if (ret < 0)
3534 req_set_fail_links(req);
Jackie Liuec9c02a2019-11-08 23:50:36 +08003535 io_put_req_find_next(req, &nxt);
Jens Axboe89723d02019-11-05 15:32:58 -07003536 if (nxt)
Jens Axboe78912932020-01-14 22:09:06 -07003537 io_wq_assign_next(workptr, nxt);
Jens Axboe221c5eb2019-01-17 09:41:58 -07003538}
3539
Jens Axboee94f1412019-12-19 12:06:02 -07003540static void __io_poll_flush(struct io_ring_ctx *ctx, struct llist_node *nodes)
3541{
Jens Axboee94f1412019-12-19 12:06:02 -07003542 struct io_kiocb *req, *tmp;
Jens Axboe8237e042019-12-28 10:48:22 -07003543 struct req_batch rb;
Jens Axboee94f1412019-12-19 12:06:02 -07003544
Jens Axboec6ca97b302019-12-28 12:11:08 -07003545 rb.to_free = rb.need_iter = 0;
Jens Axboee94f1412019-12-19 12:06:02 -07003546 spin_lock_irq(&ctx->completion_lock);
3547 llist_for_each_entry_safe(req, tmp, nodes, llist_node) {
3548 hash_del(&req->hash_node);
3549 io_poll_complete(req, req->result, 0);
3550
Jens Axboe8237e042019-12-28 10:48:22 -07003551 if (refcount_dec_and_test(&req->refs) &&
3552 !io_req_multi_free(&rb, req)) {
3553 req->flags |= REQ_F_COMP_LOCKED;
3554 io_free_req(req);
Jens Axboee94f1412019-12-19 12:06:02 -07003555 }
3556 }
3557 spin_unlock_irq(&ctx->completion_lock);
3558
3559 io_cqring_ev_posted(ctx);
Jens Axboe8237e042019-12-28 10:48:22 -07003560 io_free_req_many(ctx, &rb);
Jens Axboee94f1412019-12-19 12:06:02 -07003561}
3562
3563static void io_poll_flush(struct io_wq_work **workptr)
3564{
3565 struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);
3566 struct llist_node *nodes;
3567
3568 nodes = llist_del_all(&req->ctx->poll_llist);
3569 if (nodes)
3570 __io_poll_flush(req->ctx, nodes);
3571}
3572
Jens Axboef0b493e2020-02-01 21:30:11 -07003573static void io_poll_trigger_evfd(struct io_wq_work **workptr)
3574{
3575 struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);
3576
3577 eventfd_signal(req->ctx->cq_ev_fd, 1);
3578 io_put_req(req);
3579}
3580
Jens Axboe221c5eb2019-01-17 09:41:58 -07003581static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
3582 void *key)
3583{
Jens Axboee9444752019-11-26 15:02:04 -07003584 struct io_poll_iocb *poll = wait->private;
Jens Axboe221c5eb2019-01-17 09:41:58 -07003585 struct io_kiocb *req = container_of(poll, struct io_kiocb, poll);
3586 struct io_ring_ctx *ctx = req->ctx;
3587 __poll_t mask = key_to_poll(key);
Jens Axboe221c5eb2019-01-17 09:41:58 -07003588
3589 /* for instances that support it check for an event match first: */
Jens Axboe8c838782019-03-12 15:48:16 -06003590 if (mask && !(mask & poll->events))
3591 return 0;
Jens Axboe221c5eb2019-01-17 09:41:58 -07003592
Jens Axboe392edb42019-12-09 17:52:20 -07003593 list_del_init(&poll->wait.entry);
Jens Axboe8c838782019-03-12 15:48:16 -06003594
Jens Axboe7c9e7f02019-11-12 08:15:53 -07003595 /*
3596 * Run completion inline if we can. We're using trylock here because
3597 * we are violating the completion_lock -> poll wq lock ordering.
3598 * If we have a link timeout we're going to need the completion_lock
3599 * for finalizing the request, mark us as having grabbed that already.
3600 */
Jens Axboee94f1412019-12-19 12:06:02 -07003601 if (mask) {
3602 unsigned long flags;
Jens Axboe8c838782019-03-12 15:48:16 -06003603
Jens Axboee94f1412019-12-19 12:06:02 -07003604 if (llist_empty(&ctx->poll_llist) &&
3605 spin_trylock_irqsave(&ctx->completion_lock, flags)) {
Jens Axboef0b493e2020-02-01 21:30:11 -07003606 bool trigger_ev;
3607
Jens Axboee94f1412019-12-19 12:06:02 -07003608 hash_del(&req->hash_node);
3609 io_poll_complete(req, mask, 0);
Jens Axboee94f1412019-12-19 12:06:02 -07003610
Jens Axboef0b493e2020-02-01 21:30:11 -07003611 trigger_ev = io_should_trigger_evfd(ctx);
3612 if (trigger_ev && eventfd_signal_count()) {
3613 trigger_ev = false;
3614 req->work.func = io_poll_trigger_evfd;
3615 } else {
3616 req->flags |= REQ_F_COMP_LOCKED;
3617 io_put_req(req);
3618 req = NULL;
3619 }
3620 spin_unlock_irqrestore(&ctx->completion_lock, flags);
3621 __io_cqring_ev_posted(ctx, trigger_ev);
Jens Axboee94f1412019-12-19 12:06:02 -07003622 } else {
3623 req->result = mask;
3624 req->llist_node.next = NULL;
3625 /* if the list wasn't empty, we're done */
3626 if (!llist_add(&req->llist_node, &ctx->poll_llist))
3627 req = NULL;
3628 else
3629 req->work.func = io_poll_flush;
3630 }
Jens Axboe8c838782019-03-12 15:48:16 -06003631 }
Jens Axboee94f1412019-12-19 12:06:02 -07003632 if (req)
3633 io_queue_async_work(req);
Jens Axboe8c838782019-03-12 15:48:16 -06003634
Jens Axboe221c5eb2019-01-17 09:41:58 -07003635 return 1;
3636}
3637
3638struct io_poll_table {
3639 struct poll_table_struct pt;
3640 struct io_kiocb *req;
3641 int error;
3642};
3643
3644static void io_poll_queue_proc(struct file *file, struct wait_queue_head *head,
3645 struct poll_table_struct *p)
3646{
3647 struct io_poll_table *pt = container_of(p, struct io_poll_table, pt);
3648
3649 if (unlikely(pt->req->poll.head)) {
3650 pt->error = -EINVAL;
3651 return;
3652 }
3653
3654 pt->error = 0;
3655 pt->req->poll.head = head;
Jens Axboe392edb42019-12-09 17:52:20 -07003656 add_wait_queue(head, &pt->req->poll.wait);
Jens Axboe221c5eb2019-01-17 09:41:58 -07003657}
3658
Jens Axboeeac406c2019-11-14 12:09:58 -07003659static void io_poll_req_insert(struct io_kiocb *req)
3660{
3661 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe78076bb2019-12-04 19:56:40 -07003662 struct hlist_head *list;
Jens Axboeeac406c2019-11-14 12:09:58 -07003663
Jens Axboe78076bb2019-12-04 19:56:40 -07003664 list = &ctx->cancel_hash[hash_long(req->user_data, ctx->cancel_hash_bits)];
3665 hlist_add_head(&req->hash_node, list);
Jens Axboeeac406c2019-11-14 12:09:58 -07003666}
3667
Jens Axboe3529d8c2019-12-19 18:24:38 -07003668static int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboe221c5eb2019-01-17 09:41:58 -07003669{
3670 struct io_poll_iocb *poll = &req->poll;
Jens Axboe221c5eb2019-01-17 09:41:58 -07003671 u16 events;
Jens Axboe221c5eb2019-01-17 09:41:58 -07003672
3673 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3674 return -EINVAL;
3675 if (sqe->addr || sqe->ioprio || sqe->off || sqe->len || sqe->buf_index)
3676 return -EINVAL;
Jens Axboe09bb8392019-03-13 12:39:28 -06003677 if (!poll->file)
3678 return -EBADF;
Jens Axboe221c5eb2019-01-17 09:41:58 -07003679
Jens Axboe221c5eb2019-01-17 09:41:58 -07003680 events = READ_ONCE(sqe->poll_events);
3681 poll->events = demangle_poll(events) | EPOLLERR | EPOLLHUP;
Jens Axboe0969e782019-12-17 18:40:57 -07003682 return 0;
3683}
3684
3685static int io_poll_add(struct io_kiocb *req, struct io_kiocb **nxt)
3686{
3687 struct io_poll_iocb *poll = &req->poll;
3688 struct io_ring_ctx *ctx = req->ctx;
3689 struct io_poll_table ipt;
3690 bool cancel = false;
3691 __poll_t mask;
Jens Axboe0969e782019-12-17 18:40:57 -07003692
3693 INIT_IO_WORK(&req->work, io_poll_complete_work);
Jens Axboe78076bb2019-12-04 19:56:40 -07003694 INIT_HLIST_NODE(&req->hash_node);
Jens Axboe221c5eb2019-01-17 09:41:58 -07003695
Jens Axboe221c5eb2019-01-17 09:41:58 -07003696 poll->head = NULL;
Jens Axboe8c838782019-03-12 15:48:16 -06003697 poll->done = false;
Jens Axboe221c5eb2019-01-17 09:41:58 -07003698 poll->canceled = false;
3699
3700 ipt.pt._qproc = io_poll_queue_proc;
3701 ipt.pt._key = poll->events;
3702 ipt.req = req;
3703 ipt.error = -EINVAL; /* same as no support for IOCB_CMD_POLL */
3704
3705 /* initialized the list so that we can do list_empty checks */
Jens Axboe392edb42019-12-09 17:52:20 -07003706 INIT_LIST_HEAD(&poll->wait.entry);
3707 init_waitqueue_func_entry(&poll->wait, io_poll_wake);
3708 poll->wait.private = poll;
Jens Axboe221c5eb2019-01-17 09:41:58 -07003709
Jens Axboe36703242019-07-25 10:20:18 -06003710 INIT_LIST_HEAD(&req->list);
3711
Jens Axboe221c5eb2019-01-17 09:41:58 -07003712 mask = vfs_poll(poll->file, &ipt.pt) & poll->events;
Jens Axboe221c5eb2019-01-17 09:41:58 -07003713
3714 spin_lock_irq(&ctx->completion_lock);
Jens Axboe8c838782019-03-12 15:48:16 -06003715 if (likely(poll->head)) {
3716 spin_lock(&poll->head->lock);
Jens Axboe392edb42019-12-09 17:52:20 -07003717 if (unlikely(list_empty(&poll->wait.entry))) {
Jens Axboe8c838782019-03-12 15:48:16 -06003718 if (ipt.error)
3719 cancel = true;
3720 ipt.error = 0;
3721 mask = 0;
3722 }
3723 if (mask || ipt.error)
Jens Axboe392edb42019-12-09 17:52:20 -07003724 list_del_init(&poll->wait.entry);
Jens Axboe8c838782019-03-12 15:48:16 -06003725 else if (cancel)
3726 WRITE_ONCE(poll->canceled, true);
3727 else if (!poll->done) /* actually waiting for an event */
Jens Axboeeac406c2019-11-14 12:09:58 -07003728 io_poll_req_insert(req);
Jens Axboe8c838782019-03-12 15:48:16 -06003729 spin_unlock(&poll->head->lock);
Jens Axboe221c5eb2019-01-17 09:41:58 -07003730 }
Jens Axboe8c838782019-03-12 15:48:16 -06003731 if (mask) { /* no async, we'd stolen it */
Jens Axboe8c838782019-03-12 15:48:16 -06003732 ipt.error = 0;
Jens Axboeb0dd8a42019-11-18 12:14:54 -07003733 io_poll_complete(req, mask, 0);
Jens Axboe8c838782019-03-12 15:48:16 -06003734 }
Jens Axboe221c5eb2019-01-17 09:41:58 -07003735 spin_unlock_irq(&ctx->completion_lock);
3736
Jens Axboe8c838782019-03-12 15:48:16 -06003737 if (mask) {
3738 io_cqring_ev_posted(ctx);
Jackie Liuec9c02a2019-11-08 23:50:36 +08003739 io_put_req_find_next(req, nxt);
Jens Axboe221c5eb2019-01-17 09:41:58 -07003740 }
Jens Axboe8c838782019-03-12 15:48:16 -06003741 return ipt.error;
Jens Axboe221c5eb2019-01-17 09:41:58 -07003742}
3743
Jens Axboe5262f562019-09-17 12:26:57 -06003744static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer)
3745{
Jens Axboead8a48a2019-11-15 08:49:11 -07003746 struct io_timeout_data *data = container_of(timer,
3747 struct io_timeout_data, timer);
3748 struct io_kiocb *req = data->req;
3749 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe5262f562019-09-17 12:26:57 -06003750 unsigned long flags;
3751
Jens Axboe5262f562019-09-17 12:26:57 -06003752 atomic_inc(&ctx->cq_timeouts);
3753
3754 spin_lock_irqsave(&ctx->completion_lock, flags);
zhangyi (F)ef036812019-10-23 15:10:08 +08003755 /*
Jens Axboe11365042019-10-16 09:08:32 -06003756 * We could be racing with timeout deletion. If the list is empty,
3757 * then timeout lookup already found it and will be handling it.
zhangyi (F)ef036812019-10-23 15:10:08 +08003758 */
Jens Axboe842f9612019-10-29 12:34:10 -06003759 if (!list_empty(&req->list)) {
Jens Axboe11365042019-10-16 09:08:32 -06003760 struct io_kiocb *prev;
Jens Axboe5262f562019-09-17 12:26:57 -06003761
Jens Axboe11365042019-10-16 09:08:32 -06003762 /*
3763 * Adjust the reqs sequence before the current one because it
Brian Gianforcarod195a662019-12-13 03:09:50 -08003764 * will consume a slot in the cq_ring and the cq_tail
Jens Axboe11365042019-10-16 09:08:32 -06003765 * pointer will be increased, otherwise other timeout reqs may
3766 * return in advance without waiting for enough wait_nr.
3767 */
3768 prev = req;
3769 list_for_each_entry_continue_reverse(prev, &ctx->timeout_list, list)
3770 prev->sequence++;
Jens Axboe11365042019-10-16 09:08:32 -06003771 list_del_init(&req->list);
Jens Axboe11365042019-10-16 09:08:32 -06003772 }
Jens Axboe842f9612019-10-29 12:34:10 -06003773
Jens Axboe78e19bb2019-11-06 15:21:34 -07003774 io_cqring_fill_event(req, -ETIME);
Jens Axboe5262f562019-09-17 12:26:57 -06003775 io_commit_cqring(ctx);
3776 spin_unlock_irqrestore(&ctx->completion_lock, flags);
3777
3778 io_cqring_ev_posted(ctx);
Jens Axboe4e88d6e2019-12-07 20:59:47 -07003779 req_set_fail_links(req);
Jens Axboe5262f562019-09-17 12:26:57 -06003780 io_put_req(req);
3781 return HRTIMER_NORESTART;
3782}
3783
Jens Axboe47f46762019-11-09 17:43:02 -07003784static int io_timeout_cancel(struct io_ring_ctx *ctx, __u64 user_data)
3785{
3786 struct io_kiocb *req;
3787 int ret = -ENOENT;
3788
3789 list_for_each_entry(req, &ctx->timeout_list, list) {
3790 if (user_data == req->user_data) {
3791 list_del_init(&req->list);
3792 ret = 0;
3793 break;
3794 }
3795 }
3796
3797 if (ret == -ENOENT)
3798 return ret;
3799
Jens Axboe2d283902019-12-04 11:08:05 -07003800 ret = hrtimer_try_to_cancel(&req->io->timeout.timer);
Jens Axboe47f46762019-11-09 17:43:02 -07003801 if (ret == -1)
3802 return -EALREADY;
3803
Jens Axboe4e88d6e2019-12-07 20:59:47 -07003804 req_set_fail_links(req);
Jens Axboe47f46762019-11-09 17:43:02 -07003805 io_cqring_fill_event(req, -ECANCELED);
3806 io_put_req(req);
3807 return 0;
3808}
3809
Jens Axboe3529d8c2019-12-19 18:24:38 -07003810static int io_timeout_remove_prep(struct io_kiocb *req,
3811 const struct io_uring_sqe *sqe)
Jens Axboeb29472e2019-12-17 18:50:29 -07003812{
Jens Axboeb29472e2019-12-17 18:50:29 -07003813 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3814 return -EINVAL;
3815 if (sqe->flags || sqe->ioprio || sqe->buf_index || sqe->len)
3816 return -EINVAL;
3817
3818 req->timeout.addr = READ_ONCE(sqe->addr);
3819 req->timeout.flags = READ_ONCE(sqe->timeout_flags);
3820 if (req->timeout.flags)
3821 return -EINVAL;
3822
Jens Axboeb29472e2019-12-17 18:50:29 -07003823 return 0;
3824}
3825
Jens Axboe11365042019-10-16 09:08:32 -06003826/*
3827 * Remove or update an existing timeout command
3828 */
Jens Axboefc4df992019-12-10 14:38:45 -07003829static int io_timeout_remove(struct io_kiocb *req)
Jens Axboe11365042019-10-16 09:08:32 -06003830{
3831 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe47f46762019-11-09 17:43:02 -07003832 int ret;
Jens Axboe11365042019-10-16 09:08:32 -06003833
Jens Axboe11365042019-10-16 09:08:32 -06003834 spin_lock_irq(&ctx->completion_lock);
Jens Axboeb29472e2019-12-17 18:50:29 -07003835 ret = io_timeout_cancel(ctx, req->timeout.addr);
Jens Axboe11365042019-10-16 09:08:32 -06003836
Jens Axboe47f46762019-11-09 17:43:02 -07003837 io_cqring_fill_event(req, ret);
Jens Axboe11365042019-10-16 09:08:32 -06003838 io_commit_cqring(ctx);
3839 spin_unlock_irq(&ctx->completion_lock);
Jens Axboe5262f562019-09-17 12:26:57 -06003840 io_cqring_ev_posted(ctx);
Jens Axboe4e88d6e2019-12-07 20:59:47 -07003841 if (ret < 0)
3842 req_set_fail_links(req);
Jackie Liuec9c02a2019-11-08 23:50:36 +08003843 io_put_req(req);
Jens Axboe11365042019-10-16 09:08:32 -06003844 return 0;
Jens Axboe5262f562019-09-17 12:26:57 -06003845}
3846
Jens Axboe3529d8c2019-12-19 18:24:38 -07003847static int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe,
Jens Axboe2d283902019-12-04 11:08:05 -07003848 bool is_timeout_link)
Jens Axboe5262f562019-09-17 12:26:57 -06003849{
Jens Axboead8a48a2019-11-15 08:49:11 -07003850 struct io_timeout_data *data;
Jens Axboea41525a2019-10-15 16:48:15 -06003851 unsigned flags;
Jens Axboe5262f562019-09-17 12:26:57 -06003852
Jens Axboead8a48a2019-11-15 08:49:11 -07003853 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
Jens Axboe5262f562019-09-17 12:26:57 -06003854 return -EINVAL;
Jens Axboead8a48a2019-11-15 08:49:11 -07003855 if (sqe->ioprio || sqe->buf_index || sqe->len != 1)
Jens Axboea41525a2019-10-15 16:48:15 -06003856 return -EINVAL;
Jens Axboe2d283902019-12-04 11:08:05 -07003857 if (sqe->off && is_timeout_link)
3858 return -EINVAL;
Jens Axboea41525a2019-10-15 16:48:15 -06003859 flags = READ_ONCE(sqe->timeout_flags);
3860 if (flags & ~IORING_TIMEOUT_ABS)
Jens Axboe5262f562019-09-17 12:26:57 -06003861 return -EINVAL;
Arnd Bergmannbdf20072019-10-01 09:53:29 -06003862
Jens Axboe26a61672019-12-20 09:02:01 -07003863 req->timeout.count = READ_ONCE(sqe->off);
3864
Jens Axboe3529d8c2019-12-19 18:24:38 -07003865 if (!req->io && io_alloc_async_ctx(req))
Jens Axboe26a61672019-12-20 09:02:01 -07003866 return -ENOMEM;
3867
3868 data = &req->io->timeout;
Jens Axboead8a48a2019-11-15 08:49:11 -07003869 data->req = req;
Jens Axboead8a48a2019-11-15 08:49:11 -07003870 req->flags |= REQ_F_TIMEOUT;
3871
3872 if (get_timespec64(&data->ts, u64_to_user_ptr(sqe->addr)))
Jens Axboe5262f562019-09-17 12:26:57 -06003873 return -EFAULT;
3874
Jens Axboe11365042019-10-16 09:08:32 -06003875 if (flags & IORING_TIMEOUT_ABS)
Jens Axboead8a48a2019-11-15 08:49:11 -07003876 data->mode = HRTIMER_MODE_ABS;
Jens Axboe11365042019-10-16 09:08:32 -06003877 else
Jens Axboead8a48a2019-11-15 08:49:11 -07003878 data->mode = HRTIMER_MODE_REL;
Jens Axboe11365042019-10-16 09:08:32 -06003879
Jens Axboead8a48a2019-11-15 08:49:11 -07003880 hrtimer_init(&data->timer, CLOCK_MONOTONIC, data->mode);
3881 return 0;
3882}
3883
Jens Axboefc4df992019-12-10 14:38:45 -07003884static int io_timeout(struct io_kiocb *req)
Jens Axboead8a48a2019-11-15 08:49:11 -07003885{
3886 unsigned count;
3887 struct io_ring_ctx *ctx = req->ctx;
3888 struct io_timeout_data *data;
3889 struct list_head *entry;
3890 unsigned span = 0;
Jens Axboead8a48a2019-11-15 08:49:11 -07003891
Jens Axboe2d283902019-12-04 11:08:05 -07003892 data = &req->io->timeout;
Jens Axboe93bd25b2019-11-11 23:34:31 -07003893
Jens Axboe5262f562019-09-17 12:26:57 -06003894 /*
3895 * sqe->off holds how many events that need to occur for this
Jens Axboe93bd25b2019-11-11 23:34:31 -07003896 * timeout event to be satisfied. If it isn't set, then this is
3897 * a pure timeout request, sequence isn't used.
Jens Axboe5262f562019-09-17 12:26:57 -06003898 */
Jens Axboe26a61672019-12-20 09:02:01 -07003899 count = req->timeout.count;
Jens Axboe93bd25b2019-11-11 23:34:31 -07003900 if (!count) {
3901 req->flags |= REQ_F_TIMEOUT_NOSEQ;
3902 spin_lock_irq(&ctx->completion_lock);
3903 entry = ctx->timeout_list.prev;
3904 goto add;
3905 }
Jens Axboe5262f562019-09-17 12:26:57 -06003906
3907 req->sequence = ctx->cached_sq_head + count - 1;
Jens Axboe2d283902019-12-04 11:08:05 -07003908 data->seq_offset = count;
Jens Axboe5262f562019-09-17 12:26:57 -06003909
3910 /*
3911 * Insertion sort, ensuring the first entry in the list is always
3912 * the one we need first.
3913 */
Jens Axboe5262f562019-09-17 12:26:57 -06003914 spin_lock_irq(&ctx->completion_lock);
3915 list_for_each_prev(entry, &ctx->timeout_list) {
3916 struct io_kiocb *nxt = list_entry(entry, struct io_kiocb, list);
yangerkun5da0fb12019-10-15 21:59:29 +08003917 unsigned nxt_sq_head;
3918 long long tmp, tmp_nxt;
Jens Axboe2d283902019-12-04 11:08:05 -07003919 u32 nxt_offset = nxt->io->timeout.seq_offset;
Jens Axboe5262f562019-09-17 12:26:57 -06003920
Jens Axboe93bd25b2019-11-11 23:34:31 -07003921 if (nxt->flags & REQ_F_TIMEOUT_NOSEQ)
3922 continue;
3923
yangerkun5da0fb12019-10-15 21:59:29 +08003924 /*
3925 * Since cached_sq_head + count - 1 can overflow, use type long
3926 * long to store it.
3927 */
3928 tmp = (long long)ctx->cached_sq_head + count - 1;
Pavel Begunkovcc42e0a2019-11-25 23:14:38 +03003929 nxt_sq_head = nxt->sequence - nxt_offset + 1;
3930 tmp_nxt = (long long)nxt_sq_head + nxt_offset - 1;
yangerkun5da0fb12019-10-15 21:59:29 +08003931
3932 /*
3933 * cached_sq_head may overflow, and it will never overflow twice
3934 * once there is some timeout req still be valid.
3935 */
3936 if (ctx->cached_sq_head < nxt_sq_head)
yangerkun8b07a652019-10-17 12:12:35 +08003937 tmp += UINT_MAX;
yangerkun5da0fb12019-10-15 21:59:29 +08003938
zhangyi (F)a1f58ba2019-10-23 15:10:09 +08003939 if (tmp > tmp_nxt)
Jens Axboe5262f562019-09-17 12:26:57 -06003940 break;
zhangyi (F)a1f58ba2019-10-23 15:10:09 +08003941
3942 /*
3943 * Sequence of reqs after the insert one and itself should
3944 * be adjusted because each timeout req consumes a slot.
3945 */
3946 span++;
3947 nxt->sequence++;
Jens Axboe5262f562019-09-17 12:26:57 -06003948 }
zhangyi (F)a1f58ba2019-10-23 15:10:09 +08003949 req->sequence -= span;
Jens Axboe93bd25b2019-11-11 23:34:31 -07003950add:
Jens Axboe5262f562019-09-17 12:26:57 -06003951 list_add(&req->list, entry);
Jens Axboead8a48a2019-11-15 08:49:11 -07003952 data->timer.function = io_timeout_fn;
3953 hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), data->mode);
Jens Axboe842f9612019-10-29 12:34:10 -06003954 spin_unlock_irq(&ctx->completion_lock);
Jens Axboe5262f562019-09-17 12:26:57 -06003955 return 0;
3956}
3957
Jens Axboe62755e32019-10-28 21:49:21 -06003958static bool io_cancel_cb(struct io_wq_work *work, void *data)
Jens Axboede0617e2019-04-06 21:51:27 -06003959{
Jens Axboe62755e32019-10-28 21:49:21 -06003960 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
Jens Axboede0617e2019-04-06 21:51:27 -06003961
Jens Axboe62755e32019-10-28 21:49:21 -06003962 return req->user_data == (unsigned long) data;
3963}
3964
Jens Axboee977d6d2019-11-05 12:39:45 -07003965static int io_async_cancel_one(struct io_ring_ctx *ctx, void *sqe_addr)
Jens Axboe62755e32019-10-28 21:49:21 -06003966{
Jens Axboe62755e32019-10-28 21:49:21 -06003967 enum io_wq_cancel cancel_ret;
Jens Axboe62755e32019-10-28 21:49:21 -06003968 int ret = 0;
3969
Jens Axboe62755e32019-10-28 21:49:21 -06003970 cancel_ret = io_wq_cancel_cb(ctx->io_wq, io_cancel_cb, sqe_addr);
3971 switch (cancel_ret) {
3972 case IO_WQ_CANCEL_OK:
3973 ret = 0;
3974 break;
3975 case IO_WQ_CANCEL_RUNNING:
3976 ret = -EALREADY;
3977 break;
3978 case IO_WQ_CANCEL_NOTFOUND:
3979 ret = -ENOENT;
3980 break;
3981 }
3982
Jens Axboee977d6d2019-11-05 12:39:45 -07003983 return ret;
3984}
3985
Jens Axboe47f46762019-11-09 17:43:02 -07003986static void io_async_find_and_cancel(struct io_ring_ctx *ctx,
3987 struct io_kiocb *req, __u64 sqe_addr,
Jens Axboeb0dd8a42019-11-18 12:14:54 -07003988 struct io_kiocb **nxt, int success_ret)
Jens Axboe47f46762019-11-09 17:43:02 -07003989{
3990 unsigned long flags;
3991 int ret;
3992
3993 ret = io_async_cancel_one(ctx, (void *) (unsigned long) sqe_addr);
3994 if (ret != -ENOENT) {
3995 spin_lock_irqsave(&ctx->completion_lock, flags);
3996 goto done;
3997 }
3998
3999 spin_lock_irqsave(&ctx->completion_lock, flags);
4000 ret = io_timeout_cancel(ctx, sqe_addr);
4001 if (ret != -ENOENT)
4002 goto done;
4003 ret = io_poll_cancel(ctx, sqe_addr);
4004done:
Jens Axboeb0dd8a42019-11-18 12:14:54 -07004005 if (!ret)
4006 ret = success_ret;
Jens Axboe47f46762019-11-09 17:43:02 -07004007 io_cqring_fill_event(req, ret);
4008 io_commit_cqring(ctx);
4009 spin_unlock_irqrestore(&ctx->completion_lock, flags);
4010 io_cqring_ev_posted(ctx);
4011
Jens Axboe4e88d6e2019-12-07 20:59:47 -07004012 if (ret < 0)
4013 req_set_fail_links(req);
Jens Axboe47f46762019-11-09 17:43:02 -07004014 io_put_req_find_next(req, nxt);
4015}
4016
Jens Axboe3529d8c2019-12-19 18:24:38 -07004017static int io_async_cancel_prep(struct io_kiocb *req,
4018 const struct io_uring_sqe *sqe)
Jens Axboee977d6d2019-11-05 12:39:45 -07004019{
Jens Axboefbf23842019-12-17 18:45:56 -07004020 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
Jens Axboee977d6d2019-11-05 12:39:45 -07004021 return -EINVAL;
4022 if (sqe->flags || sqe->ioprio || sqe->off || sqe->len ||
4023 sqe->cancel_flags)
4024 return -EINVAL;
4025
Jens Axboefbf23842019-12-17 18:45:56 -07004026 req->cancel.addr = READ_ONCE(sqe->addr);
4027 return 0;
4028}
4029
4030static int io_async_cancel(struct io_kiocb *req, struct io_kiocb **nxt)
4031{
4032 struct io_ring_ctx *ctx = req->ctx;
Jens Axboefbf23842019-12-17 18:45:56 -07004033
4034 io_async_find_and_cancel(ctx, req, req->cancel.addr, nxt, 0);
Jens Axboe62755e32019-10-28 21:49:21 -06004035 return 0;
4036}
4037
Jens Axboe05f3fb32019-12-09 11:22:50 -07004038static int io_files_update_prep(struct io_kiocb *req,
4039 const struct io_uring_sqe *sqe)
4040{
4041 if (sqe->flags || sqe->ioprio || sqe->rw_flags)
4042 return -EINVAL;
4043
4044 req->files_update.offset = READ_ONCE(sqe->off);
4045 req->files_update.nr_args = READ_ONCE(sqe->len);
4046 if (!req->files_update.nr_args)
4047 return -EINVAL;
4048 req->files_update.arg = READ_ONCE(sqe->addr);
4049 return 0;
4050}
4051
4052static int io_files_update(struct io_kiocb *req, bool force_nonblock)
4053{
4054 struct io_ring_ctx *ctx = req->ctx;
4055 struct io_uring_files_update up;
4056 int ret;
4057
Jens Axboef86cd202020-01-29 13:46:44 -07004058 if (force_nonblock)
Jens Axboe05f3fb32019-12-09 11:22:50 -07004059 return -EAGAIN;
Jens Axboe05f3fb32019-12-09 11:22:50 -07004060
4061 up.offset = req->files_update.offset;
4062 up.fds = req->files_update.arg;
4063
4064 mutex_lock(&ctx->uring_lock);
4065 ret = __io_sqe_files_update(ctx, &up, req->files_update.nr_args);
4066 mutex_unlock(&ctx->uring_lock);
4067
4068 if (ret < 0)
4069 req_set_fail_links(req);
4070 io_cqring_add_event(req, ret);
4071 io_put_req(req);
4072 return 0;
4073}
4074
Jens Axboe3529d8c2019-12-19 18:24:38 -07004075static int io_req_defer_prep(struct io_kiocb *req,
4076 const struct io_uring_sqe *sqe)
Jens Axboef67676d2019-12-02 11:03:47 -07004077{
Jens Axboee7815732019-12-17 19:45:06 -07004078 ssize_t ret = 0;
Jens Axboef67676d2019-12-02 11:03:47 -07004079
Jens Axboef86cd202020-01-29 13:46:44 -07004080 if (io_op_defs[req->opcode].file_table) {
4081 ret = io_grab_files(req);
4082 if (unlikely(ret))
4083 return ret;
4084 }
4085
Jens Axboecccf0ee2020-01-27 16:34:48 -07004086 io_req_work_grab_env(req, &io_op_defs[req->opcode]);
4087
Jens Axboed625c6e2019-12-17 19:53:05 -07004088 switch (req->opcode) {
Jens Axboee7815732019-12-17 19:45:06 -07004089 case IORING_OP_NOP:
4090 break;
Jens Axboef67676d2019-12-02 11:03:47 -07004091 case IORING_OP_READV:
4092 case IORING_OP_READ_FIXED:
Jens Axboe3a6820f2019-12-22 15:19:35 -07004093 case IORING_OP_READ:
Jens Axboe3529d8c2019-12-19 18:24:38 -07004094 ret = io_read_prep(req, sqe, true);
Jens Axboef67676d2019-12-02 11:03:47 -07004095 break;
4096 case IORING_OP_WRITEV:
4097 case IORING_OP_WRITE_FIXED:
Jens Axboe3a6820f2019-12-22 15:19:35 -07004098 case IORING_OP_WRITE:
Jens Axboe3529d8c2019-12-19 18:24:38 -07004099 ret = io_write_prep(req, sqe, true);
Jens Axboef67676d2019-12-02 11:03:47 -07004100 break;
Jens Axboe0969e782019-12-17 18:40:57 -07004101 case IORING_OP_POLL_ADD:
Jens Axboe3529d8c2019-12-19 18:24:38 -07004102 ret = io_poll_add_prep(req, sqe);
Jens Axboe0969e782019-12-17 18:40:57 -07004103 break;
4104 case IORING_OP_POLL_REMOVE:
Jens Axboe3529d8c2019-12-19 18:24:38 -07004105 ret = io_poll_remove_prep(req, sqe);
Jens Axboe0969e782019-12-17 18:40:57 -07004106 break;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004107 case IORING_OP_FSYNC:
Jens Axboe3529d8c2019-12-19 18:24:38 -07004108 ret = io_prep_fsync(req, sqe);
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004109 break;
4110 case IORING_OP_SYNC_FILE_RANGE:
Jens Axboe3529d8c2019-12-19 18:24:38 -07004111 ret = io_prep_sfr(req, sqe);
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004112 break;
Jens Axboe03b12302019-12-02 18:50:25 -07004113 case IORING_OP_SENDMSG:
Jens Axboefddafac2020-01-04 20:19:44 -07004114 case IORING_OP_SEND:
Jens Axboe3529d8c2019-12-19 18:24:38 -07004115 ret = io_sendmsg_prep(req, sqe);
Jens Axboe03b12302019-12-02 18:50:25 -07004116 break;
4117 case IORING_OP_RECVMSG:
Jens Axboefddafac2020-01-04 20:19:44 -07004118 case IORING_OP_RECV:
Jens Axboe3529d8c2019-12-19 18:24:38 -07004119 ret = io_recvmsg_prep(req, sqe);
Jens Axboe03b12302019-12-02 18:50:25 -07004120 break;
Jens Axboef499a022019-12-02 16:28:46 -07004121 case IORING_OP_CONNECT:
Jens Axboe3529d8c2019-12-19 18:24:38 -07004122 ret = io_connect_prep(req, sqe);
Jens Axboef499a022019-12-02 16:28:46 -07004123 break;
Jens Axboe2d283902019-12-04 11:08:05 -07004124 case IORING_OP_TIMEOUT:
Jens Axboe3529d8c2019-12-19 18:24:38 -07004125 ret = io_timeout_prep(req, sqe, false);
Jens Axboeb7bb4f72019-12-15 22:13:43 -07004126 break;
Jens Axboeb29472e2019-12-17 18:50:29 -07004127 case IORING_OP_TIMEOUT_REMOVE:
Jens Axboe3529d8c2019-12-19 18:24:38 -07004128 ret = io_timeout_remove_prep(req, sqe);
Jens Axboeb29472e2019-12-17 18:50:29 -07004129 break;
Jens Axboefbf23842019-12-17 18:45:56 -07004130 case IORING_OP_ASYNC_CANCEL:
Jens Axboe3529d8c2019-12-19 18:24:38 -07004131 ret = io_async_cancel_prep(req, sqe);
Jens Axboefbf23842019-12-17 18:45:56 -07004132 break;
Jens Axboe2d283902019-12-04 11:08:05 -07004133 case IORING_OP_LINK_TIMEOUT:
Jens Axboe3529d8c2019-12-19 18:24:38 -07004134 ret = io_timeout_prep(req, sqe, true);
Jens Axboeb7bb4f72019-12-15 22:13:43 -07004135 break;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004136 case IORING_OP_ACCEPT:
Jens Axboe3529d8c2019-12-19 18:24:38 -07004137 ret = io_accept_prep(req, sqe);
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004138 break;
Jens Axboed63d1b52019-12-10 10:38:56 -07004139 case IORING_OP_FALLOCATE:
4140 ret = io_fallocate_prep(req, sqe);
4141 break;
Jens Axboe15b71ab2019-12-11 11:20:36 -07004142 case IORING_OP_OPENAT:
4143 ret = io_openat_prep(req, sqe);
4144 break;
Jens Axboeb5dba592019-12-11 14:02:38 -07004145 case IORING_OP_CLOSE:
4146 ret = io_close_prep(req, sqe);
4147 break;
Jens Axboe05f3fb32019-12-09 11:22:50 -07004148 case IORING_OP_FILES_UPDATE:
4149 ret = io_files_update_prep(req, sqe);
4150 break;
Jens Axboeeddc7ef2019-12-13 21:18:10 -07004151 case IORING_OP_STATX:
4152 ret = io_statx_prep(req, sqe);
4153 break;
Jens Axboe4840e412019-12-25 22:03:45 -07004154 case IORING_OP_FADVISE:
4155 ret = io_fadvise_prep(req, sqe);
4156 break;
Jens Axboec1ca7572019-12-25 22:18:28 -07004157 case IORING_OP_MADVISE:
4158 ret = io_madvise_prep(req, sqe);
4159 break;
Jens Axboecebdb982020-01-08 17:59:24 -07004160 case IORING_OP_OPENAT2:
4161 ret = io_openat2_prep(req, sqe);
4162 break;
Jens Axboe3e4827b2020-01-08 15:18:09 -07004163 case IORING_OP_EPOLL_CTL:
4164 ret = io_epoll_ctl_prep(req, sqe);
4165 break;
Jens Axboef67676d2019-12-02 11:03:47 -07004166 default:
Jens Axboee7815732019-12-17 19:45:06 -07004167 printk_once(KERN_WARNING "io_uring: unhandled opcode %d\n",
4168 req->opcode);
4169 ret = -EINVAL;
Jens Axboeb7bb4f72019-12-15 22:13:43 -07004170 break;
Jens Axboef67676d2019-12-02 11:03:47 -07004171 }
4172
Jens Axboeb7bb4f72019-12-15 22:13:43 -07004173 return ret;
Jens Axboef67676d2019-12-02 11:03:47 -07004174}
4175
Jens Axboe3529d8c2019-12-19 18:24:38 -07004176static int io_req_defer(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboede0617e2019-04-06 21:51:27 -06004177{
Jackie Liua197f662019-11-08 08:09:12 -07004178 struct io_ring_ctx *ctx = req->ctx;
Jens Axboef67676d2019-12-02 11:03:47 -07004179 int ret;
Jens Axboede0617e2019-04-06 21:51:27 -06004180
Bob Liu9d858b22019-11-13 18:06:25 +08004181 /* Still need defer if there is pending req in defer list. */
4182 if (!req_need_defer(req) && list_empty(&ctx->defer_list))
Jens Axboede0617e2019-04-06 21:51:27 -06004183 return 0;
4184
Jens Axboe3529d8c2019-12-19 18:24:38 -07004185 if (!req->io && io_alloc_async_ctx(req))
Jens Axboede0617e2019-04-06 21:51:27 -06004186 return -EAGAIN;
4187
Jens Axboe3529d8c2019-12-19 18:24:38 -07004188 ret = io_req_defer_prep(req, sqe);
Jens Axboeb7bb4f72019-12-15 22:13:43 -07004189 if (ret < 0)
Jens Axboe2d283902019-12-04 11:08:05 -07004190 return ret;
Jens Axboe2d283902019-12-04 11:08:05 -07004191
Jens Axboede0617e2019-04-06 21:51:27 -06004192 spin_lock_irq(&ctx->completion_lock);
Bob Liu9d858b22019-11-13 18:06:25 +08004193 if (!req_need_defer(req) && list_empty(&ctx->defer_list)) {
Jens Axboede0617e2019-04-06 21:51:27 -06004194 spin_unlock_irq(&ctx->completion_lock);
Jens Axboede0617e2019-04-06 21:51:27 -06004195 return 0;
4196 }
4197
Jens Axboe915967f2019-11-21 09:01:20 -07004198 trace_io_uring_defer(ctx, req, req->user_data);
Jens Axboede0617e2019-04-06 21:51:27 -06004199 list_add_tail(&req->list, &ctx->defer_list);
4200 spin_unlock_irq(&ctx->completion_lock);
4201 return -EIOCBQUEUED;
4202}
4203
Jens Axboe3529d8c2019-12-19 18:24:38 -07004204static int io_issue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
4205 struct io_kiocb **nxt, bool force_nonblock)
Jens Axboe2b188cc2019-01-07 10:46:33 -07004206{
Jackie Liua197f662019-11-08 08:09:12 -07004207 struct io_ring_ctx *ctx = req->ctx;
Jens Axboed625c6e2019-12-17 19:53:05 -07004208 int ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07004209
Jens Axboed625c6e2019-12-17 19:53:05 -07004210 switch (req->opcode) {
Jens Axboe2b188cc2019-01-07 10:46:33 -07004211 case IORING_OP_NOP:
Jens Axboe78e19bb2019-11-06 15:21:34 -07004212 ret = io_nop(req);
Jens Axboe2b188cc2019-01-07 10:46:33 -07004213 break;
4214 case IORING_OP_READV:
Jens Axboe3529d8c2019-12-19 18:24:38 -07004215 case IORING_OP_READ_FIXED:
Jens Axboe3a6820f2019-12-22 15:19:35 -07004216 case IORING_OP_READ:
Jens Axboe3529d8c2019-12-19 18:24:38 -07004217 if (sqe) {
4218 ret = io_read_prep(req, sqe, force_nonblock);
4219 if (ret < 0)
4220 break;
4221 }
Pavel Begunkov267bc902019-11-07 01:41:08 +03004222 ret = io_read(req, nxt, force_nonblock);
Jens Axboe2b188cc2019-01-07 10:46:33 -07004223 break;
4224 case IORING_OP_WRITEV:
Jens Axboeedafcce2019-01-09 09:16:05 -07004225 case IORING_OP_WRITE_FIXED:
Jens Axboe3a6820f2019-12-22 15:19:35 -07004226 case IORING_OP_WRITE:
Jens Axboe3529d8c2019-12-19 18:24:38 -07004227 if (sqe) {
4228 ret = io_write_prep(req, sqe, force_nonblock);
4229 if (ret < 0)
4230 break;
4231 }
Pavel Begunkov267bc902019-11-07 01:41:08 +03004232 ret = io_write(req, nxt, force_nonblock);
Jens Axboe2b188cc2019-01-07 10:46:33 -07004233 break;
Christoph Hellwigc992fe22019-01-11 09:43:02 -07004234 case IORING_OP_FSYNC:
Jens Axboe3529d8c2019-12-19 18:24:38 -07004235 if (sqe) {
4236 ret = io_prep_fsync(req, sqe);
4237 if (ret < 0)
4238 break;
4239 }
Jens Axboefc4df992019-12-10 14:38:45 -07004240 ret = io_fsync(req, nxt, force_nonblock);
Christoph Hellwigc992fe22019-01-11 09:43:02 -07004241 break;
Jens Axboe221c5eb2019-01-17 09:41:58 -07004242 case IORING_OP_POLL_ADD:
Jens Axboe3529d8c2019-12-19 18:24:38 -07004243 if (sqe) {
4244 ret = io_poll_add_prep(req, sqe);
4245 if (ret)
4246 break;
4247 }
Jens Axboefc4df992019-12-10 14:38:45 -07004248 ret = io_poll_add(req, nxt);
Jens Axboe221c5eb2019-01-17 09:41:58 -07004249 break;
4250 case IORING_OP_POLL_REMOVE:
Jens Axboe3529d8c2019-12-19 18:24:38 -07004251 if (sqe) {
4252 ret = io_poll_remove_prep(req, sqe);
4253 if (ret < 0)
4254 break;
4255 }
Jens Axboefc4df992019-12-10 14:38:45 -07004256 ret = io_poll_remove(req);
Jens Axboe221c5eb2019-01-17 09:41:58 -07004257 break;
Jens Axboe5d17b4a2019-04-09 14:56:44 -06004258 case IORING_OP_SYNC_FILE_RANGE:
Jens Axboe3529d8c2019-12-19 18:24:38 -07004259 if (sqe) {
4260 ret = io_prep_sfr(req, sqe);
4261 if (ret < 0)
4262 break;
4263 }
Jens Axboefc4df992019-12-10 14:38:45 -07004264 ret = io_sync_file_range(req, nxt, force_nonblock);
Jens Axboe5d17b4a2019-04-09 14:56:44 -06004265 break;
Jens Axboe0fa03c62019-04-19 13:34:07 -06004266 case IORING_OP_SENDMSG:
Jens Axboefddafac2020-01-04 20:19:44 -07004267 case IORING_OP_SEND:
Jens Axboe3529d8c2019-12-19 18:24:38 -07004268 if (sqe) {
4269 ret = io_sendmsg_prep(req, sqe);
4270 if (ret < 0)
4271 break;
4272 }
Jens Axboefddafac2020-01-04 20:19:44 -07004273 if (req->opcode == IORING_OP_SENDMSG)
4274 ret = io_sendmsg(req, nxt, force_nonblock);
4275 else
4276 ret = io_send(req, nxt, force_nonblock);
Jens Axboe0fa03c62019-04-19 13:34:07 -06004277 break;
Jens Axboeaa1fa282019-04-19 13:38:09 -06004278 case IORING_OP_RECVMSG:
Jens Axboefddafac2020-01-04 20:19:44 -07004279 case IORING_OP_RECV:
Jens Axboe3529d8c2019-12-19 18:24:38 -07004280 if (sqe) {
4281 ret = io_recvmsg_prep(req, sqe);
4282 if (ret)
4283 break;
4284 }
Jens Axboefddafac2020-01-04 20:19:44 -07004285 if (req->opcode == IORING_OP_RECVMSG)
4286 ret = io_recvmsg(req, nxt, force_nonblock);
4287 else
4288 ret = io_recv(req, nxt, force_nonblock);
Jens Axboeaa1fa282019-04-19 13:38:09 -06004289 break;
Jens Axboe5262f562019-09-17 12:26:57 -06004290 case IORING_OP_TIMEOUT:
Jens Axboe3529d8c2019-12-19 18:24:38 -07004291 if (sqe) {
4292 ret = io_timeout_prep(req, sqe, false);
4293 if (ret)
4294 break;
4295 }
Jens Axboefc4df992019-12-10 14:38:45 -07004296 ret = io_timeout(req);
Jens Axboe5262f562019-09-17 12:26:57 -06004297 break;
Jens Axboe11365042019-10-16 09:08:32 -06004298 case IORING_OP_TIMEOUT_REMOVE:
Jens Axboe3529d8c2019-12-19 18:24:38 -07004299 if (sqe) {
4300 ret = io_timeout_remove_prep(req, sqe);
4301 if (ret)
4302 break;
4303 }
Jens Axboefc4df992019-12-10 14:38:45 -07004304 ret = io_timeout_remove(req);
Jens Axboe11365042019-10-16 09:08:32 -06004305 break;
Jens Axboe17f2fe32019-10-17 14:42:58 -06004306 case IORING_OP_ACCEPT:
Jens Axboe3529d8c2019-12-19 18:24:38 -07004307 if (sqe) {
4308 ret = io_accept_prep(req, sqe);
4309 if (ret)
4310 break;
4311 }
Jens Axboefc4df992019-12-10 14:38:45 -07004312 ret = io_accept(req, nxt, force_nonblock);
Jens Axboe17f2fe32019-10-17 14:42:58 -06004313 break;
Jens Axboef8e85cf2019-11-23 14:24:24 -07004314 case IORING_OP_CONNECT:
Jens Axboe3529d8c2019-12-19 18:24:38 -07004315 if (sqe) {
4316 ret = io_connect_prep(req, sqe);
4317 if (ret)
4318 break;
4319 }
Jens Axboefc4df992019-12-10 14:38:45 -07004320 ret = io_connect(req, nxt, force_nonblock);
Jens Axboef8e85cf2019-11-23 14:24:24 -07004321 break;
Jens Axboe62755e32019-10-28 21:49:21 -06004322 case IORING_OP_ASYNC_CANCEL:
Jens Axboe3529d8c2019-12-19 18:24:38 -07004323 if (sqe) {
4324 ret = io_async_cancel_prep(req, sqe);
4325 if (ret)
4326 break;
4327 }
Jens Axboefc4df992019-12-10 14:38:45 -07004328 ret = io_async_cancel(req, nxt);
Jens Axboe62755e32019-10-28 21:49:21 -06004329 break;
Jens Axboed63d1b52019-12-10 10:38:56 -07004330 case IORING_OP_FALLOCATE:
4331 if (sqe) {
4332 ret = io_fallocate_prep(req, sqe);
4333 if (ret)
4334 break;
4335 }
4336 ret = io_fallocate(req, nxt, force_nonblock);
4337 break;
Jens Axboe15b71ab2019-12-11 11:20:36 -07004338 case IORING_OP_OPENAT:
4339 if (sqe) {
4340 ret = io_openat_prep(req, sqe);
4341 if (ret)
4342 break;
4343 }
4344 ret = io_openat(req, nxt, force_nonblock);
4345 break;
Jens Axboeb5dba592019-12-11 14:02:38 -07004346 case IORING_OP_CLOSE:
4347 if (sqe) {
4348 ret = io_close_prep(req, sqe);
4349 if (ret)
4350 break;
4351 }
4352 ret = io_close(req, nxt, force_nonblock);
4353 break;
Jens Axboe05f3fb32019-12-09 11:22:50 -07004354 case IORING_OP_FILES_UPDATE:
4355 if (sqe) {
4356 ret = io_files_update_prep(req, sqe);
4357 if (ret)
4358 break;
4359 }
4360 ret = io_files_update(req, force_nonblock);
4361 break;
Jens Axboeeddc7ef2019-12-13 21:18:10 -07004362 case IORING_OP_STATX:
4363 if (sqe) {
4364 ret = io_statx_prep(req, sqe);
4365 if (ret)
4366 break;
4367 }
4368 ret = io_statx(req, nxt, force_nonblock);
4369 break;
Jens Axboe4840e412019-12-25 22:03:45 -07004370 case IORING_OP_FADVISE:
4371 if (sqe) {
4372 ret = io_fadvise_prep(req, sqe);
4373 if (ret)
4374 break;
4375 }
4376 ret = io_fadvise(req, nxt, force_nonblock);
4377 break;
Jens Axboec1ca7572019-12-25 22:18:28 -07004378 case IORING_OP_MADVISE:
4379 if (sqe) {
4380 ret = io_madvise_prep(req, sqe);
4381 if (ret)
4382 break;
4383 }
4384 ret = io_madvise(req, nxt, force_nonblock);
4385 break;
Jens Axboecebdb982020-01-08 17:59:24 -07004386 case IORING_OP_OPENAT2:
4387 if (sqe) {
4388 ret = io_openat2_prep(req, sqe);
4389 if (ret)
4390 break;
4391 }
4392 ret = io_openat2(req, nxt, force_nonblock);
4393 break;
Jens Axboe3e4827b2020-01-08 15:18:09 -07004394 case IORING_OP_EPOLL_CTL:
4395 if (sqe) {
4396 ret = io_epoll_ctl_prep(req, sqe);
4397 if (ret)
4398 break;
4399 }
4400 ret = io_epoll_ctl(req, nxt, force_nonblock);
4401 break;
Jens Axboe2b188cc2019-01-07 10:46:33 -07004402 default:
4403 ret = -EINVAL;
4404 break;
4405 }
4406
Jens Axboedef596e2019-01-09 08:59:42 -07004407 if (ret)
4408 return ret;
4409
4410 if (ctx->flags & IORING_SETUP_IOPOLL) {
Jens Axboe11ba8202020-01-15 21:51:17 -07004411 const bool in_async = io_wq_current_is_worker();
4412
Jens Axboe9e645e112019-05-10 16:07:28 -06004413 if (req->result == -EAGAIN)
Jens Axboedef596e2019-01-09 08:59:42 -07004414 return -EAGAIN;
4415
Jens Axboe11ba8202020-01-15 21:51:17 -07004416 /* workqueue context doesn't hold uring_lock, grab it now */
4417 if (in_async)
4418 mutex_lock(&ctx->uring_lock);
4419
Jens Axboedef596e2019-01-09 08:59:42 -07004420 io_iopoll_req_issued(req);
Jens Axboe11ba8202020-01-15 21:51:17 -07004421
4422 if (in_async)
4423 mutex_unlock(&ctx->uring_lock);
Jens Axboedef596e2019-01-09 08:59:42 -07004424 }
4425
4426 return 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07004427}
4428
Jens Axboe561fb042019-10-24 07:25:42 -06004429static void io_wq_submit_work(struct io_wq_work **workptr)
Jens Axboe31b51512019-01-18 22:56:34 -07004430{
Jens Axboe561fb042019-10-24 07:25:42 -06004431 struct io_wq_work *work = *workptr;
Jens Axboe2b188cc2019-01-07 10:46:33 -07004432 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
Jens Axboe561fb042019-10-24 07:25:42 -06004433 struct io_kiocb *nxt = NULL;
4434 int ret = 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07004435
Jens Axboe0c9d5cc2019-12-11 19:29:43 -07004436 /* if NO_CANCEL is set, we must still run the work */
4437 if ((work->flags & (IO_WQ_WORK_CANCEL|IO_WQ_WORK_NO_CANCEL)) ==
4438 IO_WQ_WORK_CANCEL) {
Jens Axboe561fb042019-10-24 07:25:42 -06004439 ret = -ECANCELED;
Jens Axboe0c9d5cc2019-12-11 19:29:43 -07004440 }
Jens Axboe31b51512019-01-18 22:56:34 -07004441
Jens Axboe561fb042019-10-24 07:25:42 -06004442 if (!ret) {
Pavel Begunkovcf6fd4b2019-11-25 23:14:39 +03004443 req->has_user = (work->flags & IO_WQ_WORK_HAS_MM) != 0;
4444 req->in_async = true;
Jens Axboe561fb042019-10-24 07:25:42 -06004445 do {
Jens Axboe3529d8c2019-12-19 18:24:38 -07004446 ret = io_issue_sqe(req, NULL, &nxt, false);
Jens Axboe561fb042019-10-24 07:25:42 -06004447 /*
4448 * We can get EAGAIN for polled IO even though we're
4449 * forcing a sync submission from here, since we can't
4450 * wait for request slots on the block side.
4451 */
4452 if (ret != -EAGAIN)
4453 break;
4454 cond_resched();
4455 } while (1);
4456 }
Jens Axboe31b51512019-01-18 22:56:34 -07004457
Jens Axboe561fb042019-10-24 07:25:42 -06004458 /* drop submission reference */
Jackie Liuec9c02a2019-11-08 23:50:36 +08004459 io_put_req(req);
Jens Axboe817869d2019-04-30 14:44:05 -06004460
Jens Axboe561fb042019-10-24 07:25:42 -06004461 if (ret) {
Jens Axboe4e88d6e2019-12-07 20:59:47 -07004462 req_set_fail_links(req);
Jens Axboe78e19bb2019-11-06 15:21:34 -07004463 io_cqring_add_event(req, ret);
Jens Axboe817869d2019-04-30 14:44:05 -06004464 io_put_req(req);
Jens Axboeedafcce2019-01-09 09:16:05 -07004465 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07004466
Jens Axboe561fb042019-10-24 07:25:42 -06004467 /* if a dependent link is ready, pass it back */
Jens Axboe78912932020-01-14 22:09:06 -07004468 if (!ret && nxt)
4469 io_wq_assign_next(workptr, nxt);
Jens Axboe31b51512019-01-18 22:56:34 -07004470}
Jens Axboe2b188cc2019-01-07 10:46:33 -07004471
Jens Axboe15b71ab2019-12-11 11:20:36 -07004472static int io_req_needs_file(struct io_kiocb *req, int fd)
Jens Axboe9e3aa612019-12-11 15:55:43 -07004473{
Jens Axboed3656342019-12-18 09:50:26 -07004474 if (!io_op_defs[req->opcode].needs_file)
Jens Axboe9e3aa612019-12-11 15:55:43 -07004475 return 0;
Jens Axboed3656342019-12-18 09:50:26 -07004476 if (fd == -1 && io_op_defs[req->opcode].fd_non_neg)
4477 return 0;
4478 return 1;
Jens Axboe09bb8392019-03-13 12:39:28 -06004479}
4480
Jens Axboe65e19f52019-10-26 07:20:21 -06004481static inline struct file *io_file_from_index(struct io_ring_ctx *ctx,
4482 int index)
Jens Axboe09bb8392019-03-13 12:39:28 -06004483{
Jens Axboe65e19f52019-10-26 07:20:21 -06004484 struct fixed_file_table *table;
4485
Jens Axboe05f3fb32019-12-09 11:22:50 -07004486 table = &ctx->file_data->table[index >> IORING_FILE_TABLE_SHIFT];
4487 return table->files[index & IORING_FILE_TABLE_MASK];;
Jens Axboe65e19f52019-10-26 07:20:21 -06004488}
4489
Jens Axboe3529d8c2019-12-19 18:24:38 -07004490static int io_req_set_file(struct io_submit_state *state, struct io_kiocb *req,
4491 const struct io_uring_sqe *sqe)
Jens Axboe09bb8392019-03-13 12:39:28 -06004492{
Jackie Liua197f662019-11-08 08:09:12 -07004493 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe09bb8392019-03-13 12:39:28 -06004494 unsigned flags;
Jens Axboed3656342019-12-18 09:50:26 -07004495 int fd;
Jens Axboe09bb8392019-03-13 12:39:28 -06004496
Jens Axboe3529d8c2019-12-19 18:24:38 -07004497 flags = READ_ONCE(sqe->flags);
4498 fd = READ_ONCE(sqe->fd);
Jens Axboe09bb8392019-03-13 12:39:28 -06004499
Jens Axboed3656342019-12-18 09:50:26 -07004500 if (!io_req_needs_file(req, fd))
4501 return 0;
Jens Axboe09bb8392019-03-13 12:39:28 -06004502
4503 if (flags & IOSQE_FIXED_FILE) {
Jens Axboe05f3fb32019-12-09 11:22:50 -07004504 if (unlikely(!ctx->file_data ||
Jens Axboe09bb8392019-03-13 12:39:28 -06004505 (unsigned) fd >= ctx->nr_user_files))
4506 return -EBADF;
Jens Axboeb7620122019-10-26 07:22:55 -06004507 fd = array_index_nospec(fd, ctx->nr_user_files);
Jens Axboe65e19f52019-10-26 07:20:21 -06004508 req->file = io_file_from_index(ctx, fd);
4509 if (!req->file)
Jens Axboe08a45172019-10-03 08:11:03 -06004510 return -EBADF;
Jens Axboe09bb8392019-03-13 12:39:28 -06004511 req->flags |= REQ_F_FIXED_FILE;
Jens Axboe05f3fb32019-12-09 11:22:50 -07004512 percpu_ref_get(&ctx->file_data->refs);
Jens Axboe09bb8392019-03-13 12:39:28 -06004513 } else {
Pavel Begunkovcf6fd4b2019-11-25 23:14:39 +03004514 if (req->needs_fixed_file)
Jens Axboe09bb8392019-03-13 12:39:28 -06004515 return -EBADF;
Dmitrii Dolgovc826bd72019-10-15 19:02:01 +02004516 trace_io_uring_file_get(ctx, fd);
Jens Axboe09bb8392019-03-13 12:39:28 -06004517 req->file = io_file_get(state, fd);
4518 if (unlikely(!req->file))
4519 return -EBADF;
4520 }
4521
4522 return 0;
4523}
4524
Jackie Liua197f662019-11-08 08:09:12 -07004525static int io_grab_files(struct io_kiocb *req)
Jens Axboe2b188cc2019-01-07 10:46:33 -07004526{
Jens Axboefcb323c2019-10-24 12:39:47 -06004527 int ret = -EBADF;
Jackie Liua197f662019-11-08 08:09:12 -07004528 struct io_ring_ctx *ctx = req->ctx;
Jens Axboefcb323c2019-10-24 12:39:47 -06004529
Jens Axboef86cd202020-01-29 13:46:44 -07004530 if (req->work.files)
4531 return 0;
Pavel Begunkovb14cca02020-01-17 04:45:59 +03004532 if (!ctx->ring_file)
Jens Axboeb5dba592019-12-11 14:02:38 -07004533 return -EBADF;
4534
Jens Axboefcb323c2019-10-24 12:39:47 -06004535 rcu_read_lock();
4536 spin_lock_irq(&ctx->inflight_lock);
4537 /*
4538 * We use the f_ops->flush() handler to ensure that we can flush
4539 * out work accessing these files if the fd is closed. Check if
4540 * the fd has changed since we started down this path, and disallow
4541 * this operation if it has.
4542 */
Pavel Begunkovb14cca02020-01-17 04:45:59 +03004543 if (fcheck(ctx->ring_fd) == ctx->ring_file) {
Jens Axboefcb323c2019-10-24 12:39:47 -06004544 list_add(&req->inflight_entry, &ctx->inflight_list);
4545 req->flags |= REQ_F_INFLIGHT;
4546 req->work.files = current->files;
4547 ret = 0;
4548 }
4549 spin_unlock_irq(&ctx->inflight_lock);
4550 rcu_read_unlock();
4551
4552 return ret;
4553}
4554
Jens Axboe2665abf2019-11-05 12:40:47 -07004555static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer)
4556{
Jens Axboead8a48a2019-11-15 08:49:11 -07004557 struct io_timeout_data *data = container_of(timer,
4558 struct io_timeout_data, timer);
4559 struct io_kiocb *req = data->req;
Jens Axboe2665abf2019-11-05 12:40:47 -07004560 struct io_ring_ctx *ctx = req->ctx;
4561 struct io_kiocb *prev = NULL;
4562 unsigned long flags;
Jens Axboe2665abf2019-11-05 12:40:47 -07004563
4564 spin_lock_irqsave(&ctx->completion_lock, flags);
4565
4566 /*
4567 * We don't expect the list to be empty, that will only happen if we
4568 * race with the completion of the linked work.
4569 */
Pavel Begunkov44932332019-12-05 16:16:35 +03004570 if (!list_empty(&req->link_list)) {
4571 prev = list_entry(req->link_list.prev, struct io_kiocb,
4572 link_list);
Jens Axboe5d960722019-11-19 15:31:28 -07004573 if (refcount_inc_not_zero(&prev->refs)) {
Pavel Begunkov44932332019-12-05 16:16:35 +03004574 list_del_init(&req->link_list);
Jens Axboe5d960722019-11-19 15:31:28 -07004575 prev->flags &= ~REQ_F_LINK_TIMEOUT;
4576 } else
Jens Axboe76a46e02019-11-10 23:34:16 -07004577 prev = NULL;
Jens Axboe2665abf2019-11-05 12:40:47 -07004578 }
4579
4580 spin_unlock_irqrestore(&ctx->completion_lock, flags);
4581
4582 if (prev) {
Jens Axboe4e88d6e2019-12-07 20:59:47 -07004583 req_set_fail_links(prev);
Jens Axboeb0dd8a42019-11-18 12:14:54 -07004584 io_async_find_and_cancel(ctx, req, prev->user_data, NULL,
4585 -ETIME);
Jens Axboe76a46e02019-11-10 23:34:16 -07004586 io_put_req(prev);
Jens Axboe47f46762019-11-09 17:43:02 -07004587 } else {
4588 io_cqring_add_event(req, -ETIME);
4589 io_put_req(req);
Jens Axboe2665abf2019-11-05 12:40:47 -07004590 }
Jens Axboe2665abf2019-11-05 12:40:47 -07004591 return HRTIMER_NORESTART;
4592}
4593
Jens Axboead8a48a2019-11-15 08:49:11 -07004594static void io_queue_linked_timeout(struct io_kiocb *req)
Jens Axboe2665abf2019-11-05 12:40:47 -07004595{
Jens Axboe76a46e02019-11-10 23:34:16 -07004596 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe2665abf2019-11-05 12:40:47 -07004597
Jens Axboe76a46e02019-11-10 23:34:16 -07004598 /*
4599 * If the list is now empty, then our linked request finished before
4600 * we got a chance to setup the timer
4601 */
4602 spin_lock_irq(&ctx->completion_lock);
Pavel Begunkov44932332019-12-05 16:16:35 +03004603 if (!list_empty(&req->link_list)) {
Jens Axboe2d283902019-12-04 11:08:05 -07004604 struct io_timeout_data *data = &req->io->timeout;
Jens Axboe94ae5e72019-11-14 19:39:52 -07004605
Jens Axboead8a48a2019-11-15 08:49:11 -07004606 data->timer.function = io_link_timeout_fn;
4607 hrtimer_start(&data->timer, timespec64_to_ktime(data->ts),
4608 data->mode);
Jens Axboe2665abf2019-11-05 12:40:47 -07004609 }
Jens Axboe76a46e02019-11-10 23:34:16 -07004610 spin_unlock_irq(&ctx->completion_lock);
Jens Axboe2665abf2019-11-05 12:40:47 -07004611
Jens Axboe2665abf2019-11-05 12:40:47 -07004612 /* drop submission reference */
Jens Axboe76a46e02019-11-10 23:34:16 -07004613 io_put_req(req);
Jens Axboe2665abf2019-11-05 12:40:47 -07004614}
4615
Jens Axboead8a48a2019-11-15 08:49:11 -07004616static struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req)
Jens Axboe2665abf2019-11-05 12:40:47 -07004617{
4618 struct io_kiocb *nxt;
Jens Axboe2b188cc2019-01-07 10:46:33 -07004619
Jens Axboe2665abf2019-11-05 12:40:47 -07004620 if (!(req->flags & REQ_F_LINK))
4621 return NULL;
4622
Pavel Begunkov44932332019-12-05 16:16:35 +03004623 nxt = list_first_entry_or_null(&req->link_list, struct io_kiocb,
4624 link_list);
Jens Axboed625c6e2019-12-17 19:53:05 -07004625 if (!nxt || nxt->opcode != IORING_OP_LINK_TIMEOUT)
Jens Axboe76a46e02019-11-10 23:34:16 -07004626 return NULL;
Jens Axboe2665abf2019-11-05 12:40:47 -07004627
Jens Axboe76a46e02019-11-10 23:34:16 -07004628 req->flags |= REQ_F_LINK_TIMEOUT;
Jens Axboe76a46e02019-11-10 23:34:16 -07004629 return nxt;
Jens Axboe2665abf2019-11-05 12:40:47 -07004630}
4631
Jens Axboe3529d8c2019-12-19 18:24:38 -07004632static void __io_queue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboe2b188cc2019-01-07 10:46:33 -07004633{
Jens Axboe4a0a7a12019-12-09 20:01:01 -07004634 struct io_kiocb *linked_timeout;
Pavel Begunkovf9bd67f2019-11-21 23:21:03 +03004635 struct io_kiocb *nxt = NULL;
Jens Axboe2b188cc2019-01-07 10:46:33 -07004636 int ret;
4637
Jens Axboe4a0a7a12019-12-09 20:01:01 -07004638again:
4639 linked_timeout = io_prep_linked_timeout(req);
4640
Jens Axboe3529d8c2019-12-19 18:24:38 -07004641 ret = io_issue_sqe(req, sqe, &nxt, true);
Jens Axboe491381ce2019-10-17 09:20:46 -06004642
4643 /*
4644 * We async punt it if the file wasn't marked NOWAIT, or if the file
4645 * doesn't support non-blocking read/write attempts
4646 */
4647 if (ret == -EAGAIN && (!(req->flags & REQ_F_NOWAIT) ||
4648 (req->flags & REQ_F_MUST_PUNT))) {
Pavel Begunkov86a761f2020-01-22 23:09:36 +03004649punt:
Jens Axboef86cd202020-01-29 13:46:44 -07004650 if (io_op_defs[req->opcode].file_table) {
Pavel Begunkovbbad27b2019-11-19 23:32:47 +03004651 ret = io_grab_files(req);
4652 if (ret)
4653 goto err;
Jens Axboe2b188cc2019-01-07 10:46:33 -07004654 }
Pavel Begunkovbbad27b2019-11-19 23:32:47 +03004655
4656 /*
4657 * Queued up for async execution, worker will release
4658 * submit reference when the iocb is actually submitted.
4659 */
4660 io_queue_async_work(req);
Jens Axboe4a0a7a12019-12-09 20:01:01 -07004661 goto done_req;
Jens Axboe2b188cc2019-01-07 10:46:33 -07004662 }
Jens Axboee65ef562019-03-12 10:16:44 -06004663
Jens Axboefcb323c2019-10-24 12:39:47 -06004664err:
Jens Axboee65ef562019-03-12 10:16:44 -06004665 /* drop submission reference */
4666 io_put_req(req);
4667
Pavel Begunkovf9bd67f2019-11-21 23:21:03 +03004668 if (linked_timeout) {
Jens Axboe76a46e02019-11-10 23:34:16 -07004669 if (!ret)
Pavel Begunkovf9bd67f2019-11-21 23:21:03 +03004670 io_queue_linked_timeout(linked_timeout);
Jens Axboe76a46e02019-11-10 23:34:16 -07004671 else
Pavel Begunkovf9bd67f2019-11-21 23:21:03 +03004672 io_put_req(linked_timeout);
Jens Axboe76a46e02019-11-10 23:34:16 -07004673 }
4674
Jens Axboee65ef562019-03-12 10:16:44 -06004675 /* and drop final reference, if we failed */
Jens Axboe9e645e112019-05-10 16:07:28 -06004676 if (ret) {
Jens Axboe78e19bb2019-11-06 15:21:34 -07004677 io_cqring_add_event(req, ret);
Jens Axboe4e88d6e2019-12-07 20:59:47 -07004678 req_set_fail_links(req);
Jens Axboee65ef562019-03-12 10:16:44 -06004679 io_put_req(req);
Jens Axboe9e645e112019-05-10 16:07:28 -06004680 }
Jens Axboe4a0a7a12019-12-09 20:01:01 -07004681done_req:
4682 if (nxt) {
4683 req = nxt;
4684 nxt = NULL;
Pavel Begunkov86a761f2020-01-22 23:09:36 +03004685
4686 if (req->flags & REQ_F_FORCE_ASYNC)
4687 goto punt;
Jens Axboe4a0a7a12019-12-09 20:01:01 -07004688 goto again;
4689 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07004690}
4691
Jens Axboe3529d8c2019-12-19 18:24:38 -07004692static void io_queue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jackie Liu4fe2c962019-09-09 20:50:40 +08004693{
4694 int ret;
4695
Jens Axboe3529d8c2019-12-19 18:24:38 -07004696 ret = io_req_defer(req, sqe);
Jackie Liu4fe2c962019-09-09 20:50:40 +08004697 if (ret) {
4698 if (ret != -EIOCBQUEUED) {
Pavel Begunkov11185912020-01-22 23:09:35 +03004699fail_req:
Jens Axboe78e19bb2019-11-06 15:21:34 -07004700 io_cqring_add_event(req, ret);
Jens Axboe4e88d6e2019-12-07 20:59:47 -07004701 req_set_fail_links(req);
Jens Axboe78e19bb2019-11-06 15:21:34 -07004702 io_double_put_req(req);
Jackie Liu4fe2c962019-09-09 20:50:40 +08004703 }
Pavel Begunkov25508782019-12-30 21:24:47 +03004704 } else if (req->flags & REQ_F_FORCE_ASYNC) {
Pavel Begunkov11185912020-01-22 23:09:35 +03004705 ret = io_req_defer_prep(req, sqe);
4706 if (unlikely(ret < 0))
4707 goto fail_req;
Jens Axboece35a472019-12-17 08:04:44 -07004708 /*
4709 * Never try inline submit of IOSQE_ASYNC is set, go straight
4710 * to async execution.
4711 */
4712 req->work.flags |= IO_WQ_WORK_CONCURRENT;
4713 io_queue_async_work(req);
4714 } else {
Jens Axboe3529d8c2019-12-19 18:24:38 -07004715 __io_queue_sqe(req, sqe);
Jens Axboece35a472019-12-17 08:04:44 -07004716 }
Jackie Liu4fe2c962019-09-09 20:50:40 +08004717}
4718
Pavel Begunkov1b4a51b2019-11-21 11:54:28 +03004719static inline void io_queue_link_head(struct io_kiocb *req)
Jackie Liu4fe2c962019-09-09 20:50:40 +08004720{
Jens Axboe94ae5e72019-11-14 19:39:52 -07004721 if (unlikely(req->flags & REQ_F_FAIL_LINK)) {
Pavel Begunkov1b4a51b2019-11-21 11:54:28 +03004722 io_cqring_add_event(req, -ECANCELED);
4723 io_double_put_req(req);
4724 } else
Jens Axboe3529d8c2019-12-19 18:24:38 -07004725 io_queue_sqe(req, NULL);
Jackie Liu4fe2c962019-09-09 20:50:40 +08004726}
4727
Jens Axboe4e88d6e2019-12-07 20:59:47 -07004728#define SQE_VALID_FLAGS (IOSQE_FIXED_FILE|IOSQE_IO_DRAIN|IOSQE_IO_LINK| \
Jens Axboece35a472019-12-17 08:04:44 -07004729 IOSQE_IO_HARDLINK | IOSQE_ASYNC)
Jens Axboe9e645e112019-05-10 16:07:28 -06004730
Jens Axboe3529d8c2019-12-19 18:24:38 -07004731static bool io_submit_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
4732 struct io_submit_state *state, struct io_kiocb **link)
Jens Axboe9e645e112019-05-10 16:07:28 -06004733{
Jens Axboe75c6a032020-01-28 10:15:23 -07004734 const struct cred *old_creds = NULL;
Jackie Liua197f662019-11-08 08:09:12 -07004735 struct io_ring_ctx *ctx = req->ctx;
Pavel Begunkov32fe5252019-12-17 22:26:58 +03004736 unsigned int sqe_flags;
Jens Axboe75c6a032020-01-28 10:15:23 -07004737 int ret, id;
Jens Axboe9e645e112019-05-10 16:07:28 -06004738
Pavel Begunkov32fe5252019-12-17 22:26:58 +03004739 sqe_flags = READ_ONCE(sqe->flags);
Jens Axboe9e645e112019-05-10 16:07:28 -06004740
4741 /* enforce forwards compatibility on users */
Pavel Begunkov32fe5252019-12-17 22:26:58 +03004742 if (unlikely(sqe_flags & ~SQE_VALID_FLAGS)) {
Jens Axboe9e645e112019-05-10 16:07:28 -06004743 ret = -EINVAL;
Pavel Begunkov196be952019-11-07 01:41:06 +03004744 goto err_req;
Jens Axboe9e645e112019-05-10 16:07:28 -06004745 }
4746
Jens Axboe75c6a032020-01-28 10:15:23 -07004747 id = READ_ONCE(sqe->personality);
4748 if (id) {
4749 const struct cred *personality_creds;
4750
4751 personality_creds = idr_find(&ctx->personality_idr, id);
4752 if (unlikely(!personality_creds)) {
4753 ret = -EINVAL;
4754 goto err_req;
4755 }
4756 old_creds = override_creds(personality_creds);
4757 }
4758
Pavel Begunkov6b47ee62020-01-18 20:22:41 +03004759 /* same numerical values with corresponding REQ_F_*, safe to copy */
4760 req->flags |= sqe_flags & (IOSQE_IO_DRAIN|IOSQE_IO_HARDLINK|
4761 IOSQE_ASYNC);
Jens Axboe9e645e112019-05-10 16:07:28 -06004762
Jens Axboe3529d8c2019-12-19 18:24:38 -07004763 ret = io_req_set_file(state, req, sqe);
Jens Axboe9e645e112019-05-10 16:07:28 -06004764 if (unlikely(ret)) {
4765err_req:
Jens Axboe78e19bb2019-11-06 15:21:34 -07004766 io_cqring_add_event(req, ret);
4767 io_double_put_req(req);
Jens Axboe75c6a032020-01-28 10:15:23 -07004768 if (old_creds)
4769 revert_creds(old_creds);
Pavel Begunkov2e6e1fd2019-12-05 16:15:45 +03004770 return false;
Jens Axboe9e645e112019-05-10 16:07:28 -06004771 }
4772
Jens Axboe9e645e112019-05-10 16:07:28 -06004773 /*
4774 * If we already have a head request, queue this one for async
4775 * submittal once the head completes. If we don't have a head but
4776 * IOSQE_IO_LINK is set in the sqe, start a new head. This one will be
4777 * submitted sync once the chain is complete. If none of those
4778 * conditions are true (normal request), then just queue it.
4779 */
4780 if (*link) {
Pavel Begunkov9d763772019-12-17 02:22:07 +03004781 struct io_kiocb *head = *link;
Jens Axboe9e645e112019-05-10 16:07:28 -06004782
Pavel Begunkov8cdf2192020-01-25 00:40:24 +03004783 /*
4784 * Taking sequential execution of a link, draining both sides
4785 * of the link also fullfils IOSQE_IO_DRAIN semantics for all
4786 * requests in the link. So, it drains the head and the
4787 * next after the link request. The last one is done via
4788 * drain_next flag to persist the effect across calls.
4789 */
Pavel Begunkov711be032020-01-17 03:57:59 +03004790 if (sqe_flags & IOSQE_IO_DRAIN) {
4791 head->flags |= REQ_F_IO_DRAIN;
4792 ctx->drain_next = 1;
4793 }
Jens Axboeb7bb4f72019-12-15 22:13:43 -07004794 if (io_alloc_async_ctx(req)) {
Jens Axboe9e645e112019-05-10 16:07:28 -06004795 ret = -EAGAIN;
4796 goto err_req;
4797 }
4798
Jens Axboe3529d8c2019-12-19 18:24:38 -07004799 ret = io_req_defer_prep(req, sqe);
Jens Axboe2d283902019-12-04 11:08:05 -07004800 if (ret) {
Jens Axboe4e88d6e2019-12-07 20:59:47 -07004801 /* fail even hard links since we don't submit */
Pavel Begunkov9d763772019-12-17 02:22:07 +03004802 head->flags |= REQ_F_FAIL_LINK;
Jens Axboef67676d2019-12-02 11:03:47 -07004803 goto err_req;
Jens Axboe2d283902019-12-04 11:08:05 -07004804 }
Pavel Begunkov9d763772019-12-17 02:22:07 +03004805 trace_io_uring_link(ctx, req, head);
4806 list_add_tail(&req->link_list, &head->link_list);
Jens Axboe9e645e112019-05-10 16:07:28 -06004807
Pavel Begunkov32fe5252019-12-17 22:26:58 +03004808 /* last request of a link, enqueue the link */
4809 if (!(sqe_flags & (IOSQE_IO_LINK|IOSQE_IO_HARDLINK))) {
4810 io_queue_link_head(head);
4811 *link = NULL;
4812 }
Jens Axboe9e645e112019-05-10 16:07:28 -06004813 } else {
Pavel Begunkov711be032020-01-17 03:57:59 +03004814 if (unlikely(ctx->drain_next)) {
4815 req->flags |= REQ_F_IO_DRAIN;
4816 req->ctx->drain_next = 0;
4817 }
4818 if (sqe_flags & (IOSQE_IO_LINK|IOSQE_IO_HARDLINK)) {
4819 req->flags |= REQ_F_LINK;
Pavel Begunkov711be032020-01-17 03:57:59 +03004820 INIT_LIST_HEAD(&req->link_list);
4821 ret = io_req_defer_prep(req, sqe);
4822 if (ret)
4823 req->flags |= REQ_F_FAIL_LINK;
4824 *link = req;
4825 } else {
4826 io_queue_sqe(req, sqe);
4827 }
Jens Axboe9e645e112019-05-10 16:07:28 -06004828 }
Pavel Begunkov2e6e1fd2019-12-05 16:15:45 +03004829
Jens Axboe75c6a032020-01-28 10:15:23 -07004830 if (old_creds)
4831 revert_creds(old_creds);
Pavel Begunkov2e6e1fd2019-12-05 16:15:45 +03004832 return true;
Jens Axboe9e645e112019-05-10 16:07:28 -06004833}
4834
Jens Axboe9a56a232019-01-09 09:06:50 -07004835/*
4836 * Batched submission is done, ensure local IO is flushed out.
4837 */
4838static void io_submit_state_end(struct io_submit_state *state)
4839{
4840 blk_finish_plug(&state->plug);
Jens Axboe3d6770f2019-04-13 11:50:54 -06004841 io_file_put(state);
Jens Axboe2579f912019-01-09 09:10:43 -07004842 if (state->free_reqs)
4843 kmem_cache_free_bulk(req_cachep, state->free_reqs,
4844 &state->reqs[state->cur_req]);
Jens Axboe9a56a232019-01-09 09:06:50 -07004845}
4846
4847/*
4848 * Start submission side cache.
4849 */
4850static void io_submit_state_start(struct io_submit_state *state,
Jackie Liu22efde52019-12-02 17:14:52 +08004851 unsigned int max_ios)
Jens Axboe9a56a232019-01-09 09:06:50 -07004852{
4853 blk_start_plug(&state->plug);
Jens Axboe2579f912019-01-09 09:10:43 -07004854 state->free_reqs = 0;
Jens Axboe9a56a232019-01-09 09:06:50 -07004855 state->file = NULL;
4856 state->ios_left = max_ios;
4857}
4858
Jens Axboe2b188cc2019-01-07 10:46:33 -07004859static void io_commit_sqring(struct io_ring_ctx *ctx)
4860{
Hristo Venev75b28af2019-08-26 17:23:46 +00004861 struct io_rings *rings = ctx->rings;
Jens Axboe2b188cc2019-01-07 10:46:33 -07004862
Pavel Begunkovcaf582c2019-12-30 21:24:46 +03004863 /*
4864 * Ensure any loads from the SQEs are done at this point,
4865 * since once we write the new head, the application could
4866 * write new data to them.
4867 */
4868 smp_store_release(&rings->sq.head, ctx->cached_sq_head);
Jens Axboe2b188cc2019-01-07 10:46:33 -07004869}
4870
4871/*
Jens Axboe3529d8c2019-12-19 18:24:38 -07004872 * Fetch an sqe, if one is available. Note that sqe_ptr will point to memory
Jens Axboe2b188cc2019-01-07 10:46:33 -07004873 * that is mapped by userspace. This means that care needs to be taken to
4874 * ensure that reads are stable, as we cannot rely on userspace always
4875 * being a good citizen. If members of the sqe are validated and then later
4876 * used, it's important that those reads are done through READ_ONCE() to
4877 * prevent a re-load down the line.
4878 */
Jens Axboe3529d8c2019-12-19 18:24:38 -07004879static bool io_get_sqring(struct io_ring_ctx *ctx, struct io_kiocb *req,
4880 const struct io_uring_sqe **sqe_ptr)
Jens Axboe2b188cc2019-01-07 10:46:33 -07004881{
Hristo Venev75b28af2019-08-26 17:23:46 +00004882 u32 *sq_array = ctx->sq_array;
Jens Axboe2b188cc2019-01-07 10:46:33 -07004883 unsigned head;
4884
4885 /*
4886 * The cached sq head (or cq tail) serves two purposes:
4887 *
4888 * 1) allows us to batch the cost of updating the user visible
4889 * head updates.
4890 * 2) allows the kernel side to track the head on its own, even
4891 * though the application is the one updating it.
4892 */
Pavel Begunkovee7d46d2019-12-30 21:24:45 +03004893 head = READ_ONCE(sq_array[ctx->cached_sq_head & ctx->sq_mask]);
Pavel Begunkov9835d6f2019-11-21 21:24:56 +03004894 if (likely(head < ctx->sq_entries)) {
Pavel Begunkovcf6fd4b2019-11-25 23:14:39 +03004895 /*
4896 * All io need record the previous position, if LINK vs DARIN,
4897 * it can be used to mark the position of the first IO in the
4898 * link list.
4899 */
4900 req->sequence = ctx->cached_sq_head;
Jens Axboe3529d8c2019-12-19 18:24:38 -07004901 *sqe_ptr = &ctx->sq_sqes[head];
4902 req->opcode = READ_ONCE((*sqe_ptr)->opcode);
4903 req->user_data = READ_ONCE((*sqe_ptr)->user_data);
Jens Axboe2b188cc2019-01-07 10:46:33 -07004904 ctx->cached_sq_head++;
4905 return true;
4906 }
4907
4908 /* drop invalid entries */
4909 ctx->cached_sq_head++;
Jens Axboe498ccd92019-10-25 10:04:25 -06004910 ctx->cached_sq_dropped++;
Pavel Begunkovee7d46d2019-12-30 21:24:45 +03004911 WRITE_ONCE(ctx->rings->sq_dropped, ctx->cached_sq_dropped);
Jens Axboe2b188cc2019-01-07 10:46:33 -07004912 return false;
4913}
4914
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03004915static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr,
Pavel Begunkovae9428c2019-11-06 00:22:14 +03004916 struct file *ring_file, int ring_fd,
4917 struct mm_struct **mm, bool async)
Jens Axboe6c271ce2019-01-10 11:22:30 -07004918{
4919 struct io_submit_state state, *statep = NULL;
Jens Axboe9e645e112019-05-10 16:07:28 -06004920 struct io_kiocb *link = NULL;
Jens Axboe9e645e112019-05-10 16:07:28 -06004921 int i, submitted = 0;
Pavel Begunkov95a1b3ff2019-10-27 23:15:41 +03004922 bool mm_fault = false;
Jens Axboe6c271ce2019-01-10 11:22:30 -07004923
Jens Axboec4a2ed72019-11-21 21:01:26 -07004924 /* if we have a backlog and couldn't flush it all, return BUSY */
Jens Axboead3eb2c2019-12-18 17:12:20 -07004925 if (test_bit(0, &ctx->sq_check_overflow)) {
4926 if (!list_empty(&ctx->cq_overflow_list) &&
4927 !io_cqring_overflow_flush(ctx, false))
4928 return -EBUSY;
4929 }
Jens Axboe6c271ce2019-01-10 11:22:30 -07004930
Pavel Begunkovee7d46d2019-12-30 21:24:45 +03004931 /* make sure SQ entry isn't read before tail */
4932 nr = min3(nr, ctx->sq_entries, io_sqring_entries(ctx));
Pavel Begunkov9ef4f122019-12-30 21:24:44 +03004933
Pavel Begunkov2b85edf2019-12-28 14:13:03 +03004934 if (!percpu_ref_tryget_many(&ctx->refs, nr))
4935 return -EAGAIN;
Jens Axboe6c271ce2019-01-10 11:22:30 -07004936
4937 if (nr > IO_PLUG_THRESHOLD) {
Jackie Liu22efde52019-12-02 17:14:52 +08004938 io_submit_state_start(&state, nr);
Jens Axboe6c271ce2019-01-10 11:22:30 -07004939 statep = &state;
4940 }
4941
Pavel Begunkovb14cca02020-01-17 04:45:59 +03004942 ctx->ring_fd = ring_fd;
4943 ctx->ring_file = ring_file;
4944
Jens Axboe6c271ce2019-01-10 11:22:30 -07004945 for (i = 0; i < nr; i++) {
Jens Axboe3529d8c2019-12-19 18:24:38 -07004946 const struct io_uring_sqe *sqe;
Pavel Begunkov196be952019-11-07 01:41:06 +03004947 struct io_kiocb *req;
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03004948
Pavel Begunkov196be952019-11-07 01:41:06 +03004949 req = io_get_req(ctx, statep);
4950 if (unlikely(!req)) {
4951 if (!submitted)
4952 submitted = -EAGAIN;
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03004953 break;
Jens Axboe9e645e112019-05-10 16:07:28 -06004954 }
Jens Axboe3529d8c2019-12-19 18:24:38 -07004955 if (!io_get_sqring(ctx, req, &sqe)) {
Pavel Begunkov2b85edf2019-12-28 14:13:03 +03004956 __io_req_do_free(req);
Pavel Begunkov196be952019-11-07 01:41:06 +03004957 break;
4958 }
Jens Axboe9e645e112019-05-10 16:07:28 -06004959
Jens Axboed3656342019-12-18 09:50:26 -07004960 /* will complete beyond this point, count as submitted */
4961 submitted++;
4962
4963 if (unlikely(req->opcode >= IORING_OP_LAST)) {
4964 io_cqring_add_event(req, -EINVAL);
4965 io_double_put_req(req);
4966 break;
4967 }
4968
4969 if (io_op_defs[req->opcode].needs_mm && !*mm) {
Pavel Begunkov95a1b3ff2019-10-27 23:15:41 +03004970 mm_fault = mm_fault || !mmget_not_zero(ctx->sqo_mm);
4971 if (!mm_fault) {
4972 use_mm(ctx->sqo_mm);
4973 *mm = ctx->sqo_mm;
4974 }
4975 }
4976
Pavel Begunkovcf6fd4b2019-11-25 23:14:39 +03004977 req->has_user = *mm != NULL;
4978 req->in_async = async;
4979 req->needs_fixed_file = async;
Jens Axboe354420f2020-01-08 18:55:15 -07004980 trace_io_uring_submit_sqe(ctx, req->opcode, req->user_data,
4981 true, async);
Jens Axboe3529d8c2019-12-19 18:24:38 -07004982 if (!io_submit_sqe(req, sqe, statep, &link))
Pavel Begunkov2e6e1fd2019-12-05 16:15:45 +03004983 break;
Jens Axboe6c271ce2019-01-10 11:22:30 -07004984 }
4985
Pavel Begunkov9466f432020-01-25 22:34:01 +03004986 if (unlikely(submitted != nr)) {
4987 int ref_used = (submitted == -EAGAIN) ? 0 : submitted;
4988
4989 percpu_ref_put_many(&ctx->refs, nr - ref_used);
4990 }
Jens Axboe9e645e112019-05-10 16:07:28 -06004991 if (link)
Pavel Begunkov1b4a51b2019-11-21 11:54:28 +03004992 io_queue_link_head(link);
Jens Axboe6c271ce2019-01-10 11:22:30 -07004993 if (statep)
4994 io_submit_state_end(&state);
4995
Pavel Begunkovae9428c2019-11-06 00:22:14 +03004996 /* Commit SQ ring head once we've consumed and submitted all SQEs */
4997 io_commit_sqring(ctx);
4998
Jens Axboe6c271ce2019-01-10 11:22:30 -07004999 return submitted;
5000}
5001
5002static int io_sq_thread(void *data)
5003{
Jens Axboe6c271ce2019-01-10 11:22:30 -07005004 struct io_ring_ctx *ctx = data;
5005 struct mm_struct *cur_mm = NULL;
Jens Axboe181e4482019-11-25 08:52:30 -07005006 const struct cred *old_cred;
Jens Axboe6c271ce2019-01-10 11:22:30 -07005007 mm_segment_t old_fs;
5008 DEFINE_WAIT(wait);
5009 unsigned inflight;
5010 unsigned long timeout;
Jens Axboec1edbf52019-11-10 16:56:04 -07005011 int ret;
Jens Axboe6c271ce2019-01-10 11:22:30 -07005012
Jens Axboe206aefd2019-11-07 18:27:42 -07005013 complete(&ctx->completions[1]);
Jackie Liua4c0b3d2019-07-08 13:41:12 +08005014
Jens Axboe6c271ce2019-01-10 11:22:30 -07005015 old_fs = get_fs();
5016 set_fs(USER_DS);
Jens Axboe181e4482019-11-25 08:52:30 -07005017 old_cred = override_creds(ctx->creds);
Jens Axboe6c271ce2019-01-10 11:22:30 -07005018
Jens Axboec1edbf52019-11-10 16:56:04 -07005019 ret = timeout = inflight = 0;
Roman Penyaev2bbcd6d2019-05-16 10:53:57 +02005020 while (!kthread_should_park()) {
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03005021 unsigned int to_submit;
Jens Axboe6c271ce2019-01-10 11:22:30 -07005022
5023 if (inflight) {
5024 unsigned nr_events = 0;
5025
5026 if (ctx->flags & IORING_SETUP_IOPOLL) {
Jens Axboe2b2ed972019-10-25 10:06:15 -06005027 /*
5028 * inflight is the count of the maximum possible
5029 * entries we submitted, but it can be smaller
5030 * if we dropped some of them. If we don't have
5031 * poll entries available, then we know that we
5032 * have nothing left to poll for. Reset the
5033 * inflight count to zero in that case.
5034 */
5035 mutex_lock(&ctx->uring_lock);
5036 if (!list_empty(&ctx->poll_list))
5037 __io_iopoll_check(ctx, &nr_events, 0);
5038 else
5039 inflight = 0;
5040 mutex_unlock(&ctx->uring_lock);
Jens Axboe6c271ce2019-01-10 11:22:30 -07005041 } else {
5042 /*
5043 * Normal IO, just pretend everything completed.
5044 * We don't have to poll completions for that.
5045 */
5046 nr_events = inflight;
5047 }
5048
5049 inflight -= nr_events;
5050 if (!inflight)
5051 timeout = jiffies + ctx->sq_thread_idle;
5052 }
5053
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03005054 to_submit = io_sqring_entries(ctx);
Jens Axboec1edbf52019-11-10 16:56:04 -07005055
5056 /*
5057 * If submit got -EBUSY, flag us as needing the application
5058 * to enter the kernel to reap and flush events.
5059 */
5060 if (!to_submit || ret == -EBUSY) {
Jens Axboe6c271ce2019-01-10 11:22:30 -07005061 /*
5062 * We're polling. If we're within the defined idle
5063 * period, then let us spin without work before going
Jens Axboec1edbf52019-11-10 16:56:04 -07005064 * to sleep. The exception is if we got EBUSY doing
5065 * more IO, we should wait for the application to
5066 * reap events and wake us up.
Jens Axboe6c271ce2019-01-10 11:22:30 -07005067 */
Jens Axboec1edbf52019-11-10 16:56:04 -07005068 if (inflight ||
5069 (!time_after(jiffies, timeout) && ret != -EBUSY)) {
Jens Axboe9831a902019-09-19 09:48:55 -06005070 cond_resched();
Jens Axboe6c271ce2019-01-10 11:22:30 -07005071 continue;
5072 }
5073
5074 /*
5075 * Drop cur_mm before scheduling, we can't hold it for
5076 * long periods (or over schedule()). Do this before
5077 * adding ourselves to the waitqueue, as the unuse/drop
5078 * may sleep.
5079 */
5080 if (cur_mm) {
5081 unuse_mm(cur_mm);
5082 mmput(cur_mm);
5083 cur_mm = NULL;
5084 }
5085
5086 prepare_to_wait(&ctx->sqo_wait, &wait,
5087 TASK_INTERRUPTIBLE);
5088
5089 /* Tell userspace we may need a wakeup call */
Hristo Venev75b28af2019-08-26 17:23:46 +00005090 ctx->rings->sq_flags |= IORING_SQ_NEED_WAKEUP;
Stefan Bühler0d7bae62019-04-19 11:57:45 +02005091 /* make sure to read SQ tail after writing flags */
5092 smp_mb();
Jens Axboe6c271ce2019-01-10 11:22:30 -07005093
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03005094 to_submit = io_sqring_entries(ctx);
Jens Axboec1edbf52019-11-10 16:56:04 -07005095 if (!to_submit || ret == -EBUSY) {
Roman Penyaev2bbcd6d2019-05-16 10:53:57 +02005096 if (kthread_should_park()) {
Jens Axboe6c271ce2019-01-10 11:22:30 -07005097 finish_wait(&ctx->sqo_wait, &wait);
5098 break;
5099 }
5100 if (signal_pending(current))
5101 flush_signals(current);
5102 schedule();
5103 finish_wait(&ctx->sqo_wait, &wait);
5104
Hristo Venev75b28af2019-08-26 17:23:46 +00005105 ctx->rings->sq_flags &= ~IORING_SQ_NEED_WAKEUP;
Jens Axboe6c271ce2019-01-10 11:22:30 -07005106 continue;
5107 }
5108 finish_wait(&ctx->sqo_wait, &wait);
5109
Hristo Venev75b28af2019-08-26 17:23:46 +00005110 ctx->rings->sq_flags &= ~IORING_SQ_NEED_WAKEUP;
Jens Axboe6c271ce2019-01-10 11:22:30 -07005111 }
5112
Jens Axboe8a4955f2019-12-09 14:52:35 -07005113 mutex_lock(&ctx->uring_lock);
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07005114 ret = io_submit_sqes(ctx, to_submit, NULL, -1, &cur_mm, true);
Jens Axboe8a4955f2019-12-09 14:52:35 -07005115 mutex_unlock(&ctx->uring_lock);
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07005116 if (ret > 0)
5117 inflight += ret;
Jens Axboe6c271ce2019-01-10 11:22:30 -07005118 }
5119
5120 set_fs(old_fs);
5121 if (cur_mm) {
5122 unuse_mm(cur_mm);
5123 mmput(cur_mm);
5124 }
Jens Axboe181e4482019-11-25 08:52:30 -07005125 revert_creds(old_cred);
Jens Axboe06058632019-04-13 09:26:03 -06005126
Roman Penyaev2bbcd6d2019-05-16 10:53:57 +02005127 kthread_parkme();
Jens Axboe06058632019-04-13 09:26:03 -06005128
Jens Axboe6c271ce2019-01-10 11:22:30 -07005129 return 0;
5130}
5131
Jens Axboebda52162019-09-24 13:47:15 -06005132struct io_wait_queue {
5133 struct wait_queue_entry wq;
5134 struct io_ring_ctx *ctx;
5135 unsigned to_wait;
5136 unsigned nr_timeouts;
5137};
5138
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07005139static inline bool io_should_wake(struct io_wait_queue *iowq, bool noflush)
Jens Axboebda52162019-09-24 13:47:15 -06005140{
5141 struct io_ring_ctx *ctx = iowq->ctx;
5142
5143 /*
Brian Gianforcarod195a662019-12-13 03:09:50 -08005144 * Wake up if we have enough events, or if a timeout occurred since we
Jens Axboebda52162019-09-24 13:47:15 -06005145 * started waiting. For timeouts, we always want to return to userspace,
5146 * regardless of event count.
5147 */
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07005148 return io_cqring_events(ctx, noflush) >= iowq->to_wait ||
Jens Axboebda52162019-09-24 13:47:15 -06005149 atomic_read(&ctx->cq_timeouts) != iowq->nr_timeouts;
5150}
5151
5152static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode,
5153 int wake_flags, void *key)
5154{
5155 struct io_wait_queue *iowq = container_of(curr, struct io_wait_queue,
5156 wq);
5157
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07005158 /* use noflush == true, as we can't safely rely on locking context */
5159 if (!io_should_wake(iowq, true))
Jens Axboebda52162019-09-24 13:47:15 -06005160 return -1;
5161
5162 return autoremove_wake_function(curr, mode, wake_flags, key);
5163}
5164
Jens Axboe2b188cc2019-01-07 10:46:33 -07005165/*
5166 * Wait until events become available, if we don't already have some. The
5167 * application must reap them itself, as they reside on the shared cq ring.
5168 */
5169static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events,
5170 const sigset_t __user *sig, size_t sigsz)
5171{
Jens Axboebda52162019-09-24 13:47:15 -06005172 struct io_wait_queue iowq = {
5173 .wq = {
5174 .private = current,
5175 .func = io_wake_function,
5176 .entry = LIST_HEAD_INIT(iowq.wq.entry),
5177 },
5178 .ctx = ctx,
5179 .to_wait = min_events,
5180 };
Hristo Venev75b28af2019-08-26 17:23:46 +00005181 struct io_rings *rings = ctx->rings;
Jackie Liue9ffa5c2019-10-29 11:16:42 +08005182 int ret = 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07005183
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07005184 if (io_cqring_events(ctx, false) >= min_events)
Jens Axboe2b188cc2019-01-07 10:46:33 -07005185 return 0;
5186
5187 if (sig) {
Arnd Bergmann9e75ad52019-03-25 15:34:53 +01005188#ifdef CONFIG_COMPAT
5189 if (in_compat_syscall())
5190 ret = set_compat_user_sigmask((const compat_sigset_t __user *)sig,
Oleg Nesterovb7724342019-07-16 16:29:53 -07005191 sigsz);
Arnd Bergmann9e75ad52019-03-25 15:34:53 +01005192 else
5193#endif
Oleg Nesterovb7724342019-07-16 16:29:53 -07005194 ret = set_user_sigmask(sig, sigsz);
Arnd Bergmann9e75ad52019-03-25 15:34:53 +01005195
Jens Axboe2b188cc2019-01-07 10:46:33 -07005196 if (ret)
5197 return ret;
5198 }
5199
Jens Axboebda52162019-09-24 13:47:15 -06005200 iowq.nr_timeouts = atomic_read(&ctx->cq_timeouts);
Dmitrii Dolgovc826bd72019-10-15 19:02:01 +02005201 trace_io_uring_cqring_wait(ctx, min_events);
Jens Axboebda52162019-09-24 13:47:15 -06005202 do {
5203 prepare_to_wait_exclusive(&ctx->wait, &iowq.wq,
5204 TASK_INTERRUPTIBLE);
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07005205 if (io_should_wake(&iowq, false))
Jens Axboebda52162019-09-24 13:47:15 -06005206 break;
5207 schedule();
5208 if (signal_pending(current)) {
Jackie Liue9ffa5c2019-10-29 11:16:42 +08005209 ret = -EINTR;
Jens Axboebda52162019-09-24 13:47:15 -06005210 break;
5211 }
5212 } while (1);
5213 finish_wait(&ctx->wait, &iowq.wq);
5214
Jackie Liue9ffa5c2019-10-29 11:16:42 +08005215 restore_saved_sigmask_unless(ret == -EINTR);
Jens Axboe2b188cc2019-01-07 10:46:33 -07005216
Hristo Venev75b28af2019-08-26 17:23:46 +00005217 return READ_ONCE(rings->cq.head) == READ_ONCE(rings->cq.tail) ? ret : 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07005218}
5219
Jens Axboe6b063142019-01-10 22:13:58 -07005220static void __io_sqe_files_unregister(struct io_ring_ctx *ctx)
5221{
5222#if defined(CONFIG_UNIX)
5223 if (ctx->ring_sock) {
5224 struct sock *sock = ctx->ring_sock->sk;
5225 struct sk_buff *skb;
5226
5227 while ((skb = skb_dequeue(&sock->sk_receive_queue)) != NULL)
5228 kfree_skb(skb);
5229 }
5230#else
5231 int i;
5232
Jens Axboe65e19f52019-10-26 07:20:21 -06005233 for (i = 0; i < ctx->nr_user_files; i++) {
5234 struct file *file;
5235
5236 file = io_file_from_index(ctx, i);
5237 if (file)
5238 fput(file);
5239 }
Jens Axboe6b063142019-01-10 22:13:58 -07005240#endif
5241}
5242
Jens Axboe05f3fb32019-12-09 11:22:50 -07005243static void io_file_ref_kill(struct percpu_ref *ref)
5244{
5245 struct fixed_file_data *data;
5246
5247 data = container_of(ref, struct fixed_file_data, refs);
5248 complete(&data->done);
5249}
5250
Jens Axboe6b063142019-01-10 22:13:58 -07005251static int io_sqe_files_unregister(struct io_ring_ctx *ctx)
5252{
Jens Axboe05f3fb32019-12-09 11:22:50 -07005253 struct fixed_file_data *data = ctx->file_data;
Jens Axboe65e19f52019-10-26 07:20:21 -06005254 unsigned nr_tables, i;
5255
Jens Axboe05f3fb32019-12-09 11:22:50 -07005256 if (!data)
Jens Axboe6b063142019-01-10 22:13:58 -07005257 return -ENXIO;
5258
Jens Axboe05f3fb32019-12-09 11:22:50 -07005259 /* protect against inflight atomic switch, which drops the ref */
Jens Axboe05f3fb32019-12-09 11:22:50 -07005260 percpu_ref_get(&data->refs);
Jens Axboee46a7952020-01-17 11:15:34 -07005261 /* wait for existing switches */
5262 flush_work(&data->ref_work);
Jens Axboe05f3fb32019-12-09 11:22:50 -07005263 percpu_ref_kill_and_confirm(&data->refs, io_file_ref_kill);
5264 wait_for_completion(&data->done);
5265 percpu_ref_put(&data->refs);
Jens Axboee46a7952020-01-17 11:15:34 -07005266 /* flush potential new switch */
5267 flush_work(&data->ref_work);
Jens Axboe05f3fb32019-12-09 11:22:50 -07005268 percpu_ref_exit(&data->refs);
5269
Jens Axboe6b063142019-01-10 22:13:58 -07005270 __io_sqe_files_unregister(ctx);
Jens Axboe65e19f52019-10-26 07:20:21 -06005271 nr_tables = DIV_ROUND_UP(ctx->nr_user_files, IORING_MAX_FILES_TABLE);
5272 for (i = 0; i < nr_tables; i++)
Jens Axboe05f3fb32019-12-09 11:22:50 -07005273 kfree(data->table[i].files);
5274 kfree(data->table);
5275 kfree(data);
5276 ctx->file_data = NULL;
Jens Axboe6b063142019-01-10 22:13:58 -07005277 ctx->nr_user_files = 0;
5278 return 0;
5279}
5280
Jens Axboe6c271ce2019-01-10 11:22:30 -07005281static void io_sq_thread_stop(struct io_ring_ctx *ctx)
5282{
5283 if (ctx->sqo_thread) {
Jens Axboe206aefd2019-11-07 18:27:42 -07005284 wait_for_completion(&ctx->completions[1]);
Roman Penyaev2bbcd6d2019-05-16 10:53:57 +02005285 /*
5286 * The park is a bit of a work-around, without it we get
5287 * warning spews on shutdown with SQPOLL set and affinity
5288 * set to a single CPU.
5289 */
Jens Axboe06058632019-04-13 09:26:03 -06005290 kthread_park(ctx->sqo_thread);
Jens Axboe6c271ce2019-01-10 11:22:30 -07005291 kthread_stop(ctx->sqo_thread);
5292 ctx->sqo_thread = NULL;
5293 }
5294}
5295
Jens Axboe6b063142019-01-10 22:13:58 -07005296static void io_finish_async(struct io_ring_ctx *ctx)
5297{
Jens Axboe6c271ce2019-01-10 11:22:30 -07005298 io_sq_thread_stop(ctx);
5299
Jens Axboe561fb042019-10-24 07:25:42 -06005300 if (ctx->io_wq) {
5301 io_wq_destroy(ctx->io_wq);
5302 ctx->io_wq = NULL;
Jens Axboe6b063142019-01-10 22:13:58 -07005303 }
5304}
5305
5306#if defined(CONFIG_UNIX)
Jens Axboe6b063142019-01-10 22:13:58 -07005307/*
5308 * Ensure the UNIX gc is aware of our file set, so we are certain that
5309 * the io_uring can be safely unregistered on process exit, even if we have
5310 * loops in the file referencing.
5311 */
5312static int __io_sqe_files_scm(struct io_ring_ctx *ctx, int nr, int offset)
5313{
5314 struct sock *sk = ctx->ring_sock->sk;
5315 struct scm_fp_list *fpl;
5316 struct sk_buff *skb;
Jens Axboe08a45172019-10-03 08:11:03 -06005317 int i, nr_files;
Jens Axboe6b063142019-01-10 22:13:58 -07005318
5319 if (!capable(CAP_SYS_RESOURCE) && !capable(CAP_SYS_ADMIN)) {
5320 unsigned long inflight = ctx->user->unix_inflight + nr;
5321
5322 if (inflight > task_rlimit(current, RLIMIT_NOFILE))
5323 return -EMFILE;
5324 }
5325
5326 fpl = kzalloc(sizeof(*fpl), GFP_KERNEL);
5327 if (!fpl)
5328 return -ENOMEM;
5329
5330 skb = alloc_skb(0, GFP_KERNEL);
5331 if (!skb) {
5332 kfree(fpl);
5333 return -ENOMEM;
5334 }
5335
5336 skb->sk = sk;
Jens Axboe6b063142019-01-10 22:13:58 -07005337
Jens Axboe08a45172019-10-03 08:11:03 -06005338 nr_files = 0;
Jens Axboe6b063142019-01-10 22:13:58 -07005339 fpl->user = get_uid(ctx->user);
5340 for (i = 0; i < nr; i++) {
Jens Axboe65e19f52019-10-26 07:20:21 -06005341 struct file *file = io_file_from_index(ctx, i + offset);
5342
5343 if (!file)
Jens Axboe08a45172019-10-03 08:11:03 -06005344 continue;
Jens Axboe65e19f52019-10-26 07:20:21 -06005345 fpl->fp[nr_files] = get_file(file);
Jens Axboe08a45172019-10-03 08:11:03 -06005346 unix_inflight(fpl->user, fpl->fp[nr_files]);
5347 nr_files++;
Jens Axboe6b063142019-01-10 22:13:58 -07005348 }
5349
Jens Axboe08a45172019-10-03 08:11:03 -06005350 if (nr_files) {
5351 fpl->max = SCM_MAX_FD;
5352 fpl->count = nr_files;
5353 UNIXCB(skb).fp = fpl;
Jens Axboe05f3fb32019-12-09 11:22:50 -07005354 skb->destructor = unix_destruct_scm;
Jens Axboe08a45172019-10-03 08:11:03 -06005355 refcount_add(skb->truesize, &sk->sk_wmem_alloc);
5356 skb_queue_head(&sk->sk_receive_queue, skb);
Jens Axboe6b063142019-01-10 22:13:58 -07005357
Jens Axboe08a45172019-10-03 08:11:03 -06005358 for (i = 0; i < nr_files; i++)
5359 fput(fpl->fp[i]);
5360 } else {
5361 kfree_skb(skb);
5362 kfree(fpl);
5363 }
Jens Axboe6b063142019-01-10 22:13:58 -07005364
5365 return 0;
5366}
5367
5368/*
5369 * If UNIX sockets are enabled, fd passing can cause a reference cycle which
5370 * causes regular reference counting to break down. We rely on the UNIX
5371 * garbage collection to take care of this problem for us.
5372 */
5373static int io_sqe_files_scm(struct io_ring_ctx *ctx)
5374{
5375 unsigned left, total;
5376 int ret = 0;
5377
5378 total = 0;
5379 left = ctx->nr_user_files;
5380 while (left) {
5381 unsigned this_files = min_t(unsigned, left, SCM_MAX_FD);
Jens Axboe6b063142019-01-10 22:13:58 -07005382
5383 ret = __io_sqe_files_scm(ctx, this_files, total);
5384 if (ret)
5385 break;
5386 left -= this_files;
5387 total += this_files;
5388 }
5389
5390 if (!ret)
5391 return 0;
5392
5393 while (total < ctx->nr_user_files) {
Jens Axboe65e19f52019-10-26 07:20:21 -06005394 struct file *file = io_file_from_index(ctx, total);
5395
5396 if (file)
5397 fput(file);
Jens Axboe6b063142019-01-10 22:13:58 -07005398 total++;
5399 }
5400
5401 return ret;
5402}
5403#else
5404static int io_sqe_files_scm(struct io_ring_ctx *ctx)
5405{
5406 return 0;
5407}
5408#endif
5409
Jens Axboe65e19f52019-10-26 07:20:21 -06005410static int io_sqe_alloc_file_tables(struct io_ring_ctx *ctx, unsigned nr_tables,
5411 unsigned nr_files)
5412{
5413 int i;
5414
5415 for (i = 0; i < nr_tables; i++) {
Jens Axboe05f3fb32019-12-09 11:22:50 -07005416 struct fixed_file_table *table = &ctx->file_data->table[i];
Jens Axboe65e19f52019-10-26 07:20:21 -06005417 unsigned this_files;
5418
5419 this_files = min(nr_files, IORING_MAX_FILES_TABLE);
5420 table->files = kcalloc(this_files, sizeof(struct file *),
5421 GFP_KERNEL);
5422 if (!table->files)
5423 break;
5424 nr_files -= this_files;
5425 }
5426
5427 if (i == nr_tables)
5428 return 0;
5429
5430 for (i = 0; i < nr_tables; i++) {
Jens Axboe05f3fb32019-12-09 11:22:50 -07005431 struct fixed_file_table *table = &ctx->file_data->table[i];
Jens Axboe65e19f52019-10-26 07:20:21 -06005432 kfree(table->files);
5433 }
5434 return 1;
5435}
5436
Jens Axboe05f3fb32019-12-09 11:22:50 -07005437static void io_ring_file_put(struct io_ring_ctx *ctx, struct file *file)
Jens Axboec3a31e62019-10-03 13:59:56 -06005438{
5439#if defined(CONFIG_UNIX)
Jens Axboec3a31e62019-10-03 13:59:56 -06005440 struct sock *sock = ctx->ring_sock->sk;
5441 struct sk_buff_head list, *head = &sock->sk_receive_queue;
5442 struct sk_buff *skb;
5443 int i;
5444
5445 __skb_queue_head_init(&list);
5446
5447 /*
5448 * Find the skb that holds this file in its SCM_RIGHTS. When found,
5449 * remove this entry and rearrange the file array.
5450 */
5451 skb = skb_dequeue(head);
5452 while (skb) {
5453 struct scm_fp_list *fp;
5454
5455 fp = UNIXCB(skb).fp;
5456 for (i = 0; i < fp->count; i++) {
5457 int left;
5458
5459 if (fp->fp[i] != file)
5460 continue;
5461
5462 unix_notinflight(fp->user, fp->fp[i]);
5463 left = fp->count - 1 - i;
5464 if (left) {
5465 memmove(&fp->fp[i], &fp->fp[i + 1],
5466 left * sizeof(struct file *));
5467 }
5468 fp->count--;
5469 if (!fp->count) {
5470 kfree_skb(skb);
5471 skb = NULL;
5472 } else {
5473 __skb_queue_tail(&list, skb);
5474 }
5475 fput(file);
5476 file = NULL;
5477 break;
5478 }
5479
5480 if (!file)
5481 break;
5482
5483 __skb_queue_tail(&list, skb);
5484
5485 skb = skb_dequeue(head);
5486 }
5487
5488 if (skb_peek(&list)) {
5489 spin_lock_irq(&head->lock);
5490 while ((skb = __skb_dequeue(&list)) != NULL)
5491 __skb_queue_tail(head, skb);
5492 spin_unlock_irq(&head->lock);
5493 }
5494#else
Jens Axboe05f3fb32019-12-09 11:22:50 -07005495 fput(file);
Jens Axboec3a31e62019-10-03 13:59:56 -06005496#endif
5497}
5498
Jens Axboe05f3fb32019-12-09 11:22:50 -07005499struct io_file_put {
5500 struct llist_node llist;
5501 struct file *file;
5502 struct completion *done;
5503};
5504
5505static void io_ring_file_ref_switch(struct work_struct *work)
5506{
5507 struct io_file_put *pfile, *tmp;
5508 struct fixed_file_data *data;
5509 struct llist_node *node;
5510
5511 data = container_of(work, struct fixed_file_data, ref_work);
5512
5513 while ((node = llist_del_all(&data->put_llist)) != NULL) {
5514 llist_for_each_entry_safe(pfile, tmp, node, llist) {
5515 io_ring_file_put(data->ctx, pfile->file);
5516 if (pfile->done)
5517 complete(pfile->done);
5518 else
5519 kfree(pfile);
5520 }
5521 }
5522
5523 percpu_ref_get(&data->refs);
5524 percpu_ref_switch_to_percpu(&data->refs);
5525}
5526
5527static void io_file_data_ref_zero(struct percpu_ref *ref)
5528{
5529 struct fixed_file_data *data;
5530
5531 data = container_of(ref, struct fixed_file_data, refs);
5532
5533 /* we can't safely switch from inside this context, punt to wq */
5534 queue_work(system_wq, &data->ref_work);
5535}
5536
5537static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg,
5538 unsigned nr_args)
5539{
5540 __s32 __user *fds = (__s32 __user *) arg;
5541 unsigned nr_tables;
5542 struct file *file;
5543 int fd, ret = 0;
5544 unsigned i;
5545
5546 if (ctx->file_data)
5547 return -EBUSY;
5548 if (!nr_args)
5549 return -EINVAL;
5550 if (nr_args > IORING_MAX_FIXED_FILES)
5551 return -EMFILE;
5552
5553 ctx->file_data = kzalloc(sizeof(*ctx->file_data), GFP_KERNEL);
5554 if (!ctx->file_data)
5555 return -ENOMEM;
5556 ctx->file_data->ctx = ctx;
5557 init_completion(&ctx->file_data->done);
5558
5559 nr_tables = DIV_ROUND_UP(nr_args, IORING_MAX_FILES_TABLE);
5560 ctx->file_data->table = kcalloc(nr_tables,
5561 sizeof(struct fixed_file_table),
5562 GFP_KERNEL);
5563 if (!ctx->file_data->table) {
5564 kfree(ctx->file_data);
5565 ctx->file_data = NULL;
5566 return -ENOMEM;
5567 }
5568
5569 if (percpu_ref_init(&ctx->file_data->refs, io_file_data_ref_zero,
5570 PERCPU_REF_ALLOW_REINIT, GFP_KERNEL)) {
5571 kfree(ctx->file_data->table);
5572 kfree(ctx->file_data);
5573 ctx->file_data = NULL;
5574 return -ENOMEM;
5575 }
5576 ctx->file_data->put_llist.first = NULL;
5577 INIT_WORK(&ctx->file_data->ref_work, io_ring_file_ref_switch);
5578
5579 if (io_sqe_alloc_file_tables(ctx, nr_tables, nr_args)) {
5580 percpu_ref_exit(&ctx->file_data->refs);
5581 kfree(ctx->file_data->table);
5582 kfree(ctx->file_data);
5583 ctx->file_data = NULL;
5584 return -ENOMEM;
5585 }
5586
5587 for (i = 0; i < nr_args; i++, ctx->nr_user_files++) {
5588 struct fixed_file_table *table;
5589 unsigned index;
5590
5591 ret = -EFAULT;
5592 if (copy_from_user(&fd, &fds[i], sizeof(fd)))
5593 break;
5594 /* allow sparse sets */
5595 if (fd == -1) {
5596 ret = 0;
5597 continue;
5598 }
5599
5600 table = &ctx->file_data->table[i >> IORING_FILE_TABLE_SHIFT];
5601 index = i & IORING_FILE_TABLE_MASK;
5602 file = fget(fd);
5603
5604 ret = -EBADF;
5605 if (!file)
5606 break;
5607
5608 /*
5609 * Don't allow io_uring instances to be registered. If UNIX
5610 * isn't enabled, then this causes a reference cycle and this
5611 * instance can never get freed. If UNIX is enabled we'll
5612 * handle it just fine, but there's still no point in allowing
5613 * a ring fd as it doesn't support regular read/write anyway.
5614 */
5615 if (file->f_op == &io_uring_fops) {
5616 fput(file);
5617 break;
5618 }
5619 ret = 0;
5620 table->files[index] = file;
5621 }
5622
5623 if (ret) {
5624 for (i = 0; i < ctx->nr_user_files; i++) {
5625 file = io_file_from_index(ctx, i);
5626 if (file)
5627 fput(file);
5628 }
5629 for (i = 0; i < nr_tables; i++)
5630 kfree(ctx->file_data->table[i].files);
5631
5632 kfree(ctx->file_data->table);
5633 kfree(ctx->file_data);
5634 ctx->file_data = NULL;
5635 ctx->nr_user_files = 0;
5636 return ret;
5637 }
5638
5639 ret = io_sqe_files_scm(ctx);
5640 if (ret)
5641 io_sqe_files_unregister(ctx);
5642
5643 return ret;
5644}
5645
Jens Axboec3a31e62019-10-03 13:59:56 -06005646static int io_sqe_file_register(struct io_ring_ctx *ctx, struct file *file,
5647 int index)
5648{
5649#if defined(CONFIG_UNIX)
5650 struct sock *sock = ctx->ring_sock->sk;
5651 struct sk_buff_head *head = &sock->sk_receive_queue;
5652 struct sk_buff *skb;
5653
5654 /*
5655 * See if we can merge this file into an existing skb SCM_RIGHTS
5656 * file set. If there's no room, fall back to allocating a new skb
5657 * and filling it in.
5658 */
5659 spin_lock_irq(&head->lock);
5660 skb = skb_peek(head);
5661 if (skb) {
5662 struct scm_fp_list *fpl = UNIXCB(skb).fp;
5663
5664 if (fpl->count < SCM_MAX_FD) {
5665 __skb_unlink(skb, head);
5666 spin_unlock_irq(&head->lock);
5667 fpl->fp[fpl->count] = get_file(file);
5668 unix_inflight(fpl->user, fpl->fp[fpl->count]);
5669 fpl->count++;
5670 spin_lock_irq(&head->lock);
5671 __skb_queue_head(head, skb);
5672 } else {
5673 skb = NULL;
5674 }
5675 }
5676 spin_unlock_irq(&head->lock);
5677
5678 if (skb) {
5679 fput(file);
5680 return 0;
5681 }
5682
5683 return __io_sqe_files_scm(ctx, 1, index);
5684#else
5685 return 0;
5686#endif
5687}
5688
Jens Axboe05f3fb32019-12-09 11:22:50 -07005689static void io_atomic_switch(struct percpu_ref *ref)
Jens Axboec3a31e62019-10-03 13:59:56 -06005690{
Jens Axboe05f3fb32019-12-09 11:22:50 -07005691 struct fixed_file_data *data;
5692
5693 data = container_of(ref, struct fixed_file_data, refs);
5694 clear_bit(FFD_F_ATOMIC, &data->state);
5695}
5696
5697static bool io_queue_file_removal(struct fixed_file_data *data,
5698 struct file *file)
5699{
5700 struct io_file_put *pfile, pfile_stack;
5701 DECLARE_COMPLETION_ONSTACK(done);
5702
5703 /*
5704 * If we fail allocating the struct we need for doing async reomval
5705 * of this file, just punt to sync and wait for it.
5706 */
5707 pfile = kzalloc(sizeof(*pfile), GFP_KERNEL);
5708 if (!pfile) {
5709 pfile = &pfile_stack;
5710 pfile->done = &done;
5711 }
5712
5713 pfile->file = file;
5714 llist_add(&pfile->llist, &data->put_llist);
5715
5716 if (pfile == &pfile_stack) {
5717 if (!test_and_set_bit(FFD_F_ATOMIC, &data->state)) {
5718 percpu_ref_put(&data->refs);
5719 percpu_ref_switch_to_atomic(&data->refs,
5720 io_atomic_switch);
5721 }
5722 wait_for_completion(&done);
5723 flush_work(&data->ref_work);
5724 return false;
5725 }
5726
5727 return true;
5728}
5729
5730static int __io_sqe_files_update(struct io_ring_ctx *ctx,
5731 struct io_uring_files_update *up,
5732 unsigned nr_args)
5733{
5734 struct fixed_file_data *data = ctx->file_data;
5735 bool ref_switch = false;
5736 struct file *file;
Jens Axboec3a31e62019-10-03 13:59:56 -06005737 __s32 __user *fds;
5738 int fd, i, err;
5739 __u32 done;
5740
Jens Axboe05f3fb32019-12-09 11:22:50 -07005741 if (check_add_overflow(up->offset, nr_args, &done))
Jens Axboec3a31e62019-10-03 13:59:56 -06005742 return -EOVERFLOW;
5743 if (done > ctx->nr_user_files)
5744 return -EINVAL;
5745
5746 done = 0;
Jens Axboe05f3fb32019-12-09 11:22:50 -07005747 fds = u64_to_user_ptr(up->fds);
Jens Axboec3a31e62019-10-03 13:59:56 -06005748 while (nr_args) {
Jens Axboe65e19f52019-10-26 07:20:21 -06005749 struct fixed_file_table *table;
5750 unsigned index;
5751
Jens Axboec3a31e62019-10-03 13:59:56 -06005752 err = 0;
5753 if (copy_from_user(&fd, &fds[done], sizeof(fd))) {
5754 err = -EFAULT;
5755 break;
5756 }
Jens Axboe05f3fb32019-12-09 11:22:50 -07005757 i = array_index_nospec(up->offset, ctx->nr_user_files);
5758 table = &ctx->file_data->table[i >> IORING_FILE_TABLE_SHIFT];
Jens Axboe65e19f52019-10-26 07:20:21 -06005759 index = i & IORING_FILE_TABLE_MASK;
5760 if (table->files[index]) {
Jens Axboe05f3fb32019-12-09 11:22:50 -07005761 file = io_file_from_index(ctx, index);
Jens Axboe65e19f52019-10-26 07:20:21 -06005762 table->files[index] = NULL;
Jens Axboe05f3fb32019-12-09 11:22:50 -07005763 if (io_queue_file_removal(data, file))
5764 ref_switch = true;
Jens Axboec3a31e62019-10-03 13:59:56 -06005765 }
5766 if (fd != -1) {
Jens Axboec3a31e62019-10-03 13:59:56 -06005767 file = fget(fd);
5768 if (!file) {
5769 err = -EBADF;
5770 break;
5771 }
5772 /*
5773 * Don't allow io_uring instances to be registered. If
5774 * UNIX isn't enabled, then this causes a reference
5775 * cycle and this instance can never get freed. If UNIX
5776 * is enabled we'll handle it just fine, but there's
5777 * still no point in allowing a ring fd as it doesn't
5778 * support regular read/write anyway.
5779 */
5780 if (file->f_op == &io_uring_fops) {
5781 fput(file);
5782 err = -EBADF;
5783 break;
5784 }
Jens Axboe65e19f52019-10-26 07:20:21 -06005785 table->files[index] = file;
Jens Axboec3a31e62019-10-03 13:59:56 -06005786 err = io_sqe_file_register(ctx, file, i);
5787 if (err)
5788 break;
5789 }
5790 nr_args--;
5791 done++;
Jens Axboe05f3fb32019-12-09 11:22:50 -07005792 up->offset++;
5793 }
5794
5795 if (ref_switch && !test_and_set_bit(FFD_F_ATOMIC, &data->state)) {
5796 percpu_ref_put(&data->refs);
5797 percpu_ref_switch_to_atomic(&data->refs, io_atomic_switch);
Jens Axboec3a31e62019-10-03 13:59:56 -06005798 }
5799
5800 return done ? done : err;
5801}
Jens Axboe05f3fb32019-12-09 11:22:50 -07005802static int io_sqe_files_update(struct io_ring_ctx *ctx, void __user *arg,
5803 unsigned nr_args)
5804{
5805 struct io_uring_files_update up;
5806
5807 if (!ctx->file_data)
5808 return -ENXIO;
5809 if (!nr_args)
5810 return -EINVAL;
5811 if (copy_from_user(&up, arg, sizeof(up)))
5812 return -EFAULT;
5813 if (up.resv)
5814 return -EINVAL;
5815
5816 return __io_sqe_files_update(ctx, &up, nr_args);
5817}
Jens Axboec3a31e62019-10-03 13:59:56 -06005818
Jens Axboe7d723062019-11-12 22:31:31 -07005819static void io_put_work(struct io_wq_work *work)
5820{
5821 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
5822
5823 io_put_req(req);
5824}
5825
5826static void io_get_work(struct io_wq_work *work)
5827{
5828 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
5829
5830 refcount_inc(&req->refs);
5831}
5832
Pavel Begunkov24369c22020-01-28 03:15:48 +03005833static int io_init_wq_offload(struct io_ring_ctx *ctx,
5834 struct io_uring_params *p)
5835{
5836 struct io_wq_data data;
5837 struct fd f;
5838 struct io_ring_ctx *ctx_attach;
5839 unsigned int concurrency;
5840 int ret = 0;
5841
5842 data.user = ctx->user;
5843 data.get_work = io_get_work;
5844 data.put_work = io_put_work;
5845
5846 if (!(p->flags & IORING_SETUP_ATTACH_WQ)) {
5847 /* Do QD, or 4 * CPUS, whatever is smallest */
5848 concurrency = min(ctx->sq_entries, 4 * num_online_cpus());
5849
5850 ctx->io_wq = io_wq_create(concurrency, &data);
5851 if (IS_ERR(ctx->io_wq)) {
5852 ret = PTR_ERR(ctx->io_wq);
5853 ctx->io_wq = NULL;
5854 }
5855 return ret;
5856 }
5857
5858 f = fdget(p->wq_fd);
5859 if (!f.file)
5860 return -EBADF;
5861
5862 if (f.file->f_op != &io_uring_fops) {
5863 ret = -EINVAL;
5864 goto out_fput;
5865 }
5866
5867 ctx_attach = f.file->private_data;
5868 /* @io_wq is protected by holding the fd */
5869 if (!io_wq_get(ctx_attach->io_wq, &data)) {
5870 ret = -EINVAL;
5871 goto out_fput;
5872 }
5873
5874 ctx->io_wq = ctx_attach->io_wq;
5875out_fput:
5876 fdput(f);
5877 return ret;
5878}
5879
Jens Axboe6c271ce2019-01-10 11:22:30 -07005880static int io_sq_offload_start(struct io_ring_ctx *ctx,
5881 struct io_uring_params *p)
Jens Axboe2b188cc2019-01-07 10:46:33 -07005882{
5883 int ret;
5884
Jens Axboe6c271ce2019-01-10 11:22:30 -07005885 init_waitqueue_head(&ctx->sqo_wait);
Jens Axboe2b188cc2019-01-07 10:46:33 -07005886 mmgrab(current->mm);
5887 ctx->sqo_mm = current->mm;
5888
Jens Axboe6c271ce2019-01-10 11:22:30 -07005889 if (ctx->flags & IORING_SETUP_SQPOLL) {
Jens Axboe3ec482d2019-04-08 10:51:01 -06005890 ret = -EPERM;
5891 if (!capable(CAP_SYS_ADMIN))
5892 goto err;
5893
Jens Axboe917257d2019-04-13 09:28:55 -06005894 ctx->sq_thread_idle = msecs_to_jiffies(p->sq_thread_idle);
5895 if (!ctx->sq_thread_idle)
5896 ctx->sq_thread_idle = HZ;
5897
Jens Axboe6c271ce2019-01-10 11:22:30 -07005898 if (p->flags & IORING_SETUP_SQ_AFF) {
Jens Axboe44a9bd12019-05-14 20:00:30 -06005899 int cpu = p->sq_thread_cpu;
Jens Axboe6c271ce2019-01-10 11:22:30 -07005900
Jens Axboe917257d2019-04-13 09:28:55 -06005901 ret = -EINVAL;
Jens Axboe44a9bd12019-05-14 20:00:30 -06005902 if (cpu >= nr_cpu_ids)
5903 goto err;
Shenghui Wang7889f442019-05-07 16:03:19 +08005904 if (!cpu_online(cpu))
Jens Axboe917257d2019-04-13 09:28:55 -06005905 goto err;
5906
Jens Axboe6c271ce2019-01-10 11:22:30 -07005907 ctx->sqo_thread = kthread_create_on_cpu(io_sq_thread,
5908 ctx, cpu,
5909 "io_uring-sq");
5910 } else {
5911 ctx->sqo_thread = kthread_create(io_sq_thread, ctx,
5912 "io_uring-sq");
5913 }
5914 if (IS_ERR(ctx->sqo_thread)) {
5915 ret = PTR_ERR(ctx->sqo_thread);
5916 ctx->sqo_thread = NULL;
5917 goto err;
5918 }
5919 wake_up_process(ctx->sqo_thread);
5920 } else if (p->flags & IORING_SETUP_SQ_AFF) {
5921 /* Can't have SQ_AFF without SQPOLL */
5922 ret = -EINVAL;
5923 goto err;
5924 }
5925
Pavel Begunkov24369c22020-01-28 03:15:48 +03005926 ret = io_init_wq_offload(ctx, p);
5927 if (ret)
Jens Axboe2b188cc2019-01-07 10:46:33 -07005928 goto err;
Jens Axboe2b188cc2019-01-07 10:46:33 -07005929
5930 return 0;
5931err:
Jens Axboe54a91f32019-09-10 09:15:04 -06005932 io_finish_async(ctx);
Jens Axboe2b188cc2019-01-07 10:46:33 -07005933 mmdrop(ctx->sqo_mm);
5934 ctx->sqo_mm = NULL;
5935 return ret;
5936}
5937
5938static void io_unaccount_mem(struct user_struct *user, unsigned long nr_pages)
5939{
5940 atomic_long_sub(nr_pages, &user->locked_vm);
5941}
5942
5943static int io_account_mem(struct user_struct *user, unsigned long nr_pages)
5944{
5945 unsigned long page_limit, cur_pages, new_pages;
5946
5947 /* Don't allow more pages than we can safely lock */
5948 page_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
5949
5950 do {
5951 cur_pages = atomic_long_read(&user->locked_vm);
5952 new_pages = cur_pages + nr_pages;
5953 if (new_pages > page_limit)
5954 return -ENOMEM;
5955 } while (atomic_long_cmpxchg(&user->locked_vm, cur_pages,
5956 new_pages) != cur_pages);
5957
5958 return 0;
5959}
5960
5961static void io_mem_free(void *ptr)
5962{
Mark Rutland52e04ef2019-04-30 17:30:21 +01005963 struct page *page;
Jens Axboe2b188cc2019-01-07 10:46:33 -07005964
Mark Rutland52e04ef2019-04-30 17:30:21 +01005965 if (!ptr)
5966 return;
5967
5968 page = virt_to_head_page(ptr);
Jens Axboe2b188cc2019-01-07 10:46:33 -07005969 if (put_page_testzero(page))
5970 free_compound_page(page);
5971}
5972
5973static void *io_mem_alloc(size_t size)
5974{
5975 gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN | __GFP_COMP |
5976 __GFP_NORETRY;
5977
5978 return (void *) __get_free_pages(gfp_flags, get_order(size));
5979}
5980
Hristo Venev75b28af2019-08-26 17:23:46 +00005981static unsigned long rings_size(unsigned sq_entries, unsigned cq_entries,
5982 size_t *sq_offset)
5983{
5984 struct io_rings *rings;
5985 size_t off, sq_array_size;
5986
5987 off = struct_size(rings, cqes, cq_entries);
5988 if (off == SIZE_MAX)
5989 return SIZE_MAX;
5990
5991#ifdef CONFIG_SMP
5992 off = ALIGN(off, SMP_CACHE_BYTES);
5993 if (off == 0)
5994 return SIZE_MAX;
5995#endif
5996
5997 sq_array_size = array_size(sizeof(u32), sq_entries);
5998 if (sq_array_size == SIZE_MAX)
5999 return SIZE_MAX;
6000
6001 if (check_add_overflow(off, sq_array_size, &off))
6002 return SIZE_MAX;
6003
6004 if (sq_offset)
6005 *sq_offset = off;
6006
6007 return off;
6008}
6009
Jens Axboe2b188cc2019-01-07 10:46:33 -07006010static unsigned long ring_pages(unsigned sq_entries, unsigned cq_entries)
6011{
Hristo Venev75b28af2019-08-26 17:23:46 +00006012 size_t pages;
Jens Axboe2b188cc2019-01-07 10:46:33 -07006013
Hristo Venev75b28af2019-08-26 17:23:46 +00006014 pages = (size_t)1 << get_order(
6015 rings_size(sq_entries, cq_entries, NULL));
6016 pages += (size_t)1 << get_order(
6017 array_size(sizeof(struct io_uring_sqe), sq_entries));
Jens Axboe2b188cc2019-01-07 10:46:33 -07006018
Hristo Venev75b28af2019-08-26 17:23:46 +00006019 return pages;
Jens Axboe2b188cc2019-01-07 10:46:33 -07006020}
6021
Jens Axboeedafcce2019-01-09 09:16:05 -07006022static int io_sqe_buffer_unregister(struct io_ring_ctx *ctx)
6023{
6024 int i, j;
6025
6026 if (!ctx->user_bufs)
6027 return -ENXIO;
6028
6029 for (i = 0; i < ctx->nr_user_bufs; i++) {
6030 struct io_mapped_ubuf *imu = &ctx->user_bufs[i];
6031
6032 for (j = 0; j < imu->nr_bvecs; j++)
John Hubbard27c4d3a2019-08-04 19:32:06 -07006033 put_user_page(imu->bvec[j].bv_page);
Jens Axboeedafcce2019-01-09 09:16:05 -07006034
6035 if (ctx->account_mem)
6036 io_unaccount_mem(ctx->user, imu->nr_bvecs);
Mark Rutlandd4ef6472019-05-01 16:59:16 +01006037 kvfree(imu->bvec);
Jens Axboeedafcce2019-01-09 09:16:05 -07006038 imu->nr_bvecs = 0;
6039 }
6040
6041 kfree(ctx->user_bufs);
6042 ctx->user_bufs = NULL;
6043 ctx->nr_user_bufs = 0;
6044 return 0;
6045}
6046
6047static int io_copy_iov(struct io_ring_ctx *ctx, struct iovec *dst,
6048 void __user *arg, unsigned index)
6049{
6050 struct iovec __user *src;
6051
6052#ifdef CONFIG_COMPAT
6053 if (ctx->compat) {
6054 struct compat_iovec __user *ciovs;
6055 struct compat_iovec ciov;
6056
6057 ciovs = (struct compat_iovec __user *) arg;
6058 if (copy_from_user(&ciov, &ciovs[index], sizeof(ciov)))
6059 return -EFAULT;
6060
Jens Axboed55e5f52019-12-11 16:12:15 -07006061 dst->iov_base = u64_to_user_ptr((u64)ciov.iov_base);
Jens Axboeedafcce2019-01-09 09:16:05 -07006062 dst->iov_len = ciov.iov_len;
6063 return 0;
6064 }
6065#endif
6066 src = (struct iovec __user *) arg;
6067 if (copy_from_user(dst, &src[index], sizeof(*dst)))
6068 return -EFAULT;
6069 return 0;
6070}
6071
6072static int io_sqe_buffer_register(struct io_ring_ctx *ctx, void __user *arg,
6073 unsigned nr_args)
6074{
6075 struct vm_area_struct **vmas = NULL;
6076 struct page **pages = NULL;
6077 int i, j, got_pages = 0;
6078 int ret = -EINVAL;
6079
6080 if (ctx->user_bufs)
6081 return -EBUSY;
6082 if (!nr_args || nr_args > UIO_MAXIOV)
6083 return -EINVAL;
6084
6085 ctx->user_bufs = kcalloc(nr_args, sizeof(struct io_mapped_ubuf),
6086 GFP_KERNEL);
6087 if (!ctx->user_bufs)
6088 return -ENOMEM;
6089
6090 for (i = 0; i < nr_args; i++) {
6091 struct io_mapped_ubuf *imu = &ctx->user_bufs[i];
6092 unsigned long off, start, end, ubuf;
6093 int pret, nr_pages;
6094 struct iovec iov;
6095 size_t size;
6096
6097 ret = io_copy_iov(ctx, &iov, arg, i);
6098 if (ret)
Pavel Begunkova2786822019-05-26 12:35:47 +03006099 goto err;
Jens Axboeedafcce2019-01-09 09:16:05 -07006100
6101 /*
6102 * Don't impose further limits on the size and buffer
6103 * constraints here, we'll -EINVAL later when IO is
6104 * submitted if they are wrong.
6105 */
6106 ret = -EFAULT;
6107 if (!iov.iov_base || !iov.iov_len)
6108 goto err;
6109
6110 /* arbitrary limit, but we need something */
6111 if (iov.iov_len > SZ_1G)
6112 goto err;
6113
6114 ubuf = (unsigned long) iov.iov_base;
6115 end = (ubuf + iov.iov_len + PAGE_SIZE - 1) >> PAGE_SHIFT;
6116 start = ubuf >> PAGE_SHIFT;
6117 nr_pages = end - start;
6118
6119 if (ctx->account_mem) {
6120 ret = io_account_mem(ctx->user, nr_pages);
6121 if (ret)
6122 goto err;
6123 }
6124
6125 ret = 0;
6126 if (!pages || nr_pages > got_pages) {
6127 kfree(vmas);
6128 kfree(pages);
Mark Rutlandd4ef6472019-05-01 16:59:16 +01006129 pages = kvmalloc_array(nr_pages, sizeof(struct page *),
Jens Axboeedafcce2019-01-09 09:16:05 -07006130 GFP_KERNEL);
Mark Rutlandd4ef6472019-05-01 16:59:16 +01006131 vmas = kvmalloc_array(nr_pages,
Jens Axboeedafcce2019-01-09 09:16:05 -07006132 sizeof(struct vm_area_struct *),
6133 GFP_KERNEL);
6134 if (!pages || !vmas) {
6135 ret = -ENOMEM;
6136 if (ctx->account_mem)
6137 io_unaccount_mem(ctx->user, nr_pages);
6138 goto err;
6139 }
6140 got_pages = nr_pages;
6141 }
6142
Mark Rutlandd4ef6472019-05-01 16:59:16 +01006143 imu->bvec = kvmalloc_array(nr_pages, sizeof(struct bio_vec),
Jens Axboeedafcce2019-01-09 09:16:05 -07006144 GFP_KERNEL);
6145 ret = -ENOMEM;
6146 if (!imu->bvec) {
6147 if (ctx->account_mem)
6148 io_unaccount_mem(ctx->user, nr_pages);
6149 goto err;
6150 }
6151
6152 ret = 0;
6153 down_read(&current->mm->mmap_sem);
Ira Weiny932f4a62019-05-13 17:17:03 -07006154 pret = get_user_pages(ubuf, nr_pages,
6155 FOLL_WRITE | FOLL_LONGTERM,
6156 pages, vmas);
Jens Axboeedafcce2019-01-09 09:16:05 -07006157 if (pret == nr_pages) {
6158 /* don't support file backed memory */
6159 for (j = 0; j < nr_pages; j++) {
6160 struct vm_area_struct *vma = vmas[j];
6161
6162 if (vma->vm_file &&
6163 !is_file_hugepages(vma->vm_file)) {
6164 ret = -EOPNOTSUPP;
6165 break;
6166 }
6167 }
6168 } else {
6169 ret = pret < 0 ? pret : -EFAULT;
6170 }
6171 up_read(&current->mm->mmap_sem);
6172 if (ret) {
6173 /*
6174 * if we did partial map, or found file backed vmas,
6175 * release any pages we did get
6176 */
John Hubbard27c4d3a2019-08-04 19:32:06 -07006177 if (pret > 0)
6178 put_user_pages(pages, pret);
Jens Axboeedafcce2019-01-09 09:16:05 -07006179 if (ctx->account_mem)
6180 io_unaccount_mem(ctx->user, nr_pages);
Mark Rutlandd4ef6472019-05-01 16:59:16 +01006181 kvfree(imu->bvec);
Jens Axboeedafcce2019-01-09 09:16:05 -07006182 goto err;
6183 }
6184
6185 off = ubuf & ~PAGE_MASK;
6186 size = iov.iov_len;
6187 for (j = 0; j < nr_pages; j++) {
6188 size_t vec_len;
6189
6190 vec_len = min_t(size_t, size, PAGE_SIZE - off);
6191 imu->bvec[j].bv_page = pages[j];
6192 imu->bvec[j].bv_len = vec_len;
6193 imu->bvec[j].bv_offset = off;
6194 off = 0;
6195 size -= vec_len;
6196 }
6197 /* store original address for later verification */
6198 imu->ubuf = ubuf;
6199 imu->len = iov.iov_len;
6200 imu->nr_bvecs = nr_pages;
6201
6202 ctx->nr_user_bufs++;
6203 }
Mark Rutlandd4ef6472019-05-01 16:59:16 +01006204 kvfree(pages);
6205 kvfree(vmas);
Jens Axboeedafcce2019-01-09 09:16:05 -07006206 return 0;
6207err:
Mark Rutlandd4ef6472019-05-01 16:59:16 +01006208 kvfree(pages);
6209 kvfree(vmas);
Jens Axboeedafcce2019-01-09 09:16:05 -07006210 io_sqe_buffer_unregister(ctx);
6211 return ret;
6212}
6213
Jens Axboe9b402842019-04-11 11:45:41 -06006214static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg)
6215{
6216 __s32 __user *fds = arg;
6217 int fd;
6218
6219 if (ctx->cq_ev_fd)
6220 return -EBUSY;
6221
6222 if (copy_from_user(&fd, fds, sizeof(*fds)))
6223 return -EFAULT;
6224
6225 ctx->cq_ev_fd = eventfd_ctx_fdget(fd);
6226 if (IS_ERR(ctx->cq_ev_fd)) {
6227 int ret = PTR_ERR(ctx->cq_ev_fd);
6228 ctx->cq_ev_fd = NULL;
6229 return ret;
6230 }
6231
6232 return 0;
6233}
6234
6235static int io_eventfd_unregister(struct io_ring_ctx *ctx)
6236{
6237 if (ctx->cq_ev_fd) {
6238 eventfd_ctx_put(ctx->cq_ev_fd);
6239 ctx->cq_ev_fd = NULL;
6240 return 0;
6241 }
6242
6243 return -ENXIO;
6244}
6245
Jens Axboe2b188cc2019-01-07 10:46:33 -07006246static void io_ring_ctx_free(struct io_ring_ctx *ctx)
6247{
Jens Axboe6b063142019-01-10 22:13:58 -07006248 io_finish_async(ctx);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006249 if (ctx->sqo_mm)
6250 mmdrop(ctx->sqo_mm);
Jens Axboedef596e2019-01-09 08:59:42 -07006251
6252 io_iopoll_reap_events(ctx);
Jens Axboeedafcce2019-01-09 09:16:05 -07006253 io_sqe_buffer_unregister(ctx);
Jens Axboe6b063142019-01-10 22:13:58 -07006254 io_sqe_files_unregister(ctx);
Jens Axboe9b402842019-04-11 11:45:41 -06006255 io_eventfd_unregister(ctx);
Jens Axboedef596e2019-01-09 08:59:42 -07006256
Jens Axboe2b188cc2019-01-07 10:46:33 -07006257#if defined(CONFIG_UNIX)
Eric Biggers355e8d22019-06-12 14:58:43 -07006258 if (ctx->ring_sock) {
6259 ctx->ring_sock->file = NULL; /* so that iput() is called */
Jens Axboe2b188cc2019-01-07 10:46:33 -07006260 sock_release(ctx->ring_sock);
Eric Biggers355e8d22019-06-12 14:58:43 -07006261 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07006262#endif
6263
Hristo Venev75b28af2019-08-26 17:23:46 +00006264 io_mem_free(ctx->rings);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006265 io_mem_free(ctx->sq_sqes);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006266
6267 percpu_ref_exit(&ctx->refs);
6268 if (ctx->account_mem)
6269 io_unaccount_mem(ctx->user,
6270 ring_pages(ctx->sq_entries, ctx->cq_entries));
6271 free_uid(ctx->user);
Jens Axboe181e4482019-11-25 08:52:30 -07006272 put_cred(ctx->creds);
Jens Axboe206aefd2019-11-07 18:27:42 -07006273 kfree(ctx->completions);
Jens Axboe78076bb2019-12-04 19:56:40 -07006274 kfree(ctx->cancel_hash);
Jens Axboe0ddf92e2019-11-08 08:52:53 -07006275 kmem_cache_free(req_cachep, ctx->fallback_req);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006276 kfree(ctx);
6277}
6278
6279static __poll_t io_uring_poll(struct file *file, poll_table *wait)
6280{
6281 struct io_ring_ctx *ctx = file->private_data;
6282 __poll_t mask = 0;
6283
6284 poll_wait(file, &ctx->cq_wait, wait);
Stefan Bühler4f7067c2019-04-24 23:54:17 +02006285 /*
6286 * synchronizes with barrier from wq_has_sleeper call in
6287 * io_commit_cqring
6288 */
Jens Axboe2b188cc2019-01-07 10:46:33 -07006289 smp_rmb();
Hristo Venev75b28af2019-08-26 17:23:46 +00006290 if (READ_ONCE(ctx->rings->sq.tail) - ctx->cached_sq_head !=
6291 ctx->rings->sq_ring_entries)
Jens Axboe2b188cc2019-01-07 10:46:33 -07006292 mask |= EPOLLOUT | EPOLLWRNORM;
yangerkundaa5de52019-09-24 20:53:34 +08006293 if (READ_ONCE(ctx->rings->cq.head) != ctx->cached_cq_tail)
Jens Axboe2b188cc2019-01-07 10:46:33 -07006294 mask |= EPOLLIN | EPOLLRDNORM;
6295
6296 return mask;
6297}
6298
6299static int io_uring_fasync(int fd, struct file *file, int on)
6300{
6301 struct io_ring_ctx *ctx = file->private_data;
6302
6303 return fasync_helper(fd, file, on, &ctx->cq_fasync);
6304}
6305
Jens Axboe071698e2020-01-28 10:04:42 -07006306static int io_remove_personalities(int id, void *p, void *data)
6307{
6308 struct io_ring_ctx *ctx = data;
6309 const struct cred *cred;
6310
6311 cred = idr_remove(&ctx->personality_idr, id);
6312 if (cred)
6313 put_cred(cred);
6314 return 0;
6315}
6316
Jens Axboe2b188cc2019-01-07 10:46:33 -07006317static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx)
6318{
6319 mutex_lock(&ctx->uring_lock);
6320 percpu_ref_kill(&ctx->refs);
6321 mutex_unlock(&ctx->uring_lock);
6322
Jens Axboe5262f562019-09-17 12:26:57 -06006323 io_kill_timeouts(ctx);
Jens Axboe221c5eb2019-01-17 09:41:58 -07006324 io_poll_remove_all(ctx);
Jens Axboe561fb042019-10-24 07:25:42 -06006325
6326 if (ctx->io_wq)
6327 io_wq_cancel_all(ctx->io_wq);
6328
Jens Axboedef596e2019-01-09 08:59:42 -07006329 io_iopoll_reap_events(ctx);
Jens Axboe15dff282019-11-13 09:09:23 -07006330 /* if we failed setting up the ctx, we might not have any rings */
6331 if (ctx->rings)
6332 io_cqring_overflow_flush(ctx, true);
Jens Axboe071698e2020-01-28 10:04:42 -07006333 idr_for_each(&ctx->personality_idr, io_remove_personalities, ctx);
Jens Axboe206aefd2019-11-07 18:27:42 -07006334 wait_for_completion(&ctx->completions[0]);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006335 io_ring_ctx_free(ctx);
6336}
6337
6338static int io_uring_release(struct inode *inode, struct file *file)
6339{
6340 struct io_ring_ctx *ctx = file->private_data;
6341
6342 file->private_data = NULL;
6343 io_ring_ctx_wait_and_kill(ctx);
6344 return 0;
6345}
6346
Jens Axboefcb323c2019-10-24 12:39:47 -06006347static void io_uring_cancel_files(struct io_ring_ctx *ctx,
6348 struct files_struct *files)
6349{
6350 struct io_kiocb *req;
6351 DEFINE_WAIT(wait);
6352
6353 while (!list_empty_careful(&ctx->inflight_list)) {
Jens Axboe768134d2019-11-10 20:30:53 -07006354 struct io_kiocb *cancel_req = NULL;
Jens Axboefcb323c2019-10-24 12:39:47 -06006355
6356 spin_lock_irq(&ctx->inflight_lock);
6357 list_for_each_entry(req, &ctx->inflight_list, inflight_entry) {
Jens Axboe768134d2019-11-10 20:30:53 -07006358 if (req->work.files != files)
6359 continue;
6360 /* req is being completed, ignore */
6361 if (!refcount_inc_not_zero(&req->refs))
6362 continue;
6363 cancel_req = req;
6364 break;
Jens Axboefcb323c2019-10-24 12:39:47 -06006365 }
Jens Axboe768134d2019-11-10 20:30:53 -07006366 if (cancel_req)
Jens Axboefcb323c2019-10-24 12:39:47 -06006367 prepare_to_wait(&ctx->inflight_wait, &wait,
Jens Axboe768134d2019-11-10 20:30:53 -07006368 TASK_UNINTERRUPTIBLE);
Jens Axboefcb323c2019-10-24 12:39:47 -06006369 spin_unlock_irq(&ctx->inflight_lock);
6370
Jens Axboe768134d2019-11-10 20:30:53 -07006371 /* We need to keep going until we don't find a matching req */
6372 if (!cancel_req)
Jens Axboefcb323c2019-10-24 12:39:47 -06006373 break;
Bob Liu2f6d9b92019-11-13 18:06:24 +08006374
6375 io_wq_cancel_work(ctx->io_wq, &cancel_req->work);
6376 io_put_req(cancel_req);
Jens Axboefcb323c2019-10-24 12:39:47 -06006377 schedule();
6378 }
Jens Axboe768134d2019-11-10 20:30:53 -07006379 finish_wait(&ctx->inflight_wait, &wait);
Jens Axboefcb323c2019-10-24 12:39:47 -06006380}
6381
6382static int io_uring_flush(struct file *file, void *data)
6383{
6384 struct io_ring_ctx *ctx = file->private_data;
6385
6386 io_uring_cancel_files(ctx, data);
Jens Axboefcb323c2019-10-24 12:39:47 -06006387 return 0;
6388}
6389
Roman Penyaev6c5c2402019-11-28 12:53:22 +01006390static void *io_uring_validate_mmap_request(struct file *file,
6391 loff_t pgoff, size_t sz)
Jens Axboe2b188cc2019-01-07 10:46:33 -07006392{
Jens Axboe2b188cc2019-01-07 10:46:33 -07006393 struct io_ring_ctx *ctx = file->private_data;
Roman Penyaev6c5c2402019-11-28 12:53:22 +01006394 loff_t offset = pgoff << PAGE_SHIFT;
Jens Axboe2b188cc2019-01-07 10:46:33 -07006395 struct page *page;
6396 void *ptr;
6397
6398 switch (offset) {
6399 case IORING_OFF_SQ_RING:
Hristo Venev75b28af2019-08-26 17:23:46 +00006400 case IORING_OFF_CQ_RING:
6401 ptr = ctx->rings;
Jens Axboe2b188cc2019-01-07 10:46:33 -07006402 break;
6403 case IORING_OFF_SQES:
6404 ptr = ctx->sq_sqes;
6405 break;
Jens Axboe2b188cc2019-01-07 10:46:33 -07006406 default:
Roman Penyaev6c5c2402019-11-28 12:53:22 +01006407 return ERR_PTR(-EINVAL);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006408 }
6409
6410 page = virt_to_head_page(ptr);
Matthew Wilcox (Oracle)a50b8542019-09-23 15:34:25 -07006411 if (sz > page_size(page))
Roman Penyaev6c5c2402019-11-28 12:53:22 +01006412 return ERR_PTR(-EINVAL);
6413
6414 return ptr;
6415}
6416
6417#ifdef CONFIG_MMU
6418
6419static int io_uring_mmap(struct file *file, struct vm_area_struct *vma)
6420{
6421 size_t sz = vma->vm_end - vma->vm_start;
6422 unsigned long pfn;
6423 void *ptr;
6424
6425 ptr = io_uring_validate_mmap_request(file, vma->vm_pgoff, sz);
6426 if (IS_ERR(ptr))
6427 return PTR_ERR(ptr);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006428
6429 pfn = virt_to_phys(ptr) >> PAGE_SHIFT;
6430 return remap_pfn_range(vma, vma->vm_start, pfn, sz, vma->vm_page_prot);
6431}
6432
Roman Penyaev6c5c2402019-11-28 12:53:22 +01006433#else /* !CONFIG_MMU */
6434
6435static int io_uring_mmap(struct file *file, struct vm_area_struct *vma)
6436{
6437 return vma->vm_flags & (VM_SHARED | VM_MAYSHARE) ? 0 : -EINVAL;
6438}
6439
6440static unsigned int io_uring_nommu_mmap_capabilities(struct file *file)
6441{
6442 return NOMMU_MAP_DIRECT | NOMMU_MAP_READ | NOMMU_MAP_WRITE;
6443}
6444
6445static unsigned long io_uring_nommu_get_unmapped_area(struct file *file,
6446 unsigned long addr, unsigned long len,
6447 unsigned long pgoff, unsigned long flags)
6448{
6449 void *ptr;
6450
6451 ptr = io_uring_validate_mmap_request(file, pgoff, len);
6452 if (IS_ERR(ptr))
6453 return PTR_ERR(ptr);
6454
6455 return (unsigned long) ptr;
6456}
6457
6458#endif /* !CONFIG_MMU */
6459
Jens Axboe2b188cc2019-01-07 10:46:33 -07006460SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
6461 u32, min_complete, u32, flags, const sigset_t __user *, sig,
6462 size_t, sigsz)
6463{
6464 struct io_ring_ctx *ctx;
6465 long ret = -EBADF;
6466 int submitted = 0;
6467 struct fd f;
6468
Jens Axboe6c271ce2019-01-10 11:22:30 -07006469 if (flags & ~(IORING_ENTER_GETEVENTS | IORING_ENTER_SQ_WAKEUP))
Jens Axboe2b188cc2019-01-07 10:46:33 -07006470 return -EINVAL;
6471
6472 f = fdget(fd);
6473 if (!f.file)
6474 return -EBADF;
6475
6476 ret = -EOPNOTSUPP;
6477 if (f.file->f_op != &io_uring_fops)
6478 goto out_fput;
6479
6480 ret = -ENXIO;
6481 ctx = f.file->private_data;
6482 if (!percpu_ref_tryget(&ctx->refs))
6483 goto out_fput;
6484
Jens Axboe6c271ce2019-01-10 11:22:30 -07006485 /*
6486 * For SQ polling, the thread will do all submissions and completions.
6487 * Just return the requested submit count, and wake the thread if
6488 * we were asked to.
6489 */
Jens Axboeb2a9ead2019-09-12 14:19:16 -06006490 ret = 0;
Jens Axboe6c271ce2019-01-10 11:22:30 -07006491 if (ctx->flags & IORING_SETUP_SQPOLL) {
Jens Axboec1edbf52019-11-10 16:56:04 -07006492 if (!list_empty_careful(&ctx->cq_overflow_list))
6493 io_cqring_overflow_flush(ctx, false);
Jens Axboe6c271ce2019-01-10 11:22:30 -07006494 if (flags & IORING_ENTER_SQ_WAKEUP)
6495 wake_up(&ctx->sqo_wait);
6496 submitted = to_submit;
Jens Axboeb2a9ead2019-09-12 14:19:16 -06006497 } else if (to_submit) {
Pavel Begunkovae9428c2019-11-06 00:22:14 +03006498 struct mm_struct *cur_mm;
Jens Axboe2b188cc2019-01-07 10:46:33 -07006499
6500 mutex_lock(&ctx->uring_lock);
Pavel Begunkovae9428c2019-11-06 00:22:14 +03006501 /* already have mm, so io_submit_sqes() won't try to grab it */
6502 cur_mm = ctx->sqo_mm;
6503 submitted = io_submit_sqes(ctx, to_submit, f.file, fd,
6504 &cur_mm, false);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006505 mutex_unlock(&ctx->uring_lock);
Pavel Begunkov7c504e652019-12-18 19:53:45 +03006506
6507 if (submitted != to_submit)
6508 goto out;
Jens Axboe2b188cc2019-01-07 10:46:33 -07006509 }
6510 if (flags & IORING_ENTER_GETEVENTS) {
Jens Axboedef596e2019-01-09 08:59:42 -07006511 unsigned nr_events = 0;
6512
Jens Axboe2b188cc2019-01-07 10:46:33 -07006513 min_complete = min(min_complete, ctx->cq_entries);
6514
Jens Axboedef596e2019-01-09 08:59:42 -07006515 if (ctx->flags & IORING_SETUP_IOPOLL) {
Jens Axboedef596e2019-01-09 08:59:42 -07006516 ret = io_iopoll_check(ctx, &nr_events, min_complete);
Jens Axboedef596e2019-01-09 08:59:42 -07006517 } else {
6518 ret = io_cqring_wait(ctx, min_complete, sig, sigsz);
6519 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07006520 }
6521
Pavel Begunkov7c504e652019-12-18 19:53:45 +03006522out:
Pavel Begunkov6805b322019-10-08 02:18:42 +03006523 percpu_ref_put(&ctx->refs);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006524out_fput:
6525 fdput(f);
6526 return submitted ? submitted : ret;
6527}
6528
Jens Axboe87ce9552020-01-30 08:25:34 -07006529static int io_uring_show_cred(int id, void *p, void *data)
6530{
6531 const struct cred *cred = p;
6532 struct seq_file *m = data;
6533 struct user_namespace *uns = seq_user_ns(m);
6534 struct group_info *gi;
6535 kernel_cap_t cap;
6536 unsigned __capi;
6537 int g;
6538
6539 seq_printf(m, "%5d\n", id);
6540 seq_put_decimal_ull(m, "\tUid:\t", from_kuid_munged(uns, cred->uid));
6541 seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->euid));
6542 seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->suid));
6543 seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->fsuid));
6544 seq_put_decimal_ull(m, "\n\tGid:\t", from_kgid_munged(uns, cred->gid));
6545 seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->egid));
6546 seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->sgid));
6547 seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->fsgid));
6548 seq_puts(m, "\n\tGroups:\t");
6549 gi = cred->group_info;
6550 for (g = 0; g < gi->ngroups; g++) {
6551 seq_put_decimal_ull(m, g ? " " : "",
6552 from_kgid_munged(uns, gi->gid[g]));
6553 }
6554 seq_puts(m, "\n\tCapEff:\t");
6555 cap = cred->cap_effective;
6556 CAP_FOR_EACH_U32(__capi)
6557 seq_put_hex_ll(m, NULL, cap.cap[CAP_LAST_U32 - __capi], 8);
6558 seq_putc(m, '\n');
6559 return 0;
6560}
6561
6562static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m)
6563{
6564 int i;
6565
6566 mutex_lock(&ctx->uring_lock);
6567 seq_printf(m, "UserFiles:\t%u\n", ctx->nr_user_files);
6568 for (i = 0; i < ctx->nr_user_files; i++) {
6569 struct fixed_file_table *table;
6570 struct file *f;
6571
6572 table = &ctx->file_data->table[i >> IORING_FILE_TABLE_SHIFT];
6573 f = table->files[i & IORING_FILE_TABLE_MASK];
6574 if (f)
6575 seq_printf(m, "%5u: %s\n", i, file_dentry(f)->d_iname);
6576 else
6577 seq_printf(m, "%5u: <none>\n", i);
6578 }
6579 seq_printf(m, "UserBufs:\t%u\n", ctx->nr_user_bufs);
6580 for (i = 0; i < ctx->nr_user_bufs; i++) {
6581 struct io_mapped_ubuf *buf = &ctx->user_bufs[i];
6582
6583 seq_printf(m, "%5u: 0x%llx/%u\n", i, buf->ubuf,
6584 (unsigned int) buf->len);
6585 }
6586 if (!idr_is_empty(&ctx->personality_idr)) {
6587 seq_printf(m, "Personalities:\n");
6588 idr_for_each(&ctx->personality_idr, io_uring_show_cred, m);
6589 }
6590 mutex_unlock(&ctx->uring_lock);
6591}
6592
6593static void io_uring_show_fdinfo(struct seq_file *m, struct file *f)
6594{
6595 struct io_ring_ctx *ctx = f->private_data;
6596
6597 if (percpu_ref_tryget(&ctx->refs)) {
6598 __io_uring_show_fdinfo(ctx, m);
6599 percpu_ref_put(&ctx->refs);
6600 }
6601}
6602
Jens Axboe2b188cc2019-01-07 10:46:33 -07006603static const struct file_operations io_uring_fops = {
6604 .release = io_uring_release,
Jens Axboefcb323c2019-10-24 12:39:47 -06006605 .flush = io_uring_flush,
Jens Axboe2b188cc2019-01-07 10:46:33 -07006606 .mmap = io_uring_mmap,
Roman Penyaev6c5c2402019-11-28 12:53:22 +01006607#ifndef CONFIG_MMU
6608 .get_unmapped_area = io_uring_nommu_get_unmapped_area,
6609 .mmap_capabilities = io_uring_nommu_mmap_capabilities,
6610#endif
Jens Axboe2b188cc2019-01-07 10:46:33 -07006611 .poll = io_uring_poll,
6612 .fasync = io_uring_fasync,
Jens Axboe87ce9552020-01-30 08:25:34 -07006613 .show_fdinfo = io_uring_show_fdinfo,
Jens Axboe2b188cc2019-01-07 10:46:33 -07006614};
6615
6616static int io_allocate_scq_urings(struct io_ring_ctx *ctx,
6617 struct io_uring_params *p)
6618{
Hristo Venev75b28af2019-08-26 17:23:46 +00006619 struct io_rings *rings;
6620 size_t size, sq_array_offset;
Jens Axboe2b188cc2019-01-07 10:46:33 -07006621
Hristo Venev75b28af2019-08-26 17:23:46 +00006622 size = rings_size(p->sq_entries, p->cq_entries, &sq_array_offset);
6623 if (size == SIZE_MAX)
6624 return -EOVERFLOW;
6625
6626 rings = io_mem_alloc(size);
6627 if (!rings)
Jens Axboe2b188cc2019-01-07 10:46:33 -07006628 return -ENOMEM;
6629
Hristo Venev75b28af2019-08-26 17:23:46 +00006630 ctx->rings = rings;
6631 ctx->sq_array = (u32 *)((char *)rings + sq_array_offset);
6632 rings->sq_ring_mask = p->sq_entries - 1;
6633 rings->cq_ring_mask = p->cq_entries - 1;
6634 rings->sq_ring_entries = p->sq_entries;
6635 rings->cq_ring_entries = p->cq_entries;
6636 ctx->sq_mask = rings->sq_ring_mask;
6637 ctx->cq_mask = rings->cq_ring_mask;
6638 ctx->sq_entries = rings->sq_ring_entries;
6639 ctx->cq_entries = rings->cq_ring_entries;
Jens Axboe2b188cc2019-01-07 10:46:33 -07006640
6641 size = array_size(sizeof(struct io_uring_sqe), p->sq_entries);
Jens Axboeeb065d32019-11-20 09:26:29 -07006642 if (size == SIZE_MAX) {
6643 io_mem_free(ctx->rings);
6644 ctx->rings = NULL;
Jens Axboe2b188cc2019-01-07 10:46:33 -07006645 return -EOVERFLOW;
Jens Axboeeb065d32019-11-20 09:26:29 -07006646 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07006647
6648 ctx->sq_sqes = io_mem_alloc(size);
Jens Axboeeb065d32019-11-20 09:26:29 -07006649 if (!ctx->sq_sqes) {
6650 io_mem_free(ctx->rings);
6651 ctx->rings = NULL;
Jens Axboe2b188cc2019-01-07 10:46:33 -07006652 return -ENOMEM;
Jens Axboeeb065d32019-11-20 09:26:29 -07006653 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07006654
Jens Axboe2b188cc2019-01-07 10:46:33 -07006655 return 0;
6656}
6657
6658/*
6659 * Allocate an anonymous fd, this is what constitutes the application
6660 * visible backing of an io_uring instance. The application mmaps this
6661 * fd to gain access to the SQ/CQ ring details. If UNIX sockets are enabled,
6662 * we have to tie this fd to a socket for file garbage collection purposes.
6663 */
6664static int io_uring_get_fd(struct io_ring_ctx *ctx)
6665{
6666 struct file *file;
6667 int ret;
6668
6669#if defined(CONFIG_UNIX)
6670 ret = sock_create_kern(&init_net, PF_UNIX, SOCK_RAW, IPPROTO_IP,
6671 &ctx->ring_sock);
6672 if (ret)
6673 return ret;
6674#endif
6675
6676 ret = get_unused_fd_flags(O_RDWR | O_CLOEXEC);
6677 if (ret < 0)
6678 goto err;
6679
6680 file = anon_inode_getfile("[io_uring]", &io_uring_fops, ctx,
6681 O_RDWR | O_CLOEXEC);
6682 if (IS_ERR(file)) {
6683 put_unused_fd(ret);
6684 ret = PTR_ERR(file);
6685 goto err;
6686 }
6687
6688#if defined(CONFIG_UNIX)
6689 ctx->ring_sock->file = file;
6690#endif
6691 fd_install(ret, file);
6692 return ret;
6693err:
6694#if defined(CONFIG_UNIX)
6695 sock_release(ctx->ring_sock);
6696 ctx->ring_sock = NULL;
6697#endif
6698 return ret;
6699}
6700
6701static int io_uring_create(unsigned entries, struct io_uring_params *p)
6702{
6703 struct user_struct *user = NULL;
6704 struct io_ring_ctx *ctx;
6705 bool account_mem;
6706 int ret;
6707
Jens Axboe8110c1a2019-12-28 15:39:54 -07006708 if (!entries)
Jens Axboe2b188cc2019-01-07 10:46:33 -07006709 return -EINVAL;
Jens Axboe8110c1a2019-12-28 15:39:54 -07006710 if (entries > IORING_MAX_ENTRIES) {
6711 if (!(p->flags & IORING_SETUP_CLAMP))
6712 return -EINVAL;
6713 entries = IORING_MAX_ENTRIES;
6714 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07006715
6716 /*
6717 * Use twice as many entries for the CQ ring. It's possible for the
6718 * application to drive a higher depth than the size of the SQ ring,
6719 * since the sqes are only used at submission time. This allows for
Jens Axboe33a107f2019-10-04 12:10:03 -06006720 * some flexibility in overcommitting a bit. If the application has
6721 * set IORING_SETUP_CQSIZE, it will have passed in the desired number
6722 * of CQ ring entries manually.
Jens Axboe2b188cc2019-01-07 10:46:33 -07006723 */
6724 p->sq_entries = roundup_pow_of_two(entries);
Jens Axboe33a107f2019-10-04 12:10:03 -06006725 if (p->flags & IORING_SETUP_CQSIZE) {
6726 /*
6727 * If IORING_SETUP_CQSIZE is set, we do the same roundup
6728 * to a power-of-two, if it isn't already. We do NOT impose
6729 * any cq vs sq ring sizing.
6730 */
Jens Axboe8110c1a2019-12-28 15:39:54 -07006731 if (p->cq_entries < p->sq_entries)
Jens Axboe33a107f2019-10-04 12:10:03 -06006732 return -EINVAL;
Jens Axboe8110c1a2019-12-28 15:39:54 -07006733 if (p->cq_entries > IORING_MAX_CQ_ENTRIES) {
6734 if (!(p->flags & IORING_SETUP_CLAMP))
6735 return -EINVAL;
6736 p->cq_entries = IORING_MAX_CQ_ENTRIES;
6737 }
Jens Axboe33a107f2019-10-04 12:10:03 -06006738 p->cq_entries = roundup_pow_of_two(p->cq_entries);
6739 } else {
6740 p->cq_entries = 2 * p->sq_entries;
6741 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07006742
6743 user = get_uid(current_user());
6744 account_mem = !capable(CAP_IPC_LOCK);
6745
6746 if (account_mem) {
6747 ret = io_account_mem(user,
6748 ring_pages(p->sq_entries, p->cq_entries));
6749 if (ret) {
6750 free_uid(user);
6751 return ret;
6752 }
6753 }
6754
6755 ctx = io_ring_ctx_alloc(p);
6756 if (!ctx) {
6757 if (account_mem)
6758 io_unaccount_mem(user, ring_pages(p->sq_entries,
6759 p->cq_entries));
6760 free_uid(user);
6761 return -ENOMEM;
6762 }
6763 ctx->compat = in_compat_syscall();
6764 ctx->account_mem = account_mem;
6765 ctx->user = user;
Jens Axboe0b8c0ec2019-12-02 08:50:00 -07006766 ctx->creds = get_current_cred();
Jens Axboe2b188cc2019-01-07 10:46:33 -07006767
6768 ret = io_allocate_scq_urings(ctx, p);
6769 if (ret)
6770 goto err;
6771
Jens Axboe6c271ce2019-01-10 11:22:30 -07006772 ret = io_sq_offload_start(ctx, p);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006773 if (ret)
6774 goto err;
6775
Jens Axboe2b188cc2019-01-07 10:46:33 -07006776 memset(&p->sq_off, 0, sizeof(p->sq_off));
Hristo Venev75b28af2019-08-26 17:23:46 +00006777 p->sq_off.head = offsetof(struct io_rings, sq.head);
6778 p->sq_off.tail = offsetof(struct io_rings, sq.tail);
6779 p->sq_off.ring_mask = offsetof(struct io_rings, sq_ring_mask);
6780 p->sq_off.ring_entries = offsetof(struct io_rings, sq_ring_entries);
6781 p->sq_off.flags = offsetof(struct io_rings, sq_flags);
6782 p->sq_off.dropped = offsetof(struct io_rings, sq_dropped);
6783 p->sq_off.array = (char *)ctx->sq_array - (char *)ctx->rings;
Jens Axboe2b188cc2019-01-07 10:46:33 -07006784
6785 memset(&p->cq_off, 0, sizeof(p->cq_off));
Hristo Venev75b28af2019-08-26 17:23:46 +00006786 p->cq_off.head = offsetof(struct io_rings, cq.head);
6787 p->cq_off.tail = offsetof(struct io_rings, cq.tail);
6788 p->cq_off.ring_mask = offsetof(struct io_rings, cq_ring_mask);
6789 p->cq_off.ring_entries = offsetof(struct io_rings, cq_ring_entries);
6790 p->cq_off.overflow = offsetof(struct io_rings, cq_overflow);
6791 p->cq_off.cqes = offsetof(struct io_rings, cqes);
Jens Axboeac90f242019-09-06 10:26:21 -06006792
Jens Axboe044c1ab2019-10-28 09:15:33 -06006793 /*
6794 * Install ring fd as the very last thing, so we don't risk someone
6795 * having closed it before we finish setup
6796 */
6797 ret = io_uring_get_fd(ctx);
6798 if (ret < 0)
6799 goto err;
6800
Jens Axboeda8c9692019-12-02 18:51:26 -07006801 p->features = IORING_FEAT_SINGLE_MMAP | IORING_FEAT_NODROP |
Jens Axboecccf0ee2020-01-27 16:34:48 -07006802 IORING_FEAT_SUBMIT_STABLE | IORING_FEAT_RW_CUR_POS |
6803 IORING_FEAT_CUR_PERSONALITY;
Dmitrii Dolgovc826bd72019-10-15 19:02:01 +02006804 trace_io_uring_create(ret, ctx, p->sq_entries, p->cq_entries, p->flags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006805 return ret;
6806err:
6807 io_ring_ctx_wait_and_kill(ctx);
6808 return ret;
6809}
6810
6811/*
6812 * Sets up an aio uring context, and returns the fd. Applications asks for a
6813 * ring size, we return the actual sq/cq ring sizes (among other things) in the
6814 * params structure passed in.
6815 */
6816static long io_uring_setup(u32 entries, struct io_uring_params __user *params)
6817{
6818 struct io_uring_params p;
6819 long ret;
6820 int i;
6821
6822 if (copy_from_user(&p, params, sizeof(p)))
6823 return -EFAULT;
6824 for (i = 0; i < ARRAY_SIZE(p.resv); i++) {
6825 if (p.resv[i])
6826 return -EINVAL;
6827 }
6828
Jens Axboe6c271ce2019-01-10 11:22:30 -07006829 if (p.flags & ~(IORING_SETUP_IOPOLL | IORING_SETUP_SQPOLL |
Jens Axboe8110c1a2019-12-28 15:39:54 -07006830 IORING_SETUP_SQ_AFF | IORING_SETUP_CQSIZE |
Pavel Begunkov24369c22020-01-28 03:15:48 +03006831 IORING_SETUP_CLAMP | IORING_SETUP_ATTACH_WQ))
Jens Axboe2b188cc2019-01-07 10:46:33 -07006832 return -EINVAL;
6833
6834 ret = io_uring_create(entries, &p);
6835 if (ret < 0)
6836 return ret;
6837
6838 if (copy_to_user(params, &p, sizeof(p)))
6839 return -EFAULT;
6840
6841 return ret;
6842}
6843
6844SYSCALL_DEFINE2(io_uring_setup, u32, entries,
6845 struct io_uring_params __user *, params)
6846{
6847 return io_uring_setup(entries, params);
6848}
6849
Jens Axboe66f4af92020-01-16 15:36:52 -07006850static int io_probe(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args)
6851{
6852 struct io_uring_probe *p;
6853 size_t size;
6854 int i, ret;
6855
6856 size = struct_size(p, ops, nr_args);
6857 if (size == SIZE_MAX)
6858 return -EOVERFLOW;
6859 p = kzalloc(size, GFP_KERNEL);
6860 if (!p)
6861 return -ENOMEM;
6862
6863 ret = -EFAULT;
6864 if (copy_from_user(p, arg, size))
6865 goto out;
6866 ret = -EINVAL;
6867 if (memchr_inv(p, 0, size))
6868 goto out;
6869
6870 p->last_op = IORING_OP_LAST - 1;
6871 if (nr_args > IORING_OP_LAST)
6872 nr_args = IORING_OP_LAST;
6873
6874 for (i = 0; i < nr_args; i++) {
6875 p->ops[i].op = i;
6876 if (!io_op_defs[i].not_supported)
6877 p->ops[i].flags = IO_URING_OP_SUPPORTED;
6878 }
6879 p->ops_len = i;
6880
6881 ret = 0;
6882 if (copy_to_user(arg, p, size))
6883 ret = -EFAULT;
6884out:
6885 kfree(p);
6886 return ret;
6887}
6888
Jens Axboe071698e2020-01-28 10:04:42 -07006889static int io_register_personality(struct io_ring_ctx *ctx)
6890{
6891 const struct cred *creds = get_current_cred();
6892 int id;
6893
6894 id = idr_alloc_cyclic(&ctx->personality_idr, (void *) creds, 1,
6895 USHRT_MAX, GFP_KERNEL);
6896 if (id < 0)
6897 put_cred(creds);
6898 return id;
6899}
6900
6901static int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id)
6902{
6903 const struct cred *old_creds;
6904
6905 old_creds = idr_remove(&ctx->personality_idr, id);
6906 if (old_creds) {
6907 put_cred(old_creds);
6908 return 0;
6909 }
6910
6911 return -EINVAL;
6912}
6913
6914static bool io_register_op_must_quiesce(int op)
6915{
6916 switch (op) {
6917 case IORING_UNREGISTER_FILES:
6918 case IORING_REGISTER_FILES_UPDATE:
6919 case IORING_REGISTER_PROBE:
6920 case IORING_REGISTER_PERSONALITY:
6921 case IORING_UNREGISTER_PERSONALITY:
6922 return false;
6923 default:
6924 return true;
6925 }
6926}
6927
Jens Axboeedafcce2019-01-09 09:16:05 -07006928static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,
6929 void __user *arg, unsigned nr_args)
Jens Axboeb19062a2019-04-15 10:49:38 -06006930 __releases(ctx->uring_lock)
6931 __acquires(ctx->uring_lock)
Jens Axboeedafcce2019-01-09 09:16:05 -07006932{
6933 int ret;
6934
Jens Axboe35fa71a2019-04-22 10:23:23 -06006935 /*
6936 * We're inside the ring mutex, if the ref is already dying, then
6937 * someone else killed the ctx or is already going through
6938 * io_uring_register().
6939 */
6940 if (percpu_ref_is_dying(&ctx->refs))
6941 return -ENXIO;
6942
Jens Axboe071698e2020-01-28 10:04:42 -07006943 if (io_register_op_must_quiesce(opcode)) {
Jens Axboe05f3fb32019-12-09 11:22:50 -07006944 percpu_ref_kill(&ctx->refs);
Jens Axboeb19062a2019-04-15 10:49:38 -06006945
Jens Axboe05f3fb32019-12-09 11:22:50 -07006946 /*
6947 * Drop uring mutex before waiting for references to exit. If
6948 * another thread is currently inside io_uring_enter() it might
6949 * need to grab the uring_lock to make progress. If we hold it
6950 * here across the drain wait, then we can deadlock. It's safe
6951 * to drop the mutex here, since no new references will come in
6952 * after we've killed the percpu ref.
6953 */
6954 mutex_unlock(&ctx->uring_lock);
Jens Axboec1503682020-01-08 08:26:07 -07006955 ret = wait_for_completion_interruptible(&ctx->completions[0]);
Jens Axboe05f3fb32019-12-09 11:22:50 -07006956 mutex_lock(&ctx->uring_lock);
Jens Axboec1503682020-01-08 08:26:07 -07006957 if (ret) {
6958 percpu_ref_resurrect(&ctx->refs);
6959 ret = -EINTR;
6960 goto out;
6961 }
Jens Axboe05f3fb32019-12-09 11:22:50 -07006962 }
Jens Axboeedafcce2019-01-09 09:16:05 -07006963
6964 switch (opcode) {
6965 case IORING_REGISTER_BUFFERS:
6966 ret = io_sqe_buffer_register(ctx, arg, nr_args);
6967 break;
6968 case IORING_UNREGISTER_BUFFERS:
6969 ret = -EINVAL;
6970 if (arg || nr_args)
6971 break;
6972 ret = io_sqe_buffer_unregister(ctx);
6973 break;
Jens Axboe6b063142019-01-10 22:13:58 -07006974 case IORING_REGISTER_FILES:
6975 ret = io_sqe_files_register(ctx, arg, nr_args);
6976 break;
6977 case IORING_UNREGISTER_FILES:
6978 ret = -EINVAL;
6979 if (arg || nr_args)
6980 break;
6981 ret = io_sqe_files_unregister(ctx);
6982 break;
Jens Axboec3a31e62019-10-03 13:59:56 -06006983 case IORING_REGISTER_FILES_UPDATE:
6984 ret = io_sqe_files_update(ctx, arg, nr_args);
6985 break;
Jens Axboe9b402842019-04-11 11:45:41 -06006986 case IORING_REGISTER_EVENTFD:
Jens Axboef2842ab2020-01-08 11:04:00 -07006987 case IORING_REGISTER_EVENTFD_ASYNC:
Jens Axboe9b402842019-04-11 11:45:41 -06006988 ret = -EINVAL;
6989 if (nr_args != 1)
6990 break;
6991 ret = io_eventfd_register(ctx, arg);
Jens Axboef2842ab2020-01-08 11:04:00 -07006992 if (ret)
6993 break;
6994 if (opcode == IORING_REGISTER_EVENTFD_ASYNC)
6995 ctx->eventfd_async = 1;
6996 else
6997 ctx->eventfd_async = 0;
Jens Axboe9b402842019-04-11 11:45:41 -06006998 break;
6999 case IORING_UNREGISTER_EVENTFD:
7000 ret = -EINVAL;
7001 if (arg || nr_args)
7002 break;
7003 ret = io_eventfd_unregister(ctx);
7004 break;
Jens Axboe66f4af92020-01-16 15:36:52 -07007005 case IORING_REGISTER_PROBE:
7006 ret = -EINVAL;
7007 if (!arg || nr_args > 256)
7008 break;
7009 ret = io_probe(ctx, arg, nr_args);
7010 break;
Jens Axboe071698e2020-01-28 10:04:42 -07007011 case IORING_REGISTER_PERSONALITY:
7012 ret = -EINVAL;
7013 if (arg || nr_args)
7014 break;
7015 ret = io_register_personality(ctx);
7016 break;
7017 case IORING_UNREGISTER_PERSONALITY:
7018 ret = -EINVAL;
7019 if (arg)
7020 break;
7021 ret = io_unregister_personality(ctx, nr_args);
7022 break;
Jens Axboeedafcce2019-01-09 09:16:05 -07007023 default:
7024 ret = -EINVAL;
7025 break;
7026 }
7027
Jens Axboe071698e2020-01-28 10:04:42 -07007028 if (io_register_op_must_quiesce(opcode)) {
Jens Axboe05f3fb32019-12-09 11:22:50 -07007029 /* bring the ctx back to life */
Jens Axboe05f3fb32019-12-09 11:22:50 -07007030 percpu_ref_reinit(&ctx->refs);
Jens Axboec1503682020-01-08 08:26:07 -07007031out:
7032 reinit_completion(&ctx->completions[0]);
Jens Axboe05f3fb32019-12-09 11:22:50 -07007033 }
Jens Axboeedafcce2019-01-09 09:16:05 -07007034 return ret;
7035}
7036
7037SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode,
7038 void __user *, arg, unsigned int, nr_args)
7039{
7040 struct io_ring_ctx *ctx;
7041 long ret = -EBADF;
7042 struct fd f;
7043
7044 f = fdget(fd);
7045 if (!f.file)
7046 return -EBADF;
7047
7048 ret = -EOPNOTSUPP;
7049 if (f.file->f_op != &io_uring_fops)
7050 goto out_fput;
7051
7052 ctx = f.file->private_data;
7053
7054 mutex_lock(&ctx->uring_lock);
7055 ret = __io_uring_register(ctx, opcode, arg, nr_args);
7056 mutex_unlock(&ctx->uring_lock);
Dmitrii Dolgovc826bd72019-10-15 19:02:01 +02007057 trace_io_uring_register(ctx, opcode, ctx->nr_user_files, ctx->nr_user_bufs,
7058 ctx->cq_ev_fd != NULL, ret);
Jens Axboeedafcce2019-01-09 09:16:05 -07007059out_fput:
7060 fdput(f);
7061 return ret;
7062}
7063
Jens Axboe2b188cc2019-01-07 10:46:33 -07007064static int __init io_uring_init(void)
7065{
Stefan Metzmacherd7f62e82020-01-29 14:39:41 +01007066#define __BUILD_BUG_VERIFY_ELEMENT(stype, eoffset, etype, ename) do { \
7067 BUILD_BUG_ON(offsetof(stype, ename) != eoffset); \
7068 BUILD_BUG_ON(sizeof(etype) != sizeof_field(stype, ename)); \
7069} while (0)
7070
7071#define BUILD_BUG_SQE_ELEM(eoffset, etype, ename) \
7072 __BUILD_BUG_VERIFY_ELEMENT(struct io_uring_sqe, eoffset, etype, ename)
7073 BUILD_BUG_ON(sizeof(struct io_uring_sqe) != 64);
7074 BUILD_BUG_SQE_ELEM(0, __u8, opcode);
7075 BUILD_BUG_SQE_ELEM(1, __u8, flags);
7076 BUILD_BUG_SQE_ELEM(2, __u16, ioprio);
7077 BUILD_BUG_SQE_ELEM(4, __s32, fd);
7078 BUILD_BUG_SQE_ELEM(8, __u64, off);
7079 BUILD_BUG_SQE_ELEM(8, __u64, addr2);
7080 BUILD_BUG_SQE_ELEM(16, __u64, addr);
7081 BUILD_BUG_SQE_ELEM(24, __u32, len);
7082 BUILD_BUG_SQE_ELEM(28, __kernel_rwf_t, rw_flags);
7083 BUILD_BUG_SQE_ELEM(28, /* compat */ int, rw_flags);
7084 BUILD_BUG_SQE_ELEM(28, /* compat */ __u32, rw_flags);
7085 BUILD_BUG_SQE_ELEM(28, __u32, fsync_flags);
7086 BUILD_BUG_SQE_ELEM(28, __u16, poll_events);
7087 BUILD_BUG_SQE_ELEM(28, __u32, sync_range_flags);
7088 BUILD_BUG_SQE_ELEM(28, __u32, msg_flags);
7089 BUILD_BUG_SQE_ELEM(28, __u32, timeout_flags);
7090 BUILD_BUG_SQE_ELEM(28, __u32, accept_flags);
7091 BUILD_BUG_SQE_ELEM(28, __u32, cancel_flags);
7092 BUILD_BUG_SQE_ELEM(28, __u32, open_flags);
7093 BUILD_BUG_SQE_ELEM(28, __u32, statx_flags);
7094 BUILD_BUG_SQE_ELEM(28, __u32, fadvise_advice);
7095 BUILD_BUG_SQE_ELEM(32, __u64, user_data);
7096 BUILD_BUG_SQE_ELEM(40, __u16, buf_index);
7097 BUILD_BUG_SQE_ELEM(42, __u16, personality);
7098
Jens Axboed3656342019-12-18 09:50:26 -07007099 BUILD_BUG_ON(ARRAY_SIZE(io_op_defs) != IORING_OP_LAST);
Jens Axboe2b188cc2019-01-07 10:46:33 -07007100 req_cachep = KMEM_CACHE(io_kiocb, SLAB_HWCACHE_ALIGN | SLAB_PANIC);
7101 return 0;
7102};
7103__initcall(io_uring_init);