blob: 147d1f0e13cc126c10842c27aca19e45f2d74d94 [file] [log] [blame]
Jens Axboe2b188cc2019-01-07 10:46:33 -07001// SPDX-License-Identifier: GPL-2.0
2/*
3 * Shared application/kernel submission and completion ring pairs, for
4 * supporting fast/efficient IO.
5 *
6 * A note on the read/write ordering memory barriers that are matched between
Stefan Bühler1e84b972019-04-24 23:54:16 +02007 * the application and kernel side.
8 *
9 * After the application reads the CQ ring tail, it must use an
10 * appropriate smp_rmb() to pair with the smp_wmb() the kernel uses
11 * before writing the tail (using smp_load_acquire to read the tail will
12 * do). It also needs a smp_mb() before updating CQ head (ordering the
13 * entry load(s) with the head store), pairing with an implicit barrier
14 * through a control-dependency in io_get_cqring (smp_store_release to
15 * store head will do). Failure to do so could lead to reading invalid
16 * CQ entries.
17 *
18 * Likewise, the application must use an appropriate smp_wmb() before
19 * writing the SQ tail (ordering SQ entry stores with the tail store),
20 * which pairs with smp_load_acquire in io_get_sqring (smp_store_release
21 * to store the tail will do). And it needs a barrier ordering the SQ
22 * head load before writing new SQ entries (smp_load_acquire to read
23 * head will do).
24 *
25 * When using the SQ poll thread (IORING_SETUP_SQPOLL), the application
26 * needs to check the SQ flags for IORING_SQ_NEED_WAKEUP *after*
27 * updating the SQ tail; a full memory barrier smp_mb() is needed
28 * between.
Jens Axboe2b188cc2019-01-07 10:46:33 -070029 *
30 * Also see the examples in the liburing library:
31 *
32 * git://git.kernel.dk/liburing
33 *
34 * io_uring also uses READ/WRITE_ONCE() for _any_ store or load that happens
35 * from data shared between the kernel and application. This is done both
36 * for ordering purposes, but also to ensure that once a value is loaded from
37 * data that the application could potentially modify, it remains stable.
38 *
39 * Copyright (C) 2018-2019 Jens Axboe
Christoph Hellwigc992fe22019-01-11 09:43:02 -070040 * Copyright (c) 2018-2019 Christoph Hellwig
Jens Axboe2b188cc2019-01-07 10:46:33 -070041 */
42#include <linux/kernel.h>
43#include <linux/init.h>
44#include <linux/errno.h>
45#include <linux/syscalls.h>
46#include <linux/compat.h>
47#include <linux/refcount.h>
48#include <linux/uio.h>
49
50#include <linux/sched/signal.h>
51#include <linux/fs.h>
52#include <linux/file.h>
53#include <linux/fdtable.h>
54#include <linux/mm.h>
55#include <linux/mman.h>
56#include <linux/mmu_context.h>
57#include <linux/percpu.h>
58#include <linux/slab.h>
Jens Axboe6c271ce2019-01-10 11:22:30 -070059#include <linux/kthread.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070060#include <linux/blkdev.h>
Jens Axboeedafcce2019-01-09 09:16:05 -070061#include <linux/bvec.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070062#include <linux/net.h>
63#include <net/sock.h>
64#include <net/af_unix.h>
Jens Axboe6b063142019-01-10 22:13:58 -070065#include <net/scm.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070066#include <linux/anon_inodes.h>
67#include <linux/sched/mm.h>
68#include <linux/uaccess.h>
69#include <linux/nospec.h>
Jens Axboeedafcce2019-01-09 09:16:05 -070070#include <linux/sizes.h>
71#include <linux/hugetlb.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070072
Dmitrii Dolgovc826bd72019-10-15 19:02:01 +020073#define CREATE_TRACE_POINTS
74#include <trace/events/io_uring.h>
75
Jens Axboe2b188cc2019-01-07 10:46:33 -070076#include <uapi/linux/io_uring.h>
77
78#include "internal.h"
Jens Axboe561fb042019-10-24 07:25:42 -060079#include "io-wq.h"
Jens Axboe2b188cc2019-01-07 10:46:33 -070080
Daniel Xu5277dea2019-09-14 14:23:45 -070081#define IORING_MAX_ENTRIES 32768
Jens Axboe33a107f2019-10-04 12:10:03 -060082#define IORING_MAX_CQ_ENTRIES (2 * IORING_MAX_ENTRIES)
Jens Axboe65e19f52019-10-26 07:20:21 -060083
84/*
85 * Shift of 9 is 512 entries, or exactly one page on 64-bit archs
86 */
87#define IORING_FILE_TABLE_SHIFT 9
88#define IORING_MAX_FILES_TABLE (1U << IORING_FILE_TABLE_SHIFT)
89#define IORING_FILE_TABLE_MASK (IORING_MAX_FILES_TABLE - 1)
90#define IORING_MAX_FIXED_FILES (64 * IORING_MAX_FILES_TABLE)
Jens Axboe2b188cc2019-01-07 10:46:33 -070091
92struct io_uring {
93 u32 head ____cacheline_aligned_in_smp;
94 u32 tail ____cacheline_aligned_in_smp;
95};
96
Stefan Bühler1e84b972019-04-24 23:54:16 +020097/*
Hristo Venev75b28af2019-08-26 17:23:46 +000098 * This data is shared with the application through the mmap at offsets
99 * IORING_OFF_SQ_RING and IORING_OFF_CQ_RING.
Stefan Bühler1e84b972019-04-24 23:54:16 +0200100 *
101 * The offsets to the member fields are published through struct
102 * io_sqring_offsets when calling io_uring_setup.
103 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000104struct io_rings {
Stefan Bühler1e84b972019-04-24 23:54:16 +0200105 /*
106 * Head and tail offsets into the ring; the offsets need to be
107 * masked to get valid indices.
108 *
Hristo Venev75b28af2019-08-26 17:23:46 +0000109 * The kernel controls head of the sq ring and the tail of the cq ring,
110 * and the application controls tail of the sq ring and the head of the
111 * cq ring.
Stefan Bühler1e84b972019-04-24 23:54:16 +0200112 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000113 struct io_uring sq, cq;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200114 /*
Hristo Venev75b28af2019-08-26 17:23:46 +0000115 * Bitmasks to apply to head and tail offsets (constant, equals
Stefan Bühler1e84b972019-04-24 23:54:16 +0200116 * ring_entries - 1)
117 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000118 u32 sq_ring_mask, cq_ring_mask;
119 /* Ring sizes (constant, power of 2) */
120 u32 sq_ring_entries, cq_ring_entries;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200121 /*
122 * Number of invalid entries dropped by the kernel due to
123 * invalid index stored in array
124 *
125 * Written by the kernel, shouldn't be modified by the
126 * application (i.e. get number of "new events" by comparing to
127 * cached value).
128 *
129 * After a new SQ head value was read by the application this
130 * counter includes all submissions that were dropped reaching
131 * the new SQ head (and possibly more).
132 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000133 u32 sq_dropped;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200134 /*
135 * Runtime flags
136 *
137 * Written by the kernel, shouldn't be modified by the
138 * application.
139 *
140 * The application needs a full memory barrier before checking
141 * for IORING_SQ_NEED_WAKEUP after updating the sq tail.
142 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000143 u32 sq_flags;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200144 /*
145 * Number of completion events lost because the queue was full;
146 * this should be avoided by the application by making sure
147 * there are not more requests pending thatn there is space in
148 * the completion queue.
149 *
150 * Written by the kernel, shouldn't be modified by the
151 * application (i.e. get number of "new events" by comparing to
152 * cached value).
153 *
154 * As completion events come in out of order this counter is not
155 * ordered with any other data.
156 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000157 u32 cq_overflow;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200158 /*
159 * Ring buffer of completion events.
160 *
161 * The kernel writes completion events fresh every time they are
162 * produced, so the application is allowed to modify pending
163 * entries.
164 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000165 struct io_uring_cqe cqes[] ____cacheline_aligned_in_smp;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700166};
167
Jens Axboeedafcce2019-01-09 09:16:05 -0700168struct io_mapped_ubuf {
169 u64 ubuf;
170 size_t len;
171 struct bio_vec *bvec;
172 unsigned int nr_bvecs;
173};
174
Jens Axboe65e19f52019-10-26 07:20:21 -0600175struct fixed_file_table {
176 struct file **files;
177};
178
Jens Axboe2b188cc2019-01-07 10:46:33 -0700179struct io_ring_ctx {
180 struct {
181 struct percpu_ref refs;
182 } ____cacheline_aligned_in_smp;
183
184 struct {
185 unsigned int flags;
186 bool compat;
187 bool account_mem;
Jens Axboe1d7bb1d2019-11-06 11:31:17 -0700188 bool cq_overflow_flushed;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700189
Hristo Venev75b28af2019-08-26 17:23:46 +0000190 /*
191 * Ring buffer of indices into array of io_uring_sqe, which is
192 * mmapped by the application using the IORING_OFF_SQES offset.
193 *
194 * This indirection could e.g. be used to assign fixed
195 * io_uring_sqe entries to operations and only submit them to
196 * the queue when needed.
197 *
198 * The kernel modifies neither the indices array nor the entries
199 * array.
200 */
201 u32 *sq_array;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700202 unsigned cached_sq_head;
203 unsigned sq_entries;
204 unsigned sq_mask;
Jens Axboe6c271ce2019-01-10 11:22:30 -0700205 unsigned sq_thread_idle;
Jens Axboe498ccd92019-10-25 10:04:25 -0600206 unsigned cached_sq_dropped;
Jens Axboe206aefd2019-11-07 18:27:42 -0700207 atomic_t cached_cq_overflow;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700208 struct io_uring_sqe *sq_sqes;
Jens Axboede0617e2019-04-06 21:51:27 -0600209
210 struct list_head defer_list;
Jens Axboe5262f562019-09-17 12:26:57 -0600211 struct list_head timeout_list;
Jens Axboe1d7bb1d2019-11-06 11:31:17 -0700212 struct list_head cq_overflow_list;
Jens Axboefcb323c2019-10-24 12:39:47 -0600213
214 wait_queue_head_t inflight_wait;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700215 } ____cacheline_aligned_in_smp;
216
Jens Axboe206aefd2019-11-07 18:27:42 -0700217 struct io_rings *rings;
218
Jens Axboe2b188cc2019-01-07 10:46:33 -0700219 /* IO offload */
Jens Axboe561fb042019-10-24 07:25:42 -0600220 struct io_wq *io_wq;
Jens Axboe6c271ce2019-01-10 11:22:30 -0700221 struct task_struct *sqo_thread; /* if using sq thread polling */
Jens Axboe2b188cc2019-01-07 10:46:33 -0700222 struct mm_struct *sqo_mm;
Jens Axboe6c271ce2019-01-10 11:22:30 -0700223 wait_queue_head_t sqo_wait;
Hristo Venev75b28af2019-08-26 17:23:46 +0000224
Jens Axboe6b063142019-01-10 22:13:58 -0700225 /*
226 * If used, fixed file set. Writers must ensure that ->refs is dead,
227 * readers must ensure that ->refs is alive as long as the file* is
228 * used. Only updated through io_uring_register(2).
229 */
Jens Axboe65e19f52019-10-26 07:20:21 -0600230 struct fixed_file_table *file_table;
Jens Axboe6b063142019-01-10 22:13:58 -0700231 unsigned nr_user_files;
232
Jens Axboeedafcce2019-01-09 09:16:05 -0700233 /* if used, fixed mapped user buffers */
234 unsigned nr_user_bufs;
235 struct io_mapped_ubuf *user_bufs;
236
Jens Axboe2b188cc2019-01-07 10:46:33 -0700237 struct user_struct *user;
238
Jens Axboe206aefd2019-11-07 18:27:42 -0700239 /* 0 is for ctx quiesce/reinit/free, 1 is for sqo_thread started */
240 struct completion *completions;
241
242#if defined(CONFIG_UNIX)
243 struct socket *ring_sock;
244#endif
245
246 struct {
247 unsigned cached_cq_tail;
248 unsigned cq_entries;
249 unsigned cq_mask;
250 atomic_t cq_timeouts;
251 struct wait_queue_head cq_wait;
252 struct fasync_struct *cq_fasync;
253 struct eventfd_ctx *cq_ev_fd;
254 } ____cacheline_aligned_in_smp;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700255
256 struct {
257 struct mutex uring_lock;
258 wait_queue_head_t wait;
259 } ____cacheline_aligned_in_smp;
260
261 struct {
262 spinlock_t completion_lock;
Jens Axboedef596e2019-01-09 08:59:42 -0700263 bool poll_multi_file;
264 /*
265 * ->poll_list is protected by the ctx->uring_lock for
266 * io_uring instances that don't use IORING_SETUP_SQPOLL.
267 * For SQPOLL, only the single threaded io_sq_thread() will
268 * manipulate the list, hence no extra locking is needed there.
269 */
270 struct list_head poll_list;
Jens Axboe221c5eb2019-01-17 09:41:58 -0700271 struct list_head cancel_list;
Jens Axboefcb323c2019-10-24 12:39:47 -0600272
273 spinlock_t inflight_lock;
274 struct list_head inflight_list;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700275 } ____cacheline_aligned_in_smp;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700276};
277
278struct sqe_submit {
279 const struct io_uring_sqe *sqe;
Jens Axboefcb323c2019-10-24 12:39:47 -0600280 struct file *ring_file;
281 int ring_fd;
Jackie Liu8776f3f2019-09-09 20:50:39 +0800282 u32 sequence;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700283 bool has_user;
Jackie Liuba5290c2019-10-09 09:19:59 +0800284 bool in_async;
Jens Axboe6c271ce2019-01-10 11:22:30 -0700285 bool needs_fixed_file;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700286};
287
Jens Axboe09bb8392019-03-13 12:39:28 -0600288/*
289 * First field must be the file pointer in all the
290 * iocb unions! See also 'struct kiocb' in <linux/fs.h>
291 */
Jens Axboe221c5eb2019-01-17 09:41:58 -0700292struct io_poll_iocb {
293 struct file *file;
294 struct wait_queue_head *head;
295 __poll_t events;
Jens Axboe8c838782019-03-12 15:48:16 -0600296 bool done;
Jens Axboe221c5eb2019-01-17 09:41:58 -0700297 bool canceled;
298 struct wait_queue_entry wait;
299};
300
Jens Axboe5262f562019-09-17 12:26:57 -0600301struct io_timeout {
302 struct file *file;
303 struct hrtimer timer;
304};
305
Jens Axboe09bb8392019-03-13 12:39:28 -0600306/*
307 * NOTE! Each of the iocb union members has the file pointer
308 * as the first entry in their struct definition. So you can
309 * access the file pointer through any of the sub-structs,
310 * or directly as just 'ki_filp' in this struct.
311 */
Jens Axboe2b188cc2019-01-07 10:46:33 -0700312struct io_kiocb {
Jens Axboe221c5eb2019-01-17 09:41:58 -0700313 union {
Jens Axboe09bb8392019-03-13 12:39:28 -0600314 struct file *file;
Jens Axboe221c5eb2019-01-17 09:41:58 -0700315 struct kiocb rw;
316 struct io_poll_iocb poll;
Jens Axboe5262f562019-09-17 12:26:57 -0600317 struct io_timeout timeout;
Jens Axboe221c5eb2019-01-17 09:41:58 -0700318 };
Jens Axboe2b188cc2019-01-07 10:46:33 -0700319
320 struct sqe_submit submit;
321
322 struct io_ring_ctx *ctx;
323 struct list_head list;
Jens Axboe9e645e112019-05-10 16:07:28 -0600324 struct list_head link_list;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700325 unsigned int flags;
Jens Axboec16361c2019-01-17 08:39:48 -0700326 refcount_t refs;
Stefan Bühler8449eed2019-04-27 20:34:19 +0200327#define REQ_F_NOWAIT 1 /* must not punt to workers */
Jens Axboedef596e2019-01-09 08:59:42 -0700328#define REQ_F_IOPOLL_COMPLETED 2 /* polled IO has completed */
Jens Axboe6b063142019-01-10 22:13:58 -0700329#define REQ_F_FIXED_FILE 4 /* ctx owns file */
Jens Axboe31b51512019-01-18 22:56:34 -0700330#define REQ_F_SEQ_PREV 8 /* sequential with previous */
Stefan Bühlere2033e32019-05-11 19:08:01 +0200331#define REQ_F_IO_DRAIN 16 /* drain existing IO first */
332#define REQ_F_IO_DRAINED 32 /* drain done */
Jens Axboe9e645e112019-05-10 16:07:28 -0600333#define REQ_F_LINK 64 /* linked sqes */
Jens Axboe2665abf2019-11-05 12:40:47 -0700334#define REQ_F_LINK_TIMEOUT 128 /* has linked timeout */
Zhengyuan Liuf7b76ac2019-07-16 23:26:14 +0800335#define REQ_F_FAIL_LINK 256 /* fail rest of links */
Jackie Liu4fe2c962019-09-09 20:50:40 +0800336#define REQ_F_SHADOW_DRAIN 512 /* link-drain shadow req */
Jens Axboe5262f562019-09-17 12:26:57 -0600337#define REQ_F_TIMEOUT 1024 /* timeout request */
Jens Axboe491381ce2019-10-17 09:20:46 -0600338#define REQ_F_ISREG 2048 /* regular file */
339#define REQ_F_MUST_PUNT 4096 /* must be punted even for NONBLOCK */
Jens Axboefcb323c2019-10-24 12:39:47 -0600340#define REQ_F_INFLIGHT 8192 /* on inflight list */
Jens Axboe2b188cc2019-01-07 10:46:33 -0700341 u64 user_data;
Jens Axboe9e645e112019-05-10 16:07:28 -0600342 u32 result;
Jens Axboede0617e2019-04-06 21:51:27 -0600343 u32 sequence;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700344
Jens Axboefcb323c2019-10-24 12:39:47 -0600345 struct list_head inflight_entry;
346
Jens Axboe561fb042019-10-24 07:25:42 -0600347 struct io_wq_work work;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700348};
349
350#define IO_PLUG_THRESHOLD 2
Jens Axboedef596e2019-01-09 08:59:42 -0700351#define IO_IOPOLL_BATCH 8
Jens Axboe2b188cc2019-01-07 10:46:33 -0700352
Jens Axboe9a56a232019-01-09 09:06:50 -0700353struct io_submit_state {
354 struct blk_plug plug;
355
356 /*
Jens Axboe2579f912019-01-09 09:10:43 -0700357 * io_kiocb alloc cache
358 */
359 void *reqs[IO_IOPOLL_BATCH];
360 unsigned int free_reqs;
361 unsigned int cur_req;
362
363 /*
Jens Axboe9a56a232019-01-09 09:06:50 -0700364 * File reference cache
365 */
366 struct file *file;
367 unsigned int fd;
368 unsigned int has_refs;
369 unsigned int used_refs;
370 unsigned int ios_left;
371};
372
Jens Axboe561fb042019-10-24 07:25:42 -0600373static void io_wq_submit_work(struct io_wq_work **workptr);
Jens Axboe78e19bb2019-11-06 15:21:34 -0700374static void io_cqring_fill_event(struct io_kiocb *req, long res);
Jackie Liu4fe2c962019-09-09 20:50:40 +0800375static void __io_free_req(struct io_kiocb *req);
Jens Axboe2665abf2019-11-05 12:40:47 -0700376static void io_put_req(struct io_kiocb *req, struct io_kiocb **nxtptr);
Jens Axboe78e19bb2019-11-06 15:21:34 -0700377static void io_double_put_req(struct io_kiocb *req);
Jens Axboede0617e2019-04-06 21:51:27 -0600378
Jens Axboe2b188cc2019-01-07 10:46:33 -0700379static struct kmem_cache *req_cachep;
380
381static const struct file_operations io_uring_fops;
382
383struct sock *io_uring_get_socket(struct file *file)
384{
385#if defined(CONFIG_UNIX)
386 if (file->f_op == &io_uring_fops) {
387 struct io_ring_ctx *ctx = file->private_data;
388
389 return ctx->ring_sock->sk;
390 }
391#endif
392 return NULL;
393}
394EXPORT_SYMBOL(io_uring_get_socket);
395
396static void io_ring_ctx_ref_free(struct percpu_ref *ref)
397{
398 struct io_ring_ctx *ctx = container_of(ref, struct io_ring_ctx, refs);
399
Jens Axboe206aefd2019-11-07 18:27:42 -0700400 complete(&ctx->completions[0]);
Jens Axboe2b188cc2019-01-07 10:46:33 -0700401}
402
403static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
404{
405 struct io_ring_ctx *ctx;
406
407 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
408 if (!ctx)
409 return NULL;
410
Jens Axboe206aefd2019-11-07 18:27:42 -0700411 ctx->completions = kmalloc(2 * sizeof(struct completion), GFP_KERNEL);
412 if (!ctx->completions)
413 goto err;
414
Roman Gushchin21482892019-05-07 10:01:48 -0700415 if (percpu_ref_init(&ctx->refs, io_ring_ctx_ref_free,
Jens Axboe206aefd2019-11-07 18:27:42 -0700416 PERCPU_REF_ALLOW_REINIT, GFP_KERNEL))
417 goto err;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700418
419 ctx->flags = p->flags;
420 init_waitqueue_head(&ctx->cq_wait);
Jens Axboe1d7bb1d2019-11-06 11:31:17 -0700421 INIT_LIST_HEAD(&ctx->cq_overflow_list);
Jens Axboe206aefd2019-11-07 18:27:42 -0700422 init_completion(&ctx->completions[0]);
423 init_completion(&ctx->completions[1]);
Jens Axboe2b188cc2019-01-07 10:46:33 -0700424 mutex_init(&ctx->uring_lock);
425 init_waitqueue_head(&ctx->wait);
426 spin_lock_init(&ctx->completion_lock);
Jens Axboedef596e2019-01-09 08:59:42 -0700427 INIT_LIST_HEAD(&ctx->poll_list);
Jens Axboe221c5eb2019-01-17 09:41:58 -0700428 INIT_LIST_HEAD(&ctx->cancel_list);
Jens Axboede0617e2019-04-06 21:51:27 -0600429 INIT_LIST_HEAD(&ctx->defer_list);
Jens Axboe5262f562019-09-17 12:26:57 -0600430 INIT_LIST_HEAD(&ctx->timeout_list);
Jens Axboefcb323c2019-10-24 12:39:47 -0600431 init_waitqueue_head(&ctx->inflight_wait);
432 spin_lock_init(&ctx->inflight_lock);
433 INIT_LIST_HEAD(&ctx->inflight_list);
Jens Axboe2b188cc2019-01-07 10:46:33 -0700434 return ctx;
Jens Axboe206aefd2019-11-07 18:27:42 -0700435err:
436 kfree(ctx->completions);
437 kfree(ctx);
438 return NULL;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700439}
440
Jackie Liua197f662019-11-08 08:09:12 -0700441static inline bool __io_sequence_defer(struct io_kiocb *req)
Jens Axboede0617e2019-04-06 21:51:27 -0600442{
Jackie Liua197f662019-11-08 08:09:12 -0700443 struct io_ring_ctx *ctx = req->ctx;
444
Jens Axboe498ccd92019-10-25 10:04:25 -0600445 return req->sequence != ctx->cached_cq_tail + ctx->cached_sq_dropped
446 + atomic_read(&ctx->cached_cq_overflow);
Jens Axboede0617e2019-04-06 21:51:27 -0600447}
448
Jackie Liua197f662019-11-08 08:09:12 -0700449static inline bool io_sequence_defer(struct io_kiocb *req)
Jens Axboe7adf4ea2019-10-10 21:42:58 -0600450{
451 if ((req->flags & (REQ_F_IO_DRAIN|REQ_F_IO_DRAINED)) != REQ_F_IO_DRAIN)
452 return false;
453
Jackie Liua197f662019-11-08 08:09:12 -0700454 return __io_sequence_defer(req);
Jens Axboe7adf4ea2019-10-10 21:42:58 -0600455}
456
457static struct io_kiocb *io_get_deferred_req(struct io_ring_ctx *ctx)
Jens Axboede0617e2019-04-06 21:51:27 -0600458{
459 struct io_kiocb *req;
460
Jens Axboe7adf4ea2019-10-10 21:42:58 -0600461 req = list_first_entry_or_null(&ctx->defer_list, struct io_kiocb, list);
Jackie Liua197f662019-11-08 08:09:12 -0700462 if (req && !io_sequence_defer(req)) {
Jens Axboede0617e2019-04-06 21:51:27 -0600463 list_del_init(&req->list);
464 return req;
465 }
466
467 return NULL;
468}
469
Jens Axboe5262f562019-09-17 12:26:57 -0600470static struct io_kiocb *io_get_timeout_req(struct io_ring_ctx *ctx)
471{
Jens Axboe7adf4ea2019-10-10 21:42:58 -0600472 struct io_kiocb *req;
473
474 req = list_first_entry_or_null(&ctx->timeout_list, struct io_kiocb, list);
Jackie Liua197f662019-11-08 08:09:12 -0700475 if (req && !__io_sequence_defer(req)) {
Jens Axboe7adf4ea2019-10-10 21:42:58 -0600476 list_del_init(&req->list);
477 return req;
478 }
479
480 return NULL;
Jens Axboe5262f562019-09-17 12:26:57 -0600481}
482
Jens Axboede0617e2019-04-06 21:51:27 -0600483static void __io_commit_cqring(struct io_ring_ctx *ctx)
Jens Axboe2b188cc2019-01-07 10:46:33 -0700484{
Hristo Venev75b28af2019-08-26 17:23:46 +0000485 struct io_rings *rings = ctx->rings;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700486
Hristo Venev75b28af2019-08-26 17:23:46 +0000487 if (ctx->cached_cq_tail != READ_ONCE(rings->cq.tail)) {
Jens Axboe2b188cc2019-01-07 10:46:33 -0700488 /* order cqe stores with ring update */
Hristo Venev75b28af2019-08-26 17:23:46 +0000489 smp_store_release(&rings->cq.tail, ctx->cached_cq_tail);
Jens Axboe2b188cc2019-01-07 10:46:33 -0700490
Jens Axboe2b188cc2019-01-07 10:46:33 -0700491 if (wq_has_sleeper(&ctx->cq_wait)) {
492 wake_up_interruptible(&ctx->cq_wait);
493 kill_fasync(&ctx->cq_fasync, SIGIO, POLL_IN);
494 }
495 }
496}
497
Jens Axboe561fb042019-10-24 07:25:42 -0600498static inline bool io_sqe_needs_user(const struct io_uring_sqe *sqe)
Jens Axboe18d9be12019-09-10 09:13:05 -0600499{
Jens Axboe561fb042019-10-24 07:25:42 -0600500 u8 opcode = READ_ONCE(sqe->opcode);
501
502 return !(opcode == IORING_OP_READ_FIXED ||
503 opcode == IORING_OP_WRITE_FIXED);
504}
505
506static inline bool io_prep_async_work(struct io_kiocb *req)
507{
508 bool do_hashed = false;
Jens Axboe54a91f32019-09-10 09:15:04 -0600509
Jens Axboe6cc47d12019-09-18 11:18:23 -0600510 if (req->submit.sqe) {
511 switch (req->submit.sqe->opcode) {
512 case IORING_OP_WRITEV:
513 case IORING_OP_WRITE_FIXED:
Jens Axboe561fb042019-10-24 07:25:42 -0600514 do_hashed = true;
Jens Axboe5f8fd2d32019-11-07 10:57:36 -0700515 /* fall-through */
516 case IORING_OP_READV:
517 case IORING_OP_READ_FIXED:
518 case IORING_OP_SENDMSG:
519 case IORING_OP_RECVMSG:
520 case IORING_OP_ACCEPT:
521 case IORING_OP_POLL_ADD:
522 /*
523 * We know REQ_F_ISREG is not set on some of these
524 * opcodes, but this enables us to keep the check in
525 * just one place.
526 */
527 if (!(req->flags & REQ_F_ISREG))
528 req->work.flags |= IO_WQ_WORK_UNBOUND;
Jens Axboe6cc47d12019-09-18 11:18:23 -0600529 break;
530 }
Jens Axboe561fb042019-10-24 07:25:42 -0600531 if (io_sqe_needs_user(req->submit.sqe))
532 req->work.flags |= IO_WQ_WORK_NEEDS_USER;
Jens Axboe54a91f32019-09-10 09:15:04 -0600533 }
534
Jens Axboe561fb042019-10-24 07:25:42 -0600535 return do_hashed;
536}
537
Jackie Liua197f662019-11-08 08:09:12 -0700538static inline void io_queue_async_work(struct io_kiocb *req)
Jens Axboe561fb042019-10-24 07:25:42 -0600539{
540 bool do_hashed = io_prep_async_work(req);
Jackie Liua197f662019-11-08 08:09:12 -0700541 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe561fb042019-10-24 07:25:42 -0600542
543 trace_io_uring_queue_async_work(ctx, do_hashed, req, &req->work,
544 req->flags);
545 if (!do_hashed) {
546 io_wq_enqueue(ctx->io_wq, &req->work);
547 } else {
548 io_wq_enqueue_hashed(ctx->io_wq, &req->work,
549 file_inode(req->file));
550 }
Jens Axboe18d9be12019-09-10 09:13:05 -0600551}
552
Jens Axboe5262f562019-09-17 12:26:57 -0600553static void io_kill_timeout(struct io_kiocb *req)
554{
555 int ret;
556
557 ret = hrtimer_try_to_cancel(&req->timeout.timer);
558 if (ret != -1) {
559 atomic_inc(&req->ctx->cq_timeouts);
Jens Axboe842f9612019-10-29 12:34:10 -0600560 list_del_init(&req->list);
Jens Axboe78e19bb2019-11-06 15:21:34 -0700561 io_cqring_fill_event(req, 0);
562 io_put_req(req, NULL);
Jens Axboe5262f562019-09-17 12:26:57 -0600563 }
564}
565
566static void io_kill_timeouts(struct io_ring_ctx *ctx)
567{
568 struct io_kiocb *req, *tmp;
569
570 spin_lock_irq(&ctx->completion_lock);
571 list_for_each_entry_safe(req, tmp, &ctx->timeout_list, list)
572 io_kill_timeout(req);
573 spin_unlock_irq(&ctx->completion_lock);
574}
575
Jens Axboede0617e2019-04-06 21:51:27 -0600576static void io_commit_cqring(struct io_ring_ctx *ctx)
577{
578 struct io_kiocb *req;
579
Jens Axboe5262f562019-09-17 12:26:57 -0600580 while ((req = io_get_timeout_req(ctx)) != NULL)
581 io_kill_timeout(req);
582
Jens Axboede0617e2019-04-06 21:51:27 -0600583 __io_commit_cqring(ctx);
584
585 while ((req = io_get_deferred_req(ctx)) != NULL) {
Jackie Liu4fe2c962019-09-09 20:50:40 +0800586 if (req->flags & REQ_F_SHADOW_DRAIN) {
587 /* Just for drain, free it. */
588 __io_free_req(req);
589 continue;
590 }
Jens Axboede0617e2019-04-06 21:51:27 -0600591 req->flags |= REQ_F_IO_DRAINED;
Jackie Liua197f662019-11-08 08:09:12 -0700592 io_queue_async_work(req);
Jens Axboede0617e2019-04-06 21:51:27 -0600593 }
594}
595
Jens Axboe2b188cc2019-01-07 10:46:33 -0700596static struct io_uring_cqe *io_get_cqring(struct io_ring_ctx *ctx)
597{
Hristo Venev75b28af2019-08-26 17:23:46 +0000598 struct io_rings *rings = ctx->rings;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700599 unsigned tail;
600
601 tail = ctx->cached_cq_tail;
Stefan Bühler115e12e2019-04-24 23:54:18 +0200602 /*
603 * writes to the cq entry need to come after reading head; the
604 * control dependency is enough as we're using WRITE_ONCE to
605 * fill the cq entry
606 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000607 if (tail - READ_ONCE(rings->cq.head) == rings->cq_ring_entries)
Jens Axboe2b188cc2019-01-07 10:46:33 -0700608 return NULL;
609
610 ctx->cached_cq_tail++;
Hristo Venev75b28af2019-08-26 17:23:46 +0000611 return &rings->cqes[tail & ctx->cq_mask];
Jens Axboe2b188cc2019-01-07 10:46:33 -0700612}
613
Jens Axboe1d7bb1d2019-11-06 11:31:17 -0700614static void io_cqring_ev_posted(struct io_ring_ctx *ctx)
615{
616 if (waitqueue_active(&ctx->wait))
617 wake_up(&ctx->wait);
618 if (waitqueue_active(&ctx->sqo_wait))
619 wake_up(&ctx->sqo_wait);
620 if (ctx->cq_ev_fd)
621 eventfd_signal(ctx->cq_ev_fd, 1);
622}
623
624static void io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force)
625{
626 struct io_rings *rings = ctx->rings;
627 struct io_uring_cqe *cqe;
628 struct io_kiocb *req;
629 unsigned long flags;
630 LIST_HEAD(list);
631
632 if (!force) {
633 if (list_empty_careful(&ctx->cq_overflow_list))
634 return;
635 if ((ctx->cached_cq_tail - READ_ONCE(rings->cq.head) ==
636 rings->cq_ring_entries))
637 return;
638 }
639
640 spin_lock_irqsave(&ctx->completion_lock, flags);
641
642 /* if force is set, the ring is going away. always drop after that */
643 if (force)
644 ctx->cq_overflow_flushed = true;
645
646 while (!list_empty(&ctx->cq_overflow_list)) {
647 cqe = io_get_cqring(ctx);
648 if (!cqe && !force)
649 break;
650
651 req = list_first_entry(&ctx->cq_overflow_list, struct io_kiocb,
652 list);
653 list_move(&req->list, &list);
654 if (cqe) {
655 WRITE_ONCE(cqe->user_data, req->user_data);
656 WRITE_ONCE(cqe->res, req->result);
657 WRITE_ONCE(cqe->flags, 0);
658 } else {
659 WRITE_ONCE(ctx->rings->cq_overflow,
660 atomic_inc_return(&ctx->cached_cq_overflow));
661 }
662 }
663
664 io_commit_cqring(ctx);
665 spin_unlock_irqrestore(&ctx->completion_lock, flags);
666 io_cqring_ev_posted(ctx);
667
668 while (!list_empty(&list)) {
669 req = list_first_entry(&list, struct io_kiocb, list);
670 list_del(&req->list);
671 io_put_req(req, NULL);
672 }
673}
674
Jens Axboe78e19bb2019-11-06 15:21:34 -0700675static void io_cqring_fill_event(struct io_kiocb *req, long res)
Jens Axboe2b188cc2019-01-07 10:46:33 -0700676{
Jens Axboe78e19bb2019-11-06 15:21:34 -0700677 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700678 struct io_uring_cqe *cqe;
679
Jens Axboe78e19bb2019-11-06 15:21:34 -0700680 trace_io_uring_complete(ctx, req->user_data, res);
Jens Axboe51c3ff62019-11-03 06:52:50 -0700681
Jens Axboe2b188cc2019-01-07 10:46:33 -0700682 /*
683 * If we can't get a cq entry, userspace overflowed the
684 * submission (by quite a lot). Increment the overflow count in
685 * the ring.
686 */
687 cqe = io_get_cqring(ctx);
Jens Axboe1d7bb1d2019-11-06 11:31:17 -0700688 if (likely(cqe)) {
Jens Axboe78e19bb2019-11-06 15:21:34 -0700689 WRITE_ONCE(cqe->user_data, req->user_data);
Jens Axboe2b188cc2019-01-07 10:46:33 -0700690 WRITE_ONCE(cqe->res, res);
Jens Axboec71ffb62019-05-13 20:58:29 -0600691 WRITE_ONCE(cqe->flags, 0);
Jens Axboe1d7bb1d2019-11-06 11:31:17 -0700692 } else if (ctx->cq_overflow_flushed) {
Jens Axboe498ccd92019-10-25 10:04:25 -0600693 WRITE_ONCE(ctx->rings->cq_overflow,
694 atomic_inc_return(&ctx->cached_cq_overflow));
Jens Axboe1d7bb1d2019-11-06 11:31:17 -0700695 } else {
696 refcount_inc(&req->refs);
697 req->result = res;
698 list_add_tail(&req->list, &ctx->cq_overflow_list);
Jens Axboe2b188cc2019-01-07 10:46:33 -0700699 }
700}
701
Jens Axboe78e19bb2019-11-06 15:21:34 -0700702static void io_cqring_add_event(struct io_kiocb *req, long res)
Jens Axboe2b188cc2019-01-07 10:46:33 -0700703{
Jens Axboe78e19bb2019-11-06 15:21:34 -0700704 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700705 unsigned long flags;
706
707 spin_lock_irqsave(&ctx->completion_lock, flags);
Jens Axboe78e19bb2019-11-06 15:21:34 -0700708 io_cqring_fill_event(req, res);
Jens Axboe2b188cc2019-01-07 10:46:33 -0700709 io_commit_cqring(ctx);
710 spin_unlock_irqrestore(&ctx->completion_lock, flags);
711
Jens Axboe8c838782019-03-12 15:48:16 -0600712 io_cqring_ev_posted(ctx);
Jens Axboe2b188cc2019-01-07 10:46:33 -0700713}
714
Jens Axboe2579f912019-01-09 09:10:43 -0700715static struct io_kiocb *io_get_req(struct io_ring_ctx *ctx,
716 struct io_submit_state *state)
Jens Axboe2b188cc2019-01-07 10:46:33 -0700717{
Jens Axboefd6fab22019-03-14 16:30:06 -0600718 gfp_t gfp = GFP_KERNEL | __GFP_NOWARN;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700719 struct io_kiocb *req;
720
721 if (!percpu_ref_tryget(&ctx->refs))
722 return NULL;
723
Jens Axboe2579f912019-01-09 09:10:43 -0700724 if (!state) {
Jens Axboefd6fab22019-03-14 16:30:06 -0600725 req = kmem_cache_alloc(req_cachep, gfp);
Jens Axboe2579f912019-01-09 09:10:43 -0700726 if (unlikely(!req))
727 goto out;
728 } else if (!state->free_reqs) {
729 size_t sz;
730 int ret;
731
732 sz = min_t(size_t, state->ios_left, ARRAY_SIZE(state->reqs));
Jens Axboefd6fab22019-03-14 16:30:06 -0600733 ret = kmem_cache_alloc_bulk(req_cachep, gfp, sz, state->reqs);
734
735 /*
736 * Bulk alloc is all-or-nothing. If we fail to get a batch,
737 * retry single alloc to be on the safe side.
738 */
739 if (unlikely(ret <= 0)) {
740 state->reqs[0] = kmem_cache_alloc(req_cachep, gfp);
741 if (!state->reqs[0])
742 goto out;
743 ret = 1;
744 }
Jens Axboe2579f912019-01-09 09:10:43 -0700745 state->free_reqs = ret - 1;
746 state->cur_req = 1;
747 req = state->reqs[0];
748 } else {
749 req = state->reqs[state->cur_req];
750 state->free_reqs--;
751 state->cur_req++;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700752 }
753
Jens Axboe60c112b2019-06-21 10:20:18 -0600754 req->file = NULL;
Jens Axboe2579f912019-01-09 09:10:43 -0700755 req->ctx = ctx;
756 req->flags = 0;
Jens Axboee65ef562019-03-12 10:16:44 -0600757 /* one is dropped after submission, the other at completion */
758 refcount_set(&req->refs, 2);
Jens Axboe9e645e112019-05-10 16:07:28 -0600759 req->result = 0;
Jens Axboe561fb042019-10-24 07:25:42 -0600760 INIT_IO_WORK(&req->work, io_wq_submit_work);
Jens Axboe2579f912019-01-09 09:10:43 -0700761 return req;
762out:
Pavel Begunkov6805b322019-10-08 02:18:42 +0300763 percpu_ref_put(&ctx->refs);
Jens Axboe2b188cc2019-01-07 10:46:33 -0700764 return NULL;
765}
766
Jens Axboedef596e2019-01-09 08:59:42 -0700767static void io_free_req_many(struct io_ring_ctx *ctx, void **reqs, int *nr)
768{
769 if (*nr) {
770 kmem_cache_free_bulk(req_cachep, *nr, reqs);
Pavel Begunkov6805b322019-10-08 02:18:42 +0300771 percpu_ref_put_many(&ctx->refs, *nr);
Jens Axboedef596e2019-01-09 08:59:42 -0700772 *nr = 0;
773 }
774}
775
Jens Axboe9e645e112019-05-10 16:07:28 -0600776static void __io_free_req(struct io_kiocb *req)
Jens Axboe2b188cc2019-01-07 10:46:33 -0700777{
Jens Axboefcb323c2019-10-24 12:39:47 -0600778 struct io_ring_ctx *ctx = req->ctx;
779
Jens Axboe09bb8392019-03-13 12:39:28 -0600780 if (req->file && !(req->flags & REQ_F_FIXED_FILE))
781 fput(req->file);
Jens Axboefcb323c2019-10-24 12:39:47 -0600782 if (req->flags & REQ_F_INFLIGHT) {
783 unsigned long flags;
784
785 spin_lock_irqsave(&ctx->inflight_lock, flags);
786 list_del(&req->inflight_entry);
787 if (waitqueue_active(&ctx->inflight_wait))
788 wake_up(&ctx->inflight_wait);
789 spin_unlock_irqrestore(&ctx->inflight_lock, flags);
790 }
791 percpu_ref_put(&ctx->refs);
Jens Axboee65ef562019-03-12 10:16:44 -0600792 kmem_cache_free(req_cachep, req);
793}
794
Jackie Liua197f662019-11-08 08:09:12 -0700795static bool io_link_cancel_timeout(struct io_kiocb *req)
Jens Axboe2665abf2019-11-05 12:40:47 -0700796{
Jackie Liua197f662019-11-08 08:09:12 -0700797 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe2665abf2019-11-05 12:40:47 -0700798 int ret;
799
800 ret = hrtimer_try_to_cancel(&req->timeout.timer);
801 if (ret != -1) {
Jens Axboe78e19bb2019-11-06 15:21:34 -0700802 io_cqring_fill_event(req, -ECANCELED);
Jens Axboe2665abf2019-11-05 12:40:47 -0700803 io_commit_cqring(ctx);
804 req->flags &= ~REQ_F_LINK;
Jens Axboe78e19bb2019-11-06 15:21:34 -0700805 io_put_req(req, NULL);
Jens Axboe2665abf2019-11-05 12:40:47 -0700806 return true;
807 }
808
809 return false;
810}
811
Jens Axboeba816ad2019-09-28 11:36:45 -0600812static void io_req_link_next(struct io_kiocb *req, struct io_kiocb **nxtptr)
Jens Axboe9e645e112019-05-10 16:07:28 -0600813{
Jens Axboe2665abf2019-11-05 12:40:47 -0700814 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe9e645e112019-05-10 16:07:28 -0600815 struct io_kiocb *nxt;
Jens Axboe2665abf2019-11-05 12:40:47 -0700816 bool wake_ev = false;
Jens Axboe9e645e112019-05-10 16:07:28 -0600817
818 /*
819 * The list should never be empty when we are called here. But could
820 * potentially happen if the chain is messed up, check to be on the
821 * safe side.
822 */
823 nxt = list_first_entry_or_null(&req->link_list, struct io_kiocb, list);
Jens Axboe2665abf2019-11-05 12:40:47 -0700824 while (nxt) {
Jens Axboe9e645e112019-05-10 16:07:28 -0600825 list_del(&nxt->list);
826 if (!list_empty(&req->link_list)) {
827 INIT_LIST_HEAD(&nxt->link_list);
828 list_splice(&req->link_list, &nxt->link_list);
829 nxt->flags |= REQ_F_LINK;
830 }
831
Jens Axboeba816ad2019-09-28 11:36:45 -0600832 /*
833 * If we're in async work, we can continue processing the chain
834 * in this context instead of having to queue up new async work.
835 */
Jens Axboe2665abf2019-11-05 12:40:47 -0700836 if (req->flags & REQ_F_LINK_TIMEOUT) {
Jackie Liua197f662019-11-08 08:09:12 -0700837 wake_ev = io_link_cancel_timeout(nxt);
Jens Axboe2665abf2019-11-05 12:40:47 -0700838
839 /* we dropped this link, get next */
840 nxt = list_first_entry_or_null(&req->link_list,
841 struct io_kiocb, list);
842 } else if (nxtptr && current_work()) {
Jens Axboeba816ad2019-09-28 11:36:45 -0600843 *nxtptr = nxt;
Jens Axboe2665abf2019-11-05 12:40:47 -0700844 break;
845 } else {
Jackie Liua197f662019-11-08 08:09:12 -0700846 io_queue_async_work(nxt);
Jens Axboe2665abf2019-11-05 12:40:47 -0700847 break;
848 }
Jens Axboe9e645e112019-05-10 16:07:28 -0600849 }
Jens Axboe2665abf2019-11-05 12:40:47 -0700850
851 if (wake_ev)
852 io_cqring_ev_posted(ctx);
Jens Axboe9e645e112019-05-10 16:07:28 -0600853}
854
855/*
856 * Called if REQ_F_LINK is set, and we fail the head request
857 */
858static void io_fail_links(struct io_kiocb *req)
859{
Jens Axboe2665abf2019-11-05 12:40:47 -0700860 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe9e645e112019-05-10 16:07:28 -0600861 struct io_kiocb *link;
Jens Axboe2665abf2019-11-05 12:40:47 -0700862 unsigned long flags;
863
864 spin_lock_irqsave(&ctx->completion_lock, flags);
Jens Axboe9e645e112019-05-10 16:07:28 -0600865
866 while (!list_empty(&req->link_list)) {
867 link = list_first_entry(&req->link_list, struct io_kiocb, list);
Jens Axboe2665abf2019-11-05 12:40:47 -0700868 list_del_init(&link->list);
Jens Axboe9e645e112019-05-10 16:07:28 -0600869
Dmitrii Dolgovc826bd72019-10-15 19:02:01 +0200870 trace_io_uring_fail_link(req, link);
Jens Axboe2665abf2019-11-05 12:40:47 -0700871
872 if ((req->flags & REQ_F_LINK_TIMEOUT) &&
873 link->submit.sqe->opcode == IORING_OP_LINK_TIMEOUT) {
Jackie Liua197f662019-11-08 08:09:12 -0700874 io_link_cancel_timeout(link);
Jens Axboe2665abf2019-11-05 12:40:47 -0700875 } else {
Jens Axboe78e19bb2019-11-06 15:21:34 -0700876 io_cqring_fill_event(link, -ECANCELED);
877 io_double_put_req(link);
Jens Axboe2665abf2019-11-05 12:40:47 -0700878 }
Jens Axboe9e645e112019-05-10 16:07:28 -0600879 }
Jens Axboe2665abf2019-11-05 12:40:47 -0700880
881 io_commit_cqring(ctx);
882 spin_unlock_irqrestore(&ctx->completion_lock, flags);
883 io_cqring_ev_posted(ctx);
Jens Axboe9e645e112019-05-10 16:07:28 -0600884}
885
Jens Axboeba816ad2019-09-28 11:36:45 -0600886static void io_free_req(struct io_kiocb *req, struct io_kiocb **nxt)
Jens Axboe9e645e112019-05-10 16:07:28 -0600887{
Jens Axboe2665abf2019-11-05 12:40:47 -0700888 if (likely(!(req->flags & REQ_F_LINK))) {
889 __io_free_req(req);
890 return;
891 }
892
Jens Axboe9e645e112019-05-10 16:07:28 -0600893 /*
894 * If LINK is set, we have dependent requests in this chain. If we
895 * didn't fail this request, queue the first one up, moving any other
896 * dependencies to the next request. In case of failure, fail the rest
897 * of the chain.
898 */
Jens Axboe2665abf2019-11-05 12:40:47 -0700899 if (req->flags & REQ_F_FAIL_LINK) {
900 io_fail_links(req);
901 } else if (req->flags & REQ_F_LINK_TIMEOUT) {
902 struct io_ring_ctx *ctx = req->ctx;
903 unsigned long flags;
904
905 /*
906 * If this is a timeout link, we could be racing with the
907 * timeout timer. Grab the completion lock for this case to
908 * protection against that.
909 */
910 spin_lock_irqsave(&ctx->completion_lock, flags);
911 io_req_link_next(req, nxt);
912 spin_unlock_irqrestore(&ctx->completion_lock, flags);
913 } else {
914 io_req_link_next(req, nxt);
Jens Axboe9e645e112019-05-10 16:07:28 -0600915 }
916
917 __io_free_req(req);
918}
919
Jens Axboeba816ad2019-09-28 11:36:45 -0600920/*
921 * Drop reference to request, return next in chain (if there is one) if this
922 * was the last reference to this request.
923 */
924static struct io_kiocb *io_put_req_find_next(struct io_kiocb *req)
Jens Axboee65ef562019-03-12 10:16:44 -0600925{
Jens Axboeba816ad2019-09-28 11:36:45 -0600926 struct io_kiocb *nxt = NULL;
927
Jens Axboee65ef562019-03-12 10:16:44 -0600928 if (refcount_dec_and_test(&req->refs))
Jens Axboeba816ad2019-09-28 11:36:45 -0600929 io_free_req(req, &nxt);
930
931 return nxt;
932}
933
934static void io_put_req(struct io_kiocb *req, struct io_kiocb **nxtptr)
935{
936 struct io_kiocb *nxt;
937
938 nxt = io_put_req_find_next(req);
939 if (nxt) {
Jens Axboe561fb042019-10-24 07:25:42 -0600940 if (nxtptr)
Jens Axboeba816ad2019-09-28 11:36:45 -0600941 *nxtptr = nxt;
Jens Axboe561fb042019-10-24 07:25:42 -0600942 else
Jackie Liua197f662019-11-08 08:09:12 -0700943 io_queue_async_work(nxt);
Jens Axboeba816ad2019-09-28 11:36:45 -0600944 }
Jens Axboe2b188cc2019-01-07 10:46:33 -0700945}
946
Jens Axboe78e19bb2019-11-06 15:21:34 -0700947static void io_double_put_req(struct io_kiocb *req)
948{
949 /* drop both submit and complete references */
950 if (refcount_sub_and_test(2, &req->refs))
951 __io_free_req(req);
952}
953
Jens Axboe1d7bb1d2019-11-06 11:31:17 -0700954static unsigned io_cqring_events(struct io_ring_ctx *ctx, bool noflush)
Jens Axboea3a0e432019-08-20 11:03:11 -0600955{
Jens Axboe84f97dc2019-11-06 11:27:53 -0700956 struct io_rings *rings = ctx->rings;
957
Jens Axboe1d7bb1d2019-11-06 11:31:17 -0700958 /*
959 * noflush == true is from the waitqueue handler, just ensure we wake
960 * up the task, and the next invocation will flush the entries. We
961 * cannot safely to it from here.
962 */
963 if (noflush && !list_empty(&ctx->cq_overflow_list))
964 return -1U;
965
966 io_cqring_overflow_flush(ctx, false);
967
Jens Axboea3a0e432019-08-20 11:03:11 -0600968 /* See comment at the top of this file */
969 smp_rmb();
Hristo Venev75b28af2019-08-26 17:23:46 +0000970 return READ_ONCE(rings->cq.tail) - READ_ONCE(rings->cq.head);
Jens Axboea3a0e432019-08-20 11:03:11 -0600971}
972
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +0300973static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx)
974{
975 struct io_rings *rings = ctx->rings;
976
977 /* make sure SQ entry isn't read before tail */
978 return smp_load_acquire(&rings->sq.tail) - ctx->cached_sq_head;
979}
980
Jens Axboedef596e2019-01-09 08:59:42 -0700981/*
982 * Find and free completed poll iocbs
983 */
984static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events,
985 struct list_head *done)
986{
987 void *reqs[IO_IOPOLL_BATCH];
988 struct io_kiocb *req;
Jens Axboe09bb8392019-03-13 12:39:28 -0600989 int to_free;
Jens Axboedef596e2019-01-09 08:59:42 -0700990
Jens Axboe09bb8392019-03-13 12:39:28 -0600991 to_free = 0;
Jens Axboedef596e2019-01-09 08:59:42 -0700992 while (!list_empty(done)) {
993 req = list_first_entry(done, struct io_kiocb, list);
994 list_del(&req->list);
995
Jens Axboe78e19bb2019-11-06 15:21:34 -0700996 io_cqring_fill_event(req, req->result);
Jens Axboedef596e2019-01-09 08:59:42 -0700997 (*nr_events)++;
998
Jens Axboe09bb8392019-03-13 12:39:28 -0600999 if (refcount_dec_and_test(&req->refs)) {
1000 /* If we're not using fixed files, we have to pair the
1001 * completion part with the file put. Use regular
1002 * completions for those, only batch free for fixed
Jens Axboe9e645e112019-05-10 16:07:28 -06001003 * file and non-linked commands.
Jens Axboe09bb8392019-03-13 12:39:28 -06001004 */
Jens Axboe9e645e112019-05-10 16:07:28 -06001005 if ((req->flags & (REQ_F_FIXED_FILE|REQ_F_LINK)) ==
1006 REQ_F_FIXED_FILE) {
Jens Axboe09bb8392019-03-13 12:39:28 -06001007 reqs[to_free++] = req;
1008 if (to_free == ARRAY_SIZE(reqs))
1009 io_free_req_many(ctx, reqs, &to_free);
Jens Axboe6b063142019-01-10 22:13:58 -07001010 } else {
Jens Axboeba816ad2019-09-28 11:36:45 -06001011 io_free_req(req, NULL);
Jens Axboe6b063142019-01-10 22:13:58 -07001012 }
Jens Axboe9a56a232019-01-09 09:06:50 -07001013 }
Jens Axboedef596e2019-01-09 08:59:42 -07001014 }
Jens Axboedef596e2019-01-09 08:59:42 -07001015
Jens Axboe09bb8392019-03-13 12:39:28 -06001016 io_commit_cqring(ctx);
Jens Axboedef596e2019-01-09 08:59:42 -07001017 io_free_req_many(ctx, reqs, &to_free);
1018}
1019
1020static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events,
1021 long min)
1022{
1023 struct io_kiocb *req, *tmp;
1024 LIST_HEAD(done);
1025 bool spin;
1026 int ret;
1027
1028 /*
1029 * Only spin for completions if we don't have multiple devices hanging
1030 * off our complete list, and we're under the requested amount.
1031 */
1032 spin = !ctx->poll_multi_file && *nr_events < min;
1033
1034 ret = 0;
1035 list_for_each_entry_safe(req, tmp, &ctx->poll_list, list) {
1036 struct kiocb *kiocb = &req->rw;
1037
1038 /*
1039 * Move completed entries to our local list. If we find a
1040 * request that requires polling, break out and complete
1041 * the done list first, if we have entries there.
1042 */
1043 if (req->flags & REQ_F_IOPOLL_COMPLETED) {
1044 list_move_tail(&req->list, &done);
1045 continue;
1046 }
1047 if (!list_empty(&done))
1048 break;
1049
1050 ret = kiocb->ki_filp->f_op->iopoll(kiocb, spin);
1051 if (ret < 0)
1052 break;
1053
1054 if (ret && spin)
1055 spin = false;
1056 ret = 0;
1057 }
1058
1059 if (!list_empty(&done))
1060 io_iopoll_complete(ctx, nr_events, &done);
1061
1062 return ret;
1063}
1064
1065/*
1066 * Poll for a mininum of 'min' events. Note that if min == 0 we consider that a
1067 * non-spinning poll check - we'll still enter the driver poll loop, but only
1068 * as a non-spinning completion check.
1069 */
1070static int io_iopoll_getevents(struct io_ring_ctx *ctx, unsigned int *nr_events,
1071 long min)
1072{
Jens Axboe08f54392019-08-21 22:19:11 -06001073 while (!list_empty(&ctx->poll_list) && !need_resched()) {
Jens Axboedef596e2019-01-09 08:59:42 -07001074 int ret;
1075
1076 ret = io_do_iopoll(ctx, nr_events, min);
1077 if (ret < 0)
1078 return ret;
1079 if (!min || *nr_events >= min)
1080 return 0;
1081 }
1082
1083 return 1;
1084}
1085
1086/*
1087 * We can't just wait for polled events to come to us, we have to actively
1088 * find and complete them.
1089 */
1090static void io_iopoll_reap_events(struct io_ring_ctx *ctx)
1091{
1092 if (!(ctx->flags & IORING_SETUP_IOPOLL))
1093 return;
1094
1095 mutex_lock(&ctx->uring_lock);
1096 while (!list_empty(&ctx->poll_list)) {
1097 unsigned int nr_events = 0;
1098
1099 io_iopoll_getevents(ctx, &nr_events, 1);
Jens Axboe08f54392019-08-21 22:19:11 -06001100
1101 /*
1102 * Ensure we allow local-to-the-cpu processing to take place,
1103 * in this case we need to ensure that we reap all events.
1104 */
1105 cond_resched();
Jens Axboedef596e2019-01-09 08:59:42 -07001106 }
1107 mutex_unlock(&ctx->uring_lock);
1108}
1109
Jens Axboe2b2ed972019-10-25 10:06:15 -06001110static int __io_iopoll_check(struct io_ring_ctx *ctx, unsigned *nr_events,
1111 long min)
Jens Axboedef596e2019-01-09 08:59:42 -07001112{
Jens Axboe2b2ed972019-10-25 10:06:15 -06001113 int iters = 0, ret = 0;
Jens Axboedef596e2019-01-09 08:59:42 -07001114
1115 do {
1116 int tmin = 0;
1117
Jens Axboe500f9fb2019-08-19 12:15:59 -06001118 /*
Jens Axboea3a0e432019-08-20 11:03:11 -06001119 * Don't enter poll loop if we already have events pending.
1120 * If we do, we can potentially be spinning for commands that
1121 * already triggered a CQE (eg in error).
1122 */
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001123 if (io_cqring_events(ctx, false))
Jens Axboea3a0e432019-08-20 11:03:11 -06001124 break;
1125
1126 /*
Jens Axboe500f9fb2019-08-19 12:15:59 -06001127 * If a submit got punted to a workqueue, we can have the
1128 * application entering polling for a command before it gets
1129 * issued. That app will hold the uring_lock for the duration
1130 * of the poll right here, so we need to take a breather every
1131 * now and then to ensure that the issue has a chance to add
1132 * the poll to the issued list. Otherwise we can spin here
1133 * forever, while the workqueue is stuck trying to acquire the
1134 * very same mutex.
1135 */
1136 if (!(++iters & 7)) {
1137 mutex_unlock(&ctx->uring_lock);
1138 mutex_lock(&ctx->uring_lock);
1139 }
1140
Jens Axboedef596e2019-01-09 08:59:42 -07001141 if (*nr_events < min)
1142 tmin = min - *nr_events;
1143
1144 ret = io_iopoll_getevents(ctx, nr_events, tmin);
1145 if (ret <= 0)
1146 break;
1147 ret = 0;
1148 } while (min && !*nr_events && !need_resched());
1149
Jens Axboe2b2ed972019-10-25 10:06:15 -06001150 return ret;
1151}
1152
1153static int io_iopoll_check(struct io_ring_ctx *ctx, unsigned *nr_events,
1154 long min)
1155{
1156 int ret;
1157
1158 /*
1159 * We disallow the app entering submit/complete with polling, but we
1160 * still need to lock the ring to prevent racing with polled issue
1161 * that got punted to a workqueue.
1162 */
1163 mutex_lock(&ctx->uring_lock);
1164 ret = __io_iopoll_check(ctx, nr_events, min);
Jens Axboe500f9fb2019-08-19 12:15:59 -06001165 mutex_unlock(&ctx->uring_lock);
Jens Axboedef596e2019-01-09 08:59:42 -07001166 return ret;
1167}
1168
Jens Axboe491381ce2019-10-17 09:20:46 -06001169static void kiocb_end_write(struct io_kiocb *req)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001170{
Jens Axboe491381ce2019-10-17 09:20:46 -06001171 /*
1172 * Tell lockdep we inherited freeze protection from submission
1173 * thread.
1174 */
1175 if (req->flags & REQ_F_ISREG) {
1176 struct inode *inode = file_inode(req->file);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001177
Jens Axboe491381ce2019-10-17 09:20:46 -06001178 __sb_writers_acquired(inode->i_sb, SB_FREEZE_WRITE);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001179 }
Jens Axboe491381ce2019-10-17 09:20:46 -06001180 file_end_write(req->file);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001181}
1182
Jens Axboeba816ad2019-09-28 11:36:45 -06001183static void io_complete_rw_common(struct kiocb *kiocb, long res)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001184{
1185 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw);
1186
Jens Axboe491381ce2019-10-17 09:20:46 -06001187 if (kiocb->ki_flags & IOCB_WRITE)
1188 kiocb_end_write(req);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001189
Jens Axboe9e645e112019-05-10 16:07:28 -06001190 if ((req->flags & REQ_F_LINK) && res != req->result)
1191 req->flags |= REQ_F_FAIL_LINK;
Jens Axboe78e19bb2019-11-06 15:21:34 -07001192 io_cqring_add_event(req, res);
Jens Axboeba816ad2019-09-28 11:36:45 -06001193}
1194
1195static void io_complete_rw(struct kiocb *kiocb, long res, long res2)
1196{
1197 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw);
1198
1199 io_complete_rw_common(kiocb, res);
1200 io_put_req(req, NULL);
1201}
1202
1203static struct io_kiocb *__io_complete_rw(struct kiocb *kiocb, long res)
1204{
1205 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw);
1206
1207 io_complete_rw_common(kiocb, res);
1208 return io_put_req_find_next(req);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001209}
1210
Jens Axboedef596e2019-01-09 08:59:42 -07001211static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2)
1212{
1213 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw);
1214
Jens Axboe491381ce2019-10-17 09:20:46 -06001215 if (kiocb->ki_flags & IOCB_WRITE)
1216 kiocb_end_write(req);
Jens Axboedef596e2019-01-09 08:59:42 -07001217
Jens Axboe9e645e112019-05-10 16:07:28 -06001218 if ((req->flags & REQ_F_LINK) && res != req->result)
1219 req->flags |= REQ_F_FAIL_LINK;
1220 req->result = res;
Jens Axboedef596e2019-01-09 08:59:42 -07001221 if (res != -EAGAIN)
1222 req->flags |= REQ_F_IOPOLL_COMPLETED;
1223}
1224
1225/*
1226 * After the iocb has been issued, it's safe to be found on the poll list.
1227 * Adding the kiocb to the list AFTER submission ensures that we don't
1228 * find it from a io_iopoll_getevents() thread before the issuer is done
1229 * accessing the kiocb cookie.
1230 */
1231static void io_iopoll_req_issued(struct io_kiocb *req)
1232{
1233 struct io_ring_ctx *ctx = req->ctx;
1234
1235 /*
1236 * Track whether we have multiple files in our lists. This will impact
1237 * how we do polling eventually, not spinning if we're on potentially
1238 * different devices.
1239 */
1240 if (list_empty(&ctx->poll_list)) {
1241 ctx->poll_multi_file = false;
1242 } else if (!ctx->poll_multi_file) {
1243 struct io_kiocb *list_req;
1244
1245 list_req = list_first_entry(&ctx->poll_list, struct io_kiocb,
1246 list);
1247 if (list_req->rw.ki_filp != req->rw.ki_filp)
1248 ctx->poll_multi_file = true;
1249 }
1250
1251 /*
1252 * For fast devices, IO may have already completed. If it has, add
1253 * it to the front so we find it first.
1254 */
1255 if (req->flags & REQ_F_IOPOLL_COMPLETED)
1256 list_add(&req->list, &ctx->poll_list);
1257 else
1258 list_add_tail(&req->list, &ctx->poll_list);
1259}
1260
Jens Axboe3d6770f2019-04-13 11:50:54 -06001261static void io_file_put(struct io_submit_state *state)
Jens Axboe9a56a232019-01-09 09:06:50 -07001262{
Jens Axboe3d6770f2019-04-13 11:50:54 -06001263 if (state->file) {
Jens Axboe9a56a232019-01-09 09:06:50 -07001264 int diff = state->has_refs - state->used_refs;
1265
1266 if (diff)
1267 fput_many(state->file, diff);
1268 state->file = NULL;
1269 }
1270}
1271
1272/*
1273 * Get as many references to a file as we have IOs left in this submission,
1274 * assuming most submissions are for one file, or at least that each file
1275 * has more than one submission.
1276 */
1277static struct file *io_file_get(struct io_submit_state *state, int fd)
1278{
1279 if (!state)
1280 return fget(fd);
1281
1282 if (state->file) {
1283 if (state->fd == fd) {
1284 state->used_refs++;
1285 state->ios_left--;
1286 return state->file;
1287 }
Jens Axboe3d6770f2019-04-13 11:50:54 -06001288 io_file_put(state);
Jens Axboe9a56a232019-01-09 09:06:50 -07001289 }
1290 state->file = fget_many(fd, state->ios_left);
1291 if (!state->file)
1292 return NULL;
1293
1294 state->fd = fd;
1295 state->has_refs = state->ios_left;
1296 state->used_refs = 1;
1297 state->ios_left--;
1298 return state->file;
1299}
1300
Jens Axboe2b188cc2019-01-07 10:46:33 -07001301/*
1302 * If we tracked the file through the SCM inflight mechanism, we could support
1303 * any file. For now, just ensure that anything potentially problematic is done
1304 * inline.
1305 */
1306static bool io_file_supports_async(struct file *file)
1307{
1308 umode_t mode = file_inode(file)->i_mode;
1309
1310 if (S_ISBLK(mode) || S_ISCHR(mode))
1311 return true;
1312 if (S_ISREG(mode) && file->f_op != &io_uring_fops)
1313 return true;
1314
1315 return false;
1316}
1317
Pavel Begunkov267bc902019-11-07 01:41:08 +03001318static int io_prep_rw(struct io_kiocb *req, bool force_nonblock)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001319{
Pavel Begunkov267bc902019-11-07 01:41:08 +03001320 const struct io_uring_sqe *sqe = req->submit.sqe;
Jens Axboedef596e2019-01-09 08:59:42 -07001321 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001322 struct kiocb *kiocb = &req->rw;
Jens Axboe09bb8392019-03-13 12:39:28 -06001323 unsigned ioprio;
1324 int ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001325
Jens Axboe09bb8392019-03-13 12:39:28 -06001326 if (!req->file)
1327 return -EBADF;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001328
Jens Axboe491381ce2019-10-17 09:20:46 -06001329 if (S_ISREG(file_inode(req->file)->i_mode))
1330 req->flags |= REQ_F_ISREG;
1331
1332 /*
1333 * If the file doesn't support async, mark it as REQ_F_MUST_PUNT so
1334 * we know to async punt it even if it was opened O_NONBLOCK
1335 */
1336 if (force_nonblock && !io_file_supports_async(req->file)) {
1337 req->flags |= REQ_F_MUST_PUNT;
1338 return -EAGAIN;
1339 }
Jens Axboe6b063142019-01-10 22:13:58 -07001340
Jens Axboe2b188cc2019-01-07 10:46:33 -07001341 kiocb->ki_pos = READ_ONCE(sqe->off);
1342 kiocb->ki_flags = iocb_flags(kiocb->ki_filp);
1343 kiocb->ki_hint = ki_hint_validate(file_write_hint(kiocb->ki_filp));
1344
1345 ioprio = READ_ONCE(sqe->ioprio);
1346 if (ioprio) {
1347 ret = ioprio_check_cap(ioprio);
1348 if (ret)
Jens Axboe09bb8392019-03-13 12:39:28 -06001349 return ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001350
1351 kiocb->ki_ioprio = ioprio;
1352 } else
1353 kiocb->ki_ioprio = get_current_ioprio();
1354
1355 ret = kiocb_set_rw_flags(kiocb, READ_ONCE(sqe->rw_flags));
1356 if (unlikely(ret))
Jens Axboe09bb8392019-03-13 12:39:28 -06001357 return ret;
Stefan Bühler8449eed2019-04-27 20:34:19 +02001358
1359 /* don't allow async punt if RWF_NOWAIT was requested */
Jens Axboe491381ce2019-10-17 09:20:46 -06001360 if ((kiocb->ki_flags & IOCB_NOWAIT) ||
1361 (req->file->f_flags & O_NONBLOCK))
Stefan Bühler8449eed2019-04-27 20:34:19 +02001362 req->flags |= REQ_F_NOWAIT;
1363
1364 if (force_nonblock)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001365 kiocb->ki_flags |= IOCB_NOWAIT;
Stefan Bühler8449eed2019-04-27 20:34:19 +02001366
Jens Axboedef596e2019-01-09 08:59:42 -07001367 if (ctx->flags & IORING_SETUP_IOPOLL) {
Jens Axboedef596e2019-01-09 08:59:42 -07001368 if (!(kiocb->ki_flags & IOCB_DIRECT) ||
1369 !kiocb->ki_filp->f_op->iopoll)
Jens Axboe09bb8392019-03-13 12:39:28 -06001370 return -EOPNOTSUPP;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001371
Jens Axboedef596e2019-01-09 08:59:42 -07001372 kiocb->ki_flags |= IOCB_HIPRI;
1373 kiocb->ki_complete = io_complete_rw_iopoll;
1374 } else {
Jens Axboe09bb8392019-03-13 12:39:28 -06001375 if (kiocb->ki_flags & IOCB_HIPRI)
1376 return -EINVAL;
Jens Axboedef596e2019-01-09 08:59:42 -07001377 kiocb->ki_complete = io_complete_rw;
1378 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07001379 return 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001380}
1381
1382static inline void io_rw_done(struct kiocb *kiocb, ssize_t ret)
1383{
1384 switch (ret) {
1385 case -EIOCBQUEUED:
1386 break;
1387 case -ERESTARTSYS:
1388 case -ERESTARTNOINTR:
1389 case -ERESTARTNOHAND:
1390 case -ERESTART_RESTARTBLOCK:
1391 /*
1392 * We can't just restart the syscall, since previously
1393 * submitted sqes may already be in progress. Just fail this
1394 * IO with EINTR.
1395 */
1396 ret = -EINTR;
1397 /* fall through */
1398 default:
1399 kiocb->ki_complete(kiocb, ret, 0);
1400 }
1401}
1402
Jens Axboeba816ad2019-09-28 11:36:45 -06001403static void kiocb_done(struct kiocb *kiocb, ssize_t ret, struct io_kiocb **nxt,
1404 bool in_async)
1405{
1406 if (in_async && ret >= 0 && nxt && kiocb->ki_complete == io_complete_rw)
1407 *nxt = __io_complete_rw(kiocb, ret);
1408 else
1409 io_rw_done(kiocb, ret);
1410}
1411
Jens Axboeedafcce2019-01-09 09:16:05 -07001412static int io_import_fixed(struct io_ring_ctx *ctx, int rw,
1413 const struct io_uring_sqe *sqe,
1414 struct iov_iter *iter)
1415{
1416 size_t len = READ_ONCE(sqe->len);
1417 struct io_mapped_ubuf *imu;
1418 unsigned index, buf_index;
1419 size_t offset;
1420 u64 buf_addr;
1421
1422 /* attempt to use fixed buffers without having provided iovecs */
1423 if (unlikely(!ctx->user_bufs))
1424 return -EFAULT;
1425
1426 buf_index = READ_ONCE(sqe->buf_index);
1427 if (unlikely(buf_index >= ctx->nr_user_bufs))
1428 return -EFAULT;
1429
1430 index = array_index_nospec(buf_index, ctx->nr_user_bufs);
1431 imu = &ctx->user_bufs[index];
1432 buf_addr = READ_ONCE(sqe->addr);
1433
1434 /* overflow */
1435 if (buf_addr + len < buf_addr)
1436 return -EFAULT;
1437 /* not inside the mapped region */
1438 if (buf_addr < imu->ubuf || buf_addr + len > imu->ubuf + imu->len)
1439 return -EFAULT;
1440
1441 /*
1442 * May not be a start of buffer, set size appropriately
1443 * and advance us to the beginning.
1444 */
1445 offset = buf_addr - imu->ubuf;
1446 iov_iter_bvec(iter, rw, imu->bvec, imu->nr_bvecs, offset + len);
Jens Axboebd11b3a2019-07-20 08:37:31 -06001447
1448 if (offset) {
1449 /*
1450 * Don't use iov_iter_advance() here, as it's really slow for
1451 * using the latter parts of a big fixed buffer - it iterates
1452 * over each segment manually. We can cheat a bit here, because
1453 * we know that:
1454 *
1455 * 1) it's a BVEC iter, we set it up
1456 * 2) all bvecs are PAGE_SIZE in size, except potentially the
1457 * first and last bvec
1458 *
1459 * So just find our index, and adjust the iterator afterwards.
1460 * If the offset is within the first bvec (or the whole first
1461 * bvec, just use iov_iter_advance(). This makes it easier
1462 * since we can just skip the first segment, which may not
1463 * be PAGE_SIZE aligned.
1464 */
1465 const struct bio_vec *bvec = imu->bvec;
1466
1467 if (offset <= bvec->bv_len) {
1468 iov_iter_advance(iter, offset);
1469 } else {
1470 unsigned long seg_skip;
1471
1472 /* skip first vec */
1473 offset -= bvec->bv_len;
1474 seg_skip = 1 + (offset >> PAGE_SHIFT);
1475
1476 iter->bvec = bvec + seg_skip;
1477 iter->nr_segs -= seg_skip;
Aleix Roca Nonell99c79f62019-08-15 14:03:22 +02001478 iter->count -= bvec->bv_len + offset;
Jens Axboebd11b3a2019-07-20 08:37:31 -06001479 iter->iov_offset = offset & ~PAGE_MASK;
Jens Axboebd11b3a2019-07-20 08:37:31 -06001480 }
1481 }
1482
Jens Axboeedafcce2019-01-09 09:16:05 -07001483 return 0;
1484}
1485
Jens Axboe87e5e6d2019-05-14 16:02:22 -06001486static ssize_t io_import_iovec(struct io_ring_ctx *ctx, int rw,
1487 const struct sqe_submit *s, struct iovec **iovec,
1488 struct iov_iter *iter)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001489{
1490 const struct io_uring_sqe *sqe = s->sqe;
1491 void __user *buf = u64_to_user_ptr(READ_ONCE(sqe->addr));
1492 size_t sqe_len = READ_ONCE(sqe->len);
Jens Axboeedafcce2019-01-09 09:16:05 -07001493 u8 opcode;
1494
1495 /*
1496 * We're reading ->opcode for the second time, but the first read
1497 * doesn't care whether it's _FIXED or not, so it doesn't matter
1498 * whether ->opcode changes concurrently. The first read does care
1499 * about whether it is a READ or a WRITE, so we don't trust this read
1500 * for that purpose and instead let the caller pass in the read/write
1501 * flag.
1502 */
1503 opcode = READ_ONCE(sqe->opcode);
1504 if (opcode == IORING_OP_READ_FIXED ||
1505 opcode == IORING_OP_WRITE_FIXED) {
Jens Axboe87e5e6d2019-05-14 16:02:22 -06001506 ssize_t ret = io_import_fixed(ctx, rw, sqe, iter);
Jens Axboeedafcce2019-01-09 09:16:05 -07001507 *iovec = NULL;
1508 return ret;
1509 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07001510
1511 if (!s->has_user)
1512 return -EFAULT;
1513
1514#ifdef CONFIG_COMPAT
1515 if (ctx->compat)
1516 return compat_import_iovec(rw, buf, sqe_len, UIO_FASTIOV,
1517 iovec, iter);
1518#endif
1519
1520 return import_iovec(rw, buf, sqe_len, UIO_FASTIOV, iovec, iter);
1521}
1522
Jens Axboe32960612019-09-23 11:05:34 -06001523/*
1524 * For files that don't have ->read_iter() and ->write_iter(), handle them
1525 * by looping over ->read() or ->write() manually.
1526 */
1527static ssize_t loop_rw_iter(int rw, struct file *file, struct kiocb *kiocb,
1528 struct iov_iter *iter)
1529{
1530 ssize_t ret = 0;
1531
1532 /*
1533 * Don't support polled IO through this interface, and we can't
1534 * support non-blocking either. For the latter, this just causes
1535 * the kiocb to be handled from an async context.
1536 */
1537 if (kiocb->ki_flags & IOCB_HIPRI)
1538 return -EOPNOTSUPP;
1539 if (kiocb->ki_flags & IOCB_NOWAIT)
1540 return -EAGAIN;
1541
1542 while (iov_iter_count(iter)) {
1543 struct iovec iovec = iov_iter_iovec(iter);
1544 ssize_t nr;
1545
1546 if (rw == READ) {
1547 nr = file->f_op->read(file, iovec.iov_base,
1548 iovec.iov_len, &kiocb->ki_pos);
1549 } else {
1550 nr = file->f_op->write(file, iovec.iov_base,
1551 iovec.iov_len, &kiocb->ki_pos);
1552 }
1553
1554 if (nr < 0) {
1555 if (!ret)
1556 ret = nr;
1557 break;
1558 }
1559 ret += nr;
1560 if (nr != iovec.iov_len)
1561 break;
1562 iov_iter_advance(iter, nr);
1563 }
1564
1565 return ret;
1566}
1567
Pavel Begunkov267bc902019-11-07 01:41:08 +03001568static int io_read(struct io_kiocb *req, struct io_kiocb **nxt,
1569 bool force_nonblock)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001570{
1571 struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
1572 struct kiocb *kiocb = &req->rw;
1573 struct iov_iter iter;
1574 struct file *file;
Jens Axboe31b51512019-01-18 22:56:34 -07001575 size_t iov_count;
Jens Axboe9d93a3f2019-05-15 13:53:07 -06001576 ssize_t read_size, ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001577
Pavel Begunkov267bc902019-11-07 01:41:08 +03001578 ret = io_prep_rw(req, force_nonblock);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001579 if (ret)
1580 return ret;
1581 file = kiocb->ki_filp;
1582
Jens Axboe2b188cc2019-01-07 10:46:33 -07001583 if (unlikely(!(file->f_mode & FMODE_READ)))
Jens Axboe09bb8392019-03-13 12:39:28 -06001584 return -EBADF;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001585
Pavel Begunkov267bc902019-11-07 01:41:08 +03001586 ret = io_import_iovec(req->ctx, READ, &req->submit, &iovec, &iter);
Jens Axboe87e5e6d2019-05-14 16:02:22 -06001587 if (ret < 0)
Jens Axboe09bb8392019-03-13 12:39:28 -06001588 return ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001589
Jens Axboe9d93a3f2019-05-15 13:53:07 -06001590 read_size = ret;
Jens Axboe9e645e112019-05-10 16:07:28 -06001591 if (req->flags & REQ_F_LINK)
1592 req->result = read_size;
1593
Jens Axboe31b51512019-01-18 22:56:34 -07001594 iov_count = iov_iter_count(&iter);
1595 ret = rw_verify_area(READ, file, &kiocb->ki_pos, iov_count);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001596 if (!ret) {
1597 ssize_t ret2;
1598
Jens Axboe32960612019-09-23 11:05:34 -06001599 if (file->f_op->read_iter)
1600 ret2 = call_read_iter(file, kiocb, &iter);
1601 else
1602 ret2 = loop_rw_iter(READ, file, kiocb, &iter);
1603
Jens Axboe9d93a3f2019-05-15 13:53:07 -06001604 /*
1605 * In case of a short read, punt to async. This can happen
1606 * if we have data partially cached. Alternatively we can
1607 * return the short read, in which case the application will
1608 * need to issue another SQE and wait for it. That SQE will
1609 * need async punt anyway, so it's more efficient to do it
1610 * here.
1611 */
Jens Axboe491381ce2019-10-17 09:20:46 -06001612 if (force_nonblock && !(req->flags & REQ_F_NOWAIT) &&
1613 (req->flags & REQ_F_ISREG) &&
1614 ret2 > 0 && ret2 < read_size)
Jens Axboe9d93a3f2019-05-15 13:53:07 -06001615 ret2 = -EAGAIN;
1616 /* Catch -EAGAIN return for forced non-blocking submission */
Jens Axboe561fb042019-10-24 07:25:42 -06001617 if (!force_nonblock || ret2 != -EAGAIN)
Pavel Begunkov267bc902019-11-07 01:41:08 +03001618 kiocb_done(kiocb, ret2, nxt, req->submit.in_async);
Jens Axboe561fb042019-10-24 07:25:42 -06001619 else
Jens Axboe2b188cc2019-01-07 10:46:33 -07001620 ret = -EAGAIN;
1621 }
1622 kfree(iovec);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001623 return ret;
1624}
1625
Pavel Begunkov267bc902019-11-07 01:41:08 +03001626static int io_write(struct io_kiocb *req, struct io_kiocb **nxt,
1627 bool force_nonblock)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001628{
1629 struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
1630 struct kiocb *kiocb = &req->rw;
1631 struct iov_iter iter;
1632 struct file *file;
Jens Axboe31b51512019-01-18 22:56:34 -07001633 size_t iov_count;
Jens Axboe87e5e6d2019-05-14 16:02:22 -06001634 ssize_t ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001635
Pavel Begunkov267bc902019-11-07 01:41:08 +03001636 ret = io_prep_rw(req, force_nonblock);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001637 if (ret)
1638 return ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001639
Jens Axboe2b188cc2019-01-07 10:46:33 -07001640 file = kiocb->ki_filp;
1641 if (unlikely(!(file->f_mode & FMODE_WRITE)))
Jens Axboe09bb8392019-03-13 12:39:28 -06001642 return -EBADF;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001643
Pavel Begunkov267bc902019-11-07 01:41:08 +03001644 ret = io_import_iovec(req->ctx, WRITE, &req->submit, &iovec, &iter);
Jens Axboe87e5e6d2019-05-14 16:02:22 -06001645 if (ret < 0)
Jens Axboe09bb8392019-03-13 12:39:28 -06001646 return ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001647
Jens Axboe9e645e112019-05-10 16:07:28 -06001648 if (req->flags & REQ_F_LINK)
1649 req->result = ret;
1650
Jens Axboe31b51512019-01-18 22:56:34 -07001651 iov_count = iov_iter_count(&iter);
1652
1653 ret = -EAGAIN;
Jens Axboe561fb042019-10-24 07:25:42 -06001654 if (force_nonblock && !(kiocb->ki_flags & IOCB_DIRECT))
Jens Axboe31b51512019-01-18 22:56:34 -07001655 goto out_free;
Jens Axboe31b51512019-01-18 22:56:34 -07001656
1657 ret = rw_verify_area(WRITE, file, &kiocb->ki_pos, iov_count);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001658 if (!ret) {
Roman Penyaev9bf79332019-03-25 20:09:24 +01001659 ssize_t ret2;
1660
Jens Axboe2b188cc2019-01-07 10:46:33 -07001661 /*
1662 * Open-code file_start_write here to grab freeze protection,
1663 * which will be released by another thread in
1664 * io_complete_rw(). Fool lockdep by telling it the lock got
1665 * released so that it doesn't complain about the held lock when
1666 * we return to userspace.
1667 */
Jens Axboe491381ce2019-10-17 09:20:46 -06001668 if (req->flags & REQ_F_ISREG) {
Jens Axboe2b188cc2019-01-07 10:46:33 -07001669 __sb_start_write(file_inode(file)->i_sb,
1670 SB_FREEZE_WRITE, true);
1671 __sb_writers_release(file_inode(file)->i_sb,
1672 SB_FREEZE_WRITE);
1673 }
1674 kiocb->ki_flags |= IOCB_WRITE;
Roman Penyaev9bf79332019-03-25 20:09:24 +01001675
Jens Axboe32960612019-09-23 11:05:34 -06001676 if (file->f_op->write_iter)
1677 ret2 = call_write_iter(file, kiocb, &iter);
1678 else
1679 ret2 = loop_rw_iter(WRITE, file, kiocb, &iter);
Jens Axboe561fb042019-10-24 07:25:42 -06001680 if (!force_nonblock || ret2 != -EAGAIN)
Pavel Begunkov267bc902019-11-07 01:41:08 +03001681 kiocb_done(kiocb, ret2, nxt, req->submit.in_async);
Jens Axboe561fb042019-10-24 07:25:42 -06001682 else
Roman Penyaev9bf79332019-03-25 20:09:24 +01001683 ret = -EAGAIN;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001684 }
Jens Axboe31b51512019-01-18 22:56:34 -07001685out_free:
Jens Axboe2b188cc2019-01-07 10:46:33 -07001686 kfree(iovec);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001687 return ret;
1688}
1689
1690/*
1691 * IORING_OP_NOP just posts a completion event, nothing else.
1692 */
Jens Axboe78e19bb2019-11-06 15:21:34 -07001693static int io_nop(struct io_kiocb *req)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001694{
1695 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001696
Jens Axboedef596e2019-01-09 08:59:42 -07001697 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
1698 return -EINVAL;
1699
Jens Axboe78e19bb2019-11-06 15:21:34 -07001700 io_cqring_add_event(req, 0);
Jens Axboeba816ad2019-09-28 11:36:45 -06001701 io_put_req(req, NULL);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001702 return 0;
1703}
1704
Christoph Hellwigc992fe22019-01-11 09:43:02 -07001705static int io_prep_fsync(struct io_kiocb *req, const struct io_uring_sqe *sqe)
1706{
Jens Axboe6b063142019-01-10 22:13:58 -07001707 struct io_ring_ctx *ctx = req->ctx;
Christoph Hellwigc992fe22019-01-11 09:43:02 -07001708
Jens Axboe09bb8392019-03-13 12:39:28 -06001709 if (!req->file)
1710 return -EBADF;
Christoph Hellwigc992fe22019-01-11 09:43:02 -07001711
Jens Axboe6b063142019-01-10 22:13:58 -07001712 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
Jens Axboedef596e2019-01-09 08:59:42 -07001713 return -EINVAL;
Jens Axboeedafcce2019-01-09 09:16:05 -07001714 if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index))
Christoph Hellwigc992fe22019-01-11 09:43:02 -07001715 return -EINVAL;
1716
Christoph Hellwigc992fe22019-01-11 09:43:02 -07001717 return 0;
1718}
1719
1720static int io_fsync(struct io_kiocb *req, const struct io_uring_sqe *sqe,
Jens Axboeba816ad2019-09-28 11:36:45 -06001721 struct io_kiocb **nxt, bool force_nonblock)
Christoph Hellwigc992fe22019-01-11 09:43:02 -07001722{
1723 loff_t sqe_off = READ_ONCE(sqe->off);
1724 loff_t sqe_len = READ_ONCE(sqe->len);
1725 loff_t end = sqe_off + sqe_len;
1726 unsigned fsync_flags;
1727 int ret;
1728
1729 fsync_flags = READ_ONCE(sqe->fsync_flags);
1730 if (unlikely(fsync_flags & ~IORING_FSYNC_DATASYNC))
1731 return -EINVAL;
1732
1733 ret = io_prep_fsync(req, sqe);
1734 if (ret)
1735 return ret;
1736
1737 /* fsync always requires a blocking context */
1738 if (force_nonblock)
1739 return -EAGAIN;
1740
1741 ret = vfs_fsync_range(req->rw.ki_filp, sqe_off,
1742 end > 0 ? end : LLONG_MAX,
1743 fsync_flags & IORING_FSYNC_DATASYNC);
1744
Jens Axboe9e645e112019-05-10 16:07:28 -06001745 if (ret < 0 && (req->flags & REQ_F_LINK))
1746 req->flags |= REQ_F_FAIL_LINK;
Jens Axboe78e19bb2019-11-06 15:21:34 -07001747 io_cqring_add_event(req, ret);
Jens Axboeba816ad2019-09-28 11:36:45 -06001748 io_put_req(req, nxt);
Christoph Hellwigc992fe22019-01-11 09:43:02 -07001749 return 0;
1750}
1751
Jens Axboe5d17b4a2019-04-09 14:56:44 -06001752static int io_prep_sfr(struct io_kiocb *req, const struct io_uring_sqe *sqe)
1753{
1754 struct io_ring_ctx *ctx = req->ctx;
1755 int ret = 0;
1756
1757 if (!req->file)
1758 return -EBADF;
Jens Axboe5d17b4a2019-04-09 14:56:44 -06001759
1760 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
1761 return -EINVAL;
1762 if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index))
1763 return -EINVAL;
1764
Jens Axboe5d17b4a2019-04-09 14:56:44 -06001765 return ret;
1766}
1767
1768static int io_sync_file_range(struct io_kiocb *req,
1769 const struct io_uring_sqe *sqe,
Jens Axboeba816ad2019-09-28 11:36:45 -06001770 struct io_kiocb **nxt,
Jens Axboe5d17b4a2019-04-09 14:56:44 -06001771 bool force_nonblock)
1772{
1773 loff_t sqe_off;
1774 loff_t sqe_len;
1775 unsigned flags;
1776 int ret;
1777
1778 ret = io_prep_sfr(req, sqe);
1779 if (ret)
1780 return ret;
1781
1782 /* sync_file_range always requires a blocking context */
1783 if (force_nonblock)
1784 return -EAGAIN;
1785
1786 sqe_off = READ_ONCE(sqe->off);
1787 sqe_len = READ_ONCE(sqe->len);
1788 flags = READ_ONCE(sqe->sync_range_flags);
1789
1790 ret = sync_file_range(req->rw.ki_filp, sqe_off, sqe_len, flags);
1791
Jens Axboe9e645e112019-05-10 16:07:28 -06001792 if (ret < 0 && (req->flags & REQ_F_LINK))
1793 req->flags |= REQ_F_FAIL_LINK;
Jens Axboe78e19bb2019-11-06 15:21:34 -07001794 io_cqring_add_event(req, ret);
Jens Axboeba816ad2019-09-28 11:36:45 -06001795 io_put_req(req, nxt);
Jens Axboe5d17b4a2019-04-09 14:56:44 -06001796 return 0;
1797}
1798
Jens Axboe0fa03c62019-04-19 13:34:07 -06001799#if defined(CONFIG_NET)
Jens Axboeaa1fa282019-04-19 13:38:09 -06001800static int io_send_recvmsg(struct io_kiocb *req, const struct io_uring_sqe *sqe,
Jens Axboeba816ad2019-09-28 11:36:45 -06001801 struct io_kiocb **nxt, bool force_nonblock,
Jens Axboeaa1fa282019-04-19 13:38:09 -06001802 long (*fn)(struct socket *, struct user_msghdr __user *,
1803 unsigned int))
1804{
Jens Axboe0fa03c62019-04-19 13:34:07 -06001805 struct socket *sock;
1806 int ret;
1807
1808 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
1809 return -EINVAL;
1810
1811 sock = sock_from_file(req->file, &ret);
1812 if (sock) {
1813 struct user_msghdr __user *msg;
1814 unsigned flags;
1815
1816 flags = READ_ONCE(sqe->msg_flags);
1817 if (flags & MSG_DONTWAIT)
1818 req->flags |= REQ_F_NOWAIT;
1819 else if (force_nonblock)
1820 flags |= MSG_DONTWAIT;
1821
1822 msg = (struct user_msghdr __user *) (unsigned long)
1823 READ_ONCE(sqe->addr);
1824
Jens Axboeaa1fa282019-04-19 13:38:09 -06001825 ret = fn(sock, msg, flags);
Jens Axboe0fa03c62019-04-19 13:34:07 -06001826 if (force_nonblock && ret == -EAGAIN)
1827 return ret;
1828 }
1829
Jens Axboe78e19bb2019-11-06 15:21:34 -07001830 io_cqring_add_event(req, ret);
Jens Axboef1f40852019-11-05 20:33:16 -07001831 if (ret < 0 && (req->flags & REQ_F_LINK))
1832 req->flags |= REQ_F_FAIL_LINK;
Jens Axboeba816ad2019-09-28 11:36:45 -06001833 io_put_req(req, nxt);
Jens Axboe0fa03c62019-04-19 13:34:07 -06001834 return 0;
Jens Axboeaa1fa282019-04-19 13:38:09 -06001835}
1836#endif
1837
1838static int io_sendmsg(struct io_kiocb *req, const struct io_uring_sqe *sqe,
Jens Axboeba816ad2019-09-28 11:36:45 -06001839 struct io_kiocb **nxt, bool force_nonblock)
Jens Axboeaa1fa282019-04-19 13:38:09 -06001840{
1841#if defined(CONFIG_NET)
Jens Axboeba816ad2019-09-28 11:36:45 -06001842 return io_send_recvmsg(req, sqe, nxt, force_nonblock,
1843 __sys_sendmsg_sock);
Jens Axboeaa1fa282019-04-19 13:38:09 -06001844#else
1845 return -EOPNOTSUPP;
1846#endif
1847}
1848
1849static int io_recvmsg(struct io_kiocb *req, const struct io_uring_sqe *sqe,
Jens Axboeba816ad2019-09-28 11:36:45 -06001850 struct io_kiocb **nxt, bool force_nonblock)
Jens Axboeaa1fa282019-04-19 13:38:09 -06001851{
1852#if defined(CONFIG_NET)
Jens Axboeba816ad2019-09-28 11:36:45 -06001853 return io_send_recvmsg(req, sqe, nxt, force_nonblock,
1854 __sys_recvmsg_sock);
Jens Axboe0fa03c62019-04-19 13:34:07 -06001855#else
1856 return -EOPNOTSUPP;
1857#endif
1858}
1859
Jens Axboe17f2fe32019-10-17 14:42:58 -06001860static int io_accept(struct io_kiocb *req, const struct io_uring_sqe *sqe,
1861 struct io_kiocb **nxt, bool force_nonblock)
1862{
1863#if defined(CONFIG_NET)
1864 struct sockaddr __user *addr;
1865 int __user *addr_len;
1866 unsigned file_flags;
1867 int flags, ret;
1868
1869 if (unlikely(req->ctx->flags & (IORING_SETUP_IOPOLL|IORING_SETUP_SQPOLL)))
1870 return -EINVAL;
1871 if (sqe->ioprio || sqe->off || sqe->len || sqe->buf_index)
1872 return -EINVAL;
1873
1874 addr = (struct sockaddr __user *) (unsigned long) READ_ONCE(sqe->addr);
1875 addr_len = (int __user *) (unsigned long) READ_ONCE(sqe->addr2);
1876 flags = READ_ONCE(sqe->accept_flags);
1877 file_flags = force_nonblock ? O_NONBLOCK : 0;
1878
1879 ret = __sys_accept4_file(req->file, file_flags, addr, addr_len, flags);
1880 if (ret == -EAGAIN && force_nonblock) {
1881 req->work.flags |= IO_WQ_WORK_NEEDS_FILES;
1882 return -EAGAIN;
1883 }
1884 if (ret < 0 && (req->flags & REQ_F_LINK))
1885 req->flags |= REQ_F_FAIL_LINK;
Jens Axboe78e19bb2019-11-06 15:21:34 -07001886 io_cqring_add_event(req, ret);
Jens Axboe17f2fe32019-10-17 14:42:58 -06001887 io_put_req(req, nxt);
1888 return 0;
1889#else
1890 return -EOPNOTSUPP;
1891#endif
1892}
1893
Jens Axboe221c5eb2019-01-17 09:41:58 -07001894static void io_poll_remove_one(struct io_kiocb *req)
1895{
1896 struct io_poll_iocb *poll = &req->poll;
1897
1898 spin_lock(&poll->head->lock);
1899 WRITE_ONCE(poll->canceled, true);
1900 if (!list_empty(&poll->wait.entry)) {
1901 list_del_init(&poll->wait.entry);
Jackie Liua197f662019-11-08 08:09:12 -07001902 io_queue_async_work(req);
Jens Axboe221c5eb2019-01-17 09:41:58 -07001903 }
1904 spin_unlock(&poll->head->lock);
1905
1906 list_del_init(&req->list);
1907}
1908
1909static void io_poll_remove_all(struct io_ring_ctx *ctx)
1910{
1911 struct io_kiocb *req;
1912
1913 spin_lock_irq(&ctx->completion_lock);
1914 while (!list_empty(&ctx->cancel_list)) {
1915 req = list_first_entry(&ctx->cancel_list, struct io_kiocb,list);
1916 io_poll_remove_one(req);
1917 }
1918 spin_unlock_irq(&ctx->completion_lock);
1919}
1920
1921/*
1922 * Find a running poll command that matches one specified in sqe->addr,
1923 * and remove it if found.
1924 */
1925static int io_poll_remove(struct io_kiocb *req, const struct io_uring_sqe *sqe)
1926{
1927 struct io_ring_ctx *ctx = req->ctx;
1928 struct io_kiocb *poll_req, *next;
1929 int ret = -ENOENT;
1930
1931 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
1932 return -EINVAL;
1933 if (sqe->ioprio || sqe->off || sqe->len || sqe->buf_index ||
1934 sqe->poll_events)
1935 return -EINVAL;
1936
1937 spin_lock_irq(&ctx->completion_lock);
1938 list_for_each_entry_safe(poll_req, next, &ctx->cancel_list, list) {
1939 if (READ_ONCE(sqe->addr) == poll_req->user_data) {
1940 io_poll_remove_one(poll_req);
1941 ret = 0;
1942 break;
1943 }
1944 }
1945 spin_unlock_irq(&ctx->completion_lock);
1946
Jens Axboe78e19bb2019-11-06 15:21:34 -07001947 io_cqring_add_event(req, ret);
Jens Axboef1f40852019-11-05 20:33:16 -07001948 if (ret < 0 && (req->flags & REQ_F_LINK))
1949 req->flags |= REQ_F_FAIL_LINK;
Jens Axboeba816ad2019-09-28 11:36:45 -06001950 io_put_req(req, NULL);
Jens Axboe221c5eb2019-01-17 09:41:58 -07001951 return 0;
1952}
1953
Jackie Liua197f662019-11-08 08:09:12 -07001954static void io_poll_complete(struct io_kiocb *req, __poll_t mask)
Jens Axboe221c5eb2019-01-17 09:41:58 -07001955{
Jackie Liua197f662019-11-08 08:09:12 -07001956 struct io_ring_ctx *ctx = req->ctx;
1957
Jens Axboe8c838782019-03-12 15:48:16 -06001958 req->poll.done = true;
Jens Axboe78e19bb2019-11-06 15:21:34 -07001959 io_cqring_fill_event(req, mangle_poll(mask));
Jens Axboe8c838782019-03-12 15:48:16 -06001960 io_commit_cqring(ctx);
Jens Axboe221c5eb2019-01-17 09:41:58 -07001961}
1962
Jens Axboe561fb042019-10-24 07:25:42 -06001963static void io_poll_complete_work(struct io_wq_work **workptr)
Jens Axboe221c5eb2019-01-17 09:41:58 -07001964{
Jens Axboe561fb042019-10-24 07:25:42 -06001965 struct io_wq_work *work = *workptr;
Jens Axboe221c5eb2019-01-17 09:41:58 -07001966 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
1967 struct io_poll_iocb *poll = &req->poll;
1968 struct poll_table_struct pt = { ._key = poll->events };
1969 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe89723d02019-11-05 15:32:58 -07001970 struct io_kiocb *nxt = NULL;
Jens Axboe221c5eb2019-01-17 09:41:58 -07001971 __poll_t mask = 0;
1972
Jens Axboe561fb042019-10-24 07:25:42 -06001973 if (work->flags & IO_WQ_WORK_CANCEL)
1974 WRITE_ONCE(poll->canceled, true);
1975
Jens Axboe221c5eb2019-01-17 09:41:58 -07001976 if (!READ_ONCE(poll->canceled))
1977 mask = vfs_poll(poll->file, &pt) & poll->events;
1978
1979 /*
1980 * Note that ->ki_cancel callers also delete iocb from active_reqs after
1981 * calling ->ki_cancel. We need the ctx_lock roundtrip here to
1982 * synchronize with them. In the cancellation case the list_del_init
1983 * itself is not actually needed, but harmless so we keep it in to
1984 * avoid further branches in the fast path.
1985 */
1986 spin_lock_irq(&ctx->completion_lock);
1987 if (!mask && !READ_ONCE(poll->canceled)) {
1988 add_wait_queue(poll->head, &poll->wait);
1989 spin_unlock_irq(&ctx->completion_lock);
1990 return;
1991 }
1992 list_del_init(&req->list);
Jackie Liua197f662019-11-08 08:09:12 -07001993 io_poll_complete(req, mask);
Jens Axboe221c5eb2019-01-17 09:41:58 -07001994 spin_unlock_irq(&ctx->completion_lock);
1995
Jens Axboe8c838782019-03-12 15:48:16 -06001996 io_cqring_ev_posted(ctx);
Jens Axboe89723d02019-11-05 15:32:58 -07001997
1998 io_put_req(req, &nxt);
1999 if (nxt)
2000 *workptr = &nxt->work;
Jens Axboe221c5eb2019-01-17 09:41:58 -07002001}
2002
2003static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
2004 void *key)
2005{
2006 struct io_poll_iocb *poll = container_of(wait, struct io_poll_iocb,
2007 wait);
2008 struct io_kiocb *req = container_of(poll, struct io_kiocb, poll);
2009 struct io_ring_ctx *ctx = req->ctx;
2010 __poll_t mask = key_to_poll(key);
Jens Axboe8c838782019-03-12 15:48:16 -06002011 unsigned long flags;
Jens Axboe221c5eb2019-01-17 09:41:58 -07002012
2013 /* for instances that support it check for an event match first: */
Jens Axboe8c838782019-03-12 15:48:16 -06002014 if (mask && !(mask & poll->events))
2015 return 0;
Jens Axboe221c5eb2019-01-17 09:41:58 -07002016
2017 list_del_init(&poll->wait.entry);
Jens Axboe8c838782019-03-12 15:48:16 -06002018
2019 if (mask && spin_trylock_irqsave(&ctx->completion_lock, flags)) {
2020 list_del(&req->list);
Jackie Liua197f662019-11-08 08:09:12 -07002021 io_poll_complete(req, mask);
Jens Axboe8c838782019-03-12 15:48:16 -06002022 spin_unlock_irqrestore(&ctx->completion_lock, flags);
2023
2024 io_cqring_ev_posted(ctx);
Jens Axboeba816ad2019-09-28 11:36:45 -06002025 io_put_req(req, NULL);
Jens Axboe8c838782019-03-12 15:48:16 -06002026 } else {
Jackie Liua197f662019-11-08 08:09:12 -07002027 io_queue_async_work(req);
Jens Axboe8c838782019-03-12 15:48:16 -06002028 }
2029
Jens Axboe221c5eb2019-01-17 09:41:58 -07002030 return 1;
2031}
2032
2033struct io_poll_table {
2034 struct poll_table_struct pt;
2035 struct io_kiocb *req;
2036 int error;
2037};
2038
2039static void io_poll_queue_proc(struct file *file, struct wait_queue_head *head,
2040 struct poll_table_struct *p)
2041{
2042 struct io_poll_table *pt = container_of(p, struct io_poll_table, pt);
2043
2044 if (unlikely(pt->req->poll.head)) {
2045 pt->error = -EINVAL;
2046 return;
2047 }
2048
2049 pt->error = 0;
2050 pt->req->poll.head = head;
2051 add_wait_queue(head, &pt->req->poll.wait);
2052}
2053
Jens Axboe89723d02019-11-05 15:32:58 -07002054static int io_poll_add(struct io_kiocb *req, const struct io_uring_sqe *sqe,
2055 struct io_kiocb **nxt)
Jens Axboe221c5eb2019-01-17 09:41:58 -07002056{
2057 struct io_poll_iocb *poll = &req->poll;
2058 struct io_ring_ctx *ctx = req->ctx;
2059 struct io_poll_table ipt;
Jens Axboe8c838782019-03-12 15:48:16 -06002060 bool cancel = false;
Jens Axboe221c5eb2019-01-17 09:41:58 -07002061 __poll_t mask;
2062 u16 events;
Jens Axboe221c5eb2019-01-17 09:41:58 -07002063
2064 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
2065 return -EINVAL;
2066 if (sqe->addr || sqe->ioprio || sqe->off || sqe->len || sqe->buf_index)
2067 return -EINVAL;
Jens Axboe09bb8392019-03-13 12:39:28 -06002068 if (!poll->file)
2069 return -EBADF;
Jens Axboe221c5eb2019-01-17 09:41:58 -07002070
Jens Axboe6cc47d12019-09-18 11:18:23 -06002071 req->submit.sqe = NULL;
Jens Axboe561fb042019-10-24 07:25:42 -06002072 INIT_IO_WORK(&req->work, io_poll_complete_work);
Jens Axboe221c5eb2019-01-17 09:41:58 -07002073 events = READ_ONCE(sqe->poll_events);
2074 poll->events = demangle_poll(events) | EPOLLERR | EPOLLHUP;
2075
Jens Axboe221c5eb2019-01-17 09:41:58 -07002076 poll->head = NULL;
Jens Axboe8c838782019-03-12 15:48:16 -06002077 poll->done = false;
Jens Axboe221c5eb2019-01-17 09:41:58 -07002078 poll->canceled = false;
2079
2080 ipt.pt._qproc = io_poll_queue_proc;
2081 ipt.pt._key = poll->events;
2082 ipt.req = req;
2083 ipt.error = -EINVAL; /* same as no support for IOCB_CMD_POLL */
2084
2085 /* initialized the list so that we can do list_empty checks */
2086 INIT_LIST_HEAD(&poll->wait.entry);
2087 init_waitqueue_func_entry(&poll->wait, io_poll_wake);
2088
Jens Axboe36703242019-07-25 10:20:18 -06002089 INIT_LIST_HEAD(&req->list);
2090
Jens Axboe221c5eb2019-01-17 09:41:58 -07002091 mask = vfs_poll(poll->file, &ipt.pt) & poll->events;
Jens Axboe221c5eb2019-01-17 09:41:58 -07002092
2093 spin_lock_irq(&ctx->completion_lock);
Jens Axboe8c838782019-03-12 15:48:16 -06002094 if (likely(poll->head)) {
2095 spin_lock(&poll->head->lock);
2096 if (unlikely(list_empty(&poll->wait.entry))) {
2097 if (ipt.error)
2098 cancel = true;
2099 ipt.error = 0;
2100 mask = 0;
2101 }
2102 if (mask || ipt.error)
2103 list_del_init(&poll->wait.entry);
2104 else if (cancel)
2105 WRITE_ONCE(poll->canceled, true);
2106 else if (!poll->done) /* actually waiting for an event */
2107 list_add_tail(&req->list, &ctx->cancel_list);
2108 spin_unlock(&poll->head->lock);
Jens Axboe221c5eb2019-01-17 09:41:58 -07002109 }
Jens Axboe8c838782019-03-12 15:48:16 -06002110 if (mask) { /* no async, we'd stolen it */
Jens Axboe8c838782019-03-12 15:48:16 -06002111 ipt.error = 0;
Jackie Liua197f662019-11-08 08:09:12 -07002112 io_poll_complete(req, mask);
Jens Axboe8c838782019-03-12 15:48:16 -06002113 }
Jens Axboe221c5eb2019-01-17 09:41:58 -07002114 spin_unlock_irq(&ctx->completion_lock);
2115
Jens Axboe8c838782019-03-12 15:48:16 -06002116 if (mask) {
2117 io_cqring_ev_posted(ctx);
Jens Axboe89723d02019-11-05 15:32:58 -07002118 io_put_req(req, nxt);
Jens Axboe221c5eb2019-01-17 09:41:58 -07002119 }
Jens Axboe8c838782019-03-12 15:48:16 -06002120 return ipt.error;
Jens Axboe221c5eb2019-01-17 09:41:58 -07002121}
2122
Jens Axboe5262f562019-09-17 12:26:57 -06002123static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer)
2124{
2125 struct io_ring_ctx *ctx;
Jens Axboe11365042019-10-16 09:08:32 -06002126 struct io_kiocb *req;
Jens Axboe5262f562019-09-17 12:26:57 -06002127 unsigned long flags;
2128
2129 req = container_of(timer, struct io_kiocb, timeout.timer);
2130 ctx = req->ctx;
2131 atomic_inc(&ctx->cq_timeouts);
2132
2133 spin_lock_irqsave(&ctx->completion_lock, flags);
zhangyi (F)ef036812019-10-23 15:10:08 +08002134 /*
Jens Axboe11365042019-10-16 09:08:32 -06002135 * We could be racing with timeout deletion. If the list is empty,
2136 * then timeout lookup already found it and will be handling it.
zhangyi (F)ef036812019-10-23 15:10:08 +08002137 */
Jens Axboe842f9612019-10-29 12:34:10 -06002138 if (!list_empty(&req->list)) {
Jens Axboe11365042019-10-16 09:08:32 -06002139 struct io_kiocb *prev;
Jens Axboe5262f562019-09-17 12:26:57 -06002140
Jens Axboe11365042019-10-16 09:08:32 -06002141 /*
2142 * Adjust the reqs sequence before the current one because it
2143 * will consume a slot in the cq_ring and the the cq_tail
2144 * pointer will be increased, otherwise other timeout reqs may
2145 * return in advance without waiting for enough wait_nr.
2146 */
2147 prev = req;
2148 list_for_each_entry_continue_reverse(prev, &ctx->timeout_list, list)
2149 prev->sequence++;
Jens Axboe11365042019-10-16 09:08:32 -06002150 list_del_init(&req->list);
Jens Axboe11365042019-10-16 09:08:32 -06002151 }
Jens Axboe842f9612019-10-29 12:34:10 -06002152
Jens Axboe78e19bb2019-11-06 15:21:34 -07002153 io_cqring_fill_event(req, -ETIME);
Jens Axboe842f9612019-10-29 12:34:10 -06002154 io_commit_cqring(ctx);
Jens Axboe5262f562019-09-17 12:26:57 -06002155 spin_unlock_irqrestore(&ctx->completion_lock, flags);
2156
Jens Axboe842f9612019-10-29 12:34:10 -06002157 io_cqring_ev_posted(ctx);
Jens Axboef1f40852019-11-05 20:33:16 -07002158 if (req->flags & REQ_F_LINK)
2159 req->flags |= REQ_F_FAIL_LINK;
Jens Axboe842f9612019-10-29 12:34:10 -06002160 io_put_req(req, NULL);
Jens Axboe11365042019-10-16 09:08:32 -06002161 return HRTIMER_NORESTART;
2162}
2163
2164/*
2165 * Remove or update an existing timeout command
2166 */
2167static int io_timeout_remove(struct io_kiocb *req,
2168 const struct io_uring_sqe *sqe)
2169{
2170 struct io_ring_ctx *ctx = req->ctx;
2171 struct io_kiocb *treq;
2172 int ret = -ENOENT;
2173 __u64 user_data;
2174 unsigned flags;
2175
2176 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
2177 return -EINVAL;
2178 if (sqe->flags || sqe->ioprio || sqe->buf_index || sqe->len)
2179 return -EINVAL;
2180 flags = READ_ONCE(sqe->timeout_flags);
2181 if (flags)
2182 return -EINVAL;
2183
2184 user_data = READ_ONCE(sqe->addr);
2185 spin_lock_irq(&ctx->completion_lock);
2186 list_for_each_entry(treq, &ctx->timeout_list, list) {
2187 if (user_data == treq->user_data) {
2188 list_del_init(&treq->list);
2189 ret = 0;
2190 break;
2191 }
2192 }
2193
2194 /* didn't find timeout */
2195 if (ret) {
2196fill_ev:
Jens Axboe78e19bb2019-11-06 15:21:34 -07002197 io_cqring_fill_event(req, ret);
Jens Axboe11365042019-10-16 09:08:32 -06002198 io_commit_cqring(ctx);
2199 spin_unlock_irq(&ctx->completion_lock);
2200 io_cqring_ev_posted(ctx);
Jens Axboef1f40852019-11-05 20:33:16 -07002201 if (req->flags & REQ_F_LINK)
2202 req->flags |= REQ_F_FAIL_LINK;
Jens Axboe11365042019-10-16 09:08:32 -06002203 io_put_req(req, NULL);
2204 return 0;
2205 }
2206
2207 ret = hrtimer_try_to_cancel(&treq->timeout.timer);
2208 if (ret == -1) {
2209 ret = -EBUSY;
2210 goto fill_ev;
2211 }
2212
Jens Axboe78e19bb2019-11-06 15:21:34 -07002213 io_cqring_fill_event(req, 0);
2214 io_cqring_fill_event(treq, -ECANCELED);
Jens Axboe11365042019-10-16 09:08:32 -06002215 io_commit_cqring(ctx);
2216 spin_unlock_irq(&ctx->completion_lock);
Jens Axboe5262f562019-09-17 12:26:57 -06002217 io_cqring_ev_posted(ctx);
2218
Jens Axboe11365042019-10-16 09:08:32 -06002219 io_put_req(treq, NULL);
Jens Axboeba816ad2019-09-28 11:36:45 -06002220 io_put_req(req, NULL);
Jens Axboe11365042019-10-16 09:08:32 -06002221 return 0;
Jens Axboe5262f562019-09-17 12:26:57 -06002222}
2223
2224static int io_timeout(struct io_kiocb *req, const struct io_uring_sqe *sqe)
2225{
yangerkun5da0fb12019-10-15 21:59:29 +08002226 unsigned count;
Jens Axboe5262f562019-09-17 12:26:57 -06002227 struct io_ring_ctx *ctx = req->ctx;
2228 struct list_head *entry;
Jens Axboea41525a2019-10-15 16:48:15 -06002229 enum hrtimer_mode mode;
Arnd Bergmannbdf20072019-10-01 09:53:29 -06002230 struct timespec64 ts;
zhangyi (F)a1f58ba2019-10-23 15:10:09 +08002231 unsigned span = 0;
Jens Axboea41525a2019-10-15 16:48:15 -06002232 unsigned flags;
Jens Axboe5262f562019-09-17 12:26:57 -06002233
2234 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
2235 return -EINVAL;
Jens Axboea41525a2019-10-15 16:48:15 -06002236 if (sqe->flags || sqe->ioprio || sqe->buf_index || sqe->len != 1)
2237 return -EINVAL;
2238 flags = READ_ONCE(sqe->timeout_flags);
2239 if (flags & ~IORING_TIMEOUT_ABS)
Jens Axboe5262f562019-09-17 12:26:57 -06002240 return -EINVAL;
Arnd Bergmannbdf20072019-10-01 09:53:29 -06002241
2242 if (get_timespec64(&ts, u64_to_user_ptr(sqe->addr)))
Jens Axboe5262f562019-09-17 12:26:57 -06002243 return -EFAULT;
2244
Jens Axboe11365042019-10-16 09:08:32 -06002245 if (flags & IORING_TIMEOUT_ABS)
2246 mode = HRTIMER_MODE_ABS;
2247 else
2248 mode = HRTIMER_MODE_REL;
2249
2250 hrtimer_init(&req->timeout.timer, CLOCK_MONOTONIC, mode);
2251
Jens Axboe5262f562019-09-17 12:26:57 -06002252 /*
2253 * sqe->off holds how many events that need to occur for this
2254 * timeout event to be satisfied.
2255 */
2256 count = READ_ONCE(sqe->off);
2257 if (!count)
2258 count = 1;
2259
2260 req->sequence = ctx->cached_sq_head + count - 1;
yangerkun5da0fb12019-10-15 21:59:29 +08002261 /* reuse it to store the count */
2262 req->submit.sequence = count;
Jens Axboe5262f562019-09-17 12:26:57 -06002263 req->flags |= REQ_F_TIMEOUT;
2264
2265 /*
2266 * Insertion sort, ensuring the first entry in the list is always
2267 * the one we need first.
2268 */
Jens Axboe5262f562019-09-17 12:26:57 -06002269 spin_lock_irq(&ctx->completion_lock);
2270 list_for_each_prev(entry, &ctx->timeout_list) {
2271 struct io_kiocb *nxt = list_entry(entry, struct io_kiocb, list);
yangerkun5da0fb12019-10-15 21:59:29 +08002272 unsigned nxt_sq_head;
2273 long long tmp, tmp_nxt;
Jens Axboe5262f562019-09-17 12:26:57 -06002274
yangerkun5da0fb12019-10-15 21:59:29 +08002275 /*
2276 * Since cached_sq_head + count - 1 can overflow, use type long
2277 * long to store it.
2278 */
2279 tmp = (long long)ctx->cached_sq_head + count - 1;
2280 nxt_sq_head = nxt->sequence - nxt->submit.sequence + 1;
2281 tmp_nxt = (long long)nxt_sq_head + nxt->submit.sequence - 1;
2282
2283 /*
2284 * cached_sq_head may overflow, and it will never overflow twice
2285 * once there is some timeout req still be valid.
2286 */
2287 if (ctx->cached_sq_head < nxt_sq_head)
yangerkun8b07a652019-10-17 12:12:35 +08002288 tmp += UINT_MAX;
yangerkun5da0fb12019-10-15 21:59:29 +08002289
zhangyi (F)a1f58ba2019-10-23 15:10:09 +08002290 if (tmp > tmp_nxt)
Jens Axboe5262f562019-09-17 12:26:57 -06002291 break;
zhangyi (F)a1f58ba2019-10-23 15:10:09 +08002292
2293 /*
2294 * Sequence of reqs after the insert one and itself should
2295 * be adjusted because each timeout req consumes a slot.
2296 */
2297 span++;
2298 nxt->sequence++;
Jens Axboe5262f562019-09-17 12:26:57 -06002299 }
zhangyi (F)a1f58ba2019-10-23 15:10:09 +08002300 req->sequence -= span;
Jens Axboe5262f562019-09-17 12:26:57 -06002301 list_add(&req->list, entry);
Jens Axboe5262f562019-09-17 12:26:57 -06002302 req->timeout.timer.function = io_timeout_fn;
Jens Axboea41525a2019-10-15 16:48:15 -06002303 hrtimer_start(&req->timeout.timer, timespec64_to_ktime(ts), mode);
Jens Axboe842f9612019-10-29 12:34:10 -06002304 spin_unlock_irq(&ctx->completion_lock);
Jens Axboe5262f562019-09-17 12:26:57 -06002305 return 0;
2306}
2307
Jens Axboe62755e32019-10-28 21:49:21 -06002308static bool io_cancel_cb(struct io_wq_work *work, void *data)
2309{
2310 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
2311
2312 return req->user_data == (unsigned long) data;
2313}
2314
Jens Axboee977d6d2019-11-05 12:39:45 -07002315static int io_async_cancel_one(struct io_ring_ctx *ctx, void *sqe_addr)
Jens Axboe62755e32019-10-28 21:49:21 -06002316{
Jens Axboe62755e32019-10-28 21:49:21 -06002317 enum io_wq_cancel cancel_ret;
Jens Axboe62755e32019-10-28 21:49:21 -06002318 int ret = 0;
2319
Jens Axboe62755e32019-10-28 21:49:21 -06002320 cancel_ret = io_wq_cancel_cb(ctx->io_wq, io_cancel_cb, sqe_addr);
2321 switch (cancel_ret) {
2322 case IO_WQ_CANCEL_OK:
2323 ret = 0;
2324 break;
2325 case IO_WQ_CANCEL_RUNNING:
2326 ret = -EALREADY;
2327 break;
2328 case IO_WQ_CANCEL_NOTFOUND:
2329 ret = -ENOENT;
2330 break;
2331 }
2332
Jens Axboee977d6d2019-11-05 12:39:45 -07002333 return ret;
2334}
2335
2336static int io_async_cancel(struct io_kiocb *req, const struct io_uring_sqe *sqe,
2337 struct io_kiocb **nxt)
2338{
2339 struct io_ring_ctx *ctx = req->ctx;
2340 void *sqe_addr;
2341 int ret;
2342
2343 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
2344 return -EINVAL;
2345 if (sqe->flags || sqe->ioprio || sqe->off || sqe->len ||
2346 sqe->cancel_flags)
2347 return -EINVAL;
2348
2349 sqe_addr = (void *) (unsigned long) READ_ONCE(sqe->addr);
2350 ret = io_async_cancel_one(ctx, sqe_addr);
2351
Jens Axboe62755e32019-10-28 21:49:21 -06002352 if (ret < 0 && (req->flags & REQ_F_LINK))
2353 req->flags |= REQ_F_FAIL_LINK;
Jens Axboe78e19bb2019-11-06 15:21:34 -07002354 io_cqring_add_event(req, ret);
Jens Axboe62755e32019-10-28 21:49:21 -06002355 io_put_req(req, nxt);
2356 return 0;
2357}
2358
Jackie Liua197f662019-11-08 08:09:12 -07002359static int io_req_defer(struct io_kiocb *req)
Jens Axboede0617e2019-04-06 21:51:27 -06002360{
Pavel Begunkov267bc902019-11-07 01:41:08 +03002361 const struct io_uring_sqe *sqe = req->submit.sqe;
Jens Axboede0617e2019-04-06 21:51:27 -06002362 struct io_uring_sqe *sqe_copy;
Jackie Liua197f662019-11-08 08:09:12 -07002363 struct io_ring_ctx *ctx = req->ctx;
Jens Axboede0617e2019-04-06 21:51:27 -06002364
Jackie Liua197f662019-11-08 08:09:12 -07002365 if (!io_sequence_defer(req) && list_empty(&ctx->defer_list))
Jens Axboede0617e2019-04-06 21:51:27 -06002366 return 0;
2367
2368 sqe_copy = kmalloc(sizeof(*sqe_copy), GFP_KERNEL);
2369 if (!sqe_copy)
2370 return -EAGAIN;
2371
2372 spin_lock_irq(&ctx->completion_lock);
Jackie Liua197f662019-11-08 08:09:12 -07002373 if (!io_sequence_defer(req) && list_empty(&ctx->defer_list)) {
Jens Axboede0617e2019-04-06 21:51:27 -06002374 spin_unlock_irq(&ctx->completion_lock);
2375 kfree(sqe_copy);
2376 return 0;
2377 }
2378
2379 memcpy(sqe_copy, sqe, sizeof(*sqe_copy));
2380 req->submit.sqe = sqe_copy;
2381
Dmitrii Dolgovc826bd72019-10-15 19:02:01 +02002382 trace_io_uring_defer(ctx, req, false);
Jens Axboede0617e2019-04-06 21:51:27 -06002383 list_add_tail(&req->list, &ctx->defer_list);
2384 spin_unlock_irq(&ctx->completion_lock);
2385 return -EIOCBQUEUED;
2386}
2387
Jackie Liua197f662019-11-08 08:09:12 -07002388static int __io_submit_sqe(struct io_kiocb *req, struct io_kiocb **nxt,
2389 bool force_nonblock)
Jens Axboe2b188cc2019-01-07 10:46:33 -07002390{
Jens Axboee0c5c572019-03-12 10:18:47 -06002391 int ret, opcode;
Pavel Begunkov267bc902019-11-07 01:41:08 +03002392 struct sqe_submit *s = &req->submit;
Jackie Liua197f662019-11-08 08:09:12 -07002393 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002394
Jens Axboe2b188cc2019-01-07 10:46:33 -07002395 opcode = READ_ONCE(s->sqe->opcode);
2396 switch (opcode) {
2397 case IORING_OP_NOP:
Jens Axboe78e19bb2019-11-06 15:21:34 -07002398 ret = io_nop(req);
Jens Axboe2b188cc2019-01-07 10:46:33 -07002399 break;
2400 case IORING_OP_READV:
Jens Axboeedafcce2019-01-09 09:16:05 -07002401 if (unlikely(s->sqe->buf_index))
2402 return -EINVAL;
Pavel Begunkov267bc902019-11-07 01:41:08 +03002403 ret = io_read(req, nxt, force_nonblock);
Jens Axboe2b188cc2019-01-07 10:46:33 -07002404 break;
2405 case IORING_OP_WRITEV:
Jens Axboeedafcce2019-01-09 09:16:05 -07002406 if (unlikely(s->sqe->buf_index))
2407 return -EINVAL;
Pavel Begunkov267bc902019-11-07 01:41:08 +03002408 ret = io_write(req, nxt, force_nonblock);
Jens Axboeedafcce2019-01-09 09:16:05 -07002409 break;
2410 case IORING_OP_READ_FIXED:
Pavel Begunkov267bc902019-11-07 01:41:08 +03002411 ret = io_read(req, nxt, force_nonblock);
Jens Axboeedafcce2019-01-09 09:16:05 -07002412 break;
2413 case IORING_OP_WRITE_FIXED:
Pavel Begunkov267bc902019-11-07 01:41:08 +03002414 ret = io_write(req, nxt, force_nonblock);
Jens Axboe2b188cc2019-01-07 10:46:33 -07002415 break;
Christoph Hellwigc992fe22019-01-11 09:43:02 -07002416 case IORING_OP_FSYNC:
Jens Axboeba816ad2019-09-28 11:36:45 -06002417 ret = io_fsync(req, s->sqe, nxt, force_nonblock);
Christoph Hellwigc992fe22019-01-11 09:43:02 -07002418 break;
Jens Axboe221c5eb2019-01-17 09:41:58 -07002419 case IORING_OP_POLL_ADD:
Jens Axboe89723d02019-11-05 15:32:58 -07002420 ret = io_poll_add(req, s->sqe, nxt);
Jens Axboe221c5eb2019-01-17 09:41:58 -07002421 break;
2422 case IORING_OP_POLL_REMOVE:
2423 ret = io_poll_remove(req, s->sqe);
2424 break;
Jens Axboe5d17b4a2019-04-09 14:56:44 -06002425 case IORING_OP_SYNC_FILE_RANGE:
Jens Axboeba816ad2019-09-28 11:36:45 -06002426 ret = io_sync_file_range(req, s->sqe, nxt, force_nonblock);
Jens Axboe5d17b4a2019-04-09 14:56:44 -06002427 break;
Jens Axboe0fa03c62019-04-19 13:34:07 -06002428 case IORING_OP_SENDMSG:
Jens Axboeba816ad2019-09-28 11:36:45 -06002429 ret = io_sendmsg(req, s->sqe, nxt, force_nonblock);
Jens Axboe0fa03c62019-04-19 13:34:07 -06002430 break;
Jens Axboeaa1fa282019-04-19 13:38:09 -06002431 case IORING_OP_RECVMSG:
Jens Axboeba816ad2019-09-28 11:36:45 -06002432 ret = io_recvmsg(req, s->sqe, nxt, force_nonblock);
Jens Axboeaa1fa282019-04-19 13:38:09 -06002433 break;
Jens Axboe5262f562019-09-17 12:26:57 -06002434 case IORING_OP_TIMEOUT:
2435 ret = io_timeout(req, s->sqe);
2436 break;
Jens Axboe11365042019-10-16 09:08:32 -06002437 case IORING_OP_TIMEOUT_REMOVE:
2438 ret = io_timeout_remove(req, s->sqe);
2439 break;
Jens Axboe17f2fe32019-10-17 14:42:58 -06002440 case IORING_OP_ACCEPT:
2441 ret = io_accept(req, s->sqe, nxt, force_nonblock);
2442 break;
Jens Axboe62755e32019-10-28 21:49:21 -06002443 case IORING_OP_ASYNC_CANCEL:
2444 ret = io_async_cancel(req, s->sqe, nxt);
2445 break;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002446 default:
2447 ret = -EINVAL;
2448 break;
2449 }
2450
Jens Axboedef596e2019-01-09 08:59:42 -07002451 if (ret)
2452 return ret;
2453
2454 if (ctx->flags & IORING_SETUP_IOPOLL) {
Jens Axboe9e645e112019-05-10 16:07:28 -06002455 if (req->result == -EAGAIN)
Jens Axboedef596e2019-01-09 08:59:42 -07002456 return -EAGAIN;
2457
2458 /* workqueue context doesn't hold uring_lock, grab it now */
Jackie Liuba5290c2019-10-09 09:19:59 +08002459 if (s->in_async)
Jens Axboedef596e2019-01-09 08:59:42 -07002460 mutex_lock(&ctx->uring_lock);
2461 io_iopoll_req_issued(req);
Jackie Liuba5290c2019-10-09 09:19:59 +08002462 if (s->in_async)
Jens Axboedef596e2019-01-09 08:59:42 -07002463 mutex_unlock(&ctx->uring_lock);
2464 }
2465
2466 return 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002467}
2468
Jens Axboe561fb042019-10-24 07:25:42 -06002469static void io_wq_submit_work(struct io_wq_work **workptr)
Jens Axboe31b51512019-01-18 22:56:34 -07002470{
Jens Axboe561fb042019-10-24 07:25:42 -06002471 struct io_wq_work *work = *workptr;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002472 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
Jens Axboe561fb042019-10-24 07:25:42 -06002473 struct sqe_submit *s = &req->submit;
2474 const struct io_uring_sqe *sqe = s->sqe;
2475 struct io_kiocb *nxt = NULL;
2476 int ret = 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002477
Jens Axboe561fb042019-10-24 07:25:42 -06002478 /* Ensure we clear previously set non-block flag */
2479 req->rw.ki_flags &= ~IOCB_NOWAIT;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002480
Jens Axboe561fb042019-10-24 07:25:42 -06002481 if (work->flags & IO_WQ_WORK_CANCEL)
2482 ret = -ECANCELED;
Jens Axboe31b51512019-01-18 22:56:34 -07002483
Jens Axboe561fb042019-10-24 07:25:42 -06002484 if (!ret) {
2485 s->has_user = (work->flags & IO_WQ_WORK_HAS_MM) != 0;
2486 s->in_async = true;
2487 do {
Jackie Liua197f662019-11-08 08:09:12 -07002488 ret = __io_submit_sqe(req, &nxt, false);
Jens Axboe561fb042019-10-24 07:25:42 -06002489 /*
2490 * We can get EAGAIN for polled IO even though we're
2491 * forcing a sync submission from here, since we can't
2492 * wait for request slots on the block side.
2493 */
2494 if (ret != -EAGAIN)
2495 break;
2496 cond_resched();
2497 } while (1);
2498 }
Jens Axboe31b51512019-01-18 22:56:34 -07002499
Jens Axboe561fb042019-10-24 07:25:42 -06002500 /* drop submission reference */
2501 io_put_req(req, NULL);
Jens Axboe817869d2019-04-30 14:44:05 -06002502
Jens Axboe561fb042019-10-24 07:25:42 -06002503 if (ret) {
Jens Axboef1f40852019-11-05 20:33:16 -07002504 if (req->flags & REQ_F_LINK)
2505 req->flags |= REQ_F_FAIL_LINK;
Jens Axboe78e19bb2019-11-06 15:21:34 -07002506 io_cqring_add_event(req, ret);
Jens Axboeba816ad2019-09-28 11:36:45 -06002507 io_put_req(req, NULL);
Jens Axboeedafcce2019-01-09 09:16:05 -07002508 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07002509
Jens Axboe561fb042019-10-24 07:25:42 -06002510 /* async context always use a copy of the sqe */
2511 kfree(sqe);
2512
2513 /* if a dependent link is ready, pass it back */
2514 if (!ret && nxt) {
2515 io_prep_async_work(nxt);
2516 *workptr = &nxt->work;
Jens Axboeedafcce2019-01-09 09:16:05 -07002517 }
Jens Axboe31b51512019-01-18 22:56:34 -07002518}
Jens Axboe2b188cc2019-01-07 10:46:33 -07002519
Jens Axboe09bb8392019-03-13 12:39:28 -06002520static bool io_op_needs_file(const struct io_uring_sqe *sqe)
2521{
2522 int op = READ_ONCE(sqe->opcode);
2523
2524 switch (op) {
2525 case IORING_OP_NOP:
2526 case IORING_OP_POLL_REMOVE:
2527 return false;
2528 default:
2529 return true;
2530 }
2531}
2532
Jens Axboe65e19f52019-10-26 07:20:21 -06002533static inline struct file *io_file_from_index(struct io_ring_ctx *ctx,
2534 int index)
2535{
2536 struct fixed_file_table *table;
2537
2538 table = &ctx->file_table[index >> IORING_FILE_TABLE_SHIFT];
2539 return table->files[index & IORING_FILE_TABLE_MASK];
2540}
2541
Jackie Liua197f662019-11-08 08:09:12 -07002542static int io_req_set_file(struct io_submit_state *state, struct io_kiocb *req)
Jens Axboe09bb8392019-03-13 12:39:28 -06002543{
Pavel Begunkov267bc902019-11-07 01:41:08 +03002544 struct sqe_submit *s = &req->submit;
Jackie Liua197f662019-11-08 08:09:12 -07002545 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe09bb8392019-03-13 12:39:28 -06002546 unsigned flags;
2547 int fd;
2548
2549 flags = READ_ONCE(s->sqe->flags);
2550 fd = READ_ONCE(s->sqe->fd);
2551
Jackie Liu4fe2c962019-09-09 20:50:40 +08002552 if (flags & IOSQE_IO_DRAIN)
Jens Axboede0617e2019-04-06 21:51:27 -06002553 req->flags |= REQ_F_IO_DRAIN;
Jackie Liu4fe2c962019-09-09 20:50:40 +08002554 /*
2555 * All io need record the previous position, if LINK vs DARIN,
2556 * it can be used to mark the position of the first IO in the
2557 * link list.
2558 */
2559 req->sequence = s->sequence;
Jens Axboede0617e2019-04-06 21:51:27 -06002560
Jens Axboe60c112b2019-06-21 10:20:18 -06002561 if (!io_op_needs_file(s->sqe))
Jens Axboe09bb8392019-03-13 12:39:28 -06002562 return 0;
Jens Axboe09bb8392019-03-13 12:39:28 -06002563
2564 if (flags & IOSQE_FIXED_FILE) {
Jens Axboe65e19f52019-10-26 07:20:21 -06002565 if (unlikely(!ctx->file_table ||
Jens Axboe09bb8392019-03-13 12:39:28 -06002566 (unsigned) fd >= ctx->nr_user_files))
2567 return -EBADF;
Jens Axboeb7620122019-10-26 07:22:55 -06002568 fd = array_index_nospec(fd, ctx->nr_user_files);
Jens Axboe65e19f52019-10-26 07:20:21 -06002569 req->file = io_file_from_index(ctx, fd);
2570 if (!req->file)
Jens Axboe08a45172019-10-03 08:11:03 -06002571 return -EBADF;
Jens Axboe09bb8392019-03-13 12:39:28 -06002572 req->flags |= REQ_F_FIXED_FILE;
2573 } else {
2574 if (s->needs_fixed_file)
2575 return -EBADF;
Dmitrii Dolgovc826bd72019-10-15 19:02:01 +02002576 trace_io_uring_file_get(ctx, fd);
Jens Axboe09bb8392019-03-13 12:39:28 -06002577 req->file = io_file_get(state, fd);
2578 if (unlikely(!req->file))
2579 return -EBADF;
2580 }
2581
2582 return 0;
2583}
2584
Jackie Liua197f662019-11-08 08:09:12 -07002585static int io_grab_files(struct io_kiocb *req)
Jens Axboefcb323c2019-10-24 12:39:47 -06002586{
2587 int ret = -EBADF;
Jackie Liua197f662019-11-08 08:09:12 -07002588 struct io_ring_ctx *ctx = req->ctx;
Jens Axboefcb323c2019-10-24 12:39:47 -06002589
2590 rcu_read_lock();
2591 spin_lock_irq(&ctx->inflight_lock);
2592 /*
2593 * We use the f_ops->flush() handler to ensure that we can flush
2594 * out work accessing these files if the fd is closed. Check if
2595 * the fd has changed since we started down this path, and disallow
2596 * this operation if it has.
2597 */
2598 if (fcheck(req->submit.ring_fd) == req->submit.ring_file) {
2599 list_add(&req->inflight_entry, &ctx->inflight_list);
2600 req->flags |= REQ_F_INFLIGHT;
2601 req->work.files = current->files;
2602 ret = 0;
2603 }
2604 spin_unlock_irq(&ctx->inflight_lock);
2605 rcu_read_unlock();
2606
2607 return ret;
2608}
2609
Jens Axboe2665abf2019-11-05 12:40:47 -07002610static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer)
2611{
2612 struct io_kiocb *req = container_of(timer, struct io_kiocb,
2613 timeout.timer);
2614 struct io_ring_ctx *ctx = req->ctx;
2615 struct io_kiocb *prev = NULL;
2616 unsigned long flags;
2617 int ret = -ETIME;
2618
2619 spin_lock_irqsave(&ctx->completion_lock, flags);
2620
2621 /*
2622 * We don't expect the list to be empty, that will only happen if we
2623 * race with the completion of the linked work.
2624 */
2625 if (!list_empty(&req->list)) {
2626 prev = list_entry(req->list.prev, struct io_kiocb, link_list);
2627 list_del_init(&req->list);
2628 }
2629
2630 spin_unlock_irqrestore(&ctx->completion_lock, flags);
2631
2632 if (prev) {
2633 void *user_data = (void *) (unsigned long) prev->user_data;
2634 ret = io_async_cancel_one(ctx, user_data);
2635 }
2636
Jens Axboe78e19bb2019-11-06 15:21:34 -07002637 io_cqring_add_event(req, ret);
Jens Axboe2665abf2019-11-05 12:40:47 -07002638 io_put_req(req, NULL);
2639 return HRTIMER_NORESTART;
2640}
2641
2642static int io_queue_linked_timeout(struct io_kiocb *req, struct io_kiocb *nxt)
2643{
2644 const struct io_uring_sqe *sqe = nxt->submit.sqe;
2645 enum hrtimer_mode mode;
2646 struct timespec64 ts;
2647 int ret = -EINVAL;
2648
2649 if (sqe->ioprio || sqe->buf_index || sqe->len != 1 || sqe->off)
2650 goto err;
2651 if (sqe->timeout_flags & ~IORING_TIMEOUT_ABS)
2652 goto err;
2653 if (get_timespec64(&ts, u64_to_user_ptr(sqe->addr))) {
2654 ret = -EFAULT;
2655 goto err;
2656 }
2657
2658 req->flags |= REQ_F_LINK_TIMEOUT;
2659
2660 if (sqe->timeout_flags & IORING_TIMEOUT_ABS)
2661 mode = HRTIMER_MODE_ABS;
2662 else
2663 mode = HRTIMER_MODE_REL;
2664 hrtimer_init(&nxt->timeout.timer, CLOCK_MONOTONIC, mode);
2665 nxt->timeout.timer.function = io_link_timeout_fn;
2666 hrtimer_start(&nxt->timeout.timer, timespec64_to_ktime(ts), mode);
2667 ret = 0;
2668err:
2669 /* drop submission reference */
2670 io_put_req(nxt, NULL);
2671
2672 if (ret) {
2673 struct io_ring_ctx *ctx = req->ctx;
2674
2675 /*
2676 * Break the link and fail linked timeout, parent will get
2677 * failed by the regular submission path.
2678 */
2679 list_del(&nxt->list);
Jens Axboe78e19bb2019-11-06 15:21:34 -07002680 io_cqring_fill_event(nxt, ret);
Jens Axboe2665abf2019-11-05 12:40:47 -07002681 trace_io_uring_fail_link(req, nxt);
2682 io_commit_cqring(ctx);
2683 io_put_req(nxt, NULL);
2684 ret = -ECANCELED;
2685 }
2686
2687 return ret;
2688}
2689
2690static inline struct io_kiocb *io_get_linked_timeout(struct io_kiocb *req)
2691{
2692 struct io_kiocb *nxt;
2693
2694 if (!(req->flags & REQ_F_LINK))
2695 return NULL;
2696
2697 nxt = list_first_entry_or_null(&req->link_list, struct io_kiocb, list);
2698 if (nxt && nxt->submit.sqe->opcode == IORING_OP_LINK_TIMEOUT)
2699 return nxt;
2700
2701 return NULL;
2702}
2703
Jackie Liua197f662019-11-08 08:09:12 -07002704static int __io_queue_sqe(struct io_kiocb *req)
Jens Axboe2b188cc2019-01-07 10:46:33 -07002705{
Jens Axboe2665abf2019-11-05 12:40:47 -07002706 struct io_kiocb *nxt;
Jens Axboee0c5c572019-03-12 10:18:47 -06002707 int ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002708
Jens Axboe2665abf2019-11-05 12:40:47 -07002709 nxt = io_get_linked_timeout(req);
2710 if (unlikely(nxt)) {
2711 ret = io_queue_linked_timeout(req, nxt);
2712 if (ret)
2713 goto err;
2714 }
2715
Jackie Liua197f662019-11-08 08:09:12 -07002716 ret = __io_submit_sqe(req, NULL, true);
Jens Axboe491381ce2019-10-17 09:20:46 -06002717
2718 /*
2719 * We async punt it if the file wasn't marked NOWAIT, or if the file
2720 * doesn't support non-blocking read/write attempts
2721 */
2722 if (ret == -EAGAIN && (!(req->flags & REQ_F_NOWAIT) ||
2723 (req->flags & REQ_F_MUST_PUNT))) {
Pavel Begunkov267bc902019-11-07 01:41:08 +03002724 struct sqe_submit *s = &req->submit;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002725 struct io_uring_sqe *sqe_copy;
2726
Jackie Liu954dab12019-09-18 10:37:52 +08002727 sqe_copy = kmemdup(s->sqe, sizeof(*sqe_copy), GFP_KERNEL);
Jens Axboe2b188cc2019-01-07 10:46:33 -07002728 if (sqe_copy) {
Jens Axboe2b188cc2019-01-07 10:46:33 -07002729 s->sqe = sqe_copy;
Jens Axboefcb323c2019-10-24 12:39:47 -06002730 if (req->work.flags & IO_WQ_WORK_NEEDS_FILES) {
Jackie Liua197f662019-11-08 08:09:12 -07002731 ret = io_grab_files(req);
Jens Axboefcb323c2019-10-24 12:39:47 -06002732 if (ret) {
2733 kfree(sqe_copy);
2734 goto err;
2735 }
2736 }
Jens Axboee65ef562019-03-12 10:16:44 -06002737
2738 /*
2739 * Queued up for async execution, worker will release
Jens Axboe9e645e112019-05-10 16:07:28 -06002740 * submit reference when the iocb is actually submitted.
Jens Axboee65ef562019-03-12 10:16:44 -06002741 */
Jackie Liua197f662019-11-08 08:09:12 -07002742 io_queue_async_work(req);
Jens Axboee65ef562019-03-12 10:16:44 -06002743 return 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002744 }
2745 }
Jens Axboee65ef562019-03-12 10:16:44 -06002746
2747 /* drop submission reference */
Jens Axboefcb323c2019-10-24 12:39:47 -06002748err:
Jens Axboeba816ad2019-09-28 11:36:45 -06002749 io_put_req(req, NULL);
Jens Axboee65ef562019-03-12 10:16:44 -06002750
2751 /* and drop final reference, if we failed */
Jens Axboe9e645e112019-05-10 16:07:28 -06002752 if (ret) {
Jens Axboe78e19bb2019-11-06 15:21:34 -07002753 io_cqring_add_event(req, ret);
Jens Axboe9e645e112019-05-10 16:07:28 -06002754 if (req->flags & REQ_F_LINK)
2755 req->flags |= REQ_F_FAIL_LINK;
Jens Axboeba816ad2019-09-28 11:36:45 -06002756 io_put_req(req, NULL);
Jens Axboe9e645e112019-05-10 16:07:28 -06002757 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07002758
2759 return ret;
2760}
2761
Jackie Liua197f662019-11-08 08:09:12 -07002762static int io_queue_sqe(struct io_kiocb *req)
Jackie Liu4fe2c962019-09-09 20:50:40 +08002763{
2764 int ret;
2765
Jackie Liua197f662019-11-08 08:09:12 -07002766 ret = io_req_defer(req);
Jackie Liu4fe2c962019-09-09 20:50:40 +08002767 if (ret) {
2768 if (ret != -EIOCBQUEUED) {
Jens Axboe78e19bb2019-11-06 15:21:34 -07002769 io_cqring_add_event(req, ret);
2770 io_double_put_req(req);
Jackie Liu4fe2c962019-09-09 20:50:40 +08002771 }
2772 return 0;
2773 }
2774
Jackie Liua197f662019-11-08 08:09:12 -07002775 return __io_queue_sqe(req);
Jackie Liu4fe2c962019-09-09 20:50:40 +08002776}
2777
Jackie Liua197f662019-11-08 08:09:12 -07002778static int io_queue_link_head(struct io_kiocb *req, struct io_kiocb *shadow)
Jackie Liu4fe2c962019-09-09 20:50:40 +08002779{
2780 int ret;
2781 int need_submit = false;
Jackie Liua197f662019-11-08 08:09:12 -07002782 struct io_ring_ctx *ctx = req->ctx;
Jackie Liu4fe2c962019-09-09 20:50:40 +08002783
2784 if (!shadow)
Jackie Liua197f662019-11-08 08:09:12 -07002785 return io_queue_sqe(req);
Jackie Liu4fe2c962019-09-09 20:50:40 +08002786
2787 /*
2788 * Mark the first IO in link list as DRAIN, let all the following
2789 * IOs enter the defer list. all IO needs to be completed before link
2790 * list.
2791 */
2792 req->flags |= REQ_F_IO_DRAIN;
Jackie Liua197f662019-11-08 08:09:12 -07002793 ret = io_req_defer(req);
Jackie Liu4fe2c962019-09-09 20:50:40 +08002794 if (ret) {
2795 if (ret != -EIOCBQUEUED) {
Jens Axboe78e19bb2019-11-06 15:21:34 -07002796 io_cqring_add_event(req, ret);
2797 io_double_put_req(req);
Pavel Begunkov7b202382019-10-27 22:10:36 +03002798 __io_free_req(shadow);
Jackie Liu4fe2c962019-09-09 20:50:40 +08002799 return 0;
2800 }
2801 } else {
2802 /*
2803 * If ret == 0 means that all IOs in front of link io are
2804 * running done. let's queue link head.
2805 */
2806 need_submit = true;
2807 }
2808
2809 /* Insert shadow req to defer_list, blocking next IOs */
2810 spin_lock_irq(&ctx->completion_lock);
Dmitrii Dolgovc826bd72019-10-15 19:02:01 +02002811 trace_io_uring_defer(ctx, shadow, true);
Jackie Liu4fe2c962019-09-09 20:50:40 +08002812 list_add_tail(&shadow->list, &ctx->defer_list);
2813 spin_unlock_irq(&ctx->completion_lock);
2814
2815 if (need_submit)
Jackie Liua197f662019-11-08 08:09:12 -07002816 return __io_queue_sqe(req);
Jackie Liu4fe2c962019-09-09 20:50:40 +08002817
2818 return 0;
2819}
2820
Jens Axboe9e645e112019-05-10 16:07:28 -06002821#define SQE_VALID_FLAGS (IOSQE_FIXED_FILE|IOSQE_IO_DRAIN|IOSQE_IO_LINK)
2822
Jackie Liua197f662019-11-08 08:09:12 -07002823static void io_submit_sqe(struct io_kiocb *req, struct io_submit_state *state,
2824 struct io_kiocb **link)
Jens Axboe9e645e112019-05-10 16:07:28 -06002825{
2826 struct io_uring_sqe *sqe_copy;
Pavel Begunkov267bc902019-11-07 01:41:08 +03002827 struct sqe_submit *s = &req->submit;
Jackie Liua197f662019-11-08 08:09:12 -07002828 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe9e645e112019-05-10 16:07:28 -06002829 int ret;
2830
Jens Axboe78e19bb2019-11-06 15:21:34 -07002831 req->user_data = s->sqe->user_data;
2832
Jens Axboe9e645e112019-05-10 16:07:28 -06002833 /* enforce forwards compatibility on users */
2834 if (unlikely(s->sqe->flags & ~SQE_VALID_FLAGS)) {
2835 ret = -EINVAL;
Pavel Begunkov196be952019-11-07 01:41:06 +03002836 goto err_req;
Jens Axboe9e645e112019-05-10 16:07:28 -06002837 }
2838
Jackie Liua197f662019-11-08 08:09:12 -07002839 ret = io_req_set_file(state, req);
Jens Axboe9e645e112019-05-10 16:07:28 -06002840 if (unlikely(ret)) {
2841err_req:
Jens Axboe78e19bb2019-11-06 15:21:34 -07002842 io_cqring_add_event(req, ret);
2843 io_double_put_req(req);
Jens Axboe9e645e112019-05-10 16:07:28 -06002844 return;
2845 }
2846
Jens Axboe9e645e112019-05-10 16:07:28 -06002847 /*
2848 * If we already have a head request, queue this one for async
2849 * submittal once the head completes. If we don't have a head but
2850 * IOSQE_IO_LINK is set in the sqe, start a new head. This one will be
2851 * submitted sync once the chain is complete. If none of those
2852 * conditions are true (normal request), then just queue it.
2853 */
2854 if (*link) {
2855 struct io_kiocb *prev = *link;
2856
2857 sqe_copy = kmemdup(s->sqe, sizeof(*sqe_copy), GFP_KERNEL);
2858 if (!sqe_copy) {
2859 ret = -EAGAIN;
2860 goto err_req;
2861 }
2862
2863 s->sqe = sqe_copy;
Dmitrii Dolgovc826bd72019-10-15 19:02:01 +02002864 trace_io_uring_link(ctx, req, prev);
Jens Axboe9e645e112019-05-10 16:07:28 -06002865 list_add_tail(&req->list, &prev->link_list);
2866 } else if (s->sqe->flags & IOSQE_IO_LINK) {
2867 req->flags |= REQ_F_LINK;
2868
Jens Axboe9e645e112019-05-10 16:07:28 -06002869 INIT_LIST_HEAD(&req->link_list);
2870 *link = req;
Jens Axboe2665abf2019-11-05 12:40:47 -07002871 } else if (READ_ONCE(s->sqe->opcode) == IORING_OP_LINK_TIMEOUT) {
2872 /* Only valid as a linked SQE */
2873 ret = -EINVAL;
2874 goto err_req;
Jens Axboe9e645e112019-05-10 16:07:28 -06002875 } else {
Jackie Liua197f662019-11-08 08:09:12 -07002876 io_queue_sqe(req);
Jens Axboe9e645e112019-05-10 16:07:28 -06002877 }
2878}
2879
Jens Axboe9a56a232019-01-09 09:06:50 -07002880/*
2881 * Batched submission is done, ensure local IO is flushed out.
2882 */
2883static void io_submit_state_end(struct io_submit_state *state)
2884{
2885 blk_finish_plug(&state->plug);
Jens Axboe3d6770f2019-04-13 11:50:54 -06002886 io_file_put(state);
Jens Axboe2579f912019-01-09 09:10:43 -07002887 if (state->free_reqs)
2888 kmem_cache_free_bulk(req_cachep, state->free_reqs,
2889 &state->reqs[state->cur_req]);
Jens Axboe9a56a232019-01-09 09:06:50 -07002890}
2891
2892/*
2893 * Start submission side cache.
2894 */
2895static void io_submit_state_start(struct io_submit_state *state,
2896 struct io_ring_ctx *ctx, unsigned max_ios)
2897{
2898 blk_start_plug(&state->plug);
Jens Axboe2579f912019-01-09 09:10:43 -07002899 state->free_reqs = 0;
Jens Axboe9a56a232019-01-09 09:06:50 -07002900 state->file = NULL;
2901 state->ios_left = max_ios;
2902}
2903
Jens Axboe2b188cc2019-01-07 10:46:33 -07002904static void io_commit_sqring(struct io_ring_ctx *ctx)
2905{
Hristo Venev75b28af2019-08-26 17:23:46 +00002906 struct io_rings *rings = ctx->rings;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002907
Hristo Venev75b28af2019-08-26 17:23:46 +00002908 if (ctx->cached_sq_head != READ_ONCE(rings->sq.head)) {
Jens Axboe2b188cc2019-01-07 10:46:33 -07002909 /*
2910 * Ensure any loads from the SQEs are done at this point,
2911 * since once we write the new head, the application could
2912 * write new data to them.
2913 */
Hristo Venev75b28af2019-08-26 17:23:46 +00002914 smp_store_release(&rings->sq.head, ctx->cached_sq_head);
Jens Axboe2b188cc2019-01-07 10:46:33 -07002915 }
2916}
2917
2918/*
Jens Axboe2b188cc2019-01-07 10:46:33 -07002919 * Fetch an sqe, if one is available. Note that s->sqe will point to memory
2920 * that is mapped by userspace. This means that care needs to be taken to
2921 * ensure that reads are stable, as we cannot rely on userspace always
2922 * being a good citizen. If members of the sqe are validated and then later
2923 * used, it's important that those reads are done through READ_ONCE() to
2924 * prevent a re-load down the line.
2925 */
2926static bool io_get_sqring(struct io_ring_ctx *ctx, struct sqe_submit *s)
2927{
Hristo Venev75b28af2019-08-26 17:23:46 +00002928 struct io_rings *rings = ctx->rings;
2929 u32 *sq_array = ctx->sq_array;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002930 unsigned head;
2931
2932 /*
2933 * The cached sq head (or cq tail) serves two purposes:
2934 *
2935 * 1) allows us to batch the cost of updating the user visible
2936 * head updates.
2937 * 2) allows the kernel side to track the head on its own, even
2938 * though the application is the one updating it.
2939 */
2940 head = ctx->cached_sq_head;
Stefan Bühlere523a292019-04-19 11:57:44 +02002941 /* make sure SQ entry isn't read before tail */
Hristo Venev75b28af2019-08-26 17:23:46 +00002942 if (head == smp_load_acquire(&rings->sq.tail))
Jens Axboe2b188cc2019-01-07 10:46:33 -07002943 return false;
2944
Hristo Venev75b28af2019-08-26 17:23:46 +00002945 head = READ_ONCE(sq_array[head & ctx->sq_mask]);
Jens Axboe2b188cc2019-01-07 10:46:33 -07002946 if (head < ctx->sq_entries) {
Jens Axboefcb323c2019-10-24 12:39:47 -06002947 s->ring_file = NULL;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002948 s->sqe = &ctx->sq_sqes[head];
Jackie Liu8776f3f2019-09-09 20:50:39 +08002949 s->sequence = ctx->cached_sq_head;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002950 ctx->cached_sq_head++;
2951 return true;
2952 }
2953
2954 /* drop invalid entries */
2955 ctx->cached_sq_head++;
Jens Axboe498ccd92019-10-25 10:04:25 -06002956 ctx->cached_sq_dropped++;
2957 WRITE_ONCE(rings->sq_dropped, ctx->cached_sq_dropped);
Jens Axboe2b188cc2019-01-07 10:46:33 -07002958 return false;
2959}
2960
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03002961static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr,
Pavel Begunkovae9428c2019-11-06 00:22:14 +03002962 struct file *ring_file, int ring_fd,
2963 struct mm_struct **mm, bool async)
Jens Axboe6c271ce2019-01-10 11:22:30 -07002964{
2965 struct io_submit_state state, *statep = NULL;
Jens Axboe9e645e112019-05-10 16:07:28 -06002966 struct io_kiocb *link = NULL;
Jackie Liu4fe2c962019-09-09 20:50:40 +08002967 struct io_kiocb *shadow_req = NULL;
Jens Axboe9e645e112019-05-10 16:07:28 -06002968 int i, submitted = 0;
Pavel Begunkov95a1b3ff2019-10-27 23:15:41 +03002969 bool mm_fault = false;
Jens Axboe6c271ce2019-01-10 11:22:30 -07002970
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07002971 if (!list_empty(&ctx->cq_overflow_list)) {
2972 io_cqring_overflow_flush(ctx, false);
2973 return -EBUSY;
2974 }
2975
Jens Axboe6c271ce2019-01-10 11:22:30 -07002976 if (nr > IO_PLUG_THRESHOLD) {
2977 io_submit_state_start(&state, ctx, nr);
2978 statep = &state;
2979 }
2980
2981 for (i = 0; i < nr; i++) {
Pavel Begunkov196be952019-11-07 01:41:06 +03002982 struct io_kiocb *req;
Pavel Begunkov50585b92019-11-07 01:41:07 +03002983 unsigned int sqe_flags;
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03002984
Pavel Begunkov196be952019-11-07 01:41:06 +03002985 req = io_get_req(ctx, statep);
2986 if (unlikely(!req)) {
2987 if (!submitted)
2988 submitted = -EAGAIN;
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03002989 break;
Pavel Begunkov196be952019-11-07 01:41:06 +03002990 }
Pavel Begunkov50585b92019-11-07 01:41:07 +03002991 if (!io_get_sqring(ctx, &req->submit)) {
Pavel Begunkov196be952019-11-07 01:41:06 +03002992 __io_free_req(req);
2993 break;
2994 }
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03002995
Pavel Begunkov50585b92019-11-07 01:41:07 +03002996 if (io_sqe_needs_user(req->submit.sqe) && !*mm) {
Pavel Begunkov95a1b3ff2019-10-27 23:15:41 +03002997 mm_fault = mm_fault || !mmget_not_zero(ctx->sqo_mm);
2998 if (!mm_fault) {
2999 use_mm(ctx->sqo_mm);
3000 *mm = ctx->sqo_mm;
3001 }
3002 }
3003
Pavel Begunkov50585b92019-11-07 01:41:07 +03003004 sqe_flags = req->submit.sqe->flags;
3005
3006 if (link && (sqe_flags & IOSQE_IO_DRAIN)) {
Jackie Liu4fe2c962019-09-09 20:50:40 +08003007 if (!shadow_req) {
3008 shadow_req = io_get_req(ctx, NULL);
Jackie Liua1041c22019-09-18 17:25:52 +08003009 if (unlikely(!shadow_req))
3010 goto out;
Jackie Liu4fe2c962019-09-09 20:50:40 +08003011 shadow_req->flags |= (REQ_F_IO_DRAIN | REQ_F_SHADOW_DRAIN);
3012 refcount_dec(&shadow_req->refs);
3013 }
Pavel Begunkov50585b92019-11-07 01:41:07 +03003014 shadow_req->sequence = req->submit.sequence;
Jackie Liu4fe2c962019-09-09 20:50:40 +08003015 }
3016
Jackie Liua1041c22019-09-18 17:25:52 +08003017out:
Pavel Begunkov50585b92019-11-07 01:41:07 +03003018 req->submit.ring_file = ring_file;
3019 req->submit.ring_fd = ring_fd;
3020 req->submit.has_user = *mm != NULL;
3021 req->submit.in_async = async;
3022 req->submit.needs_fixed_file = async;
3023 trace_io_uring_submit_sqe(ctx, req->submit.sqe->user_data,
3024 true, async);
Jackie Liua197f662019-11-08 08:09:12 -07003025 io_submit_sqe(req, statep, &link);
Pavel Begunkov95a1b3ff2019-10-27 23:15:41 +03003026 submitted++;
Pavel Begunkove5eb6362019-11-06 00:22:15 +03003027
3028 /*
3029 * If previous wasn't linked and we have a linked command,
3030 * that's the end of the chain. Submit the previous link.
3031 */
Pavel Begunkov50585b92019-11-07 01:41:07 +03003032 if (!(sqe_flags & IOSQE_IO_LINK) && link) {
Jackie Liua197f662019-11-08 08:09:12 -07003033 io_queue_link_head(link, shadow_req);
Pavel Begunkove5eb6362019-11-06 00:22:15 +03003034 link = NULL;
3035 shadow_req = NULL;
3036 }
Jens Axboe6c271ce2019-01-10 11:22:30 -07003037 }
3038
Jens Axboe9e645e112019-05-10 16:07:28 -06003039 if (link)
Jackie Liua197f662019-11-08 08:09:12 -07003040 io_queue_link_head(link, shadow_req);
Jens Axboe6c271ce2019-01-10 11:22:30 -07003041 if (statep)
3042 io_submit_state_end(&state);
3043
Pavel Begunkovae9428c2019-11-06 00:22:14 +03003044 /* Commit SQ ring head once we've consumed and submitted all SQEs */
3045 io_commit_sqring(ctx);
3046
Jens Axboe6c271ce2019-01-10 11:22:30 -07003047 return submitted;
3048}
3049
3050static int io_sq_thread(void *data)
3051{
Jens Axboe6c271ce2019-01-10 11:22:30 -07003052 struct io_ring_ctx *ctx = data;
3053 struct mm_struct *cur_mm = NULL;
3054 mm_segment_t old_fs;
3055 DEFINE_WAIT(wait);
3056 unsigned inflight;
3057 unsigned long timeout;
3058
Jens Axboe206aefd2019-11-07 18:27:42 -07003059 complete(&ctx->completions[1]);
Jackie Liua4c0b3d2019-07-08 13:41:12 +08003060
Jens Axboe6c271ce2019-01-10 11:22:30 -07003061 old_fs = get_fs();
3062 set_fs(USER_DS);
3063
3064 timeout = inflight = 0;
Roman Penyaev2bbcd6d2019-05-16 10:53:57 +02003065 while (!kthread_should_park()) {
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03003066 unsigned int to_submit;
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07003067 int ret;
Jens Axboe6c271ce2019-01-10 11:22:30 -07003068
3069 if (inflight) {
3070 unsigned nr_events = 0;
3071
3072 if (ctx->flags & IORING_SETUP_IOPOLL) {
Jens Axboe2b2ed972019-10-25 10:06:15 -06003073 /*
3074 * inflight is the count of the maximum possible
3075 * entries we submitted, but it can be smaller
3076 * if we dropped some of them. If we don't have
3077 * poll entries available, then we know that we
3078 * have nothing left to poll for. Reset the
3079 * inflight count to zero in that case.
3080 */
3081 mutex_lock(&ctx->uring_lock);
3082 if (!list_empty(&ctx->poll_list))
3083 __io_iopoll_check(ctx, &nr_events, 0);
3084 else
3085 inflight = 0;
3086 mutex_unlock(&ctx->uring_lock);
Jens Axboe6c271ce2019-01-10 11:22:30 -07003087 } else {
3088 /*
3089 * Normal IO, just pretend everything completed.
3090 * We don't have to poll completions for that.
3091 */
3092 nr_events = inflight;
3093 }
3094
3095 inflight -= nr_events;
3096 if (!inflight)
3097 timeout = jiffies + ctx->sq_thread_idle;
3098 }
3099
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03003100 to_submit = io_sqring_entries(ctx);
3101 if (!to_submit) {
Jens Axboe6c271ce2019-01-10 11:22:30 -07003102 /*
3103 * We're polling. If we're within the defined idle
3104 * period, then let us spin without work before going
3105 * to sleep.
3106 */
3107 if (inflight || !time_after(jiffies, timeout)) {
Jens Axboe9831a902019-09-19 09:48:55 -06003108 cond_resched();
Jens Axboe6c271ce2019-01-10 11:22:30 -07003109 continue;
3110 }
3111
3112 /*
3113 * Drop cur_mm before scheduling, we can't hold it for
3114 * long periods (or over schedule()). Do this before
3115 * adding ourselves to the waitqueue, as the unuse/drop
3116 * may sleep.
3117 */
3118 if (cur_mm) {
3119 unuse_mm(cur_mm);
3120 mmput(cur_mm);
3121 cur_mm = NULL;
3122 }
3123
3124 prepare_to_wait(&ctx->sqo_wait, &wait,
3125 TASK_INTERRUPTIBLE);
3126
3127 /* Tell userspace we may need a wakeup call */
Hristo Venev75b28af2019-08-26 17:23:46 +00003128 ctx->rings->sq_flags |= IORING_SQ_NEED_WAKEUP;
Stefan Bühler0d7bae62019-04-19 11:57:45 +02003129 /* make sure to read SQ tail after writing flags */
3130 smp_mb();
Jens Axboe6c271ce2019-01-10 11:22:30 -07003131
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03003132 to_submit = io_sqring_entries(ctx);
3133 if (!to_submit) {
Roman Penyaev2bbcd6d2019-05-16 10:53:57 +02003134 if (kthread_should_park()) {
Jens Axboe6c271ce2019-01-10 11:22:30 -07003135 finish_wait(&ctx->sqo_wait, &wait);
3136 break;
3137 }
3138 if (signal_pending(current))
3139 flush_signals(current);
3140 schedule();
3141 finish_wait(&ctx->sqo_wait, &wait);
3142
Hristo Venev75b28af2019-08-26 17:23:46 +00003143 ctx->rings->sq_flags &= ~IORING_SQ_NEED_WAKEUP;
Jens Axboe6c271ce2019-01-10 11:22:30 -07003144 continue;
3145 }
3146 finish_wait(&ctx->sqo_wait, &wait);
3147
Hristo Venev75b28af2019-08-26 17:23:46 +00003148 ctx->rings->sq_flags &= ~IORING_SQ_NEED_WAKEUP;
Jens Axboe6c271ce2019-01-10 11:22:30 -07003149 }
3150
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03003151 to_submit = min(to_submit, ctx->sq_entries);
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07003152 ret = io_submit_sqes(ctx, to_submit, NULL, -1, &cur_mm, true);
3153 if (ret > 0)
3154 inflight += ret;
Jens Axboe6c271ce2019-01-10 11:22:30 -07003155 }
3156
3157 set_fs(old_fs);
3158 if (cur_mm) {
3159 unuse_mm(cur_mm);
3160 mmput(cur_mm);
3161 }
Jens Axboe06058632019-04-13 09:26:03 -06003162
Roman Penyaev2bbcd6d2019-05-16 10:53:57 +02003163 kthread_parkme();
Jens Axboe06058632019-04-13 09:26:03 -06003164
Jens Axboe6c271ce2019-01-10 11:22:30 -07003165 return 0;
3166}
3167
Jens Axboebda52162019-09-24 13:47:15 -06003168struct io_wait_queue {
3169 struct wait_queue_entry wq;
3170 struct io_ring_ctx *ctx;
3171 unsigned to_wait;
3172 unsigned nr_timeouts;
3173};
3174
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07003175static inline bool io_should_wake(struct io_wait_queue *iowq, bool noflush)
Jens Axboebda52162019-09-24 13:47:15 -06003176{
3177 struct io_ring_ctx *ctx = iowq->ctx;
3178
3179 /*
3180 * Wake up if we have enough events, or if a timeout occured since we
3181 * started waiting. For timeouts, we always want to return to userspace,
3182 * regardless of event count.
3183 */
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07003184 return io_cqring_events(ctx, noflush) >= iowq->to_wait ||
Jens Axboebda52162019-09-24 13:47:15 -06003185 atomic_read(&ctx->cq_timeouts) != iowq->nr_timeouts;
3186}
3187
3188static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode,
3189 int wake_flags, void *key)
3190{
3191 struct io_wait_queue *iowq = container_of(curr, struct io_wait_queue,
3192 wq);
3193
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07003194 /* use noflush == true, as we can't safely rely on locking context */
3195 if (!io_should_wake(iowq, true))
Jens Axboebda52162019-09-24 13:47:15 -06003196 return -1;
3197
3198 return autoremove_wake_function(curr, mode, wake_flags, key);
3199}
3200
Jens Axboe2b188cc2019-01-07 10:46:33 -07003201/*
3202 * Wait until events become available, if we don't already have some. The
3203 * application must reap them itself, as they reside on the shared cq ring.
3204 */
3205static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events,
3206 const sigset_t __user *sig, size_t sigsz)
3207{
Jens Axboebda52162019-09-24 13:47:15 -06003208 struct io_wait_queue iowq = {
3209 .wq = {
3210 .private = current,
3211 .func = io_wake_function,
3212 .entry = LIST_HEAD_INIT(iowq.wq.entry),
3213 },
3214 .ctx = ctx,
3215 .to_wait = min_events,
3216 };
Hristo Venev75b28af2019-08-26 17:23:46 +00003217 struct io_rings *rings = ctx->rings;
Jackie Liue9ffa5c2019-10-29 11:16:42 +08003218 int ret = 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003219
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07003220 if (io_cqring_events(ctx, false) >= min_events)
Jens Axboe2b188cc2019-01-07 10:46:33 -07003221 return 0;
3222
3223 if (sig) {
Arnd Bergmann9e75ad52019-03-25 15:34:53 +01003224#ifdef CONFIG_COMPAT
3225 if (in_compat_syscall())
3226 ret = set_compat_user_sigmask((const compat_sigset_t __user *)sig,
Oleg Nesterovb7724342019-07-16 16:29:53 -07003227 sigsz);
Arnd Bergmann9e75ad52019-03-25 15:34:53 +01003228 else
3229#endif
Oleg Nesterovb7724342019-07-16 16:29:53 -07003230 ret = set_user_sigmask(sig, sigsz);
Arnd Bergmann9e75ad52019-03-25 15:34:53 +01003231
Jens Axboe2b188cc2019-01-07 10:46:33 -07003232 if (ret)
3233 return ret;
3234 }
3235
Jens Axboebda52162019-09-24 13:47:15 -06003236 iowq.nr_timeouts = atomic_read(&ctx->cq_timeouts);
Dmitrii Dolgovc826bd72019-10-15 19:02:01 +02003237 trace_io_uring_cqring_wait(ctx, min_events);
Jens Axboebda52162019-09-24 13:47:15 -06003238 do {
3239 prepare_to_wait_exclusive(&ctx->wait, &iowq.wq,
3240 TASK_INTERRUPTIBLE);
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07003241 if (io_should_wake(&iowq, false))
Jens Axboebda52162019-09-24 13:47:15 -06003242 break;
3243 schedule();
3244 if (signal_pending(current)) {
Jackie Liue9ffa5c2019-10-29 11:16:42 +08003245 ret = -EINTR;
Jens Axboebda52162019-09-24 13:47:15 -06003246 break;
3247 }
3248 } while (1);
3249 finish_wait(&ctx->wait, &iowq.wq);
3250
Jackie Liue9ffa5c2019-10-29 11:16:42 +08003251 restore_saved_sigmask_unless(ret == -EINTR);
Jens Axboe2b188cc2019-01-07 10:46:33 -07003252
Hristo Venev75b28af2019-08-26 17:23:46 +00003253 return READ_ONCE(rings->cq.head) == READ_ONCE(rings->cq.tail) ? ret : 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003254}
3255
Jens Axboe6b063142019-01-10 22:13:58 -07003256static void __io_sqe_files_unregister(struct io_ring_ctx *ctx)
3257{
3258#if defined(CONFIG_UNIX)
3259 if (ctx->ring_sock) {
3260 struct sock *sock = ctx->ring_sock->sk;
3261 struct sk_buff *skb;
3262
3263 while ((skb = skb_dequeue(&sock->sk_receive_queue)) != NULL)
3264 kfree_skb(skb);
3265 }
3266#else
3267 int i;
3268
Jens Axboe65e19f52019-10-26 07:20:21 -06003269 for (i = 0; i < ctx->nr_user_files; i++) {
3270 struct file *file;
3271
3272 file = io_file_from_index(ctx, i);
3273 if (file)
3274 fput(file);
3275 }
Jens Axboe6b063142019-01-10 22:13:58 -07003276#endif
3277}
3278
3279static int io_sqe_files_unregister(struct io_ring_ctx *ctx)
3280{
Jens Axboe65e19f52019-10-26 07:20:21 -06003281 unsigned nr_tables, i;
3282
3283 if (!ctx->file_table)
Jens Axboe6b063142019-01-10 22:13:58 -07003284 return -ENXIO;
3285
3286 __io_sqe_files_unregister(ctx);
Jens Axboe65e19f52019-10-26 07:20:21 -06003287 nr_tables = DIV_ROUND_UP(ctx->nr_user_files, IORING_MAX_FILES_TABLE);
3288 for (i = 0; i < nr_tables; i++)
3289 kfree(ctx->file_table[i].files);
3290 kfree(ctx->file_table);
3291 ctx->file_table = NULL;
Jens Axboe6b063142019-01-10 22:13:58 -07003292 ctx->nr_user_files = 0;
3293 return 0;
3294}
3295
Jens Axboe6c271ce2019-01-10 11:22:30 -07003296static void io_sq_thread_stop(struct io_ring_ctx *ctx)
3297{
3298 if (ctx->sqo_thread) {
Jens Axboe206aefd2019-11-07 18:27:42 -07003299 wait_for_completion(&ctx->completions[1]);
Roman Penyaev2bbcd6d2019-05-16 10:53:57 +02003300 /*
3301 * The park is a bit of a work-around, without it we get
3302 * warning spews on shutdown with SQPOLL set and affinity
3303 * set to a single CPU.
3304 */
Jens Axboe06058632019-04-13 09:26:03 -06003305 kthread_park(ctx->sqo_thread);
Jens Axboe6c271ce2019-01-10 11:22:30 -07003306 kthread_stop(ctx->sqo_thread);
3307 ctx->sqo_thread = NULL;
3308 }
3309}
3310
Jens Axboe6b063142019-01-10 22:13:58 -07003311static void io_finish_async(struct io_ring_ctx *ctx)
3312{
Jens Axboe6c271ce2019-01-10 11:22:30 -07003313 io_sq_thread_stop(ctx);
3314
Jens Axboe561fb042019-10-24 07:25:42 -06003315 if (ctx->io_wq) {
3316 io_wq_destroy(ctx->io_wq);
3317 ctx->io_wq = NULL;
Jens Axboe6b063142019-01-10 22:13:58 -07003318 }
3319}
3320
3321#if defined(CONFIG_UNIX)
3322static void io_destruct_skb(struct sk_buff *skb)
3323{
3324 struct io_ring_ctx *ctx = skb->sk->sk_user_data;
3325
Jens Axboe561fb042019-10-24 07:25:42 -06003326 if (ctx->io_wq)
3327 io_wq_flush(ctx->io_wq);
Jens Axboe8a997342019-10-09 14:40:13 -06003328
Jens Axboe6b063142019-01-10 22:13:58 -07003329 unix_destruct_scm(skb);
3330}
3331
3332/*
3333 * Ensure the UNIX gc is aware of our file set, so we are certain that
3334 * the io_uring can be safely unregistered on process exit, even if we have
3335 * loops in the file referencing.
3336 */
3337static int __io_sqe_files_scm(struct io_ring_ctx *ctx, int nr, int offset)
3338{
3339 struct sock *sk = ctx->ring_sock->sk;
3340 struct scm_fp_list *fpl;
3341 struct sk_buff *skb;
Jens Axboe08a45172019-10-03 08:11:03 -06003342 int i, nr_files;
Jens Axboe6b063142019-01-10 22:13:58 -07003343
3344 if (!capable(CAP_SYS_RESOURCE) && !capable(CAP_SYS_ADMIN)) {
3345 unsigned long inflight = ctx->user->unix_inflight + nr;
3346
3347 if (inflight > task_rlimit(current, RLIMIT_NOFILE))
3348 return -EMFILE;
3349 }
3350
3351 fpl = kzalloc(sizeof(*fpl), GFP_KERNEL);
3352 if (!fpl)
3353 return -ENOMEM;
3354
3355 skb = alloc_skb(0, GFP_KERNEL);
3356 if (!skb) {
3357 kfree(fpl);
3358 return -ENOMEM;
3359 }
3360
3361 skb->sk = sk;
Jens Axboe6b063142019-01-10 22:13:58 -07003362
Jens Axboe08a45172019-10-03 08:11:03 -06003363 nr_files = 0;
Jens Axboe6b063142019-01-10 22:13:58 -07003364 fpl->user = get_uid(ctx->user);
3365 for (i = 0; i < nr; i++) {
Jens Axboe65e19f52019-10-26 07:20:21 -06003366 struct file *file = io_file_from_index(ctx, i + offset);
3367
3368 if (!file)
Jens Axboe08a45172019-10-03 08:11:03 -06003369 continue;
Jens Axboe65e19f52019-10-26 07:20:21 -06003370 fpl->fp[nr_files] = get_file(file);
Jens Axboe08a45172019-10-03 08:11:03 -06003371 unix_inflight(fpl->user, fpl->fp[nr_files]);
3372 nr_files++;
Jens Axboe6b063142019-01-10 22:13:58 -07003373 }
3374
Jens Axboe08a45172019-10-03 08:11:03 -06003375 if (nr_files) {
3376 fpl->max = SCM_MAX_FD;
3377 fpl->count = nr_files;
3378 UNIXCB(skb).fp = fpl;
3379 skb->destructor = io_destruct_skb;
3380 refcount_add(skb->truesize, &sk->sk_wmem_alloc);
3381 skb_queue_head(&sk->sk_receive_queue, skb);
Jens Axboe6b063142019-01-10 22:13:58 -07003382
Jens Axboe08a45172019-10-03 08:11:03 -06003383 for (i = 0; i < nr_files; i++)
3384 fput(fpl->fp[i]);
3385 } else {
3386 kfree_skb(skb);
3387 kfree(fpl);
3388 }
Jens Axboe6b063142019-01-10 22:13:58 -07003389
3390 return 0;
3391}
3392
3393/*
3394 * If UNIX sockets are enabled, fd passing can cause a reference cycle which
3395 * causes regular reference counting to break down. We rely on the UNIX
3396 * garbage collection to take care of this problem for us.
3397 */
3398static int io_sqe_files_scm(struct io_ring_ctx *ctx)
3399{
3400 unsigned left, total;
3401 int ret = 0;
3402
3403 total = 0;
3404 left = ctx->nr_user_files;
3405 while (left) {
3406 unsigned this_files = min_t(unsigned, left, SCM_MAX_FD);
Jens Axboe6b063142019-01-10 22:13:58 -07003407
3408 ret = __io_sqe_files_scm(ctx, this_files, total);
3409 if (ret)
3410 break;
3411 left -= this_files;
3412 total += this_files;
3413 }
3414
3415 if (!ret)
3416 return 0;
3417
3418 while (total < ctx->nr_user_files) {
Jens Axboe65e19f52019-10-26 07:20:21 -06003419 struct file *file = io_file_from_index(ctx, total);
3420
3421 if (file)
3422 fput(file);
Jens Axboe6b063142019-01-10 22:13:58 -07003423 total++;
3424 }
3425
3426 return ret;
3427}
3428#else
3429static int io_sqe_files_scm(struct io_ring_ctx *ctx)
3430{
3431 return 0;
3432}
3433#endif
3434
Jens Axboe65e19f52019-10-26 07:20:21 -06003435static int io_sqe_alloc_file_tables(struct io_ring_ctx *ctx, unsigned nr_tables,
3436 unsigned nr_files)
3437{
3438 int i;
3439
3440 for (i = 0; i < nr_tables; i++) {
3441 struct fixed_file_table *table = &ctx->file_table[i];
3442 unsigned this_files;
3443
3444 this_files = min(nr_files, IORING_MAX_FILES_TABLE);
3445 table->files = kcalloc(this_files, sizeof(struct file *),
3446 GFP_KERNEL);
3447 if (!table->files)
3448 break;
3449 nr_files -= this_files;
3450 }
3451
3452 if (i == nr_tables)
3453 return 0;
3454
3455 for (i = 0; i < nr_tables; i++) {
3456 struct fixed_file_table *table = &ctx->file_table[i];
3457 kfree(table->files);
3458 }
3459 return 1;
3460}
3461
Jens Axboe6b063142019-01-10 22:13:58 -07003462static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg,
3463 unsigned nr_args)
3464{
3465 __s32 __user *fds = (__s32 __user *) arg;
Jens Axboe65e19f52019-10-26 07:20:21 -06003466 unsigned nr_tables;
Jens Axboe6b063142019-01-10 22:13:58 -07003467 int fd, ret = 0;
3468 unsigned i;
3469
Jens Axboe65e19f52019-10-26 07:20:21 -06003470 if (ctx->file_table)
Jens Axboe6b063142019-01-10 22:13:58 -07003471 return -EBUSY;
3472 if (!nr_args)
3473 return -EINVAL;
3474 if (nr_args > IORING_MAX_FIXED_FILES)
3475 return -EMFILE;
3476
Jens Axboe65e19f52019-10-26 07:20:21 -06003477 nr_tables = DIV_ROUND_UP(nr_args, IORING_MAX_FILES_TABLE);
3478 ctx->file_table = kcalloc(nr_tables, sizeof(struct fixed_file_table),
3479 GFP_KERNEL);
3480 if (!ctx->file_table)
Jens Axboe6b063142019-01-10 22:13:58 -07003481 return -ENOMEM;
3482
Jens Axboe65e19f52019-10-26 07:20:21 -06003483 if (io_sqe_alloc_file_tables(ctx, nr_tables, nr_args)) {
3484 kfree(ctx->file_table);
3485 return -ENOMEM;
3486 }
3487
Jens Axboe08a45172019-10-03 08:11:03 -06003488 for (i = 0; i < nr_args; i++, ctx->nr_user_files++) {
Jens Axboe65e19f52019-10-26 07:20:21 -06003489 struct fixed_file_table *table;
3490 unsigned index;
3491
Jens Axboe6b063142019-01-10 22:13:58 -07003492 ret = -EFAULT;
3493 if (copy_from_user(&fd, &fds[i], sizeof(fd)))
3494 break;
Jens Axboe08a45172019-10-03 08:11:03 -06003495 /* allow sparse sets */
3496 if (fd == -1) {
3497 ret = 0;
3498 continue;
3499 }
Jens Axboe6b063142019-01-10 22:13:58 -07003500
Jens Axboe65e19f52019-10-26 07:20:21 -06003501 table = &ctx->file_table[i >> IORING_FILE_TABLE_SHIFT];
3502 index = i & IORING_FILE_TABLE_MASK;
3503 table->files[index] = fget(fd);
Jens Axboe6b063142019-01-10 22:13:58 -07003504
3505 ret = -EBADF;
Jens Axboe65e19f52019-10-26 07:20:21 -06003506 if (!table->files[index])
Jens Axboe6b063142019-01-10 22:13:58 -07003507 break;
3508 /*
3509 * Don't allow io_uring instances to be registered. If UNIX
3510 * isn't enabled, then this causes a reference cycle and this
3511 * instance can never get freed. If UNIX is enabled we'll
3512 * handle it just fine, but there's still no point in allowing
3513 * a ring fd as it doesn't support regular read/write anyway.
3514 */
Jens Axboe65e19f52019-10-26 07:20:21 -06003515 if (table->files[index]->f_op == &io_uring_fops) {
3516 fput(table->files[index]);
Jens Axboe6b063142019-01-10 22:13:58 -07003517 break;
3518 }
Jens Axboe6b063142019-01-10 22:13:58 -07003519 ret = 0;
3520 }
3521
3522 if (ret) {
Jens Axboe65e19f52019-10-26 07:20:21 -06003523 for (i = 0; i < ctx->nr_user_files; i++) {
3524 struct file *file;
Jens Axboe6b063142019-01-10 22:13:58 -07003525
Jens Axboe65e19f52019-10-26 07:20:21 -06003526 file = io_file_from_index(ctx, i);
3527 if (file)
3528 fput(file);
3529 }
3530 for (i = 0; i < nr_tables; i++)
3531 kfree(ctx->file_table[i].files);
3532
3533 kfree(ctx->file_table);
3534 ctx->file_table = NULL;
Jens Axboe6b063142019-01-10 22:13:58 -07003535 ctx->nr_user_files = 0;
3536 return ret;
3537 }
3538
3539 ret = io_sqe_files_scm(ctx);
3540 if (ret)
3541 io_sqe_files_unregister(ctx);
3542
3543 return ret;
3544}
3545
Jens Axboec3a31e62019-10-03 13:59:56 -06003546static void io_sqe_file_unregister(struct io_ring_ctx *ctx, int index)
3547{
3548#if defined(CONFIG_UNIX)
Jens Axboe65e19f52019-10-26 07:20:21 -06003549 struct file *file = io_file_from_index(ctx, index);
Jens Axboec3a31e62019-10-03 13:59:56 -06003550 struct sock *sock = ctx->ring_sock->sk;
3551 struct sk_buff_head list, *head = &sock->sk_receive_queue;
3552 struct sk_buff *skb;
3553 int i;
3554
3555 __skb_queue_head_init(&list);
3556
3557 /*
3558 * Find the skb that holds this file in its SCM_RIGHTS. When found,
3559 * remove this entry and rearrange the file array.
3560 */
3561 skb = skb_dequeue(head);
3562 while (skb) {
3563 struct scm_fp_list *fp;
3564
3565 fp = UNIXCB(skb).fp;
3566 for (i = 0; i < fp->count; i++) {
3567 int left;
3568
3569 if (fp->fp[i] != file)
3570 continue;
3571
3572 unix_notinflight(fp->user, fp->fp[i]);
3573 left = fp->count - 1 - i;
3574 if (left) {
3575 memmove(&fp->fp[i], &fp->fp[i + 1],
3576 left * sizeof(struct file *));
3577 }
3578 fp->count--;
3579 if (!fp->count) {
3580 kfree_skb(skb);
3581 skb = NULL;
3582 } else {
3583 __skb_queue_tail(&list, skb);
3584 }
3585 fput(file);
3586 file = NULL;
3587 break;
3588 }
3589
3590 if (!file)
3591 break;
3592
3593 __skb_queue_tail(&list, skb);
3594
3595 skb = skb_dequeue(head);
3596 }
3597
3598 if (skb_peek(&list)) {
3599 spin_lock_irq(&head->lock);
3600 while ((skb = __skb_dequeue(&list)) != NULL)
3601 __skb_queue_tail(head, skb);
3602 spin_unlock_irq(&head->lock);
3603 }
3604#else
Jens Axboe65e19f52019-10-26 07:20:21 -06003605 fput(io_file_from_index(ctx, index));
Jens Axboec3a31e62019-10-03 13:59:56 -06003606#endif
3607}
3608
3609static int io_sqe_file_register(struct io_ring_ctx *ctx, struct file *file,
3610 int index)
3611{
3612#if defined(CONFIG_UNIX)
3613 struct sock *sock = ctx->ring_sock->sk;
3614 struct sk_buff_head *head = &sock->sk_receive_queue;
3615 struct sk_buff *skb;
3616
3617 /*
3618 * See if we can merge this file into an existing skb SCM_RIGHTS
3619 * file set. If there's no room, fall back to allocating a new skb
3620 * and filling it in.
3621 */
3622 spin_lock_irq(&head->lock);
3623 skb = skb_peek(head);
3624 if (skb) {
3625 struct scm_fp_list *fpl = UNIXCB(skb).fp;
3626
3627 if (fpl->count < SCM_MAX_FD) {
3628 __skb_unlink(skb, head);
3629 spin_unlock_irq(&head->lock);
3630 fpl->fp[fpl->count] = get_file(file);
3631 unix_inflight(fpl->user, fpl->fp[fpl->count]);
3632 fpl->count++;
3633 spin_lock_irq(&head->lock);
3634 __skb_queue_head(head, skb);
3635 } else {
3636 skb = NULL;
3637 }
3638 }
3639 spin_unlock_irq(&head->lock);
3640
3641 if (skb) {
3642 fput(file);
3643 return 0;
3644 }
3645
3646 return __io_sqe_files_scm(ctx, 1, index);
3647#else
3648 return 0;
3649#endif
3650}
3651
3652static int io_sqe_files_update(struct io_ring_ctx *ctx, void __user *arg,
3653 unsigned nr_args)
3654{
3655 struct io_uring_files_update up;
3656 __s32 __user *fds;
3657 int fd, i, err;
3658 __u32 done;
3659
Jens Axboe65e19f52019-10-26 07:20:21 -06003660 if (!ctx->file_table)
Jens Axboec3a31e62019-10-03 13:59:56 -06003661 return -ENXIO;
3662 if (!nr_args)
3663 return -EINVAL;
3664 if (copy_from_user(&up, arg, sizeof(up)))
3665 return -EFAULT;
3666 if (check_add_overflow(up.offset, nr_args, &done))
3667 return -EOVERFLOW;
3668 if (done > ctx->nr_user_files)
3669 return -EINVAL;
3670
3671 done = 0;
3672 fds = (__s32 __user *) up.fds;
3673 while (nr_args) {
Jens Axboe65e19f52019-10-26 07:20:21 -06003674 struct fixed_file_table *table;
3675 unsigned index;
3676
Jens Axboec3a31e62019-10-03 13:59:56 -06003677 err = 0;
3678 if (copy_from_user(&fd, &fds[done], sizeof(fd))) {
3679 err = -EFAULT;
3680 break;
3681 }
3682 i = array_index_nospec(up.offset, ctx->nr_user_files);
Jens Axboe65e19f52019-10-26 07:20:21 -06003683 table = &ctx->file_table[i >> IORING_FILE_TABLE_SHIFT];
3684 index = i & IORING_FILE_TABLE_MASK;
3685 if (table->files[index]) {
Jens Axboec3a31e62019-10-03 13:59:56 -06003686 io_sqe_file_unregister(ctx, i);
Jens Axboe65e19f52019-10-26 07:20:21 -06003687 table->files[index] = NULL;
Jens Axboec3a31e62019-10-03 13:59:56 -06003688 }
3689 if (fd != -1) {
3690 struct file *file;
3691
3692 file = fget(fd);
3693 if (!file) {
3694 err = -EBADF;
3695 break;
3696 }
3697 /*
3698 * Don't allow io_uring instances to be registered. If
3699 * UNIX isn't enabled, then this causes a reference
3700 * cycle and this instance can never get freed. If UNIX
3701 * is enabled we'll handle it just fine, but there's
3702 * still no point in allowing a ring fd as it doesn't
3703 * support regular read/write anyway.
3704 */
3705 if (file->f_op == &io_uring_fops) {
3706 fput(file);
3707 err = -EBADF;
3708 break;
3709 }
Jens Axboe65e19f52019-10-26 07:20:21 -06003710 table->files[index] = file;
Jens Axboec3a31e62019-10-03 13:59:56 -06003711 err = io_sqe_file_register(ctx, file, i);
3712 if (err)
3713 break;
3714 }
3715 nr_args--;
3716 done++;
3717 up.offset++;
3718 }
3719
3720 return done ? done : err;
3721}
3722
Jens Axboe6c271ce2019-01-10 11:22:30 -07003723static int io_sq_offload_start(struct io_ring_ctx *ctx,
3724 struct io_uring_params *p)
Jens Axboe2b188cc2019-01-07 10:46:33 -07003725{
Jens Axboe561fb042019-10-24 07:25:42 -06003726 unsigned concurrency;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003727 int ret;
3728
Jens Axboe6c271ce2019-01-10 11:22:30 -07003729 init_waitqueue_head(&ctx->sqo_wait);
Jens Axboe2b188cc2019-01-07 10:46:33 -07003730 mmgrab(current->mm);
3731 ctx->sqo_mm = current->mm;
3732
Jens Axboe6c271ce2019-01-10 11:22:30 -07003733 if (ctx->flags & IORING_SETUP_SQPOLL) {
Jens Axboe3ec482d2019-04-08 10:51:01 -06003734 ret = -EPERM;
3735 if (!capable(CAP_SYS_ADMIN))
3736 goto err;
3737
Jens Axboe917257d2019-04-13 09:28:55 -06003738 ctx->sq_thread_idle = msecs_to_jiffies(p->sq_thread_idle);
3739 if (!ctx->sq_thread_idle)
3740 ctx->sq_thread_idle = HZ;
3741
Jens Axboe6c271ce2019-01-10 11:22:30 -07003742 if (p->flags & IORING_SETUP_SQ_AFF) {
Jens Axboe44a9bd12019-05-14 20:00:30 -06003743 int cpu = p->sq_thread_cpu;
Jens Axboe6c271ce2019-01-10 11:22:30 -07003744
Jens Axboe917257d2019-04-13 09:28:55 -06003745 ret = -EINVAL;
Jens Axboe44a9bd12019-05-14 20:00:30 -06003746 if (cpu >= nr_cpu_ids)
3747 goto err;
Shenghui Wang7889f442019-05-07 16:03:19 +08003748 if (!cpu_online(cpu))
Jens Axboe917257d2019-04-13 09:28:55 -06003749 goto err;
3750
Jens Axboe6c271ce2019-01-10 11:22:30 -07003751 ctx->sqo_thread = kthread_create_on_cpu(io_sq_thread,
3752 ctx, cpu,
3753 "io_uring-sq");
3754 } else {
3755 ctx->sqo_thread = kthread_create(io_sq_thread, ctx,
3756 "io_uring-sq");
3757 }
3758 if (IS_ERR(ctx->sqo_thread)) {
3759 ret = PTR_ERR(ctx->sqo_thread);
3760 ctx->sqo_thread = NULL;
3761 goto err;
3762 }
3763 wake_up_process(ctx->sqo_thread);
3764 } else if (p->flags & IORING_SETUP_SQ_AFF) {
3765 /* Can't have SQ_AFF without SQPOLL */
3766 ret = -EINVAL;
3767 goto err;
3768 }
3769
Jens Axboe561fb042019-10-24 07:25:42 -06003770 /* Do QD, or 4 * CPUS, whatever is smallest */
3771 concurrency = min(ctx->sq_entries, 4 * num_online_cpus());
Jens Axboe5f8fd2d32019-11-07 10:57:36 -07003772 ctx->io_wq = io_wq_create(concurrency, ctx->sqo_mm, ctx->user);
Jens Axboe975c99a52019-10-30 08:42:56 -06003773 if (IS_ERR(ctx->io_wq)) {
3774 ret = PTR_ERR(ctx->io_wq);
3775 ctx->io_wq = NULL;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003776 goto err;
3777 }
3778
3779 return 0;
3780err:
Jens Axboe54a91f32019-09-10 09:15:04 -06003781 io_finish_async(ctx);
Jens Axboe2b188cc2019-01-07 10:46:33 -07003782 mmdrop(ctx->sqo_mm);
3783 ctx->sqo_mm = NULL;
3784 return ret;
3785}
3786
3787static void io_unaccount_mem(struct user_struct *user, unsigned long nr_pages)
3788{
3789 atomic_long_sub(nr_pages, &user->locked_vm);
3790}
3791
3792static int io_account_mem(struct user_struct *user, unsigned long nr_pages)
3793{
3794 unsigned long page_limit, cur_pages, new_pages;
3795
3796 /* Don't allow more pages than we can safely lock */
3797 page_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
3798
3799 do {
3800 cur_pages = atomic_long_read(&user->locked_vm);
3801 new_pages = cur_pages + nr_pages;
3802 if (new_pages > page_limit)
3803 return -ENOMEM;
3804 } while (atomic_long_cmpxchg(&user->locked_vm, cur_pages,
3805 new_pages) != cur_pages);
3806
3807 return 0;
3808}
3809
3810static void io_mem_free(void *ptr)
3811{
Mark Rutland52e04ef2019-04-30 17:30:21 +01003812 struct page *page;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003813
Mark Rutland52e04ef2019-04-30 17:30:21 +01003814 if (!ptr)
3815 return;
3816
3817 page = virt_to_head_page(ptr);
Jens Axboe2b188cc2019-01-07 10:46:33 -07003818 if (put_page_testzero(page))
3819 free_compound_page(page);
3820}
3821
3822static void *io_mem_alloc(size_t size)
3823{
3824 gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN | __GFP_COMP |
3825 __GFP_NORETRY;
3826
3827 return (void *) __get_free_pages(gfp_flags, get_order(size));
3828}
3829
Hristo Venev75b28af2019-08-26 17:23:46 +00003830static unsigned long rings_size(unsigned sq_entries, unsigned cq_entries,
3831 size_t *sq_offset)
3832{
3833 struct io_rings *rings;
3834 size_t off, sq_array_size;
3835
3836 off = struct_size(rings, cqes, cq_entries);
3837 if (off == SIZE_MAX)
3838 return SIZE_MAX;
3839
3840#ifdef CONFIG_SMP
3841 off = ALIGN(off, SMP_CACHE_BYTES);
3842 if (off == 0)
3843 return SIZE_MAX;
3844#endif
3845
3846 sq_array_size = array_size(sizeof(u32), sq_entries);
3847 if (sq_array_size == SIZE_MAX)
3848 return SIZE_MAX;
3849
3850 if (check_add_overflow(off, sq_array_size, &off))
3851 return SIZE_MAX;
3852
3853 if (sq_offset)
3854 *sq_offset = off;
3855
3856 return off;
3857}
3858
Jens Axboe2b188cc2019-01-07 10:46:33 -07003859static unsigned long ring_pages(unsigned sq_entries, unsigned cq_entries)
3860{
Hristo Venev75b28af2019-08-26 17:23:46 +00003861 size_t pages;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003862
Hristo Venev75b28af2019-08-26 17:23:46 +00003863 pages = (size_t)1 << get_order(
3864 rings_size(sq_entries, cq_entries, NULL));
3865 pages += (size_t)1 << get_order(
3866 array_size(sizeof(struct io_uring_sqe), sq_entries));
Jens Axboe2b188cc2019-01-07 10:46:33 -07003867
Hristo Venev75b28af2019-08-26 17:23:46 +00003868 return pages;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003869}
3870
Jens Axboeedafcce2019-01-09 09:16:05 -07003871static int io_sqe_buffer_unregister(struct io_ring_ctx *ctx)
3872{
3873 int i, j;
3874
3875 if (!ctx->user_bufs)
3876 return -ENXIO;
3877
3878 for (i = 0; i < ctx->nr_user_bufs; i++) {
3879 struct io_mapped_ubuf *imu = &ctx->user_bufs[i];
3880
3881 for (j = 0; j < imu->nr_bvecs; j++)
John Hubbard27c4d3a2019-08-04 19:32:06 -07003882 put_user_page(imu->bvec[j].bv_page);
Jens Axboeedafcce2019-01-09 09:16:05 -07003883
3884 if (ctx->account_mem)
3885 io_unaccount_mem(ctx->user, imu->nr_bvecs);
Mark Rutlandd4ef6472019-05-01 16:59:16 +01003886 kvfree(imu->bvec);
Jens Axboeedafcce2019-01-09 09:16:05 -07003887 imu->nr_bvecs = 0;
3888 }
3889
3890 kfree(ctx->user_bufs);
3891 ctx->user_bufs = NULL;
3892 ctx->nr_user_bufs = 0;
3893 return 0;
3894}
3895
3896static int io_copy_iov(struct io_ring_ctx *ctx, struct iovec *dst,
3897 void __user *arg, unsigned index)
3898{
3899 struct iovec __user *src;
3900
3901#ifdef CONFIG_COMPAT
3902 if (ctx->compat) {
3903 struct compat_iovec __user *ciovs;
3904 struct compat_iovec ciov;
3905
3906 ciovs = (struct compat_iovec __user *) arg;
3907 if (copy_from_user(&ciov, &ciovs[index], sizeof(ciov)))
3908 return -EFAULT;
3909
3910 dst->iov_base = (void __user *) (unsigned long) ciov.iov_base;
3911 dst->iov_len = ciov.iov_len;
3912 return 0;
3913 }
3914#endif
3915 src = (struct iovec __user *) arg;
3916 if (copy_from_user(dst, &src[index], sizeof(*dst)))
3917 return -EFAULT;
3918 return 0;
3919}
3920
3921static int io_sqe_buffer_register(struct io_ring_ctx *ctx, void __user *arg,
3922 unsigned nr_args)
3923{
3924 struct vm_area_struct **vmas = NULL;
3925 struct page **pages = NULL;
3926 int i, j, got_pages = 0;
3927 int ret = -EINVAL;
3928
3929 if (ctx->user_bufs)
3930 return -EBUSY;
3931 if (!nr_args || nr_args > UIO_MAXIOV)
3932 return -EINVAL;
3933
3934 ctx->user_bufs = kcalloc(nr_args, sizeof(struct io_mapped_ubuf),
3935 GFP_KERNEL);
3936 if (!ctx->user_bufs)
3937 return -ENOMEM;
3938
3939 for (i = 0; i < nr_args; i++) {
3940 struct io_mapped_ubuf *imu = &ctx->user_bufs[i];
3941 unsigned long off, start, end, ubuf;
3942 int pret, nr_pages;
3943 struct iovec iov;
3944 size_t size;
3945
3946 ret = io_copy_iov(ctx, &iov, arg, i);
3947 if (ret)
Pavel Begunkova2786822019-05-26 12:35:47 +03003948 goto err;
Jens Axboeedafcce2019-01-09 09:16:05 -07003949
3950 /*
3951 * Don't impose further limits on the size and buffer
3952 * constraints here, we'll -EINVAL later when IO is
3953 * submitted if they are wrong.
3954 */
3955 ret = -EFAULT;
3956 if (!iov.iov_base || !iov.iov_len)
3957 goto err;
3958
3959 /* arbitrary limit, but we need something */
3960 if (iov.iov_len > SZ_1G)
3961 goto err;
3962
3963 ubuf = (unsigned long) iov.iov_base;
3964 end = (ubuf + iov.iov_len + PAGE_SIZE - 1) >> PAGE_SHIFT;
3965 start = ubuf >> PAGE_SHIFT;
3966 nr_pages = end - start;
3967
3968 if (ctx->account_mem) {
3969 ret = io_account_mem(ctx->user, nr_pages);
3970 if (ret)
3971 goto err;
3972 }
3973
3974 ret = 0;
3975 if (!pages || nr_pages > got_pages) {
3976 kfree(vmas);
3977 kfree(pages);
Mark Rutlandd4ef6472019-05-01 16:59:16 +01003978 pages = kvmalloc_array(nr_pages, sizeof(struct page *),
Jens Axboeedafcce2019-01-09 09:16:05 -07003979 GFP_KERNEL);
Mark Rutlandd4ef6472019-05-01 16:59:16 +01003980 vmas = kvmalloc_array(nr_pages,
Jens Axboeedafcce2019-01-09 09:16:05 -07003981 sizeof(struct vm_area_struct *),
3982 GFP_KERNEL);
3983 if (!pages || !vmas) {
3984 ret = -ENOMEM;
3985 if (ctx->account_mem)
3986 io_unaccount_mem(ctx->user, nr_pages);
3987 goto err;
3988 }
3989 got_pages = nr_pages;
3990 }
3991
Mark Rutlandd4ef6472019-05-01 16:59:16 +01003992 imu->bvec = kvmalloc_array(nr_pages, sizeof(struct bio_vec),
Jens Axboeedafcce2019-01-09 09:16:05 -07003993 GFP_KERNEL);
3994 ret = -ENOMEM;
3995 if (!imu->bvec) {
3996 if (ctx->account_mem)
3997 io_unaccount_mem(ctx->user, nr_pages);
3998 goto err;
3999 }
4000
4001 ret = 0;
4002 down_read(&current->mm->mmap_sem);
Ira Weiny932f4a62019-05-13 17:17:03 -07004003 pret = get_user_pages(ubuf, nr_pages,
4004 FOLL_WRITE | FOLL_LONGTERM,
4005 pages, vmas);
Jens Axboeedafcce2019-01-09 09:16:05 -07004006 if (pret == nr_pages) {
4007 /* don't support file backed memory */
4008 for (j = 0; j < nr_pages; j++) {
4009 struct vm_area_struct *vma = vmas[j];
4010
4011 if (vma->vm_file &&
4012 !is_file_hugepages(vma->vm_file)) {
4013 ret = -EOPNOTSUPP;
4014 break;
4015 }
4016 }
4017 } else {
4018 ret = pret < 0 ? pret : -EFAULT;
4019 }
4020 up_read(&current->mm->mmap_sem);
4021 if (ret) {
4022 /*
4023 * if we did partial map, or found file backed vmas,
4024 * release any pages we did get
4025 */
John Hubbard27c4d3a2019-08-04 19:32:06 -07004026 if (pret > 0)
4027 put_user_pages(pages, pret);
Jens Axboeedafcce2019-01-09 09:16:05 -07004028 if (ctx->account_mem)
4029 io_unaccount_mem(ctx->user, nr_pages);
Mark Rutlandd4ef6472019-05-01 16:59:16 +01004030 kvfree(imu->bvec);
Jens Axboeedafcce2019-01-09 09:16:05 -07004031 goto err;
4032 }
4033
4034 off = ubuf & ~PAGE_MASK;
4035 size = iov.iov_len;
4036 for (j = 0; j < nr_pages; j++) {
4037 size_t vec_len;
4038
4039 vec_len = min_t(size_t, size, PAGE_SIZE - off);
4040 imu->bvec[j].bv_page = pages[j];
4041 imu->bvec[j].bv_len = vec_len;
4042 imu->bvec[j].bv_offset = off;
4043 off = 0;
4044 size -= vec_len;
4045 }
4046 /* store original address for later verification */
4047 imu->ubuf = ubuf;
4048 imu->len = iov.iov_len;
4049 imu->nr_bvecs = nr_pages;
4050
4051 ctx->nr_user_bufs++;
4052 }
Mark Rutlandd4ef6472019-05-01 16:59:16 +01004053 kvfree(pages);
4054 kvfree(vmas);
Jens Axboeedafcce2019-01-09 09:16:05 -07004055 return 0;
4056err:
Mark Rutlandd4ef6472019-05-01 16:59:16 +01004057 kvfree(pages);
4058 kvfree(vmas);
Jens Axboeedafcce2019-01-09 09:16:05 -07004059 io_sqe_buffer_unregister(ctx);
4060 return ret;
4061}
4062
Jens Axboe9b402842019-04-11 11:45:41 -06004063static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg)
4064{
4065 __s32 __user *fds = arg;
4066 int fd;
4067
4068 if (ctx->cq_ev_fd)
4069 return -EBUSY;
4070
4071 if (copy_from_user(&fd, fds, sizeof(*fds)))
4072 return -EFAULT;
4073
4074 ctx->cq_ev_fd = eventfd_ctx_fdget(fd);
4075 if (IS_ERR(ctx->cq_ev_fd)) {
4076 int ret = PTR_ERR(ctx->cq_ev_fd);
4077 ctx->cq_ev_fd = NULL;
4078 return ret;
4079 }
4080
4081 return 0;
4082}
4083
4084static int io_eventfd_unregister(struct io_ring_ctx *ctx)
4085{
4086 if (ctx->cq_ev_fd) {
4087 eventfd_ctx_put(ctx->cq_ev_fd);
4088 ctx->cq_ev_fd = NULL;
4089 return 0;
4090 }
4091
4092 return -ENXIO;
4093}
4094
Jens Axboe2b188cc2019-01-07 10:46:33 -07004095static void io_ring_ctx_free(struct io_ring_ctx *ctx)
4096{
Jens Axboe6b063142019-01-10 22:13:58 -07004097 io_finish_async(ctx);
Jens Axboe2b188cc2019-01-07 10:46:33 -07004098 if (ctx->sqo_mm)
4099 mmdrop(ctx->sqo_mm);
Jens Axboedef596e2019-01-09 08:59:42 -07004100
4101 io_iopoll_reap_events(ctx);
Jens Axboeedafcce2019-01-09 09:16:05 -07004102 io_sqe_buffer_unregister(ctx);
Jens Axboe6b063142019-01-10 22:13:58 -07004103 io_sqe_files_unregister(ctx);
Jens Axboe9b402842019-04-11 11:45:41 -06004104 io_eventfd_unregister(ctx);
Jens Axboedef596e2019-01-09 08:59:42 -07004105
Jens Axboe2b188cc2019-01-07 10:46:33 -07004106#if defined(CONFIG_UNIX)
Eric Biggers355e8d22019-06-12 14:58:43 -07004107 if (ctx->ring_sock) {
4108 ctx->ring_sock->file = NULL; /* so that iput() is called */
Jens Axboe2b188cc2019-01-07 10:46:33 -07004109 sock_release(ctx->ring_sock);
Eric Biggers355e8d22019-06-12 14:58:43 -07004110 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07004111#endif
4112
Hristo Venev75b28af2019-08-26 17:23:46 +00004113 io_mem_free(ctx->rings);
Jens Axboe2b188cc2019-01-07 10:46:33 -07004114 io_mem_free(ctx->sq_sqes);
Jens Axboe2b188cc2019-01-07 10:46:33 -07004115
4116 percpu_ref_exit(&ctx->refs);
4117 if (ctx->account_mem)
4118 io_unaccount_mem(ctx->user,
4119 ring_pages(ctx->sq_entries, ctx->cq_entries));
4120 free_uid(ctx->user);
Jens Axboe206aefd2019-11-07 18:27:42 -07004121 kfree(ctx->completions);
Jens Axboe2b188cc2019-01-07 10:46:33 -07004122 kfree(ctx);
4123}
4124
4125static __poll_t io_uring_poll(struct file *file, poll_table *wait)
4126{
4127 struct io_ring_ctx *ctx = file->private_data;
4128 __poll_t mask = 0;
4129
4130 poll_wait(file, &ctx->cq_wait, wait);
Stefan Bühler4f7067c2019-04-24 23:54:17 +02004131 /*
4132 * synchronizes with barrier from wq_has_sleeper call in
4133 * io_commit_cqring
4134 */
Jens Axboe2b188cc2019-01-07 10:46:33 -07004135 smp_rmb();
Hristo Venev75b28af2019-08-26 17:23:46 +00004136 if (READ_ONCE(ctx->rings->sq.tail) - ctx->cached_sq_head !=
4137 ctx->rings->sq_ring_entries)
Jens Axboe2b188cc2019-01-07 10:46:33 -07004138 mask |= EPOLLOUT | EPOLLWRNORM;
yangerkundaa5de52019-09-24 20:53:34 +08004139 if (READ_ONCE(ctx->rings->cq.head) != ctx->cached_cq_tail)
Jens Axboe2b188cc2019-01-07 10:46:33 -07004140 mask |= EPOLLIN | EPOLLRDNORM;
4141
4142 return mask;
4143}
4144
4145static int io_uring_fasync(int fd, struct file *file, int on)
4146{
4147 struct io_ring_ctx *ctx = file->private_data;
4148
4149 return fasync_helper(fd, file, on, &ctx->cq_fasync);
4150}
4151
4152static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx)
4153{
4154 mutex_lock(&ctx->uring_lock);
4155 percpu_ref_kill(&ctx->refs);
4156 mutex_unlock(&ctx->uring_lock);
4157
Jens Axboe5262f562019-09-17 12:26:57 -06004158 io_kill_timeouts(ctx);
Jens Axboe221c5eb2019-01-17 09:41:58 -07004159 io_poll_remove_all(ctx);
Jens Axboe561fb042019-10-24 07:25:42 -06004160
4161 if (ctx->io_wq)
4162 io_wq_cancel_all(ctx->io_wq);
4163
Jens Axboedef596e2019-01-09 08:59:42 -07004164 io_iopoll_reap_events(ctx);
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07004165 io_cqring_overflow_flush(ctx, true);
Jens Axboe206aefd2019-11-07 18:27:42 -07004166 wait_for_completion(&ctx->completions[0]);
Jens Axboe2b188cc2019-01-07 10:46:33 -07004167 io_ring_ctx_free(ctx);
4168}
4169
4170static int io_uring_release(struct inode *inode, struct file *file)
4171{
4172 struct io_ring_ctx *ctx = file->private_data;
4173
4174 file->private_data = NULL;
4175 io_ring_ctx_wait_and_kill(ctx);
4176 return 0;
4177}
4178
Jens Axboefcb323c2019-10-24 12:39:47 -06004179static void io_uring_cancel_files(struct io_ring_ctx *ctx,
4180 struct files_struct *files)
4181{
4182 struct io_kiocb *req;
4183 DEFINE_WAIT(wait);
4184
4185 while (!list_empty_careful(&ctx->inflight_list)) {
4186 enum io_wq_cancel ret = IO_WQ_CANCEL_NOTFOUND;
4187
4188 spin_lock_irq(&ctx->inflight_lock);
4189 list_for_each_entry(req, &ctx->inflight_list, inflight_entry) {
4190 if (req->work.files == files) {
4191 ret = io_wq_cancel_work(ctx->io_wq, &req->work);
4192 break;
4193 }
4194 }
4195 if (ret == IO_WQ_CANCEL_RUNNING)
4196 prepare_to_wait(&ctx->inflight_wait, &wait,
4197 TASK_UNINTERRUPTIBLE);
4198
4199 spin_unlock_irq(&ctx->inflight_lock);
4200
4201 /*
4202 * We need to keep going until we get NOTFOUND. We only cancel
4203 * one work at the time.
4204 *
4205 * If we get CANCEL_RUNNING, then wait for a work to complete
4206 * before continuing.
4207 */
4208 if (ret == IO_WQ_CANCEL_OK)
4209 continue;
4210 else if (ret != IO_WQ_CANCEL_RUNNING)
4211 break;
4212 schedule();
4213 }
4214}
4215
4216static int io_uring_flush(struct file *file, void *data)
4217{
4218 struct io_ring_ctx *ctx = file->private_data;
4219
4220 io_uring_cancel_files(ctx, data);
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07004221 if (fatal_signal_pending(current) || (current->flags & PF_EXITING)) {
4222 io_cqring_overflow_flush(ctx, true);
Jens Axboefcb323c2019-10-24 12:39:47 -06004223 io_wq_cancel_all(ctx->io_wq);
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07004224 }
Jens Axboefcb323c2019-10-24 12:39:47 -06004225 return 0;
4226}
4227
Jens Axboe2b188cc2019-01-07 10:46:33 -07004228static int io_uring_mmap(struct file *file, struct vm_area_struct *vma)
4229{
4230 loff_t offset = (loff_t) vma->vm_pgoff << PAGE_SHIFT;
4231 unsigned long sz = vma->vm_end - vma->vm_start;
4232 struct io_ring_ctx *ctx = file->private_data;
4233 unsigned long pfn;
4234 struct page *page;
4235 void *ptr;
4236
4237 switch (offset) {
4238 case IORING_OFF_SQ_RING:
Hristo Venev75b28af2019-08-26 17:23:46 +00004239 case IORING_OFF_CQ_RING:
4240 ptr = ctx->rings;
Jens Axboe2b188cc2019-01-07 10:46:33 -07004241 break;
4242 case IORING_OFF_SQES:
4243 ptr = ctx->sq_sqes;
4244 break;
Jens Axboe2b188cc2019-01-07 10:46:33 -07004245 default:
4246 return -EINVAL;
4247 }
4248
4249 page = virt_to_head_page(ptr);
Matthew Wilcox (Oracle)a50b8542019-09-23 15:34:25 -07004250 if (sz > page_size(page))
Jens Axboe2b188cc2019-01-07 10:46:33 -07004251 return -EINVAL;
4252
4253 pfn = virt_to_phys(ptr) >> PAGE_SHIFT;
4254 return remap_pfn_range(vma, vma->vm_start, pfn, sz, vma->vm_page_prot);
4255}
4256
4257SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
4258 u32, min_complete, u32, flags, const sigset_t __user *, sig,
4259 size_t, sigsz)
4260{
4261 struct io_ring_ctx *ctx;
4262 long ret = -EBADF;
4263 int submitted = 0;
4264 struct fd f;
4265
Jens Axboe6c271ce2019-01-10 11:22:30 -07004266 if (flags & ~(IORING_ENTER_GETEVENTS | IORING_ENTER_SQ_WAKEUP))
Jens Axboe2b188cc2019-01-07 10:46:33 -07004267 return -EINVAL;
4268
4269 f = fdget(fd);
4270 if (!f.file)
4271 return -EBADF;
4272
4273 ret = -EOPNOTSUPP;
4274 if (f.file->f_op != &io_uring_fops)
4275 goto out_fput;
4276
4277 ret = -ENXIO;
4278 ctx = f.file->private_data;
4279 if (!percpu_ref_tryget(&ctx->refs))
4280 goto out_fput;
4281
Jens Axboe6c271ce2019-01-10 11:22:30 -07004282 /*
4283 * For SQ polling, the thread will do all submissions and completions.
4284 * Just return the requested submit count, and wake the thread if
4285 * we were asked to.
4286 */
Jens Axboeb2a9ead2019-09-12 14:19:16 -06004287 ret = 0;
Jens Axboe6c271ce2019-01-10 11:22:30 -07004288 if (ctx->flags & IORING_SETUP_SQPOLL) {
4289 if (flags & IORING_ENTER_SQ_WAKEUP)
4290 wake_up(&ctx->sqo_wait);
4291 submitted = to_submit;
Jens Axboeb2a9ead2019-09-12 14:19:16 -06004292 } else if (to_submit) {
Pavel Begunkovae9428c2019-11-06 00:22:14 +03004293 struct mm_struct *cur_mm;
Jens Axboe2b188cc2019-01-07 10:46:33 -07004294
Pavel Begunkovae9428c2019-11-06 00:22:14 +03004295 to_submit = min(to_submit, ctx->sq_entries);
Jens Axboe2b188cc2019-01-07 10:46:33 -07004296 mutex_lock(&ctx->uring_lock);
Pavel Begunkovae9428c2019-11-06 00:22:14 +03004297 /* already have mm, so io_submit_sqes() won't try to grab it */
4298 cur_mm = ctx->sqo_mm;
4299 submitted = io_submit_sqes(ctx, to_submit, f.file, fd,
4300 &cur_mm, false);
Jens Axboe2b188cc2019-01-07 10:46:33 -07004301 mutex_unlock(&ctx->uring_lock);
Jens Axboe2b188cc2019-01-07 10:46:33 -07004302 }
4303 if (flags & IORING_ENTER_GETEVENTS) {
Jens Axboedef596e2019-01-09 08:59:42 -07004304 unsigned nr_events = 0;
4305
Jens Axboe2b188cc2019-01-07 10:46:33 -07004306 min_complete = min(min_complete, ctx->cq_entries);
4307
Jens Axboedef596e2019-01-09 08:59:42 -07004308 if (ctx->flags & IORING_SETUP_IOPOLL) {
Jens Axboedef596e2019-01-09 08:59:42 -07004309 ret = io_iopoll_check(ctx, &nr_events, min_complete);
Jens Axboedef596e2019-01-09 08:59:42 -07004310 } else {
4311 ret = io_cqring_wait(ctx, min_complete, sig, sigsz);
4312 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07004313 }
4314
Pavel Begunkov6805b322019-10-08 02:18:42 +03004315 percpu_ref_put(&ctx->refs);
Jens Axboe2b188cc2019-01-07 10:46:33 -07004316out_fput:
4317 fdput(f);
4318 return submitted ? submitted : ret;
4319}
4320
4321static const struct file_operations io_uring_fops = {
4322 .release = io_uring_release,
Jens Axboefcb323c2019-10-24 12:39:47 -06004323 .flush = io_uring_flush,
Jens Axboe2b188cc2019-01-07 10:46:33 -07004324 .mmap = io_uring_mmap,
4325 .poll = io_uring_poll,
4326 .fasync = io_uring_fasync,
4327};
4328
4329static int io_allocate_scq_urings(struct io_ring_ctx *ctx,
4330 struct io_uring_params *p)
4331{
Hristo Venev75b28af2019-08-26 17:23:46 +00004332 struct io_rings *rings;
4333 size_t size, sq_array_offset;
Jens Axboe2b188cc2019-01-07 10:46:33 -07004334
Hristo Venev75b28af2019-08-26 17:23:46 +00004335 size = rings_size(p->sq_entries, p->cq_entries, &sq_array_offset);
4336 if (size == SIZE_MAX)
4337 return -EOVERFLOW;
4338
4339 rings = io_mem_alloc(size);
4340 if (!rings)
Jens Axboe2b188cc2019-01-07 10:46:33 -07004341 return -ENOMEM;
4342
Hristo Venev75b28af2019-08-26 17:23:46 +00004343 ctx->rings = rings;
4344 ctx->sq_array = (u32 *)((char *)rings + sq_array_offset);
4345 rings->sq_ring_mask = p->sq_entries - 1;
4346 rings->cq_ring_mask = p->cq_entries - 1;
4347 rings->sq_ring_entries = p->sq_entries;
4348 rings->cq_ring_entries = p->cq_entries;
4349 ctx->sq_mask = rings->sq_ring_mask;
4350 ctx->cq_mask = rings->cq_ring_mask;
4351 ctx->sq_entries = rings->sq_ring_entries;
4352 ctx->cq_entries = rings->cq_ring_entries;
Jens Axboe2b188cc2019-01-07 10:46:33 -07004353
4354 size = array_size(sizeof(struct io_uring_sqe), p->sq_entries);
4355 if (size == SIZE_MAX)
4356 return -EOVERFLOW;
4357
4358 ctx->sq_sqes = io_mem_alloc(size);
Mark Rutland52e04ef2019-04-30 17:30:21 +01004359 if (!ctx->sq_sqes)
Jens Axboe2b188cc2019-01-07 10:46:33 -07004360 return -ENOMEM;
Jens Axboe2b188cc2019-01-07 10:46:33 -07004361
Jens Axboe2b188cc2019-01-07 10:46:33 -07004362 return 0;
4363}
4364
4365/*
4366 * Allocate an anonymous fd, this is what constitutes the application
4367 * visible backing of an io_uring instance. The application mmaps this
4368 * fd to gain access to the SQ/CQ ring details. If UNIX sockets are enabled,
4369 * we have to tie this fd to a socket for file garbage collection purposes.
4370 */
4371static int io_uring_get_fd(struct io_ring_ctx *ctx)
4372{
4373 struct file *file;
4374 int ret;
4375
4376#if defined(CONFIG_UNIX)
4377 ret = sock_create_kern(&init_net, PF_UNIX, SOCK_RAW, IPPROTO_IP,
4378 &ctx->ring_sock);
4379 if (ret)
4380 return ret;
4381#endif
4382
4383 ret = get_unused_fd_flags(O_RDWR | O_CLOEXEC);
4384 if (ret < 0)
4385 goto err;
4386
4387 file = anon_inode_getfile("[io_uring]", &io_uring_fops, ctx,
4388 O_RDWR | O_CLOEXEC);
4389 if (IS_ERR(file)) {
4390 put_unused_fd(ret);
4391 ret = PTR_ERR(file);
4392 goto err;
4393 }
4394
4395#if defined(CONFIG_UNIX)
4396 ctx->ring_sock->file = file;
Jens Axboe6b063142019-01-10 22:13:58 -07004397 ctx->ring_sock->sk->sk_user_data = ctx;
Jens Axboe2b188cc2019-01-07 10:46:33 -07004398#endif
4399 fd_install(ret, file);
4400 return ret;
4401err:
4402#if defined(CONFIG_UNIX)
4403 sock_release(ctx->ring_sock);
4404 ctx->ring_sock = NULL;
4405#endif
4406 return ret;
4407}
4408
4409static int io_uring_create(unsigned entries, struct io_uring_params *p)
4410{
4411 struct user_struct *user = NULL;
4412 struct io_ring_ctx *ctx;
4413 bool account_mem;
4414 int ret;
4415
4416 if (!entries || entries > IORING_MAX_ENTRIES)
4417 return -EINVAL;
4418
4419 /*
4420 * Use twice as many entries for the CQ ring. It's possible for the
4421 * application to drive a higher depth than the size of the SQ ring,
4422 * since the sqes are only used at submission time. This allows for
Jens Axboe33a107f2019-10-04 12:10:03 -06004423 * some flexibility in overcommitting a bit. If the application has
4424 * set IORING_SETUP_CQSIZE, it will have passed in the desired number
4425 * of CQ ring entries manually.
Jens Axboe2b188cc2019-01-07 10:46:33 -07004426 */
4427 p->sq_entries = roundup_pow_of_two(entries);
Jens Axboe33a107f2019-10-04 12:10:03 -06004428 if (p->flags & IORING_SETUP_CQSIZE) {
4429 /*
4430 * If IORING_SETUP_CQSIZE is set, we do the same roundup
4431 * to a power-of-two, if it isn't already. We do NOT impose
4432 * any cq vs sq ring sizing.
4433 */
4434 if (p->cq_entries < p->sq_entries || p->cq_entries > IORING_MAX_CQ_ENTRIES)
4435 return -EINVAL;
4436 p->cq_entries = roundup_pow_of_two(p->cq_entries);
4437 } else {
4438 p->cq_entries = 2 * p->sq_entries;
4439 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07004440
4441 user = get_uid(current_user());
4442 account_mem = !capable(CAP_IPC_LOCK);
4443
4444 if (account_mem) {
4445 ret = io_account_mem(user,
4446 ring_pages(p->sq_entries, p->cq_entries));
4447 if (ret) {
4448 free_uid(user);
4449 return ret;
4450 }
4451 }
4452
4453 ctx = io_ring_ctx_alloc(p);
4454 if (!ctx) {
4455 if (account_mem)
4456 io_unaccount_mem(user, ring_pages(p->sq_entries,
4457 p->cq_entries));
4458 free_uid(user);
4459 return -ENOMEM;
4460 }
4461 ctx->compat = in_compat_syscall();
4462 ctx->account_mem = account_mem;
4463 ctx->user = user;
4464
4465 ret = io_allocate_scq_urings(ctx, p);
4466 if (ret)
4467 goto err;
4468
Jens Axboe6c271ce2019-01-10 11:22:30 -07004469 ret = io_sq_offload_start(ctx, p);
Jens Axboe2b188cc2019-01-07 10:46:33 -07004470 if (ret)
4471 goto err;
4472
Jens Axboe2b188cc2019-01-07 10:46:33 -07004473 memset(&p->sq_off, 0, sizeof(p->sq_off));
Hristo Venev75b28af2019-08-26 17:23:46 +00004474 p->sq_off.head = offsetof(struct io_rings, sq.head);
4475 p->sq_off.tail = offsetof(struct io_rings, sq.tail);
4476 p->sq_off.ring_mask = offsetof(struct io_rings, sq_ring_mask);
4477 p->sq_off.ring_entries = offsetof(struct io_rings, sq_ring_entries);
4478 p->sq_off.flags = offsetof(struct io_rings, sq_flags);
4479 p->sq_off.dropped = offsetof(struct io_rings, sq_dropped);
4480 p->sq_off.array = (char *)ctx->sq_array - (char *)ctx->rings;
Jens Axboe2b188cc2019-01-07 10:46:33 -07004481
4482 memset(&p->cq_off, 0, sizeof(p->cq_off));
Hristo Venev75b28af2019-08-26 17:23:46 +00004483 p->cq_off.head = offsetof(struct io_rings, cq.head);
4484 p->cq_off.tail = offsetof(struct io_rings, cq.tail);
4485 p->cq_off.ring_mask = offsetof(struct io_rings, cq_ring_mask);
4486 p->cq_off.ring_entries = offsetof(struct io_rings, cq_ring_entries);
4487 p->cq_off.overflow = offsetof(struct io_rings, cq_overflow);
4488 p->cq_off.cqes = offsetof(struct io_rings, cqes);
Jens Axboeac90f242019-09-06 10:26:21 -06004489
Jens Axboe044c1ab2019-10-28 09:15:33 -06004490 /*
4491 * Install ring fd as the very last thing, so we don't risk someone
4492 * having closed it before we finish setup
4493 */
4494 ret = io_uring_get_fd(ctx);
4495 if (ret < 0)
4496 goto err;
4497
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07004498 p->features = IORING_FEAT_SINGLE_MMAP | IORING_FEAT_NODROP;
Dmitrii Dolgovc826bd72019-10-15 19:02:01 +02004499 trace_io_uring_create(ret, ctx, p->sq_entries, p->cq_entries, p->flags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07004500 return ret;
4501err:
4502 io_ring_ctx_wait_and_kill(ctx);
4503 return ret;
4504}
4505
4506/*
4507 * Sets up an aio uring context, and returns the fd. Applications asks for a
4508 * ring size, we return the actual sq/cq ring sizes (among other things) in the
4509 * params structure passed in.
4510 */
4511static long io_uring_setup(u32 entries, struct io_uring_params __user *params)
4512{
4513 struct io_uring_params p;
4514 long ret;
4515 int i;
4516
4517 if (copy_from_user(&p, params, sizeof(p)))
4518 return -EFAULT;
4519 for (i = 0; i < ARRAY_SIZE(p.resv); i++) {
4520 if (p.resv[i])
4521 return -EINVAL;
4522 }
4523
Jens Axboe6c271ce2019-01-10 11:22:30 -07004524 if (p.flags & ~(IORING_SETUP_IOPOLL | IORING_SETUP_SQPOLL |
Jens Axboe33a107f2019-10-04 12:10:03 -06004525 IORING_SETUP_SQ_AFF | IORING_SETUP_CQSIZE))
Jens Axboe2b188cc2019-01-07 10:46:33 -07004526 return -EINVAL;
4527
4528 ret = io_uring_create(entries, &p);
4529 if (ret < 0)
4530 return ret;
4531
4532 if (copy_to_user(params, &p, sizeof(p)))
4533 return -EFAULT;
4534
4535 return ret;
4536}
4537
4538SYSCALL_DEFINE2(io_uring_setup, u32, entries,
4539 struct io_uring_params __user *, params)
4540{
4541 return io_uring_setup(entries, params);
4542}
4543
Jens Axboeedafcce2019-01-09 09:16:05 -07004544static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,
4545 void __user *arg, unsigned nr_args)
Jens Axboeb19062a2019-04-15 10:49:38 -06004546 __releases(ctx->uring_lock)
4547 __acquires(ctx->uring_lock)
Jens Axboeedafcce2019-01-09 09:16:05 -07004548{
4549 int ret;
4550
Jens Axboe35fa71a2019-04-22 10:23:23 -06004551 /*
4552 * We're inside the ring mutex, if the ref is already dying, then
4553 * someone else killed the ctx or is already going through
4554 * io_uring_register().
4555 */
4556 if (percpu_ref_is_dying(&ctx->refs))
4557 return -ENXIO;
4558
Jens Axboeedafcce2019-01-09 09:16:05 -07004559 percpu_ref_kill(&ctx->refs);
Jens Axboeb19062a2019-04-15 10:49:38 -06004560
4561 /*
4562 * Drop uring mutex before waiting for references to exit. If another
4563 * thread is currently inside io_uring_enter() it might need to grab
4564 * the uring_lock to make progress. If we hold it here across the drain
4565 * wait, then we can deadlock. It's safe to drop the mutex here, since
4566 * no new references will come in after we've killed the percpu ref.
4567 */
4568 mutex_unlock(&ctx->uring_lock);
Jens Axboe206aefd2019-11-07 18:27:42 -07004569 wait_for_completion(&ctx->completions[0]);
Jens Axboeb19062a2019-04-15 10:49:38 -06004570 mutex_lock(&ctx->uring_lock);
Jens Axboeedafcce2019-01-09 09:16:05 -07004571
4572 switch (opcode) {
4573 case IORING_REGISTER_BUFFERS:
4574 ret = io_sqe_buffer_register(ctx, arg, nr_args);
4575 break;
4576 case IORING_UNREGISTER_BUFFERS:
4577 ret = -EINVAL;
4578 if (arg || nr_args)
4579 break;
4580 ret = io_sqe_buffer_unregister(ctx);
4581 break;
Jens Axboe6b063142019-01-10 22:13:58 -07004582 case IORING_REGISTER_FILES:
4583 ret = io_sqe_files_register(ctx, arg, nr_args);
4584 break;
4585 case IORING_UNREGISTER_FILES:
4586 ret = -EINVAL;
4587 if (arg || nr_args)
4588 break;
4589 ret = io_sqe_files_unregister(ctx);
4590 break;
Jens Axboec3a31e62019-10-03 13:59:56 -06004591 case IORING_REGISTER_FILES_UPDATE:
4592 ret = io_sqe_files_update(ctx, arg, nr_args);
4593 break;
Jens Axboe9b402842019-04-11 11:45:41 -06004594 case IORING_REGISTER_EVENTFD:
4595 ret = -EINVAL;
4596 if (nr_args != 1)
4597 break;
4598 ret = io_eventfd_register(ctx, arg);
4599 break;
4600 case IORING_UNREGISTER_EVENTFD:
4601 ret = -EINVAL;
4602 if (arg || nr_args)
4603 break;
4604 ret = io_eventfd_unregister(ctx);
4605 break;
Jens Axboeedafcce2019-01-09 09:16:05 -07004606 default:
4607 ret = -EINVAL;
4608 break;
4609 }
4610
4611 /* bring the ctx back to life */
Jens Axboe206aefd2019-11-07 18:27:42 -07004612 reinit_completion(&ctx->completions[0]);
Jens Axboeedafcce2019-01-09 09:16:05 -07004613 percpu_ref_reinit(&ctx->refs);
4614 return ret;
4615}
4616
4617SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode,
4618 void __user *, arg, unsigned int, nr_args)
4619{
4620 struct io_ring_ctx *ctx;
4621 long ret = -EBADF;
4622 struct fd f;
4623
4624 f = fdget(fd);
4625 if (!f.file)
4626 return -EBADF;
4627
4628 ret = -EOPNOTSUPP;
4629 if (f.file->f_op != &io_uring_fops)
4630 goto out_fput;
4631
4632 ctx = f.file->private_data;
4633
4634 mutex_lock(&ctx->uring_lock);
4635 ret = __io_uring_register(ctx, opcode, arg, nr_args);
4636 mutex_unlock(&ctx->uring_lock);
Dmitrii Dolgovc826bd72019-10-15 19:02:01 +02004637 trace_io_uring_register(ctx, opcode, ctx->nr_user_files, ctx->nr_user_bufs,
4638 ctx->cq_ev_fd != NULL, ret);
Jens Axboeedafcce2019-01-09 09:16:05 -07004639out_fput:
4640 fdput(f);
4641 return ret;
4642}
4643
Jens Axboe2b188cc2019-01-07 10:46:33 -07004644static int __init io_uring_init(void)
4645{
4646 req_cachep = KMEM_CACHE(io_kiocb, SLAB_HWCACHE_ALIGN | SLAB_PANIC);
4647 return 0;
4648};
4649__initcall(io_uring_init);