blob: 7426e4f23f9be3914fdda88898ebff8c6ce714ce [file] [log] [blame]
Jens Axboe2b188cc2019-01-07 10:46:33 -07001// SPDX-License-Identifier: GPL-2.0
2/*
3 * Shared application/kernel submission and completion ring pairs, for
4 * supporting fast/efficient IO.
5 *
6 * A note on the read/write ordering memory barriers that are matched between
Stefan Bühler1e84b972019-04-24 23:54:16 +02007 * the application and kernel side.
8 *
9 * After the application reads the CQ ring tail, it must use an
10 * appropriate smp_rmb() to pair with the smp_wmb() the kernel uses
11 * before writing the tail (using smp_load_acquire to read the tail will
12 * do). It also needs a smp_mb() before updating CQ head (ordering the
13 * entry load(s) with the head store), pairing with an implicit barrier
14 * through a control-dependency in io_get_cqring (smp_store_release to
15 * store head will do). Failure to do so could lead to reading invalid
16 * CQ entries.
17 *
18 * Likewise, the application must use an appropriate smp_wmb() before
19 * writing the SQ tail (ordering SQ entry stores with the tail store),
20 * which pairs with smp_load_acquire in io_get_sqring (smp_store_release
21 * to store the tail will do). And it needs a barrier ordering the SQ
22 * head load before writing new SQ entries (smp_load_acquire to read
23 * head will do).
24 *
25 * When using the SQ poll thread (IORING_SETUP_SQPOLL), the application
26 * needs to check the SQ flags for IORING_SQ_NEED_WAKEUP *after*
27 * updating the SQ tail; a full memory barrier smp_mb() is needed
28 * between.
Jens Axboe2b188cc2019-01-07 10:46:33 -070029 *
30 * Also see the examples in the liburing library:
31 *
32 * git://git.kernel.dk/liburing
33 *
34 * io_uring also uses READ/WRITE_ONCE() for _any_ store or load that happens
35 * from data shared between the kernel and application. This is done both
36 * for ordering purposes, but also to ensure that once a value is loaded from
37 * data that the application could potentially modify, it remains stable.
38 *
39 * Copyright (C) 2018-2019 Jens Axboe
Christoph Hellwigc992fe22019-01-11 09:43:02 -070040 * Copyright (c) 2018-2019 Christoph Hellwig
Jens Axboe2b188cc2019-01-07 10:46:33 -070041 */
42#include <linux/kernel.h>
43#include <linux/init.h>
44#include <linux/errno.h>
45#include <linux/syscalls.h>
46#include <linux/compat.h>
Jens Axboe52de1fe2020-02-27 10:15:42 -070047#include <net/compat.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070048#include <linux/refcount.h>
49#include <linux/uio.h>
Pavel Begunkov6b47ee62020-01-18 20:22:41 +030050#include <linux/bits.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070051
52#include <linux/sched/signal.h>
53#include <linux/fs.h>
54#include <linux/file.h>
55#include <linux/fdtable.h>
56#include <linux/mm.h>
57#include <linux/mman.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070058#include <linux/percpu.h>
59#include <linux/slab.h>
Jens Axboe6c271ce2019-01-10 11:22:30 -070060#include <linux/kthread.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070061#include <linux/blkdev.h>
Jens Axboeedafcce2019-01-09 09:16:05 -070062#include <linux/bvec.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070063#include <linux/net.h>
64#include <net/sock.h>
65#include <net/af_unix.h>
Jens Axboe6b063142019-01-10 22:13:58 -070066#include <net/scm.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070067#include <linux/anon_inodes.h>
68#include <linux/sched/mm.h>
69#include <linux/uaccess.h>
70#include <linux/nospec.h>
Jens Axboeedafcce2019-01-09 09:16:05 -070071#include <linux/sizes.h>
72#include <linux/hugetlb.h>
Jens Axboeaa4c3962019-11-29 10:14:00 -070073#include <linux/highmem.h>
Jens Axboe15b71ab2019-12-11 11:20:36 -070074#include <linux/namei.h>
75#include <linux/fsnotify.h>
Jens Axboe4840e412019-12-25 22:03:45 -070076#include <linux/fadvise.h>
Jens Axboe3e4827b2020-01-08 15:18:09 -070077#include <linux/eventpoll.h>
Jens Axboeff002b32020-02-07 16:05:21 -070078#include <linux/fs_struct.h>
Pavel Begunkov7d67af22020-02-24 11:32:45 +030079#include <linux/splice.h>
Jens Axboeb41e9852020-02-17 09:52:41 -070080#include <linux/task_work.h>
Jens Axboebcf5a062020-05-22 09:24:42 -060081#include <linux/pagemap.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070082
Dmitrii Dolgovc826bd72019-10-15 19:02:01 +020083#define CREATE_TRACE_POINTS
84#include <trace/events/io_uring.h>
85
Jens Axboe2b188cc2019-01-07 10:46:33 -070086#include <uapi/linux/io_uring.h>
87
88#include "internal.h"
Jens Axboe561fb042019-10-24 07:25:42 -060089#include "io-wq.h"
Jens Axboe2b188cc2019-01-07 10:46:33 -070090
Daniel Xu5277dea2019-09-14 14:23:45 -070091#define IORING_MAX_ENTRIES 32768
Jens Axboe33a107f2019-10-04 12:10:03 -060092#define IORING_MAX_CQ_ENTRIES (2 * IORING_MAX_ENTRIES)
Jens Axboe65e19f52019-10-26 07:20:21 -060093
94/*
95 * Shift of 9 is 512 entries, or exactly one page on 64-bit archs
96 */
97#define IORING_FILE_TABLE_SHIFT 9
98#define IORING_MAX_FILES_TABLE (1U << IORING_FILE_TABLE_SHIFT)
99#define IORING_FILE_TABLE_MASK (IORING_MAX_FILES_TABLE - 1)
100#define IORING_MAX_FIXED_FILES (64 * IORING_MAX_FILES_TABLE)
Jens Axboe2b188cc2019-01-07 10:46:33 -0700101
102struct io_uring {
103 u32 head ____cacheline_aligned_in_smp;
104 u32 tail ____cacheline_aligned_in_smp;
105};
106
Stefan Bühler1e84b972019-04-24 23:54:16 +0200107/*
Hristo Venev75b28af2019-08-26 17:23:46 +0000108 * This data is shared with the application through the mmap at offsets
109 * IORING_OFF_SQ_RING and IORING_OFF_CQ_RING.
Stefan Bühler1e84b972019-04-24 23:54:16 +0200110 *
111 * The offsets to the member fields are published through struct
112 * io_sqring_offsets when calling io_uring_setup.
113 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000114struct io_rings {
Stefan Bühler1e84b972019-04-24 23:54:16 +0200115 /*
116 * Head and tail offsets into the ring; the offsets need to be
117 * masked to get valid indices.
118 *
Hristo Venev75b28af2019-08-26 17:23:46 +0000119 * The kernel controls head of the sq ring and the tail of the cq ring,
120 * and the application controls tail of the sq ring and the head of the
121 * cq ring.
Stefan Bühler1e84b972019-04-24 23:54:16 +0200122 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000123 struct io_uring sq, cq;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200124 /*
Hristo Venev75b28af2019-08-26 17:23:46 +0000125 * Bitmasks to apply to head and tail offsets (constant, equals
Stefan Bühler1e84b972019-04-24 23:54:16 +0200126 * ring_entries - 1)
127 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000128 u32 sq_ring_mask, cq_ring_mask;
129 /* Ring sizes (constant, power of 2) */
130 u32 sq_ring_entries, cq_ring_entries;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200131 /*
132 * Number of invalid entries dropped by the kernel due to
133 * invalid index stored in array
134 *
135 * Written by the kernel, shouldn't be modified by the
136 * application (i.e. get number of "new events" by comparing to
137 * cached value).
138 *
139 * After a new SQ head value was read by the application this
140 * counter includes all submissions that were dropped reaching
141 * the new SQ head (and possibly more).
142 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000143 u32 sq_dropped;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200144 /*
Stefano Garzarella0d9b5b32020-05-15 18:38:04 +0200145 * Runtime SQ flags
Stefan Bühler1e84b972019-04-24 23:54:16 +0200146 *
147 * Written by the kernel, shouldn't be modified by the
148 * application.
149 *
150 * The application needs a full memory barrier before checking
151 * for IORING_SQ_NEED_WAKEUP after updating the sq tail.
152 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000153 u32 sq_flags;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200154 /*
Stefano Garzarella0d9b5b32020-05-15 18:38:04 +0200155 * Runtime CQ flags
156 *
157 * Written by the application, shouldn't be modified by the
158 * kernel.
159 */
160 u32 cq_flags;
161 /*
Stefan Bühler1e84b972019-04-24 23:54:16 +0200162 * Number of completion events lost because the queue was full;
163 * this should be avoided by the application by making sure
LimingWu0b4295b2019-12-05 20:18:18 +0800164 * there are not more requests pending than there is space in
Stefan Bühler1e84b972019-04-24 23:54:16 +0200165 * the completion queue.
166 *
167 * Written by the kernel, shouldn't be modified by the
168 * application (i.e. get number of "new events" by comparing to
169 * cached value).
170 *
171 * As completion events come in out of order this counter is not
172 * ordered with any other data.
173 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000174 u32 cq_overflow;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200175 /*
176 * Ring buffer of completion events.
177 *
178 * The kernel writes completion events fresh every time they are
179 * produced, so the application is allowed to modify pending
180 * entries.
181 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000182 struct io_uring_cqe cqes[] ____cacheline_aligned_in_smp;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700183};
184
Jens Axboeedafcce2019-01-09 09:16:05 -0700185struct io_mapped_ubuf {
186 u64 ubuf;
187 size_t len;
188 struct bio_vec *bvec;
189 unsigned int nr_bvecs;
190};
191
Jens Axboe65e19f52019-10-26 07:20:21 -0600192struct fixed_file_table {
193 struct file **files;
Jens Axboe31b51512019-01-18 22:56:34 -0700194};
195
Xiaoguang Wang05589552020-03-31 14:05:18 +0800196struct fixed_file_ref_node {
197 struct percpu_ref refs;
198 struct list_head node;
199 struct list_head file_list;
200 struct fixed_file_data *file_data;
Jens Axboe4a38aed22020-05-14 17:21:15 -0600201 struct llist_node llist;
Xiaoguang Wang05589552020-03-31 14:05:18 +0800202};
203
Jens Axboe05f3fb32019-12-09 11:22:50 -0700204struct fixed_file_data {
205 struct fixed_file_table *table;
206 struct io_ring_ctx *ctx;
207
Xiaoguang Wang05589552020-03-31 14:05:18 +0800208 struct percpu_ref *cur_refs;
Jens Axboe05f3fb32019-12-09 11:22:50 -0700209 struct percpu_ref refs;
Jens Axboe05f3fb32019-12-09 11:22:50 -0700210 struct completion done;
Xiaoguang Wang05589552020-03-31 14:05:18 +0800211 struct list_head ref_list;
212 spinlock_t lock;
Jens Axboe05f3fb32019-12-09 11:22:50 -0700213};
214
Jens Axboe5a2e7452020-02-23 16:23:11 -0700215struct io_buffer {
216 struct list_head list;
217 __u64 addr;
218 __s32 len;
219 __u16 bid;
220};
221
Jens Axboe2b188cc2019-01-07 10:46:33 -0700222struct io_ring_ctx {
223 struct {
224 struct percpu_ref refs;
225 } ____cacheline_aligned_in_smp;
226
227 struct {
228 unsigned int flags;
Randy Dunlape1d85332020-02-05 20:57:10 -0800229 unsigned int compat: 1;
Bijan Mottahedehaad5d8d2020-06-16 16:36:08 -0700230 unsigned int limit_mem: 1;
Randy Dunlape1d85332020-02-05 20:57:10 -0800231 unsigned int cq_overflow_flushed: 1;
232 unsigned int drain_next: 1;
233 unsigned int eventfd_async: 1;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700234
Hristo Venev75b28af2019-08-26 17:23:46 +0000235 /*
236 * Ring buffer of indices into array of io_uring_sqe, which is
237 * mmapped by the application using the IORING_OFF_SQES offset.
238 *
239 * This indirection could e.g. be used to assign fixed
240 * io_uring_sqe entries to operations and only submit them to
241 * the queue when needed.
242 *
243 * The kernel modifies neither the indices array nor the entries
244 * array.
245 */
246 u32 *sq_array;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700247 unsigned cached_sq_head;
248 unsigned sq_entries;
249 unsigned sq_mask;
Jens Axboe6c271ce2019-01-10 11:22:30 -0700250 unsigned sq_thread_idle;
Jens Axboe498ccd92019-10-25 10:04:25 -0600251 unsigned cached_sq_dropped;
Jens Axboe206aefd2019-11-07 18:27:42 -0700252 atomic_t cached_cq_overflow;
Jens Axboead3eb2c2019-12-18 17:12:20 -0700253 unsigned long sq_check_overflow;
Jens Axboede0617e2019-04-06 21:51:27 -0600254
255 struct list_head defer_list;
Jens Axboe5262f562019-09-17 12:26:57 -0600256 struct list_head timeout_list;
Jens Axboe1d7bb1d2019-11-06 11:31:17 -0700257 struct list_head cq_overflow_list;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700258
Jens Axboefcb323c2019-10-24 12:39:47 -0600259 wait_queue_head_t inflight_wait;
Jens Axboead3eb2c2019-12-18 17:12:20 -0700260 struct io_uring_sqe *sq_sqes;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700261 } ____cacheline_aligned_in_smp;
262
Hristo Venev75b28af2019-08-26 17:23:46 +0000263 struct io_rings *rings;
264
Jens Axboe2b188cc2019-01-07 10:46:33 -0700265 /* IO offload */
Jens Axboe561fb042019-10-24 07:25:42 -0600266 struct io_wq *io_wq;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700267 struct task_struct *sqo_thread; /* if using sq thread polling */
268 struct mm_struct *sqo_mm;
269 wait_queue_head_t sqo_wait;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700270
Jens Axboe6b063142019-01-10 22:13:58 -0700271 /*
272 * If used, fixed file set. Writers must ensure that ->refs is dead,
273 * readers must ensure that ->refs is alive as long as the file* is
274 * used. Only updated through io_uring_register(2).
275 */
Jens Axboe05f3fb32019-12-09 11:22:50 -0700276 struct fixed_file_data *file_data;
Jens Axboe6b063142019-01-10 22:13:58 -0700277 unsigned nr_user_files;
Pavel Begunkovb14cca02020-01-17 04:45:59 +0300278 int ring_fd;
279 struct file *ring_file;
Jens Axboe6b063142019-01-10 22:13:58 -0700280
Jens Axboeedafcce2019-01-09 09:16:05 -0700281 /* if used, fixed mapped user buffers */
282 unsigned nr_user_bufs;
283 struct io_mapped_ubuf *user_bufs;
284
Jens Axboe2b188cc2019-01-07 10:46:33 -0700285 struct user_struct *user;
286
Jens Axboe0b8c0ec2019-12-02 08:50:00 -0700287 const struct cred *creds;
Jens Axboe181e4482019-11-25 08:52:30 -0700288
Jens Axboe0f158b42020-05-14 17:18:39 -0600289 struct completion ref_comp;
290 struct completion sq_thread_comp;
Jens Axboe206aefd2019-11-07 18:27:42 -0700291
Jens Axboe0ddf92e2019-11-08 08:52:53 -0700292 /* if all else fails... */
293 struct io_kiocb *fallback_req;
294
Jens Axboe206aefd2019-11-07 18:27:42 -0700295#if defined(CONFIG_UNIX)
296 struct socket *ring_sock;
297#endif
298
Jens Axboe5a2e7452020-02-23 16:23:11 -0700299 struct idr io_buffer_idr;
300
Jens Axboe071698e2020-01-28 10:04:42 -0700301 struct idr personality_idr;
302
Jens Axboe206aefd2019-11-07 18:27:42 -0700303 struct {
304 unsigned cached_cq_tail;
305 unsigned cq_entries;
306 unsigned cq_mask;
307 atomic_t cq_timeouts;
Jens Axboead3eb2c2019-12-18 17:12:20 -0700308 unsigned long cq_check_overflow;
Jens Axboe206aefd2019-11-07 18:27:42 -0700309 struct wait_queue_head cq_wait;
310 struct fasync_struct *cq_fasync;
311 struct eventfd_ctx *cq_ev_fd;
312 } ____cacheline_aligned_in_smp;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700313
314 struct {
315 struct mutex uring_lock;
316 wait_queue_head_t wait;
317 } ____cacheline_aligned_in_smp;
318
319 struct {
320 spinlock_t completion_lock;
Jens Axboee94f1412019-12-19 12:06:02 -0700321
Jens Axboedef596e2019-01-09 08:59:42 -0700322 /*
323 * ->poll_list is protected by the ctx->uring_lock for
324 * io_uring instances that don't use IORING_SETUP_SQPOLL.
325 * For SQPOLL, only the single threaded io_sq_thread() will
326 * manipulate the list, hence no extra locking is needed there.
327 */
328 struct list_head poll_list;
Jens Axboe78076bb2019-12-04 19:56:40 -0700329 struct hlist_head *cancel_hash;
330 unsigned cancel_hash_bits;
Jens Axboee94f1412019-12-19 12:06:02 -0700331 bool poll_multi_file;
Jens Axboefcb323c2019-10-24 12:39:47 -0600332
333 spinlock_t inflight_lock;
334 struct list_head inflight_list;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700335 } ____cacheline_aligned_in_smp;
Jens Axboe85faa7b2020-04-09 18:14:00 -0600336
Jens Axboe4a38aed22020-05-14 17:21:15 -0600337 struct delayed_work file_put_work;
338 struct llist_head file_put_llist;
339
Jens Axboe85faa7b2020-04-09 18:14:00 -0600340 struct work_struct exit_work;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700341};
342
Jens Axboe09bb8392019-03-13 12:39:28 -0600343/*
344 * First field must be the file pointer in all the
345 * iocb unions! See also 'struct kiocb' in <linux/fs.h>
346 */
Jens Axboe221c5eb2019-01-17 09:41:58 -0700347struct io_poll_iocb {
348 struct file *file;
Jens Axboe0969e782019-12-17 18:40:57 -0700349 union {
350 struct wait_queue_head *head;
351 u64 addr;
352 };
Jens Axboe221c5eb2019-01-17 09:41:58 -0700353 __poll_t events;
Jens Axboe8c838782019-03-12 15:48:16 -0600354 bool done;
Jens Axboe221c5eb2019-01-17 09:41:58 -0700355 bool canceled;
Jens Axboe392edb42019-12-09 17:52:20 -0700356 struct wait_queue_entry wait;
Jens Axboe221c5eb2019-01-17 09:41:58 -0700357};
358
Jens Axboeb5dba592019-12-11 14:02:38 -0700359struct io_close {
360 struct file *file;
361 struct file *put_file;
362 int fd;
363};
364
Jens Axboead8a48a2019-11-15 08:49:11 -0700365struct io_timeout_data {
366 struct io_kiocb *req;
367 struct hrtimer timer;
368 struct timespec64 ts;
369 enum hrtimer_mode mode;
370};
371
Jens Axboe8ed8d3c2019-12-16 11:55:28 -0700372struct io_accept {
373 struct file *file;
374 struct sockaddr __user *addr;
375 int __user *addr_len;
376 int flags;
Jens Axboe09952e32020-03-19 20:16:56 -0600377 unsigned long nofile;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -0700378};
379
380struct io_sync {
381 struct file *file;
382 loff_t len;
383 loff_t off;
384 int flags;
Jens Axboed63d1b52019-12-10 10:38:56 -0700385 int mode;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -0700386};
387
Jens Axboefbf23842019-12-17 18:45:56 -0700388struct io_cancel {
389 struct file *file;
390 u64 addr;
391};
392
Jens Axboeb29472e2019-12-17 18:50:29 -0700393struct io_timeout {
394 struct file *file;
395 u64 addr;
396 int flags;
Pavel Begunkovbfe68a22020-05-30 14:54:18 +0300397 u32 off;
398 u32 target_seq;
Jens Axboeb29472e2019-12-17 18:50:29 -0700399};
400
Jens Axboe9adbd452019-12-20 08:45:55 -0700401struct io_rw {
402 /* NOTE: kiocb has the file as the first member, so don't do it here */
403 struct kiocb kiocb;
404 u64 addr;
405 u64 len;
406};
407
Jens Axboe3fbb51c2019-12-20 08:51:52 -0700408struct io_connect {
409 struct file *file;
410 struct sockaddr __user *addr;
411 int addr_len;
412};
413
Jens Axboee47293f2019-12-20 08:58:21 -0700414struct io_sr_msg {
415 struct file *file;
Jens Axboefddafac2020-01-04 20:19:44 -0700416 union {
417 struct user_msghdr __user *msg;
418 void __user *buf;
419 };
Jens Axboee47293f2019-12-20 08:58:21 -0700420 int msg_flags;
Jens Axboebcda7ba2020-02-23 16:42:51 -0700421 int bgid;
Jens Axboefddafac2020-01-04 20:19:44 -0700422 size_t len;
Jens Axboebcda7ba2020-02-23 16:42:51 -0700423 struct io_buffer *kbuf;
Jens Axboee47293f2019-12-20 08:58:21 -0700424};
425
Jens Axboe15b71ab2019-12-11 11:20:36 -0700426struct io_open {
427 struct file *file;
428 int dfd;
Jens Axboe15b71ab2019-12-11 11:20:36 -0700429 struct filename *filename;
Jens Axboec12cedf2020-01-08 17:41:21 -0700430 struct open_how how;
Jens Axboe4022e7a2020-03-19 19:23:18 -0600431 unsigned long nofile;
Jens Axboe15b71ab2019-12-11 11:20:36 -0700432};
433
Jens Axboe05f3fb32019-12-09 11:22:50 -0700434struct io_files_update {
435 struct file *file;
436 u64 arg;
437 u32 nr_args;
438 u32 offset;
439};
440
Jens Axboe4840e412019-12-25 22:03:45 -0700441struct io_fadvise {
442 struct file *file;
443 u64 offset;
444 u32 len;
445 u32 advice;
446};
447
Jens Axboec1ca7572019-12-25 22:18:28 -0700448struct io_madvise {
449 struct file *file;
450 u64 addr;
451 u32 len;
452 u32 advice;
453};
454
Jens Axboe3e4827b2020-01-08 15:18:09 -0700455struct io_epoll {
456 struct file *file;
457 int epfd;
458 int op;
459 int fd;
460 struct epoll_event event;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700461};
462
Pavel Begunkov7d67af22020-02-24 11:32:45 +0300463struct io_splice {
464 struct file *file_out;
465 struct file *file_in;
466 loff_t off_out;
467 loff_t off_in;
468 u64 len;
469 unsigned int flags;
470};
471
Jens Axboeddf0322d2020-02-23 16:41:33 -0700472struct io_provide_buf {
473 struct file *file;
474 __u64 addr;
475 __s32 len;
476 __u32 bgid;
477 __u16 nbufs;
478 __u16 bid;
479};
480
Bijan Mottahedeh1d9e1282020-05-22 21:31:16 -0700481struct io_statx {
482 struct file *file;
483 int dfd;
484 unsigned int mask;
485 unsigned int flags;
Bijan Mottahedehe62753e2020-05-22 21:31:18 -0700486 const char __user *filename;
Bijan Mottahedeh1d9e1282020-05-22 21:31:16 -0700487 struct statx __user *buffer;
488};
489
Jens Axboef499a022019-12-02 16:28:46 -0700490struct io_async_connect {
491 struct sockaddr_storage address;
492};
493
Jens Axboe03b12302019-12-02 18:50:25 -0700494struct io_async_msghdr {
495 struct iovec fast_iov[UIO_FASTIOV];
496 struct iovec *iov;
497 struct sockaddr __user *uaddr;
498 struct msghdr msg;
Jens Axboeb5379162020-02-09 11:29:15 -0700499 struct sockaddr_storage addr;
Jens Axboe03b12302019-12-02 18:50:25 -0700500};
501
Jens Axboef67676d2019-12-02 11:03:47 -0700502struct io_async_rw {
503 struct iovec fast_iov[UIO_FASTIOV];
504 struct iovec *iov;
505 ssize_t nr_segs;
506 ssize_t size;
Jens Axboebcf5a062020-05-22 09:24:42 -0600507 struct wait_page_queue wpq;
508 struct callback_head task_work;
Jens Axboef67676d2019-12-02 11:03:47 -0700509};
510
Jens Axboe1a6b74f2019-12-02 10:33:15 -0700511struct io_async_ctx {
Jens Axboef67676d2019-12-02 11:03:47 -0700512 union {
513 struct io_async_rw rw;
Jens Axboe03b12302019-12-02 18:50:25 -0700514 struct io_async_msghdr msg;
Jens Axboef499a022019-12-02 16:28:46 -0700515 struct io_async_connect connect;
Jens Axboe2d283902019-12-04 11:08:05 -0700516 struct io_timeout_data timeout;
Jens Axboef67676d2019-12-02 11:03:47 -0700517 };
Jens Axboe1a6b74f2019-12-02 10:33:15 -0700518};
519
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300520enum {
521 REQ_F_FIXED_FILE_BIT = IOSQE_FIXED_FILE_BIT,
522 REQ_F_IO_DRAIN_BIT = IOSQE_IO_DRAIN_BIT,
523 REQ_F_LINK_BIT = IOSQE_IO_LINK_BIT,
524 REQ_F_HARDLINK_BIT = IOSQE_IO_HARDLINK_BIT,
525 REQ_F_FORCE_ASYNC_BIT = IOSQE_ASYNC_BIT,
Jens Axboebcda7ba2020-02-23 16:42:51 -0700526 REQ_F_BUFFER_SELECT_BIT = IOSQE_BUFFER_SELECT_BIT,
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300527
Pavel Begunkovdea3b492020-04-12 02:05:04 +0300528 REQ_F_LINK_HEAD_BIT,
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300529 REQ_F_FAIL_LINK_BIT,
530 REQ_F_INFLIGHT_BIT,
531 REQ_F_CUR_POS_BIT,
532 REQ_F_NOWAIT_BIT,
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300533 REQ_F_LINK_TIMEOUT_BIT,
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300534 REQ_F_ISREG_BIT,
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300535 REQ_F_COMP_LOCKED_BIT,
Pavel Begunkov99bc4c32020-02-07 22:04:45 +0300536 REQ_F_NEED_CLEANUP_BIT,
Jens Axboe2ca10252020-02-13 17:17:35 -0700537 REQ_F_OVERFLOW_BIT,
Jens Axboed7718a92020-02-14 22:23:12 -0700538 REQ_F_POLLED_BIT,
Jens Axboebcda7ba2020-02-23 16:42:51 -0700539 REQ_F_BUFFER_SELECTED_BIT,
Jens Axboe5b0bbee2020-04-27 10:41:22 -0600540 REQ_F_NO_FILE_TABLE_BIT,
Pavel Begunkovd4c81f32020-06-08 21:08:19 +0300541 REQ_F_QUEUE_TIMEOUT_BIT,
Xiaoguang Wang7cdaf582020-06-10 19:41:19 +0800542 REQ_F_WORK_INITIALIZED_BIT,
Pavel Begunkov4dd28242020-06-15 10:33:13 +0300543 REQ_F_TASK_PINNED_BIT,
Jens Axboe84557872020-03-03 15:28:17 -0700544
545 /* not a real bit, just to check we're not overflowing the space */
546 __REQ_F_LAST_BIT,
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300547};
548
549enum {
550 /* ctx owns file */
551 REQ_F_FIXED_FILE = BIT(REQ_F_FIXED_FILE_BIT),
552 /* drain existing IO first */
553 REQ_F_IO_DRAIN = BIT(REQ_F_IO_DRAIN_BIT),
554 /* linked sqes */
555 REQ_F_LINK = BIT(REQ_F_LINK_BIT),
556 /* doesn't sever on completion < 0 */
557 REQ_F_HARDLINK = BIT(REQ_F_HARDLINK_BIT),
558 /* IOSQE_ASYNC */
559 REQ_F_FORCE_ASYNC = BIT(REQ_F_FORCE_ASYNC_BIT),
Jens Axboebcda7ba2020-02-23 16:42:51 -0700560 /* IOSQE_BUFFER_SELECT */
561 REQ_F_BUFFER_SELECT = BIT(REQ_F_BUFFER_SELECT_BIT),
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300562
Pavel Begunkovdea3b492020-04-12 02:05:04 +0300563 /* head of a link */
564 REQ_F_LINK_HEAD = BIT(REQ_F_LINK_HEAD_BIT),
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300565 /* fail rest of links */
566 REQ_F_FAIL_LINK = BIT(REQ_F_FAIL_LINK_BIT),
567 /* on inflight list */
568 REQ_F_INFLIGHT = BIT(REQ_F_INFLIGHT_BIT),
569 /* read/write uses file position */
570 REQ_F_CUR_POS = BIT(REQ_F_CUR_POS_BIT),
571 /* must not punt to workers */
572 REQ_F_NOWAIT = BIT(REQ_F_NOWAIT_BIT),
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300573 /* has linked timeout */
574 REQ_F_LINK_TIMEOUT = BIT(REQ_F_LINK_TIMEOUT_BIT),
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300575 /* regular file */
576 REQ_F_ISREG = BIT(REQ_F_ISREG_BIT),
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300577 /* completion under lock */
578 REQ_F_COMP_LOCKED = BIT(REQ_F_COMP_LOCKED_BIT),
Pavel Begunkov99bc4c32020-02-07 22:04:45 +0300579 /* needs cleanup */
580 REQ_F_NEED_CLEANUP = BIT(REQ_F_NEED_CLEANUP_BIT),
Jens Axboe2ca10252020-02-13 17:17:35 -0700581 /* in overflow list */
582 REQ_F_OVERFLOW = BIT(REQ_F_OVERFLOW_BIT),
Jens Axboed7718a92020-02-14 22:23:12 -0700583 /* already went through poll handler */
584 REQ_F_POLLED = BIT(REQ_F_POLLED_BIT),
Jens Axboebcda7ba2020-02-23 16:42:51 -0700585 /* buffer already selected */
586 REQ_F_BUFFER_SELECTED = BIT(REQ_F_BUFFER_SELECTED_BIT),
Jens Axboe5b0bbee2020-04-27 10:41:22 -0600587 /* doesn't need file table for this request */
588 REQ_F_NO_FILE_TABLE = BIT(REQ_F_NO_FILE_TABLE_BIT),
Pavel Begunkovd4c81f32020-06-08 21:08:19 +0300589 /* needs to queue linked timeout */
590 REQ_F_QUEUE_TIMEOUT = BIT(REQ_F_QUEUE_TIMEOUT_BIT),
Xiaoguang Wang7cdaf582020-06-10 19:41:19 +0800591 /* io_wq_work is initialized */
592 REQ_F_WORK_INITIALIZED = BIT(REQ_F_WORK_INITIALIZED_BIT),
Pavel Begunkov4dd28242020-06-15 10:33:13 +0300593 /* req->task is refcounted */
594 REQ_F_TASK_PINNED = BIT(REQ_F_TASK_PINNED_BIT),
Jens Axboed7718a92020-02-14 22:23:12 -0700595};
596
597struct async_poll {
598 struct io_poll_iocb poll;
599 struct io_wq_work work;
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300600};
601
Jens Axboe09bb8392019-03-13 12:39:28 -0600602/*
603 * NOTE! Each of the iocb union members has the file pointer
604 * as the first entry in their struct definition. So you can
605 * access the file pointer through any of the sub-structs,
606 * or directly as just 'ki_filp' in this struct.
607 */
Jens Axboe2b188cc2019-01-07 10:46:33 -0700608struct io_kiocb {
Jens Axboe221c5eb2019-01-17 09:41:58 -0700609 union {
Jens Axboe09bb8392019-03-13 12:39:28 -0600610 struct file *file;
Jens Axboe9adbd452019-12-20 08:45:55 -0700611 struct io_rw rw;
Jens Axboe221c5eb2019-01-17 09:41:58 -0700612 struct io_poll_iocb poll;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -0700613 struct io_accept accept;
614 struct io_sync sync;
Jens Axboefbf23842019-12-17 18:45:56 -0700615 struct io_cancel cancel;
Jens Axboeb29472e2019-12-17 18:50:29 -0700616 struct io_timeout timeout;
Jens Axboe3fbb51c2019-12-20 08:51:52 -0700617 struct io_connect connect;
Jens Axboee47293f2019-12-20 08:58:21 -0700618 struct io_sr_msg sr_msg;
Jens Axboe15b71ab2019-12-11 11:20:36 -0700619 struct io_open open;
Jens Axboeb5dba592019-12-11 14:02:38 -0700620 struct io_close close;
Jens Axboe05f3fb32019-12-09 11:22:50 -0700621 struct io_files_update files_update;
Jens Axboe4840e412019-12-25 22:03:45 -0700622 struct io_fadvise fadvise;
Jens Axboec1ca7572019-12-25 22:18:28 -0700623 struct io_madvise madvise;
Jens Axboe3e4827b2020-01-08 15:18:09 -0700624 struct io_epoll epoll;
Pavel Begunkov7d67af22020-02-24 11:32:45 +0300625 struct io_splice splice;
Jens Axboeddf0322d2020-02-23 16:41:33 -0700626 struct io_provide_buf pbuf;
Bijan Mottahedeh1d9e1282020-05-22 21:31:16 -0700627 struct io_statx statx;
Jens Axboe221c5eb2019-01-17 09:41:58 -0700628 };
Jens Axboe2b188cc2019-01-07 10:46:33 -0700629
Jens Axboe1a6b74f2019-12-02 10:33:15 -0700630 struct io_async_ctx *io;
Pavel Begunkovc398ecb2020-04-09 08:17:59 +0300631 int cflags;
Jens Axboed625c6e2019-12-17 19:53:05 -0700632 u8 opcode;
Xiaoguang Wang65a65432020-06-11 23:39:36 +0800633 /* polled IO has completed */
634 u8 iopoll_completed;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700635
Bijan Mottahedeh4f4eeba2020-05-19 14:52:49 -0700636 u16 buf_index;
637
Jens Axboe2b188cc2019-01-07 10:46:33 -0700638 struct io_ring_ctx *ctx;
Jens Axboed7718a92020-02-14 22:23:12 -0700639 struct list_head list;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700640 unsigned int flags;
Jens Axboec16361c2019-01-17 08:39:48 -0700641 refcount_t refs;
Jens Axboe3537b6a2020-04-03 11:19:06 -0600642 struct task_struct *task;
643 unsigned long fsize;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700644 u64 user_data;
Jens Axboe9e645e112019-05-10 16:07:28 -0600645 u32 result;
Jens Axboede0617e2019-04-06 21:51:27 -0600646 u32 sequence;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700647
Jens Axboed7718a92020-02-14 22:23:12 -0700648 struct list_head link_list;
649
Jens Axboefcb323c2019-10-24 12:39:47 -0600650 struct list_head inflight_entry;
651
Xiaoguang Wang05589552020-03-31 14:05:18 +0800652 struct percpu_ref *fixed_file_refs;
653
Jens Axboeb41e9852020-02-17 09:52:41 -0700654 union {
655 /*
656 * Only commands that never go async can use the below fields,
Jens Axboed7718a92020-02-14 22:23:12 -0700657 * obviously. Right now only IORING_OP_POLL_ADD uses them, and
658 * async armed poll handlers for regular commands. The latter
659 * restore the work, if needed.
Jens Axboeb41e9852020-02-17 09:52:41 -0700660 */
661 struct {
Jens Axboed7718a92020-02-14 22:23:12 -0700662 struct hlist_node hash_node;
663 struct async_poll *apoll;
Jens Axboeb41e9852020-02-17 09:52:41 -0700664 };
665 struct io_wq_work work;
666 };
Pavel Begunkov8ef77762020-06-27 14:04:59 +0300667 struct callback_head task_work;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700668};
669
Jens Axboedef596e2019-01-09 08:59:42 -0700670#define IO_IOPOLL_BATCH 8
Jens Axboe2b188cc2019-01-07 10:46:33 -0700671
Jens Axboe013538b2020-06-22 09:29:15 -0600672struct io_comp_state {
673 unsigned int nr;
674 struct list_head list;
675 struct io_ring_ctx *ctx;
676};
677
Jens Axboe9a56a232019-01-09 09:06:50 -0700678struct io_submit_state {
679 struct blk_plug plug;
680
681 /*
Jens Axboe2579f912019-01-09 09:10:43 -0700682 * io_kiocb alloc cache
683 */
684 void *reqs[IO_IOPOLL_BATCH];
Pavel Begunkov6c8a3132020-02-01 03:58:00 +0300685 unsigned int free_reqs;
Jens Axboe2579f912019-01-09 09:10:43 -0700686
687 /*
Jens Axboe013538b2020-06-22 09:29:15 -0600688 * Batch completion logic
689 */
690 struct io_comp_state comp;
691
692 /*
Jens Axboe9a56a232019-01-09 09:06:50 -0700693 * File reference cache
694 */
695 struct file *file;
696 unsigned int fd;
697 unsigned int has_refs;
698 unsigned int used_refs;
699 unsigned int ios_left;
700};
701
Jens Axboed3656342019-12-18 09:50:26 -0700702struct io_op_def {
703 /* needs req->io allocated for deferral/async */
704 unsigned async_ctx : 1;
705 /* needs current->mm setup, does mm access */
706 unsigned needs_mm : 1;
707 /* needs req->file assigned */
708 unsigned needs_file : 1;
Jens Axboefd2206e2020-06-02 16:40:47 -0600709 /* don't fail if file grab fails */
710 unsigned needs_file_no_error : 1;
Jens Axboed3656342019-12-18 09:50:26 -0700711 /* hash wq insertion if file is a regular file */
712 unsigned hash_reg_file : 1;
713 /* unbound wq insertion if file is a non-regular file */
714 unsigned unbound_nonreg_file : 1;
Jens Axboe66f4af92020-01-16 15:36:52 -0700715 /* opcode is not supported by this kernel */
716 unsigned not_supported : 1;
Jens Axboef86cd202020-01-29 13:46:44 -0700717 /* needs file table */
718 unsigned file_table : 1;
Jens Axboeff002b32020-02-07 16:05:21 -0700719 /* needs ->fs */
720 unsigned needs_fs : 1;
Jens Axboe8a727582020-02-20 09:59:44 -0700721 /* set if opcode supports polled "wait" */
722 unsigned pollin : 1;
723 unsigned pollout : 1;
Jens Axboebcda7ba2020-02-23 16:42:51 -0700724 /* op supports buffer selection */
725 unsigned buffer_select : 1;
Jens Axboed3656342019-12-18 09:50:26 -0700726};
727
728static const struct io_op_def io_op_defs[] = {
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300729 [IORING_OP_NOP] = {},
730 [IORING_OP_READV] = {
Jens Axboed3656342019-12-18 09:50:26 -0700731 .async_ctx = 1,
732 .needs_mm = 1,
733 .needs_file = 1,
734 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -0700735 .pollin = 1,
Jens Axboe4d954c22020-02-27 07:31:19 -0700736 .buffer_select = 1,
Jens Axboed3656342019-12-18 09:50:26 -0700737 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300738 [IORING_OP_WRITEV] = {
Jens Axboed3656342019-12-18 09:50:26 -0700739 .async_ctx = 1,
740 .needs_mm = 1,
741 .needs_file = 1,
742 .hash_reg_file = 1,
743 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -0700744 .pollout = 1,
Jens Axboed3656342019-12-18 09:50:26 -0700745 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300746 [IORING_OP_FSYNC] = {
Jens Axboed3656342019-12-18 09:50:26 -0700747 .needs_file = 1,
748 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300749 [IORING_OP_READ_FIXED] = {
Jens Axboed3656342019-12-18 09:50:26 -0700750 .needs_file = 1,
751 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -0700752 .pollin = 1,
Jens Axboed3656342019-12-18 09:50:26 -0700753 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300754 [IORING_OP_WRITE_FIXED] = {
Jens Axboed3656342019-12-18 09:50:26 -0700755 .needs_file = 1,
756 .hash_reg_file = 1,
757 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -0700758 .pollout = 1,
Jens Axboed3656342019-12-18 09:50:26 -0700759 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300760 [IORING_OP_POLL_ADD] = {
Jens Axboed3656342019-12-18 09:50:26 -0700761 .needs_file = 1,
762 .unbound_nonreg_file = 1,
763 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300764 [IORING_OP_POLL_REMOVE] = {},
765 [IORING_OP_SYNC_FILE_RANGE] = {
Jens Axboed3656342019-12-18 09:50:26 -0700766 .needs_file = 1,
767 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300768 [IORING_OP_SENDMSG] = {
Jens Axboed3656342019-12-18 09:50:26 -0700769 .async_ctx = 1,
770 .needs_mm = 1,
771 .needs_file = 1,
772 .unbound_nonreg_file = 1,
Jens Axboeff002b32020-02-07 16:05:21 -0700773 .needs_fs = 1,
Jens Axboe8a727582020-02-20 09:59:44 -0700774 .pollout = 1,
Jens Axboed3656342019-12-18 09:50:26 -0700775 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300776 [IORING_OP_RECVMSG] = {
Jens Axboed3656342019-12-18 09:50:26 -0700777 .async_ctx = 1,
778 .needs_mm = 1,
779 .needs_file = 1,
780 .unbound_nonreg_file = 1,
Jens Axboeff002b32020-02-07 16:05:21 -0700781 .needs_fs = 1,
Jens Axboe8a727582020-02-20 09:59:44 -0700782 .pollin = 1,
Jens Axboe52de1fe2020-02-27 10:15:42 -0700783 .buffer_select = 1,
Jens Axboed3656342019-12-18 09:50:26 -0700784 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300785 [IORING_OP_TIMEOUT] = {
Jens Axboed3656342019-12-18 09:50:26 -0700786 .async_ctx = 1,
787 .needs_mm = 1,
788 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300789 [IORING_OP_TIMEOUT_REMOVE] = {},
790 [IORING_OP_ACCEPT] = {
Jens Axboed3656342019-12-18 09:50:26 -0700791 .needs_mm = 1,
792 .needs_file = 1,
793 .unbound_nonreg_file = 1,
Jens Axboef86cd202020-01-29 13:46:44 -0700794 .file_table = 1,
Jens Axboe8a727582020-02-20 09:59:44 -0700795 .pollin = 1,
Jens Axboed3656342019-12-18 09:50:26 -0700796 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300797 [IORING_OP_ASYNC_CANCEL] = {},
798 [IORING_OP_LINK_TIMEOUT] = {
Jens Axboed3656342019-12-18 09:50:26 -0700799 .async_ctx = 1,
800 .needs_mm = 1,
801 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300802 [IORING_OP_CONNECT] = {
Jens Axboed3656342019-12-18 09:50:26 -0700803 .async_ctx = 1,
804 .needs_mm = 1,
805 .needs_file = 1,
806 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -0700807 .pollout = 1,
Jens Axboed3656342019-12-18 09:50:26 -0700808 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300809 [IORING_OP_FALLOCATE] = {
Jens Axboed3656342019-12-18 09:50:26 -0700810 .needs_file = 1,
811 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300812 [IORING_OP_OPENAT] = {
Jens Axboef86cd202020-01-29 13:46:44 -0700813 .file_table = 1,
Jens Axboeff002b32020-02-07 16:05:21 -0700814 .needs_fs = 1,
Jens Axboed3656342019-12-18 09:50:26 -0700815 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300816 [IORING_OP_CLOSE] = {
Jens Axboefd2206e2020-06-02 16:40:47 -0600817 .needs_file = 1,
818 .needs_file_no_error = 1,
Jens Axboef86cd202020-01-29 13:46:44 -0700819 .file_table = 1,
Jens Axboed3656342019-12-18 09:50:26 -0700820 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300821 [IORING_OP_FILES_UPDATE] = {
Jens Axboed3656342019-12-18 09:50:26 -0700822 .needs_mm = 1,
Jens Axboef86cd202020-01-29 13:46:44 -0700823 .file_table = 1,
Jens Axboed3656342019-12-18 09:50:26 -0700824 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300825 [IORING_OP_STATX] = {
Jens Axboed3656342019-12-18 09:50:26 -0700826 .needs_mm = 1,
Jens Axboeff002b32020-02-07 16:05:21 -0700827 .needs_fs = 1,
Jens Axboe5b0bbee2020-04-27 10:41:22 -0600828 .file_table = 1,
Jens Axboed3656342019-12-18 09:50:26 -0700829 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300830 [IORING_OP_READ] = {
Jens Axboe3a6820f2019-12-22 15:19:35 -0700831 .needs_mm = 1,
832 .needs_file = 1,
833 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -0700834 .pollin = 1,
Jens Axboebcda7ba2020-02-23 16:42:51 -0700835 .buffer_select = 1,
Jens Axboe3a6820f2019-12-22 15:19:35 -0700836 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300837 [IORING_OP_WRITE] = {
Jens Axboe3a6820f2019-12-22 15:19:35 -0700838 .needs_mm = 1,
839 .needs_file = 1,
840 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -0700841 .pollout = 1,
Jens Axboe3a6820f2019-12-22 15:19:35 -0700842 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300843 [IORING_OP_FADVISE] = {
Jens Axboe4840e412019-12-25 22:03:45 -0700844 .needs_file = 1,
845 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300846 [IORING_OP_MADVISE] = {
Jens Axboec1ca7572019-12-25 22:18:28 -0700847 .needs_mm = 1,
848 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300849 [IORING_OP_SEND] = {
Jens Axboefddafac2020-01-04 20:19:44 -0700850 .needs_mm = 1,
851 .needs_file = 1,
852 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -0700853 .pollout = 1,
Jens Axboefddafac2020-01-04 20:19:44 -0700854 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300855 [IORING_OP_RECV] = {
Jens Axboefddafac2020-01-04 20:19:44 -0700856 .needs_mm = 1,
857 .needs_file = 1,
858 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -0700859 .pollin = 1,
Jens Axboebcda7ba2020-02-23 16:42:51 -0700860 .buffer_select = 1,
Jens Axboefddafac2020-01-04 20:19:44 -0700861 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300862 [IORING_OP_OPENAT2] = {
Jens Axboef86cd202020-01-29 13:46:44 -0700863 .file_table = 1,
Jens Axboeff002b32020-02-07 16:05:21 -0700864 .needs_fs = 1,
Jens Axboecebdb982020-01-08 17:59:24 -0700865 },
Jens Axboe3e4827b2020-01-08 15:18:09 -0700866 [IORING_OP_EPOLL_CTL] = {
867 .unbound_nonreg_file = 1,
868 .file_table = 1,
869 },
Pavel Begunkov7d67af22020-02-24 11:32:45 +0300870 [IORING_OP_SPLICE] = {
871 .needs_file = 1,
872 .hash_reg_file = 1,
873 .unbound_nonreg_file = 1,
Jens Axboeddf0322d2020-02-23 16:41:33 -0700874 },
875 [IORING_OP_PROVIDE_BUFFERS] = {},
Jens Axboe067524e2020-03-02 16:32:28 -0700876 [IORING_OP_REMOVE_BUFFERS] = {},
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +0300877 [IORING_OP_TEE] = {
878 .needs_file = 1,
879 .hash_reg_file = 1,
880 .unbound_nonreg_file = 1,
881 },
Jens Axboed3656342019-12-18 09:50:26 -0700882};
883
Bijan Mottahedeh2e0464d2020-06-16 16:36:10 -0700884enum io_mem_account {
885 ACCT_LOCKED,
886 ACCT_PINNED,
887};
888
Pavel Begunkovf3a6fa22020-06-28 12:52:38 +0300889static bool io_rw_reissue(struct io_kiocb *req, long res);
Jens Axboe78e19bb2019-11-06 15:21:34 -0700890static void io_cqring_fill_event(struct io_kiocb *req, long res);
Jackie Liuec9c02a2019-11-08 23:50:36 +0800891static void io_put_req(struct io_kiocb *req);
Jens Axboec40f6372020-06-25 15:39:59 -0600892static void io_double_put_req(struct io_kiocb *req);
Jens Axboe978db572019-11-14 22:39:04 -0700893static void __io_double_put_req(struct io_kiocb *req);
Jens Axboe94ae5e72019-11-14 19:39:52 -0700894static struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req);
895static void io_queue_linked_timeout(struct io_kiocb *req);
Jens Axboe05f3fb32019-12-09 11:22:50 -0700896static int __io_sqe_files_update(struct io_ring_ctx *ctx,
897 struct io_uring_files_update *ip,
898 unsigned nr_args);
Jens Axboef86cd202020-01-29 13:46:44 -0700899static int io_grab_files(struct io_kiocb *req);
Jens Axboe2237d762020-06-26 13:44:16 -0600900static void io_complete_rw_common(struct kiocb *kiocb, long res,
901 struct io_comp_state *cs);
Pavel Begunkov99bc4c32020-02-07 22:04:45 +0300902static void io_cleanup_req(struct io_kiocb *req);
Jens Axboeb41e9852020-02-17 09:52:41 -0700903static int io_file_get(struct io_submit_state *state, struct io_kiocb *req,
904 int fd, struct file **out_file, bool fixed);
905static void __io_queue_sqe(struct io_kiocb *req,
Jens Axboef13fad72020-06-22 09:34:30 -0600906 const struct io_uring_sqe *sqe,
907 struct io_comp_state *cs);
Jens Axboede0617e2019-04-06 21:51:27 -0600908
Jens Axboeb63534c2020-06-04 11:28:00 -0600909static ssize_t io_import_iovec(int rw, struct io_kiocb *req,
910 struct iovec **iovec, struct iov_iter *iter,
911 bool needs_lock);
912static int io_setup_async_rw(struct io_kiocb *req, ssize_t io_size,
913 struct iovec *iovec, struct iovec *fast_iov,
914 struct iov_iter *iter);
915
Jens Axboe2b188cc2019-01-07 10:46:33 -0700916static struct kmem_cache *req_cachep;
917
918static const struct file_operations io_uring_fops;
919
920struct sock *io_uring_get_socket(struct file *file)
921{
922#if defined(CONFIG_UNIX)
923 if (file->f_op == &io_uring_fops) {
924 struct io_ring_ctx *ctx = file->private_data;
925
926 return ctx->ring_sock->sk;
927 }
928#endif
929 return NULL;
930}
931EXPORT_SYMBOL(io_uring_get_socket);
932
Pavel Begunkov4dd28242020-06-15 10:33:13 +0300933static void io_get_req_task(struct io_kiocb *req)
934{
935 if (req->flags & REQ_F_TASK_PINNED)
936 return;
937 get_task_struct(req->task);
938 req->flags |= REQ_F_TASK_PINNED;
939}
940
941/* not idempotent -- it doesn't clear REQ_F_TASK_PINNED */
942static void __io_put_req_task(struct io_kiocb *req)
943{
944 if (req->flags & REQ_F_TASK_PINNED)
945 put_task_struct(req->task);
946}
947
Jens Axboec40f6372020-06-25 15:39:59 -0600948static void io_sq_thread_drop_mm(struct io_ring_ctx *ctx)
949{
950 struct mm_struct *mm = current->mm;
951
952 if (mm) {
953 kthread_unuse_mm(mm);
954 mmput(mm);
955 }
956}
957
958static int __io_sq_thread_acquire_mm(struct io_ring_ctx *ctx)
959{
960 if (!current->mm) {
Pavel Begunkov8eb06d72020-06-30 15:20:39 +0300961 if (unlikely(!ctx->sqo_mm || !mmget_not_zero(ctx->sqo_mm)))
Jens Axboec40f6372020-06-25 15:39:59 -0600962 return -EFAULT;
963 kthread_use_mm(ctx->sqo_mm);
964 }
965
966 return 0;
967}
968
969static int io_sq_thread_acquire_mm(struct io_ring_ctx *ctx,
970 struct io_kiocb *req)
971{
972 if (!io_op_defs[req->opcode].needs_mm)
973 return 0;
974 return __io_sq_thread_acquire_mm(ctx);
975}
976
977static inline void req_set_fail_links(struct io_kiocb *req)
978{
979 if ((req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) == REQ_F_LINK)
980 req->flags |= REQ_F_FAIL_LINK;
981}
982
Jens Axboe4a38aed22020-05-14 17:21:15 -0600983static void io_file_put_work(struct work_struct *work);
984
Xiaoguang Wang7cdaf582020-06-10 19:41:19 +0800985/*
986 * Note: must call io_req_init_async() for the first time you
987 * touch any members of io_wq_work.
988 */
989static inline void io_req_init_async(struct io_kiocb *req)
990{
991 if (req->flags & REQ_F_WORK_INITIALIZED)
992 return;
993
994 memset(&req->work, 0, sizeof(req->work));
995 req->flags |= REQ_F_WORK_INITIALIZED;
996}
997
Pavel Begunkov0cdaf762020-05-17 14:13:40 +0300998static inline bool io_async_submit(struct io_ring_ctx *ctx)
999{
1000 return ctx->flags & IORING_SETUP_SQPOLL;
1001}
1002
Jens Axboe2b188cc2019-01-07 10:46:33 -07001003static void io_ring_ctx_ref_free(struct percpu_ref *ref)
1004{
1005 struct io_ring_ctx *ctx = container_of(ref, struct io_ring_ctx, refs);
1006
Jens Axboe0f158b42020-05-14 17:18:39 -06001007 complete(&ctx->ref_comp);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001008}
1009
Pavel Begunkov8eb7e2d2020-06-29 13:13:02 +03001010static inline bool io_is_timeout_noseq(struct io_kiocb *req)
1011{
1012 return !req->timeout.off;
1013}
1014
Jens Axboe2b188cc2019-01-07 10:46:33 -07001015static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
1016{
1017 struct io_ring_ctx *ctx;
Jens Axboe78076bb2019-12-04 19:56:40 -07001018 int hash_bits;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001019
1020 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
1021 if (!ctx)
1022 return NULL;
1023
Jens Axboe0ddf92e2019-11-08 08:52:53 -07001024 ctx->fallback_req = kmem_cache_alloc(req_cachep, GFP_KERNEL);
1025 if (!ctx->fallback_req)
1026 goto err;
1027
Jens Axboe78076bb2019-12-04 19:56:40 -07001028 /*
1029 * Use 5 bits less than the max cq entries, that should give us around
1030 * 32 entries per hash list if totally full and uniformly spread.
1031 */
1032 hash_bits = ilog2(p->cq_entries);
1033 hash_bits -= 5;
1034 if (hash_bits <= 0)
1035 hash_bits = 1;
1036 ctx->cancel_hash_bits = hash_bits;
1037 ctx->cancel_hash = kmalloc((1U << hash_bits) * sizeof(struct hlist_head),
1038 GFP_KERNEL);
1039 if (!ctx->cancel_hash)
1040 goto err;
1041 __hash_init(ctx->cancel_hash, 1U << hash_bits);
1042
Roman Gushchin21482892019-05-07 10:01:48 -07001043 if (percpu_ref_init(&ctx->refs, io_ring_ctx_ref_free,
Jens Axboe206aefd2019-11-07 18:27:42 -07001044 PERCPU_REF_ALLOW_REINIT, GFP_KERNEL))
1045 goto err;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001046
1047 ctx->flags = p->flags;
Jens Axboe583863e2020-05-17 09:20:00 -06001048 init_waitqueue_head(&ctx->sqo_wait);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001049 init_waitqueue_head(&ctx->cq_wait);
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001050 INIT_LIST_HEAD(&ctx->cq_overflow_list);
Jens Axboe0f158b42020-05-14 17:18:39 -06001051 init_completion(&ctx->ref_comp);
1052 init_completion(&ctx->sq_thread_comp);
Jens Axboe5a2e7452020-02-23 16:23:11 -07001053 idr_init(&ctx->io_buffer_idr);
Jens Axboe071698e2020-01-28 10:04:42 -07001054 idr_init(&ctx->personality_idr);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001055 mutex_init(&ctx->uring_lock);
1056 init_waitqueue_head(&ctx->wait);
1057 spin_lock_init(&ctx->completion_lock);
Jens Axboedef596e2019-01-09 08:59:42 -07001058 INIT_LIST_HEAD(&ctx->poll_list);
Jens Axboede0617e2019-04-06 21:51:27 -06001059 INIT_LIST_HEAD(&ctx->defer_list);
Jens Axboe5262f562019-09-17 12:26:57 -06001060 INIT_LIST_HEAD(&ctx->timeout_list);
Jens Axboefcb323c2019-10-24 12:39:47 -06001061 init_waitqueue_head(&ctx->inflight_wait);
1062 spin_lock_init(&ctx->inflight_lock);
1063 INIT_LIST_HEAD(&ctx->inflight_list);
Jens Axboe4a38aed22020-05-14 17:21:15 -06001064 INIT_DELAYED_WORK(&ctx->file_put_work, io_file_put_work);
1065 init_llist_head(&ctx->file_put_llist);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001066 return ctx;
Jens Axboe206aefd2019-11-07 18:27:42 -07001067err:
Jens Axboe0ddf92e2019-11-08 08:52:53 -07001068 if (ctx->fallback_req)
1069 kmem_cache_free(req_cachep, ctx->fallback_req);
Jens Axboe78076bb2019-12-04 19:56:40 -07001070 kfree(ctx->cancel_hash);
Jens Axboe206aefd2019-11-07 18:27:42 -07001071 kfree(ctx);
1072 return NULL;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001073}
1074
Bob Liu9d858b22019-11-13 18:06:25 +08001075static inline bool __req_need_defer(struct io_kiocb *req)
Jens Axboede0617e2019-04-06 21:51:27 -06001076{
Jackie Liua197f662019-11-08 08:09:12 -07001077 struct io_ring_ctx *ctx = req->ctx;
1078
Pavel Begunkov31af27c2020-04-15 00:39:50 +03001079 return req->sequence != ctx->cached_cq_tail
1080 + atomic_read(&ctx->cached_cq_overflow);
Jens Axboede0617e2019-04-06 21:51:27 -06001081}
1082
Bob Liu9d858b22019-11-13 18:06:25 +08001083static inline bool req_need_defer(struct io_kiocb *req)
Jens Axboe7adf4ea2019-10-10 21:42:58 -06001084{
Pavel Begunkov87987892020-01-18 01:22:30 +03001085 if (unlikely(req->flags & REQ_F_IO_DRAIN))
Bob Liu9d858b22019-11-13 18:06:25 +08001086 return __req_need_defer(req);
Jens Axboe7adf4ea2019-10-10 21:42:58 -06001087
Bob Liu9d858b22019-11-13 18:06:25 +08001088 return false;
Jens Axboe7adf4ea2019-10-10 21:42:58 -06001089}
1090
Jens Axboede0617e2019-04-06 21:51:27 -06001091static void __io_commit_cqring(struct io_ring_ctx *ctx)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001092{
Hristo Venev75b28af2019-08-26 17:23:46 +00001093 struct io_rings *rings = ctx->rings;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001094
Pavel Begunkov07910152020-01-17 03:52:46 +03001095 /* order cqe stores with ring update */
1096 smp_store_release(&rings->cq.tail, ctx->cached_cq_tail);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001097
Pavel Begunkov07910152020-01-17 03:52:46 +03001098 if (wq_has_sleeper(&ctx->cq_wait)) {
1099 wake_up_interruptible(&ctx->cq_wait);
1100 kill_fasync(&ctx->cq_fasync, SIGIO, POLL_IN);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001101 }
1102}
1103
Pavel Begunkovcbdcb432020-06-29 19:18:43 +03001104static void io_req_work_grab_env(struct io_kiocb *req)
Jens Axboe18d9be12019-09-10 09:13:05 -06001105{
Pavel Begunkov351fd532020-06-29 19:18:40 +03001106 const struct io_op_def *def = &io_op_defs[req->opcode];
1107
Pavel Begunkovedcdfcc2020-06-29 19:18:41 +03001108 io_req_init_async(req);
1109
Jens Axboecccf0ee2020-01-27 16:34:48 -07001110 if (!req->work.mm && def->needs_mm) {
1111 mmgrab(current->mm);
1112 req->work.mm = current->mm;
1113 }
1114 if (!req->work.creds)
1115 req->work.creds = get_current_cred();
Jens Axboeff002b32020-02-07 16:05:21 -07001116 if (!req->work.fs && def->needs_fs) {
1117 spin_lock(&current->fs->lock);
1118 if (!current->fs->in_exec) {
1119 req->work.fs = current->fs;
1120 req->work.fs->users++;
1121 } else {
1122 req->work.flags |= IO_WQ_WORK_CANCEL;
1123 }
1124 spin_unlock(&current->fs->lock);
1125 }
Jens Axboecccf0ee2020-01-27 16:34:48 -07001126}
1127
1128static inline void io_req_work_drop_env(struct io_kiocb *req)
1129{
Xiaoguang Wang7cdaf582020-06-10 19:41:19 +08001130 if (!(req->flags & REQ_F_WORK_INITIALIZED))
1131 return;
1132
Jens Axboecccf0ee2020-01-27 16:34:48 -07001133 if (req->work.mm) {
1134 mmdrop(req->work.mm);
1135 req->work.mm = NULL;
1136 }
1137 if (req->work.creds) {
1138 put_cred(req->work.creds);
1139 req->work.creds = NULL;
1140 }
Jens Axboeff002b32020-02-07 16:05:21 -07001141 if (req->work.fs) {
1142 struct fs_struct *fs = req->work.fs;
1143
1144 spin_lock(&req->work.fs->lock);
1145 if (--fs->users)
1146 fs = NULL;
1147 spin_unlock(&req->work.fs->lock);
1148 if (fs)
1149 free_fs_struct(fs);
1150 }
Jens Axboe561fb042019-10-24 07:25:42 -06001151}
1152
Pavel Begunkovcbdcb432020-06-29 19:18:43 +03001153static void io_prep_async_work(struct io_kiocb *req)
Jens Axboe561fb042019-10-24 07:25:42 -06001154{
Jens Axboed3656342019-12-18 09:50:26 -07001155 const struct io_op_def *def = &io_op_defs[req->opcode];
Jens Axboe54a91f32019-09-10 09:15:04 -06001156
Jens Axboed3656342019-12-18 09:50:26 -07001157 if (req->flags & REQ_F_ISREG) {
1158 if (def->hash_reg_file)
Pavel Begunkov8766dd52020-03-14 00:31:04 +03001159 io_wq_hash_work(&req->work, file_inode(req->file));
Jens Axboed3656342019-12-18 09:50:26 -07001160 } else {
1161 if (def->unbound_nonreg_file)
Jens Axboe3529d8c2019-12-19 18:24:38 -07001162 req->work.flags |= IO_WQ_WORK_UNBOUND;
Jens Axboe54a91f32019-09-10 09:15:04 -06001163 }
Jens Axboecccf0ee2020-01-27 16:34:48 -07001164
Pavel Begunkov351fd532020-06-29 19:18:40 +03001165 io_req_work_grab_env(req);
Jens Axboe561fb042019-10-24 07:25:42 -06001166}
1167
Pavel Begunkovcbdcb432020-06-29 19:18:43 +03001168static void io_prep_async_link(struct io_kiocb *req)
1169{
1170 struct io_kiocb *cur;
1171
1172 io_prep_async_work(req);
1173 if (req->flags & REQ_F_LINK_HEAD)
1174 list_for_each_entry(cur, &req->link_list, link_list)
1175 io_prep_async_work(cur);
1176}
1177
1178static void __io_queue_async_work(struct io_kiocb *req)
Jens Axboe561fb042019-10-24 07:25:42 -06001179{
Jackie Liua197f662019-11-08 08:09:12 -07001180 struct io_ring_ctx *ctx = req->ctx;
Pavel Begunkovcbdcb432020-06-29 19:18:43 +03001181 struct io_kiocb *link = io_prep_linked_timeout(req);
Jens Axboe561fb042019-10-24 07:25:42 -06001182
Pavel Begunkov8766dd52020-03-14 00:31:04 +03001183 trace_io_uring_queue_async_work(ctx, io_wq_is_hashed(&req->work), req,
1184 &req->work, req->flags);
1185 io_wq_enqueue(ctx->io_wq, &req->work);
Jens Axboe94ae5e72019-11-14 19:39:52 -07001186
1187 if (link)
1188 io_queue_linked_timeout(link);
Jens Axboe18d9be12019-09-10 09:13:05 -06001189}
1190
Pavel Begunkovcbdcb432020-06-29 19:18:43 +03001191static void io_queue_async_work(struct io_kiocb *req)
1192{
1193 /* init ->work of the whole link before punting */
1194 io_prep_async_link(req);
1195 __io_queue_async_work(req);
1196}
1197
Jens Axboe5262f562019-09-17 12:26:57 -06001198static void io_kill_timeout(struct io_kiocb *req)
1199{
1200 int ret;
1201
Jens Axboe2d283902019-12-04 11:08:05 -07001202 ret = hrtimer_try_to_cancel(&req->io->timeout.timer);
Jens Axboe5262f562019-09-17 12:26:57 -06001203 if (ret != -1) {
1204 atomic_inc(&req->ctx->cq_timeouts);
Jens Axboe842f9612019-10-29 12:34:10 -06001205 list_del_init(&req->list);
Pavel Begunkovf0e20b82020-03-07 01:15:22 +03001206 req->flags |= REQ_F_COMP_LOCKED;
Jens Axboe78e19bb2019-11-06 15:21:34 -07001207 io_cqring_fill_event(req, 0);
Jackie Liuec9c02a2019-11-08 23:50:36 +08001208 io_put_req(req);
Jens Axboe5262f562019-09-17 12:26:57 -06001209 }
1210}
1211
1212static void io_kill_timeouts(struct io_ring_ctx *ctx)
1213{
1214 struct io_kiocb *req, *tmp;
1215
1216 spin_lock_irq(&ctx->completion_lock);
1217 list_for_each_entry_safe(req, tmp, &ctx->timeout_list, list)
1218 io_kill_timeout(req);
1219 spin_unlock_irq(&ctx->completion_lock);
1220}
1221
Pavel Begunkov04518942020-05-26 20:34:05 +03001222static void __io_queue_deferred(struct io_ring_ctx *ctx)
1223{
1224 do {
1225 struct io_kiocb *req = list_first_entry(&ctx->defer_list,
1226 struct io_kiocb, list);
1227
1228 if (req_need_defer(req))
1229 break;
1230 list_del_init(&req->list);
Pavel Begunkovcbdcb432020-06-29 19:18:43 +03001231 /* punt-init is done before queueing for defer */
1232 __io_queue_async_work(req);
Pavel Begunkov04518942020-05-26 20:34:05 +03001233 } while (!list_empty(&ctx->defer_list));
1234}
1235
Pavel Begunkov360428f2020-05-30 14:54:17 +03001236static void io_flush_timeouts(struct io_ring_ctx *ctx)
1237{
1238 while (!list_empty(&ctx->timeout_list)) {
1239 struct io_kiocb *req = list_first_entry(&ctx->timeout_list,
1240 struct io_kiocb, list);
1241
Pavel Begunkov8eb7e2d2020-06-29 13:13:02 +03001242 if (io_is_timeout_noseq(req))
Pavel Begunkov360428f2020-05-30 14:54:17 +03001243 break;
Pavel Begunkovbfe68a22020-05-30 14:54:18 +03001244 if (req->timeout.target_seq != ctx->cached_cq_tail
1245 - atomic_read(&ctx->cq_timeouts))
Pavel Begunkov360428f2020-05-30 14:54:17 +03001246 break;
Pavel Begunkovbfe68a22020-05-30 14:54:18 +03001247
Pavel Begunkov360428f2020-05-30 14:54:17 +03001248 list_del_init(&req->list);
1249 io_kill_timeout(req);
1250 }
1251}
1252
Jens Axboede0617e2019-04-06 21:51:27 -06001253static void io_commit_cqring(struct io_ring_ctx *ctx)
1254{
Pavel Begunkov360428f2020-05-30 14:54:17 +03001255 io_flush_timeouts(ctx);
Jens Axboede0617e2019-04-06 21:51:27 -06001256 __io_commit_cqring(ctx);
1257
Pavel Begunkov04518942020-05-26 20:34:05 +03001258 if (unlikely(!list_empty(&ctx->defer_list)))
1259 __io_queue_deferred(ctx);
Jens Axboede0617e2019-04-06 21:51:27 -06001260}
1261
Jens Axboe2b188cc2019-01-07 10:46:33 -07001262static struct io_uring_cqe *io_get_cqring(struct io_ring_ctx *ctx)
1263{
Hristo Venev75b28af2019-08-26 17:23:46 +00001264 struct io_rings *rings = ctx->rings;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001265 unsigned tail;
1266
1267 tail = ctx->cached_cq_tail;
Stefan Bühler115e12e2019-04-24 23:54:18 +02001268 /*
1269 * writes to the cq entry need to come after reading head; the
1270 * control dependency is enough as we're using WRITE_ONCE to
1271 * fill the cq entry
1272 */
Hristo Venev75b28af2019-08-26 17:23:46 +00001273 if (tail - READ_ONCE(rings->cq.head) == rings->cq_ring_entries)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001274 return NULL;
1275
1276 ctx->cached_cq_tail++;
Hristo Venev75b28af2019-08-26 17:23:46 +00001277 return &rings->cqes[tail & ctx->cq_mask];
Jens Axboe2b188cc2019-01-07 10:46:33 -07001278}
1279
Jens Axboef2842ab2020-01-08 11:04:00 -07001280static inline bool io_should_trigger_evfd(struct io_ring_ctx *ctx)
1281{
Jens Axboef0b493e2020-02-01 21:30:11 -07001282 if (!ctx->cq_ev_fd)
1283 return false;
Stefano Garzarella7e55a192020-05-15 18:38:05 +02001284 if (READ_ONCE(ctx->rings->cq_flags) & IORING_CQ_EVENTFD_DISABLED)
1285 return false;
Jens Axboef2842ab2020-01-08 11:04:00 -07001286 if (!ctx->eventfd_async)
1287 return true;
Jens Axboeb41e9852020-02-17 09:52:41 -07001288 return io_wq_current_is_worker();
Jens Axboef2842ab2020-01-08 11:04:00 -07001289}
1290
Jens Axboeb41e9852020-02-17 09:52:41 -07001291static void io_cqring_ev_posted(struct io_ring_ctx *ctx)
Jens Axboe8c838782019-03-12 15:48:16 -06001292{
1293 if (waitqueue_active(&ctx->wait))
1294 wake_up(&ctx->wait);
1295 if (waitqueue_active(&ctx->sqo_wait))
1296 wake_up(&ctx->sqo_wait);
Jens Axboeb41e9852020-02-17 09:52:41 -07001297 if (io_should_trigger_evfd(ctx))
Jens Axboe9b402842019-04-11 11:45:41 -06001298 eventfd_signal(ctx->cq_ev_fd, 1);
Jens Axboe8c838782019-03-12 15:48:16 -06001299}
1300
Jens Axboec4a2ed72019-11-21 21:01:26 -07001301/* Returns true if there are no backlogged entries after the flush */
1302static bool io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001303{
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001304 struct io_rings *rings = ctx->rings;
1305 struct io_uring_cqe *cqe;
1306 struct io_kiocb *req;
1307 unsigned long flags;
1308 LIST_HEAD(list);
1309
1310 if (!force) {
1311 if (list_empty_careful(&ctx->cq_overflow_list))
Jens Axboec4a2ed72019-11-21 21:01:26 -07001312 return true;
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001313 if ((ctx->cached_cq_tail - READ_ONCE(rings->cq.head) ==
1314 rings->cq_ring_entries))
Jens Axboec4a2ed72019-11-21 21:01:26 -07001315 return false;
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001316 }
1317
1318 spin_lock_irqsave(&ctx->completion_lock, flags);
1319
1320 /* if force is set, the ring is going away. always drop after that */
1321 if (force)
Jens Axboe69b3e542020-01-08 11:01:46 -07001322 ctx->cq_overflow_flushed = 1;
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001323
Jens Axboec4a2ed72019-11-21 21:01:26 -07001324 cqe = NULL;
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001325 while (!list_empty(&ctx->cq_overflow_list)) {
1326 cqe = io_get_cqring(ctx);
1327 if (!cqe && !force)
1328 break;
1329
1330 req = list_first_entry(&ctx->cq_overflow_list, struct io_kiocb,
1331 list);
1332 list_move(&req->list, &list);
Jens Axboe2ca10252020-02-13 17:17:35 -07001333 req->flags &= ~REQ_F_OVERFLOW;
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001334 if (cqe) {
1335 WRITE_ONCE(cqe->user_data, req->user_data);
1336 WRITE_ONCE(cqe->res, req->result);
Jens Axboebcda7ba2020-02-23 16:42:51 -07001337 WRITE_ONCE(cqe->flags, req->cflags);
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001338 } else {
1339 WRITE_ONCE(ctx->rings->cq_overflow,
1340 atomic_inc_return(&ctx->cached_cq_overflow));
1341 }
1342 }
1343
1344 io_commit_cqring(ctx);
Jens Axboead3eb2c2019-12-18 17:12:20 -07001345 if (cqe) {
1346 clear_bit(0, &ctx->sq_check_overflow);
1347 clear_bit(0, &ctx->cq_check_overflow);
1348 }
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001349 spin_unlock_irqrestore(&ctx->completion_lock, flags);
1350 io_cqring_ev_posted(ctx);
1351
1352 while (!list_empty(&list)) {
1353 req = list_first_entry(&list, struct io_kiocb, list);
1354 list_del(&req->list);
Jackie Liuec9c02a2019-11-08 23:50:36 +08001355 io_put_req(req);
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001356 }
Jens Axboec4a2ed72019-11-21 21:01:26 -07001357
1358 return cqe != NULL;
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001359}
1360
Jens Axboebcda7ba2020-02-23 16:42:51 -07001361static void __io_cqring_fill_event(struct io_kiocb *req, long res, long cflags)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001362{
Jens Axboe78e19bb2019-11-06 15:21:34 -07001363 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001364 struct io_uring_cqe *cqe;
1365
Jens Axboe78e19bb2019-11-06 15:21:34 -07001366 trace_io_uring_complete(ctx, req->user_data, res);
Jens Axboe51c3ff62019-11-03 06:52:50 -07001367
Jens Axboe2b188cc2019-01-07 10:46:33 -07001368 /*
1369 * If we can't get a cq entry, userspace overflowed the
1370 * submission (by quite a lot). Increment the overflow count in
1371 * the ring.
1372 */
1373 cqe = io_get_cqring(ctx);
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001374 if (likely(cqe)) {
Jens Axboe78e19bb2019-11-06 15:21:34 -07001375 WRITE_ONCE(cqe->user_data, req->user_data);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001376 WRITE_ONCE(cqe->res, res);
Jens Axboebcda7ba2020-02-23 16:42:51 -07001377 WRITE_ONCE(cqe->flags, cflags);
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001378 } else if (ctx->cq_overflow_flushed) {
Jens Axboe2b188cc2019-01-07 10:46:33 -07001379 WRITE_ONCE(ctx->rings->cq_overflow,
1380 atomic_inc_return(&ctx->cached_cq_overflow));
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001381 } else {
Jens Axboead3eb2c2019-12-18 17:12:20 -07001382 if (list_empty(&ctx->cq_overflow_list)) {
1383 set_bit(0, &ctx->sq_check_overflow);
1384 set_bit(0, &ctx->cq_check_overflow);
1385 }
Jens Axboe2ca10252020-02-13 17:17:35 -07001386 req->flags |= REQ_F_OVERFLOW;
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001387 refcount_inc(&req->refs);
1388 req->result = res;
Jens Axboebcda7ba2020-02-23 16:42:51 -07001389 req->cflags = cflags;
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001390 list_add_tail(&req->list, &ctx->cq_overflow_list);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001391 }
1392}
1393
Jens Axboebcda7ba2020-02-23 16:42:51 -07001394static void io_cqring_fill_event(struct io_kiocb *req, long res)
1395{
1396 __io_cqring_fill_event(req, res, 0);
1397}
1398
Jens Axboee1e16092020-06-22 09:17:17 -06001399static void io_cqring_add_event(struct io_kiocb *req, long res, long cflags)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001400{
Jens Axboe78e19bb2019-11-06 15:21:34 -07001401 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001402 unsigned long flags;
1403
1404 spin_lock_irqsave(&ctx->completion_lock, flags);
Jens Axboebcda7ba2020-02-23 16:42:51 -07001405 __io_cqring_fill_event(req, res, cflags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001406 io_commit_cqring(ctx);
1407 spin_unlock_irqrestore(&ctx->completion_lock, flags);
1408
Jens Axboe8c838782019-03-12 15:48:16 -06001409 io_cqring_ev_posted(ctx);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001410}
1411
Jens Axboe229a7b62020-06-22 10:13:11 -06001412static void io_submit_flush_completions(struct io_comp_state *cs)
Jens Axboebcda7ba2020-02-23 16:42:51 -07001413{
Jens Axboe229a7b62020-06-22 10:13:11 -06001414 struct io_ring_ctx *ctx = cs->ctx;
1415
1416 spin_lock_irq(&ctx->completion_lock);
1417 while (!list_empty(&cs->list)) {
1418 struct io_kiocb *req;
1419
1420 req = list_first_entry(&cs->list, struct io_kiocb, list);
1421 list_del(&req->list);
1422 io_cqring_fill_event(req, req->result);
1423 if (!(req->flags & REQ_F_LINK_HEAD)) {
1424 req->flags |= REQ_F_COMP_LOCKED;
1425 io_put_req(req);
1426 } else {
1427 spin_unlock_irq(&ctx->completion_lock);
1428 io_put_req(req);
1429 spin_lock_irq(&ctx->completion_lock);
1430 }
1431 }
1432 io_commit_cqring(ctx);
1433 spin_unlock_irq(&ctx->completion_lock);
1434
1435 io_cqring_ev_posted(ctx);
1436 cs->nr = 0;
1437}
1438
1439static void __io_req_complete(struct io_kiocb *req, long res, unsigned cflags,
1440 struct io_comp_state *cs)
1441{
1442 if (!cs) {
1443 io_cqring_add_event(req, res, cflags);
1444 io_put_req(req);
1445 } else {
1446 req->result = res;
1447 list_add_tail(&req->list, &cs->list);
1448 if (++cs->nr >= 32)
1449 io_submit_flush_completions(cs);
1450 }
Jens Axboee1e16092020-06-22 09:17:17 -06001451}
1452
1453static void io_req_complete(struct io_kiocb *req, long res)
1454{
Jens Axboe229a7b62020-06-22 10:13:11 -06001455 __io_req_complete(req, res, 0, NULL);
Jens Axboebcda7ba2020-02-23 16:42:51 -07001456}
1457
Jens Axboe0ddf92e2019-11-08 08:52:53 -07001458static inline bool io_is_fallback_req(struct io_kiocb *req)
1459{
1460 return req == (struct io_kiocb *)
1461 ((unsigned long) req->ctx->fallback_req & ~1UL);
1462}
1463
1464static struct io_kiocb *io_get_fallback_req(struct io_ring_ctx *ctx)
1465{
1466 struct io_kiocb *req;
1467
1468 req = ctx->fallback_req;
Bijan Mottahedehdd461af2020-04-29 17:47:50 -07001469 if (!test_and_set_bit_lock(0, (unsigned long *) &ctx->fallback_req))
Jens Axboe0ddf92e2019-11-08 08:52:53 -07001470 return req;
1471
1472 return NULL;
1473}
1474
Pavel Begunkov0553b8b2020-04-08 08:58:45 +03001475static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx,
1476 struct io_submit_state *state)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001477{
Jens Axboefd6fab22019-03-14 16:30:06 -06001478 gfp_t gfp = GFP_KERNEL | __GFP_NOWARN;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001479 struct io_kiocb *req;
1480
Pavel Begunkovf6b6c7d2020-06-21 13:09:53 +03001481 if (!state->free_reqs) {
Jens Axboe2579f912019-01-09 09:10:43 -07001482 size_t sz;
1483 int ret;
1484
1485 sz = min_t(size_t, state->ios_left, ARRAY_SIZE(state->reqs));
Jens Axboefd6fab22019-03-14 16:30:06 -06001486 ret = kmem_cache_alloc_bulk(req_cachep, gfp, sz, state->reqs);
1487
1488 /*
1489 * Bulk alloc is all-or-nothing. If we fail to get a batch,
1490 * retry single alloc to be on the safe side.
1491 */
1492 if (unlikely(ret <= 0)) {
1493 state->reqs[0] = kmem_cache_alloc(req_cachep, gfp);
1494 if (!state->reqs[0])
Jens Axboe0ddf92e2019-11-08 08:52:53 -07001495 goto fallback;
Jens Axboefd6fab22019-03-14 16:30:06 -06001496 ret = 1;
1497 }
Jens Axboe2579f912019-01-09 09:10:43 -07001498 state->free_reqs = ret - 1;
Pavel Begunkov6c8a3132020-02-01 03:58:00 +03001499 req = state->reqs[ret - 1];
Jens Axboe2579f912019-01-09 09:10:43 -07001500 } else {
Jens Axboe2579f912019-01-09 09:10:43 -07001501 state->free_reqs--;
Pavel Begunkov6c8a3132020-02-01 03:58:00 +03001502 req = state->reqs[state->free_reqs];
Jens Axboe2b188cc2019-01-07 10:46:33 -07001503 }
1504
Jens Axboe2579f912019-01-09 09:10:43 -07001505 return req;
Jens Axboe0ddf92e2019-11-08 08:52:53 -07001506fallback:
Pavel Begunkov0553b8b2020-04-08 08:58:45 +03001507 return io_get_fallback_req(ctx);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001508}
1509
Pavel Begunkov8da11c12020-02-24 11:32:44 +03001510static inline void io_put_file(struct io_kiocb *req, struct file *file,
1511 bool fixed)
1512{
1513 if (fixed)
Xiaoguang Wang05589552020-03-31 14:05:18 +08001514 percpu_ref_put(req->fixed_file_refs);
Pavel Begunkov8da11c12020-02-24 11:32:44 +03001515 else
1516 fput(file);
1517}
1518
Pavel Begunkove6543a82020-06-28 12:52:30 +03001519static void io_dismantle_req(struct io_kiocb *req)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001520{
Pavel Begunkov929a3af2020-02-19 00:19:09 +03001521 if (req->flags & REQ_F_NEED_CLEANUP)
1522 io_cleanup_req(req);
1523
YueHaibing96fd84d2020-01-07 22:22:44 +08001524 kfree(req->io);
Pavel Begunkov8da11c12020-02-24 11:32:44 +03001525 if (req->file)
1526 io_put_file(req, req->file, (req->flags & REQ_F_FIXED_FILE));
Pavel Begunkov4dd28242020-06-15 10:33:13 +03001527 __io_put_req_task(req);
Jens Axboecccf0ee2020-01-27 16:34:48 -07001528 io_req_work_drop_env(req);
Jens Axboefcb323c2019-10-24 12:39:47 -06001529
Jens Axboefcb323c2019-10-24 12:39:47 -06001530 if (req->flags & REQ_F_INFLIGHT) {
Jens Axboec6ca97b302019-12-28 12:11:08 -07001531 struct io_ring_ctx *ctx = req->ctx;
Jens Axboefcb323c2019-10-24 12:39:47 -06001532 unsigned long flags;
1533
1534 spin_lock_irqsave(&ctx->inflight_lock, flags);
1535 list_del(&req->inflight_entry);
1536 if (waitqueue_active(&ctx->inflight_wait))
1537 wake_up(&ctx->inflight_wait);
1538 spin_unlock_irqrestore(&ctx->inflight_lock, flags);
1539 }
Pavel Begunkove6543a82020-06-28 12:52:30 +03001540}
Pavel Begunkov2b85edf2019-12-28 14:13:03 +03001541
Pavel Begunkove6543a82020-06-28 12:52:30 +03001542static void __io_free_req(struct io_kiocb *req)
1543{
Pavel Begunkovecfc5172020-06-29 13:13:03 +03001544 struct io_ring_ctx *ctx;
1545
Pavel Begunkove6543a82020-06-28 12:52:30 +03001546 io_dismantle_req(req);
Pavel Begunkovecfc5172020-06-29 13:13:03 +03001547 ctx = req->ctx;
Pavel Begunkovb1e50e52020-04-08 08:58:44 +03001548 if (likely(!io_is_fallback_req(req)))
1549 kmem_cache_free(req_cachep, req);
1550 else
Pavel Begunkovecfc5172020-06-29 13:13:03 +03001551 clear_bit_unlock(0, (unsigned long *) &ctx->fallback_req);
1552 percpu_ref_put(&ctx->refs);
Jens Axboee65ef562019-03-12 10:16:44 -06001553}
1554
Jackie Liua197f662019-11-08 08:09:12 -07001555static bool io_link_cancel_timeout(struct io_kiocb *req)
Jens Axboe9e645e112019-05-10 16:07:28 -06001556{
Jackie Liua197f662019-11-08 08:09:12 -07001557 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe2665abf2019-11-05 12:40:47 -07001558 int ret;
1559
Jens Axboe2d283902019-12-04 11:08:05 -07001560 ret = hrtimer_try_to_cancel(&req->io->timeout.timer);
Jens Axboe2665abf2019-11-05 12:40:47 -07001561 if (ret != -1) {
Jens Axboe78e19bb2019-11-06 15:21:34 -07001562 io_cqring_fill_event(req, -ECANCELED);
Jens Axboe2665abf2019-11-05 12:40:47 -07001563 io_commit_cqring(ctx);
Pavel Begunkovdea3b492020-04-12 02:05:04 +03001564 req->flags &= ~REQ_F_LINK_HEAD;
Jackie Liuec9c02a2019-11-08 23:50:36 +08001565 io_put_req(req);
Jens Axboe2665abf2019-11-05 12:40:47 -07001566 return true;
1567 }
1568
1569 return false;
1570}
1571
Jens Axboeab0b6452020-06-30 08:43:15 -06001572static bool __io_kill_linked_timeout(struct io_kiocb *req)
Jens Axboe9e645e112019-05-10 16:07:28 -06001573{
Pavel Begunkov7c86ffe2020-06-29 13:12:59 +03001574 struct io_kiocb *link;
Jens Axboeab0b6452020-06-30 08:43:15 -06001575 bool wake_ev;
Pavel Begunkov7c86ffe2020-06-29 13:12:59 +03001576
1577 if (list_empty(&req->link_list))
Jens Axboeab0b6452020-06-30 08:43:15 -06001578 return false;
Pavel Begunkov7c86ffe2020-06-29 13:12:59 +03001579 link = list_first_entry(&req->link_list, struct io_kiocb, link_list);
1580 if (link->opcode != IORING_OP_LINK_TIMEOUT)
Jens Axboeab0b6452020-06-30 08:43:15 -06001581 return false;
Pavel Begunkov7c86ffe2020-06-29 13:12:59 +03001582
1583 list_del_init(&link->link_list);
1584 wake_ev = io_link_cancel_timeout(link);
1585 req->flags &= ~REQ_F_LINK_TIMEOUT;
Jens Axboeab0b6452020-06-30 08:43:15 -06001586 return wake_ev;
1587}
1588
1589static void io_kill_linked_timeout(struct io_kiocb *req)
1590{
1591 struct io_ring_ctx *ctx = req->ctx;
1592 bool wake_ev;
1593
1594 if (!(req->flags & REQ_F_COMP_LOCKED)) {
1595 unsigned long flags;
1596
1597 spin_lock_irqsave(&ctx->completion_lock, flags);
1598 wake_ev = __io_kill_linked_timeout(req);
Pavel Begunkov7c86ffe2020-06-29 13:12:59 +03001599 spin_unlock_irqrestore(&ctx->completion_lock, flags);
Jens Axboeab0b6452020-06-30 08:43:15 -06001600 } else {
1601 wake_ev = __io_kill_linked_timeout(req);
1602 }
1603
Pavel Begunkov7c86ffe2020-06-29 13:12:59 +03001604 if (wake_ev)
1605 io_cqring_ev_posted(ctx);
1606}
1607
Pavel Begunkov9b5f7bd2020-06-29 13:13:00 +03001608static struct io_kiocb *io_req_link_next(struct io_kiocb *req)
Pavel Begunkov7c86ffe2020-06-29 13:12:59 +03001609{
1610 struct io_kiocb *nxt;
Jens Axboe9e645e112019-05-10 16:07:28 -06001611
1612 /*
1613 * The list should never be empty when we are called here. But could
1614 * potentially happen if the chain is messed up, check to be on the
1615 * safe side.
1616 */
Pavel Begunkov7c86ffe2020-06-29 13:12:59 +03001617 if (unlikely(list_empty(&req->link_list)))
Pavel Begunkov9b5f7bd2020-06-29 13:13:00 +03001618 return NULL;
Jens Axboe94ae5e72019-11-14 19:39:52 -07001619
Pavel Begunkov7c86ffe2020-06-29 13:12:59 +03001620 nxt = list_first_entry(&req->link_list, struct io_kiocb, link_list);
1621 list_del_init(&req->link_list);
1622 if (!list_empty(&nxt->link_list))
1623 nxt->flags |= REQ_F_LINK_HEAD;
Pavel Begunkov9b5f7bd2020-06-29 13:13:00 +03001624 return nxt;
Jens Axboe9e645e112019-05-10 16:07:28 -06001625}
1626
1627/*
Pavel Begunkovdea3b492020-04-12 02:05:04 +03001628 * Called if REQ_F_LINK_HEAD is set, and we fail the head request
Jens Axboe9e645e112019-05-10 16:07:28 -06001629 */
Pavel Begunkov7c86ffe2020-06-29 13:12:59 +03001630static void __io_fail_links(struct io_kiocb *req)
Jens Axboe9e645e112019-05-10 16:07:28 -06001631{
Jens Axboe2665abf2019-11-05 12:40:47 -07001632 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe9e645e112019-05-10 16:07:28 -06001633
1634 while (!list_empty(&req->link_list)) {
Pavel Begunkov44932332019-12-05 16:16:35 +03001635 struct io_kiocb *link = list_first_entry(&req->link_list,
1636 struct io_kiocb, link_list);
Jens Axboe9e645e112019-05-10 16:07:28 -06001637
Pavel Begunkov44932332019-12-05 16:16:35 +03001638 list_del_init(&link->link_list);
Dmitrii Dolgovc826bd72019-10-15 19:02:01 +02001639 trace_io_uring_fail_link(req, link);
Jens Axboe2665abf2019-11-05 12:40:47 -07001640
Pavel Begunkov7c86ffe2020-06-29 13:12:59 +03001641 io_cqring_fill_event(link, -ECANCELED);
1642 __io_double_put_req(link);
Jens Axboe5d960722019-11-19 15:31:28 -07001643 req->flags &= ~REQ_F_LINK_TIMEOUT;
Jens Axboe9e645e112019-05-10 16:07:28 -06001644 }
Jens Axboe2665abf2019-11-05 12:40:47 -07001645
1646 io_commit_cqring(ctx);
Pavel Begunkov7c86ffe2020-06-29 13:12:59 +03001647 io_cqring_ev_posted(ctx);
1648}
1649
1650static void io_fail_links(struct io_kiocb *req)
1651{
1652 struct io_ring_ctx *ctx = req->ctx;
1653
1654 if (!(req->flags & REQ_F_COMP_LOCKED)) {
1655 unsigned long flags;
1656
1657 spin_lock_irqsave(&ctx->completion_lock, flags);
1658 __io_fail_links(req);
1659 spin_unlock_irqrestore(&ctx->completion_lock, flags);
1660 } else {
1661 __io_fail_links(req);
1662 }
1663
Jens Axboe2665abf2019-11-05 12:40:47 -07001664 io_cqring_ev_posted(ctx);
Jens Axboe9e645e112019-05-10 16:07:28 -06001665}
1666
Pavel Begunkov3fa5e0f2020-06-30 15:20:43 +03001667static struct io_kiocb *__io_req_find_next(struct io_kiocb *req)
Jens Axboe9e645e112019-05-10 16:07:28 -06001668{
Pavel Begunkov9b0d9112020-06-28 12:52:34 +03001669 req->flags &= ~REQ_F_LINK_HEAD;
Pavel Begunkov7c86ffe2020-06-29 13:12:59 +03001670 if (req->flags & REQ_F_LINK_TIMEOUT)
1671 io_kill_linked_timeout(req);
1672
Jens Axboe9e645e112019-05-10 16:07:28 -06001673 /*
1674 * If LINK is set, we have dependent requests in this chain. If we
1675 * didn't fail this request, queue the first one up, moving any other
1676 * dependencies to the next request. In case of failure, fail the rest
1677 * of the chain.
1678 */
Pavel Begunkov9b5f7bd2020-06-29 13:13:00 +03001679 if (likely(!(req->flags & REQ_F_FAIL_LINK)))
1680 return io_req_link_next(req);
1681 io_fail_links(req);
1682 return NULL;
Jens Axboe4d7dd462019-11-20 13:03:52 -07001683}
Jens Axboe9e645e112019-05-10 16:07:28 -06001684
Pavel Begunkov3fa5e0f2020-06-30 15:20:43 +03001685static struct io_kiocb *io_req_find_next(struct io_kiocb *req)
1686{
1687 if (likely(!(req->flags & REQ_F_LINK_HEAD)))
1688 return NULL;
1689 return __io_req_find_next(req);
1690}
1691
Jens Axboec40f6372020-06-25 15:39:59 -06001692static void __io_req_task_cancel(struct io_kiocb *req, int error)
1693{
1694 struct io_ring_ctx *ctx = req->ctx;
1695
1696 spin_lock_irq(&ctx->completion_lock);
1697 io_cqring_fill_event(req, error);
1698 io_commit_cqring(ctx);
1699 spin_unlock_irq(&ctx->completion_lock);
1700
1701 io_cqring_ev_posted(ctx);
1702 req_set_fail_links(req);
1703 io_double_put_req(req);
1704}
1705
1706static void io_req_task_cancel(struct callback_head *cb)
1707{
1708 struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
1709
1710 __io_req_task_cancel(req, -ECANCELED);
1711}
1712
1713static void __io_req_task_submit(struct io_kiocb *req)
1714{
1715 struct io_ring_ctx *ctx = req->ctx;
1716
1717 __set_current_state(TASK_RUNNING);
1718 if (!__io_sq_thread_acquire_mm(ctx)) {
1719 mutex_lock(&ctx->uring_lock);
1720 __io_queue_sqe(req, NULL, NULL);
1721 mutex_unlock(&ctx->uring_lock);
1722 } else {
1723 __io_req_task_cancel(req, -EFAULT);
1724 }
1725}
1726
1727static void io_req_task_submit(struct callback_head *cb)
1728{
1729 struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
1730
1731 __io_req_task_submit(req);
1732}
1733
1734static void io_req_task_queue(struct io_kiocb *req)
1735{
1736 struct task_struct *tsk = req->task;
1737 int ret;
1738
1739 init_task_work(&req->task_work, io_req_task_submit);
1740
1741 ret = task_work_add(tsk, &req->task_work, true);
1742 if (unlikely(ret)) {
1743 init_task_work(&req->task_work, io_req_task_cancel);
1744 tsk = io_wq_get_task(req->ctx->io_wq);
1745 task_work_add(tsk, &req->task_work, true);
1746 }
1747 wake_up_process(tsk);
1748}
1749
Pavel Begunkovc3524382020-06-28 12:52:32 +03001750static void io_queue_next(struct io_kiocb *req)
Jackie Liuc69f8db2019-11-09 11:00:08 +08001751{
Pavel Begunkov9b5f7bd2020-06-29 13:13:00 +03001752 struct io_kiocb *nxt = io_req_find_next(req);
Pavel Begunkov944e58b2019-11-21 23:21:01 +03001753
Pavel Begunkov906a8c32020-06-27 14:04:55 +03001754 if (nxt)
1755 io_req_task_queue(nxt);
Jackie Liuc69f8db2019-11-09 11:00:08 +08001756}
1757
Pavel Begunkovc3524382020-06-28 12:52:32 +03001758static void io_free_req(struct io_kiocb *req)
1759{
1760 io_queue_next(req);
1761 __io_free_req(req);
1762}
1763
Pavel Begunkov2d6500d2020-06-28 12:52:33 +03001764struct req_batch {
1765 void *reqs[IO_IOPOLL_BATCH];
1766 int to_free;
1767};
1768
1769static void __io_req_free_batch_flush(struct io_ring_ctx *ctx,
1770 struct req_batch *rb)
1771{
1772 kmem_cache_free_bulk(req_cachep, rb->to_free, rb->reqs);
1773 percpu_ref_put_many(&ctx->refs, rb->to_free);
1774 rb->to_free = 0;
1775}
1776
1777static void io_req_free_batch_finish(struct io_ring_ctx *ctx,
1778 struct req_batch *rb)
1779{
1780 if (rb->to_free)
1781 __io_req_free_batch_flush(ctx, rb);
1782}
1783
1784static void io_req_free_batch(struct req_batch *rb, struct io_kiocb *req)
1785{
1786 if (unlikely(io_is_fallback_req(req))) {
1787 io_free_req(req);
1788 return;
1789 }
1790 if (req->flags & REQ_F_LINK_HEAD)
1791 io_queue_next(req);
1792
1793 io_dismantle_req(req);
1794 rb->reqs[rb->to_free++] = req;
1795 if (unlikely(rb->to_free == ARRAY_SIZE(rb->reqs)))
1796 __io_req_free_batch_flush(req->ctx, rb);
1797}
1798
Jens Axboeba816ad2019-09-28 11:36:45 -06001799/*
1800 * Drop reference to request, return next in chain (if there is one) if this
1801 * was the last reference to this request.
1802 */
Pavel Begunkov9b5f7bd2020-06-29 13:13:00 +03001803static struct io_kiocb *io_put_req_find_next(struct io_kiocb *req)
Jens Axboee65ef562019-03-12 10:16:44 -06001804{
Pavel Begunkov9b5f7bd2020-06-29 13:13:00 +03001805 struct io_kiocb *nxt = NULL;
1806
Jens Axboe2a44f462020-02-25 13:25:41 -07001807 if (refcount_dec_and_test(&req->refs)) {
Pavel Begunkov9b5f7bd2020-06-29 13:13:00 +03001808 nxt = io_req_find_next(req);
Jens Axboe4d7dd462019-11-20 13:03:52 -07001809 __io_free_req(req);
Jens Axboe2a44f462020-02-25 13:25:41 -07001810 }
Pavel Begunkov9b5f7bd2020-06-29 13:13:00 +03001811 return nxt;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001812}
1813
Jens Axboe2b188cc2019-01-07 10:46:33 -07001814static void io_put_req(struct io_kiocb *req)
1815{
Jens Axboedef596e2019-01-09 08:59:42 -07001816 if (refcount_dec_and_test(&req->refs))
1817 io_free_req(req);
1818}
1819
Pavel Begunkovf4db7182020-06-25 18:20:54 +03001820static struct io_wq_work *io_steal_work(struct io_kiocb *req)
Pavel Begunkov7a743e22020-03-03 21:33:13 +03001821{
Pavel Begunkovcbdcb432020-06-29 19:18:43 +03001822 struct io_kiocb *timeout, *nxt = NULL;
Pavel Begunkov7a743e22020-03-03 21:33:13 +03001823
Pavel Begunkovf4db7182020-06-25 18:20:54 +03001824 /*
1825 * A ref is owned by io-wq in which context we're. So, if that's the
1826 * last one, it's safe to steal next work. False negatives are Ok,
1827 * it just will be re-punted async in io_put_work()
1828 */
1829 if (refcount_read(&req->refs) != 1)
1830 return NULL;
1831
Pavel Begunkov9b5f7bd2020-06-29 13:13:00 +03001832 nxt = io_req_find_next(req);
Pavel Begunkovf4db7182020-06-25 18:20:54 +03001833 if (!nxt)
1834 return NULL;
1835
Pavel Begunkovcbdcb432020-06-29 19:18:43 +03001836 timeout = io_prep_linked_timeout(nxt);
1837 if (timeout)
1838 nxt->flags |= REQ_F_QUEUE_TIMEOUT;
1839 return &nxt->work;
Pavel Begunkov7a743e22020-03-03 21:33:13 +03001840}
1841
Jens Axboe978db572019-11-14 22:39:04 -07001842/*
1843 * Must only be used if we don't need to care about links, usually from
1844 * within the completion handling itself.
1845 */
1846static void __io_double_put_req(struct io_kiocb *req)
Jens Axboea3a0e432019-08-20 11:03:11 -06001847{
Jens Axboe78e19bb2019-11-06 15:21:34 -07001848 /* drop both submit and complete references */
1849 if (refcount_sub_and_test(2, &req->refs))
1850 __io_free_req(req);
1851}
1852
Jens Axboe978db572019-11-14 22:39:04 -07001853static void io_double_put_req(struct io_kiocb *req)
1854{
1855 /* drop both submit and complete references */
1856 if (refcount_sub_and_test(2, &req->refs))
1857 io_free_req(req);
1858}
1859
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001860static unsigned io_cqring_events(struct io_ring_ctx *ctx, bool noflush)
Jens Axboea3a0e432019-08-20 11:03:11 -06001861{
Jens Axboe84f97dc2019-11-06 11:27:53 -07001862 struct io_rings *rings = ctx->rings;
1863
Jens Axboead3eb2c2019-12-18 17:12:20 -07001864 if (test_bit(0, &ctx->cq_check_overflow)) {
1865 /*
1866 * noflush == true is from the waitqueue handler, just ensure
1867 * we wake up the task, and the next invocation will flush the
1868 * entries. We cannot safely to it from here.
1869 */
1870 if (noflush && !list_empty(&ctx->cq_overflow_list))
1871 return -1U;
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001872
Jens Axboead3eb2c2019-12-18 17:12:20 -07001873 io_cqring_overflow_flush(ctx, false);
1874 }
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001875
Jens Axboea3a0e432019-08-20 11:03:11 -06001876 /* See comment at the top of this file */
1877 smp_rmb();
Jens Axboead3eb2c2019-12-18 17:12:20 -07001878 return ctx->cached_cq_tail - READ_ONCE(rings->cq.head);
Jens Axboea3a0e432019-08-20 11:03:11 -06001879}
1880
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03001881static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx)
1882{
1883 struct io_rings *rings = ctx->rings;
1884
1885 /* make sure SQ entry isn't read before tail */
1886 return smp_load_acquire(&rings->sq.tail) - ctx->cached_sq_head;
1887}
1888
Jens Axboebcda7ba2020-02-23 16:42:51 -07001889static int io_put_kbuf(struct io_kiocb *req)
1890{
Jens Axboe4d954c22020-02-27 07:31:19 -07001891 struct io_buffer *kbuf;
Jens Axboebcda7ba2020-02-23 16:42:51 -07001892 int cflags;
1893
Jens Axboe4d954c22020-02-27 07:31:19 -07001894 kbuf = (struct io_buffer *) (unsigned long) req->rw.addr;
Jens Axboebcda7ba2020-02-23 16:42:51 -07001895 cflags = kbuf->bid << IORING_CQE_BUFFER_SHIFT;
1896 cflags |= IORING_CQE_F_BUFFER;
1897 req->rw.addr = 0;
1898 kfree(kbuf);
1899 return cflags;
1900}
1901
Xiaoguang Wangbbde0172020-06-16 02:06:38 +08001902static void io_iopoll_queue(struct list_head *again)
1903{
1904 struct io_kiocb *req;
1905
1906 do {
1907 req = list_first_entry(again, struct io_kiocb, list);
1908 list_del(&req->list);
Pavel Begunkovcf2f5422020-06-30 15:20:40 +03001909 if (!io_rw_reissue(req, -EAGAIN))
Jens Axboe2237d762020-06-26 13:44:16 -06001910 io_complete_rw_common(&req->rw.kiocb, -EAGAIN, NULL);
Xiaoguang Wangbbde0172020-06-16 02:06:38 +08001911 } while (!list_empty(again));
1912}
1913
Jens Axboedef596e2019-01-09 08:59:42 -07001914/*
1915 * Find and free completed poll iocbs
1916 */
1917static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events,
1918 struct list_head *done)
1919{
Jens Axboe8237e042019-12-28 10:48:22 -07001920 struct req_batch rb;
Jens Axboedef596e2019-01-09 08:59:42 -07001921 struct io_kiocb *req;
Xiaoguang Wangbbde0172020-06-16 02:06:38 +08001922 LIST_HEAD(again);
1923
1924 /* order with ->result store in io_complete_rw_iopoll() */
1925 smp_rmb();
Jens Axboedef596e2019-01-09 08:59:42 -07001926
Pavel Begunkov2757a232020-06-28 12:52:31 +03001927 rb.to_free = 0;
Jens Axboedef596e2019-01-09 08:59:42 -07001928 while (!list_empty(done)) {
Jens Axboebcda7ba2020-02-23 16:42:51 -07001929 int cflags = 0;
1930
Jens Axboedef596e2019-01-09 08:59:42 -07001931 req = list_first_entry(done, struct io_kiocb, list);
Xiaoguang Wangbbde0172020-06-16 02:06:38 +08001932 if (READ_ONCE(req->result) == -EAGAIN) {
1933 req->iopoll_completed = 0;
1934 list_move_tail(&req->list, &again);
1935 continue;
1936 }
Jens Axboedef596e2019-01-09 08:59:42 -07001937 list_del(&req->list);
1938
Jens Axboebcda7ba2020-02-23 16:42:51 -07001939 if (req->flags & REQ_F_BUFFER_SELECTED)
1940 cflags = io_put_kbuf(req);
1941
1942 __io_cqring_fill_event(req, req->result, cflags);
Jens Axboedef596e2019-01-09 08:59:42 -07001943 (*nr_events)++;
1944
Pavel Begunkovc3524382020-06-28 12:52:32 +03001945 if (refcount_dec_and_test(&req->refs))
Pavel Begunkov2d6500d2020-06-28 12:52:33 +03001946 io_req_free_batch(&rb, req);
Jens Axboedef596e2019-01-09 08:59:42 -07001947 }
Jens Axboedef596e2019-01-09 08:59:42 -07001948
Jens Axboe09bb8392019-03-13 12:39:28 -06001949 io_commit_cqring(ctx);
Xiaoguang Wang32b22442020-03-11 09:26:09 +08001950 if (ctx->flags & IORING_SETUP_SQPOLL)
1951 io_cqring_ev_posted(ctx);
Pavel Begunkov2d6500d2020-06-28 12:52:33 +03001952 io_req_free_batch_finish(ctx, &rb);
Jens Axboedef596e2019-01-09 08:59:42 -07001953
Xiaoguang Wangbbde0172020-06-16 02:06:38 +08001954 if (!list_empty(&again))
1955 io_iopoll_queue(&again);
Bijan Mottahedeh581f9812020-04-03 13:51:33 -07001956}
1957
Jens Axboedef596e2019-01-09 08:59:42 -07001958static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events,
1959 long min)
1960{
1961 struct io_kiocb *req, *tmp;
1962 LIST_HEAD(done);
1963 bool spin;
1964 int ret;
1965
1966 /*
1967 * Only spin for completions if we don't have multiple devices hanging
1968 * off our complete list, and we're under the requested amount.
1969 */
1970 spin = !ctx->poll_multi_file && *nr_events < min;
1971
1972 ret = 0;
1973 list_for_each_entry_safe(req, tmp, &ctx->poll_list, list) {
Jens Axboe9adbd452019-12-20 08:45:55 -07001974 struct kiocb *kiocb = &req->rw.kiocb;
Jens Axboedef596e2019-01-09 08:59:42 -07001975
1976 /*
Bijan Mottahedeh581f9812020-04-03 13:51:33 -07001977 * Move completed and retryable entries to our local lists.
1978 * If we find a request that requires polling, break out
1979 * and complete those lists first, if we have entries there.
Jens Axboedef596e2019-01-09 08:59:42 -07001980 */
Xiaoguang Wang65a65432020-06-11 23:39:36 +08001981 if (READ_ONCE(req->iopoll_completed)) {
Jens Axboedef596e2019-01-09 08:59:42 -07001982 list_move_tail(&req->list, &done);
1983 continue;
1984 }
1985 if (!list_empty(&done))
1986 break;
1987
1988 ret = kiocb->ki_filp->f_op->iopoll(kiocb, spin);
1989 if (ret < 0)
1990 break;
1991
1992 if (ret && spin)
1993 spin = false;
1994 ret = 0;
1995 }
1996
1997 if (!list_empty(&done))
1998 io_iopoll_complete(ctx, nr_events, &done);
1999
2000 return ret;
2001}
2002
2003/*
Brian Gianforcarod195a662019-12-13 03:09:50 -08002004 * Poll for a minimum of 'min' events. Note that if min == 0 we consider that a
Jens Axboedef596e2019-01-09 08:59:42 -07002005 * non-spinning poll check - we'll still enter the driver poll loop, but only
2006 * as a non-spinning completion check.
2007 */
2008static int io_iopoll_getevents(struct io_ring_ctx *ctx, unsigned int *nr_events,
2009 long min)
2010{
Jens Axboe08f54392019-08-21 22:19:11 -06002011 while (!list_empty(&ctx->poll_list) && !need_resched()) {
Jens Axboedef596e2019-01-09 08:59:42 -07002012 int ret;
2013
2014 ret = io_do_iopoll(ctx, nr_events, min);
2015 if (ret < 0)
2016 return ret;
2017 if (!min || *nr_events >= min)
2018 return 0;
2019 }
2020
2021 return 1;
2022}
2023
2024/*
2025 * We can't just wait for polled events to come to us, we have to actively
2026 * find and complete them.
2027 */
2028static void io_iopoll_reap_events(struct io_ring_ctx *ctx)
2029{
2030 if (!(ctx->flags & IORING_SETUP_IOPOLL))
2031 return;
2032
2033 mutex_lock(&ctx->uring_lock);
2034 while (!list_empty(&ctx->poll_list)) {
2035 unsigned int nr_events = 0;
2036
2037 io_iopoll_getevents(ctx, &nr_events, 1);
Jens Axboe08f54392019-08-21 22:19:11 -06002038
2039 /*
2040 * Ensure we allow local-to-the-cpu processing to take place,
2041 * in this case we need to ensure that we reap all events.
2042 */
2043 cond_resched();
Jens Axboedef596e2019-01-09 08:59:42 -07002044 }
2045 mutex_unlock(&ctx->uring_lock);
2046}
2047
Xiaoguang Wangc7849be2020-02-22 14:46:05 +08002048static int io_iopoll_check(struct io_ring_ctx *ctx, unsigned *nr_events,
2049 long min)
Jens Axboedef596e2019-01-09 08:59:42 -07002050{
Jens Axboe2b2ed972019-10-25 10:06:15 -06002051 int iters = 0, ret = 0;
Jens Axboedef596e2019-01-09 08:59:42 -07002052
Xiaoguang Wangc7849be2020-02-22 14:46:05 +08002053 /*
2054 * We disallow the app entering submit/complete with polling, but we
2055 * still need to lock the ring to prevent racing with polled issue
2056 * that got punted to a workqueue.
2057 */
2058 mutex_lock(&ctx->uring_lock);
Jens Axboedef596e2019-01-09 08:59:42 -07002059 do {
2060 int tmin = 0;
2061
Jens Axboe500f9fb2019-08-19 12:15:59 -06002062 /*
Jens Axboea3a0e432019-08-20 11:03:11 -06002063 * Don't enter poll loop if we already have events pending.
2064 * If we do, we can potentially be spinning for commands that
2065 * already triggered a CQE (eg in error).
2066 */
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07002067 if (io_cqring_events(ctx, false))
Jens Axboea3a0e432019-08-20 11:03:11 -06002068 break;
2069
2070 /*
Jens Axboe500f9fb2019-08-19 12:15:59 -06002071 * If a submit got punted to a workqueue, we can have the
2072 * application entering polling for a command before it gets
2073 * issued. That app will hold the uring_lock for the duration
2074 * of the poll right here, so we need to take a breather every
2075 * now and then to ensure that the issue has a chance to add
2076 * the poll to the issued list. Otherwise we can spin here
2077 * forever, while the workqueue is stuck trying to acquire the
2078 * very same mutex.
2079 */
2080 if (!(++iters & 7)) {
2081 mutex_unlock(&ctx->uring_lock);
Pavel Begunkov3adfeca2020-06-28 12:52:37 +03002082 if (current->task_works)
2083 task_work_run();
Jens Axboe500f9fb2019-08-19 12:15:59 -06002084 mutex_lock(&ctx->uring_lock);
2085 }
2086
Jens Axboedef596e2019-01-09 08:59:42 -07002087 if (*nr_events < min)
2088 tmin = min - *nr_events;
2089
2090 ret = io_iopoll_getevents(ctx, nr_events, tmin);
2091 if (ret <= 0)
2092 break;
2093 ret = 0;
2094 } while (min && !*nr_events && !need_resched());
2095
Jens Axboe500f9fb2019-08-19 12:15:59 -06002096 mutex_unlock(&ctx->uring_lock);
Jens Axboedef596e2019-01-09 08:59:42 -07002097 return ret;
2098}
2099
Jens Axboe491381ce2019-10-17 09:20:46 -06002100static void kiocb_end_write(struct io_kiocb *req)
Jens Axboe2b188cc2019-01-07 10:46:33 -07002101{
Jens Axboe491381ce2019-10-17 09:20:46 -06002102 /*
2103 * Tell lockdep we inherited freeze protection from submission
2104 * thread.
2105 */
2106 if (req->flags & REQ_F_ISREG) {
2107 struct inode *inode = file_inode(req->file);
Jens Axboe2b188cc2019-01-07 10:46:33 -07002108
Jens Axboe491381ce2019-10-17 09:20:46 -06002109 __sb_writers_acquired(inode->i_sb, SB_FREEZE_WRITE);
Jens Axboe2b188cc2019-01-07 10:46:33 -07002110 }
Jens Axboe491381ce2019-10-17 09:20:46 -06002111 file_end_write(req->file);
Jens Axboe2b188cc2019-01-07 10:46:33 -07002112}
2113
Jens Axboea1d7c392020-06-22 11:09:46 -06002114static void io_complete_rw_common(struct kiocb *kiocb, long res,
2115 struct io_comp_state *cs)
Jens Axboe2b188cc2019-01-07 10:46:33 -07002116{
Jens Axboe9adbd452019-12-20 08:45:55 -07002117 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
Jens Axboebcda7ba2020-02-23 16:42:51 -07002118 int cflags = 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002119
Jens Axboe491381ce2019-10-17 09:20:46 -06002120 if (kiocb->ki_flags & IOCB_WRITE)
2121 kiocb_end_write(req);
Jens Axboe2b188cc2019-01-07 10:46:33 -07002122
Jens Axboe4e88d6e2019-12-07 20:59:47 -07002123 if (res != req->result)
2124 req_set_fail_links(req);
Jens Axboebcda7ba2020-02-23 16:42:51 -07002125 if (req->flags & REQ_F_BUFFER_SELECTED)
2126 cflags = io_put_kbuf(req);
Jens Axboea1d7c392020-06-22 11:09:46 -06002127 __io_req_complete(req, res, cflags, cs);
Jens Axboeba816ad2019-09-28 11:36:45 -06002128}
2129
Jens Axboeb63534c2020-06-04 11:28:00 -06002130#ifdef CONFIG_BLOCK
2131static bool io_resubmit_prep(struct io_kiocb *req, int error)
2132{
2133 struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
2134 ssize_t ret = -ECANCELED;
2135 struct iov_iter iter;
2136 int rw;
2137
2138 if (error) {
2139 ret = error;
2140 goto end_req;
2141 }
2142
2143 switch (req->opcode) {
2144 case IORING_OP_READV:
2145 case IORING_OP_READ_FIXED:
2146 case IORING_OP_READ:
2147 rw = READ;
2148 break;
2149 case IORING_OP_WRITEV:
2150 case IORING_OP_WRITE_FIXED:
2151 case IORING_OP_WRITE:
2152 rw = WRITE;
2153 break;
2154 default:
2155 printk_once(KERN_WARNING "io_uring: bad opcode in resubmit %d\n",
2156 req->opcode);
2157 goto end_req;
2158 }
2159
2160 ret = io_import_iovec(rw, req, &iovec, &iter, false);
2161 if (ret < 0)
2162 goto end_req;
2163 ret = io_setup_async_rw(req, ret, iovec, inline_vecs, &iter);
2164 if (!ret)
2165 return true;
2166 kfree(iovec);
2167end_req:
Jens Axboeb63534c2020-06-04 11:28:00 -06002168 req_set_fail_links(req);
Jens Axboee1e16092020-06-22 09:17:17 -06002169 io_req_complete(req, ret);
Jens Axboeb63534c2020-06-04 11:28:00 -06002170 return false;
2171}
2172
2173static void io_rw_resubmit(struct callback_head *cb)
2174{
2175 struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
2176 struct io_ring_ctx *ctx = req->ctx;
2177 int err;
2178
2179 __set_current_state(TASK_RUNNING);
2180
2181 err = io_sq_thread_acquire_mm(ctx, req);
2182
2183 if (io_resubmit_prep(req, err)) {
2184 refcount_inc(&req->refs);
2185 io_queue_async_work(req);
2186 }
2187}
2188#endif
2189
2190static bool io_rw_reissue(struct io_kiocb *req, long res)
2191{
2192#ifdef CONFIG_BLOCK
2193 struct task_struct *tsk;
2194 int ret;
2195
2196 if ((res != -EAGAIN && res != -EOPNOTSUPP) || io_wq_current_is_worker())
2197 return false;
2198
2199 tsk = req->task;
2200 init_task_work(&req->task_work, io_rw_resubmit);
2201 ret = task_work_add(tsk, &req->task_work, true);
Pavel Begunkovfb492782020-06-29 12:59:48 +03002202 if (!ret) {
2203 wake_up_process(tsk);
Jens Axboeb63534c2020-06-04 11:28:00 -06002204 return true;
Pavel Begunkovfb492782020-06-29 12:59:48 +03002205 }
Jens Axboeb63534c2020-06-04 11:28:00 -06002206#endif
2207 return false;
2208}
2209
Jens Axboea1d7c392020-06-22 11:09:46 -06002210static void __io_complete_rw(struct io_kiocb *req, long res, long res2,
2211 struct io_comp_state *cs)
2212{
2213 if (!io_rw_reissue(req, res))
2214 io_complete_rw_common(&req->rw.kiocb, res, cs);
2215}
2216
Jens Axboeba816ad2019-09-28 11:36:45 -06002217static void io_complete_rw(struct kiocb *kiocb, long res, long res2)
2218{
Jens Axboe9adbd452019-12-20 08:45:55 -07002219 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
Jens Axboeba816ad2019-09-28 11:36:45 -06002220
Jens Axboea1d7c392020-06-22 11:09:46 -06002221 __io_complete_rw(req, res, res2, NULL);
Jens Axboe2b188cc2019-01-07 10:46:33 -07002222}
2223
Jens Axboedef596e2019-01-09 08:59:42 -07002224static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2)
2225{
Jens Axboe9adbd452019-12-20 08:45:55 -07002226 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
Jens Axboedef596e2019-01-09 08:59:42 -07002227
Jens Axboe491381ce2019-10-17 09:20:46 -06002228 if (kiocb->ki_flags & IOCB_WRITE)
2229 kiocb_end_write(req);
Jens Axboedef596e2019-01-09 08:59:42 -07002230
Xiaoguang Wang2d7d6792020-06-16 02:06:37 +08002231 if (res != -EAGAIN && res != req->result)
Jens Axboe4e88d6e2019-12-07 20:59:47 -07002232 req_set_fail_links(req);
Xiaoguang Wangbbde0172020-06-16 02:06:38 +08002233
2234 WRITE_ONCE(req->result, res);
2235 /* order with io_poll_complete() checking ->result */
Pavel Begunkovcd664b02020-06-25 12:37:10 +03002236 smp_wmb();
2237 WRITE_ONCE(req->iopoll_completed, 1);
Jens Axboedef596e2019-01-09 08:59:42 -07002238}
2239
2240/*
2241 * After the iocb has been issued, it's safe to be found on the poll list.
2242 * Adding the kiocb to the list AFTER submission ensures that we don't
2243 * find it from a io_iopoll_getevents() thread before the issuer is done
2244 * accessing the kiocb cookie.
2245 */
2246static void io_iopoll_req_issued(struct io_kiocb *req)
2247{
2248 struct io_ring_ctx *ctx = req->ctx;
2249
2250 /*
2251 * Track whether we have multiple files in our lists. This will impact
2252 * how we do polling eventually, not spinning if we're on potentially
2253 * different devices.
2254 */
2255 if (list_empty(&ctx->poll_list)) {
2256 ctx->poll_multi_file = false;
2257 } else if (!ctx->poll_multi_file) {
2258 struct io_kiocb *list_req;
2259
2260 list_req = list_first_entry(&ctx->poll_list, struct io_kiocb,
2261 list);
Jens Axboe9adbd452019-12-20 08:45:55 -07002262 if (list_req->file != req->file)
Jens Axboedef596e2019-01-09 08:59:42 -07002263 ctx->poll_multi_file = true;
2264 }
2265
2266 /*
2267 * For fast devices, IO may have already completed. If it has, add
2268 * it to the front so we find it first.
2269 */
Xiaoguang Wang65a65432020-06-11 23:39:36 +08002270 if (READ_ONCE(req->iopoll_completed))
Jens Axboedef596e2019-01-09 08:59:42 -07002271 list_add(&req->list, &ctx->poll_list);
2272 else
2273 list_add_tail(&req->list, &ctx->poll_list);
Xiaoguang Wangbdcd3ea2020-02-25 22:12:08 +08002274
2275 if ((ctx->flags & IORING_SETUP_SQPOLL) &&
2276 wq_has_sleeper(&ctx->sqo_wait))
2277 wake_up(&ctx->sqo_wait);
Jens Axboedef596e2019-01-09 08:59:42 -07002278}
2279
Pavel Begunkov9f13c352020-05-17 14:13:41 +03002280static void __io_state_file_put(struct io_submit_state *state)
Jens Axboe9a56a232019-01-09 09:06:50 -07002281{
Pavel Begunkov9f13c352020-05-17 14:13:41 +03002282 int diff = state->has_refs - state->used_refs;
Jens Axboe9a56a232019-01-09 09:06:50 -07002283
Pavel Begunkov9f13c352020-05-17 14:13:41 +03002284 if (diff)
2285 fput_many(state->file, diff);
2286 state->file = NULL;
2287}
2288
2289static inline void io_state_file_put(struct io_submit_state *state)
2290{
2291 if (state->file)
2292 __io_state_file_put(state);
Jens Axboe9a56a232019-01-09 09:06:50 -07002293}
2294
2295/*
2296 * Get as many references to a file as we have IOs left in this submission,
2297 * assuming most submissions are for one file, or at least that each file
2298 * has more than one submission.
2299 */
Pavel Begunkov8da11c12020-02-24 11:32:44 +03002300static struct file *__io_file_get(struct io_submit_state *state, int fd)
Jens Axboe9a56a232019-01-09 09:06:50 -07002301{
2302 if (!state)
2303 return fget(fd);
2304
2305 if (state->file) {
2306 if (state->fd == fd) {
2307 state->used_refs++;
2308 state->ios_left--;
2309 return state->file;
2310 }
Pavel Begunkov9f13c352020-05-17 14:13:41 +03002311 __io_state_file_put(state);
Jens Axboe9a56a232019-01-09 09:06:50 -07002312 }
2313 state->file = fget_many(fd, state->ios_left);
2314 if (!state->file)
2315 return NULL;
2316
2317 state->fd = fd;
2318 state->has_refs = state->ios_left;
2319 state->used_refs = 1;
2320 state->ios_left--;
2321 return state->file;
2322}
2323
Jens Axboe4503b762020-06-01 10:00:27 -06002324static bool io_bdev_nowait(struct block_device *bdev)
2325{
2326#ifdef CONFIG_BLOCK
2327 return !bdev || queue_is_mq(bdev_get_queue(bdev));
2328#else
2329 return true;
2330#endif
2331}
2332
Jens Axboe2b188cc2019-01-07 10:46:33 -07002333/*
2334 * If we tracked the file through the SCM inflight mechanism, we could support
2335 * any file. For now, just ensure that anything potentially problematic is done
2336 * inline.
2337 */
Jens Axboeaf197f52020-04-28 13:15:06 -06002338static bool io_file_supports_async(struct file *file, int rw)
Jens Axboe2b188cc2019-01-07 10:46:33 -07002339{
2340 umode_t mode = file_inode(file)->i_mode;
2341
Jens Axboe4503b762020-06-01 10:00:27 -06002342 if (S_ISBLK(mode)) {
2343 if (io_bdev_nowait(file->f_inode->i_bdev))
2344 return true;
2345 return false;
2346 }
2347 if (S_ISCHR(mode) || S_ISSOCK(mode))
Jens Axboe2b188cc2019-01-07 10:46:33 -07002348 return true;
Jens Axboe4503b762020-06-01 10:00:27 -06002349 if (S_ISREG(mode)) {
2350 if (io_bdev_nowait(file->f_inode->i_sb->s_bdev) &&
2351 file->f_op != &io_uring_fops)
2352 return true;
2353 return false;
2354 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07002355
Jens Axboec5b85622020-06-09 19:23:05 -06002356 /* any ->read/write should understand O_NONBLOCK */
2357 if (file->f_flags & O_NONBLOCK)
2358 return true;
2359
Jens Axboeaf197f52020-04-28 13:15:06 -06002360 if (!(file->f_mode & FMODE_NOWAIT))
2361 return false;
2362
2363 if (rw == READ)
2364 return file->f_op->read_iter != NULL;
2365
2366 return file->f_op->write_iter != NULL;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002367}
2368
Jens Axboe3529d8c2019-12-19 18:24:38 -07002369static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,
2370 bool force_nonblock)
Jens Axboe2b188cc2019-01-07 10:46:33 -07002371{
Jens Axboedef596e2019-01-09 08:59:42 -07002372 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe9adbd452019-12-20 08:45:55 -07002373 struct kiocb *kiocb = &req->rw.kiocb;
Jens Axboe09bb8392019-03-13 12:39:28 -06002374 unsigned ioprio;
2375 int ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002376
Jens Axboe491381ce2019-10-17 09:20:46 -06002377 if (S_ISREG(file_inode(req->file)->i_mode))
2378 req->flags |= REQ_F_ISREG;
2379
Jens Axboe2b188cc2019-01-07 10:46:33 -07002380 kiocb->ki_pos = READ_ONCE(sqe->off);
Jens Axboeba042912019-12-25 16:33:42 -07002381 if (kiocb->ki_pos == -1 && !(req->file->f_mode & FMODE_STREAM)) {
2382 req->flags |= REQ_F_CUR_POS;
2383 kiocb->ki_pos = req->file->f_pos;
2384 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07002385 kiocb->ki_hint = ki_hint_validate(file_write_hint(kiocb->ki_filp));
Pavel Begunkov3e577dc2020-02-01 03:58:42 +03002386 kiocb->ki_flags = iocb_flags(kiocb->ki_filp);
2387 ret = kiocb_set_rw_flags(kiocb, READ_ONCE(sqe->rw_flags));
2388 if (unlikely(ret))
2389 return ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002390
2391 ioprio = READ_ONCE(sqe->ioprio);
2392 if (ioprio) {
2393 ret = ioprio_check_cap(ioprio);
2394 if (ret)
Jens Axboe09bb8392019-03-13 12:39:28 -06002395 return ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002396
2397 kiocb->ki_ioprio = ioprio;
2398 } else
2399 kiocb->ki_ioprio = get_current_ioprio();
2400
Stefan Bühler8449eed2019-04-27 20:34:19 +02002401 /* don't allow async punt if RWF_NOWAIT was requested */
Jens Axboec5b85622020-06-09 19:23:05 -06002402 if (kiocb->ki_flags & IOCB_NOWAIT)
Stefan Bühler8449eed2019-04-27 20:34:19 +02002403 req->flags |= REQ_F_NOWAIT;
2404
Jens Axboeb63534c2020-06-04 11:28:00 -06002405 if (kiocb->ki_flags & IOCB_DIRECT)
2406 io_get_req_task(req);
2407
Stefan Bühler8449eed2019-04-27 20:34:19 +02002408 if (force_nonblock)
Jens Axboe2b188cc2019-01-07 10:46:33 -07002409 kiocb->ki_flags |= IOCB_NOWAIT;
Stefan Bühler8449eed2019-04-27 20:34:19 +02002410
Jens Axboedef596e2019-01-09 08:59:42 -07002411 if (ctx->flags & IORING_SETUP_IOPOLL) {
Jens Axboedef596e2019-01-09 08:59:42 -07002412 if (!(kiocb->ki_flags & IOCB_DIRECT) ||
2413 !kiocb->ki_filp->f_op->iopoll)
Jens Axboe09bb8392019-03-13 12:39:28 -06002414 return -EOPNOTSUPP;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002415
Jens Axboedef596e2019-01-09 08:59:42 -07002416 kiocb->ki_flags |= IOCB_HIPRI;
2417 kiocb->ki_complete = io_complete_rw_iopoll;
Xiaoguang Wang65a65432020-06-11 23:39:36 +08002418 req->iopoll_completed = 0;
Pavel Begunkovf3a6fa22020-06-28 12:52:38 +03002419 io_get_req_task(req);
Jens Axboedef596e2019-01-09 08:59:42 -07002420 } else {
Jens Axboe09bb8392019-03-13 12:39:28 -06002421 if (kiocb->ki_flags & IOCB_HIPRI)
2422 return -EINVAL;
Jens Axboedef596e2019-01-09 08:59:42 -07002423 kiocb->ki_complete = io_complete_rw;
2424 }
Jens Axboe9adbd452019-12-20 08:45:55 -07002425
Jens Axboe3529d8c2019-12-19 18:24:38 -07002426 req->rw.addr = READ_ONCE(sqe->addr);
2427 req->rw.len = READ_ONCE(sqe->len);
Bijan Mottahedeh4f4eeba2020-05-19 14:52:49 -07002428 req->buf_index = READ_ONCE(sqe->buf_index);
Jens Axboe2b188cc2019-01-07 10:46:33 -07002429 return 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002430}
2431
2432static inline void io_rw_done(struct kiocb *kiocb, ssize_t ret)
2433{
2434 switch (ret) {
2435 case -EIOCBQUEUED:
2436 break;
2437 case -ERESTARTSYS:
2438 case -ERESTARTNOINTR:
2439 case -ERESTARTNOHAND:
2440 case -ERESTART_RESTARTBLOCK:
2441 /*
2442 * We can't just restart the syscall, since previously
2443 * submitted sqes may already be in progress. Just fail this
2444 * IO with EINTR.
2445 */
2446 ret = -EINTR;
2447 /* fall through */
2448 default:
2449 kiocb->ki_complete(kiocb, ret, 0);
2450 }
2451}
2452
Jens Axboea1d7c392020-06-22 11:09:46 -06002453static void kiocb_done(struct kiocb *kiocb, ssize_t ret,
2454 struct io_comp_state *cs)
Jens Axboeba816ad2019-09-28 11:36:45 -06002455{
Jens Axboeba042912019-12-25 16:33:42 -07002456 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
2457
2458 if (req->flags & REQ_F_CUR_POS)
2459 req->file->f_pos = kiocb->ki_pos;
Pavel Begunkovbcaec082020-02-24 11:30:18 +03002460 if (ret >= 0 && kiocb->ki_complete == io_complete_rw)
Jens Axboea1d7c392020-06-22 11:09:46 -06002461 __io_complete_rw(req, ret, 0, cs);
Jens Axboeba816ad2019-09-28 11:36:45 -06002462 else
2463 io_rw_done(kiocb, ret);
2464}
2465
Jens Axboe9adbd452019-12-20 08:45:55 -07002466static ssize_t io_import_fixed(struct io_kiocb *req, int rw,
Pavel Begunkov7d009162019-11-25 23:14:40 +03002467 struct iov_iter *iter)
Jens Axboeedafcce2019-01-09 09:16:05 -07002468{
Jens Axboe9adbd452019-12-20 08:45:55 -07002469 struct io_ring_ctx *ctx = req->ctx;
2470 size_t len = req->rw.len;
Jens Axboeedafcce2019-01-09 09:16:05 -07002471 struct io_mapped_ubuf *imu;
Bijan Mottahedeh4f4eeba2020-05-19 14:52:49 -07002472 u16 index, buf_index;
Jens Axboeedafcce2019-01-09 09:16:05 -07002473 size_t offset;
2474 u64 buf_addr;
2475
2476 /* attempt to use fixed buffers without having provided iovecs */
2477 if (unlikely(!ctx->user_bufs))
2478 return -EFAULT;
2479
Bijan Mottahedeh4f4eeba2020-05-19 14:52:49 -07002480 buf_index = req->buf_index;
Jens Axboeedafcce2019-01-09 09:16:05 -07002481 if (unlikely(buf_index >= ctx->nr_user_bufs))
2482 return -EFAULT;
2483
2484 index = array_index_nospec(buf_index, ctx->nr_user_bufs);
2485 imu = &ctx->user_bufs[index];
Jens Axboe9adbd452019-12-20 08:45:55 -07002486 buf_addr = req->rw.addr;
Jens Axboeedafcce2019-01-09 09:16:05 -07002487
2488 /* overflow */
2489 if (buf_addr + len < buf_addr)
2490 return -EFAULT;
2491 /* not inside the mapped region */
2492 if (buf_addr < imu->ubuf || buf_addr + len > imu->ubuf + imu->len)
2493 return -EFAULT;
2494
2495 /*
2496 * May not be a start of buffer, set size appropriately
2497 * and advance us to the beginning.
2498 */
2499 offset = buf_addr - imu->ubuf;
2500 iov_iter_bvec(iter, rw, imu->bvec, imu->nr_bvecs, offset + len);
Jens Axboebd11b3a2019-07-20 08:37:31 -06002501
2502 if (offset) {
2503 /*
2504 * Don't use iov_iter_advance() here, as it's really slow for
2505 * using the latter parts of a big fixed buffer - it iterates
2506 * over each segment manually. We can cheat a bit here, because
2507 * we know that:
2508 *
2509 * 1) it's a BVEC iter, we set it up
2510 * 2) all bvecs are PAGE_SIZE in size, except potentially the
2511 * first and last bvec
2512 *
2513 * So just find our index, and adjust the iterator afterwards.
2514 * If the offset is within the first bvec (or the whole first
2515 * bvec, just use iov_iter_advance(). This makes it easier
2516 * since we can just skip the first segment, which may not
2517 * be PAGE_SIZE aligned.
2518 */
2519 const struct bio_vec *bvec = imu->bvec;
2520
2521 if (offset <= bvec->bv_len) {
2522 iov_iter_advance(iter, offset);
2523 } else {
2524 unsigned long seg_skip;
2525
2526 /* skip first vec */
2527 offset -= bvec->bv_len;
2528 seg_skip = 1 + (offset >> PAGE_SHIFT);
2529
2530 iter->bvec = bvec + seg_skip;
2531 iter->nr_segs -= seg_skip;
Aleix Roca Nonell99c79f62019-08-15 14:03:22 +02002532 iter->count -= bvec->bv_len + offset;
Jens Axboebd11b3a2019-07-20 08:37:31 -06002533 iter->iov_offset = offset & ~PAGE_MASK;
Jens Axboebd11b3a2019-07-20 08:37:31 -06002534 }
2535 }
2536
Jens Axboe5e559562019-11-13 16:12:46 -07002537 return len;
Jens Axboeedafcce2019-01-09 09:16:05 -07002538}
2539
Jens Axboebcda7ba2020-02-23 16:42:51 -07002540static void io_ring_submit_unlock(struct io_ring_ctx *ctx, bool needs_lock)
2541{
2542 if (needs_lock)
2543 mutex_unlock(&ctx->uring_lock);
2544}
2545
2546static void io_ring_submit_lock(struct io_ring_ctx *ctx, bool needs_lock)
2547{
2548 /*
2549 * "Normal" inline submissions always hold the uring_lock, since we
2550 * grab it from the system call. Same is true for the SQPOLL offload.
2551 * The only exception is when we've detached the request and issue it
2552 * from an async worker thread, grab the lock for that case.
2553 */
2554 if (needs_lock)
2555 mutex_lock(&ctx->uring_lock);
2556}
2557
2558static struct io_buffer *io_buffer_select(struct io_kiocb *req, size_t *len,
2559 int bgid, struct io_buffer *kbuf,
2560 bool needs_lock)
2561{
2562 struct io_buffer *head;
2563
2564 if (req->flags & REQ_F_BUFFER_SELECTED)
2565 return kbuf;
2566
2567 io_ring_submit_lock(req->ctx, needs_lock);
2568
2569 lockdep_assert_held(&req->ctx->uring_lock);
2570
2571 head = idr_find(&req->ctx->io_buffer_idr, bgid);
2572 if (head) {
2573 if (!list_empty(&head->list)) {
2574 kbuf = list_last_entry(&head->list, struct io_buffer,
2575 list);
2576 list_del(&kbuf->list);
2577 } else {
2578 kbuf = head;
2579 idr_remove(&req->ctx->io_buffer_idr, bgid);
2580 }
2581 if (*len > kbuf->len)
2582 *len = kbuf->len;
2583 } else {
2584 kbuf = ERR_PTR(-ENOBUFS);
2585 }
2586
2587 io_ring_submit_unlock(req->ctx, needs_lock);
2588
2589 return kbuf;
2590}
2591
Jens Axboe4d954c22020-02-27 07:31:19 -07002592static void __user *io_rw_buffer_select(struct io_kiocb *req, size_t *len,
2593 bool needs_lock)
2594{
2595 struct io_buffer *kbuf;
Bijan Mottahedeh4f4eeba2020-05-19 14:52:49 -07002596 u16 bgid;
Jens Axboe4d954c22020-02-27 07:31:19 -07002597
2598 kbuf = (struct io_buffer *) (unsigned long) req->rw.addr;
Bijan Mottahedeh4f4eeba2020-05-19 14:52:49 -07002599 bgid = req->buf_index;
Jens Axboe4d954c22020-02-27 07:31:19 -07002600 kbuf = io_buffer_select(req, len, bgid, kbuf, needs_lock);
2601 if (IS_ERR(kbuf))
2602 return kbuf;
2603 req->rw.addr = (u64) (unsigned long) kbuf;
2604 req->flags |= REQ_F_BUFFER_SELECTED;
2605 return u64_to_user_ptr(kbuf->addr);
2606}
2607
2608#ifdef CONFIG_COMPAT
2609static ssize_t io_compat_import(struct io_kiocb *req, struct iovec *iov,
2610 bool needs_lock)
2611{
2612 struct compat_iovec __user *uiov;
2613 compat_ssize_t clen;
2614 void __user *buf;
2615 ssize_t len;
2616
2617 uiov = u64_to_user_ptr(req->rw.addr);
2618 if (!access_ok(uiov, sizeof(*uiov)))
2619 return -EFAULT;
2620 if (__get_user(clen, &uiov->iov_len))
2621 return -EFAULT;
2622 if (clen < 0)
2623 return -EINVAL;
2624
2625 len = clen;
2626 buf = io_rw_buffer_select(req, &len, needs_lock);
2627 if (IS_ERR(buf))
2628 return PTR_ERR(buf);
2629 iov[0].iov_base = buf;
2630 iov[0].iov_len = (compat_size_t) len;
2631 return 0;
2632}
2633#endif
2634
2635static ssize_t __io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov,
2636 bool needs_lock)
2637{
2638 struct iovec __user *uiov = u64_to_user_ptr(req->rw.addr);
2639 void __user *buf;
2640 ssize_t len;
2641
2642 if (copy_from_user(iov, uiov, sizeof(*uiov)))
2643 return -EFAULT;
2644
2645 len = iov[0].iov_len;
2646 if (len < 0)
2647 return -EINVAL;
2648 buf = io_rw_buffer_select(req, &len, needs_lock);
2649 if (IS_ERR(buf))
2650 return PTR_ERR(buf);
2651 iov[0].iov_base = buf;
2652 iov[0].iov_len = len;
2653 return 0;
2654}
2655
2656static ssize_t io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov,
2657 bool needs_lock)
2658{
Jens Axboedddb3e22020-06-04 11:27:01 -06002659 if (req->flags & REQ_F_BUFFER_SELECTED) {
2660 struct io_buffer *kbuf;
2661
2662 kbuf = (struct io_buffer *) (unsigned long) req->rw.addr;
2663 iov[0].iov_base = u64_to_user_ptr(kbuf->addr);
2664 iov[0].iov_len = kbuf->len;
Jens Axboe4d954c22020-02-27 07:31:19 -07002665 return 0;
Jens Axboedddb3e22020-06-04 11:27:01 -06002666 }
Jens Axboe4d954c22020-02-27 07:31:19 -07002667 if (!req->rw.len)
2668 return 0;
2669 else if (req->rw.len > 1)
2670 return -EINVAL;
2671
2672#ifdef CONFIG_COMPAT
2673 if (req->ctx->compat)
2674 return io_compat_import(req, iov, needs_lock);
2675#endif
2676
2677 return __io_iov_buffer_select(req, iov, needs_lock);
2678}
2679
Pavel Begunkovcf6fd4b2019-11-25 23:14:39 +03002680static ssize_t io_import_iovec(int rw, struct io_kiocb *req,
Jens Axboebcda7ba2020-02-23 16:42:51 -07002681 struct iovec **iovec, struct iov_iter *iter,
2682 bool needs_lock)
Jens Axboe2b188cc2019-01-07 10:46:33 -07002683{
Jens Axboe9adbd452019-12-20 08:45:55 -07002684 void __user *buf = u64_to_user_ptr(req->rw.addr);
2685 size_t sqe_len = req->rw.len;
Jens Axboe4d954c22020-02-27 07:31:19 -07002686 ssize_t ret;
Jens Axboeedafcce2019-01-09 09:16:05 -07002687 u8 opcode;
2688
Jens Axboed625c6e2019-12-17 19:53:05 -07002689 opcode = req->opcode;
Pavel Begunkov7d009162019-11-25 23:14:40 +03002690 if (opcode == IORING_OP_READ_FIXED || opcode == IORING_OP_WRITE_FIXED) {
Jens Axboeedafcce2019-01-09 09:16:05 -07002691 *iovec = NULL;
Jens Axboe9adbd452019-12-20 08:45:55 -07002692 return io_import_fixed(req, rw, iter);
Jens Axboeedafcce2019-01-09 09:16:05 -07002693 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07002694
Jens Axboebcda7ba2020-02-23 16:42:51 -07002695 /* buffer index only valid with fixed read/write, or buffer select */
Bijan Mottahedeh4f4eeba2020-05-19 14:52:49 -07002696 if (req->buf_index && !(req->flags & REQ_F_BUFFER_SELECT))
Jens Axboe9adbd452019-12-20 08:45:55 -07002697 return -EINVAL;
2698
Jens Axboe3a6820f2019-12-22 15:19:35 -07002699 if (opcode == IORING_OP_READ || opcode == IORING_OP_WRITE) {
Jens Axboebcda7ba2020-02-23 16:42:51 -07002700 if (req->flags & REQ_F_BUFFER_SELECT) {
Jens Axboe4d954c22020-02-27 07:31:19 -07002701 buf = io_rw_buffer_select(req, &sqe_len, needs_lock);
2702 if (IS_ERR(buf)) {
Jens Axboebcda7ba2020-02-23 16:42:51 -07002703 *iovec = NULL;
Jens Axboe4d954c22020-02-27 07:31:19 -07002704 return PTR_ERR(buf);
Jens Axboebcda7ba2020-02-23 16:42:51 -07002705 }
Jens Axboe3f9d6442020-03-11 12:27:04 -06002706 req->rw.len = sqe_len;
Jens Axboebcda7ba2020-02-23 16:42:51 -07002707 }
2708
Jens Axboe3a6820f2019-12-22 15:19:35 -07002709 ret = import_single_range(rw, buf, sqe_len, *iovec, iter);
2710 *iovec = NULL;
Jens Axboe3a901592020-02-25 17:48:55 -07002711 return ret < 0 ? ret : sqe_len;
Jens Axboe3a6820f2019-12-22 15:19:35 -07002712 }
2713
Jens Axboef67676d2019-12-02 11:03:47 -07002714 if (req->io) {
2715 struct io_async_rw *iorw = &req->io->rw;
2716
2717 *iovec = iorw->iov;
2718 iov_iter_init(iter, rw, *iovec, iorw->nr_segs, iorw->size);
2719 if (iorw->iov == iorw->fast_iov)
2720 *iovec = NULL;
2721 return iorw->size;
2722 }
2723
Jens Axboe4d954c22020-02-27 07:31:19 -07002724 if (req->flags & REQ_F_BUFFER_SELECT) {
2725 ret = io_iov_buffer_select(req, *iovec, needs_lock);
Jens Axboe3f9d6442020-03-11 12:27:04 -06002726 if (!ret) {
2727 ret = (*iovec)->iov_len;
2728 iov_iter_init(iter, rw, *iovec, 1, ret);
2729 }
Jens Axboe4d954c22020-02-27 07:31:19 -07002730 *iovec = NULL;
2731 return ret;
2732 }
2733
Jens Axboe2b188cc2019-01-07 10:46:33 -07002734#ifdef CONFIG_COMPAT
Pavel Begunkovcf6fd4b2019-11-25 23:14:39 +03002735 if (req->ctx->compat)
Jens Axboe2b188cc2019-01-07 10:46:33 -07002736 return compat_import_iovec(rw, buf, sqe_len, UIO_FASTIOV,
2737 iovec, iter);
2738#endif
2739
2740 return import_iovec(rw, buf, sqe_len, UIO_FASTIOV, iovec, iter);
2741}
2742
Jens Axboe32960612019-09-23 11:05:34 -06002743/*
2744 * For files that don't have ->read_iter() and ->write_iter(), handle them
2745 * by looping over ->read() or ->write() manually.
2746 */
2747static ssize_t loop_rw_iter(int rw, struct file *file, struct kiocb *kiocb,
2748 struct iov_iter *iter)
2749{
2750 ssize_t ret = 0;
2751
2752 /*
2753 * Don't support polled IO through this interface, and we can't
2754 * support non-blocking either. For the latter, this just causes
2755 * the kiocb to be handled from an async context.
2756 */
2757 if (kiocb->ki_flags & IOCB_HIPRI)
2758 return -EOPNOTSUPP;
2759 if (kiocb->ki_flags & IOCB_NOWAIT)
2760 return -EAGAIN;
2761
2762 while (iov_iter_count(iter)) {
Pavel Begunkov311ae9e2019-11-24 11:58:24 +03002763 struct iovec iovec;
Jens Axboe32960612019-09-23 11:05:34 -06002764 ssize_t nr;
2765
Pavel Begunkov311ae9e2019-11-24 11:58:24 +03002766 if (!iov_iter_is_bvec(iter)) {
2767 iovec = iov_iter_iovec(iter);
2768 } else {
2769 /* fixed buffers import bvec */
2770 iovec.iov_base = kmap(iter->bvec->bv_page)
2771 + iter->iov_offset;
2772 iovec.iov_len = min(iter->count,
2773 iter->bvec->bv_len - iter->iov_offset);
2774 }
2775
Jens Axboe32960612019-09-23 11:05:34 -06002776 if (rw == READ) {
2777 nr = file->f_op->read(file, iovec.iov_base,
2778 iovec.iov_len, &kiocb->ki_pos);
2779 } else {
2780 nr = file->f_op->write(file, iovec.iov_base,
2781 iovec.iov_len, &kiocb->ki_pos);
2782 }
2783
Pavel Begunkov311ae9e2019-11-24 11:58:24 +03002784 if (iov_iter_is_bvec(iter))
2785 kunmap(iter->bvec->bv_page);
2786
Jens Axboe32960612019-09-23 11:05:34 -06002787 if (nr < 0) {
2788 if (!ret)
2789 ret = nr;
2790 break;
2791 }
2792 ret += nr;
2793 if (nr != iovec.iov_len)
2794 break;
2795 iov_iter_advance(iter, nr);
2796 }
2797
2798 return ret;
2799}
2800
Jens Axboeb7bb4f72019-12-15 22:13:43 -07002801static void io_req_map_rw(struct io_kiocb *req, ssize_t io_size,
Jens Axboef67676d2019-12-02 11:03:47 -07002802 struct iovec *iovec, struct iovec *fast_iov,
2803 struct iov_iter *iter)
2804{
2805 req->io->rw.nr_segs = iter->nr_segs;
2806 req->io->rw.size = io_size;
2807 req->io->rw.iov = iovec;
2808 if (!req->io->rw.iov) {
2809 req->io->rw.iov = req->io->rw.fast_iov;
Xiaoguang Wang45097da2020-04-08 22:29:58 +08002810 if (req->io->rw.iov != fast_iov)
2811 memcpy(req->io->rw.iov, fast_iov,
2812 sizeof(struct iovec) * iter->nr_segs);
Pavel Begunkov99bc4c32020-02-07 22:04:45 +03002813 } else {
2814 req->flags |= REQ_F_NEED_CLEANUP;
Jens Axboef67676d2019-12-02 11:03:47 -07002815 }
2816}
2817
Xiaoguang Wang3d9932a2020-03-27 15:36:52 +08002818static inline int __io_alloc_async_ctx(struct io_kiocb *req)
2819{
2820 req->io = kmalloc(sizeof(*req->io), GFP_KERNEL);
2821 return req->io == NULL;
2822}
2823
Jens Axboeb7bb4f72019-12-15 22:13:43 -07002824static int io_alloc_async_ctx(struct io_kiocb *req)
Jens Axboef67676d2019-12-02 11:03:47 -07002825{
Jens Axboed3656342019-12-18 09:50:26 -07002826 if (!io_op_defs[req->opcode].async_ctx)
2827 return 0;
Xiaoguang Wang3d9932a2020-03-27 15:36:52 +08002828
2829 return __io_alloc_async_ctx(req);
Jens Axboeb7bb4f72019-12-15 22:13:43 -07002830}
2831
Jens Axboeb7bb4f72019-12-15 22:13:43 -07002832static int io_setup_async_rw(struct io_kiocb *req, ssize_t io_size,
2833 struct iovec *iovec, struct iovec *fast_iov,
2834 struct iov_iter *iter)
2835{
Jens Axboe980ad262020-01-24 23:08:54 -07002836 if (!io_op_defs[req->opcode].async_ctx)
Jens Axboe74566df2020-01-13 19:23:24 -07002837 return 0;
Jens Axboe5d204bc2020-01-31 12:06:52 -07002838 if (!req->io) {
Xiaoguang Wang3d9932a2020-03-27 15:36:52 +08002839 if (__io_alloc_async_ctx(req))
Jens Axboe5d204bc2020-01-31 12:06:52 -07002840 return -ENOMEM;
Jens Axboeb7bb4f72019-12-15 22:13:43 -07002841
Jens Axboe5d204bc2020-01-31 12:06:52 -07002842 io_req_map_rw(req, io_size, iovec, fast_iov, iter);
2843 }
Jens Axboeb7bb4f72019-12-15 22:13:43 -07002844 return 0;
Jens Axboef67676d2019-12-02 11:03:47 -07002845}
2846
Jens Axboe3529d8c2019-12-19 18:24:38 -07002847static int io_read_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe,
2848 bool force_nonblock)
Jens Axboef67676d2019-12-02 11:03:47 -07002849{
Jens Axboe3529d8c2019-12-19 18:24:38 -07002850 struct io_async_ctx *io;
2851 struct iov_iter iter;
Jens Axboef67676d2019-12-02 11:03:47 -07002852 ssize_t ret;
2853
Jens Axboe3529d8c2019-12-19 18:24:38 -07002854 ret = io_prep_rw(req, sqe, force_nonblock);
2855 if (ret)
2856 return ret;
Jens Axboef67676d2019-12-02 11:03:47 -07002857
Jens Axboe3529d8c2019-12-19 18:24:38 -07002858 if (unlikely(!(req->file->f_mode & FMODE_READ)))
2859 return -EBADF;
Jens Axboef67676d2019-12-02 11:03:47 -07002860
Pavel Begunkov5f798be2020-02-08 13:28:02 +03002861 /* either don't need iovec imported or already have it */
2862 if (!req->io || req->flags & REQ_F_NEED_CLEANUP)
Jens Axboe3529d8c2019-12-19 18:24:38 -07002863 return 0;
2864
2865 io = req->io;
2866 io->rw.iov = io->rw.fast_iov;
2867 req->io = NULL;
Jens Axboebcda7ba2020-02-23 16:42:51 -07002868 ret = io_import_iovec(READ, req, &io->rw.iov, &iter, !force_nonblock);
Jens Axboe3529d8c2019-12-19 18:24:38 -07002869 req->io = io;
2870 if (ret < 0)
2871 return ret;
2872
2873 io_req_map_rw(req, ret, io->rw.iov, io->rw.fast_iov, &iter);
2874 return 0;
Jens Axboef67676d2019-12-02 11:03:47 -07002875}
2876
Jens Axboebcf5a062020-05-22 09:24:42 -06002877static void io_async_buf_cancel(struct callback_head *cb)
2878{
2879 struct io_async_rw *rw;
2880 struct io_kiocb *req;
2881
2882 rw = container_of(cb, struct io_async_rw, task_work);
2883 req = rw->wpq.wait.private;
Jens Axboec40f6372020-06-25 15:39:59 -06002884 __io_req_task_cancel(req, -ECANCELED);
Jens Axboebcf5a062020-05-22 09:24:42 -06002885}
2886
2887static void io_async_buf_retry(struct callback_head *cb)
2888{
2889 struct io_async_rw *rw;
Jens Axboebcf5a062020-05-22 09:24:42 -06002890 struct io_kiocb *req;
2891
2892 rw = container_of(cb, struct io_async_rw, task_work);
2893 req = rw->wpq.wait.private;
Jens Axboebcf5a062020-05-22 09:24:42 -06002894
Jens Axboec40f6372020-06-25 15:39:59 -06002895 __io_req_task_submit(req);
Jens Axboebcf5a062020-05-22 09:24:42 -06002896}
2897
2898static int io_async_buf_func(struct wait_queue_entry *wait, unsigned mode,
2899 int sync, void *arg)
2900{
2901 struct wait_page_queue *wpq;
2902 struct io_kiocb *req = wait->private;
2903 struct io_async_rw *rw = &req->io->rw;
2904 struct wait_page_key *key = arg;
2905 struct task_struct *tsk;
2906 int ret;
2907
2908 wpq = container_of(wait, struct wait_page_queue, wait);
2909
2910 ret = wake_page_match(wpq, key);
2911 if (ret != 1)
2912 return ret;
2913
2914 list_del_init(&wait->entry);
2915
2916 init_task_work(&rw->task_work, io_async_buf_retry);
2917 /* submit ref gets dropped, acquire a new one */
2918 refcount_inc(&req->refs);
2919 tsk = req->task;
2920 ret = task_work_add(tsk, &rw->task_work, true);
2921 if (unlikely(ret)) {
2922 /* queue just for cancelation */
2923 init_task_work(&rw->task_work, io_async_buf_cancel);
2924 tsk = io_wq_get_task(req->ctx->io_wq);
2925 task_work_add(tsk, &rw->task_work, true);
2926 }
2927 wake_up_process(tsk);
2928 return 1;
2929}
2930
2931static bool io_rw_should_retry(struct io_kiocb *req)
2932{
2933 struct kiocb *kiocb = &req->rw.kiocb;
2934 int ret;
2935
2936 /* never retry for NOWAIT, we just complete with -EAGAIN */
2937 if (req->flags & REQ_F_NOWAIT)
2938 return false;
2939
2940 /* already tried, or we're doing O_DIRECT */
2941 if (kiocb->ki_flags & (IOCB_DIRECT | IOCB_WAITQ))
2942 return false;
2943 /*
2944 * just use poll if we can, and don't attempt if the fs doesn't
2945 * support callback based unlocks
2946 */
2947 if (file_can_poll(req->file) || !(req->file->f_mode & FMODE_BUF_RASYNC))
2948 return false;
2949
2950 /*
2951 * If request type doesn't require req->io to defer in general,
2952 * we need to allocate it here
2953 */
2954 if (!req->io && __io_alloc_async_ctx(req))
2955 return false;
2956
2957 ret = kiocb_wait_page_queue_init(kiocb, &req->io->rw.wpq,
2958 io_async_buf_func, req);
2959 if (!ret) {
2960 io_get_req_task(req);
2961 return true;
2962 }
2963
2964 return false;
2965}
2966
2967static int io_iter_do_read(struct io_kiocb *req, struct iov_iter *iter)
2968{
2969 if (req->file->f_op->read_iter)
2970 return call_read_iter(req->file, &req->rw.kiocb, iter);
2971 return loop_rw_iter(READ, req->file, &req->rw.kiocb, iter);
2972}
2973
Jens Axboea1d7c392020-06-22 11:09:46 -06002974static int io_read(struct io_kiocb *req, bool force_nonblock,
2975 struct io_comp_state *cs)
Jens Axboe2b188cc2019-01-07 10:46:33 -07002976{
2977 struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
Jens Axboe9adbd452019-12-20 08:45:55 -07002978 struct kiocb *kiocb = &req->rw.kiocb;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002979 struct iov_iter iter;
Jens Axboe31b51512019-01-18 22:56:34 -07002980 size_t iov_count;
Jens Axboef67676d2019-12-02 11:03:47 -07002981 ssize_t io_size, ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002982
Jens Axboebcda7ba2020-02-23 16:42:51 -07002983 ret = io_import_iovec(READ, req, &iovec, &iter, !force_nonblock);
Jens Axboe06b76d42019-12-19 14:44:26 -07002984 if (ret < 0)
2985 return ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002986
Jens Axboefd6c2e42019-12-18 12:19:41 -07002987 /* Ensure we clear previously set non-block flag */
2988 if (!force_nonblock)
Jens Axboe29de5f62020-02-20 09:56:08 -07002989 kiocb->ki_flags &= ~IOCB_NOWAIT;
Jens Axboefd6c2e42019-12-18 12:19:41 -07002990
Jens Axboef67676d2019-12-02 11:03:47 -07002991 io_size = ret;
Pavel Begunkov6795c5a2020-06-28 12:52:35 +03002992 req->result = io_size;
Jens Axboef67676d2019-12-02 11:03:47 -07002993
Pavel Begunkov24c74672020-06-21 13:09:51 +03002994 /* If the file doesn't support async, just async punt */
Jens Axboeaf197f52020-04-28 13:15:06 -06002995 if (force_nonblock && !io_file_supports_async(req->file, READ))
Jens Axboef67676d2019-12-02 11:03:47 -07002996 goto copy_iov;
Jens Axboe9e645e112019-05-10 16:07:28 -06002997
Jens Axboe31b51512019-01-18 22:56:34 -07002998 iov_count = iov_iter_count(&iter);
Jens Axboe9adbd452019-12-20 08:45:55 -07002999 ret = rw_verify_area(READ, req->file, &kiocb->ki_pos, iov_count);
Jens Axboe2b188cc2019-01-07 10:46:33 -07003000 if (!ret) {
Jens Axboeb63534c2020-06-04 11:28:00 -06003001 unsigned long nr_segs = iter.nr_segs;
Jens Axboe4503b762020-06-01 10:00:27 -06003002 ssize_t ret2 = 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003003
Jens Axboebcf5a062020-05-22 09:24:42 -06003004 ret2 = io_iter_do_read(req, &iter);
Jens Axboe32960612019-09-23 11:05:34 -06003005
Jens Axboe9d93a3f2019-05-15 13:53:07 -06003006 /* Catch -EAGAIN return for forced non-blocking submission */
Jens Axboe4503b762020-06-01 10:00:27 -06003007 if (!force_nonblock || (ret2 != -EAGAIN && ret2 != -EIO)) {
Jens Axboea1d7c392020-06-22 11:09:46 -06003008 kiocb_done(kiocb, ret2, cs);
Jens Axboef67676d2019-12-02 11:03:47 -07003009 } else {
Jens Axboeb63534c2020-06-04 11:28:00 -06003010 iter.count = iov_count;
3011 iter.nr_segs = nr_segs;
Jens Axboef67676d2019-12-02 11:03:47 -07003012copy_iov:
Jens Axboeb7bb4f72019-12-15 22:13:43 -07003013 ret = io_setup_async_rw(req, io_size, iovec,
Jens Axboef67676d2019-12-02 11:03:47 -07003014 inline_vecs, &iter);
3015 if (ret)
3016 goto out_free;
Jens Axboebcf5a062020-05-22 09:24:42 -06003017 /* if we can retry, do so with the callbacks armed */
3018 if (io_rw_should_retry(req)) {
3019 ret2 = io_iter_do_read(req, &iter);
3020 if (ret2 == -EIOCBQUEUED) {
3021 goto out_free;
3022 } else if (ret2 != -EAGAIN) {
Jens Axboea1d7c392020-06-22 11:09:46 -06003023 kiocb_done(kiocb, ret2, cs);
Jens Axboebcf5a062020-05-22 09:24:42 -06003024 goto out_free;
3025 }
3026 }
3027 kiocb->ki_flags &= ~IOCB_WAITQ;
Jens Axboef67676d2019-12-02 11:03:47 -07003028 return -EAGAIN;
3029 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07003030 }
Jens Axboef67676d2019-12-02 11:03:47 -07003031out_free:
Xiaoguang Wang6f2cc162020-06-18 15:01:56 +08003032 if (!(req->flags & REQ_F_NEED_CLEANUP))
3033 kfree(iovec);
Jens Axboe2b188cc2019-01-07 10:46:33 -07003034 return ret;
3035}
3036
Jens Axboe3529d8c2019-12-19 18:24:38 -07003037static int io_write_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe,
3038 bool force_nonblock)
Jens Axboef67676d2019-12-02 11:03:47 -07003039{
Jens Axboe3529d8c2019-12-19 18:24:38 -07003040 struct io_async_ctx *io;
3041 struct iov_iter iter;
Jens Axboef67676d2019-12-02 11:03:47 -07003042 ssize_t ret;
3043
Jens Axboe3529d8c2019-12-19 18:24:38 -07003044 ret = io_prep_rw(req, sqe, force_nonblock);
3045 if (ret)
3046 return ret;
Jens Axboef67676d2019-12-02 11:03:47 -07003047
Jens Axboe3529d8c2019-12-19 18:24:38 -07003048 if (unlikely(!(req->file->f_mode & FMODE_WRITE)))
3049 return -EBADF;
Jens Axboef67676d2019-12-02 11:03:47 -07003050
Jens Axboe4ed734b2020-03-20 11:23:41 -06003051 req->fsize = rlimit(RLIMIT_FSIZE);
3052
Pavel Begunkov5f798be2020-02-08 13:28:02 +03003053 /* either don't need iovec imported or already have it */
3054 if (!req->io || req->flags & REQ_F_NEED_CLEANUP)
Jens Axboe3529d8c2019-12-19 18:24:38 -07003055 return 0;
3056
3057 io = req->io;
3058 io->rw.iov = io->rw.fast_iov;
3059 req->io = NULL;
Jens Axboebcda7ba2020-02-23 16:42:51 -07003060 ret = io_import_iovec(WRITE, req, &io->rw.iov, &iter, !force_nonblock);
Jens Axboe3529d8c2019-12-19 18:24:38 -07003061 req->io = io;
3062 if (ret < 0)
3063 return ret;
3064
3065 io_req_map_rw(req, ret, io->rw.iov, io->rw.fast_iov, &iter);
3066 return 0;
Jens Axboef67676d2019-12-02 11:03:47 -07003067}
3068
Jens Axboea1d7c392020-06-22 11:09:46 -06003069static int io_write(struct io_kiocb *req, bool force_nonblock,
3070 struct io_comp_state *cs)
Jens Axboe2b188cc2019-01-07 10:46:33 -07003071{
3072 struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
Jens Axboe9adbd452019-12-20 08:45:55 -07003073 struct kiocb *kiocb = &req->rw.kiocb;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003074 struct iov_iter iter;
Jens Axboe31b51512019-01-18 22:56:34 -07003075 size_t iov_count;
Jens Axboef67676d2019-12-02 11:03:47 -07003076 ssize_t ret, io_size;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003077
Jens Axboebcda7ba2020-02-23 16:42:51 -07003078 ret = io_import_iovec(WRITE, req, &iovec, &iter, !force_nonblock);
Jens Axboe06b76d42019-12-19 14:44:26 -07003079 if (ret < 0)
3080 return ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003081
Jens Axboefd6c2e42019-12-18 12:19:41 -07003082 /* Ensure we clear previously set non-block flag */
3083 if (!force_nonblock)
Jens Axboe9adbd452019-12-20 08:45:55 -07003084 req->rw.kiocb.ki_flags &= ~IOCB_NOWAIT;
Jens Axboefd6c2e42019-12-18 12:19:41 -07003085
Jens Axboef67676d2019-12-02 11:03:47 -07003086 io_size = ret;
Pavel Begunkov6795c5a2020-06-28 12:52:35 +03003087 req->result = io_size;
Jens Axboef67676d2019-12-02 11:03:47 -07003088
Pavel Begunkov24c74672020-06-21 13:09:51 +03003089 /* If the file doesn't support async, just async punt */
Jens Axboeaf197f52020-04-28 13:15:06 -06003090 if (force_nonblock && !io_file_supports_async(req->file, WRITE))
Jens Axboef67676d2019-12-02 11:03:47 -07003091 goto copy_iov;
Jens Axboef67676d2019-12-02 11:03:47 -07003092
Jens Axboe10d59342019-12-09 20:16:22 -07003093 /* file path doesn't support NOWAIT for non-direct_IO */
3094 if (force_nonblock && !(kiocb->ki_flags & IOCB_DIRECT) &&
3095 (req->flags & REQ_F_ISREG))
Jens Axboef67676d2019-12-02 11:03:47 -07003096 goto copy_iov;
Jens Axboe9e645e112019-05-10 16:07:28 -06003097
Jens Axboe31b51512019-01-18 22:56:34 -07003098 iov_count = iov_iter_count(&iter);
Jens Axboe9adbd452019-12-20 08:45:55 -07003099 ret = rw_verify_area(WRITE, req->file, &kiocb->ki_pos, iov_count);
Jens Axboe2b188cc2019-01-07 10:46:33 -07003100 if (!ret) {
Jens Axboeb63534c2020-06-04 11:28:00 -06003101 unsigned long nr_segs = iter.nr_segs;
Roman Penyaev9bf79332019-03-25 20:09:24 +01003102 ssize_t ret2;
3103
Jens Axboe2b188cc2019-01-07 10:46:33 -07003104 /*
3105 * Open-code file_start_write here to grab freeze protection,
3106 * which will be released by another thread in
3107 * io_complete_rw(). Fool lockdep by telling it the lock got
3108 * released so that it doesn't complain about the held lock when
3109 * we return to userspace.
3110 */
Jens Axboe491381ce2019-10-17 09:20:46 -06003111 if (req->flags & REQ_F_ISREG) {
Jens Axboe9adbd452019-12-20 08:45:55 -07003112 __sb_start_write(file_inode(req->file)->i_sb,
Jens Axboe2b188cc2019-01-07 10:46:33 -07003113 SB_FREEZE_WRITE, true);
Jens Axboe9adbd452019-12-20 08:45:55 -07003114 __sb_writers_release(file_inode(req->file)->i_sb,
Jens Axboe2b188cc2019-01-07 10:46:33 -07003115 SB_FREEZE_WRITE);
3116 }
3117 kiocb->ki_flags |= IOCB_WRITE;
Roman Penyaev9bf79332019-03-25 20:09:24 +01003118
Jens Axboe4ed734b2020-03-20 11:23:41 -06003119 if (!force_nonblock)
3120 current->signal->rlim[RLIMIT_FSIZE].rlim_cur = req->fsize;
3121
Jens Axboe9adbd452019-12-20 08:45:55 -07003122 if (req->file->f_op->write_iter)
3123 ret2 = call_write_iter(req->file, kiocb, &iter);
Jens Axboe32960612019-09-23 11:05:34 -06003124 else
Jens Axboe9adbd452019-12-20 08:45:55 -07003125 ret2 = loop_rw_iter(WRITE, req->file, kiocb, &iter);
Jens Axboe4ed734b2020-03-20 11:23:41 -06003126
3127 if (!force_nonblock)
3128 current->signal->rlim[RLIMIT_FSIZE].rlim_cur = RLIM_INFINITY;
3129
Jens Axboefaac9962020-02-07 15:45:22 -07003130 /*
Chucheng Luobff60352020-03-25 11:31:38 +08003131 * Raw bdev writes will return -EOPNOTSUPP for IOCB_NOWAIT. Just
Jens Axboefaac9962020-02-07 15:45:22 -07003132 * retry them without IOCB_NOWAIT.
3133 */
3134 if (ret2 == -EOPNOTSUPP && (kiocb->ki_flags & IOCB_NOWAIT))
3135 ret2 = -EAGAIN;
Jens Axboef67676d2019-12-02 11:03:47 -07003136 if (!force_nonblock || ret2 != -EAGAIN) {
Jens Axboea1d7c392020-06-22 11:09:46 -06003137 kiocb_done(kiocb, ret2, cs);
Jens Axboef67676d2019-12-02 11:03:47 -07003138 } else {
Jens Axboeb63534c2020-06-04 11:28:00 -06003139 iter.count = iov_count;
3140 iter.nr_segs = nr_segs;
Jens Axboef67676d2019-12-02 11:03:47 -07003141copy_iov:
Jens Axboeb7bb4f72019-12-15 22:13:43 -07003142 ret = io_setup_async_rw(req, io_size, iovec,
Jens Axboef67676d2019-12-02 11:03:47 -07003143 inline_vecs, &iter);
3144 if (ret)
3145 goto out_free;
3146 return -EAGAIN;
3147 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07003148 }
Jens Axboe31b51512019-01-18 22:56:34 -07003149out_free:
Xiaoguang Wang6f2cc162020-06-18 15:01:56 +08003150 if (!(req->flags & REQ_F_NEED_CLEANUP))
3151 kfree(iovec);
Jens Axboe2b188cc2019-01-07 10:46:33 -07003152 return ret;
3153}
3154
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03003155static int __io_splice_prep(struct io_kiocb *req,
3156 const struct io_uring_sqe *sqe)
Pavel Begunkov7d67af22020-02-24 11:32:45 +03003157{
3158 struct io_splice* sp = &req->splice;
3159 unsigned int valid_flags = SPLICE_F_FD_IN_FIXED | SPLICE_F_ALL;
3160 int ret;
3161
3162 if (req->flags & REQ_F_NEED_CLEANUP)
3163 return 0;
Pavel Begunkov3232dd02020-06-03 18:03:22 +03003164 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3165 return -EINVAL;
Pavel Begunkov7d67af22020-02-24 11:32:45 +03003166
3167 sp->file_in = NULL;
Pavel Begunkov7d67af22020-02-24 11:32:45 +03003168 sp->len = READ_ONCE(sqe->len);
3169 sp->flags = READ_ONCE(sqe->splice_flags);
3170
3171 if (unlikely(sp->flags & ~valid_flags))
3172 return -EINVAL;
3173
3174 ret = io_file_get(NULL, req, READ_ONCE(sqe->splice_fd_in), &sp->file_in,
3175 (sp->flags & SPLICE_F_FD_IN_FIXED));
3176 if (ret)
3177 return ret;
3178 req->flags |= REQ_F_NEED_CLEANUP;
3179
Xiaoguang Wang7cdaf582020-06-10 19:41:19 +08003180 if (!S_ISREG(file_inode(sp->file_in)->i_mode)) {
3181 /*
3182 * Splice operation will be punted aync, and here need to
3183 * modify io_wq_work.flags, so initialize io_wq_work firstly.
3184 */
3185 io_req_init_async(req);
Pavel Begunkov7d67af22020-02-24 11:32:45 +03003186 req->work.flags |= IO_WQ_WORK_UNBOUND;
Xiaoguang Wang7cdaf582020-06-10 19:41:19 +08003187 }
Pavel Begunkov7d67af22020-02-24 11:32:45 +03003188
3189 return 0;
3190}
3191
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03003192static int io_tee_prep(struct io_kiocb *req,
3193 const struct io_uring_sqe *sqe)
3194{
3195 if (READ_ONCE(sqe->splice_off_in) || READ_ONCE(sqe->off))
3196 return -EINVAL;
3197 return __io_splice_prep(req, sqe);
3198}
3199
3200static int io_tee(struct io_kiocb *req, bool force_nonblock)
3201{
3202 struct io_splice *sp = &req->splice;
3203 struct file *in = sp->file_in;
3204 struct file *out = sp->file_out;
3205 unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED;
3206 long ret = 0;
3207
3208 if (force_nonblock)
3209 return -EAGAIN;
3210 if (sp->len)
3211 ret = do_tee(in, out, sp->len, flags);
3212
3213 io_put_file(req, in, (sp->flags & SPLICE_F_FD_IN_FIXED));
3214 req->flags &= ~REQ_F_NEED_CLEANUP;
3215
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03003216 if (ret != sp->len)
3217 req_set_fail_links(req);
Jens Axboee1e16092020-06-22 09:17:17 -06003218 io_req_complete(req, ret);
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03003219 return 0;
3220}
3221
3222static int io_splice_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
3223{
3224 struct io_splice* sp = &req->splice;
3225
3226 sp->off_in = READ_ONCE(sqe->splice_off_in);
3227 sp->off_out = READ_ONCE(sqe->off);
3228 return __io_splice_prep(req, sqe);
3229}
3230
Pavel Begunkov014db002020-03-03 21:33:12 +03003231static int io_splice(struct io_kiocb *req, bool force_nonblock)
Pavel Begunkov7d67af22020-02-24 11:32:45 +03003232{
3233 struct io_splice *sp = &req->splice;
3234 struct file *in = sp->file_in;
3235 struct file *out = sp->file_out;
3236 unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED;
3237 loff_t *poff_in, *poff_out;
Pavel Begunkovc9687422020-05-04 23:00:54 +03003238 long ret = 0;
Pavel Begunkov7d67af22020-02-24 11:32:45 +03003239
Pavel Begunkov2fb3e822020-05-01 17:09:38 +03003240 if (force_nonblock)
3241 return -EAGAIN;
Pavel Begunkov7d67af22020-02-24 11:32:45 +03003242
3243 poff_in = (sp->off_in == -1) ? NULL : &sp->off_in;
3244 poff_out = (sp->off_out == -1) ? NULL : &sp->off_out;
Pavel Begunkovc9687422020-05-04 23:00:54 +03003245
Jens Axboe948a7742020-05-17 14:21:38 -06003246 if (sp->len)
Pavel Begunkovc9687422020-05-04 23:00:54 +03003247 ret = do_splice(in, poff_in, out, poff_out, sp->len, flags);
Pavel Begunkov7d67af22020-02-24 11:32:45 +03003248
3249 io_put_file(req, in, (sp->flags & SPLICE_F_FD_IN_FIXED));
3250 req->flags &= ~REQ_F_NEED_CLEANUP;
3251
Pavel Begunkov7d67af22020-02-24 11:32:45 +03003252 if (ret != sp->len)
3253 req_set_fail_links(req);
Jens Axboee1e16092020-06-22 09:17:17 -06003254 io_req_complete(req, ret);
Pavel Begunkov7d67af22020-02-24 11:32:45 +03003255 return 0;
3256}
3257
Jens Axboe2b188cc2019-01-07 10:46:33 -07003258/*
3259 * IORING_OP_NOP just posts a completion event, nothing else.
3260 */
Jens Axboe229a7b62020-06-22 10:13:11 -06003261static int io_nop(struct io_kiocb *req, struct io_comp_state *cs)
Jens Axboe2b188cc2019-01-07 10:46:33 -07003262{
3263 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003264
Jens Axboedef596e2019-01-09 08:59:42 -07003265 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
3266 return -EINVAL;
3267
Jens Axboe229a7b62020-06-22 10:13:11 -06003268 __io_req_complete(req, 0, 0, cs);
Jens Axboe2b188cc2019-01-07 10:46:33 -07003269 return 0;
3270}
3271
Jens Axboe3529d8c2019-12-19 18:24:38 -07003272static int io_prep_fsync(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Christoph Hellwigc992fe22019-01-11 09:43:02 -07003273{
Jens Axboe6b063142019-01-10 22:13:58 -07003274 struct io_ring_ctx *ctx = req->ctx;
Christoph Hellwigc992fe22019-01-11 09:43:02 -07003275
Jens Axboe09bb8392019-03-13 12:39:28 -06003276 if (!req->file)
3277 return -EBADF;
Christoph Hellwigc992fe22019-01-11 09:43:02 -07003278
Jens Axboe6b063142019-01-10 22:13:58 -07003279 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
Jens Axboedef596e2019-01-09 08:59:42 -07003280 return -EINVAL;
Jens Axboeedafcce2019-01-09 09:16:05 -07003281 if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index))
Christoph Hellwigc992fe22019-01-11 09:43:02 -07003282 return -EINVAL;
3283
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07003284 req->sync.flags = READ_ONCE(sqe->fsync_flags);
3285 if (unlikely(req->sync.flags & ~IORING_FSYNC_DATASYNC))
3286 return -EINVAL;
3287
3288 req->sync.off = READ_ONCE(sqe->off);
3289 req->sync.len = READ_ONCE(sqe->len);
Christoph Hellwigc992fe22019-01-11 09:43:02 -07003290 return 0;
3291}
3292
Pavel Begunkovac45abc2020-06-08 21:08:18 +03003293static int io_fsync(struct io_kiocb *req, bool force_nonblock)
Jens Axboe78912932020-01-14 22:09:06 -07003294{
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07003295 loff_t end = req->sync.off + req->sync.len;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07003296 int ret;
3297
Pavel Begunkovac45abc2020-06-08 21:08:18 +03003298 /* fsync always requires a blocking context */
3299 if (force_nonblock)
3300 return -EAGAIN;
3301
Jens Axboe9adbd452019-12-20 08:45:55 -07003302 ret = vfs_fsync_range(req->file, req->sync.off,
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07003303 end > 0 ? end : LLONG_MAX,
3304 req->sync.flags & IORING_FSYNC_DATASYNC);
3305 if (ret < 0)
3306 req_set_fail_links(req);
Jens Axboee1e16092020-06-22 09:17:17 -06003307 io_req_complete(req, ret);
Christoph Hellwigc992fe22019-01-11 09:43:02 -07003308 return 0;
3309}
3310
Jens Axboed63d1b52019-12-10 10:38:56 -07003311static int io_fallocate_prep(struct io_kiocb *req,
3312 const struct io_uring_sqe *sqe)
3313{
3314 if (sqe->ioprio || sqe->buf_index || sqe->rw_flags)
3315 return -EINVAL;
Pavel Begunkov3232dd02020-06-03 18:03:22 +03003316 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3317 return -EINVAL;
Jens Axboed63d1b52019-12-10 10:38:56 -07003318
3319 req->sync.off = READ_ONCE(sqe->off);
3320 req->sync.len = READ_ONCE(sqe->addr);
3321 req->sync.mode = READ_ONCE(sqe->len);
Jens Axboe4ed734b2020-03-20 11:23:41 -06003322 req->fsize = rlimit(RLIMIT_FSIZE);
Jens Axboed63d1b52019-12-10 10:38:56 -07003323 return 0;
3324}
3325
Pavel Begunkov014db002020-03-03 21:33:12 +03003326static int io_fallocate(struct io_kiocb *req, bool force_nonblock)
Jens Axboed63d1b52019-12-10 10:38:56 -07003327{
Pavel Begunkovac45abc2020-06-08 21:08:18 +03003328 int ret;
Jens Axboed63d1b52019-12-10 10:38:56 -07003329
Pavel Begunkovac45abc2020-06-08 21:08:18 +03003330 /* fallocate always requiring blocking context */
3331 if (force_nonblock)
3332 return -EAGAIN;
3333
3334 current->signal->rlim[RLIMIT_FSIZE].rlim_cur = req->fsize;
3335 ret = vfs_fallocate(req->file, req->sync.mode, req->sync.off,
3336 req->sync.len);
3337 current->signal->rlim[RLIMIT_FSIZE].rlim_cur = RLIM_INFINITY;
3338 if (ret < 0)
3339 req_set_fail_links(req);
Jens Axboee1e16092020-06-22 09:17:17 -06003340 io_req_complete(req, ret);
Jens Axboed63d1b52019-12-10 10:38:56 -07003341 return 0;
3342}
3343
Pavel Begunkovec65fea2020-06-03 18:03:24 +03003344static int __io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboe15b71ab2019-12-11 11:20:36 -07003345{
Jens Axboef8748882020-01-08 17:47:02 -07003346 const char __user *fname;
Jens Axboe15b71ab2019-12-11 11:20:36 -07003347 int ret;
3348
Pavel Begunkov3232dd02020-06-03 18:03:22 +03003349 if (unlikely(req->ctx->flags & (IORING_SETUP_IOPOLL|IORING_SETUP_SQPOLL)))
Jens Axboe15b71ab2019-12-11 11:20:36 -07003350 return -EINVAL;
Pavel Begunkovec65fea2020-06-03 18:03:24 +03003351 if (unlikely(sqe->ioprio || sqe->buf_index))
Jens Axboe15b71ab2019-12-11 11:20:36 -07003352 return -EINVAL;
Pavel Begunkovec65fea2020-06-03 18:03:24 +03003353 if (unlikely(req->flags & REQ_F_FIXED_FILE))
Jens Axboecf3040c2020-02-06 21:31:40 -07003354 return -EBADF;
Jens Axboe15b71ab2019-12-11 11:20:36 -07003355
Pavel Begunkovec65fea2020-06-03 18:03:24 +03003356 /* open.how should be already initialised */
3357 if (!(req->open.how.flags & O_PATH) && force_o_largefile())
Jens Axboe08a1d26eb2020-04-08 09:20:54 -06003358 req->open.how.flags |= O_LARGEFILE;
Jens Axboe15b71ab2019-12-11 11:20:36 -07003359
Pavel Begunkov25e72d12020-06-03 18:03:23 +03003360 req->open.dfd = READ_ONCE(sqe->fd);
3361 fname = u64_to_user_ptr(READ_ONCE(sqe->addr));
Jens Axboef8748882020-01-08 17:47:02 -07003362 req->open.filename = getname(fname);
Jens Axboe15b71ab2019-12-11 11:20:36 -07003363 if (IS_ERR(req->open.filename)) {
3364 ret = PTR_ERR(req->open.filename);
3365 req->open.filename = NULL;
3366 return ret;
3367 }
Jens Axboe4022e7a2020-03-19 19:23:18 -06003368 req->open.nofile = rlimit(RLIMIT_NOFILE);
Pavel Begunkov8fef80b2020-02-07 23:59:53 +03003369 req->flags |= REQ_F_NEED_CLEANUP;
Jens Axboe15b71ab2019-12-11 11:20:36 -07003370 return 0;
3371}
3372
Pavel Begunkovec65fea2020-06-03 18:03:24 +03003373static int io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
3374{
3375 u64 flags, mode;
3376
3377 if (req->flags & REQ_F_NEED_CLEANUP)
3378 return 0;
3379 mode = READ_ONCE(sqe->len);
3380 flags = READ_ONCE(sqe->open_flags);
3381 req->open.how = build_open_how(flags, mode);
3382 return __io_openat_prep(req, sqe);
3383}
3384
Jens Axboecebdb982020-01-08 17:59:24 -07003385static int io_openat2_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
3386{
3387 struct open_how __user *how;
Jens Axboecebdb982020-01-08 17:59:24 -07003388 size_t len;
3389 int ret;
3390
Pavel Begunkov0bdbdd02020-02-08 13:28:03 +03003391 if (req->flags & REQ_F_NEED_CLEANUP)
3392 return 0;
Jens Axboecebdb982020-01-08 17:59:24 -07003393 how = u64_to_user_ptr(READ_ONCE(sqe->addr2));
3394 len = READ_ONCE(sqe->len);
Jens Axboecebdb982020-01-08 17:59:24 -07003395 if (len < OPEN_HOW_SIZE_VER0)
3396 return -EINVAL;
3397
3398 ret = copy_struct_from_user(&req->open.how, sizeof(req->open.how), how,
3399 len);
3400 if (ret)
3401 return ret;
3402
Pavel Begunkovec65fea2020-06-03 18:03:24 +03003403 return __io_openat_prep(req, sqe);
Jens Axboecebdb982020-01-08 17:59:24 -07003404}
3405
Pavel Begunkov014db002020-03-03 21:33:12 +03003406static int io_openat2(struct io_kiocb *req, bool force_nonblock)
Jens Axboe15b71ab2019-12-11 11:20:36 -07003407{
3408 struct open_flags op;
Jens Axboe15b71ab2019-12-11 11:20:36 -07003409 struct file *file;
3410 int ret;
3411
Jens Axboef86cd202020-01-29 13:46:44 -07003412 if (force_nonblock)
Jens Axboe15b71ab2019-12-11 11:20:36 -07003413 return -EAGAIN;
Jens Axboe15b71ab2019-12-11 11:20:36 -07003414
Jens Axboecebdb982020-01-08 17:59:24 -07003415 ret = build_open_flags(&req->open.how, &op);
Jens Axboe15b71ab2019-12-11 11:20:36 -07003416 if (ret)
3417 goto err;
3418
Jens Axboe4022e7a2020-03-19 19:23:18 -06003419 ret = __get_unused_fd_flags(req->open.how.flags, req->open.nofile);
Jens Axboe15b71ab2019-12-11 11:20:36 -07003420 if (ret < 0)
3421 goto err;
3422
3423 file = do_filp_open(req->open.dfd, req->open.filename, &op);
3424 if (IS_ERR(file)) {
3425 put_unused_fd(ret);
3426 ret = PTR_ERR(file);
3427 } else {
3428 fsnotify_open(file);
3429 fd_install(ret, file);
3430 }
3431err:
3432 putname(req->open.filename);
Pavel Begunkov8fef80b2020-02-07 23:59:53 +03003433 req->flags &= ~REQ_F_NEED_CLEANUP;
Jens Axboe15b71ab2019-12-11 11:20:36 -07003434 if (ret < 0)
3435 req_set_fail_links(req);
Jens Axboee1e16092020-06-22 09:17:17 -06003436 io_req_complete(req, ret);
Jens Axboe15b71ab2019-12-11 11:20:36 -07003437 return 0;
3438}
3439
Pavel Begunkov014db002020-03-03 21:33:12 +03003440static int io_openat(struct io_kiocb *req, bool force_nonblock)
Jens Axboecebdb982020-01-08 17:59:24 -07003441{
Pavel Begunkov014db002020-03-03 21:33:12 +03003442 return io_openat2(req, force_nonblock);
Jens Axboecebdb982020-01-08 17:59:24 -07003443}
3444
Jens Axboe067524e2020-03-02 16:32:28 -07003445static int io_remove_buffers_prep(struct io_kiocb *req,
3446 const struct io_uring_sqe *sqe)
3447{
3448 struct io_provide_buf *p = &req->pbuf;
3449 u64 tmp;
3450
3451 if (sqe->ioprio || sqe->rw_flags || sqe->addr || sqe->len || sqe->off)
3452 return -EINVAL;
3453
3454 tmp = READ_ONCE(sqe->fd);
3455 if (!tmp || tmp > USHRT_MAX)
3456 return -EINVAL;
3457
3458 memset(p, 0, sizeof(*p));
3459 p->nbufs = tmp;
3460 p->bgid = READ_ONCE(sqe->buf_group);
3461 return 0;
3462}
3463
3464static int __io_remove_buffers(struct io_ring_ctx *ctx, struct io_buffer *buf,
3465 int bgid, unsigned nbufs)
3466{
3467 unsigned i = 0;
3468
3469 /* shouldn't happen */
3470 if (!nbufs)
3471 return 0;
3472
3473 /* the head kbuf is the list itself */
3474 while (!list_empty(&buf->list)) {
3475 struct io_buffer *nxt;
3476
3477 nxt = list_first_entry(&buf->list, struct io_buffer, list);
3478 list_del(&nxt->list);
3479 kfree(nxt);
3480 if (++i == nbufs)
3481 return i;
3482 }
3483 i++;
3484 kfree(buf);
3485 idr_remove(&ctx->io_buffer_idr, bgid);
3486
3487 return i;
3488}
3489
Jens Axboe229a7b62020-06-22 10:13:11 -06003490static int io_remove_buffers(struct io_kiocb *req, bool force_nonblock,
3491 struct io_comp_state *cs)
Jens Axboe067524e2020-03-02 16:32:28 -07003492{
3493 struct io_provide_buf *p = &req->pbuf;
3494 struct io_ring_ctx *ctx = req->ctx;
3495 struct io_buffer *head;
3496 int ret = 0;
3497
3498 io_ring_submit_lock(ctx, !force_nonblock);
3499
3500 lockdep_assert_held(&ctx->uring_lock);
3501
3502 ret = -ENOENT;
3503 head = idr_find(&ctx->io_buffer_idr, p->bgid);
3504 if (head)
3505 ret = __io_remove_buffers(ctx, head, p->bgid, p->nbufs);
3506
3507 io_ring_submit_lock(ctx, !force_nonblock);
3508 if (ret < 0)
3509 req_set_fail_links(req);
Jens Axboe229a7b62020-06-22 10:13:11 -06003510 __io_req_complete(req, ret, 0, cs);
Jens Axboe067524e2020-03-02 16:32:28 -07003511 return 0;
3512}
3513
Jens Axboeddf0322d2020-02-23 16:41:33 -07003514static int io_provide_buffers_prep(struct io_kiocb *req,
3515 const struct io_uring_sqe *sqe)
3516{
3517 struct io_provide_buf *p = &req->pbuf;
3518 u64 tmp;
3519
3520 if (sqe->ioprio || sqe->rw_flags)
3521 return -EINVAL;
3522
3523 tmp = READ_ONCE(sqe->fd);
3524 if (!tmp || tmp > USHRT_MAX)
3525 return -E2BIG;
3526 p->nbufs = tmp;
3527 p->addr = READ_ONCE(sqe->addr);
3528 p->len = READ_ONCE(sqe->len);
3529
Bijan Mottahedehefe68c12020-06-04 18:01:52 -07003530 if (!access_ok(u64_to_user_ptr(p->addr), (p->len * p->nbufs)))
Jens Axboeddf0322d2020-02-23 16:41:33 -07003531 return -EFAULT;
3532
3533 p->bgid = READ_ONCE(sqe->buf_group);
3534 tmp = READ_ONCE(sqe->off);
3535 if (tmp > USHRT_MAX)
3536 return -E2BIG;
3537 p->bid = tmp;
3538 return 0;
3539}
3540
3541static int io_add_buffers(struct io_provide_buf *pbuf, struct io_buffer **head)
3542{
3543 struct io_buffer *buf;
3544 u64 addr = pbuf->addr;
3545 int i, bid = pbuf->bid;
3546
3547 for (i = 0; i < pbuf->nbufs; i++) {
3548 buf = kmalloc(sizeof(*buf), GFP_KERNEL);
3549 if (!buf)
3550 break;
3551
3552 buf->addr = addr;
3553 buf->len = pbuf->len;
3554 buf->bid = bid;
3555 addr += pbuf->len;
3556 bid++;
3557 if (!*head) {
3558 INIT_LIST_HEAD(&buf->list);
3559 *head = buf;
3560 } else {
3561 list_add_tail(&buf->list, &(*head)->list);
3562 }
3563 }
3564
3565 return i ? i : -ENOMEM;
3566}
3567
Jens Axboe229a7b62020-06-22 10:13:11 -06003568static int io_provide_buffers(struct io_kiocb *req, bool force_nonblock,
3569 struct io_comp_state *cs)
Jens Axboeddf0322d2020-02-23 16:41:33 -07003570{
3571 struct io_provide_buf *p = &req->pbuf;
3572 struct io_ring_ctx *ctx = req->ctx;
3573 struct io_buffer *head, *list;
3574 int ret = 0;
3575
3576 io_ring_submit_lock(ctx, !force_nonblock);
3577
3578 lockdep_assert_held(&ctx->uring_lock);
3579
3580 list = head = idr_find(&ctx->io_buffer_idr, p->bgid);
3581
3582 ret = io_add_buffers(p, &head);
3583 if (ret < 0)
3584 goto out;
3585
3586 if (!list) {
3587 ret = idr_alloc(&ctx->io_buffer_idr, head, p->bgid, p->bgid + 1,
3588 GFP_KERNEL);
3589 if (ret < 0) {
Jens Axboe067524e2020-03-02 16:32:28 -07003590 __io_remove_buffers(ctx, head, p->bgid, -1U);
Jens Axboeddf0322d2020-02-23 16:41:33 -07003591 goto out;
3592 }
3593 }
3594out:
3595 io_ring_submit_unlock(ctx, !force_nonblock);
3596 if (ret < 0)
3597 req_set_fail_links(req);
Jens Axboe229a7b62020-06-22 10:13:11 -06003598 __io_req_complete(req, ret, 0, cs);
Jens Axboeddf0322d2020-02-23 16:41:33 -07003599 return 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003600}
3601
Jens Axboe3e4827b2020-01-08 15:18:09 -07003602static int io_epoll_ctl_prep(struct io_kiocb *req,
3603 const struct io_uring_sqe *sqe)
3604{
3605#if defined(CONFIG_EPOLL)
3606 if (sqe->ioprio || sqe->buf_index)
3607 return -EINVAL;
Pavel Begunkov3232dd02020-06-03 18:03:22 +03003608 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3609 return -EINVAL;
Jens Axboe3e4827b2020-01-08 15:18:09 -07003610
3611 req->epoll.epfd = READ_ONCE(sqe->fd);
3612 req->epoll.op = READ_ONCE(sqe->len);
3613 req->epoll.fd = READ_ONCE(sqe->off);
3614
3615 if (ep_op_has_event(req->epoll.op)) {
3616 struct epoll_event __user *ev;
3617
3618 ev = u64_to_user_ptr(READ_ONCE(sqe->addr));
3619 if (copy_from_user(&req->epoll.event, ev, sizeof(*ev)))
3620 return -EFAULT;
3621 }
3622
3623 return 0;
3624#else
3625 return -EOPNOTSUPP;
3626#endif
3627}
3628
Jens Axboe229a7b62020-06-22 10:13:11 -06003629static int io_epoll_ctl(struct io_kiocb *req, bool force_nonblock,
3630 struct io_comp_state *cs)
Jens Axboe3e4827b2020-01-08 15:18:09 -07003631{
3632#if defined(CONFIG_EPOLL)
3633 struct io_epoll *ie = &req->epoll;
3634 int ret;
3635
3636 ret = do_epoll_ctl(ie->epfd, ie->op, ie->fd, &ie->event, force_nonblock);
3637 if (force_nonblock && ret == -EAGAIN)
3638 return -EAGAIN;
3639
3640 if (ret < 0)
3641 req_set_fail_links(req);
Jens Axboe229a7b62020-06-22 10:13:11 -06003642 __io_req_complete(req, ret, 0, cs);
Jens Axboe3e4827b2020-01-08 15:18:09 -07003643 return 0;
3644#else
3645 return -EOPNOTSUPP;
3646#endif
3647}
3648
Jens Axboec1ca7572019-12-25 22:18:28 -07003649static int io_madvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
3650{
3651#if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU)
3652 if (sqe->ioprio || sqe->buf_index || sqe->off)
3653 return -EINVAL;
Pavel Begunkov3232dd02020-06-03 18:03:22 +03003654 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3655 return -EINVAL;
Jens Axboec1ca7572019-12-25 22:18:28 -07003656
3657 req->madvise.addr = READ_ONCE(sqe->addr);
3658 req->madvise.len = READ_ONCE(sqe->len);
3659 req->madvise.advice = READ_ONCE(sqe->fadvise_advice);
3660 return 0;
3661#else
3662 return -EOPNOTSUPP;
3663#endif
3664}
3665
Pavel Begunkov014db002020-03-03 21:33:12 +03003666static int io_madvise(struct io_kiocb *req, bool force_nonblock)
Jens Axboec1ca7572019-12-25 22:18:28 -07003667{
3668#if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU)
3669 struct io_madvise *ma = &req->madvise;
3670 int ret;
3671
3672 if (force_nonblock)
3673 return -EAGAIN;
3674
3675 ret = do_madvise(ma->addr, ma->len, ma->advice);
3676 if (ret < 0)
3677 req_set_fail_links(req);
Jens Axboee1e16092020-06-22 09:17:17 -06003678 io_req_complete(req, ret);
Jens Axboec1ca7572019-12-25 22:18:28 -07003679 return 0;
3680#else
3681 return -EOPNOTSUPP;
3682#endif
3683}
3684
Jens Axboe4840e412019-12-25 22:03:45 -07003685static int io_fadvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
3686{
3687 if (sqe->ioprio || sqe->buf_index || sqe->addr)
3688 return -EINVAL;
Pavel Begunkov3232dd02020-06-03 18:03:22 +03003689 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3690 return -EINVAL;
Jens Axboe4840e412019-12-25 22:03:45 -07003691
3692 req->fadvise.offset = READ_ONCE(sqe->off);
3693 req->fadvise.len = READ_ONCE(sqe->len);
3694 req->fadvise.advice = READ_ONCE(sqe->fadvise_advice);
3695 return 0;
3696}
3697
Pavel Begunkov014db002020-03-03 21:33:12 +03003698static int io_fadvise(struct io_kiocb *req, bool force_nonblock)
Jens Axboe4840e412019-12-25 22:03:45 -07003699{
3700 struct io_fadvise *fa = &req->fadvise;
3701 int ret;
3702
Jens Axboe3e694262020-02-01 09:22:49 -07003703 if (force_nonblock) {
3704 switch (fa->advice) {
3705 case POSIX_FADV_NORMAL:
3706 case POSIX_FADV_RANDOM:
3707 case POSIX_FADV_SEQUENTIAL:
3708 break;
3709 default:
3710 return -EAGAIN;
3711 }
3712 }
Jens Axboe4840e412019-12-25 22:03:45 -07003713
3714 ret = vfs_fadvise(req->file, fa->offset, fa->len, fa->advice);
3715 if (ret < 0)
3716 req_set_fail_links(req);
Jens Axboee1e16092020-06-22 09:17:17 -06003717 io_req_complete(req, ret);
Jens Axboe4840e412019-12-25 22:03:45 -07003718 return 0;
3719}
3720
Jens Axboeeddc7ef2019-12-13 21:18:10 -07003721static int io_statx_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
3722{
Pavel Begunkov3232dd02020-06-03 18:03:22 +03003723 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3724 return -EINVAL;
Jens Axboeeddc7ef2019-12-13 21:18:10 -07003725 if (sqe->ioprio || sqe->buf_index)
3726 return -EINVAL;
Pavel Begunkov9c280f92020-04-08 08:58:46 +03003727 if (req->flags & REQ_F_FIXED_FILE)
Jens Axboecf3040c2020-02-06 21:31:40 -07003728 return -EBADF;
Jens Axboeeddc7ef2019-12-13 21:18:10 -07003729
Bijan Mottahedeh1d9e1282020-05-22 21:31:16 -07003730 req->statx.dfd = READ_ONCE(sqe->fd);
3731 req->statx.mask = READ_ONCE(sqe->len);
Bijan Mottahedehe62753e2020-05-22 21:31:18 -07003732 req->statx.filename = u64_to_user_ptr(READ_ONCE(sqe->addr));
Bijan Mottahedeh1d9e1282020-05-22 21:31:16 -07003733 req->statx.buffer = u64_to_user_ptr(READ_ONCE(sqe->addr2));
3734 req->statx.flags = READ_ONCE(sqe->statx_flags);
Jens Axboeeddc7ef2019-12-13 21:18:10 -07003735
Jens Axboeeddc7ef2019-12-13 21:18:10 -07003736 return 0;
3737}
3738
Pavel Begunkov014db002020-03-03 21:33:12 +03003739static int io_statx(struct io_kiocb *req, bool force_nonblock)
Jens Axboeeddc7ef2019-12-13 21:18:10 -07003740{
Bijan Mottahedeh1d9e1282020-05-22 21:31:16 -07003741 struct io_statx *ctx = &req->statx;
Jens Axboeeddc7ef2019-12-13 21:18:10 -07003742 int ret;
3743
Jens Axboe5b0bbee2020-04-27 10:41:22 -06003744 if (force_nonblock) {
3745 /* only need file table for an actual valid fd */
3746 if (ctx->dfd == -1 || ctx->dfd == AT_FDCWD)
3747 req->flags |= REQ_F_NO_FILE_TABLE;
Jens Axboeeddc7ef2019-12-13 21:18:10 -07003748 return -EAGAIN;
Jens Axboe5b0bbee2020-04-27 10:41:22 -06003749 }
Jens Axboeeddc7ef2019-12-13 21:18:10 -07003750
Bijan Mottahedehe62753e2020-05-22 21:31:18 -07003751 ret = do_statx(ctx->dfd, ctx->filename, ctx->flags, ctx->mask,
3752 ctx->buffer);
Jens Axboeeddc7ef2019-12-13 21:18:10 -07003753
Jens Axboeeddc7ef2019-12-13 21:18:10 -07003754 if (ret < 0)
3755 req_set_fail_links(req);
Jens Axboee1e16092020-06-22 09:17:17 -06003756 io_req_complete(req, ret);
Jens Axboeeddc7ef2019-12-13 21:18:10 -07003757 return 0;
3758}
3759
Jens Axboeb5dba592019-12-11 14:02:38 -07003760static int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
3761{
3762 /*
3763 * If we queue this for async, it must not be cancellable. That would
Xiaoguang Wang7cdaf582020-06-10 19:41:19 +08003764 * leave the 'file' in an undeterminate state, and here need to modify
3765 * io_wq_work.flags, so initialize io_wq_work firstly.
Jens Axboeb5dba592019-12-11 14:02:38 -07003766 */
Xiaoguang Wang7cdaf582020-06-10 19:41:19 +08003767 io_req_init_async(req);
Jens Axboeb5dba592019-12-11 14:02:38 -07003768 req->work.flags |= IO_WQ_WORK_NO_CANCEL;
3769
Pavel Begunkov3232dd02020-06-03 18:03:22 +03003770 if (unlikely(req->ctx->flags & (IORING_SETUP_IOPOLL|IORING_SETUP_SQPOLL)))
3771 return -EINVAL;
Jens Axboeb5dba592019-12-11 14:02:38 -07003772 if (sqe->ioprio || sqe->off || sqe->addr || sqe->len ||
3773 sqe->rw_flags || sqe->buf_index)
3774 return -EINVAL;
Pavel Begunkov9c280f92020-04-08 08:58:46 +03003775 if (req->flags & REQ_F_FIXED_FILE)
Jens Axboecf3040c2020-02-06 21:31:40 -07003776 return -EBADF;
Jens Axboeb5dba592019-12-11 14:02:38 -07003777
3778 req->close.fd = READ_ONCE(sqe->fd);
Jens Axboefd2206e2020-06-02 16:40:47 -06003779 if ((req->file && req->file->f_op == &io_uring_fops) ||
3780 req->close.fd == req->ctx->ring_fd)
3781 return -EBADF;
3782
Pavel Begunkov3af73b22020-06-08 21:08:17 +03003783 req->close.put_file = NULL;
Jens Axboeb5dba592019-12-11 14:02:38 -07003784 return 0;
3785}
3786
Jens Axboe229a7b62020-06-22 10:13:11 -06003787static int io_close(struct io_kiocb *req, bool force_nonblock,
3788 struct io_comp_state *cs)
Jens Axboeb5dba592019-12-11 14:02:38 -07003789{
Pavel Begunkov3af73b22020-06-08 21:08:17 +03003790 struct io_close *close = &req->close;
Jens Axboeb5dba592019-12-11 14:02:38 -07003791 int ret;
3792
Pavel Begunkov3af73b22020-06-08 21:08:17 +03003793 /* might be already done during nonblock submission */
3794 if (!close->put_file) {
3795 ret = __close_fd_get_file(close->fd, &close->put_file);
3796 if (ret < 0)
3797 return (ret == -ENOENT) ? -EBADF : ret;
3798 }
Jens Axboeb5dba592019-12-11 14:02:38 -07003799
3800 /* if the file has a flush method, be safe and punt to async */
Pavel Begunkov3af73b22020-06-08 21:08:17 +03003801 if (close->put_file->f_op->flush && force_nonblock) {
Pavel Begunkov24c74672020-06-21 13:09:51 +03003802 /* was never set, but play safe */
3803 req->flags &= ~REQ_F_NOWAIT;
Pavel Begunkov0bf0eef2020-05-26 20:34:06 +03003804 /* avoid grabbing files - we don't need the files */
Pavel Begunkov24c74672020-06-21 13:09:51 +03003805 req->flags |= REQ_F_NO_FILE_TABLE;
Pavel Begunkov0bf0eef2020-05-26 20:34:06 +03003806 return -EAGAIN;
Pavel Begunkova2100672020-03-02 23:45:16 +03003807 }
Jens Axboeb5dba592019-12-11 14:02:38 -07003808
Pavel Begunkov3af73b22020-06-08 21:08:17 +03003809 /* No ->flush() or already async, safely close from here */
3810 ret = filp_close(close->put_file, req->work.files);
3811 if (ret < 0)
3812 req_set_fail_links(req);
Pavel Begunkov3af73b22020-06-08 21:08:17 +03003813 fput(close->put_file);
3814 close->put_file = NULL;
Jens Axboe229a7b62020-06-22 10:13:11 -06003815 __io_req_complete(req, ret, 0, cs);
Jens Axboe1a417f42020-01-31 17:16:48 -07003816 return 0;
Jens Axboeb5dba592019-12-11 14:02:38 -07003817}
3818
Jens Axboe3529d8c2019-12-19 18:24:38 -07003819static int io_prep_sfr(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboe5d17b4a2019-04-09 14:56:44 -06003820{
3821 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe5d17b4a2019-04-09 14:56:44 -06003822
3823 if (!req->file)
3824 return -EBADF;
Jens Axboe5d17b4a2019-04-09 14:56:44 -06003825
3826 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
3827 return -EINVAL;
3828 if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index))
3829 return -EINVAL;
3830
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07003831 req->sync.off = READ_ONCE(sqe->off);
3832 req->sync.len = READ_ONCE(sqe->len);
3833 req->sync.flags = READ_ONCE(sqe->sync_range_flags);
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07003834 return 0;
3835}
3836
Pavel Begunkovac45abc2020-06-08 21:08:18 +03003837static int io_sync_file_range(struct io_kiocb *req, bool force_nonblock)
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07003838{
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07003839 int ret;
3840
Pavel Begunkovac45abc2020-06-08 21:08:18 +03003841 /* sync_file_range always requires a blocking context */
3842 if (force_nonblock)
3843 return -EAGAIN;
3844
Jens Axboe9adbd452019-12-20 08:45:55 -07003845 ret = sync_file_range(req->file, req->sync.off, req->sync.len,
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07003846 req->sync.flags);
3847 if (ret < 0)
3848 req_set_fail_links(req);
Jens Axboee1e16092020-06-22 09:17:17 -06003849 io_req_complete(req, ret);
Jens Axboe5d17b4a2019-04-09 14:56:44 -06003850 return 0;
3851}
3852
YueHaibing469956e2020-03-04 15:53:52 +08003853#if defined(CONFIG_NET)
Pavel Begunkov02d27d82020-02-28 10:36:36 +03003854static int io_setup_async_msg(struct io_kiocb *req,
3855 struct io_async_msghdr *kmsg)
3856{
3857 if (req->io)
3858 return -EAGAIN;
3859 if (io_alloc_async_ctx(req)) {
3860 if (kmsg->iov != kmsg->fast_iov)
3861 kfree(kmsg->iov);
3862 return -ENOMEM;
3863 }
3864 req->flags |= REQ_F_NEED_CLEANUP;
3865 memcpy(&req->io->msg, kmsg, sizeof(*kmsg));
3866 return -EAGAIN;
3867}
3868
Jens Axboe3529d8c2019-12-19 18:24:38 -07003869static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboeaa1fa282019-04-19 13:38:09 -06003870{
Jens Axboee47293f2019-12-20 08:58:21 -07003871 struct io_sr_msg *sr = &req->sr_msg;
Jens Axboe3529d8c2019-12-19 18:24:38 -07003872 struct io_async_ctx *io = req->io;
Pavel Begunkov99bc4c32020-02-07 22:04:45 +03003873 int ret;
Jens Axboe03b12302019-12-02 18:50:25 -07003874
Pavel Begunkovd2b6f482020-06-03 18:03:25 +03003875 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3876 return -EINVAL;
3877
Jens Axboee47293f2019-12-20 08:58:21 -07003878 sr->msg_flags = READ_ONCE(sqe->msg_flags);
3879 sr->msg = u64_to_user_ptr(READ_ONCE(sqe->addr));
Jens Axboefddafac2020-01-04 20:19:44 -07003880 sr->len = READ_ONCE(sqe->len);
Jens Axboe3529d8c2019-12-19 18:24:38 -07003881
Jens Axboed8768362020-02-27 14:17:49 -07003882#ifdef CONFIG_COMPAT
3883 if (req->ctx->compat)
3884 sr->msg_flags |= MSG_CMSG_COMPAT;
3885#endif
3886
Jens Axboefddafac2020-01-04 20:19:44 -07003887 if (!io || req->opcode == IORING_OP_SEND)
Jens Axboe3529d8c2019-12-19 18:24:38 -07003888 return 0;
Pavel Begunkov5f798be2020-02-08 13:28:02 +03003889 /* iovec is already imported */
3890 if (req->flags & REQ_F_NEED_CLEANUP)
3891 return 0;
Jens Axboe3529d8c2019-12-19 18:24:38 -07003892
Jens Axboed9688562019-12-09 19:35:20 -07003893 io->msg.iov = io->msg.fast_iov;
Pavel Begunkov99bc4c32020-02-07 22:04:45 +03003894 ret = sendmsg_copy_msghdr(&io->msg.msg, sr->msg, sr->msg_flags,
Jens Axboee47293f2019-12-20 08:58:21 -07003895 &io->msg.iov);
Pavel Begunkov99bc4c32020-02-07 22:04:45 +03003896 if (!ret)
3897 req->flags |= REQ_F_NEED_CLEANUP;
3898 return ret;
Jens Axboe03b12302019-12-02 18:50:25 -07003899}
3900
Jens Axboe229a7b62020-06-22 10:13:11 -06003901static int io_sendmsg(struct io_kiocb *req, bool force_nonblock,
3902 struct io_comp_state *cs)
Jens Axboe03b12302019-12-02 18:50:25 -07003903{
Jens Axboe0b416c32019-12-15 10:57:46 -07003904 struct io_async_msghdr *kmsg = NULL;
Jens Axboe03b12302019-12-02 18:50:25 -07003905 struct socket *sock;
3906 int ret;
3907
Jens Axboe03b12302019-12-02 18:50:25 -07003908 sock = sock_from_file(req->file, &ret);
3909 if (sock) {
Jens Axboeb7bb4f72019-12-15 22:13:43 -07003910 struct io_async_ctx io;
Jens Axboe03b12302019-12-02 18:50:25 -07003911 unsigned flags;
3912
Jens Axboe03b12302019-12-02 18:50:25 -07003913 if (req->io) {
Jens Axboe0b416c32019-12-15 10:57:46 -07003914 kmsg = &req->io->msg;
Jens Axboeb5379162020-02-09 11:29:15 -07003915 kmsg->msg.msg_name = &req->io->msg.addr;
Jens Axboe0b416c32019-12-15 10:57:46 -07003916 /* if iov is set, it's allocated already */
3917 if (!kmsg->iov)
3918 kmsg->iov = kmsg->fast_iov;
3919 kmsg->msg.msg_iter.iov = kmsg->iov;
Jens Axboe03b12302019-12-02 18:50:25 -07003920 } else {
Jens Axboe3529d8c2019-12-19 18:24:38 -07003921 struct io_sr_msg *sr = &req->sr_msg;
3922
Jens Axboe0b416c32019-12-15 10:57:46 -07003923 kmsg = &io.msg;
Jens Axboeb5379162020-02-09 11:29:15 -07003924 kmsg->msg.msg_name = &io.msg.addr;
Jens Axboe3529d8c2019-12-19 18:24:38 -07003925
3926 io.msg.iov = io.msg.fast_iov;
3927 ret = sendmsg_copy_msghdr(&io.msg.msg, sr->msg,
3928 sr->msg_flags, &io.msg.iov);
Jens Axboe03b12302019-12-02 18:50:25 -07003929 if (ret)
Jens Axboe3529d8c2019-12-19 18:24:38 -07003930 return ret;
Jens Axboe03b12302019-12-02 18:50:25 -07003931 }
3932
Jens Axboee47293f2019-12-20 08:58:21 -07003933 flags = req->sr_msg.msg_flags;
3934 if (flags & MSG_DONTWAIT)
3935 req->flags |= REQ_F_NOWAIT;
3936 else if (force_nonblock)
3937 flags |= MSG_DONTWAIT;
3938
Jens Axboe0b416c32019-12-15 10:57:46 -07003939 ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags);
Pavel Begunkov02d27d82020-02-28 10:36:36 +03003940 if (force_nonblock && ret == -EAGAIN)
3941 return io_setup_async_msg(req, kmsg);
Jens Axboe03b12302019-12-02 18:50:25 -07003942 if (ret == -ERESTARTSYS)
3943 ret = -EINTR;
3944 }
3945
Pavel Begunkov1e950812020-02-06 19:51:16 +03003946 if (kmsg && kmsg->iov != kmsg->fast_iov)
Jens Axboe0b416c32019-12-15 10:57:46 -07003947 kfree(kmsg->iov);
Pavel Begunkov99bc4c32020-02-07 22:04:45 +03003948 req->flags &= ~REQ_F_NEED_CLEANUP;
Jens Axboe4e88d6e2019-12-07 20:59:47 -07003949 if (ret < 0)
3950 req_set_fail_links(req);
Jens Axboe229a7b62020-06-22 10:13:11 -06003951 __io_req_complete(req, ret, 0, cs);
Jens Axboe03b12302019-12-02 18:50:25 -07003952 return 0;
Jens Axboe03b12302019-12-02 18:50:25 -07003953}
3954
Jens Axboe229a7b62020-06-22 10:13:11 -06003955static int io_send(struct io_kiocb *req, bool force_nonblock,
3956 struct io_comp_state *cs)
Jens Axboefddafac2020-01-04 20:19:44 -07003957{
Jens Axboefddafac2020-01-04 20:19:44 -07003958 struct socket *sock;
3959 int ret;
3960
Jens Axboefddafac2020-01-04 20:19:44 -07003961 sock = sock_from_file(req->file, &ret);
3962 if (sock) {
3963 struct io_sr_msg *sr = &req->sr_msg;
3964 struct msghdr msg;
3965 struct iovec iov;
3966 unsigned flags;
3967
3968 ret = import_single_range(WRITE, sr->buf, sr->len, &iov,
3969 &msg.msg_iter);
3970 if (ret)
3971 return ret;
3972
3973 msg.msg_name = NULL;
3974 msg.msg_control = NULL;
3975 msg.msg_controllen = 0;
3976 msg.msg_namelen = 0;
3977
3978 flags = req->sr_msg.msg_flags;
3979 if (flags & MSG_DONTWAIT)
3980 req->flags |= REQ_F_NOWAIT;
3981 else if (force_nonblock)
3982 flags |= MSG_DONTWAIT;
3983
Jens Axboe0b7b21e2020-01-31 08:34:59 -07003984 msg.msg_flags = flags;
3985 ret = sock_sendmsg(sock, &msg);
Jens Axboefddafac2020-01-04 20:19:44 -07003986 if (force_nonblock && ret == -EAGAIN)
3987 return -EAGAIN;
3988 if (ret == -ERESTARTSYS)
3989 ret = -EINTR;
3990 }
3991
Jens Axboefddafac2020-01-04 20:19:44 -07003992 if (ret < 0)
3993 req_set_fail_links(req);
Jens Axboe229a7b62020-06-22 10:13:11 -06003994 __io_req_complete(req, ret, 0, cs);
Jens Axboefddafac2020-01-04 20:19:44 -07003995 return 0;
Jens Axboefddafac2020-01-04 20:19:44 -07003996}
3997
Jens Axboe52de1fe2020-02-27 10:15:42 -07003998static int __io_recvmsg_copy_hdr(struct io_kiocb *req, struct io_async_ctx *io)
3999{
4000 struct io_sr_msg *sr = &req->sr_msg;
4001 struct iovec __user *uiov;
4002 size_t iov_len;
4003 int ret;
4004
4005 ret = __copy_msghdr_from_user(&io->msg.msg, sr->msg, &io->msg.uaddr,
4006 &uiov, &iov_len);
4007 if (ret)
4008 return ret;
4009
4010 if (req->flags & REQ_F_BUFFER_SELECT) {
4011 if (iov_len > 1)
4012 return -EINVAL;
4013 if (copy_from_user(io->msg.iov, uiov, sizeof(*uiov)))
4014 return -EFAULT;
4015 sr->len = io->msg.iov[0].iov_len;
4016 iov_iter_init(&io->msg.msg.msg_iter, READ, io->msg.iov, 1,
4017 sr->len);
4018 io->msg.iov = NULL;
4019 } else {
4020 ret = import_iovec(READ, uiov, iov_len, UIO_FASTIOV,
4021 &io->msg.iov, &io->msg.msg.msg_iter);
4022 if (ret > 0)
4023 ret = 0;
4024 }
4025
4026 return ret;
4027}
4028
4029#ifdef CONFIG_COMPAT
4030static int __io_compat_recvmsg_copy_hdr(struct io_kiocb *req,
4031 struct io_async_ctx *io)
4032{
4033 struct compat_msghdr __user *msg_compat;
4034 struct io_sr_msg *sr = &req->sr_msg;
4035 struct compat_iovec __user *uiov;
4036 compat_uptr_t ptr;
4037 compat_size_t len;
4038 int ret;
4039
4040 msg_compat = (struct compat_msghdr __user *) sr->msg;
4041 ret = __get_compat_msghdr(&io->msg.msg, msg_compat, &io->msg.uaddr,
4042 &ptr, &len);
4043 if (ret)
4044 return ret;
4045
4046 uiov = compat_ptr(ptr);
4047 if (req->flags & REQ_F_BUFFER_SELECT) {
4048 compat_ssize_t clen;
4049
4050 if (len > 1)
4051 return -EINVAL;
4052 if (!access_ok(uiov, sizeof(*uiov)))
4053 return -EFAULT;
4054 if (__get_user(clen, &uiov->iov_len))
4055 return -EFAULT;
4056 if (clen < 0)
4057 return -EINVAL;
4058 sr->len = io->msg.iov[0].iov_len;
4059 io->msg.iov = NULL;
4060 } else {
4061 ret = compat_import_iovec(READ, uiov, len, UIO_FASTIOV,
4062 &io->msg.iov,
4063 &io->msg.msg.msg_iter);
4064 if (ret < 0)
4065 return ret;
4066 }
4067
4068 return 0;
4069}
Jens Axboe03b12302019-12-02 18:50:25 -07004070#endif
Jens Axboe52de1fe2020-02-27 10:15:42 -07004071
4072static int io_recvmsg_copy_hdr(struct io_kiocb *req, struct io_async_ctx *io)
4073{
4074 io->msg.iov = io->msg.fast_iov;
4075
4076#ifdef CONFIG_COMPAT
4077 if (req->ctx->compat)
4078 return __io_compat_recvmsg_copy_hdr(req, io);
4079#endif
4080
4081 return __io_recvmsg_copy_hdr(req, io);
4082}
4083
Jens Axboebcda7ba2020-02-23 16:42:51 -07004084static struct io_buffer *io_recv_buffer_select(struct io_kiocb *req,
4085 int *cflags, bool needs_lock)
4086{
4087 struct io_sr_msg *sr = &req->sr_msg;
4088 struct io_buffer *kbuf;
4089
4090 if (!(req->flags & REQ_F_BUFFER_SELECT))
4091 return NULL;
4092
4093 kbuf = io_buffer_select(req, &sr->len, sr->bgid, sr->kbuf, needs_lock);
4094 if (IS_ERR(kbuf))
4095 return kbuf;
4096
4097 sr->kbuf = kbuf;
4098 req->flags |= REQ_F_BUFFER_SELECTED;
4099
4100 *cflags = kbuf->bid << IORING_CQE_BUFFER_SHIFT;
4101 *cflags |= IORING_CQE_F_BUFFER;
4102 return kbuf;
Jens Axboe03b12302019-12-02 18:50:25 -07004103}
4104
Jens Axboe3529d8c2019-12-19 18:24:38 -07004105static int io_recvmsg_prep(struct io_kiocb *req,
4106 const struct io_uring_sqe *sqe)
Jens Axboe03b12302019-12-02 18:50:25 -07004107{
Jens Axboee47293f2019-12-20 08:58:21 -07004108 struct io_sr_msg *sr = &req->sr_msg;
Jens Axboe3529d8c2019-12-19 18:24:38 -07004109 struct io_async_ctx *io = req->io;
Pavel Begunkov99bc4c32020-02-07 22:04:45 +03004110 int ret;
Jens Axboe06b76d42019-12-19 14:44:26 -07004111
Pavel Begunkovd2b6f482020-06-03 18:03:25 +03004112 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4113 return -EINVAL;
4114
Jens Axboe3529d8c2019-12-19 18:24:38 -07004115 sr->msg_flags = READ_ONCE(sqe->msg_flags);
4116 sr->msg = u64_to_user_ptr(READ_ONCE(sqe->addr));
Jens Axboe0b7b21e2020-01-31 08:34:59 -07004117 sr->len = READ_ONCE(sqe->len);
Jens Axboebcda7ba2020-02-23 16:42:51 -07004118 sr->bgid = READ_ONCE(sqe->buf_group);
Jens Axboe3529d8c2019-12-19 18:24:38 -07004119
Jens Axboed8768362020-02-27 14:17:49 -07004120#ifdef CONFIG_COMPAT
4121 if (req->ctx->compat)
4122 sr->msg_flags |= MSG_CMSG_COMPAT;
4123#endif
4124
Jens Axboefddafac2020-01-04 20:19:44 -07004125 if (!io || req->opcode == IORING_OP_RECV)
Jens Axboe06b76d42019-12-19 14:44:26 -07004126 return 0;
Pavel Begunkov5f798be2020-02-08 13:28:02 +03004127 /* iovec is already imported */
4128 if (req->flags & REQ_F_NEED_CLEANUP)
4129 return 0;
Jens Axboe03b12302019-12-02 18:50:25 -07004130
Jens Axboe52de1fe2020-02-27 10:15:42 -07004131 ret = io_recvmsg_copy_hdr(req, io);
Pavel Begunkov99bc4c32020-02-07 22:04:45 +03004132 if (!ret)
4133 req->flags |= REQ_F_NEED_CLEANUP;
4134 return ret;
Jens Axboe03b12302019-12-02 18:50:25 -07004135}
4136
Jens Axboe229a7b62020-06-22 10:13:11 -06004137static int io_recvmsg(struct io_kiocb *req, bool force_nonblock,
4138 struct io_comp_state *cs)
Jens Axboe03b12302019-12-02 18:50:25 -07004139{
Jens Axboe0b416c32019-12-15 10:57:46 -07004140 struct io_async_msghdr *kmsg = NULL;
Jens Axboe0fa03c62019-04-19 13:34:07 -06004141 struct socket *sock;
Jens Axboe52de1fe2020-02-27 10:15:42 -07004142 int ret, cflags = 0;
Jens Axboe0fa03c62019-04-19 13:34:07 -06004143
Jens Axboe0fa03c62019-04-19 13:34:07 -06004144 sock = sock_from_file(req->file, &ret);
4145 if (sock) {
Jens Axboe52de1fe2020-02-27 10:15:42 -07004146 struct io_buffer *kbuf;
Jens Axboeb7bb4f72019-12-15 22:13:43 -07004147 struct io_async_ctx io;
Jens Axboe0fa03c62019-04-19 13:34:07 -06004148 unsigned flags;
4149
Jens Axboe03b12302019-12-02 18:50:25 -07004150 if (req->io) {
Jens Axboe0b416c32019-12-15 10:57:46 -07004151 kmsg = &req->io->msg;
Jens Axboeb5379162020-02-09 11:29:15 -07004152 kmsg->msg.msg_name = &req->io->msg.addr;
Jens Axboe0b416c32019-12-15 10:57:46 -07004153 /* if iov is set, it's allocated already */
4154 if (!kmsg->iov)
4155 kmsg->iov = kmsg->fast_iov;
4156 kmsg->msg.msg_iter.iov = kmsg->iov;
Jens Axboe03b12302019-12-02 18:50:25 -07004157 } else {
Jens Axboe0b416c32019-12-15 10:57:46 -07004158 kmsg = &io.msg;
Jens Axboeb5379162020-02-09 11:29:15 -07004159 kmsg->msg.msg_name = &io.msg.addr;
Jens Axboe3529d8c2019-12-19 18:24:38 -07004160
Jens Axboe52de1fe2020-02-27 10:15:42 -07004161 ret = io_recvmsg_copy_hdr(req, &io);
Jens Axboe03b12302019-12-02 18:50:25 -07004162 if (ret)
Jens Axboe3529d8c2019-12-19 18:24:38 -07004163 return ret;
Jens Axboe03b12302019-12-02 18:50:25 -07004164 }
Jens Axboe0fa03c62019-04-19 13:34:07 -06004165
Jens Axboe52de1fe2020-02-27 10:15:42 -07004166 kbuf = io_recv_buffer_select(req, &cflags, !force_nonblock);
4167 if (IS_ERR(kbuf)) {
4168 return PTR_ERR(kbuf);
4169 } else if (kbuf) {
4170 kmsg->fast_iov[0].iov_base = u64_to_user_ptr(kbuf->addr);
4171 iov_iter_init(&kmsg->msg.msg_iter, READ, kmsg->iov,
4172 1, req->sr_msg.len);
4173 }
4174
Jens Axboee47293f2019-12-20 08:58:21 -07004175 flags = req->sr_msg.msg_flags;
4176 if (flags & MSG_DONTWAIT)
4177 req->flags |= REQ_F_NOWAIT;
4178 else if (force_nonblock)
4179 flags |= MSG_DONTWAIT;
4180
4181 ret = __sys_recvmsg_sock(sock, &kmsg->msg, req->sr_msg.msg,
4182 kmsg->uaddr, flags);
Pavel Begunkov02d27d82020-02-28 10:36:36 +03004183 if (force_nonblock && ret == -EAGAIN)
4184 return io_setup_async_msg(req, kmsg);
Jens Axboe441cdbd2019-12-02 18:49:10 -07004185 if (ret == -ERESTARTSYS)
4186 ret = -EINTR;
Jens Axboe0fa03c62019-04-19 13:34:07 -06004187 }
4188
Pavel Begunkov1e950812020-02-06 19:51:16 +03004189 if (kmsg && kmsg->iov != kmsg->fast_iov)
Jens Axboe0b416c32019-12-15 10:57:46 -07004190 kfree(kmsg->iov);
Pavel Begunkov99bc4c32020-02-07 22:04:45 +03004191 req->flags &= ~REQ_F_NEED_CLEANUP;
Jens Axboe4e88d6e2019-12-07 20:59:47 -07004192 if (ret < 0)
4193 req_set_fail_links(req);
Jens Axboe229a7b62020-06-22 10:13:11 -06004194 __io_req_complete(req, ret, cflags, cs);
Jens Axboe0fa03c62019-04-19 13:34:07 -06004195 return 0;
Jens Axboe0fa03c62019-04-19 13:34:07 -06004196}
4197
Jens Axboe229a7b62020-06-22 10:13:11 -06004198static int io_recv(struct io_kiocb *req, bool force_nonblock,
4199 struct io_comp_state *cs)
Jens Axboefddafac2020-01-04 20:19:44 -07004200{
Jens Axboebcda7ba2020-02-23 16:42:51 -07004201 struct io_buffer *kbuf = NULL;
Jens Axboefddafac2020-01-04 20:19:44 -07004202 struct socket *sock;
Jens Axboebcda7ba2020-02-23 16:42:51 -07004203 int ret, cflags = 0;
Jens Axboefddafac2020-01-04 20:19:44 -07004204
Jens Axboefddafac2020-01-04 20:19:44 -07004205 sock = sock_from_file(req->file, &ret);
4206 if (sock) {
4207 struct io_sr_msg *sr = &req->sr_msg;
Jens Axboebcda7ba2020-02-23 16:42:51 -07004208 void __user *buf = sr->buf;
Jens Axboefddafac2020-01-04 20:19:44 -07004209 struct msghdr msg;
4210 struct iovec iov;
4211 unsigned flags;
4212
Jens Axboebcda7ba2020-02-23 16:42:51 -07004213 kbuf = io_recv_buffer_select(req, &cflags, !force_nonblock);
4214 if (IS_ERR(kbuf))
4215 return PTR_ERR(kbuf);
4216 else if (kbuf)
4217 buf = u64_to_user_ptr(kbuf->addr);
Jens Axboefddafac2020-01-04 20:19:44 -07004218
Jens Axboebcda7ba2020-02-23 16:42:51 -07004219 ret = import_single_range(READ, buf, sr->len, &iov,
4220 &msg.msg_iter);
4221 if (ret) {
4222 kfree(kbuf);
4223 return ret;
4224 }
4225
4226 req->flags |= REQ_F_NEED_CLEANUP;
Jens Axboefddafac2020-01-04 20:19:44 -07004227 msg.msg_name = NULL;
4228 msg.msg_control = NULL;
4229 msg.msg_controllen = 0;
4230 msg.msg_namelen = 0;
4231 msg.msg_iocb = NULL;
4232 msg.msg_flags = 0;
4233
4234 flags = req->sr_msg.msg_flags;
4235 if (flags & MSG_DONTWAIT)
4236 req->flags |= REQ_F_NOWAIT;
4237 else if (force_nonblock)
4238 flags |= MSG_DONTWAIT;
4239
Jens Axboe0b7b21e2020-01-31 08:34:59 -07004240 ret = sock_recvmsg(sock, &msg, flags);
Jens Axboefddafac2020-01-04 20:19:44 -07004241 if (force_nonblock && ret == -EAGAIN)
4242 return -EAGAIN;
4243 if (ret == -ERESTARTSYS)
4244 ret = -EINTR;
4245 }
4246
Jens Axboebcda7ba2020-02-23 16:42:51 -07004247 kfree(kbuf);
4248 req->flags &= ~REQ_F_NEED_CLEANUP;
Jens Axboefddafac2020-01-04 20:19:44 -07004249 if (ret < 0)
4250 req_set_fail_links(req);
Jens Axboe229a7b62020-06-22 10:13:11 -06004251 __io_req_complete(req, ret, cflags, cs);
Jens Axboefddafac2020-01-04 20:19:44 -07004252 return 0;
Jens Axboefddafac2020-01-04 20:19:44 -07004253}
4254
Jens Axboe3529d8c2019-12-19 18:24:38 -07004255static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboe17f2fe32019-10-17 14:42:58 -06004256{
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004257 struct io_accept *accept = &req->accept;
4258
Jens Axboe17f2fe32019-10-17 14:42:58 -06004259 if (unlikely(req->ctx->flags & (IORING_SETUP_IOPOLL|IORING_SETUP_SQPOLL)))
4260 return -EINVAL;
Hrvoje Zeba8042d6c2019-11-25 14:40:22 -05004261 if (sqe->ioprio || sqe->len || sqe->buf_index)
Jens Axboe17f2fe32019-10-17 14:42:58 -06004262 return -EINVAL;
4263
Jens Axboed55e5f52019-12-11 16:12:15 -07004264 accept->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));
4265 accept->addr_len = u64_to_user_ptr(READ_ONCE(sqe->addr2));
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004266 accept->flags = READ_ONCE(sqe->accept_flags);
Jens Axboe09952e32020-03-19 20:16:56 -06004267 accept->nofile = rlimit(RLIMIT_NOFILE);
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004268 return 0;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004269}
Jens Axboe17f2fe32019-10-17 14:42:58 -06004270
Jens Axboe229a7b62020-06-22 10:13:11 -06004271static int io_accept(struct io_kiocb *req, bool force_nonblock,
4272 struct io_comp_state *cs)
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004273{
4274 struct io_accept *accept = &req->accept;
Pavel Begunkovac45abc2020-06-08 21:08:18 +03004275 unsigned int file_flags = force_nonblock ? O_NONBLOCK : 0;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004276 int ret;
4277
Jiufei Xuee697dee2020-06-10 13:41:59 +08004278 if (req->file->f_flags & O_NONBLOCK)
4279 req->flags |= REQ_F_NOWAIT;
4280
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004281 ret = __sys_accept4_file(req->file, file_flags, accept->addr,
Jens Axboe09952e32020-03-19 20:16:56 -06004282 accept->addr_len, accept->flags,
4283 accept->nofile);
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004284 if (ret == -EAGAIN && force_nonblock)
Jens Axboe17f2fe32019-10-17 14:42:58 -06004285 return -EAGAIN;
Pavel Begunkovac45abc2020-06-08 21:08:18 +03004286 if (ret < 0) {
4287 if (ret == -ERESTARTSYS)
4288 ret = -EINTR;
Jens Axboe4e88d6e2019-12-07 20:59:47 -07004289 req_set_fail_links(req);
Pavel Begunkovac45abc2020-06-08 21:08:18 +03004290 }
Jens Axboe229a7b62020-06-22 10:13:11 -06004291 __io_req_complete(req, ret, 0, cs);
Jens Axboe17f2fe32019-10-17 14:42:58 -06004292 return 0;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004293}
4294
Jens Axboe3529d8c2019-12-19 18:24:38 -07004295static int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboef499a022019-12-02 16:28:46 -07004296{
Jens Axboe3529d8c2019-12-19 18:24:38 -07004297 struct io_connect *conn = &req->connect;
4298 struct io_async_ctx *io = req->io;
Jens Axboef499a022019-12-02 16:28:46 -07004299
Jens Axboe3fbb51c2019-12-20 08:51:52 -07004300 if (unlikely(req->ctx->flags & (IORING_SETUP_IOPOLL|IORING_SETUP_SQPOLL)))
4301 return -EINVAL;
4302 if (sqe->ioprio || sqe->len || sqe->buf_index || sqe->rw_flags)
4303 return -EINVAL;
4304
Jens Axboe3529d8c2019-12-19 18:24:38 -07004305 conn->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));
4306 conn->addr_len = READ_ONCE(sqe->addr2);
4307
4308 if (!io)
4309 return 0;
4310
4311 return move_addr_to_kernel(conn->addr, conn->addr_len,
Jens Axboe3fbb51c2019-12-20 08:51:52 -07004312 &io->connect.address);
Jens Axboef499a022019-12-02 16:28:46 -07004313}
4314
Jens Axboe229a7b62020-06-22 10:13:11 -06004315static int io_connect(struct io_kiocb *req, bool force_nonblock,
4316 struct io_comp_state *cs)
Jens Axboef8e85cf2019-11-23 14:24:24 -07004317{
Jens Axboef499a022019-12-02 16:28:46 -07004318 struct io_async_ctx __io, *io;
Jens Axboef8e85cf2019-11-23 14:24:24 -07004319 unsigned file_flags;
Jens Axboe3fbb51c2019-12-20 08:51:52 -07004320 int ret;
Jens Axboef8e85cf2019-11-23 14:24:24 -07004321
Jens Axboef499a022019-12-02 16:28:46 -07004322 if (req->io) {
4323 io = req->io;
4324 } else {
Jens Axboe3529d8c2019-12-19 18:24:38 -07004325 ret = move_addr_to_kernel(req->connect.addr,
4326 req->connect.addr_len,
4327 &__io.connect.address);
Jens Axboef499a022019-12-02 16:28:46 -07004328 if (ret)
4329 goto out;
4330 io = &__io;
4331 }
4332
Jens Axboe3fbb51c2019-12-20 08:51:52 -07004333 file_flags = force_nonblock ? O_NONBLOCK : 0;
4334
4335 ret = __sys_connect_file(req->file, &io->connect.address,
4336 req->connect.addr_len, file_flags);
Jens Axboe87f80d62019-12-03 11:23:54 -07004337 if ((ret == -EAGAIN || ret == -EINPROGRESS) && force_nonblock) {
Jens Axboeb7bb4f72019-12-15 22:13:43 -07004338 if (req->io)
4339 return -EAGAIN;
4340 if (io_alloc_async_ctx(req)) {
Jens Axboef499a022019-12-02 16:28:46 -07004341 ret = -ENOMEM;
4342 goto out;
4343 }
Jens Axboeb7bb4f72019-12-15 22:13:43 -07004344 memcpy(&req->io->connect, &__io.connect, sizeof(__io.connect));
Jens Axboef8e85cf2019-11-23 14:24:24 -07004345 return -EAGAIN;
Jens Axboef499a022019-12-02 16:28:46 -07004346 }
Jens Axboef8e85cf2019-11-23 14:24:24 -07004347 if (ret == -ERESTARTSYS)
4348 ret = -EINTR;
Jens Axboef499a022019-12-02 16:28:46 -07004349out:
Jens Axboe4e88d6e2019-12-07 20:59:47 -07004350 if (ret < 0)
4351 req_set_fail_links(req);
Jens Axboe229a7b62020-06-22 10:13:11 -06004352 __io_req_complete(req, ret, 0, cs);
Jens Axboef8e85cf2019-11-23 14:24:24 -07004353 return 0;
Jens Axboef8e85cf2019-11-23 14:24:24 -07004354}
YueHaibing469956e2020-03-04 15:53:52 +08004355#else /* !CONFIG_NET */
4356static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4357{
Jens Axboef8e85cf2019-11-23 14:24:24 -07004358 return -EOPNOTSUPP;
Jens Axboef8e85cf2019-11-23 14:24:24 -07004359}
4360
Randy Dunlap1e16c2f2020-06-26 16:32:50 -07004361static int io_sendmsg(struct io_kiocb *req, bool force_nonblock,
4362 struct io_comp_state *cs)
Jens Axboe221c5eb2019-01-17 09:41:58 -07004363{
YueHaibing469956e2020-03-04 15:53:52 +08004364 return -EOPNOTSUPP;
4365}
4366
Randy Dunlap1e16c2f2020-06-26 16:32:50 -07004367static int io_send(struct io_kiocb *req, bool force_nonblock,
4368 struct io_comp_state *cs)
YueHaibing469956e2020-03-04 15:53:52 +08004369{
4370 return -EOPNOTSUPP;
4371}
4372
4373static int io_recvmsg_prep(struct io_kiocb *req,
4374 const struct io_uring_sqe *sqe)
4375{
4376 return -EOPNOTSUPP;
4377}
4378
Randy Dunlap1e16c2f2020-06-26 16:32:50 -07004379static int io_recvmsg(struct io_kiocb *req, bool force_nonblock,
4380 struct io_comp_state *cs)
YueHaibing469956e2020-03-04 15:53:52 +08004381{
4382 return -EOPNOTSUPP;
4383}
4384
Randy Dunlap1e16c2f2020-06-26 16:32:50 -07004385static int io_recv(struct io_kiocb *req, bool force_nonblock,
4386 struct io_comp_state *cs)
YueHaibing469956e2020-03-04 15:53:52 +08004387{
4388 return -EOPNOTSUPP;
4389}
4390
4391static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4392{
4393 return -EOPNOTSUPP;
4394}
4395
Randy Dunlap1e16c2f2020-06-26 16:32:50 -07004396static int io_accept(struct io_kiocb *req, bool force_nonblock,
4397 struct io_comp_state *cs)
YueHaibing469956e2020-03-04 15:53:52 +08004398{
4399 return -EOPNOTSUPP;
4400}
4401
4402static int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4403{
4404 return -EOPNOTSUPP;
4405}
4406
Randy Dunlap1e16c2f2020-06-26 16:32:50 -07004407static int io_connect(struct io_kiocb *req, bool force_nonblock,
4408 struct io_comp_state *cs)
YueHaibing469956e2020-03-04 15:53:52 +08004409{
4410 return -EOPNOTSUPP;
4411}
4412#endif /* CONFIG_NET */
Jens Axboe2b188cc2019-01-07 10:46:33 -07004413
Jens Axboed7718a92020-02-14 22:23:12 -07004414struct io_poll_table {
4415 struct poll_table_struct pt;
4416 struct io_kiocb *req;
4417 int error;
4418};
4419
Jens Axboeb7db41c2020-07-04 08:55:50 -06004420static int io_req_task_work_add(struct io_kiocb *req, struct callback_head *cb)
Jens Axboece593a62020-06-30 12:39:05 -06004421{
4422 struct task_struct *tsk = req->task;
Jens Axboeb7db41c2020-07-04 08:55:50 -06004423 struct io_ring_ctx *ctx = req->ctx;
4424 int ret, notify = TWA_RESUME;
Jens Axboece593a62020-06-30 12:39:05 -06004425
Jens Axboeb7db41c2020-07-04 08:55:50 -06004426 /*
4427 * SQPOLL kernel thread doesn't need notification, just a wakeup.
4428 * If we're not using an eventfd, then TWA_RESUME is always fine,
4429 * as we won't have dependencies between request completions for
4430 * other kernel wait conditions.
4431 */
4432 if (ctx->flags & IORING_SETUP_SQPOLL)
Jens Axboece593a62020-06-30 12:39:05 -06004433 notify = 0;
Jens Axboeb7db41c2020-07-04 08:55:50 -06004434 else if (ctx->cq_ev_fd)
4435 notify = TWA_SIGNAL;
Jens Axboece593a62020-06-30 12:39:05 -06004436
4437 ret = task_work_add(tsk, cb, notify);
4438 if (!ret)
4439 wake_up_process(tsk);
4440 return ret;
4441}
4442
Jens Axboed7718a92020-02-14 22:23:12 -07004443static int __io_async_wake(struct io_kiocb *req, struct io_poll_iocb *poll,
4444 __poll_t mask, task_work_func_t func)
4445{
4446 struct task_struct *tsk;
Jens Axboeaa96bf82020-04-03 11:26:26 -06004447 int ret;
Jens Axboed7718a92020-02-14 22:23:12 -07004448
4449 /* for instances that support it check for an event match first: */
4450 if (mask && !(mask & poll->events))
4451 return 0;
4452
4453 trace_io_uring_task_add(req->ctx, req->opcode, req->user_data, mask);
4454
4455 list_del_init(&poll->wait.entry);
4456
4457 tsk = req->task;
4458 req->result = mask;
4459 init_task_work(&req->task_work, func);
4460 /*
Jens Axboee3aabf92020-05-18 11:04:17 -06004461 * If this fails, then the task is exiting. When a task exits, the
4462 * work gets canceled, so just cancel this request as well instead
4463 * of executing it. We can't safely execute it anyway, as we may not
4464 * have the needed state needed for it anyway.
Jens Axboed7718a92020-02-14 22:23:12 -07004465 */
Jens Axboeb7db41c2020-07-04 08:55:50 -06004466 ret = io_req_task_work_add(req, &req->task_work);
Jens Axboeaa96bf82020-04-03 11:26:26 -06004467 if (unlikely(ret)) {
Jens Axboee3aabf92020-05-18 11:04:17 -06004468 WRITE_ONCE(poll->canceled, true);
Jens Axboeaa96bf82020-04-03 11:26:26 -06004469 tsk = io_wq_get_task(req->ctx->io_wq);
Jens Axboece593a62020-06-30 12:39:05 -06004470 task_work_add(tsk, &req->task_work, 0);
4471 wake_up_process(tsk);
Jens Axboeaa96bf82020-04-03 11:26:26 -06004472 }
Jens Axboed7718a92020-02-14 22:23:12 -07004473 return 1;
4474}
4475
Jens Axboe74ce6ce2020-04-13 11:09:12 -06004476static bool io_poll_rewait(struct io_kiocb *req, struct io_poll_iocb *poll)
4477 __acquires(&req->ctx->completion_lock)
4478{
4479 struct io_ring_ctx *ctx = req->ctx;
4480
4481 if (!req->result && !READ_ONCE(poll->canceled)) {
4482 struct poll_table_struct pt = { ._key = poll->events };
4483
4484 req->result = vfs_poll(req->file, &pt) & poll->events;
4485 }
4486
4487 spin_lock_irq(&ctx->completion_lock);
4488 if (!req->result && !READ_ONCE(poll->canceled)) {
4489 add_wait_queue(poll->head, &poll->wait);
4490 return true;
4491 }
4492
4493 return false;
4494}
4495
Jens Axboe18bceab2020-05-15 11:56:54 -06004496static void io_poll_remove_double(struct io_kiocb *req)
4497{
4498 struct io_poll_iocb *poll = (struct io_poll_iocb *) req->io;
4499
4500 lockdep_assert_held(&req->ctx->completion_lock);
4501
4502 if (poll && poll->head) {
4503 struct wait_queue_head *head = poll->head;
4504
4505 spin_lock(&head->lock);
4506 list_del_init(&poll->wait.entry);
4507 if (poll->wait.private)
4508 refcount_dec(&req->refs);
4509 poll->head = NULL;
4510 spin_unlock(&head->lock);
4511 }
4512}
4513
4514static void io_poll_complete(struct io_kiocb *req, __poll_t mask, int error)
4515{
4516 struct io_ring_ctx *ctx = req->ctx;
4517
4518 io_poll_remove_double(req);
4519 req->poll.done = true;
4520 io_cqring_fill_event(req, error ? error : mangle_poll(mask));
4521 io_commit_cqring(ctx);
4522}
4523
4524static void io_poll_task_handler(struct io_kiocb *req, struct io_kiocb **nxt)
4525{
4526 struct io_ring_ctx *ctx = req->ctx;
4527
4528 if (io_poll_rewait(req, &req->poll)) {
4529 spin_unlock_irq(&ctx->completion_lock);
4530 return;
4531 }
4532
4533 hash_del(&req->hash_node);
4534 io_poll_complete(req, req->result, 0);
4535 req->flags |= REQ_F_COMP_LOCKED;
Pavel Begunkov9b5f7bd2020-06-29 13:13:00 +03004536 *nxt = io_put_req_find_next(req);
Jens Axboe18bceab2020-05-15 11:56:54 -06004537 spin_unlock_irq(&ctx->completion_lock);
4538
4539 io_cqring_ev_posted(ctx);
4540}
4541
4542static void io_poll_task_func(struct callback_head *cb)
4543{
4544 struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
4545 struct io_kiocb *nxt = NULL;
4546
4547 io_poll_task_handler(req, &nxt);
Pavel Begunkovea1164e2020-06-30 15:20:41 +03004548 if (nxt)
4549 __io_req_task_submit(nxt);
Jens Axboe18bceab2020-05-15 11:56:54 -06004550}
4551
4552static int io_poll_double_wake(struct wait_queue_entry *wait, unsigned mode,
4553 int sync, void *key)
4554{
4555 struct io_kiocb *req = wait->private;
4556 struct io_poll_iocb *poll = (struct io_poll_iocb *) req->io;
4557 __poll_t mask = key_to_poll(key);
4558
4559 /* for instances that support it check for an event match first: */
4560 if (mask && !(mask & poll->events))
4561 return 0;
4562
4563 if (req->poll.head) {
4564 bool done;
4565
4566 spin_lock(&req->poll.head->lock);
4567 done = list_empty(&req->poll.wait.entry);
4568 if (!done)
4569 list_del_init(&req->poll.wait.entry);
4570 spin_unlock(&req->poll.head->lock);
4571 if (!done)
4572 __io_async_wake(req, poll, mask, io_poll_task_func);
4573 }
4574 refcount_dec(&req->refs);
4575 return 1;
4576}
4577
4578static void io_init_poll_iocb(struct io_poll_iocb *poll, __poll_t events,
4579 wait_queue_func_t wake_func)
4580{
4581 poll->head = NULL;
4582 poll->done = false;
4583 poll->canceled = false;
4584 poll->events = events;
4585 INIT_LIST_HEAD(&poll->wait.entry);
4586 init_waitqueue_func_entry(&poll->wait, wake_func);
4587}
4588
4589static void __io_queue_proc(struct io_poll_iocb *poll, struct io_poll_table *pt,
4590 struct wait_queue_head *head)
4591{
4592 struct io_kiocb *req = pt->req;
4593
4594 /*
4595 * If poll->head is already set, it's because the file being polled
4596 * uses multiple waitqueues for poll handling (eg one for read, one
4597 * for write). Setup a separate io_poll_iocb if this happens.
4598 */
4599 if (unlikely(poll->head)) {
4600 /* already have a 2nd entry, fail a third attempt */
4601 if (req->io) {
4602 pt->error = -EINVAL;
4603 return;
4604 }
4605 poll = kmalloc(sizeof(*poll), GFP_ATOMIC);
4606 if (!poll) {
4607 pt->error = -ENOMEM;
4608 return;
4609 }
4610 io_init_poll_iocb(poll, req->poll.events, io_poll_double_wake);
4611 refcount_inc(&req->refs);
4612 poll->wait.private = req;
4613 req->io = (void *) poll;
4614 }
4615
4616 pt->error = 0;
4617 poll->head = head;
Jiufei Xuea31eb4a2020-06-17 17:53:56 +08004618
4619 if (poll->events & EPOLLEXCLUSIVE)
4620 add_wait_queue_exclusive(head, &poll->wait);
4621 else
4622 add_wait_queue(head, &poll->wait);
Jens Axboe18bceab2020-05-15 11:56:54 -06004623}
4624
4625static void io_async_queue_proc(struct file *file, struct wait_queue_head *head,
4626 struct poll_table_struct *p)
4627{
4628 struct io_poll_table *pt = container_of(p, struct io_poll_table, pt);
4629
4630 __io_queue_proc(&pt->req->apoll->poll, pt, head);
4631}
4632
Jens Axboed7718a92020-02-14 22:23:12 -07004633static void io_async_task_func(struct callback_head *cb)
4634{
4635 struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
4636 struct async_poll *apoll = req->apoll;
4637 struct io_ring_ctx *ctx = req->ctx;
4638
4639 trace_io_uring_task_run(req->ctx, req->opcode, req->user_data);
4640
Jens Axboe74ce6ce2020-04-13 11:09:12 -06004641 if (io_poll_rewait(req, &apoll->poll)) {
Jens Axboed7718a92020-02-14 22:23:12 -07004642 spin_unlock_irq(&ctx->completion_lock);
Jens Axboe74ce6ce2020-04-13 11:09:12 -06004643 return;
Jens Axboed7718a92020-02-14 22:23:12 -07004644 }
4645
Jens Axboe31067252020-05-17 17:43:31 -06004646 /* If req is still hashed, it cannot have been canceled. Don't check. */
Pavel Begunkov0be0b0e2020-06-30 15:20:42 +03004647 if (hash_hashed(&req->hash_node))
Jens Axboe74ce6ce2020-04-13 11:09:12 -06004648 hash_del(&req->hash_node);
Jens Axboe2bae0472020-04-13 11:16:34 -06004649
Jens Axboe74ce6ce2020-04-13 11:09:12 -06004650 spin_unlock_irq(&ctx->completion_lock);
4651
Xiaoguang Wang44575a62020-04-19 10:06:55 +08004652 /* restore ->work in case we need to retry again */
Xiaoguang Wang405a5d22020-06-10 19:41:20 +08004653 if (req->flags & REQ_F_WORK_INITIALIZED)
4654 memcpy(&req->work, &apoll->work, sizeof(req->work));
Jens Axboe31067252020-05-17 17:43:31 -06004655 kfree(apoll);
Xiaoguang Wang44575a62020-04-19 10:06:55 +08004656
Pavel Begunkov0be0b0e2020-06-30 15:20:42 +03004657 if (!READ_ONCE(apoll->poll.canceled))
4658 __io_req_task_submit(req);
4659 else
4660 __io_req_task_cancel(req, -ECANCELED);
Jens Axboed7718a92020-02-14 22:23:12 -07004661}
4662
4663static int io_async_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
4664 void *key)
4665{
4666 struct io_kiocb *req = wait->private;
4667 struct io_poll_iocb *poll = &req->apoll->poll;
4668
4669 trace_io_uring_poll_wake(req->ctx, req->opcode, req->user_data,
4670 key_to_poll(key));
4671
4672 return __io_async_wake(req, poll, key_to_poll(key), io_async_task_func);
4673}
4674
4675static void io_poll_req_insert(struct io_kiocb *req)
4676{
4677 struct io_ring_ctx *ctx = req->ctx;
4678 struct hlist_head *list;
4679
4680 list = &ctx->cancel_hash[hash_long(req->user_data, ctx->cancel_hash_bits)];
4681 hlist_add_head(&req->hash_node, list);
4682}
4683
4684static __poll_t __io_arm_poll_handler(struct io_kiocb *req,
4685 struct io_poll_iocb *poll,
4686 struct io_poll_table *ipt, __poll_t mask,
4687 wait_queue_func_t wake_func)
4688 __acquires(&ctx->completion_lock)
4689{
4690 struct io_ring_ctx *ctx = req->ctx;
4691 bool cancel = false;
4692
Jens Axboe18bceab2020-05-15 11:56:54 -06004693 io_init_poll_iocb(poll, mask, wake_func);
Pavel Begunkovb90cd192020-06-21 13:09:52 +03004694 poll->file = req->file;
Jens Axboe18bceab2020-05-15 11:56:54 -06004695 poll->wait.private = req;
Jens Axboed7718a92020-02-14 22:23:12 -07004696
4697 ipt->pt._key = mask;
4698 ipt->req = req;
4699 ipt->error = -EINVAL;
4700
Jens Axboed7718a92020-02-14 22:23:12 -07004701 mask = vfs_poll(req->file, &ipt->pt) & poll->events;
4702
4703 spin_lock_irq(&ctx->completion_lock);
4704 if (likely(poll->head)) {
4705 spin_lock(&poll->head->lock);
4706 if (unlikely(list_empty(&poll->wait.entry))) {
4707 if (ipt->error)
4708 cancel = true;
4709 ipt->error = 0;
4710 mask = 0;
4711 }
4712 if (mask || ipt->error)
4713 list_del_init(&poll->wait.entry);
4714 else if (cancel)
4715 WRITE_ONCE(poll->canceled, true);
4716 else if (!poll->done) /* actually waiting for an event */
4717 io_poll_req_insert(req);
4718 spin_unlock(&poll->head->lock);
4719 }
4720
4721 return mask;
4722}
4723
4724static bool io_arm_poll_handler(struct io_kiocb *req)
4725{
4726 const struct io_op_def *def = &io_op_defs[req->opcode];
4727 struct io_ring_ctx *ctx = req->ctx;
4728 struct async_poll *apoll;
4729 struct io_poll_table ipt;
4730 __poll_t mask, ret;
Jens Axboe18bceab2020-05-15 11:56:54 -06004731 bool had_io;
Jens Axboed7718a92020-02-14 22:23:12 -07004732
4733 if (!req->file || !file_can_poll(req->file))
4734 return false;
Pavel Begunkov24c74672020-06-21 13:09:51 +03004735 if (req->flags & REQ_F_POLLED)
Jens Axboed7718a92020-02-14 22:23:12 -07004736 return false;
4737 if (!def->pollin && !def->pollout)
4738 return false;
4739
4740 apoll = kmalloc(sizeof(*apoll), GFP_ATOMIC);
4741 if (unlikely(!apoll))
4742 return false;
4743
4744 req->flags |= REQ_F_POLLED;
Xiaoguang Wang405a5d22020-06-10 19:41:20 +08004745 if (req->flags & REQ_F_WORK_INITIALIZED)
4746 memcpy(&apoll->work, &req->work, sizeof(req->work));
Jens Axboe18bceab2020-05-15 11:56:54 -06004747 had_io = req->io != NULL;
Jens Axboed7718a92020-02-14 22:23:12 -07004748
Pavel Begunkov4dd28242020-06-15 10:33:13 +03004749 io_get_req_task(req);
Jens Axboed7718a92020-02-14 22:23:12 -07004750 req->apoll = apoll;
4751 INIT_HLIST_NODE(&req->hash_node);
4752
Nathan Chancellor8755d972020-03-02 16:01:19 -07004753 mask = 0;
Jens Axboed7718a92020-02-14 22:23:12 -07004754 if (def->pollin)
Nathan Chancellor8755d972020-03-02 16:01:19 -07004755 mask |= POLLIN | POLLRDNORM;
Jens Axboed7718a92020-02-14 22:23:12 -07004756 if (def->pollout)
4757 mask |= POLLOUT | POLLWRNORM;
4758 mask |= POLLERR | POLLPRI;
4759
4760 ipt.pt._qproc = io_async_queue_proc;
4761
4762 ret = __io_arm_poll_handler(req, &apoll->poll, &ipt, mask,
4763 io_async_wake);
4764 if (ret) {
4765 ipt.error = 0;
Jens Axboe18bceab2020-05-15 11:56:54 -06004766 /* only remove double add if we did it here */
4767 if (!had_io)
4768 io_poll_remove_double(req);
Jens Axboed7718a92020-02-14 22:23:12 -07004769 spin_unlock_irq(&ctx->completion_lock);
Xiaoguang Wang405a5d22020-06-10 19:41:20 +08004770 if (req->flags & REQ_F_WORK_INITIALIZED)
4771 memcpy(&req->work, &apoll->work, sizeof(req->work));
Jens Axboed7718a92020-02-14 22:23:12 -07004772 kfree(apoll);
4773 return false;
4774 }
4775 spin_unlock_irq(&ctx->completion_lock);
4776 trace_io_uring_poll_arm(ctx, req->opcode, req->user_data, mask,
4777 apoll->poll.events);
4778 return true;
4779}
4780
4781static bool __io_poll_remove_one(struct io_kiocb *req,
4782 struct io_poll_iocb *poll)
4783{
Jens Axboeb41e9852020-02-17 09:52:41 -07004784 bool do_complete = false;
Jens Axboe221c5eb2019-01-17 09:41:58 -07004785
4786 spin_lock(&poll->head->lock);
4787 WRITE_ONCE(poll->canceled, true);
Jens Axboe392edb42019-12-09 17:52:20 -07004788 if (!list_empty(&poll->wait.entry)) {
4789 list_del_init(&poll->wait.entry);
Jens Axboeb41e9852020-02-17 09:52:41 -07004790 do_complete = true;
Jens Axboe221c5eb2019-01-17 09:41:58 -07004791 }
4792 spin_unlock(&poll->head->lock);
Jens Axboe3bfa5bc2020-05-17 13:54:12 -06004793 hash_del(&req->hash_node);
Jens Axboed7718a92020-02-14 22:23:12 -07004794 return do_complete;
4795}
4796
4797static bool io_poll_remove_one(struct io_kiocb *req)
4798{
4799 bool do_complete;
4800
4801 if (req->opcode == IORING_OP_POLL_ADD) {
Jens Axboe18bceab2020-05-15 11:56:54 -06004802 io_poll_remove_double(req);
Jens Axboed7718a92020-02-14 22:23:12 -07004803 do_complete = __io_poll_remove_one(req, &req->poll);
4804 } else {
Jens Axboe3bfa5bc2020-05-17 13:54:12 -06004805 struct async_poll *apoll = req->apoll;
4806
Jens Axboed7718a92020-02-14 22:23:12 -07004807 /* non-poll requests have submit ref still */
Jens Axboe3bfa5bc2020-05-17 13:54:12 -06004808 do_complete = __io_poll_remove_one(req, &apoll->poll);
4809 if (do_complete) {
Jens Axboed7718a92020-02-14 22:23:12 -07004810 io_put_req(req);
Jens Axboe3bfa5bc2020-05-17 13:54:12 -06004811 /*
4812 * restore ->work because we will call
4813 * io_req_work_drop_env below when dropping the
4814 * final reference.
4815 */
Xiaoguang Wang405a5d22020-06-10 19:41:20 +08004816 if (req->flags & REQ_F_WORK_INITIALIZED)
4817 memcpy(&req->work, &apoll->work,
4818 sizeof(req->work));
Jens Axboe3bfa5bc2020-05-17 13:54:12 -06004819 kfree(apoll);
4820 }
Xiaoguang Wangb1f573b2020-04-12 14:50:54 +08004821 }
4822
Jens Axboeb41e9852020-02-17 09:52:41 -07004823 if (do_complete) {
4824 io_cqring_fill_event(req, -ECANCELED);
4825 io_commit_cqring(req->ctx);
4826 req->flags |= REQ_F_COMP_LOCKED;
4827 io_put_req(req);
4828 }
4829
4830 return do_complete;
Jens Axboe221c5eb2019-01-17 09:41:58 -07004831}
4832
4833static void io_poll_remove_all(struct io_ring_ctx *ctx)
4834{
Jens Axboe78076bb2019-12-04 19:56:40 -07004835 struct hlist_node *tmp;
Jens Axboe221c5eb2019-01-17 09:41:58 -07004836 struct io_kiocb *req;
Jens Axboe8e2e1fa2020-04-13 17:05:14 -06004837 int posted = 0, i;
Jens Axboe221c5eb2019-01-17 09:41:58 -07004838
4839 spin_lock_irq(&ctx->completion_lock);
Jens Axboe78076bb2019-12-04 19:56:40 -07004840 for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) {
4841 struct hlist_head *list;
4842
4843 list = &ctx->cancel_hash[i];
4844 hlist_for_each_entry_safe(req, tmp, list, hash_node)
Jens Axboe8e2e1fa2020-04-13 17:05:14 -06004845 posted += io_poll_remove_one(req);
Jens Axboe221c5eb2019-01-17 09:41:58 -07004846 }
4847 spin_unlock_irq(&ctx->completion_lock);
Jens Axboeb41e9852020-02-17 09:52:41 -07004848
Jens Axboe8e2e1fa2020-04-13 17:05:14 -06004849 if (posted)
4850 io_cqring_ev_posted(ctx);
Jens Axboe221c5eb2019-01-17 09:41:58 -07004851}
4852
Jens Axboe47f46762019-11-09 17:43:02 -07004853static int io_poll_cancel(struct io_ring_ctx *ctx, __u64 sqe_addr)
4854{
Jens Axboe78076bb2019-12-04 19:56:40 -07004855 struct hlist_head *list;
Jens Axboe47f46762019-11-09 17:43:02 -07004856 struct io_kiocb *req;
4857
Jens Axboe78076bb2019-12-04 19:56:40 -07004858 list = &ctx->cancel_hash[hash_long(sqe_addr, ctx->cancel_hash_bits)];
4859 hlist_for_each_entry(req, list, hash_node) {
Jens Axboeb41e9852020-02-17 09:52:41 -07004860 if (sqe_addr != req->user_data)
4861 continue;
4862 if (io_poll_remove_one(req))
Jens Axboeeac406c2019-11-14 12:09:58 -07004863 return 0;
Jens Axboeb41e9852020-02-17 09:52:41 -07004864 return -EALREADY;
Jens Axboe47f46762019-11-09 17:43:02 -07004865 }
4866
4867 return -ENOENT;
4868}
4869
Jens Axboe3529d8c2019-12-19 18:24:38 -07004870static int io_poll_remove_prep(struct io_kiocb *req,
4871 const struct io_uring_sqe *sqe)
Jens Axboe221c5eb2019-01-17 09:41:58 -07004872{
Jens Axboe221c5eb2019-01-17 09:41:58 -07004873 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4874 return -EINVAL;
4875 if (sqe->ioprio || sqe->off || sqe->len || sqe->buf_index ||
4876 sqe->poll_events)
4877 return -EINVAL;
4878
Jens Axboe0969e782019-12-17 18:40:57 -07004879 req->poll.addr = READ_ONCE(sqe->addr);
Jens Axboe0969e782019-12-17 18:40:57 -07004880 return 0;
4881}
4882
4883/*
4884 * Find a running poll command that matches one specified in sqe->addr,
4885 * and remove it if found.
4886 */
4887static int io_poll_remove(struct io_kiocb *req)
4888{
4889 struct io_ring_ctx *ctx = req->ctx;
4890 u64 addr;
4891 int ret;
4892
Jens Axboe0969e782019-12-17 18:40:57 -07004893 addr = req->poll.addr;
Jens Axboe221c5eb2019-01-17 09:41:58 -07004894 spin_lock_irq(&ctx->completion_lock);
Jens Axboe0969e782019-12-17 18:40:57 -07004895 ret = io_poll_cancel(ctx, addr);
Jens Axboe221c5eb2019-01-17 09:41:58 -07004896 spin_unlock_irq(&ctx->completion_lock);
4897
Jens Axboe4e88d6e2019-12-07 20:59:47 -07004898 if (ret < 0)
4899 req_set_fail_links(req);
Jens Axboee1e16092020-06-22 09:17:17 -06004900 io_req_complete(req, ret);
Jens Axboe221c5eb2019-01-17 09:41:58 -07004901 return 0;
4902}
4903
Jens Axboe221c5eb2019-01-17 09:41:58 -07004904static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
4905 void *key)
4906{
Jens Axboec2f2eb72020-02-10 09:07:05 -07004907 struct io_kiocb *req = wait->private;
4908 struct io_poll_iocb *poll = &req->poll;
Jens Axboe221c5eb2019-01-17 09:41:58 -07004909
Jens Axboed7718a92020-02-14 22:23:12 -07004910 return __io_async_wake(req, poll, key_to_poll(key), io_poll_task_func);
Jens Axboe221c5eb2019-01-17 09:41:58 -07004911}
4912
Jens Axboe221c5eb2019-01-17 09:41:58 -07004913static void io_poll_queue_proc(struct file *file, struct wait_queue_head *head,
4914 struct poll_table_struct *p)
4915{
4916 struct io_poll_table *pt = container_of(p, struct io_poll_table, pt);
4917
Jens Axboed7718a92020-02-14 22:23:12 -07004918 __io_queue_proc(&pt->req->poll, pt, head);
Jens Axboeeac406c2019-11-14 12:09:58 -07004919}
4920
Jens Axboe3529d8c2019-12-19 18:24:38 -07004921static int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboe221c5eb2019-01-17 09:41:58 -07004922{
4923 struct io_poll_iocb *poll = &req->poll;
Jiufei Xue5769a352020-06-17 17:53:55 +08004924 u32 events;
Jens Axboe221c5eb2019-01-17 09:41:58 -07004925
4926 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4927 return -EINVAL;
4928 if (sqe->addr || sqe->ioprio || sqe->off || sqe->len || sqe->buf_index)
4929 return -EINVAL;
Jens Axboe09bb8392019-03-13 12:39:28 -06004930 if (!poll->file)
4931 return -EBADF;
Jens Axboe221c5eb2019-01-17 09:41:58 -07004932
Jiufei Xue5769a352020-06-17 17:53:55 +08004933 events = READ_ONCE(sqe->poll32_events);
4934#ifdef __BIG_ENDIAN
4935 events = swahw32(events);
4936#endif
Jiufei Xuea31eb4a2020-06-17 17:53:56 +08004937 poll->events = demangle_poll(events) | EPOLLERR | EPOLLHUP |
4938 (events & EPOLLEXCLUSIVE);
Jens Axboeb41e9852020-02-17 09:52:41 -07004939
Pavel Begunkov4dd28242020-06-15 10:33:13 +03004940 io_get_req_task(req);
Jens Axboe0969e782019-12-17 18:40:57 -07004941 return 0;
4942}
4943
Pavel Begunkov014db002020-03-03 21:33:12 +03004944static int io_poll_add(struct io_kiocb *req)
Jens Axboe0969e782019-12-17 18:40:57 -07004945{
4946 struct io_poll_iocb *poll = &req->poll;
4947 struct io_ring_ctx *ctx = req->ctx;
4948 struct io_poll_table ipt;
Jens Axboe0969e782019-12-17 18:40:57 -07004949 __poll_t mask;
Jens Axboe0969e782019-12-17 18:40:57 -07004950
Jens Axboe78076bb2019-12-04 19:56:40 -07004951 INIT_HLIST_NODE(&req->hash_node);
Jens Axboe36703242019-07-25 10:20:18 -06004952 INIT_LIST_HEAD(&req->list);
Jens Axboed7718a92020-02-14 22:23:12 -07004953 ipt.pt._qproc = io_poll_queue_proc;
Jens Axboe36703242019-07-25 10:20:18 -06004954
Jens Axboed7718a92020-02-14 22:23:12 -07004955 mask = __io_arm_poll_handler(req, &req->poll, &ipt, poll->events,
4956 io_poll_wake);
Jens Axboe221c5eb2019-01-17 09:41:58 -07004957
Jens Axboe8c838782019-03-12 15:48:16 -06004958 if (mask) { /* no async, we'd stolen it */
Jens Axboe8c838782019-03-12 15:48:16 -06004959 ipt.error = 0;
Jens Axboeb0dd8a42019-11-18 12:14:54 -07004960 io_poll_complete(req, mask, 0);
Jens Axboe8c838782019-03-12 15:48:16 -06004961 }
Jens Axboe221c5eb2019-01-17 09:41:58 -07004962 spin_unlock_irq(&ctx->completion_lock);
4963
Jens Axboe8c838782019-03-12 15:48:16 -06004964 if (mask) {
4965 io_cqring_ev_posted(ctx);
Pavel Begunkov014db002020-03-03 21:33:12 +03004966 io_put_req(req);
Jens Axboe221c5eb2019-01-17 09:41:58 -07004967 }
Jens Axboe8c838782019-03-12 15:48:16 -06004968 return ipt.error;
Jens Axboe221c5eb2019-01-17 09:41:58 -07004969}
4970
Jens Axboe5262f562019-09-17 12:26:57 -06004971static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer)
4972{
Jens Axboead8a48a2019-11-15 08:49:11 -07004973 struct io_timeout_data *data = container_of(timer,
4974 struct io_timeout_data, timer);
4975 struct io_kiocb *req = data->req;
4976 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe5262f562019-09-17 12:26:57 -06004977 unsigned long flags;
4978
Jens Axboe5262f562019-09-17 12:26:57 -06004979 atomic_inc(&ctx->cq_timeouts);
4980
4981 spin_lock_irqsave(&ctx->completion_lock, flags);
zhangyi (F)ef036812019-10-23 15:10:08 +08004982 /*
Jens Axboe11365042019-10-16 09:08:32 -06004983 * We could be racing with timeout deletion. If the list is empty,
4984 * then timeout lookup already found it and will be handling it.
zhangyi (F)ef036812019-10-23 15:10:08 +08004985 */
Pavel Begunkovbfe68a22020-05-30 14:54:18 +03004986 if (!list_empty(&req->list))
Jens Axboe11365042019-10-16 09:08:32 -06004987 list_del_init(&req->list);
Jens Axboe842f9612019-10-29 12:34:10 -06004988
Jens Axboe78e19bb2019-11-06 15:21:34 -07004989 io_cqring_fill_event(req, -ETIME);
Jens Axboe5262f562019-09-17 12:26:57 -06004990 io_commit_cqring(ctx);
4991 spin_unlock_irqrestore(&ctx->completion_lock, flags);
4992
4993 io_cqring_ev_posted(ctx);
Jens Axboe4e88d6e2019-12-07 20:59:47 -07004994 req_set_fail_links(req);
Jens Axboe5262f562019-09-17 12:26:57 -06004995 io_put_req(req);
4996 return HRTIMER_NORESTART;
4997}
4998
Jens Axboe47f46762019-11-09 17:43:02 -07004999static int io_timeout_cancel(struct io_ring_ctx *ctx, __u64 user_data)
5000{
5001 struct io_kiocb *req;
5002 int ret = -ENOENT;
5003
5004 list_for_each_entry(req, &ctx->timeout_list, list) {
5005 if (user_data == req->user_data) {
5006 list_del_init(&req->list);
5007 ret = 0;
5008 break;
5009 }
5010 }
5011
5012 if (ret == -ENOENT)
5013 return ret;
5014
Jens Axboe2d283902019-12-04 11:08:05 -07005015 ret = hrtimer_try_to_cancel(&req->io->timeout.timer);
Jens Axboe47f46762019-11-09 17:43:02 -07005016 if (ret == -1)
5017 return -EALREADY;
5018
Jens Axboe4e88d6e2019-12-07 20:59:47 -07005019 req_set_fail_links(req);
Jens Axboe47f46762019-11-09 17:43:02 -07005020 io_cqring_fill_event(req, -ECANCELED);
5021 io_put_req(req);
5022 return 0;
5023}
5024
Jens Axboe3529d8c2019-12-19 18:24:38 -07005025static int io_timeout_remove_prep(struct io_kiocb *req,
5026 const struct io_uring_sqe *sqe)
Jens Axboeb29472e2019-12-17 18:50:29 -07005027{
Jens Axboeb29472e2019-12-17 18:50:29 -07005028 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
5029 return -EINVAL;
5030 if (sqe->flags || sqe->ioprio || sqe->buf_index || sqe->len)
5031 return -EINVAL;
5032
5033 req->timeout.addr = READ_ONCE(sqe->addr);
5034 req->timeout.flags = READ_ONCE(sqe->timeout_flags);
5035 if (req->timeout.flags)
5036 return -EINVAL;
5037
Jens Axboeb29472e2019-12-17 18:50:29 -07005038 return 0;
5039}
5040
Jens Axboe11365042019-10-16 09:08:32 -06005041/*
5042 * Remove or update an existing timeout command
5043 */
Jens Axboefc4df992019-12-10 14:38:45 -07005044static int io_timeout_remove(struct io_kiocb *req)
Jens Axboe11365042019-10-16 09:08:32 -06005045{
5046 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe47f46762019-11-09 17:43:02 -07005047 int ret;
Jens Axboe11365042019-10-16 09:08:32 -06005048
Jens Axboe11365042019-10-16 09:08:32 -06005049 spin_lock_irq(&ctx->completion_lock);
Jens Axboeb29472e2019-12-17 18:50:29 -07005050 ret = io_timeout_cancel(ctx, req->timeout.addr);
Jens Axboe11365042019-10-16 09:08:32 -06005051
Jens Axboe47f46762019-11-09 17:43:02 -07005052 io_cqring_fill_event(req, ret);
Jens Axboe11365042019-10-16 09:08:32 -06005053 io_commit_cqring(ctx);
5054 spin_unlock_irq(&ctx->completion_lock);
Jens Axboe5262f562019-09-17 12:26:57 -06005055 io_cqring_ev_posted(ctx);
Jens Axboe4e88d6e2019-12-07 20:59:47 -07005056 if (ret < 0)
5057 req_set_fail_links(req);
Jackie Liuec9c02a2019-11-08 23:50:36 +08005058 io_put_req(req);
Jens Axboe11365042019-10-16 09:08:32 -06005059 return 0;
Jens Axboe5262f562019-09-17 12:26:57 -06005060}
5061
Jens Axboe3529d8c2019-12-19 18:24:38 -07005062static int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe,
Jens Axboe2d283902019-12-04 11:08:05 -07005063 bool is_timeout_link)
Jens Axboe5262f562019-09-17 12:26:57 -06005064{
Jens Axboead8a48a2019-11-15 08:49:11 -07005065 struct io_timeout_data *data;
Jens Axboea41525a2019-10-15 16:48:15 -06005066 unsigned flags;
Pavel Begunkov56080b02020-05-26 20:34:04 +03005067 u32 off = READ_ONCE(sqe->off);
Jens Axboe5262f562019-09-17 12:26:57 -06005068
Jens Axboead8a48a2019-11-15 08:49:11 -07005069 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
Jens Axboe5262f562019-09-17 12:26:57 -06005070 return -EINVAL;
Jens Axboead8a48a2019-11-15 08:49:11 -07005071 if (sqe->ioprio || sqe->buf_index || sqe->len != 1)
Jens Axboea41525a2019-10-15 16:48:15 -06005072 return -EINVAL;
Pavel Begunkov56080b02020-05-26 20:34:04 +03005073 if (off && is_timeout_link)
Jens Axboe2d283902019-12-04 11:08:05 -07005074 return -EINVAL;
Jens Axboea41525a2019-10-15 16:48:15 -06005075 flags = READ_ONCE(sqe->timeout_flags);
5076 if (flags & ~IORING_TIMEOUT_ABS)
Jens Axboe5262f562019-09-17 12:26:57 -06005077 return -EINVAL;
Arnd Bergmannbdf20072019-10-01 09:53:29 -06005078
Pavel Begunkovbfe68a22020-05-30 14:54:18 +03005079 req->timeout.off = off;
Jens Axboe26a61672019-12-20 09:02:01 -07005080
Jens Axboe3529d8c2019-12-19 18:24:38 -07005081 if (!req->io && io_alloc_async_ctx(req))
Jens Axboe26a61672019-12-20 09:02:01 -07005082 return -ENOMEM;
5083
5084 data = &req->io->timeout;
Jens Axboead8a48a2019-11-15 08:49:11 -07005085 data->req = req;
Jens Axboead8a48a2019-11-15 08:49:11 -07005086
5087 if (get_timespec64(&data->ts, u64_to_user_ptr(sqe->addr)))
Jens Axboe5262f562019-09-17 12:26:57 -06005088 return -EFAULT;
5089
Jens Axboe11365042019-10-16 09:08:32 -06005090 if (flags & IORING_TIMEOUT_ABS)
Jens Axboead8a48a2019-11-15 08:49:11 -07005091 data->mode = HRTIMER_MODE_ABS;
Jens Axboe11365042019-10-16 09:08:32 -06005092 else
Jens Axboead8a48a2019-11-15 08:49:11 -07005093 data->mode = HRTIMER_MODE_REL;
Jens Axboe11365042019-10-16 09:08:32 -06005094
Jens Axboead8a48a2019-11-15 08:49:11 -07005095 hrtimer_init(&data->timer, CLOCK_MONOTONIC, data->mode);
5096 return 0;
5097}
5098
Jens Axboefc4df992019-12-10 14:38:45 -07005099static int io_timeout(struct io_kiocb *req)
Jens Axboead8a48a2019-11-15 08:49:11 -07005100{
Jens Axboead8a48a2019-11-15 08:49:11 -07005101 struct io_ring_ctx *ctx = req->ctx;
Pavel Begunkovbfe68a22020-05-30 14:54:18 +03005102 struct io_timeout_data *data = &req->io->timeout;
Jens Axboead8a48a2019-11-15 08:49:11 -07005103 struct list_head *entry;
Pavel Begunkovbfe68a22020-05-30 14:54:18 +03005104 u32 tail, off = req->timeout.off;
Jens Axboead8a48a2019-11-15 08:49:11 -07005105
Pavel Begunkov733f5c92020-05-26 20:34:03 +03005106 spin_lock_irq(&ctx->completion_lock);
Jens Axboe93bd25b2019-11-11 23:34:31 -07005107
Jens Axboe5262f562019-09-17 12:26:57 -06005108 /*
5109 * sqe->off holds how many events that need to occur for this
Jens Axboe93bd25b2019-11-11 23:34:31 -07005110 * timeout event to be satisfied. If it isn't set, then this is
5111 * a pure timeout request, sequence isn't used.
Jens Axboe5262f562019-09-17 12:26:57 -06005112 */
Pavel Begunkov8eb7e2d2020-06-29 13:13:02 +03005113 if (io_is_timeout_noseq(req)) {
Jens Axboe93bd25b2019-11-11 23:34:31 -07005114 entry = ctx->timeout_list.prev;
5115 goto add;
5116 }
Jens Axboe5262f562019-09-17 12:26:57 -06005117
Pavel Begunkovbfe68a22020-05-30 14:54:18 +03005118 tail = ctx->cached_cq_tail - atomic_read(&ctx->cq_timeouts);
5119 req->timeout.target_seq = tail + off;
Jens Axboe5262f562019-09-17 12:26:57 -06005120
5121 /*
5122 * Insertion sort, ensuring the first entry in the list is always
5123 * the one we need first.
5124 */
Jens Axboe5262f562019-09-17 12:26:57 -06005125 list_for_each_prev(entry, &ctx->timeout_list) {
5126 struct io_kiocb *nxt = list_entry(entry, struct io_kiocb, list);
Jens Axboe5262f562019-09-17 12:26:57 -06005127
Pavel Begunkov8eb7e2d2020-06-29 13:13:02 +03005128 if (io_is_timeout_noseq(nxt))
Jens Axboe93bd25b2019-11-11 23:34:31 -07005129 continue;
Pavel Begunkovbfe68a22020-05-30 14:54:18 +03005130 /* nxt.seq is behind @tail, otherwise would've been completed */
5131 if (off >= nxt->timeout.target_seq - tail)
Jens Axboe5262f562019-09-17 12:26:57 -06005132 break;
5133 }
Jens Axboe93bd25b2019-11-11 23:34:31 -07005134add:
Jens Axboe5262f562019-09-17 12:26:57 -06005135 list_add(&req->list, entry);
Jens Axboead8a48a2019-11-15 08:49:11 -07005136 data->timer.function = io_timeout_fn;
5137 hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), data->mode);
Jens Axboe842f9612019-10-29 12:34:10 -06005138 spin_unlock_irq(&ctx->completion_lock);
Jens Axboe5262f562019-09-17 12:26:57 -06005139 return 0;
5140}
5141
Jens Axboe62755e32019-10-28 21:49:21 -06005142static bool io_cancel_cb(struct io_wq_work *work, void *data)
Jens Axboede0617e2019-04-06 21:51:27 -06005143{
Jens Axboe62755e32019-10-28 21:49:21 -06005144 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
Jens Axboede0617e2019-04-06 21:51:27 -06005145
Jens Axboe62755e32019-10-28 21:49:21 -06005146 return req->user_data == (unsigned long) data;
5147}
5148
Jens Axboee977d6d2019-11-05 12:39:45 -07005149static int io_async_cancel_one(struct io_ring_ctx *ctx, void *sqe_addr)
Jens Axboe62755e32019-10-28 21:49:21 -06005150{
Jens Axboe62755e32019-10-28 21:49:21 -06005151 enum io_wq_cancel cancel_ret;
Jens Axboe62755e32019-10-28 21:49:21 -06005152 int ret = 0;
5153
Pavel Begunkov4f26bda2020-06-15 10:24:03 +03005154 cancel_ret = io_wq_cancel_cb(ctx->io_wq, io_cancel_cb, sqe_addr, false);
Jens Axboe62755e32019-10-28 21:49:21 -06005155 switch (cancel_ret) {
5156 case IO_WQ_CANCEL_OK:
5157 ret = 0;
5158 break;
5159 case IO_WQ_CANCEL_RUNNING:
5160 ret = -EALREADY;
5161 break;
5162 case IO_WQ_CANCEL_NOTFOUND:
5163 ret = -ENOENT;
5164 break;
5165 }
5166
Jens Axboee977d6d2019-11-05 12:39:45 -07005167 return ret;
5168}
5169
Jens Axboe47f46762019-11-09 17:43:02 -07005170static void io_async_find_and_cancel(struct io_ring_ctx *ctx,
5171 struct io_kiocb *req, __u64 sqe_addr,
Pavel Begunkov014db002020-03-03 21:33:12 +03005172 int success_ret)
Jens Axboe47f46762019-11-09 17:43:02 -07005173{
5174 unsigned long flags;
5175 int ret;
5176
5177 ret = io_async_cancel_one(ctx, (void *) (unsigned long) sqe_addr);
5178 if (ret != -ENOENT) {
5179 spin_lock_irqsave(&ctx->completion_lock, flags);
5180 goto done;
5181 }
5182
5183 spin_lock_irqsave(&ctx->completion_lock, flags);
5184 ret = io_timeout_cancel(ctx, sqe_addr);
5185 if (ret != -ENOENT)
5186 goto done;
5187 ret = io_poll_cancel(ctx, sqe_addr);
5188done:
Jens Axboeb0dd8a42019-11-18 12:14:54 -07005189 if (!ret)
5190 ret = success_ret;
Jens Axboe47f46762019-11-09 17:43:02 -07005191 io_cqring_fill_event(req, ret);
5192 io_commit_cqring(ctx);
5193 spin_unlock_irqrestore(&ctx->completion_lock, flags);
5194 io_cqring_ev_posted(ctx);
5195
Jens Axboe4e88d6e2019-12-07 20:59:47 -07005196 if (ret < 0)
5197 req_set_fail_links(req);
Pavel Begunkov014db002020-03-03 21:33:12 +03005198 io_put_req(req);
Jens Axboe47f46762019-11-09 17:43:02 -07005199}
5200
Jens Axboe3529d8c2019-12-19 18:24:38 -07005201static int io_async_cancel_prep(struct io_kiocb *req,
5202 const struct io_uring_sqe *sqe)
Jens Axboee977d6d2019-11-05 12:39:45 -07005203{
Jens Axboefbf23842019-12-17 18:45:56 -07005204 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
Jens Axboee977d6d2019-11-05 12:39:45 -07005205 return -EINVAL;
5206 if (sqe->flags || sqe->ioprio || sqe->off || sqe->len ||
5207 sqe->cancel_flags)
5208 return -EINVAL;
5209
Jens Axboefbf23842019-12-17 18:45:56 -07005210 req->cancel.addr = READ_ONCE(sqe->addr);
5211 return 0;
5212}
5213
Pavel Begunkov014db002020-03-03 21:33:12 +03005214static int io_async_cancel(struct io_kiocb *req)
Jens Axboefbf23842019-12-17 18:45:56 -07005215{
5216 struct io_ring_ctx *ctx = req->ctx;
Jens Axboefbf23842019-12-17 18:45:56 -07005217
Pavel Begunkov014db002020-03-03 21:33:12 +03005218 io_async_find_and_cancel(ctx, req, req->cancel.addr, 0);
Jens Axboe62755e32019-10-28 21:49:21 -06005219 return 0;
5220}
5221
Jens Axboe05f3fb32019-12-09 11:22:50 -07005222static int io_files_update_prep(struct io_kiocb *req,
5223 const struct io_uring_sqe *sqe)
5224{
5225 if (sqe->flags || sqe->ioprio || sqe->rw_flags)
5226 return -EINVAL;
5227
5228 req->files_update.offset = READ_ONCE(sqe->off);
5229 req->files_update.nr_args = READ_ONCE(sqe->len);
5230 if (!req->files_update.nr_args)
5231 return -EINVAL;
5232 req->files_update.arg = READ_ONCE(sqe->addr);
5233 return 0;
5234}
5235
Jens Axboe229a7b62020-06-22 10:13:11 -06005236static int io_files_update(struct io_kiocb *req, bool force_nonblock,
5237 struct io_comp_state *cs)
Jens Axboe05f3fb32019-12-09 11:22:50 -07005238{
5239 struct io_ring_ctx *ctx = req->ctx;
5240 struct io_uring_files_update up;
5241 int ret;
5242
Jens Axboef86cd202020-01-29 13:46:44 -07005243 if (force_nonblock)
Jens Axboe05f3fb32019-12-09 11:22:50 -07005244 return -EAGAIN;
Jens Axboe05f3fb32019-12-09 11:22:50 -07005245
5246 up.offset = req->files_update.offset;
5247 up.fds = req->files_update.arg;
5248
5249 mutex_lock(&ctx->uring_lock);
5250 ret = __io_sqe_files_update(ctx, &up, req->files_update.nr_args);
5251 mutex_unlock(&ctx->uring_lock);
5252
5253 if (ret < 0)
5254 req_set_fail_links(req);
Jens Axboe229a7b62020-06-22 10:13:11 -06005255 __io_req_complete(req, ret, 0, cs);
Jens Axboe05f3fb32019-12-09 11:22:50 -07005256 return 0;
5257}
5258
Jens Axboe3529d8c2019-12-19 18:24:38 -07005259static int io_req_defer_prep(struct io_kiocb *req,
Pavel Begunkovdebb85f2020-06-29 19:18:42 +03005260 const struct io_uring_sqe *sqe)
Jens Axboef67676d2019-12-02 11:03:47 -07005261{
Jens Axboee7815732019-12-17 19:45:06 -07005262 ssize_t ret = 0;
Jens Axboef67676d2019-12-02 11:03:47 -07005263
Pavel Begunkovf1d96a82020-03-13 22:29:14 +03005264 if (!sqe)
5265 return 0;
5266
Pavel Begunkov710c2bf2020-06-27 14:04:58 +03005267 if (io_op_defs[req->opcode].file_table) {
5268 io_req_init_async(req);
5269 ret = io_grab_files(req);
5270 if (unlikely(ret))
5271 return ret;
5272 }
5273
Jens Axboed625c6e2019-12-17 19:53:05 -07005274 switch (req->opcode) {
Jens Axboee7815732019-12-17 19:45:06 -07005275 case IORING_OP_NOP:
5276 break;
Jens Axboef67676d2019-12-02 11:03:47 -07005277 case IORING_OP_READV:
5278 case IORING_OP_READ_FIXED:
Jens Axboe3a6820f2019-12-22 15:19:35 -07005279 case IORING_OP_READ:
Jens Axboe3529d8c2019-12-19 18:24:38 -07005280 ret = io_read_prep(req, sqe, true);
Jens Axboef67676d2019-12-02 11:03:47 -07005281 break;
5282 case IORING_OP_WRITEV:
5283 case IORING_OP_WRITE_FIXED:
Jens Axboe3a6820f2019-12-22 15:19:35 -07005284 case IORING_OP_WRITE:
Jens Axboe3529d8c2019-12-19 18:24:38 -07005285 ret = io_write_prep(req, sqe, true);
Jens Axboef67676d2019-12-02 11:03:47 -07005286 break;
Jens Axboe0969e782019-12-17 18:40:57 -07005287 case IORING_OP_POLL_ADD:
Jens Axboe3529d8c2019-12-19 18:24:38 -07005288 ret = io_poll_add_prep(req, sqe);
Jens Axboe0969e782019-12-17 18:40:57 -07005289 break;
5290 case IORING_OP_POLL_REMOVE:
Jens Axboe3529d8c2019-12-19 18:24:38 -07005291 ret = io_poll_remove_prep(req, sqe);
Jens Axboe0969e782019-12-17 18:40:57 -07005292 break;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07005293 case IORING_OP_FSYNC:
Jens Axboe3529d8c2019-12-19 18:24:38 -07005294 ret = io_prep_fsync(req, sqe);
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07005295 break;
5296 case IORING_OP_SYNC_FILE_RANGE:
Jens Axboe3529d8c2019-12-19 18:24:38 -07005297 ret = io_prep_sfr(req, sqe);
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07005298 break;
Jens Axboe03b12302019-12-02 18:50:25 -07005299 case IORING_OP_SENDMSG:
Jens Axboefddafac2020-01-04 20:19:44 -07005300 case IORING_OP_SEND:
Jens Axboe3529d8c2019-12-19 18:24:38 -07005301 ret = io_sendmsg_prep(req, sqe);
Jens Axboe03b12302019-12-02 18:50:25 -07005302 break;
5303 case IORING_OP_RECVMSG:
Jens Axboefddafac2020-01-04 20:19:44 -07005304 case IORING_OP_RECV:
Jens Axboe3529d8c2019-12-19 18:24:38 -07005305 ret = io_recvmsg_prep(req, sqe);
Jens Axboe03b12302019-12-02 18:50:25 -07005306 break;
Jens Axboef499a022019-12-02 16:28:46 -07005307 case IORING_OP_CONNECT:
Jens Axboe3529d8c2019-12-19 18:24:38 -07005308 ret = io_connect_prep(req, sqe);
Jens Axboef499a022019-12-02 16:28:46 -07005309 break;
Jens Axboe2d283902019-12-04 11:08:05 -07005310 case IORING_OP_TIMEOUT:
Jens Axboe3529d8c2019-12-19 18:24:38 -07005311 ret = io_timeout_prep(req, sqe, false);
Jens Axboeb7bb4f72019-12-15 22:13:43 -07005312 break;
Jens Axboeb29472e2019-12-17 18:50:29 -07005313 case IORING_OP_TIMEOUT_REMOVE:
Jens Axboe3529d8c2019-12-19 18:24:38 -07005314 ret = io_timeout_remove_prep(req, sqe);
Jens Axboeb29472e2019-12-17 18:50:29 -07005315 break;
Jens Axboefbf23842019-12-17 18:45:56 -07005316 case IORING_OP_ASYNC_CANCEL:
Jens Axboe3529d8c2019-12-19 18:24:38 -07005317 ret = io_async_cancel_prep(req, sqe);
Jens Axboefbf23842019-12-17 18:45:56 -07005318 break;
Jens Axboe2d283902019-12-04 11:08:05 -07005319 case IORING_OP_LINK_TIMEOUT:
Jens Axboe3529d8c2019-12-19 18:24:38 -07005320 ret = io_timeout_prep(req, sqe, true);
Jens Axboeb7bb4f72019-12-15 22:13:43 -07005321 break;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07005322 case IORING_OP_ACCEPT:
Jens Axboe3529d8c2019-12-19 18:24:38 -07005323 ret = io_accept_prep(req, sqe);
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07005324 break;
Jens Axboed63d1b52019-12-10 10:38:56 -07005325 case IORING_OP_FALLOCATE:
5326 ret = io_fallocate_prep(req, sqe);
5327 break;
Jens Axboe15b71ab2019-12-11 11:20:36 -07005328 case IORING_OP_OPENAT:
5329 ret = io_openat_prep(req, sqe);
5330 break;
Jens Axboeb5dba592019-12-11 14:02:38 -07005331 case IORING_OP_CLOSE:
5332 ret = io_close_prep(req, sqe);
5333 break;
Jens Axboe05f3fb32019-12-09 11:22:50 -07005334 case IORING_OP_FILES_UPDATE:
5335 ret = io_files_update_prep(req, sqe);
5336 break;
Jens Axboeeddc7ef2019-12-13 21:18:10 -07005337 case IORING_OP_STATX:
5338 ret = io_statx_prep(req, sqe);
5339 break;
Jens Axboe4840e412019-12-25 22:03:45 -07005340 case IORING_OP_FADVISE:
5341 ret = io_fadvise_prep(req, sqe);
5342 break;
Jens Axboec1ca7572019-12-25 22:18:28 -07005343 case IORING_OP_MADVISE:
5344 ret = io_madvise_prep(req, sqe);
5345 break;
Jens Axboecebdb982020-01-08 17:59:24 -07005346 case IORING_OP_OPENAT2:
5347 ret = io_openat2_prep(req, sqe);
5348 break;
Jens Axboe3e4827b2020-01-08 15:18:09 -07005349 case IORING_OP_EPOLL_CTL:
5350 ret = io_epoll_ctl_prep(req, sqe);
5351 break;
Pavel Begunkov7d67af22020-02-24 11:32:45 +03005352 case IORING_OP_SPLICE:
5353 ret = io_splice_prep(req, sqe);
5354 break;
Jens Axboeddf0322d2020-02-23 16:41:33 -07005355 case IORING_OP_PROVIDE_BUFFERS:
5356 ret = io_provide_buffers_prep(req, sqe);
5357 break;
Jens Axboe067524e2020-03-02 16:32:28 -07005358 case IORING_OP_REMOVE_BUFFERS:
5359 ret = io_remove_buffers_prep(req, sqe);
5360 break;
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03005361 case IORING_OP_TEE:
5362 ret = io_tee_prep(req, sqe);
5363 break;
Jens Axboef67676d2019-12-02 11:03:47 -07005364 default:
Jens Axboee7815732019-12-17 19:45:06 -07005365 printk_once(KERN_WARNING "io_uring: unhandled opcode %d\n",
5366 req->opcode);
5367 ret = -EINVAL;
Jens Axboeb7bb4f72019-12-15 22:13:43 -07005368 break;
Jens Axboef67676d2019-12-02 11:03:47 -07005369 }
5370
Jens Axboeb7bb4f72019-12-15 22:13:43 -07005371 return ret;
Jens Axboef67676d2019-12-02 11:03:47 -07005372}
5373
Jens Axboe3529d8c2019-12-19 18:24:38 -07005374static int io_req_defer(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboede0617e2019-04-06 21:51:27 -06005375{
Jackie Liua197f662019-11-08 08:09:12 -07005376 struct io_ring_ctx *ctx = req->ctx;
Jens Axboef67676d2019-12-02 11:03:47 -07005377 int ret;
Jens Axboede0617e2019-04-06 21:51:27 -06005378
Bob Liu9d858b22019-11-13 18:06:25 +08005379 /* Still need defer if there is pending req in defer list. */
Pavel Begunkov4ee36312020-05-01 17:09:37 +03005380 if (!req_need_defer(req) && list_empty_careful(&ctx->defer_list))
Jens Axboede0617e2019-04-06 21:51:27 -06005381 return 0;
5382
Pavel Begunkov650b5482020-05-17 14:02:11 +03005383 if (!req->io) {
5384 if (io_alloc_async_ctx(req))
5385 return -EAGAIN;
Pavel Begunkovdebb85f2020-06-29 19:18:42 +03005386 ret = io_req_defer_prep(req, sqe);
Pavel Begunkov650b5482020-05-17 14:02:11 +03005387 if (ret < 0)
5388 return ret;
5389 }
Pavel Begunkovcbdcb432020-06-29 19:18:43 +03005390 io_prep_async_link(req);
Jens Axboe2d283902019-12-04 11:08:05 -07005391
Jens Axboede0617e2019-04-06 21:51:27 -06005392 spin_lock_irq(&ctx->completion_lock);
Bob Liu9d858b22019-11-13 18:06:25 +08005393 if (!req_need_defer(req) && list_empty(&ctx->defer_list)) {
Jens Axboede0617e2019-04-06 21:51:27 -06005394 spin_unlock_irq(&ctx->completion_lock);
Jens Axboede0617e2019-04-06 21:51:27 -06005395 return 0;
5396 }
5397
Jens Axboe915967f2019-11-21 09:01:20 -07005398 trace_io_uring_defer(ctx, req, req->user_data);
Jens Axboede0617e2019-04-06 21:51:27 -06005399 list_add_tail(&req->list, &ctx->defer_list);
5400 spin_unlock_irq(&ctx->completion_lock);
5401 return -EIOCBQUEUED;
5402}
5403
Pavel Begunkov99bc4c32020-02-07 22:04:45 +03005404static void io_cleanup_req(struct io_kiocb *req)
5405{
5406 struct io_async_ctx *io = req->io;
5407
5408 switch (req->opcode) {
5409 case IORING_OP_READV:
5410 case IORING_OP_READ_FIXED:
5411 case IORING_OP_READ:
Jens Axboebcda7ba2020-02-23 16:42:51 -07005412 if (req->flags & REQ_F_BUFFER_SELECTED)
5413 kfree((void *)(unsigned long)req->rw.addr);
5414 /* fallthrough */
Pavel Begunkov99bc4c32020-02-07 22:04:45 +03005415 case IORING_OP_WRITEV:
5416 case IORING_OP_WRITE_FIXED:
5417 case IORING_OP_WRITE:
5418 if (io->rw.iov != io->rw.fast_iov)
5419 kfree(io->rw.iov);
5420 break;
Pavel Begunkov99bc4c32020-02-07 22:04:45 +03005421 case IORING_OP_RECVMSG:
Jens Axboe52de1fe2020-02-27 10:15:42 -07005422 if (req->flags & REQ_F_BUFFER_SELECTED)
5423 kfree(req->sr_msg.kbuf);
5424 /* fallthrough */
5425 case IORING_OP_SENDMSG:
Pavel Begunkov99bc4c32020-02-07 22:04:45 +03005426 if (io->msg.iov != io->msg.fast_iov)
5427 kfree(io->msg.iov);
5428 break;
Jens Axboebcda7ba2020-02-23 16:42:51 -07005429 case IORING_OP_RECV:
5430 if (req->flags & REQ_F_BUFFER_SELECTED)
5431 kfree(req->sr_msg.kbuf);
5432 break;
Pavel Begunkov8fef80b2020-02-07 23:59:53 +03005433 case IORING_OP_OPENAT:
5434 case IORING_OP_OPENAT2:
Pavel Begunkov8fef80b2020-02-07 23:59:53 +03005435 break;
Pavel Begunkov7d67af22020-02-24 11:32:45 +03005436 case IORING_OP_SPLICE:
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03005437 case IORING_OP_TEE:
Pavel Begunkov7d67af22020-02-24 11:32:45 +03005438 io_put_file(req, req->splice.file_in,
5439 (req->splice.flags & SPLICE_F_FD_IN_FIXED));
5440 break;
Pavel Begunkov99bc4c32020-02-07 22:04:45 +03005441 }
5442
5443 req->flags &= ~REQ_F_NEED_CLEANUP;
5444}
5445
Jens Axboe3529d8c2019-12-19 18:24:38 -07005446static int io_issue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
Jens Axboef13fad72020-06-22 09:34:30 -06005447 bool force_nonblock, struct io_comp_state *cs)
Jens Axboe2b188cc2019-01-07 10:46:33 -07005448{
Jackie Liua197f662019-11-08 08:09:12 -07005449 struct io_ring_ctx *ctx = req->ctx;
Jens Axboed625c6e2019-12-17 19:53:05 -07005450 int ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07005451
Jens Axboed625c6e2019-12-17 19:53:05 -07005452 switch (req->opcode) {
Jens Axboe2b188cc2019-01-07 10:46:33 -07005453 case IORING_OP_NOP:
Jens Axboe229a7b62020-06-22 10:13:11 -06005454 ret = io_nop(req, cs);
Jens Axboe2b188cc2019-01-07 10:46:33 -07005455 break;
5456 case IORING_OP_READV:
Jens Axboe3529d8c2019-12-19 18:24:38 -07005457 case IORING_OP_READ_FIXED:
Jens Axboe3a6820f2019-12-22 15:19:35 -07005458 case IORING_OP_READ:
Jens Axboe3529d8c2019-12-19 18:24:38 -07005459 if (sqe) {
5460 ret = io_read_prep(req, sqe, force_nonblock);
5461 if (ret < 0)
5462 break;
5463 }
Jens Axboea1d7c392020-06-22 11:09:46 -06005464 ret = io_read(req, force_nonblock, cs);
Jens Axboe2b188cc2019-01-07 10:46:33 -07005465 break;
5466 case IORING_OP_WRITEV:
Jens Axboeedafcce2019-01-09 09:16:05 -07005467 case IORING_OP_WRITE_FIXED:
Jens Axboe3a6820f2019-12-22 15:19:35 -07005468 case IORING_OP_WRITE:
Jens Axboe3529d8c2019-12-19 18:24:38 -07005469 if (sqe) {
5470 ret = io_write_prep(req, sqe, force_nonblock);
5471 if (ret < 0)
5472 break;
5473 }
Jens Axboea1d7c392020-06-22 11:09:46 -06005474 ret = io_write(req, force_nonblock, cs);
Jens Axboe2b188cc2019-01-07 10:46:33 -07005475 break;
Christoph Hellwigc992fe22019-01-11 09:43:02 -07005476 case IORING_OP_FSYNC:
Jens Axboe3529d8c2019-12-19 18:24:38 -07005477 if (sqe) {
5478 ret = io_prep_fsync(req, sqe);
5479 if (ret < 0)
5480 break;
5481 }
Pavel Begunkov014db002020-03-03 21:33:12 +03005482 ret = io_fsync(req, force_nonblock);
Christoph Hellwigc992fe22019-01-11 09:43:02 -07005483 break;
Jens Axboe221c5eb2019-01-17 09:41:58 -07005484 case IORING_OP_POLL_ADD:
Jens Axboe3529d8c2019-12-19 18:24:38 -07005485 if (sqe) {
5486 ret = io_poll_add_prep(req, sqe);
5487 if (ret)
5488 break;
5489 }
Pavel Begunkov014db002020-03-03 21:33:12 +03005490 ret = io_poll_add(req);
Jens Axboe221c5eb2019-01-17 09:41:58 -07005491 break;
5492 case IORING_OP_POLL_REMOVE:
Jens Axboe3529d8c2019-12-19 18:24:38 -07005493 if (sqe) {
5494 ret = io_poll_remove_prep(req, sqe);
5495 if (ret < 0)
5496 break;
5497 }
Jens Axboefc4df992019-12-10 14:38:45 -07005498 ret = io_poll_remove(req);
Jens Axboe221c5eb2019-01-17 09:41:58 -07005499 break;
Jens Axboe5d17b4a2019-04-09 14:56:44 -06005500 case IORING_OP_SYNC_FILE_RANGE:
Jens Axboe3529d8c2019-12-19 18:24:38 -07005501 if (sqe) {
5502 ret = io_prep_sfr(req, sqe);
5503 if (ret < 0)
5504 break;
5505 }
Pavel Begunkov014db002020-03-03 21:33:12 +03005506 ret = io_sync_file_range(req, force_nonblock);
Jens Axboe5d17b4a2019-04-09 14:56:44 -06005507 break;
Jens Axboe0fa03c62019-04-19 13:34:07 -06005508 case IORING_OP_SENDMSG:
Jens Axboefddafac2020-01-04 20:19:44 -07005509 case IORING_OP_SEND:
Jens Axboe3529d8c2019-12-19 18:24:38 -07005510 if (sqe) {
5511 ret = io_sendmsg_prep(req, sqe);
5512 if (ret < 0)
5513 break;
5514 }
Jens Axboefddafac2020-01-04 20:19:44 -07005515 if (req->opcode == IORING_OP_SENDMSG)
Jens Axboe229a7b62020-06-22 10:13:11 -06005516 ret = io_sendmsg(req, force_nonblock, cs);
Jens Axboefddafac2020-01-04 20:19:44 -07005517 else
Jens Axboe229a7b62020-06-22 10:13:11 -06005518 ret = io_send(req, force_nonblock, cs);
Jens Axboe0fa03c62019-04-19 13:34:07 -06005519 break;
Jens Axboeaa1fa282019-04-19 13:38:09 -06005520 case IORING_OP_RECVMSG:
Jens Axboefddafac2020-01-04 20:19:44 -07005521 case IORING_OP_RECV:
Jens Axboe3529d8c2019-12-19 18:24:38 -07005522 if (sqe) {
5523 ret = io_recvmsg_prep(req, sqe);
5524 if (ret)
5525 break;
5526 }
Jens Axboefddafac2020-01-04 20:19:44 -07005527 if (req->opcode == IORING_OP_RECVMSG)
Jens Axboe229a7b62020-06-22 10:13:11 -06005528 ret = io_recvmsg(req, force_nonblock, cs);
Jens Axboefddafac2020-01-04 20:19:44 -07005529 else
Jens Axboe229a7b62020-06-22 10:13:11 -06005530 ret = io_recv(req, force_nonblock, cs);
Jens Axboeaa1fa282019-04-19 13:38:09 -06005531 break;
Jens Axboe5262f562019-09-17 12:26:57 -06005532 case IORING_OP_TIMEOUT:
Jens Axboe3529d8c2019-12-19 18:24:38 -07005533 if (sqe) {
5534 ret = io_timeout_prep(req, sqe, false);
5535 if (ret)
5536 break;
5537 }
Jens Axboefc4df992019-12-10 14:38:45 -07005538 ret = io_timeout(req);
Jens Axboe5262f562019-09-17 12:26:57 -06005539 break;
Jens Axboe11365042019-10-16 09:08:32 -06005540 case IORING_OP_TIMEOUT_REMOVE:
Jens Axboe3529d8c2019-12-19 18:24:38 -07005541 if (sqe) {
5542 ret = io_timeout_remove_prep(req, sqe);
5543 if (ret)
5544 break;
5545 }
Jens Axboefc4df992019-12-10 14:38:45 -07005546 ret = io_timeout_remove(req);
Jens Axboe11365042019-10-16 09:08:32 -06005547 break;
Jens Axboe17f2fe32019-10-17 14:42:58 -06005548 case IORING_OP_ACCEPT:
Jens Axboe3529d8c2019-12-19 18:24:38 -07005549 if (sqe) {
5550 ret = io_accept_prep(req, sqe);
5551 if (ret)
5552 break;
5553 }
Jens Axboe229a7b62020-06-22 10:13:11 -06005554 ret = io_accept(req, force_nonblock, cs);
Jens Axboe17f2fe32019-10-17 14:42:58 -06005555 break;
Jens Axboef8e85cf2019-11-23 14:24:24 -07005556 case IORING_OP_CONNECT:
Jens Axboe3529d8c2019-12-19 18:24:38 -07005557 if (sqe) {
5558 ret = io_connect_prep(req, sqe);
5559 if (ret)
5560 break;
5561 }
Jens Axboe229a7b62020-06-22 10:13:11 -06005562 ret = io_connect(req, force_nonblock, cs);
Jens Axboef8e85cf2019-11-23 14:24:24 -07005563 break;
Jens Axboe62755e32019-10-28 21:49:21 -06005564 case IORING_OP_ASYNC_CANCEL:
Jens Axboe3529d8c2019-12-19 18:24:38 -07005565 if (sqe) {
5566 ret = io_async_cancel_prep(req, sqe);
5567 if (ret)
5568 break;
5569 }
Pavel Begunkov014db002020-03-03 21:33:12 +03005570 ret = io_async_cancel(req);
Jens Axboe62755e32019-10-28 21:49:21 -06005571 break;
Jens Axboed63d1b52019-12-10 10:38:56 -07005572 case IORING_OP_FALLOCATE:
5573 if (sqe) {
5574 ret = io_fallocate_prep(req, sqe);
5575 if (ret)
5576 break;
5577 }
Pavel Begunkov014db002020-03-03 21:33:12 +03005578 ret = io_fallocate(req, force_nonblock);
Jens Axboed63d1b52019-12-10 10:38:56 -07005579 break;
Jens Axboe15b71ab2019-12-11 11:20:36 -07005580 case IORING_OP_OPENAT:
5581 if (sqe) {
5582 ret = io_openat_prep(req, sqe);
5583 if (ret)
5584 break;
5585 }
Pavel Begunkov014db002020-03-03 21:33:12 +03005586 ret = io_openat(req, force_nonblock);
Jens Axboe15b71ab2019-12-11 11:20:36 -07005587 break;
Jens Axboeb5dba592019-12-11 14:02:38 -07005588 case IORING_OP_CLOSE:
5589 if (sqe) {
5590 ret = io_close_prep(req, sqe);
5591 if (ret)
5592 break;
5593 }
Jens Axboe229a7b62020-06-22 10:13:11 -06005594 ret = io_close(req, force_nonblock, cs);
Jens Axboeb5dba592019-12-11 14:02:38 -07005595 break;
Jens Axboe05f3fb32019-12-09 11:22:50 -07005596 case IORING_OP_FILES_UPDATE:
5597 if (sqe) {
5598 ret = io_files_update_prep(req, sqe);
5599 if (ret)
5600 break;
5601 }
Jens Axboe229a7b62020-06-22 10:13:11 -06005602 ret = io_files_update(req, force_nonblock, cs);
Jens Axboe05f3fb32019-12-09 11:22:50 -07005603 break;
Jens Axboeeddc7ef2019-12-13 21:18:10 -07005604 case IORING_OP_STATX:
5605 if (sqe) {
5606 ret = io_statx_prep(req, sqe);
5607 if (ret)
5608 break;
5609 }
Pavel Begunkov014db002020-03-03 21:33:12 +03005610 ret = io_statx(req, force_nonblock);
Jens Axboeeddc7ef2019-12-13 21:18:10 -07005611 break;
Jens Axboe4840e412019-12-25 22:03:45 -07005612 case IORING_OP_FADVISE:
5613 if (sqe) {
5614 ret = io_fadvise_prep(req, sqe);
5615 if (ret)
5616 break;
5617 }
Pavel Begunkov014db002020-03-03 21:33:12 +03005618 ret = io_fadvise(req, force_nonblock);
Jens Axboe4840e412019-12-25 22:03:45 -07005619 break;
Jens Axboec1ca7572019-12-25 22:18:28 -07005620 case IORING_OP_MADVISE:
5621 if (sqe) {
5622 ret = io_madvise_prep(req, sqe);
5623 if (ret)
5624 break;
5625 }
Pavel Begunkov014db002020-03-03 21:33:12 +03005626 ret = io_madvise(req, force_nonblock);
Jens Axboec1ca7572019-12-25 22:18:28 -07005627 break;
Jens Axboecebdb982020-01-08 17:59:24 -07005628 case IORING_OP_OPENAT2:
5629 if (sqe) {
5630 ret = io_openat2_prep(req, sqe);
5631 if (ret)
5632 break;
5633 }
Pavel Begunkov014db002020-03-03 21:33:12 +03005634 ret = io_openat2(req, force_nonblock);
Jens Axboecebdb982020-01-08 17:59:24 -07005635 break;
Jens Axboe3e4827b2020-01-08 15:18:09 -07005636 case IORING_OP_EPOLL_CTL:
5637 if (sqe) {
5638 ret = io_epoll_ctl_prep(req, sqe);
5639 if (ret)
5640 break;
5641 }
Jens Axboe229a7b62020-06-22 10:13:11 -06005642 ret = io_epoll_ctl(req, force_nonblock, cs);
Jens Axboe3e4827b2020-01-08 15:18:09 -07005643 break;
Pavel Begunkov7d67af22020-02-24 11:32:45 +03005644 case IORING_OP_SPLICE:
5645 if (sqe) {
5646 ret = io_splice_prep(req, sqe);
5647 if (ret < 0)
5648 break;
5649 }
Pavel Begunkov014db002020-03-03 21:33:12 +03005650 ret = io_splice(req, force_nonblock);
Pavel Begunkov7d67af22020-02-24 11:32:45 +03005651 break;
Jens Axboeddf0322d2020-02-23 16:41:33 -07005652 case IORING_OP_PROVIDE_BUFFERS:
5653 if (sqe) {
5654 ret = io_provide_buffers_prep(req, sqe);
5655 if (ret)
5656 break;
5657 }
Jens Axboe229a7b62020-06-22 10:13:11 -06005658 ret = io_provide_buffers(req, force_nonblock, cs);
Jens Axboeddf0322d2020-02-23 16:41:33 -07005659 break;
Jens Axboe067524e2020-03-02 16:32:28 -07005660 case IORING_OP_REMOVE_BUFFERS:
5661 if (sqe) {
5662 ret = io_remove_buffers_prep(req, sqe);
5663 if (ret)
5664 break;
5665 }
Jens Axboe229a7b62020-06-22 10:13:11 -06005666 ret = io_remove_buffers(req, force_nonblock, cs);
Jens Axboe31b51512019-01-18 22:56:34 -07005667 break;
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03005668 case IORING_OP_TEE:
5669 if (sqe) {
5670 ret = io_tee_prep(req, sqe);
5671 if (ret < 0)
5672 break;
5673 }
5674 ret = io_tee(req, force_nonblock);
5675 break;
Jens Axboe2b188cc2019-01-07 10:46:33 -07005676 default:
5677 ret = -EINVAL;
5678 break;
5679 }
5680
5681 if (ret)
5682 return ret;
5683
Jens Axboeb5325762020-05-19 21:20:27 -06005684 /* If the op doesn't have a file, we're not polling for it */
5685 if ((ctx->flags & IORING_SETUP_IOPOLL) && req->file) {
Jens Axboe11ba8202020-01-15 21:51:17 -07005686 const bool in_async = io_wq_current_is_worker();
5687
Jens Axboe11ba8202020-01-15 21:51:17 -07005688 /* workqueue context doesn't hold uring_lock, grab it now */
5689 if (in_async)
5690 mutex_lock(&ctx->uring_lock);
5691
Jens Axboe2b188cc2019-01-07 10:46:33 -07005692 io_iopoll_req_issued(req);
Jens Axboe11ba8202020-01-15 21:51:17 -07005693
5694 if (in_async)
5695 mutex_unlock(&ctx->uring_lock);
Jens Axboedef596e2019-01-09 08:59:42 -07005696 }
5697
5698 return 0;
5699}
5700
Pavel Begunkovd4c81f32020-06-08 21:08:19 +03005701static void io_arm_async_linked_timeout(struct io_kiocb *req)
5702{
5703 struct io_kiocb *link;
5704
5705 /* link head's timeout is queued in io_queue_async_work() */
5706 if (!(req->flags & REQ_F_QUEUE_TIMEOUT))
5707 return;
5708
5709 link = list_first_entry(&req->link_list, struct io_kiocb, link_list);
5710 io_queue_linked_timeout(link);
5711}
5712
Pavel Begunkovf4db7182020-06-25 18:20:54 +03005713static struct io_wq_work *io_wq_submit_work(struct io_wq_work *work)
Jens Axboedef596e2019-01-09 08:59:42 -07005714{
Jens Axboe2b188cc2019-01-07 10:46:33 -07005715 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
Jens Axboe561fb042019-10-24 07:25:42 -06005716 int ret = 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07005717
Pavel Begunkovd4c81f32020-06-08 21:08:19 +03005718 io_arm_async_linked_timeout(req);
5719
Jens Axboe0c9d5cc2019-12-11 19:29:43 -07005720 /* if NO_CANCEL is set, we must still run the work */
5721 if ((work->flags & (IO_WQ_WORK_CANCEL|IO_WQ_WORK_NO_CANCEL)) ==
5722 IO_WQ_WORK_CANCEL) {
Jens Axboe561fb042019-10-24 07:25:42 -06005723 ret = -ECANCELED;
Jens Axboe0c9d5cc2019-12-11 19:29:43 -07005724 }
Jens Axboe31b51512019-01-18 22:56:34 -07005725
Jens Axboe561fb042019-10-24 07:25:42 -06005726 if (!ret) {
Jens Axboe561fb042019-10-24 07:25:42 -06005727 do {
Jens Axboef13fad72020-06-22 09:34:30 -06005728 ret = io_issue_sqe(req, NULL, false, NULL);
Jens Axboe561fb042019-10-24 07:25:42 -06005729 /*
5730 * We can get EAGAIN for polled IO even though we're
5731 * forcing a sync submission from here, since we can't
5732 * wait for request slots on the block side.
5733 */
5734 if (ret != -EAGAIN)
5735 break;
5736 cond_resched();
5737 } while (1);
5738 }
Jens Axboe31b51512019-01-18 22:56:34 -07005739
Jens Axboe561fb042019-10-24 07:25:42 -06005740 if (ret) {
Jens Axboe4e88d6e2019-12-07 20:59:47 -07005741 req_set_fail_links(req);
Jens Axboee1e16092020-06-22 09:17:17 -06005742 io_req_complete(req, ret);
Jens Axboeedafcce2019-01-09 09:16:05 -07005743 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07005744
Pavel Begunkovf4db7182020-06-25 18:20:54 +03005745 return io_steal_work(req);
Jens Axboe31b51512019-01-18 22:56:34 -07005746}
Jens Axboe2b188cc2019-01-07 10:46:33 -07005747
Jens Axboe65e19f52019-10-26 07:20:21 -06005748static inline struct file *io_file_from_index(struct io_ring_ctx *ctx,
5749 int index)
Jens Axboe09bb8392019-03-13 12:39:28 -06005750{
Jens Axboe65e19f52019-10-26 07:20:21 -06005751 struct fixed_file_table *table;
5752
Jens Axboe05f3fb32019-12-09 11:22:50 -07005753 table = &ctx->file_data->table[index >> IORING_FILE_TABLE_SHIFT];
Xiaoming Ni84695082020-05-11 19:25:43 +08005754 return table->files[index & IORING_FILE_TABLE_MASK];
Jens Axboe65e19f52019-10-26 07:20:21 -06005755}
5756
Pavel Begunkov8da11c12020-02-24 11:32:44 +03005757static int io_file_get(struct io_submit_state *state, struct io_kiocb *req,
5758 int fd, struct file **out_file, bool fixed)
5759{
5760 struct io_ring_ctx *ctx = req->ctx;
5761 struct file *file;
5762
5763 if (fixed) {
5764 if (unlikely(!ctx->file_data ||
5765 (unsigned) fd >= ctx->nr_user_files))
5766 return -EBADF;
5767 fd = array_index_nospec(fd, ctx->nr_user_files);
5768 file = io_file_from_index(ctx, fd);
Jens Axboefd2206e2020-06-02 16:40:47 -06005769 if (file) {
5770 req->fixed_file_refs = ctx->file_data->cur_refs;
5771 percpu_ref_get(req->fixed_file_refs);
5772 }
Pavel Begunkov8da11c12020-02-24 11:32:44 +03005773 } else {
5774 trace_io_uring_file_get(ctx, fd);
5775 file = __io_file_get(state, fd);
Pavel Begunkov8da11c12020-02-24 11:32:44 +03005776 }
5777
Jens Axboefd2206e2020-06-02 16:40:47 -06005778 if (file || io_op_defs[req->opcode].needs_file_no_error) {
5779 *out_file = file;
5780 return 0;
5781 }
5782 return -EBADF;
Pavel Begunkov8da11c12020-02-24 11:32:44 +03005783}
5784
Jens Axboe3529d8c2019-12-19 18:24:38 -07005785static int io_req_set_file(struct io_submit_state *state, struct io_kiocb *req,
Jens Axboe63ff8222020-05-07 14:56:15 -06005786 int fd)
Jens Axboe09bb8392019-03-13 12:39:28 -06005787{
Pavel Begunkov8da11c12020-02-24 11:32:44 +03005788 bool fixed;
Jens Axboe09bb8392019-03-13 12:39:28 -06005789
Jens Axboe63ff8222020-05-07 14:56:15 -06005790 fixed = (req->flags & REQ_F_FIXED_FILE) != 0;
Pavel Begunkov0cdaf762020-05-17 14:13:40 +03005791 if (unlikely(!fixed && io_async_submit(req->ctx)))
Pavel Begunkov8da11c12020-02-24 11:32:44 +03005792 return -EBADF;
Jens Axboe09bb8392019-03-13 12:39:28 -06005793
Pavel Begunkov8da11c12020-02-24 11:32:44 +03005794 return io_file_get(state, req, fd, &req->file, fixed);
Jens Axboe09bb8392019-03-13 12:39:28 -06005795}
5796
Jackie Liua197f662019-11-08 08:09:12 -07005797static int io_grab_files(struct io_kiocb *req)
Jens Axboe2b188cc2019-01-07 10:46:33 -07005798{
Jens Axboefcb323c2019-10-24 12:39:47 -06005799 int ret = -EBADF;
Jackie Liua197f662019-11-08 08:09:12 -07005800 struct io_ring_ctx *ctx = req->ctx;
Jens Axboefcb323c2019-10-24 12:39:47 -06005801
Jens Axboe5b0bbee2020-04-27 10:41:22 -06005802 if (req->work.files || (req->flags & REQ_F_NO_FILE_TABLE))
Jens Axboef86cd202020-01-29 13:46:44 -07005803 return 0;
Pavel Begunkovb14cca02020-01-17 04:45:59 +03005804 if (!ctx->ring_file)
Jens Axboeb5dba592019-12-11 14:02:38 -07005805 return -EBADF;
5806
Jens Axboefcb323c2019-10-24 12:39:47 -06005807 rcu_read_lock();
5808 spin_lock_irq(&ctx->inflight_lock);
5809 /*
5810 * We use the f_ops->flush() handler to ensure that we can flush
5811 * out work accessing these files if the fd is closed. Check if
5812 * the fd has changed since we started down this path, and disallow
5813 * this operation if it has.
5814 */
Pavel Begunkovb14cca02020-01-17 04:45:59 +03005815 if (fcheck(ctx->ring_fd) == ctx->ring_file) {
Jens Axboefcb323c2019-10-24 12:39:47 -06005816 list_add(&req->inflight_entry, &ctx->inflight_list);
5817 req->flags |= REQ_F_INFLIGHT;
5818 req->work.files = current->files;
5819 ret = 0;
5820 }
5821 spin_unlock_irq(&ctx->inflight_lock);
5822 rcu_read_unlock();
5823
5824 return ret;
5825}
5826
Jens Axboe2665abf2019-11-05 12:40:47 -07005827static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer)
5828{
Jens Axboead8a48a2019-11-15 08:49:11 -07005829 struct io_timeout_data *data = container_of(timer,
5830 struct io_timeout_data, timer);
5831 struct io_kiocb *req = data->req;
Jens Axboe2665abf2019-11-05 12:40:47 -07005832 struct io_ring_ctx *ctx = req->ctx;
5833 struct io_kiocb *prev = NULL;
5834 unsigned long flags;
Jens Axboe2665abf2019-11-05 12:40:47 -07005835
5836 spin_lock_irqsave(&ctx->completion_lock, flags);
5837
5838 /*
5839 * We don't expect the list to be empty, that will only happen if we
5840 * race with the completion of the linked work.
5841 */
Pavel Begunkov44932332019-12-05 16:16:35 +03005842 if (!list_empty(&req->link_list)) {
5843 prev = list_entry(req->link_list.prev, struct io_kiocb,
5844 link_list);
Jens Axboe5d960722019-11-19 15:31:28 -07005845 if (refcount_inc_not_zero(&prev->refs)) {
Pavel Begunkov44932332019-12-05 16:16:35 +03005846 list_del_init(&req->link_list);
Jens Axboe5d960722019-11-19 15:31:28 -07005847 prev->flags &= ~REQ_F_LINK_TIMEOUT;
5848 } else
Jens Axboe76a46e02019-11-10 23:34:16 -07005849 prev = NULL;
Jens Axboe2665abf2019-11-05 12:40:47 -07005850 }
5851
5852 spin_unlock_irqrestore(&ctx->completion_lock, flags);
5853
5854 if (prev) {
Jens Axboe4e88d6e2019-12-07 20:59:47 -07005855 req_set_fail_links(prev);
Pavel Begunkov014db002020-03-03 21:33:12 +03005856 io_async_find_and_cancel(ctx, req, prev->user_data, -ETIME);
Jens Axboe76a46e02019-11-10 23:34:16 -07005857 io_put_req(prev);
Jens Axboe47f46762019-11-09 17:43:02 -07005858 } else {
Jens Axboee1e16092020-06-22 09:17:17 -06005859 io_req_complete(req, -ETIME);
Jens Axboe2665abf2019-11-05 12:40:47 -07005860 }
Jens Axboe2665abf2019-11-05 12:40:47 -07005861 return HRTIMER_NORESTART;
5862}
5863
Jens Axboead8a48a2019-11-15 08:49:11 -07005864static void io_queue_linked_timeout(struct io_kiocb *req)
Jens Axboe2665abf2019-11-05 12:40:47 -07005865{
Jens Axboe76a46e02019-11-10 23:34:16 -07005866 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe2665abf2019-11-05 12:40:47 -07005867
Jens Axboe76a46e02019-11-10 23:34:16 -07005868 /*
5869 * If the list is now empty, then our linked request finished before
5870 * we got a chance to setup the timer
5871 */
5872 spin_lock_irq(&ctx->completion_lock);
Pavel Begunkov44932332019-12-05 16:16:35 +03005873 if (!list_empty(&req->link_list)) {
Jens Axboe2d283902019-12-04 11:08:05 -07005874 struct io_timeout_data *data = &req->io->timeout;
Jens Axboe94ae5e72019-11-14 19:39:52 -07005875
Jens Axboead8a48a2019-11-15 08:49:11 -07005876 data->timer.function = io_link_timeout_fn;
5877 hrtimer_start(&data->timer, timespec64_to_ktime(data->ts),
5878 data->mode);
Jens Axboe2665abf2019-11-05 12:40:47 -07005879 }
Jens Axboe76a46e02019-11-10 23:34:16 -07005880 spin_unlock_irq(&ctx->completion_lock);
Jens Axboe2665abf2019-11-05 12:40:47 -07005881
Jens Axboe2665abf2019-11-05 12:40:47 -07005882 /* drop submission reference */
Jens Axboe76a46e02019-11-10 23:34:16 -07005883 io_put_req(req);
Jens Axboe2665abf2019-11-05 12:40:47 -07005884}
5885
Jens Axboead8a48a2019-11-15 08:49:11 -07005886static struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req)
Jens Axboe2665abf2019-11-05 12:40:47 -07005887{
5888 struct io_kiocb *nxt;
Jens Axboe2b188cc2019-01-07 10:46:33 -07005889
Pavel Begunkovdea3b492020-04-12 02:05:04 +03005890 if (!(req->flags & REQ_F_LINK_HEAD))
Jens Axboe2665abf2019-11-05 12:40:47 -07005891 return NULL;
Jens Axboed7718a92020-02-14 22:23:12 -07005892 /* for polled retry, if flag is set, we already went through here */
5893 if (req->flags & REQ_F_POLLED)
5894 return NULL;
Jens Axboe2665abf2019-11-05 12:40:47 -07005895
Pavel Begunkov44932332019-12-05 16:16:35 +03005896 nxt = list_first_entry_or_null(&req->link_list, struct io_kiocb,
5897 link_list);
Jens Axboed625c6e2019-12-17 19:53:05 -07005898 if (!nxt || nxt->opcode != IORING_OP_LINK_TIMEOUT)
Jens Axboe76a46e02019-11-10 23:34:16 -07005899 return NULL;
Jens Axboe2665abf2019-11-05 12:40:47 -07005900
Jens Axboe76a46e02019-11-10 23:34:16 -07005901 req->flags |= REQ_F_LINK_TIMEOUT;
Jens Axboe76a46e02019-11-10 23:34:16 -07005902 return nxt;
Jens Axboe2665abf2019-11-05 12:40:47 -07005903}
5904
Jens Axboef13fad72020-06-22 09:34:30 -06005905static void __io_queue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
5906 struct io_comp_state *cs)
Jens Axboe2b188cc2019-01-07 10:46:33 -07005907{
Jens Axboe4a0a7a12019-12-09 20:01:01 -07005908 struct io_kiocb *linked_timeout;
Pavel Begunkov4bc44942020-02-29 22:48:24 +03005909 struct io_kiocb *nxt;
Jens Axboe193155c2020-02-22 23:22:19 -07005910 const struct cred *old_creds = NULL;
Jens Axboe2b188cc2019-01-07 10:46:33 -07005911 int ret;
5912
Jens Axboe4a0a7a12019-12-09 20:01:01 -07005913again:
5914 linked_timeout = io_prep_linked_timeout(req);
5915
Xiaoguang Wang7cdaf582020-06-10 19:41:19 +08005916 if ((req->flags & REQ_F_WORK_INITIALIZED) && req->work.creds &&
5917 req->work.creds != current_cred()) {
Jens Axboe193155c2020-02-22 23:22:19 -07005918 if (old_creds)
5919 revert_creds(old_creds);
5920 if (old_creds == req->work.creds)
5921 old_creds = NULL; /* restored original creds */
5922 else
5923 old_creds = override_creds(req->work.creds);
5924 }
5925
Jens Axboef13fad72020-06-22 09:34:30 -06005926 ret = io_issue_sqe(req, sqe, true, cs);
Jens Axboe491381ce2019-10-17 09:20:46 -06005927
5928 /*
5929 * We async punt it if the file wasn't marked NOWAIT, or if the file
5930 * doesn't support non-blocking read/write attempts
5931 */
Pavel Begunkov24c74672020-06-21 13:09:51 +03005932 if (ret == -EAGAIN && !(req->flags & REQ_F_NOWAIT)) {
Jens Axboed7718a92020-02-14 22:23:12 -07005933 if (io_arm_poll_handler(req)) {
5934 if (linked_timeout)
5935 io_queue_linked_timeout(linked_timeout);
Pavel Begunkov4bc44942020-02-29 22:48:24 +03005936 goto exit;
Jens Axboed7718a92020-02-14 22:23:12 -07005937 }
Pavel Begunkov86a761f2020-01-22 23:09:36 +03005938punt:
Xiaoguang Wang7cdaf582020-06-10 19:41:19 +08005939 io_req_init_async(req);
5940
Jens Axboef86cd202020-01-29 13:46:44 -07005941 if (io_op_defs[req->opcode].file_table) {
Pavel Begunkovbbad27b2019-11-19 23:32:47 +03005942 ret = io_grab_files(req);
5943 if (ret)
5944 goto err;
Jens Axboe2b188cc2019-01-07 10:46:33 -07005945 }
Pavel Begunkovbbad27b2019-11-19 23:32:47 +03005946
5947 /*
5948 * Queued up for async execution, worker will release
5949 * submit reference when the iocb is actually submitted.
5950 */
5951 io_queue_async_work(req);
Pavel Begunkov4bc44942020-02-29 22:48:24 +03005952 goto exit;
Jens Axboe2b188cc2019-01-07 10:46:33 -07005953 }
Jens Axboee65ef562019-03-12 10:16:44 -06005954
Jens Axboefcb323c2019-10-24 12:39:47 -06005955err:
Jens Axboee65ef562019-03-12 10:16:44 -06005956 /* drop submission reference */
Pavel Begunkov9b5f7bd2020-06-29 13:13:00 +03005957 nxt = io_put_req_find_next(req);
Jens Axboee65ef562019-03-12 10:16:44 -06005958
Pavel Begunkovf9bd67f2019-11-21 23:21:03 +03005959 if (linked_timeout) {
Jens Axboe76a46e02019-11-10 23:34:16 -07005960 if (!ret)
Pavel Begunkovf9bd67f2019-11-21 23:21:03 +03005961 io_queue_linked_timeout(linked_timeout);
Jens Axboe76a46e02019-11-10 23:34:16 -07005962 else
Pavel Begunkovf9bd67f2019-11-21 23:21:03 +03005963 io_put_req(linked_timeout);
Jens Axboe76a46e02019-11-10 23:34:16 -07005964 }
5965
Jens Axboee65ef562019-03-12 10:16:44 -06005966 /* and drop final reference, if we failed */
Jens Axboe9e645e112019-05-10 16:07:28 -06005967 if (ret) {
Jens Axboe4e88d6e2019-12-07 20:59:47 -07005968 req_set_fail_links(req);
Jens Axboee1e16092020-06-22 09:17:17 -06005969 io_req_complete(req, ret);
Jens Axboe9e645e112019-05-10 16:07:28 -06005970 }
Jens Axboe4a0a7a12019-12-09 20:01:01 -07005971 if (nxt) {
5972 req = nxt;
Pavel Begunkov86a761f2020-01-22 23:09:36 +03005973
5974 if (req->flags & REQ_F_FORCE_ASYNC)
5975 goto punt;
Jens Axboe4a0a7a12019-12-09 20:01:01 -07005976 goto again;
5977 }
Pavel Begunkov4bc44942020-02-29 22:48:24 +03005978exit:
Jens Axboe193155c2020-02-22 23:22:19 -07005979 if (old_creds)
5980 revert_creds(old_creds);
Jens Axboe2b188cc2019-01-07 10:46:33 -07005981}
5982
Jens Axboef13fad72020-06-22 09:34:30 -06005983static void io_queue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
5984 struct io_comp_state *cs)
Jackie Liu4fe2c962019-09-09 20:50:40 +08005985{
5986 int ret;
5987
Jens Axboe3529d8c2019-12-19 18:24:38 -07005988 ret = io_req_defer(req, sqe);
Jackie Liu4fe2c962019-09-09 20:50:40 +08005989 if (ret) {
5990 if (ret != -EIOCBQUEUED) {
Pavel Begunkov11185912020-01-22 23:09:35 +03005991fail_req:
Jens Axboe4e88d6e2019-12-07 20:59:47 -07005992 req_set_fail_links(req);
Jens Axboee1e16092020-06-22 09:17:17 -06005993 io_put_req(req);
5994 io_req_complete(req, ret);
Jackie Liu4fe2c962019-09-09 20:50:40 +08005995 }
Pavel Begunkov25508782019-12-30 21:24:47 +03005996 } else if (req->flags & REQ_F_FORCE_ASYNC) {
Pavel Begunkovbd2ab182020-05-17 14:02:12 +03005997 if (!req->io) {
5998 ret = -EAGAIN;
5999 if (io_alloc_async_ctx(req))
6000 goto fail_req;
Pavel Begunkovdebb85f2020-06-29 19:18:42 +03006001 ret = io_req_defer_prep(req, sqe);
Pavel Begunkovbd2ab182020-05-17 14:02:12 +03006002 if (unlikely(ret < 0))
6003 goto fail_req;
6004 }
6005
Jens Axboece35a472019-12-17 08:04:44 -07006006 /*
6007 * Never try inline submit of IOSQE_ASYNC is set, go straight
6008 * to async execution.
6009 */
6010 req->work.flags |= IO_WQ_WORK_CONCURRENT;
6011 io_queue_async_work(req);
6012 } else {
Jens Axboef13fad72020-06-22 09:34:30 -06006013 __io_queue_sqe(req, sqe, cs);
Jens Axboece35a472019-12-17 08:04:44 -07006014 }
Jackie Liu4fe2c962019-09-09 20:50:40 +08006015}
6016
Jens Axboef13fad72020-06-22 09:34:30 -06006017static inline void io_queue_link_head(struct io_kiocb *req,
6018 struct io_comp_state *cs)
Jackie Liu4fe2c962019-09-09 20:50:40 +08006019{
Jens Axboe94ae5e72019-11-14 19:39:52 -07006020 if (unlikely(req->flags & REQ_F_FAIL_LINK)) {
Jens Axboee1e16092020-06-22 09:17:17 -06006021 io_put_req(req);
6022 io_req_complete(req, -ECANCELED);
Pavel Begunkov1b4a51b2019-11-21 11:54:28 +03006023 } else
Jens Axboef13fad72020-06-22 09:34:30 -06006024 io_queue_sqe(req, NULL, cs);
Jackie Liu4fe2c962019-09-09 20:50:40 +08006025}
6026
Pavel Begunkov1d4240c2020-04-12 02:05:03 +03006027static int io_submit_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
Jens Axboef13fad72020-06-22 09:34:30 -06006028 struct io_kiocb **link, struct io_comp_state *cs)
Jens Axboe9e645e112019-05-10 16:07:28 -06006029{
Jackie Liua197f662019-11-08 08:09:12 -07006030 struct io_ring_ctx *ctx = req->ctx;
Pavel Begunkovef4ff582020-04-12 02:05:05 +03006031 int ret;
Jens Axboe9e645e112019-05-10 16:07:28 -06006032
Jens Axboe9e645e112019-05-10 16:07:28 -06006033 /*
6034 * If we already have a head request, queue this one for async
6035 * submittal once the head completes. If we don't have a head but
6036 * IOSQE_IO_LINK is set in the sqe, start a new head. This one will be
6037 * submitted sync once the chain is complete. If none of those
6038 * conditions are true (normal request), then just queue it.
6039 */
6040 if (*link) {
Pavel Begunkov9d763772019-12-17 02:22:07 +03006041 struct io_kiocb *head = *link;
Jens Axboe9e645e112019-05-10 16:07:28 -06006042
Pavel Begunkov8cdf2192020-01-25 00:40:24 +03006043 /*
6044 * Taking sequential execution of a link, draining both sides
6045 * of the link also fullfils IOSQE_IO_DRAIN semantics for all
6046 * requests in the link. So, it drains the head and the
6047 * next after the link request. The last one is done via
6048 * drain_next flag to persist the effect across calls.
6049 */
Pavel Begunkovef4ff582020-04-12 02:05:05 +03006050 if (req->flags & REQ_F_IO_DRAIN) {
Pavel Begunkov711be032020-01-17 03:57:59 +03006051 head->flags |= REQ_F_IO_DRAIN;
6052 ctx->drain_next = 1;
6053 }
Pavel Begunkov1d4240c2020-04-12 02:05:03 +03006054 if (io_alloc_async_ctx(req))
6055 return -EAGAIN;
Jens Axboe9e645e112019-05-10 16:07:28 -06006056
Pavel Begunkovdebb85f2020-06-29 19:18:42 +03006057 ret = io_req_defer_prep(req, sqe);
Jens Axboe2d283902019-12-04 11:08:05 -07006058 if (ret) {
Jens Axboe4e88d6e2019-12-07 20:59:47 -07006059 /* fail even hard links since we don't submit */
Pavel Begunkov9d763772019-12-17 02:22:07 +03006060 head->flags |= REQ_F_FAIL_LINK;
Pavel Begunkov1d4240c2020-04-12 02:05:03 +03006061 return ret;
Jens Axboe2d283902019-12-04 11:08:05 -07006062 }
Pavel Begunkov9d763772019-12-17 02:22:07 +03006063 trace_io_uring_link(ctx, req, head);
Jens Axboec40f6372020-06-25 15:39:59 -06006064 io_get_req_task(req);
Pavel Begunkov9d763772019-12-17 02:22:07 +03006065 list_add_tail(&req->link_list, &head->link_list);
Jens Axboe9e645e112019-05-10 16:07:28 -06006066
Pavel Begunkov32fe5252019-12-17 22:26:58 +03006067 /* last request of a link, enqueue the link */
Pavel Begunkovef4ff582020-04-12 02:05:05 +03006068 if (!(req->flags & (REQ_F_LINK | REQ_F_HARDLINK))) {
Jens Axboef13fad72020-06-22 09:34:30 -06006069 io_queue_link_head(head, cs);
Pavel Begunkov32fe5252019-12-17 22:26:58 +03006070 *link = NULL;
6071 }
Jens Axboe9e645e112019-05-10 16:07:28 -06006072 } else {
Pavel Begunkov711be032020-01-17 03:57:59 +03006073 if (unlikely(ctx->drain_next)) {
6074 req->flags |= REQ_F_IO_DRAIN;
Pavel Begunkovef4ff582020-04-12 02:05:05 +03006075 ctx->drain_next = 0;
Pavel Begunkov711be032020-01-17 03:57:59 +03006076 }
Pavel Begunkovef4ff582020-04-12 02:05:05 +03006077 if (req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) {
Pavel Begunkovdea3b492020-04-12 02:05:04 +03006078 req->flags |= REQ_F_LINK_HEAD;
Pavel Begunkov711be032020-01-17 03:57:59 +03006079 INIT_LIST_HEAD(&req->link_list);
Pavel Begunkovf1d96a82020-03-13 22:29:14 +03006080
Pavel Begunkov1d4240c2020-04-12 02:05:03 +03006081 if (io_alloc_async_ctx(req))
6082 return -EAGAIN;
6083
Pavel Begunkovdebb85f2020-06-29 19:18:42 +03006084 ret = io_req_defer_prep(req, sqe);
Pavel Begunkov711be032020-01-17 03:57:59 +03006085 if (ret)
6086 req->flags |= REQ_F_FAIL_LINK;
6087 *link = req;
6088 } else {
Jens Axboef13fad72020-06-22 09:34:30 -06006089 io_queue_sqe(req, sqe, cs);
Pavel Begunkov711be032020-01-17 03:57:59 +03006090 }
Jens Axboe9e645e112019-05-10 16:07:28 -06006091 }
Pavel Begunkov2e6e1fd2019-12-05 16:15:45 +03006092
Pavel Begunkov1d4240c2020-04-12 02:05:03 +03006093 return 0;
Jens Axboe9e645e112019-05-10 16:07:28 -06006094}
6095
Jens Axboe9a56a232019-01-09 09:06:50 -07006096/*
6097 * Batched submission is done, ensure local IO is flushed out.
6098 */
6099static void io_submit_state_end(struct io_submit_state *state)
6100{
Jens Axboef13fad72020-06-22 09:34:30 -06006101 if (!list_empty(&state->comp.list))
6102 io_submit_flush_completions(&state->comp);
Jens Axboe9a56a232019-01-09 09:06:50 -07006103 blk_finish_plug(&state->plug);
Pavel Begunkov9f13c352020-05-17 14:13:41 +03006104 io_state_file_put(state);
Jens Axboe2579f912019-01-09 09:10:43 -07006105 if (state->free_reqs)
Pavel Begunkov6c8a3132020-02-01 03:58:00 +03006106 kmem_cache_free_bulk(req_cachep, state->free_reqs, state->reqs);
Jens Axboe9a56a232019-01-09 09:06:50 -07006107}
6108
6109/*
6110 * Start submission side cache.
6111 */
6112static void io_submit_state_start(struct io_submit_state *state,
Jens Axboe013538b2020-06-22 09:29:15 -06006113 struct io_ring_ctx *ctx, unsigned int max_ios)
Jens Axboe9a56a232019-01-09 09:06:50 -07006114{
6115 blk_start_plug(&state->plug);
Jens Axboeb63534c2020-06-04 11:28:00 -06006116#ifdef CONFIG_BLOCK
6117 state->plug.nowait = true;
6118#endif
Jens Axboe013538b2020-06-22 09:29:15 -06006119 state->comp.nr = 0;
6120 INIT_LIST_HEAD(&state->comp.list);
6121 state->comp.ctx = ctx;
Jens Axboe2579f912019-01-09 09:10:43 -07006122 state->free_reqs = 0;
Jens Axboe9a56a232019-01-09 09:06:50 -07006123 state->file = NULL;
6124 state->ios_left = max_ios;
6125}
6126
Jens Axboe2b188cc2019-01-07 10:46:33 -07006127static void io_commit_sqring(struct io_ring_ctx *ctx)
6128{
Hristo Venev75b28af2019-08-26 17:23:46 +00006129 struct io_rings *rings = ctx->rings;
Jens Axboe2b188cc2019-01-07 10:46:33 -07006130
Pavel Begunkovcaf582c2019-12-30 21:24:46 +03006131 /*
6132 * Ensure any loads from the SQEs are done at this point,
6133 * since once we write the new head, the application could
6134 * write new data to them.
6135 */
6136 smp_store_release(&rings->sq.head, ctx->cached_sq_head);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006137}
6138
6139/*
Jens Axboe3529d8c2019-12-19 18:24:38 -07006140 * Fetch an sqe, if one is available. Note that sqe_ptr will point to memory
Jens Axboe2b188cc2019-01-07 10:46:33 -07006141 * that is mapped by userspace. This means that care needs to be taken to
6142 * ensure that reads are stable, as we cannot rely on userspace always
6143 * being a good citizen. If members of the sqe are validated and then later
6144 * used, it's important that those reads are done through READ_ONCE() to
6145 * prevent a re-load down the line.
6146 */
Pavel Begunkov709b3022020-04-08 08:58:43 +03006147static const struct io_uring_sqe *io_get_sqe(struct io_ring_ctx *ctx)
Jens Axboe2b188cc2019-01-07 10:46:33 -07006148{
Hristo Venev75b28af2019-08-26 17:23:46 +00006149 u32 *sq_array = ctx->sq_array;
Jens Axboe2b188cc2019-01-07 10:46:33 -07006150 unsigned head;
6151
6152 /*
6153 * The cached sq head (or cq tail) serves two purposes:
6154 *
6155 * 1) allows us to batch the cost of updating the user visible
6156 * head updates.
6157 * 2) allows the kernel side to track the head on its own, even
6158 * though the application is the one updating it.
6159 */
Pavel Begunkovee7d46d2019-12-30 21:24:45 +03006160 head = READ_ONCE(sq_array[ctx->cached_sq_head & ctx->sq_mask]);
Pavel Begunkov709b3022020-04-08 08:58:43 +03006161 if (likely(head < ctx->sq_entries))
6162 return &ctx->sq_sqes[head];
Jens Axboe2b188cc2019-01-07 10:46:33 -07006163
6164 /* drop invalid entries */
Jens Axboe498ccd92019-10-25 10:04:25 -06006165 ctx->cached_sq_dropped++;
Pavel Begunkovee7d46d2019-12-30 21:24:45 +03006166 WRITE_ONCE(ctx->rings->sq_dropped, ctx->cached_sq_dropped);
Pavel Begunkov709b3022020-04-08 08:58:43 +03006167 return NULL;
6168}
6169
6170static inline void io_consume_sqe(struct io_ring_ctx *ctx)
6171{
6172 ctx->cached_sq_head++;
Jens Axboe2b188cc2019-01-07 10:46:33 -07006173}
6174
Pavel Begunkovef4ff582020-04-12 02:05:05 +03006175#define SQE_VALID_FLAGS (IOSQE_FIXED_FILE|IOSQE_IO_DRAIN|IOSQE_IO_LINK| \
6176 IOSQE_IO_HARDLINK | IOSQE_ASYNC | \
6177 IOSQE_BUFFER_SELECT)
6178
6179static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,
6180 const struct io_uring_sqe *sqe,
Pavel Begunkov0cdaf762020-05-17 14:13:40 +03006181 struct io_submit_state *state)
Pavel Begunkov0553b8b2020-04-08 08:58:45 +03006182{
Pavel Begunkovef4ff582020-04-12 02:05:05 +03006183 unsigned int sqe_flags;
Jens Axboe63ff8222020-05-07 14:56:15 -06006184 int id;
Pavel Begunkovef4ff582020-04-12 02:05:05 +03006185
Pavel Begunkov0553b8b2020-04-08 08:58:45 +03006186 /*
6187 * All io need record the previous position, if LINK vs DARIN,
6188 * it can be used to mark the position of the first IO in the
6189 * link list.
6190 */
Pavel Begunkov31af27c2020-04-15 00:39:50 +03006191 req->sequence = ctx->cached_sq_head - ctx->cached_sq_dropped;
Pavel Begunkov0553b8b2020-04-08 08:58:45 +03006192 req->opcode = READ_ONCE(sqe->opcode);
6193 req->user_data = READ_ONCE(sqe->user_data);
6194 req->io = NULL;
6195 req->file = NULL;
6196 req->ctx = ctx;
6197 req->flags = 0;
6198 /* one is dropped after submission, the other at completion */
6199 refcount_set(&req->refs, 2);
Pavel Begunkov4dd28242020-06-15 10:33:13 +03006200 req->task = current;
Pavel Begunkov0553b8b2020-04-08 08:58:45 +03006201 req->result = 0;
Pavel Begunkovef4ff582020-04-12 02:05:05 +03006202
6203 if (unlikely(req->opcode >= IORING_OP_LAST))
6204 return -EINVAL;
6205
Jens Axboe9d8426a2020-06-16 18:42:49 -06006206 if (unlikely(io_sq_thread_acquire_mm(ctx, req)))
6207 return -EFAULT;
Pavel Begunkovef4ff582020-04-12 02:05:05 +03006208
6209 sqe_flags = READ_ONCE(sqe->flags);
6210 /* enforce forwards compatibility on users */
6211 if (unlikely(sqe_flags & ~SQE_VALID_FLAGS))
6212 return -EINVAL;
6213
6214 if ((sqe_flags & IOSQE_BUFFER_SELECT) &&
6215 !io_op_defs[req->opcode].buffer_select)
6216 return -EOPNOTSUPP;
6217
6218 id = READ_ONCE(sqe->personality);
6219 if (id) {
Xiaoguang Wang7cdaf582020-06-10 19:41:19 +08006220 io_req_init_async(req);
Pavel Begunkovef4ff582020-04-12 02:05:05 +03006221 req->work.creds = idr_find(&ctx->personality_idr, id);
6222 if (unlikely(!req->work.creds))
6223 return -EINVAL;
6224 get_cred(req->work.creds);
6225 }
6226
6227 /* same numerical values with corresponding REQ_F_*, safe to copy */
Pavel Begunkovc11368a52020-05-17 14:13:42 +03006228 req->flags |= sqe_flags;
Pavel Begunkovef4ff582020-04-12 02:05:05 +03006229
Jens Axboe63ff8222020-05-07 14:56:15 -06006230 if (!io_op_defs[req->opcode].needs_file)
6231 return 0;
6232
6233 return io_req_set_file(state, req, READ_ONCE(sqe->fd));
Pavel Begunkov0553b8b2020-04-08 08:58:45 +03006234}
6235
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03006236static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr,
Pavel Begunkov0cdaf762020-05-17 14:13:40 +03006237 struct file *ring_file, int ring_fd)
Jens Axboe6c271ce2019-01-10 11:22:30 -07006238{
Jens Axboeac8691c2020-06-01 08:30:41 -06006239 struct io_submit_state state;
Jens Axboe9e645e112019-05-10 16:07:28 -06006240 struct io_kiocb *link = NULL;
Jens Axboe9e645e112019-05-10 16:07:28 -06006241 int i, submitted = 0;
Jens Axboe6c271ce2019-01-10 11:22:30 -07006242
Jens Axboec4a2ed72019-11-21 21:01:26 -07006243 /* if we have a backlog and couldn't flush it all, return BUSY */
Jens Axboead3eb2c2019-12-18 17:12:20 -07006244 if (test_bit(0, &ctx->sq_check_overflow)) {
6245 if (!list_empty(&ctx->cq_overflow_list) &&
6246 !io_cqring_overflow_flush(ctx, false))
6247 return -EBUSY;
6248 }
Jens Axboe6c271ce2019-01-10 11:22:30 -07006249
Pavel Begunkovee7d46d2019-12-30 21:24:45 +03006250 /* make sure SQ entry isn't read before tail */
6251 nr = min3(nr, ctx->sq_entries, io_sqring_entries(ctx));
Pavel Begunkov9ef4f122019-12-30 21:24:44 +03006252
Pavel Begunkov2b85edf2019-12-28 14:13:03 +03006253 if (!percpu_ref_tryget_many(&ctx->refs, nr))
6254 return -EAGAIN;
Jens Axboe6c271ce2019-01-10 11:22:30 -07006255
Jens Axboe013538b2020-06-22 09:29:15 -06006256 io_submit_state_start(&state, ctx, nr);
Jens Axboe6c271ce2019-01-10 11:22:30 -07006257
Pavel Begunkovb14cca02020-01-17 04:45:59 +03006258 ctx->ring_fd = ring_fd;
6259 ctx->ring_file = ring_file;
6260
Jens Axboe6c271ce2019-01-10 11:22:30 -07006261 for (i = 0; i < nr; i++) {
Jens Axboe3529d8c2019-12-19 18:24:38 -07006262 const struct io_uring_sqe *sqe;
Pavel Begunkov196be952019-11-07 01:41:06 +03006263 struct io_kiocb *req;
Pavel Begunkov1cb1edb2020-02-06 21:16:09 +03006264 int err;
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03006265
Pavel Begunkovb1e50e52020-04-08 08:58:44 +03006266 sqe = io_get_sqe(ctx);
6267 if (unlikely(!sqe)) {
6268 io_consume_sqe(ctx);
6269 break;
6270 }
Jens Axboeac8691c2020-06-01 08:30:41 -06006271 req = io_alloc_req(ctx, &state);
Pavel Begunkov196be952019-11-07 01:41:06 +03006272 if (unlikely(!req)) {
6273 if (!submitted)
6274 submitted = -EAGAIN;
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03006275 break;
Jens Axboe9e645e112019-05-10 16:07:28 -06006276 }
Jens Axboe9e645e112019-05-10 16:07:28 -06006277
Jens Axboeac8691c2020-06-01 08:30:41 -06006278 err = io_init_req(ctx, req, sqe, &state);
Pavel Begunkov709b3022020-04-08 08:58:43 +03006279 io_consume_sqe(ctx);
Jens Axboed3656342019-12-18 09:50:26 -07006280 /* will complete beyond this point, count as submitted */
6281 submitted++;
6282
Pavel Begunkovef4ff582020-04-12 02:05:05 +03006283 if (unlikely(err)) {
Pavel Begunkov1cb1edb2020-02-06 21:16:09 +03006284fail_req:
Jens Axboee1e16092020-06-22 09:17:17 -06006285 io_put_req(req);
6286 io_req_complete(req, err);
Jens Axboed3656342019-12-18 09:50:26 -07006287 break;
6288 }
6289
Jens Axboe354420f2020-01-08 18:55:15 -07006290 trace_io_uring_submit_sqe(ctx, req->opcode, req->user_data,
Pavel Begunkov0cdaf762020-05-17 14:13:40 +03006291 true, io_async_submit(ctx));
Jens Axboef13fad72020-06-22 09:34:30 -06006292 err = io_submit_sqe(req, sqe, &link, &state.comp);
Pavel Begunkov1d4240c2020-04-12 02:05:03 +03006293 if (err)
6294 goto fail_req;
Jens Axboe6c271ce2019-01-10 11:22:30 -07006295 }
6296
Pavel Begunkov9466f432020-01-25 22:34:01 +03006297 if (unlikely(submitted != nr)) {
6298 int ref_used = (submitted == -EAGAIN) ? 0 : submitted;
6299
6300 percpu_ref_put_many(&ctx->refs, nr - ref_used);
6301 }
Jens Axboe9e645e112019-05-10 16:07:28 -06006302 if (link)
Jens Axboef13fad72020-06-22 09:34:30 -06006303 io_queue_link_head(link, &state.comp);
Jens Axboeac8691c2020-06-01 08:30:41 -06006304 io_submit_state_end(&state);
Jens Axboe6c271ce2019-01-10 11:22:30 -07006305
Pavel Begunkovae9428c2019-11-06 00:22:14 +03006306 /* Commit SQ ring head once we've consumed and submitted all SQEs */
6307 io_commit_sqring(ctx);
6308
Jens Axboe6c271ce2019-01-10 11:22:30 -07006309 return submitted;
6310}
6311
6312static int io_sq_thread(void *data)
6313{
Jens Axboe6c271ce2019-01-10 11:22:30 -07006314 struct io_ring_ctx *ctx = data;
Jens Axboe181e4482019-11-25 08:52:30 -07006315 const struct cred *old_cred;
Jens Axboe6c271ce2019-01-10 11:22:30 -07006316 DEFINE_WAIT(wait);
Jens Axboe6c271ce2019-01-10 11:22:30 -07006317 unsigned long timeout;
Xiaoguang Wangbdcd3ea2020-02-25 22:12:08 +08006318 int ret = 0;
Jens Axboe6c271ce2019-01-10 11:22:30 -07006319
Jens Axboe0f158b42020-05-14 17:18:39 -06006320 complete(&ctx->sq_thread_comp);
Jackie Liua4c0b3d2019-07-08 13:41:12 +08006321
Jens Axboe181e4482019-11-25 08:52:30 -07006322 old_cred = override_creds(ctx->creds);
Jens Axboe6c271ce2019-01-10 11:22:30 -07006323
Xiaoguang Wangbdcd3ea2020-02-25 22:12:08 +08006324 timeout = jiffies + ctx->sq_thread_idle;
Roman Penyaev2bbcd6d2019-05-16 10:53:57 +02006325 while (!kthread_should_park()) {
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03006326 unsigned int to_submit;
Jens Axboe6c271ce2019-01-10 11:22:30 -07006327
Xiaoguang Wangbdcd3ea2020-02-25 22:12:08 +08006328 if (!list_empty(&ctx->poll_list)) {
Jens Axboe6c271ce2019-01-10 11:22:30 -07006329 unsigned nr_events = 0;
6330
Xiaoguang Wangbdcd3ea2020-02-25 22:12:08 +08006331 mutex_lock(&ctx->uring_lock);
6332 if (!list_empty(&ctx->poll_list))
6333 io_iopoll_getevents(ctx, &nr_events, 0);
6334 else
Jens Axboe6c271ce2019-01-10 11:22:30 -07006335 timeout = jiffies + ctx->sq_thread_idle;
Xiaoguang Wangbdcd3ea2020-02-25 22:12:08 +08006336 mutex_unlock(&ctx->uring_lock);
Jens Axboe6c271ce2019-01-10 11:22:30 -07006337 }
6338
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03006339 to_submit = io_sqring_entries(ctx);
Jens Axboec1edbf52019-11-10 16:56:04 -07006340
6341 /*
6342 * If submit got -EBUSY, flag us as needing the application
6343 * to enter the kernel to reap and flush events.
6344 */
Xuan Zhuob772f072020-06-23 19:34:06 +08006345 if (!to_submit || ret == -EBUSY || need_resched()) {
Jens Axboe6c271ce2019-01-10 11:22:30 -07006346 /*
Stefano Garzarella7143b5a2020-02-21 16:42:16 +01006347 * Drop cur_mm before scheduling, we can't hold it for
6348 * long periods (or over schedule()). Do this before
6349 * adding ourselves to the waitqueue, as the unuse/drop
6350 * may sleep.
6351 */
Pavel Begunkovbf9c2f12020-04-12 02:05:02 +03006352 io_sq_thread_drop_mm(ctx);
Stefano Garzarella7143b5a2020-02-21 16:42:16 +01006353
6354 /*
Jens Axboe6c271ce2019-01-10 11:22:30 -07006355 * We're polling. If we're within the defined idle
6356 * period, then let us spin without work before going
Jens Axboec1edbf52019-11-10 16:56:04 -07006357 * to sleep. The exception is if we got EBUSY doing
6358 * more IO, we should wait for the application to
6359 * reap events and wake us up.
Jens Axboe6c271ce2019-01-10 11:22:30 -07006360 */
Xuan Zhuob772f072020-06-23 19:34:06 +08006361 if (!list_empty(&ctx->poll_list) || need_resched() ||
Jens Axboedf069d82020-02-04 16:48:34 -07006362 (!time_after(jiffies, timeout) && ret != -EBUSY &&
6363 !percpu_ref_is_dying(&ctx->refs))) {
Jens Axboeb41e9852020-02-17 09:52:41 -07006364 if (current->task_works)
6365 task_work_run();
Jens Axboe9831a902019-09-19 09:48:55 -06006366 cond_resched();
Jens Axboe6c271ce2019-01-10 11:22:30 -07006367 continue;
6368 }
6369
Jens Axboe6c271ce2019-01-10 11:22:30 -07006370 prepare_to_wait(&ctx->sqo_wait, &wait,
6371 TASK_INTERRUPTIBLE);
6372
Xiaoguang Wangbdcd3ea2020-02-25 22:12:08 +08006373 /*
6374 * While doing polled IO, before going to sleep, we need
6375 * to check if there are new reqs added to poll_list, it
6376 * is because reqs may have been punted to io worker and
6377 * will be added to poll_list later, hence check the
6378 * poll_list again.
6379 */
6380 if ((ctx->flags & IORING_SETUP_IOPOLL) &&
6381 !list_empty_careful(&ctx->poll_list)) {
6382 finish_wait(&ctx->sqo_wait, &wait);
6383 continue;
6384 }
6385
Jens Axboe6c271ce2019-01-10 11:22:30 -07006386 /* Tell userspace we may need a wakeup call */
Hristo Venev75b28af2019-08-26 17:23:46 +00006387 ctx->rings->sq_flags |= IORING_SQ_NEED_WAKEUP;
Stefan Bühler0d7bae62019-04-19 11:57:45 +02006388 /* make sure to read SQ tail after writing flags */
6389 smp_mb();
Jens Axboe6c271ce2019-01-10 11:22:30 -07006390
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03006391 to_submit = io_sqring_entries(ctx);
Jens Axboec1edbf52019-11-10 16:56:04 -07006392 if (!to_submit || ret == -EBUSY) {
Roman Penyaev2bbcd6d2019-05-16 10:53:57 +02006393 if (kthread_should_park()) {
Jens Axboe6c271ce2019-01-10 11:22:30 -07006394 finish_wait(&ctx->sqo_wait, &wait);
6395 break;
6396 }
Jens Axboeb41e9852020-02-17 09:52:41 -07006397 if (current->task_works) {
6398 task_work_run();
Hillf Danton10bea962020-04-01 17:19:33 +08006399 finish_wait(&ctx->sqo_wait, &wait);
Jens Axboeb41e9852020-02-17 09:52:41 -07006400 continue;
6401 }
Jens Axboe6c271ce2019-01-10 11:22:30 -07006402 if (signal_pending(current))
6403 flush_signals(current);
6404 schedule();
6405 finish_wait(&ctx->sqo_wait, &wait);
6406
Hristo Venev75b28af2019-08-26 17:23:46 +00006407 ctx->rings->sq_flags &= ~IORING_SQ_NEED_WAKEUP;
Xiaoguang Wangd4ae2712020-05-20 21:24:35 +08006408 ret = 0;
Jens Axboe6c271ce2019-01-10 11:22:30 -07006409 continue;
6410 }
6411 finish_wait(&ctx->sqo_wait, &wait);
6412
Hristo Venev75b28af2019-08-26 17:23:46 +00006413 ctx->rings->sq_flags &= ~IORING_SQ_NEED_WAKEUP;
Jens Axboe6c271ce2019-01-10 11:22:30 -07006414 }
6415
Jens Axboe8a4955f2019-12-09 14:52:35 -07006416 mutex_lock(&ctx->uring_lock);
Xiaoguang Wang6b668c92020-05-20 15:35:03 +08006417 if (likely(!percpu_ref_is_dying(&ctx->refs)))
6418 ret = io_submit_sqes(ctx, to_submit, NULL, -1);
Jens Axboe8a4955f2019-12-09 14:52:35 -07006419 mutex_unlock(&ctx->uring_lock);
Xiaoguang Wangbdcd3ea2020-02-25 22:12:08 +08006420 timeout = jiffies + ctx->sq_thread_idle;
Jens Axboe6c271ce2019-01-10 11:22:30 -07006421 }
6422
Jens Axboeb41e9852020-02-17 09:52:41 -07006423 if (current->task_works)
6424 task_work_run();
6425
Pavel Begunkovbf9c2f12020-04-12 02:05:02 +03006426 io_sq_thread_drop_mm(ctx);
Jens Axboe181e4482019-11-25 08:52:30 -07006427 revert_creds(old_cred);
Jens Axboe06058632019-04-13 09:26:03 -06006428
Roman Penyaev2bbcd6d2019-05-16 10:53:57 +02006429 kthread_parkme();
Jens Axboe06058632019-04-13 09:26:03 -06006430
Jens Axboe6c271ce2019-01-10 11:22:30 -07006431 return 0;
6432}
6433
Jens Axboebda52162019-09-24 13:47:15 -06006434struct io_wait_queue {
6435 struct wait_queue_entry wq;
6436 struct io_ring_ctx *ctx;
6437 unsigned to_wait;
6438 unsigned nr_timeouts;
6439};
6440
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07006441static inline bool io_should_wake(struct io_wait_queue *iowq, bool noflush)
Jens Axboebda52162019-09-24 13:47:15 -06006442{
6443 struct io_ring_ctx *ctx = iowq->ctx;
6444
6445 /*
Brian Gianforcarod195a662019-12-13 03:09:50 -08006446 * Wake up if we have enough events, or if a timeout occurred since we
Jens Axboebda52162019-09-24 13:47:15 -06006447 * started waiting. For timeouts, we always want to return to userspace,
6448 * regardless of event count.
6449 */
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07006450 return io_cqring_events(ctx, noflush) >= iowq->to_wait ||
Jens Axboebda52162019-09-24 13:47:15 -06006451 atomic_read(&ctx->cq_timeouts) != iowq->nr_timeouts;
6452}
6453
6454static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode,
6455 int wake_flags, void *key)
6456{
6457 struct io_wait_queue *iowq = container_of(curr, struct io_wait_queue,
6458 wq);
6459
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07006460 /* use noflush == true, as we can't safely rely on locking context */
6461 if (!io_should_wake(iowq, true))
Jens Axboebda52162019-09-24 13:47:15 -06006462 return -1;
6463
6464 return autoremove_wake_function(curr, mode, wake_flags, key);
6465}
6466
Jens Axboe2b188cc2019-01-07 10:46:33 -07006467/*
6468 * Wait until events become available, if we don't already have some. The
6469 * application must reap them itself, as they reside on the shared cq ring.
6470 */
6471static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events,
6472 const sigset_t __user *sig, size_t sigsz)
6473{
Jens Axboebda52162019-09-24 13:47:15 -06006474 struct io_wait_queue iowq = {
6475 .wq = {
6476 .private = current,
6477 .func = io_wake_function,
6478 .entry = LIST_HEAD_INIT(iowq.wq.entry),
6479 },
6480 .ctx = ctx,
6481 .to_wait = min_events,
6482 };
Hristo Venev75b28af2019-08-26 17:23:46 +00006483 struct io_rings *rings = ctx->rings;
Jackie Liue9ffa5c2019-10-29 11:16:42 +08006484 int ret = 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07006485
Jens Axboeb41e9852020-02-17 09:52:41 -07006486 do {
6487 if (io_cqring_events(ctx, false) >= min_events)
6488 return 0;
6489 if (!current->task_works)
6490 break;
6491 task_work_run();
6492 } while (1);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006493
6494 if (sig) {
Arnd Bergmann9e75ad52019-03-25 15:34:53 +01006495#ifdef CONFIG_COMPAT
6496 if (in_compat_syscall())
6497 ret = set_compat_user_sigmask((const compat_sigset_t __user *)sig,
Oleg Nesterovb7724342019-07-16 16:29:53 -07006498 sigsz);
Arnd Bergmann9e75ad52019-03-25 15:34:53 +01006499 else
6500#endif
Oleg Nesterovb7724342019-07-16 16:29:53 -07006501 ret = set_user_sigmask(sig, sigsz);
Arnd Bergmann9e75ad52019-03-25 15:34:53 +01006502
Jens Axboe2b188cc2019-01-07 10:46:33 -07006503 if (ret)
6504 return ret;
6505 }
6506
Jens Axboebda52162019-09-24 13:47:15 -06006507 iowq.nr_timeouts = atomic_read(&ctx->cq_timeouts);
Dmitrii Dolgovc826bd72019-10-15 19:02:01 +02006508 trace_io_uring_cqring_wait(ctx, min_events);
Jens Axboebda52162019-09-24 13:47:15 -06006509 do {
6510 prepare_to_wait_exclusive(&ctx->wait, &iowq.wq,
6511 TASK_INTERRUPTIBLE);
Jens Axboece593a62020-06-30 12:39:05 -06006512 /* make sure we run task_work before checking for signals */
Jens Axboeb41e9852020-02-17 09:52:41 -07006513 if (current->task_works)
6514 task_work_run();
Jens Axboebda52162019-09-24 13:47:15 -06006515 if (signal_pending(current)) {
Jens Axboeb7db41c2020-07-04 08:55:50 -06006516 if (current->jobctl & JOBCTL_TASK_WORK) {
6517 spin_lock_irq(&current->sighand->siglock);
6518 current->jobctl &= ~JOBCTL_TASK_WORK;
6519 recalc_sigpending();
6520 spin_unlock_irq(&current->sighand->siglock);
6521 continue;
6522 }
Jackie Liue9ffa5c2019-10-29 11:16:42 +08006523 ret = -EINTR;
Jens Axboebda52162019-09-24 13:47:15 -06006524 break;
6525 }
Jens Axboebda52162019-09-24 13:47:15 -06006526 if (io_should_wake(&iowq, false))
6527 break;
6528 schedule();
Jens Axboebda52162019-09-24 13:47:15 -06006529 } while (1);
6530 finish_wait(&ctx->wait, &iowq.wq);
6531
Jackie Liue9ffa5c2019-10-29 11:16:42 +08006532 restore_saved_sigmask_unless(ret == -EINTR);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006533
Hristo Venev75b28af2019-08-26 17:23:46 +00006534 return READ_ONCE(rings->cq.head) == READ_ONCE(rings->cq.tail) ? ret : 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07006535}
6536
Jens Axboe6b063142019-01-10 22:13:58 -07006537static void __io_sqe_files_unregister(struct io_ring_ctx *ctx)
6538{
6539#if defined(CONFIG_UNIX)
6540 if (ctx->ring_sock) {
6541 struct sock *sock = ctx->ring_sock->sk;
6542 struct sk_buff *skb;
6543
6544 while ((skb = skb_dequeue(&sock->sk_receive_queue)) != NULL)
6545 kfree_skb(skb);
6546 }
6547#else
6548 int i;
6549
Jens Axboe65e19f52019-10-26 07:20:21 -06006550 for (i = 0; i < ctx->nr_user_files; i++) {
6551 struct file *file;
6552
6553 file = io_file_from_index(ctx, i);
6554 if (file)
6555 fput(file);
6556 }
Jens Axboe6b063142019-01-10 22:13:58 -07006557#endif
6558}
6559
Jens Axboe05f3fb32019-12-09 11:22:50 -07006560static void io_file_ref_kill(struct percpu_ref *ref)
6561{
6562 struct fixed_file_data *data;
6563
6564 data = container_of(ref, struct fixed_file_data, refs);
6565 complete(&data->done);
6566}
6567
Jens Axboe6b063142019-01-10 22:13:58 -07006568static int io_sqe_files_unregister(struct io_ring_ctx *ctx)
6569{
Jens Axboe05f3fb32019-12-09 11:22:50 -07006570 struct fixed_file_data *data = ctx->file_data;
Xiaoguang Wang05589552020-03-31 14:05:18 +08006571 struct fixed_file_ref_node *ref_node = NULL;
Jens Axboe65e19f52019-10-26 07:20:21 -06006572 unsigned nr_tables, i;
6573
Jens Axboe05f3fb32019-12-09 11:22:50 -07006574 if (!data)
Jens Axboe6b063142019-01-10 22:13:58 -07006575 return -ENXIO;
6576
Jens Axboe6a4d07c2020-05-15 14:30:38 -06006577 spin_lock(&data->lock);
Xiaoguang Wang05589552020-03-31 14:05:18 +08006578 if (!list_empty(&data->ref_list))
6579 ref_node = list_first_entry(&data->ref_list,
6580 struct fixed_file_ref_node, node);
Jens Axboe6a4d07c2020-05-15 14:30:38 -06006581 spin_unlock(&data->lock);
Xiaoguang Wang05589552020-03-31 14:05:18 +08006582 if (ref_node)
6583 percpu_ref_kill(&ref_node->refs);
6584
6585 percpu_ref_kill(&data->refs);
6586
6587 /* wait for all refs nodes to complete */
Jens Axboe4a38aed22020-05-14 17:21:15 -06006588 flush_delayed_work(&ctx->file_put_work);
Jens Axboe2faf8522020-02-04 19:54:55 -07006589 wait_for_completion(&data->done);
Jens Axboe05f3fb32019-12-09 11:22:50 -07006590
Jens Axboe6b063142019-01-10 22:13:58 -07006591 __io_sqe_files_unregister(ctx);
Jens Axboe65e19f52019-10-26 07:20:21 -06006592 nr_tables = DIV_ROUND_UP(ctx->nr_user_files, IORING_MAX_FILES_TABLE);
6593 for (i = 0; i < nr_tables; i++)
Jens Axboe05f3fb32019-12-09 11:22:50 -07006594 kfree(data->table[i].files);
6595 kfree(data->table);
Xiaoguang Wang05589552020-03-31 14:05:18 +08006596 percpu_ref_exit(&data->refs);
6597 kfree(data);
Jens Axboe05f3fb32019-12-09 11:22:50 -07006598 ctx->file_data = NULL;
Jens Axboe6b063142019-01-10 22:13:58 -07006599 ctx->nr_user_files = 0;
6600 return 0;
6601}
6602
Jens Axboe6c271ce2019-01-10 11:22:30 -07006603static void io_sq_thread_stop(struct io_ring_ctx *ctx)
6604{
6605 if (ctx->sqo_thread) {
Jens Axboe0f158b42020-05-14 17:18:39 -06006606 wait_for_completion(&ctx->sq_thread_comp);
Roman Penyaev2bbcd6d2019-05-16 10:53:57 +02006607 /*
6608 * The park is a bit of a work-around, without it we get
6609 * warning spews on shutdown with SQPOLL set and affinity
6610 * set to a single CPU.
6611 */
Jens Axboe06058632019-04-13 09:26:03 -06006612 kthread_park(ctx->sqo_thread);
Jens Axboe6c271ce2019-01-10 11:22:30 -07006613 kthread_stop(ctx->sqo_thread);
6614 ctx->sqo_thread = NULL;
6615 }
6616}
6617
Jens Axboe6b063142019-01-10 22:13:58 -07006618static void io_finish_async(struct io_ring_ctx *ctx)
6619{
Jens Axboe6c271ce2019-01-10 11:22:30 -07006620 io_sq_thread_stop(ctx);
6621
Jens Axboe561fb042019-10-24 07:25:42 -06006622 if (ctx->io_wq) {
6623 io_wq_destroy(ctx->io_wq);
6624 ctx->io_wq = NULL;
Jens Axboe6b063142019-01-10 22:13:58 -07006625 }
6626}
6627
6628#if defined(CONFIG_UNIX)
Jens Axboe6b063142019-01-10 22:13:58 -07006629/*
6630 * Ensure the UNIX gc is aware of our file set, so we are certain that
6631 * the io_uring can be safely unregistered on process exit, even if we have
6632 * loops in the file referencing.
6633 */
6634static int __io_sqe_files_scm(struct io_ring_ctx *ctx, int nr, int offset)
6635{
6636 struct sock *sk = ctx->ring_sock->sk;
6637 struct scm_fp_list *fpl;
6638 struct sk_buff *skb;
Jens Axboe08a45172019-10-03 08:11:03 -06006639 int i, nr_files;
Jens Axboe6b063142019-01-10 22:13:58 -07006640
Jens Axboe6b063142019-01-10 22:13:58 -07006641 fpl = kzalloc(sizeof(*fpl), GFP_KERNEL);
6642 if (!fpl)
6643 return -ENOMEM;
6644
6645 skb = alloc_skb(0, GFP_KERNEL);
6646 if (!skb) {
6647 kfree(fpl);
6648 return -ENOMEM;
6649 }
6650
6651 skb->sk = sk;
Jens Axboe6b063142019-01-10 22:13:58 -07006652
Jens Axboe08a45172019-10-03 08:11:03 -06006653 nr_files = 0;
Jens Axboe6b063142019-01-10 22:13:58 -07006654 fpl->user = get_uid(ctx->user);
6655 for (i = 0; i < nr; i++) {
Jens Axboe65e19f52019-10-26 07:20:21 -06006656 struct file *file = io_file_from_index(ctx, i + offset);
6657
6658 if (!file)
Jens Axboe08a45172019-10-03 08:11:03 -06006659 continue;
Jens Axboe65e19f52019-10-26 07:20:21 -06006660 fpl->fp[nr_files] = get_file(file);
Jens Axboe08a45172019-10-03 08:11:03 -06006661 unix_inflight(fpl->user, fpl->fp[nr_files]);
6662 nr_files++;
Jens Axboe6b063142019-01-10 22:13:58 -07006663 }
6664
Jens Axboe08a45172019-10-03 08:11:03 -06006665 if (nr_files) {
6666 fpl->max = SCM_MAX_FD;
6667 fpl->count = nr_files;
6668 UNIXCB(skb).fp = fpl;
Jens Axboe05f3fb32019-12-09 11:22:50 -07006669 skb->destructor = unix_destruct_scm;
Jens Axboe08a45172019-10-03 08:11:03 -06006670 refcount_add(skb->truesize, &sk->sk_wmem_alloc);
6671 skb_queue_head(&sk->sk_receive_queue, skb);
Jens Axboe6b063142019-01-10 22:13:58 -07006672
Jens Axboe08a45172019-10-03 08:11:03 -06006673 for (i = 0; i < nr_files; i++)
6674 fput(fpl->fp[i]);
6675 } else {
6676 kfree_skb(skb);
6677 kfree(fpl);
6678 }
Jens Axboe6b063142019-01-10 22:13:58 -07006679
6680 return 0;
6681}
6682
6683/*
6684 * If UNIX sockets are enabled, fd passing can cause a reference cycle which
6685 * causes regular reference counting to break down. We rely on the UNIX
6686 * garbage collection to take care of this problem for us.
6687 */
6688static int io_sqe_files_scm(struct io_ring_ctx *ctx)
6689{
6690 unsigned left, total;
6691 int ret = 0;
6692
6693 total = 0;
6694 left = ctx->nr_user_files;
6695 while (left) {
6696 unsigned this_files = min_t(unsigned, left, SCM_MAX_FD);
Jens Axboe6b063142019-01-10 22:13:58 -07006697
6698 ret = __io_sqe_files_scm(ctx, this_files, total);
6699 if (ret)
6700 break;
6701 left -= this_files;
6702 total += this_files;
6703 }
6704
6705 if (!ret)
6706 return 0;
6707
6708 while (total < ctx->nr_user_files) {
Jens Axboe65e19f52019-10-26 07:20:21 -06006709 struct file *file = io_file_from_index(ctx, total);
6710
6711 if (file)
6712 fput(file);
Jens Axboe6b063142019-01-10 22:13:58 -07006713 total++;
6714 }
6715
6716 return ret;
6717}
6718#else
6719static int io_sqe_files_scm(struct io_ring_ctx *ctx)
6720{
6721 return 0;
6722}
6723#endif
6724
Jens Axboe65e19f52019-10-26 07:20:21 -06006725static int io_sqe_alloc_file_tables(struct io_ring_ctx *ctx, unsigned nr_tables,
6726 unsigned nr_files)
6727{
6728 int i;
6729
6730 for (i = 0; i < nr_tables; i++) {
Jens Axboe05f3fb32019-12-09 11:22:50 -07006731 struct fixed_file_table *table = &ctx->file_data->table[i];
Jens Axboe65e19f52019-10-26 07:20:21 -06006732 unsigned this_files;
6733
6734 this_files = min(nr_files, IORING_MAX_FILES_TABLE);
6735 table->files = kcalloc(this_files, sizeof(struct file *),
6736 GFP_KERNEL);
6737 if (!table->files)
6738 break;
6739 nr_files -= this_files;
6740 }
6741
6742 if (i == nr_tables)
6743 return 0;
6744
6745 for (i = 0; i < nr_tables; i++) {
Jens Axboe05f3fb32019-12-09 11:22:50 -07006746 struct fixed_file_table *table = &ctx->file_data->table[i];
Jens Axboe65e19f52019-10-26 07:20:21 -06006747 kfree(table->files);
6748 }
6749 return 1;
6750}
6751
Jens Axboe05f3fb32019-12-09 11:22:50 -07006752static void io_ring_file_put(struct io_ring_ctx *ctx, struct file *file)
Jens Axboec3a31e62019-10-03 13:59:56 -06006753{
6754#if defined(CONFIG_UNIX)
Jens Axboec3a31e62019-10-03 13:59:56 -06006755 struct sock *sock = ctx->ring_sock->sk;
6756 struct sk_buff_head list, *head = &sock->sk_receive_queue;
6757 struct sk_buff *skb;
6758 int i;
6759
6760 __skb_queue_head_init(&list);
6761
6762 /*
6763 * Find the skb that holds this file in its SCM_RIGHTS. When found,
6764 * remove this entry and rearrange the file array.
6765 */
6766 skb = skb_dequeue(head);
6767 while (skb) {
6768 struct scm_fp_list *fp;
6769
6770 fp = UNIXCB(skb).fp;
6771 for (i = 0; i < fp->count; i++) {
6772 int left;
6773
6774 if (fp->fp[i] != file)
6775 continue;
6776
6777 unix_notinflight(fp->user, fp->fp[i]);
6778 left = fp->count - 1 - i;
6779 if (left) {
6780 memmove(&fp->fp[i], &fp->fp[i + 1],
6781 left * sizeof(struct file *));
6782 }
6783 fp->count--;
6784 if (!fp->count) {
6785 kfree_skb(skb);
6786 skb = NULL;
6787 } else {
6788 __skb_queue_tail(&list, skb);
6789 }
6790 fput(file);
6791 file = NULL;
6792 break;
6793 }
6794
6795 if (!file)
6796 break;
6797
6798 __skb_queue_tail(&list, skb);
6799
6800 skb = skb_dequeue(head);
6801 }
6802
6803 if (skb_peek(&list)) {
6804 spin_lock_irq(&head->lock);
6805 while ((skb = __skb_dequeue(&list)) != NULL)
6806 __skb_queue_tail(head, skb);
6807 spin_unlock_irq(&head->lock);
6808 }
6809#else
Jens Axboe05f3fb32019-12-09 11:22:50 -07006810 fput(file);
Jens Axboec3a31e62019-10-03 13:59:56 -06006811#endif
6812}
6813
Jens Axboe05f3fb32019-12-09 11:22:50 -07006814struct io_file_put {
Xiaoguang Wang05589552020-03-31 14:05:18 +08006815 struct list_head list;
Jens Axboe05f3fb32019-12-09 11:22:50 -07006816 struct file *file;
Jens Axboe05f3fb32019-12-09 11:22:50 -07006817};
6818
Jens Axboe4a38aed22020-05-14 17:21:15 -06006819static void __io_file_put_work(struct fixed_file_ref_node *ref_node)
Jens Axboe05f3fb32019-12-09 11:22:50 -07006820{
Jens Axboe4a38aed22020-05-14 17:21:15 -06006821 struct fixed_file_data *file_data = ref_node->file_data;
6822 struct io_ring_ctx *ctx = file_data->ctx;
Jens Axboe05f3fb32019-12-09 11:22:50 -07006823 struct io_file_put *pfile, *tmp;
Xiaoguang Wang05589552020-03-31 14:05:18 +08006824
6825 list_for_each_entry_safe(pfile, tmp, &ref_node->file_list, list) {
Jens Axboe6a4d07c2020-05-15 14:30:38 -06006826 list_del(&pfile->list);
Xiaoguang Wang05589552020-03-31 14:05:18 +08006827 io_ring_file_put(ctx, pfile->file);
6828 kfree(pfile);
Jens Axboe05f3fb32019-12-09 11:22:50 -07006829 }
6830
Jens Axboe6a4d07c2020-05-15 14:30:38 -06006831 spin_lock(&file_data->lock);
6832 list_del(&ref_node->node);
6833 spin_unlock(&file_data->lock);
Jens Axboe2faf8522020-02-04 19:54:55 -07006834
Xiaoguang Wang05589552020-03-31 14:05:18 +08006835 percpu_ref_exit(&ref_node->refs);
6836 kfree(ref_node);
6837 percpu_ref_put(&file_data->refs);
Jens Axboe05f3fb32019-12-09 11:22:50 -07006838}
6839
Jens Axboe4a38aed22020-05-14 17:21:15 -06006840static void io_file_put_work(struct work_struct *work)
6841{
6842 struct io_ring_ctx *ctx;
6843 struct llist_node *node;
6844
6845 ctx = container_of(work, struct io_ring_ctx, file_put_work.work);
6846 node = llist_del_all(&ctx->file_put_llist);
6847
6848 while (node) {
6849 struct fixed_file_ref_node *ref_node;
6850 struct llist_node *next = node->next;
6851
6852 ref_node = llist_entry(node, struct fixed_file_ref_node, llist);
6853 __io_file_put_work(ref_node);
6854 node = next;
6855 }
6856}
6857
Jens Axboe05f3fb32019-12-09 11:22:50 -07006858static void io_file_data_ref_zero(struct percpu_ref *ref)
6859{
Xiaoguang Wang05589552020-03-31 14:05:18 +08006860 struct fixed_file_ref_node *ref_node;
Jens Axboe4a38aed22020-05-14 17:21:15 -06006861 struct io_ring_ctx *ctx;
6862 bool first_add;
6863 int delay = HZ;
Jens Axboe05f3fb32019-12-09 11:22:50 -07006864
Xiaoguang Wang05589552020-03-31 14:05:18 +08006865 ref_node = container_of(ref, struct fixed_file_ref_node, refs);
Jens Axboe4a38aed22020-05-14 17:21:15 -06006866 ctx = ref_node->file_data->ctx;
Jens Axboe05f3fb32019-12-09 11:22:50 -07006867
Jens Axboe4a38aed22020-05-14 17:21:15 -06006868 if (percpu_ref_is_dying(&ctx->file_data->refs))
6869 delay = 0;
6870
6871 first_add = llist_add(&ref_node->llist, &ctx->file_put_llist);
6872 if (!delay)
6873 mod_delayed_work(system_wq, &ctx->file_put_work, 0);
6874 else if (first_add)
6875 queue_delayed_work(system_wq, &ctx->file_put_work, delay);
Xiaoguang Wang05589552020-03-31 14:05:18 +08006876}
6877
6878static struct fixed_file_ref_node *alloc_fixed_file_ref_node(
6879 struct io_ring_ctx *ctx)
6880{
6881 struct fixed_file_ref_node *ref_node;
6882
6883 ref_node = kzalloc(sizeof(*ref_node), GFP_KERNEL);
6884 if (!ref_node)
6885 return ERR_PTR(-ENOMEM);
6886
6887 if (percpu_ref_init(&ref_node->refs, io_file_data_ref_zero,
6888 0, GFP_KERNEL)) {
6889 kfree(ref_node);
6890 return ERR_PTR(-ENOMEM);
6891 }
6892 INIT_LIST_HEAD(&ref_node->node);
6893 INIT_LIST_HEAD(&ref_node->file_list);
Xiaoguang Wang05589552020-03-31 14:05:18 +08006894 ref_node->file_data = ctx->file_data;
6895 return ref_node;
Xiaoguang Wang05589552020-03-31 14:05:18 +08006896}
6897
6898static void destroy_fixed_file_ref_node(struct fixed_file_ref_node *ref_node)
6899{
6900 percpu_ref_exit(&ref_node->refs);
6901 kfree(ref_node);
Jens Axboe05f3fb32019-12-09 11:22:50 -07006902}
6903
6904static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg,
6905 unsigned nr_args)
6906{
6907 __s32 __user *fds = (__s32 __user *) arg;
6908 unsigned nr_tables;
6909 struct file *file;
6910 int fd, ret = 0;
6911 unsigned i;
Xiaoguang Wang05589552020-03-31 14:05:18 +08006912 struct fixed_file_ref_node *ref_node;
Jens Axboe05f3fb32019-12-09 11:22:50 -07006913
6914 if (ctx->file_data)
6915 return -EBUSY;
6916 if (!nr_args)
6917 return -EINVAL;
6918 if (nr_args > IORING_MAX_FIXED_FILES)
6919 return -EMFILE;
6920
6921 ctx->file_data = kzalloc(sizeof(*ctx->file_data), GFP_KERNEL);
6922 if (!ctx->file_data)
6923 return -ENOMEM;
6924 ctx->file_data->ctx = ctx;
6925 init_completion(&ctx->file_data->done);
Xiaoguang Wang05589552020-03-31 14:05:18 +08006926 INIT_LIST_HEAD(&ctx->file_data->ref_list);
Xiaoguang Wangf7fe9342020-04-07 20:02:31 +08006927 spin_lock_init(&ctx->file_data->lock);
Jens Axboe05f3fb32019-12-09 11:22:50 -07006928
6929 nr_tables = DIV_ROUND_UP(nr_args, IORING_MAX_FILES_TABLE);
6930 ctx->file_data->table = kcalloc(nr_tables,
6931 sizeof(struct fixed_file_table),
6932 GFP_KERNEL);
6933 if (!ctx->file_data->table) {
6934 kfree(ctx->file_data);
6935 ctx->file_data = NULL;
6936 return -ENOMEM;
6937 }
6938
Xiaoguang Wang05589552020-03-31 14:05:18 +08006939 if (percpu_ref_init(&ctx->file_data->refs, io_file_ref_kill,
Jens Axboe05f3fb32019-12-09 11:22:50 -07006940 PERCPU_REF_ALLOW_REINIT, GFP_KERNEL)) {
6941 kfree(ctx->file_data->table);
6942 kfree(ctx->file_data);
6943 ctx->file_data = NULL;
6944 return -ENOMEM;
6945 }
Jens Axboe05f3fb32019-12-09 11:22:50 -07006946
6947 if (io_sqe_alloc_file_tables(ctx, nr_tables, nr_args)) {
6948 percpu_ref_exit(&ctx->file_data->refs);
6949 kfree(ctx->file_data->table);
6950 kfree(ctx->file_data);
6951 ctx->file_data = NULL;
6952 return -ENOMEM;
6953 }
6954
6955 for (i = 0; i < nr_args; i++, ctx->nr_user_files++) {
6956 struct fixed_file_table *table;
6957 unsigned index;
6958
6959 ret = -EFAULT;
6960 if (copy_from_user(&fd, &fds[i], sizeof(fd)))
6961 break;
6962 /* allow sparse sets */
6963 if (fd == -1) {
6964 ret = 0;
6965 continue;
6966 }
6967
6968 table = &ctx->file_data->table[i >> IORING_FILE_TABLE_SHIFT];
6969 index = i & IORING_FILE_TABLE_MASK;
6970 file = fget(fd);
6971
6972 ret = -EBADF;
6973 if (!file)
6974 break;
6975
6976 /*
6977 * Don't allow io_uring instances to be registered. If UNIX
6978 * isn't enabled, then this causes a reference cycle and this
6979 * instance can never get freed. If UNIX is enabled we'll
6980 * handle it just fine, but there's still no point in allowing
6981 * a ring fd as it doesn't support regular read/write anyway.
6982 */
6983 if (file->f_op == &io_uring_fops) {
6984 fput(file);
6985 break;
6986 }
6987 ret = 0;
6988 table->files[index] = file;
6989 }
6990
6991 if (ret) {
6992 for (i = 0; i < ctx->nr_user_files; i++) {
6993 file = io_file_from_index(ctx, i);
6994 if (file)
6995 fput(file);
6996 }
6997 for (i = 0; i < nr_tables; i++)
6998 kfree(ctx->file_data->table[i].files);
6999
7000 kfree(ctx->file_data->table);
7001 kfree(ctx->file_data);
7002 ctx->file_data = NULL;
7003 ctx->nr_user_files = 0;
7004 return ret;
7005 }
7006
7007 ret = io_sqe_files_scm(ctx);
Xiaoguang Wang05589552020-03-31 14:05:18 +08007008 if (ret) {
Jens Axboe05f3fb32019-12-09 11:22:50 -07007009 io_sqe_files_unregister(ctx);
Xiaoguang Wang05589552020-03-31 14:05:18 +08007010 return ret;
7011 }
Jens Axboe05f3fb32019-12-09 11:22:50 -07007012
Xiaoguang Wang05589552020-03-31 14:05:18 +08007013 ref_node = alloc_fixed_file_ref_node(ctx);
7014 if (IS_ERR(ref_node)) {
7015 io_sqe_files_unregister(ctx);
7016 return PTR_ERR(ref_node);
7017 }
7018
7019 ctx->file_data->cur_refs = &ref_node->refs;
Jens Axboe6a4d07c2020-05-15 14:30:38 -06007020 spin_lock(&ctx->file_data->lock);
Xiaoguang Wang05589552020-03-31 14:05:18 +08007021 list_add(&ref_node->node, &ctx->file_data->ref_list);
Jens Axboe6a4d07c2020-05-15 14:30:38 -06007022 spin_unlock(&ctx->file_data->lock);
Xiaoguang Wang05589552020-03-31 14:05:18 +08007023 percpu_ref_get(&ctx->file_data->refs);
Jens Axboe05f3fb32019-12-09 11:22:50 -07007024 return ret;
7025}
7026
Jens Axboec3a31e62019-10-03 13:59:56 -06007027static int io_sqe_file_register(struct io_ring_ctx *ctx, struct file *file,
7028 int index)
7029{
7030#if defined(CONFIG_UNIX)
7031 struct sock *sock = ctx->ring_sock->sk;
7032 struct sk_buff_head *head = &sock->sk_receive_queue;
7033 struct sk_buff *skb;
7034
7035 /*
7036 * See if we can merge this file into an existing skb SCM_RIGHTS
7037 * file set. If there's no room, fall back to allocating a new skb
7038 * and filling it in.
7039 */
7040 spin_lock_irq(&head->lock);
7041 skb = skb_peek(head);
7042 if (skb) {
7043 struct scm_fp_list *fpl = UNIXCB(skb).fp;
7044
7045 if (fpl->count < SCM_MAX_FD) {
7046 __skb_unlink(skb, head);
7047 spin_unlock_irq(&head->lock);
7048 fpl->fp[fpl->count] = get_file(file);
7049 unix_inflight(fpl->user, fpl->fp[fpl->count]);
7050 fpl->count++;
7051 spin_lock_irq(&head->lock);
7052 __skb_queue_head(head, skb);
7053 } else {
7054 skb = NULL;
7055 }
7056 }
7057 spin_unlock_irq(&head->lock);
7058
7059 if (skb) {
7060 fput(file);
7061 return 0;
7062 }
7063
7064 return __io_sqe_files_scm(ctx, 1, index);
7065#else
7066 return 0;
7067#endif
7068}
7069
Hillf Dantona5318d32020-03-23 17:47:15 +08007070static int io_queue_file_removal(struct fixed_file_data *data,
Xiaoguang Wang05589552020-03-31 14:05:18 +08007071 struct file *file)
Jens Axboe05f3fb32019-12-09 11:22:50 -07007072{
Hillf Dantona5318d32020-03-23 17:47:15 +08007073 struct io_file_put *pfile;
Xiaoguang Wang05589552020-03-31 14:05:18 +08007074 struct percpu_ref *refs = data->cur_refs;
7075 struct fixed_file_ref_node *ref_node;
Jens Axboe05f3fb32019-12-09 11:22:50 -07007076
Jens Axboe05f3fb32019-12-09 11:22:50 -07007077 pfile = kzalloc(sizeof(*pfile), GFP_KERNEL);
Hillf Dantona5318d32020-03-23 17:47:15 +08007078 if (!pfile)
7079 return -ENOMEM;
Jens Axboe05f3fb32019-12-09 11:22:50 -07007080
Xiaoguang Wang05589552020-03-31 14:05:18 +08007081 ref_node = container_of(refs, struct fixed_file_ref_node, refs);
Jens Axboe05f3fb32019-12-09 11:22:50 -07007082 pfile->file = file;
Xiaoguang Wang05589552020-03-31 14:05:18 +08007083 list_add(&pfile->list, &ref_node->file_list);
7084
Hillf Dantona5318d32020-03-23 17:47:15 +08007085 return 0;
Jens Axboe05f3fb32019-12-09 11:22:50 -07007086}
7087
7088static int __io_sqe_files_update(struct io_ring_ctx *ctx,
7089 struct io_uring_files_update *up,
7090 unsigned nr_args)
7091{
7092 struct fixed_file_data *data = ctx->file_data;
Xiaoguang Wang05589552020-03-31 14:05:18 +08007093 struct fixed_file_ref_node *ref_node;
Jens Axboe05f3fb32019-12-09 11:22:50 -07007094 struct file *file;
Jens Axboec3a31e62019-10-03 13:59:56 -06007095 __s32 __user *fds;
7096 int fd, i, err;
7097 __u32 done;
Xiaoguang Wang05589552020-03-31 14:05:18 +08007098 bool needs_switch = false;
Jens Axboec3a31e62019-10-03 13:59:56 -06007099
Jens Axboe05f3fb32019-12-09 11:22:50 -07007100 if (check_add_overflow(up->offset, nr_args, &done))
Jens Axboec3a31e62019-10-03 13:59:56 -06007101 return -EOVERFLOW;
7102 if (done > ctx->nr_user_files)
7103 return -EINVAL;
7104
Xiaoguang Wang05589552020-03-31 14:05:18 +08007105 ref_node = alloc_fixed_file_ref_node(ctx);
7106 if (IS_ERR(ref_node))
7107 return PTR_ERR(ref_node);
7108
Jens Axboec3a31e62019-10-03 13:59:56 -06007109 done = 0;
Jens Axboe05f3fb32019-12-09 11:22:50 -07007110 fds = u64_to_user_ptr(up->fds);
Jens Axboec3a31e62019-10-03 13:59:56 -06007111 while (nr_args) {
Jens Axboe65e19f52019-10-26 07:20:21 -06007112 struct fixed_file_table *table;
7113 unsigned index;
7114
Jens Axboec3a31e62019-10-03 13:59:56 -06007115 err = 0;
7116 if (copy_from_user(&fd, &fds[done], sizeof(fd))) {
7117 err = -EFAULT;
7118 break;
7119 }
Jens Axboe05f3fb32019-12-09 11:22:50 -07007120 i = array_index_nospec(up->offset, ctx->nr_user_files);
7121 table = &ctx->file_data->table[i >> IORING_FILE_TABLE_SHIFT];
Jens Axboe65e19f52019-10-26 07:20:21 -06007122 index = i & IORING_FILE_TABLE_MASK;
7123 if (table->files[index]) {
Jens Axboe05f3fb32019-12-09 11:22:50 -07007124 file = io_file_from_index(ctx, index);
Hillf Dantona5318d32020-03-23 17:47:15 +08007125 err = io_queue_file_removal(data, file);
7126 if (err)
7127 break;
Jens Axboe65e19f52019-10-26 07:20:21 -06007128 table->files[index] = NULL;
Xiaoguang Wang05589552020-03-31 14:05:18 +08007129 needs_switch = true;
Jens Axboec3a31e62019-10-03 13:59:56 -06007130 }
7131 if (fd != -1) {
Jens Axboec3a31e62019-10-03 13:59:56 -06007132 file = fget(fd);
7133 if (!file) {
7134 err = -EBADF;
7135 break;
7136 }
7137 /*
7138 * Don't allow io_uring instances to be registered. If
7139 * UNIX isn't enabled, then this causes a reference
7140 * cycle and this instance can never get freed. If UNIX
7141 * is enabled we'll handle it just fine, but there's
7142 * still no point in allowing a ring fd as it doesn't
7143 * support regular read/write anyway.
7144 */
7145 if (file->f_op == &io_uring_fops) {
7146 fput(file);
7147 err = -EBADF;
7148 break;
7149 }
Jens Axboe65e19f52019-10-26 07:20:21 -06007150 table->files[index] = file;
Jens Axboec3a31e62019-10-03 13:59:56 -06007151 err = io_sqe_file_register(ctx, file, i);
7152 if (err)
7153 break;
7154 }
7155 nr_args--;
7156 done++;
Jens Axboe05f3fb32019-12-09 11:22:50 -07007157 up->offset++;
7158 }
7159
Xiaoguang Wang05589552020-03-31 14:05:18 +08007160 if (needs_switch) {
7161 percpu_ref_kill(data->cur_refs);
Jens Axboe6a4d07c2020-05-15 14:30:38 -06007162 spin_lock(&data->lock);
Xiaoguang Wang05589552020-03-31 14:05:18 +08007163 list_add(&ref_node->node, &data->ref_list);
7164 data->cur_refs = &ref_node->refs;
Jens Axboe6a4d07c2020-05-15 14:30:38 -06007165 spin_unlock(&data->lock);
Xiaoguang Wang05589552020-03-31 14:05:18 +08007166 percpu_ref_get(&ctx->file_data->refs);
7167 } else
7168 destroy_fixed_file_ref_node(ref_node);
Jens Axboec3a31e62019-10-03 13:59:56 -06007169
7170 return done ? done : err;
7171}
Xiaoguang Wang05589552020-03-31 14:05:18 +08007172
Jens Axboe05f3fb32019-12-09 11:22:50 -07007173static int io_sqe_files_update(struct io_ring_ctx *ctx, void __user *arg,
7174 unsigned nr_args)
7175{
7176 struct io_uring_files_update up;
7177
7178 if (!ctx->file_data)
7179 return -ENXIO;
7180 if (!nr_args)
7181 return -EINVAL;
7182 if (copy_from_user(&up, arg, sizeof(up)))
7183 return -EFAULT;
7184 if (up.resv)
7185 return -EINVAL;
7186
7187 return __io_sqe_files_update(ctx, &up, nr_args);
7188}
Jens Axboec3a31e62019-10-03 13:59:56 -06007189
Pavel Begunkove9fd9392020-03-04 16:14:12 +03007190static void io_free_work(struct io_wq_work *work)
Jens Axboe7d723062019-11-12 22:31:31 -07007191{
7192 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
7193
Pavel Begunkove9fd9392020-03-04 16:14:12 +03007194 /* Consider that io_steal_work() relies on this ref */
Jens Axboe7d723062019-11-12 22:31:31 -07007195 io_put_req(req);
7196}
7197
Pavel Begunkov24369c22020-01-28 03:15:48 +03007198static int io_init_wq_offload(struct io_ring_ctx *ctx,
7199 struct io_uring_params *p)
7200{
7201 struct io_wq_data data;
7202 struct fd f;
7203 struct io_ring_ctx *ctx_attach;
7204 unsigned int concurrency;
7205 int ret = 0;
7206
7207 data.user = ctx->user;
Pavel Begunkove9fd9392020-03-04 16:14:12 +03007208 data.free_work = io_free_work;
Pavel Begunkovf5fa38c2020-06-08 21:08:20 +03007209 data.do_work = io_wq_submit_work;
Pavel Begunkov24369c22020-01-28 03:15:48 +03007210
7211 if (!(p->flags & IORING_SETUP_ATTACH_WQ)) {
7212 /* Do QD, or 4 * CPUS, whatever is smallest */
7213 concurrency = min(ctx->sq_entries, 4 * num_online_cpus());
7214
7215 ctx->io_wq = io_wq_create(concurrency, &data);
7216 if (IS_ERR(ctx->io_wq)) {
7217 ret = PTR_ERR(ctx->io_wq);
7218 ctx->io_wq = NULL;
7219 }
7220 return ret;
7221 }
7222
7223 f = fdget(p->wq_fd);
7224 if (!f.file)
7225 return -EBADF;
7226
7227 if (f.file->f_op != &io_uring_fops) {
7228 ret = -EINVAL;
7229 goto out_fput;
7230 }
7231
7232 ctx_attach = f.file->private_data;
7233 /* @io_wq is protected by holding the fd */
7234 if (!io_wq_get(ctx_attach->io_wq, &data)) {
7235 ret = -EINVAL;
7236 goto out_fput;
7237 }
7238
7239 ctx->io_wq = ctx_attach->io_wq;
7240out_fput:
7241 fdput(f);
7242 return ret;
7243}
7244
Jens Axboe6c271ce2019-01-10 11:22:30 -07007245static int io_sq_offload_start(struct io_ring_ctx *ctx,
7246 struct io_uring_params *p)
Jens Axboe2b188cc2019-01-07 10:46:33 -07007247{
7248 int ret;
7249
Jens Axboe6c271ce2019-01-10 11:22:30 -07007250 if (ctx->flags & IORING_SETUP_SQPOLL) {
Pavel Begunkov8eb06d72020-06-30 15:20:39 +03007251 mmgrab(current->mm);
7252 ctx->sqo_mm = current->mm;
7253
Jens Axboe3ec482d2019-04-08 10:51:01 -06007254 ret = -EPERM;
7255 if (!capable(CAP_SYS_ADMIN))
7256 goto err;
7257
Jens Axboe917257d2019-04-13 09:28:55 -06007258 ctx->sq_thread_idle = msecs_to_jiffies(p->sq_thread_idle);
7259 if (!ctx->sq_thread_idle)
7260 ctx->sq_thread_idle = HZ;
7261
Jens Axboe6c271ce2019-01-10 11:22:30 -07007262 if (p->flags & IORING_SETUP_SQ_AFF) {
Jens Axboe44a9bd12019-05-14 20:00:30 -06007263 int cpu = p->sq_thread_cpu;
Jens Axboe6c271ce2019-01-10 11:22:30 -07007264
Jens Axboe917257d2019-04-13 09:28:55 -06007265 ret = -EINVAL;
Jens Axboe44a9bd12019-05-14 20:00:30 -06007266 if (cpu >= nr_cpu_ids)
7267 goto err;
Shenghui Wang7889f442019-05-07 16:03:19 +08007268 if (!cpu_online(cpu))
Jens Axboe917257d2019-04-13 09:28:55 -06007269 goto err;
7270
Jens Axboe6c271ce2019-01-10 11:22:30 -07007271 ctx->sqo_thread = kthread_create_on_cpu(io_sq_thread,
7272 ctx, cpu,
7273 "io_uring-sq");
7274 } else {
7275 ctx->sqo_thread = kthread_create(io_sq_thread, ctx,
7276 "io_uring-sq");
7277 }
7278 if (IS_ERR(ctx->sqo_thread)) {
7279 ret = PTR_ERR(ctx->sqo_thread);
7280 ctx->sqo_thread = NULL;
7281 goto err;
7282 }
7283 wake_up_process(ctx->sqo_thread);
7284 } else if (p->flags & IORING_SETUP_SQ_AFF) {
7285 /* Can't have SQ_AFF without SQPOLL */
7286 ret = -EINVAL;
7287 goto err;
7288 }
7289
Pavel Begunkov24369c22020-01-28 03:15:48 +03007290 ret = io_init_wq_offload(ctx, p);
7291 if (ret)
Jens Axboe2b188cc2019-01-07 10:46:33 -07007292 goto err;
Jens Axboe2b188cc2019-01-07 10:46:33 -07007293
7294 return 0;
7295err:
Jens Axboe54a91f32019-09-10 09:15:04 -06007296 io_finish_async(ctx);
Pavel Begunkov8eb06d72020-06-30 15:20:39 +03007297 if (ctx->sqo_mm) {
7298 mmdrop(ctx->sqo_mm);
7299 ctx->sqo_mm = NULL;
7300 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07007301 return ret;
7302}
7303
Bijan Mottahedeha087e2b2020-06-16 16:36:07 -07007304static inline void __io_unaccount_mem(struct user_struct *user,
7305 unsigned long nr_pages)
Jens Axboe2b188cc2019-01-07 10:46:33 -07007306{
7307 atomic_long_sub(nr_pages, &user->locked_vm);
7308}
7309
Bijan Mottahedeha087e2b2020-06-16 16:36:07 -07007310static inline int __io_account_mem(struct user_struct *user,
7311 unsigned long nr_pages)
Jens Axboe2b188cc2019-01-07 10:46:33 -07007312{
7313 unsigned long page_limit, cur_pages, new_pages;
7314
7315 /* Don't allow more pages than we can safely lock */
7316 page_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
7317
7318 do {
7319 cur_pages = atomic_long_read(&user->locked_vm);
7320 new_pages = cur_pages + nr_pages;
7321 if (new_pages > page_limit)
7322 return -ENOMEM;
7323 } while (atomic_long_cmpxchg(&user->locked_vm, cur_pages,
7324 new_pages) != cur_pages);
7325
7326 return 0;
7327}
7328
Bijan Mottahedeh2e0464d2020-06-16 16:36:10 -07007329static void io_unaccount_mem(struct io_ring_ctx *ctx, unsigned long nr_pages,
7330 enum io_mem_account acct)
Bijan Mottahedeha087e2b2020-06-16 16:36:07 -07007331{
Bijan Mottahedehaad5d8d2020-06-16 16:36:08 -07007332 if (ctx->limit_mem)
Bijan Mottahedeha087e2b2020-06-16 16:36:07 -07007333 __io_unaccount_mem(ctx->user, nr_pages);
Bijan Mottahedeh30975822020-06-16 16:36:09 -07007334
Bijan Mottahedeh2e0464d2020-06-16 16:36:10 -07007335 if (ctx->sqo_mm) {
7336 if (acct == ACCT_LOCKED)
7337 ctx->sqo_mm->locked_vm -= nr_pages;
7338 else if (acct == ACCT_PINNED)
7339 atomic64_sub(nr_pages, &ctx->sqo_mm->pinned_vm);
7340 }
Bijan Mottahedeha087e2b2020-06-16 16:36:07 -07007341}
7342
Bijan Mottahedeh2e0464d2020-06-16 16:36:10 -07007343static int io_account_mem(struct io_ring_ctx *ctx, unsigned long nr_pages,
7344 enum io_mem_account acct)
Bijan Mottahedeha087e2b2020-06-16 16:36:07 -07007345{
Bijan Mottahedeh30975822020-06-16 16:36:09 -07007346 int ret;
7347
7348 if (ctx->limit_mem) {
7349 ret = __io_account_mem(ctx->user, nr_pages);
7350 if (ret)
7351 return ret;
7352 }
7353
Bijan Mottahedeh2e0464d2020-06-16 16:36:10 -07007354 if (ctx->sqo_mm) {
7355 if (acct == ACCT_LOCKED)
7356 ctx->sqo_mm->locked_vm += nr_pages;
7357 else if (acct == ACCT_PINNED)
7358 atomic64_add(nr_pages, &ctx->sqo_mm->pinned_vm);
7359 }
Bijan Mottahedeha087e2b2020-06-16 16:36:07 -07007360
7361 return 0;
7362}
7363
Jens Axboe2b188cc2019-01-07 10:46:33 -07007364static void io_mem_free(void *ptr)
7365{
Mark Rutland52e04ef2019-04-30 17:30:21 +01007366 struct page *page;
Jens Axboe2b188cc2019-01-07 10:46:33 -07007367
Mark Rutland52e04ef2019-04-30 17:30:21 +01007368 if (!ptr)
7369 return;
7370
7371 page = virt_to_head_page(ptr);
Jens Axboe2b188cc2019-01-07 10:46:33 -07007372 if (put_page_testzero(page))
7373 free_compound_page(page);
7374}
7375
7376static void *io_mem_alloc(size_t size)
7377{
7378 gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN | __GFP_COMP |
7379 __GFP_NORETRY;
7380
7381 return (void *) __get_free_pages(gfp_flags, get_order(size));
7382}
7383
Hristo Venev75b28af2019-08-26 17:23:46 +00007384static unsigned long rings_size(unsigned sq_entries, unsigned cq_entries,
7385 size_t *sq_offset)
7386{
7387 struct io_rings *rings;
7388 size_t off, sq_array_size;
7389
7390 off = struct_size(rings, cqes, cq_entries);
7391 if (off == SIZE_MAX)
7392 return SIZE_MAX;
7393
7394#ifdef CONFIG_SMP
7395 off = ALIGN(off, SMP_CACHE_BYTES);
7396 if (off == 0)
7397 return SIZE_MAX;
7398#endif
7399
7400 sq_array_size = array_size(sizeof(u32), sq_entries);
7401 if (sq_array_size == SIZE_MAX)
7402 return SIZE_MAX;
7403
7404 if (check_add_overflow(off, sq_array_size, &off))
7405 return SIZE_MAX;
7406
7407 if (sq_offset)
7408 *sq_offset = off;
7409
7410 return off;
7411}
7412
Jens Axboe2b188cc2019-01-07 10:46:33 -07007413static unsigned long ring_pages(unsigned sq_entries, unsigned cq_entries)
7414{
Hristo Venev75b28af2019-08-26 17:23:46 +00007415 size_t pages;
Jens Axboe2b188cc2019-01-07 10:46:33 -07007416
Hristo Venev75b28af2019-08-26 17:23:46 +00007417 pages = (size_t)1 << get_order(
7418 rings_size(sq_entries, cq_entries, NULL));
7419 pages += (size_t)1 << get_order(
7420 array_size(sizeof(struct io_uring_sqe), sq_entries));
Jens Axboe2b188cc2019-01-07 10:46:33 -07007421
Hristo Venev75b28af2019-08-26 17:23:46 +00007422 return pages;
Jens Axboe2b188cc2019-01-07 10:46:33 -07007423}
7424
Jens Axboeedafcce2019-01-09 09:16:05 -07007425static int io_sqe_buffer_unregister(struct io_ring_ctx *ctx)
7426{
7427 int i, j;
7428
7429 if (!ctx->user_bufs)
7430 return -ENXIO;
7431
7432 for (i = 0; i < ctx->nr_user_bufs; i++) {
7433 struct io_mapped_ubuf *imu = &ctx->user_bufs[i];
7434
7435 for (j = 0; j < imu->nr_bvecs; j++)
John Hubbardf1f6a7d2020-01-30 22:13:35 -08007436 unpin_user_page(imu->bvec[j].bv_page);
Jens Axboeedafcce2019-01-09 09:16:05 -07007437
Bijan Mottahedeh2e0464d2020-06-16 16:36:10 -07007438 io_unaccount_mem(ctx, imu->nr_bvecs, ACCT_PINNED);
Mark Rutlandd4ef6472019-05-01 16:59:16 +01007439 kvfree(imu->bvec);
Jens Axboeedafcce2019-01-09 09:16:05 -07007440 imu->nr_bvecs = 0;
7441 }
7442
7443 kfree(ctx->user_bufs);
7444 ctx->user_bufs = NULL;
7445 ctx->nr_user_bufs = 0;
7446 return 0;
7447}
7448
7449static int io_copy_iov(struct io_ring_ctx *ctx, struct iovec *dst,
7450 void __user *arg, unsigned index)
7451{
7452 struct iovec __user *src;
7453
7454#ifdef CONFIG_COMPAT
7455 if (ctx->compat) {
7456 struct compat_iovec __user *ciovs;
7457 struct compat_iovec ciov;
7458
7459 ciovs = (struct compat_iovec __user *) arg;
7460 if (copy_from_user(&ciov, &ciovs[index], sizeof(ciov)))
7461 return -EFAULT;
7462
Jens Axboed55e5f52019-12-11 16:12:15 -07007463 dst->iov_base = u64_to_user_ptr((u64)ciov.iov_base);
Jens Axboeedafcce2019-01-09 09:16:05 -07007464 dst->iov_len = ciov.iov_len;
7465 return 0;
7466 }
7467#endif
7468 src = (struct iovec __user *) arg;
7469 if (copy_from_user(dst, &src[index], sizeof(*dst)))
7470 return -EFAULT;
7471 return 0;
7472}
7473
7474static int io_sqe_buffer_register(struct io_ring_ctx *ctx, void __user *arg,
7475 unsigned nr_args)
7476{
7477 struct vm_area_struct **vmas = NULL;
7478 struct page **pages = NULL;
7479 int i, j, got_pages = 0;
7480 int ret = -EINVAL;
7481
7482 if (ctx->user_bufs)
7483 return -EBUSY;
7484 if (!nr_args || nr_args > UIO_MAXIOV)
7485 return -EINVAL;
7486
7487 ctx->user_bufs = kcalloc(nr_args, sizeof(struct io_mapped_ubuf),
7488 GFP_KERNEL);
7489 if (!ctx->user_bufs)
7490 return -ENOMEM;
7491
7492 for (i = 0; i < nr_args; i++) {
7493 struct io_mapped_ubuf *imu = &ctx->user_bufs[i];
7494 unsigned long off, start, end, ubuf;
7495 int pret, nr_pages;
7496 struct iovec iov;
7497 size_t size;
7498
7499 ret = io_copy_iov(ctx, &iov, arg, i);
7500 if (ret)
Pavel Begunkova2786822019-05-26 12:35:47 +03007501 goto err;
Jens Axboeedafcce2019-01-09 09:16:05 -07007502
7503 /*
7504 * Don't impose further limits on the size and buffer
7505 * constraints here, we'll -EINVAL later when IO is
7506 * submitted if they are wrong.
7507 */
7508 ret = -EFAULT;
7509 if (!iov.iov_base || !iov.iov_len)
7510 goto err;
7511
7512 /* arbitrary limit, but we need something */
7513 if (iov.iov_len > SZ_1G)
7514 goto err;
7515
7516 ubuf = (unsigned long) iov.iov_base;
7517 end = (ubuf + iov.iov_len + PAGE_SIZE - 1) >> PAGE_SHIFT;
7518 start = ubuf >> PAGE_SHIFT;
7519 nr_pages = end - start;
7520
Bijan Mottahedeh2e0464d2020-06-16 16:36:10 -07007521 ret = io_account_mem(ctx, nr_pages, ACCT_PINNED);
Bijan Mottahedeha087e2b2020-06-16 16:36:07 -07007522 if (ret)
7523 goto err;
Jens Axboeedafcce2019-01-09 09:16:05 -07007524
7525 ret = 0;
7526 if (!pages || nr_pages > got_pages) {
Denis Efremova8c73c12020-06-05 12:32:03 +03007527 kvfree(vmas);
7528 kvfree(pages);
Mark Rutlandd4ef6472019-05-01 16:59:16 +01007529 pages = kvmalloc_array(nr_pages, sizeof(struct page *),
Jens Axboeedafcce2019-01-09 09:16:05 -07007530 GFP_KERNEL);
Mark Rutlandd4ef6472019-05-01 16:59:16 +01007531 vmas = kvmalloc_array(nr_pages,
Jens Axboeedafcce2019-01-09 09:16:05 -07007532 sizeof(struct vm_area_struct *),
7533 GFP_KERNEL);
7534 if (!pages || !vmas) {
7535 ret = -ENOMEM;
Bijan Mottahedeh2e0464d2020-06-16 16:36:10 -07007536 io_unaccount_mem(ctx, nr_pages, ACCT_PINNED);
Jens Axboeedafcce2019-01-09 09:16:05 -07007537 goto err;
7538 }
7539 got_pages = nr_pages;
7540 }
7541
Mark Rutlandd4ef6472019-05-01 16:59:16 +01007542 imu->bvec = kvmalloc_array(nr_pages, sizeof(struct bio_vec),
Jens Axboeedafcce2019-01-09 09:16:05 -07007543 GFP_KERNEL);
7544 ret = -ENOMEM;
7545 if (!imu->bvec) {
Bijan Mottahedeh2e0464d2020-06-16 16:36:10 -07007546 io_unaccount_mem(ctx, nr_pages, ACCT_PINNED);
Jens Axboeedafcce2019-01-09 09:16:05 -07007547 goto err;
7548 }
7549
7550 ret = 0;
Michel Lespinassed8ed45c2020-06-08 21:33:25 -07007551 mmap_read_lock(current->mm);
John Hubbard2113b052020-01-30 22:13:13 -08007552 pret = pin_user_pages(ubuf, nr_pages,
Ira Weiny932f4a62019-05-13 17:17:03 -07007553 FOLL_WRITE | FOLL_LONGTERM,
7554 pages, vmas);
Jens Axboeedafcce2019-01-09 09:16:05 -07007555 if (pret == nr_pages) {
7556 /* don't support file backed memory */
7557 for (j = 0; j < nr_pages; j++) {
7558 struct vm_area_struct *vma = vmas[j];
7559
7560 if (vma->vm_file &&
7561 !is_file_hugepages(vma->vm_file)) {
7562 ret = -EOPNOTSUPP;
7563 break;
7564 }
7565 }
7566 } else {
7567 ret = pret < 0 ? pret : -EFAULT;
7568 }
Michel Lespinassed8ed45c2020-06-08 21:33:25 -07007569 mmap_read_unlock(current->mm);
Jens Axboeedafcce2019-01-09 09:16:05 -07007570 if (ret) {
7571 /*
7572 * if we did partial map, or found file backed vmas,
7573 * release any pages we did get
7574 */
John Hubbard27c4d3a2019-08-04 19:32:06 -07007575 if (pret > 0)
John Hubbardf1f6a7d2020-01-30 22:13:35 -08007576 unpin_user_pages(pages, pret);
Bijan Mottahedeh2e0464d2020-06-16 16:36:10 -07007577 io_unaccount_mem(ctx, nr_pages, ACCT_PINNED);
Mark Rutlandd4ef6472019-05-01 16:59:16 +01007578 kvfree(imu->bvec);
Jens Axboeedafcce2019-01-09 09:16:05 -07007579 goto err;
7580 }
7581
7582 off = ubuf & ~PAGE_MASK;
7583 size = iov.iov_len;
7584 for (j = 0; j < nr_pages; j++) {
7585 size_t vec_len;
7586
7587 vec_len = min_t(size_t, size, PAGE_SIZE - off);
7588 imu->bvec[j].bv_page = pages[j];
7589 imu->bvec[j].bv_len = vec_len;
7590 imu->bvec[j].bv_offset = off;
7591 off = 0;
7592 size -= vec_len;
7593 }
7594 /* store original address for later verification */
7595 imu->ubuf = ubuf;
7596 imu->len = iov.iov_len;
7597 imu->nr_bvecs = nr_pages;
7598
7599 ctx->nr_user_bufs++;
7600 }
Mark Rutlandd4ef6472019-05-01 16:59:16 +01007601 kvfree(pages);
7602 kvfree(vmas);
Jens Axboeedafcce2019-01-09 09:16:05 -07007603 return 0;
7604err:
Mark Rutlandd4ef6472019-05-01 16:59:16 +01007605 kvfree(pages);
7606 kvfree(vmas);
Jens Axboeedafcce2019-01-09 09:16:05 -07007607 io_sqe_buffer_unregister(ctx);
7608 return ret;
7609}
7610
Jens Axboe9b402842019-04-11 11:45:41 -06007611static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg)
7612{
7613 __s32 __user *fds = arg;
7614 int fd;
7615
7616 if (ctx->cq_ev_fd)
7617 return -EBUSY;
7618
7619 if (copy_from_user(&fd, fds, sizeof(*fds)))
7620 return -EFAULT;
7621
7622 ctx->cq_ev_fd = eventfd_ctx_fdget(fd);
7623 if (IS_ERR(ctx->cq_ev_fd)) {
7624 int ret = PTR_ERR(ctx->cq_ev_fd);
7625 ctx->cq_ev_fd = NULL;
7626 return ret;
7627 }
7628
7629 return 0;
7630}
7631
7632static int io_eventfd_unregister(struct io_ring_ctx *ctx)
7633{
7634 if (ctx->cq_ev_fd) {
7635 eventfd_ctx_put(ctx->cq_ev_fd);
7636 ctx->cq_ev_fd = NULL;
7637 return 0;
7638 }
7639
7640 return -ENXIO;
7641}
7642
Jens Axboe5a2e7452020-02-23 16:23:11 -07007643static int __io_destroy_buffers(int id, void *p, void *data)
7644{
7645 struct io_ring_ctx *ctx = data;
7646 struct io_buffer *buf = p;
7647
Jens Axboe067524e2020-03-02 16:32:28 -07007648 __io_remove_buffers(ctx, buf, id, -1U);
Jens Axboe5a2e7452020-02-23 16:23:11 -07007649 return 0;
7650}
7651
7652static void io_destroy_buffers(struct io_ring_ctx *ctx)
7653{
7654 idr_for_each(&ctx->io_buffer_idr, __io_destroy_buffers, ctx);
7655 idr_destroy(&ctx->io_buffer_idr);
7656}
7657
Jens Axboe2b188cc2019-01-07 10:46:33 -07007658static void io_ring_ctx_free(struct io_ring_ctx *ctx)
7659{
Jens Axboe6b063142019-01-10 22:13:58 -07007660 io_finish_async(ctx);
Bijan Mottahedeh30975822020-06-16 16:36:09 -07007661 if (ctx->sqo_mm) {
Jens Axboe2b188cc2019-01-07 10:46:33 -07007662 mmdrop(ctx->sqo_mm);
Bijan Mottahedeh30975822020-06-16 16:36:09 -07007663 ctx->sqo_mm = NULL;
7664 }
Jens Axboedef596e2019-01-09 08:59:42 -07007665
7666 io_iopoll_reap_events(ctx);
Jens Axboeedafcce2019-01-09 09:16:05 -07007667 io_sqe_buffer_unregister(ctx);
Jens Axboe6b063142019-01-10 22:13:58 -07007668 io_sqe_files_unregister(ctx);
Jens Axboe9b402842019-04-11 11:45:41 -06007669 io_eventfd_unregister(ctx);
Jens Axboe5a2e7452020-02-23 16:23:11 -07007670 io_destroy_buffers(ctx);
Jens Axboe41726c92020-02-23 13:11:42 -07007671 idr_destroy(&ctx->personality_idr);
Jens Axboedef596e2019-01-09 08:59:42 -07007672
Jens Axboe2b188cc2019-01-07 10:46:33 -07007673#if defined(CONFIG_UNIX)
Eric Biggers355e8d22019-06-12 14:58:43 -07007674 if (ctx->ring_sock) {
7675 ctx->ring_sock->file = NULL; /* so that iput() is called */
Jens Axboe2b188cc2019-01-07 10:46:33 -07007676 sock_release(ctx->ring_sock);
Eric Biggers355e8d22019-06-12 14:58:43 -07007677 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07007678#endif
7679
Hristo Venev75b28af2019-08-26 17:23:46 +00007680 io_mem_free(ctx->rings);
Jens Axboe2b188cc2019-01-07 10:46:33 -07007681 io_mem_free(ctx->sq_sqes);
Jens Axboe2b188cc2019-01-07 10:46:33 -07007682
7683 percpu_ref_exit(&ctx->refs);
Bijan Mottahedeh2e0464d2020-06-16 16:36:10 -07007684 io_unaccount_mem(ctx, ring_pages(ctx->sq_entries, ctx->cq_entries),
7685 ACCT_LOCKED);
Jens Axboe2b188cc2019-01-07 10:46:33 -07007686 free_uid(ctx->user);
Jens Axboe181e4482019-11-25 08:52:30 -07007687 put_cred(ctx->creds);
Jens Axboe78076bb2019-12-04 19:56:40 -07007688 kfree(ctx->cancel_hash);
Jens Axboe0ddf92e2019-11-08 08:52:53 -07007689 kmem_cache_free(req_cachep, ctx->fallback_req);
Jens Axboe2b188cc2019-01-07 10:46:33 -07007690 kfree(ctx);
7691}
7692
7693static __poll_t io_uring_poll(struct file *file, poll_table *wait)
7694{
7695 struct io_ring_ctx *ctx = file->private_data;
7696 __poll_t mask = 0;
7697
7698 poll_wait(file, &ctx->cq_wait, wait);
Stefan Bühler4f7067c2019-04-24 23:54:17 +02007699 /*
7700 * synchronizes with barrier from wq_has_sleeper call in
7701 * io_commit_cqring
7702 */
Jens Axboe2b188cc2019-01-07 10:46:33 -07007703 smp_rmb();
Hristo Venev75b28af2019-08-26 17:23:46 +00007704 if (READ_ONCE(ctx->rings->sq.tail) - ctx->cached_sq_head !=
7705 ctx->rings->sq_ring_entries)
Jens Axboe2b188cc2019-01-07 10:46:33 -07007706 mask |= EPOLLOUT | EPOLLWRNORM;
Stefano Garzarella63e5d812020-02-07 13:18:28 +01007707 if (io_cqring_events(ctx, false))
Jens Axboe2b188cc2019-01-07 10:46:33 -07007708 mask |= EPOLLIN | EPOLLRDNORM;
7709
7710 return mask;
7711}
7712
7713static int io_uring_fasync(int fd, struct file *file, int on)
7714{
7715 struct io_ring_ctx *ctx = file->private_data;
7716
7717 return fasync_helper(fd, file, on, &ctx->cq_fasync);
7718}
7719
Jens Axboe071698e2020-01-28 10:04:42 -07007720static int io_remove_personalities(int id, void *p, void *data)
7721{
7722 struct io_ring_ctx *ctx = data;
7723 const struct cred *cred;
7724
7725 cred = idr_remove(&ctx->personality_idr, id);
7726 if (cred)
7727 put_cred(cred);
7728 return 0;
7729}
7730
Jens Axboe85faa7b2020-04-09 18:14:00 -06007731static void io_ring_exit_work(struct work_struct *work)
7732{
7733 struct io_ring_ctx *ctx;
7734
7735 ctx = container_of(work, struct io_ring_ctx, exit_work);
7736 if (ctx->rings)
7737 io_cqring_overflow_flush(ctx, true);
7738
Jens Axboe56952e92020-06-17 15:00:04 -06007739 /*
7740 * If we're doing polled IO and end up having requests being
7741 * submitted async (out-of-line), then completions can come in while
7742 * we're waiting for refs to drop. We need to reap these manually,
7743 * as nobody else will be looking for them.
7744 */
7745 while (!wait_for_completion_timeout(&ctx->ref_comp, HZ/20)) {
7746 io_iopoll_reap_events(ctx);
7747 if (ctx->rings)
7748 io_cqring_overflow_flush(ctx, true);
7749 }
Jens Axboe85faa7b2020-04-09 18:14:00 -06007750 io_ring_ctx_free(ctx);
7751}
7752
Jens Axboe2b188cc2019-01-07 10:46:33 -07007753static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx)
7754{
7755 mutex_lock(&ctx->uring_lock);
7756 percpu_ref_kill(&ctx->refs);
7757 mutex_unlock(&ctx->uring_lock);
7758
Jens Axboe5262f562019-09-17 12:26:57 -06007759 io_kill_timeouts(ctx);
Jens Axboe221c5eb2019-01-17 09:41:58 -07007760 io_poll_remove_all(ctx);
Jens Axboe561fb042019-10-24 07:25:42 -06007761
7762 if (ctx->io_wq)
7763 io_wq_cancel_all(ctx->io_wq);
7764
Jens Axboedef596e2019-01-09 08:59:42 -07007765 io_iopoll_reap_events(ctx);
Jens Axboe15dff282019-11-13 09:09:23 -07007766 /* if we failed setting up the ctx, we might not have any rings */
7767 if (ctx->rings)
7768 io_cqring_overflow_flush(ctx, true);
Jens Axboe071698e2020-01-28 10:04:42 -07007769 idr_for_each(&ctx->personality_idr, io_remove_personalities, ctx);
Jens Axboe85faa7b2020-04-09 18:14:00 -06007770 INIT_WORK(&ctx->exit_work, io_ring_exit_work);
7771 queue_work(system_wq, &ctx->exit_work);
Jens Axboe2b188cc2019-01-07 10:46:33 -07007772}
7773
7774static int io_uring_release(struct inode *inode, struct file *file)
7775{
7776 struct io_ring_ctx *ctx = file->private_data;
7777
7778 file->private_data = NULL;
7779 io_ring_ctx_wait_and_kill(ctx);
7780 return 0;
7781}
7782
Pavel Begunkov67c4d9e2020-06-15 10:24:05 +03007783static bool io_wq_files_match(struct io_wq_work *work, void *data)
7784{
7785 struct files_struct *files = data;
7786
7787 return work->files == files;
7788}
7789
Jens Axboefcb323c2019-10-24 12:39:47 -06007790static void io_uring_cancel_files(struct io_ring_ctx *ctx,
7791 struct files_struct *files)
7792{
Pavel Begunkov67c4d9e2020-06-15 10:24:05 +03007793 if (list_empty_careful(&ctx->inflight_list))
7794 return;
7795
7796 /* cancel all at once, should be faster than doing it one by one*/
7797 io_wq_cancel_cb(ctx->io_wq, io_wq_files_match, files, true);
7798
Jens Axboefcb323c2019-10-24 12:39:47 -06007799 while (!list_empty_careful(&ctx->inflight_list)) {
Xiaoguang Wangd8f1b972020-04-26 15:54:43 +08007800 struct io_kiocb *cancel_req = NULL, *req;
7801 DEFINE_WAIT(wait);
Jens Axboefcb323c2019-10-24 12:39:47 -06007802
7803 spin_lock_irq(&ctx->inflight_lock);
7804 list_for_each_entry(req, &ctx->inflight_list, inflight_entry) {
Jens Axboe768134d2019-11-10 20:30:53 -07007805 if (req->work.files != files)
7806 continue;
7807 /* req is being completed, ignore */
7808 if (!refcount_inc_not_zero(&req->refs))
7809 continue;
7810 cancel_req = req;
7811 break;
Jens Axboefcb323c2019-10-24 12:39:47 -06007812 }
Jens Axboe768134d2019-11-10 20:30:53 -07007813 if (cancel_req)
Jens Axboefcb323c2019-10-24 12:39:47 -06007814 prepare_to_wait(&ctx->inflight_wait, &wait,
Jens Axboe768134d2019-11-10 20:30:53 -07007815 TASK_UNINTERRUPTIBLE);
Jens Axboefcb323c2019-10-24 12:39:47 -06007816 spin_unlock_irq(&ctx->inflight_lock);
7817
Jens Axboe768134d2019-11-10 20:30:53 -07007818 /* We need to keep going until we don't find a matching req */
7819 if (!cancel_req)
Jens Axboefcb323c2019-10-24 12:39:47 -06007820 break;
Bob Liu2f6d9b92019-11-13 18:06:24 +08007821
Jens Axboe2ca10252020-02-13 17:17:35 -07007822 if (cancel_req->flags & REQ_F_OVERFLOW) {
7823 spin_lock_irq(&ctx->completion_lock);
7824 list_del(&cancel_req->list);
7825 cancel_req->flags &= ~REQ_F_OVERFLOW;
7826 if (list_empty(&ctx->cq_overflow_list)) {
7827 clear_bit(0, &ctx->sq_check_overflow);
7828 clear_bit(0, &ctx->cq_check_overflow);
7829 }
7830 spin_unlock_irq(&ctx->completion_lock);
7831
7832 WRITE_ONCE(ctx->rings->cq_overflow,
7833 atomic_inc_return(&ctx->cached_cq_overflow));
7834
7835 /*
7836 * Put inflight ref and overflow ref. If that's
7837 * all we had, then we're done with this request.
7838 */
7839 if (refcount_sub_and_test(2, &cancel_req->refs)) {
Pavel Begunkov4518a3c2020-05-26 20:34:02 +03007840 io_free_req(cancel_req);
Xiaoguang Wangd8f1b972020-04-26 15:54:43 +08007841 finish_wait(&ctx->inflight_wait, &wait);
Jens Axboe2ca10252020-02-13 17:17:35 -07007842 continue;
7843 }
Pavel Begunkov7b53d592020-05-30 14:19:15 +03007844 } else {
7845 io_wq_cancel_work(ctx->io_wq, &cancel_req->work);
7846 io_put_req(cancel_req);
Jens Axboe2ca10252020-02-13 17:17:35 -07007847 }
7848
Jens Axboefcb323c2019-10-24 12:39:47 -06007849 schedule();
Xiaoguang Wangd8f1b972020-04-26 15:54:43 +08007850 finish_wait(&ctx->inflight_wait, &wait);
Jens Axboefcb323c2019-10-24 12:39:47 -06007851 }
7852}
7853
Pavel Begunkov801dd572020-06-15 10:33:14 +03007854static bool io_cancel_task_cb(struct io_wq_work *work, void *data)
Pavel Begunkov44e728b2020-06-15 10:24:04 +03007855{
Pavel Begunkov801dd572020-06-15 10:33:14 +03007856 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
7857 struct task_struct *task = data;
Pavel Begunkov44e728b2020-06-15 10:24:04 +03007858
Pavel Begunkov801dd572020-06-15 10:33:14 +03007859 return req->task == task;
Pavel Begunkov44e728b2020-06-15 10:24:04 +03007860}
7861
Jens Axboefcb323c2019-10-24 12:39:47 -06007862static int io_uring_flush(struct file *file, void *data)
7863{
7864 struct io_ring_ctx *ctx = file->private_data;
7865
7866 io_uring_cancel_files(ctx, data);
Jens Axboe6ab23142020-02-08 20:23:59 -07007867
7868 /*
7869 * If the task is going away, cancel work it may have pending
7870 */
Pavel Begunkov801dd572020-06-15 10:33:14 +03007871 if (fatal_signal_pending(current) || (current->flags & PF_EXITING))
7872 io_wq_cancel_cb(ctx->io_wq, io_cancel_task_cb, current, true);
Jens Axboe6ab23142020-02-08 20:23:59 -07007873
Jens Axboefcb323c2019-10-24 12:39:47 -06007874 return 0;
7875}
7876
Roman Penyaev6c5c2402019-11-28 12:53:22 +01007877static void *io_uring_validate_mmap_request(struct file *file,
7878 loff_t pgoff, size_t sz)
Jens Axboe2b188cc2019-01-07 10:46:33 -07007879{
Jens Axboe2b188cc2019-01-07 10:46:33 -07007880 struct io_ring_ctx *ctx = file->private_data;
Roman Penyaev6c5c2402019-11-28 12:53:22 +01007881 loff_t offset = pgoff << PAGE_SHIFT;
Jens Axboe2b188cc2019-01-07 10:46:33 -07007882 struct page *page;
7883 void *ptr;
7884
7885 switch (offset) {
7886 case IORING_OFF_SQ_RING:
Hristo Venev75b28af2019-08-26 17:23:46 +00007887 case IORING_OFF_CQ_RING:
7888 ptr = ctx->rings;
Jens Axboe2b188cc2019-01-07 10:46:33 -07007889 break;
7890 case IORING_OFF_SQES:
7891 ptr = ctx->sq_sqes;
7892 break;
Jens Axboe2b188cc2019-01-07 10:46:33 -07007893 default:
Roman Penyaev6c5c2402019-11-28 12:53:22 +01007894 return ERR_PTR(-EINVAL);
Jens Axboe2b188cc2019-01-07 10:46:33 -07007895 }
7896
7897 page = virt_to_head_page(ptr);
Matthew Wilcox (Oracle)a50b8542019-09-23 15:34:25 -07007898 if (sz > page_size(page))
Roman Penyaev6c5c2402019-11-28 12:53:22 +01007899 return ERR_PTR(-EINVAL);
7900
7901 return ptr;
7902}
7903
7904#ifdef CONFIG_MMU
7905
7906static int io_uring_mmap(struct file *file, struct vm_area_struct *vma)
7907{
7908 size_t sz = vma->vm_end - vma->vm_start;
7909 unsigned long pfn;
7910 void *ptr;
7911
7912 ptr = io_uring_validate_mmap_request(file, vma->vm_pgoff, sz);
7913 if (IS_ERR(ptr))
7914 return PTR_ERR(ptr);
Jens Axboe2b188cc2019-01-07 10:46:33 -07007915
7916 pfn = virt_to_phys(ptr) >> PAGE_SHIFT;
7917 return remap_pfn_range(vma, vma->vm_start, pfn, sz, vma->vm_page_prot);
7918}
7919
Roman Penyaev6c5c2402019-11-28 12:53:22 +01007920#else /* !CONFIG_MMU */
7921
7922static int io_uring_mmap(struct file *file, struct vm_area_struct *vma)
7923{
7924 return vma->vm_flags & (VM_SHARED | VM_MAYSHARE) ? 0 : -EINVAL;
7925}
7926
7927static unsigned int io_uring_nommu_mmap_capabilities(struct file *file)
7928{
7929 return NOMMU_MAP_DIRECT | NOMMU_MAP_READ | NOMMU_MAP_WRITE;
7930}
7931
7932static unsigned long io_uring_nommu_get_unmapped_area(struct file *file,
7933 unsigned long addr, unsigned long len,
7934 unsigned long pgoff, unsigned long flags)
7935{
7936 void *ptr;
7937
7938 ptr = io_uring_validate_mmap_request(file, pgoff, len);
7939 if (IS_ERR(ptr))
7940 return PTR_ERR(ptr);
7941
7942 return (unsigned long) ptr;
7943}
7944
7945#endif /* !CONFIG_MMU */
7946
Jens Axboe2b188cc2019-01-07 10:46:33 -07007947SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
7948 u32, min_complete, u32, flags, const sigset_t __user *, sig,
7949 size_t, sigsz)
7950{
7951 struct io_ring_ctx *ctx;
7952 long ret = -EBADF;
7953 int submitted = 0;
7954 struct fd f;
7955
Jens Axboeb41e9852020-02-17 09:52:41 -07007956 if (current->task_works)
7957 task_work_run();
7958
Jens Axboe6c271ce2019-01-10 11:22:30 -07007959 if (flags & ~(IORING_ENTER_GETEVENTS | IORING_ENTER_SQ_WAKEUP))
Jens Axboe2b188cc2019-01-07 10:46:33 -07007960 return -EINVAL;
7961
7962 f = fdget(fd);
7963 if (!f.file)
7964 return -EBADF;
7965
7966 ret = -EOPNOTSUPP;
7967 if (f.file->f_op != &io_uring_fops)
7968 goto out_fput;
7969
7970 ret = -ENXIO;
7971 ctx = f.file->private_data;
7972 if (!percpu_ref_tryget(&ctx->refs))
7973 goto out_fput;
7974
Jens Axboe6c271ce2019-01-10 11:22:30 -07007975 /*
7976 * For SQ polling, the thread will do all submissions and completions.
7977 * Just return the requested submit count, and wake the thread if
7978 * we were asked to.
7979 */
Jens Axboeb2a9ead2019-09-12 14:19:16 -06007980 ret = 0;
Jens Axboe6c271ce2019-01-10 11:22:30 -07007981 if (ctx->flags & IORING_SETUP_SQPOLL) {
Jens Axboec1edbf52019-11-10 16:56:04 -07007982 if (!list_empty_careful(&ctx->cq_overflow_list))
7983 io_cqring_overflow_flush(ctx, false);
Jens Axboe6c271ce2019-01-10 11:22:30 -07007984 if (flags & IORING_ENTER_SQ_WAKEUP)
7985 wake_up(&ctx->sqo_wait);
7986 submitted = to_submit;
Jens Axboeb2a9ead2019-09-12 14:19:16 -06007987 } else if (to_submit) {
Jens Axboe2b188cc2019-01-07 10:46:33 -07007988 mutex_lock(&ctx->uring_lock);
Pavel Begunkov0cdaf762020-05-17 14:13:40 +03007989 submitted = io_submit_sqes(ctx, to_submit, f.file, fd);
Jens Axboe2b188cc2019-01-07 10:46:33 -07007990 mutex_unlock(&ctx->uring_lock);
Pavel Begunkov7c504e652019-12-18 19:53:45 +03007991
7992 if (submitted != to_submit)
7993 goto out;
Jens Axboe2b188cc2019-01-07 10:46:33 -07007994 }
7995 if (flags & IORING_ENTER_GETEVENTS) {
Jens Axboedef596e2019-01-09 08:59:42 -07007996 unsigned nr_events = 0;
7997
Jens Axboe2b188cc2019-01-07 10:46:33 -07007998 min_complete = min(min_complete, ctx->cq_entries);
7999
Xiaoguang Wang32b22442020-03-11 09:26:09 +08008000 /*
8001 * When SETUP_IOPOLL and SETUP_SQPOLL are both enabled, user
8002 * space applications don't need to do io completion events
8003 * polling again, they can rely on io_sq_thread to do polling
8004 * work, which can reduce cpu usage and uring_lock contention.
8005 */
8006 if (ctx->flags & IORING_SETUP_IOPOLL &&
8007 !(ctx->flags & IORING_SETUP_SQPOLL)) {
Jens Axboedef596e2019-01-09 08:59:42 -07008008 ret = io_iopoll_check(ctx, &nr_events, min_complete);
Jens Axboedef596e2019-01-09 08:59:42 -07008009 } else {
8010 ret = io_cqring_wait(ctx, min_complete, sig, sigsz);
8011 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07008012 }
8013
Pavel Begunkov7c504e652019-12-18 19:53:45 +03008014out:
Pavel Begunkov6805b322019-10-08 02:18:42 +03008015 percpu_ref_put(&ctx->refs);
Jens Axboe2b188cc2019-01-07 10:46:33 -07008016out_fput:
8017 fdput(f);
8018 return submitted ? submitted : ret;
8019}
8020
Tobias Klauserbebdb652020-02-26 18:38:32 +01008021#ifdef CONFIG_PROC_FS
Jens Axboe87ce9552020-01-30 08:25:34 -07008022static int io_uring_show_cred(int id, void *p, void *data)
8023{
8024 const struct cred *cred = p;
8025 struct seq_file *m = data;
8026 struct user_namespace *uns = seq_user_ns(m);
8027 struct group_info *gi;
8028 kernel_cap_t cap;
8029 unsigned __capi;
8030 int g;
8031
8032 seq_printf(m, "%5d\n", id);
8033 seq_put_decimal_ull(m, "\tUid:\t", from_kuid_munged(uns, cred->uid));
8034 seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->euid));
8035 seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->suid));
8036 seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->fsuid));
8037 seq_put_decimal_ull(m, "\n\tGid:\t", from_kgid_munged(uns, cred->gid));
8038 seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->egid));
8039 seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->sgid));
8040 seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->fsgid));
8041 seq_puts(m, "\n\tGroups:\t");
8042 gi = cred->group_info;
8043 for (g = 0; g < gi->ngroups; g++) {
8044 seq_put_decimal_ull(m, g ? " " : "",
8045 from_kgid_munged(uns, gi->gid[g]));
8046 }
8047 seq_puts(m, "\n\tCapEff:\t");
8048 cap = cred->cap_effective;
8049 CAP_FOR_EACH_U32(__capi)
8050 seq_put_hex_ll(m, NULL, cap.cap[CAP_LAST_U32 - __capi], 8);
8051 seq_putc(m, '\n');
8052 return 0;
8053}
8054
8055static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m)
8056{
8057 int i;
8058
8059 mutex_lock(&ctx->uring_lock);
8060 seq_printf(m, "UserFiles:\t%u\n", ctx->nr_user_files);
8061 for (i = 0; i < ctx->nr_user_files; i++) {
8062 struct fixed_file_table *table;
8063 struct file *f;
8064
8065 table = &ctx->file_data->table[i >> IORING_FILE_TABLE_SHIFT];
8066 f = table->files[i & IORING_FILE_TABLE_MASK];
8067 if (f)
8068 seq_printf(m, "%5u: %s\n", i, file_dentry(f)->d_iname);
8069 else
8070 seq_printf(m, "%5u: <none>\n", i);
8071 }
8072 seq_printf(m, "UserBufs:\t%u\n", ctx->nr_user_bufs);
8073 for (i = 0; i < ctx->nr_user_bufs; i++) {
8074 struct io_mapped_ubuf *buf = &ctx->user_bufs[i];
8075
8076 seq_printf(m, "%5u: 0x%llx/%u\n", i, buf->ubuf,
8077 (unsigned int) buf->len);
8078 }
8079 if (!idr_is_empty(&ctx->personality_idr)) {
8080 seq_printf(m, "Personalities:\n");
8081 idr_for_each(&ctx->personality_idr, io_uring_show_cred, m);
8082 }
Jens Axboed7718a92020-02-14 22:23:12 -07008083 seq_printf(m, "PollList:\n");
8084 spin_lock_irq(&ctx->completion_lock);
8085 for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) {
8086 struct hlist_head *list = &ctx->cancel_hash[i];
8087 struct io_kiocb *req;
8088
8089 hlist_for_each_entry(req, list, hash_node)
8090 seq_printf(m, " op=%d, task_works=%d\n", req->opcode,
8091 req->task->task_works != NULL);
8092 }
8093 spin_unlock_irq(&ctx->completion_lock);
Jens Axboe87ce9552020-01-30 08:25:34 -07008094 mutex_unlock(&ctx->uring_lock);
8095}
8096
8097static void io_uring_show_fdinfo(struct seq_file *m, struct file *f)
8098{
8099 struct io_ring_ctx *ctx = f->private_data;
8100
8101 if (percpu_ref_tryget(&ctx->refs)) {
8102 __io_uring_show_fdinfo(ctx, m);
8103 percpu_ref_put(&ctx->refs);
8104 }
8105}
Tobias Klauserbebdb652020-02-26 18:38:32 +01008106#endif
Jens Axboe87ce9552020-01-30 08:25:34 -07008107
Jens Axboe2b188cc2019-01-07 10:46:33 -07008108static const struct file_operations io_uring_fops = {
8109 .release = io_uring_release,
Jens Axboefcb323c2019-10-24 12:39:47 -06008110 .flush = io_uring_flush,
Jens Axboe2b188cc2019-01-07 10:46:33 -07008111 .mmap = io_uring_mmap,
Roman Penyaev6c5c2402019-11-28 12:53:22 +01008112#ifndef CONFIG_MMU
8113 .get_unmapped_area = io_uring_nommu_get_unmapped_area,
8114 .mmap_capabilities = io_uring_nommu_mmap_capabilities,
8115#endif
Jens Axboe2b188cc2019-01-07 10:46:33 -07008116 .poll = io_uring_poll,
8117 .fasync = io_uring_fasync,
Tobias Klauserbebdb652020-02-26 18:38:32 +01008118#ifdef CONFIG_PROC_FS
Jens Axboe87ce9552020-01-30 08:25:34 -07008119 .show_fdinfo = io_uring_show_fdinfo,
Tobias Klauserbebdb652020-02-26 18:38:32 +01008120#endif
Jens Axboe2b188cc2019-01-07 10:46:33 -07008121};
8122
8123static int io_allocate_scq_urings(struct io_ring_ctx *ctx,
8124 struct io_uring_params *p)
8125{
Hristo Venev75b28af2019-08-26 17:23:46 +00008126 struct io_rings *rings;
8127 size_t size, sq_array_offset;
Jens Axboe2b188cc2019-01-07 10:46:33 -07008128
Hristo Venev75b28af2019-08-26 17:23:46 +00008129 size = rings_size(p->sq_entries, p->cq_entries, &sq_array_offset);
8130 if (size == SIZE_MAX)
8131 return -EOVERFLOW;
8132
8133 rings = io_mem_alloc(size);
8134 if (!rings)
Jens Axboe2b188cc2019-01-07 10:46:33 -07008135 return -ENOMEM;
8136
Hristo Venev75b28af2019-08-26 17:23:46 +00008137 ctx->rings = rings;
8138 ctx->sq_array = (u32 *)((char *)rings + sq_array_offset);
8139 rings->sq_ring_mask = p->sq_entries - 1;
8140 rings->cq_ring_mask = p->cq_entries - 1;
8141 rings->sq_ring_entries = p->sq_entries;
8142 rings->cq_ring_entries = p->cq_entries;
8143 ctx->sq_mask = rings->sq_ring_mask;
8144 ctx->cq_mask = rings->cq_ring_mask;
8145 ctx->sq_entries = rings->sq_ring_entries;
8146 ctx->cq_entries = rings->cq_ring_entries;
Jens Axboe2b188cc2019-01-07 10:46:33 -07008147
8148 size = array_size(sizeof(struct io_uring_sqe), p->sq_entries);
Jens Axboeeb065d32019-11-20 09:26:29 -07008149 if (size == SIZE_MAX) {
8150 io_mem_free(ctx->rings);
8151 ctx->rings = NULL;
Jens Axboe2b188cc2019-01-07 10:46:33 -07008152 return -EOVERFLOW;
Jens Axboeeb065d32019-11-20 09:26:29 -07008153 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07008154
8155 ctx->sq_sqes = io_mem_alloc(size);
Jens Axboeeb065d32019-11-20 09:26:29 -07008156 if (!ctx->sq_sqes) {
8157 io_mem_free(ctx->rings);
8158 ctx->rings = NULL;
Jens Axboe2b188cc2019-01-07 10:46:33 -07008159 return -ENOMEM;
Jens Axboeeb065d32019-11-20 09:26:29 -07008160 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07008161
Jens Axboe2b188cc2019-01-07 10:46:33 -07008162 return 0;
8163}
8164
8165/*
8166 * Allocate an anonymous fd, this is what constitutes the application
8167 * visible backing of an io_uring instance. The application mmaps this
8168 * fd to gain access to the SQ/CQ ring details. If UNIX sockets are enabled,
8169 * we have to tie this fd to a socket for file garbage collection purposes.
8170 */
8171static int io_uring_get_fd(struct io_ring_ctx *ctx)
8172{
8173 struct file *file;
8174 int ret;
8175
8176#if defined(CONFIG_UNIX)
8177 ret = sock_create_kern(&init_net, PF_UNIX, SOCK_RAW, IPPROTO_IP,
8178 &ctx->ring_sock);
8179 if (ret)
8180 return ret;
8181#endif
8182
8183 ret = get_unused_fd_flags(O_RDWR | O_CLOEXEC);
8184 if (ret < 0)
8185 goto err;
8186
8187 file = anon_inode_getfile("[io_uring]", &io_uring_fops, ctx,
8188 O_RDWR | O_CLOEXEC);
8189 if (IS_ERR(file)) {
8190 put_unused_fd(ret);
8191 ret = PTR_ERR(file);
8192 goto err;
8193 }
8194
8195#if defined(CONFIG_UNIX)
8196 ctx->ring_sock->file = file;
8197#endif
8198 fd_install(ret, file);
8199 return ret;
8200err:
8201#if defined(CONFIG_UNIX)
8202 sock_release(ctx->ring_sock);
8203 ctx->ring_sock = NULL;
8204#endif
8205 return ret;
8206}
8207
Xiaoguang Wang7f136572020-05-05 16:28:53 +08008208static int io_uring_create(unsigned entries, struct io_uring_params *p,
8209 struct io_uring_params __user *params)
Jens Axboe2b188cc2019-01-07 10:46:33 -07008210{
8211 struct user_struct *user = NULL;
8212 struct io_ring_ctx *ctx;
Bijan Mottahedehaad5d8d2020-06-16 16:36:08 -07008213 bool limit_mem;
Jens Axboe2b188cc2019-01-07 10:46:33 -07008214 int ret;
8215
Jens Axboe8110c1a2019-12-28 15:39:54 -07008216 if (!entries)
Jens Axboe2b188cc2019-01-07 10:46:33 -07008217 return -EINVAL;
Jens Axboe8110c1a2019-12-28 15:39:54 -07008218 if (entries > IORING_MAX_ENTRIES) {
8219 if (!(p->flags & IORING_SETUP_CLAMP))
8220 return -EINVAL;
8221 entries = IORING_MAX_ENTRIES;
8222 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07008223
8224 /*
8225 * Use twice as many entries for the CQ ring. It's possible for the
8226 * application to drive a higher depth than the size of the SQ ring,
8227 * since the sqes are only used at submission time. This allows for
Jens Axboe33a107f2019-10-04 12:10:03 -06008228 * some flexibility in overcommitting a bit. If the application has
8229 * set IORING_SETUP_CQSIZE, it will have passed in the desired number
8230 * of CQ ring entries manually.
Jens Axboe2b188cc2019-01-07 10:46:33 -07008231 */
8232 p->sq_entries = roundup_pow_of_two(entries);
Jens Axboe33a107f2019-10-04 12:10:03 -06008233 if (p->flags & IORING_SETUP_CQSIZE) {
8234 /*
8235 * If IORING_SETUP_CQSIZE is set, we do the same roundup
8236 * to a power-of-two, if it isn't already. We do NOT impose
8237 * any cq vs sq ring sizing.
8238 */
Jens Axboe8110c1a2019-12-28 15:39:54 -07008239 if (p->cq_entries < p->sq_entries)
Jens Axboe33a107f2019-10-04 12:10:03 -06008240 return -EINVAL;
Jens Axboe8110c1a2019-12-28 15:39:54 -07008241 if (p->cq_entries > IORING_MAX_CQ_ENTRIES) {
8242 if (!(p->flags & IORING_SETUP_CLAMP))
8243 return -EINVAL;
8244 p->cq_entries = IORING_MAX_CQ_ENTRIES;
8245 }
Jens Axboe33a107f2019-10-04 12:10:03 -06008246 p->cq_entries = roundup_pow_of_two(p->cq_entries);
8247 } else {
8248 p->cq_entries = 2 * p->sq_entries;
8249 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07008250
8251 user = get_uid(current_user());
Bijan Mottahedehaad5d8d2020-06-16 16:36:08 -07008252 limit_mem = !capable(CAP_IPC_LOCK);
Jens Axboe2b188cc2019-01-07 10:46:33 -07008253
Bijan Mottahedehaad5d8d2020-06-16 16:36:08 -07008254 if (limit_mem) {
Bijan Mottahedeha087e2b2020-06-16 16:36:07 -07008255 ret = __io_account_mem(user,
Jens Axboe2b188cc2019-01-07 10:46:33 -07008256 ring_pages(p->sq_entries, p->cq_entries));
8257 if (ret) {
8258 free_uid(user);
8259 return ret;
8260 }
8261 }
8262
8263 ctx = io_ring_ctx_alloc(p);
8264 if (!ctx) {
Bijan Mottahedehaad5d8d2020-06-16 16:36:08 -07008265 if (limit_mem)
Bijan Mottahedeha087e2b2020-06-16 16:36:07 -07008266 __io_unaccount_mem(user, ring_pages(p->sq_entries,
Jens Axboe2b188cc2019-01-07 10:46:33 -07008267 p->cq_entries));
8268 free_uid(user);
8269 return -ENOMEM;
8270 }
8271 ctx->compat = in_compat_syscall();
Jens Axboe2b188cc2019-01-07 10:46:33 -07008272 ctx->user = user;
Jens Axboe0b8c0ec2019-12-02 08:50:00 -07008273 ctx->creds = get_current_cred();
Jens Axboe2b188cc2019-01-07 10:46:33 -07008274
8275 ret = io_allocate_scq_urings(ctx, p);
8276 if (ret)
8277 goto err;
8278
Jens Axboe6c271ce2019-01-10 11:22:30 -07008279 ret = io_sq_offload_start(ctx, p);
Jens Axboe2b188cc2019-01-07 10:46:33 -07008280 if (ret)
8281 goto err;
8282
Jens Axboe2b188cc2019-01-07 10:46:33 -07008283 memset(&p->sq_off, 0, sizeof(p->sq_off));
Hristo Venev75b28af2019-08-26 17:23:46 +00008284 p->sq_off.head = offsetof(struct io_rings, sq.head);
8285 p->sq_off.tail = offsetof(struct io_rings, sq.tail);
8286 p->sq_off.ring_mask = offsetof(struct io_rings, sq_ring_mask);
8287 p->sq_off.ring_entries = offsetof(struct io_rings, sq_ring_entries);
8288 p->sq_off.flags = offsetof(struct io_rings, sq_flags);
8289 p->sq_off.dropped = offsetof(struct io_rings, sq_dropped);
8290 p->sq_off.array = (char *)ctx->sq_array - (char *)ctx->rings;
Jens Axboe2b188cc2019-01-07 10:46:33 -07008291
8292 memset(&p->cq_off, 0, sizeof(p->cq_off));
Hristo Venev75b28af2019-08-26 17:23:46 +00008293 p->cq_off.head = offsetof(struct io_rings, cq.head);
8294 p->cq_off.tail = offsetof(struct io_rings, cq.tail);
8295 p->cq_off.ring_mask = offsetof(struct io_rings, cq_ring_mask);
8296 p->cq_off.ring_entries = offsetof(struct io_rings, cq_ring_entries);
8297 p->cq_off.overflow = offsetof(struct io_rings, cq_overflow);
8298 p->cq_off.cqes = offsetof(struct io_rings, cqes);
Stefano Garzarella0d9b5b32020-05-15 18:38:04 +02008299 p->cq_off.flags = offsetof(struct io_rings, cq_flags);
Jens Axboeac90f242019-09-06 10:26:21 -06008300
Xiaoguang Wang7f136572020-05-05 16:28:53 +08008301 p->features = IORING_FEAT_SINGLE_MMAP | IORING_FEAT_NODROP |
8302 IORING_FEAT_SUBMIT_STABLE | IORING_FEAT_RW_CUR_POS |
Jiufei Xue5769a352020-06-17 17:53:55 +08008303 IORING_FEAT_CUR_PERSONALITY | IORING_FEAT_FAST_POLL |
8304 IORING_FEAT_POLL_32BITS;
Xiaoguang Wang7f136572020-05-05 16:28:53 +08008305
8306 if (copy_to_user(params, p, sizeof(*p))) {
8307 ret = -EFAULT;
8308 goto err;
8309 }
Jens Axboe044c1ab2019-10-28 09:15:33 -06008310 /*
8311 * Install ring fd as the very last thing, so we don't risk someone
8312 * having closed it before we finish setup
8313 */
8314 ret = io_uring_get_fd(ctx);
8315 if (ret < 0)
8316 goto err;
8317
Dmitrii Dolgovc826bd72019-10-15 19:02:01 +02008318 trace_io_uring_create(ret, ctx, p->sq_entries, p->cq_entries, p->flags);
Bijan Mottahedeh2e0464d2020-06-16 16:36:10 -07008319 io_account_mem(ctx, ring_pages(p->sq_entries, p->cq_entries),
8320 ACCT_LOCKED);
Bijan Mottahedeh30975822020-06-16 16:36:09 -07008321 ctx->limit_mem = limit_mem;
Jens Axboe2b188cc2019-01-07 10:46:33 -07008322 return ret;
8323err:
8324 io_ring_ctx_wait_and_kill(ctx);
8325 return ret;
8326}
8327
8328/*
8329 * Sets up an aio uring context, and returns the fd. Applications asks for a
8330 * ring size, we return the actual sq/cq ring sizes (among other things) in the
8331 * params structure passed in.
8332 */
8333static long io_uring_setup(u32 entries, struct io_uring_params __user *params)
8334{
8335 struct io_uring_params p;
Jens Axboe2b188cc2019-01-07 10:46:33 -07008336 int i;
8337
8338 if (copy_from_user(&p, params, sizeof(p)))
8339 return -EFAULT;
8340 for (i = 0; i < ARRAY_SIZE(p.resv); i++) {
8341 if (p.resv[i])
8342 return -EINVAL;
8343 }
8344
Jens Axboe6c271ce2019-01-10 11:22:30 -07008345 if (p.flags & ~(IORING_SETUP_IOPOLL | IORING_SETUP_SQPOLL |
Jens Axboe8110c1a2019-12-28 15:39:54 -07008346 IORING_SETUP_SQ_AFF | IORING_SETUP_CQSIZE |
Pavel Begunkov24369c22020-01-28 03:15:48 +03008347 IORING_SETUP_CLAMP | IORING_SETUP_ATTACH_WQ))
Jens Axboe2b188cc2019-01-07 10:46:33 -07008348 return -EINVAL;
8349
Xiaoguang Wang7f136572020-05-05 16:28:53 +08008350 return io_uring_create(entries, &p, params);
Jens Axboe2b188cc2019-01-07 10:46:33 -07008351}
8352
8353SYSCALL_DEFINE2(io_uring_setup, u32, entries,
8354 struct io_uring_params __user *, params)
8355{
8356 return io_uring_setup(entries, params);
8357}
8358
Jens Axboe66f4af92020-01-16 15:36:52 -07008359static int io_probe(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args)
8360{
8361 struct io_uring_probe *p;
8362 size_t size;
8363 int i, ret;
8364
8365 size = struct_size(p, ops, nr_args);
8366 if (size == SIZE_MAX)
8367 return -EOVERFLOW;
8368 p = kzalloc(size, GFP_KERNEL);
8369 if (!p)
8370 return -ENOMEM;
8371
8372 ret = -EFAULT;
8373 if (copy_from_user(p, arg, size))
8374 goto out;
8375 ret = -EINVAL;
8376 if (memchr_inv(p, 0, size))
8377 goto out;
8378
8379 p->last_op = IORING_OP_LAST - 1;
8380 if (nr_args > IORING_OP_LAST)
8381 nr_args = IORING_OP_LAST;
8382
8383 for (i = 0; i < nr_args; i++) {
8384 p->ops[i].op = i;
8385 if (!io_op_defs[i].not_supported)
8386 p->ops[i].flags = IO_URING_OP_SUPPORTED;
8387 }
8388 p->ops_len = i;
8389
8390 ret = 0;
8391 if (copy_to_user(arg, p, size))
8392 ret = -EFAULT;
8393out:
8394 kfree(p);
8395 return ret;
8396}
8397
Jens Axboe071698e2020-01-28 10:04:42 -07008398static int io_register_personality(struct io_ring_ctx *ctx)
8399{
8400 const struct cred *creds = get_current_cred();
8401 int id;
8402
8403 id = idr_alloc_cyclic(&ctx->personality_idr, (void *) creds, 1,
8404 USHRT_MAX, GFP_KERNEL);
8405 if (id < 0)
8406 put_cred(creds);
8407 return id;
8408}
8409
8410static int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id)
8411{
8412 const struct cred *old_creds;
8413
8414 old_creds = idr_remove(&ctx->personality_idr, id);
8415 if (old_creds) {
8416 put_cred(old_creds);
8417 return 0;
8418 }
8419
8420 return -EINVAL;
8421}
8422
8423static bool io_register_op_must_quiesce(int op)
8424{
8425 switch (op) {
8426 case IORING_UNREGISTER_FILES:
8427 case IORING_REGISTER_FILES_UPDATE:
8428 case IORING_REGISTER_PROBE:
8429 case IORING_REGISTER_PERSONALITY:
8430 case IORING_UNREGISTER_PERSONALITY:
8431 return false;
8432 default:
8433 return true;
8434 }
8435}
8436
Jens Axboeedafcce2019-01-09 09:16:05 -07008437static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,
8438 void __user *arg, unsigned nr_args)
Jens Axboeb19062a2019-04-15 10:49:38 -06008439 __releases(ctx->uring_lock)
8440 __acquires(ctx->uring_lock)
Jens Axboeedafcce2019-01-09 09:16:05 -07008441{
8442 int ret;
8443
Jens Axboe35fa71a2019-04-22 10:23:23 -06008444 /*
8445 * We're inside the ring mutex, if the ref is already dying, then
8446 * someone else killed the ctx or is already going through
8447 * io_uring_register().
8448 */
8449 if (percpu_ref_is_dying(&ctx->refs))
8450 return -ENXIO;
8451
Jens Axboe071698e2020-01-28 10:04:42 -07008452 if (io_register_op_must_quiesce(opcode)) {
Jens Axboe05f3fb32019-12-09 11:22:50 -07008453 percpu_ref_kill(&ctx->refs);
Jens Axboeb19062a2019-04-15 10:49:38 -06008454
Jens Axboe05f3fb32019-12-09 11:22:50 -07008455 /*
8456 * Drop uring mutex before waiting for references to exit. If
8457 * another thread is currently inside io_uring_enter() it might
8458 * need to grab the uring_lock to make progress. If we hold it
8459 * here across the drain wait, then we can deadlock. It's safe
8460 * to drop the mutex here, since no new references will come in
8461 * after we've killed the percpu ref.
8462 */
8463 mutex_unlock(&ctx->uring_lock);
Jens Axboe0f158b42020-05-14 17:18:39 -06008464 ret = wait_for_completion_interruptible(&ctx->ref_comp);
Jens Axboe05f3fb32019-12-09 11:22:50 -07008465 mutex_lock(&ctx->uring_lock);
Jens Axboec1503682020-01-08 08:26:07 -07008466 if (ret) {
8467 percpu_ref_resurrect(&ctx->refs);
8468 ret = -EINTR;
8469 goto out;
8470 }
Jens Axboe05f3fb32019-12-09 11:22:50 -07008471 }
Jens Axboeedafcce2019-01-09 09:16:05 -07008472
8473 switch (opcode) {
8474 case IORING_REGISTER_BUFFERS:
8475 ret = io_sqe_buffer_register(ctx, arg, nr_args);
8476 break;
8477 case IORING_UNREGISTER_BUFFERS:
8478 ret = -EINVAL;
8479 if (arg || nr_args)
8480 break;
8481 ret = io_sqe_buffer_unregister(ctx);
8482 break;
Jens Axboe6b063142019-01-10 22:13:58 -07008483 case IORING_REGISTER_FILES:
8484 ret = io_sqe_files_register(ctx, arg, nr_args);
8485 break;
8486 case IORING_UNREGISTER_FILES:
8487 ret = -EINVAL;
8488 if (arg || nr_args)
8489 break;
8490 ret = io_sqe_files_unregister(ctx);
8491 break;
Jens Axboec3a31e62019-10-03 13:59:56 -06008492 case IORING_REGISTER_FILES_UPDATE:
8493 ret = io_sqe_files_update(ctx, arg, nr_args);
8494 break;
Jens Axboe9b402842019-04-11 11:45:41 -06008495 case IORING_REGISTER_EVENTFD:
Jens Axboef2842ab2020-01-08 11:04:00 -07008496 case IORING_REGISTER_EVENTFD_ASYNC:
Jens Axboe9b402842019-04-11 11:45:41 -06008497 ret = -EINVAL;
8498 if (nr_args != 1)
8499 break;
8500 ret = io_eventfd_register(ctx, arg);
Jens Axboef2842ab2020-01-08 11:04:00 -07008501 if (ret)
8502 break;
8503 if (opcode == IORING_REGISTER_EVENTFD_ASYNC)
8504 ctx->eventfd_async = 1;
8505 else
8506 ctx->eventfd_async = 0;
Jens Axboe9b402842019-04-11 11:45:41 -06008507 break;
8508 case IORING_UNREGISTER_EVENTFD:
8509 ret = -EINVAL;
8510 if (arg || nr_args)
8511 break;
8512 ret = io_eventfd_unregister(ctx);
8513 break;
Jens Axboe66f4af92020-01-16 15:36:52 -07008514 case IORING_REGISTER_PROBE:
8515 ret = -EINVAL;
8516 if (!arg || nr_args > 256)
8517 break;
8518 ret = io_probe(ctx, arg, nr_args);
8519 break;
Jens Axboe071698e2020-01-28 10:04:42 -07008520 case IORING_REGISTER_PERSONALITY:
8521 ret = -EINVAL;
8522 if (arg || nr_args)
8523 break;
8524 ret = io_register_personality(ctx);
8525 break;
8526 case IORING_UNREGISTER_PERSONALITY:
8527 ret = -EINVAL;
8528 if (arg)
8529 break;
8530 ret = io_unregister_personality(ctx, nr_args);
8531 break;
Jens Axboeedafcce2019-01-09 09:16:05 -07008532 default:
8533 ret = -EINVAL;
8534 break;
8535 }
8536
Jens Axboe071698e2020-01-28 10:04:42 -07008537 if (io_register_op_must_quiesce(opcode)) {
Jens Axboe05f3fb32019-12-09 11:22:50 -07008538 /* bring the ctx back to life */
Jens Axboe05f3fb32019-12-09 11:22:50 -07008539 percpu_ref_reinit(&ctx->refs);
Jens Axboec1503682020-01-08 08:26:07 -07008540out:
Jens Axboe0f158b42020-05-14 17:18:39 -06008541 reinit_completion(&ctx->ref_comp);
Jens Axboe05f3fb32019-12-09 11:22:50 -07008542 }
Jens Axboeedafcce2019-01-09 09:16:05 -07008543 return ret;
8544}
8545
8546SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode,
8547 void __user *, arg, unsigned int, nr_args)
8548{
8549 struct io_ring_ctx *ctx;
8550 long ret = -EBADF;
8551 struct fd f;
8552
8553 f = fdget(fd);
8554 if (!f.file)
8555 return -EBADF;
8556
8557 ret = -EOPNOTSUPP;
8558 if (f.file->f_op != &io_uring_fops)
8559 goto out_fput;
8560
8561 ctx = f.file->private_data;
8562
8563 mutex_lock(&ctx->uring_lock);
8564 ret = __io_uring_register(ctx, opcode, arg, nr_args);
8565 mutex_unlock(&ctx->uring_lock);
Dmitrii Dolgovc826bd72019-10-15 19:02:01 +02008566 trace_io_uring_register(ctx, opcode, ctx->nr_user_files, ctx->nr_user_bufs,
8567 ctx->cq_ev_fd != NULL, ret);
Jens Axboeedafcce2019-01-09 09:16:05 -07008568out_fput:
8569 fdput(f);
8570 return ret;
8571}
8572
Jens Axboe2b188cc2019-01-07 10:46:33 -07008573static int __init io_uring_init(void)
8574{
Stefan Metzmacherd7f62e82020-01-29 14:39:41 +01008575#define __BUILD_BUG_VERIFY_ELEMENT(stype, eoffset, etype, ename) do { \
8576 BUILD_BUG_ON(offsetof(stype, ename) != eoffset); \
8577 BUILD_BUG_ON(sizeof(etype) != sizeof_field(stype, ename)); \
8578} while (0)
8579
8580#define BUILD_BUG_SQE_ELEM(eoffset, etype, ename) \
8581 __BUILD_BUG_VERIFY_ELEMENT(struct io_uring_sqe, eoffset, etype, ename)
8582 BUILD_BUG_ON(sizeof(struct io_uring_sqe) != 64);
8583 BUILD_BUG_SQE_ELEM(0, __u8, opcode);
8584 BUILD_BUG_SQE_ELEM(1, __u8, flags);
8585 BUILD_BUG_SQE_ELEM(2, __u16, ioprio);
8586 BUILD_BUG_SQE_ELEM(4, __s32, fd);
8587 BUILD_BUG_SQE_ELEM(8, __u64, off);
8588 BUILD_BUG_SQE_ELEM(8, __u64, addr2);
8589 BUILD_BUG_SQE_ELEM(16, __u64, addr);
Pavel Begunkov7d67af22020-02-24 11:32:45 +03008590 BUILD_BUG_SQE_ELEM(16, __u64, splice_off_in);
Stefan Metzmacherd7f62e82020-01-29 14:39:41 +01008591 BUILD_BUG_SQE_ELEM(24, __u32, len);
8592 BUILD_BUG_SQE_ELEM(28, __kernel_rwf_t, rw_flags);
8593 BUILD_BUG_SQE_ELEM(28, /* compat */ int, rw_flags);
8594 BUILD_BUG_SQE_ELEM(28, /* compat */ __u32, rw_flags);
8595 BUILD_BUG_SQE_ELEM(28, __u32, fsync_flags);
Jiufei Xue5769a352020-06-17 17:53:55 +08008596 BUILD_BUG_SQE_ELEM(28, /* compat */ __u16, poll_events);
8597 BUILD_BUG_SQE_ELEM(28, __u32, poll32_events);
Stefan Metzmacherd7f62e82020-01-29 14:39:41 +01008598 BUILD_BUG_SQE_ELEM(28, __u32, sync_range_flags);
8599 BUILD_BUG_SQE_ELEM(28, __u32, msg_flags);
8600 BUILD_BUG_SQE_ELEM(28, __u32, timeout_flags);
8601 BUILD_BUG_SQE_ELEM(28, __u32, accept_flags);
8602 BUILD_BUG_SQE_ELEM(28, __u32, cancel_flags);
8603 BUILD_BUG_SQE_ELEM(28, __u32, open_flags);
8604 BUILD_BUG_SQE_ELEM(28, __u32, statx_flags);
8605 BUILD_BUG_SQE_ELEM(28, __u32, fadvise_advice);
Pavel Begunkov7d67af22020-02-24 11:32:45 +03008606 BUILD_BUG_SQE_ELEM(28, __u32, splice_flags);
Stefan Metzmacherd7f62e82020-01-29 14:39:41 +01008607 BUILD_BUG_SQE_ELEM(32, __u64, user_data);
8608 BUILD_BUG_SQE_ELEM(40, __u16, buf_index);
8609 BUILD_BUG_SQE_ELEM(42, __u16, personality);
Pavel Begunkov7d67af22020-02-24 11:32:45 +03008610 BUILD_BUG_SQE_ELEM(44, __s32, splice_fd_in);
Stefan Metzmacherd7f62e82020-01-29 14:39:41 +01008611
Jens Axboed3656342019-12-18 09:50:26 -07008612 BUILD_BUG_ON(ARRAY_SIZE(io_op_defs) != IORING_OP_LAST);
Jens Axboe84557872020-03-03 15:28:17 -07008613 BUILD_BUG_ON(__REQ_F_LAST_BIT >= 8 * sizeof(int));
Jens Axboe2b188cc2019-01-07 10:46:33 -07008614 req_cachep = KMEM_CACHE(io_kiocb, SLAB_HWCACHE_ALIGN | SLAB_PANIC);
8615 return 0;
8616};
8617__initcall(io_uring_init);