blob: d78a05ecbf683c0482bb7de7c34406c2e21bdc00 [file] [log] [blame]
Jens Axboe2b188cc2019-01-07 10:46:33 -07001// SPDX-License-Identifier: GPL-2.0
2/*
3 * Shared application/kernel submission and completion ring pairs, for
4 * supporting fast/efficient IO.
5 *
6 * A note on the read/write ordering memory barriers that are matched between
Stefan Bühler1e84b972019-04-24 23:54:16 +02007 * the application and kernel side.
8 *
9 * After the application reads the CQ ring tail, it must use an
10 * appropriate smp_rmb() to pair with the smp_wmb() the kernel uses
11 * before writing the tail (using smp_load_acquire to read the tail will
12 * do). It also needs a smp_mb() before updating CQ head (ordering the
13 * entry load(s) with the head store), pairing with an implicit barrier
Pavel Begunkovd068b502021-05-16 22:58:11 +010014 * through a control-dependency in io_get_cqe (smp_store_release to
Stefan Bühler1e84b972019-04-24 23:54:16 +020015 * store head will do). Failure to do so could lead to reading invalid
16 * CQ entries.
17 *
18 * Likewise, the application must use an appropriate smp_wmb() before
19 * writing the SQ tail (ordering SQ entry stores with the tail store),
20 * which pairs with smp_load_acquire in io_get_sqring (smp_store_release
21 * to store the tail will do). And it needs a barrier ordering the SQ
22 * head load before writing new SQ entries (smp_load_acquire to read
23 * head will do).
24 *
25 * When using the SQ poll thread (IORING_SETUP_SQPOLL), the application
26 * needs to check the SQ flags for IORING_SQ_NEED_WAKEUP *after*
27 * updating the SQ tail; a full memory barrier smp_mb() is needed
28 * between.
Jens Axboe2b188cc2019-01-07 10:46:33 -070029 *
30 * Also see the examples in the liburing library:
31 *
32 * git://git.kernel.dk/liburing
33 *
34 * io_uring also uses READ/WRITE_ONCE() for _any_ store or load that happens
35 * from data shared between the kernel and application. This is done both
36 * for ordering purposes, but also to ensure that once a value is loaded from
37 * data that the application could potentially modify, it remains stable.
38 *
39 * Copyright (C) 2018-2019 Jens Axboe
Christoph Hellwigc992fe22019-01-11 09:43:02 -070040 * Copyright (c) 2018-2019 Christoph Hellwig
Jens Axboe2b188cc2019-01-07 10:46:33 -070041 */
42#include <linux/kernel.h>
43#include <linux/init.h>
44#include <linux/errno.h>
45#include <linux/syscalls.h>
46#include <linux/compat.h>
Jens Axboe52de1fe2020-02-27 10:15:42 -070047#include <net/compat.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070048#include <linux/refcount.h>
49#include <linux/uio.h>
Pavel Begunkov6b47ee62020-01-18 20:22:41 +030050#include <linux/bits.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070051
52#include <linux/sched/signal.h>
53#include <linux/fs.h>
54#include <linux/file.h>
55#include <linux/fdtable.h>
56#include <linux/mm.h>
57#include <linux/mman.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070058#include <linux/percpu.h>
59#include <linux/slab.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070060#include <linux/blkdev.h>
Jens Axboeedafcce2019-01-09 09:16:05 -070061#include <linux/bvec.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070062#include <linux/net.h>
63#include <net/sock.h>
64#include <net/af_unix.h>
Jens Axboe6b063142019-01-10 22:13:58 -070065#include <net/scm.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070066#include <linux/anon_inodes.h>
67#include <linux/sched/mm.h>
68#include <linux/uaccess.h>
69#include <linux/nospec.h>
Jens Axboeedafcce2019-01-09 09:16:05 -070070#include <linux/sizes.h>
71#include <linux/hugetlb.h>
Jens Axboeaa4c3962019-11-29 10:14:00 -070072#include <linux/highmem.h>
Jens Axboe15b71ab2019-12-11 11:20:36 -070073#include <linux/namei.h>
74#include <linux/fsnotify.h>
Jens Axboe4840e412019-12-25 22:03:45 -070075#include <linux/fadvise.h>
Jens Axboe3e4827b2020-01-08 15:18:09 -070076#include <linux/eventpoll.h>
Pavel Begunkov7d67af22020-02-24 11:32:45 +030077#include <linux/splice.h>
Jens Axboeb41e9852020-02-17 09:52:41 -070078#include <linux/task_work.h>
Jens Axboebcf5a062020-05-22 09:24:42 -060079#include <linux/pagemap.h>
Jens Axboe0f212202020-09-13 13:09:39 -060080#include <linux/io_uring.h>
Nadav Amitef98eb02021-08-07 17:13:41 -070081#include <linux/tracehook.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070082
Dmitrii Dolgovc826bd72019-10-15 19:02:01 +020083#define CREATE_TRACE_POINTS
84#include <trace/events/io_uring.h>
85
Jens Axboe2b188cc2019-01-07 10:46:33 -070086#include <uapi/linux/io_uring.h>
87
88#include "internal.h"
Jens Axboe561fb042019-10-24 07:25:42 -060089#include "io-wq.h"
Jens Axboe2b188cc2019-01-07 10:46:33 -070090
Daniel Xu5277dea2019-09-14 14:23:45 -070091#define IORING_MAX_ENTRIES 32768
Jens Axboe33a107f2019-10-04 12:10:03 -060092#define IORING_MAX_CQ_ENTRIES (2 * IORING_MAX_ENTRIES)
Olivier Langlois4ce8ad92021-06-23 11:50:18 -070093#define IORING_SQPOLL_CAP_ENTRIES_VALUE 8
Jens Axboe65e19f52019-10-26 07:20:21 -060094
Pavel Begunkov042b0d82021-08-09 13:04:01 +010095/* 512 entries per page on 64-bit archs, 64 pages max */
96#define IORING_MAX_FIXED_FILES (1U << 15)
Stefano Garzarella21b55db2020-08-27 16:58:30 +020097#define IORING_MAX_RESTRICTIONS (IORING_RESTRICTION_LAST + \
98 IORING_REGISTER_LAST + IORING_OP_LAST)
Jens Axboe2b188cc2019-01-07 10:46:33 -070099
Pavel Begunkov2d091d62021-06-14 02:36:21 +0100100#define IO_RSRC_TAG_TABLE_SHIFT 9
101#define IO_RSRC_TAG_TABLE_MAX (1U << IO_RSRC_TAG_TABLE_SHIFT)
102#define IO_RSRC_TAG_TABLE_MASK (IO_RSRC_TAG_TABLE_MAX - 1)
103
Pavel Begunkov489809e2021-05-14 12:06:44 +0100104#define IORING_MAX_REG_BUFFERS (1U << 14)
105
Pavel Begunkovb16fed662021-02-18 18:29:40 +0000106#define SQE_VALID_FLAGS (IOSQE_FIXED_FILE|IOSQE_IO_DRAIN|IOSQE_IO_LINK| \
107 IOSQE_IO_HARDLINK | IOSQE_ASYNC | \
108 IOSQE_BUFFER_SELECT)
Pavel Begunkovc8543572021-06-17 18:14:04 +0100109#define IO_REQ_CLEAN_FLAGS (REQ_F_BUFFER_SELECTED | REQ_F_NEED_CLEANUP | \
110 REQ_F_POLLED | REQ_F_INFLIGHT | REQ_F_CREDS)
Pavel Begunkovb16fed662021-02-18 18:29:40 +0000111
Pavel Begunkov09899b12021-06-14 02:36:22 +0100112#define IO_TCTX_REFS_CACHE_NR (1U << 10)
113
Jens Axboe2b188cc2019-01-07 10:46:33 -0700114struct io_uring {
115 u32 head ____cacheline_aligned_in_smp;
116 u32 tail ____cacheline_aligned_in_smp;
117};
118
Stefan Bühler1e84b972019-04-24 23:54:16 +0200119/*
Hristo Venev75b28af2019-08-26 17:23:46 +0000120 * This data is shared with the application through the mmap at offsets
121 * IORING_OFF_SQ_RING and IORING_OFF_CQ_RING.
Stefan Bühler1e84b972019-04-24 23:54:16 +0200122 *
123 * The offsets to the member fields are published through struct
124 * io_sqring_offsets when calling io_uring_setup.
125 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000126struct io_rings {
Stefan Bühler1e84b972019-04-24 23:54:16 +0200127 /*
128 * Head and tail offsets into the ring; the offsets need to be
129 * masked to get valid indices.
130 *
Hristo Venev75b28af2019-08-26 17:23:46 +0000131 * The kernel controls head of the sq ring and the tail of the cq ring,
132 * and the application controls tail of the sq ring and the head of the
133 * cq ring.
Stefan Bühler1e84b972019-04-24 23:54:16 +0200134 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000135 struct io_uring sq, cq;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200136 /*
Hristo Venev75b28af2019-08-26 17:23:46 +0000137 * Bitmasks to apply to head and tail offsets (constant, equals
Stefan Bühler1e84b972019-04-24 23:54:16 +0200138 * ring_entries - 1)
139 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000140 u32 sq_ring_mask, cq_ring_mask;
141 /* Ring sizes (constant, power of 2) */
142 u32 sq_ring_entries, cq_ring_entries;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200143 /*
144 * Number of invalid entries dropped by the kernel due to
145 * invalid index stored in array
146 *
147 * Written by the kernel, shouldn't be modified by the
148 * application (i.e. get number of "new events" by comparing to
149 * cached value).
150 *
151 * After a new SQ head value was read by the application this
152 * counter includes all submissions that were dropped reaching
153 * the new SQ head (and possibly more).
154 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000155 u32 sq_dropped;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200156 /*
Stefano Garzarella0d9b5b32020-05-15 18:38:04 +0200157 * Runtime SQ flags
Stefan Bühler1e84b972019-04-24 23:54:16 +0200158 *
159 * Written by the kernel, shouldn't be modified by the
160 * application.
161 *
162 * The application needs a full memory barrier before checking
163 * for IORING_SQ_NEED_WAKEUP after updating the sq tail.
164 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000165 u32 sq_flags;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200166 /*
Stefano Garzarella0d9b5b32020-05-15 18:38:04 +0200167 * Runtime CQ flags
168 *
169 * Written by the application, shouldn't be modified by the
170 * kernel.
171 */
Pavel Begunkovfe7e3252021-06-24 15:09:57 +0100172 u32 cq_flags;
Stefano Garzarella0d9b5b32020-05-15 18:38:04 +0200173 /*
Stefan Bühler1e84b972019-04-24 23:54:16 +0200174 * Number of completion events lost because the queue was full;
175 * this should be avoided by the application by making sure
LimingWu0b4295b2019-12-05 20:18:18 +0800176 * there are not more requests pending than there is space in
Stefan Bühler1e84b972019-04-24 23:54:16 +0200177 * the completion queue.
178 *
179 * Written by the kernel, shouldn't be modified by the
180 * application (i.e. get number of "new events" by comparing to
181 * cached value).
182 *
183 * As completion events come in out of order this counter is not
184 * ordered with any other data.
185 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000186 u32 cq_overflow;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200187 /*
188 * Ring buffer of completion events.
189 *
190 * The kernel writes completion events fresh every time they are
191 * produced, so the application is allowed to modify pending
192 * entries.
193 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000194 struct io_uring_cqe cqes[] ____cacheline_aligned_in_smp;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700195};
196
Pavel Begunkov45d189c2021-02-10 00:03:07 +0000197enum io_uring_cmd_flags {
198 IO_URING_F_NONBLOCK = 1,
Pavel Begunkov889fca72021-02-10 00:03:09 +0000199 IO_URING_F_COMPLETE_DEFER = 2,
Pavel Begunkov45d189c2021-02-10 00:03:07 +0000200};
201
Jens Axboeedafcce2019-01-09 09:16:05 -0700202struct io_mapped_ubuf {
203 u64 ubuf;
Pavel Begunkov4751f532021-04-01 15:43:55 +0100204 u64 ubuf_end;
Jens Axboeedafcce2019-01-09 09:16:05 -0700205 unsigned int nr_bvecs;
Jens Axboede293932020-09-17 16:19:16 -0600206 unsigned long acct_pages;
Pavel Begunkov41edf1a2021-04-25 14:32:23 +0100207 struct bio_vec bvec[];
Jens Axboeedafcce2019-01-09 09:16:05 -0700208};
209
Bijan Mottahedeh50238532021-01-15 17:37:45 +0000210struct io_ring_ctx;
211
Pavel Begunkov6c2450a2021-02-23 12:40:22 +0000212struct io_overflow_cqe {
213 struct io_uring_cqe cqe;
214 struct list_head list;
215};
216
Pavel Begunkova04b0ac2021-04-01 15:44:04 +0100217struct io_fixed_file {
218 /* file * with additional FFS_* flags */
219 unsigned long file_ptr;
220};
221
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +0000222struct io_rsrc_put {
223 struct list_head list;
Pavel Begunkovb60c8dc2021-04-25 14:32:18 +0100224 u64 tag;
Bijan Mottahedeh50238532021-01-15 17:37:45 +0000225 union {
226 void *rsrc;
227 struct file *file;
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +0100228 struct io_mapped_ubuf *buf;
Bijan Mottahedeh50238532021-01-15 17:37:45 +0000229 };
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +0000230};
231
Pavel Begunkovaeca2412021-04-11 01:46:37 +0100232struct io_file_table {
Pavel Begunkov042b0d82021-08-09 13:04:01 +0100233 struct io_fixed_file *files;
Jens Axboe31b51512019-01-18 22:56:34 -0700234};
235
Pavel Begunkovb895c9a2021-04-01 15:43:40 +0100236struct io_rsrc_node {
Xiaoguang Wang05589552020-03-31 14:05:18 +0800237 struct percpu_ref refs;
238 struct list_head node;
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +0000239 struct list_head rsrc_list;
Pavel Begunkovb895c9a2021-04-01 15:43:40 +0100240 struct io_rsrc_data *rsrc_data;
Jens Axboe4a38aed22020-05-14 17:21:15 -0600241 struct llist_node llist;
Pavel Begunkove2978222020-11-18 14:56:26 +0000242 bool done;
Xiaoguang Wang05589552020-03-31 14:05:18 +0800243};
244
Pavel Begunkov40ae0ff2021-04-01 15:43:44 +0100245typedef void (rsrc_put_fn)(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc);
246
Pavel Begunkovb895c9a2021-04-01 15:43:40 +0100247struct io_rsrc_data {
Jens Axboe05f3fb32019-12-09 11:22:50 -0700248 struct io_ring_ctx *ctx;
249
Pavel Begunkov2d091d62021-06-14 02:36:21 +0100250 u64 **tags;
251 unsigned int nr;
Pavel Begunkov40ae0ff2021-04-01 15:43:44 +0100252 rsrc_put_fn *do_put;
Pavel Begunkov3e942492021-04-11 01:46:34 +0100253 atomic_t refs;
Jens Axboe05f3fb32019-12-09 11:22:50 -0700254 struct completion done;
Hao Xu8bad28d2021-02-19 17:19:36 +0800255 bool quiesce;
Jens Axboe05f3fb32019-12-09 11:22:50 -0700256};
257
Jens Axboe5a2e7452020-02-23 16:23:11 -0700258struct io_buffer {
259 struct list_head list;
260 __u64 addr;
Thadeu Lima de Souza Cascardod1f82802021-05-05 09:47:06 -0300261 __u32 len;
Jens Axboe5a2e7452020-02-23 16:23:11 -0700262 __u16 bid;
263};
264
Stefano Garzarella21b55db2020-08-27 16:58:30 +0200265struct io_restriction {
266 DECLARE_BITMAP(register_op, IORING_REGISTER_LAST);
267 DECLARE_BITMAP(sqe_op, IORING_OP_LAST);
268 u8 sqe_flags_allowed;
269 u8 sqe_flags_required;
Stefano Garzarella7e84e1c2020-08-27 16:58:31 +0200270 bool registered;
Stefano Garzarella21b55db2020-08-27 16:58:30 +0200271};
272
Jens Axboe37d1e2e2021-02-17 21:03:43 -0700273enum {
274 IO_SQ_THREAD_SHOULD_STOP = 0,
275 IO_SQ_THREAD_SHOULD_PARK,
276};
277
Jens Axboe534ca6d2020-09-02 13:52:19 -0600278struct io_sq_data {
279 refcount_t refs;
Pavel Begunkov9e138a42021-03-14 20:57:12 +0000280 atomic_t park_pending;
Pavel Begunkov09a6f4e2021-03-14 20:57:10 +0000281 struct mutex lock;
Jens Axboe69fb2132020-09-14 11:16:23 -0600282
283 /* ctx's that are using this sqd */
284 struct list_head ctx_list;
Jens Axboe69fb2132020-09-14 11:16:23 -0600285
Jens Axboe534ca6d2020-09-02 13:52:19 -0600286 struct task_struct *thread;
287 struct wait_queue_head wait;
Xiaoguang Wang08369242020-11-03 14:15:59 +0800288
289 unsigned sq_thread_idle;
Jens Axboe37d1e2e2021-02-17 21:03:43 -0700290 int sq_cpu;
291 pid_t task_pid;
Jens Axboe5c2469e2021-03-11 10:17:56 -0700292 pid_t task_tgid;
Jens Axboe37d1e2e2021-02-17 21:03:43 -0700293
294 unsigned long state;
Jens Axboe37d1e2e2021-02-17 21:03:43 -0700295 struct completion exited;
Jens Axboe534ca6d2020-09-02 13:52:19 -0600296};
297
Pavel Begunkov6dd0be12021-02-10 00:03:13 +0000298#define IO_COMPL_BATCH 32
Pavel Begunkov6ff119a2021-02-10 00:03:18 +0000299#define IO_REQ_CACHE_SIZE 32
Pavel Begunkovbf019da2021-02-10 00:03:17 +0000300#define IO_REQ_ALLOC_BATCH 8
Pavel Begunkov258b29a2021-02-10 00:03:10 +0000301
Pavel Begunkova1ab7b32021-02-18 18:29:42 +0000302struct io_submit_link {
303 struct io_kiocb *head;
304 struct io_kiocb *last;
305};
306
Pavel Begunkov258b29a2021-02-10 00:03:10 +0000307struct io_submit_state {
308 struct blk_plug plug;
Pavel Begunkova1ab7b32021-02-18 18:29:42 +0000309 struct io_submit_link link;
Pavel Begunkov258b29a2021-02-10 00:03:10 +0000310
311 /*
312 * io_kiocb alloc cache
313 */
Pavel Begunkovbf019da2021-02-10 00:03:17 +0000314 void *reqs[IO_REQ_CACHE_SIZE];
Pavel Begunkov258b29a2021-02-10 00:03:10 +0000315 unsigned int free_reqs;
316
317 bool plug_started;
318
319 /*
320 * Batch completion logic
321 */
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +0100322 struct io_kiocb *compl_reqs[IO_COMPL_BATCH];
323 unsigned int compl_nr;
324 /* inline/task_work completion list, under ->uring_lock */
325 struct list_head free_list;
Pavel Begunkov258b29a2021-02-10 00:03:10 +0000326
Pavel Begunkov258b29a2021-02-10 00:03:10 +0000327 unsigned int ios_left;
328};
329
Jens Axboe2b188cc2019-01-07 10:46:33 -0700330struct io_ring_ctx {
Pavel Begunkovb52ecf82021-06-14 23:37:21 +0100331 /* const or read-mostly hot data */
Jens Axboe2b188cc2019-01-07 10:46:33 -0700332 struct {
333 struct percpu_ref refs;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700334
Pavel Begunkovb52ecf82021-06-14 23:37:21 +0100335 struct io_rings *rings;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700336 unsigned int flags;
Randy Dunlape1d85332020-02-05 20:57:10 -0800337 unsigned int compat: 1;
Randy Dunlape1d85332020-02-05 20:57:10 -0800338 unsigned int drain_next: 1;
339 unsigned int eventfd_async: 1;
Stefano Garzarella21b55db2020-08-27 16:58:30 +0200340 unsigned int restricted: 1;
Pavel Begunkovf18ee4c2021-06-14 23:37:25 +0100341 unsigned int off_timeout_used: 1;
Pavel Begunkov10c66902021-06-15 16:47:56 +0100342 unsigned int drain_active: 1;
Pavel Begunkovb52ecf82021-06-14 23:37:21 +0100343 } ____cacheline_aligned_in_smp;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700344
Pavel Begunkov7f1129d2021-06-14 23:37:22 +0100345 /* submission data */
Pavel Begunkovb52ecf82021-06-14 23:37:21 +0100346 struct {
Pavel Begunkov0499e582021-06-14 23:37:29 +0100347 struct mutex uring_lock;
348
Hristo Venev75b28af2019-08-26 17:23:46 +0000349 /*
350 * Ring buffer of indices into array of io_uring_sqe, which is
351 * mmapped by the application using the IORING_OFF_SQES offset.
352 *
353 * This indirection could e.g. be used to assign fixed
354 * io_uring_sqe entries to operations and only submit them to
355 * the queue when needed.
356 *
357 * The kernel modifies neither the indices array nor the entries
358 * array.
359 */
360 u32 *sq_array;
Pavel Begunkovc7af47c2021-06-14 23:37:20 +0100361 struct io_uring_sqe *sq_sqes;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700362 unsigned cached_sq_head;
363 unsigned sq_entries;
Jens Axboede0617e2019-04-06 21:51:27 -0600364 struct list_head defer_list;
Pavel Begunkov7f1129d2021-06-14 23:37:22 +0100365
366 /*
367 * Fixed resources fast path, should be accessed only under
368 * uring_lock, and updated through io_uring_register(2)
369 */
370 struct io_rsrc_node *rsrc_node;
371 struct io_file_table file_table;
372 unsigned nr_user_files;
373 unsigned nr_user_bufs;
374 struct io_mapped_ubuf **user_bufs;
375
376 struct io_submit_state submit_state;
Jens Axboe5262f562019-09-17 12:26:57 -0600377 struct list_head timeout_list;
Jens Axboe1d7bb1d2019-11-06 11:31:17 -0700378 struct list_head cq_overflow_list;
Pavel Begunkov7f1129d2021-06-14 23:37:22 +0100379 struct xarray io_buffers;
380 struct xarray personalities;
381 u32 pers_next;
382 unsigned sq_thread_idle;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700383 } ____cacheline_aligned_in_smp;
384
Pavel Begunkovd0acdee2021-05-16 22:58:12 +0100385 /* IRQ completion list, under ->completion_lock */
386 struct list_head locked_free_list;
387 unsigned int locked_free_nr;
Jens Axboe3c1a2ea2021-02-11 10:48:03 -0700388
Stefan Metzmacher7c30f36a2021-03-07 11:54:28 +0100389 const struct cred *sq_creds; /* cred used for __io_sq_thread() */
Jens Axboe534ca6d2020-09-02 13:52:19 -0600390 struct io_sq_data *sq_data; /* if using sq thread polling */
391
Jens Axboe90554202020-09-03 12:12:41 -0600392 struct wait_queue_head sqo_sq_wait;
Jens Axboe69fb2132020-09-14 11:16:23 -0600393 struct list_head sqd_list;
Hristo Venev75b28af2019-08-26 17:23:46 +0000394
Pavel Begunkov5ed7a372021-06-14 23:37:27 +0100395 unsigned long check_cq_overflow;
396
Jens Axboe206aefd2019-11-07 18:27:42 -0700397 struct {
398 unsigned cached_cq_tail;
399 unsigned cq_entries;
Jens Axboe206aefd2019-11-07 18:27:42 -0700400 struct eventfd_ctx *cq_ev_fd;
Pavel Begunkov0499e582021-06-14 23:37:29 +0100401 struct wait_queue_head poll_wait;
402 struct wait_queue_head cq_wait;
403 unsigned cq_extra;
404 atomic_t cq_timeouts;
405 struct fasync_struct *cq_fasync;
406 unsigned cq_last_tm_flush;
Jens Axboe206aefd2019-11-07 18:27:42 -0700407 } ____cacheline_aligned_in_smp;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700408
409 struct {
Jens Axboe2b188cc2019-01-07 10:46:33 -0700410 spinlock_t completion_lock;
Jens Axboee94f1412019-12-19 12:06:02 -0700411
Jens Axboe89850fc2021-08-10 15:11:51 -0600412 spinlock_t timeout_lock;
413
Jens Axboedef596e2019-01-09 08:59:42 -0700414 /*
Pavel Begunkov540e32a2020-07-13 23:37:09 +0300415 * ->iopoll_list is protected by the ctx->uring_lock for
Jens Axboedef596e2019-01-09 08:59:42 -0700416 * io_uring instances that don't use IORING_SETUP_SQPOLL.
417 * For SQPOLL, only the single threaded io_sq_thread() will
418 * manipulate the list, hence no extra locking is needed there.
419 */
Pavel Begunkov540e32a2020-07-13 23:37:09 +0300420 struct list_head iopoll_list;
Jens Axboe78076bb2019-12-04 19:56:40 -0700421 struct hlist_head *cancel_hash;
422 unsigned cancel_hash_bits;
Hao Xu915b3dd2021-06-28 05:37:30 +0800423 bool poll_multi_queue;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700424 } ____cacheline_aligned_in_smp;
Jens Axboe85faa7b2020-04-09 18:14:00 -0600425
Stefano Garzarella21b55db2020-08-27 16:58:30 +0200426 struct io_restriction restrictions;
Jens Axboe3c1a2ea2021-02-11 10:48:03 -0700427
Pavel Begunkovb13a8912021-05-16 22:58:07 +0100428 /* slow path rsrc auxilary data, used by update/register */
429 struct {
430 struct io_rsrc_node *rsrc_backup_node;
431 struct io_mapped_ubuf *dummy_ubuf;
432 struct io_rsrc_data *file_data;
433 struct io_rsrc_data *buf_data;
434
435 struct delayed_work rsrc_put_work;
436 struct llist_head rsrc_put_llist;
437 struct list_head rsrc_ref_list;
438 spinlock_t rsrc_ref_lock;
439 };
440
Jens Axboe3c1a2ea2021-02-11 10:48:03 -0700441 /* Keep this last, we don't need it for the fast path */
Pavel Begunkovb986af72021-05-16 22:58:06 +0100442 struct {
443 #if defined(CONFIG_UNIX)
444 struct socket *ring_sock;
445 #endif
446 /* hashed buffered write serialization */
447 struct io_wq_hash *hash_map;
448
449 /* Only used for accounting purposes */
450 struct user_struct *user;
451 struct mm_struct *mm_account;
452
453 /* ctx exit and cancelation */
Pavel Begunkov9011bf92021-06-30 21:54:03 +0100454 struct llist_head fallback_llist;
455 struct delayed_work fallback_work;
Pavel Begunkovb986af72021-05-16 22:58:06 +0100456 struct work_struct exit_work;
457 struct list_head tctx_list;
458 struct completion ref_comp;
459 };
Jens Axboe2b188cc2019-01-07 10:46:33 -0700460};
461
Stefan Metzmacher53e043b2021-03-15 12:56:56 +0100462struct io_uring_task {
463 /* submission side */
Pavel Begunkov09899b12021-06-14 02:36:22 +0100464 int cached_refs;
Stefan Metzmacher53e043b2021-03-15 12:56:56 +0100465 struct xarray xa;
466 struct wait_queue_head wait;
Stefan Metzmacheree53fb22021-03-15 12:56:57 +0100467 const struct io_ring_ctx *last;
468 struct io_wq *io_wq;
Stefan Metzmacher53e043b2021-03-15 12:56:56 +0100469 struct percpu_counter inflight;
Pavel Begunkovb303fe22021-04-11 01:46:26 +0100470 atomic_t inflight_tracked;
Stefan Metzmacher53e043b2021-03-15 12:56:56 +0100471 atomic_t in_idle;
Stefan Metzmacher53e043b2021-03-15 12:56:56 +0100472
473 spinlock_t task_lock;
474 struct io_wq_work_list task_list;
Stefan Metzmacher53e043b2021-03-15 12:56:56 +0100475 struct callback_head task_work;
Pavel Begunkov6294f362021-08-10 17:53:55 +0100476 bool task_running;
Stefan Metzmacher53e043b2021-03-15 12:56:56 +0100477};
478
Jens Axboe09bb8392019-03-13 12:39:28 -0600479/*
480 * First field must be the file pointer in all the
481 * iocb unions! See also 'struct kiocb' in <linux/fs.h>
482 */
Jens Axboe221c5eb2019-01-17 09:41:58 -0700483struct io_poll_iocb {
484 struct file *file;
Pavel Begunkov018043b2020-10-27 23:17:18 +0000485 struct wait_queue_head *head;
Jens Axboe221c5eb2019-01-17 09:41:58 -0700486 __poll_t events;
Jens Axboe8c838782019-03-12 15:48:16 -0600487 bool done;
Jens Axboe221c5eb2019-01-17 09:41:58 -0700488 bool canceled;
Jens Axboe392edb42019-12-09 17:52:20 -0700489 struct wait_queue_entry wait;
Jens Axboe221c5eb2019-01-17 09:41:58 -0700490};
491
Pavel Begunkov9d805892021-04-13 02:58:40 +0100492struct io_poll_update {
Pavel Begunkov018043b2020-10-27 23:17:18 +0000493 struct file *file;
Pavel Begunkov9d805892021-04-13 02:58:40 +0100494 u64 old_user_data;
495 u64 new_user_data;
496 __poll_t events;
Jens Axboeb69de282021-03-17 08:37:41 -0600497 bool update_events;
498 bool update_user_data;
Pavel Begunkov018043b2020-10-27 23:17:18 +0000499};
500
Jens Axboeb5dba592019-12-11 14:02:38 -0700501struct io_close {
502 struct file *file;
Jens Axboeb5dba592019-12-11 14:02:38 -0700503 int fd;
504};
505
Jens Axboead8a48a2019-11-15 08:49:11 -0700506struct io_timeout_data {
507 struct io_kiocb *req;
508 struct hrtimer timer;
509 struct timespec64 ts;
510 enum hrtimer_mode mode;
511};
512
Jens Axboe8ed8d3c2019-12-16 11:55:28 -0700513struct io_accept {
514 struct file *file;
515 struct sockaddr __user *addr;
516 int __user *addr_len;
517 int flags;
Jens Axboe09952e32020-03-19 20:16:56 -0600518 unsigned long nofile;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -0700519};
520
521struct io_sync {
522 struct file *file;
523 loff_t len;
524 loff_t off;
525 int flags;
Jens Axboed63d1b52019-12-10 10:38:56 -0700526 int mode;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -0700527};
528
Jens Axboefbf23842019-12-17 18:45:56 -0700529struct io_cancel {
530 struct file *file;
531 u64 addr;
532};
533
Jens Axboeb29472e2019-12-17 18:50:29 -0700534struct io_timeout {
535 struct file *file;
Pavel Begunkovbfe68a22020-05-30 14:54:18 +0300536 u32 off;
537 u32 target_seq;
Pavel Begunkov135fcde2020-07-13 23:37:12 +0300538 struct list_head list;
Pavel Begunkov90cd7e42020-10-27 23:25:36 +0000539 /* head of the link, used by linked timeouts only */
540 struct io_kiocb *head;
Jens Axboe89b263f2021-08-10 15:14:18 -0600541 /* for linked completions */
542 struct io_kiocb *prev;
Jens Axboeb29472e2019-12-17 18:50:29 -0700543};
544
Pavel Begunkov0bdf7a22020-10-10 18:34:10 +0100545struct io_timeout_rem {
546 struct file *file;
547 u64 addr;
Pavel Begunkov9c8e11b2020-11-30 19:11:16 +0000548
549 /* timeout update */
550 struct timespec64 ts;
551 u32 flags;
Pavel Begunkov0bdf7a22020-10-10 18:34:10 +0100552};
553
Jens Axboe9adbd452019-12-20 08:45:55 -0700554struct io_rw {
555 /* NOTE: kiocb has the file as the first member, so don't do it here */
556 struct kiocb kiocb;
557 u64 addr;
558 u64 len;
559};
560
Jens Axboe3fbb51c2019-12-20 08:51:52 -0700561struct io_connect {
562 struct file *file;
563 struct sockaddr __user *addr;
564 int addr_len;
565};
566
Jens Axboee47293f2019-12-20 08:58:21 -0700567struct io_sr_msg {
568 struct file *file;
Jens Axboefddafac2020-01-04 20:19:44 -0700569 union {
Pavel Begunkov4af34172021-04-11 01:46:30 +0100570 struct compat_msghdr __user *umsg_compat;
571 struct user_msghdr __user *umsg;
572 void __user *buf;
Jens Axboefddafac2020-01-04 20:19:44 -0700573 };
Jens Axboee47293f2019-12-20 08:58:21 -0700574 int msg_flags;
Jens Axboebcda7ba2020-02-23 16:42:51 -0700575 int bgid;
Jens Axboefddafac2020-01-04 20:19:44 -0700576 size_t len;
Jens Axboebcda7ba2020-02-23 16:42:51 -0700577 struct io_buffer *kbuf;
Jens Axboee47293f2019-12-20 08:58:21 -0700578};
579
Jens Axboe15b71ab2019-12-11 11:20:36 -0700580struct io_open {
581 struct file *file;
582 int dfd;
Jens Axboe15b71ab2019-12-11 11:20:36 -0700583 struct filename *filename;
Jens Axboec12cedf2020-01-08 17:41:21 -0700584 struct open_how how;
Jens Axboe4022e7a2020-03-19 19:23:18 -0600585 unsigned long nofile;
Jens Axboe15b71ab2019-12-11 11:20:36 -0700586};
587
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +0000588struct io_rsrc_update {
Jens Axboe05f3fb32019-12-09 11:22:50 -0700589 struct file *file;
590 u64 arg;
591 u32 nr_args;
592 u32 offset;
593};
594
Jens Axboe4840e412019-12-25 22:03:45 -0700595struct io_fadvise {
596 struct file *file;
597 u64 offset;
598 u32 len;
599 u32 advice;
600};
601
Jens Axboec1ca7572019-12-25 22:18:28 -0700602struct io_madvise {
603 struct file *file;
604 u64 addr;
605 u32 len;
606 u32 advice;
607};
608
Jens Axboe3e4827b2020-01-08 15:18:09 -0700609struct io_epoll {
610 struct file *file;
611 int epfd;
612 int op;
613 int fd;
614 struct epoll_event event;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700615};
616
Pavel Begunkov7d67af22020-02-24 11:32:45 +0300617struct io_splice {
618 struct file *file_out;
619 struct file *file_in;
620 loff_t off_out;
621 loff_t off_in;
622 u64 len;
623 unsigned int flags;
624};
625
Jens Axboeddf0322d2020-02-23 16:41:33 -0700626struct io_provide_buf {
627 struct file *file;
628 __u64 addr;
Pavel Begunkov38134ad2021-04-15 13:07:39 +0100629 __u32 len;
Jens Axboeddf0322d2020-02-23 16:41:33 -0700630 __u32 bgid;
631 __u16 nbufs;
632 __u16 bid;
633};
634
Bijan Mottahedeh1d9e1282020-05-22 21:31:16 -0700635struct io_statx {
636 struct file *file;
637 int dfd;
638 unsigned int mask;
639 unsigned int flags;
Bijan Mottahedehe62753e2020-05-22 21:31:18 -0700640 const char __user *filename;
Bijan Mottahedeh1d9e1282020-05-22 21:31:16 -0700641 struct statx __user *buffer;
642};
643
Jens Axboe36f4fa62020-09-05 11:14:22 -0600644struct io_shutdown {
645 struct file *file;
646 int how;
647};
648
Jens Axboe80a261f2020-09-28 14:23:58 -0600649struct io_rename {
650 struct file *file;
651 int old_dfd;
652 int new_dfd;
653 struct filename *oldpath;
654 struct filename *newpath;
655 int flags;
656};
657
Jens Axboe14a11432020-09-28 14:27:37 -0600658struct io_unlink {
659 struct file *file;
660 int dfd;
661 int flags;
662 struct filename *filename;
663};
664
Pavel Begunkov3ca405e2020-07-13 23:37:08 +0300665struct io_completion {
666 struct file *file;
Pavel Begunkov8c3f9cd2021-02-28 22:35:15 +0000667 u32 cflags;
Pavel Begunkov3ca405e2020-07-13 23:37:08 +0300668};
669
Jens Axboef499a022019-12-02 16:28:46 -0700670struct io_async_connect {
671 struct sockaddr_storage address;
672};
673
Jens Axboe03b12302019-12-02 18:50:25 -0700674struct io_async_msghdr {
675 struct iovec fast_iov[UIO_FASTIOV];
Pavel Begunkov257e84a2021-02-05 00:58:00 +0000676 /* points to an allocated iov, if NULL we use fast_iov instead */
677 struct iovec *free_iov;
Jens Axboe03b12302019-12-02 18:50:25 -0700678 struct sockaddr __user *uaddr;
679 struct msghdr msg;
Jens Axboeb5379162020-02-09 11:29:15 -0700680 struct sockaddr_storage addr;
Jens Axboe03b12302019-12-02 18:50:25 -0700681};
682
Jens Axboef67676d2019-12-02 11:03:47 -0700683struct io_async_rw {
684 struct iovec fast_iov[UIO_FASTIOV];
Jens Axboeff6165b2020-08-13 09:47:43 -0600685 const struct iovec *free_iovec;
686 struct iov_iter iter;
Jens Axboe227c0c92020-08-13 11:51:40 -0600687 size_t bytes_done;
Jens Axboebcf5a062020-05-22 09:24:42 -0600688 struct wait_page_queue wpq;
Jens Axboef67676d2019-12-02 11:03:47 -0700689};
690
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300691enum {
692 REQ_F_FIXED_FILE_BIT = IOSQE_FIXED_FILE_BIT,
693 REQ_F_IO_DRAIN_BIT = IOSQE_IO_DRAIN_BIT,
694 REQ_F_LINK_BIT = IOSQE_IO_LINK_BIT,
695 REQ_F_HARDLINK_BIT = IOSQE_IO_HARDLINK_BIT,
696 REQ_F_FORCE_ASYNC_BIT = IOSQE_ASYNC_BIT,
Jens Axboebcda7ba2020-02-23 16:42:51 -0700697 REQ_F_BUFFER_SELECT_BIT = IOSQE_BUFFER_SELECT_BIT,
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300698
Pavel Begunkovdddca222021-04-27 16:13:52 +0100699 /* first byte is taken by user flags, shift it to not overlap */
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +0100700 REQ_F_FAIL_BIT = 8,
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300701 REQ_F_INFLIGHT_BIT,
702 REQ_F_CUR_POS_BIT,
703 REQ_F_NOWAIT_BIT,
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300704 REQ_F_LINK_TIMEOUT_BIT,
Pavel Begunkov99bc4c32020-02-07 22:04:45 +0300705 REQ_F_NEED_CLEANUP_BIT,
Jens Axboed7718a92020-02-14 22:23:12 -0700706 REQ_F_POLLED_BIT,
Jens Axboebcda7ba2020-02-23 16:42:51 -0700707 REQ_F_BUFFER_SELECTED_BIT,
Pavel Begunkove342c802021-01-19 13:32:47 +0000708 REQ_F_COMPLETE_INLINE_BIT,
Jens Axboe230d50d2021-04-01 20:41:15 -0600709 REQ_F_REISSUE_BIT,
Pavel Begunkov8c130822021-03-22 01:58:32 +0000710 REQ_F_DONT_REISSUE_BIT,
Pavel Begunkovb8e64b52021-06-17 18:14:02 +0100711 REQ_F_CREDS_BIT,
Pavel Begunkov20e60a32021-08-11 19:28:30 +0100712 REQ_F_REFCOUNT_BIT,
Pavel Begunkov4d13d1a2021-08-15 10:40:24 +0100713 REQ_F_ARM_LTIMEOUT_BIT,
Jens Axboe7b29f922021-03-12 08:30:14 -0700714 /* keep async read/write and isreg together and in order */
Pavel Begunkovb191e2d2021-08-09 13:04:03 +0100715 REQ_F_NOWAIT_READ_BIT,
716 REQ_F_NOWAIT_WRITE_BIT,
Jens Axboe7b29f922021-03-12 08:30:14 -0700717 REQ_F_ISREG_BIT,
Jens Axboe84557872020-03-03 15:28:17 -0700718
719 /* not a real bit, just to check we're not overflowing the space */
720 __REQ_F_LAST_BIT,
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300721};
722
723enum {
724 /* ctx owns file */
725 REQ_F_FIXED_FILE = BIT(REQ_F_FIXED_FILE_BIT),
726 /* drain existing IO first */
727 REQ_F_IO_DRAIN = BIT(REQ_F_IO_DRAIN_BIT),
728 /* linked sqes */
729 REQ_F_LINK = BIT(REQ_F_LINK_BIT),
730 /* doesn't sever on completion < 0 */
731 REQ_F_HARDLINK = BIT(REQ_F_HARDLINK_BIT),
732 /* IOSQE_ASYNC */
733 REQ_F_FORCE_ASYNC = BIT(REQ_F_FORCE_ASYNC_BIT),
Jens Axboebcda7ba2020-02-23 16:42:51 -0700734 /* IOSQE_BUFFER_SELECT */
735 REQ_F_BUFFER_SELECT = BIT(REQ_F_BUFFER_SELECT_BIT),
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300736
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300737 /* fail rest of links */
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +0100738 REQ_F_FAIL = BIT(REQ_F_FAIL_BIT),
Pavel Begunkovb05a1bc2021-03-04 13:59:24 +0000739 /* on inflight list, should be cancelled and waited on exit reliably */
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300740 REQ_F_INFLIGHT = BIT(REQ_F_INFLIGHT_BIT),
741 /* read/write uses file position */
742 REQ_F_CUR_POS = BIT(REQ_F_CUR_POS_BIT),
743 /* must not punt to workers */
744 REQ_F_NOWAIT = BIT(REQ_F_NOWAIT_BIT),
Pavel Begunkov900fad42020-10-19 16:39:16 +0100745 /* has or had linked timeout */
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300746 REQ_F_LINK_TIMEOUT = BIT(REQ_F_LINK_TIMEOUT_BIT),
Pavel Begunkov99bc4c32020-02-07 22:04:45 +0300747 /* needs cleanup */
748 REQ_F_NEED_CLEANUP = BIT(REQ_F_NEED_CLEANUP_BIT),
Jens Axboed7718a92020-02-14 22:23:12 -0700749 /* already went through poll handler */
750 REQ_F_POLLED = BIT(REQ_F_POLLED_BIT),
Jens Axboebcda7ba2020-02-23 16:42:51 -0700751 /* buffer already selected */
752 REQ_F_BUFFER_SELECTED = BIT(REQ_F_BUFFER_SELECTED_BIT),
Pavel Begunkove342c802021-01-19 13:32:47 +0000753 /* completion is deferred through io_comp_state */
754 REQ_F_COMPLETE_INLINE = BIT(REQ_F_COMPLETE_INLINE_BIT),
Jens Axboe230d50d2021-04-01 20:41:15 -0600755 /* caller should reissue async */
756 REQ_F_REISSUE = BIT(REQ_F_REISSUE_BIT),
Pavel Begunkov8c130822021-03-22 01:58:32 +0000757 /* don't attempt request reissue, see io_rw_reissue() */
758 REQ_F_DONT_REISSUE = BIT(REQ_F_DONT_REISSUE_BIT),
Jens Axboe7b29f922021-03-12 08:30:14 -0700759 /* supports async reads */
Pavel Begunkovb191e2d2021-08-09 13:04:03 +0100760 REQ_F_NOWAIT_READ = BIT(REQ_F_NOWAIT_READ_BIT),
Jens Axboe7b29f922021-03-12 08:30:14 -0700761 /* supports async writes */
Pavel Begunkovb191e2d2021-08-09 13:04:03 +0100762 REQ_F_NOWAIT_WRITE = BIT(REQ_F_NOWAIT_WRITE_BIT),
Jens Axboe7b29f922021-03-12 08:30:14 -0700763 /* regular file */
764 REQ_F_ISREG = BIT(REQ_F_ISREG_BIT),
Pavel Begunkovb8e64b52021-06-17 18:14:02 +0100765 /* has creds assigned */
766 REQ_F_CREDS = BIT(REQ_F_CREDS_BIT),
Pavel Begunkov20e60a32021-08-11 19:28:30 +0100767 /* skip refcounting if not set */
768 REQ_F_REFCOUNT = BIT(REQ_F_REFCOUNT_BIT),
Pavel Begunkov4d13d1a2021-08-15 10:40:24 +0100769 /* there is a linked timeout that has to be armed */
770 REQ_F_ARM_LTIMEOUT = BIT(REQ_F_ARM_LTIMEOUT_BIT),
Jens Axboed7718a92020-02-14 22:23:12 -0700771};
772
773struct async_poll {
774 struct io_poll_iocb poll;
Jens Axboe807abcb2020-07-17 17:09:27 -0600775 struct io_poll_iocb *double_poll;
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300776};
777
Pavel Begunkov5b0a6ac2021-06-30 21:54:04 +0100778typedef void (*io_req_tw_func_t)(struct io_kiocb *req);
779
Jens Axboe7cbf1722021-02-10 00:03:20 +0000780struct io_task_work {
Pavel Begunkov5b0a6ac2021-06-30 21:54:04 +0100781 union {
782 struct io_wq_work_node node;
783 struct llist_node fallback_node;
784 };
785 io_req_tw_func_t func;
Jens Axboe7cbf1722021-02-10 00:03:20 +0000786};
787
Pavel Begunkov992da012021-06-10 16:37:37 +0100788enum {
789 IORING_RSRC_FILE = 0,
790 IORING_RSRC_BUFFER = 1,
791};
792
Jens Axboe09bb8392019-03-13 12:39:28 -0600793/*
794 * NOTE! Each of the iocb union members has the file pointer
795 * as the first entry in their struct definition. So you can
796 * access the file pointer through any of the sub-structs,
797 * or directly as just 'ki_filp' in this struct.
798 */
Jens Axboe2b188cc2019-01-07 10:46:33 -0700799struct io_kiocb {
Jens Axboe221c5eb2019-01-17 09:41:58 -0700800 union {
Jens Axboe09bb8392019-03-13 12:39:28 -0600801 struct file *file;
Jens Axboe9adbd452019-12-20 08:45:55 -0700802 struct io_rw rw;
Jens Axboe221c5eb2019-01-17 09:41:58 -0700803 struct io_poll_iocb poll;
Pavel Begunkov9d805892021-04-13 02:58:40 +0100804 struct io_poll_update poll_update;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -0700805 struct io_accept accept;
806 struct io_sync sync;
Jens Axboefbf23842019-12-17 18:45:56 -0700807 struct io_cancel cancel;
Jens Axboeb29472e2019-12-17 18:50:29 -0700808 struct io_timeout timeout;
Pavel Begunkov0bdf7a22020-10-10 18:34:10 +0100809 struct io_timeout_rem timeout_rem;
Jens Axboe3fbb51c2019-12-20 08:51:52 -0700810 struct io_connect connect;
Jens Axboee47293f2019-12-20 08:58:21 -0700811 struct io_sr_msg sr_msg;
Jens Axboe15b71ab2019-12-11 11:20:36 -0700812 struct io_open open;
Jens Axboeb5dba592019-12-11 14:02:38 -0700813 struct io_close close;
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +0000814 struct io_rsrc_update rsrc_update;
Jens Axboe4840e412019-12-25 22:03:45 -0700815 struct io_fadvise fadvise;
Jens Axboec1ca7572019-12-25 22:18:28 -0700816 struct io_madvise madvise;
Jens Axboe3e4827b2020-01-08 15:18:09 -0700817 struct io_epoll epoll;
Pavel Begunkov7d67af22020-02-24 11:32:45 +0300818 struct io_splice splice;
Jens Axboeddf0322d2020-02-23 16:41:33 -0700819 struct io_provide_buf pbuf;
Bijan Mottahedeh1d9e1282020-05-22 21:31:16 -0700820 struct io_statx statx;
Jens Axboe36f4fa62020-09-05 11:14:22 -0600821 struct io_shutdown shutdown;
Jens Axboe80a261f2020-09-28 14:23:58 -0600822 struct io_rename rename;
Jens Axboe14a11432020-09-28 14:27:37 -0600823 struct io_unlink unlink;
Pavel Begunkov3ca405e2020-07-13 23:37:08 +0300824 /* use only after cleaning per-op data, see io_clean_op() */
825 struct io_completion compl;
Jens Axboe221c5eb2019-01-17 09:41:58 -0700826 };
Jens Axboe2b188cc2019-01-07 10:46:33 -0700827
Jens Axboee8c2bc12020-08-15 18:44:09 -0700828 /* opcode allocated if it needs to store data for async defer */
829 void *async_data;
Jens Axboed625c6e2019-12-17 19:53:05 -0700830 u8 opcode;
Xiaoguang Wang65a65432020-06-11 23:39:36 +0800831 /* polled IO has completed */
832 u8 iopoll_completed;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700833
Bijan Mottahedeh4f4eeba2020-05-19 14:52:49 -0700834 u16 buf_index;
Pavel Begunkov9cf7c102020-07-13 23:37:15 +0300835 u32 result;
Bijan Mottahedeh4f4eeba2020-05-19 14:52:49 -0700836
Pavel Begunkov010e8e62020-07-30 18:43:45 +0300837 struct io_ring_ctx *ctx;
838 unsigned int flags;
Jens Axboeabc54d62021-02-24 13:32:30 -0700839 atomic_t refs;
Pavel Begunkov010e8e62020-07-30 18:43:45 +0300840 struct task_struct *task;
841 u64 user_data;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700842
Pavel Begunkovf2f87372020-10-27 23:25:37 +0000843 struct io_kiocb *link;
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +0000844 struct percpu_ref *fixed_rsrc_refs;
Jens Axboed7718a92020-02-14 22:23:12 -0700845
Pavel Begunkovb303fe22021-04-11 01:46:26 +0100846 /* used with ctx->iopoll_list with reads/writes */
Pavel Begunkov010e8e62020-07-30 18:43:45 +0300847 struct list_head inflight_entry;
Pavel Begunkov5b0a6ac2021-06-30 21:54:04 +0100848 struct io_task_work io_task_work;
Pavel Begunkov010e8e62020-07-30 18:43:45 +0300849 /* for polled requests, i.e. IORING_OP_POLL_ADD and async armed poll */
850 struct hlist_node hash_node;
851 struct async_poll *apoll;
852 struct io_wq_work work;
Pavel Begunkovfe7e3252021-06-24 15:09:57 +0100853 const struct cred *creds;
Pavel Begunkovc10d1f92021-06-17 18:14:01 +0100854
Pavel Begunkoveae071c2021-04-25 14:32:24 +0100855 /* store used ubuf, so we can prevent reloading */
856 struct io_mapped_ubuf *imu;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700857};
858
Pavel Begunkov13bf43f2021-03-06 11:02:12 +0000859struct io_tctx_node {
860 struct list_head ctx_node;
861 struct task_struct *task;
Pavel Begunkov13bf43f2021-03-06 11:02:12 +0000862 struct io_ring_ctx *ctx;
863};
864
Pavel Begunkov27dc8332020-07-13 23:37:14 +0300865struct io_defer_entry {
866 struct list_head list;
867 struct io_kiocb *req;
Pavel Begunkov9cf7c102020-07-13 23:37:15 +0300868 u32 seq;
Pavel Begunkov27dc8332020-07-13 23:37:14 +0300869};
870
Jens Axboed3656342019-12-18 09:50:26 -0700871struct io_op_def {
Jens Axboed3656342019-12-18 09:50:26 -0700872 /* needs req->file assigned */
873 unsigned needs_file : 1;
Jens Axboed3656342019-12-18 09:50:26 -0700874 /* hash wq insertion if file is a regular file */
875 unsigned hash_reg_file : 1;
876 /* unbound wq insertion if file is a non-regular file */
877 unsigned unbound_nonreg_file : 1;
Jens Axboe66f4af92020-01-16 15:36:52 -0700878 /* opcode is not supported by this kernel */
879 unsigned not_supported : 1;
Jens Axboe8a727582020-02-20 09:59:44 -0700880 /* set if opcode supports polled "wait" */
881 unsigned pollin : 1;
882 unsigned pollout : 1;
Jens Axboebcda7ba2020-02-23 16:42:51 -0700883 /* op supports buffer selection */
884 unsigned buffer_select : 1;
Pavel Begunkov26f05052021-02-28 22:35:18 +0000885 /* do prep async if is going to be punted */
886 unsigned needs_async_setup : 1;
Jens Axboe27926b62020-10-28 09:33:23 -0600887 /* should block plug */
888 unsigned plug : 1;
Jens Axboee8c2bc12020-08-15 18:44:09 -0700889 /* size of async data needed, if any */
890 unsigned short async_size;
Jens Axboed3656342019-12-18 09:50:26 -0700891};
892
Jens Axboe09186822020-10-13 15:01:40 -0600893static const struct io_op_def io_op_defs[] = {
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300894 [IORING_OP_NOP] = {},
895 [IORING_OP_READV] = {
Jens Axboed3656342019-12-18 09:50:26 -0700896 .needs_file = 1,
897 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -0700898 .pollin = 1,
Jens Axboe4d954c22020-02-27 07:31:19 -0700899 .buffer_select = 1,
Pavel Begunkov26f05052021-02-28 22:35:18 +0000900 .needs_async_setup = 1,
Jens Axboe27926b62020-10-28 09:33:23 -0600901 .plug = 1,
Jens Axboee8c2bc12020-08-15 18:44:09 -0700902 .async_size = sizeof(struct io_async_rw),
Jens Axboed3656342019-12-18 09:50:26 -0700903 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300904 [IORING_OP_WRITEV] = {
Jens Axboed3656342019-12-18 09:50:26 -0700905 .needs_file = 1,
906 .hash_reg_file = 1,
907 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -0700908 .pollout = 1,
Pavel Begunkov26f05052021-02-28 22:35:18 +0000909 .needs_async_setup = 1,
Jens Axboe27926b62020-10-28 09:33:23 -0600910 .plug = 1,
Jens Axboee8c2bc12020-08-15 18:44:09 -0700911 .async_size = sizeof(struct io_async_rw),
Jens Axboed3656342019-12-18 09:50:26 -0700912 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300913 [IORING_OP_FSYNC] = {
Jens Axboed3656342019-12-18 09:50:26 -0700914 .needs_file = 1,
915 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300916 [IORING_OP_READ_FIXED] = {
Jens Axboed3656342019-12-18 09:50:26 -0700917 .needs_file = 1,
918 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -0700919 .pollin = 1,
Jens Axboe27926b62020-10-28 09:33:23 -0600920 .plug = 1,
Jens Axboee8c2bc12020-08-15 18:44:09 -0700921 .async_size = sizeof(struct io_async_rw),
Jens Axboed3656342019-12-18 09:50:26 -0700922 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300923 [IORING_OP_WRITE_FIXED] = {
Jens Axboed3656342019-12-18 09:50:26 -0700924 .needs_file = 1,
925 .hash_reg_file = 1,
926 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -0700927 .pollout = 1,
Jens Axboe27926b62020-10-28 09:33:23 -0600928 .plug = 1,
Jens Axboee8c2bc12020-08-15 18:44:09 -0700929 .async_size = sizeof(struct io_async_rw),
Jens Axboed3656342019-12-18 09:50:26 -0700930 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300931 [IORING_OP_POLL_ADD] = {
Jens Axboed3656342019-12-18 09:50:26 -0700932 .needs_file = 1,
933 .unbound_nonreg_file = 1,
934 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300935 [IORING_OP_POLL_REMOVE] = {},
936 [IORING_OP_SYNC_FILE_RANGE] = {
Jens Axboed3656342019-12-18 09:50:26 -0700937 .needs_file = 1,
938 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300939 [IORING_OP_SENDMSG] = {
Jens Axboed3656342019-12-18 09:50:26 -0700940 .needs_file = 1,
941 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -0700942 .pollout = 1,
Pavel Begunkov26f05052021-02-28 22:35:18 +0000943 .needs_async_setup = 1,
Jens Axboee8c2bc12020-08-15 18:44:09 -0700944 .async_size = sizeof(struct io_async_msghdr),
Jens Axboed3656342019-12-18 09:50:26 -0700945 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300946 [IORING_OP_RECVMSG] = {
Jens Axboed3656342019-12-18 09:50:26 -0700947 .needs_file = 1,
948 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -0700949 .pollin = 1,
Jens Axboe52de1fe2020-02-27 10:15:42 -0700950 .buffer_select = 1,
Pavel Begunkov26f05052021-02-28 22:35:18 +0000951 .needs_async_setup = 1,
Jens Axboee8c2bc12020-08-15 18:44:09 -0700952 .async_size = sizeof(struct io_async_msghdr),
Jens Axboed3656342019-12-18 09:50:26 -0700953 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300954 [IORING_OP_TIMEOUT] = {
Jens Axboee8c2bc12020-08-15 18:44:09 -0700955 .async_size = sizeof(struct io_timeout_data),
Jens Axboed3656342019-12-18 09:50:26 -0700956 },
Pavel Begunkov9c8e11b2020-11-30 19:11:16 +0000957 [IORING_OP_TIMEOUT_REMOVE] = {
958 /* used by timeout updates' prep() */
Pavel Begunkov9c8e11b2020-11-30 19:11:16 +0000959 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300960 [IORING_OP_ACCEPT] = {
Jens Axboed3656342019-12-18 09:50:26 -0700961 .needs_file = 1,
962 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -0700963 .pollin = 1,
Jens Axboed3656342019-12-18 09:50:26 -0700964 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300965 [IORING_OP_ASYNC_CANCEL] = {},
966 [IORING_OP_LINK_TIMEOUT] = {
Jens Axboee8c2bc12020-08-15 18:44:09 -0700967 .async_size = sizeof(struct io_timeout_data),
Jens Axboed3656342019-12-18 09:50:26 -0700968 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300969 [IORING_OP_CONNECT] = {
Jens Axboed3656342019-12-18 09:50:26 -0700970 .needs_file = 1,
971 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -0700972 .pollout = 1,
Pavel Begunkov26f05052021-02-28 22:35:18 +0000973 .needs_async_setup = 1,
Jens Axboee8c2bc12020-08-15 18:44:09 -0700974 .async_size = sizeof(struct io_async_connect),
Jens Axboed3656342019-12-18 09:50:26 -0700975 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300976 [IORING_OP_FALLOCATE] = {
Jens Axboed3656342019-12-18 09:50:26 -0700977 .needs_file = 1,
978 },
Jens Axboe44526be2021-02-15 13:32:18 -0700979 [IORING_OP_OPENAT] = {},
980 [IORING_OP_CLOSE] = {},
981 [IORING_OP_FILES_UPDATE] = {},
982 [IORING_OP_STATX] = {},
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300983 [IORING_OP_READ] = {
Jens Axboe3a6820f2019-12-22 15:19:35 -0700984 .needs_file = 1,
985 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -0700986 .pollin = 1,
Jens Axboebcda7ba2020-02-23 16:42:51 -0700987 .buffer_select = 1,
Jens Axboe27926b62020-10-28 09:33:23 -0600988 .plug = 1,
Jens Axboee8c2bc12020-08-15 18:44:09 -0700989 .async_size = sizeof(struct io_async_rw),
Jens Axboe3a6820f2019-12-22 15:19:35 -0700990 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300991 [IORING_OP_WRITE] = {
Jens Axboe3a6820f2019-12-22 15:19:35 -0700992 .needs_file = 1,
993 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -0700994 .pollout = 1,
Jens Axboe27926b62020-10-28 09:33:23 -0600995 .plug = 1,
Jens Axboee8c2bc12020-08-15 18:44:09 -0700996 .async_size = sizeof(struct io_async_rw),
Jens Axboe3a6820f2019-12-22 15:19:35 -0700997 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300998 [IORING_OP_FADVISE] = {
Jens Axboe4840e412019-12-25 22:03:45 -0700999 .needs_file = 1,
1000 },
Jens Axboe44526be2021-02-15 13:32:18 -07001001 [IORING_OP_MADVISE] = {},
Pavel Begunkov0463b6c2020-01-18 21:35:38 +03001002 [IORING_OP_SEND] = {
Jens Axboefddafac2020-01-04 20:19:44 -07001003 .needs_file = 1,
1004 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -07001005 .pollout = 1,
Jens Axboefddafac2020-01-04 20:19:44 -07001006 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +03001007 [IORING_OP_RECV] = {
Jens Axboefddafac2020-01-04 20:19:44 -07001008 .needs_file = 1,
1009 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -07001010 .pollin = 1,
Jens Axboebcda7ba2020-02-23 16:42:51 -07001011 .buffer_select = 1,
Jens Axboefddafac2020-01-04 20:19:44 -07001012 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +03001013 [IORING_OP_OPENAT2] = {
Jens Axboecebdb982020-01-08 17:59:24 -07001014 },
Jens Axboe3e4827b2020-01-08 15:18:09 -07001015 [IORING_OP_EPOLL_CTL] = {
1016 .unbound_nonreg_file = 1,
Jens Axboe3e4827b2020-01-08 15:18:09 -07001017 },
Pavel Begunkov7d67af22020-02-24 11:32:45 +03001018 [IORING_OP_SPLICE] = {
1019 .needs_file = 1,
1020 .hash_reg_file = 1,
1021 .unbound_nonreg_file = 1,
Jens Axboeddf0322d2020-02-23 16:41:33 -07001022 },
1023 [IORING_OP_PROVIDE_BUFFERS] = {},
Jens Axboe067524e2020-03-02 16:32:28 -07001024 [IORING_OP_REMOVE_BUFFERS] = {},
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03001025 [IORING_OP_TEE] = {
1026 .needs_file = 1,
1027 .hash_reg_file = 1,
1028 .unbound_nonreg_file = 1,
1029 },
Jens Axboe36f4fa62020-09-05 11:14:22 -06001030 [IORING_OP_SHUTDOWN] = {
1031 .needs_file = 1,
1032 },
Jens Axboe44526be2021-02-15 13:32:18 -07001033 [IORING_OP_RENAMEAT] = {},
1034 [IORING_OP_UNLINKAT] = {},
Jens Axboed3656342019-12-18 09:50:26 -07001035};
1036
Pavel Begunkov0756a862021-08-15 10:40:25 +01001037/* requests with any of those set should undergo io_disarm_next() */
1038#define IO_DISARM_MASK (REQ_F_ARM_LTIMEOUT | REQ_F_LINK_TIMEOUT | REQ_F_FAIL)
1039
Pavel Begunkov7a612352021-03-09 00:37:59 +00001040static bool io_disarm_next(struct io_kiocb *req);
Pavel Begunkoveef51da2021-06-14 02:36:15 +01001041static void io_uring_del_tctx_node(unsigned long index);
Pavel Begunkov9936c7c2021-02-04 13:51:56 +00001042static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
1043 struct task_struct *task,
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01001044 bool cancel_all);
Pavel Begunkov78cc6872021-06-14 02:36:23 +01001045static void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd);
Pavel Begunkov1ffc5422020-12-30 21:34:15 +00001046
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01001047static bool io_cqring_fill_event(struct io_ring_ctx *ctx, u64 user_data,
1048 long res, unsigned int cflags);
Jackie Liuec9c02a2019-11-08 23:50:36 +08001049static void io_put_req(struct io_kiocb *req);
Pavel Begunkov91c2f692021-08-11 19:28:28 +01001050static void io_put_req_deferred(struct io_kiocb *req);
Jens Axboec7dae4b2021-02-09 19:53:37 -07001051static void io_dismantle_req(struct io_kiocb *req);
Jens Axboe94ae5e72019-11-14 19:39:52 -07001052static void io_queue_linked_timeout(struct io_kiocb *req);
Pavel Begunkovfdecb662021-04-25 14:32:20 +01001053static int __io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned type,
Pavel Begunkovc3bdad02021-04-25 14:32:22 +01001054 struct io_uring_rsrc_update2 *up,
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +01001055 unsigned nr_args);
Pavel Begunkov68fb8972021-03-19 17:22:41 +00001056static void io_clean_op(struct io_kiocb *req);
Pavel Begunkovac177052021-08-09 13:04:02 +01001057static struct file *io_file_get(struct io_ring_ctx *ctx,
Pavel Begunkov8371adf2020-10-10 18:34:08 +01001058 struct io_kiocb *req, int fd, bool fixed);
Pavel Begunkovc5eef2b2021-02-10 00:03:22 +00001059static void __io_queue_sqe(struct io_kiocb *req);
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00001060static void io_rsrc_put_work(struct work_struct *work);
Jens Axboede0617e2019-04-06 21:51:27 -06001061
Pavel Begunkov907d1df2021-01-26 23:35:10 +00001062static void io_req_task_queue(struct io_kiocb *req);
Pavel Begunkov2a2758f2021-06-17 18:14:00 +01001063static void io_submit_flush_completions(struct io_ring_ctx *ctx);
Pavel Begunkov179ae0d2021-02-28 22:35:20 +00001064static int io_req_prep_async(struct io_kiocb *req);
Jens Axboe9a56a232019-01-09 09:06:50 -07001065
Jens Axboe2b188cc2019-01-07 10:46:33 -07001066static struct kmem_cache *req_cachep;
1067
Jens Axboe09186822020-10-13 15:01:40 -06001068static const struct file_operations io_uring_fops;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001069
1070struct sock *io_uring_get_socket(struct file *file)
1071{
1072#if defined(CONFIG_UNIX)
1073 if (file->f_op == &io_uring_fops) {
1074 struct io_ring_ctx *ctx = file->private_data;
1075
1076 return ctx->ring_sock->sk;
1077 }
1078#endif
1079 return NULL;
1080}
1081EXPORT_SYMBOL(io_uring_get_socket);
1082
Pavel Begunkovf2f87372020-10-27 23:25:37 +00001083#define io_for_each_link(pos, head) \
1084 for (pos = (head); pos; pos = pos->link)
1085
Pavel Begunkov21c843d2021-08-11 19:28:27 +01001086/*
1087 * Shamelessly stolen from the mm implementation of page reference checking,
1088 * see commit f958d7b528b1 for details.
1089 */
1090#define req_ref_zero_or_close_to_overflow(req) \
1091 ((unsigned int) atomic_read(&(req->refs)) + 127u <= 127u)
1092
1093static inline bool req_ref_inc_not_zero(struct io_kiocb *req)
1094{
Pavel Begunkov20e60a32021-08-11 19:28:30 +01001095 WARN_ON_ONCE(!(req->flags & REQ_F_REFCOUNT));
Pavel Begunkov21c843d2021-08-11 19:28:27 +01001096 return atomic_inc_not_zero(&req->refs);
1097}
1098
Pavel Begunkov21c843d2021-08-11 19:28:27 +01001099static inline bool req_ref_put_and_test(struct io_kiocb *req)
1100{
Pavel Begunkov20e60a32021-08-11 19:28:30 +01001101 if (likely(!(req->flags & REQ_F_REFCOUNT)))
1102 return true;
1103
Pavel Begunkov21c843d2021-08-11 19:28:27 +01001104 WARN_ON_ONCE(req_ref_zero_or_close_to_overflow(req));
1105 return atomic_dec_and_test(&req->refs);
1106}
1107
1108static inline void req_ref_put(struct io_kiocb *req)
1109{
Pavel Begunkov20e60a32021-08-11 19:28:30 +01001110 WARN_ON_ONCE(!(req->flags & REQ_F_REFCOUNT));
Pavel Begunkov21c843d2021-08-11 19:28:27 +01001111 WARN_ON_ONCE(req_ref_put_and_test(req));
1112}
1113
1114static inline void req_ref_get(struct io_kiocb *req)
1115{
Pavel Begunkov20e60a32021-08-11 19:28:30 +01001116 WARN_ON_ONCE(!(req->flags & REQ_F_REFCOUNT));
Pavel Begunkov21c843d2021-08-11 19:28:27 +01001117 WARN_ON_ONCE(req_ref_zero_or_close_to_overflow(req));
1118 atomic_inc(&req->refs);
1119}
1120
Pavel Begunkov48dcd382021-08-15 10:40:18 +01001121static inline void __io_req_set_refcount(struct io_kiocb *req, int nr)
Pavel Begunkov20e60a32021-08-11 19:28:30 +01001122{
1123 if (!(req->flags & REQ_F_REFCOUNT)) {
1124 req->flags |= REQ_F_REFCOUNT;
Pavel Begunkov48dcd382021-08-15 10:40:18 +01001125 atomic_set(&req->refs, nr);
Pavel Begunkov20e60a32021-08-11 19:28:30 +01001126 }
1127}
1128
Pavel Begunkov48dcd382021-08-15 10:40:18 +01001129static inline void io_req_set_refcount(struct io_kiocb *req)
1130{
1131 __io_req_set_refcount(req, 1);
1132}
1133
Pavel Begunkovb895c9a2021-04-01 15:43:40 +01001134static inline void io_req_set_rsrc_node(struct io_kiocb *req)
Jens Axboec40f6372020-06-25 15:39:59 -06001135{
Pavel Begunkov36f72fe2020-11-18 19:57:26 +00001136 struct io_ring_ctx *ctx = req->ctx;
1137
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00001138 if (!req->fixed_rsrc_refs) {
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01001139 req->fixed_rsrc_refs = &ctx->rsrc_node->refs;
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00001140 percpu_ref_get(req->fixed_rsrc_refs);
Pavel Begunkov36f72fe2020-11-18 19:57:26 +00001141 }
1142}
1143
Pavel Begunkovf70865d2021-04-11 01:46:40 +01001144static void io_refs_resurrect(struct percpu_ref *ref, struct completion *compl)
1145{
1146 bool got = percpu_ref_tryget(ref);
1147
1148 /* already at zero, wait for ->release() */
1149 if (!got)
1150 wait_for_completion(compl);
1151 percpu_ref_resurrect(ref);
1152 if (got)
1153 percpu_ref_put(ref);
1154}
1155
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01001156static bool io_match_task(struct io_kiocb *head, struct task_struct *task,
1157 bool cancel_all)
Pavel Begunkov08d23632020-11-06 13:00:22 +00001158{
1159 struct io_kiocb *req;
1160
Pavel Begunkov68207682021-03-22 01:58:25 +00001161 if (task && head->task != task)
Pavel Begunkov08d23632020-11-06 13:00:22 +00001162 return false;
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01001163 if (cancel_all)
Pavel Begunkov08d23632020-11-06 13:00:22 +00001164 return true;
1165
1166 io_for_each_link(req, head) {
Pavel Begunkovb05a1bc2021-03-04 13:59:24 +00001167 if (req->flags & REQ_F_INFLIGHT)
Jens Axboe02a13672021-01-23 15:49:31 -07001168 return true;
Pavel Begunkov08d23632020-11-06 13:00:22 +00001169 }
1170 return false;
1171}
1172
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01001173static inline void req_set_fail(struct io_kiocb *req)
Jens Axboec40f6372020-06-25 15:39:59 -06001174{
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01001175 req->flags |= REQ_F_FAIL;
Jens Axboec40f6372020-06-25 15:39:59 -06001176}
Jens Axboe4a38aed22020-05-14 17:21:15 -06001177
Jens Axboe2b188cc2019-01-07 10:46:33 -07001178static void io_ring_ctx_ref_free(struct percpu_ref *ref)
1179{
1180 struct io_ring_ctx *ctx = container_of(ref, struct io_ring_ctx, refs);
1181
Jens Axboe0f158b42020-05-14 17:18:39 -06001182 complete(&ctx->ref_comp);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001183}
1184
Pavel Begunkov8eb7e2d2020-06-29 13:13:02 +03001185static inline bool io_is_timeout_noseq(struct io_kiocb *req)
1186{
1187 return !req->timeout.off;
1188}
1189
Pavel Begunkovf56165e2021-08-09 20:18:07 +01001190static void io_fallback_req_func(struct work_struct *work)
1191{
1192 struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx,
1193 fallback_work.work);
1194 struct llist_node *node = llist_del_all(&ctx->fallback_llist);
1195 struct io_kiocb *req, *tmp;
1196
1197 percpu_ref_get(&ctx->refs);
1198 llist_for_each_entry_safe(req, tmp, node, io_task_work.fallback_node)
1199 req->io_task_work.func(req);
1200 percpu_ref_put(&ctx->refs);
1201}
1202
Jens Axboe2b188cc2019-01-07 10:46:33 -07001203static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
1204{
1205 struct io_ring_ctx *ctx;
Jens Axboe78076bb2019-12-04 19:56:40 -07001206 int hash_bits;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001207
1208 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
1209 if (!ctx)
1210 return NULL;
1211
Jens Axboe78076bb2019-12-04 19:56:40 -07001212 /*
1213 * Use 5 bits less than the max cq entries, that should give us around
1214 * 32 entries per hash list if totally full and uniformly spread.
1215 */
1216 hash_bits = ilog2(p->cq_entries);
1217 hash_bits -= 5;
1218 if (hash_bits <= 0)
1219 hash_bits = 1;
1220 ctx->cancel_hash_bits = hash_bits;
1221 ctx->cancel_hash = kmalloc((1U << hash_bits) * sizeof(struct hlist_head),
1222 GFP_KERNEL);
1223 if (!ctx->cancel_hash)
1224 goto err;
1225 __hash_init(ctx->cancel_hash, 1U << hash_bits);
1226
Pavel Begunkov62248432021-04-28 13:11:29 +01001227 ctx->dummy_ubuf = kzalloc(sizeof(*ctx->dummy_ubuf), GFP_KERNEL);
1228 if (!ctx->dummy_ubuf)
1229 goto err;
1230 /* set invalid range, so io_import_fixed() fails meeting it */
1231 ctx->dummy_ubuf->ubuf = -1UL;
1232
Roman Gushchin21482892019-05-07 10:01:48 -07001233 if (percpu_ref_init(&ctx->refs, io_ring_ctx_ref_free,
Jens Axboe206aefd2019-11-07 18:27:42 -07001234 PERCPU_REF_ALLOW_REINIT, GFP_KERNEL))
1235 goto err;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001236
1237 ctx->flags = p->flags;
Jens Axboe90554202020-09-03 12:12:41 -06001238 init_waitqueue_head(&ctx->sqo_sq_wait);
Jens Axboe69fb2132020-09-14 11:16:23 -06001239 INIT_LIST_HEAD(&ctx->sqd_list);
Pavel Begunkov311997b2021-06-14 23:37:28 +01001240 init_waitqueue_head(&ctx->poll_wait);
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001241 INIT_LIST_HEAD(&ctx->cq_overflow_list);
Jens Axboe0f158b42020-05-14 17:18:39 -06001242 init_completion(&ctx->ref_comp);
Jens Axboe9e15c3a2021-03-13 12:29:43 -07001243 xa_init_flags(&ctx->io_buffers, XA_FLAGS_ALLOC1);
Matthew Wilcox (Oracle)61cf9372021-03-08 14:16:16 +00001244 xa_init_flags(&ctx->personalities, XA_FLAGS_ALLOC1);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001245 mutex_init(&ctx->uring_lock);
Pavel Begunkov311997b2021-06-14 23:37:28 +01001246 init_waitqueue_head(&ctx->cq_wait);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001247 spin_lock_init(&ctx->completion_lock);
Jens Axboe89850fc2021-08-10 15:11:51 -06001248 spin_lock_init(&ctx->timeout_lock);
Pavel Begunkov540e32a2020-07-13 23:37:09 +03001249 INIT_LIST_HEAD(&ctx->iopoll_list);
Jens Axboede0617e2019-04-06 21:51:27 -06001250 INIT_LIST_HEAD(&ctx->defer_list);
Jens Axboe5262f562019-09-17 12:26:57 -06001251 INIT_LIST_HEAD(&ctx->timeout_list);
Bijan Mottahedehd67d2262021-01-15 17:37:46 +00001252 spin_lock_init(&ctx->rsrc_ref_lock);
1253 INIT_LIST_HEAD(&ctx->rsrc_ref_list);
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00001254 INIT_DELAYED_WORK(&ctx->rsrc_put_work, io_rsrc_put_work);
1255 init_llist_head(&ctx->rsrc_put_llist);
Pavel Begunkov13bf43f2021-03-06 11:02:12 +00001256 INIT_LIST_HEAD(&ctx->tctx_list);
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01001257 INIT_LIST_HEAD(&ctx->submit_state.free_list);
Pavel Begunkovd0acdee2021-05-16 22:58:12 +01001258 INIT_LIST_HEAD(&ctx->locked_free_list);
Pavel Begunkov9011bf92021-06-30 21:54:03 +01001259 INIT_DELAYED_WORK(&ctx->fallback_work, io_fallback_req_func);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001260 return ctx;
Jens Axboe206aefd2019-11-07 18:27:42 -07001261err:
Pavel Begunkov62248432021-04-28 13:11:29 +01001262 kfree(ctx->dummy_ubuf);
Jens Axboe78076bb2019-12-04 19:56:40 -07001263 kfree(ctx->cancel_hash);
Jens Axboe206aefd2019-11-07 18:27:42 -07001264 kfree(ctx);
1265 return NULL;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001266}
1267
Pavel Begunkov8f6ed492021-05-16 22:58:10 +01001268static void io_account_cq_overflow(struct io_ring_ctx *ctx)
1269{
1270 struct io_rings *r = ctx->rings;
1271
1272 WRITE_ONCE(r->cq_overflow, READ_ONCE(r->cq_overflow) + 1);
1273 ctx->cq_extra--;
1274}
1275
Pavel Begunkov9cf7c102020-07-13 23:37:15 +03001276static bool req_need_defer(struct io_kiocb *req, u32 seq)
Jens Axboede0617e2019-04-06 21:51:27 -06001277{
Jens Axboe2bc99302020-07-09 09:43:27 -06001278 if (unlikely(req->flags & REQ_F_IO_DRAIN)) {
1279 struct io_ring_ctx *ctx = req->ctx;
Jackie Liua197f662019-11-08 08:09:12 -07001280
Pavel Begunkov8f6ed492021-05-16 22:58:10 +01001281 return seq + READ_ONCE(ctx->cq_extra) != ctx->cached_cq_tail;
Jens Axboe2bc99302020-07-09 09:43:27 -06001282 }
Jens Axboe7adf4ea2019-10-10 21:42:58 -06001283
Bob Liu9d858b22019-11-13 18:06:25 +08001284 return false;
Jens Axboe7adf4ea2019-10-10 21:42:58 -06001285}
1286
Pavel Begunkovc97d8a02021-08-09 13:04:04 +01001287#define FFS_ASYNC_READ 0x1UL
1288#define FFS_ASYNC_WRITE 0x2UL
1289#ifdef CONFIG_64BIT
1290#define FFS_ISREG 0x4UL
1291#else
1292#define FFS_ISREG 0x0UL
1293#endif
1294#define FFS_MASK ~(FFS_ASYNC_READ|FFS_ASYNC_WRITE|FFS_ISREG)
1295
1296static inline bool io_req_ffs_set(struct io_kiocb *req)
1297{
1298 return IS_ENABLED(CONFIG_64BIT) && (req->flags & REQ_F_FIXED_FILE);
1299}
1300
Pavel Begunkovce3d5aa2021-02-01 18:59:55 +00001301static void io_req_track_inflight(struct io_kiocb *req)
1302{
Pavel Begunkovce3d5aa2021-02-01 18:59:55 +00001303 if (!(req->flags & REQ_F_INFLIGHT)) {
Pavel Begunkovce3d5aa2021-02-01 18:59:55 +00001304 req->flags |= REQ_F_INFLIGHT;
Pavel Begunkovb303fe22021-04-11 01:46:26 +01001305 atomic_inc(&current->io_uring->inflight_tracked);
Pavel Begunkovce3d5aa2021-02-01 18:59:55 +00001306 }
1307}
1308
Pavel Begunkov906c6ca2021-08-15 10:40:26 +01001309static inline void io_unprep_linked_timeout(struct io_kiocb *req)
1310{
1311 req->flags &= ~REQ_F_LINK_TIMEOUT;
1312}
1313
Pavel Begunkovfd08e532021-08-11 19:28:31 +01001314static struct io_kiocb *__io_prep_linked_timeout(struct io_kiocb *req)
1315{
Pavel Begunkov906c6ca2021-08-15 10:40:26 +01001316 if (WARN_ON_ONCE(!req->link))
1317 return NULL;
1318
Pavel Begunkov4d13d1a2021-08-15 10:40:24 +01001319 req->flags &= ~REQ_F_ARM_LTIMEOUT;
1320 req->flags |= REQ_F_LINK_TIMEOUT;
Pavel Begunkovfd08e532021-08-11 19:28:31 +01001321
1322 /* linked timeouts should have two refs once prep'ed */
Pavel Begunkov48dcd382021-08-15 10:40:18 +01001323 io_req_set_refcount(req);
Pavel Begunkov4d13d1a2021-08-15 10:40:24 +01001324 __io_req_set_refcount(req->link, 2);
1325 return req->link;
Pavel Begunkovfd08e532021-08-11 19:28:31 +01001326}
1327
1328static inline struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req)
1329{
Pavel Begunkov4d13d1a2021-08-15 10:40:24 +01001330 if (likely(!(req->flags & REQ_F_ARM_LTIMEOUT)))
Pavel Begunkovfd08e532021-08-11 19:28:31 +01001331 return NULL;
1332 return __io_prep_linked_timeout(req);
1333}
1334
Pavel Begunkovcbdcb432020-06-29 19:18:43 +03001335static void io_prep_async_work(struct io_kiocb *req)
Jens Axboe561fb042019-10-24 07:25:42 -06001336{
Jens Axboed3656342019-12-18 09:50:26 -07001337 const struct io_op_def *def = &io_op_defs[req->opcode];
Pavel Begunkov23329512020-10-10 18:34:06 +01001338 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe54a91f32019-09-10 09:15:04 -06001339
Pavel Begunkovb8e64b52021-06-17 18:14:02 +01001340 if (!(req->flags & REQ_F_CREDS)) {
1341 req->flags |= REQ_F_CREDS;
Pavel Begunkovc10d1f92021-06-17 18:14:01 +01001342 req->creds = get_current_cred();
Pavel Begunkovb8e64b52021-06-17 18:14:02 +01001343 }
Jens Axboe003e8dc2021-03-06 09:22:27 -07001344
Pavel Begunkove1d675d2021-03-22 01:58:29 +00001345 req->work.list.next = NULL;
1346 req->work.flags = 0;
Pavel Begunkovfeaadc42020-10-22 16:47:16 +01001347 if (req->flags & REQ_F_FORCE_ASYNC)
1348 req->work.flags |= IO_WQ_WORK_CONCURRENT;
1349
Jens Axboed3656342019-12-18 09:50:26 -07001350 if (req->flags & REQ_F_ISREG) {
Pavel Begunkov23329512020-10-10 18:34:06 +01001351 if (def->hash_reg_file || (ctx->flags & IORING_SETUP_IOPOLL))
Pavel Begunkov8766dd52020-03-14 00:31:04 +03001352 io_wq_hash_work(&req->work, file_inode(req->file));
Jens Axboe4b982bd2021-04-01 08:38:34 -06001353 } else if (!req->file || !S_ISBLK(file_inode(req->file)->i_mode)) {
Jens Axboed3656342019-12-18 09:50:26 -07001354 if (def->unbound_nonreg_file)
Jens Axboe3529d8c2019-12-19 18:24:38 -07001355 req->work.flags |= IO_WQ_WORK_UNBOUND;
Jens Axboe54a91f32019-09-10 09:15:04 -06001356 }
Pavel Begunkove1d675d2021-03-22 01:58:29 +00001357
1358 switch (req->opcode) {
1359 case IORING_OP_SPLICE:
1360 case IORING_OP_TEE:
Pavel Begunkove1d675d2021-03-22 01:58:29 +00001361 if (!S_ISREG(file_inode(req->splice.file_in)->i_mode))
1362 req->work.flags |= IO_WQ_WORK_UNBOUND;
1363 break;
1364 }
Jens Axboe561fb042019-10-24 07:25:42 -06001365}
1366
Pavel Begunkovcbdcb432020-06-29 19:18:43 +03001367static void io_prep_async_link(struct io_kiocb *req)
1368{
1369 struct io_kiocb *cur;
1370
Pavel Begunkov44eff402021-07-26 14:14:31 +01001371 if (req->flags & REQ_F_LINK_TIMEOUT) {
1372 struct io_ring_ctx *ctx = req->ctx;
1373
Jens Axboe79ebeae2021-08-10 15:18:27 -06001374 spin_lock(&ctx->completion_lock);
Pavel Begunkov44eff402021-07-26 14:14:31 +01001375 io_for_each_link(cur, req)
1376 io_prep_async_work(cur);
Jens Axboe79ebeae2021-08-10 15:18:27 -06001377 spin_unlock(&ctx->completion_lock);
Pavel Begunkov44eff402021-07-26 14:14:31 +01001378 } else {
1379 io_for_each_link(cur, req)
1380 io_prep_async_work(cur);
1381 }
Pavel Begunkovcbdcb432020-06-29 19:18:43 +03001382}
1383
Pavel Begunkovebf93662021-03-01 18:20:47 +00001384static void io_queue_async_work(struct io_kiocb *req)
Jens Axboe561fb042019-10-24 07:25:42 -06001385{
Jackie Liua197f662019-11-08 08:09:12 -07001386 struct io_ring_ctx *ctx = req->ctx;
Pavel Begunkovcbdcb432020-06-29 19:18:43 +03001387 struct io_kiocb *link = io_prep_linked_timeout(req);
Jens Axboe5aa75ed2021-02-16 12:56:50 -07001388 struct io_uring_task *tctx = req->task->io_uring;
Jens Axboe561fb042019-10-24 07:25:42 -06001389
Jens Axboe3bfe6102021-02-16 14:15:30 -07001390 BUG_ON(!tctx);
1391 BUG_ON(!tctx->io_wq);
Jens Axboe561fb042019-10-24 07:25:42 -06001392
Pavel Begunkovcbdcb432020-06-29 19:18:43 +03001393 /* init ->work of the whole link before punting */
1394 io_prep_async_link(req);
Jens Axboe991468d2021-07-23 11:53:54 -06001395
1396 /*
1397 * Not expected to happen, but if we do have a bug where this _can_
1398 * happen, catch it here and ensure the request is marked as
1399 * canceled. That will make io-wq go through the usual work cancel
1400 * procedure rather than attempt to run this request (or create a new
1401 * worker for it).
1402 */
1403 if (WARN_ON_ONCE(!same_thread_group(req->task, current)))
1404 req->work.flags |= IO_WQ_WORK_CANCEL;
1405
Pavel Begunkovd07f1e8a2021-03-22 01:45:58 +00001406 trace_io_uring_queue_async_work(ctx, io_wq_is_hashed(&req->work), req,
1407 &req->work, req->flags);
Pavel Begunkovebf93662021-03-01 18:20:47 +00001408 io_wq_enqueue(tctx->io_wq, &req->work);
Jens Axboe7271ef32020-08-10 09:55:22 -06001409 if (link)
1410 io_queue_linked_timeout(link);
Pavel Begunkovcbdcb432020-06-29 19:18:43 +03001411}
1412
Pavel Begunkov1ee41602021-03-25 18:32:42 +00001413static void io_kill_timeout(struct io_kiocb *req, int status)
Pavel Begunkov8c855882021-04-13 02:58:41 +01001414 __must_hold(&req->ctx->completion_lock)
Jens Axboe89850fc2021-08-10 15:11:51 -06001415 __must_hold(&req->ctx->timeout_lock)
Jens Axboe5262f562019-09-17 12:26:57 -06001416{
Jens Axboee8c2bc12020-08-15 18:44:09 -07001417 struct io_timeout_data *io = req->async_data;
Jens Axboe5262f562019-09-17 12:26:57 -06001418
Pavel Begunkovfd9c7bc2021-04-13 02:58:42 +01001419 if (hrtimer_try_to_cancel(&io->timer) != -1) {
Pavel Begunkov01cec8c2020-07-30 18:43:50 +03001420 atomic_set(&req->ctx->cq_timeouts,
1421 atomic_read(&req->ctx->cq_timeouts) + 1);
Pavel Begunkov135fcde2020-07-13 23:37:12 +03001422 list_del_init(&req->timeout.list);
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01001423 io_cqring_fill_event(req->ctx, req->user_data, status, 0);
Pavel Begunkov91c2f692021-08-11 19:28:28 +01001424 io_put_req_deferred(req);
Jens Axboe5262f562019-09-17 12:26:57 -06001425 }
1426}
1427
Pavel Begunkov441b8a72021-06-14 23:37:31 +01001428static void io_queue_deferred(struct io_ring_ctx *ctx)
Pavel Begunkov04518942020-05-26 20:34:05 +03001429{
Pavel Begunkov441b8a72021-06-14 23:37:31 +01001430 while (!list_empty(&ctx->defer_list)) {
Pavel Begunkov27dc8332020-07-13 23:37:14 +03001431 struct io_defer_entry *de = list_first_entry(&ctx->defer_list,
1432 struct io_defer_entry, list);
Pavel Begunkov04518942020-05-26 20:34:05 +03001433
Pavel Begunkov9cf7c102020-07-13 23:37:15 +03001434 if (req_need_defer(de->req, de->seq))
Pavel Begunkov04518942020-05-26 20:34:05 +03001435 break;
Pavel Begunkov27dc8332020-07-13 23:37:14 +03001436 list_del_init(&de->list);
Pavel Begunkov907d1df2021-01-26 23:35:10 +00001437 io_req_task_queue(de->req);
Pavel Begunkov27dc8332020-07-13 23:37:14 +03001438 kfree(de);
Pavel Begunkov441b8a72021-06-14 23:37:31 +01001439 }
Pavel Begunkov04518942020-05-26 20:34:05 +03001440}
1441
Pavel Begunkov360428f2020-05-30 14:54:17 +03001442static void io_flush_timeouts(struct io_ring_ctx *ctx)
Jens Axboe89850fc2021-08-10 15:11:51 -06001443 __must_hold(&ctx->completion_lock)
Pavel Begunkov360428f2020-05-30 14:54:17 +03001444{
Pavel Begunkov441b8a72021-06-14 23:37:31 +01001445 u32 seq = ctx->cached_cq_tail - atomic_read(&ctx->cq_timeouts);
Marcelo Diop-Gonzalezf0105052021-01-15 11:54:40 -05001446
Jens Axboe79ebeae2021-08-10 15:18:27 -06001447 spin_lock_irq(&ctx->timeout_lock);
Pavel Begunkovf18ee4c2021-06-14 23:37:25 +01001448 while (!list_empty(&ctx->timeout_list)) {
Marcelo Diop-Gonzalezf0105052021-01-15 11:54:40 -05001449 u32 events_needed, events_got;
Pavel Begunkov360428f2020-05-30 14:54:17 +03001450 struct io_kiocb *req = list_first_entry(&ctx->timeout_list,
Pavel Begunkov135fcde2020-07-13 23:37:12 +03001451 struct io_kiocb, timeout.list);
Pavel Begunkov360428f2020-05-30 14:54:17 +03001452
Pavel Begunkov8eb7e2d2020-06-29 13:13:02 +03001453 if (io_is_timeout_noseq(req))
Pavel Begunkov360428f2020-05-30 14:54:17 +03001454 break;
Marcelo Diop-Gonzalezf0105052021-01-15 11:54:40 -05001455
1456 /*
1457 * Since seq can easily wrap around over time, subtract
1458 * the last seq at which timeouts were flushed before comparing.
1459 * Assuming not more than 2^31-1 events have happened since,
1460 * these subtractions won't have wrapped, so we can check if
1461 * target is in [last_seq, current_seq] by comparing the two.
1462 */
1463 events_needed = req->timeout.target_seq - ctx->cq_last_tm_flush;
1464 events_got = seq - ctx->cq_last_tm_flush;
1465 if (events_got < events_needed)
Pavel Begunkov360428f2020-05-30 14:54:17 +03001466 break;
Pavel Begunkovbfe68a22020-05-30 14:54:18 +03001467
Pavel Begunkov135fcde2020-07-13 23:37:12 +03001468 list_del_init(&req->timeout.list);
Pavel Begunkov1ee41602021-03-25 18:32:42 +00001469 io_kill_timeout(req, 0);
Pavel Begunkovf18ee4c2021-06-14 23:37:25 +01001470 }
Marcelo Diop-Gonzalezf0105052021-01-15 11:54:40 -05001471 ctx->cq_last_tm_flush = seq;
Jens Axboe79ebeae2021-08-10 15:18:27 -06001472 spin_unlock_irq(&ctx->timeout_lock);
Pavel Begunkov360428f2020-05-30 14:54:17 +03001473}
1474
Pavel Begunkov2335f6f2021-06-15 16:47:58 +01001475static void __io_commit_cqring_flush(struct io_ring_ctx *ctx)
Jens Axboede0617e2019-04-06 21:51:27 -06001476{
Pavel Begunkov2335f6f2021-06-15 16:47:58 +01001477 if (ctx->off_timeout_used)
1478 io_flush_timeouts(ctx);
1479 if (ctx->drain_active)
1480 io_queue_deferred(ctx);
1481}
1482
1483static inline void io_commit_cqring(struct io_ring_ctx *ctx)
1484{
1485 if (unlikely(ctx->off_timeout_used || ctx->drain_active))
1486 __io_commit_cqring_flush(ctx);
Pavel Begunkovec30e042021-01-19 13:32:38 +00001487 /* order cqe stores with ring update */
1488 smp_store_release(&ctx->rings->cq.tail, ctx->cached_cq_tail);
Jens Axboede0617e2019-04-06 21:51:27 -06001489}
1490
Jens Axboe90554202020-09-03 12:12:41 -06001491static inline bool io_sqring_full(struct io_ring_ctx *ctx)
1492{
1493 struct io_rings *r = ctx->rings;
1494
Pavel Begunkova566c552021-05-16 22:58:08 +01001495 return READ_ONCE(r->sq.tail) - ctx->cached_sq_head == ctx->sq_entries;
Jens Axboe90554202020-09-03 12:12:41 -06001496}
1497
Pavel Begunkov888aae22021-01-19 13:32:39 +00001498static inline unsigned int __io_cqring_events(struct io_ring_ctx *ctx)
1499{
1500 return ctx->cached_cq_tail - READ_ONCE(ctx->rings->cq.head);
1501}
1502
Pavel Begunkovd068b502021-05-16 22:58:11 +01001503static inline struct io_uring_cqe *io_get_cqe(struct io_ring_ctx *ctx)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001504{
Hristo Venev75b28af2019-08-26 17:23:46 +00001505 struct io_rings *rings = ctx->rings;
Pavel Begunkovea5ab3b2021-05-16 22:58:09 +01001506 unsigned tail, mask = ctx->cq_entries - 1;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001507
Stefan Bühler115e12e2019-04-24 23:54:18 +02001508 /*
1509 * writes to the cq entry need to come after reading head; the
1510 * control dependency is enough as we're using WRITE_ONCE to
1511 * fill the cq entry
1512 */
Pavel Begunkova566c552021-05-16 22:58:08 +01001513 if (__io_cqring_events(ctx) == ctx->cq_entries)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001514 return NULL;
1515
Pavel Begunkov888aae22021-01-19 13:32:39 +00001516 tail = ctx->cached_cq_tail++;
Pavel Begunkovea5ab3b2021-05-16 22:58:09 +01001517 return &rings->cqes[tail & mask];
Jens Axboe2b188cc2019-01-07 10:46:33 -07001518}
1519
Jens Axboef2842ab2020-01-08 11:04:00 -07001520static inline bool io_should_trigger_evfd(struct io_ring_ctx *ctx)
1521{
Pavel Begunkov44c769d2021-04-11 01:46:31 +01001522 if (likely(!ctx->cq_ev_fd))
Jens Axboef0b493e2020-02-01 21:30:11 -07001523 return false;
Stefano Garzarella7e55a192020-05-15 18:38:05 +02001524 if (READ_ONCE(ctx->rings->cq_flags) & IORING_CQ_EVENTFD_DISABLED)
1525 return false;
Pavel Begunkov44c769d2021-04-11 01:46:31 +01001526 return !ctx->eventfd_async || io_wq_current_is_worker();
Jens Axboef2842ab2020-01-08 11:04:00 -07001527}
1528
Jens Axboeb41e9852020-02-17 09:52:41 -07001529static void io_cqring_ev_posted(struct io_ring_ctx *ctx)
Jens Axboe8c838782019-03-12 15:48:16 -06001530{
Jens Axboe5fd46172021-08-06 14:04:31 -06001531 /*
1532 * wake_up_all() may seem excessive, but io_wake_function() and
1533 * io_should_wake() handle the termination of the loop and only
1534 * wake as many waiters as we need to.
1535 */
1536 if (wq_has_sleeper(&ctx->cq_wait))
1537 wake_up_all(&ctx->cq_wait);
Jens Axboe534ca6d2020-09-02 13:52:19 -06001538 if (ctx->sq_data && waitqueue_active(&ctx->sq_data->wait))
1539 wake_up(&ctx->sq_data->wait);
Jens Axboeb41e9852020-02-17 09:52:41 -07001540 if (io_should_trigger_evfd(ctx))
Jens Axboe9b402842019-04-11 11:45:41 -06001541 eventfd_signal(ctx->cq_ev_fd, 1);
Pavel Begunkov311997b2021-06-14 23:37:28 +01001542 if (waitqueue_active(&ctx->poll_wait)) {
1543 wake_up_interruptible(&ctx->poll_wait);
Pavel Begunkov4aa84f22021-01-07 03:15:42 +00001544 kill_fasync(&ctx->cq_fasync, SIGIO, POLL_IN);
1545 }
Jens Axboe8c838782019-03-12 15:48:16 -06001546}
1547
Pavel Begunkov80c18e42021-01-07 03:15:41 +00001548static void io_cqring_ev_posted_iopoll(struct io_ring_ctx *ctx)
1549{
1550 if (ctx->flags & IORING_SETUP_SQPOLL) {
Jens Axboe5fd46172021-08-06 14:04:31 -06001551 if (wq_has_sleeper(&ctx->cq_wait))
1552 wake_up_all(&ctx->cq_wait);
Pavel Begunkov80c18e42021-01-07 03:15:41 +00001553 }
1554 if (io_should_trigger_evfd(ctx))
1555 eventfd_signal(ctx->cq_ev_fd, 1);
Pavel Begunkov311997b2021-06-14 23:37:28 +01001556 if (waitqueue_active(&ctx->poll_wait)) {
1557 wake_up_interruptible(&ctx->poll_wait);
Pavel Begunkov4aa84f22021-01-07 03:15:42 +00001558 kill_fasync(&ctx->cq_fasync, SIGIO, POLL_IN);
1559 }
Pavel Begunkov80c18e42021-01-07 03:15:41 +00001560}
1561
Jens Axboec4a2ed72019-11-21 21:01:26 -07001562/* Returns true if there are no backlogged entries after the flush */
Pavel Begunkov6c2450a2021-02-23 12:40:22 +00001563static bool __io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001564{
Jens Axboeb18032b2021-01-24 16:58:56 -07001565 bool all_flushed, posted;
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001566
Pavel Begunkova566c552021-05-16 22:58:08 +01001567 if (!force && __io_cqring_events(ctx) == ctx->cq_entries)
Pavel Begunkove23de152020-12-17 00:24:37 +00001568 return false;
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001569
Jens Axboeb18032b2021-01-24 16:58:56 -07001570 posted = false;
Jens Axboe79ebeae2021-08-10 15:18:27 -06001571 spin_lock(&ctx->completion_lock);
Pavel Begunkov6c2450a2021-02-23 12:40:22 +00001572 while (!list_empty(&ctx->cq_overflow_list)) {
Pavel Begunkovd068b502021-05-16 22:58:11 +01001573 struct io_uring_cqe *cqe = io_get_cqe(ctx);
Pavel Begunkov6c2450a2021-02-23 12:40:22 +00001574 struct io_overflow_cqe *ocqe;
Jens Axboee6c8aa92020-09-28 13:10:13 -06001575
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001576 if (!cqe && !force)
1577 break;
Pavel Begunkov6c2450a2021-02-23 12:40:22 +00001578 ocqe = list_first_entry(&ctx->cq_overflow_list,
1579 struct io_overflow_cqe, list);
1580 if (cqe)
1581 memcpy(cqe, &ocqe->cqe, sizeof(*cqe));
1582 else
Pavel Begunkov8f6ed492021-05-16 22:58:10 +01001583 io_account_cq_overflow(ctx);
1584
Jens Axboeb18032b2021-01-24 16:58:56 -07001585 posted = true;
Pavel Begunkov6c2450a2021-02-23 12:40:22 +00001586 list_del(&ocqe->list);
1587 kfree(ocqe);
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001588 }
1589
Pavel Begunkov09e88402020-12-17 00:24:38 +00001590 all_flushed = list_empty(&ctx->cq_overflow_list);
1591 if (all_flushed) {
Pavel Begunkov5ed7a372021-06-14 23:37:27 +01001592 clear_bit(0, &ctx->check_cq_overflow);
Nadav Amit20c0b382021-08-07 17:13:42 -07001593 WRITE_ONCE(ctx->rings->sq_flags,
1594 ctx->rings->sq_flags & ~IORING_SQ_CQ_OVERFLOW);
Pavel Begunkov09e88402020-12-17 00:24:38 +00001595 }
Pavel Begunkov46930142020-07-30 18:43:49 +03001596
Jens Axboeb18032b2021-01-24 16:58:56 -07001597 if (posted)
1598 io_commit_cqring(ctx);
Jens Axboe79ebeae2021-08-10 15:18:27 -06001599 spin_unlock(&ctx->completion_lock);
Jens Axboeb18032b2021-01-24 16:58:56 -07001600 if (posted)
1601 io_cqring_ev_posted(ctx);
Pavel Begunkov09e88402020-12-17 00:24:38 +00001602 return all_flushed;
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001603}
1604
Pavel Begunkov90f67362021-08-09 20:18:12 +01001605static bool io_cqring_overflow_flush(struct io_ring_ctx *ctx)
Pavel Begunkov6c503152021-01-04 20:36:36 +00001606{
Jens Axboeca0a2652021-03-04 17:15:48 -07001607 bool ret = true;
1608
Pavel Begunkov5ed7a372021-06-14 23:37:27 +01001609 if (test_bit(0, &ctx->check_cq_overflow)) {
Pavel Begunkov6c503152021-01-04 20:36:36 +00001610 /* iopoll syncs against uring_lock, not completion_lock */
1611 if (ctx->flags & IORING_SETUP_IOPOLL)
1612 mutex_lock(&ctx->uring_lock);
Pavel Begunkov90f67362021-08-09 20:18:12 +01001613 ret = __io_cqring_overflow_flush(ctx, false);
Pavel Begunkov6c503152021-01-04 20:36:36 +00001614 if (ctx->flags & IORING_SETUP_IOPOLL)
1615 mutex_unlock(&ctx->uring_lock);
1616 }
Jens Axboeca0a2652021-03-04 17:15:48 -07001617
1618 return ret;
Pavel Begunkov6c503152021-01-04 20:36:36 +00001619}
1620
Pavel Begunkov6a290a12021-08-09 13:04:13 +01001621/* must to be called somewhat shortly after putting a request */
1622static inline void io_put_task(struct task_struct *task, int nr)
1623{
1624 struct io_uring_task *tctx = task->io_uring;
1625
1626 percpu_counter_sub(&tctx->inflight, nr);
1627 if (unlikely(atomic_read(&tctx->in_idle)))
1628 wake_up(&tctx->wait);
1629 put_task_struct_many(task, nr);
1630}
1631
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01001632static bool io_cqring_event_overflow(struct io_ring_ctx *ctx, u64 user_data,
1633 long res, unsigned int cflags)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001634{
Pavel Begunkovcce4b8b2021-04-13 02:58:44 +01001635 struct io_overflow_cqe *ocqe;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001636
Pavel Begunkovcce4b8b2021-04-13 02:58:44 +01001637 ocqe = kmalloc(sizeof(*ocqe), GFP_ATOMIC | __GFP_ACCOUNT);
1638 if (!ocqe) {
1639 /*
1640 * If we're in ring overflow flush mode, or in task cancel mode,
1641 * or cannot allocate an overflow entry, then we need to drop it
1642 * on the floor.
1643 */
Pavel Begunkov8f6ed492021-05-16 22:58:10 +01001644 io_account_cq_overflow(ctx);
Pavel Begunkovcce4b8b2021-04-13 02:58:44 +01001645 return false;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001646 }
Pavel Begunkovcce4b8b2021-04-13 02:58:44 +01001647 if (list_empty(&ctx->cq_overflow_list)) {
Pavel Begunkov5ed7a372021-06-14 23:37:27 +01001648 set_bit(0, &ctx->check_cq_overflow);
Nadav Amit20c0b382021-08-07 17:13:42 -07001649 WRITE_ONCE(ctx->rings->sq_flags,
1650 ctx->rings->sq_flags | IORING_SQ_CQ_OVERFLOW);
1651
Pavel Begunkovcce4b8b2021-04-13 02:58:44 +01001652 }
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01001653 ocqe->cqe.user_data = user_data;
Pavel Begunkovcce4b8b2021-04-13 02:58:44 +01001654 ocqe->cqe.res = res;
1655 ocqe->cqe.flags = cflags;
1656 list_add_tail(&ocqe->list, &ctx->cq_overflow_list);
1657 return true;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001658}
1659
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01001660static inline bool __io_cqring_fill_event(struct io_ring_ctx *ctx, u64 user_data,
1661 long res, unsigned int cflags)
Pavel Begunkov8d133262021-04-11 01:46:33 +01001662{
Jens Axboe2b188cc2019-01-07 10:46:33 -07001663 struct io_uring_cqe *cqe;
1664
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01001665 trace_io_uring_complete(ctx, user_data, res, cflags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001666
1667 /*
1668 * If we can't get a cq entry, userspace overflowed the
1669 * submission (by quite a lot). Increment the overflow count in
1670 * the ring.
1671 */
Pavel Begunkovd068b502021-05-16 22:58:11 +01001672 cqe = io_get_cqe(ctx);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001673 if (likely(cqe)) {
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01001674 WRITE_ONCE(cqe->user_data, user_data);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001675 WRITE_ONCE(cqe->res, res);
1676 WRITE_ONCE(cqe->flags, cflags);
Pavel Begunkov8d133262021-04-11 01:46:33 +01001677 return true;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001678 }
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01001679 return io_cqring_event_overflow(ctx, user_data, res, cflags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001680}
1681
Pavel Begunkov8d133262021-04-11 01:46:33 +01001682/* not as hot to bloat with inlining */
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01001683static noinline bool io_cqring_fill_event(struct io_ring_ctx *ctx, u64 user_data,
1684 long res, unsigned int cflags)
Jens Axboebcda7ba2020-02-23 16:42:51 -07001685{
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01001686 return __io_cqring_fill_event(ctx, user_data, res, cflags);
Jens Axboebcda7ba2020-02-23 16:42:51 -07001687}
1688
Pavel Begunkov7a612352021-03-09 00:37:59 +00001689static void io_req_complete_post(struct io_kiocb *req, long res,
1690 unsigned int cflags)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001691{
Jens Axboe78e19bb2019-11-06 15:21:34 -07001692 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001693
Jens Axboe79ebeae2021-08-10 15:18:27 -06001694 spin_lock(&ctx->completion_lock);
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01001695 __io_cqring_fill_event(ctx, req->user_data, res, cflags);
Jens Axboec7dae4b2021-02-09 19:53:37 -07001696 /*
1697 * If we're the last reference to this request, add to our locked
1698 * free_list cache.
1699 */
Jens Axboede9b4cc2021-02-24 13:28:27 -07001700 if (req_ref_put_and_test(req)) {
Pavel Begunkov7a612352021-03-09 00:37:59 +00001701 if (req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) {
Pavel Begunkov0756a862021-08-15 10:40:25 +01001702 if (req->flags & IO_DISARM_MASK)
Pavel Begunkov7a612352021-03-09 00:37:59 +00001703 io_disarm_next(req);
1704 if (req->link) {
1705 io_req_task_queue(req->link);
1706 req->link = NULL;
1707 }
1708 }
Jens Axboec7dae4b2021-02-09 19:53:37 -07001709 io_dismantle_req(req);
1710 io_put_task(req->task, 1);
Pavel Begunkovbb943b82021-08-09 20:18:10 +01001711 list_add(&req->inflight_entry, &ctx->locked_free_list);
Pavel Begunkovd0acdee2021-05-16 22:58:12 +01001712 ctx->locked_free_nr++;
Pavel Begunkov180f8292021-03-14 20:57:09 +00001713 } else {
1714 if (!percpu_ref_tryget(&ctx->refs))
1715 req = NULL;
1716 }
Pavel Begunkov7a612352021-03-09 00:37:59 +00001717 io_commit_cqring(ctx);
Jens Axboe79ebeae2021-08-10 15:18:27 -06001718 spin_unlock(&ctx->completion_lock);
Pavel Begunkov7a612352021-03-09 00:37:59 +00001719
Pavel Begunkov180f8292021-03-14 20:57:09 +00001720 if (req) {
1721 io_cqring_ev_posted(ctx);
Jens Axboec7dae4b2021-02-09 19:53:37 -07001722 percpu_ref_put(&ctx->refs);
Pavel Begunkov180f8292021-03-14 20:57:09 +00001723 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07001724}
1725
Jens Axboe4e3d9ff2021-04-15 17:44:34 -06001726static inline bool io_req_needs_clean(struct io_kiocb *req)
1727{
Pavel Begunkovc8543572021-06-17 18:14:04 +01001728 return req->flags & IO_REQ_CLEAN_FLAGS;
Jens Axboe4e3d9ff2021-04-15 17:44:34 -06001729}
1730
Pavel Begunkova38d68d2021-01-19 13:32:45 +00001731static void io_req_complete_state(struct io_kiocb *req, long res,
Pavel Begunkov889fca72021-02-10 00:03:09 +00001732 unsigned int cflags)
Jens Axboebcda7ba2020-02-23 16:42:51 -07001733{
Jens Axboe4e3d9ff2021-04-15 17:44:34 -06001734 if (io_req_needs_clean(req))
Pavel Begunkov68fb8972021-03-19 17:22:41 +00001735 io_clean_op(req);
Pavel Begunkova38d68d2021-01-19 13:32:45 +00001736 req->result = res;
1737 req->compl.cflags = cflags;
Pavel Begunkove342c802021-01-19 13:32:47 +00001738 req->flags |= REQ_F_COMPLETE_INLINE;
Jens Axboee1e16092020-06-22 09:17:17 -06001739}
Jens Axboe2b188cc2019-01-07 10:46:33 -07001740
Pavel Begunkov889fca72021-02-10 00:03:09 +00001741static inline void __io_req_complete(struct io_kiocb *req, unsigned issue_flags,
1742 long res, unsigned cflags)
Pavel Begunkova38d68d2021-01-19 13:32:45 +00001743{
Pavel Begunkov889fca72021-02-10 00:03:09 +00001744 if (issue_flags & IO_URING_F_COMPLETE_DEFER)
1745 io_req_complete_state(req, res, cflags);
Pavel Begunkova38d68d2021-01-19 13:32:45 +00001746 else
Jens Axboec7dae4b2021-02-09 19:53:37 -07001747 io_req_complete_post(req, res, cflags);
Pavel Begunkova38d68d2021-01-19 13:32:45 +00001748}
Jens Axboebcda7ba2020-02-23 16:42:51 -07001749
Pavel Begunkova38d68d2021-01-19 13:32:45 +00001750static inline void io_req_complete(struct io_kiocb *req, long res)
Jens Axboee1e16092020-06-22 09:17:17 -06001751{
Pavel Begunkov889fca72021-02-10 00:03:09 +00001752 __io_req_complete(req, 0, res, 0);
Jens Axboebcda7ba2020-02-23 16:42:51 -07001753}
1754
Pavel Begunkovf41db2732021-02-28 22:35:12 +00001755static void io_req_complete_failed(struct io_kiocb *req, long res)
1756{
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01001757 req_set_fail(req);
Pavel Begunkovf41db2732021-02-28 22:35:12 +00001758 io_req_complete_post(req, res, 0);
1759}
1760
Pavel Begunkov864ea922021-08-09 13:04:08 +01001761/*
1762 * Don't initialise the fields below on every allocation, but do that in
1763 * advance and keep them valid across allocations.
1764 */
1765static void io_preinit_req(struct io_kiocb *req, struct io_ring_ctx *ctx)
1766{
1767 req->ctx = ctx;
1768 req->link = NULL;
1769 req->async_data = NULL;
1770 /* not necessary, but safer to zero */
1771 req->result = 0;
1772}
1773
Pavel Begunkovdac7a092021-03-19 17:22:39 +00001774static void io_flush_cached_locked_reqs(struct io_ring_ctx *ctx,
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01001775 struct io_submit_state *state)
Pavel Begunkovdac7a092021-03-19 17:22:39 +00001776{
Jens Axboe79ebeae2021-08-10 15:18:27 -06001777 spin_lock(&ctx->completion_lock);
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01001778 list_splice_init(&ctx->locked_free_list, &state->free_list);
Pavel Begunkovd0acdee2021-05-16 22:58:12 +01001779 ctx->locked_free_nr = 0;
Jens Axboe79ebeae2021-08-10 15:18:27 -06001780 spin_unlock(&ctx->completion_lock);
Pavel Begunkovdac7a092021-03-19 17:22:39 +00001781}
1782
Pavel Begunkovdd78f492021-03-19 17:22:35 +00001783/* Returns true IFF there are requests in the cache */
Jens Axboec7dae4b2021-02-09 19:53:37 -07001784static bool io_flush_cached_reqs(struct io_ring_ctx *ctx)
Pavel Begunkove5d1bc02021-02-10 00:03:23 +00001785{
Jens Axboec7dae4b2021-02-09 19:53:37 -07001786 struct io_submit_state *state = &ctx->submit_state;
Pavel Begunkovdd78f492021-03-19 17:22:35 +00001787 int nr;
Pavel Begunkove5d1bc02021-02-10 00:03:23 +00001788
Jens Axboec7dae4b2021-02-09 19:53:37 -07001789 /*
1790 * If we have more than a batch's worth of requests in our IRQ side
1791 * locked cache, grab the lock and move them over to our submission
1792 * side cache.
1793 */
Pavel Begunkovd0acdee2021-05-16 22:58:12 +01001794 if (READ_ONCE(ctx->locked_free_nr) > IO_COMPL_BATCH)
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01001795 io_flush_cached_locked_reqs(ctx, state);
Jens Axboec7dae4b2021-02-09 19:53:37 -07001796
Pavel Begunkovdd78f492021-03-19 17:22:35 +00001797 nr = state->free_reqs;
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01001798 while (!list_empty(&state->free_list)) {
1799 struct io_kiocb *req = list_first_entry(&state->free_list,
Pavel Begunkovbb943b82021-08-09 20:18:10 +01001800 struct io_kiocb, inflight_entry);
Pavel Begunkovdd78f492021-03-19 17:22:35 +00001801
Pavel Begunkovbb943b82021-08-09 20:18:10 +01001802 list_del(&req->inflight_entry);
Pavel Begunkovdd78f492021-03-19 17:22:35 +00001803 state->reqs[nr++] = req;
1804 if (nr == ARRAY_SIZE(state->reqs))
Pavel Begunkove5d1bc02021-02-10 00:03:23 +00001805 break;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001806 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07001807
Pavel Begunkovdd78f492021-03-19 17:22:35 +00001808 state->free_reqs = nr;
1809 return nr != 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001810}
1811
Pavel Begunkov5d5901a2021-08-11 19:28:29 +01001812/*
1813 * A request might get retired back into the request caches even before opcode
1814 * handlers and io_issue_sqe() are done with it, e.g. inline completion path.
1815 * Because of that, io_alloc_req() should be called only under ->uring_lock
1816 * and with extra caution to not get a request that is still worked on.
1817 */
Pavel Begunkov258b29a2021-02-10 00:03:10 +00001818static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx)
Pavel Begunkov5d5901a2021-08-11 19:28:29 +01001819 __must_hold(&ctx->uring_lock)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001820{
Pavel Begunkov258b29a2021-02-10 00:03:10 +00001821 struct io_submit_state *state = &ctx->submit_state;
Pavel Begunkov864ea922021-08-09 13:04:08 +01001822 gfp_t gfp = GFP_KERNEL | __GFP_NOWARN;
1823 int ret, i;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001824
Pavel Begunkovfe7e3252021-06-24 15:09:57 +01001825 BUILD_BUG_ON(ARRAY_SIZE(state->reqs) < IO_REQ_ALLOC_BATCH);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001826
Pavel Begunkov864ea922021-08-09 13:04:08 +01001827 if (likely(state->free_reqs || io_flush_cached_reqs(ctx)))
1828 goto got_req;
Jens Axboe2579f912019-01-09 09:10:43 -07001829
Pavel Begunkov864ea922021-08-09 13:04:08 +01001830 ret = kmem_cache_alloc_bulk(req_cachep, gfp, IO_REQ_ALLOC_BATCH,
1831 state->reqs);
Pavel Begunkove5d1bc02021-02-10 00:03:23 +00001832
Pavel Begunkov864ea922021-08-09 13:04:08 +01001833 /*
1834 * Bulk alloc is all-or-nothing. If we fail to get a batch,
1835 * retry single alloc to be on the safe side.
1836 */
1837 if (unlikely(ret <= 0)) {
1838 state->reqs[0] = kmem_cache_alloc(req_cachep, gfp);
1839 if (!state->reqs[0])
1840 return NULL;
1841 ret = 1;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001842 }
Pavel Begunkov864ea922021-08-09 13:04:08 +01001843
1844 for (i = 0; i < ret; i++)
1845 io_preinit_req(state->reqs[i], ctx);
1846 state->free_reqs = ret;
Pavel Begunkove5d1bc02021-02-10 00:03:23 +00001847got_req:
Pavel Begunkov291b2822020-09-30 22:57:01 +03001848 state->free_reqs--;
1849 return state->reqs[state->free_reqs];
Jens Axboe2b188cc2019-01-07 10:46:33 -07001850}
1851
Pavel Begunkove1d767f2021-03-19 17:22:43 +00001852static inline void io_put_file(struct file *file)
Pavel Begunkov8da11c12020-02-24 11:32:44 +03001853{
Pavel Begunkove1d767f2021-03-19 17:22:43 +00001854 if (file)
Pavel Begunkov8da11c12020-02-24 11:32:44 +03001855 fput(file);
1856}
1857
Pavel Begunkov4edf20f2020-10-13 09:43:59 +01001858static void io_dismantle_req(struct io_kiocb *req)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001859{
Pavel Begunkov094bae42021-03-19 17:22:42 +00001860 unsigned int flags = req->flags;
Pavel Begunkov929a3af2020-02-19 00:19:09 +03001861
Pavel Begunkov3a0a6902021-04-20 12:03:31 +01001862 if (io_req_needs_clean(req))
1863 io_clean_op(req);
Pavel Begunkove1d767f2021-03-19 17:22:43 +00001864 if (!(flags & REQ_F_FIXED_FILE))
1865 io_put_file(req->file);
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00001866 if (req->fixed_rsrc_refs)
1867 percpu_ref_put(req->fixed_rsrc_refs);
Pavel Begunkov99ebe4e2021-06-26 21:40:49 +01001868 if (req->async_data) {
Pavel Begunkov094bae42021-03-19 17:22:42 +00001869 kfree(req->async_data);
Pavel Begunkov99ebe4e2021-06-26 21:40:49 +01001870 req->async_data = NULL;
1871 }
Pavel Begunkove6543a82020-06-28 12:52:30 +03001872}
Pavel Begunkov2b85edf2019-12-28 14:13:03 +03001873
Pavel Begunkov216578e2020-10-13 09:44:00 +01001874static void __io_free_req(struct io_kiocb *req)
Pavel Begunkove6543a82020-06-28 12:52:30 +03001875{
Jens Axboe51a4cc12020-08-10 10:55:56 -06001876 struct io_ring_ctx *ctx = req->ctx;
Pavel Begunkovecfc5172020-06-29 13:13:03 +03001877
Pavel Begunkov216578e2020-10-13 09:44:00 +01001878 io_dismantle_req(req);
Pavel Begunkov7c660732021-01-25 11:42:21 +00001879 io_put_task(req->task, 1);
Pavel Begunkove6543a82020-06-28 12:52:30 +03001880
Jens Axboe79ebeae2021-08-10 15:18:27 -06001881 spin_lock(&ctx->completion_lock);
Pavel Begunkovbb943b82021-08-09 20:18:10 +01001882 list_add(&req->inflight_entry, &ctx->locked_free_list);
Pavel Begunkovc34b0252021-08-09 20:18:08 +01001883 ctx->locked_free_nr++;
Jens Axboe79ebeae2021-08-10 15:18:27 -06001884 spin_unlock(&ctx->completion_lock);
Pavel Begunkovc34b0252021-08-09 20:18:08 +01001885
Pavel Begunkovecfc5172020-06-29 13:13:03 +03001886 percpu_ref_put(&ctx->refs);
Jens Axboee65ef562019-03-12 10:16:44 -06001887}
1888
Pavel Begunkovf2f87372020-10-27 23:25:37 +00001889static inline void io_remove_next_linked(struct io_kiocb *req)
1890{
1891 struct io_kiocb *nxt = req->link;
1892
1893 req->link = nxt->link;
1894 nxt->link = NULL;
1895}
1896
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00001897static bool io_kill_linked_timeout(struct io_kiocb *req)
1898 __must_hold(&req->ctx->completion_lock)
Jens Axboe89b263f2021-08-10 15:14:18 -06001899 __must_hold(&req->ctx->timeout_lock)
Jens Axboe9e645e112019-05-10 16:07:28 -06001900{
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00001901 struct io_kiocb *link = req->link;
Pavel Begunkovf2f87372020-10-27 23:25:37 +00001902
Pavel Begunkovb97e7362021-08-15 10:40:23 +01001903 if (link && link->opcode == IORING_OP_LINK_TIMEOUT) {
Pavel Begunkovc9abd7a2020-10-22 16:43:11 +01001904 struct io_timeout_data *io = link->async_data;
Pavel Begunkov7c86ffe2020-06-29 13:12:59 +03001905
Pavel Begunkovf2f87372020-10-27 23:25:37 +00001906 io_remove_next_linked(req);
Pavel Begunkov90cd7e42020-10-27 23:25:36 +00001907 link->timeout.head = NULL;
Pavel Begunkovfd9c7bc2021-04-13 02:58:42 +01001908 if (hrtimer_try_to_cancel(&io->timer) != -1) {
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01001909 io_cqring_fill_event(link->ctx, link->user_data,
1910 -ECANCELED, 0);
Pavel Begunkov91c2f692021-08-11 19:28:28 +01001911 io_put_req_deferred(link);
Pavel Begunkovd4729fb2021-03-22 01:58:24 +00001912 return true;
Pavel Begunkovc9abd7a2020-10-22 16:43:11 +01001913 }
1914 }
Pavel Begunkovd4729fb2021-03-22 01:58:24 +00001915 return false;
Pavel Begunkov7c86ffe2020-06-29 13:12:59 +03001916}
1917
Pavel Begunkovd148ca42020-10-18 10:17:39 +01001918static void io_fail_links(struct io_kiocb *req)
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00001919 __must_hold(&req->ctx->completion_lock)
Jens Axboe9e645e112019-05-10 16:07:28 -06001920{
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00001921 struct io_kiocb *nxt, *link = req->link;
Jens Axboe9e645e112019-05-10 16:07:28 -06001922
Pavel Begunkovf2f87372020-10-27 23:25:37 +00001923 req->link = NULL;
Pavel Begunkovf2f87372020-10-27 23:25:37 +00001924 while (link) {
1925 nxt = link->link;
1926 link->link = NULL;
1927
Dmitrii Dolgovc826bd72019-10-15 19:02:01 +02001928 trace_io_uring_fail_link(req, link);
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01001929 io_cqring_fill_event(link->ctx, link->user_data, -ECANCELED, 0);
Pavel Begunkov91c2f692021-08-11 19:28:28 +01001930 io_put_req_deferred(link);
Pavel Begunkovf2f87372020-10-27 23:25:37 +00001931 link = nxt;
Jens Axboe9e645e112019-05-10 16:07:28 -06001932 }
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00001933}
Pavel Begunkov7c86ffe2020-06-29 13:12:59 +03001934
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00001935static bool io_disarm_next(struct io_kiocb *req)
1936 __must_hold(&req->ctx->completion_lock)
1937{
1938 bool posted = false;
1939
Pavel Begunkov0756a862021-08-15 10:40:25 +01001940 if (req->flags & REQ_F_ARM_LTIMEOUT) {
1941 struct io_kiocb *link = req->link;
1942
Pavel Begunkov906c6ca2021-08-15 10:40:26 +01001943 req->flags &= ~REQ_F_ARM_LTIMEOUT;
Pavel Begunkov0756a862021-08-15 10:40:25 +01001944 if (link && link->opcode == IORING_OP_LINK_TIMEOUT) {
1945 io_remove_next_linked(req);
1946 io_cqring_fill_event(link->ctx, link->user_data,
1947 -ECANCELED, 0);
1948 io_put_req_deferred(link);
1949 posted = true;
1950 }
1951 } else if (req->flags & REQ_F_LINK_TIMEOUT) {
Jens Axboe89b263f2021-08-10 15:14:18 -06001952 struct io_ring_ctx *ctx = req->ctx;
1953
1954 spin_lock_irq(&ctx->timeout_lock);
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00001955 posted = io_kill_linked_timeout(req);
Jens Axboe89b263f2021-08-10 15:14:18 -06001956 spin_unlock_irq(&ctx->timeout_lock);
1957 }
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01001958 if (unlikely((req->flags & REQ_F_FAIL) &&
Pavel Begunkove4335ed2021-04-11 01:46:39 +01001959 !(req->flags & REQ_F_HARDLINK))) {
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00001960 posted |= (req->link != NULL);
1961 io_fail_links(req);
1962 }
1963 return posted;
Jens Axboe9e645e112019-05-10 16:07:28 -06001964}
1965
Pavel Begunkov3fa5e0f2020-06-30 15:20:43 +03001966static struct io_kiocb *__io_req_find_next(struct io_kiocb *req)
Jens Axboe9e645e112019-05-10 16:07:28 -06001967{
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00001968 struct io_kiocb *nxt;
Jens Axboe2665abf2019-11-05 12:40:47 -07001969
Jens Axboe9e645e112019-05-10 16:07:28 -06001970 /*
1971 * If LINK is set, we have dependent requests in this chain. If we
1972 * didn't fail this request, queue the first one up, moving any other
1973 * dependencies to the next request. In case of failure, fail the rest
1974 * of the chain.
1975 */
Pavel Begunkov0756a862021-08-15 10:40:25 +01001976 if (req->flags & IO_DISARM_MASK) {
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00001977 struct io_ring_ctx *ctx = req->ctx;
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00001978 bool posted;
Pavel Begunkovf2f87372020-10-27 23:25:37 +00001979
Jens Axboe79ebeae2021-08-10 15:18:27 -06001980 spin_lock(&ctx->completion_lock);
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00001981 posted = io_disarm_next(req);
1982 if (posted)
1983 io_commit_cqring(req->ctx);
Jens Axboe79ebeae2021-08-10 15:18:27 -06001984 spin_unlock(&ctx->completion_lock);
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00001985 if (posted)
1986 io_cqring_ev_posted(ctx);
Pavel Begunkovf2f87372020-10-27 23:25:37 +00001987 }
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00001988 nxt = req->link;
1989 req->link = NULL;
1990 return nxt;
Jens Axboe4d7dd462019-11-20 13:03:52 -07001991}
Jens Axboe2665abf2019-11-05 12:40:47 -07001992
Pavel Begunkovf2f87372020-10-27 23:25:37 +00001993static inline struct io_kiocb *io_req_find_next(struct io_kiocb *req)
Pavel Begunkov3fa5e0f2020-06-30 15:20:43 +03001994{
Pavel Begunkovcdbff982021-02-12 18:41:16 +00001995 if (likely(!(req->flags & (REQ_F_LINK|REQ_F_HARDLINK))))
Pavel Begunkov3fa5e0f2020-06-30 15:20:43 +03001996 return NULL;
1997 return __io_req_find_next(req);
1998}
1999
Pavel Begunkov2c323952021-02-28 22:04:53 +00002000static void ctx_flush_and_put(struct io_ring_ctx *ctx)
2001{
2002 if (!ctx)
2003 return;
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01002004 if (ctx->submit_state.compl_nr) {
Pavel Begunkov2c323952021-02-28 22:04:53 +00002005 mutex_lock(&ctx->uring_lock);
Pavel Begunkov2a2758f2021-06-17 18:14:00 +01002006 io_submit_flush_completions(ctx);
Pavel Begunkov2c323952021-02-28 22:04:53 +00002007 mutex_unlock(&ctx->uring_lock);
2008 }
2009 percpu_ref_put(&ctx->refs);
2010}
2011
Jens Axboe7cbf1722021-02-10 00:03:20 +00002012static void tctx_task_work(struct callback_head *cb)
2013{
Pavel Begunkovebd0df22021-06-17 18:14:07 +01002014 struct io_ring_ctx *ctx = NULL;
Pavel Begunkov3f184072021-06-17 18:14:06 +01002015 struct io_uring_task *tctx = container_of(cb, struct io_uring_task,
2016 task_work);
Jens Axboe7cbf1722021-02-10 00:03:20 +00002017
Pavel Begunkov16f72072021-06-17 18:14:09 +01002018 while (1) {
Pavel Begunkov3f184072021-06-17 18:14:06 +01002019 struct io_wq_work_node *node;
2020
2021 spin_lock_irq(&tctx->task_lock);
Pavel Begunkovc6538be2021-06-17 18:14:08 +01002022 node = tctx->task_list.first;
Pavel Begunkov3f184072021-06-17 18:14:06 +01002023 INIT_WQ_LIST(&tctx->task_list);
Pavel Begunkov6294f362021-08-10 17:53:55 +01002024 if (!node)
2025 tctx->task_running = false;
Pavel Begunkov3f184072021-06-17 18:14:06 +01002026 spin_unlock_irq(&tctx->task_lock);
Pavel Begunkov6294f362021-08-10 17:53:55 +01002027 if (!node)
2028 break;
Pavel Begunkov3f184072021-06-17 18:14:06 +01002029
Pavel Begunkov6294f362021-08-10 17:53:55 +01002030 do {
Pavel Begunkov3f184072021-06-17 18:14:06 +01002031 struct io_wq_work_node *next = node->next;
2032 struct io_kiocb *req = container_of(node, struct io_kiocb,
2033 io_task_work.node);
2034
2035 if (req->ctx != ctx) {
2036 ctx_flush_and_put(ctx);
2037 ctx = req->ctx;
2038 percpu_ref_get(&ctx->refs);
2039 }
Pavel Begunkov5b0a6ac2021-06-30 21:54:04 +01002040 req->io_task_work.func(req);
Pavel Begunkov3f184072021-06-17 18:14:06 +01002041 node = next;
Pavel Begunkov6294f362021-08-10 17:53:55 +01002042 } while (node);
2043
Jens Axboe7cbf1722021-02-10 00:03:20 +00002044 cond_resched();
Pavel Begunkov3f184072021-06-17 18:14:06 +01002045 }
Pavel Begunkovebd0df22021-06-17 18:14:07 +01002046
2047 ctx_flush_and_put(ctx);
Jens Axboe7cbf1722021-02-10 00:03:20 +00002048}
2049
Pavel Begunkove09ee512021-07-01 13:26:05 +01002050static void io_req_task_work_add(struct io_kiocb *req)
Jens Axboe7cbf1722021-02-10 00:03:20 +00002051{
Pavel Begunkovc15b79d2021-03-19 17:22:44 +00002052 struct task_struct *tsk = req->task;
Jens Axboe7cbf1722021-02-10 00:03:20 +00002053 struct io_uring_task *tctx = tsk->io_uring;
Pavel Begunkovc15b79d2021-03-19 17:22:44 +00002054 enum task_work_notify_mode notify;
Pavel Begunkove09ee512021-07-01 13:26:05 +01002055 struct io_wq_work_node *node;
Jens Axboe0b81e802021-02-16 10:33:53 -07002056 unsigned long flags;
Pavel Begunkov6294f362021-08-10 17:53:55 +01002057 bool running;
Jens Axboe7cbf1722021-02-10 00:03:20 +00002058
2059 WARN_ON_ONCE(!tctx);
2060
Jens Axboe0b81e802021-02-16 10:33:53 -07002061 spin_lock_irqsave(&tctx->task_lock, flags);
Jens Axboe7cbf1722021-02-10 00:03:20 +00002062 wq_list_add_tail(&req->io_task_work.node, &tctx->task_list);
Pavel Begunkov6294f362021-08-10 17:53:55 +01002063 running = tctx->task_running;
2064 if (!running)
2065 tctx->task_running = true;
Jens Axboe0b81e802021-02-16 10:33:53 -07002066 spin_unlock_irqrestore(&tctx->task_lock, flags);
Jens Axboe7cbf1722021-02-10 00:03:20 +00002067
2068 /* task_work already pending, we're done */
Pavel Begunkov6294f362021-08-10 17:53:55 +01002069 if (running)
Pavel Begunkove09ee512021-07-01 13:26:05 +01002070 return;
Jens Axboe7cbf1722021-02-10 00:03:20 +00002071
Pavel Begunkovc15b79d2021-03-19 17:22:44 +00002072 /*
2073 * SQPOLL kernel thread doesn't need notification, just a wakeup. For
2074 * all other cases, use TWA_SIGNAL unconditionally to ensure we're
2075 * processing task_work. There's no reliable way to tell if TWA_RESUME
2076 * will do the job.
2077 */
2078 notify = (req->ctx->flags & IORING_SETUP_SQPOLL) ? TWA_NONE : TWA_SIGNAL;
Pavel Begunkovc15b79d2021-03-19 17:22:44 +00002079 if (!task_work_add(tsk, &tctx->task_work, notify)) {
2080 wake_up_process(tsk);
Pavel Begunkove09ee512021-07-01 13:26:05 +01002081 return;
Pavel Begunkovc15b79d2021-03-19 17:22:44 +00002082 }
Pavel Begunkov2215bed2021-08-09 13:04:06 +01002083
Pavel Begunkove09ee512021-07-01 13:26:05 +01002084 spin_lock_irqsave(&tctx->task_lock, flags);
Pavel Begunkov6294f362021-08-10 17:53:55 +01002085 tctx->task_running = false;
Pavel Begunkove09ee512021-07-01 13:26:05 +01002086 node = tctx->task_list.first;
2087 INIT_WQ_LIST(&tctx->task_list);
2088 spin_unlock_irqrestore(&tctx->task_lock, flags);
Jens Axboe7cbf1722021-02-10 00:03:20 +00002089
Pavel Begunkove09ee512021-07-01 13:26:05 +01002090 while (node) {
2091 req = container_of(node, struct io_kiocb, io_task_work.node);
2092 node = node->next;
2093 if (llist_add(&req->io_task_work.fallback_node,
2094 &req->ctx->fallback_llist))
2095 schedule_delayed_work(&req->ctx->fallback_work, 1);
2096 }
Pavel Begunkoveab30c42021-01-19 13:32:42 +00002097}
2098
Pavel Begunkov5b0a6ac2021-06-30 21:54:04 +01002099static void io_req_task_cancel(struct io_kiocb *req)
Jens Axboec40f6372020-06-25 15:39:59 -06002100{
Jens Axboe87ceb6a2020-09-14 08:20:12 -06002101 struct io_ring_ctx *ctx = req->ctx;
Jens Axboec40f6372020-06-25 15:39:59 -06002102
Pavel Begunkove83acd72021-02-28 22:35:09 +00002103 /* ctx is guaranteed to stay alive while we hold uring_lock */
Pavel Begunkov792bb6e2021-02-18 22:32:51 +00002104 mutex_lock(&ctx->uring_lock);
Pavel Begunkov25935532021-03-19 17:22:40 +00002105 io_req_complete_failed(req, req->result);
Pavel Begunkov792bb6e2021-02-18 22:32:51 +00002106 mutex_unlock(&ctx->uring_lock);
Jens Axboec40f6372020-06-25 15:39:59 -06002107}
2108
Pavel Begunkov5b0a6ac2021-06-30 21:54:04 +01002109static void io_req_task_submit(struct io_kiocb *req)
Jens Axboec40f6372020-06-25 15:39:59 -06002110{
2111 struct io_ring_ctx *ctx = req->ctx;
2112
Pavel Begunkov04fc6c82021-02-12 03:23:54 +00002113 /* ctx stays valid until unlock, even if we drop all ours ctx->refs */
Pavel Begunkov81b6d052021-01-04 20:36:35 +00002114 mutex_lock(&ctx->uring_lock);
Pavel Begunkovaf066f32021-08-09 13:04:19 +01002115 if (likely(!(req->task->flags & PF_EXITING)))
Pavel Begunkovc5eef2b2021-02-10 00:03:22 +00002116 __io_queue_sqe(req);
Pavel Begunkov81b6d052021-01-04 20:36:35 +00002117 else
Pavel Begunkov25935532021-03-19 17:22:40 +00002118 io_req_complete_failed(req, -EFAULT);
Pavel Begunkov81b6d052021-01-04 20:36:35 +00002119 mutex_unlock(&ctx->uring_lock);
Jens Axboe9e645e112019-05-10 16:07:28 -06002120}
2121
Pavel Begunkova3df76982021-02-18 22:32:52 +00002122static void io_req_task_queue_fail(struct io_kiocb *req, int ret)
2123{
Pavel Begunkova3df76982021-02-18 22:32:52 +00002124 req->result = ret;
Pavel Begunkov5b0a6ac2021-06-30 21:54:04 +01002125 req->io_task_work.func = io_req_task_cancel;
Pavel Begunkove09ee512021-07-01 13:26:05 +01002126 io_req_task_work_add(req);
Pavel Begunkova3df76982021-02-18 22:32:52 +00002127}
2128
Pavel Begunkov2c4b8eb2021-02-28 22:35:10 +00002129static void io_req_task_queue(struct io_kiocb *req)
2130{
Pavel Begunkov5b0a6ac2021-06-30 21:54:04 +01002131 req->io_task_work.func = io_req_task_submit;
Pavel Begunkove09ee512021-07-01 13:26:05 +01002132 io_req_task_work_add(req);
Pavel Begunkov2c4b8eb2021-02-28 22:35:10 +00002133}
2134
Jens Axboe773af692021-07-27 10:25:55 -06002135static void io_req_task_queue_reissue(struct io_kiocb *req)
2136{
2137 req->io_task_work.func = io_queue_async_work;
2138 io_req_task_work_add(req);
2139}
2140
Pavel Begunkovf2f87372020-10-27 23:25:37 +00002141static inline void io_queue_next(struct io_kiocb *req)
Jackie Liuc69f8db2019-11-09 11:00:08 +08002142{
Pavel Begunkov9b5f7bd2020-06-29 13:13:00 +03002143 struct io_kiocb *nxt = io_req_find_next(req);
Pavel Begunkov944e58b2019-11-21 23:21:01 +03002144
Pavel Begunkov906a8c32020-06-27 14:04:55 +03002145 if (nxt)
2146 io_req_task_queue(nxt);
Jackie Liuc69f8db2019-11-09 11:00:08 +08002147}
2148
Jens Axboe9e645e112019-05-10 16:07:28 -06002149static void io_free_req(struct io_kiocb *req)
2150{
Pavel Begunkovc3524382020-06-28 12:52:32 +03002151 io_queue_next(req);
Jens Axboe9e645e112019-05-10 16:07:28 -06002152 __io_free_req(req);
Jens Axboee65ef562019-03-12 10:16:44 -06002153}
2154
Pavel Begunkov2d6500d2020-06-28 12:52:33 +03002155struct req_batch {
Pavel Begunkov5af1d132020-07-18 11:32:52 +03002156 struct task_struct *task;
2157 int task_refs;
Jens Axboe1b4c3512021-02-10 00:03:19 +00002158 int ctx_refs;
Pavel Begunkov2d6500d2020-06-28 12:52:33 +03002159};
2160
Pavel Begunkov5af1d132020-07-18 11:32:52 +03002161static inline void io_init_req_batch(struct req_batch *rb)
Pavel Begunkov7a743e22020-03-03 21:33:13 +03002162{
Pavel Begunkov5af1d132020-07-18 11:32:52 +03002163 rb->task_refs = 0;
Pavel Begunkov9ae72462021-02-10 00:03:16 +00002164 rb->ctx_refs = 0;
Pavel Begunkov5af1d132020-07-18 11:32:52 +03002165 rb->task = NULL;
2166}
Pavel Begunkov8766dd52020-03-14 00:31:04 +03002167
Pavel Begunkov2d6500d2020-06-28 12:52:33 +03002168static void io_req_free_batch_finish(struct io_ring_ctx *ctx,
2169 struct req_batch *rb)
2170{
Pavel Begunkov9ae72462021-02-10 00:03:16 +00002171 if (rb->ctx_refs)
2172 percpu_ref_put_many(&ctx->refs, rb->ctx_refs);
Pavel Begunkove9dbe222021-08-09 13:04:20 +01002173 if (rb->task == current)
2174 current->io_uring->cached_refs += rb->task_refs;
2175 else if (rb->task)
2176 io_put_task(rb->task, rb->task_refs);
Pavel Begunkov2d6500d2020-06-28 12:52:33 +03002177}
2178
Pavel Begunkov6ff119a2021-02-10 00:03:18 +00002179static void io_req_free_batch(struct req_batch *rb, struct io_kiocb *req,
2180 struct io_submit_state *state)
Pavel Begunkov2d6500d2020-06-28 12:52:33 +03002181{
Pavel Begunkovf2f87372020-10-27 23:25:37 +00002182 io_queue_next(req);
Pavel Begunkov96670652021-03-19 17:22:32 +00002183 io_dismantle_req(req);
Pavel Begunkov2d6500d2020-06-28 12:52:33 +03002184
Jens Axboee3bc8e92020-09-24 08:45:57 -06002185 if (req->task != rb->task) {
Pavel Begunkov7c660732021-01-25 11:42:21 +00002186 if (rb->task)
2187 io_put_task(rb->task, rb->task_refs);
Jens Axboee3bc8e92020-09-24 08:45:57 -06002188 rb->task = req->task;
2189 rb->task_refs = 0;
Pavel Begunkov5af1d132020-07-18 11:32:52 +03002190 }
Jens Axboee3bc8e92020-09-24 08:45:57 -06002191 rb->task_refs++;
Pavel Begunkov9ae72462021-02-10 00:03:16 +00002192 rb->ctx_refs++;
Pavel Begunkov5af1d132020-07-18 11:32:52 +03002193
Pavel Begunkovbd759042021-02-12 03:23:50 +00002194 if (state->free_reqs != ARRAY_SIZE(state->reqs))
Pavel Begunkov6ff119a2021-02-10 00:03:18 +00002195 state->reqs[state->free_reqs++] = req;
Pavel Begunkovbd759042021-02-12 03:23:50 +00002196 else
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01002197 list_add(&req->inflight_entry, &state->free_list);
Pavel Begunkov7a743e22020-03-03 21:33:13 +03002198}
2199
Pavel Begunkov2a2758f2021-06-17 18:14:00 +01002200static void io_submit_flush_completions(struct io_ring_ctx *ctx)
Jens Axboea141dd82021-08-12 12:48:34 -06002201 __must_hold(&ctx->uring_lock)
Pavel Begunkov905c1722021-02-10 00:03:14 +00002202{
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01002203 struct io_submit_state *state = &ctx->submit_state;
2204 int i, nr = state->compl_nr;
Pavel Begunkov905c1722021-02-10 00:03:14 +00002205 struct req_batch rb;
2206
Jens Axboe79ebeae2021-08-10 15:18:27 -06002207 spin_lock(&ctx->completion_lock);
Pavel Begunkov905c1722021-02-10 00:03:14 +00002208 for (i = 0; i < nr; i++) {
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01002209 struct io_kiocb *req = state->compl_reqs[i];
Pavel Begunkov5182ed22021-06-26 21:40:48 +01002210
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01002211 __io_cqring_fill_event(ctx, req->user_data, req->result,
2212 req->compl.cflags);
Pavel Begunkov905c1722021-02-10 00:03:14 +00002213 }
2214 io_commit_cqring(ctx);
Jens Axboe79ebeae2021-08-10 15:18:27 -06002215 spin_unlock(&ctx->completion_lock);
Pavel Begunkov905c1722021-02-10 00:03:14 +00002216 io_cqring_ev_posted(ctx);
Pavel Begunkov5182ed22021-06-26 21:40:48 +01002217
2218 io_init_req_batch(&rb);
Pavel Begunkov905c1722021-02-10 00:03:14 +00002219 for (i = 0; i < nr; i++) {
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01002220 struct io_kiocb *req = state->compl_reqs[i];
Pavel Begunkov905c1722021-02-10 00:03:14 +00002221
Pavel Begunkov91c2f692021-08-11 19:28:28 +01002222 if (req_ref_put_and_test(req))
Pavel Begunkov6ff119a2021-02-10 00:03:18 +00002223 io_req_free_batch(&rb, req, &ctx->submit_state);
Pavel Begunkov905c1722021-02-10 00:03:14 +00002224 }
2225
2226 io_req_free_batch_finish(ctx, &rb);
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01002227 state->compl_nr = 0;
Jens Axboee65ef562019-03-12 10:16:44 -06002228}
2229
Jens Axboeba816ad2019-09-28 11:36:45 -06002230/*
2231 * Drop reference to request, return next in chain (if there is one) if this
2232 * was the last reference to this request.
2233 */
Pavel Begunkov0d850352021-03-19 17:22:37 +00002234static inline struct io_kiocb *io_put_req_find_next(struct io_kiocb *req)
Jens Axboee65ef562019-03-12 10:16:44 -06002235{
Pavel Begunkov9b5f7bd2020-06-29 13:13:00 +03002236 struct io_kiocb *nxt = NULL;
2237
Jens Axboede9b4cc2021-02-24 13:28:27 -07002238 if (req_ref_put_and_test(req)) {
Pavel Begunkov9b5f7bd2020-06-29 13:13:00 +03002239 nxt = io_req_find_next(req);
Jens Axboe4d7dd462019-11-20 13:03:52 -07002240 __io_free_req(req);
Jens Axboe2a44f462020-02-25 13:25:41 -07002241 }
Pavel Begunkov9b5f7bd2020-06-29 13:13:00 +03002242 return nxt;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002243}
2244
Pavel Begunkov0d850352021-03-19 17:22:37 +00002245static inline void io_put_req(struct io_kiocb *req)
Jens Axboe2b188cc2019-01-07 10:46:33 -07002246{
Jens Axboede9b4cc2021-02-24 13:28:27 -07002247 if (req_ref_put_and_test(req))
Jens Axboedef596e2019-01-09 08:59:42 -07002248 io_free_req(req);
2249}
2250
Pavel Begunkov91c2f692021-08-11 19:28:28 +01002251static inline void io_put_req_deferred(struct io_kiocb *req)
Pavel Begunkov216578e2020-10-13 09:44:00 +01002252{
Pavel Begunkov91c2f692021-08-11 19:28:28 +01002253 if (req_ref_put_and_test(req)) {
Pavel Begunkov543af3a2021-08-09 13:04:15 +01002254 req->io_task_work.func = io_free_req;
2255 io_req_task_work_add(req);
2256 }
Pavel Begunkov216578e2020-10-13 09:44:00 +01002257}
2258
Pavel Begunkov6c503152021-01-04 20:36:36 +00002259static unsigned io_cqring_events(struct io_ring_ctx *ctx)
Jens Axboea3a0e432019-08-20 11:03:11 -06002260{
2261 /* See comment at the top of this file */
2262 smp_rmb();
Pavel Begunkove23de152020-12-17 00:24:37 +00002263 return __io_cqring_events(ctx);
Jens Axboea3a0e432019-08-20 11:03:11 -06002264}
2265
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03002266static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx)
2267{
2268 struct io_rings *rings = ctx->rings;
2269
2270 /* make sure SQ entry isn't read before tail */
2271 return smp_load_acquire(&rings->sq.tail) - ctx->cached_sq_head;
2272}
2273
Pavel Begunkov8ff069b2020-07-16 23:28:04 +03002274static unsigned int io_put_kbuf(struct io_kiocb *req, struct io_buffer *kbuf)
Jens Axboee94f1412019-12-19 12:06:02 -07002275{
Pavel Begunkov8ff069b2020-07-16 23:28:04 +03002276 unsigned int cflags;
Jens Axboee94f1412019-12-19 12:06:02 -07002277
Jens Axboebcda7ba2020-02-23 16:42:51 -07002278 cflags = kbuf->bid << IORING_CQE_BUFFER_SHIFT;
2279 cflags |= IORING_CQE_F_BUFFER;
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03002280 req->flags &= ~REQ_F_BUFFER_SELECTED;
Jens Axboebcda7ba2020-02-23 16:42:51 -07002281 kfree(kbuf);
2282 return cflags;
2283}
2284
Pavel Begunkov8ff069b2020-07-16 23:28:04 +03002285static inline unsigned int io_put_rw_kbuf(struct io_kiocb *req)
2286{
2287 struct io_buffer *kbuf;
2288
2289 kbuf = (struct io_buffer *) (unsigned long) req->rw.addr;
2290 return io_put_kbuf(req, kbuf);
2291}
2292
Jens Axboe4c6e2772020-07-01 11:29:10 -06002293static inline bool io_run_task_work(void)
2294{
Nadav Amitef98eb02021-08-07 17:13:41 -07002295 if (test_thread_flag(TIF_NOTIFY_SIGNAL) || current->task_works) {
Jens Axboe4c6e2772020-07-01 11:29:10 -06002296 __set_current_state(TASK_RUNNING);
Nadav Amitef98eb02021-08-07 17:13:41 -07002297 tracehook_notify_signal();
Jens Axboe4c6e2772020-07-01 11:29:10 -06002298 return true;
2299 }
2300
2301 return false;
2302}
2303
Jens Axboedef596e2019-01-09 08:59:42 -07002304/*
2305 * Find and free completed poll iocbs
2306 */
2307static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events,
Pavel Begunkova8576af2021-08-15 10:40:21 +01002308 struct list_head *done)
Jens Axboedef596e2019-01-09 08:59:42 -07002309{
Jens Axboe8237e042019-12-28 10:48:22 -07002310 struct req_batch rb;
Jens Axboedef596e2019-01-09 08:59:42 -07002311 struct io_kiocb *req;
Xiaoguang Wangbbde0172020-06-16 02:06:38 +08002312
2313 /* order with ->result store in io_complete_rw_iopoll() */
2314 smp_rmb();
Jens Axboedef596e2019-01-09 08:59:42 -07002315
Pavel Begunkov5af1d132020-07-18 11:32:52 +03002316 io_init_req_batch(&rb);
Jens Axboedef596e2019-01-09 08:59:42 -07002317 while (!list_empty(done)) {
Jens Axboebcda7ba2020-02-23 16:42:51 -07002318 int cflags = 0;
2319
Pavel Begunkovd21ffe72020-07-13 23:37:10 +03002320 req = list_first_entry(done, struct io_kiocb, inflight_entry);
Pavel Begunkovd21ffe72020-07-13 23:37:10 +03002321 list_del(&req->inflight_entry);
Jens Axboedef596e2019-01-09 08:59:42 -07002322
Pavel Begunkova8576af2021-08-15 10:40:21 +01002323 if (READ_ONCE(req->result) == -EAGAIN &&
Pavel Begunkov8c130822021-03-22 01:58:32 +00002324 !(req->flags & REQ_F_DONT_REISSUE)) {
Pavel Begunkovf1613402021-02-11 18:28:21 +00002325 req->iopoll_completed = 0;
Jens Axboe773af692021-07-27 10:25:55 -06002326 io_req_task_queue_reissue(req);
Pavel Begunkov8c130822021-03-22 01:58:32 +00002327 continue;
Pavel Begunkovf1613402021-02-11 18:28:21 +00002328 }
2329
Jens Axboebcda7ba2020-02-23 16:42:51 -07002330 if (req->flags & REQ_F_BUFFER_SELECTED)
Pavel Begunkov8ff069b2020-07-16 23:28:04 +03002331 cflags = io_put_rw_kbuf(req);
Jens Axboebcda7ba2020-02-23 16:42:51 -07002332
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01002333 __io_cqring_fill_event(ctx, req->user_data, req->result, cflags);
Jens Axboedef596e2019-01-09 08:59:42 -07002334 (*nr_events)++;
2335
Jens Axboede9b4cc2021-02-24 13:28:27 -07002336 if (req_ref_put_and_test(req))
Pavel Begunkov6ff119a2021-02-10 00:03:18 +00002337 io_req_free_batch(&rb, req, &ctx->submit_state);
Jens Axboedef596e2019-01-09 08:59:42 -07002338 }
Jens Axboedef596e2019-01-09 08:59:42 -07002339
Jens Axboe09bb8392019-03-13 12:39:28 -06002340 io_commit_cqring(ctx);
Pavel Begunkov80c18e42021-01-07 03:15:41 +00002341 io_cqring_ev_posted_iopoll(ctx);
Pavel Begunkov2d6500d2020-06-28 12:52:33 +03002342 io_req_free_batch_finish(ctx, &rb);
Bijan Mottahedeh581f9812020-04-03 13:51:33 -07002343}
2344
Jens Axboedef596e2019-01-09 08:59:42 -07002345static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events,
Pavel Begunkova8576af2021-08-15 10:40:21 +01002346 long min)
Jens Axboedef596e2019-01-09 08:59:42 -07002347{
2348 struct io_kiocb *req, *tmp;
2349 LIST_HEAD(done);
2350 bool spin;
Jens Axboedef596e2019-01-09 08:59:42 -07002351
2352 /*
2353 * Only spin for completions if we don't have multiple devices hanging
2354 * off our complete list, and we're under the requested amount.
2355 */
Hao Xu915b3dd2021-06-28 05:37:30 +08002356 spin = !ctx->poll_multi_queue && *nr_events < min;
Jens Axboedef596e2019-01-09 08:59:42 -07002357
Pavel Begunkovd21ffe72020-07-13 23:37:10 +03002358 list_for_each_entry_safe(req, tmp, &ctx->iopoll_list, inflight_entry) {
Jens Axboe9adbd452019-12-20 08:45:55 -07002359 struct kiocb *kiocb = &req->rw.kiocb;
Pavel Begunkova2416e12021-08-09 13:04:09 +01002360 int ret;
Jens Axboedef596e2019-01-09 08:59:42 -07002361
2362 /*
Bijan Mottahedeh581f9812020-04-03 13:51:33 -07002363 * Move completed and retryable entries to our local lists.
2364 * If we find a request that requires polling, break out
2365 * and complete those lists first, if we have entries there.
Jens Axboedef596e2019-01-09 08:59:42 -07002366 */
Xiaoguang Wang65a65432020-06-11 23:39:36 +08002367 if (READ_ONCE(req->iopoll_completed)) {
Pavel Begunkovd21ffe72020-07-13 23:37:10 +03002368 list_move_tail(&req->inflight_entry, &done);
Jens Axboedef596e2019-01-09 08:59:42 -07002369 continue;
2370 }
2371 if (!list_empty(&done))
2372 break;
2373
2374 ret = kiocb->ki_filp->f_op->iopoll(kiocb, spin);
Pavel Begunkova2416e12021-08-09 13:04:09 +01002375 if (unlikely(ret < 0))
2376 return ret;
2377 else if (ret)
2378 spin = false;
Jens Axboedef596e2019-01-09 08:59:42 -07002379
Pavel Begunkov3aadc232020-07-06 17:59:29 +03002380 /* iopoll may have completed current req */
2381 if (READ_ONCE(req->iopoll_completed))
Pavel Begunkovd21ffe72020-07-13 23:37:10 +03002382 list_move_tail(&req->inflight_entry, &done);
Jens Axboedef596e2019-01-09 08:59:42 -07002383 }
2384
2385 if (!list_empty(&done))
Pavel Begunkova8576af2021-08-15 10:40:21 +01002386 io_iopoll_complete(ctx, nr_events, &done);
Jens Axboedef596e2019-01-09 08:59:42 -07002387
Pavel Begunkova2416e12021-08-09 13:04:09 +01002388 return 0;
Jens Axboedef596e2019-01-09 08:59:42 -07002389}
2390
2391/*
Jens Axboedef596e2019-01-09 08:59:42 -07002392 * We can't just wait for polled events to come to us, we have to actively
2393 * find and complete them.
2394 */
Pavel Begunkovb2edc0a2020-07-07 16:36:22 +03002395static void io_iopoll_try_reap_events(struct io_ring_ctx *ctx)
Jens Axboedef596e2019-01-09 08:59:42 -07002396{
2397 if (!(ctx->flags & IORING_SETUP_IOPOLL))
2398 return;
2399
2400 mutex_lock(&ctx->uring_lock);
Pavel Begunkov540e32a2020-07-13 23:37:09 +03002401 while (!list_empty(&ctx->iopoll_list)) {
Jens Axboedef596e2019-01-09 08:59:42 -07002402 unsigned int nr_events = 0;
2403
Pavel Begunkova8576af2021-08-15 10:40:21 +01002404 io_do_iopoll(ctx, &nr_events, 0);
Jens Axboe08f54392019-08-21 22:19:11 -06002405
Pavel Begunkovb2edc0a2020-07-07 16:36:22 +03002406 /* let it sleep and repeat later if can't complete a request */
2407 if (nr_events == 0)
2408 break;
Jens Axboe08f54392019-08-21 22:19:11 -06002409 /*
2410 * Ensure we allow local-to-the-cpu processing to take place,
2411 * in this case we need to ensure that we reap all events.
Pavel Begunkov3fcee5a2020-07-06 17:59:31 +03002412 * Also let task_work, etc. to progress by releasing the mutex
Jens Axboe08f54392019-08-21 22:19:11 -06002413 */
Pavel Begunkov3fcee5a2020-07-06 17:59:31 +03002414 if (need_resched()) {
2415 mutex_unlock(&ctx->uring_lock);
2416 cond_resched();
2417 mutex_lock(&ctx->uring_lock);
2418 }
Jens Axboedef596e2019-01-09 08:59:42 -07002419 }
2420 mutex_unlock(&ctx->uring_lock);
2421}
2422
Pavel Begunkov7668b922020-07-07 16:36:21 +03002423static int io_iopoll_check(struct io_ring_ctx *ctx, long min)
Jens Axboedef596e2019-01-09 08:59:42 -07002424{
Pavel Begunkov7668b922020-07-07 16:36:21 +03002425 unsigned int nr_events = 0;
Pavel Begunkove9979b32021-04-13 02:58:45 +01002426 int ret = 0;
Jens Axboedef596e2019-01-09 08:59:42 -07002427
Xiaoguang Wangc7849be2020-02-22 14:46:05 +08002428 /*
2429 * We disallow the app entering submit/complete with polling, but we
2430 * still need to lock the ring to prevent racing with polled issue
2431 * that got punted to a workqueue.
2432 */
2433 mutex_lock(&ctx->uring_lock);
Pavel Begunkovf39c8a52021-04-13 02:58:46 +01002434 /*
2435 * Don't enter poll loop if we already have events pending.
2436 * If we do, we can potentially be spinning for commands that
2437 * already triggered a CQE (eg in error).
2438 */
Pavel Begunkov5ed7a372021-06-14 23:37:27 +01002439 if (test_bit(0, &ctx->check_cq_overflow))
Pavel Begunkovf39c8a52021-04-13 02:58:46 +01002440 __io_cqring_overflow_flush(ctx, false);
2441 if (io_cqring_events(ctx))
2442 goto out;
Jens Axboedef596e2019-01-09 08:59:42 -07002443 do {
Jens Axboe500f9fb2019-08-19 12:15:59 -06002444 /*
2445 * If a submit got punted to a workqueue, we can have the
2446 * application entering polling for a command before it gets
2447 * issued. That app will hold the uring_lock for the duration
2448 * of the poll right here, so we need to take a breather every
2449 * now and then to ensure that the issue has a chance to add
2450 * the poll to the issued list. Otherwise we can spin here
2451 * forever, while the workqueue is stuck trying to acquire the
2452 * very same mutex.
2453 */
Pavel Begunkove9979b32021-04-13 02:58:45 +01002454 if (list_empty(&ctx->iopoll_list)) {
Pavel Begunkov8f487ef2021-07-08 13:37:06 +01002455 u32 tail = ctx->cached_cq_tail;
2456
Jens Axboe500f9fb2019-08-19 12:15:59 -06002457 mutex_unlock(&ctx->uring_lock);
Jens Axboe4c6e2772020-07-01 11:29:10 -06002458 io_run_task_work();
Jens Axboe500f9fb2019-08-19 12:15:59 -06002459 mutex_lock(&ctx->uring_lock);
Pavel Begunkove9979b32021-04-13 02:58:45 +01002460
Pavel Begunkov8f487ef2021-07-08 13:37:06 +01002461 /* some requests don't go through iopoll_list */
2462 if (tail != ctx->cached_cq_tail ||
2463 list_empty(&ctx->iopoll_list))
Pavel Begunkove9979b32021-04-13 02:58:45 +01002464 break;
Jens Axboe500f9fb2019-08-19 12:15:59 -06002465 }
Pavel Begunkova8576af2021-08-15 10:40:21 +01002466 ret = io_do_iopoll(ctx, &nr_events, min);
Pavel Begunkovf39c8a52021-04-13 02:58:46 +01002467 } while (!ret && nr_events < min && !need_resched());
2468out:
Jens Axboe500f9fb2019-08-19 12:15:59 -06002469 mutex_unlock(&ctx->uring_lock);
Jens Axboedef596e2019-01-09 08:59:42 -07002470 return ret;
2471}
2472
Jens Axboe491381ce2019-10-17 09:20:46 -06002473static void kiocb_end_write(struct io_kiocb *req)
Jens Axboe2b188cc2019-01-07 10:46:33 -07002474{
Jens Axboe491381ce2019-10-17 09:20:46 -06002475 /*
2476 * Tell lockdep we inherited freeze protection from submission
2477 * thread.
2478 */
2479 if (req->flags & REQ_F_ISREG) {
Pavel Begunkov1c986792021-03-22 01:58:31 +00002480 struct super_block *sb = file_inode(req->file)->i_sb;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002481
Pavel Begunkov1c986792021-03-22 01:58:31 +00002482 __sb_writers_acquired(sb, SB_FREEZE_WRITE);
2483 sb_end_write(sb);
Jens Axboe2b188cc2019-01-07 10:46:33 -07002484 }
2485}
2486
Jens Axboeb63534c2020-06-04 11:28:00 -06002487#ifdef CONFIG_BLOCK
Pavel Begunkovdc2a6e92021-01-19 13:32:35 +00002488static bool io_resubmit_prep(struct io_kiocb *req)
Jens Axboeb63534c2020-06-04 11:28:00 -06002489{
Pavel Begunkovab454432021-03-22 01:58:33 +00002490 struct io_async_rw *rw = req->async_data;
Jens Axboeb63534c2020-06-04 11:28:00 -06002491
Pavel Begunkovab454432021-03-22 01:58:33 +00002492 if (!rw)
2493 return !io_req_prep_async(req);
2494 /* may have left rw->iter inconsistent on -EIOCBQUEUED */
2495 iov_iter_revert(&rw->iter, req->result - iov_iter_count(&rw->iter));
2496 return true;
Jens Axboeb63534c2020-06-04 11:28:00 -06002497}
Jens Axboeb63534c2020-06-04 11:28:00 -06002498
Jens Axboe3e6a0d32021-03-01 13:56:00 -07002499static bool io_rw_should_reissue(struct io_kiocb *req)
Jens Axboeb63534c2020-06-04 11:28:00 -06002500{
Jens Axboe355afae2020-09-02 09:30:31 -06002501 umode_t mode = file_inode(req->file)->i_mode;
Jens Axboe3e6a0d32021-03-01 13:56:00 -07002502 struct io_ring_ctx *ctx = req->ctx;
Jens Axboeb63534c2020-06-04 11:28:00 -06002503
Jens Axboe355afae2020-09-02 09:30:31 -06002504 if (!S_ISBLK(mode) && !S_ISREG(mode))
2505 return false;
Jens Axboe3e6a0d32021-03-01 13:56:00 -07002506 if ((req->flags & REQ_F_NOWAIT) || (io_wq_current_is_worker() &&
2507 !(ctx->flags & IORING_SETUP_IOPOLL)))
Jens Axboeb63534c2020-06-04 11:28:00 -06002508 return false;
Jens Axboe7c977a52021-02-23 19:17:35 -07002509 /*
2510 * If ref is dying, we might be running poll reap from the exit work.
2511 * Don't attempt to reissue from that path, just let it fail with
2512 * -EAGAIN.
2513 */
Jens Axboe3e6a0d32021-03-01 13:56:00 -07002514 if (percpu_ref_is_dying(&ctx->refs))
2515 return false;
Jens Axboeef046882021-07-27 10:50:31 -06002516 /*
2517 * Play it safe and assume not safe to re-import and reissue if we're
2518 * not in the original thread group (or in task context).
2519 */
2520 if (!same_thread_group(req->task, current) || !in_task())
2521 return false;
Jens Axboe3e6a0d32021-03-01 13:56:00 -07002522 return true;
2523}
Jens Axboee82ad482021-04-02 19:45:34 -06002524#else
Jens Axboea1ff1e32021-04-12 06:40:02 -06002525static bool io_resubmit_prep(struct io_kiocb *req)
2526{
2527 return false;
2528}
Jens Axboee82ad482021-04-02 19:45:34 -06002529static bool io_rw_should_reissue(struct io_kiocb *req)
2530{
2531 return false;
2532}
Jens Axboe3e6a0d32021-03-01 13:56:00 -07002533#endif
2534
Jens Axboe8ef12ef2021-08-10 15:15:25 -06002535static bool __io_complete_rw_common(struct io_kiocb *req, long res)
Jens Axboea1d7c392020-06-22 11:09:46 -06002536{
Pavel Begunkovb65c1282021-03-22 01:45:59 +00002537 if (req->rw.kiocb.ki_flags & IOCB_WRITE)
2538 kiocb_end_write(req);
Pavel Begunkov9532b992021-03-22 01:58:34 +00002539 if (res != req->result) {
2540 if ((res == -EAGAIN || res == -EOPNOTSUPP) &&
2541 io_rw_should_reissue(req)) {
2542 req->flags |= REQ_F_REISSUE;
Jens Axboe8ef12ef2021-08-10 15:15:25 -06002543 return true;
Pavel Begunkov9532b992021-03-22 01:58:34 +00002544 }
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01002545 req_set_fail(req);
Jens Axboe8ef12ef2021-08-10 15:15:25 -06002546 req->result = res;
Pavel Begunkov9532b992021-03-22 01:58:34 +00002547 }
Jens Axboe8ef12ef2021-08-10 15:15:25 -06002548 return false;
2549}
2550
2551static void io_req_task_complete(struct io_kiocb *req)
2552{
2553 int cflags = 0;
2554
Pavel Begunkov2f8e45f2021-02-11 18:28:23 +00002555 if (req->flags & REQ_F_BUFFER_SELECTED)
2556 cflags = io_put_rw_kbuf(req);
Jens Axboe8ef12ef2021-08-10 15:15:25 -06002557 __io_req_complete(req, 0, req->result, cflags);
2558}
2559
2560static void __io_complete_rw(struct io_kiocb *req, long res, long res2,
2561 unsigned int issue_flags)
2562{
2563 if (__io_complete_rw_common(req, res))
2564 return;
2565 io_req_task_complete(req);
Jens Axboeba816ad2019-09-28 11:36:45 -06002566}
2567
2568static void io_complete_rw(struct kiocb *kiocb, long res, long res2)
2569{
Jens Axboe9adbd452019-12-20 08:45:55 -07002570 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
Jens Axboeba816ad2019-09-28 11:36:45 -06002571
Jens Axboe8ef12ef2021-08-10 15:15:25 -06002572 if (__io_complete_rw_common(req, res))
2573 return;
2574 req->result = res;
2575 req->io_task_work.func = io_req_task_complete;
2576 io_req_task_work_add(req);
Jens Axboe2b188cc2019-01-07 10:46:33 -07002577}
2578
Jens Axboedef596e2019-01-09 08:59:42 -07002579static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2)
2580{
Jens Axboe9adbd452019-12-20 08:45:55 -07002581 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
Jens Axboedef596e2019-01-09 08:59:42 -07002582
Jens Axboe491381ce2019-10-17 09:20:46 -06002583 if (kiocb->ki_flags & IOCB_WRITE)
2584 kiocb_end_write(req);
Pavel Begunkov9532b992021-03-22 01:58:34 +00002585 if (unlikely(res != req->result)) {
Jens Axboea1ff1e32021-04-12 06:40:02 -06002586 if (!(res == -EAGAIN && io_rw_should_reissue(req) &&
2587 io_resubmit_prep(req))) {
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01002588 req_set_fail(req);
Pavel Begunkov9532b992021-03-22 01:58:34 +00002589 req->flags |= REQ_F_DONT_REISSUE;
2590 }
Pavel Begunkov8c130822021-03-22 01:58:32 +00002591 }
Xiaoguang Wangbbde0172020-06-16 02:06:38 +08002592
2593 WRITE_ONCE(req->result, res);
Jens Axboeb9b0e0d2021-02-23 08:18:36 -07002594 /* order with io_iopoll_complete() checking ->result */
Pavel Begunkovcd664b02020-06-25 12:37:10 +03002595 smp_wmb();
2596 WRITE_ONCE(req->iopoll_completed, 1);
Jens Axboedef596e2019-01-09 08:59:42 -07002597}
2598
2599/*
2600 * After the iocb has been issued, it's safe to be found on the poll list.
2601 * Adding the kiocb to the list AFTER submission ensures that we don't
Pavel Begunkovf39c8a52021-04-13 02:58:46 +01002602 * find it from a io_do_iopoll() thread before the issuer is done
Jens Axboedef596e2019-01-09 08:59:42 -07002603 * accessing the kiocb cookie.
2604 */
Pavel Begunkovcb3d8972021-06-14 02:36:14 +01002605static void io_iopoll_req_issued(struct io_kiocb *req)
Jens Axboedef596e2019-01-09 08:59:42 -07002606{
2607 struct io_ring_ctx *ctx = req->ctx;
Pavel Begunkovcb3d8972021-06-14 02:36:14 +01002608 const bool in_async = io_wq_current_is_worker();
2609
2610 /* workqueue context doesn't hold uring_lock, grab it now */
2611 if (unlikely(in_async))
2612 mutex_lock(&ctx->uring_lock);
Jens Axboedef596e2019-01-09 08:59:42 -07002613
2614 /*
2615 * Track whether we have multiple files in our lists. This will impact
2616 * how we do polling eventually, not spinning if we're on potentially
2617 * different devices.
2618 */
Pavel Begunkov540e32a2020-07-13 23:37:09 +03002619 if (list_empty(&ctx->iopoll_list)) {
Hao Xu915b3dd2021-06-28 05:37:30 +08002620 ctx->poll_multi_queue = false;
2621 } else if (!ctx->poll_multi_queue) {
Jens Axboedef596e2019-01-09 08:59:42 -07002622 struct io_kiocb *list_req;
Hao Xu915b3dd2021-06-28 05:37:30 +08002623 unsigned int queue_num0, queue_num1;
Jens Axboedef596e2019-01-09 08:59:42 -07002624
Pavel Begunkov540e32a2020-07-13 23:37:09 +03002625 list_req = list_first_entry(&ctx->iopoll_list, struct io_kiocb,
Pavel Begunkovd21ffe72020-07-13 23:37:10 +03002626 inflight_entry);
Hao Xu915b3dd2021-06-28 05:37:30 +08002627
2628 if (list_req->file != req->file) {
2629 ctx->poll_multi_queue = true;
2630 } else {
2631 queue_num0 = blk_qc_t_to_queue_num(list_req->rw.kiocb.ki_cookie);
2632 queue_num1 = blk_qc_t_to_queue_num(req->rw.kiocb.ki_cookie);
2633 if (queue_num0 != queue_num1)
2634 ctx->poll_multi_queue = true;
2635 }
Jens Axboedef596e2019-01-09 08:59:42 -07002636 }
2637
2638 /*
2639 * For fast devices, IO may have already completed. If it has, add
2640 * it to the front so we find it first.
2641 */
Xiaoguang Wang65a65432020-06-11 23:39:36 +08002642 if (READ_ONCE(req->iopoll_completed))
Pavel Begunkovd21ffe72020-07-13 23:37:10 +03002643 list_add(&req->inflight_entry, &ctx->iopoll_list);
Jens Axboedef596e2019-01-09 08:59:42 -07002644 else
Pavel Begunkovd21ffe72020-07-13 23:37:10 +03002645 list_add_tail(&req->inflight_entry, &ctx->iopoll_list);
Xiaoguang Wangbdcd3ea2020-02-25 22:12:08 +08002646
Pavel Begunkovcb3d8972021-06-14 02:36:14 +01002647 if (unlikely(in_async)) {
2648 /*
2649 * If IORING_SETUP_SQPOLL is enabled, sqes are either handle
2650 * in sq thread task context or in io worker task context. If
2651 * current task context is sq thread, we don't need to check
2652 * whether should wake up sq thread.
2653 */
2654 if ((ctx->flags & IORING_SETUP_SQPOLL) &&
2655 wq_has_sleeper(&ctx->sq_data->wait))
2656 wake_up(&ctx->sq_data->wait);
2657
2658 mutex_unlock(&ctx->uring_lock);
2659 }
Jens Axboedef596e2019-01-09 08:59:42 -07002660}
2661
Jens Axboe4503b762020-06-01 10:00:27 -06002662static bool io_bdev_nowait(struct block_device *bdev)
2663{
Jeffle Xu9ba0d0c2020-10-19 16:59:42 +08002664 return !bdev || blk_queue_nowait(bdev_get_queue(bdev));
Jens Axboe4503b762020-06-01 10:00:27 -06002665}
2666
Jens Axboe2b188cc2019-01-07 10:46:33 -07002667/*
2668 * If we tracked the file through the SCM inflight mechanism, we could support
2669 * any file. For now, just ensure that anything potentially problematic is done
2670 * inline.
2671 */
Pavel Begunkovb191e2d2021-08-09 13:04:03 +01002672static bool __io_file_supports_nowait(struct file *file, int rw)
Jens Axboe2b188cc2019-01-07 10:46:33 -07002673{
2674 umode_t mode = file_inode(file)->i_mode;
2675
Jens Axboe4503b762020-06-01 10:00:27 -06002676 if (S_ISBLK(mode)) {
Christoph Hellwig4e7b5672020-11-23 13:38:40 +01002677 if (IS_ENABLED(CONFIG_BLOCK) &&
2678 io_bdev_nowait(I_BDEV(file->f_mapping->host)))
Jens Axboe4503b762020-06-01 10:00:27 -06002679 return true;
2680 return false;
2681 }
Pavel Begunkov976517f2021-06-09 12:07:25 +01002682 if (S_ISSOCK(mode))
Jens Axboe2b188cc2019-01-07 10:46:33 -07002683 return true;
Jens Axboe4503b762020-06-01 10:00:27 -06002684 if (S_ISREG(mode)) {
Christoph Hellwig4e7b5672020-11-23 13:38:40 +01002685 if (IS_ENABLED(CONFIG_BLOCK) &&
2686 io_bdev_nowait(file->f_inode->i_sb->s_bdev) &&
Jens Axboe4503b762020-06-01 10:00:27 -06002687 file->f_op != &io_uring_fops)
2688 return true;
2689 return false;
2690 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07002691
Jens Axboec5b85622020-06-09 19:23:05 -06002692 /* any ->read/write should understand O_NONBLOCK */
2693 if (file->f_flags & O_NONBLOCK)
2694 return true;
2695
Jens Axboeaf197f52020-04-28 13:15:06 -06002696 if (!(file->f_mode & FMODE_NOWAIT))
2697 return false;
2698
2699 if (rw == READ)
2700 return file->f_op->read_iter != NULL;
2701
2702 return file->f_op->write_iter != NULL;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002703}
2704
Pavel Begunkovb191e2d2021-08-09 13:04:03 +01002705static bool io_file_supports_nowait(struct io_kiocb *req, int rw)
Jens Axboe7b29f922021-03-12 08:30:14 -07002706{
Pavel Begunkovb191e2d2021-08-09 13:04:03 +01002707 if (rw == READ && (req->flags & REQ_F_NOWAIT_READ))
Jens Axboe7b29f922021-03-12 08:30:14 -07002708 return true;
Pavel Begunkovb191e2d2021-08-09 13:04:03 +01002709 else if (rw == WRITE && (req->flags & REQ_F_NOWAIT_WRITE))
Jens Axboe7b29f922021-03-12 08:30:14 -07002710 return true;
2711
Pavel Begunkovb191e2d2021-08-09 13:04:03 +01002712 return __io_file_supports_nowait(req->file, rw);
Jens Axboe7b29f922021-03-12 08:30:14 -07002713}
2714
Pavel Begunkova88fc402020-09-30 22:57:53 +03002715static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboe2b188cc2019-01-07 10:46:33 -07002716{
Jens Axboedef596e2019-01-09 08:59:42 -07002717 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe9adbd452019-12-20 08:45:55 -07002718 struct kiocb *kiocb = &req->rw.kiocb;
Pavel Begunkov75c668c2021-02-04 13:52:05 +00002719 struct file *file = req->file;
Jens Axboe09bb8392019-03-13 12:39:28 -06002720 unsigned ioprio;
2721 int ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002722
Pavel Begunkovc97d8a02021-08-09 13:04:04 +01002723 if (!io_req_ffs_set(req) && S_ISREG(file_inode(file)->i_mode))
Jens Axboe491381ce2019-10-17 09:20:46 -06002724 req->flags |= REQ_F_ISREG;
2725
Jens Axboe2b188cc2019-01-07 10:46:33 -07002726 kiocb->ki_pos = READ_ONCE(sqe->off);
Pavel Begunkov75c668c2021-02-04 13:52:05 +00002727 if (kiocb->ki_pos == -1 && !(file->f_mode & FMODE_STREAM)) {
Jens Axboeba042912019-12-25 16:33:42 -07002728 req->flags |= REQ_F_CUR_POS;
Pavel Begunkov75c668c2021-02-04 13:52:05 +00002729 kiocb->ki_pos = file->f_pos;
Jens Axboeba042912019-12-25 16:33:42 -07002730 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07002731 kiocb->ki_hint = ki_hint_validate(file_write_hint(kiocb->ki_filp));
Pavel Begunkov3e577dc2020-02-01 03:58:42 +03002732 kiocb->ki_flags = iocb_flags(kiocb->ki_filp);
2733 ret = kiocb_set_rw_flags(kiocb, READ_ONCE(sqe->rw_flags));
2734 if (unlikely(ret))
2735 return ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002736
Pavel Begunkov75c668c2021-02-04 13:52:05 +00002737 /* don't allow async punt for O_NONBLOCK or RWF_NOWAIT */
2738 if ((kiocb->ki_flags & IOCB_NOWAIT) || (file->f_flags & O_NONBLOCK))
2739 req->flags |= REQ_F_NOWAIT;
2740
Jens Axboe2b188cc2019-01-07 10:46:33 -07002741 ioprio = READ_ONCE(sqe->ioprio);
2742 if (ioprio) {
2743 ret = ioprio_check_cap(ioprio);
2744 if (ret)
Jens Axboe09bb8392019-03-13 12:39:28 -06002745 return ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002746
2747 kiocb->ki_ioprio = ioprio;
2748 } else
2749 kiocb->ki_ioprio = get_current_ioprio();
2750
Jens Axboedef596e2019-01-09 08:59:42 -07002751 if (ctx->flags & IORING_SETUP_IOPOLL) {
Jens Axboedef596e2019-01-09 08:59:42 -07002752 if (!(kiocb->ki_flags & IOCB_DIRECT) ||
2753 !kiocb->ki_filp->f_op->iopoll)
Jens Axboe09bb8392019-03-13 12:39:28 -06002754 return -EOPNOTSUPP;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002755
Jens Axboedef596e2019-01-09 08:59:42 -07002756 kiocb->ki_flags |= IOCB_HIPRI;
2757 kiocb->ki_complete = io_complete_rw_iopoll;
Xiaoguang Wang65a65432020-06-11 23:39:36 +08002758 req->iopoll_completed = 0;
Jens Axboedef596e2019-01-09 08:59:42 -07002759 } else {
Jens Axboe09bb8392019-03-13 12:39:28 -06002760 if (kiocb->ki_flags & IOCB_HIPRI)
2761 return -EINVAL;
Jens Axboedef596e2019-01-09 08:59:42 -07002762 kiocb->ki_complete = io_complete_rw;
2763 }
Jens Axboe9adbd452019-12-20 08:45:55 -07002764
Pavel Begunkoveae071c2021-04-25 14:32:24 +01002765 if (req->opcode == IORING_OP_READ_FIXED ||
2766 req->opcode == IORING_OP_WRITE_FIXED) {
2767 req->imu = NULL;
2768 io_req_set_rsrc_node(req);
2769 }
2770
Jens Axboe3529d8c2019-12-19 18:24:38 -07002771 req->rw.addr = READ_ONCE(sqe->addr);
2772 req->rw.len = READ_ONCE(sqe->len);
Bijan Mottahedeh4f4eeba2020-05-19 14:52:49 -07002773 req->buf_index = READ_ONCE(sqe->buf_index);
Jens Axboe2b188cc2019-01-07 10:46:33 -07002774 return 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002775}
2776
2777static inline void io_rw_done(struct kiocb *kiocb, ssize_t ret)
2778{
2779 switch (ret) {
2780 case -EIOCBQUEUED:
2781 break;
2782 case -ERESTARTSYS:
2783 case -ERESTARTNOINTR:
2784 case -ERESTARTNOHAND:
2785 case -ERESTART_RESTARTBLOCK:
2786 /*
2787 * We can't just restart the syscall, since previously
2788 * submitted sqes may already be in progress. Just fail this
2789 * IO with EINTR.
2790 */
2791 ret = -EINTR;
Gustavo A. R. Silvadf561f662020-08-23 17:36:59 -05002792 fallthrough;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002793 default:
2794 kiocb->ki_complete(kiocb, ret, 0);
2795 }
2796}
2797
Jens Axboea1d7c392020-06-22 11:09:46 -06002798static void kiocb_done(struct kiocb *kiocb, ssize_t ret,
Pavel Begunkov889fca72021-02-10 00:03:09 +00002799 unsigned int issue_flags)
Jens Axboeba816ad2019-09-28 11:36:45 -06002800{
Jens Axboeba042912019-12-25 16:33:42 -07002801 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
Jens Axboee8c2bc12020-08-15 18:44:09 -07002802 struct io_async_rw *io = req->async_data;
Pavel Begunkov97284632021-04-08 19:28:03 +01002803 bool check_reissue = kiocb->ki_complete == io_complete_rw;
Jens Axboeba042912019-12-25 16:33:42 -07002804
Jens Axboe227c0c92020-08-13 11:51:40 -06002805 /* add previously done IO, if any */
Jens Axboee8c2bc12020-08-15 18:44:09 -07002806 if (io && io->bytes_done > 0) {
Jens Axboe227c0c92020-08-13 11:51:40 -06002807 if (ret < 0)
Jens Axboee8c2bc12020-08-15 18:44:09 -07002808 ret = io->bytes_done;
Jens Axboe227c0c92020-08-13 11:51:40 -06002809 else
Jens Axboee8c2bc12020-08-15 18:44:09 -07002810 ret += io->bytes_done;
Jens Axboe227c0c92020-08-13 11:51:40 -06002811 }
2812
Jens Axboeba042912019-12-25 16:33:42 -07002813 if (req->flags & REQ_F_CUR_POS)
2814 req->file->f_pos = kiocb->ki_pos;
Hao Xue149bd742021-06-28 05:48:05 +08002815 if (ret >= 0 && check_reissue)
Pavel Begunkov889fca72021-02-10 00:03:09 +00002816 __io_complete_rw(req, ret, 0, issue_flags);
Jens Axboeba816ad2019-09-28 11:36:45 -06002817 else
2818 io_rw_done(kiocb, ret);
Pavel Begunkov97284632021-04-08 19:28:03 +01002819
Pavel Begunkovfe7e3252021-06-24 15:09:57 +01002820 if (check_reissue && (req->flags & REQ_F_REISSUE)) {
Pavel Begunkov97284632021-04-08 19:28:03 +01002821 req->flags &= ~REQ_F_REISSUE;
Jens Axboea7be7c22021-04-15 11:31:14 -06002822 if (io_resubmit_prep(req)) {
Jens Axboe773af692021-07-27 10:25:55 -06002823 io_req_task_queue_reissue(req);
Pavel Begunkov8c130822021-03-22 01:58:32 +00002824 } else {
Pavel Begunkov97284632021-04-08 19:28:03 +01002825 int cflags = 0;
2826
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01002827 req_set_fail(req);
Pavel Begunkov97284632021-04-08 19:28:03 +01002828 if (req->flags & REQ_F_BUFFER_SELECTED)
2829 cflags = io_put_rw_kbuf(req);
2830 __io_req_complete(req, issue_flags, ret, cflags);
2831 }
2832 }
Jens Axboeba816ad2019-09-28 11:36:45 -06002833}
2834
Pavel Begunkoveae071c2021-04-25 14:32:24 +01002835static int __io_import_fixed(struct io_kiocb *req, int rw, struct iov_iter *iter,
2836 struct io_mapped_ubuf *imu)
Jens Axboeedafcce2019-01-09 09:16:05 -07002837{
Jens Axboe9adbd452019-12-20 08:45:55 -07002838 size_t len = req->rw.len;
Pavel Begunkov75769e32021-04-01 15:43:54 +01002839 u64 buf_end, buf_addr = req->rw.addr;
Jens Axboeedafcce2019-01-09 09:16:05 -07002840 size_t offset;
Jens Axboeedafcce2019-01-09 09:16:05 -07002841
Pavel Begunkov75769e32021-04-01 15:43:54 +01002842 if (unlikely(check_add_overflow(buf_addr, (u64)len, &buf_end)))
Jens Axboeedafcce2019-01-09 09:16:05 -07002843 return -EFAULT;
2844 /* not inside the mapped region */
Pavel Begunkov4751f532021-04-01 15:43:55 +01002845 if (unlikely(buf_addr < imu->ubuf || buf_end > imu->ubuf_end))
Jens Axboeedafcce2019-01-09 09:16:05 -07002846 return -EFAULT;
2847
2848 /*
2849 * May not be a start of buffer, set size appropriately
2850 * and advance us to the beginning.
2851 */
2852 offset = buf_addr - imu->ubuf;
2853 iov_iter_bvec(iter, rw, imu->bvec, imu->nr_bvecs, offset + len);
Jens Axboebd11b3a2019-07-20 08:37:31 -06002854
2855 if (offset) {
2856 /*
2857 * Don't use iov_iter_advance() here, as it's really slow for
2858 * using the latter parts of a big fixed buffer - it iterates
2859 * over each segment manually. We can cheat a bit here, because
2860 * we know that:
2861 *
2862 * 1) it's a BVEC iter, we set it up
2863 * 2) all bvecs are PAGE_SIZE in size, except potentially the
2864 * first and last bvec
2865 *
2866 * So just find our index, and adjust the iterator afterwards.
2867 * If the offset is within the first bvec (or the whole first
2868 * bvec, just use iov_iter_advance(). This makes it easier
2869 * since we can just skip the first segment, which may not
2870 * be PAGE_SIZE aligned.
2871 */
2872 const struct bio_vec *bvec = imu->bvec;
2873
2874 if (offset <= bvec->bv_len) {
2875 iov_iter_advance(iter, offset);
2876 } else {
2877 unsigned long seg_skip;
2878
2879 /* skip first vec */
2880 offset -= bvec->bv_len;
2881 seg_skip = 1 + (offset >> PAGE_SHIFT);
2882
2883 iter->bvec = bvec + seg_skip;
2884 iter->nr_segs -= seg_skip;
Aleix Roca Nonell99c79f62019-08-15 14:03:22 +02002885 iter->count -= bvec->bv_len + offset;
Jens Axboebd11b3a2019-07-20 08:37:31 -06002886 iter->iov_offset = offset & ~PAGE_MASK;
Jens Axboebd11b3a2019-07-20 08:37:31 -06002887 }
2888 }
2889
Pavel Begunkov847595d2021-02-04 13:52:06 +00002890 return 0;
Jens Axboeedafcce2019-01-09 09:16:05 -07002891}
2892
Pavel Begunkoveae071c2021-04-25 14:32:24 +01002893static int io_import_fixed(struct io_kiocb *req, int rw, struct iov_iter *iter)
2894{
2895 struct io_ring_ctx *ctx = req->ctx;
2896 struct io_mapped_ubuf *imu = req->imu;
2897 u16 index, buf_index = req->buf_index;
2898
2899 if (likely(!imu)) {
2900 if (unlikely(buf_index >= ctx->nr_user_bufs))
2901 return -EFAULT;
2902 index = array_index_nospec(buf_index, ctx->nr_user_bufs);
2903 imu = READ_ONCE(ctx->user_bufs[index]);
2904 req->imu = imu;
2905 }
2906 return __io_import_fixed(req, rw, iter, imu);
2907}
2908
Jens Axboebcda7ba2020-02-23 16:42:51 -07002909static void io_ring_submit_unlock(struct io_ring_ctx *ctx, bool needs_lock)
2910{
2911 if (needs_lock)
2912 mutex_unlock(&ctx->uring_lock);
2913}
2914
2915static void io_ring_submit_lock(struct io_ring_ctx *ctx, bool needs_lock)
2916{
2917 /*
2918 * "Normal" inline submissions always hold the uring_lock, since we
2919 * grab it from the system call. Same is true for the SQPOLL offload.
2920 * The only exception is when we've detached the request and issue it
2921 * from an async worker thread, grab the lock for that case.
2922 */
2923 if (needs_lock)
2924 mutex_lock(&ctx->uring_lock);
2925}
2926
2927static struct io_buffer *io_buffer_select(struct io_kiocb *req, size_t *len,
2928 int bgid, struct io_buffer *kbuf,
2929 bool needs_lock)
2930{
2931 struct io_buffer *head;
2932
2933 if (req->flags & REQ_F_BUFFER_SELECTED)
2934 return kbuf;
2935
2936 io_ring_submit_lock(req->ctx, needs_lock);
2937
2938 lockdep_assert_held(&req->ctx->uring_lock);
2939
Jens Axboe9e15c3a2021-03-13 12:29:43 -07002940 head = xa_load(&req->ctx->io_buffers, bgid);
Jens Axboebcda7ba2020-02-23 16:42:51 -07002941 if (head) {
2942 if (!list_empty(&head->list)) {
2943 kbuf = list_last_entry(&head->list, struct io_buffer,
2944 list);
2945 list_del(&kbuf->list);
2946 } else {
2947 kbuf = head;
Jens Axboe9e15c3a2021-03-13 12:29:43 -07002948 xa_erase(&req->ctx->io_buffers, bgid);
Jens Axboebcda7ba2020-02-23 16:42:51 -07002949 }
2950 if (*len > kbuf->len)
2951 *len = kbuf->len;
2952 } else {
2953 kbuf = ERR_PTR(-ENOBUFS);
2954 }
2955
2956 io_ring_submit_unlock(req->ctx, needs_lock);
2957
2958 return kbuf;
2959}
2960
Jens Axboe4d954c22020-02-27 07:31:19 -07002961static void __user *io_rw_buffer_select(struct io_kiocb *req, size_t *len,
2962 bool needs_lock)
2963{
2964 struct io_buffer *kbuf;
Bijan Mottahedeh4f4eeba2020-05-19 14:52:49 -07002965 u16 bgid;
Jens Axboe4d954c22020-02-27 07:31:19 -07002966
2967 kbuf = (struct io_buffer *) (unsigned long) req->rw.addr;
Bijan Mottahedeh4f4eeba2020-05-19 14:52:49 -07002968 bgid = req->buf_index;
Jens Axboe4d954c22020-02-27 07:31:19 -07002969 kbuf = io_buffer_select(req, len, bgid, kbuf, needs_lock);
2970 if (IS_ERR(kbuf))
2971 return kbuf;
2972 req->rw.addr = (u64) (unsigned long) kbuf;
2973 req->flags |= REQ_F_BUFFER_SELECTED;
2974 return u64_to_user_ptr(kbuf->addr);
2975}
2976
2977#ifdef CONFIG_COMPAT
2978static ssize_t io_compat_import(struct io_kiocb *req, struct iovec *iov,
2979 bool needs_lock)
2980{
2981 struct compat_iovec __user *uiov;
2982 compat_ssize_t clen;
2983 void __user *buf;
2984 ssize_t len;
2985
2986 uiov = u64_to_user_ptr(req->rw.addr);
2987 if (!access_ok(uiov, sizeof(*uiov)))
2988 return -EFAULT;
2989 if (__get_user(clen, &uiov->iov_len))
2990 return -EFAULT;
2991 if (clen < 0)
2992 return -EINVAL;
2993
2994 len = clen;
2995 buf = io_rw_buffer_select(req, &len, needs_lock);
2996 if (IS_ERR(buf))
2997 return PTR_ERR(buf);
2998 iov[0].iov_base = buf;
2999 iov[0].iov_len = (compat_size_t) len;
3000 return 0;
3001}
3002#endif
3003
3004static ssize_t __io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov,
3005 bool needs_lock)
3006{
3007 struct iovec __user *uiov = u64_to_user_ptr(req->rw.addr);
3008 void __user *buf;
3009 ssize_t len;
3010
3011 if (copy_from_user(iov, uiov, sizeof(*uiov)))
3012 return -EFAULT;
3013
3014 len = iov[0].iov_len;
3015 if (len < 0)
3016 return -EINVAL;
3017 buf = io_rw_buffer_select(req, &len, needs_lock);
3018 if (IS_ERR(buf))
3019 return PTR_ERR(buf);
3020 iov[0].iov_base = buf;
3021 iov[0].iov_len = len;
3022 return 0;
3023}
3024
3025static ssize_t io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov,
3026 bool needs_lock)
3027{
Jens Axboedddb3e22020-06-04 11:27:01 -06003028 if (req->flags & REQ_F_BUFFER_SELECTED) {
3029 struct io_buffer *kbuf;
3030
3031 kbuf = (struct io_buffer *) (unsigned long) req->rw.addr;
3032 iov[0].iov_base = u64_to_user_ptr(kbuf->addr);
3033 iov[0].iov_len = kbuf->len;
Jens Axboe4d954c22020-02-27 07:31:19 -07003034 return 0;
Jens Axboedddb3e22020-06-04 11:27:01 -06003035 }
Pavel Begunkovdd201662020-12-19 03:15:43 +00003036 if (req->rw.len != 1)
Jens Axboe4d954c22020-02-27 07:31:19 -07003037 return -EINVAL;
3038
3039#ifdef CONFIG_COMPAT
3040 if (req->ctx->compat)
3041 return io_compat_import(req, iov, needs_lock);
3042#endif
3043
3044 return __io_iov_buffer_select(req, iov, needs_lock);
3045}
3046
Pavel Begunkov847595d2021-02-04 13:52:06 +00003047static int io_import_iovec(int rw, struct io_kiocb *req, struct iovec **iovec,
3048 struct iov_iter *iter, bool needs_lock)
Jens Axboe2b188cc2019-01-07 10:46:33 -07003049{
Jens Axboe9adbd452019-12-20 08:45:55 -07003050 void __user *buf = u64_to_user_ptr(req->rw.addr);
3051 size_t sqe_len = req->rw.len;
Pavel Begunkov847595d2021-02-04 13:52:06 +00003052 u8 opcode = req->opcode;
Jens Axboe4d954c22020-02-27 07:31:19 -07003053 ssize_t ret;
Jens Axboeedafcce2019-01-09 09:16:05 -07003054
Pavel Begunkov7d009162019-11-25 23:14:40 +03003055 if (opcode == IORING_OP_READ_FIXED || opcode == IORING_OP_WRITE_FIXED) {
Jens Axboeedafcce2019-01-09 09:16:05 -07003056 *iovec = NULL;
Jens Axboe9adbd452019-12-20 08:45:55 -07003057 return io_import_fixed(req, rw, iter);
Jens Axboeedafcce2019-01-09 09:16:05 -07003058 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07003059
Jens Axboebcda7ba2020-02-23 16:42:51 -07003060 /* buffer index only valid with fixed read/write, or buffer select */
Bijan Mottahedeh4f4eeba2020-05-19 14:52:49 -07003061 if (req->buf_index && !(req->flags & REQ_F_BUFFER_SELECT))
Jens Axboe9adbd452019-12-20 08:45:55 -07003062 return -EINVAL;
3063
Jens Axboe3a6820f2019-12-22 15:19:35 -07003064 if (opcode == IORING_OP_READ || opcode == IORING_OP_WRITE) {
Jens Axboebcda7ba2020-02-23 16:42:51 -07003065 if (req->flags & REQ_F_BUFFER_SELECT) {
Jens Axboe4d954c22020-02-27 07:31:19 -07003066 buf = io_rw_buffer_select(req, &sqe_len, needs_lock);
Pavel Begunkov867a23e2020-08-20 11:34:39 +03003067 if (IS_ERR(buf))
Jens Axboe4d954c22020-02-27 07:31:19 -07003068 return PTR_ERR(buf);
Jens Axboe3f9d6442020-03-11 12:27:04 -06003069 req->rw.len = sqe_len;
Jens Axboebcda7ba2020-02-23 16:42:51 -07003070 }
3071
Jens Axboe3a6820f2019-12-22 15:19:35 -07003072 ret = import_single_range(rw, buf, sqe_len, *iovec, iter);
3073 *iovec = NULL;
David Laight10fc72e2020-11-07 13:16:25 +00003074 return ret;
Jens Axboe3a6820f2019-12-22 15:19:35 -07003075 }
3076
Jens Axboe4d954c22020-02-27 07:31:19 -07003077 if (req->flags & REQ_F_BUFFER_SELECT) {
3078 ret = io_iov_buffer_select(req, *iovec, needs_lock);
Pavel Begunkov847595d2021-02-04 13:52:06 +00003079 if (!ret)
3080 iov_iter_init(iter, rw, *iovec, 1, (*iovec)->iov_len);
Jens Axboe4d954c22020-02-27 07:31:19 -07003081 *iovec = NULL;
3082 return ret;
3083 }
3084
Christoph Hellwig89cd35c2020-09-25 06:51:41 +02003085 return __import_iovec(rw, buf, sqe_len, UIO_FASTIOV, iovec, iter,
3086 req->ctx->compat);
Jens Axboe2b188cc2019-01-07 10:46:33 -07003087}
3088
Jens Axboe0fef9482020-08-26 10:36:20 -06003089static inline loff_t *io_kiocb_ppos(struct kiocb *kiocb)
3090{
Pavel Begunkov5b09e372020-09-30 22:57:15 +03003091 return (kiocb->ki_filp->f_mode & FMODE_STREAM) ? NULL : &kiocb->ki_pos;
Jens Axboe0fef9482020-08-26 10:36:20 -06003092}
3093
Jens Axboe32960612019-09-23 11:05:34 -06003094/*
3095 * For files that don't have ->read_iter() and ->write_iter(), handle them
3096 * by looping over ->read() or ->write() manually.
3097 */
Jens Axboe4017eb92020-10-22 14:14:12 -06003098static ssize_t loop_rw_iter(int rw, struct io_kiocb *req, struct iov_iter *iter)
Jens Axboe32960612019-09-23 11:05:34 -06003099{
Jens Axboe4017eb92020-10-22 14:14:12 -06003100 struct kiocb *kiocb = &req->rw.kiocb;
3101 struct file *file = req->file;
Jens Axboe32960612019-09-23 11:05:34 -06003102 ssize_t ret = 0;
3103
3104 /*
3105 * Don't support polled IO through this interface, and we can't
3106 * support non-blocking either. For the latter, this just causes
3107 * the kiocb to be handled from an async context.
3108 */
3109 if (kiocb->ki_flags & IOCB_HIPRI)
3110 return -EOPNOTSUPP;
3111 if (kiocb->ki_flags & IOCB_NOWAIT)
3112 return -EAGAIN;
3113
3114 while (iov_iter_count(iter)) {
Pavel Begunkov311ae9e2019-11-24 11:58:24 +03003115 struct iovec iovec;
Jens Axboe32960612019-09-23 11:05:34 -06003116 ssize_t nr;
3117
Pavel Begunkov311ae9e2019-11-24 11:58:24 +03003118 if (!iov_iter_is_bvec(iter)) {
3119 iovec = iov_iter_iovec(iter);
3120 } else {
Jens Axboe4017eb92020-10-22 14:14:12 -06003121 iovec.iov_base = u64_to_user_ptr(req->rw.addr);
3122 iovec.iov_len = req->rw.len;
Pavel Begunkov311ae9e2019-11-24 11:58:24 +03003123 }
3124
Jens Axboe32960612019-09-23 11:05:34 -06003125 if (rw == READ) {
3126 nr = file->f_op->read(file, iovec.iov_base,
Jens Axboe0fef9482020-08-26 10:36:20 -06003127 iovec.iov_len, io_kiocb_ppos(kiocb));
Jens Axboe32960612019-09-23 11:05:34 -06003128 } else {
3129 nr = file->f_op->write(file, iovec.iov_base,
Jens Axboe0fef9482020-08-26 10:36:20 -06003130 iovec.iov_len, io_kiocb_ppos(kiocb));
Jens Axboe32960612019-09-23 11:05:34 -06003131 }
3132
3133 if (nr < 0) {
3134 if (!ret)
3135 ret = nr;
3136 break;
3137 }
3138 ret += nr;
3139 if (nr != iovec.iov_len)
3140 break;
Jens Axboe4017eb92020-10-22 14:14:12 -06003141 req->rw.len -= nr;
3142 req->rw.addr += nr;
Jens Axboe32960612019-09-23 11:05:34 -06003143 iov_iter_advance(iter, nr);
3144 }
3145
3146 return ret;
3147}
3148
Jens Axboeff6165b2020-08-13 09:47:43 -06003149static void io_req_map_rw(struct io_kiocb *req, const struct iovec *iovec,
3150 const struct iovec *fast_iov, struct iov_iter *iter)
Jens Axboef67676d2019-12-02 11:03:47 -07003151{
Jens Axboee8c2bc12020-08-15 18:44:09 -07003152 struct io_async_rw *rw = req->async_data;
Pavel Begunkovb64e3442020-07-13 22:59:18 +03003153
Jens Axboeff6165b2020-08-13 09:47:43 -06003154 memcpy(&rw->iter, iter, sizeof(*iter));
Pavel Begunkovafb87652020-09-06 00:45:46 +03003155 rw->free_iovec = iovec;
Jens Axboe227c0c92020-08-13 11:51:40 -06003156 rw->bytes_done = 0;
Jens Axboeff6165b2020-08-13 09:47:43 -06003157 /* can only be fixed buffers, no need to do anything */
Pavel Begunkov9c3a2052020-11-23 23:20:27 +00003158 if (iov_iter_is_bvec(iter))
Jens Axboeff6165b2020-08-13 09:47:43 -06003159 return;
Pavel Begunkovb64e3442020-07-13 22:59:18 +03003160 if (!iovec) {
Jens Axboeff6165b2020-08-13 09:47:43 -06003161 unsigned iov_off = 0;
3162
3163 rw->iter.iov = rw->fast_iov;
3164 if (iter->iov != fast_iov) {
3165 iov_off = iter->iov - fast_iov;
3166 rw->iter.iov += iov_off;
3167 }
3168 if (rw->fast_iov != fast_iov)
3169 memcpy(rw->fast_iov + iov_off, fast_iov + iov_off,
Xiaoguang Wang45097da2020-04-08 22:29:58 +08003170 sizeof(struct iovec) * iter->nr_segs);
Pavel Begunkov99bc4c32020-02-07 22:04:45 +03003171 } else {
3172 req->flags |= REQ_F_NEED_CLEANUP;
Jens Axboef67676d2019-12-02 11:03:47 -07003173 }
3174}
3175
Pavel Begunkov6cb78682021-02-28 22:35:17 +00003176static inline int io_alloc_async_data(struct io_kiocb *req)
Xiaoguang Wang3d9932a2020-03-27 15:36:52 +08003177{
Jens Axboee8c2bc12020-08-15 18:44:09 -07003178 WARN_ON_ONCE(!io_op_defs[req->opcode].async_size);
3179 req->async_data = kmalloc(io_op_defs[req->opcode].async_size, GFP_KERNEL);
3180 return req->async_data == NULL;
Xiaoguang Wang3d9932a2020-03-27 15:36:52 +08003181}
3182
Jens Axboeff6165b2020-08-13 09:47:43 -06003183static int io_setup_async_rw(struct io_kiocb *req, const struct iovec *iovec,
3184 const struct iovec *fast_iov,
Jens Axboe227c0c92020-08-13 11:51:40 -06003185 struct iov_iter *iter, bool force)
Jens Axboeb7bb4f72019-12-15 22:13:43 -07003186{
Pavel Begunkov26f05052021-02-28 22:35:18 +00003187 if (!force && !io_op_defs[req->opcode].needs_async_setup)
Jens Axboe74566df2020-01-13 19:23:24 -07003188 return 0;
Jens Axboee8c2bc12020-08-15 18:44:09 -07003189 if (!req->async_data) {
Pavel Begunkov6cb78682021-02-28 22:35:17 +00003190 if (io_alloc_async_data(req)) {
Pavel Begunkov6bf985d2021-02-04 13:52:01 +00003191 kfree(iovec);
Jens Axboe5d204bc2020-01-31 12:06:52 -07003192 return -ENOMEM;
Pavel Begunkov6bf985d2021-02-04 13:52:01 +00003193 }
Jens Axboeb7bb4f72019-12-15 22:13:43 -07003194
Jens Axboeff6165b2020-08-13 09:47:43 -06003195 io_req_map_rw(req, iovec, fast_iov, iter);
Jens Axboe5d204bc2020-01-31 12:06:52 -07003196 }
Jens Axboeb7bb4f72019-12-15 22:13:43 -07003197 return 0;
Jens Axboef67676d2019-12-02 11:03:47 -07003198}
3199
Pavel Begunkov73debe62020-09-30 22:57:54 +03003200static inline int io_rw_prep_async(struct io_kiocb *req, int rw)
Pavel Begunkovc3e330a2020-07-13 22:59:19 +03003201{
Jens Axboee8c2bc12020-08-15 18:44:09 -07003202 struct io_async_rw *iorw = req->async_data;
Pavel Begunkovf4bff102020-09-06 00:45:45 +03003203 struct iovec *iov = iorw->fast_iov;
Pavel Begunkov847595d2021-02-04 13:52:06 +00003204 int ret;
Pavel Begunkovc3e330a2020-07-13 22:59:19 +03003205
Pavel Begunkov2846c482020-11-07 13:16:27 +00003206 ret = io_import_iovec(rw, req, &iov, &iorw->iter, false);
Pavel Begunkovc3e330a2020-07-13 22:59:19 +03003207 if (unlikely(ret < 0))
3208 return ret;
3209
Pavel Begunkovab0b1962020-09-06 00:45:47 +03003210 iorw->bytes_done = 0;
3211 iorw->free_iovec = iov;
3212 if (iov)
3213 req->flags |= REQ_F_NEED_CLEANUP;
Pavel Begunkovc3e330a2020-07-13 22:59:19 +03003214 return 0;
3215}
3216
Pavel Begunkov73debe62020-09-30 22:57:54 +03003217static int io_read_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboef67676d2019-12-02 11:03:47 -07003218{
Jens Axboe3529d8c2019-12-19 18:24:38 -07003219 if (unlikely(!(req->file->f_mode & FMODE_READ)))
3220 return -EBADF;
Pavel Begunkov93642ef2021-02-18 18:29:44 +00003221 return io_prep_rw(req, sqe);
Jens Axboef67676d2019-12-02 11:03:47 -07003222}
3223
Jens Axboec1dd91d2020-08-03 16:43:59 -06003224/*
3225 * This is our waitqueue callback handler, registered through lock_page_async()
3226 * when we initially tried to do the IO with the iocb armed our waitqueue.
3227 * This gets called when the page is unlocked, and we generally expect that to
3228 * happen when the page IO is completed and the page is now uptodate. This will
3229 * queue a task_work based retry of the operation, attempting to copy the data
3230 * again. If the latter fails because the page was NOT uptodate, then we will
3231 * do a thread based blocking retry of the operation. That's the unexpected
3232 * slow path.
3233 */
Jens Axboebcf5a062020-05-22 09:24:42 -06003234static int io_async_buf_func(struct wait_queue_entry *wait, unsigned mode,
3235 int sync, void *arg)
3236{
3237 struct wait_page_queue *wpq;
3238 struct io_kiocb *req = wait->private;
Jens Axboebcf5a062020-05-22 09:24:42 -06003239 struct wait_page_key *key = arg;
Jens Axboebcf5a062020-05-22 09:24:42 -06003240
3241 wpq = container_of(wait, struct wait_page_queue, wait);
3242
Linus Torvaldscdc8fcb2020-08-03 13:01:22 -07003243 if (!wake_page_match(wpq, key))
3244 return 0;
3245
Hao Xuc8d317a2020-09-29 20:00:45 +08003246 req->rw.kiocb.ki_flags &= ~IOCB_WAITQ;
Jens Axboebcf5a062020-05-22 09:24:42 -06003247 list_del_init(&wait->entry);
Pavel Begunkov921b9052021-02-12 03:23:53 +00003248 io_req_task_queue(req);
Jens Axboebcf5a062020-05-22 09:24:42 -06003249 return 1;
3250}
3251
Jens Axboec1dd91d2020-08-03 16:43:59 -06003252/*
3253 * This controls whether a given IO request should be armed for async page
3254 * based retry. If we return false here, the request is handed to the async
3255 * worker threads for retry. If we're doing buffered reads on a regular file,
3256 * we prepare a private wait_page_queue entry and retry the operation. This
3257 * will either succeed because the page is now uptodate and unlocked, or it
3258 * will register a callback when the page is unlocked at IO completion. Through
3259 * that callback, io_uring uses task_work to setup a retry of the operation.
3260 * That retry will attempt the buffered read again. The retry will generally
3261 * succeed, or in rare cases where it fails, we then fall back to using the
3262 * async worker threads for a blocking retry.
3263 */
Jens Axboe227c0c92020-08-13 11:51:40 -06003264static bool io_rw_should_retry(struct io_kiocb *req)
Jens Axboebcf5a062020-05-22 09:24:42 -06003265{
Jens Axboee8c2bc12020-08-15 18:44:09 -07003266 struct io_async_rw *rw = req->async_data;
3267 struct wait_page_queue *wait = &rw->wpq;
Jens Axboebcf5a062020-05-22 09:24:42 -06003268 struct kiocb *kiocb = &req->rw.kiocb;
Jens Axboebcf5a062020-05-22 09:24:42 -06003269
3270 /* never retry for NOWAIT, we just complete with -EAGAIN */
3271 if (req->flags & REQ_F_NOWAIT)
3272 return false;
3273
Jens Axboe227c0c92020-08-13 11:51:40 -06003274 /* Only for buffered IO */
Jens Axboe3b2a4432020-08-16 10:58:43 -07003275 if (kiocb->ki_flags & (IOCB_DIRECT | IOCB_HIPRI))
Jens Axboebcf5a062020-05-22 09:24:42 -06003276 return false;
Jens Axboe3b2a4432020-08-16 10:58:43 -07003277
Jens Axboebcf5a062020-05-22 09:24:42 -06003278 /*
3279 * just use poll if we can, and don't attempt if the fs doesn't
3280 * support callback based unlocks
3281 */
3282 if (file_can_poll(req->file) || !(req->file->f_mode & FMODE_BUF_RASYNC))
3283 return false;
3284
Jens Axboe3b2a4432020-08-16 10:58:43 -07003285 wait->wait.func = io_async_buf_func;
3286 wait->wait.private = req;
3287 wait->wait.flags = 0;
3288 INIT_LIST_HEAD(&wait->wait.entry);
3289 kiocb->ki_flags |= IOCB_WAITQ;
Hao Xuc8d317a2020-09-29 20:00:45 +08003290 kiocb->ki_flags &= ~IOCB_NOWAIT;
Jens Axboe3b2a4432020-08-16 10:58:43 -07003291 kiocb->ki_waitq = wait;
Jens Axboe3b2a4432020-08-16 10:58:43 -07003292 return true;
Jens Axboebcf5a062020-05-22 09:24:42 -06003293}
3294
Pavel Begunkovaeab9502021-06-14 02:36:24 +01003295static inline int io_iter_do_read(struct io_kiocb *req, struct iov_iter *iter)
Jens Axboebcf5a062020-05-22 09:24:42 -06003296{
3297 if (req->file->f_op->read_iter)
3298 return call_read_iter(req->file, &req->rw.kiocb, iter);
Guoyu Huang2dd21112020-08-05 03:53:50 -07003299 else if (req->file->f_op->read)
Jens Axboe4017eb92020-10-22 14:14:12 -06003300 return loop_rw_iter(READ, req, iter);
Guoyu Huang2dd21112020-08-05 03:53:50 -07003301 else
3302 return -EINVAL;
Jens Axboebcf5a062020-05-22 09:24:42 -06003303}
3304
Pavel Begunkov889fca72021-02-10 00:03:09 +00003305static int io_read(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe2b188cc2019-01-07 10:46:33 -07003306{
3307 struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
Jens Axboe9adbd452019-12-20 08:45:55 -07003308 struct kiocb *kiocb = &req->rw.kiocb;
Jens Axboeff6165b2020-08-13 09:47:43 -06003309 struct iov_iter __iter, *iter = &__iter;
Jens Axboee8c2bc12020-08-15 18:44:09 -07003310 struct io_async_rw *rw = req->async_data;
Jens Axboe227c0c92020-08-13 11:51:40 -06003311 ssize_t io_size, ret, ret2;
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003312 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003313
Pavel Begunkov2846c482020-11-07 13:16:27 +00003314 if (rw) {
Jens Axboee8c2bc12020-08-15 18:44:09 -07003315 iter = &rw->iter;
Pavel Begunkov2846c482020-11-07 13:16:27 +00003316 iovec = NULL;
3317 } else {
3318 ret = io_import_iovec(READ, req, &iovec, iter, !force_nonblock);
3319 if (ret < 0)
3320 return ret;
3321 }
Pavel Begunkov632546c2020-11-07 13:16:26 +00003322 io_size = iov_iter_count(iter);
Pavel Begunkovfa15baf2020-08-01 13:50:02 +03003323 req->result = io_size;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003324
Jens Axboefd6c2e42019-12-18 12:19:41 -07003325 /* Ensure we clear previously set non-block flag */
3326 if (!force_nonblock)
Jens Axboe29de5f62020-02-20 09:56:08 -07003327 kiocb->ki_flags &= ~IOCB_NOWAIT;
Pavel Begunkova88fc402020-09-30 22:57:53 +03003328 else
3329 kiocb->ki_flags |= IOCB_NOWAIT;
3330
Pavel Begunkov24c74672020-06-21 13:09:51 +03003331 /* If the file doesn't support async, just async punt */
Pavel Begunkovb191e2d2021-08-09 13:04:03 +01003332 if (force_nonblock && !io_file_supports_nowait(req, READ)) {
Pavel Begunkov6713e7a2021-02-04 13:51:59 +00003333 ret = io_setup_async_rw(req, iovec, inline_vecs, iter, true);
Pavel Begunkov6bf985d2021-02-04 13:52:01 +00003334 return ret ?: -EAGAIN;
Pavel Begunkov6713e7a2021-02-04 13:51:59 +00003335 }
Jens Axboe9e645e112019-05-10 16:07:28 -06003336
Pavel Begunkov632546c2020-11-07 13:16:26 +00003337 ret = rw_verify_area(READ, req->file, io_kiocb_ppos(kiocb), io_size);
Pavel Begunkov5ea5dd42021-02-04 13:52:03 +00003338 if (unlikely(ret)) {
3339 kfree(iovec);
3340 return ret;
3341 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07003342
Jens Axboe227c0c92020-08-13 11:51:40 -06003343 ret = io_iter_do_read(req, iter);
Jens Axboe32960612019-09-23 11:05:34 -06003344
Jens Axboe230d50d2021-04-01 20:41:15 -06003345 if (ret == -EAGAIN || (req->flags & REQ_F_REISSUE)) {
Pavel Begunkov6ad7f232021-04-08 01:54:39 +01003346 req->flags &= ~REQ_F_REISSUE;
Jens Axboeeefdf302020-08-27 16:40:19 -06003347 /* IOPOLL retry should happen for io-wq threads */
3348 if (!force_nonblock && !(req->ctx->flags & IORING_SETUP_IOPOLL))
Jens Axboef91daf52020-08-15 15:58:42 -07003349 goto done;
Pavel Begunkov75c668c2021-02-04 13:52:05 +00003350 /* no retry on NONBLOCK nor RWF_NOWAIT */
3351 if (req->flags & REQ_F_NOWAIT)
Jens Axboe355afae2020-09-02 09:30:31 -06003352 goto done;
Jens Axboe84216312020-08-24 11:45:26 -06003353 /* some cases will consume bytes even on error returns */
Pavel Begunkov632546c2020-11-07 13:16:26 +00003354 iov_iter_revert(iter, io_size - iov_iter_count(iter));
Jens Axboef38c7e32020-09-25 15:23:43 -06003355 ret = 0;
Jens Axboe230d50d2021-04-01 20:41:15 -06003356 } else if (ret == -EIOCBQUEUED) {
3357 goto out_free;
Pavel Begunkov7335e3b2021-02-04 13:52:02 +00003358 } else if (ret <= 0 || ret == io_size || !force_nonblock ||
Pavel Begunkov75c668c2021-02-04 13:52:05 +00003359 (req->flags & REQ_F_NOWAIT) || !(req->flags & REQ_F_ISREG)) {
Pavel Begunkov7335e3b2021-02-04 13:52:02 +00003360 /* read all, failed, already did sync or don't want to retry */
Jens Axboe00d23d52020-08-25 12:59:22 -06003361 goto done;
Jens Axboe227c0c92020-08-13 11:51:40 -06003362 }
3363
Jens Axboe227c0c92020-08-13 11:51:40 -06003364 ret2 = io_setup_async_rw(req, iovec, inline_vecs, iter, true);
Pavel Begunkov6bf985d2021-02-04 13:52:01 +00003365 if (ret2)
3366 return ret2;
3367
Pavel Begunkovfe1cdd52021-02-17 21:02:36 +00003368 iovec = NULL;
Jens Axboee8c2bc12020-08-15 18:44:09 -07003369 rw = req->async_data;
Jens Axboe227c0c92020-08-13 11:51:40 -06003370 /* now use our persistent iterator, if we aren't already */
Jens Axboee8c2bc12020-08-15 18:44:09 -07003371 iter = &rw->iter;
Jens Axboe227c0c92020-08-13 11:51:40 -06003372
Pavel Begunkovb23df912021-02-04 13:52:04 +00003373 do {
3374 io_size -= ret;
3375 rw->bytes_done += ret;
3376 /* if we can retry, do so with the callbacks armed */
3377 if (!io_rw_should_retry(req)) {
3378 kiocb->ki_flags &= ~IOCB_WAITQ;
3379 return -EAGAIN;
3380 }
3381
3382 /*
3383 * Now retry read with the IOCB_WAITQ parts set in the iocb. If
3384 * we get -EIOCBQUEUED, then we'll get a notification when the
3385 * desired page gets unlocked. We can also get a partial read
3386 * here, and if we do, then just retry at the new offset.
3387 */
3388 ret = io_iter_do_read(req, iter);
3389 if (ret == -EIOCBQUEUED)
3390 return 0;
Jens Axboe227c0c92020-08-13 11:51:40 -06003391 /* we got some bytes, but not all. retry. */
Jens Axboeb5b0ecb2021-03-04 21:02:58 -07003392 kiocb->ki_flags &= ~IOCB_WAITQ;
Pavel Begunkovb23df912021-02-04 13:52:04 +00003393 } while (ret > 0 && ret < io_size);
Jens Axboe227c0c92020-08-13 11:51:40 -06003394done:
Pavel Begunkov889fca72021-02-10 00:03:09 +00003395 kiocb_done(kiocb, ret, issue_flags);
Pavel Begunkovfe1cdd52021-02-17 21:02:36 +00003396out_free:
3397 /* it's faster to check here then delegate to kfree */
3398 if (iovec)
3399 kfree(iovec);
Pavel Begunkov5ea5dd42021-02-04 13:52:03 +00003400 return 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003401}
3402
Pavel Begunkov73debe62020-09-30 22:57:54 +03003403static int io_write_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboef67676d2019-12-02 11:03:47 -07003404{
Jens Axboe3529d8c2019-12-19 18:24:38 -07003405 if (unlikely(!(req->file->f_mode & FMODE_WRITE)))
3406 return -EBADF;
Pavel Begunkov93642ef2021-02-18 18:29:44 +00003407 return io_prep_rw(req, sqe);
Jens Axboef67676d2019-12-02 11:03:47 -07003408}
3409
Pavel Begunkov889fca72021-02-10 00:03:09 +00003410static int io_write(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe2b188cc2019-01-07 10:46:33 -07003411{
3412 struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
Jens Axboe9adbd452019-12-20 08:45:55 -07003413 struct kiocb *kiocb = &req->rw.kiocb;
Jens Axboeff6165b2020-08-13 09:47:43 -06003414 struct iov_iter __iter, *iter = &__iter;
Jens Axboee8c2bc12020-08-15 18:44:09 -07003415 struct io_async_rw *rw = req->async_data;
Pavel Begunkovfa15baf2020-08-01 13:50:02 +03003416 ssize_t ret, ret2, io_size;
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003417 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003418
Pavel Begunkov2846c482020-11-07 13:16:27 +00003419 if (rw) {
Jens Axboee8c2bc12020-08-15 18:44:09 -07003420 iter = &rw->iter;
Pavel Begunkov2846c482020-11-07 13:16:27 +00003421 iovec = NULL;
3422 } else {
3423 ret = io_import_iovec(WRITE, req, &iovec, iter, !force_nonblock);
3424 if (ret < 0)
3425 return ret;
3426 }
Pavel Begunkov632546c2020-11-07 13:16:26 +00003427 io_size = iov_iter_count(iter);
Pavel Begunkovfa15baf2020-08-01 13:50:02 +03003428 req->result = io_size;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003429
Jens Axboefd6c2e42019-12-18 12:19:41 -07003430 /* Ensure we clear previously set non-block flag */
3431 if (!force_nonblock)
Pavel Begunkova88fc402020-09-30 22:57:53 +03003432 kiocb->ki_flags &= ~IOCB_NOWAIT;
3433 else
3434 kiocb->ki_flags |= IOCB_NOWAIT;
Jens Axboefd6c2e42019-12-18 12:19:41 -07003435
Pavel Begunkov24c74672020-06-21 13:09:51 +03003436 /* If the file doesn't support async, just async punt */
Pavel Begunkovb191e2d2021-08-09 13:04:03 +01003437 if (force_nonblock && !io_file_supports_nowait(req, WRITE))
Jens Axboef67676d2019-12-02 11:03:47 -07003438 goto copy_iov;
Jens Axboef67676d2019-12-02 11:03:47 -07003439
Jens Axboe10d59342019-12-09 20:16:22 -07003440 /* file path doesn't support NOWAIT for non-direct_IO */
3441 if (force_nonblock && !(kiocb->ki_flags & IOCB_DIRECT) &&
3442 (req->flags & REQ_F_ISREG))
Jens Axboef67676d2019-12-02 11:03:47 -07003443 goto copy_iov;
Jens Axboe9e645e112019-05-10 16:07:28 -06003444
Pavel Begunkov632546c2020-11-07 13:16:26 +00003445 ret = rw_verify_area(WRITE, req->file, io_kiocb_ppos(kiocb), io_size);
Pavel Begunkovfa15baf2020-08-01 13:50:02 +03003446 if (unlikely(ret))
3447 goto out_free;
Roman Penyaev9bf79332019-03-25 20:09:24 +01003448
Pavel Begunkovfa15baf2020-08-01 13:50:02 +03003449 /*
3450 * Open-code file_start_write here to grab freeze protection,
3451 * which will be released by another thread in
3452 * io_complete_rw(). Fool lockdep by telling it the lock got
3453 * released so that it doesn't complain about the held lock when
3454 * we return to userspace.
3455 */
3456 if (req->flags & REQ_F_ISREG) {
Darrick J. Wong8a3c84b2020-11-10 16:50:21 -08003457 sb_start_write(file_inode(req->file)->i_sb);
Pavel Begunkovfa15baf2020-08-01 13:50:02 +03003458 __sb_writers_release(file_inode(req->file)->i_sb,
3459 SB_FREEZE_WRITE);
3460 }
3461 kiocb->ki_flags |= IOCB_WRITE;
Roman Penyaev9bf79332019-03-25 20:09:24 +01003462
Pavel Begunkovfa15baf2020-08-01 13:50:02 +03003463 if (req->file->f_op->write_iter)
Jens Axboeff6165b2020-08-13 09:47:43 -06003464 ret2 = call_write_iter(req->file, kiocb, iter);
Guoyu Huang2dd21112020-08-05 03:53:50 -07003465 else if (req->file->f_op->write)
Jens Axboe4017eb92020-10-22 14:14:12 -06003466 ret2 = loop_rw_iter(WRITE, req, iter);
Guoyu Huang2dd21112020-08-05 03:53:50 -07003467 else
3468 ret2 = -EINVAL;
Jens Axboe4ed734b2020-03-20 11:23:41 -06003469
Pavel Begunkov6ad7f232021-04-08 01:54:39 +01003470 if (req->flags & REQ_F_REISSUE) {
3471 req->flags &= ~REQ_F_REISSUE;
Jens Axboe230d50d2021-04-01 20:41:15 -06003472 ret2 = -EAGAIN;
Pavel Begunkov6ad7f232021-04-08 01:54:39 +01003473 }
Jens Axboe230d50d2021-04-01 20:41:15 -06003474
Pavel Begunkovfa15baf2020-08-01 13:50:02 +03003475 /*
3476 * Raw bdev writes will return -EOPNOTSUPP for IOCB_NOWAIT. Just
3477 * retry them without IOCB_NOWAIT.
3478 */
3479 if (ret2 == -EOPNOTSUPP && (kiocb->ki_flags & IOCB_NOWAIT))
3480 ret2 = -EAGAIN;
Pavel Begunkov75c668c2021-02-04 13:52:05 +00003481 /* no retry on NONBLOCK nor RWF_NOWAIT */
3482 if (ret2 == -EAGAIN && (req->flags & REQ_F_NOWAIT))
Jens Axboe355afae2020-09-02 09:30:31 -06003483 goto done;
Pavel Begunkovfa15baf2020-08-01 13:50:02 +03003484 if (!force_nonblock || ret2 != -EAGAIN) {
Jens Axboeeefdf302020-08-27 16:40:19 -06003485 /* IOPOLL retry should happen for io-wq threads */
3486 if ((req->ctx->flags & IORING_SETUP_IOPOLL) && ret2 == -EAGAIN)
3487 goto copy_iov;
Jens Axboe355afae2020-09-02 09:30:31 -06003488done:
Pavel Begunkov889fca72021-02-10 00:03:09 +00003489 kiocb_done(kiocb, ret2, issue_flags);
Pavel Begunkovfa15baf2020-08-01 13:50:02 +03003490 } else {
Jens Axboef67676d2019-12-02 11:03:47 -07003491copy_iov:
Jens Axboe84216312020-08-24 11:45:26 -06003492 /* some cases will consume bytes even on error returns */
Pavel Begunkov632546c2020-11-07 13:16:26 +00003493 iov_iter_revert(iter, io_size - iov_iter_count(iter));
Jens Axboe227c0c92020-08-13 11:51:40 -06003494 ret = io_setup_async_rw(req, iovec, inline_vecs, iter, false);
Pavel Begunkov6bf985d2021-02-04 13:52:01 +00003495 return ret ?: -EAGAIN;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003496 }
Jens Axboe31b51512019-01-18 22:56:34 -07003497out_free:
Pavel Begunkovf261c162020-08-20 11:34:10 +03003498 /* it's reportedly faster than delegating the null check to kfree() */
Pavel Begunkov252917c2020-07-13 22:59:20 +03003499 if (iovec)
Xiaoguang Wang6f2cc162020-06-18 15:01:56 +08003500 kfree(iovec);
Jens Axboe2b188cc2019-01-07 10:46:33 -07003501 return ret;
3502}
3503
Jens Axboe80a261f2020-09-28 14:23:58 -06003504static int io_renameat_prep(struct io_kiocb *req,
3505 const struct io_uring_sqe *sqe)
3506{
3507 struct io_rename *ren = &req->rename;
3508 const char __user *oldf, *newf;
3509
Jens Axboeed7eb252021-06-23 09:04:13 -06003510 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3511 return -EINVAL;
3512 if (sqe->ioprio || sqe->buf_index)
3513 return -EINVAL;
Jens Axboe80a261f2020-09-28 14:23:58 -06003514 if (unlikely(req->flags & REQ_F_FIXED_FILE))
3515 return -EBADF;
3516
3517 ren->old_dfd = READ_ONCE(sqe->fd);
3518 oldf = u64_to_user_ptr(READ_ONCE(sqe->addr));
3519 newf = u64_to_user_ptr(READ_ONCE(sqe->addr2));
3520 ren->new_dfd = READ_ONCE(sqe->len);
3521 ren->flags = READ_ONCE(sqe->rename_flags);
3522
3523 ren->oldpath = getname(oldf);
3524 if (IS_ERR(ren->oldpath))
3525 return PTR_ERR(ren->oldpath);
3526
3527 ren->newpath = getname(newf);
3528 if (IS_ERR(ren->newpath)) {
3529 putname(ren->oldpath);
3530 return PTR_ERR(ren->newpath);
3531 }
3532
3533 req->flags |= REQ_F_NEED_CLEANUP;
3534 return 0;
3535}
3536
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003537static int io_renameat(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe80a261f2020-09-28 14:23:58 -06003538{
3539 struct io_rename *ren = &req->rename;
3540 int ret;
3541
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003542 if (issue_flags & IO_URING_F_NONBLOCK)
Jens Axboe80a261f2020-09-28 14:23:58 -06003543 return -EAGAIN;
3544
3545 ret = do_renameat2(ren->old_dfd, ren->oldpath, ren->new_dfd,
3546 ren->newpath, ren->flags);
3547
3548 req->flags &= ~REQ_F_NEED_CLEANUP;
3549 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01003550 req_set_fail(req);
Jens Axboe80a261f2020-09-28 14:23:58 -06003551 io_req_complete(req, ret);
3552 return 0;
3553}
3554
Jens Axboe14a11432020-09-28 14:27:37 -06003555static int io_unlinkat_prep(struct io_kiocb *req,
3556 const struct io_uring_sqe *sqe)
3557{
3558 struct io_unlink *un = &req->unlink;
3559 const char __user *fname;
3560
Jens Axboe22634bc2021-06-23 09:07:45 -06003561 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3562 return -EINVAL;
3563 if (sqe->ioprio || sqe->off || sqe->len || sqe->buf_index)
3564 return -EINVAL;
Jens Axboe14a11432020-09-28 14:27:37 -06003565 if (unlikely(req->flags & REQ_F_FIXED_FILE))
3566 return -EBADF;
3567
3568 un->dfd = READ_ONCE(sqe->fd);
3569
3570 un->flags = READ_ONCE(sqe->unlink_flags);
3571 if (un->flags & ~AT_REMOVEDIR)
3572 return -EINVAL;
3573
3574 fname = u64_to_user_ptr(READ_ONCE(sqe->addr));
3575 un->filename = getname(fname);
3576 if (IS_ERR(un->filename))
3577 return PTR_ERR(un->filename);
3578
3579 req->flags |= REQ_F_NEED_CLEANUP;
3580 return 0;
3581}
3582
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003583static int io_unlinkat(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe14a11432020-09-28 14:27:37 -06003584{
3585 struct io_unlink *un = &req->unlink;
3586 int ret;
3587
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003588 if (issue_flags & IO_URING_F_NONBLOCK)
Jens Axboe14a11432020-09-28 14:27:37 -06003589 return -EAGAIN;
3590
3591 if (un->flags & AT_REMOVEDIR)
3592 ret = do_rmdir(un->dfd, un->filename);
3593 else
3594 ret = do_unlinkat(un->dfd, un->filename);
3595
3596 req->flags &= ~REQ_F_NEED_CLEANUP;
3597 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01003598 req_set_fail(req);
Jens Axboe14a11432020-09-28 14:27:37 -06003599 io_req_complete(req, ret);
3600 return 0;
3601}
3602
Jens Axboe36f4fa62020-09-05 11:14:22 -06003603static int io_shutdown_prep(struct io_kiocb *req,
3604 const struct io_uring_sqe *sqe)
3605{
3606#if defined(CONFIG_NET)
3607 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3608 return -EINVAL;
3609 if (sqe->ioprio || sqe->off || sqe->addr || sqe->rw_flags ||
3610 sqe->buf_index)
3611 return -EINVAL;
3612
3613 req->shutdown.how = READ_ONCE(sqe->len);
3614 return 0;
3615#else
3616 return -EOPNOTSUPP;
3617#endif
3618}
3619
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003620static int io_shutdown(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe36f4fa62020-09-05 11:14:22 -06003621{
3622#if defined(CONFIG_NET)
3623 struct socket *sock;
3624 int ret;
3625
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003626 if (issue_flags & IO_URING_F_NONBLOCK)
Jens Axboe36f4fa62020-09-05 11:14:22 -06003627 return -EAGAIN;
3628
Linus Torvalds48aba792020-12-16 12:44:05 -08003629 sock = sock_from_file(req->file);
Jens Axboe36f4fa62020-09-05 11:14:22 -06003630 if (unlikely(!sock))
Linus Torvalds48aba792020-12-16 12:44:05 -08003631 return -ENOTSOCK;
Jens Axboe36f4fa62020-09-05 11:14:22 -06003632
3633 ret = __sys_shutdown_sock(sock, req->shutdown.how);
Jens Axboea1464682020-12-14 20:57:27 -07003634 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01003635 req_set_fail(req);
Jens Axboe36f4fa62020-09-05 11:14:22 -06003636 io_req_complete(req, ret);
3637 return 0;
3638#else
3639 return -EOPNOTSUPP;
3640#endif
3641}
3642
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03003643static int __io_splice_prep(struct io_kiocb *req,
3644 const struct io_uring_sqe *sqe)
Pavel Begunkov7d67af22020-02-24 11:32:45 +03003645{
Pavel Begunkovfe7e3252021-06-24 15:09:57 +01003646 struct io_splice *sp = &req->splice;
Pavel Begunkov7d67af22020-02-24 11:32:45 +03003647 unsigned int valid_flags = SPLICE_F_FD_IN_FIXED | SPLICE_F_ALL;
Pavel Begunkov7d67af22020-02-24 11:32:45 +03003648
Pavel Begunkov3232dd02020-06-03 18:03:22 +03003649 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3650 return -EINVAL;
Pavel Begunkov7d67af22020-02-24 11:32:45 +03003651
3652 sp->file_in = NULL;
Pavel Begunkov7d67af22020-02-24 11:32:45 +03003653 sp->len = READ_ONCE(sqe->len);
3654 sp->flags = READ_ONCE(sqe->splice_flags);
3655
3656 if (unlikely(sp->flags & ~valid_flags))
3657 return -EINVAL;
3658
Pavel Begunkov62906e82021-08-10 14:52:47 +01003659 sp->file_in = io_file_get(req->ctx, req, READ_ONCE(sqe->splice_fd_in),
Pavel Begunkov8371adf2020-10-10 18:34:08 +01003660 (sp->flags & SPLICE_F_FD_IN_FIXED));
3661 if (!sp->file_in)
3662 return -EBADF;
Pavel Begunkov7d67af22020-02-24 11:32:45 +03003663 req->flags |= REQ_F_NEED_CLEANUP;
Pavel Begunkov7d67af22020-02-24 11:32:45 +03003664 return 0;
3665}
3666
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03003667static int io_tee_prep(struct io_kiocb *req,
3668 const struct io_uring_sqe *sqe)
3669{
3670 if (READ_ONCE(sqe->splice_off_in) || READ_ONCE(sqe->off))
3671 return -EINVAL;
3672 return __io_splice_prep(req, sqe);
3673}
3674
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003675static int io_tee(struct io_kiocb *req, unsigned int issue_flags)
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03003676{
3677 struct io_splice *sp = &req->splice;
3678 struct file *in = sp->file_in;
3679 struct file *out = sp->file_out;
3680 unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED;
3681 long ret = 0;
3682
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003683 if (issue_flags & IO_URING_F_NONBLOCK)
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03003684 return -EAGAIN;
3685 if (sp->len)
3686 ret = do_tee(in, out, sp->len, flags);
3687
Pavel Begunkove1d767f2021-03-19 17:22:43 +00003688 if (!(sp->flags & SPLICE_F_FD_IN_FIXED))
3689 io_put_file(in);
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03003690 req->flags &= ~REQ_F_NEED_CLEANUP;
3691
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03003692 if (ret != sp->len)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01003693 req_set_fail(req);
Jens Axboee1e16092020-06-22 09:17:17 -06003694 io_req_complete(req, ret);
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03003695 return 0;
3696}
3697
3698static int io_splice_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
3699{
Pavel Begunkovfe7e3252021-06-24 15:09:57 +01003700 struct io_splice *sp = &req->splice;
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03003701
3702 sp->off_in = READ_ONCE(sqe->splice_off_in);
3703 sp->off_out = READ_ONCE(sqe->off);
3704 return __io_splice_prep(req, sqe);
3705}
3706
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003707static int io_splice(struct io_kiocb *req, unsigned int issue_flags)
Pavel Begunkov7d67af22020-02-24 11:32:45 +03003708{
3709 struct io_splice *sp = &req->splice;
3710 struct file *in = sp->file_in;
3711 struct file *out = sp->file_out;
3712 unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED;
3713 loff_t *poff_in, *poff_out;
Pavel Begunkovc9687422020-05-04 23:00:54 +03003714 long ret = 0;
Pavel Begunkov7d67af22020-02-24 11:32:45 +03003715
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003716 if (issue_flags & IO_URING_F_NONBLOCK)
Pavel Begunkov2fb3e822020-05-01 17:09:38 +03003717 return -EAGAIN;
Pavel Begunkov7d67af22020-02-24 11:32:45 +03003718
3719 poff_in = (sp->off_in == -1) ? NULL : &sp->off_in;
3720 poff_out = (sp->off_out == -1) ? NULL : &sp->off_out;
Pavel Begunkovc9687422020-05-04 23:00:54 +03003721
Jens Axboe948a7742020-05-17 14:21:38 -06003722 if (sp->len)
Pavel Begunkovc9687422020-05-04 23:00:54 +03003723 ret = do_splice(in, poff_in, out, poff_out, sp->len, flags);
Pavel Begunkov7d67af22020-02-24 11:32:45 +03003724
Pavel Begunkove1d767f2021-03-19 17:22:43 +00003725 if (!(sp->flags & SPLICE_F_FD_IN_FIXED))
3726 io_put_file(in);
Pavel Begunkov7d67af22020-02-24 11:32:45 +03003727 req->flags &= ~REQ_F_NEED_CLEANUP;
3728
Pavel Begunkov7d67af22020-02-24 11:32:45 +03003729 if (ret != sp->len)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01003730 req_set_fail(req);
Jens Axboee1e16092020-06-22 09:17:17 -06003731 io_req_complete(req, ret);
Pavel Begunkov7d67af22020-02-24 11:32:45 +03003732 return 0;
3733}
3734
Jens Axboe2b188cc2019-01-07 10:46:33 -07003735/*
3736 * IORING_OP_NOP just posts a completion event, nothing else.
3737 */
Pavel Begunkov889fca72021-02-10 00:03:09 +00003738static int io_nop(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe2b188cc2019-01-07 10:46:33 -07003739{
3740 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003741
Jens Axboedef596e2019-01-09 08:59:42 -07003742 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
3743 return -EINVAL;
3744
Pavel Begunkov889fca72021-02-10 00:03:09 +00003745 __io_req_complete(req, issue_flags, 0, 0);
Jens Axboe2b188cc2019-01-07 10:46:33 -07003746 return 0;
3747}
3748
Pavel Begunkov1155c762021-02-18 18:29:38 +00003749static int io_fsync_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Christoph Hellwigc992fe22019-01-11 09:43:02 -07003750{
Jens Axboe6b063142019-01-10 22:13:58 -07003751 struct io_ring_ctx *ctx = req->ctx;
Christoph Hellwigc992fe22019-01-11 09:43:02 -07003752
Jens Axboe09bb8392019-03-13 12:39:28 -06003753 if (!req->file)
3754 return -EBADF;
Christoph Hellwigc992fe22019-01-11 09:43:02 -07003755
Jens Axboe6b063142019-01-10 22:13:58 -07003756 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
Jens Axboedef596e2019-01-09 08:59:42 -07003757 return -EINVAL;
Jens Axboeedafcce2019-01-09 09:16:05 -07003758 if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index))
Christoph Hellwigc992fe22019-01-11 09:43:02 -07003759 return -EINVAL;
3760
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07003761 req->sync.flags = READ_ONCE(sqe->fsync_flags);
3762 if (unlikely(req->sync.flags & ~IORING_FSYNC_DATASYNC))
3763 return -EINVAL;
3764
3765 req->sync.off = READ_ONCE(sqe->off);
3766 req->sync.len = READ_ONCE(sqe->len);
Christoph Hellwigc992fe22019-01-11 09:43:02 -07003767 return 0;
3768}
3769
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003770static int io_fsync(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe78912932020-01-14 22:09:06 -07003771{
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07003772 loff_t end = req->sync.off + req->sync.len;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07003773 int ret;
3774
Pavel Begunkovac45abc2020-06-08 21:08:18 +03003775 /* fsync always requires a blocking context */
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003776 if (issue_flags & IO_URING_F_NONBLOCK)
Pavel Begunkovac45abc2020-06-08 21:08:18 +03003777 return -EAGAIN;
3778
Jens Axboe9adbd452019-12-20 08:45:55 -07003779 ret = vfs_fsync_range(req->file, req->sync.off,
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07003780 end > 0 ? end : LLONG_MAX,
3781 req->sync.flags & IORING_FSYNC_DATASYNC);
3782 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01003783 req_set_fail(req);
Jens Axboee1e16092020-06-22 09:17:17 -06003784 io_req_complete(req, ret);
Christoph Hellwigc992fe22019-01-11 09:43:02 -07003785 return 0;
3786}
3787
Jens Axboed63d1b52019-12-10 10:38:56 -07003788static int io_fallocate_prep(struct io_kiocb *req,
3789 const struct io_uring_sqe *sqe)
3790{
3791 if (sqe->ioprio || sqe->buf_index || sqe->rw_flags)
3792 return -EINVAL;
Pavel Begunkov3232dd02020-06-03 18:03:22 +03003793 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3794 return -EINVAL;
Jens Axboed63d1b52019-12-10 10:38:56 -07003795
3796 req->sync.off = READ_ONCE(sqe->off);
3797 req->sync.len = READ_ONCE(sqe->addr);
3798 req->sync.mode = READ_ONCE(sqe->len);
3799 return 0;
3800}
3801
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003802static int io_fallocate(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboed63d1b52019-12-10 10:38:56 -07003803{
Pavel Begunkovac45abc2020-06-08 21:08:18 +03003804 int ret;
Jens Axboed63d1b52019-12-10 10:38:56 -07003805
Pavel Begunkovac45abc2020-06-08 21:08:18 +03003806 /* fallocate always requiring blocking context */
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003807 if (issue_flags & IO_URING_F_NONBLOCK)
Pavel Begunkovac45abc2020-06-08 21:08:18 +03003808 return -EAGAIN;
Pavel Begunkovac45abc2020-06-08 21:08:18 +03003809 ret = vfs_fallocate(req->file, req->sync.mode, req->sync.off,
3810 req->sync.len);
Pavel Begunkovac45abc2020-06-08 21:08:18 +03003811 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01003812 req_set_fail(req);
Jens Axboee1e16092020-06-22 09:17:17 -06003813 io_req_complete(req, ret);
Jens Axboed63d1b52019-12-10 10:38:56 -07003814 return 0;
3815}
3816
Pavel Begunkovec65fea2020-06-03 18:03:24 +03003817static int __io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboe15b71ab2019-12-11 11:20:36 -07003818{
Jens Axboef8748882020-01-08 17:47:02 -07003819 const char __user *fname;
Jens Axboe15b71ab2019-12-11 11:20:36 -07003820 int ret;
3821
Pavel Begunkovd3fddf62021-08-09 13:04:16 +01003822 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3823 return -EINVAL;
Pavel Begunkovec65fea2020-06-03 18:03:24 +03003824 if (unlikely(sqe->ioprio || sqe->buf_index))
Jens Axboe15b71ab2019-12-11 11:20:36 -07003825 return -EINVAL;
Pavel Begunkovec65fea2020-06-03 18:03:24 +03003826 if (unlikely(req->flags & REQ_F_FIXED_FILE))
Jens Axboecf3040c2020-02-06 21:31:40 -07003827 return -EBADF;
Jens Axboe15b71ab2019-12-11 11:20:36 -07003828
Pavel Begunkovec65fea2020-06-03 18:03:24 +03003829 /* open.how should be already initialised */
3830 if (!(req->open.how.flags & O_PATH) && force_o_largefile())
Jens Axboe08a1d26eb2020-04-08 09:20:54 -06003831 req->open.how.flags |= O_LARGEFILE;
Jens Axboe15b71ab2019-12-11 11:20:36 -07003832
Pavel Begunkov25e72d12020-06-03 18:03:23 +03003833 req->open.dfd = READ_ONCE(sqe->fd);
3834 fname = u64_to_user_ptr(READ_ONCE(sqe->addr));
Jens Axboef8748882020-01-08 17:47:02 -07003835 req->open.filename = getname(fname);
Jens Axboe15b71ab2019-12-11 11:20:36 -07003836 if (IS_ERR(req->open.filename)) {
3837 ret = PTR_ERR(req->open.filename);
3838 req->open.filename = NULL;
3839 return ret;
3840 }
Jens Axboe4022e7a2020-03-19 19:23:18 -06003841 req->open.nofile = rlimit(RLIMIT_NOFILE);
Pavel Begunkov8fef80b2020-02-07 23:59:53 +03003842 req->flags |= REQ_F_NEED_CLEANUP;
Jens Axboe15b71ab2019-12-11 11:20:36 -07003843 return 0;
3844}
3845
Pavel Begunkovec65fea2020-06-03 18:03:24 +03003846static int io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
3847{
Pavel Begunkovd3fddf62021-08-09 13:04:16 +01003848 u64 mode = READ_ONCE(sqe->len);
3849 u64 flags = READ_ONCE(sqe->open_flags);
Pavel Begunkovec65fea2020-06-03 18:03:24 +03003850
Pavel Begunkovec65fea2020-06-03 18:03:24 +03003851 req->open.how = build_open_how(flags, mode);
3852 return __io_openat_prep(req, sqe);
3853}
3854
Jens Axboecebdb982020-01-08 17:59:24 -07003855static int io_openat2_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
3856{
3857 struct open_how __user *how;
Jens Axboecebdb982020-01-08 17:59:24 -07003858 size_t len;
3859 int ret;
3860
Jens Axboecebdb982020-01-08 17:59:24 -07003861 how = u64_to_user_ptr(READ_ONCE(sqe->addr2));
3862 len = READ_ONCE(sqe->len);
Jens Axboecebdb982020-01-08 17:59:24 -07003863 if (len < OPEN_HOW_SIZE_VER0)
3864 return -EINVAL;
3865
3866 ret = copy_struct_from_user(&req->open.how, sizeof(req->open.how), how,
3867 len);
3868 if (ret)
3869 return ret;
3870
Pavel Begunkovec65fea2020-06-03 18:03:24 +03003871 return __io_openat_prep(req, sqe);
Jens Axboecebdb982020-01-08 17:59:24 -07003872}
3873
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003874static int io_openat2(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe15b71ab2019-12-11 11:20:36 -07003875{
3876 struct open_flags op;
Jens Axboe15b71ab2019-12-11 11:20:36 -07003877 struct file *file;
Jens Axboe3a81fd02020-12-10 12:25:36 -07003878 bool nonblock_set;
3879 bool resolve_nonblock;
Jens Axboe15b71ab2019-12-11 11:20:36 -07003880 int ret;
3881
Jens Axboecebdb982020-01-08 17:59:24 -07003882 ret = build_open_flags(&req->open.how, &op);
Jens Axboe15b71ab2019-12-11 11:20:36 -07003883 if (ret)
3884 goto err;
Jens Axboe3a81fd02020-12-10 12:25:36 -07003885 nonblock_set = op.open_flag & O_NONBLOCK;
3886 resolve_nonblock = req->open.how.resolve & RESOLVE_CACHED;
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003887 if (issue_flags & IO_URING_F_NONBLOCK) {
Jens Axboe3a81fd02020-12-10 12:25:36 -07003888 /*
3889 * Don't bother trying for O_TRUNC, O_CREAT, or O_TMPFILE open,
3890 * it'll always -EAGAIN
3891 */
3892 if (req->open.how.flags & (O_TRUNC | O_CREAT | O_TMPFILE))
3893 return -EAGAIN;
3894 op.lookup_flags |= LOOKUP_CACHED;
3895 op.open_flag |= O_NONBLOCK;
3896 }
Jens Axboe15b71ab2019-12-11 11:20:36 -07003897
Jens Axboe4022e7a2020-03-19 19:23:18 -06003898 ret = __get_unused_fd_flags(req->open.how.flags, req->open.nofile);
Jens Axboe15b71ab2019-12-11 11:20:36 -07003899 if (ret < 0)
3900 goto err;
3901
3902 file = do_filp_open(req->open.dfd, req->open.filename, &op);
Pavel Begunkov12dcb58a2021-06-24 15:10:00 +01003903 if (IS_ERR(file)) {
Jens Axboe3a81fd02020-12-10 12:25:36 -07003904 /*
Pavel Begunkov12dcb58a2021-06-24 15:10:00 +01003905 * We could hang on to this 'fd' on retrying, but seems like
3906 * marginal gain for something that is now known to be a slower
3907 * path. So just put it, and we'll get a new one when we retry.
Jens Axboe3a81fd02020-12-10 12:25:36 -07003908 */
3909 put_unused_fd(ret);
Pavel Begunkov12dcb58a2021-06-24 15:10:00 +01003910
3911 ret = PTR_ERR(file);
3912 /* only retry if RESOLVE_CACHED wasn't already set by application */
3913 if (ret == -EAGAIN &&
3914 (!resolve_nonblock && (issue_flags & IO_URING_F_NONBLOCK)))
3915 return -EAGAIN;
3916 goto err;
Jens Axboe3a81fd02020-12-10 12:25:36 -07003917 }
3918
Pavel Begunkov12dcb58a2021-06-24 15:10:00 +01003919 if ((issue_flags & IO_URING_F_NONBLOCK) && !nonblock_set)
3920 file->f_flags &= ~O_NONBLOCK;
3921 fsnotify_open(file);
3922 fd_install(ret, file);
Jens Axboe15b71ab2019-12-11 11:20:36 -07003923err:
3924 putname(req->open.filename);
Pavel Begunkov8fef80b2020-02-07 23:59:53 +03003925 req->flags &= ~REQ_F_NEED_CLEANUP;
Jens Axboe15b71ab2019-12-11 11:20:36 -07003926 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01003927 req_set_fail(req);
Pavel Begunkov0bdf3392021-04-11 01:46:29 +01003928 __io_req_complete(req, issue_flags, ret, 0);
Jens Axboe15b71ab2019-12-11 11:20:36 -07003929 return 0;
3930}
3931
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003932static int io_openat(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboecebdb982020-01-08 17:59:24 -07003933{
Pavel Begunkove45cff52021-02-28 22:35:14 +00003934 return io_openat2(req, issue_flags);
Jens Axboecebdb982020-01-08 17:59:24 -07003935}
3936
Jens Axboe067524e2020-03-02 16:32:28 -07003937static int io_remove_buffers_prep(struct io_kiocb *req,
3938 const struct io_uring_sqe *sqe)
3939{
3940 struct io_provide_buf *p = &req->pbuf;
3941 u64 tmp;
3942
3943 if (sqe->ioprio || sqe->rw_flags || sqe->addr || sqe->len || sqe->off)
3944 return -EINVAL;
3945
3946 tmp = READ_ONCE(sqe->fd);
3947 if (!tmp || tmp > USHRT_MAX)
3948 return -EINVAL;
3949
3950 memset(p, 0, sizeof(*p));
3951 p->nbufs = tmp;
3952 p->bgid = READ_ONCE(sqe->buf_group);
3953 return 0;
3954}
3955
3956static int __io_remove_buffers(struct io_ring_ctx *ctx, struct io_buffer *buf,
3957 int bgid, unsigned nbufs)
3958{
3959 unsigned i = 0;
3960
3961 /* shouldn't happen */
3962 if (!nbufs)
3963 return 0;
3964
3965 /* the head kbuf is the list itself */
3966 while (!list_empty(&buf->list)) {
3967 struct io_buffer *nxt;
3968
3969 nxt = list_first_entry(&buf->list, struct io_buffer, list);
3970 list_del(&nxt->list);
3971 kfree(nxt);
3972 if (++i == nbufs)
3973 return i;
3974 }
3975 i++;
3976 kfree(buf);
Jens Axboe9e15c3a2021-03-13 12:29:43 -07003977 xa_erase(&ctx->io_buffers, bgid);
Jens Axboe067524e2020-03-02 16:32:28 -07003978
3979 return i;
3980}
3981
Pavel Begunkov889fca72021-02-10 00:03:09 +00003982static int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe067524e2020-03-02 16:32:28 -07003983{
3984 struct io_provide_buf *p = &req->pbuf;
3985 struct io_ring_ctx *ctx = req->ctx;
3986 struct io_buffer *head;
3987 int ret = 0;
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003988 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
Jens Axboe067524e2020-03-02 16:32:28 -07003989
3990 io_ring_submit_lock(ctx, !force_nonblock);
3991
3992 lockdep_assert_held(&ctx->uring_lock);
3993
3994 ret = -ENOENT;
Jens Axboe9e15c3a2021-03-13 12:29:43 -07003995 head = xa_load(&ctx->io_buffers, p->bgid);
Jens Axboe067524e2020-03-02 16:32:28 -07003996 if (head)
3997 ret = __io_remove_buffers(ctx, head, p->bgid, p->nbufs);
Jens Axboe067524e2020-03-02 16:32:28 -07003998 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01003999 req_set_fail(req);
Pavel Begunkov31bff9a2020-12-06 22:22:43 +00004000
Pavel Begunkov9fb8cb42021-02-28 22:35:13 +00004001 /* complete before unlock, IOPOLL may need the lock */
4002 __io_req_complete(req, issue_flags, ret, 0);
4003 io_ring_submit_unlock(ctx, !force_nonblock);
Jens Axboe067524e2020-03-02 16:32:28 -07004004 return 0;
4005}
4006
Jens Axboeddf0322d2020-02-23 16:41:33 -07004007static int io_provide_buffers_prep(struct io_kiocb *req,
4008 const struct io_uring_sqe *sqe)
4009{
Pavel Begunkov38134ad2021-04-15 13:07:39 +01004010 unsigned long size, tmp_check;
Jens Axboeddf0322d2020-02-23 16:41:33 -07004011 struct io_provide_buf *p = &req->pbuf;
4012 u64 tmp;
4013
4014 if (sqe->ioprio || sqe->rw_flags)
4015 return -EINVAL;
4016
4017 tmp = READ_ONCE(sqe->fd);
4018 if (!tmp || tmp > USHRT_MAX)
4019 return -E2BIG;
4020 p->nbufs = tmp;
4021 p->addr = READ_ONCE(sqe->addr);
4022 p->len = READ_ONCE(sqe->len);
4023
Pavel Begunkov38134ad2021-04-15 13:07:39 +01004024 if (check_mul_overflow((unsigned long)p->len, (unsigned long)p->nbufs,
4025 &size))
4026 return -EOVERFLOW;
4027 if (check_add_overflow((unsigned long)p->addr, size, &tmp_check))
4028 return -EOVERFLOW;
4029
Pavel Begunkovd81269f2021-03-19 10:21:19 +00004030 size = (unsigned long)p->len * p->nbufs;
4031 if (!access_ok(u64_to_user_ptr(p->addr), size))
Jens Axboeddf0322d2020-02-23 16:41:33 -07004032 return -EFAULT;
4033
4034 p->bgid = READ_ONCE(sqe->buf_group);
4035 tmp = READ_ONCE(sqe->off);
4036 if (tmp > USHRT_MAX)
4037 return -E2BIG;
4038 p->bid = tmp;
4039 return 0;
4040}
4041
4042static int io_add_buffers(struct io_provide_buf *pbuf, struct io_buffer **head)
4043{
4044 struct io_buffer *buf;
4045 u64 addr = pbuf->addr;
4046 int i, bid = pbuf->bid;
4047
4048 for (i = 0; i < pbuf->nbufs; i++) {
4049 buf = kmalloc(sizeof(*buf), GFP_KERNEL);
4050 if (!buf)
4051 break;
4052
4053 buf->addr = addr;
Thadeu Lima de Souza Cascardod1f82802021-05-05 09:47:06 -03004054 buf->len = min_t(__u32, pbuf->len, MAX_RW_COUNT);
Jens Axboeddf0322d2020-02-23 16:41:33 -07004055 buf->bid = bid;
4056 addr += pbuf->len;
4057 bid++;
4058 if (!*head) {
4059 INIT_LIST_HEAD(&buf->list);
4060 *head = buf;
4061 } else {
4062 list_add_tail(&buf->list, &(*head)->list);
4063 }
4064 }
4065
4066 return i ? i : -ENOMEM;
4067}
4068
Pavel Begunkov889fca72021-02-10 00:03:09 +00004069static int io_provide_buffers(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboeddf0322d2020-02-23 16:41:33 -07004070{
4071 struct io_provide_buf *p = &req->pbuf;
4072 struct io_ring_ctx *ctx = req->ctx;
4073 struct io_buffer *head, *list;
4074 int ret = 0;
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004075 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
Jens Axboeddf0322d2020-02-23 16:41:33 -07004076
4077 io_ring_submit_lock(ctx, !force_nonblock);
4078
4079 lockdep_assert_held(&ctx->uring_lock);
4080
Jens Axboe9e15c3a2021-03-13 12:29:43 -07004081 list = head = xa_load(&ctx->io_buffers, p->bgid);
Jens Axboeddf0322d2020-02-23 16:41:33 -07004082
4083 ret = io_add_buffers(p, &head);
Jens Axboe9e15c3a2021-03-13 12:29:43 -07004084 if (ret >= 0 && !list) {
4085 ret = xa_insert(&ctx->io_buffers, p->bgid, head, GFP_KERNEL);
4086 if (ret < 0)
Jens Axboe067524e2020-03-02 16:32:28 -07004087 __io_remove_buffers(ctx, head, p->bgid, -1U);
Jens Axboeddf0322d2020-02-23 16:41:33 -07004088 }
Jens Axboeddf0322d2020-02-23 16:41:33 -07004089 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004090 req_set_fail(req);
Pavel Begunkov9fb8cb42021-02-28 22:35:13 +00004091 /* complete before unlock, IOPOLL may need the lock */
4092 __io_req_complete(req, issue_flags, ret, 0);
4093 io_ring_submit_unlock(ctx, !force_nonblock);
Jens Axboeddf0322d2020-02-23 16:41:33 -07004094 return 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07004095}
4096
Jens Axboe3e4827b2020-01-08 15:18:09 -07004097static int io_epoll_ctl_prep(struct io_kiocb *req,
4098 const struct io_uring_sqe *sqe)
4099{
4100#if defined(CONFIG_EPOLL)
4101 if (sqe->ioprio || sqe->buf_index)
4102 return -EINVAL;
Pavel Begunkov2d74d042021-05-14 12:05:46 +01004103 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
Pavel Begunkov3232dd02020-06-03 18:03:22 +03004104 return -EINVAL;
Jens Axboe3e4827b2020-01-08 15:18:09 -07004105
4106 req->epoll.epfd = READ_ONCE(sqe->fd);
4107 req->epoll.op = READ_ONCE(sqe->len);
4108 req->epoll.fd = READ_ONCE(sqe->off);
4109
4110 if (ep_op_has_event(req->epoll.op)) {
4111 struct epoll_event __user *ev;
4112
4113 ev = u64_to_user_ptr(READ_ONCE(sqe->addr));
4114 if (copy_from_user(&req->epoll.event, ev, sizeof(*ev)))
4115 return -EFAULT;
4116 }
4117
4118 return 0;
4119#else
4120 return -EOPNOTSUPP;
4121#endif
4122}
4123
Pavel Begunkov889fca72021-02-10 00:03:09 +00004124static int io_epoll_ctl(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe3e4827b2020-01-08 15:18:09 -07004125{
4126#if defined(CONFIG_EPOLL)
4127 struct io_epoll *ie = &req->epoll;
4128 int ret;
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004129 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
Jens Axboe3e4827b2020-01-08 15:18:09 -07004130
4131 ret = do_epoll_ctl(ie->epfd, ie->op, ie->fd, &ie->event, force_nonblock);
4132 if (force_nonblock && ret == -EAGAIN)
4133 return -EAGAIN;
4134
4135 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004136 req_set_fail(req);
Pavel Begunkov889fca72021-02-10 00:03:09 +00004137 __io_req_complete(req, issue_flags, ret, 0);
Jens Axboe3e4827b2020-01-08 15:18:09 -07004138 return 0;
4139#else
4140 return -EOPNOTSUPP;
4141#endif
4142}
4143
Jens Axboec1ca7572019-12-25 22:18:28 -07004144static int io_madvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4145{
4146#if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU)
4147 if (sqe->ioprio || sqe->buf_index || sqe->off)
4148 return -EINVAL;
Pavel Begunkov3232dd02020-06-03 18:03:22 +03004149 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4150 return -EINVAL;
Jens Axboec1ca7572019-12-25 22:18:28 -07004151
4152 req->madvise.addr = READ_ONCE(sqe->addr);
4153 req->madvise.len = READ_ONCE(sqe->len);
4154 req->madvise.advice = READ_ONCE(sqe->fadvise_advice);
4155 return 0;
4156#else
4157 return -EOPNOTSUPP;
4158#endif
4159}
4160
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004161static int io_madvise(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboec1ca7572019-12-25 22:18:28 -07004162{
4163#if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU)
4164 struct io_madvise *ma = &req->madvise;
4165 int ret;
4166
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004167 if (issue_flags & IO_URING_F_NONBLOCK)
Jens Axboec1ca7572019-12-25 22:18:28 -07004168 return -EAGAIN;
4169
Minchan Kim0726b012020-10-17 16:14:50 -07004170 ret = do_madvise(current->mm, ma->addr, ma->len, ma->advice);
Jens Axboec1ca7572019-12-25 22:18:28 -07004171 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004172 req_set_fail(req);
Jens Axboee1e16092020-06-22 09:17:17 -06004173 io_req_complete(req, ret);
Jens Axboec1ca7572019-12-25 22:18:28 -07004174 return 0;
4175#else
4176 return -EOPNOTSUPP;
4177#endif
4178}
4179
Jens Axboe4840e412019-12-25 22:03:45 -07004180static int io_fadvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4181{
4182 if (sqe->ioprio || sqe->buf_index || sqe->addr)
4183 return -EINVAL;
Pavel Begunkov3232dd02020-06-03 18:03:22 +03004184 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4185 return -EINVAL;
Jens Axboe4840e412019-12-25 22:03:45 -07004186
4187 req->fadvise.offset = READ_ONCE(sqe->off);
4188 req->fadvise.len = READ_ONCE(sqe->len);
4189 req->fadvise.advice = READ_ONCE(sqe->fadvise_advice);
4190 return 0;
4191}
4192
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004193static int io_fadvise(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe4840e412019-12-25 22:03:45 -07004194{
4195 struct io_fadvise *fa = &req->fadvise;
4196 int ret;
4197
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004198 if (issue_flags & IO_URING_F_NONBLOCK) {
Jens Axboe3e694262020-02-01 09:22:49 -07004199 switch (fa->advice) {
4200 case POSIX_FADV_NORMAL:
4201 case POSIX_FADV_RANDOM:
4202 case POSIX_FADV_SEQUENTIAL:
4203 break;
4204 default:
4205 return -EAGAIN;
4206 }
4207 }
Jens Axboe4840e412019-12-25 22:03:45 -07004208
4209 ret = vfs_fadvise(req->file, fa->offset, fa->len, fa->advice);
4210 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004211 req_set_fail(req);
Pavel Begunkov0bdf3392021-04-11 01:46:29 +01004212 __io_req_complete(req, issue_flags, ret, 0);
Jens Axboe4840e412019-12-25 22:03:45 -07004213 return 0;
4214}
4215
Jens Axboeeddc7ef2019-12-13 21:18:10 -07004216static int io_statx_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4217{
Pavel Begunkov2d74d042021-05-14 12:05:46 +01004218 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
Pavel Begunkov3232dd02020-06-03 18:03:22 +03004219 return -EINVAL;
Jens Axboeeddc7ef2019-12-13 21:18:10 -07004220 if (sqe->ioprio || sqe->buf_index)
4221 return -EINVAL;
Pavel Begunkov9c280f92020-04-08 08:58:46 +03004222 if (req->flags & REQ_F_FIXED_FILE)
Jens Axboecf3040c2020-02-06 21:31:40 -07004223 return -EBADF;
Jens Axboeeddc7ef2019-12-13 21:18:10 -07004224
Bijan Mottahedeh1d9e1282020-05-22 21:31:16 -07004225 req->statx.dfd = READ_ONCE(sqe->fd);
4226 req->statx.mask = READ_ONCE(sqe->len);
Bijan Mottahedehe62753e2020-05-22 21:31:18 -07004227 req->statx.filename = u64_to_user_ptr(READ_ONCE(sqe->addr));
Bijan Mottahedeh1d9e1282020-05-22 21:31:16 -07004228 req->statx.buffer = u64_to_user_ptr(READ_ONCE(sqe->addr2));
4229 req->statx.flags = READ_ONCE(sqe->statx_flags);
Jens Axboeeddc7ef2019-12-13 21:18:10 -07004230
Jens Axboeeddc7ef2019-12-13 21:18:10 -07004231 return 0;
4232}
4233
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004234static int io_statx(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboeeddc7ef2019-12-13 21:18:10 -07004235{
Bijan Mottahedeh1d9e1282020-05-22 21:31:16 -07004236 struct io_statx *ctx = &req->statx;
Jens Axboeeddc7ef2019-12-13 21:18:10 -07004237 int ret;
4238
Pavel Begunkov59d70012021-03-22 01:58:30 +00004239 if (issue_flags & IO_URING_F_NONBLOCK)
Jens Axboeeddc7ef2019-12-13 21:18:10 -07004240 return -EAGAIN;
4241
Bijan Mottahedehe62753e2020-05-22 21:31:18 -07004242 ret = do_statx(ctx->dfd, ctx->filename, ctx->flags, ctx->mask,
4243 ctx->buffer);
Jens Axboeeddc7ef2019-12-13 21:18:10 -07004244
Jens Axboeeddc7ef2019-12-13 21:18:10 -07004245 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004246 req_set_fail(req);
Jens Axboee1e16092020-06-22 09:17:17 -06004247 io_req_complete(req, ret);
Jens Axboeeddc7ef2019-12-13 21:18:10 -07004248 return 0;
4249}
4250
Jens Axboeb5dba592019-12-11 14:02:38 -07004251static int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4252{
Jens Axboe14587a462020-09-05 11:36:08 -06004253 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
Pavel Begunkov3232dd02020-06-03 18:03:22 +03004254 return -EINVAL;
Jens Axboeb5dba592019-12-11 14:02:38 -07004255 if (sqe->ioprio || sqe->off || sqe->addr || sqe->len ||
4256 sqe->rw_flags || sqe->buf_index)
4257 return -EINVAL;
Pavel Begunkov9c280f92020-04-08 08:58:46 +03004258 if (req->flags & REQ_F_FIXED_FILE)
Jens Axboecf3040c2020-02-06 21:31:40 -07004259 return -EBADF;
Jens Axboeb5dba592019-12-11 14:02:38 -07004260
4261 req->close.fd = READ_ONCE(sqe->fd);
Jens Axboeb5dba592019-12-11 14:02:38 -07004262 return 0;
4263}
4264
Pavel Begunkov889fca72021-02-10 00:03:09 +00004265static int io_close(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboeb5dba592019-12-11 14:02:38 -07004266{
Jens Axboe9eac1902021-01-19 15:50:37 -07004267 struct files_struct *files = current->files;
Pavel Begunkov3af73b22020-06-08 21:08:17 +03004268 struct io_close *close = &req->close;
Jens Axboe9eac1902021-01-19 15:50:37 -07004269 struct fdtable *fdt;
Pavel Begunkova1fde922021-04-11 01:46:28 +01004270 struct file *file = NULL;
4271 int ret = -EBADF;
Jens Axboeb5dba592019-12-11 14:02:38 -07004272
Jens Axboe9eac1902021-01-19 15:50:37 -07004273 spin_lock(&files->file_lock);
4274 fdt = files_fdtable(files);
4275 if (close->fd >= fdt->max_fds) {
4276 spin_unlock(&files->file_lock);
4277 goto err;
4278 }
4279 file = fdt->fd[close->fd];
Pavel Begunkova1fde922021-04-11 01:46:28 +01004280 if (!file || file->f_op == &io_uring_fops) {
Jens Axboe9eac1902021-01-19 15:50:37 -07004281 spin_unlock(&files->file_lock);
4282 file = NULL;
4283 goto err;
Pavel Begunkov3af73b22020-06-08 21:08:17 +03004284 }
Jens Axboeb5dba592019-12-11 14:02:38 -07004285
4286 /* if the file has a flush method, be safe and punt to async */
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004287 if (file->f_op->flush && (issue_flags & IO_URING_F_NONBLOCK)) {
Jens Axboe9eac1902021-01-19 15:50:37 -07004288 spin_unlock(&files->file_lock);
Pavel Begunkov0bf0eef2020-05-26 20:34:06 +03004289 return -EAGAIN;
Pavel Begunkova2100672020-03-02 23:45:16 +03004290 }
Jens Axboeb5dba592019-12-11 14:02:38 -07004291
Jens Axboe9eac1902021-01-19 15:50:37 -07004292 ret = __close_fd_get_file(close->fd, &file);
4293 spin_unlock(&files->file_lock);
4294 if (ret < 0) {
4295 if (ret == -ENOENT)
4296 ret = -EBADF;
4297 goto err;
4298 }
4299
Pavel Begunkov3af73b22020-06-08 21:08:17 +03004300 /* No ->flush() or already async, safely close from here */
Jens Axboe9eac1902021-01-19 15:50:37 -07004301 ret = filp_close(file, current->files);
4302err:
Pavel Begunkov3af73b22020-06-08 21:08:17 +03004303 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004304 req_set_fail(req);
Jens Axboe9eac1902021-01-19 15:50:37 -07004305 if (file)
4306 fput(file);
Pavel Begunkov889fca72021-02-10 00:03:09 +00004307 __io_req_complete(req, issue_flags, ret, 0);
Jens Axboe1a417f42020-01-31 17:16:48 -07004308 return 0;
Jens Axboeb5dba592019-12-11 14:02:38 -07004309}
4310
Pavel Begunkov1155c762021-02-18 18:29:38 +00004311static int io_sfr_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboe5d17b4a2019-04-09 14:56:44 -06004312{
4313 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe5d17b4a2019-04-09 14:56:44 -06004314
Jens Axboe5d17b4a2019-04-09 14:56:44 -06004315 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
4316 return -EINVAL;
4317 if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index))
4318 return -EINVAL;
4319
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004320 req->sync.off = READ_ONCE(sqe->off);
4321 req->sync.len = READ_ONCE(sqe->len);
4322 req->sync.flags = READ_ONCE(sqe->sync_range_flags);
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004323 return 0;
4324}
4325
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004326static int io_sync_file_range(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004327{
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004328 int ret;
4329
Pavel Begunkovac45abc2020-06-08 21:08:18 +03004330 /* sync_file_range always requires a blocking context */
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004331 if (issue_flags & IO_URING_F_NONBLOCK)
Pavel Begunkovac45abc2020-06-08 21:08:18 +03004332 return -EAGAIN;
4333
Jens Axboe9adbd452019-12-20 08:45:55 -07004334 ret = sync_file_range(req->file, req->sync.off, req->sync.len,
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004335 req->sync.flags);
4336 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004337 req_set_fail(req);
Jens Axboee1e16092020-06-22 09:17:17 -06004338 io_req_complete(req, ret);
Jens Axboe5d17b4a2019-04-09 14:56:44 -06004339 return 0;
4340}
4341
YueHaibing469956e2020-03-04 15:53:52 +08004342#if defined(CONFIG_NET)
Pavel Begunkov02d27d82020-02-28 10:36:36 +03004343static int io_setup_async_msg(struct io_kiocb *req,
4344 struct io_async_msghdr *kmsg)
4345{
Jens Axboee8c2bc12020-08-15 18:44:09 -07004346 struct io_async_msghdr *async_msg = req->async_data;
4347
4348 if (async_msg)
Pavel Begunkov02d27d82020-02-28 10:36:36 +03004349 return -EAGAIN;
Jens Axboee8c2bc12020-08-15 18:44:09 -07004350 if (io_alloc_async_data(req)) {
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004351 kfree(kmsg->free_iov);
Pavel Begunkov02d27d82020-02-28 10:36:36 +03004352 return -ENOMEM;
4353 }
Jens Axboee8c2bc12020-08-15 18:44:09 -07004354 async_msg = req->async_data;
Pavel Begunkov02d27d82020-02-28 10:36:36 +03004355 req->flags |= REQ_F_NEED_CLEANUP;
Jens Axboee8c2bc12020-08-15 18:44:09 -07004356 memcpy(async_msg, kmsg, sizeof(*kmsg));
Pavel Begunkov2a780802021-02-05 00:57:58 +00004357 async_msg->msg.msg_name = &async_msg->addr;
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004358 /* if were using fast_iov, set it to the new one */
4359 if (!async_msg->free_iov)
4360 async_msg->msg.msg_iter.iov = async_msg->fast_iov;
4361
Pavel Begunkov02d27d82020-02-28 10:36:36 +03004362 return -EAGAIN;
4363}
4364
Pavel Begunkov2ae523e2020-07-12 20:41:06 +03004365static int io_sendmsg_copy_hdr(struct io_kiocb *req,
4366 struct io_async_msghdr *iomsg)
4367{
Pavel Begunkov2ae523e2020-07-12 20:41:06 +03004368 iomsg->msg.msg_name = &iomsg->addr;
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004369 iomsg->free_iov = iomsg->fast_iov;
Pavel Begunkov2ae523e2020-07-12 20:41:06 +03004370 return sendmsg_copy_msghdr(&iomsg->msg, req->sr_msg.umsg,
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004371 req->sr_msg.msg_flags, &iomsg->free_iov);
Pavel Begunkov2ae523e2020-07-12 20:41:06 +03004372}
4373
Pavel Begunkov93642ef2021-02-18 18:29:44 +00004374static int io_sendmsg_prep_async(struct io_kiocb *req)
4375{
4376 int ret;
4377
Pavel Begunkov93642ef2021-02-18 18:29:44 +00004378 ret = io_sendmsg_copy_hdr(req, req->async_data);
4379 if (!ret)
4380 req->flags |= REQ_F_NEED_CLEANUP;
4381 return ret;
4382}
4383
Jens Axboe3529d8c2019-12-19 18:24:38 -07004384static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboeaa1fa282019-04-19 13:38:09 -06004385{
Jens Axboee47293f2019-12-20 08:58:21 -07004386 struct io_sr_msg *sr = &req->sr_msg;
Jens Axboe03b12302019-12-02 18:50:25 -07004387
Pavel Begunkovd2b6f482020-06-03 18:03:25 +03004388 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4389 return -EINVAL;
4390
Pavel Begunkov270a5942020-07-12 20:41:04 +03004391 sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
Jens Axboefddafac2020-01-04 20:19:44 -07004392 sr->len = READ_ONCE(sqe->len);
Pavel Begunkov04411802021-04-01 15:44:00 +01004393 sr->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL;
4394 if (sr->msg_flags & MSG_DONTWAIT)
4395 req->flags |= REQ_F_NOWAIT;
Jens Axboe3529d8c2019-12-19 18:24:38 -07004396
Jens Axboed8768362020-02-27 14:17:49 -07004397#ifdef CONFIG_COMPAT
4398 if (req->ctx->compat)
4399 sr->msg_flags |= MSG_CMSG_COMPAT;
4400#endif
Pavel Begunkov93642ef2021-02-18 18:29:44 +00004401 return 0;
Jens Axboe03b12302019-12-02 18:50:25 -07004402}
4403
Pavel Begunkov889fca72021-02-10 00:03:09 +00004404static int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe03b12302019-12-02 18:50:25 -07004405{
Pavel Begunkov6b754c82020-07-16 23:28:00 +03004406 struct io_async_msghdr iomsg, *kmsg;
Jens Axboe03b12302019-12-02 18:50:25 -07004407 struct socket *sock;
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004408 unsigned flags;
Stefan Metzmacher00312752021-03-20 20:33:36 +01004409 int min_ret = 0;
Jens Axboe03b12302019-12-02 18:50:25 -07004410 int ret;
4411
Florent Revestdba4a922020-12-04 12:36:04 +01004412 sock = sock_from_file(req->file);
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004413 if (unlikely(!sock))
Florent Revestdba4a922020-12-04 12:36:04 +01004414 return -ENOTSOCK;
Jens Axboe03b12302019-12-02 18:50:25 -07004415
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004416 kmsg = req->async_data;
4417 if (!kmsg) {
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004418 ret = io_sendmsg_copy_hdr(req, &iomsg);
Jens Axboefddafac2020-01-04 20:19:44 -07004419 if (ret)
4420 return ret;
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004421 kmsg = &iomsg;
Jens Axboefddafac2020-01-04 20:19:44 -07004422 }
4423
Pavel Begunkov04411802021-04-01 15:44:00 +01004424 flags = req->sr_msg.msg_flags;
4425 if (issue_flags & IO_URING_F_NONBLOCK)
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004426 flags |= MSG_DONTWAIT;
Stefan Metzmacher00312752021-03-20 20:33:36 +01004427 if (flags & MSG_WAITALL)
4428 min_ret = iov_iter_count(&kmsg->msg.msg_iter);
4429
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004430 ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags);
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004431 if ((issue_flags & IO_URING_F_NONBLOCK) && ret == -EAGAIN)
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004432 return io_setup_async_msg(req, kmsg);
4433 if (ret == -ERESTARTSYS)
4434 ret = -EINTR;
4435
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004436 /* fast path, check for non-NULL to avoid function call */
4437 if (kmsg->free_iov)
4438 kfree(kmsg->free_iov);
Jens Axboe03b12302019-12-02 18:50:25 -07004439 req->flags &= ~REQ_F_NEED_CLEANUP;
Stefan Metzmacher00312752021-03-20 20:33:36 +01004440 if (ret < min_ret)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004441 req_set_fail(req);
Pavel Begunkov889fca72021-02-10 00:03:09 +00004442 __io_req_complete(req, issue_flags, ret, 0);
Jens Axboefddafac2020-01-04 20:19:44 -07004443 return 0;
Jens Axboefddafac2020-01-04 20:19:44 -07004444}
4445
Pavel Begunkov889fca72021-02-10 00:03:09 +00004446static int io_send(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe03b12302019-12-02 18:50:25 -07004447{
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004448 struct io_sr_msg *sr = &req->sr_msg;
4449 struct msghdr msg;
4450 struct iovec iov;
Jens Axboe03b12302019-12-02 18:50:25 -07004451 struct socket *sock;
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004452 unsigned flags;
Stefan Metzmacher00312752021-03-20 20:33:36 +01004453 int min_ret = 0;
Jens Axboe03b12302019-12-02 18:50:25 -07004454 int ret;
4455
Florent Revestdba4a922020-12-04 12:36:04 +01004456 sock = sock_from_file(req->file);
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004457 if (unlikely(!sock))
Florent Revestdba4a922020-12-04 12:36:04 +01004458 return -ENOTSOCK;
Jens Axboe03b12302019-12-02 18:50:25 -07004459
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004460 ret = import_single_range(WRITE, sr->buf, sr->len, &iov, &msg.msg_iter);
4461 if (unlikely(ret))
Zheng Bin14db8412020-09-09 20:12:37 +08004462 return ret;
Jens Axboe03b12302019-12-02 18:50:25 -07004463
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004464 msg.msg_name = NULL;
4465 msg.msg_control = NULL;
4466 msg.msg_controllen = 0;
4467 msg.msg_namelen = 0;
Jens Axboe03b12302019-12-02 18:50:25 -07004468
Pavel Begunkov04411802021-04-01 15:44:00 +01004469 flags = req->sr_msg.msg_flags;
4470 if (issue_flags & IO_URING_F_NONBLOCK)
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004471 flags |= MSG_DONTWAIT;
Stefan Metzmacher00312752021-03-20 20:33:36 +01004472 if (flags & MSG_WAITALL)
4473 min_ret = iov_iter_count(&msg.msg_iter);
4474
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004475 msg.msg_flags = flags;
4476 ret = sock_sendmsg(sock, &msg);
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004477 if ((issue_flags & IO_URING_F_NONBLOCK) && ret == -EAGAIN)
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004478 return -EAGAIN;
4479 if (ret == -ERESTARTSYS)
4480 ret = -EINTR;
Jens Axboe03b12302019-12-02 18:50:25 -07004481
Stefan Metzmacher00312752021-03-20 20:33:36 +01004482 if (ret < min_ret)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004483 req_set_fail(req);
Pavel Begunkov889fca72021-02-10 00:03:09 +00004484 __io_req_complete(req, issue_flags, ret, 0);
Jens Axboe03b12302019-12-02 18:50:25 -07004485 return 0;
Jens Axboe03b12302019-12-02 18:50:25 -07004486}
4487
Pavel Begunkov1400e692020-07-12 20:41:05 +03004488static int __io_recvmsg_copy_hdr(struct io_kiocb *req,
4489 struct io_async_msghdr *iomsg)
Jens Axboe52de1fe2020-02-27 10:15:42 -07004490{
4491 struct io_sr_msg *sr = &req->sr_msg;
4492 struct iovec __user *uiov;
4493 size_t iov_len;
4494 int ret;
4495
Pavel Begunkov1400e692020-07-12 20:41:05 +03004496 ret = __copy_msghdr_from_user(&iomsg->msg, sr->umsg,
4497 &iomsg->uaddr, &uiov, &iov_len);
Jens Axboe52de1fe2020-02-27 10:15:42 -07004498 if (ret)
4499 return ret;
4500
4501 if (req->flags & REQ_F_BUFFER_SELECT) {
4502 if (iov_len > 1)
4503 return -EINVAL;
Pavel Begunkov5476dfe2021-02-05 00:57:59 +00004504 if (copy_from_user(iomsg->fast_iov, uiov, sizeof(*uiov)))
Jens Axboe52de1fe2020-02-27 10:15:42 -07004505 return -EFAULT;
Pavel Begunkov5476dfe2021-02-05 00:57:59 +00004506 sr->len = iomsg->fast_iov[0].iov_len;
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004507 iomsg->free_iov = NULL;
Jens Axboe52de1fe2020-02-27 10:15:42 -07004508 } else {
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004509 iomsg->free_iov = iomsg->fast_iov;
Christoph Hellwig89cd35c2020-09-25 06:51:41 +02004510 ret = __import_iovec(READ, uiov, iov_len, UIO_FASTIOV,
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004511 &iomsg->free_iov, &iomsg->msg.msg_iter,
Christoph Hellwig89cd35c2020-09-25 06:51:41 +02004512 false);
Jens Axboe52de1fe2020-02-27 10:15:42 -07004513 if (ret > 0)
4514 ret = 0;
4515 }
4516
4517 return ret;
4518}
4519
4520#ifdef CONFIG_COMPAT
4521static int __io_compat_recvmsg_copy_hdr(struct io_kiocb *req,
Pavel Begunkov1400e692020-07-12 20:41:05 +03004522 struct io_async_msghdr *iomsg)
Jens Axboe52de1fe2020-02-27 10:15:42 -07004523{
Jens Axboe52de1fe2020-02-27 10:15:42 -07004524 struct io_sr_msg *sr = &req->sr_msg;
4525 struct compat_iovec __user *uiov;
4526 compat_uptr_t ptr;
4527 compat_size_t len;
4528 int ret;
4529
Pavel Begunkov4af34172021-04-11 01:46:30 +01004530 ret = __get_compat_msghdr(&iomsg->msg, sr->umsg_compat, &iomsg->uaddr,
4531 &ptr, &len);
Jens Axboe52de1fe2020-02-27 10:15:42 -07004532 if (ret)
4533 return ret;
4534
4535 uiov = compat_ptr(ptr);
4536 if (req->flags & REQ_F_BUFFER_SELECT) {
4537 compat_ssize_t clen;
4538
4539 if (len > 1)
4540 return -EINVAL;
4541 if (!access_ok(uiov, sizeof(*uiov)))
4542 return -EFAULT;
4543 if (__get_user(clen, &uiov->iov_len))
4544 return -EFAULT;
4545 if (clen < 0)
4546 return -EINVAL;
Pavel Begunkov2d280bc2020-11-29 18:33:32 +00004547 sr->len = clen;
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004548 iomsg->free_iov = NULL;
Jens Axboe52de1fe2020-02-27 10:15:42 -07004549 } else {
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004550 iomsg->free_iov = iomsg->fast_iov;
Christoph Hellwig89cd35c2020-09-25 06:51:41 +02004551 ret = __import_iovec(READ, (struct iovec __user *)uiov, len,
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004552 UIO_FASTIOV, &iomsg->free_iov,
Christoph Hellwig89cd35c2020-09-25 06:51:41 +02004553 &iomsg->msg.msg_iter, true);
Jens Axboe52de1fe2020-02-27 10:15:42 -07004554 if (ret < 0)
4555 return ret;
4556 }
4557
4558 return 0;
4559}
Jens Axboe03b12302019-12-02 18:50:25 -07004560#endif
Jens Axboe52de1fe2020-02-27 10:15:42 -07004561
Pavel Begunkov1400e692020-07-12 20:41:05 +03004562static int io_recvmsg_copy_hdr(struct io_kiocb *req,
4563 struct io_async_msghdr *iomsg)
Jens Axboe52de1fe2020-02-27 10:15:42 -07004564{
Pavel Begunkov1400e692020-07-12 20:41:05 +03004565 iomsg->msg.msg_name = &iomsg->addr;
Jens Axboe52de1fe2020-02-27 10:15:42 -07004566
4567#ifdef CONFIG_COMPAT
4568 if (req->ctx->compat)
Pavel Begunkov1400e692020-07-12 20:41:05 +03004569 return __io_compat_recvmsg_copy_hdr(req, iomsg);
Jens Axboe52de1fe2020-02-27 10:15:42 -07004570#endif
4571
Pavel Begunkov1400e692020-07-12 20:41:05 +03004572 return __io_recvmsg_copy_hdr(req, iomsg);
Jens Axboe52de1fe2020-02-27 10:15:42 -07004573}
4574
Jens Axboebcda7ba2020-02-23 16:42:51 -07004575static struct io_buffer *io_recv_buffer_select(struct io_kiocb *req,
Pavel Begunkov7fbb1b52020-07-16 23:28:05 +03004576 bool needs_lock)
Jens Axboebcda7ba2020-02-23 16:42:51 -07004577{
4578 struct io_sr_msg *sr = &req->sr_msg;
4579 struct io_buffer *kbuf;
4580
Jens Axboebcda7ba2020-02-23 16:42:51 -07004581 kbuf = io_buffer_select(req, &sr->len, sr->bgid, sr->kbuf, needs_lock);
4582 if (IS_ERR(kbuf))
4583 return kbuf;
4584
4585 sr->kbuf = kbuf;
4586 req->flags |= REQ_F_BUFFER_SELECTED;
Jens Axboebcda7ba2020-02-23 16:42:51 -07004587 return kbuf;
Jens Axboe03b12302019-12-02 18:50:25 -07004588}
4589
Pavel Begunkov7fbb1b52020-07-16 23:28:05 +03004590static inline unsigned int io_put_recv_kbuf(struct io_kiocb *req)
4591{
4592 return io_put_kbuf(req, req->sr_msg.kbuf);
4593}
4594
Pavel Begunkov93642ef2021-02-18 18:29:44 +00004595static int io_recvmsg_prep_async(struct io_kiocb *req)
Jens Axboe03b12302019-12-02 18:50:25 -07004596{
Pavel Begunkov99bc4c32020-02-07 22:04:45 +03004597 int ret;
Jens Axboe06b76d42019-12-19 14:44:26 -07004598
Pavel Begunkov93642ef2021-02-18 18:29:44 +00004599 ret = io_recvmsg_copy_hdr(req, req->async_data);
4600 if (!ret)
4601 req->flags |= REQ_F_NEED_CLEANUP;
4602 return ret;
4603}
4604
4605static int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4606{
4607 struct io_sr_msg *sr = &req->sr_msg;
4608
Pavel Begunkovd2b6f482020-06-03 18:03:25 +03004609 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4610 return -EINVAL;
4611
Pavel Begunkov270a5942020-07-12 20:41:04 +03004612 sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
Jens Axboe0b7b21e2020-01-31 08:34:59 -07004613 sr->len = READ_ONCE(sqe->len);
Jens Axboebcda7ba2020-02-23 16:42:51 -07004614 sr->bgid = READ_ONCE(sqe->buf_group);
Pavel Begunkov04411802021-04-01 15:44:00 +01004615 sr->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL;
4616 if (sr->msg_flags & MSG_DONTWAIT)
4617 req->flags |= REQ_F_NOWAIT;
Jens Axboe3529d8c2019-12-19 18:24:38 -07004618
Jens Axboed8768362020-02-27 14:17:49 -07004619#ifdef CONFIG_COMPAT
4620 if (req->ctx->compat)
4621 sr->msg_flags |= MSG_CMSG_COMPAT;
4622#endif
Pavel Begunkov93642ef2021-02-18 18:29:44 +00004623 return 0;
Jens Axboe03b12302019-12-02 18:50:25 -07004624}
4625
Pavel Begunkov889fca72021-02-10 00:03:09 +00004626static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe03b12302019-12-02 18:50:25 -07004627{
Pavel Begunkov6b754c82020-07-16 23:28:00 +03004628 struct io_async_msghdr iomsg, *kmsg;
Jens Axboe0fa03c62019-04-19 13:34:07 -06004629 struct socket *sock;
Pavel Begunkov7fbb1b52020-07-16 23:28:05 +03004630 struct io_buffer *kbuf;
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004631 unsigned flags;
Stefan Metzmacher00312752021-03-20 20:33:36 +01004632 int min_ret = 0;
Jens Axboe52de1fe2020-02-27 10:15:42 -07004633 int ret, cflags = 0;
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004634 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
Jens Axboe0fa03c62019-04-19 13:34:07 -06004635
Florent Revestdba4a922020-12-04 12:36:04 +01004636 sock = sock_from_file(req->file);
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004637 if (unlikely(!sock))
Florent Revestdba4a922020-12-04 12:36:04 +01004638 return -ENOTSOCK;
Jens Axboe0fa03c62019-04-19 13:34:07 -06004639
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004640 kmsg = req->async_data;
4641 if (!kmsg) {
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004642 ret = io_recvmsg_copy_hdr(req, &iomsg);
4643 if (ret)
Pavel Begunkov681fda82020-07-15 22:20:45 +03004644 return ret;
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004645 kmsg = &iomsg;
Jens Axboe0fa03c62019-04-19 13:34:07 -06004646 }
4647
Pavel Begunkovbc02ef32020-07-16 23:28:03 +03004648 if (req->flags & REQ_F_BUFFER_SELECT) {
Pavel Begunkov7fbb1b52020-07-16 23:28:05 +03004649 kbuf = io_recv_buffer_select(req, !force_nonblock);
Pavel Begunkovbc02ef32020-07-16 23:28:03 +03004650 if (IS_ERR(kbuf))
4651 return PTR_ERR(kbuf);
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004652 kmsg->fast_iov[0].iov_base = u64_to_user_ptr(kbuf->addr);
Pavel Begunkov5476dfe2021-02-05 00:57:59 +00004653 kmsg->fast_iov[0].iov_len = req->sr_msg.len;
4654 iov_iter_init(&kmsg->msg.msg_iter, READ, kmsg->fast_iov,
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004655 1, req->sr_msg.len);
4656 }
4657
Pavel Begunkov04411802021-04-01 15:44:00 +01004658 flags = req->sr_msg.msg_flags;
4659 if (force_nonblock)
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004660 flags |= MSG_DONTWAIT;
Stefan Metzmacher00312752021-03-20 20:33:36 +01004661 if (flags & MSG_WAITALL)
4662 min_ret = iov_iter_count(&kmsg->msg.msg_iter);
4663
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004664 ret = __sys_recvmsg_sock(sock, &kmsg->msg, req->sr_msg.umsg,
4665 kmsg->uaddr, flags);
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03004666 if (force_nonblock && ret == -EAGAIN)
4667 return io_setup_async_msg(req, kmsg);
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004668 if (ret == -ERESTARTSYS)
4669 ret = -EINTR;
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03004670
Pavel Begunkov7fbb1b52020-07-16 23:28:05 +03004671 if (req->flags & REQ_F_BUFFER_SELECTED)
4672 cflags = io_put_recv_kbuf(req);
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004673 /* fast path, check for non-NULL to avoid function call */
4674 if (kmsg->free_iov)
4675 kfree(kmsg->free_iov);
Pavel Begunkov99bc4c32020-02-07 22:04:45 +03004676 req->flags &= ~REQ_F_NEED_CLEANUP;
Stefan Metzmacher00312752021-03-20 20:33:36 +01004677 if (ret < min_ret || ((flags & MSG_WAITALL) && (kmsg->msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))))
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004678 req_set_fail(req);
Pavel Begunkov889fca72021-02-10 00:03:09 +00004679 __io_req_complete(req, issue_flags, ret, cflags);
Jens Axboe0fa03c62019-04-19 13:34:07 -06004680 return 0;
Jens Axboe0fa03c62019-04-19 13:34:07 -06004681}
4682
Pavel Begunkov889fca72021-02-10 00:03:09 +00004683static int io_recv(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboefddafac2020-01-04 20:19:44 -07004684{
Pavel Begunkov6b754c82020-07-16 23:28:00 +03004685 struct io_buffer *kbuf;
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004686 struct io_sr_msg *sr = &req->sr_msg;
4687 struct msghdr msg;
4688 void __user *buf = sr->buf;
Jens Axboefddafac2020-01-04 20:19:44 -07004689 struct socket *sock;
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004690 struct iovec iov;
4691 unsigned flags;
Stefan Metzmacher00312752021-03-20 20:33:36 +01004692 int min_ret = 0;
Jens Axboebcda7ba2020-02-23 16:42:51 -07004693 int ret, cflags = 0;
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004694 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
Jens Axboefddafac2020-01-04 20:19:44 -07004695
Florent Revestdba4a922020-12-04 12:36:04 +01004696 sock = sock_from_file(req->file);
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004697 if (unlikely(!sock))
Florent Revestdba4a922020-12-04 12:36:04 +01004698 return -ENOTSOCK;
Jens Axboefddafac2020-01-04 20:19:44 -07004699
Pavel Begunkovbc02ef32020-07-16 23:28:03 +03004700 if (req->flags & REQ_F_BUFFER_SELECT) {
Pavel Begunkov7fbb1b52020-07-16 23:28:05 +03004701 kbuf = io_recv_buffer_select(req, !force_nonblock);
Jens Axboebcda7ba2020-02-23 16:42:51 -07004702 if (IS_ERR(kbuf))
4703 return PTR_ERR(kbuf);
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004704 buf = u64_to_user_ptr(kbuf->addr);
Jens Axboefddafac2020-01-04 20:19:44 -07004705 }
4706
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004707 ret = import_single_range(READ, buf, sr->len, &iov, &msg.msg_iter);
Pavel Begunkov14c32ee2020-07-16 23:28:01 +03004708 if (unlikely(ret))
4709 goto out_free;
Jens Axboefddafac2020-01-04 20:19:44 -07004710
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004711 msg.msg_name = NULL;
4712 msg.msg_control = NULL;
4713 msg.msg_controllen = 0;
4714 msg.msg_namelen = 0;
4715 msg.msg_iocb = NULL;
4716 msg.msg_flags = 0;
4717
Pavel Begunkov04411802021-04-01 15:44:00 +01004718 flags = req->sr_msg.msg_flags;
4719 if (force_nonblock)
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004720 flags |= MSG_DONTWAIT;
Stefan Metzmacher00312752021-03-20 20:33:36 +01004721 if (flags & MSG_WAITALL)
4722 min_ret = iov_iter_count(&msg.msg_iter);
4723
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004724 ret = sock_recvmsg(sock, &msg, flags);
4725 if (force_nonblock && ret == -EAGAIN)
4726 return -EAGAIN;
4727 if (ret == -ERESTARTSYS)
4728 ret = -EINTR;
Pavel Begunkov14c32ee2020-07-16 23:28:01 +03004729out_free:
Pavel Begunkov7fbb1b52020-07-16 23:28:05 +03004730 if (req->flags & REQ_F_BUFFER_SELECTED)
4731 cflags = io_put_recv_kbuf(req);
Stefan Metzmacher00312752021-03-20 20:33:36 +01004732 if (ret < min_ret || ((flags & MSG_WAITALL) && (msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))))
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004733 req_set_fail(req);
Pavel Begunkov889fca72021-02-10 00:03:09 +00004734 __io_req_complete(req, issue_flags, ret, cflags);
Jens Axboefddafac2020-01-04 20:19:44 -07004735 return 0;
Jens Axboefddafac2020-01-04 20:19:44 -07004736}
4737
Jens Axboe3529d8c2019-12-19 18:24:38 -07004738static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboe17f2fe32019-10-17 14:42:58 -06004739{
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004740 struct io_accept *accept = &req->accept;
4741
Jens Axboe14587a462020-09-05 11:36:08 -06004742 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
Jens Axboe17f2fe32019-10-17 14:42:58 -06004743 return -EINVAL;
Hrvoje Zeba8042d6c2019-11-25 14:40:22 -05004744 if (sqe->ioprio || sqe->len || sqe->buf_index)
Jens Axboe17f2fe32019-10-17 14:42:58 -06004745 return -EINVAL;
4746
Jens Axboed55e5f52019-12-11 16:12:15 -07004747 accept->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));
4748 accept->addr_len = u64_to_user_ptr(READ_ONCE(sqe->addr2));
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004749 accept->flags = READ_ONCE(sqe->accept_flags);
Jens Axboe09952e32020-03-19 20:16:56 -06004750 accept->nofile = rlimit(RLIMIT_NOFILE);
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004751 return 0;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004752}
Jens Axboe17f2fe32019-10-17 14:42:58 -06004753
Pavel Begunkov889fca72021-02-10 00:03:09 +00004754static int io_accept(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004755{
4756 struct io_accept *accept = &req->accept;
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004757 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
Pavel Begunkovac45abc2020-06-08 21:08:18 +03004758 unsigned int file_flags = force_nonblock ? O_NONBLOCK : 0;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004759 int ret;
4760
Jiufei Xuee697dee2020-06-10 13:41:59 +08004761 if (req->file->f_flags & O_NONBLOCK)
4762 req->flags |= REQ_F_NOWAIT;
4763
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004764 ret = __sys_accept4_file(req->file, file_flags, accept->addr,
Jens Axboe09952e32020-03-19 20:16:56 -06004765 accept->addr_len, accept->flags,
4766 accept->nofile);
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004767 if (ret == -EAGAIN && force_nonblock)
Jens Axboe17f2fe32019-10-17 14:42:58 -06004768 return -EAGAIN;
Pavel Begunkovac45abc2020-06-08 21:08:18 +03004769 if (ret < 0) {
4770 if (ret == -ERESTARTSYS)
4771 ret = -EINTR;
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004772 req_set_fail(req);
Pavel Begunkovac45abc2020-06-08 21:08:18 +03004773 }
Pavel Begunkov889fca72021-02-10 00:03:09 +00004774 __io_req_complete(req, issue_flags, ret, 0);
Jens Axboe17f2fe32019-10-17 14:42:58 -06004775 return 0;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004776}
4777
Pavel Begunkov93642ef2021-02-18 18:29:44 +00004778static int io_connect_prep_async(struct io_kiocb *req)
4779{
4780 struct io_async_connect *io = req->async_data;
4781 struct io_connect *conn = &req->connect;
4782
4783 return move_addr_to_kernel(conn->addr, conn->addr_len, &io->address);
4784}
4785
Jens Axboe3529d8c2019-12-19 18:24:38 -07004786static int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboef499a022019-12-02 16:28:46 -07004787{
Jens Axboe3529d8c2019-12-19 18:24:38 -07004788 struct io_connect *conn = &req->connect;
Jens Axboef499a022019-12-02 16:28:46 -07004789
Jens Axboe14587a462020-09-05 11:36:08 -06004790 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
Jens Axboe3fbb51c2019-12-20 08:51:52 -07004791 return -EINVAL;
4792 if (sqe->ioprio || sqe->len || sqe->buf_index || sqe->rw_flags)
4793 return -EINVAL;
4794
Jens Axboe3529d8c2019-12-19 18:24:38 -07004795 conn->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));
4796 conn->addr_len = READ_ONCE(sqe->addr2);
Pavel Begunkov93642ef2021-02-18 18:29:44 +00004797 return 0;
Jens Axboef499a022019-12-02 16:28:46 -07004798}
4799
Pavel Begunkov889fca72021-02-10 00:03:09 +00004800static int io_connect(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboef8e85cf2019-11-23 14:24:24 -07004801{
Jens Axboee8c2bc12020-08-15 18:44:09 -07004802 struct io_async_connect __io, *io;
Jens Axboef8e85cf2019-11-23 14:24:24 -07004803 unsigned file_flags;
Jens Axboe3fbb51c2019-12-20 08:51:52 -07004804 int ret;
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004805 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
Jens Axboef8e85cf2019-11-23 14:24:24 -07004806
Jens Axboee8c2bc12020-08-15 18:44:09 -07004807 if (req->async_data) {
4808 io = req->async_data;
Jens Axboef499a022019-12-02 16:28:46 -07004809 } else {
Jens Axboe3529d8c2019-12-19 18:24:38 -07004810 ret = move_addr_to_kernel(req->connect.addr,
4811 req->connect.addr_len,
Jens Axboee8c2bc12020-08-15 18:44:09 -07004812 &__io.address);
Jens Axboef499a022019-12-02 16:28:46 -07004813 if (ret)
4814 goto out;
4815 io = &__io;
4816 }
4817
Jens Axboe3fbb51c2019-12-20 08:51:52 -07004818 file_flags = force_nonblock ? O_NONBLOCK : 0;
4819
Jens Axboee8c2bc12020-08-15 18:44:09 -07004820 ret = __sys_connect_file(req->file, &io->address,
Jens Axboe3fbb51c2019-12-20 08:51:52 -07004821 req->connect.addr_len, file_flags);
Jens Axboe87f80d62019-12-03 11:23:54 -07004822 if ((ret == -EAGAIN || ret == -EINPROGRESS) && force_nonblock) {
Jens Axboee8c2bc12020-08-15 18:44:09 -07004823 if (req->async_data)
Jens Axboeb7bb4f72019-12-15 22:13:43 -07004824 return -EAGAIN;
Jens Axboee8c2bc12020-08-15 18:44:09 -07004825 if (io_alloc_async_data(req)) {
Jens Axboef499a022019-12-02 16:28:46 -07004826 ret = -ENOMEM;
4827 goto out;
4828 }
Jens Axboee8c2bc12020-08-15 18:44:09 -07004829 memcpy(req->async_data, &__io, sizeof(__io));
Jens Axboef8e85cf2019-11-23 14:24:24 -07004830 return -EAGAIN;
Jens Axboef499a022019-12-02 16:28:46 -07004831 }
Jens Axboef8e85cf2019-11-23 14:24:24 -07004832 if (ret == -ERESTARTSYS)
4833 ret = -EINTR;
Jens Axboef499a022019-12-02 16:28:46 -07004834out:
Jens Axboe4e88d6e2019-12-07 20:59:47 -07004835 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004836 req_set_fail(req);
Pavel Begunkov889fca72021-02-10 00:03:09 +00004837 __io_req_complete(req, issue_flags, ret, 0);
Jens Axboef8e85cf2019-11-23 14:24:24 -07004838 return 0;
Jens Axboef8e85cf2019-11-23 14:24:24 -07004839}
YueHaibing469956e2020-03-04 15:53:52 +08004840#else /* !CONFIG_NET */
Jens Axboe99a10082021-02-19 09:35:19 -07004841#define IO_NETOP_FN(op) \
4842static int io_##op(struct io_kiocb *req, unsigned int issue_flags) \
4843{ \
4844 return -EOPNOTSUPP; \
Jens Axboef8e85cf2019-11-23 14:24:24 -07004845}
4846
Jens Axboe99a10082021-02-19 09:35:19 -07004847#define IO_NETOP_PREP(op) \
4848IO_NETOP_FN(op) \
4849static int io_##op##_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) \
4850{ \
4851 return -EOPNOTSUPP; \
4852} \
4853
4854#define IO_NETOP_PREP_ASYNC(op) \
4855IO_NETOP_PREP(op) \
4856static int io_##op##_prep_async(struct io_kiocb *req) \
4857{ \
4858 return -EOPNOTSUPP; \
YueHaibing469956e2020-03-04 15:53:52 +08004859}
4860
Jens Axboe99a10082021-02-19 09:35:19 -07004861IO_NETOP_PREP_ASYNC(sendmsg);
4862IO_NETOP_PREP_ASYNC(recvmsg);
4863IO_NETOP_PREP_ASYNC(connect);
4864IO_NETOP_PREP(accept);
4865IO_NETOP_FN(send);
4866IO_NETOP_FN(recv);
YueHaibing469956e2020-03-04 15:53:52 +08004867#endif /* CONFIG_NET */
Jens Axboe17f2fe32019-10-17 14:42:58 -06004868
Jens Axboed7718a92020-02-14 22:23:12 -07004869struct io_poll_table {
4870 struct poll_table_struct pt;
4871 struct io_kiocb *req;
Pavel Begunkov68b11e82021-07-20 10:50:43 +01004872 int nr_entries;
Jens Axboed7718a92020-02-14 22:23:12 -07004873 int error;
4874};
4875
Jens Axboed7718a92020-02-14 22:23:12 -07004876static int __io_async_wake(struct io_kiocb *req, struct io_poll_iocb *poll,
Pavel Begunkov5b0a6ac2021-06-30 21:54:04 +01004877 __poll_t mask, io_req_tw_func_t func)
Jens Axboed7718a92020-02-14 22:23:12 -07004878{
Jens Axboed7718a92020-02-14 22:23:12 -07004879 /* for instances that support it check for an event match first: */
4880 if (mask && !(mask & poll->events))
4881 return 0;
4882
4883 trace_io_uring_task_add(req->ctx, req->opcode, req->user_data, mask);
4884
4885 list_del_init(&poll->wait.entry);
4886
Jens Axboed7718a92020-02-14 22:23:12 -07004887 req->result = mask;
Pavel Begunkov5b0a6ac2021-06-30 21:54:04 +01004888 req->io_task_work.func = func;
Jens Axboe6d816e02020-08-11 08:04:14 -06004889
Jens Axboed7718a92020-02-14 22:23:12 -07004890 /*
Jens Axboee3aabf92020-05-18 11:04:17 -06004891 * If this fails, then the task is exiting. When a task exits, the
4892 * work gets canceled, so just cancel this request as well instead
4893 * of executing it. We can't safely execute it anyway, as we may not
4894 * have the needed state needed for it anyway.
Jens Axboed7718a92020-02-14 22:23:12 -07004895 */
Pavel Begunkove09ee512021-07-01 13:26:05 +01004896 io_req_task_work_add(req);
Jens Axboed7718a92020-02-14 22:23:12 -07004897 return 1;
4898}
4899
Jens Axboe74ce6ce2020-04-13 11:09:12 -06004900static bool io_poll_rewait(struct io_kiocb *req, struct io_poll_iocb *poll)
4901 __acquires(&req->ctx->completion_lock)
4902{
4903 struct io_ring_ctx *ctx = req->ctx;
4904
Pavel Begunkove09ee512021-07-01 13:26:05 +01004905 if (unlikely(req->task->flags & PF_EXITING))
4906 WRITE_ONCE(poll->canceled, true);
4907
Jens Axboe74ce6ce2020-04-13 11:09:12 -06004908 if (!req->result && !READ_ONCE(poll->canceled)) {
4909 struct poll_table_struct pt = { ._key = poll->events };
4910
4911 req->result = vfs_poll(req->file, &pt) & poll->events;
4912 }
4913
Jens Axboe79ebeae2021-08-10 15:18:27 -06004914 spin_lock(&ctx->completion_lock);
Jens Axboe74ce6ce2020-04-13 11:09:12 -06004915 if (!req->result && !READ_ONCE(poll->canceled)) {
4916 add_wait_queue(poll->head, &poll->wait);
4917 return true;
4918 }
4919
4920 return false;
4921}
4922
Jens Axboed4e7cd32020-08-15 11:44:50 -07004923static struct io_poll_iocb *io_poll_get_double(struct io_kiocb *req)
Jens Axboe18bceab2020-05-15 11:56:54 -06004924{
Jens Axboee8c2bc12020-08-15 18:44:09 -07004925 /* pure poll stashes this in ->async_data, poll driven retry elsewhere */
Jens Axboed4e7cd32020-08-15 11:44:50 -07004926 if (req->opcode == IORING_OP_POLL_ADD)
Jens Axboee8c2bc12020-08-15 18:44:09 -07004927 return req->async_data;
Jens Axboed4e7cd32020-08-15 11:44:50 -07004928 return req->apoll->double_poll;
4929}
4930
4931static struct io_poll_iocb *io_poll_get_single(struct io_kiocb *req)
4932{
4933 if (req->opcode == IORING_OP_POLL_ADD)
4934 return &req->poll;
4935 return &req->apoll->poll;
4936}
4937
4938static void io_poll_remove_double(struct io_kiocb *req)
Pavel Begunkove07785b2021-04-01 15:43:57 +01004939 __must_hold(&req->ctx->completion_lock)
Jens Axboed4e7cd32020-08-15 11:44:50 -07004940{
4941 struct io_poll_iocb *poll = io_poll_get_double(req);
Jens Axboe18bceab2020-05-15 11:56:54 -06004942
4943 lockdep_assert_held(&req->ctx->completion_lock);
4944
4945 if (poll && poll->head) {
4946 struct wait_queue_head *head = poll->head;
4947
Jens Axboe79ebeae2021-08-10 15:18:27 -06004948 spin_lock_irq(&head->lock);
Jens Axboe18bceab2020-05-15 11:56:54 -06004949 list_del_init(&poll->wait.entry);
4950 if (poll->wait.private)
Jens Axboede9b4cc2021-02-24 13:28:27 -07004951 req_ref_put(req);
Jens Axboe18bceab2020-05-15 11:56:54 -06004952 poll->head = NULL;
Jens Axboe79ebeae2021-08-10 15:18:27 -06004953 spin_unlock_irq(&head->lock);
Jens Axboe18bceab2020-05-15 11:56:54 -06004954 }
4955}
4956
Pavel Begunkove27414b2021-04-09 09:13:20 +01004957static bool io_poll_complete(struct io_kiocb *req, __poll_t mask)
Pavel Begunkove07785b2021-04-01 15:43:57 +01004958 __must_hold(&req->ctx->completion_lock)
Jens Axboe18bceab2020-05-15 11:56:54 -06004959{
4960 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe88e41cf2021-02-22 22:08:01 -07004961 unsigned flags = IORING_CQE_F_MORE;
Pavel Begunkove27414b2021-04-09 09:13:20 +01004962 int error;
Jens Axboe18bceab2020-05-15 11:56:54 -06004963
Pavel Begunkove27414b2021-04-09 09:13:20 +01004964 if (READ_ONCE(req->poll.canceled)) {
Jens Axboe45ab03b2021-02-23 08:19:33 -07004965 error = -ECANCELED;
Jens Axboe88e41cf2021-02-22 22:08:01 -07004966 req->poll.events |= EPOLLONESHOT;
Pavel Begunkove27414b2021-04-09 09:13:20 +01004967 } else {
Jens Axboe50826202021-02-23 09:02:26 -07004968 error = mangle_poll(mask);
Pavel Begunkove27414b2021-04-09 09:13:20 +01004969 }
Jens Axboeb69de282021-03-17 08:37:41 -06004970 if (req->poll.events & EPOLLONESHOT)
4971 flags = 0;
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01004972 if (!io_cqring_fill_event(ctx, req->user_data, error, flags)) {
Jens Axboe88e41cf2021-02-22 22:08:01 -07004973 req->poll.done = true;
4974 flags = 0;
4975 }
Hao Xu7b289c32021-04-13 15:20:39 +08004976 if (flags & IORING_CQE_F_MORE)
4977 ctx->cq_extra++;
4978
Jens Axboe18bceab2020-05-15 11:56:54 -06004979 io_commit_cqring(ctx);
Jens Axboe88e41cf2021-02-22 22:08:01 -07004980 return !(flags & IORING_CQE_F_MORE);
Jens Axboe18bceab2020-05-15 11:56:54 -06004981}
4982
Pavel Begunkov5b0a6ac2021-06-30 21:54:04 +01004983static void io_poll_task_func(struct io_kiocb *req)
Jens Axboe18bceab2020-05-15 11:56:54 -06004984{
Jens Axboe6d816e02020-08-11 08:04:14 -06004985 struct io_ring_ctx *ctx = req->ctx;
Pavel Begunkovdd221f462020-10-18 10:17:42 +01004986 struct io_kiocb *nxt;
Jens Axboe18bceab2020-05-15 11:56:54 -06004987
Pavel Begunkovdd221f462020-10-18 10:17:42 +01004988 if (io_poll_rewait(req, &req->poll)) {
Jens Axboe79ebeae2021-08-10 15:18:27 -06004989 spin_unlock(&ctx->completion_lock);
Pavel Begunkovdd221f462020-10-18 10:17:42 +01004990 } else {
Pavel Begunkovf40b9642021-04-09 09:13:19 +01004991 bool done;
Jens Axboe88e41cf2021-02-22 22:08:01 -07004992
Pavel Begunkove27414b2021-04-09 09:13:20 +01004993 done = io_poll_complete(req, req->result);
Jens Axboe88e41cf2021-02-22 22:08:01 -07004994 if (done) {
Hao Xua890d012021-07-28 11:03:22 +08004995 io_poll_remove_double(req);
Jens Axboe88e41cf2021-02-22 22:08:01 -07004996 hash_del(&req->hash_node);
Pavel Begunkovf40b9642021-04-09 09:13:19 +01004997 } else {
Jens Axboe88e41cf2021-02-22 22:08:01 -07004998 req->result = 0;
4999 add_wait_queue(req->poll.head, &req->poll.wait);
5000 }
Jens Axboe79ebeae2021-08-10 15:18:27 -06005001 spin_unlock(&ctx->completion_lock);
Pavel Begunkovdd221f462020-10-18 10:17:42 +01005002 io_cqring_ev_posted(ctx);
Pavel Begunkovdd221f462020-10-18 10:17:42 +01005003
Jens Axboe88e41cf2021-02-22 22:08:01 -07005004 if (done) {
5005 nxt = io_put_req_find_next(req);
5006 if (nxt)
Pavel Begunkov5b0a6ac2021-06-30 21:54:04 +01005007 io_req_task_submit(nxt);
Jens Axboe88e41cf2021-02-22 22:08:01 -07005008 }
Pavel Begunkovea1164e2020-06-30 15:20:41 +03005009 }
Jens Axboe18bceab2020-05-15 11:56:54 -06005010}
5011
5012static int io_poll_double_wake(struct wait_queue_entry *wait, unsigned mode,
5013 int sync, void *key)
5014{
5015 struct io_kiocb *req = wait->private;
Jens Axboed4e7cd32020-08-15 11:44:50 -07005016 struct io_poll_iocb *poll = io_poll_get_single(req);
Jens Axboe18bceab2020-05-15 11:56:54 -06005017 __poll_t mask = key_to_poll(key);
Jens Axboe79ebeae2021-08-10 15:18:27 -06005018 unsigned long flags;
Jens Axboe18bceab2020-05-15 11:56:54 -06005019
5020 /* for instances that support it check for an event match first: */
5021 if (mask && !(mask & poll->events))
5022 return 0;
Jens Axboe88e41cf2021-02-22 22:08:01 -07005023 if (!(poll->events & EPOLLONESHOT))
5024 return poll->wait.func(&poll->wait, mode, sync, key);
Jens Axboe18bceab2020-05-15 11:56:54 -06005025
Jens Axboe8706e042020-09-28 08:38:54 -06005026 list_del_init(&wait->entry);
5027
Jens Axboe9ce85ef2021-07-09 08:20:28 -06005028 if (poll->head) {
Jens Axboe18bceab2020-05-15 11:56:54 -06005029 bool done;
5030
Jens Axboe79ebeae2021-08-10 15:18:27 -06005031 spin_lock_irqsave(&poll->head->lock, flags);
Jens Axboe807abcb2020-07-17 17:09:27 -06005032 done = list_empty(&poll->wait.entry);
Jens Axboe18bceab2020-05-15 11:56:54 -06005033 if (!done)
Jens Axboe807abcb2020-07-17 17:09:27 -06005034 list_del_init(&poll->wait.entry);
Jens Axboed4e7cd32020-08-15 11:44:50 -07005035 /* make sure double remove sees this as being gone */
5036 wait->private = NULL;
Jens Axboe79ebeae2021-08-10 15:18:27 -06005037 spin_unlock_irqrestore(&poll->head->lock, flags);
Jens Axboec8b5e262020-10-25 13:53:26 -06005038 if (!done) {
5039 /* use wait func handler, so it matches the rq type */
5040 poll->wait.func(&poll->wait, mode, sync, key);
5041 }
Jens Axboe18bceab2020-05-15 11:56:54 -06005042 }
Jens Axboede9b4cc2021-02-24 13:28:27 -07005043 req_ref_put(req);
Jens Axboe18bceab2020-05-15 11:56:54 -06005044 return 1;
5045}
5046
5047static void io_init_poll_iocb(struct io_poll_iocb *poll, __poll_t events,
5048 wait_queue_func_t wake_func)
5049{
5050 poll->head = NULL;
5051 poll->done = false;
5052 poll->canceled = false;
Jens Axboe464dca62021-03-19 14:06:24 -06005053#define IO_POLL_UNMASK (EPOLLERR|EPOLLHUP|EPOLLNVAL|EPOLLRDHUP)
5054 /* mask in events that we always want/need */
5055 poll->events = events | IO_POLL_UNMASK;
Jens Axboe18bceab2020-05-15 11:56:54 -06005056 INIT_LIST_HEAD(&poll->wait.entry);
5057 init_waitqueue_func_entry(&poll->wait, wake_func);
5058}
5059
5060static void __io_queue_proc(struct io_poll_iocb *poll, struct io_poll_table *pt,
Jens Axboe807abcb2020-07-17 17:09:27 -06005061 struct wait_queue_head *head,
5062 struct io_poll_iocb **poll_ptr)
Jens Axboe18bceab2020-05-15 11:56:54 -06005063{
5064 struct io_kiocb *req = pt->req;
5065
5066 /*
Pavel Begunkov68b11e82021-07-20 10:50:43 +01005067 * The file being polled uses multiple waitqueues for poll handling
5068 * (e.g. one for read, one for write). Setup a separate io_poll_iocb
5069 * if this happens.
Jens Axboe18bceab2020-05-15 11:56:54 -06005070 */
Pavel Begunkov68b11e82021-07-20 10:50:43 +01005071 if (unlikely(pt->nr_entries)) {
Pavel Begunkov58852d42020-10-16 20:55:56 +01005072 struct io_poll_iocb *poll_one = poll;
5073
Jens Axboe18bceab2020-05-15 11:56:54 -06005074 /* already have a 2nd entry, fail a third attempt */
Jens Axboe807abcb2020-07-17 17:09:27 -06005075 if (*poll_ptr) {
Jens Axboe18bceab2020-05-15 11:56:54 -06005076 pt->error = -EINVAL;
5077 return;
5078 }
Jens Axboeea6a693d2021-04-15 09:47:13 -06005079 /*
5080 * Can't handle multishot for double wait for now, turn it
5081 * into one-shot mode.
5082 */
Pavel Begunkov7a274722021-05-17 12:43:34 +01005083 if (!(poll_one->events & EPOLLONESHOT))
5084 poll_one->events |= EPOLLONESHOT;
Jens Axboe1c3b3e62021-02-28 16:07:30 -07005085 /* double add on the same waitqueue head, ignore */
Pavel Begunkov7a274722021-05-17 12:43:34 +01005086 if (poll_one->head == head)
Jens Axboe1c3b3e62021-02-28 16:07:30 -07005087 return;
Jens Axboe18bceab2020-05-15 11:56:54 -06005088 poll = kmalloc(sizeof(*poll), GFP_ATOMIC);
5089 if (!poll) {
5090 pt->error = -ENOMEM;
5091 return;
5092 }
Pavel Begunkov58852d42020-10-16 20:55:56 +01005093 io_init_poll_iocb(poll, poll_one->events, io_poll_double_wake);
Jens Axboede9b4cc2021-02-24 13:28:27 -07005094 req_ref_get(req);
Jens Axboe18bceab2020-05-15 11:56:54 -06005095 poll->wait.private = req;
Jens Axboe807abcb2020-07-17 17:09:27 -06005096 *poll_ptr = poll;
Jens Axboe18bceab2020-05-15 11:56:54 -06005097 }
5098
Pavel Begunkov68b11e82021-07-20 10:50:43 +01005099 pt->nr_entries++;
Jens Axboe18bceab2020-05-15 11:56:54 -06005100 poll->head = head;
Jiufei Xuea31eb4a2020-06-17 17:53:56 +08005101
5102 if (poll->events & EPOLLEXCLUSIVE)
5103 add_wait_queue_exclusive(head, &poll->wait);
5104 else
5105 add_wait_queue(head, &poll->wait);
Jens Axboe18bceab2020-05-15 11:56:54 -06005106}
5107
5108static void io_async_queue_proc(struct file *file, struct wait_queue_head *head,
5109 struct poll_table_struct *p)
5110{
5111 struct io_poll_table *pt = container_of(p, struct io_poll_table, pt);
Jens Axboe807abcb2020-07-17 17:09:27 -06005112 struct async_poll *apoll = pt->req->apoll;
Jens Axboe18bceab2020-05-15 11:56:54 -06005113
Jens Axboe807abcb2020-07-17 17:09:27 -06005114 __io_queue_proc(&apoll->poll, pt, head, &apoll->double_poll);
Jens Axboe18bceab2020-05-15 11:56:54 -06005115}
5116
Pavel Begunkov5b0a6ac2021-06-30 21:54:04 +01005117static void io_async_task_func(struct io_kiocb *req)
Jens Axboed7718a92020-02-14 22:23:12 -07005118{
Jens Axboed7718a92020-02-14 22:23:12 -07005119 struct async_poll *apoll = req->apoll;
5120 struct io_ring_ctx *ctx = req->ctx;
5121
Olivier Langlois236daeae2021-05-31 02:36:37 -04005122 trace_io_uring_task_run(req->ctx, req, req->opcode, req->user_data);
Jens Axboed7718a92020-02-14 22:23:12 -07005123
Jens Axboe74ce6ce2020-04-13 11:09:12 -06005124 if (io_poll_rewait(req, &apoll->poll)) {
Jens Axboe79ebeae2021-08-10 15:18:27 -06005125 spin_unlock(&ctx->completion_lock);
Jens Axboe74ce6ce2020-04-13 11:09:12 -06005126 return;
Jens Axboed7718a92020-02-14 22:23:12 -07005127 }
5128
Pavel Begunkov0ea13b42021-04-09 09:13:21 +01005129 hash_del(&req->hash_node);
Jens Axboed4e7cd32020-08-15 11:44:50 -07005130 io_poll_remove_double(req);
Jens Axboe79ebeae2021-08-10 15:18:27 -06005131 spin_unlock(&ctx->completion_lock);
Jens Axboe74ce6ce2020-04-13 11:09:12 -06005132
Pavel Begunkov0be0b0e2020-06-30 15:20:42 +03005133 if (!READ_ONCE(apoll->poll.canceled))
Pavel Begunkov5b0a6ac2021-06-30 21:54:04 +01005134 io_req_task_submit(req);
Pavel Begunkov0be0b0e2020-06-30 15:20:42 +03005135 else
Pavel Begunkov25935532021-03-19 17:22:40 +00005136 io_req_complete_failed(req, -ECANCELED);
Jens Axboed7718a92020-02-14 22:23:12 -07005137}
5138
5139static int io_async_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
5140 void *key)
5141{
5142 struct io_kiocb *req = wait->private;
5143 struct io_poll_iocb *poll = &req->apoll->poll;
5144
5145 trace_io_uring_poll_wake(req->ctx, req->opcode, req->user_data,
5146 key_to_poll(key));
5147
5148 return __io_async_wake(req, poll, key_to_poll(key), io_async_task_func);
5149}
5150
5151static void io_poll_req_insert(struct io_kiocb *req)
5152{
5153 struct io_ring_ctx *ctx = req->ctx;
5154 struct hlist_head *list;
5155
5156 list = &ctx->cancel_hash[hash_long(req->user_data, ctx->cancel_hash_bits)];
5157 hlist_add_head(&req->hash_node, list);
5158}
5159
5160static __poll_t __io_arm_poll_handler(struct io_kiocb *req,
5161 struct io_poll_iocb *poll,
5162 struct io_poll_table *ipt, __poll_t mask,
5163 wait_queue_func_t wake_func)
5164 __acquires(&ctx->completion_lock)
5165{
5166 struct io_ring_ctx *ctx = req->ctx;
5167 bool cancel = false;
5168
Pavel Begunkov4d52f332020-10-18 10:17:43 +01005169 INIT_HLIST_NODE(&req->hash_node);
Jens Axboe18bceab2020-05-15 11:56:54 -06005170 io_init_poll_iocb(poll, mask, wake_func);
Pavel Begunkovb90cd192020-06-21 13:09:52 +03005171 poll->file = req->file;
Jens Axboe18bceab2020-05-15 11:56:54 -06005172 poll->wait.private = req;
Jens Axboed7718a92020-02-14 22:23:12 -07005173
5174 ipt->pt._key = mask;
5175 ipt->req = req;
Pavel Begunkov68b11e82021-07-20 10:50:43 +01005176 ipt->error = 0;
5177 ipt->nr_entries = 0;
Jens Axboed7718a92020-02-14 22:23:12 -07005178
Jens Axboed7718a92020-02-14 22:23:12 -07005179 mask = vfs_poll(req->file, &ipt->pt) & poll->events;
Pavel Begunkov68b11e82021-07-20 10:50:43 +01005180 if (unlikely(!ipt->nr_entries) && !ipt->error)
5181 ipt->error = -EINVAL;
Jens Axboed7718a92020-02-14 22:23:12 -07005182
Jens Axboe79ebeae2021-08-10 15:18:27 -06005183 spin_lock(&ctx->completion_lock);
Hao Xua890d012021-07-28 11:03:22 +08005184 if (ipt->error || (mask && (poll->events & EPOLLONESHOT)))
Pavel Begunkov46fee9a2021-07-20 10:50:44 +01005185 io_poll_remove_double(req);
Jens Axboed7718a92020-02-14 22:23:12 -07005186 if (likely(poll->head)) {
Jens Axboe79ebeae2021-08-10 15:18:27 -06005187 spin_lock_irq(&poll->head->lock);
Jens Axboed7718a92020-02-14 22:23:12 -07005188 if (unlikely(list_empty(&poll->wait.entry))) {
5189 if (ipt->error)
5190 cancel = true;
5191 ipt->error = 0;
5192 mask = 0;
5193 }
Jens Axboe88e41cf2021-02-22 22:08:01 -07005194 if ((mask && (poll->events & EPOLLONESHOT)) || ipt->error)
Jens Axboed7718a92020-02-14 22:23:12 -07005195 list_del_init(&poll->wait.entry);
5196 else if (cancel)
5197 WRITE_ONCE(poll->canceled, true);
5198 else if (!poll->done) /* actually waiting for an event */
5199 io_poll_req_insert(req);
Jens Axboe79ebeae2021-08-10 15:18:27 -06005200 spin_unlock_irq(&poll->head->lock);
Jens Axboed7718a92020-02-14 22:23:12 -07005201 }
5202
5203 return mask;
5204}
5205
Olivier Langlois59b735a2021-06-22 05:17:39 -07005206enum {
5207 IO_APOLL_OK,
5208 IO_APOLL_ABORTED,
5209 IO_APOLL_READY
5210};
5211
5212static int io_arm_poll_handler(struct io_kiocb *req)
Jens Axboed7718a92020-02-14 22:23:12 -07005213{
5214 const struct io_op_def *def = &io_op_defs[req->opcode];
5215 struct io_ring_ctx *ctx = req->ctx;
5216 struct async_poll *apoll;
5217 struct io_poll_table ipt;
Pavel Begunkovb2d9c3d2021-06-26 21:40:44 +01005218 __poll_t ret, mask = EPOLLONESHOT | POLLERR | POLLPRI;
Jens Axboe9dab14b2020-08-25 12:27:50 -06005219 int rw;
Jens Axboed7718a92020-02-14 22:23:12 -07005220
5221 if (!req->file || !file_can_poll(req->file))
Olivier Langlois59b735a2021-06-22 05:17:39 -07005222 return IO_APOLL_ABORTED;
Pavel Begunkov24c74672020-06-21 13:09:51 +03005223 if (req->flags & REQ_F_POLLED)
Olivier Langlois59b735a2021-06-22 05:17:39 -07005224 return IO_APOLL_ABORTED;
Pavel Begunkovb2d9c3d2021-06-26 21:40:44 +01005225 if (!def->pollin && !def->pollout)
Olivier Langlois59b735a2021-06-22 05:17:39 -07005226 return IO_APOLL_ABORTED;
Pavel Begunkovb2d9c3d2021-06-26 21:40:44 +01005227
5228 if (def->pollin) {
5229 rw = READ;
5230 mask |= POLLIN | POLLRDNORM;
5231
5232 /* If reading from MSG_ERRQUEUE using recvmsg, ignore POLLIN */
5233 if ((req->opcode == IORING_OP_RECVMSG) &&
5234 (req->sr_msg.msg_flags & MSG_ERRQUEUE))
5235 mask &= ~POLLIN;
5236 } else {
5237 rw = WRITE;
5238 mask |= POLLOUT | POLLWRNORM;
5239 }
5240
Jens Axboe9dab14b2020-08-25 12:27:50 -06005241 /* if we can't nonblock try, then no point in arming a poll handler */
Pavel Begunkovb191e2d2021-08-09 13:04:03 +01005242 if (!io_file_supports_nowait(req, rw))
Olivier Langlois59b735a2021-06-22 05:17:39 -07005243 return IO_APOLL_ABORTED;
Jens Axboed7718a92020-02-14 22:23:12 -07005244
5245 apoll = kmalloc(sizeof(*apoll), GFP_ATOMIC);
5246 if (unlikely(!apoll))
Olivier Langlois59b735a2021-06-22 05:17:39 -07005247 return IO_APOLL_ABORTED;
Jens Axboe807abcb2020-07-17 17:09:27 -06005248 apoll->double_poll = NULL;
Jens Axboed7718a92020-02-14 22:23:12 -07005249 req->apoll = apoll;
Pavel Begunkovb2d9c3d2021-06-26 21:40:44 +01005250 req->flags |= REQ_F_POLLED;
Jens Axboed7718a92020-02-14 22:23:12 -07005251 ipt.pt._qproc = io_async_queue_proc;
Pavel Begunkov48dcd382021-08-15 10:40:18 +01005252 io_req_set_refcount(req);
Jens Axboed7718a92020-02-14 22:23:12 -07005253
5254 ret = __io_arm_poll_handler(req, &apoll->poll, &ipt, mask,
5255 io_async_wake);
Jens Axboe79ebeae2021-08-10 15:18:27 -06005256 spin_unlock(&ctx->completion_lock);
Hao Xu41a51692021-08-12 15:47:02 +08005257 if (ret || ipt.error)
5258 return ret ? IO_APOLL_READY : IO_APOLL_ABORTED;
5259
Olivier Langlois236daeae2021-05-31 02:36:37 -04005260 trace_io_uring_poll_arm(ctx, req, req->opcode, req->user_data,
5261 mask, apoll->poll.events);
Olivier Langlois59b735a2021-06-22 05:17:39 -07005262 return IO_APOLL_OK;
Jens Axboed7718a92020-02-14 22:23:12 -07005263}
5264
5265static bool __io_poll_remove_one(struct io_kiocb *req,
Jens Axboeb2e720a2021-03-31 09:03:03 -06005266 struct io_poll_iocb *poll, bool do_cancel)
Pavel Begunkove07785b2021-04-01 15:43:57 +01005267 __must_hold(&req->ctx->completion_lock)
Jens Axboed7718a92020-02-14 22:23:12 -07005268{
Jens Axboeb41e9852020-02-17 09:52:41 -07005269 bool do_complete = false;
Jens Axboe221c5eb2019-01-17 09:41:58 -07005270
Jens Axboe50826202021-02-23 09:02:26 -07005271 if (!poll->head)
5272 return false;
Jens Axboe79ebeae2021-08-10 15:18:27 -06005273 spin_lock_irq(&poll->head->lock);
Jens Axboeb2e720a2021-03-31 09:03:03 -06005274 if (do_cancel)
5275 WRITE_ONCE(poll->canceled, true);
Jens Axboe392edb42019-12-09 17:52:20 -07005276 if (!list_empty(&poll->wait.entry)) {
5277 list_del_init(&poll->wait.entry);
Jens Axboeb41e9852020-02-17 09:52:41 -07005278 do_complete = true;
Jens Axboe221c5eb2019-01-17 09:41:58 -07005279 }
Jens Axboe79ebeae2021-08-10 15:18:27 -06005280 spin_unlock_irq(&poll->head->lock);
Jens Axboe3bfa5bc2020-05-17 13:54:12 -06005281 hash_del(&req->hash_node);
Jens Axboed7718a92020-02-14 22:23:12 -07005282 return do_complete;
5283}
5284
Pavel Begunkov5d709042021-08-09 20:18:13 +01005285static bool io_poll_remove_one(struct io_kiocb *req)
Pavel Begunkove07785b2021-04-01 15:43:57 +01005286 __must_hold(&req->ctx->completion_lock)
Jens Axboed7718a92020-02-14 22:23:12 -07005287{
5288 bool do_complete;
5289
Jens Axboed4e7cd32020-08-15 11:44:50 -07005290 io_poll_remove_double(req);
Pavel Begunkove31001a2021-04-13 02:58:43 +01005291 do_complete = __io_poll_remove_one(req, io_poll_get_single(req), true);
Jens Axboed4e7cd32020-08-15 11:44:50 -07005292
Jens Axboeb41e9852020-02-17 09:52:41 -07005293 if (do_complete) {
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01005294 io_cqring_fill_event(req->ctx, req->user_data, -ECANCELED, 0);
Jens Axboeb41e9852020-02-17 09:52:41 -07005295 io_commit_cqring(req->ctx);
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01005296 req_set_fail(req);
Pavel Begunkov91c2f692021-08-11 19:28:28 +01005297 io_put_req_deferred(req);
Pavel Begunkov5d709042021-08-09 20:18:13 +01005298 }
Jens Axboeb41e9852020-02-17 09:52:41 -07005299 return do_complete;
Jens Axboe221c5eb2019-01-17 09:41:58 -07005300}
5301
Jens Axboe76e1b642020-09-26 15:05:03 -06005302/*
5303 * Returns true if we found and killed one or more poll requests
5304 */
Pavel Begunkov6b819282020-11-06 13:00:25 +00005305static bool io_poll_remove_all(struct io_ring_ctx *ctx, struct task_struct *tsk,
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01005306 bool cancel_all)
Jens Axboe221c5eb2019-01-17 09:41:58 -07005307{
Jens Axboe78076bb2019-12-04 19:56:40 -07005308 struct hlist_node *tmp;
Jens Axboe221c5eb2019-01-17 09:41:58 -07005309 struct io_kiocb *req;
Jens Axboe8e2e1fa2020-04-13 17:05:14 -06005310 int posted = 0, i;
Jens Axboe221c5eb2019-01-17 09:41:58 -07005311
Jens Axboe79ebeae2021-08-10 15:18:27 -06005312 spin_lock(&ctx->completion_lock);
Jens Axboe78076bb2019-12-04 19:56:40 -07005313 for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) {
5314 struct hlist_head *list;
5315
5316 list = &ctx->cancel_hash[i];
Jens Axboef3606e32020-09-22 08:18:24 -06005317 hlist_for_each_entry_safe(req, tmp, list, hash_node) {
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01005318 if (io_match_task(req, tsk, cancel_all))
Jens Axboef3606e32020-09-22 08:18:24 -06005319 posted += io_poll_remove_one(req);
5320 }
Jens Axboe221c5eb2019-01-17 09:41:58 -07005321 }
Jens Axboe79ebeae2021-08-10 15:18:27 -06005322 spin_unlock(&ctx->completion_lock);
Jens Axboeb41e9852020-02-17 09:52:41 -07005323
Jens Axboe8e2e1fa2020-04-13 17:05:14 -06005324 if (posted)
5325 io_cqring_ev_posted(ctx);
Jens Axboe76e1b642020-09-26 15:05:03 -06005326
5327 return posted != 0;
Jens Axboe221c5eb2019-01-17 09:41:58 -07005328}
5329
Pavel Begunkov9ba5fac2021-04-14 13:38:35 +01005330static struct io_kiocb *io_poll_find(struct io_ring_ctx *ctx, __u64 sqe_addr,
5331 bool poll_only)
Pavel Begunkove07785b2021-04-01 15:43:57 +01005332 __must_hold(&ctx->completion_lock)
Jens Axboe47f46762019-11-09 17:43:02 -07005333{
Jens Axboe78076bb2019-12-04 19:56:40 -07005334 struct hlist_head *list;
Jens Axboe47f46762019-11-09 17:43:02 -07005335 struct io_kiocb *req;
5336
Jens Axboe78076bb2019-12-04 19:56:40 -07005337 list = &ctx->cancel_hash[hash_long(sqe_addr, ctx->cancel_hash_bits)];
5338 hlist_for_each_entry(req, list, hash_node) {
Jens Axboeb41e9852020-02-17 09:52:41 -07005339 if (sqe_addr != req->user_data)
5340 continue;
Pavel Begunkov9ba5fac2021-04-14 13:38:35 +01005341 if (poll_only && req->opcode != IORING_OP_POLL_ADD)
5342 continue;
Jens Axboeb2cb8052021-03-17 08:17:19 -06005343 return req;
Jens Axboe47f46762019-11-09 17:43:02 -07005344 }
Jens Axboeb2cb8052021-03-17 08:17:19 -06005345 return NULL;
Jens Axboe47f46762019-11-09 17:43:02 -07005346}
5347
Pavel Begunkov9ba5fac2021-04-14 13:38:35 +01005348static int io_poll_cancel(struct io_ring_ctx *ctx, __u64 sqe_addr,
5349 bool poll_only)
Pavel Begunkove07785b2021-04-01 15:43:57 +01005350 __must_hold(&ctx->completion_lock)
Jens Axboeb2cb8052021-03-17 08:17:19 -06005351{
5352 struct io_kiocb *req;
5353
Pavel Begunkov9ba5fac2021-04-14 13:38:35 +01005354 req = io_poll_find(ctx, sqe_addr, poll_only);
Jens Axboeb2cb8052021-03-17 08:17:19 -06005355 if (!req)
5356 return -ENOENT;
5357 if (io_poll_remove_one(req))
5358 return 0;
5359
5360 return -EALREADY;
Jens Axboe221c5eb2019-01-17 09:41:58 -07005361}
5362
Pavel Begunkov9096af32021-04-14 13:38:36 +01005363static __poll_t io_poll_parse_events(const struct io_uring_sqe *sqe,
5364 unsigned int flags)
5365{
5366 u32 events;
5367
5368 events = READ_ONCE(sqe->poll32_events);
5369#ifdef __BIG_ENDIAN
5370 events = swahw32(events);
5371#endif
5372 if (!(flags & IORING_POLL_ADD_MULTI))
5373 events |= EPOLLONESHOT;
5374 return demangle_poll(events) | (events & (EPOLLEXCLUSIVE|EPOLLONESHOT));
5375}
5376
Pavel Begunkovc5de0032021-04-14 13:38:37 +01005377static int io_poll_update_prep(struct io_kiocb *req,
Jens Axboe3529d8c2019-12-19 18:24:38 -07005378 const struct io_uring_sqe *sqe)
Jens Axboe221c5eb2019-01-17 09:41:58 -07005379{
Pavel Begunkovc5de0032021-04-14 13:38:37 +01005380 struct io_poll_update *upd = &req->poll_update;
5381 u32 flags;
5382
Jens Axboe221c5eb2019-01-17 09:41:58 -07005383 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
5384 return -EINVAL;
Pavel Begunkovc5de0032021-04-14 13:38:37 +01005385 if (sqe->ioprio || sqe->buf_index)
5386 return -EINVAL;
5387 flags = READ_ONCE(sqe->len);
5388 if (flags & ~(IORING_POLL_UPDATE_EVENTS | IORING_POLL_UPDATE_USER_DATA |
5389 IORING_POLL_ADD_MULTI))
5390 return -EINVAL;
5391 /* meaningless without update */
5392 if (flags == IORING_POLL_ADD_MULTI)
Jens Axboe221c5eb2019-01-17 09:41:58 -07005393 return -EINVAL;
5394
Pavel Begunkovc5de0032021-04-14 13:38:37 +01005395 upd->old_user_data = READ_ONCE(sqe->addr);
5396 upd->update_events = flags & IORING_POLL_UPDATE_EVENTS;
5397 upd->update_user_data = flags & IORING_POLL_UPDATE_USER_DATA;
Jens Axboe0969e782019-12-17 18:40:57 -07005398
Pavel Begunkovc5de0032021-04-14 13:38:37 +01005399 upd->new_user_data = READ_ONCE(sqe->off);
5400 if (!upd->update_user_data && upd->new_user_data)
5401 return -EINVAL;
5402 if (upd->update_events)
5403 upd->events = io_poll_parse_events(sqe, flags);
5404 else if (sqe->poll32_events)
5405 return -EINVAL;
Jens Axboe0969e782019-12-17 18:40:57 -07005406
Jens Axboe221c5eb2019-01-17 09:41:58 -07005407 return 0;
5408}
5409
Jens Axboe221c5eb2019-01-17 09:41:58 -07005410static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
5411 void *key)
5412{
Jens Axboec2f2eb72020-02-10 09:07:05 -07005413 struct io_kiocb *req = wait->private;
5414 struct io_poll_iocb *poll = &req->poll;
Jens Axboe221c5eb2019-01-17 09:41:58 -07005415
Jens Axboed7718a92020-02-14 22:23:12 -07005416 return __io_async_wake(req, poll, key_to_poll(key), io_poll_task_func);
Jens Axboe221c5eb2019-01-17 09:41:58 -07005417}
5418
Jens Axboe221c5eb2019-01-17 09:41:58 -07005419static void io_poll_queue_proc(struct file *file, struct wait_queue_head *head,
5420 struct poll_table_struct *p)
5421{
5422 struct io_poll_table *pt = container_of(p, struct io_poll_table, pt);
5423
Jens Axboee8c2bc12020-08-15 18:44:09 -07005424 __io_queue_proc(&pt->req->poll, pt, head, (struct io_poll_iocb **) &pt->req->async_data);
Jens Axboeeac406c2019-11-14 12:09:58 -07005425}
5426
Jens Axboe3529d8c2019-12-19 18:24:38 -07005427static int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboe221c5eb2019-01-17 09:41:58 -07005428{
5429 struct io_poll_iocb *poll = &req->poll;
Pavel Begunkovc5de0032021-04-14 13:38:37 +01005430 u32 flags;
Jens Axboe221c5eb2019-01-17 09:41:58 -07005431
5432 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
5433 return -EINVAL;
Pavel Begunkovc5de0032021-04-14 13:38:37 +01005434 if (sqe->ioprio || sqe->buf_index || sqe->off || sqe->addr)
Jens Axboe88e41cf2021-02-22 22:08:01 -07005435 return -EINVAL;
5436 flags = READ_ONCE(sqe->len);
Pavel Begunkovc5de0032021-04-14 13:38:37 +01005437 if (flags & ~IORING_POLL_ADD_MULTI)
Jens Axboe221c5eb2019-01-17 09:41:58 -07005438 return -EINVAL;
5439
Pavel Begunkov48dcd382021-08-15 10:40:18 +01005440 io_req_set_refcount(req);
Pavel Begunkovc5de0032021-04-14 13:38:37 +01005441 poll->events = io_poll_parse_events(sqe, flags);
Jens Axboe0969e782019-12-17 18:40:57 -07005442 return 0;
5443}
5444
Pavel Begunkov61e98202021-02-10 00:03:08 +00005445static int io_poll_add(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe0969e782019-12-17 18:40:57 -07005446{
5447 struct io_poll_iocb *poll = &req->poll;
5448 struct io_ring_ctx *ctx = req->ctx;
5449 struct io_poll_table ipt;
Jens Axboe0969e782019-12-17 18:40:57 -07005450 __poll_t mask;
Jens Axboe0969e782019-12-17 18:40:57 -07005451
Jens Axboed7718a92020-02-14 22:23:12 -07005452 ipt.pt._qproc = io_poll_queue_proc;
Jens Axboe36703242019-07-25 10:20:18 -06005453
Jens Axboed7718a92020-02-14 22:23:12 -07005454 mask = __io_arm_poll_handler(req, &req->poll, &ipt, poll->events,
5455 io_poll_wake);
Jens Axboe221c5eb2019-01-17 09:41:58 -07005456
Jens Axboe8c838782019-03-12 15:48:16 -06005457 if (mask) { /* no async, we'd stolen it */
Jens Axboe8c838782019-03-12 15:48:16 -06005458 ipt.error = 0;
Pavel Begunkove27414b2021-04-09 09:13:20 +01005459 io_poll_complete(req, mask);
Jens Axboe8c838782019-03-12 15:48:16 -06005460 }
Jens Axboe79ebeae2021-08-10 15:18:27 -06005461 spin_unlock(&ctx->completion_lock);
Jens Axboe221c5eb2019-01-17 09:41:58 -07005462
Jens Axboe8c838782019-03-12 15:48:16 -06005463 if (mask) {
5464 io_cqring_ev_posted(ctx);
Jens Axboe88e41cf2021-02-22 22:08:01 -07005465 if (poll->events & EPOLLONESHOT)
5466 io_put_req(req);
Jens Axboe221c5eb2019-01-17 09:41:58 -07005467 }
Jens Axboe8c838782019-03-12 15:48:16 -06005468 return ipt.error;
Jens Axboe221c5eb2019-01-17 09:41:58 -07005469}
5470
Pavel Begunkovc5de0032021-04-14 13:38:37 +01005471static int io_poll_update(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboeb69de282021-03-17 08:37:41 -06005472{
5473 struct io_ring_ctx *ctx = req->ctx;
5474 struct io_kiocb *preq;
Jens Axboecb3b200e2021-04-06 09:49:31 -06005475 bool completing;
Jens Axboeb69de282021-03-17 08:37:41 -06005476 int ret;
5477
Jens Axboe79ebeae2021-08-10 15:18:27 -06005478 spin_lock(&ctx->completion_lock);
Pavel Begunkov9ba5fac2021-04-14 13:38:35 +01005479 preq = io_poll_find(ctx, req->poll_update.old_user_data, true);
Jens Axboeb69de282021-03-17 08:37:41 -06005480 if (!preq) {
5481 ret = -ENOENT;
5482 goto err;
Jens Axboeb69de282021-03-17 08:37:41 -06005483 }
Jens Axboecb3b200e2021-04-06 09:49:31 -06005484
Pavel Begunkovc5de0032021-04-14 13:38:37 +01005485 if (!req->poll_update.update_events && !req->poll_update.update_user_data) {
5486 completing = true;
5487 ret = io_poll_remove_one(preq) ? 0 : -EALREADY;
5488 goto err;
5489 }
5490
Jens Axboecb3b200e2021-04-06 09:49:31 -06005491 /*
5492 * Don't allow racy completion with singleshot, as we cannot safely
5493 * update those. For multishot, if we're racing with completion, just
5494 * let completion re-add it.
5495 */
5496 completing = !__io_poll_remove_one(preq, &preq->poll, false);
5497 if (completing && (preq->poll.events & EPOLLONESHOT)) {
5498 ret = -EALREADY;
5499 goto err;
Jens Axboeb69de282021-03-17 08:37:41 -06005500 }
5501 /* we now have a detached poll request. reissue. */
5502 ret = 0;
5503err:
Jens Axboeb69de282021-03-17 08:37:41 -06005504 if (ret < 0) {
Jens Axboe79ebeae2021-08-10 15:18:27 -06005505 spin_unlock(&ctx->completion_lock);
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01005506 req_set_fail(req);
Jens Axboeb69de282021-03-17 08:37:41 -06005507 io_req_complete(req, ret);
5508 return 0;
5509 }
5510 /* only mask one event flags, keep behavior flags */
Pavel Begunkov9d805892021-04-13 02:58:40 +01005511 if (req->poll_update.update_events) {
Jens Axboeb69de282021-03-17 08:37:41 -06005512 preq->poll.events &= ~0xffff;
Pavel Begunkov9d805892021-04-13 02:58:40 +01005513 preq->poll.events |= req->poll_update.events & 0xffff;
Jens Axboeb69de282021-03-17 08:37:41 -06005514 preq->poll.events |= IO_POLL_UNMASK;
5515 }
Pavel Begunkov9d805892021-04-13 02:58:40 +01005516 if (req->poll_update.update_user_data)
5517 preq->user_data = req->poll_update.new_user_data;
Jens Axboe79ebeae2021-08-10 15:18:27 -06005518 spin_unlock(&ctx->completion_lock);
Jens Axboecb3b200e2021-04-06 09:49:31 -06005519
Jens Axboeb69de282021-03-17 08:37:41 -06005520 /* complete update request, we're done with it */
5521 io_req_complete(req, ret);
5522
Jens Axboecb3b200e2021-04-06 09:49:31 -06005523 if (!completing) {
Pavel Begunkovc5de0032021-04-14 13:38:37 +01005524 ret = io_poll_add(preq, issue_flags);
Jens Axboecb3b200e2021-04-06 09:49:31 -06005525 if (ret < 0) {
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01005526 req_set_fail(preq);
Jens Axboecb3b200e2021-04-06 09:49:31 -06005527 io_req_complete(preq, ret);
5528 }
Jens Axboeb69de282021-03-17 08:37:41 -06005529 }
5530 return 0;
5531}
5532
Jens Axboe89850fc2021-08-10 15:11:51 -06005533static void io_req_task_timeout(struct io_kiocb *req)
5534{
5535 struct io_ring_ctx *ctx = req->ctx;
5536
Jens Axboe79ebeae2021-08-10 15:18:27 -06005537 spin_lock(&ctx->completion_lock);
Jens Axboe89850fc2021-08-10 15:11:51 -06005538 io_cqring_fill_event(ctx, req->user_data, -ETIME, 0);
5539 io_commit_cqring(ctx);
Jens Axboe79ebeae2021-08-10 15:18:27 -06005540 spin_unlock(&ctx->completion_lock);
Jens Axboe89850fc2021-08-10 15:11:51 -06005541
5542 io_cqring_ev_posted(ctx);
5543 req_set_fail(req);
5544 io_put_req(req);
5545}
5546
Jens Axboe5262f562019-09-17 12:26:57 -06005547static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer)
5548{
Jens Axboead8a48a2019-11-15 08:49:11 -07005549 struct io_timeout_data *data = container_of(timer,
5550 struct io_timeout_data, timer);
5551 struct io_kiocb *req = data->req;
5552 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe5262f562019-09-17 12:26:57 -06005553 unsigned long flags;
5554
Jens Axboe89850fc2021-08-10 15:11:51 -06005555 spin_lock_irqsave(&ctx->timeout_lock, flags);
Pavel Begunkova71976f2020-10-10 18:34:11 +01005556 list_del_init(&req->timeout.list);
Pavel Begunkov01cec8c2020-07-30 18:43:50 +03005557 atomic_set(&req->ctx->cq_timeouts,
5558 atomic_read(&req->ctx->cq_timeouts) + 1);
Jens Axboe89850fc2021-08-10 15:11:51 -06005559 spin_unlock_irqrestore(&ctx->timeout_lock, flags);
Pavel Begunkov01cec8c2020-07-30 18:43:50 +03005560
Jens Axboe89850fc2021-08-10 15:11:51 -06005561 req->io_task_work.func = io_req_task_timeout;
5562 io_req_task_work_add(req);
Jens Axboe5262f562019-09-17 12:26:57 -06005563 return HRTIMER_NORESTART;
5564}
5565
Pavel Begunkovfbd15842020-11-30 19:11:15 +00005566static struct io_kiocb *io_timeout_extract(struct io_ring_ctx *ctx,
5567 __u64 user_data)
Jens Axboe89850fc2021-08-10 15:11:51 -06005568 __must_hold(&ctx->timeout_lock)
Jens Axboe47f46762019-11-09 17:43:02 -07005569{
Pavel Begunkovfbd15842020-11-30 19:11:15 +00005570 struct io_timeout_data *io;
Jens Axboef254ac02020-08-12 17:33:30 -06005571 struct io_kiocb *req;
Pavel Begunkovfd9c7bc2021-04-13 02:58:42 +01005572 bool found = false;
Jens Axboef254ac02020-08-12 17:33:30 -06005573
5574 list_for_each_entry(req, &ctx->timeout_list, timeout.list) {
Pavel Begunkovfd9c7bc2021-04-13 02:58:42 +01005575 found = user_data == req->user_data;
5576 if (found)
Jens Axboef254ac02020-08-12 17:33:30 -06005577 break;
Jens Axboef254ac02020-08-12 17:33:30 -06005578 }
Pavel Begunkovfd9c7bc2021-04-13 02:58:42 +01005579 if (!found)
5580 return ERR_PTR(-ENOENT);
Jens Axboef254ac02020-08-12 17:33:30 -06005581
Pavel Begunkovfbd15842020-11-30 19:11:15 +00005582 io = req->async_data;
Pavel Begunkovfd9c7bc2021-04-13 02:58:42 +01005583 if (hrtimer_try_to_cancel(&io->timer) == -1)
Pavel Begunkovfbd15842020-11-30 19:11:15 +00005584 return ERR_PTR(-EALREADY);
5585 list_del_init(&req->timeout.list);
5586 return req;
5587}
5588
5589static int io_timeout_cancel(struct io_ring_ctx *ctx, __u64 user_data)
Jens Axboe89850fc2021-08-10 15:11:51 -06005590 __must_hold(&ctx->timeout_lock)
Pavel Begunkovfbd15842020-11-30 19:11:15 +00005591{
5592 struct io_kiocb *req = io_timeout_extract(ctx, user_data);
5593
5594 if (IS_ERR(req))
5595 return PTR_ERR(req);
5596
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01005597 req_set_fail(req);
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01005598 io_cqring_fill_event(ctx, req->user_data, -ECANCELED, 0);
Pavel Begunkov91c2f692021-08-11 19:28:28 +01005599 io_put_req_deferred(req);
Pavel Begunkovfbd15842020-11-30 19:11:15 +00005600 return 0;
Jens Axboef254ac02020-08-12 17:33:30 -06005601}
5602
Pavel Begunkov9c8e11b2020-11-30 19:11:16 +00005603static int io_timeout_update(struct io_ring_ctx *ctx, __u64 user_data,
5604 struct timespec64 *ts, enum hrtimer_mode mode)
Jens Axboe89850fc2021-08-10 15:11:51 -06005605 __must_hold(&ctx->timeout_lock)
Pavel Begunkov9c8e11b2020-11-30 19:11:16 +00005606{
5607 struct io_kiocb *req = io_timeout_extract(ctx, user_data);
5608 struct io_timeout_data *data;
5609
5610 if (IS_ERR(req))
5611 return PTR_ERR(req);
5612
5613 req->timeout.off = 0; /* noseq */
5614 data = req->async_data;
5615 list_add_tail(&req->timeout.list, &ctx->timeout_list);
5616 hrtimer_init(&data->timer, CLOCK_MONOTONIC, mode);
5617 data->timer.function = io_timeout_fn;
5618 hrtimer_start(&data->timer, timespec64_to_ktime(*ts), mode);
5619 return 0;
Jens Axboe47f46762019-11-09 17:43:02 -07005620}
5621
Jens Axboe3529d8c2019-12-19 18:24:38 -07005622static int io_timeout_remove_prep(struct io_kiocb *req,
5623 const struct io_uring_sqe *sqe)
Jens Axboeb29472e2019-12-17 18:50:29 -07005624{
Pavel Begunkov9c8e11b2020-11-30 19:11:16 +00005625 struct io_timeout_rem *tr = &req->timeout_rem;
5626
Jens Axboeb29472e2019-12-17 18:50:29 -07005627 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
5628 return -EINVAL;
Daniele Albano61710e42020-07-18 14:15:16 -06005629 if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT)))
5630 return -EINVAL;
Pavel Begunkov9c8e11b2020-11-30 19:11:16 +00005631 if (sqe->ioprio || sqe->buf_index || sqe->len)
Jens Axboeb29472e2019-12-17 18:50:29 -07005632 return -EINVAL;
5633
Pavel Begunkov9c8e11b2020-11-30 19:11:16 +00005634 tr->addr = READ_ONCE(sqe->addr);
5635 tr->flags = READ_ONCE(sqe->timeout_flags);
5636 if (tr->flags & IORING_TIMEOUT_UPDATE) {
5637 if (tr->flags & ~(IORING_TIMEOUT_UPDATE|IORING_TIMEOUT_ABS))
5638 return -EINVAL;
5639 if (get_timespec64(&tr->ts, u64_to_user_ptr(sqe->addr2)))
5640 return -EFAULT;
5641 } else if (tr->flags) {
5642 /* timeout removal doesn't support flags */
5643 return -EINVAL;
5644 }
5645
Jens Axboeb29472e2019-12-17 18:50:29 -07005646 return 0;
5647}
5648
Pavel Begunkov8662dae2021-01-19 13:32:44 +00005649static inline enum hrtimer_mode io_translate_timeout_mode(unsigned int flags)
5650{
5651 return (flags & IORING_TIMEOUT_ABS) ? HRTIMER_MODE_ABS
5652 : HRTIMER_MODE_REL;
5653}
5654
Jens Axboe11365042019-10-16 09:08:32 -06005655/*
5656 * Remove or update an existing timeout command
5657 */
Pavel Begunkov61e98202021-02-10 00:03:08 +00005658static int io_timeout_remove(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe11365042019-10-16 09:08:32 -06005659{
Pavel Begunkov9c8e11b2020-11-30 19:11:16 +00005660 struct io_timeout_rem *tr = &req->timeout_rem;
Jens Axboe11365042019-10-16 09:08:32 -06005661 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe47f46762019-11-09 17:43:02 -07005662 int ret;
Jens Axboe11365042019-10-16 09:08:32 -06005663
Jens Axboe89850fc2021-08-10 15:11:51 -06005664 spin_lock_irq(&ctx->timeout_lock);
Pavel Begunkov8662dae2021-01-19 13:32:44 +00005665 if (!(req->timeout_rem.flags & IORING_TIMEOUT_UPDATE))
Pavel Begunkov9c8e11b2020-11-30 19:11:16 +00005666 ret = io_timeout_cancel(ctx, tr->addr);
Pavel Begunkov8662dae2021-01-19 13:32:44 +00005667 else
5668 ret = io_timeout_update(ctx, tr->addr, &tr->ts,
5669 io_translate_timeout_mode(tr->flags));
Jens Axboe89850fc2021-08-10 15:11:51 -06005670 spin_unlock_irq(&ctx->timeout_lock);
Jens Axboe11365042019-10-16 09:08:32 -06005671
Jens Axboe79ebeae2021-08-10 15:18:27 -06005672 spin_lock(&ctx->completion_lock);
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01005673 io_cqring_fill_event(ctx, req->user_data, ret, 0);
Jens Axboe11365042019-10-16 09:08:32 -06005674 io_commit_cqring(ctx);
Jens Axboe79ebeae2021-08-10 15:18:27 -06005675 spin_unlock(&ctx->completion_lock);
Jens Axboe5262f562019-09-17 12:26:57 -06005676 io_cqring_ev_posted(ctx);
Jens Axboe4e88d6e2019-12-07 20:59:47 -07005677 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01005678 req_set_fail(req);
Jackie Liuec9c02a2019-11-08 23:50:36 +08005679 io_put_req(req);
Jens Axboe11365042019-10-16 09:08:32 -06005680 return 0;
Jens Axboe5262f562019-09-17 12:26:57 -06005681}
5682
Jens Axboe3529d8c2019-12-19 18:24:38 -07005683static int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe,
Jens Axboe2d283902019-12-04 11:08:05 -07005684 bool is_timeout_link)
Jens Axboe5262f562019-09-17 12:26:57 -06005685{
Jens Axboead8a48a2019-11-15 08:49:11 -07005686 struct io_timeout_data *data;
Jens Axboea41525a2019-10-15 16:48:15 -06005687 unsigned flags;
Pavel Begunkov56080b02020-05-26 20:34:04 +03005688 u32 off = READ_ONCE(sqe->off);
Jens Axboe5262f562019-09-17 12:26:57 -06005689
Jens Axboead8a48a2019-11-15 08:49:11 -07005690 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
Jens Axboe5262f562019-09-17 12:26:57 -06005691 return -EINVAL;
Jens Axboead8a48a2019-11-15 08:49:11 -07005692 if (sqe->ioprio || sqe->buf_index || sqe->len != 1)
Jens Axboea41525a2019-10-15 16:48:15 -06005693 return -EINVAL;
Pavel Begunkov56080b02020-05-26 20:34:04 +03005694 if (off && is_timeout_link)
Jens Axboe2d283902019-12-04 11:08:05 -07005695 return -EINVAL;
Jens Axboea41525a2019-10-15 16:48:15 -06005696 flags = READ_ONCE(sqe->timeout_flags);
5697 if (flags & ~IORING_TIMEOUT_ABS)
Jens Axboe5262f562019-09-17 12:26:57 -06005698 return -EINVAL;
Arnd Bergmannbdf20072019-10-01 09:53:29 -06005699
Pavel Begunkovbfe68a22020-05-30 14:54:18 +03005700 req->timeout.off = off;
Pavel Begunkovf18ee4c2021-06-14 23:37:25 +01005701 if (unlikely(off && !req->ctx->off_timeout_used))
5702 req->ctx->off_timeout_used = true;
Jens Axboe26a61672019-12-20 09:02:01 -07005703
Jens Axboee8c2bc12020-08-15 18:44:09 -07005704 if (!req->async_data && io_alloc_async_data(req))
Jens Axboe26a61672019-12-20 09:02:01 -07005705 return -ENOMEM;
5706
Jens Axboee8c2bc12020-08-15 18:44:09 -07005707 data = req->async_data;
Jens Axboead8a48a2019-11-15 08:49:11 -07005708 data->req = req;
Jens Axboead8a48a2019-11-15 08:49:11 -07005709
5710 if (get_timespec64(&data->ts, u64_to_user_ptr(sqe->addr)))
Jens Axboe5262f562019-09-17 12:26:57 -06005711 return -EFAULT;
5712
Pavel Begunkov8662dae2021-01-19 13:32:44 +00005713 data->mode = io_translate_timeout_mode(flags);
Jens Axboead8a48a2019-11-15 08:49:11 -07005714 hrtimer_init(&data->timer, CLOCK_MONOTONIC, data->mode);
Pavel Begunkovb97e7362021-08-15 10:40:23 +01005715
5716 if (is_timeout_link) {
5717 struct io_submit_link *link = &req->ctx->submit_state.link;
5718
5719 if (!link->head)
5720 return -EINVAL;
5721 if (link->last->opcode == IORING_OP_LINK_TIMEOUT)
5722 return -EINVAL;
Pavel Begunkov4d13d1a2021-08-15 10:40:24 +01005723 req->timeout.head = link->last;
5724 link->last->flags |= REQ_F_ARM_LTIMEOUT;
Pavel Begunkovb97e7362021-08-15 10:40:23 +01005725 }
Jens Axboead8a48a2019-11-15 08:49:11 -07005726 return 0;
5727}
5728
Pavel Begunkov61e98202021-02-10 00:03:08 +00005729static int io_timeout(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboead8a48a2019-11-15 08:49:11 -07005730{
Jens Axboead8a48a2019-11-15 08:49:11 -07005731 struct io_ring_ctx *ctx = req->ctx;
Jens Axboee8c2bc12020-08-15 18:44:09 -07005732 struct io_timeout_data *data = req->async_data;
Jens Axboead8a48a2019-11-15 08:49:11 -07005733 struct list_head *entry;
Pavel Begunkovbfe68a22020-05-30 14:54:18 +03005734 u32 tail, off = req->timeout.off;
Jens Axboead8a48a2019-11-15 08:49:11 -07005735
Jens Axboe89850fc2021-08-10 15:11:51 -06005736 spin_lock_irq(&ctx->timeout_lock);
Jens Axboe93bd25b2019-11-11 23:34:31 -07005737
Jens Axboe5262f562019-09-17 12:26:57 -06005738 /*
5739 * sqe->off holds how many events that need to occur for this
Jens Axboe93bd25b2019-11-11 23:34:31 -07005740 * timeout event to be satisfied. If it isn't set, then this is
5741 * a pure timeout request, sequence isn't used.
Jens Axboe5262f562019-09-17 12:26:57 -06005742 */
Pavel Begunkov8eb7e2d2020-06-29 13:13:02 +03005743 if (io_is_timeout_noseq(req)) {
Jens Axboe93bd25b2019-11-11 23:34:31 -07005744 entry = ctx->timeout_list.prev;
5745 goto add;
5746 }
Jens Axboe5262f562019-09-17 12:26:57 -06005747
Pavel Begunkovbfe68a22020-05-30 14:54:18 +03005748 tail = ctx->cached_cq_tail - atomic_read(&ctx->cq_timeouts);
5749 req->timeout.target_seq = tail + off;
Jens Axboe5262f562019-09-17 12:26:57 -06005750
Marcelo Diop-Gonzalezf0105052021-01-15 11:54:40 -05005751 /* Update the last seq here in case io_flush_timeouts() hasn't.
5752 * This is safe because ->completion_lock is held, and submissions
5753 * and completions are never mixed in the same ->completion_lock section.
5754 */
5755 ctx->cq_last_tm_flush = tail;
5756
Jens Axboe5262f562019-09-17 12:26:57 -06005757 /*
5758 * Insertion sort, ensuring the first entry in the list is always
5759 * the one we need first.
5760 */
Jens Axboe5262f562019-09-17 12:26:57 -06005761 list_for_each_prev(entry, &ctx->timeout_list) {
Pavel Begunkov135fcde2020-07-13 23:37:12 +03005762 struct io_kiocb *nxt = list_entry(entry, struct io_kiocb,
5763 timeout.list);
Jens Axboe5262f562019-09-17 12:26:57 -06005764
Pavel Begunkov8eb7e2d2020-06-29 13:13:02 +03005765 if (io_is_timeout_noseq(nxt))
Jens Axboe93bd25b2019-11-11 23:34:31 -07005766 continue;
Pavel Begunkovbfe68a22020-05-30 14:54:18 +03005767 /* nxt.seq is behind @tail, otherwise would've been completed */
5768 if (off >= nxt->timeout.target_seq - tail)
Jens Axboe5262f562019-09-17 12:26:57 -06005769 break;
5770 }
Jens Axboe93bd25b2019-11-11 23:34:31 -07005771add:
Pavel Begunkov135fcde2020-07-13 23:37:12 +03005772 list_add(&req->timeout.list, entry);
Jens Axboead8a48a2019-11-15 08:49:11 -07005773 data->timer.function = io_timeout_fn;
5774 hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), data->mode);
Jens Axboe89850fc2021-08-10 15:11:51 -06005775 spin_unlock_irq(&ctx->timeout_lock);
Jens Axboe5262f562019-09-17 12:26:57 -06005776 return 0;
5777}
5778
Pavel Begunkovf458dd842021-03-08 12:14:14 +00005779struct io_cancel_data {
5780 struct io_ring_ctx *ctx;
5781 u64 user_data;
5782};
5783
Jens Axboe62755e32019-10-28 21:49:21 -06005784static bool io_cancel_cb(struct io_wq_work *work, void *data)
Jens Axboede0617e2019-04-06 21:51:27 -06005785{
Jens Axboe62755e32019-10-28 21:49:21 -06005786 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
Pavel Begunkovf458dd842021-03-08 12:14:14 +00005787 struct io_cancel_data *cd = data;
Jens Axboede0617e2019-04-06 21:51:27 -06005788
Pavel Begunkovf458dd842021-03-08 12:14:14 +00005789 return req->ctx == cd->ctx && req->user_data == cd->user_data;
Jens Axboe62755e32019-10-28 21:49:21 -06005790}
5791
Pavel Begunkovf458dd842021-03-08 12:14:14 +00005792static int io_async_cancel_one(struct io_uring_task *tctx, u64 user_data,
5793 struct io_ring_ctx *ctx)
Jens Axboe62755e32019-10-28 21:49:21 -06005794{
Pavel Begunkovf458dd842021-03-08 12:14:14 +00005795 struct io_cancel_data data = { .ctx = ctx, .user_data = user_data, };
Jens Axboe62755e32019-10-28 21:49:21 -06005796 enum io_wq_cancel cancel_ret;
Jens Axboe62755e32019-10-28 21:49:21 -06005797 int ret = 0;
5798
Pavel Begunkovf458dd842021-03-08 12:14:14 +00005799 if (!tctx || !tctx->io_wq)
Jens Axboe5aa75ed2021-02-16 12:56:50 -07005800 return -ENOENT;
5801
Pavel Begunkovf458dd842021-03-08 12:14:14 +00005802 cancel_ret = io_wq_cancel_cb(tctx->io_wq, io_cancel_cb, &data, false);
Jens Axboe62755e32019-10-28 21:49:21 -06005803 switch (cancel_ret) {
5804 case IO_WQ_CANCEL_OK:
5805 ret = 0;
5806 break;
5807 case IO_WQ_CANCEL_RUNNING:
5808 ret = -EALREADY;
5809 break;
5810 case IO_WQ_CANCEL_NOTFOUND:
5811 ret = -ENOENT;
5812 break;
5813 }
5814
Jens Axboee977d6d2019-11-05 12:39:45 -07005815 return ret;
5816}
5817
Pavel Begunkov8cb01fa2021-08-15 10:40:22 +01005818static int io_try_cancel_userdata(struct io_kiocb *req, u64 sqe_addr)
5819 __acquires(&req->ctx->completion_lock)
Jens Axboe47f46762019-11-09 17:43:02 -07005820{
Pavel Begunkov8cb01fa2021-08-15 10:40:22 +01005821 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe47f46762019-11-09 17:43:02 -07005822 int ret;
5823
Pavel Begunkov8cb01fa2021-08-15 10:40:22 +01005824 WARN_ON_ONCE(req->task != current);
5825
Pavel Begunkovf458dd842021-03-08 12:14:14 +00005826 ret = io_async_cancel_one(req->task->io_uring, sqe_addr, ctx);
Jens Axboe79ebeae2021-08-10 15:18:27 -06005827 spin_lock(&ctx->completion_lock);
Pavel Begunkovdf9727a2021-04-01 15:43:59 +01005828 if (ret != -ENOENT)
Pavel Begunkov8cb01fa2021-08-15 10:40:22 +01005829 return ret;
Jens Axboe79ebeae2021-08-10 15:18:27 -06005830 spin_lock_irq(&ctx->timeout_lock);
Jens Axboe47f46762019-11-09 17:43:02 -07005831 ret = io_timeout_cancel(ctx, sqe_addr);
Jens Axboe79ebeae2021-08-10 15:18:27 -06005832 spin_unlock_irq(&ctx->timeout_lock);
Jens Axboe47f46762019-11-09 17:43:02 -07005833 if (ret != -ENOENT)
Pavel Begunkov8cb01fa2021-08-15 10:40:22 +01005834 return ret;
5835 return io_poll_cancel(ctx, sqe_addr, false);
Jens Axboe47f46762019-11-09 17:43:02 -07005836}
5837
Jens Axboe3529d8c2019-12-19 18:24:38 -07005838static int io_async_cancel_prep(struct io_kiocb *req,
5839 const struct io_uring_sqe *sqe)
Jens Axboee977d6d2019-11-05 12:39:45 -07005840{
Jens Axboefbf23842019-12-17 18:45:56 -07005841 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
Jens Axboee977d6d2019-11-05 12:39:45 -07005842 return -EINVAL;
Daniele Albano61710e42020-07-18 14:15:16 -06005843 if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT)))
5844 return -EINVAL;
5845 if (sqe->ioprio || sqe->off || sqe->len || sqe->cancel_flags)
Jens Axboee977d6d2019-11-05 12:39:45 -07005846 return -EINVAL;
5847
Jens Axboefbf23842019-12-17 18:45:56 -07005848 req->cancel.addr = READ_ONCE(sqe->addr);
5849 return 0;
5850}
5851
Pavel Begunkov61e98202021-02-10 00:03:08 +00005852static int io_async_cancel(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboefbf23842019-12-17 18:45:56 -07005853{
5854 struct io_ring_ctx *ctx = req->ctx;
Pavel Begunkov58f99372021-03-12 16:25:55 +00005855 u64 sqe_addr = req->cancel.addr;
5856 struct io_tctx_node *node;
5857 int ret;
Jens Axboefbf23842019-12-17 18:45:56 -07005858
Pavel Begunkov8cb01fa2021-08-15 10:40:22 +01005859 ret = io_try_cancel_userdata(req, sqe_addr);
Pavel Begunkov58f99372021-03-12 16:25:55 +00005860 if (ret != -ENOENT)
5861 goto done;
Jens Axboe79ebeae2021-08-10 15:18:27 -06005862 spin_unlock(&ctx->completion_lock);
Pavel Begunkov58f99372021-03-12 16:25:55 +00005863
5864 /* slow path, try all io-wq's */
5865 io_ring_submit_lock(ctx, !(issue_flags & IO_URING_F_NONBLOCK));
5866 ret = -ENOENT;
5867 list_for_each_entry(node, &ctx->tctx_list, ctx_node) {
5868 struct io_uring_task *tctx = node->task->io_uring;
5869
Pavel Begunkov58f99372021-03-12 16:25:55 +00005870 ret = io_async_cancel_one(tctx, req->cancel.addr, ctx);
5871 if (ret != -ENOENT)
5872 break;
5873 }
5874 io_ring_submit_unlock(ctx, !(issue_flags & IO_URING_F_NONBLOCK));
5875
Jens Axboe79ebeae2021-08-10 15:18:27 -06005876 spin_lock(&ctx->completion_lock);
Pavel Begunkov58f99372021-03-12 16:25:55 +00005877done:
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01005878 io_cqring_fill_event(ctx, req->user_data, ret, 0);
Pavel Begunkov58f99372021-03-12 16:25:55 +00005879 io_commit_cqring(ctx);
Jens Axboe79ebeae2021-08-10 15:18:27 -06005880 spin_unlock(&ctx->completion_lock);
Pavel Begunkov58f99372021-03-12 16:25:55 +00005881 io_cqring_ev_posted(ctx);
5882
5883 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01005884 req_set_fail(req);
Pavel Begunkov58f99372021-03-12 16:25:55 +00005885 io_put_req(req);
Jens Axboe62755e32019-10-28 21:49:21 -06005886 return 0;
5887}
5888
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00005889static int io_rsrc_update_prep(struct io_kiocb *req,
Jens Axboe05f3fb32019-12-09 11:22:50 -07005890 const struct io_uring_sqe *sqe)
5891{
Daniele Albano61710e42020-07-18 14:15:16 -06005892 if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT)))
5893 return -EINVAL;
5894 if (sqe->ioprio || sqe->rw_flags)
Jens Axboe05f3fb32019-12-09 11:22:50 -07005895 return -EINVAL;
5896
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00005897 req->rsrc_update.offset = READ_ONCE(sqe->off);
5898 req->rsrc_update.nr_args = READ_ONCE(sqe->len);
5899 if (!req->rsrc_update.nr_args)
Jens Axboe05f3fb32019-12-09 11:22:50 -07005900 return -EINVAL;
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00005901 req->rsrc_update.arg = READ_ONCE(sqe->addr);
Jens Axboe05f3fb32019-12-09 11:22:50 -07005902 return 0;
5903}
5904
Pavel Begunkov889fca72021-02-10 00:03:09 +00005905static int io_files_update(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe05f3fb32019-12-09 11:22:50 -07005906{
5907 struct io_ring_ctx *ctx = req->ctx;
Pavel Begunkovc3bdad02021-04-25 14:32:22 +01005908 struct io_uring_rsrc_update2 up;
Jens Axboe05f3fb32019-12-09 11:22:50 -07005909 int ret;
5910
Pavel Begunkov45d189c2021-02-10 00:03:07 +00005911 if (issue_flags & IO_URING_F_NONBLOCK)
Jens Axboe05f3fb32019-12-09 11:22:50 -07005912 return -EAGAIN;
Jens Axboe05f3fb32019-12-09 11:22:50 -07005913
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00005914 up.offset = req->rsrc_update.offset;
5915 up.data = req->rsrc_update.arg;
Pavel Begunkovc3bdad02021-04-25 14:32:22 +01005916 up.nr = 0;
5917 up.tags = 0;
Colin Ian King615cee42021-04-26 10:47:35 +01005918 up.resv = 0;
Jens Axboe05f3fb32019-12-09 11:22:50 -07005919
5920 mutex_lock(&ctx->uring_lock);
Pavel Begunkovfdecb662021-04-25 14:32:20 +01005921 ret = __io_register_rsrc_update(ctx, IORING_RSRC_FILE,
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +01005922 &up, req->rsrc_update.nr_args);
Jens Axboe05f3fb32019-12-09 11:22:50 -07005923 mutex_unlock(&ctx->uring_lock);
5924
5925 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01005926 req_set_fail(req);
Pavel Begunkov889fca72021-02-10 00:03:09 +00005927 __io_req_complete(req, issue_flags, ret, 0);
Jens Axboe05f3fb32019-12-09 11:22:50 -07005928 return 0;
5929}
5930
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005931static int io_req_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboef67676d2019-12-02 11:03:47 -07005932{
Jens Axboed625c6e2019-12-17 19:53:05 -07005933 switch (req->opcode) {
Jens Axboee7815732019-12-17 19:45:06 -07005934 case IORING_OP_NOP:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005935 return 0;
Jens Axboef67676d2019-12-02 11:03:47 -07005936 case IORING_OP_READV:
5937 case IORING_OP_READ_FIXED:
Jens Axboe3a6820f2019-12-22 15:19:35 -07005938 case IORING_OP_READ:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005939 return io_read_prep(req, sqe);
Jens Axboef67676d2019-12-02 11:03:47 -07005940 case IORING_OP_WRITEV:
5941 case IORING_OP_WRITE_FIXED:
Jens Axboe3a6820f2019-12-22 15:19:35 -07005942 case IORING_OP_WRITE:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005943 return io_write_prep(req, sqe);
Jens Axboe0969e782019-12-17 18:40:57 -07005944 case IORING_OP_POLL_ADD:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005945 return io_poll_add_prep(req, sqe);
Jens Axboe0969e782019-12-17 18:40:57 -07005946 case IORING_OP_POLL_REMOVE:
Pavel Begunkovc5de0032021-04-14 13:38:37 +01005947 return io_poll_update_prep(req, sqe);
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07005948 case IORING_OP_FSYNC:
Pavel Begunkov1155c762021-02-18 18:29:38 +00005949 return io_fsync_prep(req, sqe);
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07005950 case IORING_OP_SYNC_FILE_RANGE:
Pavel Begunkov1155c762021-02-18 18:29:38 +00005951 return io_sfr_prep(req, sqe);
Jens Axboe03b12302019-12-02 18:50:25 -07005952 case IORING_OP_SENDMSG:
Jens Axboefddafac2020-01-04 20:19:44 -07005953 case IORING_OP_SEND:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005954 return io_sendmsg_prep(req, sqe);
Jens Axboe03b12302019-12-02 18:50:25 -07005955 case IORING_OP_RECVMSG:
Jens Axboefddafac2020-01-04 20:19:44 -07005956 case IORING_OP_RECV:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005957 return io_recvmsg_prep(req, sqe);
Jens Axboef499a022019-12-02 16:28:46 -07005958 case IORING_OP_CONNECT:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005959 return io_connect_prep(req, sqe);
Jens Axboe2d283902019-12-04 11:08:05 -07005960 case IORING_OP_TIMEOUT:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005961 return io_timeout_prep(req, sqe, false);
Jens Axboeb29472e2019-12-17 18:50:29 -07005962 case IORING_OP_TIMEOUT_REMOVE:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005963 return io_timeout_remove_prep(req, sqe);
Jens Axboefbf23842019-12-17 18:45:56 -07005964 case IORING_OP_ASYNC_CANCEL:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005965 return io_async_cancel_prep(req, sqe);
Jens Axboe2d283902019-12-04 11:08:05 -07005966 case IORING_OP_LINK_TIMEOUT:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005967 return io_timeout_prep(req, sqe, true);
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07005968 case IORING_OP_ACCEPT:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005969 return io_accept_prep(req, sqe);
Jens Axboed63d1b52019-12-10 10:38:56 -07005970 case IORING_OP_FALLOCATE:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005971 return io_fallocate_prep(req, sqe);
Jens Axboe15b71ab2019-12-11 11:20:36 -07005972 case IORING_OP_OPENAT:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005973 return io_openat_prep(req, sqe);
Jens Axboeb5dba592019-12-11 14:02:38 -07005974 case IORING_OP_CLOSE:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005975 return io_close_prep(req, sqe);
Jens Axboe05f3fb32019-12-09 11:22:50 -07005976 case IORING_OP_FILES_UPDATE:
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00005977 return io_rsrc_update_prep(req, sqe);
Jens Axboeeddc7ef2019-12-13 21:18:10 -07005978 case IORING_OP_STATX:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005979 return io_statx_prep(req, sqe);
Jens Axboe4840e412019-12-25 22:03:45 -07005980 case IORING_OP_FADVISE:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005981 return io_fadvise_prep(req, sqe);
Jens Axboec1ca7572019-12-25 22:18:28 -07005982 case IORING_OP_MADVISE:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005983 return io_madvise_prep(req, sqe);
Jens Axboecebdb982020-01-08 17:59:24 -07005984 case IORING_OP_OPENAT2:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005985 return io_openat2_prep(req, sqe);
Jens Axboe3e4827b2020-01-08 15:18:09 -07005986 case IORING_OP_EPOLL_CTL:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005987 return io_epoll_ctl_prep(req, sqe);
Pavel Begunkov7d67af22020-02-24 11:32:45 +03005988 case IORING_OP_SPLICE:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005989 return io_splice_prep(req, sqe);
Jens Axboeddf0322d2020-02-23 16:41:33 -07005990 case IORING_OP_PROVIDE_BUFFERS:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005991 return io_provide_buffers_prep(req, sqe);
Jens Axboe067524e2020-03-02 16:32:28 -07005992 case IORING_OP_REMOVE_BUFFERS:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005993 return io_remove_buffers_prep(req, sqe);
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03005994 case IORING_OP_TEE:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005995 return io_tee_prep(req, sqe);
Jens Axboe36f4fa62020-09-05 11:14:22 -06005996 case IORING_OP_SHUTDOWN:
5997 return io_shutdown_prep(req, sqe);
Jens Axboe80a261f2020-09-28 14:23:58 -06005998 case IORING_OP_RENAMEAT:
5999 return io_renameat_prep(req, sqe);
Jens Axboe14a11432020-09-28 14:27:37 -06006000 case IORING_OP_UNLINKAT:
6001 return io_unlinkat_prep(req, sqe);
Jens Axboef67676d2019-12-02 11:03:47 -07006002 }
6003
Pavel Begunkovbfe76552020-09-30 22:57:55 +03006004 printk_once(KERN_WARNING "io_uring: unhandled opcode %d\n",
6005 req->opcode);
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01006006 return -EINVAL;
Pavel Begunkovbfe76552020-09-30 22:57:55 +03006007}
6008
Pavel Begunkov93642ef2021-02-18 18:29:44 +00006009static int io_req_prep_async(struct io_kiocb *req)
Jens Axboedef596e2019-01-09 08:59:42 -07006010{
Pavel Begunkovb7e298d2021-02-28 22:35:19 +00006011 if (!io_op_defs[req->opcode].needs_async_setup)
6012 return 0;
6013 if (WARN_ON_ONCE(req->async_data))
6014 return -EFAULT;
6015 if (io_alloc_async_data(req))
6016 return -EAGAIN;
6017
Pavel Begunkov93642ef2021-02-18 18:29:44 +00006018 switch (req->opcode) {
6019 case IORING_OP_READV:
Pavel Begunkov93642ef2021-02-18 18:29:44 +00006020 return io_rw_prep_async(req, READ);
6021 case IORING_OP_WRITEV:
Pavel Begunkov93642ef2021-02-18 18:29:44 +00006022 return io_rw_prep_async(req, WRITE);
6023 case IORING_OP_SENDMSG:
Pavel Begunkov93642ef2021-02-18 18:29:44 +00006024 return io_sendmsg_prep_async(req);
6025 case IORING_OP_RECVMSG:
Pavel Begunkov93642ef2021-02-18 18:29:44 +00006026 return io_recvmsg_prep_async(req);
6027 case IORING_OP_CONNECT:
6028 return io_connect_prep_async(req);
6029 }
Pavel Begunkovb7e298d2021-02-28 22:35:19 +00006030 printk_once(KERN_WARNING "io_uring: prep_async() bad opcode %d\n",
6031 req->opcode);
6032 return -EFAULT;
Jens Axboedef596e2019-01-09 08:59:42 -07006033}
6034
Pavel Begunkov9cf7c102020-07-13 23:37:15 +03006035static u32 io_get_sequence(struct io_kiocb *req)
6036{
Pavel Begunkova3dbdf52021-06-17 18:14:05 +01006037 u32 seq = req->ctx->cached_sq_head;
Pavel Begunkov9cf7c102020-07-13 23:37:15 +03006038
Pavel Begunkova3dbdf52021-06-17 18:14:05 +01006039 /* need original cached_sq_head, but it was increased for each req */
6040 io_for_each_link(req, req)
6041 seq--;
6042 return seq;
Pavel Begunkov9cf7c102020-07-13 23:37:15 +03006043}
6044
Pavel Begunkov76cc33d2021-06-14 23:37:30 +01006045static bool io_drain_req(struct io_kiocb *req)
Jens Axboedef596e2019-01-09 08:59:42 -07006046{
Pavel Begunkov3c199662021-06-15 16:47:57 +01006047 struct io_kiocb *pos;
Jens Axboedef596e2019-01-09 08:59:42 -07006048 struct io_ring_ctx *ctx = req->ctx;
Pavel Begunkov27dc8332020-07-13 23:37:14 +03006049 struct io_defer_entry *de;
Jens Axboedef596e2019-01-09 08:59:42 -07006050 int ret;
Pavel Begunkov9cf7c102020-07-13 23:37:15 +03006051 u32 seq;
Jens Axboedef596e2019-01-09 08:59:42 -07006052
Pavel Begunkov3c199662021-06-15 16:47:57 +01006053 /*
6054 * If we need to drain a request in the middle of a link, drain the
6055 * head request and the next request/link after the current link.
6056 * Considering sequential execution of links, IOSQE_IO_DRAIN will be
6057 * maintained for every request of our link.
6058 */
6059 if (ctx->drain_next) {
6060 req->flags |= REQ_F_IO_DRAIN;
6061 ctx->drain_next = false;
6062 }
6063 /* not interested in head, start from the first linked */
6064 io_for_each_link(pos, req->link) {
6065 if (pos->flags & REQ_F_IO_DRAIN) {
6066 ctx->drain_next = true;
6067 req->flags |= REQ_F_IO_DRAIN;
6068 break;
6069 }
6070 }
6071
Jens Axboedef596e2019-01-09 08:59:42 -07006072 /* Still need defer if there is pending req in defer list. */
Pavel Begunkov9cf7c102020-07-13 23:37:15 +03006073 if (likely(list_empty_careful(&ctx->defer_list) &&
Pavel Begunkov10c66902021-06-15 16:47:56 +01006074 !(req->flags & REQ_F_IO_DRAIN))) {
6075 ctx->drain_active = false;
Pavel Begunkov76cc33d2021-06-14 23:37:30 +01006076 return false;
Pavel Begunkov10c66902021-06-15 16:47:56 +01006077 }
Pavel Begunkov9cf7c102020-07-13 23:37:15 +03006078
6079 seq = io_get_sequence(req);
6080 /* Still a chance to pass the sequence check */
6081 if (!req_need_defer(req, seq) && list_empty_careful(&ctx->defer_list))
Pavel Begunkov76cc33d2021-06-14 23:37:30 +01006082 return false;
Jens Axboedef596e2019-01-09 08:59:42 -07006083
Pavel Begunkovb7e298d2021-02-28 22:35:19 +00006084 ret = io_req_prep_async(req);
Pavel Begunkovbe7053b2021-02-18 18:29:45 +00006085 if (ret)
Pavel Begunkov1b487732021-07-11 22:41:13 +01006086 goto fail;
Pavel Begunkovcbdcb432020-06-29 19:18:43 +03006087 io_prep_async_link(req);
Pavel Begunkov27dc8332020-07-13 23:37:14 +03006088 de = kmalloc(sizeof(*de), GFP_KERNEL);
Pavel Begunkov76cc33d2021-06-14 23:37:30 +01006089 if (!de) {
Pavel Begunkov1b487732021-07-11 22:41:13 +01006090 ret = -ENOMEM;
6091fail:
6092 io_req_complete_failed(req, ret);
Pavel Begunkov76cc33d2021-06-14 23:37:30 +01006093 return true;
6094 }
Jens Axboe31b51512019-01-18 22:56:34 -07006095
Jens Axboe79ebeae2021-08-10 15:18:27 -06006096 spin_lock(&ctx->completion_lock);
Pavel Begunkov9cf7c102020-07-13 23:37:15 +03006097 if (!req_need_defer(req, seq) && list_empty(&ctx->defer_list)) {
Jens Axboe79ebeae2021-08-10 15:18:27 -06006098 spin_unlock(&ctx->completion_lock);
Pavel Begunkov27dc8332020-07-13 23:37:14 +03006099 kfree(de);
Pavel Begunkovae348172020-07-23 20:25:20 +03006100 io_queue_async_work(req);
Pavel Begunkov76cc33d2021-06-14 23:37:30 +01006101 return true;
Jens Axboe31b51512019-01-18 22:56:34 -07006102 }
6103
6104 trace_io_uring_defer(ctx, req, req->user_data);
Pavel Begunkov27dc8332020-07-13 23:37:14 +03006105 de->req = req;
Pavel Begunkov9cf7c102020-07-13 23:37:15 +03006106 de->seq = seq;
Pavel Begunkov27dc8332020-07-13 23:37:14 +03006107 list_add_tail(&de->list, &ctx->defer_list);
Jens Axboe79ebeae2021-08-10 15:18:27 -06006108 spin_unlock(&ctx->completion_lock);
Pavel Begunkov76cc33d2021-06-14 23:37:30 +01006109 return true;
Jens Axboe31b51512019-01-18 22:56:34 -07006110}
6111
Pavel Begunkov68fb8972021-03-19 17:22:41 +00006112static void io_clean_op(struct io_kiocb *req)
Pavel Begunkov99bc4c32020-02-07 22:04:45 +03006113{
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03006114 if (req->flags & REQ_F_BUFFER_SELECTED) {
6115 switch (req->opcode) {
6116 case IORING_OP_READV:
6117 case IORING_OP_READ_FIXED:
6118 case IORING_OP_READ:
Jens Axboebcda7ba2020-02-23 16:42:51 -07006119 kfree((void *)(unsigned long)req->rw.addr);
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03006120 break;
6121 case IORING_OP_RECVMSG:
6122 case IORING_OP_RECV:
Jens Axboe52de1fe2020-02-27 10:15:42 -07006123 kfree(req->sr_msg.kbuf);
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03006124 break;
6125 }
Pavel Begunkov99bc4c32020-02-07 22:04:45 +03006126 }
6127
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03006128 if (req->flags & REQ_F_NEED_CLEANUP) {
6129 switch (req->opcode) {
6130 case IORING_OP_READV:
6131 case IORING_OP_READ_FIXED:
6132 case IORING_OP_READ:
6133 case IORING_OP_WRITEV:
6134 case IORING_OP_WRITE_FIXED:
Jens Axboee8c2bc12020-08-15 18:44:09 -07006135 case IORING_OP_WRITE: {
6136 struct io_async_rw *io = req->async_data;
Pavel Begunkov1dacb4d2021-06-17 18:14:03 +01006137
6138 kfree(io->free_iovec);
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03006139 break;
Jens Axboee8c2bc12020-08-15 18:44:09 -07006140 }
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03006141 case IORING_OP_RECVMSG:
Jens Axboee8c2bc12020-08-15 18:44:09 -07006142 case IORING_OP_SENDMSG: {
6143 struct io_async_msghdr *io = req->async_data;
Pavel Begunkov257e84a2021-02-05 00:58:00 +00006144
6145 kfree(io->free_iov);
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03006146 break;
Jens Axboee8c2bc12020-08-15 18:44:09 -07006147 }
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03006148 case IORING_OP_SPLICE:
6149 case IORING_OP_TEE:
Pavel Begunkove1d767f2021-03-19 17:22:43 +00006150 if (!(req->splice.flags & SPLICE_F_FD_IN_FIXED))
6151 io_put_file(req->splice.file_in);
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03006152 break;
Jens Axboef3cd48502020-09-24 14:55:54 -06006153 case IORING_OP_OPENAT:
6154 case IORING_OP_OPENAT2:
6155 if (req->open.filename)
6156 putname(req->open.filename);
6157 break;
Jens Axboe80a261f2020-09-28 14:23:58 -06006158 case IORING_OP_RENAMEAT:
6159 putname(req->rename.oldpath);
6160 putname(req->rename.newpath);
6161 break;
Jens Axboe14a11432020-09-28 14:27:37 -06006162 case IORING_OP_UNLINKAT:
6163 putname(req->unlink.filename);
6164 break;
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03006165 }
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03006166 }
Jens Axboe75652a302021-04-15 09:52:40 -06006167 if ((req->flags & REQ_F_POLLED) && req->apoll) {
6168 kfree(req->apoll->double_poll);
6169 kfree(req->apoll);
6170 req->apoll = NULL;
6171 }
Pavel Begunkov3a0a6902021-04-20 12:03:31 +01006172 if (req->flags & REQ_F_INFLIGHT) {
6173 struct io_uring_task *tctx = req->task->io_uring;
6174
6175 atomic_dec(&tctx->inflight_tracked);
Pavel Begunkov3a0a6902021-04-20 12:03:31 +01006176 }
Pavel Begunkovc8543572021-06-17 18:14:04 +01006177 if (req->flags & REQ_F_CREDS)
Pavel Begunkovb8e64b52021-06-17 18:14:02 +01006178 put_cred(req->creds);
Pavel Begunkovc8543572021-06-17 18:14:04 +01006179
6180 req->flags &= ~IO_REQ_CLEAN_FLAGS;
Pavel Begunkov99bc4c32020-02-07 22:04:45 +03006181}
6182
Pavel Begunkov889fca72021-02-10 00:03:09 +00006183static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboeedafcce2019-01-09 09:16:05 -07006184{
Jens Axboeedafcce2019-01-09 09:16:05 -07006185 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe5730b272021-02-27 15:57:30 -07006186 const struct cred *creds = NULL;
Jens Axboed625c6e2019-12-17 19:53:05 -07006187 int ret;
Jens Axboeedafcce2019-01-09 09:16:05 -07006188
Pavel Begunkovb8e64b52021-06-17 18:14:02 +01006189 if ((req->flags & REQ_F_CREDS) && req->creds != current_cred())
Pavel Begunkovc10d1f92021-06-17 18:14:01 +01006190 creds = override_creds(req->creds);
Jens Axboe5730b272021-02-27 15:57:30 -07006191
Jens Axboed625c6e2019-12-17 19:53:05 -07006192 switch (req->opcode) {
Jens Axboe2b188cc2019-01-07 10:46:33 -07006193 case IORING_OP_NOP:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006194 ret = io_nop(req, issue_flags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006195 break;
Jens Axboe2b188cc2019-01-07 10:46:33 -07006196 case IORING_OP_READV:
Jens Axboe3529d8c2019-12-19 18:24:38 -07006197 case IORING_OP_READ_FIXED:
Jens Axboe3a6820f2019-12-22 15:19:35 -07006198 case IORING_OP_READ:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006199 ret = io_read(req, issue_flags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006200 break;
6201 case IORING_OP_WRITEV:
Jens Axboe2b188cc2019-01-07 10:46:33 -07006202 case IORING_OP_WRITE_FIXED:
Jens Axboe3a6820f2019-12-22 15:19:35 -07006203 case IORING_OP_WRITE:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006204 ret = io_write(req, issue_flags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006205 break;
6206 case IORING_OP_FSYNC:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006207 ret = io_fsync(req, issue_flags);
Jackie Liuba5290c2019-10-09 09:19:59 +08006208 break;
Jens Axboe2b188cc2019-01-07 10:46:33 -07006209 case IORING_OP_POLL_ADD:
Pavel Begunkov61e98202021-02-10 00:03:08 +00006210 ret = io_poll_add(req, issue_flags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006211 break;
6212 case IORING_OP_POLL_REMOVE:
Pavel Begunkovc5de0032021-04-14 13:38:37 +01006213 ret = io_poll_update(req, issue_flags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006214 break;
Jens Axboeb76da702019-11-20 13:05:32 -07006215 case IORING_OP_SYNC_FILE_RANGE:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006216 ret = io_sync_file_range(req, issue_flags);
Jens Axboeb76da702019-11-20 13:05:32 -07006217 break;
6218 case IORING_OP_SENDMSG:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006219 ret = io_sendmsg(req, issue_flags);
Pavel Begunkov062d04d2020-10-10 18:34:12 +01006220 break;
Jens Axboefddafac2020-01-04 20:19:44 -07006221 case IORING_OP_SEND:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006222 ret = io_send(req, issue_flags);
Jens Axboeb76da702019-11-20 13:05:32 -07006223 break;
6224 case IORING_OP_RECVMSG:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006225 ret = io_recvmsg(req, issue_flags);
Pavel Begunkov062d04d2020-10-10 18:34:12 +01006226 break;
Jens Axboefddafac2020-01-04 20:19:44 -07006227 case IORING_OP_RECV:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006228 ret = io_recv(req, issue_flags);
Jens Axboeb76da702019-11-20 13:05:32 -07006229 break;
Jens Axboe561fb042019-10-24 07:25:42 -06006230 case IORING_OP_TIMEOUT:
Pavel Begunkov61e98202021-02-10 00:03:08 +00006231 ret = io_timeout(req, issue_flags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006232 break;
6233 case IORING_OP_TIMEOUT_REMOVE:
Pavel Begunkov61e98202021-02-10 00:03:08 +00006234 ret = io_timeout_remove(req, issue_flags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006235 break;
6236 case IORING_OP_ACCEPT:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006237 ret = io_accept(req, issue_flags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006238 break;
6239 case IORING_OP_CONNECT:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006240 ret = io_connect(req, issue_flags);
Jens Axboe31b51512019-01-18 22:56:34 -07006241 break;
6242 case IORING_OP_ASYNC_CANCEL:
Pavel Begunkov61e98202021-02-10 00:03:08 +00006243 ret = io_async_cancel(req, issue_flags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006244 break;
Jens Axboed63d1b52019-12-10 10:38:56 -07006245 case IORING_OP_FALLOCATE:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006246 ret = io_fallocate(req, issue_flags);
Jens Axboed63d1b52019-12-10 10:38:56 -07006247 break;
Jens Axboe15b71ab2019-12-11 11:20:36 -07006248 case IORING_OP_OPENAT:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006249 ret = io_openat(req, issue_flags);
Jens Axboe15b71ab2019-12-11 11:20:36 -07006250 break;
Jens Axboeb5dba592019-12-11 14:02:38 -07006251 case IORING_OP_CLOSE:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006252 ret = io_close(req, issue_flags);
Jens Axboeb5dba592019-12-11 14:02:38 -07006253 break;
Jens Axboe05f3fb32019-12-09 11:22:50 -07006254 case IORING_OP_FILES_UPDATE:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006255 ret = io_files_update(req, issue_flags);
Jens Axboe05f3fb32019-12-09 11:22:50 -07006256 break;
Jens Axboeeddc7ef2019-12-13 21:18:10 -07006257 case IORING_OP_STATX:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006258 ret = io_statx(req, issue_flags);
Jens Axboeeddc7ef2019-12-13 21:18:10 -07006259 break;
Jens Axboe4840e412019-12-25 22:03:45 -07006260 case IORING_OP_FADVISE:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006261 ret = io_fadvise(req, issue_flags);
Jens Axboe4840e412019-12-25 22:03:45 -07006262 break;
Jens Axboec1ca7572019-12-25 22:18:28 -07006263 case IORING_OP_MADVISE:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006264 ret = io_madvise(req, issue_flags);
Jens Axboec1ca7572019-12-25 22:18:28 -07006265 break;
Jens Axboecebdb982020-01-08 17:59:24 -07006266 case IORING_OP_OPENAT2:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006267 ret = io_openat2(req, issue_flags);
Jens Axboecebdb982020-01-08 17:59:24 -07006268 break;
Jens Axboe3e4827b2020-01-08 15:18:09 -07006269 case IORING_OP_EPOLL_CTL:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006270 ret = io_epoll_ctl(req, issue_flags);
Jens Axboe3e4827b2020-01-08 15:18:09 -07006271 break;
Pavel Begunkov7d67af22020-02-24 11:32:45 +03006272 case IORING_OP_SPLICE:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006273 ret = io_splice(req, issue_flags);
Pavel Begunkov7d67af22020-02-24 11:32:45 +03006274 break;
Jens Axboeddf0322d2020-02-23 16:41:33 -07006275 case IORING_OP_PROVIDE_BUFFERS:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006276 ret = io_provide_buffers(req, issue_flags);
Jens Axboeddf0322d2020-02-23 16:41:33 -07006277 break;
Jens Axboe067524e2020-03-02 16:32:28 -07006278 case IORING_OP_REMOVE_BUFFERS:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006279 ret = io_remove_buffers(req, issue_flags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006280 break;
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03006281 case IORING_OP_TEE:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006282 ret = io_tee(req, issue_flags);
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03006283 break;
Jens Axboe36f4fa62020-09-05 11:14:22 -06006284 case IORING_OP_SHUTDOWN:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006285 ret = io_shutdown(req, issue_flags);
Jens Axboe36f4fa62020-09-05 11:14:22 -06006286 break;
Jens Axboe80a261f2020-09-28 14:23:58 -06006287 case IORING_OP_RENAMEAT:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006288 ret = io_renameat(req, issue_flags);
Jens Axboe80a261f2020-09-28 14:23:58 -06006289 break;
Jens Axboe14a11432020-09-28 14:27:37 -06006290 case IORING_OP_UNLINKAT:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006291 ret = io_unlinkat(req, issue_flags);
Jens Axboe14a11432020-09-28 14:27:37 -06006292 break;
Jens Axboe2b188cc2019-01-07 10:46:33 -07006293 default:
6294 ret = -EINVAL;
6295 break;
6296 }
Jens Axboe31b51512019-01-18 22:56:34 -07006297
Jens Axboe5730b272021-02-27 15:57:30 -07006298 if (creds)
6299 revert_creds(creds);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006300 if (ret)
6301 return ret;
Jens Axboeb5325762020-05-19 21:20:27 -06006302 /* If the op doesn't have a file, we're not polling for it */
Pavel Begunkovcb3d8972021-06-14 02:36:14 +01006303 if ((ctx->flags & IORING_SETUP_IOPOLL) && req->file)
6304 io_iopoll_req_issued(req);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006305
6306 return 0;
6307}
6308
Pavel Begunkovebc11b62021-08-09 13:04:05 +01006309static struct io_wq_work *io_wq_free_work(struct io_wq_work *work)
6310{
6311 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
6312
6313 req = io_put_req_find_next(req);
6314 return req ? &req->work : NULL;
6315}
6316
Pavel Begunkov5280f7e2021-02-04 13:52:08 +00006317static void io_wq_submit_work(struct io_wq_work *work)
Pavel Begunkovd4c81f32020-06-08 21:08:19 +03006318{
Jens Axboe2b188cc2019-01-07 10:46:33 -07006319 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
Pavel Begunkov6df1db62020-07-03 22:15:06 +03006320 struct io_kiocb *timeout;
Jens Axboe561fb042019-10-24 07:25:42 -06006321 int ret = 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07006322
Pavel Begunkov48dcd382021-08-15 10:40:18 +01006323 /* one will be dropped by ->io_free_work() after returning to io-wq */
6324 if (!(req->flags & REQ_F_REFCOUNT))
6325 __io_req_set_refcount(req, 2);
6326 else
6327 req_ref_get(req);
Pavel Begunkov5d5901a2021-08-11 19:28:29 +01006328
Pavel Begunkov6df1db62020-07-03 22:15:06 +03006329 timeout = io_prep_linked_timeout(req);
6330 if (timeout)
6331 io_queue_linked_timeout(timeout);
Pavel Begunkovd4c81f32020-06-08 21:08:19 +03006332
Jens Axboe4014d942021-01-19 15:53:54 -07006333 if (work->flags & IO_WQ_WORK_CANCEL)
Jens Axboe561fb042019-10-24 07:25:42 -06006334 ret = -ECANCELED;
Jens Axboe31b51512019-01-18 22:56:34 -07006335
Jens Axboe561fb042019-10-24 07:25:42 -06006336 if (!ret) {
Jens Axboe561fb042019-10-24 07:25:42 -06006337 do {
Pavel Begunkov889fca72021-02-10 00:03:09 +00006338 ret = io_issue_sqe(req, 0);
Jens Axboe561fb042019-10-24 07:25:42 -06006339 /*
6340 * We can get EAGAIN for polled IO even though we're
6341 * forcing a sync submission from here, since we can't
6342 * wait for request slots on the block side.
6343 */
6344 if (ret != -EAGAIN)
6345 break;
6346 cond_resched();
6347 } while (1);
6348 }
Jens Axboe31b51512019-01-18 22:56:34 -07006349
Pavel Begunkova3df76982021-02-18 22:32:52 +00006350 /* avoid locking problems by failing it from a clean context */
Pavel Begunkov5d5901a2021-08-11 19:28:29 +01006351 if (ret)
Pavel Begunkova3df76982021-02-18 22:32:52 +00006352 io_req_task_queue_fail(req, ret);
Jens Axboe31b51512019-01-18 22:56:34 -07006353}
Jens Axboe2b188cc2019-01-07 10:46:33 -07006354
Pavel Begunkovaeca2412021-04-11 01:46:37 +01006355static inline struct io_fixed_file *io_fixed_file_slot(struct io_file_table *table,
Pavel Begunkov042b0d82021-08-09 13:04:01 +01006356 unsigned i)
Jens Axboe09bb8392019-03-13 12:39:28 -06006357{
Pavel Begunkov042b0d82021-08-09 13:04:01 +01006358 return &table->files[i];
Pavel Begunkovdafecf12021-02-28 22:35:11 +00006359}
6360
Jens Axboe09bb8392019-03-13 12:39:28 -06006361static inline struct file *io_file_from_index(struct io_ring_ctx *ctx,
6362 int index)
6363{
Pavel Begunkovaeca2412021-04-11 01:46:37 +01006364 struct io_fixed_file *slot = io_fixed_file_slot(&ctx->file_table, index);
Jens Axboe65e19f52019-10-26 07:20:21 -06006365
Pavel Begunkova04b0ac2021-04-01 15:44:04 +01006366 return (struct file *) (slot->file_ptr & FFS_MASK);
Jens Axboe65e19f52019-10-26 07:20:21 -06006367}
6368
Pavel Begunkova04b0ac2021-04-01 15:44:04 +01006369static void io_fixed_file_set(struct io_fixed_file *file_slot, struct file *file)
Pavel Begunkov9a321c92021-04-01 15:44:01 +01006370{
6371 unsigned long file_ptr = (unsigned long) file;
6372
Pavel Begunkovb191e2d2021-08-09 13:04:03 +01006373 if (__io_file_supports_nowait(file, READ))
Pavel Begunkov9a321c92021-04-01 15:44:01 +01006374 file_ptr |= FFS_ASYNC_READ;
Pavel Begunkovb191e2d2021-08-09 13:04:03 +01006375 if (__io_file_supports_nowait(file, WRITE))
Pavel Begunkov9a321c92021-04-01 15:44:01 +01006376 file_ptr |= FFS_ASYNC_WRITE;
6377 if (S_ISREG(file_inode(file)->i_mode))
6378 file_ptr |= FFS_ISREG;
Pavel Begunkova04b0ac2021-04-01 15:44:04 +01006379 file_slot->file_ptr = file_ptr;
Jens Axboe09bb8392019-03-13 12:39:28 -06006380}
6381
Pavel Begunkovac177052021-08-09 13:04:02 +01006382static inline struct file *io_file_get_fixed(struct io_ring_ctx *ctx,
6383 struct io_kiocb *req, int fd)
Pavel Begunkov8da11c12020-02-24 11:32:44 +03006384{
Pavel Begunkov8da11c12020-02-24 11:32:44 +03006385 struct file *file;
Pavel Begunkovac177052021-08-09 13:04:02 +01006386 unsigned long file_ptr;
Pavel Begunkov8da11c12020-02-24 11:32:44 +03006387
Pavel Begunkovac177052021-08-09 13:04:02 +01006388 if (unlikely((unsigned int)fd >= ctx->nr_user_files))
6389 return NULL;
6390 fd = array_index_nospec(fd, ctx->nr_user_files);
6391 file_ptr = io_fixed_file_slot(&ctx->file_table, fd)->file_ptr;
6392 file = (struct file *) (file_ptr & FFS_MASK);
6393 file_ptr &= ~FFS_MASK;
6394 /* mask in overlapping REQ_F and FFS bits */
Pavel Begunkovb191e2d2021-08-09 13:04:03 +01006395 req->flags |= (file_ptr << REQ_F_NOWAIT_READ_BIT);
Pavel Begunkovac177052021-08-09 13:04:02 +01006396 io_req_set_rsrc_node(req);
Pavel Begunkov8371adf2020-10-10 18:34:08 +01006397 return file;
Pavel Begunkov8da11c12020-02-24 11:32:44 +03006398}
6399
Pavel Begunkovac177052021-08-09 13:04:02 +01006400static struct file *io_file_get_normal(struct io_ring_ctx *ctx,
Pavel Begunkovac177052021-08-09 13:04:02 +01006401 struct io_kiocb *req, int fd)
6402{
Pavel Begunkov62906e82021-08-10 14:52:47 +01006403 struct file *file = fget(fd);
Pavel Begunkovac177052021-08-09 13:04:02 +01006404
6405 trace_io_uring_file_get(ctx, fd);
6406
6407 /* we don't allow fixed io_uring files */
6408 if (file && unlikely(file->f_op == &io_uring_fops))
6409 io_req_track_inflight(req);
6410 return file;
6411}
6412
6413static inline struct file *io_file_get(struct io_ring_ctx *ctx,
Pavel Begunkovac177052021-08-09 13:04:02 +01006414 struct io_kiocb *req, int fd, bool fixed)
6415{
6416 if (fixed)
6417 return io_file_get_fixed(ctx, req, fd);
6418 else
Pavel Begunkov62906e82021-08-10 14:52:47 +01006419 return io_file_get_normal(ctx, req, fd);
Pavel Begunkovac177052021-08-09 13:04:02 +01006420}
6421
Jens Axboe89b263f2021-08-10 15:14:18 -06006422static void io_req_task_link_timeout(struct io_kiocb *req)
6423{
6424 struct io_kiocb *prev = req->timeout.prev;
6425 struct io_ring_ctx *ctx = req->ctx;
Pavel Begunkov8cb01fa2021-08-15 10:40:22 +01006426 int ret;
Jens Axboe89b263f2021-08-10 15:14:18 -06006427
6428 if (prev) {
Pavel Begunkov8cb01fa2021-08-15 10:40:22 +01006429 ret = io_try_cancel_userdata(req, prev->user_data);
6430 if (!ret)
6431 ret = -ETIME;
6432 io_cqring_fill_event(ctx, req->user_data, ret, 0);
6433 io_commit_cqring(ctx);
6434 spin_unlock(&ctx->completion_lock);
6435 io_cqring_ev_posted(ctx);
6436
Jens Axboe89b263f2021-08-10 15:14:18 -06006437 io_put_req(prev);
6438 io_put_req(req);
6439 } else {
6440 io_req_complete_post(req, -ETIME, 0);
6441 }
6442}
6443
Jens Axboe2665abf2019-11-05 12:40:47 -07006444static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer)
6445{
Jens Axboead8a48a2019-11-15 08:49:11 -07006446 struct io_timeout_data *data = container_of(timer,
6447 struct io_timeout_data, timer);
Pavel Begunkov90cd7e42020-10-27 23:25:36 +00006448 struct io_kiocb *prev, *req = data->req;
Jens Axboe2665abf2019-11-05 12:40:47 -07006449 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe2665abf2019-11-05 12:40:47 -07006450 unsigned long flags;
Jens Axboe2665abf2019-11-05 12:40:47 -07006451
Jens Axboe89b263f2021-08-10 15:14:18 -06006452 spin_lock_irqsave(&ctx->timeout_lock, flags);
Pavel Begunkov90cd7e42020-10-27 23:25:36 +00006453 prev = req->timeout.head;
6454 req->timeout.head = NULL;
Jens Axboe2665abf2019-11-05 12:40:47 -07006455
6456 /*
6457 * We don't expect the list to be empty, that will only happen if we
6458 * race with the completion of the linked work.
6459 */
Pavel Begunkov447c19f2021-05-14 12:02:50 +01006460 if (prev) {
Pavel Begunkovf2f87372020-10-27 23:25:37 +00006461 io_remove_next_linked(prev);
Pavel Begunkov447c19f2021-05-14 12:02:50 +01006462 if (!req_ref_inc_not_zero(prev))
6463 prev = NULL;
6464 }
Jens Axboe89b263f2021-08-10 15:14:18 -06006465 req->timeout.prev = prev;
6466 spin_unlock_irqrestore(&ctx->timeout_lock, flags);
Jens Axboe2665abf2019-11-05 12:40:47 -07006467
Jens Axboe89b263f2021-08-10 15:14:18 -06006468 req->io_task_work.func = io_req_task_link_timeout;
6469 io_req_task_work_add(req);
Jens Axboe2665abf2019-11-05 12:40:47 -07006470 return HRTIMER_NORESTART;
6471}
6472
Pavel Begunkovde968c12021-03-19 17:22:33 +00006473static void io_queue_linked_timeout(struct io_kiocb *req)
Jens Axboe2665abf2019-11-05 12:40:47 -07006474{
Pavel Begunkovde968c12021-03-19 17:22:33 +00006475 struct io_ring_ctx *ctx = req->ctx;
6476
Jens Axboe89b263f2021-08-10 15:14:18 -06006477 spin_lock_irq(&ctx->timeout_lock);
Jens Axboe76a46e02019-11-10 23:34:16 -07006478 /*
Pavel Begunkovf2f87372020-10-27 23:25:37 +00006479 * If the back reference is NULL, then our linked request finished
6480 * before we got a chance to setup the timer
Jens Axboe76a46e02019-11-10 23:34:16 -07006481 */
Pavel Begunkov90cd7e42020-10-27 23:25:36 +00006482 if (req->timeout.head) {
Jens Axboee8c2bc12020-08-15 18:44:09 -07006483 struct io_timeout_data *data = req->async_data;
Jens Axboe94ae5e72019-11-14 19:39:52 -07006484
Jens Axboead8a48a2019-11-15 08:49:11 -07006485 data->timer.function = io_link_timeout_fn;
6486 hrtimer_start(&data->timer, timespec64_to_ktime(data->ts),
6487 data->mode);
Jens Axboe2665abf2019-11-05 12:40:47 -07006488 }
Jens Axboe89b263f2021-08-10 15:14:18 -06006489 spin_unlock_irq(&ctx->timeout_lock);
Jens Axboe2665abf2019-11-05 12:40:47 -07006490 /* drop submission reference */
Jens Axboe76a46e02019-11-10 23:34:16 -07006491 io_put_req(req);
Jens Axboe2665abf2019-11-05 12:40:47 -07006492}
6493
Pavel Begunkovc5eef2b2021-02-10 00:03:22 +00006494static void __io_queue_sqe(struct io_kiocb *req)
Pavel Begunkov282cdc82021-08-09 13:04:10 +01006495 __must_hold(&req->ctx->uring_lock)
Jens Axboe2b188cc2019-01-07 10:46:33 -07006496{
Pavel Begunkov906c6ca2021-08-15 10:40:26 +01006497 struct io_kiocb *linked_timeout;
Jens Axboee0c5c572019-03-12 10:18:47 -06006498 int ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07006499
Olivier Langlois59b735a2021-06-22 05:17:39 -07006500issue_sqe:
Pavel Begunkovc5eef2b2021-02-10 00:03:22 +00006501 ret = io_issue_sqe(req, IO_URING_F_NONBLOCK|IO_URING_F_COMPLETE_DEFER);
Jens Axboe491381ce2019-10-17 09:20:46 -06006502
6503 /*
6504 * We async punt it if the file wasn't marked NOWAIT, or if the file
6505 * doesn't support non-blocking read/write attempts
6506 */
Pavel Begunkov18400382021-03-19 17:22:34 +00006507 if (likely(!ret)) {
Pavel Begunkove342c802021-01-19 13:32:47 +00006508 if (req->flags & REQ_F_COMPLETE_INLINE) {
Pavel Begunkovc5eef2b2021-02-10 00:03:22 +00006509 struct io_ring_ctx *ctx = req->ctx;
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01006510 struct io_submit_state *state = &ctx->submit_state;
Jens Axboee65ef562019-03-12 10:16:44 -06006511
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01006512 state->compl_reqs[state->compl_nr++] = req;
6513 if (state->compl_nr == ARRAY_SIZE(state->compl_reqs))
Pavel Begunkov2a2758f2021-06-17 18:14:00 +01006514 io_submit_flush_completions(ctx);
Pavel Begunkov906c6ca2021-08-15 10:40:26 +01006515 return;
Pavel Begunkov0d63c142020-10-22 16:47:18 +01006516 }
Pavel Begunkov906c6ca2021-08-15 10:40:26 +01006517
6518 linked_timeout = io_prep_linked_timeout(req);
6519 if (linked_timeout)
6520 io_queue_linked_timeout(linked_timeout);
Pavel Begunkov18400382021-03-19 17:22:34 +00006521 } else if (ret == -EAGAIN && !(req->flags & REQ_F_NOWAIT)) {
Pavel Begunkov906c6ca2021-08-15 10:40:26 +01006522 linked_timeout = io_prep_linked_timeout(req);
6523
Olivier Langlois59b735a2021-06-22 05:17:39 -07006524 switch (io_arm_poll_handler(req)) {
6525 case IO_APOLL_READY:
Pavel Begunkov906c6ca2021-08-15 10:40:26 +01006526 if (linked_timeout)
6527 io_unprep_linked_timeout(req);
Olivier Langlois59b735a2021-06-22 05:17:39 -07006528 goto issue_sqe;
6529 case IO_APOLL_ABORTED:
Pavel Begunkov18400382021-03-19 17:22:34 +00006530 /*
6531 * Queued up for async execution, worker will release
6532 * submit reference when the iocb is actually submitted.
6533 */
6534 io_queue_async_work(req);
Olivier Langlois59b735a2021-06-22 05:17:39 -07006535 break;
Pavel Begunkov18400382021-03-19 17:22:34 +00006536 }
Pavel Begunkov906c6ca2021-08-15 10:40:26 +01006537
6538 if (linked_timeout)
6539 io_queue_linked_timeout(linked_timeout);
Pavel Begunkov0d63c142020-10-22 16:47:18 +01006540 } else {
Pavel Begunkovf41db2732021-02-28 22:35:12 +00006541 io_req_complete_failed(req, ret);
Jens Axboe9e645e112019-05-10 16:07:28 -06006542 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07006543}
6544
Pavel Begunkov441b8a72021-06-14 23:37:31 +01006545static inline void io_queue_sqe(struct io_kiocb *req)
Pavel Begunkov282cdc82021-08-09 13:04:10 +01006546 __must_hold(&req->ctx->uring_lock)
Jackie Liu4fe2c962019-09-09 20:50:40 +08006547{
Pavel Begunkov10c66902021-06-15 16:47:56 +01006548 if (unlikely(req->ctx->drain_active) && io_drain_req(req))
Pavel Begunkov76cc33d2021-06-14 23:37:30 +01006549 return;
Jackie Liu4fe2c962019-09-09 20:50:40 +08006550
Pavel Begunkov76cc33d2021-06-14 23:37:30 +01006551 if (likely(!(req->flags & REQ_F_FORCE_ASYNC))) {
Pavel Begunkovc5eef2b2021-02-10 00:03:22 +00006552 __io_queue_sqe(req);
Pavel Begunkov76cc33d2021-06-14 23:37:30 +01006553 } else {
6554 int ret = io_req_prep_async(req);
6555
6556 if (unlikely(ret))
6557 io_req_complete_failed(req, ret);
6558 else
6559 io_queue_async_work(req);
Jens Axboece35a472019-12-17 08:04:44 -07006560 }
Jackie Liu4fe2c962019-09-09 20:50:40 +08006561}
6562
Stefano Garzarella21b55db2020-08-27 16:58:30 +02006563/*
6564 * Check SQE restrictions (opcode and flags).
6565 *
6566 * Returns 'true' if SQE is allowed, 'false' otherwise.
6567 */
6568static inline bool io_check_restriction(struct io_ring_ctx *ctx,
6569 struct io_kiocb *req,
6570 unsigned int sqe_flags)
6571{
Pavel Begunkov4cfb25b2021-06-26 21:40:47 +01006572 if (likely(!ctx->restricted))
Stefano Garzarella21b55db2020-08-27 16:58:30 +02006573 return true;
6574
6575 if (!test_bit(req->opcode, ctx->restrictions.sqe_op))
6576 return false;
6577
6578 if ((sqe_flags & ctx->restrictions.sqe_flags_required) !=
6579 ctx->restrictions.sqe_flags_required)
6580 return false;
6581
6582 if (sqe_flags & ~(ctx->restrictions.sqe_flags_allowed |
6583 ctx->restrictions.sqe_flags_required))
6584 return false;
6585
6586 return true;
6587}
6588
Pavel Begunkovef4ff582020-04-12 02:05:05 +03006589static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,
Pavel Begunkov258b29a2021-02-10 00:03:10 +00006590 const struct io_uring_sqe *sqe)
Pavel Begunkov282cdc82021-08-09 13:04:10 +01006591 __must_hold(&ctx->uring_lock)
Pavel Begunkov0553b8b2020-04-08 08:58:45 +03006592{
Pavel Begunkov258b29a2021-02-10 00:03:10 +00006593 struct io_submit_state *state;
Pavel Begunkovef4ff582020-04-12 02:05:05 +03006594 unsigned int sqe_flags;
Jens Axboe003e8dc2021-03-06 09:22:27 -07006595 int personality, ret = 0;
Pavel Begunkovef4ff582020-04-12 02:05:05 +03006596
Pavel Begunkov864ea922021-08-09 13:04:08 +01006597 /* req is partially pre-initialised, see io_preinit_req() */
Pavel Begunkov0553b8b2020-04-08 08:58:45 +03006598 req->opcode = READ_ONCE(sqe->opcode);
Pavel Begunkov5be9ad12021-02-12 18:41:17 +00006599 /* same numerical values with corresponding REQ_F_*, safe to copy */
6600 req->flags = sqe_flags = READ_ONCE(sqe->flags);
Pavel Begunkov0553b8b2020-04-08 08:58:45 +03006601 req->user_data = READ_ONCE(sqe->user_data);
Pavel Begunkov0553b8b2020-04-08 08:58:45 +03006602 req->file = NULL;
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00006603 req->fixed_rsrc_refs = NULL;
Pavel Begunkov4dd28242020-06-15 10:33:13 +03006604 req->task = current;
Pavel Begunkovef4ff582020-04-12 02:05:05 +03006605
Pavel Begunkov5be9ad12021-02-12 18:41:17 +00006606 /* enforce forwards compatibility on users */
Pavel Begunkovdddca222021-04-27 16:13:52 +01006607 if (unlikely(sqe_flags & ~SQE_VALID_FLAGS))
Pavel Begunkov5be9ad12021-02-12 18:41:17 +00006608 return -EINVAL;
Pavel Begunkovef4ff582020-04-12 02:05:05 +03006609 if (unlikely(req->opcode >= IORING_OP_LAST))
6610 return -EINVAL;
Pavel Begunkov4cfb25b2021-06-26 21:40:47 +01006611 if (!io_check_restriction(ctx, req, sqe_flags))
Stefano Garzarella21b55db2020-08-27 16:58:30 +02006612 return -EACCES;
6613
Pavel Begunkovef4ff582020-04-12 02:05:05 +03006614 if ((sqe_flags & IOSQE_BUFFER_SELECT) &&
6615 !io_op_defs[req->opcode].buffer_select)
6616 return -EOPNOTSUPP;
Pavel Begunkov3c199662021-06-15 16:47:57 +01006617 if (unlikely(sqe_flags & IOSQE_IO_DRAIN))
6618 ctx->drain_active = true;
Pavel Begunkovef4ff582020-04-12 02:05:05 +03006619
Jens Axboe003e8dc2021-03-06 09:22:27 -07006620 personality = READ_ONCE(sqe->personality);
6621 if (personality) {
Pavel Begunkovc10d1f92021-06-17 18:14:01 +01006622 req->creds = xa_load(&ctx->personalities, personality);
6623 if (!req->creds)
Jens Axboe003e8dc2021-03-06 09:22:27 -07006624 return -EINVAL;
Pavel Begunkovc10d1f92021-06-17 18:14:01 +01006625 get_cred(req->creds);
Pavel Begunkovb8e64b52021-06-17 18:14:02 +01006626 req->flags |= REQ_F_CREDS;
Jens Axboe003e8dc2021-03-06 09:22:27 -07006627 }
Pavel Begunkov258b29a2021-02-10 00:03:10 +00006628 state = &ctx->submit_state;
Pavel Begunkovef4ff582020-04-12 02:05:05 +03006629
Jens Axboe27926b62020-10-28 09:33:23 -06006630 /*
6631 * Plug now if we have more than 1 IO left after this, and the target
6632 * is potentially a read/write to block based storage.
6633 */
6634 if (!state->plug_started && state->ios_left > 1 &&
6635 io_op_defs[req->opcode].plug) {
6636 blk_start_plug(&state->plug);
6637 state->plug_started = true;
6638 }
Jens Axboe63ff8222020-05-07 14:56:15 -06006639
Pavel Begunkovbd5bbda2020-11-20 15:50:51 +00006640 if (io_op_defs[req->opcode].needs_file) {
Pavel Begunkov62906e82021-08-10 14:52:47 +01006641 req->file = io_file_get(ctx, req, READ_ONCE(sqe->fd),
Pavel Begunkovac177052021-08-09 13:04:02 +01006642 (sqe_flags & IOSQE_FIXED_FILE));
Pavel Begunkovba13e232021-02-01 18:59:52 +00006643 if (unlikely(!req->file))
Pavel Begunkovbd5bbda2020-11-20 15:50:51 +00006644 ret = -EBADF;
6645 }
6646
Pavel Begunkov71b547c2020-10-10 18:34:09 +01006647 state->ios_left--;
6648 return ret;
Pavel Begunkov0553b8b2020-04-08 08:58:45 +03006649}
6650
Pavel Begunkova6b8cadc2021-02-18 18:29:41 +00006651static int io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,
Pavel Begunkova1ab7b32021-02-18 18:29:42 +00006652 const struct io_uring_sqe *sqe)
Pavel Begunkov282cdc82021-08-09 13:04:10 +01006653 __must_hold(&ctx->uring_lock)
Jens Axboe6c271ce2019-01-10 11:22:30 -07006654{
Pavel Begunkova1ab7b32021-02-18 18:29:42 +00006655 struct io_submit_link *link = &ctx->submit_state.link;
Jens Axboe6c271ce2019-01-10 11:22:30 -07006656 int ret;
6657
Pavel Begunkova6b8cadc2021-02-18 18:29:41 +00006658 ret = io_init_req(ctx, req, sqe);
6659 if (unlikely(ret)) {
6660fail_req:
Pavel Begunkovde59bc12021-02-18 18:29:47 +00006661 if (link->head) {
6662 /* fail even hard links since we don't submit */
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01006663 req_set_fail(link->head);
Pavel Begunkovf41db2732021-02-28 22:35:12 +00006664 io_req_complete_failed(link->head, -ECANCELED);
Pavel Begunkovde59bc12021-02-18 18:29:47 +00006665 link->head = NULL;
6666 }
Pavel Begunkovf41db2732021-02-28 22:35:12 +00006667 io_req_complete_failed(req, ret);
Pavel Begunkova6b8cadc2021-02-18 18:29:41 +00006668 return ret;
6669 }
Pavel Begunkov441b8a72021-06-14 23:37:31 +01006670
Pavel Begunkovbe7053b2021-02-18 18:29:45 +00006671 ret = io_req_prep(req, sqe);
6672 if (unlikely(ret))
6673 goto fail_req;
Pavel Begunkova6b8cadc2021-02-18 18:29:41 +00006674
Pavel Begunkovbe7053b2021-02-18 18:29:45 +00006675 /* don't need @sqe from now on */
Olivier Langlois236daeae2021-05-31 02:36:37 -04006676 trace_io_uring_submit_sqe(ctx, req, req->opcode, req->user_data,
6677 req->flags, true,
6678 ctx->flags & IORING_SETUP_SQPOLL);
Pavel Begunkova6b8cadc2021-02-18 18:29:41 +00006679
Jens Axboe6c271ce2019-01-10 11:22:30 -07006680 /*
6681 * If we already have a head request, queue this one for async
6682 * submittal once the head completes. If we don't have a head but
6683 * IOSQE_IO_LINK is set in the sqe, start a new head. This one will be
6684 * submitted sync once the chain is complete. If none of those
6685 * conditions are true (normal request), then just queue it.
6686 */
6687 if (link->head) {
6688 struct io_kiocb *head = link->head;
6689
Pavel Begunkovb7e298d2021-02-28 22:35:19 +00006690 ret = io_req_prep_async(req);
Pavel Begunkovcf109602021-02-18 18:29:43 +00006691 if (unlikely(ret))
Pavel Begunkova6b8cadc2021-02-18 18:29:41 +00006692 goto fail_req;
Jens Axboe6c271ce2019-01-10 11:22:30 -07006693 trace_io_uring_link(ctx, req, head);
6694 link->last->link = req;
6695 link->last = req;
6696
6697 /* last request of a link, enqueue the link */
6698 if (!(req->flags & (REQ_F_LINK | REQ_F_HARDLINK))) {
6699 link->head = NULL;
Pavel Begunkov5e159202021-06-14 23:37:26 +01006700 io_queue_sqe(head);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006701 }
Jackie Liu4fe2c962019-09-09 20:50:40 +08006702 } else {
Jens Axboe2b188cc2019-01-07 10:46:33 -07006703 if (req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) {
Jackie Liu4fe2c962019-09-09 20:50:40 +08006704 link->head = req;
6705 link->last = req;
6706 } else {
Pavel Begunkovbe7053b2021-02-18 18:29:45 +00006707 io_queue_sqe(req);
Jackie Liu4fe2c962019-09-09 20:50:40 +08006708 }
6709 }
6710
6711 return 0;
6712}
6713
6714/*
6715 * Batched submission is done, ensure local IO is flushed out.
6716 */
6717static void io_submit_state_end(struct io_submit_state *state,
6718 struct io_ring_ctx *ctx)
Pavel Begunkov1b4a51b2019-11-21 11:54:28 +03006719{
Pavel Begunkova1ab7b32021-02-18 18:29:42 +00006720 if (state->link.head)
Pavel Begunkovde59bc12021-02-18 18:29:47 +00006721 io_queue_sqe(state->link.head);
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01006722 if (state->compl_nr)
Pavel Begunkov2a2758f2021-06-17 18:14:00 +01006723 io_submit_flush_completions(ctx);
Jackie Liua197f662019-11-08 08:09:12 -07006724 if (state->plug_started)
Pavel Begunkov32fe5252019-12-17 22:26:58 +03006725 blk_finish_plug(&state->plug);
Jens Axboe9e645e112019-05-10 16:07:28 -06006726}
Pavel Begunkov32fe5252019-12-17 22:26:58 +03006727
Jens Axboe9e645e112019-05-10 16:07:28 -06006728/*
6729 * Start submission side cache.
Pavel Begunkov32fe5252019-12-17 22:26:58 +03006730 */
Jens Axboe9e645e112019-05-10 16:07:28 -06006731static void io_submit_state_start(struct io_submit_state *state,
Pavel Begunkov196be952019-11-07 01:41:06 +03006732 unsigned int max_ios)
Jens Axboe9e645e112019-05-10 16:07:28 -06006733{
6734 state->plug_started = false;
Jens Axboebcda7ba2020-02-23 16:42:51 -07006735 state->ios_left = max_ios;
Pavel Begunkova1ab7b32021-02-18 18:29:42 +00006736 /* set only head, no need to init link_last in advance */
6737 state->link.head = NULL;
Jens Axboe75c6a032020-01-28 10:15:23 -07006738}
6739
Jens Axboe193155c2020-02-22 23:22:19 -07006740static void io_commit_sqring(struct io_ring_ctx *ctx)
6741{
Jens Axboe75c6a032020-01-28 10:15:23 -07006742 struct io_rings *rings = ctx->rings;
6743
6744 /*
Jens Axboe193155c2020-02-22 23:22:19 -07006745 * Ensure any loads from the SQEs are done at this point,
Jens Axboe75c6a032020-01-28 10:15:23 -07006746 * since once we write the new head, the application could
6747 * write new data to them.
Pavel Begunkov6b47ee62020-01-18 20:22:41 +03006748 */
Pavel Begunkov8da11c12020-02-24 11:32:44 +03006749 smp_store_release(&rings->sq.head, ctx->cached_sq_head);
Jens Axboebcda7ba2020-02-23 16:42:51 -07006750}
6751
Jens Axboe9e645e112019-05-10 16:07:28 -06006752/*
Fam Zhengdd9ae8a2021-06-04 17:42:56 +01006753 * Fetch an sqe, if one is available. Note this returns a pointer to memory
Jens Axboe9e645e112019-05-10 16:07:28 -06006754 * that is mapped by userspace. This means that care needs to be taken to
6755 * ensure that reads are stable, as we cannot rely on userspace always
Jens Axboe78e19bb2019-11-06 15:21:34 -07006756 * being a good citizen. If members of the sqe are validated and then later
6757 * used, it's important that those reads are done through READ_ONCE() to
Pavel Begunkov2e6e1fd2019-12-05 16:15:45 +03006758 * prevent a re-load down the line.
Jens Axboe9e645e112019-05-10 16:07:28 -06006759 */
6760static const struct io_uring_sqe *io_get_sqe(struct io_ring_ctx *ctx)
Jens Axboe9e645e112019-05-10 16:07:28 -06006761{
Pavel Begunkovea5ab3b2021-05-16 22:58:09 +01006762 unsigned head, mask = ctx->sq_entries - 1;
Pavel Begunkov17d3aeb2021-06-14 23:37:23 +01006763 unsigned sq_idx = ctx->cached_sq_head++ & mask;
Jens Axboe9e645e112019-05-10 16:07:28 -06006764
6765 /*
6766 * The cached sq head (or cq tail) serves two purposes:
6767 *
6768 * 1) allows us to batch the cost of updating the user visible
Pavel Begunkov9d763772019-12-17 02:22:07 +03006769 * head updates.
Jens Axboe9e645e112019-05-10 16:07:28 -06006770 * 2) allows the kernel side to track the head on its own, even
Pavel Begunkov8cdf2192020-01-25 00:40:24 +03006771 * though the application is the one updating it.
6772 */
Pavel Begunkov17d3aeb2021-06-14 23:37:23 +01006773 head = READ_ONCE(ctx->sq_array[sq_idx]);
Pavel Begunkov8cdf2192020-01-25 00:40:24 +03006774 if (likely(head < ctx->sq_entries))
6775 return &ctx->sq_sqes[head];
6776
6777 /* drop invalid entries */
Pavel Begunkov15641e42021-06-14 23:37:24 +01006778 ctx->cq_extra--;
6779 WRITE_ONCE(ctx->rings->sq_dropped,
6780 READ_ONCE(ctx->rings->sq_dropped) + 1);
Pavel Begunkov711be032020-01-17 03:57:59 +03006781 return NULL;
6782}
Jens Axboeb7bb4f72019-12-15 22:13:43 -07006783
Jens Axboe0f212202020-09-13 13:09:39 -06006784static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr)
Pavel Begunkov282cdc82021-08-09 13:04:10 +01006785 __must_hold(&ctx->uring_lock)
Jens Axboe6c271ce2019-01-10 11:22:30 -07006786{
Pavel Begunkov09899b12021-06-14 02:36:22 +01006787 struct io_uring_task *tctx;
Pavel Begunkov46c4e162021-02-18 18:29:37 +00006788 int submitted = 0;
Jens Axboe6c271ce2019-01-10 11:22:30 -07006789
Pavel Begunkovee7d46d2019-12-30 21:24:45 +03006790 /* make sure SQ entry isn't read before tail */
6791 nr = min3(nr, ctx->sq_entries, io_sqring_entries(ctx));
Pavel Begunkov2b85edf2019-12-28 14:13:03 +03006792 if (!percpu_ref_tryget_many(&ctx->refs, nr))
6793 return -EAGAIN;
Jens Axboe6c271ce2019-01-10 11:22:30 -07006794
Pavel Begunkov09899b12021-06-14 02:36:22 +01006795 tctx = current->io_uring;
6796 tctx->cached_refs -= nr;
6797 if (unlikely(tctx->cached_refs < 0)) {
6798 unsigned int refill = -tctx->cached_refs + IO_TCTX_REFS_CACHE_NR;
6799
6800 percpu_counter_add(&tctx->inflight, refill);
6801 refcount_add(refill, &current->usage);
6802 tctx->cached_refs += refill;
6803 }
Pavel Begunkovba88ff12021-02-10 00:03:11 +00006804 io_submit_state_start(&ctx->submit_state, nr);
Pavel Begunkovb14cca02020-01-17 04:45:59 +03006805
Pavel Begunkov46c4e162021-02-18 18:29:37 +00006806 while (submitted < nr) {
Jens Axboe3529d8c2019-12-19 18:24:38 -07006807 const struct io_uring_sqe *sqe;
Pavel Begunkov196be952019-11-07 01:41:06 +03006808 struct io_kiocb *req;
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03006809
Pavel Begunkov258b29a2021-02-10 00:03:10 +00006810 req = io_alloc_req(ctx);
Pavel Begunkov196be952019-11-07 01:41:06 +03006811 if (unlikely(!req)) {
6812 if (!submitted)
6813 submitted = -EAGAIN;
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03006814 break;
Jens Axboe9e645e112019-05-10 16:07:28 -06006815 }
Pavel Begunkov4fccfcb2021-02-12 11:55:17 +00006816 sqe = io_get_sqe(ctx);
6817 if (unlikely(!sqe)) {
6818 kmem_cache_free(req_cachep, req);
6819 break;
6820 }
Jens Axboed3656342019-12-18 09:50:26 -07006821 /* will complete beyond this point, count as submitted */
6822 submitted++;
Pavel Begunkova1ab7b32021-02-18 18:29:42 +00006823 if (io_submit_sqe(ctx, req, sqe))
Jens Axboed3656342019-12-18 09:50:26 -07006824 break;
Jens Axboe6c271ce2019-01-10 11:22:30 -07006825 }
6826
Pavel Begunkov9466f432020-01-25 22:34:01 +03006827 if (unlikely(submitted != nr)) {
6828 int ref_used = (submitted == -EAGAIN) ? 0 : submitted;
Jens Axboed8a6df12020-10-15 16:24:45 -06006829 int unused = nr - ref_used;
Pavel Begunkov9466f432020-01-25 22:34:01 +03006830
Pavel Begunkov09899b12021-06-14 02:36:22 +01006831 current->io_uring->cached_refs += unused;
Jens Axboed8a6df12020-10-15 16:24:45 -06006832 percpu_ref_put_many(&ctx->refs, unused);
Pavel Begunkov9466f432020-01-25 22:34:01 +03006833 }
Jens Axboe6c271ce2019-01-10 11:22:30 -07006834
Pavel Begunkova1ab7b32021-02-18 18:29:42 +00006835 io_submit_state_end(&ctx->submit_state, ctx);
Pavel Begunkovae9428c2019-11-06 00:22:14 +03006836 /* Commit SQ ring head once we've consumed and submitted all SQEs */
6837 io_commit_sqring(ctx);
6838
Jens Axboe6c271ce2019-01-10 11:22:30 -07006839 return submitted;
6840}
6841
Pavel Begunkove4b6d902021-05-16 22:58:00 +01006842static inline bool io_sqd_events_pending(struct io_sq_data *sqd)
6843{
6844 return READ_ONCE(sqd->state);
6845}
6846
Xiaoguang Wang23b36282020-07-23 20:57:24 +08006847static inline void io_ring_set_wakeup_flag(struct io_ring_ctx *ctx)
6848{
6849 /* Tell userspace we may need a wakeup call */
Jens Axboe79ebeae2021-08-10 15:18:27 -06006850 spin_lock(&ctx->completion_lock);
Nadav Amit20c0b382021-08-07 17:13:42 -07006851 WRITE_ONCE(ctx->rings->sq_flags,
6852 ctx->rings->sq_flags | IORING_SQ_NEED_WAKEUP);
Jens Axboe79ebeae2021-08-10 15:18:27 -06006853 spin_unlock(&ctx->completion_lock);
Xiaoguang Wang23b36282020-07-23 20:57:24 +08006854}
6855
6856static inline void io_ring_clear_wakeup_flag(struct io_ring_ctx *ctx)
6857{
Jens Axboe79ebeae2021-08-10 15:18:27 -06006858 spin_lock(&ctx->completion_lock);
Nadav Amit20c0b382021-08-07 17:13:42 -07006859 WRITE_ONCE(ctx->rings->sq_flags,
6860 ctx->rings->sq_flags & ~IORING_SQ_NEED_WAKEUP);
Jens Axboe79ebeae2021-08-10 15:18:27 -06006861 spin_unlock(&ctx->completion_lock);
Xiaoguang Wang23b36282020-07-23 20:57:24 +08006862}
6863
Xiaoguang Wang08369242020-11-03 14:15:59 +08006864static int __io_sq_thread(struct io_ring_ctx *ctx, bool cap_entries)
Jens Axboe6c271ce2019-01-10 11:22:30 -07006865{
Jens Axboec8d1ba52020-09-14 11:07:26 -06006866 unsigned int to_submit;
Xiaoguang Wangbdcd3ea2020-02-25 22:12:08 +08006867 int ret = 0;
Jens Axboe6c271ce2019-01-10 11:22:30 -07006868
Jens Axboec8d1ba52020-09-14 11:07:26 -06006869 to_submit = io_sqring_entries(ctx);
Jens Axboee95eee22020-09-08 09:11:32 -06006870 /* if we're handling multiple rings, cap submit size for fairness */
Olivier Langlois4ce8ad92021-06-23 11:50:18 -07006871 if (cap_entries && to_submit > IORING_SQPOLL_CAP_ENTRIES_VALUE)
6872 to_submit = IORING_SQPOLL_CAP_ENTRIES_VALUE;
Jens Axboee95eee22020-09-08 09:11:32 -06006873
Xiaoguang Wang906a3c62020-11-12 14:56:00 +08006874 if (!list_empty(&ctx->iopoll_list) || to_submit) {
6875 unsigned nr_events = 0;
Pavel Begunkov948e1942021-06-24 15:09:55 +01006876 const struct cred *creds = NULL;
6877
6878 if (ctx->sq_creds != current_cred())
6879 creds = override_creds(ctx->sq_creds);
Xiaoguang Wang906a3c62020-11-12 14:56:00 +08006880
Xiaoguang Wang08369242020-11-03 14:15:59 +08006881 mutex_lock(&ctx->uring_lock);
Xiaoguang Wang906a3c62020-11-12 14:56:00 +08006882 if (!list_empty(&ctx->iopoll_list))
Pavel Begunkova8576af2021-08-15 10:40:21 +01006883 io_do_iopoll(ctx, &nr_events, 0);
Xiaoguang Wang906a3c62020-11-12 14:56:00 +08006884
Pavel Begunkov3b763ba2021-04-18 14:52:08 +01006885 /*
6886 * Don't submit if refs are dying, good for io_uring_register(),
6887 * but also it is relied upon by io_ring_exit_work()
6888 */
Pavel Begunkov0298ef92021-03-08 13:20:57 +00006889 if (to_submit && likely(!percpu_ref_is_dying(&ctx->refs)) &&
6890 !(ctx->flags & IORING_SETUP_R_DISABLED))
Xiaoguang Wang08369242020-11-03 14:15:59 +08006891 ret = io_submit_sqes(ctx, to_submit);
6892 mutex_unlock(&ctx->uring_lock);
Jens Axboe90554202020-09-03 12:12:41 -06006893
Pavel Begunkovacfb3812021-05-16 22:58:03 +01006894 if (to_submit && wq_has_sleeper(&ctx->sqo_sq_wait))
6895 wake_up(&ctx->sqo_sq_wait);
Pavel Begunkov948e1942021-06-24 15:09:55 +01006896 if (creds)
6897 revert_creds(creds);
Pavel Begunkovacfb3812021-05-16 22:58:03 +01006898 }
Jens Axboe90554202020-09-03 12:12:41 -06006899
Xiaoguang Wang08369242020-11-03 14:15:59 +08006900 return ret;
6901}
6902
6903static void io_sqd_update_thread_idle(struct io_sq_data *sqd)
6904{
6905 struct io_ring_ctx *ctx;
6906 unsigned sq_thread_idle = 0;
6907
Pavel Begunkovc9dca272021-03-10 13:13:55 +00006908 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list)
6909 sq_thread_idle = max(sq_thread_idle, ctx->sq_thread_idle);
Xiaoguang Wang08369242020-11-03 14:15:59 +08006910 sqd->sq_thread_idle = sq_thread_idle;
Jens Axboec8d1ba52020-09-14 11:07:26 -06006911}
6912
Pavel Begunkove4b6d902021-05-16 22:58:00 +01006913static bool io_sqd_handle_event(struct io_sq_data *sqd)
6914{
6915 bool did_sig = false;
6916 struct ksignal ksig;
6917
6918 if (test_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state) ||
6919 signal_pending(current)) {
6920 mutex_unlock(&sqd->lock);
6921 if (signal_pending(current))
6922 did_sig = get_signal(&ksig);
6923 cond_resched();
6924 mutex_lock(&sqd->lock);
6925 }
Pavel Begunkove4b6d902021-05-16 22:58:00 +01006926 return did_sig || test_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state);
6927}
6928
Jens Axboe6c271ce2019-01-10 11:22:30 -07006929static int io_sq_thread(void *data)
6930{
Jens Axboe69fb2132020-09-14 11:16:23 -06006931 struct io_sq_data *sqd = data;
6932 struct io_ring_ctx *ctx;
Xiaoguang Wanga0d92052020-11-12 14:55:59 +08006933 unsigned long timeout = 0;
Jens Axboe37d1e2e2021-02-17 21:03:43 -07006934 char buf[TASK_COMM_LEN];
Xiaoguang Wang08369242020-11-03 14:15:59 +08006935 DEFINE_WAIT(wait);
Jens Axboe6c271ce2019-01-10 11:22:30 -07006936
Pavel Begunkov696ee882021-04-01 09:55:04 +01006937 snprintf(buf, sizeof(buf), "iou-sqp-%d", sqd->task_pid);
Jens Axboe37d1e2e2021-02-17 21:03:43 -07006938 set_task_comm(current, buf);
Jens Axboe28cea78a2020-09-14 10:51:17 -06006939
Jens Axboe37d1e2e2021-02-17 21:03:43 -07006940 if (sqd->sq_cpu != -1)
6941 set_cpus_allowed_ptr(current, cpumask_of(sqd->sq_cpu));
6942 else
6943 set_cpus_allowed_ptr(current, cpu_online_mask);
6944 current->flags |= PF_NO_SETAFFINITY;
6945
Pavel Begunkov09a6f4e2021-03-14 20:57:10 +00006946 mutex_lock(&sqd->lock);
Pavel Begunkove4b6d902021-05-16 22:58:00 +01006947 while (1) {
Pavel Begunkov1a924a82021-06-24 15:09:56 +01006948 bool cap_entries, sqt_spin = false;
Jens Axboec1edbf52019-11-10 16:56:04 -07006949
Pavel Begunkove4b6d902021-05-16 22:58:00 +01006950 if (io_sqd_events_pending(sqd) || signal_pending(current)) {
6951 if (io_sqd_handle_event(sqd))
Pavel Begunkovc7d95612021-04-13 11:43:00 +01006952 break;
Xiaoguang Wang08369242020-11-03 14:15:59 +08006953 timeout = jiffies + sqd->sq_thread_idle;
6954 }
Pavel Begunkove4b6d902021-05-16 22:58:00 +01006955
Jens Axboee95eee22020-09-08 09:11:32 -06006956 cap_entries = !list_is_singular(&sqd->ctx_list);
Jens Axboe69fb2132020-09-14 11:16:23 -06006957 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) {
Pavel Begunkov948e1942021-06-24 15:09:55 +01006958 int ret = __io_sq_thread(ctx, cap_entries);
Stefan Metzmacher7c30f36a2021-03-07 11:54:28 +01006959
Xiaoguang Wang08369242020-11-03 14:15:59 +08006960 if (!sqt_spin && (ret > 0 || !list_empty(&ctx->iopoll_list)))
6961 sqt_spin = true;
Jens Axboe6c271ce2019-01-10 11:22:30 -07006962 }
Pavel Begunkovdd432ea52021-06-26 21:40:45 +01006963 if (io_run_task_work())
6964 sqt_spin = true;
Jens Axboe6c271ce2019-01-10 11:22:30 -07006965
Xiaoguang Wang08369242020-11-03 14:15:59 +08006966 if (sqt_spin || !time_after(jiffies, timeout)) {
Jens Axboec8d1ba52020-09-14 11:07:26 -06006967 cond_resched();
Xiaoguang Wang08369242020-11-03 14:15:59 +08006968 if (sqt_spin)
6969 timeout = jiffies + sqd->sq_thread_idle;
6970 continue;
6971 }
6972
Xiaoguang Wang08369242020-11-03 14:15:59 +08006973 prepare_to_wait(&sqd->wait, &wait, TASK_INTERRUPTIBLE);
Pavel Begunkovdd432ea52021-06-26 21:40:45 +01006974 if (!io_sqd_events_pending(sqd) && !current->task_works) {
Pavel Begunkov1a924a82021-06-24 15:09:56 +01006975 bool needs_sched = true;
6976
Hao Xu724cb4f2021-04-21 23:19:11 +08006977 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) {
Pavel Begunkovaaa9f0f2021-05-16 22:58:01 +01006978 io_ring_set_wakeup_flag(ctx);
6979
Hao Xu724cb4f2021-04-21 23:19:11 +08006980 if ((ctx->flags & IORING_SETUP_IOPOLL) &&
6981 !list_empty_careful(&ctx->iopoll_list)) {
6982 needs_sched = false;
6983 break;
6984 }
6985 if (io_sqring_entries(ctx)) {
6986 needs_sched = false;
6987 break;
6988 }
6989 }
6990
6991 if (needs_sched) {
6992 mutex_unlock(&sqd->lock);
6993 schedule();
6994 mutex_lock(&sqd->lock);
6995 }
Jens Axboe69fb2132020-09-14 11:16:23 -06006996 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list)
6997 io_ring_clear_wakeup_flag(ctx);
Jens Axboe6c271ce2019-01-10 11:22:30 -07006998 }
Xiaoguang Wang08369242020-11-03 14:15:59 +08006999
7000 finish_wait(&sqd->wait, &wait);
7001 timeout = jiffies + sqd->sq_thread_idle;
Jens Axboe6c271ce2019-01-10 11:22:30 -07007002 }
7003
Pavel Begunkov78cc6872021-06-14 02:36:23 +01007004 io_uring_cancel_generic(true, sqd);
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007005 sqd->thread = NULL;
Jens Axboe05962f92021-03-06 13:58:48 -07007006 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list)
Jens Axboe5f3f26f2021-02-25 10:17:46 -07007007 io_ring_set_wakeup_flag(ctx);
Pavel Begunkov521d6a72021-03-11 23:29:38 +00007008 io_run_task_work();
Pavel Begunkov734551d2021-04-18 14:52:09 +01007009 mutex_unlock(&sqd->lock);
7010
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007011 complete(&sqd->exited);
7012 do_exit(0);
Jens Axboe6c271ce2019-01-10 11:22:30 -07007013}
7014
Jens Axboebda52162019-09-24 13:47:15 -06007015struct io_wait_queue {
7016 struct wait_queue_entry wq;
7017 struct io_ring_ctx *ctx;
Jens Axboe5fd46172021-08-06 14:04:31 -06007018 unsigned cq_tail;
Jens Axboebda52162019-09-24 13:47:15 -06007019 unsigned nr_timeouts;
7020};
7021
Pavel Begunkov6c503152021-01-04 20:36:36 +00007022static inline bool io_should_wake(struct io_wait_queue *iowq)
Jens Axboebda52162019-09-24 13:47:15 -06007023{
7024 struct io_ring_ctx *ctx = iowq->ctx;
Jens Axboe5fd46172021-08-06 14:04:31 -06007025 int dist = ctx->cached_cq_tail - (int) iowq->cq_tail;
Jens Axboebda52162019-09-24 13:47:15 -06007026
7027 /*
Brian Gianforcarod195a662019-12-13 03:09:50 -08007028 * Wake up if we have enough events, or if a timeout occurred since we
Jens Axboebda52162019-09-24 13:47:15 -06007029 * started waiting. For timeouts, we always want to return to userspace,
7030 * regardless of event count.
7031 */
Jens Axboe5fd46172021-08-06 14:04:31 -06007032 return dist >= 0 || atomic_read(&ctx->cq_timeouts) != iowq->nr_timeouts;
Jens Axboebda52162019-09-24 13:47:15 -06007033}
7034
7035static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode,
7036 int wake_flags, void *key)
7037{
7038 struct io_wait_queue *iowq = container_of(curr, struct io_wait_queue,
7039 wq);
7040
Pavel Begunkov6c503152021-01-04 20:36:36 +00007041 /*
7042 * Cannot safely flush overflowed CQEs from here, ensure we wake up
7043 * the task, and the next invocation will do it.
7044 */
Pavel Begunkov5ed7a372021-06-14 23:37:27 +01007045 if (io_should_wake(iowq) || test_bit(0, &iowq->ctx->check_cq_overflow))
Pavel Begunkov6c503152021-01-04 20:36:36 +00007046 return autoremove_wake_function(curr, mode, wake_flags, key);
7047 return -1;
Jens Axboebda52162019-09-24 13:47:15 -06007048}
7049
Jens Axboeaf9c1a42020-09-24 13:32:18 -06007050static int io_run_task_work_sig(void)
7051{
7052 if (io_run_task_work())
7053 return 1;
7054 if (!signal_pending(current))
7055 return 0;
Jens Axboe0b8cfa92021-03-21 14:16:08 -06007056 if (test_thread_flag(TIF_NOTIFY_SIGNAL))
Jens Axboe792ee0f62020-10-22 20:17:18 -06007057 return -ERESTARTSYS;
Jens Axboeaf9c1a42020-09-24 13:32:18 -06007058 return -EINTR;
7059}
7060
Pavel Begunkoveeb60b92021-02-04 13:51:58 +00007061/* when returns >0, the caller should retry */
7062static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx,
7063 struct io_wait_queue *iowq,
7064 signed long *timeout)
7065{
7066 int ret;
7067
7068 /* make sure we run task_work before checking for signals */
7069 ret = io_run_task_work_sig();
7070 if (ret || io_should_wake(iowq))
7071 return ret;
7072 /* let the caller flush overflows, retry */
Pavel Begunkov5ed7a372021-06-14 23:37:27 +01007073 if (test_bit(0, &ctx->check_cq_overflow))
Pavel Begunkoveeb60b92021-02-04 13:51:58 +00007074 return 1;
7075
7076 *timeout = schedule_timeout(*timeout);
7077 return !*timeout ? -ETIME : 1;
7078}
7079
Jens Axboe2b188cc2019-01-07 10:46:33 -07007080/*
7081 * Wait until events become available, if we don't already have some. The
7082 * application must reap them itself, as they reside on the shared cq ring.
7083 */
7084static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events,
Hao Xuc73ebb62020-11-03 10:54:37 +08007085 const sigset_t __user *sig, size_t sigsz,
7086 struct __kernel_timespec __user *uts)
Jens Axboe2b188cc2019-01-07 10:46:33 -07007087{
Pavel Begunkov902910992021-08-09 09:07:32 -06007088 struct io_wait_queue iowq;
Hristo Venev75b28af2019-08-26 17:23:46 +00007089 struct io_rings *rings = ctx->rings;
Pavel Begunkovc1d5a222021-02-04 13:51:57 +00007090 signed long timeout = MAX_SCHEDULE_TIMEOUT;
7091 int ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07007092
Jens Axboeb41e9852020-02-17 09:52:41 -07007093 do {
Pavel Begunkov90f67362021-08-09 20:18:12 +01007094 io_cqring_overflow_flush(ctx);
Pavel Begunkov6c503152021-01-04 20:36:36 +00007095 if (io_cqring_events(ctx) >= min_events)
Jens Axboeb41e9852020-02-17 09:52:41 -07007096 return 0;
Jens Axboe4c6e2772020-07-01 11:29:10 -06007097 if (!io_run_task_work())
Jens Axboeb41e9852020-02-17 09:52:41 -07007098 break;
Jens Axboeb41e9852020-02-17 09:52:41 -07007099 } while (1);
Jens Axboe2b188cc2019-01-07 10:46:33 -07007100
7101 if (sig) {
Arnd Bergmann9e75ad52019-03-25 15:34:53 +01007102#ifdef CONFIG_COMPAT
7103 if (in_compat_syscall())
7104 ret = set_compat_user_sigmask((const compat_sigset_t __user *)sig,
Oleg Nesterovb7724342019-07-16 16:29:53 -07007105 sigsz);
Arnd Bergmann9e75ad52019-03-25 15:34:53 +01007106 else
7107#endif
Oleg Nesterovb7724342019-07-16 16:29:53 -07007108 ret = set_user_sigmask(sig, sigsz);
Arnd Bergmann9e75ad52019-03-25 15:34:53 +01007109
Jens Axboe2b188cc2019-01-07 10:46:33 -07007110 if (ret)
7111 return ret;
7112 }
7113
Hao Xuc73ebb62020-11-03 10:54:37 +08007114 if (uts) {
Pavel Begunkovc1d5a222021-02-04 13:51:57 +00007115 struct timespec64 ts;
7116
Hao Xuc73ebb62020-11-03 10:54:37 +08007117 if (get_timespec64(&ts, uts))
7118 return -EFAULT;
7119 timeout = timespec64_to_jiffies(&ts);
7120 }
7121
Pavel Begunkov902910992021-08-09 09:07:32 -06007122 init_waitqueue_func_entry(&iowq.wq, io_wake_function);
7123 iowq.wq.private = current;
7124 INIT_LIST_HEAD(&iowq.wq.entry);
7125 iowq.ctx = ctx;
Jens Axboebda52162019-09-24 13:47:15 -06007126 iowq.nr_timeouts = atomic_read(&ctx->cq_timeouts);
Jens Axboe5fd46172021-08-06 14:04:31 -06007127 iowq.cq_tail = READ_ONCE(ctx->rings->cq.head) + min_events;
Pavel Begunkov902910992021-08-09 09:07:32 -06007128
Dmitrii Dolgovc826bd72019-10-15 19:02:01 +02007129 trace_io_uring_cqring_wait(ctx, min_events);
Jens Axboebda52162019-09-24 13:47:15 -06007130 do {
Jens Axboeca0a2652021-03-04 17:15:48 -07007131 /* if we can't even flush overflow, don't wait for more */
Pavel Begunkov90f67362021-08-09 20:18:12 +01007132 if (!io_cqring_overflow_flush(ctx)) {
Jens Axboeca0a2652021-03-04 17:15:48 -07007133 ret = -EBUSY;
7134 break;
7135 }
Pavel Begunkov311997b2021-06-14 23:37:28 +01007136 prepare_to_wait_exclusive(&ctx->cq_wait, &iowq.wq,
Jens Axboebda52162019-09-24 13:47:15 -06007137 TASK_INTERRUPTIBLE);
Pavel Begunkoveeb60b92021-02-04 13:51:58 +00007138 ret = io_cqring_wait_schedule(ctx, &iowq, &timeout);
Pavel Begunkov311997b2021-06-14 23:37:28 +01007139 finish_wait(&ctx->cq_wait, &iowq.wq);
Jens Axboeca0a2652021-03-04 17:15:48 -07007140 cond_resched();
Pavel Begunkoveeb60b92021-02-04 13:51:58 +00007141 } while (ret > 0);
Jens Axboebda52162019-09-24 13:47:15 -06007142
Jens Axboeb7db41c2020-07-04 08:55:50 -06007143 restore_saved_sigmask_unless(ret == -EINTR);
Jens Axboe2b188cc2019-01-07 10:46:33 -07007144
Hristo Venev75b28af2019-08-26 17:23:46 +00007145 return READ_ONCE(rings->cq.head) == READ_ONCE(rings->cq.tail) ? ret : 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07007146}
7147
Pavel Begunkov9123c8f2021-06-14 02:36:20 +01007148static void io_free_page_table(void **table, size_t size)
Pavel Begunkov846a4ef2021-04-01 15:44:03 +01007149{
Pavel Begunkov9123c8f2021-06-14 02:36:20 +01007150 unsigned i, nr_tables = DIV_ROUND_UP(size, PAGE_SIZE);
Pavel Begunkov846a4ef2021-04-01 15:44:03 +01007151
7152 for (i = 0; i < nr_tables; i++)
Pavel Begunkov9123c8f2021-06-14 02:36:20 +01007153 kfree(table[i]);
7154 kfree(table);
7155}
7156
7157static void **io_alloc_page_table(size_t size)
7158{
7159 unsigned i, nr_tables = DIV_ROUND_UP(size, PAGE_SIZE);
7160 size_t init_size = size;
7161 void **table;
7162
7163 table = kcalloc(nr_tables, sizeof(*table), GFP_KERNEL);
7164 if (!table)
7165 return NULL;
7166
7167 for (i = 0; i < nr_tables; i++) {
Pavel Begunkov27f6b312021-06-15 13:20:13 +01007168 unsigned int this_size = min_t(size_t, size, PAGE_SIZE);
Pavel Begunkov9123c8f2021-06-14 02:36:20 +01007169
7170 table[i] = kzalloc(this_size, GFP_KERNEL);
7171 if (!table[i]) {
7172 io_free_page_table(table, init_size);
7173 return NULL;
7174 }
7175 size -= this_size;
7176 }
7177 return table;
Pavel Begunkov846a4ef2021-04-01 15:44:03 +01007178}
7179
Pavel Begunkov28a9fe22021-04-01 15:43:47 +01007180static void io_rsrc_node_destroy(struct io_rsrc_node *ref_node)
7181{
7182 percpu_ref_exit(&ref_node->refs);
7183 kfree(ref_node);
7184}
7185
Pavel Begunkovb9bd2be2021-08-09 09:09:47 -06007186static void io_rsrc_node_ref_zero(struct percpu_ref *ref)
7187{
7188 struct io_rsrc_node *node = container_of(ref, struct io_rsrc_node, refs);
7189 struct io_ring_ctx *ctx = node->rsrc_data->ctx;
7190 unsigned long flags;
7191 bool first_add = false;
7192
7193 spin_lock_irqsave(&ctx->rsrc_ref_lock, flags);
7194 node->done = true;
7195
7196 while (!list_empty(&ctx->rsrc_ref_list)) {
7197 node = list_first_entry(&ctx->rsrc_ref_list,
7198 struct io_rsrc_node, node);
7199 /* recycle ref nodes in order */
7200 if (!node->done)
7201 break;
7202 list_del(&node->node);
7203 first_add |= llist_add(&node->llist, &ctx->rsrc_put_llist);
7204 }
7205 spin_unlock_irqrestore(&ctx->rsrc_ref_lock, flags);
7206
7207 if (first_add)
7208 mod_delayed_work(system_wq, &ctx->rsrc_put_work, HZ);
7209}
7210
7211static struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx)
7212{
7213 struct io_rsrc_node *ref_node;
7214
7215 ref_node = kzalloc(sizeof(*ref_node), GFP_KERNEL);
7216 if (!ref_node)
7217 return NULL;
7218
7219 if (percpu_ref_init(&ref_node->refs, io_rsrc_node_ref_zero,
7220 0, GFP_KERNEL)) {
7221 kfree(ref_node);
7222 return NULL;
7223 }
7224 INIT_LIST_HEAD(&ref_node->node);
7225 INIT_LIST_HEAD(&ref_node->rsrc_list);
7226 ref_node->done = false;
7227 return ref_node;
7228}
7229
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01007230static void io_rsrc_node_switch(struct io_ring_ctx *ctx,
7231 struct io_rsrc_data *data_to_kill)
Pavel Begunkov1642b442020-12-30 21:34:14 +00007232{
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01007233 WARN_ON_ONCE(!ctx->rsrc_backup_node);
7234 WARN_ON_ONCE(data_to_kill && !ctx->rsrc_node);
Pavel Begunkov82fbcfa2021-04-01 15:43:43 +01007235
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01007236 if (data_to_kill) {
7237 struct io_rsrc_node *rsrc_node = ctx->rsrc_node;
Pavel Begunkov82fbcfa2021-04-01 15:43:43 +01007238
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01007239 rsrc_node->rsrc_data = data_to_kill;
Jens Axboe4956b9e2021-08-09 07:49:41 -06007240 spin_lock_irq(&ctx->rsrc_ref_lock);
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01007241 list_add_tail(&rsrc_node->node, &ctx->rsrc_ref_list);
Jens Axboe4956b9e2021-08-09 07:49:41 -06007242 spin_unlock_irq(&ctx->rsrc_ref_lock);
Pavel Begunkov82fbcfa2021-04-01 15:43:43 +01007243
Pavel Begunkov3e942492021-04-11 01:46:34 +01007244 atomic_inc(&data_to_kill->refs);
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01007245 percpu_ref_kill(&rsrc_node->refs);
7246 ctx->rsrc_node = NULL;
7247 }
7248
7249 if (!ctx->rsrc_node) {
7250 ctx->rsrc_node = ctx->rsrc_backup_node;
7251 ctx->rsrc_backup_node = NULL;
7252 }
Pavel Begunkov1642b442020-12-30 21:34:14 +00007253}
7254
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01007255static int io_rsrc_node_switch_start(struct io_ring_ctx *ctx)
Pavel Begunkov8dd03af2021-03-19 17:22:36 +00007256{
7257 if (ctx->rsrc_backup_node)
7258 return 0;
Pavel Begunkovb895c9a2021-04-01 15:43:40 +01007259 ctx->rsrc_backup_node = io_rsrc_node_alloc(ctx);
Pavel Begunkov8dd03af2021-03-19 17:22:36 +00007260 return ctx->rsrc_backup_node ? 0 : -ENOMEM;
7261}
7262
Pavel Begunkov40ae0ff2021-04-01 15:43:44 +01007263static int io_rsrc_ref_quiesce(struct io_rsrc_data *data, struct io_ring_ctx *ctx)
Hao Xu8bad28d2021-02-19 17:19:36 +08007264{
7265 int ret;
Xiaoguang Wang05589552020-03-31 14:05:18 +08007266
Pavel Begunkov215c3902021-04-01 15:43:48 +01007267 /* As we may drop ->uring_lock, other task may have started quiesce */
Hao Xu8bad28d2021-02-19 17:19:36 +08007268 if (data->quiesce)
7269 return -ENXIO;
7270
7271 data->quiesce = true;
Pavel Begunkov1ffc5422020-12-30 21:34:15 +00007272 do {
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01007273 ret = io_rsrc_node_switch_start(ctx);
Pavel Begunkov8dd03af2021-03-19 17:22:36 +00007274 if (ret)
Pavel Begunkovf2303b12021-02-20 18:03:49 +00007275 break;
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01007276 io_rsrc_node_switch(ctx, data);
7277
Pavel Begunkov3e942492021-04-11 01:46:34 +01007278 /* kill initial ref, already quiesced if zero */
7279 if (atomic_dec_and_test(&data->refs))
7280 break;
Jens Axboec018db42021-08-09 08:15:50 -06007281 mutex_unlock(&ctx->uring_lock);
Hao Xu8bad28d2021-02-19 17:19:36 +08007282 flush_delayed_work(&ctx->rsrc_put_work);
Pavel Begunkov1ffc5422020-12-30 21:34:15 +00007283 ret = wait_for_completion_interruptible(&data->done);
Jens Axboec018db42021-08-09 08:15:50 -06007284 if (!ret) {
7285 mutex_lock(&ctx->uring_lock);
Pavel Begunkov1ffc5422020-12-30 21:34:15 +00007286 break;
Jens Axboec018db42021-08-09 08:15:50 -06007287 }
Jens Axboe05f3fb32019-12-09 11:22:50 -07007288
Pavel Begunkov3e942492021-04-11 01:46:34 +01007289 atomic_inc(&data->refs);
7290 /* wait for all works potentially completing data->done */
7291 flush_delayed_work(&ctx->rsrc_put_work);
Jens Axboecb5e1b82021-02-25 07:37:35 -07007292 reinit_completion(&data->done);
Pavel Begunkov8dd03af2021-03-19 17:22:36 +00007293
Hao Xu8bad28d2021-02-19 17:19:36 +08007294 ret = io_run_task_work_sig();
7295 mutex_lock(&ctx->uring_lock);
Pavel Begunkovf2303b12021-02-20 18:03:49 +00007296 } while (ret >= 0);
Hao Xu8bad28d2021-02-19 17:19:36 +08007297 data->quiesce = false;
7298
Hao Xu8bad28d2021-02-19 17:19:36 +08007299 return ret;
Bijan Mottahedehd7954b22021-01-15 17:37:50 +00007300}
7301
Pavel Begunkov2d091d62021-06-14 02:36:21 +01007302static u64 *io_get_tag_slot(struct io_rsrc_data *data, unsigned int idx)
7303{
7304 unsigned int off = idx & IO_RSRC_TAG_TABLE_MASK;
7305 unsigned int table_idx = idx >> IO_RSRC_TAG_TABLE_SHIFT;
7306
7307 return &data->tags[table_idx][off];
7308}
7309
Pavel Begunkov44b31f22021-04-25 14:32:16 +01007310static void io_rsrc_data_free(struct io_rsrc_data *data)
7311{
Pavel Begunkov2d091d62021-06-14 02:36:21 +01007312 size_t size = data->nr * sizeof(data->tags[0][0]);
7313
7314 if (data->tags)
7315 io_free_page_table((void **)data->tags, size);
Pavel Begunkov44b31f22021-04-25 14:32:16 +01007316 kfree(data);
7317}
7318
Pavel Begunkovd878c812021-06-14 02:36:18 +01007319static int io_rsrc_data_alloc(struct io_ring_ctx *ctx, rsrc_put_fn *do_put,
7320 u64 __user *utags, unsigned nr,
7321 struct io_rsrc_data **pdata)
Bijan Mottahedeh1ad555c2021-01-15 17:37:51 +00007322{
Pavel Begunkovb895c9a2021-04-01 15:43:40 +01007323 struct io_rsrc_data *data;
Pavel Begunkov2d091d62021-06-14 02:36:21 +01007324 int ret = -ENOMEM;
Pavel Begunkovd878c812021-06-14 02:36:18 +01007325 unsigned i;
Bijan Mottahedeh1ad555c2021-01-15 17:37:51 +00007326
7327 data = kzalloc(sizeof(*data), GFP_KERNEL);
7328 if (!data)
Pavel Begunkovd878c812021-06-14 02:36:18 +01007329 return -ENOMEM;
Pavel Begunkov2d091d62021-06-14 02:36:21 +01007330 data->tags = (u64 **)io_alloc_page_table(nr * sizeof(data->tags[0][0]));
Pavel Begunkovb60c8dc2021-04-25 14:32:18 +01007331 if (!data->tags) {
7332 kfree(data);
Pavel Begunkovd878c812021-06-14 02:36:18 +01007333 return -ENOMEM;
7334 }
Pavel Begunkov2d091d62021-06-14 02:36:21 +01007335
7336 data->nr = nr;
7337 data->ctx = ctx;
7338 data->do_put = do_put;
Pavel Begunkovd878c812021-06-14 02:36:18 +01007339 if (utags) {
Pavel Begunkov2d091d62021-06-14 02:36:21 +01007340 ret = -EFAULT;
Pavel Begunkovd878c812021-06-14 02:36:18 +01007341 for (i = 0; i < nr; i++) {
Colin Ian Kingfdd1dc32021-06-15 14:00:11 +01007342 u64 *tag_slot = io_get_tag_slot(data, i);
7343
7344 if (copy_from_user(tag_slot, &utags[i],
7345 sizeof(*tag_slot)))
Pavel Begunkov2d091d62021-06-14 02:36:21 +01007346 goto fail;
Pavel Begunkovd878c812021-06-14 02:36:18 +01007347 }
Pavel Begunkovb60c8dc2021-04-25 14:32:18 +01007348 }
7349
Pavel Begunkov3e942492021-04-11 01:46:34 +01007350 atomic_set(&data->refs, 1);
Bijan Mottahedeh1ad555c2021-01-15 17:37:51 +00007351 init_completion(&data->done);
Pavel Begunkovd878c812021-06-14 02:36:18 +01007352 *pdata = data;
7353 return 0;
Pavel Begunkov2d091d62021-06-14 02:36:21 +01007354fail:
7355 io_rsrc_data_free(data);
7356 return ret;
Bijan Mottahedeh1ad555c2021-01-15 17:37:51 +00007357}
7358
Pavel Begunkov9123c8f2021-06-14 02:36:20 +01007359static bool io_alloc_file_tables(struct io_file_table *table, unsigned nr_files)
7360{
Pavel Begunkov042b0d82021-08-09 13:04:01 +01007361 table->files = kvcalloc(nr_files, sizeof(table->files[0]), GFP_KERNEL);
Pavel Begunkov9123c8f2021-06-14 02:36:20 +01007362 return !!table->files;
7363}
7364
Pavel Begunkov042b0d82021-08-09 13:04:01 +01007365static void io_free_file_tables(struct io_file_table *table)
Pavel Begunkov9123c8f2021-06-14 02:36:20 +01007366{
Pavel Begunkov042b0d82021-08-09 13:04:01 +01007367 kvfree(table->files);
Pavel Begunkov9123c8f2021-06-14 02:36:20 +01007368 table->files = NULL;
7369}
7370
Jens Axboe2b188cc2019-01-07 10:46:33 -07007371static void __io_sqe_files_unregister(struct io_ring_ctx *ctx)
7372{
7373#if defined(CONFIG_UNIX)
7374 if (ctx->ring_sock) {
7375 struct sock *sock = ctx->ring_sock->sk;
7376 struct sk_buff *skb;
7377
7378 while ((skb = skb_dequeue(&sock->sk_receive_queue)) != NULL)
7379 kfree_skb(skb);
7380 }
7381#else
7382 int i;
7383
7384 for (i = 0; i < ctx->nr_user_files; i++) {
7385 struct file *file;
7386
7387 file = io_file_from_index(ctx, i);
7388 if (file)
7389 fput(file);
7390 }
7391#endif
Pavel Begunkov042b0d82021-08-09 13:04:01 +01007392 io_free_file_tables(&ctx->file_table);
Pavel Begunkov44b31f22021-04-25 14:32:16 +01007393 io_rsrc_data_free(ctx->file_data);
Pavel Begunkovfff4db72021-04-25 14:32:15 +01007394 ctx->file_data = NULL;
7395 ctx->nr_user_files = 0;
Bijan Mottahedeh1ad555c2021-01-15 17:37:51 +00007396}
7397
Bijan Mottahedehd7954b22021-01-15 17:37:50 +00007398static int io_sqe_files_unregister(struct io_ring_ctx *ctx)
7399{
Bijan Mottahedehd7954b22021-01-15 17:37:50 +00007400 int ret;
7401
Pavel Begunkov08480402021-04-13 02:58:38 +01007402 if (!ctx->file_data)
Bijan Mottahedehd7954b22021-01-15 17:37:50 +00007403 return -ENXIO;
Pavel Begunkov08480402021-04-13 02:58:38 +01007404 ret = io_rsrc_ref_quiesce(ctx->file_data, ctx);
7405 if (!ret)
7406 __io_sqe_files_unregister(ctx);
7407 return ret;
Jens Axboe6b063142019-01-10 22:13:58 -07007408}
7409
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007410static void io_sq_thread_unpark(struct io_sq_data *sqd)
Pavel Begunkov09a6f4e2021-03-14 20:57:10 +00007411 __releases(&sqd->lock)
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007412{
Pavel Begunkov521d6a72021-03-11 23:29:38 +00007413 WARN_ON_ONCE(sqd->thread == current);
7414
Pavel Begunkov9e138a42021-03-14 20:57:12 +00007415 /*
7416 * Do the dance but not conditional clear_bit() because it'd race with
7417 * other threads incrementing park_pending and setting the bit.
7418 */
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007419 clear_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state);
Pavel Begunkov9e138a42021-03-14 20:57:12 +00007420 if (atomic_dec_return(&sqd->park_pending))
7421 set_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state);
Pavel Begunkov09a6f4e2021-03-14 20:57:10 +00007422 mutex_unlock(&sqd->lock);
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007423}
7424
Jens Axboe86e0d672021-03-05 08:44:39 -07007425static void io_sq_thread_park(struct io_sq_data *sqd)
Pavel Begunkov09a6f4e2021-03-14 20:57:10 +00007426 __acquires(&sqd->lock)
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007427{
Pavel Begunkov521d6a72021-03-11 23:29:38 +00007428 WARN_ON_ONCE(sqd->thread == current);
7429
Pavel Begunkov9e138a42021-03-14 20:57:12 +00007430 atomic_inc(&sqd->park_pending);
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007431 set_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state);
Pavel Begunkov09a6f4e2021-03-14 20:57:10 +00007432 mutex_lock(&sqd->lock);
Jens Axboe05962f92021-03-06 13:58:48 -07007433 if (sqd->thread)
Jens Axboe86e0d672021-03-05 08:44:39 -07007434 wake_up_process(sqd->thread);
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007435}
7436
7437static void io_sq_thread_stop(struct io_sq_data *sqd)
7438{
Pavel Begunkov521d6a72021-03-11 23:29:38 +00007439 WARN_ON_ONCE(sqd->thread == current);
Pavel Begunkov88885f62021-04-11 01:46:38 +01007440 WARN_ON_ONCE(test_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state));
Pavel Begunkov521d6a72021-03-11 23:29:38 +00007441
Jens Axboe05962f92021-03-06 13:58:48 -07007442 set_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state);
Pavel Begunkov88885f62021-04-11 01:46:38 +01007443 mutex_lock(&sqd->lock);
Jens Axboee8f98f242021-03-09 16:32:13 -07007444 if (sqd->thread)
7445 wake_up_process(sqd->thread);
Pavel Begunkov09a6f4e2021-03-14 20:57:10 +00007446 mutex_unlock(&sqd->lock);
Jens Axboe05962f92021-03-06 13:58:48 -07007447 wait_for_completion(&sqd->exited);
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007448}
7449
Jens Axboe534ca6d2020-09-02 13:52:19 -06007450static void io_put_sq_data(struct io_sq_data *sqd)
Jens Axboe6c271ce2019-01-10 11:22:30 -07007451{
Jens Axboe534ca6d2020-09-02 13:52:19 -06007452 if (refcount_dec_and_test(&sqd->refs)) {
Pavel Begunkov9e138a42021-03-14 20:57:12 +00007453 WARN_ON_ONCE(atomic_read(&sqd->park_pending));
7454
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007455 io_sq_thread_stop(sqd);
7456 kfree(sqd);
7457 }
7458}
7459
7460static void io_sq_thread_finish(struct io_ring_ctx *ctx)
7461{
7462 struct io_sq_data *sqd = ctx->sq_data;
7463
7464 if (sqd) {
Jens Axboe05962f92021-03-06 13:58:48 -07007465 io_sq_thread_park(sqd);
Pavel Begunkov521d6a72021-03-11 23:29:38 +00007466 list_del_init(&ctx->sqd_list);
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007467 io_sqd_update_thread_idle(sqd);
Jens Axboe05962f92021-03-06 13:58:48 -07007468 io_sq_thread_unpark(sqd);
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007469
7470 io_put_sq_data(sqd);
7471 ctx->sq_data = NULL;
Jens Axboe534ca6d2020-09-02 13:52:19 -06007472 }
7473}
7474
Jens Axboeaa061652020-09-02 14:50:27 -06007475static struct io_sq_data *io_attach_sq_data(struct io_uring_params *p)
7476{
7477 struct io_ring_ctx *ctx_attach;
7478 struct io_sq_data *sqd;
7479 struct fd f;
7480
7481 f = fdget(p->wq_fd);
7482 if (!f.file)
7483 return ERR_PTR(-ENXIO);
7484 if (f.file->f_op != &io_uring_fops) {
7485 fdput(f);
7486 return ERR_PTR(-EINVAL);
7487 }
7488
7489 ctx_attach = f.file->private_data;
7490 sqd = ctx_attach->sq_data;
7491 if (!sqd) {
7492 fdput(f);
7493 return ERR_PTR(-EINVAL);
7494 }
Jens Axboe5c2469e2021-03-11 10:17:56 -07007495 if (sqd->task_tgid != current->tgid) {
7496 fdput(f);
7497 return ERR_PTR(-EPERM);
7498 }
Jens Axboeaa061652020-09-02 14:50:27 -06007499
7500 refcount_inc(&sqd->refs);
7501 fdput(f);
7502 return sqd;
7503}
7504
Pavel Begunkov26984fb2021-03-11 23:29:37 +00007505static struct io_sq_data *io_get_sq_data(struct io_uring_params *p,
7506 bool *attached)
Jens Axboe534ca6d2020-09-02 13:52:19 -06007507{
7508 struct io_sq_data *sqd;
7509
Pavel Begunkov26984fb2021-03-11 23:29:37 +00007510 *attached = false;
Jens Axboe5c2469e2021-03-11 10:17:56 -07007511 if (p->flags & IORING_SETUP_ATTACH_WQ) {
7512 sqd = io_attach_sq_data(p);
Pavel Begunkov26984fb2021-03-11 23:29:37 +00007513 if (!IS_ERR(sqd)) {
7514 *attached = true;
Jens Axboe5c2469e2021-03-11 10:17:56 -07007515 return sqd;
Pavel Begunkov26984fb2021-03-11 23:29:37 +00007516 }
Jens Axboe5c2469e2021-03-11 10:17:56 -07007517 /* fall through for EPERM case, setup new sqd/task */
7518 if (PTR_ERR(sqd) != -EPERM)
7519 return sqd;
7520 }
Jens Axboeaa061652020-09-02 14:50:27 -06007521
Jens Axboe534ca6d2020-09-02 13:52:19 -06007522 sqd = kzalloc(sizeof(*sqd), GFP_KERNEL);
7523 if (!sqd)
7524 return ERR_PTR(-ENOMEM);
7525
Pavel Begunkov9e138a42021-03-14 20:57:12 +00007526 atomic_set(&sqd->park_pending, 0);
Jens Axboe534ca6d2020-09-02 13:52:19 -06007527 refcount_set(&sqd->refs, 1);
Jens Axboe69fb2132020-09-14 11:16:23 -06007528 INIT_LIST_HEAD(&sqd->ctx_list);
Pavel Begunkov09a6f4e2021-03-14 20:57:10 +00007529 mutex_init(&sqd->lock);
Jens Axboe534ca6d2020-09-02 13:52:19 -06007530 init_waitqueue_head(&sqd->wait);
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007531 init_completion(&sqd->exited);
Jens Axboe534ca6d2020-09-02 13:52:19 -06007532 return sqd;
7533}
7534
Jens Axboe6b063142019-01-10 22:13:58 -07007535#if defined(CONFIG_UNIX)
Jens Axboe6b063142019-01-10 22:13:58 -07007536/*
7537 * Ensure the UNIX gc is aware of our file set, so we are certain that
7538 * the io_uring can be safely unregistered on process exit, even if we have
7539 * loops in the file referencing.
7540 */
7541static int __io_sqe_files_scm(struct io_ring_ctx *ctx, int nr, int offset)
7542{
7543 struct sock *sk = ctx->ring_sock->sk;
7544 struct scm_fp_list *fpl;
7545 struct sk_buff *skb;
Jens Axboe08a45172019-10-03 08:11:03 -06007546 int i, nr_files;
Jens Axboe6b063142019-01-10 22:13:58 -07007547
Jens Axboe6b063142019-01-10 22:13:58 -07007548 fpl = kzalloc(sizeof(*fpl), GFP_KERNEL);
7549 if (!fpl)
7550 return -ENOMEM;
7551
7552 skb = alloc_skb(0, GFP_KERNEL);
7553 if (!skb) {
7554 kfree(fpl);
7555 return -ENOMEM;
7556 }
7557
7558 skb->sk = sk;
Jens Axboe6b063142019-01-10 22:13:58 -07007559
Jens Axboe08a45172019-10-03 08:11:03 -06007560 nr_files = 0;
Jens Axboe62e398b2021-02-21 16:19:37 -07007561 fpl->user = get_uid(current_user());
Jens Axboe6b063142019-01-10 22:13:58 -07007562 for (i = 0; i < nr; i++) {
Jens Axboe65e19f52019-10-26 07:20:21 -06007563 struct file *file = io_file_from_index(ctx, i + offset);
7564
7565 if (!file)
Jens Axboe08a45172019-10-03 08:11:03 -06007566 continue;
Jens Axboe65e19f52019-10-26 07:20:21 -06007567 fpl->fp[nr_files] = get_file(file);
Jens Axboe08a45172019-10-03 08:11:03 -06007568 unix_inflight(fpl->user, fpl->fp[nr_files]);
7569 nr_files++;
Jens Axboe6b063142019-01-10 22:13:58 -07007570 }
7571
Jens Axboe08a45172019-10-03 08:11:03 -06007572 if (nr_files) {
7573 fpl->max = SCM_MAX_FD;
7574 fpl->count = nr_files;
7575 UNIXCB(skb).fp = fpl;
Jens Axboe05f3fb32019-12-09 11:22:50 -07007576 skb->destructor = unix_destruct_scm;
Jens Axboe08a45172019-10-03 08:11:03 -06007577 refcount_add(skb->truesize, &sk->sk_wmem_alloc);
7578 skb_queue_head(&sk->sk_receive_queue, skb);
Jens Axboe6b063142019-01-10 22:13:58 -07007579
Jens Axboe08a45172019-10-03 08:11:03 -06007580 for (i = 0; i < nr_files; i++)
7581 fput(fpl->fp[i]);
7582 } else {
7583 kfree_skb(skb);
7584 kfree(fpl);
7585 }
Jens Axboe6b063142019-01-10 22:13:58 -07007586
7587 return 0;
7588}
7589
7590/*
7591 * If UNIX sockets are enabled, fd passing can cause a reference cycle which
7592 * causes regular reference counting to break down. We rely on the UNIX
7593 * garbage collection to take care of this problem for us.
7594 */
7595static int io_sqe_files_scm(struct io_ring_ctx *ctx)
7596{
7597 unsigned left, total;
7598 int ret = 0;
7599
7600 total = 0;
7601 left = ctx->nr_user_files;
7602 while (left) {
7603 unsigned this_files = min_t(unsigned, left, SCM_MAX_FD);
Jens Axboe6b063142019-01-10 22:13:58 -07007604
7605 ret = __io_sqe_files_scm(ctx, this_files, total);
7606 if (ret)
7607 break;
7608 left -= this_files;
7609 total += this_files;
7610 }
7611
7612 if (!ret)
7613 return 0;
7614
7615 while (total < ctx->nr_user_files) {
Jens Axboe65e19f52019-10-26 07:20:21 -06007616 struct file *file = io_file_from_index(ctx, total);
7617
7618 if (file)
7619 fput(file);
Jens Axboe6b063142019-01-10 22:13:58 -07007620 total++;
7621 }
7622
7623 return ret;
7624}
7625#else
7626static int io_sqe_files_scm(struct io_ring_ctx *ctx)
7627{
7628 return 0;
7629}
7630#endif
7631
Pavel Begunkov47e90392021-04-01 15:43:56 +01007632static void io_rsrc_file_put(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc)
Jens Axboec3a31e62019-10-03 13:59:56 -06007633{
Bijan Mottahedeh50238532021-01-15 17:37:45 +00007634 struct file *file = prsrc->file;
Jens Axboec3a31e62019-10-03 13:59:56 -06007635#if defined(CONFIG_UNIX)
Jens Axboec3a31e62019-10-03 13:59:56 -06007636 struct sock *sock = ctx->ring_sock->sk;
7637 struct sk_buff_head list, *head = &sock->sk_receive_queue;
7638 struct sk_buff *skb;
7639 int i;
7640
7641 __skb_queue_head_init(&list);
7642
7643 /*
7644 * Find the skb that holds this file in its SCM_RIGHTS. When found,
7645 * remove this entry and rearrange the file array.
7646 */
7647 skb = skb_dequeue(head);
7648 while (skb) {
7649 struct scm_fp_list *fp;
7650
7651 fp = UNIXCB(skb).fp;
7652 for (i = 0; i < fp->count; i++) {
7653 int left;
7654
7655 if (fp->fp[i] != file)
7656 continue;
7657
7658 unix_notinflight(fp->user, fp->fp[i]);
7659 left = fp->count - 1 - i;
7660 if (left) {
7661 memmove(&fp->fp[i], &fp->fp[i + 1],
7662 left * sizeof(struct file *));
7663 }
7664 fp->count--;
7665 if (!fp->count) {
7666 kfree_skb(skb);
7667 skb = NULL;
7668 } else {
7669 __skb_queue_tail(&list, skb);
7670 }
7671 fput(file);
7672 file = NULL;
7673 break;
7674 }
7675
7676 if (!file)
7677 break;
7678
7679 __skb_queue_tail(&list, skb);
7680
7681 skb = skb_dequeue(head);
7682 }
7683
7684 if (skb_peek(&list)) {
7685 spin_lock_irq(&head->lock);
7686 while ((skb = __skb_dequeue(&list)) != NULL)
7687 __skb_queue_tail(head, skb);
7688 spin_unlock_irq(&head->lock);
7689 }
7690#else
Jens Axboe05f3fb32019-12-09 11:22:50 -07007691 fput(file);
Jens Axboec3a31e62019-10-03 13:59:56 -06007692#endif
7693}
7694
Pavel Begunkovb895c9a2021-04-01 15:43:40 +01007695static void __io_rsrc_put_work(struct io_rsrc_node *ref_node)
Jens Axboe05f3fb32019-12-09 11:22:50 -07007696{
Pavel Begunkovb895c9a2021-04-01 15:43:40 +01007697 struct io_rsrc_data *rsrc_data = ref_node->rsrc_data;
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00007698 struct io_ring_ctx *ctx = rsrc_data->ctx;
7699 struct io_rsrc_put *prsrc, *tmp;
Xiaoguang Wang05589552020-03-31 14:05:18 +08007700
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00007701 list_for_each_entry_safe(prsrc, tmp, &ref_node->rsrc_list, list) {
7702 list_del(&prsrc->list);
Pavel Begunkovb60c8dc2021-04-25 14:32:18 +01007703
7704 if (prsrc->tag) {
7705 bool lock_ring = ctx->flags & IORING_SETUP_IOPOLL;
Pavel Begunkovb60c8dc2021-04-25 14:32:18 +01007706
7707 io_ring_submit_lock(ctx, lock_ring);
Jens Axboe79ebeae2021-08-10 15:18:27 -06007708 spin_lock(&ctx->completion_lock);
Pavel Begunkovb60c8dc2021-04-25 14:32:18 +01007709 io_cqring_fill_event(ctx, prsrc->tag, 0, 0);
Pavel Begunkov2840f712021-04-27 16:13:51 +01007710 ctx->cq_extra++;
Pavel Begunkovb60c8dc2021-04-25 14:32:18 +01007711 io_commit_cqring(ctx);
Jens Axboe79ebeae2021-08-10 15:18:27 -06007712 spin_unlock(&ctx->completion_lock);
Pavel Begunkovb60c8dc2021-04-25 14:32:18 +01007713 io_cqring_ev_posted(ctx);
7714 io_ring_submit_unlock(ctx, lock_ring);
7715 }
7716
Pavel Begunkov40ae0ff2021-04-01 15:43:44 +01007717 rsrc_data->do_put(ctx, prsrc);
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00007718 kfree(prsrc);
Jens Axboe05f3fb32019-12-09 11:22:50 -07007719 }
7720
Pavel Begunkov28a9fe22021-04-01 15:43:47 +01007721 io_rsrc_node_destroy(ref_node);
Pavel Begunkov3e942492021-04-11 01:46:34 +01007722 if (atomic_dec_and_test(&rsrc_data->refs))
7723 complete(&rsrc_data->done);
Jens Axboe05f3fb32019-12-09 11:22:50 -07007724}
7725
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00007726static void io_rsrc_put_work(struct work_struct *work)
Jens Axboe4a38aed22020-05-14 17:21:15 -06007727{
7728 struct io_ring_ctx *ctx;
7729 struct llist_node *node;
7730
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00007731 ctx = container_of(work, struct io_ring_ctx, rsrc_put_work.work);
7732 node = llist_del_all(&ctx->rsrc_put_llist);
Jens Axboe4a38aed22020-05-14 17:21:15 -06007733
7734 while (node) {
Pavel Begunkovb895c9a2021-04-01 15:43:40 +01007735 struct io_rsrc_node *ref_node;
Jens Axboe4a38aed22020-05-14 17:21:15 -06007736 struct llist_node *next = node->next;
7737
Pavel Begunkovb895c9a2021-04-01 15:43:40 +01007738 ref_node = llist_entry(node, struct io_rsrc_node, llist);
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00007739 __io_rsrc_put_work(ref_node);
Jens Axboe4a38aed22020-05-14 17:21:15 -06007740 node = next;
7741 }
7742}
7743
Jens Axboe05f3fb32019-12-09 11:22:50 -07007744static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg,
Pavel Begunkov792e3582021-04-25 14:32:21 +01007745 unsigned nr_args, u64 __user *tags)
Jens Axboe05f3fb32019-12-09 11:22:50 -07007746{
7747 __s32 __user *fds = (__s32 __user *) arg;
Jens Axboe05f3fb32019-12-09 11:22:50 -07007748 struct file *file;
Pavel Begunkovf3baed32021-04-01 15:43:42 +01007749 int fd, ret;
Pavel Begunkov846a4ef2021-04-01 15:44:03 +01007750 unsigned i;
Jens Axboe05f3fb32019-12-09 11:22:50 -07007751
7752 if (ctx->file_data)
7753 return -EBUSY;
7754 if (!nr_args)
7755 return -EINVAL;
7756 if (nr_args > IORING_MAX_FIXED_FILES)
7757 return -EMFILE;
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01007758 ret = io_rsrc_node_switch_start(ctx);
Pavel Begunkovf3baed32021-04-01 15:43:42 +01007759 if (ret)
7760 return ret;
Pavel Begunkovd878c812021-06-14 02:36:18 +01007761 ret = io_rsrc_data_alloc(ctx, io_rsrc_file_put, tags, nr_args,
7762 &ctx->file_data);
7763 if (ret)
7764 return ret;
Jens Axboe05f3fb32019-12-09 11:22:50 -07007765
Pavel Begunkovf3baed32021-04-01 15:43:42 +01007766 ret = -ENOMEM;
Pavel Begunkovaeca2412021-04-11 01:46:37 +01007767 if (!io_alloc_file_tables(&ctx->file_table, nr_args))
Pavel Begunkov600cf3f2020-10-10 18:34:15 +01007768 goto out_free;
Jens Axboe05f3fb32019-12-09 11:22:50 -07007769
Jens Axboe05f3fb32019-12-09 11:22:50 -07007770 for (i = 0; i < nr_args; i++, ctx->nr_user_files++) {
Pavel Begunkovd878c812021-06-14 02:36:18 +01007771 if (copy_from_user(&fd, &fds[i], sizeof(fd))) {
Pavel Begunkov600cf3f2020-10-10 18:34:15 +01007772 ret = -EFAULT;
7773 goto out_fput;
Jens Axboe05f3fb32019-12-09 11:22:50 -07007774 }
Pavel Begunkov600cf3f2020-10-10 18:34:15 +01007775 /* allow sparse sets */
Pavel Begunkov792e3582021-04-25 14:32:21 +01007776 if (fd == -1) {
7777 ret = -EINVAL;
Pavel Begunkov2d091d62021-06-14 02:36:21 +01007778 if (unlikely(*io_get_tag_slot(ctx->file_data, i)))
Pavel Begunkov792e3582021-04-25 14:32:21 +01007779 goto out_fput;
Pavel Begunkov600cf3f2020-10-10 18:34:15 +01007780 continue;
Pavel Begunkov792e3582021-04-25 14:32:21 +01007781 }
Jens Axboe05f3fb32019-12-09 11:22:50 -07007782
Jens Axboe05f3fb32019-12-09 11:22:50 -07007783 file = fget(fd);
Jens Axboe05f3fb32019-12-09 11:22:50 -07007784 ret = -EBADF;
Pavel Begunkov792e3582021-04-25 14:32:21 +01007785 if (unlikely(!file))
Pavel Begunkov600cf3f2020-10-10 18:34:15 +01007786 goto out_fput;
Jens Axboe05f3fb32019-12-09 11:22:50 -07007787
7788 /*
7789 * Don't allow io_uring instances to be registered. If UNIX
7790 * isn't enabled, then this causes a reference cycle and this
7791 * instance can never get freed. If UNIX is enabled we'll
7792 * handle it just fine, but there's still no point in allowing
7793 * a ring fd as it doesn't support regular read/write anyway.
7794 */
7795 if (file->f_op == &io_uring_fops) {
7796 fput(file);
Pavel Begunkov600cf3f2020-10-10 18:34:15 +01007797 goto out_fput;
Jens Axboe05f3fb32019-12-09 11:22:50 -07007798 }
Pavel Begunkovaeca2412021-04-11 01:46:37 +01007799 io_fixed_file_set(io_fixed_file_slot(&ctx->file_table, i), file);
Jens Axboe05f3fb32019-12-09 11:22:50 -07007800 }
7801
Jens Axboe05f3fb32019-12-09 11:22:50 -07007802 ret = io_sqe_files_scm(ctx);
Xiaoguang Wang05589552020-03-31 14:05:18 +08007803 if (ret) {
Pavel Begunkov08480402021-04-13 02:58:38 +01007804 __io_sqe_files_unregister(ctx);
Xiaoguang Wang05589552020-03-31 14:05:18 +08007805 return ret;
7806 }
Jens Axboe05f3fb32019-12-09 11:22:50 -07007807
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01007808 io_rsrc_node_switch(ctx, NULL);
Jens Axboe05f3fb32019-12-09 11:22:50 -07007809 return ret;
Pavel Begunkov600cf3f2020-10-10 18:34:15 +01007810out_fput:
7811 for (i = 0; i < ctx->nr_user_files; i++) {
7812 file = io_file_from_index(ctx, i);
7813 if (file)
7814 fput(file);
7815 }
Pavel Begunkov042b0d82021-08-09 13:04:01 +01007816 io_free_file_tables(&ctx->file_table);
Pavel Begunkov600cf3f2020-10-10 18:34:15 +01007817 ctx->nr_user_files = 0;
Pavel Begunkov600cf3f2020-10-10 18:34:15 +01007818out_free:
Pavel Begunkov44b31f22021-04-25 14:32:16 +01007819 io_rsrc_data_free(ctx->file_data);
Jens Axboe55cbc252020-10-14 07:35:57 -06007820 ctx->file_data = NULL;
Jens Axboe05f3fb32019-12-09 11:22:50 -07007821 return ret;
7822}
7823
Jens Axboec3a31e62019-10-03 13:59:56 -06007824static int io_sqe_file_register(struct io_ring_ctx *ctx, struct file *file,
7825 int index)
7826{
7827#if defined(CONFIG_UNIX)
7828 struct sock *sock = ctx->ring_sock->sk;
7829 struct sk_buff_head *head = &sock->sk_receive_queue;
7830 struct sk_buff *skb;
7831
7832 /*
7833 * See if we can merge this file into an existing skb SCM_RIGHTS
7834 * file set. If there's no room, fall back to allocating a new skb
7835 * and filling it in.
7836 */
7837 spin_lock_irq(&head->lock);
7838 skb = skb_peek(head);
7839 if (skb) {
7840 struct scm_fp_list *fpl = UNIXCB(skb).fp;
7841
7842 if (fpl->count < SCM_MAX_FD) {
7843 __skb_unlink(skb, head);
7844 spin_unlock_irq(&head->lock);
7845 fpl->fp[fpl->count] = get_file(file);
7846 unix_inflight(fpl->user, fpl->fp[fpl->count]);
7847 fpl->count++;
7848 spin_lock_irq(&head->lock);
7849 __skb_queue_head(head, skb);
7850 } else {
7851 skb = NULL;
7852 }
7853 }
7854 spin_unlock_irq(&head->lock);
7855
7856 if (skb) {
7857 fput(file);
7858 return 0;
7859 }
7860
7861 return __io_sqe_files_scm(ctx, 1, index);
7862#else
7863 return 0;
7864#endif
7865}
7866
Pavel Begunkovb60c8dc2021-04-25 14:32:18 +01007867static int io_queue_rsrc_removal(struct io_rsrc_data *data, unsigned idx,
Pavel Begunkove7c78372021-04-01 15:43:45 +01007868 struct io_rsrc_node *node, void *rsrc)
Jens Axboe05f3fb32019-12-09 11:22:50 -07007869{
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00007870 struct io_rsrc_put *prsrc;
Jens Axboe05f3fb32019-12-09 11:22:50 -07007871
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00007872 prsrc = kzalloc(sizeof(*prsrc), GFP_KERNEL);
7873 if (!prsrc)
Hillf Dantona5318d32020-03-23 17:47:15 +08007874 return -ENOMEM;
Jens Axboe05f3fb32019-12-09 11:22:50 -07007875
Pavel Begunkov2d091d62021-06-14 02:36:21 +01007876 prsrc->tag = *io_get_tag_slot(data, idx);
Bijan Mottahedeh50238532021-01-15 17:37:45 +00007877 prsrc->rsrc = rsrc;
Pavel Begunkove7c78372021-04-01 15:43:45 +01007878 list_add(&prsrc->list, &node->rsrc_list);
Hillf Dantona5318d32020-03-23 17:47:15 +08007879 return 0;
Jens Axboe05f3fb32019-12-09 11:22:50 -07007880}
7881
7882static int __io_sqe_files_update(struct io_ring_ctx *ctx,
Pavel Begunkovc3bdad02021-04-25 14:32:22 +01007883 struct io_uring_rsrc_update2 *up,
Jens Axboe05f3fb32019-12-09 11:22:50 -07007884 unsigned nr_args)
7885{
Pavel Begunkovc3bdad02021-04-25 14:32:22 +01007886 u64 __user *tags = u64_to_user_ptr(up->tags);
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +01007887 __s32 __user *fds = u64_to_user_ptr(up->data);
Pavel Begunkovb895c9a2021-04-01 15:43:40 +01007888 struct io_rsrc_data *data = ctx->file_data;
Pavel Begunkova04b0ac2021-04-01 15:44:04 +01007889 struct io_fixed_file *file_slot;
7890 struct file *file;
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +01007891 int fd, i, err = 0;
7892 unsigned int done;
Xiaoguang Wang05589552020-03-31 14:05:18 +08007893 bool needs_switch = false;
Jens Axboec3a31e62019-10-03 13:59:56 -06007894
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +01007895 if (!ctx->file_data)
7896 return -ENXIO;
7897 if (up->offset + nr_args > ctx->nr_user_files)
Jens Axboec3a31e62019-10-03 13:59:56 -06007898 return -EINVAL;
7899
Pavel Begunkov67973b92021-01-26 13:51:09 +00007900 for (done = 0; done < nr_args; done++) {
Pavel Begunkovc3bdad02021-04-25 14:32:22 +01007901 u64 tag = 0;
7902
7903 if ((tags && copy_from_user(&tag, &tags[done], sizeof(tag))) ||
7904 copy_from_user(&fd, &fds[done], sizeof(fd))) {
Jens Axboec3a31e62019-10-03 13:59:56 -06007905 err = -EFAULT;
7906 break;
7907 }
Pavel Begunkovc3bdad02021-04-25 14:32:22 +01007908 if ((fd == IORING_REGISTER_FILES_SKIP || fd == -1) && tag) {
7909 err = -EINVAL;
7910 break;
7911 }
noah4e0377a2021-01-26 15:23:28 -05007912 if (fd == IORING_REGISTER_FILES_SKIP)
7913 continue;
7914
Pavel Begunkov67973b92021-01-26 13:51:09 +00007915 i = array_index_nospec(up->offset + done, ctx->nr_user_files);
Pavel Begunkovaeca2412021-04-11 01:46:37 +01007916 file_slot = io_fixed_file_slot(&ctx->file_table, i);
Pavel Begunkovea64ec022021-02-04 13:52:07 +00007917
Pavel Begunkova04b0ac2021-04-01 15:44:04 +01007918 if (file_slot->file_ptr) {
7919 file = (struct file *)(file_slot->file_ptr & FFS_MASK);
Pavel Begunkovb60c8dc2021-04-25 14:32:18 +01007920 err = io_queue_rsrc_removal(data, up->offset + done,
7921 ctx->rsrc_node, file);
Hillf Dantona5318d32020-03-23 17:47:15 +08007922 if (err)
7923 break;
Pavel Begunkova04b0ac2021-04-01 15:44:04 +01007924 file_slot->file_ptr = 0;
Xiaoguang Wang05589552020-03-31 14:05:18 +08007925 needs_switch = true;
Jens Axboec3a31e62019-10-03 13:59:56 -06007926 }
7927 if (fd != -1) {
Jens Axboec3a31e62019-10-03 13:59:56 -06007928 file = fget(fd);
7929 if (!file) {
7930 err = -EBADF;
7931 break;
7932 }
7933 /*
7934 * Don't allow io_uring instances to be registered. If
7935 * UNIX isn't enabled, then this causes a reference
7936 * cycle and this instance can never get freed. If UNIX
7937 * is enabled we'll handle it just fine, but there's
7938 * still no point in allowing a ring fd as it doesn't
7939 * support regular read/write anyway.
7940 */
7941 if (file->f_op == &io_uring_fops) {
7942 fput(file);
7943 err = -EBADF;
7944 break;
7945 }
Pavel Begunkov2d091d62021-06-14 02:36:21 +01007946 *io_get_tag_slot(data, up->offset + done) = tag;
Pavel Begunkov9a321c92021-04-01 15:44:01 +01007947 io_fixed_file_set(file_slot, file);
Jens Axboec3a31e62019-10-03 13:59:56 -06007948 err = io_sqe_file_register(ctx, file, i);
Yang Yingliangf3bd9da2020-07-09 10:11:41 +00007949 if (err) {
Pavel Begunkova04b0ac2021-04-01 15:44:04 +01007950 file_slot->file_ptr = 0;
Yang Yingliangf3bd9da2020-07-09 10:11:41 +00007951 fput(file);
Jens Axboec3a31e62019-10-03 13:59:56 -06007952 break;
Yang Yingliangf3bd9da2020-07-09 10:11:41 +00007953 }
Jens Axboec3a31e62019-10-03 13:59:56 -06007954 }
Jens Axboe05f3fb32019-12-09 11:22:50 -07007955 }
7956
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01007957 if (needs_switch)
7958 io_rsrc_node_switch(ctx, data);
Jens Axboec3a31e62019-10-03 13:59:56 -06007959 return done ? done : err;
7960}
Xiaoguang Wang05589552020-03-31 14:05:18 +08007961
Jens Axboe685fe7f2021-03-08 09:37:51 -07007962static struct io_wq *io_init_wq_offload(struct io_ring_ctx *ctx,
7963 struct task_struct *task)
Pavel Begunkov24369c22020-01-28 03:15:48 +03007964{
Jens Axboee9418942021-02-19 12:33:30 -07007965 struct io_wq_hash *hash;
Pavel Begunkov24369c22020-01-28 03:15:48 +03007966 struct io_wq_data data;
Pavel Begunkov24369c22020-01-28 03:15:48 +03007967 unsigned int concurrency;
Pavel Begunkov24369c22020-01-28 03:15:48 +03007968
Yang Yingliang362a9e62021-07-20 16:38:05 +08007969 mutex_lock(&ctx->uring_lock);
Jens Axboee9418942021-02-19 12:33:30 -07007970 hash = ctx->hash_map;
7971 if (!hash) {
7972 hash = kzalloc(sizeof(*hash), GFP_KERNEL);
Yang Yingliang362a9e62021-07-20 16:38:05 +08007973 if (!hash) {
7974 mutex_unlock(&ctx->uring_lock);
Jens Axboee9418942021-02-19 12:33:30 -07007975 return ERR_PTR(-ENOMEM);
Yang Yingliang362a9e62021-07-20 16:38:05 +08007976 }
Jens Axboee9418942021-02-19 12:33:30 -07007977 refcount_set(&hash->refs, 1);
7978 init_waitqueue_head(&hash->wait);
7979 ctx->hash_map = hash;
7980 }
Yang Yingliang362a9e62021-07-20 16:38:05 +08007981 mutex_unlock(&ctx->uring_lock);
Jens Axboee9418942021-02-19 12:33:30 -07007982
7983 data.hash = hash;
Jens Axboe685fe7f2021-03-08 09:37:51 -07007984 data.task = task;
Pavel Begunkovebc11b62021-08-09 13:04:05 +01007985 data.free_work = io_wq_free_work;
Pavel Begunkovf5fa38c2020-06-08 21:08:20 +03007986 data.do_work = io_wq_submit_work;
Pavel Begunkov24369c22020-01-28 03:15:48 +03007987
Jens Axboed25e3a32021-02-16 11:41:41 -07007988 /* Do QD, or 4 * CPUS, whatever is smallest */
7989 concurrency = min(ctx->sq_entries, 4 * num_online_cpus());
Pavel Begunkov24369c22020-01-28 03:15:48 +03007990
Jens Axboe5aa75ed2021-02-16 12:56:50 -07007991 return io_wq_create(concurrency, &data);
Pavel Begunkov24369c22020-01-28 03:15:48 +03007992}
7993
Jens Axboe5aa75ed2021-02-16 12:56:50 -07007994static int io_uring_alloc_task_context(struct task_struct *task,
7995 struct io_ring_ctx *ctx)
Jens Axboe0f212202020-09-13 13:09:39 -06007996{
7997 struct io_uring_task *tctx;
Jens Axboed8a6df12020-10-15 16:24:45 -06007998 int ret;
Jens Axboe0f212202020-09-13 13:09:39 -06007999
Pavel Begunkov09899b12021-06-14 02:36:22 +01008000 tctx = kzalloc(sizeof(*tctx), GFP_KERNEL);
Jens Axboe0f212202020-09-13 13:09:39 -06008001 if (unlikely(!tctx))
8002 return -ENOMEM;
8003
Jens Axboed8a6df12020-10-15 16:24:45 -06008004 ret = percpu_counter_init(&tctx->inflight, 0, GFP_KERNEL);
8005 if (unlikely(ret)) {
8006 kfree(tctx);
8007 return ret;
8008 }
8009
Jens Axboe685fe7f2021-03-08 09:37:51 -07008010 tctx->io_wq = io_init_wq_offload(ctx, task);
Jens Axboe5aa75ed2021-02-16 12:56:50 -07008011 if (IS_ERR(tctx->io_wq)) {
8012 ret = PTR_ERR(tctx->io_wq);
8013 percpu_counter_destroy(&tctx->inflight);
8014 kfree(tctx);
8015 return ret;
8016 }
8017
Jens Axboe0f212202020-09-13 13:09:39 -06008018 xa_init(&tctx->xa);
8019 init_waitqueue_head(&tctx->wait);
Jens Axboefdaf0832020-10-30 09:37:30 -06008020 atomic_set(&tctx->in_idle, 0);
Pavel Begunkovb303fe22021-04-11 01:46:26 +01008021 atomic_set(&tctx->inflight_tracked, 0);
Jens Axboe0f212202020-09-13 13:09:39 -06008022 task->io_uring = tctx;
Jens Axboe7cbf1722021-02-10 00:03:20 +00008023 spin_lock_init(&tctx->task_lock);
8024 INIT_WQ_LIST(&tctx->task_list);
Jens Axboe7cbf1722021-02-10 00:03:20 +00008025 init_task_work(&tctx->task_work, tctx_task_work);
Jens Axboe0f212202020-09-13 13:09:39 -06008026 return 0;
8027}
8028
8029void __io_uring_free(struct task_struct *tsk)
8030{
8031 struct io_uring_task *tctx = tsk->io_uring;
8032
8033 WARN_ON_ONCE(!xa_empty(&tctx->xa));
Pavel Begunkovef8eaa42021-02-27 11:16:45 +00008034 WARN_ON_ONCE(tctx->io_wq);
Pavel Begunkov09899b12021-06-14 02:36:22 +01008035 WARN_ON_ONCE(tctx->cached_refs);
Pavel Begunkovef8eaa42021-02-27 11:16:45 +00008036
Jens Axboed8a6df12020-10-15 16:24:45 -06008037 percpu_counter_destroy(&tctx->inflight);
Jens Axboe0f212202020-09-13 13:09:39 -06008038 kfree(tctx);
8039 tsk->io_uring = NULL;
8040}
8041
Stefano Garzarella7e84e1c2020-08-27 16:58:31 +02008042static int io_sq_offload_create(struct io_ring_ctx *ctx,
8043 struct io_uring_params *p)
Jens Axboe2b188cc2019-01-07 10:46:33 -07008044{
8045 int ret;
8046
Jens Axboed25e3a32021-02-16 11:41:41 -07008047 /* Retain compatibility with failing for an invalid attach attempt */
8048 if ((ctx->flags & (IORING_SETUP_ATTACH_WQ | IORING_SETUP_SQPOLL)) ==
8049 IORING_SETUP_ATTACH_WQ) {
8050 struct fd f;
8051
8052 f = fdget(p->wq_fd);
8053 if (!f.file)
8054 return -ENXIO;
Jens Axboe0cc936f2021-07-22 17:08:07 -06008055 if (f.file->f_op != &io_uring_fops) {
8056 fdput(f);
Pavel Begunkovf2a48dd2021-04-20 12:03:33 +01008057 return -EINVAL;
Jens Axboe0cc936f2021-07-22 17:08:07 -06008058 }
8059 fdput(f);
Jens Axboed25e3a32021-02-16 11:41:41 -07008060 }
Jens Axboe6c271ce2019-01-10 11:22:30 -07008061 if (ctx->flags & IORING_SETUP_SQPOLL) {
Jens Axboe46fe18b2021-03-04 12:39:36 -07008062 struct task_struct *tsk;
Jens Axboe534ca6d2020-09-02 13:52:19 -06008063 struct io_sq_data *sqd;
Pavel Begunkov26984fb2021-03-11 23:29:37 +00008064 bool attached;
Jens Axboe534ca6d2020-09-02 13:52:19 -06008065
Pavel Begunkov26984fb2021-03-11 23:29:37 +00008066 sqd = io_get_sq_data(p, &attached);
Jens Axboe534ca6d2020-09-02 13:52:19 -06008067 if (IS_ERR(sqd)) {
8068 ret = PTR_ERR(sqd);
8069 goto err;
8070 }
Jens Axboe69fb2132020-09-14 11:16:23 -06008071
Stefan Metzmacher7c30f36a2021-03-07 11:54:28 +01008072 ctx->sq_creds = get_current_cred();
Jens Axboe534ca6d2020-09-02 13:52:19 -06008073 ctx->sq_data = sqd;
Jens Axboe917257d2019-04-13 09:28:55 -06008074 ctx->sq_thread_idle = msecs_to_jiffies(p->sq_thread_idle);
8075 if (!ctx->sq_thread_idle)
8076 ctx->sq_thread_idle = HZ;
8077
Pavel Begunkov78d7f6b2021-03-10 13:13:53 +00008078 io_sq_thread_park(sqd);
Pavel Begunkovde75a3d2021-03-18 11:54:35 +00008079 list_add(&ctx->sqd_list, &sqd->ctx_list);
8080 io_sqd_update_thread_idle(sqd);
Pavel Begunkov26984fb2021-03-11 23:29:37 +00008081 /* don't attach to a dying SQPOLL thread, would be racy */
Pavel Begunkovf2a48dd2021-04-20 12:03:33 +01008082 ret = (attached && !sqd->thread) ? -ENXIO : 0;
Pavel Begunkov78d7f6b2021-03-10 13:13:53 +00008083 io_sq_thread_unpark(sqd);
8084
Pavel Begunkovde75a3d2021-03-18 11:54:35 +00008085 if (ret < 0)
8086 goto err;
8087 if (attached)
Jens Axboe5aa75ed2021-02-16 12:56:50 -07008088 return 0;
Jens Axboeaa061652020-09-02 14:50:27 -06008089
Jens Axboe6c271ce2019-01-10 11:22:30 -07008090 if (p->flags & IORING_SETUP_SQ_AFF) {
Jens Axboe44a9bd12019-05-14 20:00:30 -06008091 int cpu = p->sq_thread_cpu;
Jens Axboe6c271ce2019-01-10 11:22:30 -07008092
Jens Axboe917257d2019-04-13 09:28:55 -06008093 ret = -EINVAL;
Pavel Begunkovf2a48dd2021-04-20 12:03:33 +01008094 if (cpu >= nr_cpu_ids || !cpu_online(cpu))
Jens Axboee8f98f242021-03-09 16:32:13 -07008095 goto err_sqpoll;
Jens Axboe37d1e2e2021-02-17 21:03:43 -07008096 sqd->sq_cpu = cpu;
Jens Axboe6c271ce2019-01-10 11:22:30 -07008097 } else {
Jens Axboe37d1e2e2021-02-17 21:03:43 -07008098 sqd->sq_cpu = -1;
Jens Axboe6c271ce2019-01-10 11:22:30 -07008099 }
Jens Axboe37d1e2e2021-02-17 21:03:43 -07008100
8101 sqd->task_pid = current->pid;
Jens Axboe5c2469e2021-03-11 10:17:56 -07008102 sqd->task_tgid = current->tgid;
Jens Axboe46fe18b2021-03-04 12:39:36 -07008103 tsk = create_io_thread(io_sq_thread, sqd, NUMA_NO_NODE);
8104 if (IS_ERR(tsk)) {
8105 ret = PTR_ERR(tsk);
Jens Axboee8f98f242021-03-09 16:32:13 -07008106 goto err_sqpoll;
Jens Axboe6c271ce2019-01-10 11:22:30 -07008107 }
Pavel Begunkov97a73a02021-03-08 17:30:54 +00008108
Jens Axboe46fe18b2021-03-04 12:39:36 -07008109 sqd->thread = tsk;
Pavel Begunkov97a73a02021-03-08 17:30:54 +00008110 ret = io_uring_alloc_task_context(tsk, ctx);
Jens Axboe46fe18b2021-03-04 12:39:36 -07008111 wake_up_new_task(tsk);
Jens Axboe0f212202020-09-13 13:09:39 -06008112 if (ret)
8113 goto err;
Jens Axboe6c271ce2019-01-10 11:22:30 -07008114 } else if (p->flags & IORING_SETUP_SQ_AFF) {
8115 /* Can't have SQ_AFF without SQPOLL */
8116 ret = -EINVAL;
8117 goto err;
8118 }
8119
Jens Axboe2b188cc2019-01-07 10:46:33 -07008120 return 0;
Pavel Begunkovf2a48dd2021-04-20 12:03:33 +01008121err_sqpoll:
8122 complete(&ctx->sq_data->exited);
Jens Axboe2b188cc2019-01-07 10:46:33 -07008123err:
Jens Axboe37d1e2e2021-02-17 21:03:43 -07008124 io_sq_thread_finish(ctx);
Jens Axboe2b188cc2019-01-07 10:46:33 -07008125 return ret;
8126}
8127
Bijan Mottahedeha087e2b2020-06-16 16:36:07 -07008128static inline void __io_unaccount_mem(struct user_struct *user,
8129 unsigned long nr_pages)
Jens Axboe2b188cc2019-01-07 10:46:33 -07008130{
8131 atomic_long_sub(nr_pages, &user->locked_vm);
8132}
8133
Bijan Mottahedeha087e2b2020-06-16 16:36:07 -07008134static inline int __io_account_mem(struct user_struct *user,
8135 unsigned long nr_pages)
Jens Axboe2b188cc2019-01-07 10:46:33 -07008136{
8137 unsigned long page_limit, cur_pages, new_pages;
8138
8139 /* Don't allow more pages than we can safely lock */
8140 page_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
8141
8142 do {
8143 cur_pages = atomic_long_read(&user->locked_vm);
8144 new_pages = cur_pages + nr_pages;
8145 if (new_pages > page_limit)
8146 return -ENOMEM;
8147 } while (atomic_long_cmpxchg(&user->locked_vm, cur_pages,
8148 new_pages) != cur_pages);
8149
8150 return 0;
8151}
8152
Jens Axboe26bfa89e2021-02-09 20:14:12 -07008153static void io_unaccount_mem(struct io_ring_ctx *ctx, unsigned long nr_pages)
Bijan Mottahedeha087e2b2020-06-16 16:36:07 -07008154{
Jens Axboe62e398b2021-02-21 16:19:37 -07008155 if (ctx->user)
Bijan Mottahedeha087e2b2020-06-16 16:36:07 -07008156 __io_unaccount_mem(ctx->user, nr_pages);
Bijan Mottahedeh30975822020-06-16 16:36:09 -07008157
Jens Axboe26bfa89e2021-02-09 20:14:12 -07008158 if (ctx->mm_account)
8159 atomic64_sub(nr_pages, &ctx->mm_account->pinned_vm);
Bijan Mottahedeha087e2b2020-06-16 16:36:07 -07008160}
8161
Jens Axboe26bfa89e2021-02-09 20:14:12 -07008162static int io_account_mem(struct io_ring_ctx *ctx, unsigned long nr_pages)
Bijan Mottahedeha087e2b2020-06-16 16:36:07 -07008163{
Bijan Mottahedeh30975822020-06-16 16:36:09 -07008164 int ret;
8165
Jens Axboe62e398b2021-02-21 16:19:37 -07008166 if (ctx->user) {
Bijan Mottahedeh30975822020-06-16 16:36:09 -07008167 ret = __io_account_mem(ctx->user, nr_pages);
8168 if (ret)
8169 return ret;
8170 }
8171
Jens Axboe26bfa89e2021-02-09 20:14:12 -07008172 if (ctx->mm_account)
8173 atomic64_add(nr_pages, &ctx->mm_account->pinned_vm);
Bijan Mottahedeha087e2b2020-06-16 16:36:07 -07008174
8175 return 0;
8176}
8177
Jens Axboe2b188cc2019-01-07 10:46:33 -07008178static void io_mem_free(void *ptr)
8179{
Mark Rutland52e04ef2019-04-30 17:30:21 +01008180 struct page *page;
Jens Axboe2b188cc2019-01-07 10:46:33 -07008181
Mark Rutland52e04ef2019-04-30 17:30:21 +01008182 if (!ptr)
8183 return;
8184
8185 page = virt_to_head_page(ptr);
Jens Axboe2b188cc2019-01-07 10:46:33 -07008186 if (put_page_testzero(page))
8187 free_compound_page(page);
8188}
8189
8190static void *io_mem_alloc(size_t size)
8191{
8192 gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN | __GFP_COMP |
Jens Axboe26bfa89e2021-02-09 20:14:12 -07008193 __GFP_NORETRY | __GFP_ACCOUNT;
Jens Axboe2b188cc2019-01-07 10:46:33 -07008194
8195 return (void *) __get_free_pages(gfp_flags, get_order(size));
8196}
8197
Hristo Venev75b28af2019-08-26 17:23:46 +00008198static unsigned long rings_size(unsigned sq_entries, unsigned cq_entries,
8199 size_t *sq_offset)
8200{
8201 struct io_rings *rings;
8202 size_t off, sq_array_size;
8203
8204 off = struct_size(rings, cqes, cq_entries);
8205 if (off == SIZE_MAX)
8206 return SIZE_MAX;
8207
8208#ifdef CONFIG_SMP
8209 off = ALIGN(off, SMP_CACHE_BYTES);
8210 if (off == 0)
8211 return SIZE_MAX;
8212#endif
8213
Dmitry Vyukovb36200f2020-07-11 11:31:11 +02008214 if (sq_offset)
8215 *sq_offset = off;
8216
Hristo Venev75b28af2019-08-26 17:23:46 +00008217 sq_array_size = array_size(sizeof(u32), sq_entries);
8218 if (sq_array_size == SIZE_MAX)
8219 return SIZE_MAX;
8220
8221 if (check_add_overflow(off, sq_array_size, &off))
8222 return SIZE_MAX;
8223
Hristo Venev75b28af2019-08-26 17:23:46 +00008224 return off;
8225}
8226
Pavel Begunkov41edf1a2021-04-25 14:32:23 +01008227static void io_buffer_unmap(struct io_ring_ctx *ctx, struct io_mapped_ubuf **slot)
Pavel Begunkov7f61a1e2021-04-11 01:46:35 +01008228{
Pavel Begunkov41edf1a2021-04-25 14:32:23 +01008229 struct io_mapped_ubuf *imu = *slot;
Pavel Begunkov7f61a1e2021-04-11 01:46:35 +01008230 unsigned int i;
8231
Pavel Begunkov62248432021-04-28 13:11:29 +01008232 if (imu != ctx->dummy_ubuf) {
8233 for (i = 0; i < imu->nr_bvecs; i++)
8234 unpin_user_page(imu->bvec[i].bv_page);
8235 if (imu->acct_pages)
8236 io_unaccount_mem(ctx, imu->acct_pages);
8237 kvfree(imu);
8238 }
Pavel Begunkov41edf1a2021-04-25 14:32:23 +01008239 *slot = NULL;
Pavel Begunkov7f61a1e2021-04-11 01:46:35 +01008240}
8241
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01008242static void io_rsrc_buf_put(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc)
8243{
Pavel Begunkov634d00d2021-04-25 14:32:26 +01008244 io_buffer_unmap(ctx, &prsrc->buf);
8245 prsrc->buf = NULL;
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01008246}
8247
8248static void __io_sqe_buffers_unregister(struct io_ring_ctx *ctx)
Jens Axboeedafcce2019-01-09 09:16:05 -07008249{
Pavel Begunkov7f61a1e2021-04-11 01:46:35 +01008250 unsigned int i;
Jens Axboeedafcce2019-01-09 09:16:05 -07008251
Pavel Begunkov7f61a1e2021-04-11 01:46:35 +01008252 for (i = 0; i < ctx->nr_user_bufs; i++)
8253 io_buffer_unmap(ctx, &ctx->user_bufs[i]);
Jens Axboeedafcce2019-01-09 09:16:05 -07008254 kfree(ctx->user_bufs);
Zqiangbb6659c2021-04-30 16:25:15 +08008255 io_rsrc_data_free(ctx->buf_data);
Jens Axboeedafcce2019-01-09 09:16:05 -07008256 ctx->user_bufs = NULL;
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01008257 ctx->buf_data = NULL;
Jens Axboeedafcce2019-01-09 09:16:05 -07008258 ctx->nr_user_bufs = 0;
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01008259}
8260
Jens Axboeedafcce2019-01-09 09:16:05 -07008261static int io_sqe_buffers_unregister(struct io_ring_ctx *ctx)
8262{
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01008263 int ret;
Jens Axboeedafcce2019-01-09 09:16:05 -07008264
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01008265 if (!ctx->buf_data)
Jens Axboeedafcce2019-01-09 09:16:05 -07008266 return -ENXIO;
8267
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01008268 ret = io_rsrc_ref_quiesce(ctx->buf_data, ctx);
8269 if (!ret)
8270 __io_sqe_buffers_unregister(ctx);
8271 return ret;
Jens Axboeedafcce2019-01-09 09:16:05 -07008272}
8273
8274static int io_copy_iov(struct io_ring_ctx *ctx, struct iovec *dst,
8275 void __user *arg, unsigned index)
8276{
8277 struct iovec __user *src;
8278
8279#ifdef CONFIG_COMPAT
8280 if (ctx->compat) {
8281 struct compat_iovec __user *ciovs;
8282 struct compat_iovec ciov;
8283
8284 ciovs = (struct compat_iovec __user *) arg;
8285 if (copy_from_user(&ciov, &ciovs[index], sizeof(ciov)))
8286 return -EFAULT;
8287
Jens Axboed55e5f52019-12-11 16:12:15 -07008288 dst->iov_base = u64_to_user_ptr((u64)ciov.iov_base);
Jens Axboeedafcce2019-01-09 09:16:05 -07008289 dst->iov_len = ciov.iov_len;
8290 return 0;
8291 }
8292#endif
8293 src = (struct iovec __user *) arg;
8294 if (copy_from_user(dst, &src[index], sizeof(*dst)))
8295 return -EFAULT;
8296 return 0;
8297}
8298
Jens Axboede293932020-09-17 16:19:16 -06008299/*
8300 * Not super efficient, but this is just a registration time. And we do cache
8301 * the last compound head, so generally we'll only do a full search if we don't
8302 * match that one.
8303 *
8304 * We check if the given compound head page has already been accounted, to
8305 * avoid double accounting it. This allows us to account the full size of the
8306 * page, not just the constituent pages of a huge page.
8307 */
8308static bool headpage_already_acct(struct io_ring_ctx *ctx, struct page **pages,
8309 int nr_pages, struct page *hpage)
8310{
8311 int i, j;
8312
8313 /* check current page array */
8314 for (i = 0; i < nr_pages; i++) {
8315 if (!PageCompound(pages[i]))
8316 continue;
8317 if (compound_head(pages[i]) == hpage)
8318 return true;
8319 }
8320
8321 /* check previously registered pages */
8322 for (i = 0; i < ctx->nr_user_bufs; i++) {
Pavel Begunkov41edf1a2021-04-25 14:32:23 +01008323 struct io_mapped_ubuf *imu = ctx->user_bufs[i];
Jens Axboede293932020-09-17 16:19:16 -06008324
8325 for (j = 0; j < imu->nr_bvecs; j++) {
8326 if (!PageCompound(imu->bvec[j].bv_page))
8327 continue;
8328 if (compound_head(imu->bvec[j].bv_page) == hpage)
8329 return true;
8330 }
8331 }
8332
8333 return false;
8334}
8335
8336static int io_buffer_account_pin(struct io_ring_ctx *ctx, struct page **pages,
8337 int nr_pages, struct io_mapped_ubuf *imu,
8338 struct page **last_hpage)
8339{
8340 int i, ret;
8341
Pavel Begunkov216e5832021-05-29 12:01:02 +01008342 imu->acct_pages = 0;
Jens Axboede293932020-09-17 16:19:16 -06008343 for (i = 0; i < nr_pages; i++) {
8344 if (!PageCompound(pages[i])) {
8345 imu->acct_pages++;
8346 } else {
8347 struct page *hpage;
8348
8349 hpage = compound_head(pages[i]);
8350 if (hpage == *last_hpage)
8351 continue;
8352 *last_hpage = hpage;
8353 if (headpage_already_acct(ctx, pages, i, hpage))
8354 continue;
8355 imu->acct_pages += page_size(hpage) >> PAGE_SHIFT;
8356 }
8357 }
8358
8359 if (!imu->acct_pages)
8360 return 0;
8361
Jens Axboe26bfa89e2021-02-09 20:14:12 -07008362 ret = io_account_mem(ctx, imu->acct_pages);
Jens Axboede293932020-09-17 16:19:16 -06008363 if (ret)
8364 imu->acct_pages = 0;
8365 return ret;
8366}
8367
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08008368static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov,
Pavel Begunkov41edf1a2021-04-25 14:32:23 +01008369 struct io_mapped_ubuf **pimu,
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08008370 struct page **last_hpage)
Jens Axboeedafcce2019-01-09 09:16:05 -07008371{
Pavel Begunkov41edf1a2021-04-25 14:32:23 +01008372 struct io_mapped_ubuf *imu = NULL;
Jens Axboeedafcce2019-01-09 09:16:05 -07008373 struct vm_area_struct **vmas = NULL;
8374 struct page **pages = NULL;
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08008375 unsigned long off, start, end, ubuf;
8376 size_t size;
8377 int ret, pret, nr_pages, i;
Jens Axboeedafcce2019-01-09 09:16:05 -07008378
Pavel Begunkov62248432021-04-28 13:11:29 +01008379 if (!iov->iov_base) {
8380 *pimu = ctx->dummy_ubuf;
8381 return 0;
8382 }
8383
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08008384 ubuf = (unsigned long) iov->iov_base;
8385 end = (ubuf + iov->iov_len + PAGE_SIZE - 1) >> PAGE_SHIFT;
8386 start = ubuf >> PAGE_SHIFT;
8387 nr_pages = end - start;
8388
Pavel Begunkov41edf1a2021-04-25 14:32:23 +01008389 *pimu = NULL;
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08008390 ret = -ENOMEM;
8391
8392 pages = kvmalloc_array(nr_pages, sizeof(struct page *), GFP_KERNEL);
8393 if (!pages)
8394 goto done;
8395
8396 vmas = kvmalloc_array(nr_pages, sizeof(struct vm_area_struct *),
8397 GFP_KERNEL);
8398 if (!vmas)
8399 goto done;
8400
Pavel Begunkov41edf1a2021-04-25 14:32:23 +01008401 imu = kvmalloc(struct_size(imu, bvec, nr_pages), GFP_KERNEL);
Pavel Begunkova2b41982021-04-26 00:16:31 +01008402 if (!imu)
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08008403 goto done;
8404
8405 ret = 0;
8406 mmap_read_lock(current->mm);
8407 pret = pin_user_pages(ubuf, nr_pages, FOLL_WRITE | FOLL_LONGTERM,
8408 pages, vmas);
8409 if (pret == nr_pages) {
8410 /* don't support file backed memory */
8411 for (i = 0; i < nr_pages; i++) {
8412 struct vm_area_struct *vma = vmas[i];
8413
Pavel Begunkov40dad762021-06-09 15:26:54 +01008414 if (vma_is_shmem(vma))
8415 continue;
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08008416 if (vma->vm_file &&
8417 !is_file_hugepages(vma->vm_file)) {
8418 ret = -EOPNOTSUPP;
8419 break;
8420 }
8421 }
8422 } else {
8423 ret = pret < 0 ? pret : -EFAULT;
8424 }
8425 mmap_read_unlock(current->mm);
8426 if (ret) {
8427 /*
8428 * if we did partial map, or found file backed vmas,
8429 * release any pages we did get
8430 */
8431 if (pret > 0)
8432 unpin_user_pages(pages, pret);
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08008433 goto done;
8434 }
8435
8436 ret = io_buffer_account_pin(ctx, pages, pret, imu, last_hpage);
8437 if (ret) {
8438 unpin_user_pages(pages, pret);
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08008439 goto done;
8440 }
8441
8442 off = ubuf & ~PAGE_MASK;
8443 size = iov->iov_len;
8444 for (i = 0; i < nr_pages; i++) {
8445 size_t vec_len;
8446
8447 vec_len = min_t(size_t, size, PAGE_SIZE - off);
8448 imu->bvec[i].bv_page = pages[i];
8449 imu->bvec[i].bv_len = vec_len;
8450 imu->bvec[i].bv_offset = off;
8451 off = 0;
8452 size -= vec_len;
8453 }
8454 /* store original address for later verification */
8455 imu->ubuf = ubuf;
Pavel Begunkov4751f532021-04-01 15:43:55 +01008456 imu->ubuf_end = ubuf + iov->iov_len;
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08008457 imu->nr_bvecs = nr_pages;
Pavel Begunkov41edf1a2021-04-25 14:32:23 +01008458 *pimu = imu;
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08008459 ret = 0;
8460done:
Pavel Begunkov41edf1a2021-04-25 14:32:23 +01008461 if (ret)
8462 kvfree(imu);
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08008463 kvfree(pages);
8464 kvfree(vmas);
8465 return ret;
8466}
8467
Bijan Mottahedeh2b358602021-01-06 12:39:11 -08008468static int io_buffers_map_alloc(struct io_ring_ctx *ctx, unsigned int nr_args)
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08008469{
Pavel Begunkov87094462021-04-11 01:46:36 +01008470 ctx->user_bufs = kcalloc(nr_args, sizeof(*ctx->user_bufs), GFP_KERNEL);
8471 return ctx->user_bufs ? 0 : -ENOMEM;
Bijan Mottahedeh2b358602021-01-06 12:39:11 -08008472}
8473
8474static int io_buffer_validate(struct iovec *iov)
8475{
Pavel Begunkov50e96982021-03-24 22:59:01 +00008476 unsigned long tmp, acct_len = iov->iov_len + (PAGE_SIZE - 1);
8477
Bijan Mottahedeh2b358602021-01-06 12:39:11 -08008478 /*
8479 * Don't impose further limits on the size and buffer
8480 * constraints here, we'll -EINVAL later when IO is
8481 * submitted if they are wrong.
8482 */
Pavel Begunkov62248432021-04-28 13:11:29 +01008483 if (!iov->iov_base)
8484 return iov->iov_len ? -EFAULT : 0;
8485 if (!iov->iov_len)
Bijan Mottahedeh2b358602021-01-06 12:39:11 -08008486 return -EFAULT;
8487
8488 /* arbitrary limit, but we need something */
8489 if (iov->iov_len > SZ_1G)
8490 return -EFAULT;
8491
Pavel Begunkov50e96982021-03-24 22:59:01 +00008492 if (check_add_overflow((unsigned long)iov->iov_base, acct_len, &tmp))
8493 return -EOVERFLOW;
8494
Bijan Mottahedeh2b358602021-01-06 12:39:11 -08008495 return 0;
8496}
8497
8498static int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg,
Pavel Begunkov634d00d2021-04-25 14:32:26 +01008499 unsigned int nr_args, u64 __user *tags)
Bijan Mottahedeh2b358602021-01-06 12:39:11 -08008500{
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01008501 struct page *last_hpage = NULL;
8502 struct io_rsrc_data *data;
Bijan Mottahedeh2b358602021-01-06 12:39:11 -08008503 int i, ret;
8504 struct iovec iov;
Bijan Mottahedeh2b358602021-01-06 12:39:11 -08008505
Pavel Begunkov87094462021-04-11 01:46:36 +01008506 if (ctx->user_bufs)
8507 return -EBUSY;
Pavel Begunkov489809e2021-05-14 12:06:44 +01008508 if (!nr_args || nr_args > IORING_MAX_REG_BUFFERS)
Pavel Begunkov87094462021-04-11 01:46:36 +01008509 return -EINVAL;
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01008510 ret = io_rsrc_node_switch_start(ctx);
Bijan Mottahedeh2b358602021-01-06 12:39:11 -08008511 if (ret)
8512 return ret;
Pavel Begunkovd878c812021-06-14 02:36:18 +01008513 ret = io_rsrc_data_alloc(ctx, io_rsrc_buf_put, tags, nr_args, &data);
8514 if (ret)
8515 return ret;
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01008516 ret = io_buffers_map_alloc(ctx, nr_args);
8517 if (ret) {
Zqiangbb6659c2021-04-30 16:25:15 +08008518 io_rsrc_data_free(data);
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01008519 return ret;
8520 }
Bijan Mottahedeh2b358602021-01-06 12:39:11 -08008521
Pavel Begunkov87094462021-04-11 01:46:36 +01008522 for (i = 0; i < nr_args; i++, ctx->nr_user_bufs++) {
Jens Axboeedafcce2019-01-09 09:16:05 -07008523 ret = io_copy_iov(ctx, &iov, arg, i);
8524 if (ret)
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08008525 break;
Bijan Mottahedeh2b358602021-01-06 12:39:11 -08008526 ret = io_buffer_validate(&iov);
8527 if (ret)
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08008528 break;
Pavel Begunkov2d091d62021-06-14 02:36:21 +01008529 if (!iov.iov_base && *io_get_tag_slot(data, i)) {
Colin Ian Kingcf3770e2021-04-29 11:46:02 +01008530 ret = -EINVAL;
8531 break;
8532 }
Jens Axboeedafcce2019-01-09 09:16:05 -07008533
Pavel Begunkov41edf1a2021-04-25 14:32:23 +01008534 ret = io_sqe_buffer_register(ctx, &iov, &ctx->user_bufs[i],
8535 &last_hpage);
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08008536 if (ret)
8537 break;
Jens Axboeedafcce2019-01-09 09:16:05 -07008538 }
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08008539
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01008540 WARN_ON_ONCE(ctx->buf_data);
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08008541
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01008542 ctx->buf_data = data;
8543 if (ret)
8544 __io_sqe_buffers_unregister(ctx);
8545 else
8546 io_rsrc_node_switch(ctx, NULL);
Jens Axboeedafcce2019-01-09 09:16:05 -07008547 return ret;
8548}
8549
Pavel Begunkov634d00d2021-04-25 14:32:26 +01008550static int __io_sqe_buffers_update(struct io_ring_ctx *ctx,
8551 struct io_uring_rsrc_update2 *up,
8552 unsigned int nr_args)
8553{
8554 u64 __user *tags = u64_to_user_ptr(up->tags);
8555 struct iovec iov, __user *iovs = u64_to_user_ptr(up->data);
Pavel Begunkov634d00d2021-04-25 14:32:26 +01008556 struct page *last_hpage = NULL;
8557 bool needs_switch = false;
8558 __u32 done;
8559 int i, err;
8560
8561 if (!ctx->buf_data)
8562 return -ENXIO;
8563 if (up->offset + nr_args > ctx->nr_user_bufs)
8564 return -EINVAL;
8565
8566 for (done = 0; done < nr_args; done++) {
Pavel Begunkov0b8c0e72021-04-26 15:17:38 +01008567 struct io_mapped_ubuf *imu;
8568 int offset = up->offset + done;
Pavel Begunkov634d00d2021-04-25 14:32:26 +01008569 u64 tag = 0;
8570
8571 err = io_copy_iov(ctx, &iov, iovs, done);
8572 if (err)
8573 break;
8574 if (tags && copy_from_user(&tag, &tags[done], sizeof(tag))) {
8575 err = -EFAULT;
8576 break;
8577 }
Pavel Begunkov0b8c0e72021-04-26 15:17:38 +01008578 err = io_buffer_validate(&iov);
8579 if (err)
8580 break;
Colin Ian Kingcf3770e2021-04-29 11:46:02 +01008581 if (!iov.iov_base && tag) {
8582 err = -EINVAL;
8583 break;
8584 }
Pavel Begunkov0b8c0e72021-04-26 15:17:38 +01008585 err = io_sqe_buffer_register(ctx, &iov, &imu, &last_hpage);
8586 if (err)
8587 break;
Pavel Begunkov634d00d2021-04-25 14:32:26 +01008588
Pavel Begunkov0b8c0e72021-04-26 15:17:38 +01008589 i = array_index_nospec(offset, ctx->nr_user_bufs);
Pavel Begunkov62248432021-04-28 13:11:29 +01008590 if (ctx->user_bufs[i] != ctx->dummy_ubuf) {
Pavel Begunkov0b8c0e72021-04-26 15:17:38 +01008591 err = io_queue_rsrc_removal(ctx->buf_data, offset,
8592 ctx->rsrc_node, ctx->user_bufs[i]);
8593 if (unlikely(err)) {
8594 io_buffer_unmap(ctx, &imu);
Pavel Begunkov634d00d2021-04-25 14:32:26 +01008595 break;
Pavel Begunkov0b8c0e72021-04-26 15:17:38 +01008596 }
Pavel Begunkov634d00d2021-04-25 14:32:26 +01008597 ctx->user_bufs[i] = NULL;
8598 needs_switch = true;
8599 }
8600
Pavel Begunkov0b8c0e72021-04-26 15:17:38 +01008601 ctx->user_bufs[i] = imu;
Pavel Begunkov2d091d62021-06-14 02:36:21 +01008602 *io_get_tag_slot(ctx->buf_data, offset) = tag;
Pavel Begunkov634d00d2021-04-25 14:32:26 +01008603 }
8604
8605 if (needs_switch)
8606 io_rsrc_node_switch(ctx, ctx->buf_data);
8607 return done ? done : err;
8608}
8609
Jens Axboe9b402842019-04-11 11:45:41 -06008610static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg)
8611{
8612 __s32 __user *fds = arg;
8613 int fd;
8614
8615 if (ctx->cq_ev_fd)
8616 return -EBUSY;
8617
8618 if (copy_from_user(&fd, fds, sizeof(*fds)))
8619 return -EFAULT;
8620
8621 ctx->cq_ev_fd = eventfd_ctx_fdget(fd);
8622 if (IS_ERR(ctx->cq_ev_fd)) {
8623 int ret = PTR_ERR(ctx->cq_ev_fd);
Pavel Begunkovfe7e3252021-06-24 15:09:57 +01008624
Jens Axboe9b402842019-04-11 11:45:41 -06008625 ctx->cq_ev_fd = NULL;
8626 return ret;
8627 }
8628
8629 return 0;
8630}
8631
8632static int io_eventfd_unregister(struct io_ring_ctx *ctx)
8633{
8634 if (ctx->cq_ev_fd) {
8635 eventfd_ctx_put(ctx->cq_ev_fd);
8636 ctx->cq_ev_fd = NULL;
8637 return 0;
8638 }
8639
8640 return -ENXIO;
8641}
8642
Jens Axboe5a2e7452020-02-23 16:23:11 -07008643static void io_destroy_buffers(struct io_ring_ctx *ctx)
8644{
Jens Axboe9e15c3a2021-03-13 12:29:43 -07008645 struct io_buffer *buf;
8646 unsigned long index;
8647
8648 xa_for_each(&ctx->io_buffers, index, buf)
8649 __io_remove_buffers(ctx, buf, index, -1U);
Jens Axboe5a2e7452020-02-23 16:23:11 -07008650}
8651
Pavel Begunkov72558342021-08-09 20:18:09 +01008652static void io_req_cache_free(struct list_head *list)
Jens Axboe1b4c3512021-02-10 00:03:19 +00008653{
Jens Axboe68e68ee2021-02-13 09:00:02 -07008654 struct io_kiocb *req, *nxt;
Jens Axboe1b4c3512021-02-10 00:03:19 +00008655
Pavel Begunkovbb943b82021-08-09 20:18:10 +01008656 list_for_each_entry_safe(req, nxt, list, inflight_entry) {
8657 list_del(&req->inflight_entry);
Jens Axboe1b4c3512021-02-10 00:03:19 +00008658 kmem_cache_free(req_cachep, req);
8659 }
8660}
8661
Jens Axboe4010fec2021-02-27 15:04:18 -07008662static void io_req_caches_free(struct io_ring_ctx *ctx)
Jens Axboe2b188cc2019-01-07 10:46:33 -07008663{
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01008664 struct io_submit_state *state = &ctx->submit_state;
Pavel Begunkovbf019da2021-02-10 00:03:17 +00008665
Jens Axboe9a4fdbd2021-02-13 09:09:44 -07008666 mutex_lock(&ctx->uring_lock);
8667
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01008668 if (state->free_reqs) {
8669 kmem_cache_free_bulk(req_cachep, state->free_reqs, state->reqs);
8670 state->free_reqs = 0;
Pavel Begunkov8e5c66c2021-02-22 11:45:55 +00008671 }
Jens Axboe9a4fdbd2021-02-13 09:09:44 -07008672
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01008673 io_flush_cached_locked_reqs(ctx, state);
8674 io_req_cache_free(&state->free_list);
Jens Axboe9a4fdbd2021-02-13 09:09:44 -07008675 mutex_unlock(&ctx->uring_lock);
8676}
8677
Pavel Begunkov43597aa2021-08-10 02:44:23 +01008678static void io_wait_rsrc_data(struct io_rsrc_data *data)
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01008679{
Pavel Begunkov43597aa2021-08-10 02:44:23 +01008680 if (data && !atomic_dec_and_test(&data->refs))
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01008681 wait_for_completion(&data->done);
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01008682}
8683
Jens Axboe2b188cc2019-01-07 10:46:33 -07008684static void io_ring_ctx_free(struct io_ring_ctx *ctx)
8685{
Jens Axboe37d1e2e2021-02-17 21:03:43 -07008686 io_sq_thread_finish(ctx);
Jens Axboe2aede0e2020-09-14 10:45:53 -06008687
Jens Axboe37d1e2e2021-02-17 21:03:43 -07008688 if (ctx->mm_account) {
Jens Axboe2aede0e2020-09-14 10:45:53 -06008689 mmdrop(ctx->mm_account);
8690 ctx->mm_account = NULL;
Bijan Mottahedeh30975822020-06-16 16:36:09 -07008691 }
Jens Axboedef596e2019-01-09 08:59:42 -07008692
Pavel Begunkov43597aa2021-08-10 02:44:23 +01008693 /* __io_rsrc_put_work() may need uring_lock to progress, wait w/o it */
8694 io_wait_rsrc_data(ctx->buf_data);
8695 io_wait_rsrc_data(ctx->file_data);
8696
Hao Xu8bad28d2021-02-19 17:19:36 +08008697 mutex_lock(&ctx->uring_lock);
Pavel Begunkov43597aa2021-08-10 02:44:23 +01008698 if (ctx->buf_data)
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01008699 __io_sqe_buffers_unregister(ctx);
Pavel Begunkov43597aa2021-08-10 02:44:23 +01008700 if (ctx->file_data)
Pavel Begunkov08480402021-04-13 02:58:38 +01008701 __io_sqe_files_unregister(ctx);
Pavel Begunkovc4ea0602021-04-01 15:43:58 +01008702 if (ctx->rings)
8703 __io_cqring_overflow_flush(ctx, true);
Hao Xu8bad28d2021-02-19 17:19:36 +08008704 mutex_unlock(&ctx->uring_lock);
Jens Axboe9b402842019-04-11 11:45:41 -06008705 io_eventfd_unregister(ctx);
Jens Axboe5a2e7452020-02-23 16:23:11 -07008706 io_destroy_buffers(ctx);
Pavel Begunkov07db2982021-04-20 12:03:32 +01008707 if (ctx->sq_creds)
8708 put_cred(ctx->sq_creds);
Jens Axboedef596e2019-01-09 08:59:42 -07008709
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01008710 /* there are no registered resources left, nobody uses it */
8711 if (ctx->rsrc_node)
8712 io_rsrc_node_destroy(ctx->rsrc_node);
Pavel Begunkov8dd03af2021-03-19 17:22:36 +00008713 if (ctx->rsrc_backup_node)
Pavel Begunkovb895c9a2021-04-01 15:43:40 +01008714 io_rsrc_node_destroy(ctx->rsrc_backup_node);
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01008715 flush_delayed_work(&ctx->rsrc_put_work);
8716
8717 WARN_ON_ONCE(!list_empty(&ctx->rsrc_ref_list));
8718 WARN_ON_ONCE(!llist_empty(&ctx->rsrc_put_llist));
Jens Axboe2b188cc2019-01-07 10:46:33 -07008719
8720#if defined(CONFIG_UNIX)
Eric Biggers355e8d22019-06-12 14:58:43 -07008721 if (ctx->ring_sock) {
8722 ctx->ring_sock->file = NULL; /* so that iput() is called */
Jens Axboe2b188cc2019-01-07 10:46:33 -07008723 sock_release(ctx->ring_sock);
Eric Biggers355e8d22019-06-12 14:58:43 -07008724 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07008725#endif
8726
Hristo Venev75b28af2019-08-26 17:23:46 +00008727 io_mem_free(ctx->rings);
Jens Axboe2b188cc2019-01-07 10:46:33 -07008728 io_mem_free(ctx->sq_sqes);
Jens Axboe2b188cc2019-01-07 10:46:33 -07008729
8730 percpu_ref_exit(&ctx->refs);
Jens Axboe2b188cc2019-01-07 10:46:33 -07008731 free_uid(ctx->user);
Jens Axboe4010fec2021-02-27 15:04:18 -07008732 io_req_caches_free(ctx);
Jens Axboee9418942021-02-19 12:33:30 -07008733 if (ctx->hash_map)
8734 io_wq_put_hash(ctx->hash_map);
Jens Axboe78076bb2019-12-04 19:56:40 -07008735 kfree(ctx->cancel_hash);
Pavel Begunkov62248432021-04-28 13:11:29 +01008736 kfree(ctx->dummy_ubuf);
Jens Axboe2b188cc2019-01-07 10:46:33 -07008737 kfree(ctx);
8738}
8739
8740static __poll_t io_uring_poll(struct file *file, poll_table *wait)
8741{
8742 struct io_ring_ctx *ctx = file->private_data;
8743 __poll_t mask = 0;
8744
Pavel Begunkov311997b2021-06-14 23:37:28 +01008745 poll_wait(file, &ctx->poll_wait, wait);
Stefan Bühler4f7067c2019-04-24 23:54:17 +02008746 /*
8747 * synchronizes with barrier from wq_has_sleeper call in
8748 * io_commit_cqring
8749 */
Jens Axboe2b188cc2019-01-07 10:46:33 -07008750 smp_rmb();
Jens Axboe90554202020-09-03 12:12:41 -06008751 if (!io_sqring_full(ctx))
Jens Axboe2b188cc2019-01-07 10:46:33 -07008752 mask |= EPOLLOUT | EPOLLWRNORM;
Hao Xued670c32021-02-05 16:34:21 +08008753
8754 /*
8755 * Don't flush cqring overflow list here, just do a simple check.
8756 * Otherwise there could possible be ABBA deadlock:
8757 * CPU0 CPU1
8758 * ---- ----
8759 * lock(&ctx->uring_lock);
8760 * lock(&ep->mtx);
8761 * lock(&ctx->uring_lock);
8762 * lock(&ep->mtx);
8763 *
8764 * Users may get EPOLLIN meanwhile seeing nothing in cqring, this
8765 * pushs them to do the flush.
8766 */
Pavel Begunkov5ed7a372021-06-14 23:37:27 +01008767 if (io_cqring_events(ctx) || test_bit(0, &ctx->check_cq_overflow))
Jens Axboe2b188cc2019-01-07 10:46:33 -07008768 mask |= EPOLLIN | EPOLLRDNORM;
8769
8770 return mask;
8771}
8772
8773static int io_uring_fasync(int fd, struct file *file, int on)
8774{
8775 struct io_ring_ctx *ctx = file->private_data;
8776
8777 return fasync_helper(fd, file, on, &ctx->cq_fasync);
8778}
8779
Yejune Deng0bead8c2020-12-24 11:02:20 +08008780static int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id)
Jens Axboe071698e2020-01-28 10:04:42 -07008781{
Jens Axboe4379bf82021-02-15 13:40:22 -07008782 const struct cred *creds;
Jens Axboe071698e2020-01-28 10:04:42 -07008783
Matthew Wilcox (Oracle)61cf9372021-03-08 14:16:16 +00008784 creds = xa_erase(&ctx->personalities, id);
Jens Axboe4379bf82021-02-15 13:40:22 -07008785 if (creds) {
8786 put_cred(creds);
Yejune Deng0bead8c2020-12-24 11:02:20 +08008787 return 0;
Jens Axboe1e6fa522020-10-15 08:46:24 -06008788 }
Yejune Deng0bead8c2020-12-24 11:02:20 +08008789
8790 return -EINVAL;
8791}
8792
Pavel Begunkovd56d9382021-03-06 11:02:13 +00008793struct io_tctx_exit {
8794 struct callback_head task_work;
8795 struct completion completion;
Pavel Begunkovbaf186c2021-03-06 11:02:15 +00008796 struct io_ring_ctx *ctx;
Pavel Begunkovd56d9382021-03-06 11:02:13 +00008797};
8798
8799static void io_tctx_exit_cb(struct callback_head *cb)
8800{
8801 struct io_uring_task *tctx = current->io_uring;
8802 struct io_tctx_exit *work;
8803
8804 work = container_of(cb, struct io_tctx_exit, task_work);
8805 /*
8806 * When @in_idle, we're in cancellation and it's racy to remove the
8807 * node. It'll be removed by the end of cancellation, just ignore it.
8808 */
8809 if (!atomic_read(&tctx->in_idle))
Pavel Begunkoveef51da2021-06-14 02:36:15 +01008810 io_uring_del_tctx_node((unsigned long)work->ctx);
Pavel Begunkovd56d9382021-03-06 11:02:13 +00008811 complete(&work->completion);
8812}
8813
Pavel Begunkov28090c12021-04-25 23:34:45 +01008814static bool io_cancel_ctx_cb(struct io_wq_work *work, void *data)
8815{
8816 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
8817
8818 return req->ctx == data;
8819}
8820
Jens Axboe85faa7b2020-04-09 18:14:00 -06008821static void io_ring_exit_work(struct work_struct *work)
8822{
Pavel Begunkovd56d9382021-03-06 11:02:13 +00008823 struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx, exit_work);
Pavel Begunkovb5bb3a22021-03-06 11:02:16 +00008824 unsigned long timeout = jiffies + HZ * 60 * 5;
Pavel Begunkov58d3be22021-08-09 13:04:17 +01008825 unsigned long interval = HZ / 20;
Pavel Begunkovd56d9382021-03-06 11:02:13 +00008826 struct io_tctx_exit exit;
8827 struct io_tctx_node *node;
8828 int ret;
Jens Axboe85faa7b2020-04-09 18:14:00 -06008829
Jens Axboe56952e92020-06-17 15:00:04 -06008830 /*
8831 * If we're doing polled IO and end up having requests being
8832 * submitted async (out-of-line), then completions can come in while
8833 * we're waiting for refs to drop. We need to reap these manually,
8834 * as nobody else will be looking for them.
8835 */
Pavel Begunkovb2edc0a2020-07-07 16:36:22 +03008836 do {
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01008837 io_uring_try_cancel_requests(ctx, NULL, true);
Pavel Begunkov28090c12021-04-25 23:34:45 +01008838 if (ctx->sq_data) {
8839 struct io_sq_data *sqd = ctx->sq_data;
8840 struct task_struct *tsk;
8841
8842 io_sq_thread_park(sqd);
8843 tsk = sqd->thread;
8844 if (tsk && tsk->io_uring && tsk->io_uring->io_wq)
8845 io_wq_cancel_cb(tsk->io_uring->io_wq,
8846 io_cancel_ctx_cb, ctx, true);
8847 io_sq_thread_unpark(sqd);
8848 }
Pavel Begunkovb5bb3a22021-03-06 11:02:16 +00008849
Pavel Begunkov58d3be22021-08-09 13:04:17 +01008850 if (WARN_ON_ONCE(time_after(jiffies, timeout))) {
8851 /* there is little hope left, don't run it too often */
8852 interval = HZ * 60;
8853 }
8854 } while (!wait_for_completion_timeout(&ctx->ref_comp, interval));
Pavel Begunkovd56d9382021-03-06 11:02:13 +00008855
Pavel Begunkov7f006512021-04-14 13:38:34 +01008856 init_completion(&exit.completion);
8857 init_task_work(&exit.task_work, io_tctx_exit_cb);
8858 exit.ctx = ctx;
Pavel Begunkov89b50662021-04-01 15:43:50 +01008859 /*
8860 * Some may use context even when all refs and requests have been put,
8861 * and they are free to do so while still holding uring_lock or
Pavel Begunkov5b0a6ac2021-06-30 21:54:04 +01008862 * completion_lock, see io_req_task_submit(). Apart from other work,
Pavel Begunkov89b50662021-04-01 15:43:50 +01008863 * this lock/unlock section also waits them to finish.
8864 */
Pavel Begunkovd56d9382021-03-06 11:02:13 +00008865 mutex_lock(&ctx->uring_lock);
8866 while (!list_empty(&ctx->tctx_list)) {
Pavel Begunkovb5bb3a22021-03-06 11:02:16 +00008867 WARN_ON_ONCE(time_after(jiffies, timeout));
8868
Pavel Begunkovd56d9382021-03-06 11:02:13 +00008869 node = list_first_entry(&ctx->tctx_list, struct io_tctx_node,
8870 ctx_node);
Pavel Begunkov7f006512021-04-14 13:38:34 +01008871 /* don't spin on a single task if cancellation failed */
8872 list_rotate_left(&ctx->tctx_list);
Pavel Begunkovd56d9382021-03-06 11:02:13 +00008873 ret = task_work_add(node->task, &exit.task_work, TWA_SIGNAL);
8874 if (WARN_ON_ONCE(ret))
8875 continue;
8876 wake_up_process(node->task);
8877
8878 mutex_unlock(&ctx->uring_lock);
8879 wait_for_completion(&exit.completion);
Pavel Begunkovd56d9382021-03-06 11:02:13 +00008880 mutex_lock(&ctx->uring_lock);
8881 }
8882 mutex_unlock(&ctx->uring_lock);
Jens Axboe79ebeae2021-08-10 15:18:27 -06008883 spin_lock(&ctx->completion_lock);
8884 spin_unlock(&ctx->completion_lock);
Pavel Begunkovd56d9382021-03-06 11:02:13 +00008885
Jens Axboe85faa7b2020-04-09 18:14:00 -06008886 io_ring_ctx_free(ctx);
8887}
8888
Pavel Begunkov80c4cbd2021-03-25 18:32:43 +00008889/* Returns true if we found and killed one or more timeouts */
8890static bool io_kill_timeouts(struct io_ring_ctx *ctx, struct task_struct *tsk,
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01008891 bool cancel_all)
Pavel Begunkov80c4cbd2021-03-25 18:32:43 +00008892{
8893 struct io_kiocb *req, *tmp;
8894 int canceled = 0;
8895
Jens Axboe79ebeae2021-08-10 15:18:27 -06008896 spin_lock(&ctx->completion_lock);
8897 spin_lock_irq(&ctx->timeout_lock);
Pavel Begunkov80c4cbd2021-03-25 18:32:43 +00008898 list_for_each_entry_safe(req, tmp, &ctx->timeout_list, timeout.list) {
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01008899 if (io_match_task(req, tsk, cancel_all)) {
Pavel Begunkov80c4cbd2021-03-25 18:32:43 +00008900 io_kill_timeout(req, -ECANCELED);
8901 canceled++;
8902 }
8903 }
Jens Axboe79ebeae2021-08-10 15:18:27 -06008904 spin_unlock_irq(&ctx->timeout_lock);
Pavel Begunkov51520422021-03-29 11:39:29 +01008905 if (canceled != 0)
8906 io_commit_cqring(ctx);
Jens Axboe79ebeae2021-08-10 15:18:27 -06008907 spin_unlock(&ctx->completion_lock);
Pavel Begunkov80c4cbd2021-03-25 18:32:43 +00008908 if (canceled != 0)
8909 io_cqring_ev_posted(ctx);
8910 return canceled != 0;
8911}
8912
Jens Axboe2b188cc2019-01-07 10:46:33 -07008913static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx)
8914{
Matthew Wilcox (Oracle)61cf9372021-03-08 14:16:16 +00008915 unsigned long index;
8916 struct creds *creds;
8917
Jens Axboe2b188cc2019-01-07 10:46:33 -07008918 mutex_lock(&ctx->uring_lock);
8919 percpu_ref_kill(&ctx->refs);
Pavel Begunkov634578f2020-12-06 22:22:44 +00008920 if (ctx->rings)
Pavel Begunkov6c2450a2021-02-23 12:40:22 +00008921 __io_cqring_overflow_flush(ctx, true);
Matthew Wilcox (Oracle)61cf9372021-03-08 14:16:16 +00008922 xa_for_each(&ctx->personalities, index, creds)
8923 io_unregister_personality(ctx, index);
Jens Axboe2b188cc2019-01-07 10:46:33 -07008924 mutex_unlock(&ctx->uring_lock);
8925
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01008926 io_kill_timeouts(ctx, NULL, true);
8927 io_poll_remove_all(ctx, NULL, true);
Jens Axboe561fb042019-10-24 07:25:42 -06008928
Jens Axboe15dff282019-11-13 09:09:23 -07008929 /* if we failed setting up the ctx, we might not have any rings */
Pavel Begunkovb2edc0a2020-07-07 16:36:22 +03008930 io_iopoll_try_reap_events(ctx);
Jens Axboe309fc032020-07-10 09:13:34 -06008931
Jens Axboe85faa7b2020-04-09 18:14:00 -06008932 INIT_WORK(&ctx->exit_work, io_ring_exit_work);
Jens Axboefc666772020-08-19 11:10:51 -06008933 /*
8934 * Use system_unbound_wq to avoid spawning tons of event kworkers
8935 * if we're exiting a ton of rings at the same time. It just adds
8936 * noise and overhead, there's no discernable change in runtime
8937 * over using system_wq.
8938 */
8939 queue_work(system_unbound_wq, &ctx->exit_work);
Jens Axboe2b188cc2019-01-07 10:46:33 -07008940}
8941
8942static int io_uring_release(struct inode *inode, struct file *file)
8943{
8944 struct io_ring_ctx *ctx = file->private_data;
8945
8946 file->private_data = NULL;
8947 io_ring_ctx_wait_and_kill(ctx);
8948 return 0;
8949}
8950
Pavel Begunkovf6edbab2020-11-06 13:00:26 +00008951struct io_task_cancel {
8952 struct task_struct *task;
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01008953 bool all;
Pavel Begunkovf6edbab2020-11-06 13:00:26 +00008954};
Pavel Begunkov67c4d9e2020-06-15 10:24:05 +03008955
Pavel Begunkovf6edbab2020-11-06 13:00:26 +00008956static bool io_cancel_task_cb(struct io_wq_work *work, void *data)
Jens Axboeb711d4e2020-08-16 08:23:05 -07008957{
Pavel Begunkov9a472ef2020-11-05 22:31:37 +00008958 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
Pavel Begunkovf6edbab2020-11-06 13:00:26 +00008959 struct io_task_cancel *cancel = data;
Pavel Begunkov9a472ef2020-11-05 22:31:37 +00008960 bool ret;
8961
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01008962 if (!cancel->all && (req->flags & REQ_F_LINK_TIMEOUT)) {
Pavel Begunkov9a472ef2020-11-05 22:31:37 +00008963 struct io_ring_ctx *ctx = req->ctx;
8964
8965 /* protect against races with linked timeouts */
Jens Axboe79ebeae2021-08-10 15:18:27 -06008966 spin_lock(&ctx->completion_lock);
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01008967 ret = io_match_task(req, cancel->task, cancel->all);
Jens Axboe79ebeae2021-08-10 15:18:27 -06008968 spin_unlock(&ctx->completion_lock);
Pavel Begunkov9a472ef2020-11-05 22:31:37 +00008969 } else {
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01008970 ret = io_match_task(req, cancel->task, cancel->all);
Pavel Begunkov9a472ef2020-11-05 22:31:37 +00008971 }
8972 return ret;
Jens Axboeb711d4e2020-08-16 08:23:05 -07008973}
8974
Pavel Begunkove1915f72021-03-11 23:29:35 +00008975static bool io_cancel_defer_files(struct io_ring_ctx *ctx,
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01008976 struct task_struct *task, bool cancel_all)
Pavel Begunkovb7ddce32020-09-06 00:45:14 +03008977{
Pavel Begunkove1915f72021-03-11 23:29:35 +00008978 struct io_defer_entry *de;
Pavel Begunkovb7ddce32020-09-06 00:45:14 +03008979 LIST_HEAD(list);
8980
Jens Axboe79ebeae2021-08-10 15:18:27 -06008981 spin_lock(&ctx->completion_lock);
Pavel Begunkovb7ddce32020-09-06 00:45:14 +03008982 list_for_each_entry_reverse(de, &ctx->defer_list, list) {
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01008983 if (io_match_task(de->req, task, cancel_all)) {
Pavel Begunkovb7ddce32020-09-06 00:45:14 +03008984 list_cut_position(&list, &ctx->defer_list, &de->list);
8985 break;
8986 }
8987 }
Jens Axboe79ebeae2021-08-10 15:18:27 -06008988 spin_unlock(&ctx->completion_lock);
Pavel Begunkove1915f72021-03-11 23:29:35 +00008989 if (list_empty(&list))
8990 return false;
Pavel Begunkovb7ddce32020-09-06 00:45:14 +03008991
8992 while (!list_empty(&list)) {
8993 de = list_first_entry(&list, struct io_defer_entry, list);
8994 list_del_init(&de->list);
Pavel Begunkovf41db2732021-02-28 22:35:12 +00008995 io_req_complete_failed(de->req, -ECANCELED);
Pavel Begunkovb7ddce32020-09-06 00:45:14 +03008996 kfree(de);
8997 }
Pavel Begunkove1915f72021-03-11 23:29:35 +00008998 return true;
Pavel Begunkovb7ddce32020-09-06 00:45:14 +03008999}
9000
Pavel Begunkov1b007642021-03-06 11:02:17 +00009001static bool io_uring_try_cancel_iowq(struct io_ring_ctx *ctx)
9002{
9003 struct io_tctx_node *node;
9004 enum io_wq_cancel cret;
9005 bool ret = false;
9006
9007 mutex_lock(&ctx->uring_lock);
9008 list_for_each_entry(node, &ctx->tctx_list, ctx_node) {
9009 struct io_uring_task *tctx = node->task->io_uring;
9010
9011 /*
9012 * io_wq will stay alive while we hold uring_lock, because it's
9013 * killed after ctx nodes, which requires to take the lock.
9014 */
9015 if (!tctx || !tctx->io_wq)
9016 continue;
9017 cret = io_wq_cancel_cb(tctx->io_wq, io_cancel_ctx_cb, ctx, true);
9018 ret |= (cret != IO_WQ_CANCEL_NOTFOUND);
9019 }
9020 mutex_unlock(&ctx->uring_lock);
9021
9022 return ret;
9023}
9024
Pavel Begunkov9936c7c2021-02-04 13:51:56 +00009025static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
9026 struct task_struct *task,
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01009027 bool cancel_all)
Pavel Begunkov9936c7c2021-02-04 13:51:56 +00009028{
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01009029 struct io_task_cancel cancel = { .task = task, .all = cancel_all, };
Pavel Begunkov1b007642021-03-06 11:02:17 +00009030 struct io_uring_task *tctx = task ? task->io_uring : NULL;
Pavel Begunkov9936c7c2021-02-04 13:51:56 +00009031
9032 while (1) {
9033 enum io_wq_cancel cret;
9034 bool ret = false;
9035
Pavel Begunkov1b007642021-03-06 11:02:17 +00009036 if (!task) {
9037 ret |= io_uring_try_cancel_iowq(ctx);
9038 } else if (tctx && tctx->io_wq) {
9039 /*
9040 * Cancels requests of all rings, not only @ctx, but
9041 * it's fine as the task is in exit/exec.
9042 */
Jens Axboe5aa75ed2021-02-16 12:56:50 -07009043 cret = io_wq_cancel_cb(tctx->io_wq, io_cancel_task_cb,
Pavel Begunkov9936c7c2021-02-04 13:51:56 +00009044 &cancel, true);
9045 ret |= (cret != IO_WQ_CANCEL_NOTFOUND);
9046 }
9047
9048 /* SQPOLL thread does its own polling */
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01009049 if ((!(ctx->flags & IORING_SETUP_SQPOLL) && cancel_all) ||
Jens Axboed052d1d2021-03-11 10:49:20 -07009050 (ctx->sq_data && ctx->sq_data->thread == current)) {
Pavel Begunkov9936c7c2021-02-04 13:51:56 +00009051 while (!list_empty_careful(&ctx->iopoll_list)) {
9052 io_iopoll_try_reap_events(ctx);
9053 ret = true;
9054 }
9055 }
9056
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01009057 ret |= io_cancel_defer_files(ctx, task, cancel_all);
9058 ret |= io_poll_remove_all(ctx, task, cancel_all);
9059 ret |= io_kill_timeouts(ctx, task, cancel_all);
Pavel Begunkove5dc4802021-06-26 21:40:46 +01009060 if (task)
9061 ret |= io_run_task_work();
Pavel Begunkov9936c7c2021-02-04 13:51:56 +00009062 if (!ret)
9063 break;
9064 cond_resched();
9065 }
9066}
9067
Pavel Begunkoveef51da2021-06-14 02:36:15 +01009068static int __io_uring_add_tctx_node(struct io_ring_ctx *ctx)
Jens Axboe0f212202020-09-13 13:09:39 -06009069{
Matthew Wilcox (Oracle)236434c2020-10-09 13:49:52 +01009070 struct io_uring_task *tctx = current->io_uring;
Pavel Begunkov13bf43f2021-03-06 11:02:12 +00009071 struct io_tctx_node *node;
Pavel Begunkova528b042020-12-21 18:34:04 +00009072 int ret;
Matthew Wilcox (Oracle)236434c2020-10-09 13:49:52 +01009073
9074 if (unlikely(!tctx)) {
Jens Axboe5aa75ed2021-02-16 12:56:50 -07009075 ret = io_uring_alloc_task_context(current, ctx);
Jens Axboe0f212202020-09-13 13:09:39 -06009076 if (unlikely(ret))
9077 return ret;
Matthew Wilcox (Oracle)236434c2020-10-09 13:49:52 +01009078 tctx = current->io_uring;
Jens Axboe0f212202020-09-13 13:09:39 -06009079 }
Pavel Begunkovcf27f3b2021-03-19 17:22:31 +00009080 if (!xa_load(&tctx->xa, (unsigned long)ctx)) {
9081 node = kmalloc(sizeof(*node), GFP_KERNEL);
9082 if (!node)
9083 return -ENOMEM;
9084 node->ctx = ctx;
9085 node->task = current;
Jens Axboe0f212202020-09-13 13:09:39 -06009086
Pavel Begunkovcf27f3b2021-03-19 17:22:31 +00009087 ret = xa_err(xa_store(&tctx->xa, (unsigned long)ctx,
9088 node, GFP_KERNEL));
9089 if (ret) {
9090 kfree(node);
9091 return ret;
Jens Axboe0f212202020-09-13 13:09:39 -06009092 }
Pavel Begunkovcf27f3b2021-03-19 17:22:31 +00009093
9094 mutex_lock(&ctx->uring_lock);
9095 list_add(&node->ctx_node, &ctx->tctx_list);
9096 mutex_unlock(&ctx->uring_lock);
Jens Axboe0f212202020-09-13 13:09:39 -06009097 }
Pavel Begunkovcf27f3b2021-03-19 17:22:31 +00009098 tctx->last = ctx;
Jens Axboe0f212202020-09-13 13:09:39 -06009099 return 0;
9100}
9101
9102/*
Pavel Begunkovcf27f3b2021-03-19 17:22:31 +00009103 * Note that this task has used io_uring. We use it for cancelation purposes.
9104 */
Pavel Begunkoveef51da2021-06-14 02:36:15 +01009105static inline int io_uring_add_tctx_node(struct io_ring_ctx *ctx)
Pavel Begunkovcf27f3b2021-03-19 17:22:31 +00009106{
9107 struct io_uring_task *tctx = current->io_uring;
9108
9109 if (likely(tctx && tctx->last == ctx))
9110 return 0;
Pavel Begunkoveef51da2021-06-14 02:36:15 +01009111 return __io_uring_add_tctx_node(ctx);
Pavel Begunkovcf27f3b2021-03-19 17:22:31 +00009112}
9113
9114/*
Jens Axboe0f212202020-09-13 13:09:39 -06009115 * Remove this io_uring_file -> task mapping.
9116 */
Pavel Begunkoveef51da2021-06-14 02:36:15 +01009117static void io_uring_del_tctx_node(unsigned long index)
Jens Axboe0f212202020-09-13 13:09:39 -06009118{
9119 struct io_uring_task *tctx = current->io_uring;
Pavel Begunkov13bf43f2021-03-06 11:02:12 +00009120 struct io_tctx_node *node;
Pavel Begunkov29412672021-03-06 11:02:11 +00009121
Pavel Begunkoveebd2e32021-03-06 11:02:14 +00009122 if (!tctx)
9123 return;
Pavel Begunkov13bf43f2021-03-06 11:02:12 +00009124 node = xa_erase(&tctx->xa, index);
9125 if (!node)
Pavel Begunkov29412672021-03-06 11:02:11 +00009126 return;
Jens Axboe0f212202020-09-13 13:09:39 -06009127
Pavel Begunkov13bf43f2021-03-06 11:02:12 +00009128 WARN_ON_ONCE(current != node->task);
9129 WARN_ON_ONCE(list_empty(&node->ctx_node));
9130
9131 mutex_lock(&node->ctx->uring_lock);
9132 list_del(&node->ctx_node);
9133 mutex_unlock(&node->ctx->uring_lock);
9134
Pavel Begunkovbaf186c2021-03-06 11:02:15 +00009135 if (tctx->last == node->ctx)
Jens Axboe0f212202020-09-13 13:09:39 -06009136 tctx->last = NULL;
Pavel Begunkov13bf43f2021-03-06 11:02:12 +00009137 kfree(node);
Jens Axboe0f212202020-09-13 13:09:39 -06009138}
9139
Pavel Begunkov8452d4a2021-02-27 11:16:46 +00009140static void io_uring_clean_tctx(struct io_uring_task *tctx)
Pavel Begunkovde7f1d92021-01-04 20:43:29 +00009141{
Pavel Begunkovba5ef6d2021-05-20 13:21:20 +01009142 struct io_wq *wq = tctx->io_wq;
Pavel Begunkov13bf43f2021-03-06 11:02:12 +00009143 struct io_tctx_node *node;
Pavel Begunkovde7f1d92021-01-04 20:43:29 +00009144 unsigned long index;
9145
Pavel Begunkov13bf43f2021-03-06 11:02:12 +00009146 xa_for_each(&tctx->xa, index, node)
Pavel Begunkoveef51da2021-06-14 02:36:15 +01009147 io_uring_del_tctx_node(index);
Marco Elverb16ef422021-05-27 11:25:48 +02009148 if (wq) {
9149 /*
9150 * Must be after io_uring_del_task_file() (removes nodes under
9151 * uring_lock) to avoid race with io_uring_try_cancel_iowq().
9152 */
9153 tctx->io_wq = NULL;
Pavel Begunkovba5ef6d2021-05-20 13:21:20 +01009154 io_wq_put_and_exit(wq);
Marco Elverb16ef422021-05-27 11:25:48 +02009155 }
Pavel Begunkovde7f1d92021-01-04 20:43:29 +00009156}
9157
Pavel Begunkov3f48cf12021-04-11 01:46:27 +01009158static s64 tctx_inflight(struct io_uring_task *tctx, bool tracked)
Pavel Begunkov521d6a72021-03-11 23:29:38 +00009159{
Pavel Begunkov3f48cf12021-04-11 01:46:27 +01009160 if (tracked)
9161 return atomic_read(&tctx->inflight_tracked);
Pavel Begunkov521d6a72021-03-11 23:29:38 +00009162 return percpu_counter_sum(&tctx->inflight);
9163}
9164
Pavel Begunkov09899b12021-06-14 02:36:22 +01009165static void io_uring_drop_tctx_refs(struct task_struct *task)
9166{
9167 struct io_uring_task *tctx = task->io_uring;
9168 unsigned int refs = tctx->cached_refs;
9169
Pavel Begunkove9dbe222021-08-09 13:04:20 +01009170 if (refs) {
9171 tctx->cached_refs = 0;
9172 percpu_counter_sub(&tctx->inflight, refs);
9173 put_task_struct_many(task, refs);
9174 }
Pavel Begunkov09899b12021-06-14 02:36:22 +01009175}
9176
Pavel Begunkov78cc6872021-06-14 02:36:23 +01009177/*
9178 * Find any io_uring ctx that this task has registered or done IO on, and cancel
9179 * requests. @sqd should be not-null IIF it's an SQPOLL thread cancellation.
9180 */
9181static void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd)
Pavel Begunkov0e9ddb32021-02-07 22:34:26 +00009182{
Pavel Begunkov521d6a72021-03-11 23:29:38 +00009183 struct io_uring_task *tctx = current->io_uring;
Pavel Begunkov734551d2021-04-18 14:52:09 +01009184 struct io_ring_ctx *ctx;
Jens Axboefdaf0832020-10-30 09:37:30 -06009185 s64 inflight;
Pavel Begunkov0e9ddb32021-02-07 22:34:26 +00009186 DEFINE_WAIT(wait);
Jens Axboefdaf0832020-10-30 09:37:30 -06009187
Pavel Begunkov78cc6872021-06-14 02:36:23 +01009188 WARN_ON_ONCE(sqd && sqd->thread != current);
9189
Palash Oswal6d042ff2021-04-27 18:21:49 +05309190 if (!current->io_uring)
9191 return;
Pavel Begunkov17a91052021-05-23 15:48:39 +01009192 if (tctx->io_wq)
9193 io_wq_exit_start(tctx->io_wq);
9194
Jens Axboefdaf0832020-10-30 09:37:30 -06009195 atomic_inc(&tctx->in_idle);
Jens Axboed8a6df12020-10-15 16:24:45 -06009196 do {
Pavel Begunkove9dbe222021-08-09 13:04:20 +01009197 io_uring_drop_tctx_refs(current);
Jens Axboe0f212202020-09-13 13:09:39 -06009198 /* read completions before cancelations */
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01009199 inflight = tctx_inflight(tctx, !cancel_all);
Jens Axboed8a6df12020-10-15 16:24:45 -06009200 if (!inflight)
9201 break;
Jens Axboe0f212202020-09-13 13:09:39 -06009202
Pavel Begunkov78cc6872021-06-14 02:36:23 +01009203 if (!sqd) {
9204 struct io_tctx_node *node;
9205 unsigned long index;
9206
9207 xa_for_each(&tctx->xa, index, node) {
9208 /* sqpoll task will cancel all its requests */
9209 if (node->ctx->sq_data)
9210 continue;
9211 io_uring_try_cancel_requests(node->ctx, current,
9212 cancel_all);
9213 }
9214 } else {
9215 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list)
9216 io_uring_try_cancel_requests(ctx, current,
9217 cancel_all);
9218 }
9219
9220 prepare_to_wait(&tctx->wait, &wait, TASK_UNINTERRUPTIBLE);
Pavel Begunkove9dbe222021-08-09 13:04:20 +01009221 io_uring_drop_tctx_refs(current);
Jens Axboe0f212202020-09-13 13:09:39 -06009222 /*
Pavel Begunkova1bb3cd2021-01-26 15:28:26 +00009223 * If we've seen completions, retry without waiting. This
9224 * avoids a race where a completion comes in before we did
9225 * prepare_to_wait().
Jens Axboe0f212202020-09-13 13:09:39 -06009226 */
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01009227 if (inflight == tctx_inflight(tctx, !cancel_all))
Pavel Begunkova1bb3cd2021-01-26 15:28:26 +00009228 schedule();
Pavel Begunkovf57555e2020-12-20 13:21:44 +00009229 finish_wait(&tctx->wait, &wait);
Jens Axboed8a6df12020-10-15 16:24:45 -06009230 } while (1);
Jens Axboefdaf0832020-10-30 09:37:30 -06009231 atomic_dec(&tctx->in_idle);
Pavel Begunkovde7f1d92021-01-04 20:43:29 +00009232
Pavel Begunkov8452d4a2021-02-27 11:16:46 +00009233 io_uring_clean_tctx(tctx);
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01009234 if (cancel_all) {
Pavel Begunkov3f48cf12021-04-11 01:46:27 +01009235 /* for exec all current's requests should be gone, kill tctx */
9236 __io_uring_free(current);
9237 }
Pavel Begunkov44e728b2020-06-15 10:24:04 +03009238}
9239
Hao Xuf552a272021-08-12 12:14:35 +08009240void __io_uring_cancel(bool cancel_all)
Pavel Begunkov78cc6872021-06-14 02:36:23 +01009241{
Hao Xuf552a272021-08-12 12:14:35 +08009242 io_uring_cancel_generic(cancel_all, NULL);
Pavel Begunkov78cc6872021-06-14 02:36:23 +01009243}
9244
Roman Penyaev6c5c2402019-11-28 12:53:22 +01009245static void *io_uring_validate_mmap_request(struct file *file,
9246 loff_t pgoff, size_t sz)
Jens Axboe2b188cc2019-01-07 10:46:33 -07009247{
Jens Axboe2b188cc2019-01-07 10:46:33 -07009248 struct io_ring_ctx *ctx = file->private_data;
Roman Penyaev6c5c2402019-11-28 12:53:22 +01009249 loff_t offset = pgoff << PAGE_SHIFT;
Jens Axboe2b188cc2019-01-07 10:46:33 -07009250 struct page *page;
9251 void *ptr;
9252
9253 switch (offset) {
9254 case IORING_OFF_SQ_RING:
Hristo Venev75b28af2019-08-26 17:23:46 +00009255 case IORING_OFF_CQ_RING:
9256 ptr = ctx->rings;
Jens Axboe2b188cc2019-01-07 10:46:33 -07009257 break;
9258 case IORING_OFF_SQES:
9259 ptr = ctx->sq_sqes;
9260 break;
Jens Axboe2b188cc2019-01-07 10:46:33 -07009261 default:
Roman Penyaev6c5c2402019-11-28 12:53:22 +01009262 return ERR_PTR(-EINVAL);
Jens Axboe2b188cc2019-01-07 10:46:33 -07009263 }
9264
9265 page = virt_to_head_page(ptr);
Matthew Wilcox (Oracle)a50b8542019-09-23 15:34:25 -07009266 if (sz > page_size(page))
Roman Penyaev6c5c2402019-11-28 12:53:22 +01009267 return ERR_PTR(-EINVAL);
9268
9269 return ptr;
9270}
9271
9272#ifdef CONFIG_MMU
9273
9274static int io_uring_mmap(struct file *file, struct vm_area_struct *vma)
9275{
9276 size_t sz = vma->vm_end - vma->vm_start;
9277 unsigned long pfn;
9278 void *ptr;
9279
9280 ptr = io_uring_validate_mmap_request(file, vma->vm_pgoff, sz);
9281 if (IS_ERR(ptr))
9282 return PTR_ERR(ptr);
Jens Axboe2b188cc2019-01-07 10:46:33 -07009283
9284 pfn = virt_to_phys(ptr) >> PAGE_SHIFT;
9285 return remap_pfn_range(vma, vma->vm_start, pfn, sz, vma->vm_page_prot);
9286}
9287
Roman Penyaev6c5c2402019-11-28 12:53:22 +01009288#else /* !CONFIG_MMU */
9289
9290static int io_uring_mmap(struct file *file, struct vm_area_struct *vma)
9291{
9292 return vma->vm_flags & (VM_SHARED | VM_MAYSHARE) ? 0 : -EINVAL;
9293}
9294
9295static unsigned int io_uring_nommu_mmap_capabilities(struct file *file)
9296{
9297 return NOMMU_MAP_DIRECT | NOMMU_MAP_READ | NOMMU_MAP_WRITE;
9298}
9299
9300static unsigned long io_uring_nommu_get_unmapped_area(struct file *file,
9301 unsigned long addr, unsigned long len,
9302 unsigned long pgoff, unsigned long flags)
9303{
9304 void *ptr;
9305
9306 ptr = io_uring_validate_mmap_request(file, pgoff, len);
9307 if (IS_ERR(ptr))
9308 return PTR_ERR(ptr);
9309
9310 return (unsigned long) ptr;
9311}
9312
9313#endif /* !CONFIG_MMU */
9314
Pavel Begunkovd9d05212021-01-08 20:57:25 +00009315static int io_sqpoll_wait_sq(struct io_ring_ctx *ctx)
Jens Axboe90554202020-09-03 12:12:41 -06009316{
9317 DEFINE_WAIT(wait);
9318
9319 do {
9320 if (!io_sqring_full(ctx))
9321 break;
Jens Axboe90554202020-09-03 12:12:41 -06009322 prepare_to_wait(&ctx->sqo_sq_wait, &wait, TASK_INTERRUPTIBLE);
9323
9324 if (!io_sqring_full(ctx))
9325 break;
Jens Axboe90554202020-09-03 12:12:41 -06009326 schedule();
9327 } while (!signal_pending(current));
9328
9329 finish_wait(&ctx->sqo_sq_wait, &wait);
Yang Li51993282021-03-09 14:30:41 +08009330 return 0;
Jens Axboe90554202020-09-03 12:12:41 -06009331}
9332
Hao Xuc73ebb62020-11-03 10:54:37 +08009333static int io_get_ext_arg(unsigned flags, const void __user *argp, size_t *argsz,
9334 struct __kernel_timespec __user **ts,
9335 const sigset_t __user **sig)
9336{
9337 struct io_uring_getevents_arg arg;
9338
9339 /*
9340 * If EXT_ARG isn't set, then we have no timespec and the argp pointer
9341 * is just a pointer to the sigset_t.
9342 */
9343 if (!(flags & IORING_ENTER_EXT_ARG)) {
9344 *sig = (const sigset_t __user *) argp;
9345 *ts = NULL;
9346 return 0;
9347 }
9348
9349 /*
9350 * EXT_ARG is set - ensure we agree on the size of it and copy in our
9351 * timespec and sigset_t pointers if good.
9352 */
9353 if (*argsz != sizeof(arg))
9354 return -EINVAL;
9355 if (copy_from_user(&arg, argp, sizeof(arg)))
9356 return -EFAULT;
9357 *sig = u64_to_user_ptr(arg.sigmask);
9358 *argsz = arg.sigmask_sz;
9359 *ts = u64_to_user_ptr(arg.ts);
9360 return 0;
9361}
9362
Jens Axboe2b188cc2019-01-07 10:46:33 -07009363SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
Hao Xuc73ebb62020-11-03 10:54:37 +08009364 u32, min_complete, u32, flags, const void __user *, argp,
9365 size_t, argsz)
Jens Axboe2b188cc2019-01-07 10:46:33 -07009366{
9367 struct io_ring_ctx *ctx;
Jens Axboe2b188cc2019-01-07 10:46:33 -07009368 int submitted = 0;
9369 struct fd f;
Pavel Begunkov33f993d2021-03-19 17:22:30 +00009370 long ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07009371
Jens Axboe4c6e2772020-07-01 11:29:10 -06009372 io_run_task_work();
Jens Axboeb41e9852020-02-17 09:52:41 -07009373
Pavel Begunkov33f993d2021-03-19 17:22:30 +00009374 if (unlikely(flags & ~(IORING_ENTER_GETEVENTS | IORING_ENTER_SQ_WAKEUP |
9375 IORING_ENTER_SQ_WAIT | IORING_ENTER_EXT_ARG)))
Jens Axboe2b188cc2019-01-07 10:46:33 -07009376 return -EINVAL;
9377
9378 f = fdget(fd);
Pavel Begunkov33f993d2021-03-19 17:22:30 +00009379 if (unlikely(!f.file))
Jens Axboe2b188cc2019-01-07 10:46:33 -07009380 return -EBADF;
9381
9382 ret = -EOPNOTSUPP;
Pavel Begunkov33f993d2021-03-19 17:22:30 +00009383 if (unlikely(f.file->f_op != &io_uring_fops))
Jens Axboe2b188cc2019-01-07 10:46:33 -07009384 goto out_fput;
9385
9386 ret = -ENXIO;
9387 ctx = f.file->private_data;
Pavel Begunkov33f993d2021-03-19 17:22:30 +00009388 if (unlikely(!percpu_ref_tryget(&ctx->refs)))
Jens Axboe2b188cc2019-01-07 10:46:33 -07009389 goto out_fput;
9390
Stefano Garzarella7e84e1c2020-08-27 16:58:31 +02009391 ret = -EBADFD;
Pavel Begunkov33f993d2021-03-19 17:22:30 +00009392 if (unlikely(ctx->flags & IORING_SETUP_R_DISABLED))
Stefano Garzarella7e84e1c2020-08-27 16:58:31 +02009393 goto out;
9394
Jens Axboe6c271ce2019-01-10 11:22:30 -07009395 /*
9396 * For SQ polling, the thread will do all submissions and completions.
9397 * Just return the requested submit count, and wake the thread if
9398 * we were asked to.
9399 */
Jens Axboeb2a9ead2019-09-12 14:19:16 -06009400 ret = 0;
Jens Axboe6c271ce2019-01-10 11:22:30 -07009401 if (ctx->flags & IORING_SETUP_SQPOLL) {
Pavel Begunkov90f67362021-08-09 20:18:12 +01009402 io_cqring_overflow_flush(ctx);
Pavel Begunkov89448c42020-12-17 00:24:39 +00009403
Jens Axboe21f96522021-08-14 09:04:40 -06009404 if (unlikely(ctx->sq_data->thread == NULL)) {
9405 ret = -EOWNERDEAD;
Stefan Metzmacher04147482021-03-07 11:54:29 +01009406 goto out;
Jens Axboe21f96522021-08-14 09:04:40 -06009407 }
Jens Axboe6c271ce2019-01-10 11:22:30 -07009408 if (flags & IORING_ENTER_SQ_WAKEUP)
Jens Axboe534ca6d2020-09-02 13:52:19 -06009409 wake_up(&ctx->sq_data->wait);
Pavel Begunkovd9d05212021-01-08 20:57:25 +00009410 if (flags & IORING_ENTER_SQ_WAIT) {
9411 ret = io_sqpoll_wait_sq(ctx);
9412 if (ret)
9413 goto out;
9414 }
Jens Axboe6c271ce2019-01-10 11:22:30 -07009415 submitted = to_submit;
Jens Axboeb2a9ead2019-09-12 14:19:16 -06009416 } else if (to_submit) {
Pavel Begunkoveef51da2021-06-14 02:36:15 +01009417 ret = io_uring_add_tctx_node(ctx);
Jens Axboe0f212202020-09-13 13:09:39 -06009418 if (unlikely(ret))
9419 goto out;
Jens Axboe2b188cc2019-01-07 10:46:33 -07009420 mutex_lock(&ctx->uring_lock);
Jens Axboe0f212202020-09-13 13:09:39 -06009421 submitted = io_submit_sqes(ctx, to_submit);
Jens Axboe2b188cc2019-01-07 10:46:33 -07009422 mutex_unlock(&ctx->uring_lock);
Pavel Begunkov7c504e652019-12-18 19:53:45 +03009423
9424 if (submitted != to_submit)
9425 goto out;
Jens Axboe2b188cc2019-01-07 10:46:33 -07009426 }
9427 if (flags & IORING_ENTER_GETEVENTS) {
Hao Xuc73ebb62020-11-03 10:54:37 +08009428 const sigset_t __user *sig;
9429 struct __kernel_timespec __user *ts;
9430
9431 ret = io_get_ext_arg(flags, argp, &argsz, &ts, &sig);
9432 if (unlikely(ret))
9433 goto out;
9434
Jens Axboe2b188cc2019-01-07 10:46:33 -07009435 min_complete = min(min_complete, ctx->cq_entries);
9436
Xiaoguang Wang32b22442020-03-11 09:26:09 +08009437 /*
9438 * When SETUP_IOPOLL and SETUP_SQPOLL are both enabled, user
9439 * space applications don't need to do io completion events
9440 * polling again, they can rely on io_sq_thread to do polling
9441 * work, which can reduce cpu usage and uring_lock contention.
9442 */
9443 if (ctx->flags & IORING_SETUP_IOPOLL &&
9444 !(ctx->flags & IORING_SETUP_SQPOLL)) {
Pavel Begunkov7668b922020-07-07 16:36:21 +03009445 ret = io_iopoll_check(ctx, min_complete);
Jens Axboedef596e2019-01-09 08:59:42 -07009446 } else {
Hao Xuc73ebb62020-11-03 10:54:37 +08009447 ret = io_cqring_wait(ctx, min_complete, sig, argsz, ts);
Jens Axboedef596e2019-01-09 08:59:42 -07009448 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07009449 }
9450
Pavel Begunkov7c504e652019-12-18 19:53:45 +03009451out:
Pavel Begunkov6805b322019-10-08 02:18:42 +03009452 percpu_ref_put(&ctx->refs);
Jens Axboe2b188cc2019-01-07 10:46:33 -07009453out_fput:
9454 fdput(f);
9455 return submitted ? submitted : ret;
9456}
9457
Tobias Klauserbebdb652020-02-26 18:38:32 +01009458#ifdef CONFIG_PROC_FS
Matthew Wilcox (Oracle)61cf9372021-03-08 14:16:16 +00009459static int io_uring_show_cred(struct seq_file *m, unsigned int id,
9460 const struct cred *cred)
Jens Axboe87ce9552020-01-30 08:25:34 -07009461{
Jens Axboe87ce9552020-01-30 08:25:34 -07009462 struct user_namespace *uns = seq_user_ns(m);
9463 struct group_info *gi;
9464 kernel_cap_t cap;
9465 unsigned __capi;
9466 int g;
9467
9468 seq_printf(m, "%5d\n", id);
9469 seq_put_decimal_ull(m, "\tUid:\t", from_kuid_munged(uns, cred->uid));
9470 seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->euid));
9471 seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->suid));
9472 seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->fsuid));
9473 seq_put_decimal_ull(m, "\n\tGid:\t", from_kgid_munged(uns, cred->gid));
9474 seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->egid));
9475 seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->sgid));
9476 seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->fsgid));
9477 seq_puts(m, "\n\tGroups:\t");
9478 gi = cred->group_info;
9479 for (g = 0; g < gi->ngroups; g++) {
9480 seq_put_decimal_ull(m, g ? " " : "",
9481 from_kgid_munged(uns, gi->gid[g]));
9482 }
9483 seq_puts(m, "\n\tCapEff:\t");
9484 cap = cred->cap_effective;
9485 CAP_FOR_EACH_U32(__capi)
9486 seq_put_hex_ll(m, NULL, cap.cap[CAP_LAST_U32 - __capi], 8);
9487 seq_putc(m, '\n');
9488 return 0;
9489}
9490
9491static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m)
9492{
Joseph Qidbbe9c62020-09-29 09:01:22 -06009493 struct io_sq_data *sq = NULL;
Jens Axboefad8e0d2020-09-28 08:57:48 -06009494 bool has_lock;
Jens Axboe87ce9552020-01-30 08:25:34 -07009495 int i;
9496
Jens Axboefad8e0d2020-09-28 08:57:48 -06009497 /*
9498 * Avoid ABBA deadlock between the seq lock and the io_uring mutex,
9499 * since fdinfo case grabs it in the opposite direction of normal use
9500 * cases. If we fail to get the lock, we just don't iterate any
9501 * structures that could be going away outside the io_uring mutex.
9502 */
9503 has_lock = mutex_trylock(&ctx->uring_lock);
9504
Jens Axboe5f3f26f2021-02-25 10:17:46 -07009505 if (has_lock && (ctx->flags & IORING_SETUP_SQPOLL)) {
Joseph Qidbbe9c62020-09-29 09:01:22 -06009506 sq = ctx->sq_data;
Jens Axboe5f3f26f2021-02-25 10:17:46 -07009507 if (!sq->thread)
9508 sq = NULL;
9509 }
Joseph Qidbbe9c62020-09-29 09:01:22 -06009510
9511 seq_printf(m, "SqThread:\t%d\n", sq ? task_pid_nr(sq->thread) : -1);
9512 seq_printf(m, "SqThreadCpu:\t%d\n", sq ? task_cpu(sq->thread) : -1);
Jens Axboe87ce9552020-01-30 08:25:34 -07009513 seq_printf(m, "UserFiles:\t%u\n", ctx->nr_user_files);
Jens Axboefad8e0d2020-09-28 08:57:48 -06009514 for (i = 0; has_lock && i < ctx->nr_user_files; i++) {
Jens Axboe7b29f922021-03-12 08:30:14 -07009515 struct file *f = io_file_from_index(ctx, i);
Jens Axboe87ce9552020-01-30 08:25:34 -07009516
Jens Axboe87ce9552020-01-30 08:25:34 -07009517 if (f)
9518 seq_printf(m, "%5u: %s\n", i, file_dentry(f)->d_iname);
9519 else
9520 seq_printf(m, "%5u: <none>\n", i);
9521 }
9522 seq_printf(m, "UserBufs:\t%u\n", ctx->nr_user_bufs);
Jens Axboefad8e0d2020-09-28 08:57:48 -06009523 for (i = 0; has_lock && i < ctx->nr_user_bufs; i++) {
Pavel Begunkov41edf1a2021-04-25 14:32:23 +01009524 struct io_mapped_ubuf *buf = ctx->user_bufs[i];
Pavel Begunkov4751f532021-04-01 15:43:55 +01009525 unsigned int len = buf->ubuf_end - buf->ubuf;
Jens Axboe87ce9552020-01-30 08:25:34 -07009526
Pavel Begunkov4751f532021-04-01 15:43:55 +01009527 seq_printf(m, "%5u: 0x%llx/%u\n", i, buf->ubuf, len);
Jens Axboe87ce9552020-01-30 08:25:34 -07009528 }
Matthew Wilcox (Oracle)61cf9372021-03-08 14:16:16 +00009529 if (has_lock && !xa_empty(&ctx->personalities)) {
9530 unsigned long index;
9531 const struct cred *cred;
9532
Jens Axboe87ce9552020-01-30 08:25:34 -07009533 seq_printf(m, "Personalities:\n");
Matthew Wilcox (Oracle)61cf9372021-03-08 14:16:16 +00009534 xa_for_each(&ctx->personalities, index, cred)
9535 io_uring_show_cred(m, index, cred);
Jens Axboe87ce9552020-01-30 08:25:34 -07009536 }
Jens Axboed7718a92020-02-14 22:23:12 -07009537 seq_printf(m, "PollList:\n");
Jens Axboe79ebeae2021-08-10 15:18:27 -06009538 spin_lock(&ctx->completion_lock);
Jens Axboed7718a92020-02-14 22:23:12 -07009539 for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) {
9540 struct hlist_head *list = &ctx->cancel_hash[i];
9541 struct io_kiocb *req;
9542
9543 hlist_for_each_entry(req, list, hash_node)
9544 seq_printf(m, " op=%d, task_works=%d\n", req->opcode,
9545 req->task->task_works != NULL);
9546 }
Jens Axboe79ebeae2021-08-10 15:18:27 -06009547 spin_unlock(&ctx->completion_lock);
Jens Axboefad8e0d2020-09-28 08:57:48 -06009548 if (has_lock)
9549 mutex_unlock(&ctx->uring_lock);
Jens Axboe87ce9552020-01-30 08:25:34 -07009550}
9551
9552static void io_uring_show_fdinfo(struct seq_file *m, struct file *f)
9553{
9554 struct io_ring_ctx *ctx = f->private_data;
9555
9556 if (percpu_ref_tryget(&ctx->refs)) {
9557 __io_uring_show_fdinfo(ctx, m);
9558 percpu_ref_put(&ctx->refs);
9559 }
9560}
Tobias Klauserbebdb652020-02-26 18:38:32 +01009561#endif
Jens Axboe87ce9552020-01-30 08:25:34 -07009562
Jens Axboe2b188cc2019-01-07 10:46:33 -07009563static const struct file_operations io_uring_fops = {
9564 .release = io_uring_release,
9565 .mmap = io_uring_mmap,
Roman Penyaev6c5c2402019-11-28 12:53:22 +01009566#ifndef CONFIG_MMU
9567 .get_unmapped_area = io_uring_nommu_get_unmapped_area,
9568 .mmap_capabilities = io_uring_nommu_mmap_capabilities,
9569#endif
Jens Axboe2b188cc2019-01-07 10:46:33 -07009570 .poll = io_uring_poll,
9571 .fasync = io_uring_fasync,
Tobias Klauserbebdb652020-02-26 18:38:32 +01009572#ifdef CONFIG_PROC_FS
Jens Axboe87ce9552020-01-30 08:25:34 -07009573 .show_fdinfo = io_uring_show_fdinfo,
Tobias Klauserbebdb652020-02-26 18:38:32 +01009574#endif
Jens Axboe2b188cc2019-01-07 10:46:33 -07009575};
9576
9577static int io_allocate_scq_urings(struct io_ring_ctx *ctx,
9578 struct io_uring_params *p)
9579{
Hristo Venev75b28af2019-08-26 17:23:46 +00009580 struct io_rings *rings;
9581 size_t size, sq_array_offset;
Jens Axboe2b188cc2019-01-07 10:46:33 -07009582
Jens Axboebd740482020-08-05 12:58:23 -06009583 /* make sure these are sane, as we already accounted them */
9584 ctx->sq_entries = p->sq_entries;
9585 ctx->cq_entries = p->cq_entries;
9586
Hristo Venev75b28af2019-08-26 17:23:46 +00009587 size = rings_size(p->sq_entries, p->cq_entries, &sq_array_offset);
9588 if (size == SIZE_MAX)
9589 return -EOVERFLOW;
9590
9591 rings = io_mem_alloc(size);
9592 if (!rings)
Jens Axboe2b188cc2019-01-07 10:46:33 -07009593 return -ENOMEM;
9594
Hristo Venev75b28af2019-08-26 17:23:46 +00009595 ctx->rings = rings;
9596 ctx->sq_array = (u32 *)((char *)rings + sq_array_offset);
9597 rings->sq_ring_mask = p->sq_entries - 1;
9598 rings->cq_ring_mask = p->cq_entries - 1;
9599 rings->sq_ring_entries = p->sq_entries;
9600 rings->cq_ring_entries = p->cq_entries;
Jens Axboe2b188cc2019-01-07 10:46:33 -07009601
9602 size = array_size(sizeof(struct io_uring_sqe), p->sq_entries);
Jens Axboeeb065d32019-11-20 09:26:29 -07009603 if (size == SIZE_MAX) {
9604 io_mem_free(ctx->rings);
9605 ctx->rings = NULL;
Jens Axboe2b188cc2019-01-07 10:46:33 -07009606 return -EOVERFLOW;
Jens Axboeeb065d32019-11-20 09:26:29 -07009607 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07009608
9609 ctx->sq_sqes = io_mem_alloc(size);
Jens Axboeeb065d32019-11-20 09:26:29 -07009610 if (!ctx->sq_sqes) {
9611 io_mem_free(ctx->rings);
9612 ctx->rings = NULL;
Jens Axboe2b188cc2019-01-07 10:46:33 -07009613 return -ENOMEM;
Jens Axboeeb065d32019-11-20 09:26:29 -07009614 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07009615
Jens Axboe2b188cc2019-01-07 10:46:33 -07009616 return 0;
9617}
9618
Pavel Begunkov9faadcc2020-12-21 18:34:05 +00009619static int io_uring_install_fd(struct io_ring_ctx *ctx, struct file *file)
9620{
9621 int ret, fd;
9622
9623 fd = get_unused_fd_flags(O_RDWR | O_CLOEXEC);
9624 if (fd < 0)
9625 return fd;
9626
Pavel Begunkoveef51da2021-06-14 02:36:15 +01009627 ret = io_uring_add_tctx_node(ctx);
Pavel Begunkov9faadcc2020-12-21 18:34:05 +00009628 if (ret) {
9629 put_unused_fd(fd);
9630 return ret;
9631 }
9632 fd_install(fd, file);
9633 return fd;
9634}
9635
Jens Axboe2b188cc2019-01-07 10:46:33 -07009636/*
9637 * Allocate an anonymous fd, this is what constitutes the application
9638 * visible backing of an io_uring instance. The application mmaps this
9639 * fd to gain access to the SQ/CQ ring details. If UNIX sockets are enabled,
9640 * we have to tie this fd to a socket for file garbage collection purposes.
9641 */
Pavel Begunkov9faadcc2020-12-21 18:34:05 +00009642static struct file *io_uring_get_file(struct io_ring_ctx *ctx)
Jens Axboe2b188cc2019-01-07 10:46:33 -07009643{
9644 struct file *file;
Jens Axboe2b188cc2019-01-07 10:46:33 -07009645#if defined(CONFIG_UNIX)
Pavel Begunkov9faadcc2020-12-21 18:34:05 +00009646 int ret;
9647
Jens Axboe2b188cc2019-01-07 10:46:33 -07009648 ret = sock_create_kern(&init_net, PF_UNIX, SOCK_RAW, IPPROTO_IP,
9649 &ctx->ring_sock);
9650 if (ret)
Pavel Begunkov9faadcc2020-12-21 18:34:05 +00009651 return ERR_PTR(ret);
Jens Axboe2b188cc2019-01-07 10:46:33 -07009652#endif
9653
Jens Axboe2b188cc2019-01-07 10:46:33 -07009654 file = anon_inode_getfile("[io_uring]", &io_uring_fops, ctx,
9655 O_RDWR | O_CLOEXEC);
Pavel Begunkov9faadcc2020-12-21 18:34:05 +00009656#if defined(CONFIG_UNIX)
Jens Axboe2b188cc2019-01-07 10:46:33 -07009657 if (IS_ERR(file)) {
Pavel Begunkov9faadcc2020-12-21 18:34:05 +00009658 sock_release(ctx->ring_sock);
9659 ctx->ring_sock = NULL;
9660 } else {
9661 ctx->ring_sock->file = file;
Jens Axboe2b188cc2019-01-07 10:46:33 -07009662 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07009663#endif
Pavel Begunkov9faadcc2020-12-21 18:34:05 +00009664 return file;
Jens Axboe2b188cc2019-01-07 10:46:33 -07009665}
9666
Xiaoguang Wang7f136572020-05-05 16:28:53 +08009667static int io_uring_create(unsigned entries, struct io_uring_params *p,
9668 struct io_uring_params __user *params)
Jens Axboe2b188cc2019-01-07 10:46:33 -07009669{
Jens Axboe2b188cc2019-01-07 10:46:33 -07009670 struct io_ring_ctx *ctx;
Pavel Begunkov9faadcc2020-12-21 18:34:05 +00009671 struct file *file;
Jens Axboe2b188cc2019-01-07 10:46:33 -07009672 int ret;
9673
Jens Axboe8110c1a2019-12-28 15:39:54 -07009674 if (!entries)
Jens Axboe2b188cc2019-01-07 10:46:33 -07009675 return -EINVAL;
Jens Axboe8110c1a2019-12-28 15:39:54 -07009676 if (entries > IORING_MAX_ENTRIES) {
9677 if (!(p->flags & IORING_SETUP_CLAMP))
9678 return -EINVAL;
9679 entries = IORING_MAX_ENTRIES;
9680 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07009681
9682 /*
9683 * Use twice as many entries for the CQ ring. It's possible for the
9684 * application to drive a higher depth than the size of the SQ ring,
9685 * since the sqes are only used at submission time. This allows for
Jens Axboe33a107f2019-10-04 12:10:03 -06009686 * some flexibility in overcommitting a bit. If the application has
9687 * set IORING_SETUP_CQSIZE, it will have passed in the desired number
9688 * of CQ ring entries manually.
Jens Axboe2b188cc2019-01-07 10:46:33 -07009689 */
9690 p->sq_entries = roundup_pow_of_two(entries);
Jens Axboe33a107f2019-10-04 12:10:03 -06009691 if (p->flags & IORING_SETUP_CQSIZE) {
9692 /*
9693 * If IORING_SETUP_CQSIZE is set, we do the same roundup
9694 * to a power-of-two, if it isn't already. We do NOT impose
9695 * any cq vs sq ring sizing.
9696 */
Joseph Qieb2667b32020-11-24 15:03:03 +08009697 if (!p->cq_entries)
Jens Axboe33a107f2019-10-04 12:10:03 -06009698 return -EINVAL;
Jens Axboe8110c1a2019-12-28 15:39:54 -07009699 if (p->cq_entries > IORING_MAX_CQ_ENTRIES) {
9700 if (!(p->flags & IORING_SETUP_CLAMP))
9701 return -EINVAL;
9702 p->cq_entries = IORING_MAX_CQ_ENTRIES;
9703 }
Joseph Qieb2667b32020-11-24 15:03:03 +08009704 p->cq_entries = roundup_pow_of_two(p->cq_entries);
9705 if (p->cq_entries < p->sq_entries)
9706 return -EINVAL;
Jens Axboe33a107f2019-10-04 12:10:03 -06009707 } else {
9708 p->cq_entries = 2 * p->sq_entries;
9709 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07009710
Jens Axboe2b188cc2019-01-07 10:46:33 -07009711 ctx = io_ring_ctx_alloc(p);
Jens Axboe62e398b2021-02-21 16:19:37 -07009712 if (!ctx)
Jens Axboe2b188cc2019-01-07 10:46:33 -07009713 return -ENOMEM;
Jens Axboe2b188cc2019-01-07 10:46:33 -07009714 ctx->compat = in_compat_syscall();
Jens Axboe62e398b2021-02-21 16:19:37 -07009715 if (!capable(CAP_IPC_LOCK))
9716 ctx->user = get_uid(current_user());
Jens Axboe2aede0e2020-09-14 10:45:53 -06009717
9718 /*
9719 * This is just grabbed for accounting purposes. When a process exits,
9720 * the mm is exited and dropped before the files, hence we need to hang
9721 * on to this mm purely for the purposes of being able to unaccount
9722 * memory (locked/pinned vm). It's not used for anything else.
9723 */
Jens Axboe6b7898e2020-08-25 07:58:00 -06009724 mmgrab(current->mm);
Jens Axboe2aede0e2020-09-14 10:45:53 -06009725 ctx->mm_account = current->mm;
Jens Axboe6b7898e2020-08-25 07:58:00 -06009726
Jens Axboe2b188cc2019-01-07 10:46:33 -07009727 ret = io_allocate_scq_urings(ctx, p);
9728 if (ret)
9729 goto err;
9730
Stefano Garzarella7e84e1c2020-08-27 16:58:31 +02009731 ret = io_sq_offload_create(ctx, p);
Jens Axboe2b188cc2019-01-07 10:46:33 -07009732 if (ret)
9733 goto err;
Pavel Begunkoveae071c2021-04-25 14:32:24 +01009734 /* always set a rsrc node */
Pavel Begunkov47b228c2021-04-29 11:46:48 +01009735 ret = io_rsrc_node_switch_start(ctx);
9736 if (ret)
9737 goto err;
Pavel Begunkoveae071c2021-04-25 14:32:24 +01009738 io_rsrc_node_switch(ctx, NULL);
Jens Axboe2b188cc2019-01-07 10:46:33 -07009739
Jens Axboe2b188cc2019-01-07 10:46:33 -07009740 memset(&p->sq_off, 0, sizeof(p->sq_off));
Hristo Venev75b28af2019-08-26 17:23:46 +00009741 p->sq_off.head = offsetof(struct io_rings, sq.head);
9742 p->sq_off.tail = offsetof(struct io_rings, sq.tail);
9743 p->sq_off.ring_mask = offsetof(struct io_rings, sq_ring_mask);
9744 p->sq_off.ring_entries = offsetof(struct io_rings, sq_ring_entries);
9745 p->sq_off.flags = offsetof(struct io_rings, sq_flags);
9746 p->sq_off.dropped = offsetof(struct io_rings, sq_dropped);
9747 p->sq_off.array = (char *)ctx->sq_array - (char *)ctx->rings;
Jens Axboe2b188cc2019-01-07 10:46:33 -07009748
9749 memset(&p->cq_off, 0, sizeof(p->cq_off));
Hristo Venev75b28af2019-08-26 17:23:46 +00009750 p->cq_off.head = offsetof(struct io_rings, cq.head);
9751 p->cq_off.tail = offsetof(struct io_rings, cq.tail);
9752 p->cq_off.ring_mask = offsetof(struct io_rings, cq_ring_mask);
9753 p->cq_off.ring_entries = offsetof(struct io_rings, cq_ring_entries);
9754 p->cq_off.overflow = offsetof(struct io_rings, cq_overflow);
9755 p->cq_off.cqes = offsetof(struct io_rings, cqes);
Stefano Garzarella0d9b5b32020-05-15 18:38:04 +02009756 p->cq_off.flags = offsetof(struct io_rings, cq_flags);
Jens Axboeac90f242019-09-06 10:26:21 -06009757
Xiaoguang Wang7f136572020-05-05 16:28:53 +08009758 p->features = IORING_FEAT_SINGLE_MMAP | IORING_FEAT_NODROP |
9759 IORING_FEAT_SUBMIT_STABLE | IORING_FEAT_RW_CUR_POS |
Jiufei Xue5769a352020-06-17 17:53:55 +08009760 IORING_FEAT_CUR_PERSONALITY | IORING_FEAT_FAST_POLL |
Hao Xuc73ebb62020-11-03 10:54:37 +08009761 IORING_FEAT_POLL_32BITS | IORING_FEAT_SQPOLL_NONFIXED |
Pavel Begunkov96905572021-06-10 16:37:38 +01009762 IORING_FEAT_EXT_ARG | IORING_FEAT_NATIVE_WORKERS |
9763 IORING_FEAT_RSRC_TAGS;
Xiaoguang Wang7f136572020-05-05 16:28:53 +08009764
9765 if (copy_to_user(params, p, sizeof(*p))) {
9766 ret = -EFAULT;
9767 goto err;
9768 }
Jens Axboed1719f72020-07-30 13:43:53 -06009769
Pavel Begunkov9faadcc2020-12-21 18:34:05 +00009770 file = io_uring_get_file(ctx);
9771 if (IS_ERR(file)) {
9772 ret = PTR_ERR(file);
9773 goto err;
9774 }
9775
Jens Axboed1719f72020-07-30 13:43:53 -06009776 /*
Jens Axboe044c1ab2019-10-28 09:15:33 -06009777 * Install ring fd as the very last thing, so we don't risk someone
9778 * having closed it before we finish setup
9779 */
Pavel Begunkov9faadcc2020-12-21 18:34:05 +00009780 ret = io_uring_install_fd(ctx, file);
9781 if (ret < 0) {
9782 /* fput will clean it up */
9783 fput(file);
9784 return ret;
9785 }
Jens Axboe044c1ab2019-10-28 09:15:33 -06009786
Dmitrii Dolgovc826bd72019-10-15 19:02:01 +02009787 trace_io_uring_create(ret, ctx, p->sq_entries, p->cq_entries, p->flags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07009788 return ret;
9789err:
9790 io_ring_ctx_wait_and_kill(ctx);
9791 return ret;
9792}
9793
9794/*
9795 * Sets up an aio uring context, and returns the fd. Applications asks for a
9796 * ring size, we return the actual sq/cq ring sizes (among other things) in the
9797 * params structure passed in.
9798 */
9799static long io_uring_setup(u32 entries, struct io_uring_params __user *params)
9800{
9801 struct io_uring_params p;
Jens Axboe2b188cc2019-01-07 10:46:33 -07009802 int i;
9803
9804 if (copy_from_user(&p, params, sizeof(p)))
9805 return -EFAULT;
9806 for (i = 0; i < ARRAY_SIZE(p.resv); i++) {
9807 if (p.resv[i])
9808 return -EINVAL;
9809 }
9810
Jens Axboe6c271ce2019-01-10 11:22:30 -07009811 if (p.flags & ~(IORING_SETUP_IOPOLL | IORING_SETUP_SQPOLL |
Jens Axboe8110c1a2019-12-28 15:39:54 -07009812 IORING_SETUP_SQ_AFF | IORING_SETUP_CQSIZE |
Stefano Garzarella7e84e1c2020-08-27 16:58:31 +02009813 IORING_SETUP_CLAMP | IORING_SETUP_ATTACH_WQ |
9814 IORING_SETUP_R_DISABLED))
Jens Axboe2b188cc2019-01-07 10:46:33 -07009815 return -EINVAL;
9816
Xiaoguang Wang7f136572020-05-05 16:28:53 +08009817 return io_uring_create(entries, &p, params);
Jens Axboe2b188cc2019-01-07 10:46:33 -07009818}
9819
9820SYSCALL_DEFINE2(io_uring_setup, u32, entries,
9821 struct io_uring_params __user *, params)
9822{
9823 return io_uring_setup(entries, params);
9824}
9825
Jens Axboe66f4af92020-01-16 15:36:52 -07009826static int io_probe(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args)
9827{
9828 struct io_uring_probe *p;
9829 size_t size;
9830 int i, ret;
9831
9832 size = struct_size(p, ops, nr_args);
9833 if (size == SIZE_MAX)
9834 return -EOVERFLOW;
9835 p = kzalloc(size, GFP_KERNEL);
9836 if (!p)
9837 return -ENOMEM;
9838
9839 ret = -EFAULT;
9840 if (copy_from_user(p, arg, size))
9841 goto out;
9842 ret = -EINVAL;
9843 if (memchr_inv(p, 0, size))
9844 goto out;
9845
9846 p->last_op = IORING_OP_LAST - 1;
9847 if (nr_args > IORING_OP_LAST)
9848 nr_args = IORING_OP_LAST;
9849
9850 for (i = 0; i < nr_args; i++) {
9851 p->ops[i].op = i;
9852 if (!io_op_defs[i].not_supported)
9853 p->ops[i].flags = IO_URING_OP_SUPPORTED;
9854 }
9855 p->ops_len = i;
9856
9857 ret = 0;
9858 if (copy_to_user(arg, p, size))
9859 ret = -EFAULT;
9860out:
9861 kfree(p);
9862 return ret;
9863}
9864
Jens Axboe071698e2020-01-28 10:04:42 -07009865static int io_register_personality(struct io_ring_ctx *ctx)
9866{
Jens Axboe4379bf82021-02-15 13:40:22 -07009867 const struct cred *creds;
Matthew Wilcox (Oracle)61cf9372021-03-08 14:16:16 +00009868 u32 id;
Jens Axboe1e6fa522020-10-15 08:46:24 -06009869 int ret;
Jens Axboe071698e2020-01-28 10:04:42 -07009870
Jens Axboe4379bf82021-02-15 13:40:22 -07009871 creds = get_current_cred();
Jens Axboe1e6fa522020-10-15 08:46:24 -06009872
Matthew Wilcox (Oracle)61cf9372021-03-08 14:16:16 +00009873 ret = xa_alloc_cyclic(&ctx->personalities, &id, (void *)creds,
9874 XA_LIMIT(0, USHRT_MAX), &ctx->pers_next, GFP_KERNEL);
Jens Axboea30f8952021-08-20 14:53:59 -06009875 if (ret < 0) {
9876 put_cred(creds);
9877 return ret;
9878 }
9879 return id;
Jens Axboe071698e2020-01-28 10:04:42 -07009880}
9881
Stefano Garzarella21b55db2020-08-27 16:58:30 +02009882static int io_register_restrictions(struct io_ring_ctx *ctx, void __user *arg,
9883 unsigned int nr_args)
9884{
9885 struct io_uring_restriction *res;
9886 size_t size;
9887 int i, ret;
9888
Stefano Garzarella7e84e1c2020-08-27 16:58:31 +02009889 /* Restrictions allowed only if rings started disabled */
9890 if (!(ctx->flags & IORING_SETUP_R_DISABLED))
9891 return -EBADFD;
9892
Stefano Garzarella21b55db2020-08-27 16:58:30 +02009893 /* We allow only a single restrictions registration */
Stefano Garzarella7e84e1c2020-08-27 16:58:31 +02009894 if (ctx->restrictions.registered)
Stefano Garzarella21b55db2020-08-27 16:58:30 +02009895 return -EBUSY;
9896
9897 if (!arg || nr_args > IORING_MAX_RESTRICTIONS)
9898 return -EINVAL;
9899
9900 size = array_size(nr_args, sizeof(*res));
9901 if (size == SIZE_MAX)
9902 return -EOVERFLOW;
9903
9904 res = memdup_user(arg, size);
9905 if (IS_ERR(res))
9906 return PTR_ERR(res);
9907
9908 ret = 0;
9909
9910 for (i = 0; i < nr_args; i++) {
9911 switch (res[i].opcode) {
9912 case IORING_RESTRICTION_REGISTER_OP:
9913 if (res[i].register_op >= IORING_REGISTER_LAST) {
9914 ret = -EINVAL;
9915 goto out;
9916 }
9917
9918 __set_bit(res[i].register_op,
9919 ctx->restrictions.register_op);
9920 break;
9921 case IORING_RESTRICTION_SQE_OP:
9922 if (res[i].sqe_op >= IORING_OP_LAST) {
9923 ret = -EINVAL;
9924 goto out;
9925 }
9926
9927 __set_bit(res[i].sqe_op, ctx->restrictions.sqe_op);
9928 break;
9929 case IORING_RESTRICTION_SQE_FLAGS_ALLOWED:
9930 ctx->restrictions.sqe_flags_allowed = res[i].sqe_flags;
9931 break;
9932 case IORING_RESTRICTION_SQE_FLAGS_REQUIRED:
9933 ctx->restrictions.sqe_flags_required = res[i].sqe_flags;
9934 break;
9935 default:
9936 ret = -EINVAL;
9937 goto out;
9938 }
9939 }
9940
9941out:
9942 /* Reset all restrictions if an error happened */
9943 if (ret != 0)
9944 memset(&ctx->restrictions, 0, sizeof(ctx->restrictions));
9945 else
Stefano Garzarella7e84e1c2020-08-27 16:58:31 +02009946 ctx->restrictions.registered = true;
Stefano Garzarella21b55db2020-08-27 16:58:30 +02009947
9948 kfree(res);
9949 return ret;
9950}
9951
Stefano Garzarella7e84e1c2020-08-27 16:58:31 +02009952static int io_register_enable_rings(struct io_ring_ctx *ctx)
9953{
9954 if (!(ctx->flags & IORING_SETUP_R_DISABLED))
9955 return -EBADFD;
9956
9957 if (ctx->restrictions.registered)
9958 ctx->restricted = 1;
9959
Pavel Begunkov0298ef92021-03-08 13:20:57 +00009960 ctx->flags &= ~IORING_SETUP_R_DISABLED;
9961 if (ctx->sq_data && wq_has_sleeper(&ctx->sq_data->wait))
9962 wake_up(&ctx->sq_data->wait);
Stefano Garzarella7e84e1c2020-08-27 16:58:31 +02009963 return 0;
9964}
9965
Pavel Begunkovfdecb662021-04-25 14:32:20 +01009966static int __io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned type,
Pavel Begunkovc3bdad02021-04-25 14:32:22 +01009967 struct io_uring_rsrc_update2 *up,
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +01009968 unsigned nr_args)
9969{
9970 __u32 tmp;
9971 int err;
9972
Pavel Begunkovc3bdad02021-04-25 14:32:22 +01009973 if (up->resv)
9974 return -EINVAL;
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +01009975 if (check_add_overflow(up->offset, nr_args, &tmp))
9976 return -EOVERFLOW;
9977 err = io_rsrc_node_switch_start(ctx);
9978 if (err)
9979 return err;
9980
Pavel Begunkovfdecb662021-04-25 14:32:20 +01009981 switch (type) {
9982 case IORING_RSRC_FILE:
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +01009983 return __io_sqe_files_update(ctx, up, nr_args);
Pavel Begunkov634d00d2021-04-25 14:32:26 +01009984 case IORING_RSRC_BUFFER:
9985 return __io_sqe_buffers_update(ctx, up, nr_args);
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +01009986 }
9987 return -EINVAL;
9988}
9989
Pavel Begunkovc3bdad02021-04-25 14:32:22 +01009990static int io_register_files_update(struct io_ring_ctx *ctx, void __user *arg,
9991 unsigned nr_args)
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +01009992{
Pavel Begunkovc3bdad02021-04-25 14:32:22 +01009993 struct io_uring_rsrc_update2 up;
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +01009994
9995 if (!nr_args)
9996 return -EINVAL;
Pavel Begunkovc3bdad02021-04-25 14:32:22 +01009997 memset(&up, 0, sizeof(up));
9998 if (copy_from_user(&up, arg, sizeof(struct io_uring_rsrc_update)))
9999 return -EFAULT;
10000 return __io_register_rsrc_update(ctx, IORING_RSRC_FILE, &up, nr_args);
10001}
10002
10003static int io_register_rsrc_update(struct io_ring_ctx *ctx, void __user *arg,
Pavel Begunkov992da012021-06-10 16:37:37 +010010004 unsigned size, unsigned type)
Pavel Begunkovc3bdad02021-04-25 14:32:22 +010010005{
10006 struct io_uring_rsrc_update2 up;
10007
10008 if (size != sizeof(up))
10009 return -EINVAL;
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +010010010 if (copy_from_user(&up, arg, sizeof(up)))
10011 return -EFAULT;
Pavel Begunkov992da012021-06-10 16:37:37 +010010012 if (!up.nr || up.resv)
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +010010013 return -EINVAL;
Pavel Begunkov992da012021-06-10 16:37:37 +010010014 return __io_register_rsrc_update(ctx, type, &up, up.nr);
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +010010015}
10016
Pavel Begunkov792e3582021-04-25 14:32:21 +010010017static int io_register_rsrc(struct io_ring_ctx *ctx, void __user *arg,
Pavel Begunkov992da012021-06-10 16:37:37 +010010018 unsigned int size, unsigned int type)
Pavel Begunkov792e3582021-04-25 14:32:21 +010010019{
10020 struct io_uring_rsrc_register rr;
10021
10022 /* keep it extendible */
10023 if (size != sizeof(rr))
10024 return -EINVAL;
10025
10026 memset(&rr, 0, sizeof(rr));
10027 if (copy_from_user(&rr, arg, size))
10028 return -EFAULT;
Pavel Begunkov992da012021-06-10 16:37:37 +010010029 if (!rr.nr || rr.resv || rr.resv2)
Pavel Begunkov792e3582021-04-25 14:32:21 +010010030 return -EINVAL;
10031
Pavel Begunkov992da012021-06-10 16:37:37 +010010032 switch (type) {
Pavel Begunkov792e3582021-04-25 14:32:21 +010010033 case IORING_RSRC_FILE:
10034 return io_sqe_files_register(ctx, u64_to_user_ptr(rr.data),
10035 rr.nr, u64_to_user_ptr(rr.tags));
Pavel Begunkov634d00d2021-04-25 14:32:26 +010010036 case IORING_RSRC_BUFFER:
10037 return io_sqe_buffers_register(ctx, u64_to_user_ptr(rr.data),
10038 rr.nr, u64_to_user_ptr(rr.tags));
Pavel Begunkov792e3582021-04-25 14:32:21 +010010039 }
10040 return -EINVAL;
10041}
10042
Jens Axboefe764212021-06-17 10:19:54 -060010043static int io_register_iowq_aff(struct io_ring_ctx *ctx, void __user *arg,
10044 unsigned len)
10045{
10046 struct io_uring_task *tctx = current->io_uring;
10047 cpumask_var_t new_mask;
10048 int ret;
10049
10050 if (!tctx || !tctx->io_wq)
10051 return -EINVAL;
10052
10053 if (!alloc_cpumask_var(&new_mask, GFP_KERNEL))
10054 return -ENOMEM;
10055
10056 cpumask_clear(new_mask);
10057 if (len > cpumask_size())
10058 len = cpumask_size();
10059
10060 if (copy_from_user(new_mask, arg, len)) {
10061 free_cpumask_var(new_mask);
10062 return -EFAULT;
10063 }
10064
10065 ret = io_wq_cpu_affinity(tctx->io_wq, new_mask);
10066 free_cpumask_var(new_mask);
10067 return ret;
10068}
10069
10070static int io_unregister_iowq_aff(struct io_ring_ctx *ctx)
10071{
10072 struct io_uring_task *tctx = current->io_uring;
10073
10074 if (!tctx || !tctx->io_wq)
10075 return -EINVAL;
10076
10077 return io_wq_cpu_affinity(tctx->io_wq, NULL);
10078}
10079
Jens Axboe071698e2020-01-28 10:04:42 -070010080static bool io_register_op_must_quiesce(int op)
10081{
10082 switch (op) {
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +010010083 case IORING_REGISTER_BUFFERS:
10084 case IORING_UNREGISTER_BUFFERS:
Pavel Begunkovf4f7d212021-04-01 15:44:02 +010010085 case IORING_REGISTER_FILES:
Jens Axboe071698e2020-01-28 10:04:42 -070010086 case IORING_UNREGISTER_FILES:
10087 case IORING_REGISTER_FILES_UPDATE:
10088 case IORING_REGISTER_PROBE:
10089 case IORING_REGISTER_PERSONALITY:
10090 case IORING_UNREGISTER_PERSONALITY:
Pavel Begunkov992da012021-06-10 16:37:37 +010010091 case IORING_REGISTER_FILES2:
10092 case IORING_REGISTER_FILES_UPDATE2:
10093 case IORING_REGISTER_BUFFERS2:
10094 case IORING_REGISTER_BUFFERS_UPDATE:
Jens Axboefe764212021-06-17 10:19:54 -060010095 case IORING_REGISTER_IOWQ_AFF:
10096 case IORING_UNREGISTER_IOWQ_AFF:
Jens Axboe071698e2020-01-28 10:04:42 -070010097 return false;
10098 default:
10099 return true;
10100 }
10101}
10102
Pavel Begunkove73c5c72021-08-09 13:04:12 +010010103static int io_ctx_quiesce(struct io_ring_ctx *ctx)
10104{
10105 long ret;
10106
10107 percpu_ref_kill(&ctx->refs);
10108
10109 /*
10110 * Drop uring mutex before waiting for references to exit. If another
10111 * thread is currently inside io_uring_enter() it might need to grab the
10112 * uring_lock to make progress. If we hold it here across the drain
10113 * wait, then we can deadlock. It's safe to drop the mutex here, since
10114 * no new references will come in after we've killed the percpu ref.
10115 */
10116 mutex_unlock(&ctx->uring_lock);
10117 do {
10118 ret = wait_for_completion_interruptible(&ctx->ref_comp);
10119 if (!ret)
10120 break;
10121 ret = io_run_task_work_sig();
10122 } while (ret >= 0);
10123 mutex_lock(&ctx->uring_lock);
10124
10125 if (ret)
10126 io_refs_resurrect(&ctx->refs, &ctx->ref_comp);
10127 return ret;
10128}
10129
Jens Axboeedafcce2019-01-09 09:16:05 -070010130static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,
10131 void __user *arg, unsigned nr_args)
Jens Axboeb19062a2019-04-15 10:49:38 -060010132 __releases(ctx->uring_lock)
10133 __acquires(ctx->uring_lock)
Jens Axboeedafcce2019-01-09 09:16:05 -070010134{
10135 int ret;
10136
Jens Axboe35fa71a2019-04-22 10:23:23 -060010137 /*
10138 * We're inside the ring mutex, if the ref is already dying, then
10139 * someone else killed the ctx or is already going through
10140 * io_uring_register().
10141 */
10142 if (percpu_ref_is_dying(&ctx->refs))
10143 return -ENXIO;
10144
Pavel Begunkov75c40212021-04-15 13:07:40 +010010145 if (ctx->restricted) {
10146 if (opcode >= IORING_REGISTER_LAST)
10147 return -EINVAL;
10148 opcode = array_index_nospec(opcode, IORING_REGISTER_LAST);
10149 if (!test_bit(opcode, ctx->restrictions.register_op))
10150 return -EACCES;
10151 }
10152
Jens Axboe071698e2020-01-28 10:04:42 -070010153 if (io_register_op_must_quiesce(opcode)) {
Pavel Begunkove73c5c72021-08-09 13:04:12 +010010154 ret = io_ctx_quiesce(ctx);
10155 if (ret)
Pavel Begunkovf70865d2021-04-11 01:46:40 +010010156 return ret;
Jens Axboe05f3fb32019-12-09 11:22:50 -070010157 }
Jens Axboeedafcce2019-01-09 09:16:05 -070010158
10159 switch (opcode) {
10160 case IORING_REGISTER_BUFFERS:
Pavel Begunkov634d00d2021-04-25 14:32:26 +010010161 ret = io_sqe_buffers_register(ctx, arg, nr_args, NULL);
Jens Axboeedafcce2019-01-09 09:16:05 -070010162 break;
10163 case IORING_UNREGISTER_BUFFERS:
10164 ret = -EINVAL;
10165 if (arg || nr_args)
10166 break;
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -080010167 ret = io_sqe_buffers_unregister(ctx);
Jens Axboeedafcce2019-01-09 09:16:05 -070010168 break;
Jens Axboe6b063142019-01-10 22:13:58 -070010169 case IORING_REGISTER_FILES:
Pavel Begunkov792e3582021-04-25 14:32:21 +010010170 ret = io_sqe_files_register(ctx, arg, nr_args, NULL);
Jens Axboe6b063142019-01-10 22:13:58 -070010171 break;
10172 case IORING_UNREGISTER_FILES:
10173 ret = -EINVAL;
10174 if (arg || nr_args)
10175 break;
10176 ret = io_sqe_files_unregister(ctx);
10177 break;
Jens Axboec3a31e62019-10-03 13:59:56 -060010178 case IORING_REGISTER_FILES_UPDATE:
Pavel Begunkovc3bdad02021-04-25 14:32:22 +010010179 ret = io_register_files_update(ctx, arg, nr_args);
Jens Axboec3a31e62019-10-03 13:59:56 -060010180 break;
Jens Axboe9b402842019-04-11 11:45:41 -060010181 case IORING_REGISTER_EVENTFD:
Jens Axboef2842ab2020-01-08 11:04:00 -070010182 case IORING_REGISTER_EVENTFD_ASYNC:
Jens Axboe9b402842019-04-11 11:45:41 -060010183 ret = -EINVAL;
10184 if (nr_args != 1)
10185 break;
10186 ret = io_eventfd_register(ctx, arg);
Jens Axboef2842ab2020-01-08 11:04:00 -070010187 if (ret)
10188 break;
10189 if (opcode == IORING_REGISTER_EVENTFD_ASYNC)
10190 ctx->eventfd_async = 1;
10191 else
10192 ctx->eventfd_async = 0;
Jens Axboe9b402842019-04-11 11:45:41 -060010193 break;
10194 case IORING_UNREGISTER_EVENTFD:
10195 ret = -EINVAL;
10196 if (arg || nr_args)
10197 break;
10198 ret = io_eventfd_unregister(ctx);
10199 break;
Jens Axboe66f4af92020-01-16 15:36:52 -070010200 case IORING_REGISTER_PROBE:
10201 ret = -EINVAL;
10202 if (!arg || nr_args > 256)
10203 break;
10204 ret = io_probe(ctx, arg, nr_args);
10205 break;
Jens Axboe071698e2020-01-28 10:04:42 -070010206 case IORING_REGISTER_PERSONALITY:
10207 ret = -EINVAL;
10208 if (arg || nr_args)
10209 break;
10210 ret = io_register_personality(ctx);
10211 break;
10212 case IORING_UNREGISTER_PERSONALITY:
10213 ret = -EINVAL;
10214 if (arg)
10215 break;
10216 ret = io_unregister_personality(ctx, nr_args);
10217 break;
Stefano Garzarella7e84e1c2020-08-27 16:58:31 +020010218 case IORING_REGISTER_ENABLE_RINGS:
10219 ret = -EINVAL;
10220 if (arg || nr_args)
10221 break;
10222 ret = io_register_enable_rings(ctx);
10223 break;
Stefano Garzarella21b55db2020-08-27 16:58:30 +020010224 case IORING_REGISTER_RESTRICTIONS:
10225 ret = io_register_restrictions(ctx, arg, nr_args);
10226 break;
Pavel Begunkov992da012021-06-10 16:37:37 +010010227 case IORING_REGISTER_FILES2:
10228 ret = io_register_rsrc(ctx, arg, nr_args, IORING_RSRC_FILE);
Pavel Begunkov792e3582021-04-25 14:32:21 +010010229 break;
Pavel Begunkov992da012021-06-10 16:37:37 +010010230 case IORING_REGISTER_FILES_UPDATE2:
10231 ret = io_register_rsrc_update(ctx, arg, nr_args,
10232 IORING_RSRC_FILE);
10233 break;
10234 case IORING_REGISTER_BUFFERS2:
10235 ret = io_register_rsrc(ctx, arg, nr_args, IORING_RSRC_BUFFER);
10236 break;
10237 case IORING_REGISTER_BUFFERS_UPDATE:
10238 ret = io_register_rsrc_update(ctx, arg, nr_args,
10239 IORING_RSRC_BUFFER);
Pavel Begunkovc3bdad02021-04-25 14:32:22 +010010240 break;
Jens Axboefe764212021-06-17 10:19:54 -060010241 case IORING_REGISTER_IOWQ_AFF:
10242 ret = -EINVAL;
10243 if (!arg || !nr_args)
10244 break;
10245 ret = io_register_iowq_aff(ctx, arg, nr_args);
10246 break;
10247 case IORING_UNREGISTER_IOWQ_AFF:
10248 ret = -EINVAL;
10249 if (arg || nr_args)
10250 break;
10251 ret = io_unregister_iowq_aff(ctx);
10252 break;
Jens Axboeedafcce2019-01-09 09:16:05 -070010253 default:
10254 ret = -EINVAL;
10255 break;
10256 }
10257
Jens Axboe071698e2020-01-28 10:04:42 -070010258 if (io_register_op_must_quiesce(opcode)) {
Jens Axboe05f3fb32019-12-09 11:22:50 -070010259 /* bring the ctx back to life */
Jens Axboe05f3fb32019-12-09 11:22:50 -070010260 percpu_ref_reinit(&ctx->refs);
Jens Axboe0f158b42020-05-14 17:18:39 -060010261 reinit_completion(&ctx->ref_comp);
Jens Axboe05f3fb32019-12-09 11:22:50 -070010262 }
Jens Axboeedafcce2019-01-09 09:16:05 -070010263 return ret;
10264}
10265
10266SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode,
10267 void __user *, arg, unsigned int, nr_args)
10268{
10269 struct io_ring_ctx *ctx;
10270 long ret = -EBADF;
10271 struct fd f;
10272
10273 f = fdget(fd);
10274 if (!f.file)
10275 return -EBADF;
10276
10277 ret = -EOPNOTSUPP;
10278 if (f.file->f_op != &io_uring_fops)
10279 goto out_fput;
10280
10281 ctx = f.file->private_data;
10282
Pavel Begunkovb6c23dd2021-02-20 15:17:18 +000010283 io_run_task_work();
10284
Jens Axboeedafcce2019-01-09 09:16:05 -070010285 mutex_lock(&ctx->uring_lock);
10286 ret = __io_uring_register(ctx, opcode, arg, nr_args);
10287 mutex_unlock(&ctx->uring_lock);
Dmitrii Dolgovc826bd72019-10-15 19:02:01 +020010288 trace_io_uring_register(ctx, opcode, ctx->nr_user_files, ctx->nr_user_bufs,
10289 ctx->cq_ev_fd != NULL, ret);
Jens Axboeedafcce2019-01-09 09:16:05 -070010290out_fput:
10291 fdput(f);
10292 return ret;
10293}
10294
Jens Axboe2b188cc2019-01-07 10:46:33 -070010295static int __init io_uring_init(void)
10296{
Stefan Metzmacherd7f62e82020-01-29 14:39:41 +010010297#define __BUILD_BUG_VERIFY_ELEMENT(stype, eoffset, etype, ename) do { \
10298 BUILD_BUG_ON(offsetof(stype, ename) != eoffset); \
10299 BUILD_BUG_ON(sizeof(etype) != sizeof_field(stype, ename)); \
10300} while (0)
10301
10302#define BUILD_BUG_SQE_ELEM(eoffset, etype, ename) \
10303 __BUILD_BUG_VERIFY_ELEMENT(struct io_uring_sqe, eoffset, etype, ename)
10304 BUILD_BUG_ON(sizeof(struct io_uring_sqe) != 64);
10305 BUILD_BUG_SQE_ELEM(0, __u8, opcode);
10306 BUILD_BUG_SQE_ELEM(1, __u8, flags);
10307 BUILD_BUG_SQE_ELEM(2, __u16, ioprio);
10308 BUILD_BUG_SQE_ELEM(4, __s32, fd);
10309 BUILD_BUG_SQE_ELEM(8, __u64, off);
10310 BUILD_BUG_SQE_ELEM(8, __u64, addr2);
10311 BUILD_BUG_SQE_ELEM(16, __u64, addr);
Pavel Begunkov7d67af22020-02-24 11:32:45 +030010312 BUILD_BUG_SQE_ELEM(16, __u64, splice_off_in);
Stefan Metzmacherd7f62e82020-01-29 14:39:41 +010010313 BUILD_BUG_SQE_ELEM(24, __u32, len);
10314 BUILD_BUG_SQE_ELEM(28, __kernel_rwf_t, rw_flags);
10315 BUILD_BUG_SQE_ELEM(28, /* compat */ int, rw_flags);
10316 BUILD_BUG_SQE_ELEM(28, /* compat */ __u32, rw_flags);
10317 BUILD_BUG_SQE_ELEM(28, __u32, fsync_flags);
Jiufei Xue5769a352020-06-17 17:53:55 +080010318 BUILD_BUG_SQE_ELEM(28, /* compat */ __u16, poll_events);
10319 BUILD_BUG_SQE_ELEM(28, __u32, poll32_events);
Stefan Metzmacherd7f62e82020-01-29 14:39:41 +010010320 BUILD_BUG_SQE_ELEM(28, __u32, sync_range_flags);
10321 BUILD_BUG_SQE_ELEM(28, __u32, msg_flags);
10322 BUILD_BUG_SQE_ELEM(28, __u32, timeout_flags);
10323 BUILD_BUG_SQE_ELEM(28, __u32, accept_flags);
10324 BUILD_BUG_SQE_ELEM(28, __u32, cancel_flags);
10325 BUILD_BUG_SQE_ELEM(28, __u32, open_flags);
10326 BUILD_BUG_SQE_ELEM(28, __u32, statx_flags);
10327 BUILD_BUG_SQE_ELEM(28, __u32, fadvise_advice);
Pavel Begunkov7d67af22020-02-24 11:32:45 +030010328 BUILD_BUG_SQE_ELEM(28, __u32, splice_flags);
Stefan Metzmacherd7f62e82020-01-29 14:39:41 +010010329 BUILD_BUG_SQE_ELEM(32, __u64, user_data);
10330 BUILD_BUG_SQE_ELEM(40, __u16, buf_index);
Pavel Begunkov16340ea2021-06-24 15:09:58 +010010331 BUILD_BUG_SQE_ELEM(40, __u16, buf_group);
Stefan Metzmacherd7f62e82020-01-29 14:39:41 +010010332 BUILD_BUG_SQE_ELEM(42, __u16, personality);
Pavel Begunkov7d67af22020-02-24 11:32:45 +030010333 BUILD_BUG_SQE_ELEM(44, __s32, splice_fd_in);
Stefan Metzmacherd7f62e82020-01-29 14:39:41 +010010334
Pavel Begunkovb0d658ec2021-04-27 16:13:53 +010010335 BUILD_BUG_ON(sizeof(struct io_uring_files_update) !=
10336 sizeof(struct io_uring_rsrc_update));
10337 BUILD_BUG_ON(sizeof(struct io_uring_rsrc_update) >
10338 sizeof(struct io_uring_rsrc_update2));
10339 /* should fit into one byte */
10340 BUILD_BUG_ON(SQE_VALID_FLAGS >= (1 << 8));
10341
Jens Axboed3656342019-12-18 09:50:26 -070010342 BUILD_BUG_ON(ARRAY_SIZE(io_op_defs) != IORING_OP_LAST);
Jens Axboe84557872020-03-03 15:28:17 -070010343 BUILD_BUG_ON(__REQ_F_LAST_BIT >= 8 * sizeof(int));
Pavel Begunkov16340ea2021-06-24 15:09:58 +010010344
Jens Axboe91f245d2021-02-09 13:48:50 -070010345 req_cachep = KMEM_CACHE(io_kiocb, SLAB_HWCACHE_ALIGN | SLAB_PANIC |
10346 SLAB_ACCOUNT);
Jens Axboe2b188cc2019-01-07 10:46:33 -070010347 return 0;
10348};
10349__initcall(io_uring_init);