blob: 8d263209b5085606debdf7cca1cf6e2185b4869d [file] [log] [blame]
Jens Axboe2b188cc2019-01-07 10:46:33 -07001// SPDX-License-Identifier: GPL-2.0
2/*
3 * Shared application/kernel submission and completion ring pairs, for
4 * supporting fast/efficient IO.
5 *
6 * A note on the read/write ordering memory barriers that are matched between
Stefan Bühler1e84b972019-04-24 23:54:16 +02007 * the application and kernel side.
8 *
9 * After the application reads the CQ ring tail, it must use an
10 * appropriate smp_rmb() to pair with the smp_wmb() the kernel uses
11 * before writing the tail (using smp_load_acquire to read the tail will
12 * do). It also needs a smp_mb() before updating CQ head (ordering the
13 * entry load(s) with the head store), pairing with an implicit barrier
Pavel Begunkovd068b502021-05-16 22:58:11 +010014 * through a control-dependency in io_get_cqe (smp_store_release to
Stefan Bühler1e84b972019-04-24 23:54:16 +020015 * store head will do). Failure to do so could lead to reading invalid
16 * CQ entries.
17 *
18 * Likewise, the application must use an appropriate smp_wmb() before
19 * writing the SQ tail (ordering SQ entry stores with the tail store),
20 * which pairs with smp_load_acquire in io_get_sqring (smp_store_release
21 * to store the tail will do). And it needs a barrier ordering the SQ
22 * head load before writing new SQ entries (smp_load_acquire to read
23 * head will do).
24 *
25 * When using the SQ poll thread (IORING_SETUP_SQPOLL), the application
26 * needs to check the SQ flags for IORING_SQ_NEED_WAKEUP *after*
27 * updating the SQ tail; a full memory barrier smp_mb() is needed
28 * between.
Jens Axboe2b188cc2019-01-07 10:46:33 -070029 *
30 * Also see the examples in the liburing library:
31 *
32 * git://git.kernel.dk/liburing
33 *
34 * io_uring also uses READ/WRITE_ONCE() for _any_ store or load that happens
35 * from data shared between the kernel and application. This is done both
36 * for ordering purposes, but also to ensure that once a value is loaded from
37 * data that the application could potentially modify, it remains stable.
38 *
39 * Copyright (C) 2018-2019 Jens Axboe
Christoph Hellwigc992fe22019-01-11 09:43:02 -070040 * Copyright (c) 2018-2019 Christoph Hellwig
Jens Axboe2b188cc2019-01-07 10:46:33 -070041 */
42#include <linux/kernel.h>
43#include <linux/init.h>
44#include <linux/errno.h>
45#include <linux/syscalls.h>
46#include <linux/compat.h>
Jens Axboe52de1fe2020-02-27 10:15:42 -070047#include <net/compat.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070048#include <linux/refcount.h>
49#include <linux/uio.h>
Pavel Begunkov6b47ee62020-01-18 20:22:41 +030050#include <linux/bits.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070051
52#include <linux/sched/signal.h>
53#include <linux/fs.h>
54#include <linux/file.h>
55#include <linux/fdtable.h>
56#include <linux/mm.h>
57#include <linux/mman.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070058#include <linux/percpu.h>
59#include <linux/slab.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070060#include <linux/blkdev.h>
Jens Axboeedafcce2019-01-09 09:16:05 -070061#include <linux/bvec.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070062#include <linux/net.h>
63#include <net/sock.h>
64#include <net/af_unix.h>
Jens Axboe6b063142019-01-10 22:13:58 -070065#include <net/scm.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070066#include <linux/anon_inodes.h>
67#include <linux/sched/mm.h>
68#include <linux/uaccess.h>
69#include <linux/nospec.h>
Jens Axboeedafcce2019-01-09 09:16:05 -070070#include <linux/sizes.h>
71#include <linux/hugetlb.h>
Jens Axboeaa4c3962019-11-29 10:14:00 -070072#include <linux/highmem.h>
Jens Axboe15b71ab2019-12-11 11:20:36 -070073#include <linux/namei.h>
74#include <linux/fsnotify.h>
Jens Axboe4840e412019-12-25 22:03:45 -070075#include <linux/fadvise.h>
Jens Axboe3e4827b2020-01-08 15:18:09 -070076#include <linux/eventpoll.h>
Pavel Begunkov7d67af22020-02-24 11:32:45 +030077#include <linux/splice.h>
Jens Axboeb41e9852020-02-17 09:52:41 -070078#include <linux/task_work.h>
Jens Axboebcf5a062020-05-22 09:24:42 -060079#include <linux/pagemap.h>
Jens Axboe0f212202020-09-13 13:09:39 -060080#include <linux/io_uring.h>
Nadav Amitef98eb02021-08-07 17:13:41 -070081#include <linux/tracehook.h>
Jens Axboe2b188cc2019-01-07 10:46:33 -070082
Dmitrii Dolgovc826bd72019-10-15 19:02:01 +020083#define CREATE_TRACE_POINTS
84#include <trace/events/io_uring.h>
85
Jens Axboe2b188cc2019-01-07 10:46:33 -070086#include <uapi/linux/io_uring.h>
87
88#include "internal.h"
Jens Axboe561fb042019-10-24 07:25:42 -060089#include "io-wq.h"
Jens Axboe2b188cc2019-01-07 10:46:33 -070090
Daniel Xu5277dea2019-09-14 14:23:45 -070091#define IORING_MAX_ENTRIES 32768
Jens Axboe33a107f2019-10-04 12:10:03 -060092#define IORING_MAX_CQ_ENTRIES (2 * IORING_MAX_ENTRIES)
Olivier Langlois4ce8ad92021-06-23 11:50:18 -070093#define IORING_SQPOLL_CAP_ENTRIES_VALUE 8
Jens Axboe65e19f52019-10-26 07:20:21 -060094
wangyangbo187f08c2021-08-19 13:56:57 +080095/* only define max */
Pavel Begunkov042b0d82021-08-09 13:04:01 +010096#define IORING_MAX_FIXED_FILES (1U << 15)
Stefano Garzarella21b55db2020-08-27 16:58:30 +020097#define IORING_MAX_RESTRICTIONS (IORING_RESTRICTION_LAST + \
98 IORING_REGISTER_LAST + IORING_OP_LAST)
Jens Axboe2b188cc2019-01-07 10:46:33 -070099
wangyangbo187f08c2021-08-19 13:56:57 +0800100#define IO_RSRC_TAG_TABLE_SHIFT (PAGE_SHIFT - 3)
Pavel Begunkov2d091d62021-06-14 02:36:21 +0100101#define IO_RSRC_TAG_TABLE_MAX (1U << IO_RSRC_TAG_TABLE_SHIFT)
102#define IO_RSRC_TAG_TABLE_MASK (IO_RSRC_TAG_TABLE_MAX - 1)
103
Pavel Begunkov489809e2021-05-14 12:06:44 +0100104#define IORING_MAX_REG_BUFFERS (1U << 14)
105
Pavel Begunkovb16fed662021-02-18 18:29:40 +0000106#define SQE_VALID_FLAGS (IOSQE_FIXED_FILE|IOSQE_IO_DRAIN|IOSQE_IO_LINK| \
107 IOSQE_IO_HARDLINK | IOSQE_ASYNC | \
108 IOSQE_BUFFER_SELECT)
Pavel Begunkovc8543572021-06-17 18:14:04 +0100109#define IO_REQ_CLEAN_FLAGS (REQ_F_BUFFER_SELECTED | REQ_F_NEED_CLEANUP | \
110 REQ_F_POLLED | REQ_F_INFLIGHT | REQ_F_CREDS)
Pavel Begunkovb16fed662021-02-18 18:29:40 +0000111
Pavel Begunkov09899b12021-06-14 02:36:22 +0100112#define IO_TCTX_REFS_CACHE_NR (1U << 10)
113
Jens Axboe2b188cc2019-01-07 10:46:33 -0700114struct io_uring {
115 u32 head ____cacheline_aligned_in_smp;
116 u32 tail ____cacheline_aligned_in_smp;
117};
118
Stefan Bühler1e84b972019-04-24 23:54:16 +0200119/*
Hristo Venev75b28af2019-08-26 17:23:46 +0000120 * This data is shared with the application through the mmap at offsets
121 * IORING_OFF_SQ_RING and IORING_OFF_CQ_RING.
Stefan Bühler1e84b972019-04-24 23:54:16 +0200122 *
123 * The offsets to the member fields are published through struct
124 * io_sqring_offsets when calling io_uring_setup.
125 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000126struct io_rings {
Stefan Bühler1e84b972019-04-24 23:54:16 +0200127 /*
128 * Head and tail offsets into the ring; the offsets need to be
129 * masked to get valid indices.
130 *
Hristo Venev75b28af2019-08-26 17:23:46 +0000131 * The kernel controls head of the sq ring and the tail of the cq ring,
132 * and the application controls tail of the sq ring and the head of the
133 * cq ring.
Stefan Bühler1e84b972019-04-24 23:54:16 +0200134 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000135 struct io_uring sq, cq;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200136 /*
Hristo Venev75b28af2019-08-26 17:23:46 +0000137 * Bitmasks to apply to head and tail offsets (constant, equals
Stefan Bühler1e84b972019-04-24 23:54:16 +0200138 * ring_entries - 1)
139 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000140 u32 sq_ring_mask, cq_ring_mask;
141 /* Ring sizes (constant, power of 2) */
142 u32 sq_ring_entries, cq_ring_entries;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200143 /*
144 * Number of invalid entries dropped by the kernel due to
145 * invalid index stored in array
146 *
147 * Written by the kernel, shouldn't be modified by the
148 * application (i.e. get number of "new events" by comparing to
149 * cached value).
150 *
151 * After a new SQ head value was read by the application this
152 * counter includes all submissions that were dropped reaching
153 * the new SQ head (and possibly more).
154 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000155 u32 sq_dropped;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200156 /*
Stefano Garzarella0d9b5b32020-05-15 18:38:04 +0200157 * Runtime SQ flags
Stefan Bühler1e84b972019-04-24 23:54:16 +0200158 *
159 * Written by the kernel, shouldn't be modified by the
160 * application.
161 *
162 * The application needs a full memory barrier before checking
163 * for IORING_SQ_NEED_WAKEUP after updating the sq tail.
164 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000165 u32 sq_flags;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200166 /*
Stefano Garzarella0d9b5b32020-05-15 18:38:04 +0200167 * Runtime CQ flags
168 *
169 * Written by the application, shouldn't be modified by the
170 * kernel.
171 */
Pavel Begunkovfe7e3252021-06-24 15:09:57 +0100172 u32 cq_flags;
Stefano Garzarella0d9b5b32020-05-15 18:38:04 +0200173 /*
Stefan Bühler1e84b972019-04-24 23:54:16 +0200174 * Number of completion events lost because the queue was full;
175 * this should be avoided by the application by making sure
LimingWu0b4295b2019-12-05 20:18:18 +0800176 * there are not more requests pending than there is space in
Stefan Bühler1e84b972019-04-24 23:54:16 +0200177 * the completion queue.
178 *
179 * Written by the kernel, shouldn't be modified by the
180 * application (i.e. get number of "new events" by comparing to
181 * cached value).
182 *
183 * As completion events come in out of order this counter is not
184 * ordered with any other data.
185 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000186 u32 cq_overflow;
Stefan Bühler1e84b972019-04-24 23:54:16 +0200187 /*
188 * Ring buffer of completion events.
189 *
190 * The kernel writes completion events fresh every time they are
191 * produced, so the application is allowed to modify pending
192 * entries.
193 */
Hristo Venev75b28af2019-08-26 17:23:46 +0000194 struct io_uring_cqe cqes[] ____cacheline_aligned_in_smp;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700195};
196
Pavel Begunkov45d189c2021-02-10 00:03:07 +0000197enum io_uring_cmd_flags {
198 IO_URING_F_NONBLOCK = 1,
Pavel Begunkov889fca72021-02-10 00:03:09 +0000199 IO_URING_F_COMPLETE_DEFER = 2,
Pavel Begunkov45d189c2021-02-10 00:03:07 +0000200};
201
Jens Axboeedafcce2019-01-09 09:16:05 -0700202struct io_mapped_ubuf {
203 u64 ubuf;
Pavel Begunkov4751f532021-04-01 15:43:55 +0100204 u64 ubuf_end;
Jens Axboeedafcce2019-01-09 09:16:05 -0700205 unsigned int nr_bvecs;
Jens Axboede293932020-09-17 16:19:16 -0600206 unsigned long acct_pages;
Pavel Begunkov41edf1a2021-04-25 14:32:23 +0100207 struct bio_vec bvec[];
Jens Axboeedafcce2019-01-09 09:16:05 -0700208};
209
Bijan Mottahedeh50238532021-01-15 17:37:45 +0000210struct io_ring_ctx;
211
Pavel Begunkov6c2450a2021-02-23 12:40:22 +0000212struct io_overflow_cqe {
213 struct io_uring_cqe cqe;
214 struct list_head list;
215};
216
Pavel Begunkova04b0ac2021-04-01 15:44:04 +0100217struct io_fixed_file {
218 /* file * with additional FFS_* flags */
219 unsigned long file_ptr;
220};
221
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +0000222struct io_rsrc_put {
223 struct list_head list;
Pavel Begunkovb60c8dc2021-04-25 14:32:18 +0100224 u64 tag;
Bijan Mottahedeh50238532021-01-15 17:37:45 +0000225 union {
226 void *rsrc;
227 struct file *file;
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +0100228 struct io_mapped_ubuf *buf;
Bijan Mottahedeh50238532021-01-15 17:37:45 +0000229 };
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +0000230};
231
Pavel Begunkovaeca2412021-04-11 01:46:37 +0100232struct io_file_table {
Pavel Begunkov042b0d82021-08-09 13:04:01 +0100233 struct io_fixed_file *files;
Jens Axboe31b51512019-01-18 22:56:34 -0700234};
235
Pavel Begunkovb895c9a2021-04-01 15:43:40 +0100236struct io_rsrc_node {
Xiaoguang Wang05589552020-03-31 14:05:18 +0800237 struct percpu_ref refs;
238 struct list_head node;
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +0000239 struct list_head rsrc_list;
Pavel Begunkovb895c9a2021-04-01 15:43:40 +0100240 struct io_rsrc_data *rsrc_data;
Jens Axboe4a38aed22020-05-14 17:21:15 -0600241 struct llist_node llist;
Pavel Begunkove2978222020-11-18 14:56:26 +0000242 bool done;
Xiaoguang Wang05589552020-03-31 14:05:18 +0800243};
244
Pavel Begunkov40ae0ff2021-04-01 15:43:44 +0100245typedef void (rsrc_put_fn)(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc);
246
Pavel Begunkovb895c9a2021-04-01 15:43:40 +0100247struct io_rsrc_data {
Jens Axboe05f3fb32019-12-09 11:22:50 -0700248 struct io_ring_ctx *ctx;
249
Pavel Begunkov2d091d62021-06-14 02:36:21 +0100250 u64 **tags;
251 unsigned int nr;
Pavel Begunkov40ae0ff2021-04-01 15:43:44 +0100252 rsrc_put_fn *do_put;
Pavel Begunkov3e942492021-04-11 01:46:34 +0100253 atomic_t refs;
Jens Axboe05f3fb32019-12-09 11:22:50 -0700254 struct completion done;
Hao Xu8bad28d2021-02-19 17:19:36 +0800255 bool quiesce;
Jens Axboe05f3fb32019-12-09 11:22:50 -0700256};
257
Jens Axboe5a2e7452020-02-23 16:23:11 -0700258struct io_buffer {
259 struct list_head list;
260 __u64 addr;
Thadeu Lima de Souza Cascardod1f82802021-05-05 09:47:06 -0300261 __u32 len;
Jens Axboe5a2e7452020-02-23 16:23:11 -0700262 __u16 bid;
263};
264
Stefano Garzarella21b55db2020-08-27 16:58:30 +0200265struct io_restriction {
266 DECLARE_BITMAP(register_op, IORING_REGISTER_LAST);
267 DECLARE_BITMAP(sqe_op, IORING_OP_LAST);
268 u8 sqe_flags_allowed;
269 u8 sqe_flags_required;
Stefano Garzarella7e84e1c2020-08-27 16:58:31 +0200270 bool registered;
Stefano Garzarella21b55db2020-08-27 16:58:30 +0200271};
272
Jens Axboe37d1e2e2021-02-17 21:03:43 -0700273enum {
274 IO_SQ_THREAD_SHOULD_STOP = 0,
275 IO_SQ_THREAD_SHOULD_PARK,
276};
277
Jens Axboe534ca6d2020-09-02 13:52:19 -0600278struct io_sq_data {
279 refcount_t refs;
Pavel Begunkov9e138a42021-03-14 20:57:12 +0000280 atomic_t park_pending;
Pavel Begunkov09a6f4e2021-03-14 20:57:10 +0000281 struct mutex lock;
Jens Axboe69fb2132020-09-14 11:16:23 -0600282
283 /* ctx's that are using this sqd */
284 struct list_head ctx_list;
Jens Axboe69fb2132020-09-14 11:16:23 -0600285
Jens Axboe534ca6d2020-09-02 13:52:19 -0600286 struct task_struct *thread;
287 struct wait_queue_head wait;
Xiaoguang Wang08369242020-11-03 14:15:59 +0800288
289 unsigned sq_thread_idle;
Jens Axboe37d1e2e2021-02-17 21:03:43 -0700290 int sq_cpu;
291 pid_t task_pid;
Jens Axboe5c2469e2021-03-11 10:17:56 -0700292 pid_t task_tgid;
Jens Axboe37d1e2e2021-02-17 21:03:43 -0700293
294 unsigned long state;
Jens Axboe37d1e2e2021-02-17 21:03:43 -0700295 struct completion exited;
Jens Axboe534ca6d2020-09-02 13:52:19 -0600296};
297
Pavel Begunkov6dd0be12021-02-10 00:03:13 +0000298#define IO_COMPL_BATCH 32
Pavel Begunkov6ff119a2021-02-10 00:03:18 +0000299#define IO_REQ_CACHE_SIZE 32
Pavel Begunkovbf019da2021-02-10 00:03:17 +0000300#define IO_REQ_ALLOC_BATCH 8
Pavel Begunkov258b29a2021-02-10 00:03:10 +0000301
Pavel Begunkova1ab7b32021-02-18 18:29:42 +0000302struct io_submit_link {
303 struct io_kiocb *head;
304 struct io_kiocb *last;
305};
306
Pavel Begunkov258b29a2021-02-10 00:03:10 +0000307struct io_submit_state {
308 struct blk_plug plug;
Pavel Begunkova1ab7b32021-02-18 18:29:42 +0000309 struct io_submit_link link;
Pavel Begunkov258b29a2021-02-10 00:03:10 +0000310
311 /*
312 * io_kiocb alloc cache
313 */
Pavel Begunkovbf019da2021-02-10 00:03:17 +0000314 void *reqs[IO_REQ_CACHE_SIZE];
Pavel Begunkov258b29a2021-02-10 00:03:10 +0000315 unsigned int free_reqs;
316
317 bool plug_started;
318
319 /*
320 * Batch completion logic
321 */
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +0100322 struct io_kiocb *compl_reqs[IO_COMPL_BATCH];
323 unsigned int compl_nr;
324 /* inline/task_work completion list, under ->uring_lock */
325 struct list_head free_list;
Pavel Begunkov258b29a2021-02-10 00:03:10 +0000326
Pavel Begunkov258b29a2021-02-10 00:03:10 +0000327 unsigned int ios_left;
328};
329
Jens Axboe2b188cc2019-01-07 10:46:33 -0700330struct io_ring_ctx {
Pavel Begunkovb52ecf82021-06-14 23:37:21 +0100331 /* const or read-mostly hot data */
Jens Axboe2b188cc2019-01-07 10:46:33 -0700332 struct {
333 struct percpu_ref refs;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700334
Pavel Begunkovb52ecf82021-06-14 23:37:21 +0100335 struct io_rings *rings;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700336 unsigned int flags;
Randy Dunlape1d85332020-02-05 20:57:10 -0800337 unsigned int compat: 1;
Randy Dunlape1d85332020-02-05 20:57:10 -0800338 unsigned int drain_next: 1;
339 unsigned int eventfd_async: 1;
Stefano Garzarella21b55db2020-08-27 16:58:30 +0200340 unsigned int restricted: 1;
Pavel Begunkovf18ee4c2021-06-14 23:37:25 +0100341 unsigned int off_timeout_used: 1;
Pavel Begunkov10c66902021-06-15 16:47:56 +0100342 unsigned int drain_active: 1;
Pavel Begunkovb52ecf82021-06-14 23:37:21 +0100343 } ____cacheline_aligned_in_smp;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700344
Pavel Begunkov7f1129d2021-06-14 23:37:22 +0100345 /* submission data */
Pavel Begunkovb52ecf82021-06-14 23:37:21 +0100346 struct {
Pavel Begunkov0499e582021-06-14 23:37:29 +0100347 struct mutex uring_lock;
348
Hristo Venev75b28af2019-08-26 17:23:46 +0000349 /*
350 * Ring buffer of indices into array of io_uring_sqe, which is
351 * mmapped by the application using the IORING_OFF_SQES offset.
352 *
353 * This indirection could e.g. be used to assign fixed
354 * io_uring_sqe entries to operations and only submit them to
355 * the queue when needed.
356 *
357 * The kernel modifies neither the indices array nor the entries
358 * array.
359 */
360 u32 *sq_array;
Pavel Begunkovc7af47c2021-06-14 23:37:20 +0100361 struct io_uring_sqe *sq_sqes;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700362 unsigned cached_sq_head;
363 unsigned sq_entries;
Jens Axboede0617e2019-04-06 21:51:27 -0600364 struct list_head defer_list;
Pavel Begunkov7f1129d2021-06-14 23:37:22 +0100365
366 /*
367 * Fixed resources fast path, should be accessed only under
368 * uring_lock, and updated through io_uring_register(2)
369 */
370 struct io_rsrc_node *rsrc_node;
371 struct io_file_table file_table;
372 unsigned nr_user_files;
373 unsigned nr_user_bufs;
374 struct io_mapped_ubuf **user_bufs;
375
376 struct io_submit_state submit_state;
Jens Axboe5262f562019-09-17 12:26:57 -0600377 struct list_head timeout_list;
Jens Axboe1d7bb1d2019-11-06 11:31:17 -0700378 struct list_head cq_overflow_list;
Pavel Begunkov7f1129d2021-06-14 23:37:22 +0100379 struct xarray io_buffers;
380 struct xarray personalities;
381 u32 pers_next;
382 unsigned sq_thread_idle;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700383 } ____cacheline_aligned_in_smp;
384
Pavel Begunkovd0acdee2021-05-16 22:58:12 +0100385 /* IRQ completion list, under ->completion_lock */
386 struct list_head locked_free_list;
387 unsigned int locked_free_nr;
Jens Axboe3c1a2ea2021-02-11 10:48:03 -0700388
Stefan Metzmacher7c30f36a2021-03-07 11:54:28 +0100389 const struct cred *sq_creds; /* cred used for __io_sq_thread() */
Jens Axboe534ca6d2020-09-02 13:52:19 -0600390 struct io_sq_data *sq_data; /* if using sq thread polling */
391
Jens Axboe90554202020-09-03 12:12:41 -0600392 struct wait_queue_head sqo_sq_wait;
Jens Axboe69fb2132020-09-14 11:16:23 -0600393 struct list_head sqd_list;
Hristo Venev75b28af2019-08-26 17:23:46 +0000394
Pavel Begunkov5ed7a372021-06-14 23:37:27 +0100395 unsigned long check_cq_overflow;
396
Jens Axboe206aefd2019-11-07 18:27:42 -0700397 struct {
398 unsigned cached_cq_tail;
399 unsigned cq_entries;
Jens Axboe206aefd2019-11-07 18:27:42 -0700400 struct eventfd_ctx *cq_ev_fd;
Pavel Begunkov0499e582021-06-14 23:37:29 +0100401 struct wait_queue_head poll_wait;
402 struct wait_queue_head cq_wait;
403 unsigned cq_extra;
404 atomic_t cq_timeouts;
405 struct fasync_struct *cq_fasync;
406 unsigned cq_last_tm_flush;
Jens Axboe206aefd2019-11-07 18:27:42 -0700407 } ____cacheline_aligned_in_smp;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700408
409 struct {
Jens Axboe2b188cc2019-01-07 10:46:33 -0700410 spinlock_t completion_lock;
Jens Axboee94f1412019-12-19 12:06:02 -0700411
Jens Axboe89850fc2021-08-10 15:11:51 -0600412 spinlock_t timeout_lock;
413
Jens Axboedef596e2019-01-09 08:59:42 -0700414 /*
Pavel Begunkov540e32a2020-07-13 23:37:09 +0300415 * ->iopoll_list is protected by the ctx->uring_lock for
Jens Axboedef596e2019-01-09 08:59:42 -0700416 * io_uring instances that don't use IORING_SETUP_SQPOLL.
417 * For SQPOLL, only the single threaded io_sq_thread() will
418 * manipulate the list, hence no extra locking is needed there.
419 */
Pavel Begunkov540e32a2020-07-13 23:37:09 +0300420 struct list_head iopoll_list;
Jens Axboe78076bb2019-12-04 19:56:40 -0700421 struct hlist_head *cancel_hash;
422 unsigned cancel_hash_bits;
Hao Xu915b3dd2021-06-28 05:37:30 +0800423 bool poll_multi_queue;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700424 } ____cacheline_aligned_in_smp;
Jens Axboe85faa7b2020-04-09 18:14:00 -0600425
Stefano Garzarella21b55db2020-08-27 16:58:30 +0200426 struct io_restriction restrictions;
Jens Axboe3c1a2ea2021-02-11 10:48:03 -0700427
Pavel Begunkovb13a8912021-05-16 22:58:07 +0100428 /* slow path rsrc auxilary data, used by update/register */
429 struct {
430 struct io_rsrc_node *rsrc_backup_node;
431 struct io_mapped_ubuf *dummy_ubuf;
432 struct io_rsrc_data *file_data;
433 struct io_rsrc_data *buf_data;
434
435 struct delayed_work rsrc_put_work;
436 struct llist_head rsrc_put_llist;
437 struct list_head rsrc_ref_list;
438 spinlock_t rsrc_ref_lock;
439 };
440
Jens Axboe3c1a2ea2021-02-11 10:48:03 -0700441 /* Keep this last, we don't need it for the fast path */
Pavel Begunkovb986af72021-05-16 22:58:06 +0100442 struct {
443 #if defined(CONFIG_UNIX)
444 struct socket *ring_sock;
445 #endif
446 /* hashed buffered write serialization */
447 struct io_wq_hash *hash_map;
448
449 /* Only used for accounting purposes */
450 struct user_struct *user;
451 struct mm_struct *mm_account;
452
453 /* ctx exit and cancelation */
Pavel Begunkov9011bf92021-06-30 21:54:03 +0100454 struct llist_head fallback_llist;
455 struct delayed_work fallback_work;
Pavel Begunkovb986af72021-05-16 22:58:06 +0100456 struct work_struct exit_work;
457 struct list_head tctx_list;
458 struct completion ref_comp;
459 };
Jens Axboe2b188cc2019-01-07 10:46:33 -0700460};
461
Stefan Metzmacher53e043b2021-03-15 12:56:56 +0100462struct io_uring_task {
463 /* submission side */
Pavel Begunkov09899b12021-06-14 02:36:22 +0100464 int cached_refs;
Stefan Metzmacher53e043b2021-03-15 12:56:56 +0100465 struct xarray xa;
466 struct wait_queue_head wait;
Stefan Metzmacheree53fb22021-03-15 12:56:57 +0100467 const struct io_ring_ctx *last;
468 struct io_wq *io_wq;
Stefan Metzmacher53e043b2021-03-15 12:56:56 +0100469 struct percpu_counter inflight;
Pavel Begunkovb303fe22021-04-11 01:46:26 +0100470 atomic_t inflight_tracked;
Stefan Metzmacher53e043b2021-03-15 12:56:56 +0100471 atomic_t in_idle;
Stefan Metzmacher53e043b2021-03-15 12:56:56 +0100472
473 spinlock_t task_lock;
474 struct io_wq_work_list task_list;
Stefan Metzmacher53e043b2021-03-15 12:56:56 +0100475 struct callback_head task_work;
Pavel Begunkov6294f362021-08-10 17:53:55 +0100476 bool task_running;
Stefan Metzmacher53e043b2021-03-15 12:56:56 +0100477};
478
Jens Axboe09bb8392019-03-13 12:39:28 -0600479/*
480 * First field must be the file pointer in all the
481 * iocb unions! See also 'struct kiocb' in <linux/fs.h>
482 */
Jens Axboe221c5eb2019-01-17 09:41:58 -0700483struct io_poll_iocb {
484 struct file *file;
Pavel Begunkov018043b2020-10-27 23:17:18 +0000485 struct wait_queue_head *head;
Jens Axboe221c5eb2019-01-17 09:41:58 -0700486 __poll_t events;
Jens Axboe8c838782019-03-12 15:48:16 -0600487 bool done;
Jens Axboe221c5eb2019-01-17 09:41:58 -0700488 bool canceled;
Jens Axboe392edb42019-12-09 17:52:20 -0700489 struct wait_queue_entry wait;
Jens Axboe221c5eb2019-01-17 09:41:58 -0700490};
491
Pavel Begunkov9d805892021-04-13 02:58:40 +0100492struct io_poll_update {
Pavel Begunkov018043b2020-10-27 23:17:18 +0000493 struct file *file;
Pavel Begunkov9d805892021-04-13 02:58:40 +0100494 u64 old_user_data;
495 u64 new_user_data;
496 __poll_t events;
Jens Axboeb69de282021-03-17 08:37:41 -0600497 bool update_events;
498 bool update_user_data;
Pavel Begunkov018043b2020-10-27 23:17:18 +0000499};
500
Jens Axboeb5dba592019-12-11 14:02:38 -0700501struct io_close {
502 struct file *file;
Jens Axboeb5dba592019-12-11 14:02:38 -0700503 int fd;
504};
505
Jens Axboead8a48a2019-11-15 08:49:11 -0700506struct io_timeout_data {
507 struct io_kiocb *req;
508 struct hrtimer timer;
509 struct timespec64 ts;
510 enum hrtimer_mode mode;
511};
512
Jens Axboe8ed8d3c2019-12-16 11:55:28 -0700513struct io_accept {
514 struct file *file;
515 struct sockaddr __user *addr;
516 int __user *addr_len;
517 int flags;
Jens Axboe09952e32020-03-19 20:16:56 -0600518 unsigned long nofile;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -0700519};
520
521struct io_sync {
522 struct file *file;
523 loff_t len;
524 loff_t off;
525 int flags;
Jens Axboed63d1b52019-12-10 10:38:56 -0700526 int mode;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -0700527};
528
Jens Axboefbf23842019-12-17 18:45:56 -0700529struct io_cancel {
530 struct file *file;
531 u64 addr;
532};
533
Jens Axboeb29472e2019-12-17 18:50:29 -0700534struct io_timeout {
535 struct file *file;
Pavel Begunkovbfe68a22020-05-30 14:54:18 +0300536 u32 off;
537 u32 target_seq;
Pavel Begunkov135fcde2020-07-13 23:37:12 +0300538 struct list_head list;
Pavel Begunkov90cd7e42020-10-27 23:25:36 +0000539 /* head of the link, used by linked timeouts only */
540 struct io_kiocb *head;
Jens Axboe89b263f2021-08-10 15:14:18 -0600541 /* for linked completions */
542 struct io_kiocb *prev;
Jens Axboeb29472e2019-12-17 18:50:29 -0700543};
544
Pavel Begunkov0bdf7a22020-10-10 18:34:10 +0100545struct io_timeout_rem {
546 struct file *file;
547 u64 addr;
Pavel Begunkov9c8e11b2020-11-30 19:11:16 +0000548
549 /* timeout update */
550 struct timespec64 ts;
551 u32 flags;
Pavel Begunkov0bdf7a22020-10-10 18:34:10 +0100552};
553
Jens Axboe9adbd452019-12-20 08:45:55 -0700554struct io_rw {
555 /* NOTE: kiocb has the file as the first member, so don't do it here */
556 struct kiocb kiocb;
557 u64 addr;
558 u64 len;
559};
560
Jens Axboe3fbb51c2019-12-20 08:51:52 -0700561struct io_connect {
562 struct file *file;
563 struct sockaddr __user *addr;
564 int addr_len;
565};
566
Jens Axboee47293f2019-12-20 08:58:21 -0700567struct io_sr_msg {
568 struct file *file;
Jens Axboefddafac2020-01-04 20:19:44 -0700569 union {
Pavel Begunkov4af34172021-04-11 01:46:30 +0100570 struct compat_msghdr __user *umsg_compat;
571 struct user_msghdr __user *umsg;
572 void __user *buf;
Jens Axboefddafac2020-01-04 20:19:44 -0700573 };
Jens Axboee47293f2019-12-20 08:58:21 -0700574 int msg_flags;
Jens Axboebcda7ba2020-02-23 16:42:51 -0700575 int bgid;
Jens Axboefddafac2020-01-04 20:19:44 -0700576 size_t len;
Jens Axboebcda7ba2020-02-23 16:42:51 -0700577 struct io_buffer *kbuf;
Jens Axboee47293f2019-12-20 08:58:21 -0700578};
579
Jens Axboe15b71ab2019-12-11 11:20:36 -0700580struct io_open {
581 struct file *file;
582 int dfd;
Jens Axboe15b71ab2019-12-11 11:20:36 -0700583 struct filename *filename;
Jens Axboec12cedf2020-01-08 17:41:21 -0700584 struct open_how how;
Jens Axboe4022e7a2020-03-19 19:23:18 -0600585 unsigned long nofile;
Jens Axboe15b71ab2019-12-11 11:20:36 -0700586};
587
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +0000588struct io_rsrc_update {
Jens Axboe05f3fb32019-12-09 11:22:50 -0700589 struct file *file;
590 u64 arg;
591 u32 nr_args;
592 u32 offset;
593};
594
Jens Axboe4840e412019-12-25 22:03:45 -0700595struct io_fadvise {
596 struct file *file;
597 u64 offset;
598 u32 len;
599 u32 advice;
600};
601
Jens Axboec1ca7572019-12-25 22:18:28 -0700602struct io_madvise {
603 struct file *file;
604 u64 addr;
605 u32 len;
606 u32 advice;
607};
608
Jens Axboe3e4827b2020-01-08 15:18:09 -0700609struct io_epoll {
610 struct file *file;
611 int epfd;
612 int op;
613 int fd;
614 struct epoll_event event;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700615};
616
Pavel Begunkov7d67af22020-02-24 11:32:45 +0300617struct io_splice {
618 struct file *file_out;
619 struct file *file_in;
620 loff_t off_out;
621 loff_t off_in;
622 u64 len;
623 unsigned int flags;
624};
625
Jens Axboeddf0322d2020-02-23 16:41:33 -0700626struct io_provide_buf {
627 struct file *file;
628 __u64 addr;
Pavel Begunkov38134ad2021-04-15 13:07:39 +0100629 __u32 len;
Jens Axboeddf0322d2020-02-23 16:41:33 -0700630 __u32 bgid;
631 __u16 nbufs;
632 __u16 bid;
633};
634
Bijan Mottahedeh1d9e1282020-05-22 21:31:16 -0700635struct io_statx {
636 struct file *file;
637 int dfd;
638 unsigned int mask;
639 unsigned int flags;
Bijan Mottahedehe62753e2020-05-22 21:31:18 -0700640 const char __user *filename;
Bijan Mottahedeh1d9e1282020-05-22 21:31:16 -0700641 struct statx __user *buffer;
642};
643
Jens Axboe36f4fa62020-09-05 11:14:22 -0600644struct io_shutdown {
645 struct file *file;
646 int how;
647};
648
Jens Axboe80a261f2020-09-28 14:23:58 -0600649struct io_rename {
650 struct file *file;
651 int old_dfd;
652 int new_dfd;
653 struct filename *oldpath;
654 struct filename *newpath;
655 int flags;
656};
657
Jens Axboe14a11432020-09-28 14:27:37 -0600658struct io_unlink {
659 struct file *file;
660 int dfd;
661 int flags;
662 struct filename *filename;
663};
664
Pavel Begunkov3ca405e2020-07-13 23:37:08 +0300665struct io_completion {
666 struct file *file;
Pavel Begunkov8c3f9cd2021-02-28 22:35:15 +0000667 u32 cflags;
Pavel Begunkov3ca405e2020-07-13 23:37:08 +0300668};
669
Jens Axboef499a022019-12-02 16:28:46 -0700670struct io_async_connect {
671 struct sockaddr_storage address;
672};
673
Jens Axboe03b12302019-12-02 18:50:25 -0700674struct io_async_msghdr {
675 struct iovec fast_iov[UIO_FASTIOV];
Pavel Begunkov257e84a2021-02-05 00:58:00 +0000676 /* points to an allocated iov, if NULL we use fast_iov instead */
677 struct iovec *free_iov;
Jens Axboe03b12302019-12-02 18:50:25 -0700678 struct sockaddr __user *uaddr;
679 struct msghdr msg;
Jens Axboeb5379162020-02-09 11:29:15 -0700680 struct sockaddr_storage addr;
Jens Axboe03b12302019-12-02 18:50:25 -0700681};
682
Jens Axboef67676d2019-12-02 11:03:47 -0700683struct io_async_rw {
684 struct iovec fast_iov[UIO_FASTIOV];
Jens Axboeff6165b2020-08-13 09:47:43 -0600685 const struct iovec *free_iovec;
686 struct iov_iter iter;
Jens Axboe227c0c92020-08-13 11:51:40 -0600687 size_t bytes_done;
Jens Axboebcf5a062020-05-22 09:24:42 -0600688 struct wait_page_queue wpq;
Jens Axboef67676d2019-12-02 11:03:47 -0700689};
690
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300691enum {
692 REQ_F_FIXED_FILE_BIT = IOSQE_FIXED_FILE_BIT,
693 REQ_F_IO_DRAIN_BIT = IOSQE_IO_DRAIN_BIT,
694 REQ_F_LINK_BIT = IOSQE_IO_LINK_BIT,
695 REQ_F_HARDLINK_BIT = IOSQE_IO_HARDLINK_BIT,
696 REQ_F_FORCE_ASYNC_BIT = IOSQE_ASYNC_BIT,
Jens Axboebcda7ba2020-02-23 16:42:51 -0700697 REQ_F_BUFFER_SELECT_BIT = IOSQE_BUFFER_SELECT_BIT,
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300698
Pavel Begunkovdddca222021-04-27 16:13:52 +0100699 /* first byte is taken by user flags, shift it to not overlap */
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +0100700 REQ_F_FAIL_BIT = 8,
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300701 REQ_F_INFLIGHT_BIT,
702 REQ_F_CUR_POS_BIT,
703 REQ_F_NOWAIT_BIT,
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300704 REQ_F_LINK_TIMEOUT_BIT,
Pavel Begunkov99bc4c32020-02-07 22:04:45 +0300705 REQ_F_NEED_CLEANUP_BIT,
Jens Axboed7718a92020-02-14 22:23:12 -0700706 REQ_F_POLLED_BIT,
Jens Axboebcda7ba2020-02-23 16:42:51 -0700707 REQ_F_BUFFER_SELECTED_BIT,
Pavel Begunkove342c802021-01-19 13:32:47 +0000708 REQ_F_COMPLETE_INLINE_BIT,
Jens Axboe230d50d2021-04-01 20:41:15 -0600709 REQ_F_REISSUE_BIT,
Pavel Begunkov8c130822021-03-22 01:58:32 +0000710 REQ_F_DONT_REISSUE_BIT,
Pavel Begunkovb8e64b52021-06-17 18:14:02 +0100711 REQ_F_CREDS_BIT,
Pavel Begunkov20e60a32021-08-11 19:28:30 +0100712 REQ_F_REFCOUNT_BIT,
Pavel Begunkov4d13d1a2021-08-15 10:40:24 +0100713 REQ_F_ARM_LTIMEOUT_BIT,
Jens Axboe7b29f922021-03-12 08:30:14 -0700714 /* keep async read/write and isreg together and in order */
Pavel Begunkovb191e2d2021-08-09 13:04:03 +0100715 REQ_F_NOWAIT_READ_BIT,
716 REQ_F_NOWAIT_WRITE_BIT,
Jens Axboe7b29f922021-03-12 08:30:14 -0700717 REQ_F_ISREG_BIT,
Jens Axboe84557872020-03-03 15:28:17 -0700718
719 /* not a real bit, just to check we're not overflowing the space */
720 __REQ_F_LAST_BIT,
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300721};
722
723enum {
724 /* ctx owns file */
725 REQ_F_FIXED_FILE = BIT(REQ_F_FIXED_FILE_BIT),
726 /* drain existing IO first */
727 REQ_F_IO_DRAIN = BIT(REQ_F_IO_DRAIN_BIT),
728 /* linked sqes */
729 REQ_F_LINK = BIT(REQ_F_LINK_BIT),
730 /* doesn't sever on completion < 0 */
731 REQ_F_HARDLINK = BIT(REQ_F_HARDLINK_BIT),
732 /* IOSQE_ASYNC */
733 REQ_F_FORCE_ASYNC = BIT(REQ_F_FORCE_ASYNC_BIT),
Jens Axboebcda7ba2020-02-23 16:42:51 -0700734 /* IOSQE_BUFFER_SELECT */
735 REQ_F_BUFFER_SELECT = BIT(REQ_F_BUFFER_SELECT_BIT),
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300736
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300737 /* fail rest of links */
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +0100738 REQ_F_FAIL = BIT(REQ_F_FAIL_BIT),
Pavel Begunkovb05a1bc2021-03-04 13:59:24 +0000739 /* on inflight list, should be cancelled and waited on exit reliably */
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300740 REQ_F_INFLIGHT = BIT(REQ_F_INFLIGHT_BIT),
741 /* read/write uses file position */
742 REQ_F_CUR_POS = BIT(REQ_F_CUR_POS_BIT),
743 /* must not punt to workers */
744 REQ_F_NOWAIT = BIT(REQ_F_NOWAIT_BIT),
Pavel Begunkov900fad42020-10-19 16:39:16 +0100745 /* has or had linked timeout */
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300746 REQ_F_LINK_TIMEOUT = BIT(REQ_F_LINK_TIMEOUT_BIT),
Pavel Begunkov99bc4c32020-02-07 22:04:45 +0300747 /* needs cleanup */
748 REQ_F_NEED_CLEANUP = BIT(REQ_F_NEED_CLEANUP_BIT),
Jens Axboed7718a92020-02-14 22:23:12 -0700749 /* already went through poll handler */
750 REQ_F_POLLED = BIT(REQ_F_POLLED_BIT),
Jens Axboebcda7ba2020-02-23 16:42:51 -0700751 /* buffer already selected */
752 REQ_F_BUFFER_SELECTED = BIT(REQ_F_BUFFER_SELECTED_BIT),
Pavel Begunkove342c802021-01-19 13:32:47 +0000753 /* completion is deferred through io_comp_state */
754 REQ_F_COMPLETE_INLINE = BIT(REQ_F_COMPLETE_INLINE_BIT),
Jens Axboe230d50d2021-04-01 20:41:15 -0600755 /* caller should reissue async */
756 REQ_F_REISSUE = BIT(REQ_F_REISSUE_BIT),
Pavel Begunkov8c130822021-03-22 01:58:32 +0000757 /* don't attempt request reissue, see io_rw_reissue() */
758 REQ_F_DONT_REISSUE = BIT(REQ_F_DONT_REISSUE_BIT),
Jens Axboe7b29f922021-03-12 08:30:14 -0700759 /* supports async reads */
Pavel Begunkovb191e2d2021-08-09 13:04:03 +0100760 REQ_F_NOWAIT_READ = BIT(REQ_F_NOWAIT_READ_BIT),
Jens Axboe7b29f922021-03-12 08:30:14 -0700761 /* supports async writes */
Pavel Begunkovb191e2d2021-08-09 13:04:03 +0100762 REQ_F_NOWAIT_WRITE = BIT(REQ_F_NOWAIT_WRITE_BIT),
Jens Axboe7b29f922021-03-12 08:30:14 -0700763 /* regular file */
764 REQ_F_ISREG = BIT(REQ_F_ISREG_BIT),
Pavel Begunkovb8e64b52021-06-17 18:14:02 +0100765 /* has creds assigned */
766 REQ_F_CREDS = BIT(REQ_F_CREDS_BIT),
Pavel Begunkov20e60a32021-08-11 19:28:30 +0100767 /* skip refcounting if not set */
768 REQ_F_REFCOUNT = BIT(REQ_F_REFCOUNT_BIT),
Pavel Begunkov4d13d1a2021-08-15 10:40:24 +0100769 /* there is a linked timeout that has to be armed */
770 REQ_F_ARM_LTIMEOUT = BIT(REQ_F_ARM_LTIMEOUT_BIT),
Jens Axboed7718a92020-02-14 22:23:12 -0700771};
772
773struct async_poll {
774 struct io_poll_iocb poll;
Jens Axboe807abcb2020-07-17 17:09:27 -0600775 struct io_poll_iocb *double_poll;
Pavel Begunkov6b47ee62020-01-18 20:22:41 +0300776};
777
Pavel Begunkov5b0a6ac2021-06-30 21:54:04 +0100778typedef void (*io_req_tw_func_t)(struct io_kiocb *req);
779
Jens Axboe7cbf1722021-02-10 00:03:20 +0000780struct io_task_work {
Pavel Begunkov5b0a6ac2021-06-30 21:54:04 +0100781 union {
782 struct io_wq_work_node node;
783 struct llist_node fallback_node;
784 };
785 io_req_tw_func_t func;
Jens Axboe7cbf1722021-02-10 00:03:20 +0000786};
787
Pavel Begunkov992da012021-06-10 16:37:37 +0100788enum {
789 IORING_RSRC_FILE = 0,
790 IORING_RSRC_BUFFER = 1,
791};
792
Jens Axboe09bb8392019-03-13 12:39:28 -0600793/*
794 * NOTE! Each of the iocb union members has the file pointer
795 * as the first entry in their struct definition. So you can
796 * access the file pointer through any of the sub-structs,
797 * or directly as just 'ki_filp' in this struct.
798 */
Jens Axboe2b188cc2019-01-07 10:46:33 -0700799struct io_kiocb {
Jens Axboe221c5eb2019-01-17 09:41:58 -0700800 union {
Jens Axboe09bb8392019-03-13 12:39:28 -0600801 struct file *file;
Jens Axboe9adbd452019-12-20 08:45:55 -0700802 struct io_rw rw;
Jens Axboe221c5eb2019-01-17 09:41:58 -0700803 struct io_poll_iocb poll;
Pavel Begunkov9d805892021-04-13 02:58:40 +0100804 struct io_poll_update poll_update;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -0700805 struct io_accept accept;
806 struct io_sync sync;
Jens Axboefbf23842019-12-17 18:45:56 -0700807 struct io_cancel cancel;
Jens Axboeb29472e2019-12-17 18:50:29 -0700808 struct io_timeout timeout;
Pavel Begunkov0bdf7a22020-10-10 18:34:10 +0100809 struct io_timeout_rem timeout_rem;
Jens Axboe3fbb51c2019-12-20 08:51:52 -0700810 struct io_connect connect;
Jens Axboee47293f2019-12-20 08:58:21 -0700811 struct io_sr_msg sr_msg;
Jens Axboe15b71ab2019-12-11 11:20:36 -0700812 struct io_open open;
Jens Axboeb5dba592019-12-11 14:02:38 -0700813 struct io_close close;
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +0000814 struct io_rsrc_update rsrc_update;
Jens Axboe4840e412019-12-25 22:03:45 -0700815 struct io_fadvise fadvise;
Jens Axboec1ca7572019-12-25 22:18:28 -0700816 struct io_madvise madvise;
Jens Axboe3e4827b2020-01-08 15:18:09 -0700817 struct io_epoll epoll;
Pavel Begunkov7d67af22020-02-24 11:32:45 +0300818 struct io_splice splice;
Jens Axboeddf0322d2020-02-23 16:41:33 -0700819 struct io_provide_buf pbuf;
Bijan Mottahedeh1d9e1282020-05-22 21:31:16 -0700820 struct io_statx statx;
Jens Axboe36f4fa62020-09-05 11:14:22 -0600821 struct io_shutdown shutdown;
Jens Axboe80a261f2020-09-28 14:23:58 -0600822 struct io_rename rename;
Jens Axboe14a11432020-09-28 14:27:37 -0600823 struct io_unlink unlink;
Pavel Begunkov3ca405e2020-07-13 23:37:08 +0300824 /* use only after cleaning per-op data, see io_clean_op() */
825 struct io_completion compl;
Jens Axboe221c5eb2019-01-17 09:41:58 -0700826 };
Jens Axboe2b188cc2019-01-07 10:46:33 -0700827
Jens Axboee8c2bc12020-08-15 18:44:09 -0700828 /* opcode allocated if it needs to store data for async defer */
829 void *async_data;
Jens Axboed625c6e2019-12-17 19:53:05 -0700830 u8 opcode;
Xiaoguang Wang65a65432020-06-11 23:39:36 +0800831 /* polled IO has completed */
832 u8 iopoll_completed;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700833
Bijan Mottahedeh4f4eeba2020-05-19 14:52:49 -0700834 u16 buf_index;
Pavel Begunkov9cf7c102020-07-13 23:37:15 +0300835 u32 result;
Bijan Mottahedeh4f4eeba2020-05-19 14:52:49 -0700836
Pavel Begunkov010e8e62020-07-30 18:43:45 +0300837 struct io_ring_ctx *ctx;
838 unsigned int flags;
Jens Axboeabc54d62021-02-24 13:32:30 -0700839 atomic_t refs;
Pavel Begunkov010e8e62020-07-30 18:43:45 +0300840 struct task_struct *task;
841 u64 user_data;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700842
Pavel Begunkovf2f87372020-10-27 23:25:37 +0000843 struct io_kiocb *link;
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +0000844 struct percpu_ref *fixed_rsrc_refs;
Jens Axboed7718a92020-02-14 22:23:12 -0700845
Pavel Begunkovb303fe22021-04-11 01:46:26 +0100846 /* used with ctx->iopoll_list with reads/writes */
Pavel Begunkov010e8e62020-07-30 18:43:45 +0300847 struct list_head inflight_entry;
Pavel Begunkov5b0a6ac2021-06-30 21:54:04 +0100848 struct io_task_work io_task_work;
Pavel Begunkov010e8e62020-07-30 18:43:45 +0300849 /* for polled requests, i.e. IORING_OP_POLL_ADD and async armed poll */
850 struct hlist_node hash_node;
851 struct async_poll *apoll;
852 struct io_wq_work work;
Pavel Begunkovfe7e3252021-06-24 15:09:57 +0100853 const struct cred *creds;
Pavel Begunkovc10d1f92021-06-17 18:14:01 +0100854
Pavel Begunkoveae071c2021-04-25 14:32:24 +0100855 /* store used ubuf, so we can prevent reloading */
856 struct io_mapped_ubuf *imu;
Jens Axboe2b188cc2019-01-07 10:46:33 -0700857};
858
Pavel Begunkov13bf43f2021-03-06 11:02:12 +0000859struct io_tctx_node {
860 struct list_head ctx_node;
861 struct task_struct *task;
Pavel Begunkov13bf43f2021-03-06 11:02:12 +0000862 struct io_ring_ctx *ctx;
863};
864
Pavel Begunkov27dc8332020-07-13 23:37:14 +0300865struct io_defer_entry {
866 struct list_head list;
867 struct io_kiocb *req;
Pavel Begunkov9cf7c102020-07-13 23:37:15 +0300868 u32 seq;
Pavel Begunkov27dc8332020-07-13 23:37:14 +0300869};
870
Jens Axboed3656342019-12-18 09:50:26 -0700871struct io_op_def {
Jens Axboed3656342019-12-18 09:50:26 -0700872 /* needs req->file assigned */
873 unsigned needs_file : 1;
Jens Axboed3656342019-12-18 09:50:26 -0700874 /* hash wq insertion if file is a regular file */
875 unsigned hash_reg_file : 1;
876 /* unbound wq insertion if file is a non-regular file */
877 unsigned unbound_nonreg_file : 1;
Jens Axboe66f4af92020-01-16 15:36:52 -0700878 /* opcode is not supported by this kernel */
879 unsigned not_supported : 1;
Jens Axboe8a727582020-02-20 09:59:44 -0700880 /* set if opcode supports polled "wait" */
881 unsigned pollin : 1;
882 unsigned pollout : 1;
Jens Axboebcda7ba2020-02-23 16:42:51 -0700883 /* op supports buffer selection */
884 unsigned buffer_select : 1;
Pavel Begunkov26f05052021-02-28 22:35:18 +0000885 /* do prep async if is going to be punted */
886 unsigned needs_async_setup : 1;
Jens Axboe27926b62020-10-28 09:33:23 -0600887 /* should block plug */
888 unsigned plug : 1;
Jens Axboee8c2bc12020-08-15 18:44:09 -0700889 /* size of async data needed, if any */
890 unsigned short async_size;
Jens Axboed3656342019-12-18 09:50:26 -0700891};
892
Jens Axboe09186822020-10-13 15:01:40 -0600893static const struct io_op_def io_op_defs[] = {
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300894 [IORING_OP_NOP] = {},
895 [IORING_OP_READV] = {
Jens Axboed3656342019-12-18 09:50:26 -0700896 .needs_file = 1,
897 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -0700898 .pollin = 1,
Jens Axboe4d954c22020-02-27 07:31:19 -0700899 .buffer_select = 1,
Pavel Begunkov26f05052021-02-28 22:35:18 +0000900 .needs_async_setup = 1,
Jens Axboe27926b62020-10-28 09:33:23 -0600901 .plug = 1,
Jens Axboee8c2bc12020-08-15 18:44:09 -0700902 .async_size = sizeof(struct io_async_rw),
Jens Axboed3656342019-12-18 09:50:26 -0700903 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300904 [IORING_OP_WRITEV] = {
Jens Axboed3656342019-12-18 09:50:26 -0700905 .needs_file = 1,
906 .hash_reg_file = 1,
907 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -0700908 .pollout = 1,
Pavel Begunkov26f05052021-02-28 22:35:18 +0000909 .needs_async_setup = 1,
Jens Axboe27926b62020-10-28 09:33:23 -0600910 .plug = 1,
Jens Axboee8c2bc12020-08-15 18:44:09 -0700911 .async_size = sizeof(struct io_async_rw),
Jens Axboed3656342019-12-18 09:50:26 -0700912 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300913 [IORING_OP_FSYNC] = {
Jens Axboed3656342019-12-18 09:50:26 -0700914 .needs_file = 1,
915 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300916 [IORING_OP_READ_FIXED] = {
Jens Axboed3656342019-12-18 09:50:26 -0700917 .needs_file = 1,
918 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -0700919 .pollin = 1,
Jens Axboe27926b62020-10-28 09:33:23 -0600920 .plug = 1,
Jens Axboee8c2bc12020-08-15 18:44:09 -0700921 .async_size = sizeof(struct io_async_rw),
Jens Axboed3656342019-12-18 09:50:26 -0700922 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300923 [IORING_OP_WRITE_FIXED] = {
Jens Axboed3656342019-12-18 09:50:26 -0700924 .needs_file = 1,
925 .hash_reg_file = 1,
926 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -0700927 .pollout = 1,
Jens Axboe27926b62020-10-28 09:33:23 -0600928 .plug = 1,
Jens Axboee8c2bc12020-08-15 18:44:09 -0700929 .async_size = sizeof(struct io_async_rw),
Jens Axboed3656342019-12-18 09:50:26 -0700930 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300931 [IORING_OP_POLL_ADD] = {
Jens Axboed3656342019-12-18 09:50:26 -0700932 .needs_file = 1,
933 .unbound_nonreg_file = 1,
934 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300935 [IORING_OP_POLL_REMOVE] = {},
936 [IORING_OP_SYNC_FILE_RANGE] = {
Jens Axboed3656342019-12-18 09:50:26 -0700937 .needs_file = 1,
938 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300939 [IORING_OP_SENDMSG] = {
Jens Axboed3656342019-12-18 09:50:26 -0700940 .needs_file = 1,
941 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -0700942 .pollout = 1,
Pavel Begunkov26f05052021-02-28 22:35:18 +0000943 .needs_async_setup = 1,
Jens Axboee8c2bc12020-08-15 18:44:09 -0700944 .async_size = sizeof(struct io_async_msghdr),
Jens Axboed3656342019-12-18 09:50:26 -0700945 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300946 [IORING_OP_RECVMSG] = {
Jens Axboed3656342019-12-18 09:50:26 -0700947 .needs_file = 1,
948 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -0700949 .pollin = 1,
Jens Axboe52de1fe2020-02-27 10:15:42 -0700950 .buffer_select = 1,
Pavel Begunkov26f05052021-02-28 22:35:18 +0000951 .needs_async_setup = 1,
Jens Axboee8c2bc12020-08-15 18:44:09 -0700952 .async_size = sizeof(struct io_async_msghdr),
Jens Axboed3656342019-12-18 09:50:26 -0700953 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300954 [IORING_OP_TIMEOUT] = {
Jens Axboee8c2bc12020-08-15 18:44:09 -0700955 .async_size = sizeof(struct io_timeout_data),
Jens Axboed3656342019-12-18 09:50:26 -0700956 },
Pavel Begunkov9c8e11b2020-11-30 19:11:16 +0000957 [IORING_OP_TIMEOUT_REMOVE] = {
958 /* used by timeout updates' prep() */
Pavel Begunkov9c8e11b2020-11-30 19:11:16 +0000959 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300960 [IORING_OP_ACCEPT] = {
Jens Axboed3656342019-12-18 09:50:26 -0700961 .needs_file = 1,
962 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -0700963 .pollin = 1,
Jens Axboed3656342019-12-18 09:50:26 -0700964 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300965 [IORING_OP_ASYNC_CANCEL] = {},
966 [IORING_OP_LINK_TIMEOUT] = {
Jens Axboee8c2bc12020-08-15 18:44:09 -0700967 .async_size = sizeof(struct io_timeout_data),
Jens Axboed3656342019-12-18 09:50:26 -0700968 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300969 [IORING_OP_CONNECT] = {
Jens Axboed3656342019-12-18 09:50:26 -0700970 .needs_file = 1,
971 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -0700972 .pollout = 1,
Pavel Begunkov26f05052021-02-28 22:35:18 +0000973 .needs_async_setup = 1,
Jens Axboee8c2bc12020-08-15 18:44:09 -0700974 .async_size = sizeof(struct io_async_connect),
Jens Axboed3656342019-12-18 09:50:26 -0700975 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300976 [IORING_OP_FALLOCATE] = {
Jens Axboed3656342019-12-18 09:50:26 -0700977 .needs_file = 1,
978 },
Jens Axboe44526be2021-02-15 13:32:18 -0700979 [IORING_OP_OPENAT] = {},
980 [IORING_OP_CLOSE] = {},
981 [IORING_OP_FILES_UPDATE] = {},
982 [IORING_OP_STATX] = {},
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300983 [IORING_OP_READ] = {
Jens Axboe3a6820f2019-12-22 15:19:35 -0700984 .needs_file = 1,
985 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -0700986 .pollin = 1,
Jens Axboebcda7ba2020-02-23 16:42:51 -0700987 .buffer_select = 1,
Jens Axboe27926b62020-10-28 09:33:23 -0600988 .plug = 1,
Jens Axboee8c2bc12020-08-15 18:44:09 -0700989 .async_size = sizeof(struct io_async_rw),
Jens Axboe3a6820f2019-12-22 15:19:35 -0700990 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300991 [IORING_OP_WRITE] = {
Jens Axboe3a6820f2019-12-22 15:19:35 -0700992 .needs_file = 1,
993 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -0700994 .pollout = 1,
Jens Axboe27926b62020-10-28 09:33:23 -0600995 .plug = 1,
Jens Axboee8c2bc12020-08-15 18:44:09 -0700996 .async_size = sizeof(struct io_async_rw),
Jens Axboe3a6820f2019-12-22 15:19:35 -0700997 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +0300998 [IORING_OP_FADVISE] = {
Jens Axboe4840e412019-12-25 22:03:45 -0700999 .needs_file = 1,
1000 },
Jens Axboe44526be2021-02-15 13:32:18 -07001001 [IORING_OP_MADVISE] = {},
Pavel Begunkov0463b6c2020-01-18 21:35:38 +03001002 [IORING_OP_SEND] = {
Jens Axboefddafac2020-01-04 20:19:44 -07001003 .needs_file = 1,
1004 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -07001005 .pollout = 1,
Jens Axboefddafac2020-01-04 20:19:44 -07001006 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +03001007 [IORING_OP_RECV] = {
Jens Axboefddafac2020-01-04 20:19:44 -07001008 .needs_file = 1,
1009 .unbound_nonreg_file = 1,
Jens Axboe8a727582020-02-20 09:59:44 -07001010 .pollin = 1,
Jens Axboebcda7ba2020-02-23 16:42:51 -07001011 .buffer_select = 1,
Jens Axboefddafac2020-01-04 20:19:44 -07001012 },
Pavel Begunkov0463b6c2020-01-18 21:35:38 +03001013 [IORING_OP_OPENAT2] = {
Jens Axboecebdb982020-01-08 17:59:24 -07001014 },
Jens Axboe3e4827b2020-01-08 15:18:09 -07001015 [IORING_OP_EPOLL_CTL] = {
1016 .unbound_nonreg_file = 1,
Jens Axboe3e4827b2020-01-08 15:18:09 -07001017 },
Pavel Begunkov7d67af22020-02-24 11:32:45 +03001018 [IORING_OP_SPLICE] = {
1019 .needs_file = 1,
1020 .hash_reg_file = 1,
1021 .unbound_nonreg_file = 1,
Jens Axboeddf0322d2020-02-23 16:41:33 -07001022 },
1023 [IORING_OP_PROVIDE_BUFFERS] = {},
Jens Axboe067524e2020-03-02 16:32:28 -07001024 [IORING_OP_REMOVE_BUFFERS] = {},
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03001025 [IORING_OP_TEE] = {
1026 .needs_file = 1,
1027 .hash_reg_file = 1,
1028 .unbound_nonreg_file = 1,
1029 },
Jens Axboe36f4fa62020-09-05 11:14:22 -06001030 [IORING_OP_SHUTDOWN] = {
1031 .needs_file = 1,
1032 },
Jens Axboe44526be2021-02-15 13:32:18 -07001033 [IORING_OP_RENAMEAT] = {},
1034 [IORING_OP_UNLINKAT] = {},
Jens Axboed3656342019-12-18 09:50:26 -07001035};
1036
Pavel Begunkov0756a862021-08-15 10:40:25 +01001037/* requests with any of those set should undergo io_disarm_next() */
1038#define IO_DISARM_MASK (REQ_F_ARM_LTIMEOUT | REQ_F_LINK_TIMEOUT | REQ_F_FAIL)
1039
Pavel Begunkov7a612352021-03-09 00:37:59 +00001040static bool io_disarm_next(struct io_kiocb *req);
Pavel Begunkoveef51da2021-06-14 02:36:15 +01001041static void io_uring_del_tctx_node(unsigned long index);
Pavel Begunkov9936c7c2021-02-04 13:51:56 +00001042static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
1043 struct task_struct *task,
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01001044 bool cancel_all);
Pavel Begunkov78cc6872021-06-14 02:36:23 +01001045static void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd);
Pavel Begunkov1ffc5422020-12-30 21:34:15 +00001046
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01001047static bool io_cqring_fill_event(struct io_ring_ctx *ctx, u64 user_data,
1048 long res, unsigned int cflags);
Jackie Liuec9c02a2019-11-08 23:50:36 +08001049static void io_put_req(struct io_kiocb *req);
Pavel Begunkov91c2f692021-08-11 19:28:28 +01001050static void io_put_req_deferred(struct io_kiocb *req);
Jens Axboec7dae4b2021-02-09 19:53:37 -07001051static void io_dismantle_req(struct io_kiocb *req);
Jens Axboe94ae5e72019-11-14 19:39:52 -07001052static void io_queue_linked_timeout(struct io_kiocb *req);
Pavel Begunkovfdecb662021-04-25 14:32:20 +01001053static int __io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned type,
Pavel Begunkovc3bdad02021-04-25 14:32:22 +01001054 struct io_uring_rsrc_update2 *up,
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +01001055 unsigned nr_args);
Pavel Begunkov68fb8972021-03-19 17:22:41 +00001056static void io_clean_op(struct io_kiocb *req);
Pavel Begunkovac177052021-08-09 13:04:02 +01001057static struct file *io_file_get(struct io_ring_ctx *ctx,
Pavel Begunkov8371adf2020-10-10 18:34:08 +01001058 struct io_kiocb *req, int fd, bool fixed);
Pavel Begunkovc5eef2b2021-02-10 00:03:22 +00001059static void __io_queue_sqe(struct io_kiocb *req);
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00001060static void io_rsrc_put_work(struct work_struct *work);
Jens Axboede0617e2019-04-06 21:51:27 -06001061
Pavel Begunkov907d1df2021-01-26 23:35:10 +00001062static void io_req_task_queue(struct io_kiocb *req);
Pavel Begunkov2a2758f2021-06-17 18:14:00 +01001063static void io_submit_flush_completions(struct io_ring_ctx *ctx);
Pavel Begunkov179ae0d2021-02-28 22:35:20 +00001064static int io_req_prep_async(struct io_kiocb *req);
Jens Axboe9a56a232019-01-09 09:06:50 -07001065
Jens Axboe2b188cc2019-01-07 10:46:33 -07001066static struct kmem_cache *req_cachep;
1067
Jens Axboe09186822020-10-13 15:01:40 -06001068static const struct file_operations io_uring_fops;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001069
1070struct sock *io_uring_get_socket(struct file *file)
1071{
1072#if defined(CONFIG_UNIX)
1073 if (file->f_op == &io_uring_fops) {
1074 struct io_ring_ctx *ctx = file->private_data;
1075
1076 return ctx->ring_sock->sk;
1077 }
1078#endif
1079 return NULL;
1080}
1081EXPORT_SYMBOL(io_uring_get_socket);
1082
Pavel Begunkovf2f87372020-10-27 23:25:37 +00001083#define io_for_each_link(pos, head) \
1084 for (pos = (head); pos; pos = pos->link)
1085
Pavel Begunkov21c843d2021-08-11 19:28:27 +01001086/*
1087 * Shamelessly stolen from the mm implementation of page reference checking,
1088 * see commit f958d7b528b1 for details.
1089 */
1090#define req_ref_zero_or_close_to_overflow(req) \
1091 ((unsigned int) atomic_read(&(req->refs)) + 127u <= 127u)
1092
1093static inline bool req_ref_inc_not_zero(struct io_kiocb *req)
1094{
Pavel Begunkov20e60a32021-08-11 19:28:30 +01001095 WARN_ON_ONCE(!(req->flags & REQ_F_REFCOUNT));
Pavel Begunkov21c843d2021-08-11 19:28:27 +01001096 return atomic_inc_not_zero(&req->refs);
1097}
1098
Pavel Begunkov21c843d2021-08-11 19:28:27 +01001099static inline bool req_ref_put_and_test(struct io_kiocb *req)
1100{
Pavel Begunkov20e60a32021-08-11 19:28:30 +01001101 if (likely(!(req->flags & REQ_F_REFCOUNT)))
1102 return true;
1103
Pavel Begunkov21c843d2021-08-11 19:28:27 +01001104 WARN_ON_ONCE(req_ref_zero_or_close_to_overflow(req));
1105 return atomic_dec_and_test(&req->refs);
1106}
1107
1108static inline void req_ref_put(struct io_kiocb *req)
1109{
Pavel Begunkov20e60a32021-08-11 19:28:30 +01001110 WARN_ON_ONCE(!(req->flags & REQ_F_REFCOUNT));
Pavel Begunkov21c843d2021-08-11 19:28:27 +01001111 WARN_ON_ONCE(req_ref_put_and_test(req));
1112}
1113
1114static inline void req_ref_get(struct io_kiocb *req)
1115{
Pavel Begunkov20e60a32021-08-11 19:28:30 +01001116 WARN_ON_ONCE(!(req->flags & REQ_F_REFCOUNT));
Pavel Begunkov21c843d2021-08-11 19:28:27 +01001117 WARN_ON_ONCE(req_ref_zero_or_close_to_overflow(req));
1118 atomic_inc(&req->refs);
1119}
1120
Pavel Begunkov48dcd382021-08-15 10:40:18 +01001121static inline void __io_req_set_refcount(struct io_kiocb *req, int nr)
Pavel Begunkov20e60a32021-08-11 19:28:30 +01001122{
1123 if (!(req->flags & REQ_F_REFCOUNT)) {
1124 req->flags |= REQ_F_REFCOUNT;
Pavel Begunkov48dcd382021-08-15 10:40:18 +01001125 atomic_set(&req->refs, nr);
Pavel Begunkov20e60a32021-08-11 19:28:30 +01001126 }
1127}
1128
Pavel Begunkov48dcd382021-08-15 10:40:18 +01001129static inline void io_req_set_refcount(struct io_kiocb *req)
1130{
1131 __io_req_set_refcount(req, 1);
1132}
1133
Pavel Begunkovb895c9a2021-04-01 15:43:40 +01001134static inline void io_req_set_rsrc_node(struct io_kiocb *req)
Jens Axboec40f6372020-06-25 15:39:59 -06001135{
Pavel Begunkov36f72fe2020-11-18 19:57:26 +00001136 struct io_ring_ctx *ctx = req->ctx;
1137
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00001138 if (!req->fixed_rsrc_refs) {
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01001139 req->fixed_rsrc_refs = &ctx->rsrc_node->refs;
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00001140 percpu_ref_get(req->fixed_rsrc_refs);
Pavel Begunkov36f72fe2020-11-18 19:57:26 +00001141 }
1142}
1143
Pavel Begunkovf70865d2021-04-11 01:46:40 +01001144static void io_refs_resurrect(struct percpu_ref *ref, struct completion *compl)
1145{
1146 bool got = percpu_ref_tryget(ref);
1147
1148 /* already at zero, wait for ->release() */
1149 if (!got)
1150 wait_for_completion(compl);
1151 percpu_ref_resurrect(ref);
1152 if (got)
1153 percpu_ref_put(ref);
1154}
1155
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01001156static bool io_match_task(struct io_kiocb *head, struct task_struct *task,
1157 bool cancel_all)
Pavel Begunkov08d23632020-11-06 13:00:22 +00001158{
1159 struct io_kiocb *req;
1160
Pavel Begunkov68207682021-03-22 01:58:25 +00001161 if (task && head->task != task)
Pavel Begunkov08d23632020-11-06 13:00:22 +00001162 return false;
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01001163 if (cancel_all)
Pavel Begunkov08d23632020-11-06 13:00:22 +00001164 return true;
1165
1166 io_for_each_link(req, head) {
Pavel Begunkovb05a1bc2021-03-04 13:59:24 +00001167 if (req->flags & REQ_F_INFLIGHT)
Jens Axboe02a13672021-01-23 15:49:31 -07001168 return true;
Pavel Begunkov08d23632020-11-06 13:00:22 +00001169 }
1170 return false;
1171}
1172
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01001173static inline void req_set_fail(struct io_kiocb *req)
Jens Axboec40f6372020-06-25 15:39:59 -06001174{
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01001175 req->flags |= REQ_F_FAIL;
Jens Axboec40f6372020-06-25 15:39:59 -06001176}
Jens Axboe4a38aed22020-05-14 17:21:15 -06001177
Jens Axboe2b188cc2019-01-07 10:46:33 -07001178static void io_ring_ctx_ref_free(struct percpu_ref *ref)
1179{
1180 struct io_ring_ctx *ctx = container_of(ref, struct io_ring_ctx, refs);
1181
Jens Axboe0f158b42020-05-14 17:18:39 -06001182 complete(&ctx->ref_comp);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001183}
1184
Pavel Begunkov8eb7e2d2020-06-29 13:13:02 +03001185static inline bool io_is_timeout_noseq(struct io_kiocb *req)
1186{
1187 return !req->timeout.off;
1188}
1189
Pavel Begunkovf56165e2021-08-09 20:18:07 +01001190static void io_fallback_req_func(struct work_struct *work)
1191{
1192 struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx,
1193 fallback_work.work);
1194 struct llist_node *node = llist_del_all(&ctx->fallback_llist);
1195 struct io_kiocb *req, *tmp;
1196
1197 percpu_ref_get(&ctx->refs);
1198 llist_for_each_entry_safe(req, tmp, node, io_task_work.fallback_node)
1199 req->io_task_work.func(req);
1200 percpu_ref_put(&ctx->refs);
1201}
1202
Jens Axboe2b188cc2019-01-07 10:46:33 -07001203static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
1204{
1205 struct io_ring_ctx *ctx;
Jens Axboe78076bb2019-12-04 19:56:40 -07001206 int hash_bits;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001207
1208 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
1209 if (!ctx)
1210 return NULL;
1211
Jens Axboe78076bb2019-12-04 19:56:40 -07001212 /*
1213 * Use 5 bits less than the max cq entries, that should give us around
1214 * 32 entries per hash list if totally full and uniformly spread.
1215 */
1216 hash_bits = ilog2(p->cq_entries);
1217 hash_bits -= 5;
1218 if (hash_bits <= 0)
1219 hash_bits = 1;
1220 ctx->cancel_hash_bits = hash_bits;
1221 ctx->cancel_hash = kmalloc((1U << hash_bits) * sizeof(struct hlist_head),
1222 GFP_KERNEL);
1223 if (!ctx->cancel_hash)
1224 goto err;
1225 __hash_init(ctx->cancel_hash, 1U << hash_bits);
1226
Pavel Begunkov62248432021-04-28 13:11:29 +01001227 ctx->dummy_ubuf = kzalloc(sizeof(*ctx->dummy_ubuf), GFP_KERNEL);
1228 if (!ctx->dummy_ubuf)
1229 goto err;
1230 /* set invalid range, so io_import_fixed() fails meeting it */
1231 ctx->dummy_ubuf->ubuf = -1UL;
1232
Roman Gushchin21482892019-05-07 10:01:48 -07001233 if (percpu_ref_init(&ctx->refs, io_ring_ctx_ref_free,
Jens Axboe206aefd2019-11-07 18:27:42 -07001234 PERCPU_REF_ALLOW_REINIT, GFP_KERNEL))
1235 goto err;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001236
1237 ctx->flags = p->flags;
Jens Axboe90554202020-09-03 12:12:41 -06001238 init_waitqueue_head(&ctx->sqo_sq_wait);
Jens Axboe69fb2132020-09-14 11:16:23 -06001239 INIT_LIST_HEAD(&ctx->sqd_list);
Pavel Begunkov311997b2021-06-14 23:37:28 +01001240 init_waitqueue_head(&ctx->poll_wait);
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001241 INIT_LIST_HEAD(&ctx->cq_overflow_list);
Jens Axboe0f158b42020-05-14 17:18:39 -06001242 init_completion(&ctx->ref_comp);
Jens Axboe9e15c3a2021-03-13 12:29:43 -07001243 xa_init_flags(&ctx->io_buffers, XA_FLAGS_ALLOC1);
Matthew Wilcox (Oracle)61cf9372021-03-08 14:16:16 +00001244 xa_init_flags(&ctx->personalities, XA_FLAGS_ALLOC1);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001245 mutex_init(&ctx->uring_lock);
Pavel Begunkov311997b2021-06-14 23:37:28 +01001246 init_waitqueue_head(&ctx->cq_wait);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001247 spin_lock_init(&ctx->completion_lock);
Jens Axboe89850fc2021-08-10 15:11:51 -06001248 spin_lock_init(&ctx->timeout_lock);
Pavel Begunkov540e32a2020-07-13 23:37:09 +03001249 INIT_LIST_HEAD(&ctx->iopoll_list);
Jens Axboede0617e2019-04-06 21:51:27 -06001250 INIT_LIST_HEAD(&ctx->defer_list);
Jens Axboe5262f562019-09-17 12:26:57 -06001251 INIT_LIST_HEAD(&ctx->timeout_list);
Bijan Mottahedehd67d2262021-01-15 17:37:46 +00001252 spin_lock_init(&ctx->rsrc_ref_lock);
1253 INIT_LIST_HEAD(&ctx->rsrc_ref_list);
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00001254 INIT_DELAYED_WORK(&ctx->rsrc_put_work, io_rsrc_put_work);
1255 init_llist_head(&ctx->rsrc_put_llist);
Pavel Begunkov13bf43f2021-03-06 11:02:12 +00001256 INIT_LIST_HEAD(&ctx->tctx_list);
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01001257 INIT_LIST_HEAD(&ctx->submit_state.free_list);
Pavel Begunkovd0acdee2021-05-16 22:58:12 +01001258 INIT_LIST_HEAD(&ctx->locked_free_list);
Pavel Begunkov9011bf92021-06-30 21:54:03 +01001259 INIT_DELAYED_WORK(&ctx->fallback_work, io_fallback_req_func);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001260 return ctx;
Jens Axboe206aefd2019-11-07 18:27:42 -07001261err:
Pavel Begunkov62248432021-04-28 13:11:29 +01001262 kfree(ctx->dummy_ubuf);
Jens Axboe78076bb2019-12-04 19:56:40 -07001263 kfree(ctx->cancel_hash);
Jens Axboe206aefd2019-11-07 18:27:42 -07001264 kfree(ctx);
1265 return NULL;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001266}
1267
Pavel Begunkov8f6ed492021-05-16 22:58:10 +01001268static void io_account_cq_overflow(struct io_ring_ctx *ctx)
1269{
1270 struct io_rings *r = ctx->rings;
1271
1272 WRITE_ONCE(r->cq_overflow, READ_ONCE(r->cq_overflow) + 1);
1273 ctx->cq_extra--;
1274}
1275
Pavel Begunkov9cf7c102020-07-13 23:37:15 +03001276static bool req_need_defer(struct io_kiocb *req, u32 seq)
Jens Axboede0617e2019-04-06 21:51:27 -06001277{
Jens Axboe2bc99302020-07-09 09:43:27 -06001278 if (unlikely(req->flags & REQ_F_IO_DRAIN)) {
1279 struct io_ring_ctx *ctx = req->ctx;
Jackie Liua197f662019-11-08 08:09:12 -07001280
Pavel Begunkov8f6ed492021-05-16 22:58:10 +01001281 return seq + READ_ONCE(ctx->cq_extra) != ctx->cached_cq_tail;
Jens Axboe2bc99302020-07-09 09:43:27 -06001282 }
Jens Axboe7adf4ea2019-10-10 21:42:58 -06001283
Bob Liu9d858b22019-11-13 18:06:25 +08001284 return false;
Jens Axboe7adf4ea2019-10-10 21:42:58 -06001285}
1286
Pavel Begunkovc97d8a02021-08-09 13:04:04 +01001287#define FFS_ASYNC_READ 0x1UL
1288#define FFS_ASYNC_WRITE 0x2UL
1289#ifdef CONFIG_64BIT
1290#define FFS_ISREG 0x4UL
1291#else
1292#define FFS_ISREG 0x0UL
1293#endif
1294#define FFS_MASK ~(FFS_ASYNC_READ|FFS_ASYNC_WRITE|FFS_ISREG)
1295
1296static inline bool io_req_ffs_set(struct io_kiocb *req)
1297{
1298 return IS_ENABLED(CONFIG_64BIT) && (req->flags & REQ_F_FIXED_FILE);
1299}
1300
Pavel Begunkovce3d5aa2021-02-01 18:59:55 +00001301static void io_req_track_inflight(struct io_kiocb *req)
1302{
Pavel Begunkovce3d5aa2021-02-01 18:59:55 +00001303 if (!(req->flags & REQ_F_INFLIGHT)) {
Pavel Begunkovce3d5aa2021-02-01 18:59:55 +00001304 req->flags |= REQ_F_INFLIGHT;
Pavel Begunkovb303fe22021-04-11 01:46:26 +01001305 atomic_inc(&current->io_uring->inflight_tracked);
Pavel Begunkovce3d5aa2021-02-01 18:59:55 +00001306 }
1307}
1308
Pavel Begunkov906c6ca2021-08-15 10:40:26 +01001309static inline void io_unprep_linked_timeout(struct io_kiocb *req)
1310{
1311 req->flags &= ~REQ_F_LINK_TIMEOUT;
1312}
1313
Pavel Begunkovfd08e532021-08-11 19:28:31 +01001314static struct io_kiocb *__io_prep_linked_timeout(struct io_kiocb *req)
1315{
Pavel Begunkov906c6ca2021-08-15 10:40:26 +01001316 if (WARN_ON_ONCE(!req->link))
1317 return NULL;
1318
Pavel Begunkov4d13d1a2021-08-15 10:40:24 +01001319 req->flags &= ~REQ_F_ARM_LTIMEOUT;
1320 req->flags |= REQ_F_LINK_TIMEOUT;
Pavel Begunkovfd08e532021-08-11 19:28:31 +01001321
1322 /* linked timeouts should have two refs once prep'ed */
Pavel Begunkov48dcd382021-08-15 10:40:18 +01001323 io_req_set_refcount(req);
Pavel Begunkov4d13d1a2021-08-15 10:40:24 +01001324 __io_req_set_refcount(req->link, 2);
1325 return req->link;
Pavel Begunkovfd08e532021-08-11 19:28:31 +01001326}
1327
1328static inline struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req)
1329{
Pavel Begunkov4d13d1a2021-08-15 10:40:24 +01001330 if (likely(!(req->flags & REQ_F_ARM_LTIMEOUT)))
Pavel Begunkovfd08e532021-08-11 19:28:31 +01001331 return NULL;
1332 return __io_prep_linked_timeout(req);
1333}
1334
Pavel Begunkovcbdcb432020-06-29 19:18:43 +03001335static void io_prep_async_work(struct io_kiocb *req)
Jens Axboe561fb042019-10-24 07:25:42 -06001336{
Jens Axboed3656342019-12-18 09:50:26 -07001337 const struct io_op_def *def = &io_op_defs[req->opcode];
Pavel Begunkov23329512020-10-10 18:34:06 +01001338 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe54a91f32019-09-10 09:15:04 -06001339
Pavel Begunkovb8e64b52021-06-17 18:14:02 +01001340 if (!(req->flags & REQ_F_CREDS)) {
1341 req->flags |= REQ_F_CREDS;
Pavel Begunkovc10d1f92021-06-17 18:14:01 +01001342 req->creds = get_current_cred();
Pavel Begunkovb8e64b52021-06-17 18:14:02 +01001343 }
Jens Axboe003e8dc2021-03-06 09:22:27 -07001344
Pavel Begunkove1d675d2021-03-22 01:58:29 +00001345 req->work.list.next = NULL;
1346 req->work.flags = 0;
Pavel Begunkovfeaadc42020-10-22 16:47:16 +01001347 if (req->flags & REQ_F_FORCE_ASYNC)
1348 req->work.flags |= IO_WQ_WORK_CONCURRENT;
1349
Jens Axboed3656342019-12-18 09:50:26 -07001350 if (req->flags & REQ_F_ISREG) {
Pavel Begunkov23329512020-10-10 18:34:06 +01001351 if (def->hash_reg_file || (ctx->flags & IORING_SETUP_IOPOLL))
Pavel Begunkov8766dd52020-03-14 00:31:04 +03001352 io_wq_hash_work(&req->work, file_inode(req->file));
Jens Axboe4b982bd2021-04-01 08:38:34 -06001353 } else if (!req->file || !S_ISBLK(file_inode(req->file)->i_mode)) {
Jens Axboed3656342019-12-18 09:50:26 -07001354 if (def->unbound_nonreg_file)
Jens Axboe3529d8c2019-12-19 18:24:38 -07001355 req->work.flags |= IO_WQ_WORK_UNBOUND;
Jens Axboe54a91f32019-09-10 09:15:04 -06001356 }
Pavel Begunkove1d675d2021-03-22 01:58:29 +00001357
1358 switch (req->opcode) {
1359 case IORING_OP_SPLICE:
1360 case IORING_OP_TEE:
Pavel Begunkove1d675d2021-03-22 01:58:29 +00001361 if (!S_ISREG(file_inode(req->splice.file_in)->i_mode))
1362 req->work.flags |= IO_WQ_WORK_UNBOUND;
1363 break;
1364 }
Jens Axboe561fb042019-10-24 07:25:42 -06001365}
1366
Pavel Begunkovcbdcb432020-06-29 19:18:43 +03001367static void io_prep_async_link(struct io_kiocb *req)
1368{
1369 struct io_kiocb *cur;
1370
Pavel Begunkov44eff402021-07-26 14:14:31 +01001371 if (req->flags & REQ_F_LINK_TIMEOUT) {
1372 struct io_ring_ctx *ctx = req->ctx;
1373
Jens Axboe79ebeae2021-08-10 15:18:27 -06001374 spin_lock(&ctx->completion_lock);
Pavel Begunkov44eff402021-07-26 14:14:31 +01001375 io_for_each_link(cur, req)
1376 io_prep_async_work(cur);
Jens Axboe79ebeae2021-08-10 15:18:27 -06001377 spin_unlock(&ctx->completion_lock);
Pavel Begunkov44eff402021-07-26 14:14:31 +01001378 } else {
1379 io_for_each_link(cur, req)
1380 io_prep_async_work(cur);
1381 }
Pavel Begunkovcbdcb432020-06-29 19:18:43 +03001382}
1383
Pavel Begunkovebf93662021-03-01 18:20:47 +00001384static void io_queue_async_work(struct io_kiocb *req)
Jens Axboe561fb042019-10-24 07:25:42 -06001385{
Jackie Liua197f662019-11-08 08:09:12 -07001386 struct io_ring_ctx *ctx = req->ctx;
Pavel Begunkovcbdcb432020-06-29 19:18:43 +03001387 struct io_kiocb *link = io_prep_linked_timeout(req);
Jens Axboe5aa75ed2021-02-16 12:56:50 -07001388 struct io_uring_task *tctx = req->task->io_uring;
Jens Axboe561fb042019-10-24 07:25:42 -06001389
Jens Axboe3bfe6102021-02-16 14:15:30 -07001390 BUG_ON(!tctx);
1391 BUG_ON(!tctx->io_wq);
Jens Axboe561fb042019-10-24 07:25:42 -06001392
Pavel Begunkovcbdcb432020-06-29 19:18:43 +03001393 /* init ->work of the whole link before punting */
1394 io_prep_async_link(req);
Jens Axboe991468d2021-07-23 11:53:54 -06001395
1396 /*
1397 * Not expected to happen, but if we do have a bug where this _can_
1398 * happen, catch it here and ensure the request is marked as
1399 * canceled. That will make io-wq go through the usual work cancel
1400 * procedure rather than attempt to run this request (or create a new
1401 * worker for it).
1402 */
1403 if (WARN_ON_ONCE(!same_thread_group(req->task, current)))
1404 req->work.flags |= IO_WQ_WORK_CANCEL;
1405
Pavel Begunkovd07f1e8a2021-03-22 01:45:58 +00001406 trace_io_uring_queue_async_work(ctx, io_wq_is_hashed(&req->work), req,
1407 &req->work, req->flags);
Pavel Begunkovebf93662021-03-01 18:20:47 +00001408 io_wq_enqueue(tctx->io_wq, &req->work);
Jens Axboe7271ef32020-08-10 09:55:22 -06001409 if (link)
1410 io_queue_linked_timeout(link);
Pavel Begunkovcbdcb432020-06-29 19:18:43 +03001411}
1412
Pavel Begunkov1ee41602021-03-25 18:32:42 +00001413static void io_kill_timeout(struct io_kiocb *req, int status)
Pavel Begunkov8c855882021-04-13 02:58:41 +01001414 __must_hold(&req->ctx->completion_lock)
Jens Axboe89850fc2021-08-10 15:11:51 -06001415 __must_hold(&req->ctx->timeout_lock)
Jens Axboe5262f562019-09-17 12:26:57 -06001416{
Jens Axboee8c2bc12020-08-15 18:44:09 -07001417 struct io_timeout_data *io = req->async_data;
Jens Axboe5262f562019-09-17 12:26:57 -06001418
Pavel Begunkovfd9c7bc2021-04-13 02:58:42 +01001419 if (hrtimer_try_to_cancel(&io->timer) != -1) {
Pavel Begunkov01cec8c2020-07-30 18:43:50 +03001420 atomic_set(&req->ctx->cq_timeouts,
1421 atomic_read(&req->ctx->cq_timeouts) + 1);
Pavel Begunkov135fcde2020-07-13 23:37:12 +03001422 list_del_init(&req->timeout.list);
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01001423 io_cqring_fill_event(req->ctx, req->user_data, status, 0);
Pavel Begunkov91c2f692021-08-11 19:28:28 +01001424 io_put_req_deferred(req);
Jens Axboe5262f562019-09-17 12:26:57 -06001425 }
1426}
1427
Pavel Begunkov441b8a72021-06-14 23:37:31 +01001428static void io_queue_deferred(struct io_ring_ctx *ctx)
Pavel Begunkov04518942020-05-26 20:34:05 +03001429{
Pavel Begunkov441b8a72021-06-14 23:37:31 +01001430 while (!list_empty(&ctx->defer_list)) {
Pavel Begunkov27dc8332020-07-13 23:37:14 +03001431 struct io_defer_entry *de = list_first_entry(&ctx->defer_list,
1432 struct io_defer_entry, list);
Pavel Begunkov04518942020-05-26 20:34:05 +03001433
Pavel Begunkov9cf7c102020-07-13 23:37:15 +03001434 if (req_need_defer(de->req, de->seq))
Pavel Begunkov04518942020-05-26 20:34:05 +03001435 break;
Pavel Begunkov27dc8332020-07-13 23:37:14 +03001436 list_del_init(&de->list);
Pavel Begunkov907d1df2021-01-26 23:35:10 +00001437 io_req_task_queue(de->req);
Pavel Begunkov27dc8332020-07-13 23:37:14 +03001438 kfree(de);
Pavel Begunkov441b8a72021-06-14 23:37:31 +01001439 }
Pavel Begunkov04518942020-05-26 20:34:05 +03001440}
1441
Pavel Begunkov360428f2020-05-30 14:54:17 +03001442static void io_flush_timeouts(struct io_ring_ctx *ctx)
Jens Axboe89850fc2021-08-10 15:11:51 -06001443 __must_hold(&ctx->completion_lock)
Pavel Begunkov360428f2020-05-30 14:54:17 +03001444{
Pavel Begunkov441b8a72021-06-14 23:37:31 +01001445 u32 seq = ctx->cached_cq_tail - atomic_read(&ctx->cq_timeouts);
Marcelo Diop-Gonzalezf0105052021-01-15 11:54:40 -05001446
Jens Axboe79ebeae2021-08-10 15:18:27 -06001447 spin_lock_irq(&ctx->timeout_lock);
Pavel Begunkovf18ee4c2021-06-14 23:37:25 +01001448 while (!list_empty(&ctx->timeout_list)) {
Marcelo Diop-Gonzalezf0105052021-01-15 11:54:40 -05001449 u32 events_needed, events_got;
Pavel Begunkov360428f2020-05-30 14:54:17 +03001450 struct io_kiocb *req = list_first_entry(&ctx->timeout_list,
Pavel Begunkov135fcde2020-07-13 23:37:12 +03001451 struct io_kiocb, timeout.list);
Pavel Begunkov360428f2020-05-30 14:54:17 +03001452
Pavel Begunkov8eb7e2d2020-06-29 13:13:02 +03001453 if (io_is_timeout_noseq(req))
Pavel Begunkov360428f2020-05-30 14:54:17 +03001454 break;
Marcelo Diop-Gonzalezf0105052021-01-15 11:54:40 -05001455
1456 /*
1457 * Since seq can easily wrap around over time, subtract
1458 * the last seq at which timeouts were flushed before comparing.
1459 * Assuming not more than 2^31-1 events have happened since,
1460 * these subtractions won't have wrapped, so we can check if
1461 * target is in [last_seq, current_seq] by comparing the two.
1462 */
1463 events_needed = req->timeout.target_seq - ctx->cq_last_tm_flush;
1464 events_got = seq - ctx->cq_last_tm_flush;
1465 if (events_got < events_needed)
Pavel Begunkov360428f2020-05-30 14:54:17 +03001466 break;
Pavel Begunkovbfe68a22020-05-30 14:54:18 +03001467
Pavel Begunkov135fcde2020-07-13 23:37:12 +03001468 list_del_init(&req->timeout.list);
Pavel Begunkov1ee41602021-03-25 18:32:42 +00001469 io_kill_timeout(req, 0);
Pavel Begunkovf18ee4c2021-06-14 23:37:25 +01001470 }
Marcelo Diop-Gonzalezf0105052021-01-15 11:54:40 -05001471 ctx->cq_last_tm_flush = seq;
Jens Axboe79ebeae2021-08-10 15:18:27 -06001472 spin_unlock_irq(&ctx->timeout_lock);
Pavel Begunkov360428f2020-05-30 14:54:17 +03001473}
1474
Pavel Begunkov2335f6f2021-06-15 16:47:58 +01001475static void __io_commit_cqring_flush(struct io_ring_ctx *ctx)
Jens Axboede0617e2019-04-06 21:51:27 -06001476{
Pavel Begunkov2335f6f2021-06-15 16:47:58 +01001477 if (ctx->off_timeout_used)
1478 io_flush_timeouts(ctx);
1479 if (ctx->drain_active)
1480 io_queue_deferred(ctx);
1481}
1482
1483static inline void io_commit_cqring(struct io_ring_ctx *ctx)
1484{
1485 if (unlikely(ctx->off_timeout_used || ctx->drain_active))
1486 __io_commit_cqring_flush(ctx);
Pavel Begunkovec30e042021-01-19 13:32:38 +00001487 /* order cqe stores with ring update */
1488 smp_store_release(&ctx->rings->cq.tail, ctx->cached_cq_tail);
Jens Axboede0617e2019-04-06 21:51:27 -06001489}
1490
Jens Axboe90554202020-09-03 12:12:41 -06001491static inline bool io_sqring_full(struct io_ring_ctx *ctx)
1492{
1493 struct io_rings *r = ctx->rings;
1494
Pavel Begunkova566c552021-05-16 22:58:08 +01001495 return READ_ONCE(r->sq.tail) - ctx->cached_sq_head == ctx->sq_entries;
Jens Axboe90554202020-09-03 12:12:41 -06001496}
1497
Pavel Begunkov888aae22021-01-19 13:32:39 +00001498static inline unsigned int __io_cqring_events(struct io_ring_ctx *ctx)
1499{
1500 return ctx->cached_cq_tail - READ_ONCE(ctx->rings->cq.head);
1501}
1502
Pavel Begunkovd068b502021-05-16 22:58:11 +01001503static inline struct io_uring_cqe *io_get_cqe(struct io_ring_ctx *ctx)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001504{
Hristo Venev75b28af2019-08-26 17:23:46 +00001505 struct io_rings *rings = ctx->rings;
Pavel Begunkovea5ab3b2021-05-16 22:58:09 +01001506 unsigned tail, mask = ctx->cq_entries - 1;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001507
Stefan Bühler115e12e2019-04-24 23:54:18 +02001508 /*
1509 * writes to the cq entry need to come after reading head; the
1510 * control dependency is enough as we're using WRITE_ONCE to
1511 * fill the cq entry
1512 */
Pavel Begunkova566c552021-05-16 22:58:08 +01001513 if (__io_cqring_events(ctx) == ctx->cq_entries)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001514 return NULL;
1515
Pavel Begunkov888aae22021-01-19 13:32:39 +00001516 tail = ctx->cached_cq_tail++;
Pavel Begunkovea5ab3b2021-05-16 22:58:09 +01001517 return &rings->cqes[tail & mask];
Jens Axboe2b188cc2019-01-07 10:46:33 -07001518}
1519
Jens Axboef2842ab2020-01-08 11:04:00 -07001520static inline bool io_should_trigger_evfd(struct io_ring_ctx *ctx)
1521{
Pavel Begunkov44c769d2021-04-11 01:46:31 +01001522 if (likely(!ctx->cq_ev_fd))
Jens Axboef0b493e2020-02-01 21:30:11 -07001523 return false;
Stefano Garzarella7e55a192020-05-15 18:38:05 +02001524 if (READ_ONCE(ctx->rings->cq_flags) & IORING_CQ_EVENTFD_DISABLED)
1525 return false;
Pavel Begunkov44c769d2021-04-11 01:46:31 +01001526 return !ctx->eventfd_async || io_wq_current_is_worker();
Jens Axboef2842ab2020-01-08 11:04:00 -07001527}
1528
Jens Axboe2c5d7632021-08-21 07:21:19 -06001529/*
1530 * This should only get called when at least one event has been posted.
1531 * Some applications rely on the eventfd notification count only changing
1532 * IFF a new CQE has been added to the CQ ring. There's no depedency on
1533 * 1:1 relationship between how many times this function is called (and
1534 * hence the eventfd count) and number of CQEs posted to the CQ ring.
1535 */
Jens Axboeb41e9852020-02-17 09:52:41 -07001536static void io_cqring_ev_posted(struct io_ring_ctx *ctx)
Jens Axboe8c838782019-03-12 15:48:16 -06001537{
Jens Axboe5fd46172021-08-06 14:04:31 -06001538 /*
1539 * wake_up_all() may seem excessive, but io_wake_function() and
1540 * io_should_wake() handle the termination of the loop and only
1541 * wake as many waiters as we need to.
1542 */
1543 if (wq_has_sleeper(&ctx->cq_wait))
1544 wake_up_all(&ctx->cq_wait);
Jens Axboe534ca6d2020-09-02 13:52:19 -06001545 if (ctx->sq_data && waitqueue_active(&ctx->sq_data->wait))
1546 wake_up(&ctx->sq_data->wait);
Jens Axboeb41e9852020-02-17 09:52:41 -07001547 if (io_should_trigger_evfd(ctx))
Jens Axboe9b402842019-04-11 11:45:41 -06001548 eventfd_signal(ctx->cq_ev_fd, 1);
Pavel Begunkov311997b2021-06-14 23:37:28 +01001549 if (waitqueue_active(&ctx->poll_wait)) {
1550 wake_up_interruptible(&ctx->poll_wait);
Pavel Begunkov4aa84f22021-01-07 03:15:42 +00001551 kill_fasync(&ctx->cq_fasync, SIGIO, POLL_IN);
1552 }
Jens Axboe8c838782019-03-12 15:48:16 -06001553}
1554
Pavel Begunkov80c18e42021-01-07 03:15:41 +00001555static void io_cqring_ev_posted_iopoll(struct io_ring_ctx *ctx)
1556{
1557 if (ctx->flags & IORING_SETUP_SQPOLL) {
Jens Axboe5fd46172021-08-06 14:04:31 -06001558 if (wq_has_sleeper(&ctx->cq_wait))
1559 wake_up_all(&ctx->cq_wait);
Pavel Begunkov80c18e42021-01-07 03:15:41 +00001560 }
1561 if (io_should_trigger_evfd(ctx))
1562 eventfd_signal(ctx->cq_ev_fd, 1);
Pavel Begunkov311997b2021-06-14 23:37:28 +01001563 if (waitqueue_active(&ctx->poll_wait)) {
1564 wake_up_interruptible(&ctx->poll_wait);
Pavel Begunkov4aa84f22021-01-07 03:15:42 +00001565 kill_fasync(&ctx->cq_fasync, SIGIO, POLL_IN);
1566 }
Pavel Begunkov80c18e42021-01-07 03:15:41 +00001567}
1568
Jens Axboec4a2ed72019-11-21 21:01:26 -07001569/* Returns true if there are no backlogged entries after the flush */
Pavel Begunkov6c2450a2021-02-23 12:40:22 +00001570static bool __io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001571{
Jens Axboeb18032b2021-01-24 16:58:56 -07001572 bool all_flushed, posted;
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001573
Pavel Begunkova566c552021-05-16 22:58:08 +01001574 if (!force && __io_cqring_events(ctx) == ctx->cq_entries)
Pavel Begunkove23de152020-12-17 00:24:37 +00001575 return false;
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001576
Jens Axboeb18032b2021-01-24 16:58:56 -07001577 posted = false;
Jens Axboe79ebeae2021-08-10 15:18:27 -06001578 spin_lock(&ctx->completion_lock);
Pavel Begunkov6c2450a2021-02-23 12:40:22 +00001579 while (!list_empty(&ctx->cq_overflow_list)) {
Pavel Begunkovd068b502021-05-16 22:58:11 +01001580 struct io_uring_cqe *cqe = io_get_cqe(ctx);
Pavel Begunkov6c2450a2021-02-23 12:40:22 +00001581 struct io_overflow_cqe *ocqe;
Jens Axboee6c8aa92020-09-28 13:10:13 -06001582
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001583 if (!cqe && !force)
1584 break;
Pavel Begunkov6c2450a2021-02-23 12:40:22 +00001585 ocqe = list_first_entry(&ctx->cq_overflow_list,
1586 struct io_overflow_cqe, list);
1587 if (cqe)
1588 memcpy(cqe, &ocqe->cqe, sizeof(*cqe));
1589 else
Pavel Begunkov8f6ed492021-05-16 22:58:10 +01001590 io_account_cq_overflow(ctx);
1591
Jens Axboeb18032b2021-01-24 16:58:56 -07001592 posted = true;
Pavel Begunkov6c2450a2021-02-23 12:40:22 +00001593 list_del(&ocqe->list);
1594 kfree(ocqe);
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001595 }
1596
Pavel Begunkov09e88402020-12-17 00:24:38 +00001597 all_flushed = list_empty(&ctx->cq_overflow_list);
1598 if (all_flushed) {
Pavel Begunkov5ed7a372021-06-14 23:37:27 +01001599 clear_bit(0, &ctx->check_cq_overflow);
Nadav Amit20c0b382021-08-07 17:13:42 -07001600 WRITE_ONCE(ctx->rings->sq_flags,
1601 ctx->rings->sq_flags & ~IORING_SQ_CQ_OVERFLOW);
Pavel Begunkov09e88402020-12-17 00:24:38 +00001602 }
Pavel Begunkov46930142020-07-30 18:43:49 +03001603
Jens Axboeb18032b2021-01-24 16:58:56 -07001604 if (posted)
1605 io_commit_cqring(ctx);
Jens Axboe79ebeae2021-08-10 15:18:27 -06001606 spin_unlock(&ctx->completion_lock);
Jens Axboeb18032b2021-01-24 16:58:56 -07001607 if (posted)
1608 io_cqring_ev_posted(ctx);
Pavel Begunkov09e88402020-12-17 00:24:38 +00001609 return all_flushed;
Jens Axboe1d7bb1d2019-11-06 11:31:17 -07001610}
1611
Pavel Begunkov90f67362021-08-09 20:18:12 +01001612static bool io_cqring_overflow_flush(struct io_ring_ctx *ctx)
Pavel Begunkov6c503152021-01-04 20:36:36 +00001613{
Jens Axboeca0a2652021-03-04 17:15:48 -07001614 bool ret = true;
1615
Pavel Begunkov5ed7a372021-06-14 23:37:27 +01001616 if (test_bit(0, &ctx->check_cq_overflow)) {
Pavel Begunkov6c503152021-01-04 20:36:36 +00001617 /* iopoll syncs against uring_lock, not completion_lock */
1618 if (ctx->flags & IORING_SETUP_IOPOLL)
1619 mutex_lock(&ctx->uring_lock);
Pavel Begunkov90f67362021-08-09 20:18:12 +01001620 ret = __io_cqring_overflow_flush(ctx, false);
Pavel Begunkov6c503152021-01-04 20:36:36 +00001621 if (ctx->flags & IORING_SETUP_IOPOLL)
1622 mutex_unlock(&ctx->uring_lock);
1623 }
Jens Axboeca0a2652021-03-04 17:15:48 -07001624
1625 return ret;
Pavel Begunkov6c503152021-01-04 20:36:36 +00001626}
1627
Pavel Begunkov6a290a12021-08-09 13:04:13 +01001628/* must to be called somewhat shortly after putting a request */
1629static inline void io_put_task(struct task_struct *task, int nr)
1630{
1631 struct io_uring_task *tctx = task->io_uring;
1632
Pavel Begunkove98e49b2021-08-18 17:01:43 +01001633 if (likely(task == current)) {
1634 tctx->cached_refs += nr;
1635 } else {
1636 percpu_counter_sub(&tctx->inflight, nr);
1637 if (unlikely(atomic_read(&tctx->in_idle)))
1638 wake_up(&tctx->wait);
1639 put_task_struct_many(task, nr);
1640 }
Pavel Begunkov6a290a12021-08-09 13:04:13 +01001641}
1642
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01001643static bool io_cqring_event_overflow(struct io_ring_ctx *ctx, u64 user_data,
1644 long res, unsigned int cflags)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001645{
Pavel Begunkovcce4b8b2021-04-13 02:58:44 +01001646 struct io_overflow_cqe *ocqe;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001647
Pavel Begunkovcce4b8b2021-04-13 02:58:44 +01001648 ocqe = kmalloc(sizeof(*ocqe), GFP_ATOMIC | __GFP_ACCOUNT);
1649 if (!ocqe) {
1650 /*
1651 * If we're in ring overflow flush mode, or in task cancel mode,
1652 * or cannot allocate an overflow entry, then we need to drop it
1653 * on the floor.
1654 */
Pavel Begunkov8f6ed492021-05-16 22:58:10 +01001655 io_account_cq_overflow(ctx);
Pavel Begunkovcce4b8b2021-04-13 02:58:44 +01001656 return false;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001657 }
Pavel Begunkovcce4b8b2021-04-13 02:58:44 +01001658 if (list_empty(&ctx->cq_overflow_list)) {
Pavel Begunkov5ed7a372021-06-14 23:37:27 +01001659 set_bit(0, &ctx->check_cq_overflow);
Nadav Amit20c0b382021-08-07 17:13:42 -07001660 WRITE_ONCE(ctx->rings->sq_flags,
1661 ctx->rings->sq_flags | IORING_SQ_CQ_OVERFLOW);
1662
Pavel Begunkovcce4b8b2021-04-13 02:58:44 +01001663 }
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01001664 ocqe->cqe.user_data = user_data;
Pavel Begunkovcce4b8b2021-04-13 02:58:44 +01001665 ocqe->cqe.res = res;
1666 ocqe->cqe.flags = cflags;
1667 list_add_tail(&ocqe->list, &ctx->cq_overflow_list);
1668 return true;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001669}
1670
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01001671static inline bool __io_cqring_fill_event(struct io_ring_ctx *ctx, u64 user_data,
1672 long res, unsigned int cflags)
Pavel Begunkov8d133262021-04-11 01:46:33 +01001673{
Jens Axboe2b188cc2019-01-07 10:46:33 -07001674 struct io_uring_cqe *cqe;
1675
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01001676 trace_io_uring_complete(ctx, user_data, res, cflags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001677
1678 /*
1679 * If we can't get a cq entry, userspace overflowed the
1680 * submission (by quite a lot). Increment the overflow count in
1681 * the ring.
1682 */
Pavel Begunkovd068b502021-05-16 22:58:11 +01001683 cqe = io_get_cqe(ctx);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001684 if (likely(cqe)) {
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01001685 WRITE_ONCE(cqe->user_data, user_data);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001686 WRITE_ONCE(cqe->res, res);
1687 WRITE_ONCE(cqe->flags, cflags);
Pavel Begunkov8d133262021-04-11 01:46:33 +01001688 return true;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001689 }
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01001690 return io_cqring_event_overflow(ctx, user_data, res, cflags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001691}
1692
Pavel Begunkov8d133262021-04-11 01:46:33 +01001693/* not as hot to bloat with inlining */
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01001694static noinline bool io_cqring_fill_event(struct io_ring_ctx *ctx, u64 user_data,
1695 long res, unsigned int cflags)
Jens Axboebcda7ba2020-02-23 16:42:51 -07001696{
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01001697 return __io_cqring_fill_event(ctx, user_data, res, cflags);
Jens Axboebcda7ba2020-02-23 16:42:51 -07001698}
1699
Pavel Begunkov7a612352021-03-09 00:37:59 +00001700static void io_req_complete_post(struct io_kiocb *req, long res,
1701 unsigned int cflags)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001702{
Jens Axboe78e19bb2019-11-06 15:21:34 -07001703 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001704
Jens Axboe79ebeae2021-08-10 15:18:27 -06001705 spin_lock(&ctx->completion_lock);
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01001706 __io_cqring_fill_event(ctx, req->user_data, res, cflags);
Jens Axboec7dae4b2021-02-09 19:53:37 -07001707 /*
1708 * If we're the last reference to this request, add to our locked
1709 * free_list cache.
1710 */
Jens Axboede9b4cc2021-02-24 13:28:27 -07001711 if (req_ref_put_and_test(req)) {
Pavel Begunkov7a612352021-03-09 00:37:59 +00001712 if (req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) {
Pavel Begunkov0756a862021-08-15 10:40:25 +01001713 if (req->flags & IO_DISARM_MASK)
Pavel Begunkov7a612352021-03-09 00:37:59 +00001714 io_disarm_next(req);
1715 if (req->link) {
1716 io_req_task_queue(req->link);
1717 req->link = NULL;
1718 }
1719 }
Jens Axboec7dae4b2021-02-09 19:53:37 -07001720 io_dismantle_req(req);
1721 io_put_task(req->task, 1);
Pavel Begunkovbb943b82021-08-09 20:18:10 +01001722 list_add(&req->inflight_entry, &ctx->locked_free_list);
Pavel Begunkovd0acdee2021-05-16 22:58:12 +01001723 ctx->locked_free_nr++;
Pavel Begunkov180f8292021-03-14 20:57:09 +00001724 } else {
1725 if (!percpu_ref_tryget(&ctx->refs))
1726 req = NULL;
1727 }
Pavel Begunkov7a612352021-03-09 00:37:59 +00001728 io_commit_cqring(ctx);
Jens Axboe79ebeae2021-08-10 15:18:27 -06001729 spin_unlock(&ctx->completion_lock);
Pavel Begunkov7a612352021-03-09 00:37:59 +00001730
Pavel Begunkov180f8292021-03-14 20:57:09 +00001731 if (req) {
1732 io_cqring_ev_posted(ctx);
Jens Axboec7dae4b2021-02-09 19:53:37 -07001733 percpu_ref_put(&ctx->refs);
Pavel Begunkov180f8292021-03-14 20:57:09 +00001734 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07001735}
1736
Jens Axboe4e3d9ff2021-04-15 17:44:34 -06001737static inline bool io_req_needs_clean(struct io_kiocb *req)
1738{
Pavel Begunkovc8543572021-06-17 18:14:04 +01001739 return req->flags & IO_REQ_CLEAN_FLAGS;
Jens Axboe4e3d9ff2021-04-15 17:44:34 -06001740}
1741
Pavel Begunkova38d68d2021-01-19 13:32:45 +00001742static void io_req_complete_state(struct io_kiocb *req, long res,
Pavel Begunkov889fca72021-02-10 00:03:09 +00001743 unsigned int cflags)
Jens Axboebcda7ba2020-02-23 16:42:51 -07001744{
Jens Axboe4e3d9ff2021-04-15 17:44:34 -06001745 if (io_req_needs_clean(req))
Pavel Begunkov68fb8972021-03-19 17:22:41 +00001746 io_clean_op(req);
Pavel Begunkova38d68d2021-01-19 13:32:45 +00001747 req->result = res;
1748 req->compl.cflags = cflags;
Pavel Begunkove342c802021-01-19 13:32:47 +00001749 req->flags |= REQ_F_COMPLETE_INLINE;
Jens Axboee1e16092020-06-22 09:17:17 -06001750}
Jens Axboe2b188cc2019-01-07 10:46:33 -07001751
Pavel Begunkov889fca72021-02-10 00:03:09 +00001752static inline void __io_req_complete(struct io_kiocb *req, unsigned issue_flags,
1753 long res, unsigned cflags)
Pavel Begunkova38d68d2021-01-19 13:32:45 +00001754{
Pavel Begunkov889fca72021-02-10 00:03:09 +00001755 if (issue_flags & IO_URING_F_COMPLETE_DEFER)
1756 io_req_complete_state(req, res, cflags);
Pavel Begunkova38d68d2021-01-19 13:32:45 +00001757 else
Jens Axboec7dae4b2021-02-09 19:53:37 -07001758 io_req_complete_post(req, res, cflags);
Pavel Begunkova38d68d2021-01-19 13:32:45 +00001759}
Jens Axboebcda7ba2020-02-23 16:42:51 -07001760
Pavel Begunkova38d68d2021-01-19 13:32:45 +00001761static inline void io_req_complete(struct io_kiocb *req, long res)
Jens Axboee1e16092020-06-22 09:17:17 -06001762{
Pavel Begunkov889fca72021-02-10 00:03:09 +00001763 __io_req_complete(req, 0, res, 0);
Jens Axboebcda7ba2020-02-23 16:42:51 -07001764}
1765
Pavel Begunkovf41db2732021-02-28 22:35:12 +00001766static void io_req_complete_failed(struct io_kiocb *req, long res)
1767{
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01001768 req_set_fail(req);
Pavel Begunkovf41db2732021-02-28 22:35:12 +00001769 io_req_complete_post(req, res, 0);
1770}
1771
Pavel Begunkov864ea922021-08-09 13:04:08 +01001772/*
1773 * Don't initialise the fields below on every allocation, but do that in
1774 * advance and keep them valid across allocations.
1775 */
1776static void io_preinit_req(struct io_kiocb *req, struct io_ring_ctx *ctx)
1777{
1778 req->ctx = ctx;
1779 req->link = NULL;
1780 req->async_data = NULL;
1781 /* not necessary, but safer to zero */
1782 req->result = 0;
1783}
1784
Pavel Begunkovdac7a092021-03-19 17:22:39 +00001785static void io_flush_cached_locked_reqs(struct io_ring_ctx *ctx,
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01001786 struct io_submit_state *state)
Pavel Begunkovdac7a092021-03-19 17:22:39 +00001787{
Jens Axboe79ebeae2021-08-10 15:18:27 -06001788 spin_lock(&ctx->completion_lock);
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01001789 list_splice_init(&ctx->locked_free_list, &state->free_list);
Pavel Begunkovd0acdee2021-05-16 22:58:12 +01001790 ctx->locked_free_nr = 0;
Jens Axboe79ebeae2021-08-10 15:18:27 -06001791 spin_unlock(&ctx->completion_lock);
Pavel Begunkovdac7a092021-03-19 17:22:39 +00001792}
1793
Pavel Begunkovdd78f492021-03-19 17:22:35 +00001794/* Returns true IFF there are requests in the cache */
Jens Axboec7dae4b2021-02-09 19:53:37 -07001795static bool io_flush_cached_reqs(struct io_ring_ctx *ctx)
Pavel Begunkove5d1bc02021-02-10 00:03:23 +00001796{
Jens Axboec7dae4b2021-02-09 19:53:37 -07001797 struct io_submit_state *state = &ctx->submit_state;
Pavel Begunkovdd78f492021-03-19 17:22:35 +00001798 int nr;
Pavel Begunkove5d1bc02021-02-10 00:03:23 +00001799
Jens Axboec7dae4b2021-02-09 19:53:37 -07001800 /*
1801 * If we have more than a batch's worth of requests in our IRQ side
1802 * locked cache, grab the lock and move them over to our submission
1803 * side cache.
1804 */
Pavel Begunkovd0acdee2021-05-16 22:58:12 +01001805 if (READ_ONCE(ctx->locked_free_nr) > IO_COMPL_BATCH)
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01001806 io_flush_cached_locked_reqs(ctx, state);
Jens Axboec7dae4b2021-02-09 19:53:37 -07001807
Pavel Begunkovdd78f492021-03-19 17:22:35 +00001808 nr = state->free_reqs;
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01001809 while (!list_empty(&state->free_list)) {
1810 struct io_kiocb *req = list_first_entry(&state->free_list,
Pavel Begunkovbb943b82021-08-09 20:18:10 +01001811 struct io_kiocb, inflight_entry);
Pavel Begunkovdd78f492021-03-19 17:22:35 +00001812
Pavel Begunkovbb943b82021-08-09 20:18:10 +01001813 list_del(&req->inflight_entry);
Pavel Begunkovdd78f492021-03-19 17:22:35 +00001814 state->reqs[nr++] = req;
1815 if (nr == ARRAY_SIZE(state->reqs))
Pavel Begunkove5d1bc02021-02-10 00:03:23 +00001816 break;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001817 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07001818
Pavel Begunkovdd78f492021-03-19 17:22:35 +00001819 state->free_reqs = nr;
1820 return nr != 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001821}
1822
Pavel Begunkov5d5901a2021-08-11 19:28:29 +01001823/*
1824 * A request might get retired back into the request caches even before opcode
1825 * handlers and io_issue_sqe() are done with it, e.g. inline completion path.
1826 * Because of that, io_alloc_req() should be called only under ->uring_lock
1827 * and with extra caution to not get a request that is still worked on.
1828 */
Pavel Begunkov258b29a2021-02-10 00:03:10 +00001829static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx)
Pavel Begunkov5d5901a2021-08-11 19:28:29 +01001830 __must_hold(&ctx->uring_lock)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001831{
Pavel Begunkov258b29a2021-02-10 00:03:10 +00001832 struct io_submit_state *state = &ctx->submit_state;
Pavel Begunkov864ea922021-08-09 13:04:08 +01001833 gfp_t gfp = GFP_KERNEL | __GFP_NOWARN;
1834 int ret, i;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001835
Pavel Begunkovfe7e3252021-06-24 15:09:57 +01001836 BUILD_BUG_ON(ARRAY_SIZE(state->reqs) < IO_REQ_ALLOC_BATCH);
Jens Axboe2b188cc2019-01-07 10:46:33 -07001837
Pavel Begunkov864ea922021-08-09 13:04:08 +01001838 if (likely(state->free_reqs || io_flush_cached_reqs(ctx)))
1839 goto got_req;
Jens Axboe2579f912019-01-09 09:10:43 -07001840
Pavel Begunkov864ea922021-08-09 13:04:08 +01001841 ret = kmem_cache_alloc_bulk(req_cachep, gfp, IO_REQ_ALLOC_BATCH,
1842 state->reqs);
Pavel Begunkove5d1bc02021-02-10 00:03:23 +00001843
Pavel Begunkov864ea922021-08-09 13:04:08 +01001844 /*
1845 * Bulk alloc is all-or-nothing. If we fail to get a batch,
1846 * retry single alloc to be on the safe side.
1847 */
1848 if (unlikely(ret <= 0)) {
1849 state->reqs[0] = kmem_cache_alloc(req_cachep, gfp);
1850 if (!state->reqs[0])
1851 return NULL;
1852 ret = 1;
Jens Axboe2b188cc2019-01-07 10:46:33 -07001853 }
Pavel Begunkov864ea922021-08-09 13:04:08 +01001854
1855 for (i = 0; i < ret; i++)
1856 io_preinit_req(state->reqs[i], ctx);
1857 state->free_reqs = ret;
Pavel Begunkove5d1bc02021-02-10 00:03:23 +00001858got_req:
Pavel Begunkov291b2822020-09-30 22:57:01 +03001859 state->free_reqs--;
1860 return state->reqs[state->free_reqs];
Jens Axboe2b188cc2019-01-07 10:46:33 -07001861}
1862
Pavel Begunkove1d767f2021-03-19 17:22:43 +00001863static inline void io_put_file(struct file *file)
Pavel Begunkov8da11c12020-02-24 11:32:44 +03001864{
Pavel Begunkove1d767f2021-03-19 17:22:43 +00001865 if (file)
Pavel Begunkov8da11c12020-02-24 11:32:44 +03001866 fput(file);
1867}
1868
Pavel Begunkov4edf20f2020-10-13 09:43:59 +01001869static void io_dismantle_req(struct io_kiocb *req)
Jens Axboe2b188cc2019-01-07 10:46:33 -07001870{
Pavel Begunkov094bae42021-03-19 17:22:42 +00001871 unsigned int flags = req->flags;
Pavel Begunkov929a3af2020-02-19 00:19:09 +03001872
Pavel Begunkov3a0a6902021-04-20 12:03:31 +01001873 if (io_req_needs_clean(req))
1874 io_clean_op(req);
Pavel Begunkove1d767f2021-03-19 17:22:43 +00001875 if (!(flags & REQ_F_FIXED_FILE))
1876 io_put_file(req->file);
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00001877 if (req->fixed_rsrc_refs)
1878 percpu_ref_put(req->fixed_rsrc_refs);
Pavel Begunkov99ebe4e2021-06-26 21:40:49 +01001879 if (req->async_data) {
Pavel Begunkov094bae42021-03-19 17:22:42 +00001880 kfree(req->async_data);
Pavel Begunkov99ebe4e2021-06-26 21:40:49 +01001881 req->async_data = NULL;
1882 }
Pavel Begunkove6543a82020-06-28 12:52:30 +03001883}
Pavel Begunkov2b85edf2019-12-28 14:13:03 +03001884
Pavel Begunkov216578e2020-10-13 09:44:00 +01001885static void __io_free_req(struct io_kiocb *req)
Pavel Begunkove6543a82020-06-28 12:52:30 +03001886{
Jens Axboe51a4cc12020-08-10 10:55:56 -06001887 struct io_ring_ctx *ctx = req->ctx;
Pavel Begunkovecfc5172020-06-29 13:13:03 +03001888
Pavel Begunkov216578e2020-10-13 09:44:00 +01001889 io_dismantle_req(req);
Pavel Begunkov7c660732021-01-25 11:42:21 +00001890 io_put_task(req->task, 1);
Pavel Begunkove6543a82020-06-28 12:52:30 +03001891
Jens Axboe79ebeae2021-08-10 15:18:27 -06001892 spin_lock(&ctx->completion_lock);
Pavel Begunkovbb943b82021-08-09 20:18:10 +01001893 list_add(&req->inflight_entry, &ctx->locked_free_list);
Pavel Begunkovc34b0252021-08-09 20:18:08 +01001894 ctx->locked_free_nr++;
Jens Axboe79ebeae2021-08-10 15:18:27 -06001895 spin_unlock(&ctx->completion_lock);
Pavel Begunkovc34b0252021-08-09 20:18:08 +01001896
Pavel Begunkovecfc5172020-06-29 13:13:03 +03001897 percpu_ref_put(&ctx->refs);
Jens Axboee65ef562019-03-12 10:16:44 -06001898}
1899
Pavel Begunkovf2f87372020-10-27 23:25:37 +00001900static inline void io_remove_next_linked(struct io_kiocb *req)
1901{
1902 struct io_kiocb *nxt = req->link;
1903
1904 req->link = nxt->link;
1905 nxt->link = NULL;
1906}
1907
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00001908static bool io_kill_linked_timeout(struct io_kiocb *req)
1909 __must_hold(&req->ctx->completion_lock)
Jens Axboe89b263f2021-08-10 15:14:18 -06001910 __must_hold(&req->ctx->timeout_lock)
Jens Axboe9e645e112019-05-10 16:07:28 -06001911{
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00001912 struct io_kiocb *link = req->link;
Pavel Begunkovf2f87372020-10-27 23:25:37 +00001913
Pavel Begunkovb97e7362021-08-15 10:40:23 +01001914 if (link && link->opcode == IORING_OP_LINK_TIMEOUT) {
Pavel Begunkovc9abd7a2020-10-22 16:43:11 +01001915 struct io_timeout_data *io = link->async_data;
Pavel Begunkov7c86ffe2020-06-29 13:12:59 +03001916
Pavel Begunkovf2f87372020-10-27 23:25:37 +00001917 io_remove_next_linked(req);
Pavel Begunkov90cd7e42020-10-27 23:25:36 +00001918 link->timeout.head = NULL;
Pavel Begunkovfd9c7bc2021-04-13 02:58:42 +01001919 if (hrtimer_try_to_cancel(&io->timer) != -1) {
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01001920 io_cqring_fill_event(link->ctx, link->user_data,
1921 -ECANCELED, 0);
Pavel Begunkov91c2f692021-08-11 19:28:28 +01001922 io_put_req_deferred(link);
Pavel Begunkovd4729fb2021-03-22 01:58:24 +00001923 return true;
Pavel Begunkovc9abd7a2020-10-22 16:43:11 +01001924 }
1925 }
Pavel Begunkovd4729fb2021-03-22 01:58:24 +00001926 return false;
Pavel Begunkov7c86ffe2020-06-29 13:12:59 +03001927}
1928
Pavel Begunkovd148ca42020-10-18 10:17:39 +01001929static void io_fail_links(struct io_kiocb *req)
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00001930 __must_hold(&req->ctx->completion_lock)
Jens Axboe9e645e112019-05-10 16:07:28 -06001931{
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00001932 struct io_kiocb *nxt, *link = req->link;
Jens Axboe9e645e112019-05-10 16:07:28 -06001933
Pavel Begunkovf2f87372020-10-27 23:25:37 +00001934 req->link = NULL;
Pavel Begunkovf2f87372020-10-27 23:25:37 +00001935 while (link) {
1936 nxt = link->link;
1937 link->link = NULL;
1938
Dmitrii Dolgovc826bd72019-10-15 19:02:01 +02001939 trace_io_uring_fail_link(req, link);
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01001940 io_cqring_fill_event(link->ctx, link->user_data, -ECANCELED, 0);
Pavel Begunkov91c2f692021-08-11 19:28:28 +01001941 io_put_req_deferred(link);
Pavel Begunkovf2f87372020-10-27 23:25:37 +00001942 link = nxt;
Jens Axboe9e645e112019-05-10 16:07:28 -06001943 }
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00001944}
Pavel Begunkov7c86ffe2020-06-29 13:12:59 +03001945
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00001946static bool io_disarm_next(struct io_kiocb *req)
1947 __must_hold(&req->ctx->completion_lock)
1948{
1949 bool posted = false;
1950
Pavel Begunkov0756a862021-08-15 10:40:25 +01001951 if (req->flags & REQ_F_ARM_LTIMEOUT) {
1952 struct io_kiocb *link = req->link;
1953
Pavel Begunkov906c6ca2021-08-15 10:40:26 +01001954 req->flags &= ~REQ_F_ARM_LTIMEOUT;
Pavel Begunkov0756a862021-08-15 10:40:25 +01001955 if (link && link->opcode == IORING_OP_LINK_TIMEOUT) {
1956 io_remove_next_linked(req);
1957 io_cqring_fill_event(link->ctx, link->user_data,
1958 -ECANCELED, 0);
1959 io_put_req_deferred(link);
1960 posted = true;
1961 }
1962 } else if (req->flags & REQ_F_LINK_TIMEOUT) {
Jens Axboe89b263f2021-08-10 15:14:18 -06001963 struct io_ring_ctx *ctx = req->ctx;
1964
1965 spin_lock_irq(&ctx->timeout_lock);
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00001966 posted = io_kill_linked_timeout(req);
Jens Axboe89b263f2021-08-10 15:14:18 -06001967 spin_unlock_irq(&ctx->timeout_lock);
1968 }
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01001969 if (unlikely((req->flags & REQ_F_FAIL) &&
Pavel Begunkove4335ed2021-04-11 01:46:39 +01001970 !(req->flags & REQ_F_HARDLINK))) {
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00001971 posted |= (req->link != NULL);
1972 io_fail_links(req);
1973 }
1974 return posted;
Jens Axboe9e645e112019-05-10 16:07:28 -06001975}
1976
Pavel Begunkov3fa5e0f2020-06-30 15:20:43 +03001977static struct io_kiocb *__io_req_find_next(struct io_kiocb *req)
Jens Axboe9e645e112019-05-10 16:07:28 -06001978{
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00001979 struct io_kiocb *nxt;
Jens Axboe2665abf2019-11-05 12:40:47 -07001980
Jens Axboe9e645e112019-05-10 16:07:28 -06001981 /*
1982 * If LINK is set, we have dependent requests in this chain. If we
1983 * didn't fail this request, queue the first one up, moving any other
1984 * dependencies to the next request. In case of failure, fail the rest
1985 * of the chain.
1986 */
Pavel Begunkov0756a862021-08-15 10:40:25 +01001987 if (req->flags & IO_DISARM_MASK) {
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00001988 struct io_ring_ctx *ctx = req->ctx;
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00001989 bool posted;
Pavel Begunkovf2f87372020-10-27 23:25:37 +00001990
Jens Axboe79ebeae2021-08-10 15:18:27 -06001991 spin_lock(&ctx->completion_lock);
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00001992 posted = io_disarm_next(req);
1993 if (posted)
1994 io_commit_cqring(req->ctx);
Jens Axboe79ebeae2021-08-10 15:18:27 -06001995 spin_unlock(&ctx->completion_lock);
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00001996 if (posted)
1997 io_cqring_ev_posted(ctx);
Pavel Begunkovf2f87372020-10-27 23:25:37 +00001998 }
Pavel Begunkov33cc89a2021-03-09 00:37:58 +00001999 nxt = req->link;
2000 req->link = NULL;
2001 return nxt;
Jens Axboe4d7dd462019-11-20 13:03:52 -07002002}
Jens Axboe2665abf2019-11-05 12:40:47 -07002003
Pavel Begunkovf2f87372020-10-27 23:25:37 +00002004static inline struct io_kiocb *io_req_find_next(struct io_kiocb *req)
Pavel Begunkov3fa5e0f2020-06-30 15:20:43 +03002005{
Pavel Begunkovcdbff982021-02-12 18:41:16 +00002006 if (likely(!(req->flags & (REQ_F_LINK|REQ_F_HARDLINK))))
Pavel Begunkov3fa5e0f2020-06-30 15:20:43 +03002007 return NULL;
2008 return __io_req_find_next(req);
2009}
2010
Pavel Begunkov2c323952021-02-28 22:04:53 +00002011static void ctx_flush_and_put(struct io_ring_ctx *ctx)
2012{
2013 if (!ctx)
2014 return;
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01002015 if (ctx->submit_state.compl_nr) {
Pavel Begunkov2c323952021-02-28 22:04:53 +00002016 mutex_lock(&ctx->uring_lock);
Hao Xu99c8bc52021-08-21 06:19:54 +08002017 if (ctx->submit_state.compl_nr)
2018 io_submit_flush_completions(ctx);
Pavel Begunkov2c323952021-02-28 22:04:53 +00002019 mutex_unlock(&ctx->uring_lock);
2020 }
2021 percpu_ref_put(&ctx->refs);
2022}
2023
Jens Axboe7cbf1722021-02-10 00:03:20 +00002024static void tctx_task_work(struct callback_head *cb)
2025{
Pavel Begunkovebd0df22021-06-17 18:14:07 +01002026 struct io_ring_ctx *ctx = NULL;
Pavel Begunkov3f184072021-06-17 18:14:06 +01002027 struct io_uring_task *tctx = container_of(cb, struct io_uring_task,
2028 task_work);
Jens Axboe7cbf1722021-02-10 00:03:20 +00002029
Pavel Begunkov16f72072021-06-17 18:14:09 +01002030 while (1) {
Pavel Begunkov3f184072021-06-17 18:14:06 +01002031 struct io_wq_work_node *node;
2032
2033 spin_lock_irq(&tctx->task_lock);
Pavel Begunkovc6538be2021-06-17 18:14:08 +01002034 node = tctx->task_list.first;
Pavel Begunkov3f184072021-06-17 18:14:06 +01002035 INIT_WQ_LIST(&tctx->task_list);
Pavel Begunkov6294f362021-08-10 17:53:55 +01002036 if (!node)
2037 tctx->task_running = false;
Pavel Begunkov3f184072021-06-17 18:14:06 +01002038 spin_unlock_irq(&tctx->task_lock);
Pavel Begunkov6294f362021-08-10 17:53:55 +01002039 if (!node)
2040 break;
Pavel Begunkov3f184072021-06-17 18:14:06 +01002041
Pavel Begunkov6294f362021-08-10 17:53:55 +01002042 do {
Pavel Begunkov3f184072021-06-17 18:14:06 +01002043 struct io_wq_work_node *next = node->next;
2044 struct io_kiocb *req = container_of(node, struct io_kiocb,
2045 io_task_work.node);
2046
2047 if (req->ctx != ctx) {
2048 ctx_flush_and_put(ctx);
2049 ctx = req->ctx;
2050 percpu_ref_get(&ctx->refs);
2051 }
Pavel Begunkov5b0a6ac2021-06-30 21:54:04 +01002052 req->io_task_work.func(req);
Pavel Begunkov3f184072021-06-17 18:14:06 +01002053 node = next;
Pavel Begunkov6294f362021-08-10 17:53:55 +01002054 } while (node);
2055
Jens Axboe7cbf1722021-02-10 00:03:20 +00002056 cond_resched();
Pavel Begunkov3f184072021-06-17 18:14:06 +01002057 }
Pavel Begunkovebd0df22021-06-17 18:14:07 +01002058
2059 ctx_flush_and_put(ctx);
Jens Axboe7cbf1722021-02-10 00:03:20 +00002060}
2061
Pavel Begunkove09ee512021-07-01 13:26:05 +01002062static void io_req_task_work_add(struct io_kiocb *req)
Jens Axboe7cbf1722021-02-10 00:03:20 +00002063{
Pavel Begunkovc15b79d2021-03-19 17:22:44 +00002064 struct task_struct *tsk = req->task;
Jens Axboe7cbf1722021-02-10 00:03:20 +00002065 struct io_uring_task *tctx = tsk->io_uring;
Pavel Begunkovc15b79d2021-03-19 17:22:44 +00002066 enum task_work_notify_mode notify;
Pavel Begunkove09ee512021-07-01 13:26:05 +01002067 struct io_wq_work_node *node;
Jens Axboe0b81e802021-02-16 10:33:53 -07002068 unsigned long flags;
Pavel Begunkov6294f362021-08-10 17:53:55 +01002069 bool running;
Jens Axboe7cbf1722021-02-10 00:03:20 +00002070
2071 WARN_ON_ONCE(!tctx);
2072
Jens Axboe0b81e802021-02-16 10:33:53 -07002073 spin_lock_irqsave(&tctx->task_lock, flags);
Jens Axboe7cbf1722021-02-10 00:03:20 +00002074 wq_list_add_tail(&req->io_task_work.node, &tctx->task_list);
Pavel Begunkov6294f362021-08-10 17:53:55 +01002075 running = tctx->task_running;
2076 if (!running)
2077 tctx->task_running = true;
Jens Axboe0b81e802021-02-16 10:33:53 -07002078 spin_unlock_irqrestore(&tctx->task_lock, flags);
Jens Axboe7cbf1722021-02-10 00:03:20 +00002079
2080 /* task_work already pending, we're done */
Pavel Begunkov6294f362021-08-10 17:53:55 +01002081 if (running)
Pavel Begunkove09ee512021-07-01 13:26:05 +01002082 return;
Jens Axboe7cbf1722021-02-10 00:03:20 +00002083
Pavel Begunkovc15b79d2021-03-19 17:22:44 +00002084 /*
2085 * SQPOLL kernel thread doesn't need notification, just a wakeup. For
2086 * all other cases, use TWA_SIGNAL unconditionally to ensure we're
2087 * processing task_work. There's no reliable way to tell if TWA_RESUME
2088 * will do the job.
2089 */
2090 notify = (req->ctx->flags & IORING_SETUP_SQPOLL) ? TWA_NONE : TWA_SIGNAL;
Pavel Begunkovc15b79d2021-03-19 17:22:44 +00002091 if (!task_work_add(tsk, &tctx->task_work, notify)) {
2092 wake_up_process(tsk);
Pavel Begunkove09ee512021-07-01 13:26:05 +01002093 return;
Pavel Begunkovc15b79d2021-03-19 17:22:44 +00002094 }
Pavel Begunkov2215bed2021-08-09 13:04:06 +01002095
Pavel Begunkove09ee512021-07-01 13:26:05 +01002096 spin_lock_irqsave(&tctx->task_lock, flags);
Pavel Begunkov6294f362021-08-10 17:53:55 +01002097 tctx->task_running = false;
Pavel Begunkove09ee512021-07-01 13:26:05 +01002098 node = tctx->task_list.first;
2099 INIT_WQ_LIST(&tctx->task_list);
2100 spin_unlock_irqrestore(&tctx->task_lock, flags);
Jens Axboe7cbf1722021-02-10 00:03:20 +00002101
Pavel Begunkove09ee512021-07-01 13:26:05 +01002102 while (node) {
2103 req = container_of(node, struct io_kiocb, io_task_work.node);
2104 node = node->next;
2105 if (llist_add(&req->io_task_work.fallback_node,
2106 &req->ctx->fallback_llist))
2107 schedule_delayed_work(&req->ctx->fallback_work, 1);
2108 }
Pavel Begunkoveab30c42021-01-19 13:32:42 +00002109}
2110
Pavel Begunkov5b0a6ac2021-06-30 21:54:04 +01002111static void io_req_task_cancel(struct io_kiocb *req)
Jens Axboec40f6372020-06-25 15:39:59 -06002112{
Jens Axboe87ceb6a2020-09-14 08:20:12 -06002113 struct io_ring_ctx *ctx = req->ctx;
Jens Axboec40f6372020-06-25 15:39:59 -06002114
Pavel Begunkove83acd72021-02-28 22:35:09 +00002115 /* ctx is guaranteed to stay alive while we hold uring_lock */
Pavel Begunkov792bb6e2021-02-18 22:32:51 +00002116 mutex_lock(&ctx->uring_lock);
Pavel Begunkov25935532021-03-19 17:22:40 +00002117 io_req_complete_failed(req, req->result);
Pavel Begunkov792bb6e2021-02-18 22:32:51 +00002118 mutex_unlock(&ctx->uring_lock);
Jens Axboec40f6372020-06-25 15:39:59 -06002119}
2120
Pavel Begunkov5b0a6ac2021-06-30 21:54:04 +01002121static void io_req_task_submit(struct io_kiocb *req)
Jens Axboec40f6372020-06-25 15:39:59 -06002122{
2123 struct io_ring_ctx *ctx = req->ctx;
2124
Pavel Begunkov04fc6c82021-02-12 03:23:54 +00002125 /* ctx stays valid until unlock, even if we drop all ours ctx->refs */
Pavel Begunkov81b6d052021-01-04 20:36:35 +00002126 mutex_lock(&ctx->uring_lock);
Jens Axboe316319e2021-08-19 09:41:42 -06002127 /* req->task == current here, checking PF_EXITING is safe */
Pavel Begunkovaf066f32021-08-09 13:04:19 +01002128 if (likely(!(req->task->flags & PF_EXITING)))
Pavel Begunkovc5eef2b2021-02-10 00:03:22 +00002129 __io_queue_sqe(req);
Pavel Begunkov81b6d052021-01-04 20:36:35 +00002130 else
Pavel Begunkov25935532021-03-19 17:22:40 +00002131 io_req_complete_failed(req, -EFAULT);
Pavel Begunkov81b6d052021-01-04 20:36:35 +00002132 mutex_unlock(&ctx->uring_lock);
Jens Axboe9e645e112019-05-10 16:07:28 -06002133}
2134
Pavel Begunkova3df76982021-02-18 22:32:52 +00002135static void io_req_task_queue_fail(struct io_kiocb *req, int ret)
2136{
Pavel Begunkova3df76982021-02-18 22:32:52 +00002137 req->result = ret;
Pavel Begunkov5b0a6ac2021-06-30 21:54:04 +01002138 req->io_task_work.func = io_req_task_cancel;
Pavel Begunkove09ee512021-07-01 13:26:05 +01002139 io_req_task_work_add(req);
Pavel Begunkova3df76982021-02-18 22:32:52 +00002140}
2141
Pavel Begunkov2c4b8eb2021-02-28 22:35:10 +00002142static void io_req_task_queue(struct io_kiocb *req)
2143{
Pavel Begunkov5b0a6ac2021-06-30 21:54:04 +01002144 req->io_task_work.func = io_req_task_submit;
Pavel Begunkove09ee512021-07-01 13:26:05 +01002145 io_req_task_work_add(req);
Pavel Begunkov2c4b8eb2021-02-28 22:35:10 +00002146}
2147
Jens Axboe773af692021-07-27 10:25:55 -06002148static void io_req_task_queue_reissue(struct io_kiocb *req)
2149{
2150 req->io_task_work.func = io_queue_async_work;
2151 io_req_task_work_add(req);
2152}
2153
Pavel Begunkovf2f87372020-10-27 23:25:37 +00002154static inline void io_queue_next(struct io_kiocb *req)
Jackie Liuc69f8db2019-11-09 11:00:08 +08002155{
Pavel Begunkov9b5f7bd2020-06-29 13:13:00 +03002156 struct io_kiocb *nxt = io_req_find_next(req);
Pavel Begunkov944e58b2019-11-21 23:21:01 +03002157
Pavel Begunkov906a8c32020-06-27 14:04:55 +03002158 if (nxt)
2159 io_req_task_queue(nxt);
Jackie Liuc69f8db2019-11-09 11:00:08 +08002160}
2161
Jens Axboe9e645e112019-05-10 16:07:28 -06002162static void io_free_req(struct io_kiocb *req)
2163{
Pavel Begunkovc3524382020-06-28 12:52:32 +03002164 io_queue_next(req);
Jens Axboe9e645e112019-05-10 16:07:28 -06002165 __io_free_req(req);
Jens Axboee65ef562019-03-12 10:16:44 -06002166}
2167
Pavel Begunkov2d6500d2020-06-28 12:52:33 +03002168struct req_batch {
Pavel Begunkov5af1d132020-07-18 11:32:52 +03002169 struct task_struct *task;
2170 int task_refs;
Jens Axboe1b4c3512021-02-10 00:03:19 +00002171 int ctx_refs;
Pavel Begunkov2d6500d2020-06-28 12:52:33 +03002172};
2173
Pavel Begunkov5af1d132020-07-18 11:32:52 +03002174static inline void io_init_req_batch(struct req_batch *rb)
Pavel Begunkov7a743e22020-03-03 21:33:13 +03002175{
Pavel Begunkov5af1d132020-07-18 11:32:52 +03002176 rb->task_refs = 0;
Pavel Begunkov9ae72462021-02-10 00:03:16 +00002177 rb->ctx_refs = 0;
Pavel Begunkov5af1d132020-07-18 11:32:52 +03002178 rb->task = NULL;
2179}
Pavel Begunkov8766dd52020-03-14 00:31:04 +03002180
Pavel Begunkov2d6500d2020-06-28 12:52:33 +03002181static void io_req_free_batch_finish(struct io_ring_ctx *ctx,
2182 struct req_batch *rb)
2183{
Pavel Begunkov9ae72462021-02-10 00:03:16 +00002184 if (rb->ctx_refs)
2185 percpu_ref_put_many(&ctx->refs, rb->ctx_refs);
Pavel Begunkove98e49b2021-08-18 17:01:43 +01002186 if (rb->task)
Pavel Begunkove9dbe222021-08-09 13:04:20 +01002187 io_put_task(rb->task, rb->task_refs);
Pavel Begunkov2d6500d2020-06-28 12:52:33 +03002188}
2189
Pavel Begunkov6ff119a2021-02-10 00:03:18 +00002190static void io_req_free_batch(struct req_batch *rb, struct io_kiocb *req,
2191 struct io_submit_state *state)
Pavel Begunkov2d6500d2020-06-28 12:52:33 +03002192{
Pavel Begunkovf2f87372020-10-27 23:25:37 +00002193 io_queue_next(req);
Pavel Begunkov96670652021-03-19 17:22:32 +00002194 io_dismantle_req(req);
Pavel Begunkov2d6500d2020-06-28 12:52:33 +03002195
Jens Axboee3bc8e92020-09-24 08:45:57 -06002196 if (req->task != rb->task) {
Pavel Begunkov7c660732021-01-25 11:42:21 +00002197 if (rb->task)
2198 io_put_task(rb->task, rb->task_refs);
Jens Axboee3bc8e92020-09-24 08:45:57 -06002199 rb->task = req->task;
2200 rb->task_refs = 0;
Pavel Begunkov5af1d132020-07-18 11:32:52 +03002201 }
Jens Axboee3bc8e92020-09-24 08:45:57 -06002202 rb->task_refs++;
Pavel Begunkov9ae72462021-02-10 00:03:16 +00002203 rb->ctx_refs++;
Pavel Begunkov5af1d132020-07-18 11:32:52 +03002204
Pavel Begunkovbd759042021-02-12 03:23:50 +00002205 if (state->free_reqs != ARRAY_SIZE(state->reqs))
Pavel Begunkov6ff119a2021-02-10 00:03:18 +00002206 state->reqs[state->free_reqs++] = req;
Pavel Begunkovbd759042021-02-12 03:23:50 +00002207 else
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01002208 list_add(&req->inflight_entry, &state->free_list);
Pavel Begunkov7a743e22020-03-03 21:33:13 +03002209}
2210
Pavel Begunkov2a2758f2021-06-17 18:14:00 +01002211static void io_submit_flush_completions(struct io_ring_ctx *ctx)
Jens Axboea141dd82021-08-12 12:48:34 -06002212 __must_hold(&ctx->uring_lock)
Pavel Begunkov905c1722021-02-10 00:03:14 +00002213{
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01002214 struct io_submit_state *state = &ctx->submit_state;
2215 int i, nr = state->compl_nr;
Pavel Begunkov905c1722021-02-10 00:03:14 +00002216 struct req_batch rb;
2217
Jens Axboe79ebeae2021-08-10 15:18:27 -06002218 spin_lock(&ctx->completion_lock);
Pavel Begunkov905c1722021-02-10 00:03:14 +00002219 for (i = 0; i < nr; i++) {
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01002220 struct io_kiocb *req = state->compl_reqs[i];
Pavel Begunkov5182ed22021-06-26 21:40:48 +01002221
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01002222 __io_cqring_fill_event(ctx, req->user_data, req->result,
2223 req->compl.cflags);
Pavel Begunkov905c1722021-02-10 00:03:14 +00002224 }
2225 io_commit_cqring(ctx);
Jens Axboe79ebeae2021-08-10 15:18:27 -06002226 spin_unlock(&ctx->completion_lock);
Pavel Begunkov905c1722021-02-10 00:03:14 +00002227 io_cqring_ev_posted(ctx);
Pavel Begunkov5182ed22021-06-26 21:40:48 +01002228
2229 io_init_req_batch(&rb);
Pavel Begunkov905c1722021-02-10 00:03:14 +00002230 for (i = 0; i < nr; i++) {
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01002231 struct io_kiocb *req = state->compl_reqs[i];
Pavel Begunkov905c1722021-02-10 00:03:14 +00002232
Pavel Begunkov91c2f692021-08-11 19:28:28 +01002233 if (req_ref_put_and_test(req))
Pavel Begunkov6ff119a2021-02-10 00:03:18 +00002234 io_req_free_batch(&rb, req, &ctx->submit_state);
Pavel Begunkov905c1722021-02-10 00:03:14 +00002235 }
2236
2237 io_req_free_batch_finish(ctx, &rb);
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01002238 state->compl_nr = 0;
Jens Axboee65ef562019-03-12 10:16:44 -06002239}
2240
Jens Axboeba816ad2019-09-28 11:36:45 -06002241/*
2242 * Drop reference to request, return next in chain (if there is one) if this
2243 * was the last reference to this request.
2244 */
Pavel Begunkov0d850352021-03-19 17:22:37 +00002245static inline struct io_kiocb *io_put_req_find_next(struct io_kiocb *req)
Jens Axboee65ef562019-03-12 10:16:44 -06002246{
Pavel Begunkov9b5f7bd2020-06-29 13:13:00 +03002247 struct io_kiocb *nxt = NULL;
2248
Jens Axboede9b4cc2021-02-24 13:28:27 -07002249 if (req_ref_put_and_test(req)) {
Pavel Begunkov9b5f7bd2020-06-29 13:13:00 +03002250 nxt = io_req_find_next(req);
Jens Axboe4d7dd462019-11-20 13:03:52 -07002251 __io_free_req(req);
Jens Axboe2a44f462020-02-25 13:25:41 -07002252 }
Pavel Begunkov9b5f7bd2020-06-29 13:13:00 +03002253 return nxt;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002254}
2255
Pavel Begunkov0d850352021-03-19 17:22:37 +00002256static inline void io_put_req(struct io_kiocb *req)
Jens Axboe2b188cc2019-01-07 10:46:33 -07002257{
Jens Axboede9b4cc2021-02-24 13:28:27 -07002258 if (req_ref_put_and_test(req))
Jens Axboedef596e2019-01-09 08:59:42 -07002259 io_free_req(req);
2260}
2261
Pavel Begunkov91c2f692021-08-11 19:28:28 +01002262static inline void io_put_req_deferred(struct io_kiocb *req)
Pavel Begunkov216578e2020-10-13 09:44:00 +01002263{
Pavel Begunkov91c2f692021-08-11 19:28:28 +01002264 if (req_ref_put_and_test(req)) {
Pavel Begunkov543af3a2021-08-09 13:04:15 +01002265 req->io_task_work.func = io_free_req;
2266 io_req_task_work_add(req);
2267 }
Pavel Begunkov216578e2020-10-13 09:44:00 +01002268}
2269
Pavel Begunkov6c503152021-01-04 20:36:36 +00002270static unsigned io_cqring_events(struct io_ring_ctx *ctx)
Jens Axboea3a0e432019-08-20 11:03:11 -06002271{
2272 /* See comment at the top of this file */
2273 smp_rmb();
Pavel Begunkove23de152020-12-17 00:24:37 +00002274 return __io_cqring_events(ctx);
Jens Axboea3a0e432019-08-20 11:03:11 -06002275}
2276
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03002277static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx)
2278{
2279 struct io_rings *rings = ctx->rings;
2280
2281 /* make sure SQ entry isn't read before tail */
2282 return smp_load_acquire(&rings->sq.tail) - ctx->cached_sq_head;
2283}
2284
Pavel Begunkov8ff069b2020-07-16 23:28:04 +03002285static unsigned int io_put_kbuf(struct io_kiocb *req, struct io_buffer *kbuf)
Jens Axboee94f1412019-12-19 12:06:02 -07002286{
Pavel Begunkov8ff069b2020-07-16 23:28:04 +03002287 unsigned int cflags;
Jens Axboee94f1412019-12-19 12:06:02 -07002288
Jens Axboebcda7ba2020-02-23 16:42:51 -07002289 cflags = kbuf->bid << IORING_CQE_BUFFER_SHIFT;
2290 cflags |= IORING_CQE_F_BUFFER;
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03002291 req->flags &= ~REQ_F_BUFFER_SELECTED;
Jens Axboebcda7ba2020-02-23 16:42:51 -07002292 kfree(kbuf);
2293 return cflags;
2294}
2295
Pavel Begunkov8ff069b2020-07-16 23:28:04 +03002296static inline unsigned int io_put_rw_kbuf(struct io_kiocb *req)
2297{
2298 struct io_buffer *kbuf;
2299
Pavel Begunkovae421d92021-08-17 20:28:08 +01002300 if (likely(!(req->flags & REQ_F_BUFFER_SELECTED)))
2301 return 0;
Pavel Begunkov8ff069b2020-07-16 23:28:04 +03002302 kbuf = (struct io_buffer *) (unsigned long) req->rw.addr;
2303 return io_put_kbuf(req, kbuf);
2304}
2305
Jens Axboe4c6e2772020-07-01 11:29:10 -06002306static inline bool io_run_task_work(void)
2307{
Nadav Amitef98eb02021-08-07 17:13:41 -07002308 if (test_thread_flag(TIF_NOTIFY_SIGNAL) || current->task_works) {
Jens Axboe4c6e2772020-07-01 11:29:10 -06002309 __set_current_state(TASK_RUNNING);
Nadav Amitef98eb02021-08-07 17:13:41 -07002310 tracehook_notify_signal();
Jens Axboe4c6e2772020-07-01 11:29:10 -06002311 return true;
2312 }
2313
2314 return false;
2315}
2316
Jens Axboedef596e2019-01-09 08:59:42 -07002317/*
2318 * Find and free completed poll iocbs
2319 */
2320static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events,
Pavel Begunkova8576af2021-08-15 10:40:21 +01002321 struct list_head *done)
Jens Axboedef596e2019-01-09 08:59:42 -07002322{
Jens Axboe8237e042019-12-28 10:48:22 -07002323 struct req_batch rb;
Jens Axboedef596e2019-01-09 08:59:42 -07002324 struct io_kiocb *req;
Xiaoguang Wangbbde0172020-06-16 02:06:38 +08002325
2326 /* order with ->result store in io_complete_rw_iopoll() */
2327 smp_rmb();
Jens Axboedef596e2019-01-09 08:59:42 -07002328
Pavel Begunkov5af1d132020-07-18 11:32:52 +03002329 io_init_req_batch(&rb);
Jens Axboedef596e2019-01-09 08:59:42 -07002330 while (!list_empty(done)) {
Pavel Begunkovd21ffe72020-07-13 23:37:10 +03002331 req = list_first_entry(done, struct io_kiocb, inflight_entry);
Pavel Begunkovd21ffe72020-07-13 23:37:10 +03002332 list_del(&req->inflight_entry);
Jens Axboedef596e2019-01-09 08:59:42 -07002333
Pavel Begunkova8576af2021-08-15 10:40:21 +01002334 if (READ_ONCE(req->result) == -EAGAIN &&
Pavel Begunkov8c130822021-03-22 01:58:32 +00002335 !(req->flags & REQ_F_DONT_REISSUE)) {
Pavel Begunkovf1613402021-02-11 18:28:21 +00002336 req->iopoll_completed = 0;
Jens Axboe773af692021-07-27 10:25:55 -06002337 io_req_task_queue_reissue(req);
Pavel Begunkov8c130822021-03-22 01:58:32 +00002338 continue;
Pavel Begunkovf1613402021-02-11 18:28:21 +00002339 }
2340
Pavel Begunkovae421d92021-08-17 20:28:08 +01002341 __io_cqring_fill_event(ctx, req->user_data, req->result,
2342 io_put_rw_kbuf(req));
Jens Axboedef596e2019-01-09 08:59:42 -07002343 (*nr_events)++;
2344
Jens Axboede9b4cc2021-02-24 13:28:27 -07002345 if (req_ref_put_and_test(req))
Pavel Begunkov6ff119a2021-02-10 00:03:18 +00002346 io_req_free_batch(&rb, req, &ctx->submit_state);
Jens Axboedef596e2019-01-09 08:59:42 -07002347 }
Jens Axboedef596e2019-01-09 08:59:42 -07002348
Jens Axboe09bb8392019-03-13 12:39:28 -06002349 io_commit_cqring(ctx);
Pavel Begunkov80c18e42021-01-07 03:15:41 +00002350 io_cqring_ev_posted_iopoll(ctx);
Pavel Begunkov2d6500d2020-06-28 12:52:33 +03002351 io_req_free_batch_finish(ctx, &rb);
Bijan Mottahedeh581f9812020-04-03 13:51:33 -07002352}
2353
Jens Axboedef596e2019-01-09 08:59:42 -07002354static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events,
Pavel Begunkova8576af2021-08-15 10:40:21 +01002355 long min)
Jens Axboedef596e2019-01-09 08:59:42 -07002356{
2357 struct io_kiocb *req, *tmp;
2358 LIST_HEAD(done);
2359 bool spin;
Jens Axboedef596e2019-01-09 08:59:42 -07002360
2361 /*
2362 * Only spin for completions if we don't have multiple devices hanging
2363 * off our complete list, and we're under the requested amount.
2364 */
Hao Xu915b3dd2021-06-28 05:37:30 +08002365 spin = !ctx->poll_multi_queue && *nr_events < min;
Jens Axboedef596e2019-01-09 08:59:42 -07002366
Pavel Begunkovd21ffe72020-07-13 23:37:10 +03002367 list_for_each_entry_safe(req, tmp, &ctx->iopoll_list, inflight_entry) {
Jens Axboe9adbd452019-12-20 08:45:55 -07002368 struct kiocb *kiocb = &req->rw.kiocb;
Pavel Begunkova2416e12021-08-09 13:04:09 +01002369 int ret;
Jens Axboedef596e2019-01-09 08:59:42 -07002370
2371 /*
Bijan Mottahedeh581f9812020-04-03 13:51:33 -07002372 * Move completed and retryable entries to our local lists.
2373 * If we find a request that requires polling, break out
2374 * and complete those lists first, if we have entries there.
Jens Axboedef596e2019-01-09 08:59:42 -07002375 */
Xiaoguang Wang65a65432020-06-11 23:39:36 +08002376 if (READ_ONCE(req->iopoll_completed)) {
Pavel Begunkovd21ffe72020-07-13 23:37:10 +03002377 list_move_tail(&req->inflight_entry, &done);
Jens Axboedef596e2019-01-09 08:59:42 -07002378 continue;
2379 }
2380 if (!list_empty(&done))
2381 break;
2382
2383 ret = kiocb->ki_filp->f_op->iopoll(kiocb, spin);
Pavel Begunkova2416e12021-08-09 13:04:09 +01002384 if (unlikely(ret < 0))
2385 return ret;
2386 else if (ret)
2387 spin = false;
Jens Axboedef596e2019-01-09 08:59:42 -07002388
Pavel Begunkov3aadc232020-07-06 17:59:29 +03002389 /* iopoll may have completed current req */
2390 if (READ_ONCE(req->iopoll_completed))
Pavel Begunkovd21ffe72020-07-13 23:37:10 +03002391 list_move_tail(&req->inflight_entry, &done);
Jens Axboedef596e2019-01-09 08:59:42 -07002392 }
2393
2394 if (!list_empty(&done))
Pavel Begunkova8576af2021-08-15 10:40:21 +01002395 io_iopoll_complete(ctx, nr_events, &done);
Jens Axboedef596e2019-01-09 08:59:42 -07002396
Pavel Begunkova2416e12021-08-09 13:04:09 +01002397 return 0;
Jens Axboedef596e2019-01-09 08:59:42 -07002398}
2399
2400/*
Jens Axboedef596e2019-01-09 08:59:42 -07002401 * We can't just wait for polled events to come to us, we have to actively
2402 * find and complete them.
2403 */
Pavel Begunkovb2edc0a2020-07-07 16:36:22 +03002404static void io_iopoll_try_reap_events(struct io_ring_ctx *ctx)
Jens Axboedef596e2019-01-09 08:59:42 -07002405{
2406 if (!(ctx->flags & IORING_SETUP_IOPOLL))
2407 return;
2408
2409 mutex_lock(&ctx->uring_lock);
Pavel Begunkov540e32a2020-07-13 23:37:09 +03002410 while (!list_empty(&ctx->iopoll_list)) {
Jens Axboedef596e2019-01-09 08:59:42 -07002411 unsigned int nr_events = 0;
2412
Pavel Begunkova8576af2021-08-15 10:40:21 +01002413 io_do_iopoll(ctx, &nr_events, 0);
Jens Axboe08f54392019-08-21 22:19:11 -06002414
Pavel Begunkovb2edc0a2020-07-07 16:36:22 +03002415 /* let it sleep and repeat later if can't complete a request */
2416 if (nr_events == 0)
2417 break;
Jens Axboe08f54392019-08-21 22:19:11 -06002418 /*
2419 * Ensure we allow local-to-the-cpu processing to take place,
2420 * in this case we need to ensure that we reap all events.
Pavel Begunkov3fcee5a2020-07-06 17:59:31 +03002421 * Also let task_work, etc. to progress by releasing the mutex
Jens Axboe08f54392019-08-21 22:19:11 -06002422 */
Pavel Begunkov3fcee5a2020-07-06 17:59:31 +03002423 if (need_resched()) {
2424 mutex_unlock(&ctx->uring_lock);
2425 cond_resched();
2426 mutex_lock(&ctx->uring_lock);
2427 }
Jens Axboedef596e2019-01-09 08:59:42 -07002428 }
2429 mutex_unlock(&ctx->uring_lock);
2430}
2431
Pavel Begunkov7668b922020-07-07 16:36:21 +03002432static int io_iopoll_check(struct io_ring_ctx *ctx, long min)
Jens Axboedef596e2019-01-09 08:59:42 -07002433{
Pavel Begunkov7668b922020-07-07 16:36:21 +03002434 unsigned int nr_events = 0;
Pavel Begunkove9979b32021-04-13 02:58:45 +01002435 int ret = 0;
Jens Axboedef596e2019-01-09 08:59:42 -07002436
Xiaoguang Wangc7849be2020-02-22 14:46:05 +08002437 /*
2438 * We disallow the app entering submit/complete with polling, but we
2439 * still need to lock the ring to prevent racing with polled issue
2440 * that got punted to a workqueue.
2441 */
2442 mutex_lock(&ctx->uring_lock);
Pavel Begunkovf39c8a52021-04-13 02:58:46 +01002443 /*
2444 * Don't enter poll loop if we already have events pending.
2445 * If we do, we can potentially be spinning for commands that
2446 * already triggered a CQE (eg in error).
2447 */
Pavel Begunkov5ed7a372021-06-14 23:37:27 +01002448 if (test_bit(0, &ctx->check_cq_overflow))
Pavel Begunkovf39c8a52021-04-13 02:58:46 +01002449 __io_cqring_overflow_flush(ctx, false);
2450 if (io_cqring_events(ctx))
2451 goto out;
Jens Axboedef596e2019-01-09 08:59:42 -07002452 do {
Jens Axboe500f9fb2019-08-19 12:15:59 -06002453 /*
2454 * If a submit got punted to a workqueue, we can have the
2455 * application entering polling for a command before it gets
2456 * issued. That app will hold the uring_lock for the duration
2457 * of the poll right here, so we need to take a breather every
2458 * now and then to ensure that the issue has a chance to add
2459 * the poll to the issued list. Otherwise we can spin here
2460 * forever, while the workqueue is stuck trying to acquire the
2461 * very same mutex.
2462 */
Pavel Begunkove9979b32021-04-13 02:58:45 +01002463 if (list_empty(&ctx->iopoll_list)) {
Pavel Begunkov8f487ef2021-07-08 13:37:06 +01002464 u32 tail = ctx->cached_cq_tail;
2465
Jens Axboe500f9fb2019-08-19 12:15:59 -06002466 mutex_unlock(&ctx->uring_lock);
Jens Axboe4c6e2772020-07-01 11:29:10 -06002467 io_run_task_work();
Jens Axboe500f9fb2019-08-19 12:15:59 -06002468 mutex_lock(&ctx->uring_lock);
Pavel Begunkove9979b32021-04-13 02:58:45 +01002469
Pavel Begunkov8f487ef2021-07-08 13:37:06 +01002470 /* some requests don't go through iopoll_list */
2471 if (tail != ctx->cached_cq_tail ||
2472 list_empty(&ctx->iopoll_list))
Pavel Begunkove9979b32021-04-13 02:58:45 +01002473 break;
Jens Axboe500f9fb2019-08-19 12:15:59 -06002474 }
Pavel Begunkova8576af2021-08-15 10:40:21 +01002475 ret = io_do_iopoll(ctx, &nr_events, min);
Pavel Begunkovf39c8a52021-04-13 02:58:46 +01002476 } while (!ret && nr_events < min && !need_resched());
2477out:
Jens Axboe500f9fb2019-08-19 12:15:59 -06002478 mutex_unlock(&ctx->uring_lock);
Jens Axboedef596e2019-01-09 08:59:42 -07002479 return ret;
2480}
2481
Jens Axboe491381ce2019-10-17 09:20:46 -06002482static void kiocb_end_write(struct io_kiocb *req)
Jens Axboe2b188cc2019-01-07 10:46:33 -07002483{
Jens Axboe491381ce2019-10-17 09:20:46 -06002484 /*
2485 * Tell lockdep we inherited freeze protection from submission
2486 * thread.
2487 */
2488 if (req->flags & REQ_F_ISREG) {
Pavel Begunkov1c986792021-03-22 01:58:31 +00002489 struct super_block *sb = file_inode(req->file)->i_sb;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002490
Pavel Begunkov1c986792021-03-22 01:58:31 +00002491 __sb_writers_acquired(sb, SB_FREEZE_WRITE);
2492 sb_end_write(sb);
Jens Axboe2b188cc2019-01-07 10:46:33 -07002493 }
2494}
2495
Jens Axboeb63534c2020-06-04 11:28:00 -06002496#ifdef CONFIG_BLOCK
Pavel Begunkovdc2a6e92021-01-19 13:32:35 +00002497static bool io_resubmit_prep(struct io_kiocb *req)
Jens Axboeb63534c2020-06-04 11:28:00 -06002498{
Pavel Begunkovab454432021-03-22 01:58:33 +00002499 struct io_async_rw *rw = req->async_data;
Jens Axboeb63534c2020-06-04 11:28:00 -06002500
Pavel Begunkovab454432021-03-22 01:58:33 +00002501 if (!rw)
2502 return !io_req_prep_async(req);
2503 /* may have left rw->iter inconsistent on -EIOCBQUEUED */
2504 iov_iter_revert(&rw->iter, req->result - iov_iter_count(&rw->iter));
2505 return true;
Jens Axboeb63534c2020-06-04 11:28:00 -06002506}
Jens Axboeb63534c2020-06-04 11:28:00 -06002507
Jens Axboe3e6a0d32021-03-01 13:56:00 -07002508static bool io_rw_should_reissue(struct io_kiocb *req)
Jens Axboeb63534c2020-06-04 11:28:00 -06002509{
Jens Axboe355afae2020-09-02 09:30:31 -06002510 umode_t mode = file_inode(req->file)->i_mode;
Jens Axboe3e6a0d32021-03-01 13:56:00 -07002511 struct io_ring_ctx *ctx = req->ctx;
Jens Axboeb63534c2020-06-04 11:28:00 -06002512
Jens Axboe355afae2020-09-02 09:30:31 -06002513 if (!S_ISBLK(mode) && !S_ISREG(mode))
2514 return false;
Jens Axboe3e6a0d32021-03-01 13:56:00 -07002515 if ((req->flags & REQ_F_NOWAIT) || (io_wq_current_is_worker() &&
2516 !(ctx->flags & IORING_SETUP_IOPOLL)))
Jens Axboeb63534c2020-06-04 11:28:00 -06002517 return false;
Jens Axboe7c977a52021-02-23 19:17:35 -07002518 /*
2519 * If ref is dying, we might be running poll reap from the exit work.
2520 * Don't attempt to reissue from that path, just let it fail with
2521 * -EAGAIN.
2522 */
Jens Axboe3e6a0d32021-03-01 13:56:00 -07002523 if (percpu_ref_is_dying(&ctx->refs))
2524 return false;
Jens Axboeef046882021-07-27 10:50:31 -06002525 /*
2526 * Play it safe and assume not safe to re-import and reissue if we're
2527 * not in the original thread group (or in task context).
2528 */
2529 if (!same_thread_group(req->task, current) || !in_task())
2530 return false;
Jens Axboe3e6a0d32021-03-01 13:56:00 -07002531 return true;
2532}
Jens Axboee82ad482021-04-02 19:45:34 -06002533#else
Jens Axboea1ff1e32021-04-12 06:40:02 -06002534static bool io_resubmit_prep(struct io_kiocb *req)
2535{
2536 return false;
2537}
Jens Axboee82ad482021-04-02 19:45:34 -06002538static bool io_rw_should_reissue(struct io_kiocb *req)
2539{
2540 return false;
2541}
Jens Axboe3e6a0d32021-03-01 13:56:00 -07002542#endif
2543
Jens Axboe8ef12ef2021-08-10 15:15:25 -06002544static bool __io_complete_rw_common(struct io_kiocb *req, long res)
Jens Axboea1d7c392020-06-22 11:09:46 -06002545{
Pavel Begunkovb65c1282021-03-22 01:45:59 +00002546 if (req->rw.kiocb.ki_flags & IOCB_WRITE)
2547 kiocb_end_write(req);
Pavel Begunkov9532b992021-03-22 01:58:34 +00002548 if (res != req->result) {
2549 if ((res == -EAGAIN || res == -EOPNOTSUPP) &&
2550 io_rw_should_reissue(req)) {
2551 req->flags |= REQ_F_REISSUE;
Jens Axboe8ef12ef2021-08-10 15:15:25 -06002552 return true;
Pavel Begunkov9532b992021-03-22 01:58:34 +00002553 }
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01002554 req_set_fail(req);
Jens Axboe8ef12ef2021-08-10 15:15:25 -06002555 req->result = res;
Pavel Begunkov9532b992021-03-22 01:58:34 +00002556 }
Jens Axboe8ef12ef2021-08-10 15:15:25 -06002557 return false;
2558}
2559
2560static void io_req_task_complete(struct io_kiocb *req)
2561{
Pavel Begunkovae421d92021-08-17 20:28:08 +01002562 __io_req_complete(req, 0, req->result, io_put_rw_kbuf(req));
Jens Axboe8ef12ef2021-08-10 15:15:25 -06002563}
2564
2565static void __io_complete_rw(struct io_kiocb *req, long res, long res2,
2566 unsigned int issue_flags)
2567{
2568 if (__io_complete_rw_common(req, res))
2569 return;
2570 io_req_task_complete(req);
Jens Axboeba816ad2019-09-28 11:36:45 -06002571}
2572
2573static void io_complete_rw(struct kiocb *kiocb, long res, long res2)
2574{
Jens Axboe9adbd452019-12-20 08:45:55 -07002575 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
Jens Axboeba816ad2019-09-28 11:36:45 -06002576
Jens Axboe8ef12ef2021-08-10 15:15:25 -06002577 if (__io_complete_rw_common(req, res))
2578 return;
2579 req->result = res;
2580 req->io_task_work.func = io_req_task_complete;
2581 io_req_task_work_add(req);
Jens Axboe2b188cc2019-01-07 10:46:33 -07002582}
2583
Jens Axboedef596e2019-01-09 08:59:42 -07002584static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2)
2585{
Jens Axboe9adbd452019-12-20 08:45:55 -07002586 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
Jens Axboedef596e2019-01-09 08:59:42 -07002587
Jens Axboe491381ce2019-10-17 09:20:46 -06002588 if (kiocb->ki_flags & IOCB_WRITE)
2589 kiocb_end_write(req);
Pavel Begunkov9532b992021-03-22 01:58:34 +00002590 if (unlikely(res != req->result)) {
Jens Axboea1ff1e32021-04-12 06:40:02 -06002591 if (!(res == -EAGAIN && io_rw_should_reissue(req) &&
2592 io_resubmit_prep(req))) {
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01002593 req_set_fail(req);
Pavel Begunkov9532b992021-03-22 01:58:34 +00002594 req->flags |= REQ_F_DONT_REISSUE;
2595 }
Pavel Begunkov8c130822021-03-22 01:58:32 +00002596 }
Xiaoguang Wangbbde0172020-06-16 02:06:38 +08002597
2598 WRITE_ONCE(req->result, res);
Jens Axboeb9b0e0d2021-02-23 08:18:36 -07002599 /* order with io_iopoll_complete() checking ->result */
Pavel Begunkovcd664b02020-06-25 12:37:10 +03002600 smp_wmb();
2601 WRITE_ONCE(req->iopoll_completed, 1);
Jens Axboedef596e2019-01-09 08:59:42 -07002602}
2603
2604/*
2605 * After the iocb has been issued, it's safe to be found on the poll list.
2606 * Adding the kiocb to the list AFTER submission ensures that we don't
Pavel Begunkovf39c8a52021-04-13 02:58:46 +01002607 * find it from a io_do_iopoll() thread before the issuer is done
Jens Axboedef596e2019-01-09 08:59:42 -07002608 * accessing the kiocb cookie.
2609 */
Pavel Begunkovcb3d8972021-06-14 02:36:14 +01002610static void io_iopoll_req_issued(struct io_kiocb *req)
Jens Axboedef596e2019-01-09 08:59:42 -07002611{
2612 struct io_ring_ctx *ctx = req->ctx;
Pavel Begunkovcb3d8972021-06-14 02:36:14 +01002613 const bool in_async = io_wq_current_is_worker();
2614
2615 /* workqueue context doesn't hold uring_lock, grab it now */
2616 if (unlikely(in_async))
2617 mutex_lock(&ctx->uring_lock);
Jens Axboedef596e2019-01-09 08:59:42 -07002618
2619 /*
2620 * Track whether we have multiple files in our lists. This will impact
2621 * how we do polling eventually, not spinning if we're on potentially
2622 * different devices.
2623 */
Pavel Begunkov540e32a2020-07-13 23:37:09 +03002624 if (list_empty(&ctx->iopoll_list)) {
Hao Xu915b3dd2021-06-28 05:37:30 +08002625 ctx->poll_multi_queue = false;
2626 } else if (!ctx->poll_multi_queue) {
Jens Axboedef596e2019-01-09 08:59:42 -07002627 struct io_kiocb *list_req;
Hao Xu915b3dd2021-06-28 05:37:30 +08002628 unsigned int queue_num0, queue_num1;
Jens Axboedef596e2019-01-09 08:59:42 -07002629
Pavel Begunkov540e32a2020-07-13 23:37:09 +03002630 list_req = list_first_entry(&ctx->iopoll_list, struct io_kiocb,
Pavel Begunkovd21ffe72020-07-13 23:37:10 +03002631 inflight_entry);
Hao Xu915b3dd2021-06-28 05:37:30 +08002632
2633 if (list_req->file != req->file) {
2634 ctx->poll_multi_queue = true;
2635 } else {
2636 queue_num0 = blk_qc_t_to_queue_num(list_req->rw.kiocb.ki_cookie);
2637 queue_num1 = blk_qc_t_to_queue_num(req->rw.kiocb.ki_cookie);
2638 if (queue_num0 != queue_num1)
2639 ctx->poll_multi_queue = true;
2640 }
Jens Axboedef596e2019-01-09 08:59:42 -07002641 }
2642
2643 /*
2644 * For fast devices, IO may have already completed. If it has, add
2645 * it to the front so we find it first.
2646 */
Xiaoguang Wang65a65432020-06-11 23:39:36 +08002647 if (READ_ONCE(req->iopoll_completed))
Pavel Begunkovd21ffe72020-07-13 23:37:10 +03002648 list_add(&req->inflight_entry, &ctx->iopoll_list);
Jens Axboedef596e2019-01-09 08:59:42 -07002649 else
Pavel Begunkovd21ffe72020-07-13 23:37:10 +03002650 list_add_tail(&req->inflight_entry, &ctx->iopoll_list);
Xiaoguang Wangbdcd3ea2020-02-25 22:12:08 +08002651
Pavel Begunkovcb3d8972021-06-14 02:36:14 +01002652 if (unlikely(in_async)) {
2653 /*
2654 * If IORING_SETUP_SQPOLL is enabled, sqes are either handle
2655 * in sq thread task context or in io worker task context. If
2656 * current task context is sq thread, we don't need to check
2657 * whether should wake up sq thread.
2658 */
2659 if ((ctx->flags & IORING_SETUP_SQPOLL) &&
2660 wq_has_sleeper(&ctx->sq_data->wait))
2661 wake_up(&ctx->sq_data->wait);
2662
2663 mutex_unlock(&ctx->uring_lock);
2664 }
Jens Axboedef596e2019-01-09 08:59:42 -07002665}
2666
Jens Axboe4503b762020-06-01 10:00:27 -06002667static bool io_bdev_nowait(struct block_device *bdev)
2668{
Jeffle Xu9ba0d0c2020-10-19 16:59:42 +08002669 return !bdev || blk_queue_nowait(bdev_get_queue(bdev));
Jens Axboe4503b762020-06-01 10:00:27 -06002670}
2671
Jens Axboe2b188cc2019-01-07 10:46:33 -07002672/*
2673 * If we tracked the file through the SCM inflight mechanism, we could support
2674 * any file. For now, just ensure that anything potentially problematic is done
2675 * inline.
2676 */
Pavel Begunkovb191e2d2021-08-09 13:04:03 +01002677static bool __io_file_supports_nowait(struct file *file, int rw)
Jens Axboe2b188cc2019-01-07 10:46:33 -07002678{
2679 umode_t mode = file_inode(file)->i_mode;
2680
Jens Axboe4503b762020-06-01 10:00:27 -06002681 if (S_ISBLK(mode)) {
Christoph Hellwig4e7b5672020-11-23 13:38:40 +01002682 if (IS_ENABLED(CONFIG_BLOCK) &&
2683 io_bdev_nowait(I_BDEV(file->f_mapping->host)))
Jens Axboe4503b762020-06-01 10:00:27 -06002684 return true;
2685 return false;
2686 }
Pavel Begunkov976517f2021-06-09 12:07:25 +01002687 if (S_ISSOCK(mode))
Jens Axboe2b188cc2019-01-07 10:46:33 -07002688 return true;
Jens Axboe4503b762020-06-01 10:00:27 -06002689 if (S_ISREG(mode)) {
Christoph Hellwig4e7b5672020-11-23 13:38:40 +01002690 if (IS_ENABLED(CONFIG_BLOCK) &&
2691 io_bdev_nowait(file->f_inode->i_sb->s_bdev) &&
Jens Axboe4503b762020-06-01 10:00:27 -06002692 file->f_op != &io_uring_fops)
2693 return true;
2694 return false;
2695 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07002696
Jens Axboec5b85622020-06-09 19:23:05 -06002697 /* any ->read/write should understand O_NONBLOCK */
2698 if (file->f_flags & O_NONBLOCK)
2699 return true;
2700
Jens Axboeaf197f52020-04-28 13:15:06 -06002701 if (!(file->f_mode & FMODE_NOWAIT))
2702 return false;
2703
2704 if (rw == READ)
2705 return file->f_op->read_iter != NULL;
2706
2707 return file->f_op->write_iter != NULL;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002708}
2709
Pavel Begunkovb191e2d2021-08-09 13:04:03 +01002710static bool io_file_supports_nowait(struct io_kiocb *req, int rw)
Jens Axboe7b29f922021-03-12 08:30:14 -07002711{
Pavel Begunkovb191e2d2021-08-09 13:04:03 +01002712 if (rw == READ && (req->flags & REQ_F_NOWAIT_READ))
Jens Axboe7b29f922021-03-12 08:30:14 -07002713 return true;
Pavel Begunkovb191e2d2021-08-09 13:04:03 +01002714 else if (rw == WRITE && (req->flags & REQ_F_NOWAIT_WRITE))
Jens Axboe7b29f922021-03-12 08:30:14 -07002715 return true;
2716
Pavel Begunkovb191e2d2021-08-09 13:04:03 +01002717 return __io_file_supports_nowait(req->file, rw);
Jens Axboe7b29f922021-03-12 08:30:14 -07002718}
2719
Pavel Begunkova88fc402020-09-30 22:57:53 +03002720static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboe2b188cc2019-01-07 10:46:33 -07002721{
Jens Axboedef596e2019-01-09 08:59:42 -07002722 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe9adbd452019-12-20 08:45:55 -07002723 struct kiocb *kiocb = &req->rw.kiocb;
Pavel Begunkov75c668c2021-02-04 13:52:05 +00002724 struct file *file = req->file;
Jens Axboe09bb8392019-03-13 12:39:28 -06002725 unsigned ioprio;
2726 int ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002727
Pavel Begunkovc97d8a02021-08-09 13:04:04 +01002728 if (!io_req_ffs_set(req) && S_ISREG(file_inode(file)->i_mode))
Jens Axboe491381ce2019-10-17 09:20:46 -06002729 req->flags |= REQ_F_ISREG;
2730
Jens Axboe2b188cc2019-01-07 10:46:33 -07002731 kiocb->ki_pos = READ_ONCE(sqe->off);
Pavel Begunkov75c668c2021-02-04 13:52:05 +00002732 if (kiocb->ki_pos == -1 && !(file->f_mode & FMODE_STREAM)) {
Jens Axboeba042912019-12-25 16:33:42 -07002733 req->flags |= REQ_F_CUR_POS;
Pavel Begunkov75c668c2021-02-04 13:52:05 +00002734 kiocb->ki_pos = file->f_pos;
Jens Axboeba042912019-12-25 16:33:42 -07002735 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07002736 kiocb->ki_hint = ki_hint_validate(file_write_hint(kiocb->ki_filp));
Pavel Begunkov3e577dc2020-02-01 03:58:42 +03002737 kiocb->ki_flags = iocb_flags(kiocb->ki_filp);
2738 ret = kiocb_set_rw_flags(kiocb, READ_ONCE(sqe->rw_flags));
2739 if (unlikely(ret))
2740 return ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002741
Pavel Begunkov75c668c2021-02-04 13:52:05 +00002742 /* don't allow async punt for O_NONBLOCK or RWF_NOWAIT */
2743 if ((kiocb->ki_flags & IOCB_NOWAIT) || (file->f_flags & O_NONBLOCK))
2744 req->flags |= REQ_F_NOWAIT;
2745
Jens Axboe2b188cc2019-01-07 10:46:33 -07002746 ioprio = READ_ONCE(sqe->ioprio);
2747 if (ioprio) {
2748 ret = ioprio_check_cap(ioprio);
2749 if (ret)
Jens Axboe09bb8392019-03-13 12:39:28 -06002750 return ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002751
2752 kiocb->ki_ioprio = ioprio;
2753 } else
2754 kiocb->ki_ioprio = get_current_ioprio();
2755
Jens Axboedef596e2019-01-09 08:59:42 -07002756 if (ctx->flags & IORING_SETUP_IOPOLL) {
Jens Axboedef596e2019-01-09 08:59:42 -07002757 if (!(kiocb->ki_flags & IOCB_DIRECT) ||
2758 !kiocb->ki_filp->f_op->iopoll)
Jens Axboe09bb8392019-03-13 12:39:28 -06002759 return -EOPNOTSUPP;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002760
Jens Axboedef596e2019-01-09 08:59:42 -07002761 kiocb->ki_flags |= IOCB_HIPRI;
2762 kiocb->ki_complete = io_complete_rw_iopoll;
Xiaoguang Wang65a65432020-06-11 23:39:36 +08002763 req->iopoll_completed = 0;
Jens Axboedef596e2019-01-09 08:59:42 -07002764 } else {
Jens Axboe09bb8392019-03-13 12:39:28 -06002765 if (kiocb->ki_flags & IOCB_HIPRI)
2766 return -EINVAL;
Jens Axboedef596e2019-01-09 08:59:42 -07002767 kiocb->ki_complete = io_complete_rw;
2768 }
Jens Axboe9adbd452019-12-20 08:45:55 -07002769
Pavel Begunkoveae071c2021-04-25 14:32:24 +01002770 if (req->opcode == IORING_OP_READ_FIXED ||
2771 req->opcode == IORING_OP_WRITE_FIXED) {
2772 req->imu = NULL;
2773 io_req_set_rsrc_node(req);
2774 }
2775
Jens Axboe3529d8c2019-12-19 18:24:38 -07002776 req->rw.addr = READ_ONCE(sqe->addr);
2777 req->rw.len = READ_ONCE(sqe->len);
Bijan Mottahedeh4f4eeba2020-05-19 14:52:49 -07002778 req->buf_index = READ_ONCE(sqe->buf_index);
Jens Axboe2b188cc2019-01-07 10:46:33 -07002779 return 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002780}
2781
2782static inline void io_rw_done(struct kiocb *kiocb, ssize_t ret)
2783{
2784 switch (ret) {
2785 case -EIOCBQUEUED:
2786 break;
2787 case -ERESTARTSYS:
2788 case -ERESTARTNOINTR:
2789 case -ERESTARTNOHAND:
2790 case -ERESTART_RESTARTBLOCK:
2791 /*
2792 * We can't just restart the syscall, since previously
2793 * submitted sqes may already be in progress. Just fail this
2794 * IO with EINTR.
2795 */
2796 ret = -EINTR;
Gustavo A. R. Silvadf561f662020-08-23 17:36:59 -05002797 fallthrough;
Jens Axboe2b188cc2019-01-07 10:46:33 -07002798 default:
2799 kiocb->ki_complete(kiocb, ret, 0);
2800 }
2801}
2802
Jens Axboea1d7c392020-06-22 11:09:46 -06002803static void kiocb_done(struct kiocb *kiocb, ssize_t ret,
Pavel Begunkov889fca72021-02-10 00:03:09 +00002804 unsigned int issue_flags)
Jens Axboeba816ad2019-09-28 11:36:45 -06002805{
Jens Axboeba042912019-12-25 16:33:42 -07002806 struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
Jens Axboee8c2bc12020-08-15 18:44:09 -07002807 struct io_async_rw *io = req->async_data;
Pavel Begunkov97284632021-04-08 19:28:03 +01002808 bool check_reissue = kiocb->ki_complete == io_complete_rw;
Jens Axboeba042912019-12-25 16:33:42 -07002809
Jens Axboe227c0c92020-08-13 11:51:40 -06002810 /* add previously done IO, if any */
Jens Axboee8c2bc12020-08-15 18:44:09 -07002811 if (io && io->bytes_done > 0) {
Jens Axboe227c0c92020-08-13 11:51:40 -06002812 if (ret < 0)
Jens Axboee8c2bc12020-08-15 18:44:09 -07002813 ret = io->bytes_done;
Jens Axboe227c0c92020-08-13 11:51:40 -06002814 else
Jens Axboee8c2bc12020-08-15 18:44:09 -07002815 ret += io->bytes_done;
Jens Axboe227c0c92020-08-13 11:51:40 -06002816 }
2817
Jens Axboeba042912019-12-25 16:33:42 -07002818 if (req->flags & REQ_F_CUR_POS)
2819 req->file->f_pos = kiocb->ki_pos;
Hao Xue149bd742021-06-28 05:48:05 +08002820 if (ret >= 0 && check_reissue)
Pavel Begunkov889fca72021-02-10 00:03:09 +00002821 __io_complete_rw(req, ret, 0, issue_flags);
Jens Axboeba816ad2019-09-28 11:36:45 -06002822 else
2823 io_rw_done(kiocb, ret);
Pavel Begunkov97284632021-04-08 19:28:03 +01002824
Pavel Begunkovfe7e3252021-06-24 15:09:57 +01002825 if (check_reissue && (req->flags & REQ_F_REISSUE)) {
Pavel Begunkov97284632021-04-08 19:28:03 +01002826 req->flags &= ~REQ_F_REISSUE;
Jens Axboea7be7c22021-04-15 11:31:14 -06002827 if (io_resubmit_prep(req)) {
Jens Axboe773af692021-07-27 10:25:55 -06002828 io_req_task_queue_reissue(req);
Pavel Begunkov8c130822021-03-22 01:58:32 +00002829 } else {
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01002830 req_set_fail(req);
Pavel Begunkovae421d92021-08-17 20:28:08 +01002831 __io_req_complete(req, issue_flags, ret,
2832 io_put_rw_kbuf(req));
Pavel Begunkov97284632021-04-08 19:28:03 +01002833 }
2834 }
Jens Axboeba816ad2019-09-28 11:36:45 -06002835}
2836
Pavel Begunkoveae071c2021-04-25 14:32:24 +01002837static int __io_import_fixed(struct io_kiocb *req, int rw, struct iov_iter *iter,
2838 struct io_mapped_ubuf *imu)
Jens Axboeedafcce2019-01-09 09:16:05 -07002839{
Jens Axboe9adbd452019-12-20 08:45:55 -07002840 size_t len = req->rw.len;
Pavel Begunkov75769e32021-04-01 15:43:54 +01002841 u64 buf_end, buf_addr = req->rw.addr;
Jens Axboeedafcce2019-01-09 09:16:05 -07002842 size_t offset;
Jens Axboeedafcce2019-01-09 09:16:05 -07002843
Pavel Begunkov75769e32021-04-01 15:43:54 +01002844 if (unlikely(check_add_overflow(buf_addr, (u64)len, &buf_end)))
Jens Axboeedafcce2019-01-09 09:16:05 -07002845 return -EFAULT;
2846 /* not inside the mapped region */
Pavel Begunkov4751f532021-04-01 15:43:55 +01002847 if (unlikely(buf_addr < imu->ubuf || buf_end > imu->ubuf_end))
Jens Axboeedafcce2019-01-09 09:16:05 -07002848 return -EFAULT;
2849
2850 /*
2851 * May not be a start of buffer, set size appropriately
2852 * and advance us to the beginning.
2853 */
2854 offset = buf_addr - imu->ubuf;
2855 iov_iter_bvec(iter, rw, imu->bvec, imu->nr_bvecs, offset + len);
Jens Axboebd11b3a2019-07-20 08:37:31 -06002856
2857 if (offset) {
2858 /*
2859 * Don't use iov_iter_advance() here, as it's really slow for
2860 * using the latter parts of a big fixed buffer - it iterates
2861 * over each segment manually. We can cheat a bit here, because
2862 * we know that:
2863 *
2864 * 1) it's a BVEC iter, we set it up
2865 * 2) all bvecs are PAGE_SIZE in size, except potentially the
2866 * first and last bvec
2867 *
2868 * So just find our index, and adjust the iterator afterwards.
2869 * If the offset is within the first bvec (or the whole first
2870 * bvec, just use iov_iter_advance(). This makes it easier
2871 * since we can just skip the first segment, which may not
2872 * be PAGE_SIZE aligned.
2873 */
2874 const struct bio_vec *bvec = imu->bvec;
2875
2876 if (offset <= bvec->bv_len) {
2877 iov_iter_advance(iter, offset);
2878 } else {
2879 unsigned long seg_skip;
2880
2881 /* skip first vec */
2882 offset -= bvec->bv_len;
2883 seg_skip = 1 + (offset >> PAGE_SHIFT);
2884
2885 iter->bvec = bvec + seg_skip;
2886 iter->nr_segs -= seg_skip;
Aleix Roca Nonell99c79f62019-08-15 14:03:22 +02002887 iter->count -= bvec->bv_len + offset;
Jens Axboebd11b3a2019-07-20 08:37:31 -06002888 iter->iov_offset = offset & ~PAGE_MASK;
Jens Axboebd11b3a2019-07-20 08:37:31 -06002889 }
2890 }
2891
Pavel Begunkov847595d2021-02-04 13:52:06 +00002892 return 0;
Jens Axboeedafcce2019-01-09 09:16:05 -07002893}
2894
Pavel Begunkoveae071c2021-04-25 14:32:24 +01002895static int io_import_fixed(struct io_kiocb *req, int rw, struct iov_iter *iter)
2896{
2897 struct io_ring_ctx *ctx = req->ctx;
2898 struct io_mapped_ubuf *imu = req->imu;
2899 u16 index, buf_index = req->buf_index;
2900
2901 if (likely(!imu)) {
2902 if (unlikely(buf_index >= ctx->nr_user_bufs))
2903 return -EFAULT;
2904 index = array_index_nospec(buf_index, ctx->nr_user_bufs);
2905 imu = READ_ONCE(ctx->user_bufs[index]);
2906 req->imu = imu;
2907 }
2908 return __io_import_fixed(req, rw, iter, imu);
2909}
2910
Jens Axboebcda7ba2020-02-23 16:42:51 -07002911static void io_ring_submit_unlock(struct io_ring_ctx *ctx, bool needs_lock)
2912{
2913 if (needs_lock)
2914 mutex_unlock(&ctx->uring_lock);
2915}
2916
2917static void io_ring_submit_lock(struct io_ring_ctx *ctx, bool needs_lock)
2918{
2919 /*
2920 * "Normal" inline submissions always hold the uring_lock, since we
2921 * grab it from the system call. Same is true for the SQPOLL offload.
2922 * The only exception is when we've detached the request and issue it
2923 * from an async worker thread, grab the lock for that case.
2924 */
2925 if (needs_lock)
2926 mutex_lock(&ctx->uring_lock);
2927}
2928
2929static struct io_buffer *io_buffer_select(struct io_kiocb *req, size_t *len,
2930 int bgid, struct io_buffer *kbuf,
2931 bool needs_lock)
2932{
2933 struct io_buffer *head;
2934
2935 if (req->flags & REQ_F_BUFFER_SELECTED)
2936 return kbuf;
2937
2938 io_ring_submit_lock(req->ctx, needs_lock);
2939
2940 lockdep_assert_held(&req->ctx->uring_lock);
2941
Jens Axboe9e15c3a2021-03-13 12:29:43 -07002942 head = xa_load(&req->ctx->io_buffers, bgid);
Jens Axboebcda7ba2020-02-23 16:42:51 -07002943 if (head) {
2944 if (!list_empty(&head->list)) {
2945 kbuf = list_last_entry(&head->list, struct io_buffer,
2946 list);
2947 list_del(&kbuf->list);
2948 } else {
2949 kbuf = head;
Jens Axboe9e15c3a2021-03-13 12:29:43 -07002950 xa_erase(&req->ctx->io_buffers, bgid);
Jens Axboebcda7ba2020-02-23 16:42:51 -07002951 }
2952 if (*len > kbuf->len)
2953 *len = kbuf->len;
2954 } else {
2955 kbuf = ERR_PTR(-ENOBUFS);
2956 }
2957
2958 io_ring_submit_unlock(req->ctx, needs_lock);
2959
2960 return kbuf;
2961}
2962
Jens Axboe4d954c22020-02-27 07:31:19 -07002963static void __user *io_rw_buffer_select(struct io_kiocb *req, size_t *len,
2964 bool needs_lock)
2965{
2966 struct io_buffer *kbuf;
Bijan Mottahedeh4f4eeba2020-05-19 14:52:49 -07002967 u16 bgid;
Jens Axboe4d954c22020-02-27 07:31:19 -07002968
2969 kbuf = (struct io_buffer *) (unsigned long) req->rw.addr;
Bijan Mottahedeh4f4eeba2020-05-19 14:52:49 -07002970 bgid = req->buf_index;
Jens Axboe4d954c22020-02-27 07:31:19 -07002971 kbuf = io_buffer_select(req, len, bgid, kbuf, needs_lock);
2972 if (IS_ERR(kbuf))
2973 return kbuf;
2974 req->rw.addr = (u64) (unsigned long) kbuf;
2975 req->flags |= REQ_F_BUFFER_SELECTED;
2976 return u64_to_user_ptr(kbuf->addr);
2977}
2978
2979#ifdef CONFIG_COMPAT
2980static ssize_t io_compat_import(struct io_kiocb *req, struct iovec *iov,
2981 bool needs_lock)
2982{
2983 struct compat_iovec __user *uiov;
2984 compat_ssize_t clen;
2985 void __user *buf;
2986 ssize_t len;
2987
2988 uiov = u64_to_user_ptr(req->rw.addr);
2989 if (!access_ok(uiov, sizeof(*uiov)))
2990 return -EFAULT;
2991 if (__get_user(clen, &uiov->iov_len))
2992 return -EFAULT;
2993 if (clen < 0)
2994 return -EINVAL;
2995
2996 len = clen;
2997 buf = io_rw_buffer_select(req, &len, needs_lock);
2998 if (IS_ERR(buf))
2999 return PTR_ERR(buf);
3000 iov[0].iov_base = buf;
3001 iov[0].iov_len = (compat_size_t) len;
3002 return 0;
3003}
3004#endif
3005
3006static ssize_t __io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov,
3007 bool needs_lock)
3008{
3009 struct iovec __user *uiov = u64_to_user_ptr(req->rw.addr);
3010 void __user *buf;
3011 ssize_t len;
3012
3013 if (copy_from_user(iov, uiov, sizeof(*uiov)))
3014 return -EFAULT;
3015
3016 len = iov[0].iov_len;
3017 if (len < 0)
3018 return -EINVAL;
3019 buf = io_rw_buffer_select(req, &len, needs_lock);
3020 if (IS_ERR(buf))
3021 return PTR_ERR(buf);
3022 iov[0].iov_base = buf;
3023 iov[0].iov_len = len;
3024 return 0;
3025}
3026
3027static ssize_t io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov,
3028 bool needs_lock)
3029{
Jens Axboedddb3e22020-06-04 11:27:01 -06003030 if (req->flags & REQ_F_BUFFER_SELECTED) {
3031 struct io_buffer *kbuf;
3032
3033 kbuf = (struct io_buffer *) (unsigned long) req->rw.addr;
3034 iov[0].iov_base = u64_to_user_ptr(kbuf->addr);
3035 iov[0].iov_len = kbuf->len;
Jens Axboe4d954c22020-02-27 07:31:19 -07003036 return 0;
Jens Axboedddb3e22020-06-04 11:27:01 -06003037 }
Pavel Begunkovdd201662020-12-19 03:15:43 +00003038 if (req->rw.len != 1)
Jens Axboe4d954c22020-02-27 07:31:19 -07003039 return -EINVAL;
3040
3041#ifdef CONFIG_COMPAT
3042 if (req->ctx->compat)
3043 return io_compat_import(req, iov, needs_lock);
3044#endif
3045
3046 return __io_iov_buffer_select(req, iov, needs_lock);
3047}
3048
Pavel Begunkov847595d2021-02-04 13:52:06 +00003049static int io_import_iovec(int rw, struct io_kiocb *req, struct iovec **iovec,
3050 struct iov_iter *iter, bool needs_lock)
Jens Axboe2b188cc2019-01-07 10:46:33 -07003051{
Jens Axboe9adbd452019-12-20 08:45:55 -07003052 void __user *buf = u64_to_user_ptr(req->rw.addr);
3053 size_t sqe_len = req->rw.len;
Pavel Begunkov847595d2021-02-04 13:52:06 +00003054 u8 opcode = req->opcode;
Jens Axboe4d954c22020-02-27 07:31:19 -07003055 ssize_t ret;
Jens Axboeedafcce2019-01-09 09:16:05 -07003056
Pavel Begunkov7d009162019-11-25 23:14:40 +03003057 if (opcode == IORING_OP_READ_FIXED || opcode == IORING_OP_WRITE_FIXED) {
Jens Axboeedafcce2019-01-09 09:16:05 -07003058 *iovec = NULL;
Jens Axboe9adbd452019-12-20 08:45:55 -07003059 return io_import_fixed(req, rw, iter);
Jens Axboeedafcce2019-01-09 09:16:05 -07003060 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07003061
Jens Axboebcda7ba2020-02-23 16:42:51 -07003062 /* buffer index only valid with fixed read/write, or buffer select */
Bijan Mottahedeh4f4eeba2020-05-19 14:52:49 -07003063 if (req->buf_index && !(req->flags & REQ_F_BUFFER_SELECT))
Jens Axboe9adbd452019-12-20 08:45:55 -07003064 return -EINVAL;
3065
Jens Axboe3a6820f2019-12-22 15:19:35 -07003066 if (opcode == IORING_OP_READ || opcode == IORING_OP_WRITE) {
Jens Axboebcda7ba2020-02-23 16:42:51 -07003067 if (req->flags & REQ_F_BUFFER_SELECT) {
Jens Axboe4d954c22020-02-27 07:31:19 -07003068 buf = io_rw_buffer_select(req, &sqe_len, needs_lock);
Pavel Begunkov867a23e2020-08-20 11:34:39 +03003069 if (IS_ERR(buf))
Jens Axboe4d954c22020-02-27 07:31:19 -07003070 return PTR_ERR(buf);
Jens Axboe3f9d6442020-03-11 12:27:04 -06003071 req->rw.len = sqe_len;
Jens Axboebcda7ba2020-02-23 16:42:51 -07003072 }
3073
Jens Axboe3a6820f2019-12-22 15:19:35 -07003074 ret = import_single_range(rw, buf, sqe_len, *iovec, iter);
3075 *iovec = NULL;
David Laight10fc72e2020-11-07 13:16:25 +00003076 return ret;
Jens Axboe3a6820f2019-12-22 15:19:35 -07003077 }
3078
Jens Axboe4d954c22020-02-27 07:31:19 -07003079 if (req->flags & REQ_F_BUFFER_SELECT) {
3080 ret = io_iov_buffer_select(req, *iovec, needs_lock);
Pavel Begunkov847595d2021-02-04 13:52:06 +00003081 if (!ret)
3082 iov_iter_init(iter, rw, *iovec, 1, (*iovec)->iov_len);
Jens Axboe4d954c22020-02-27 07:31:19 -07003083 *iovec = NULL;
3084 return ret;
3085 }
3086
Christoph Hellwig89cd35c2020-09-25 06:51:41 +02003087 return __import_iovec(rw, buf, sqe_len, UIO_FASTIOV, iovec, iter,
3088 req->ctx->compat);
Jens Axboe2b188cc2019-01-07 10:46:33 -07003089}
3090
Jens Axboe0fef9482020-08-26 10:36:20 -06003091static inline loff_t *io_kiocb_ppos(struct kiocb *kiocb)
3092{
Pavel Begunkov5b09e372020-09-30 22:57:15 +03003093 return (kiocb->ki_filp->f_mode & FMODE_STREAM) ? NULL : &kiocb->ki_pos;
Jens Axboe0fef9482020-08-26 10:36:20 -06003094}
3095
Jens Axboe32960612019-09-23 11:05:34 -06003096/*
3097 * For files that don't have ->read_iter() and ->write_iter(), handle them
3098 * by looping over ->read() or ->write() manually.
3099 */
Jens Axboe4017eb92020-10-22 14:14:12 -06003100static ssize_t loop_rw_iter(int rw, struct io_kiocb *req, struct iov_iter *iter)
Jens Axboe32960612019-09-23 11:05:34 -06003101{
Jens Axboe4017eb92020-10-22 14:14:12 -06003102 struct kiocb *kiocb = &req->rw.kiocb;
3103 struct file *file = req->file;
Jens Axboe32960612019-09-23 11:05:34 -06003104 ssize_t ret = 0;
3105
3106 /*
3107 * Don't support polled IO through this interface, and we can't
3108 * support non-blocking either. For the latter, this just causes
3109 * the kiocb to be handled from an async context.
3110 */
3111 if (kiocb->ki_flags & IOCB_HIPRI)
3112 return -EOPNOTSUPP;
3113 if (kiocb->ki_flags & IOCB_NOWAIT)
3114 return -EAGAIN;
3115
3116 while (iov_iter_count(iter)) {
Pavel Begunkov311ae9e2019-11-24 11:58:24 +03003117 struct iovec iovec;
Jens Axboe32960612019-09-23 11:05:34 -06003118 ssize_t nr;
3119
Pavel Begunkov311ae9e2019-11-24 11:58:24 +03003120 if (!iov_iter_is_bvec(iter)) {
3121 iovec = iov_iter_iovec(iter);
3122 } else {
Jens Axboe4017eb92020-10-22 14:14:12 -06003123 iovec.iov_base = u64_to_user_ptr(req->rw.addr);
3124 iovec.iov_len = req->rw.len;
Pavel Begunkov311ae9e2019-11-24 11:58:24 +03003125 }
3126
Jens Axboe32960612019-09-23 11:05:34 -06003127 if (rw == READ) {
3128 nr = file->f_op->read(file, iovec.iov_base,
Jens Axboe0fef9482020-08-26 10:36:20 -06003129 iovec.iov_len, io_kiocb_ppos(kiocb));
Jens Axboe32960612019-09-23 11:05:34 -06003130 } else {
3131 nr = file->f_op->write(file, iovec.iov_base,
Jens Axboe0fef9482020-08-26 10:36:20 -06003132 iovec.iov_len, io_kiocb_ppos(kiocb));
Jens Axboe32960612019-09-23 11:05:34 -06003133 }
3134
3135 if (nr < 0) {
3136 if (!ret)
3137 ret = nr;
3138 break;
3139 }
3140 ret += nr;
3141 if (nr != iovec.iov_len)
3142 break;
Jens Axboe4017eb92020-10-22 14:14:12 -06003143 req->rw.len -= nr;
3144 req->rw.addr += nr;
Jens Axboe32960612019-09-23 11:05:34 -06003145 iov_iter_advance(iter, nr);
3146 }
3147
3148 return ret;
3149}
3150
Jens Axboeff6165b2020-08-13 09:47:43 -06003151static void io_req_map_rw(struct io_kiocb *req, const struct iovec *iovec,
3152 const struct iovec *fast_iov, struct iov_iter *iter)
Jens Axboef67676d2019-12-02 11:03:47 -07003153{
Jens Axboee8c2bc12020-08-15 18:44:09 -07003154 struct io_async_rw *rw = req->async_data;
Pavel Begunkovb64e3442020-07-13 22:59:18 +03003155
Jens Axboeff6165b2020-08-13 09:47:43 -06003156 memcpy(&rw->iter, iter, sizeof(*iter));
Pavel Begunkovafb87652020-09-06 00:45:46 +03003157 rw->free_iovec = iovec;
Jens Axboe227c0c92020-08-13 11:51:40 -06003158 rw->bytes_done = 0;
Jens Axboeff6165b2020-08-13 09:47:43 -06003159 /* can only be fixed buffers, no need to do anything */
Pavel Begunkov9c3a2052020-11-23 23:20:27 +00003160 if (iov_iter_is_bvec(iter))
Jens Axboeff6165b2020-08-13 09:47:43 -06003161 return;
Pavel Begunkovb64e3442020-07-13 22:59:18 +03003162 if (!iovec) {
Jens Axboeff6165b2020-08-13 09:47:43 -06003163 unsigned iov_off = 0;
3164
3165 rw->iter.iov = rw->fast_iov;
3166 if (iter->iov != fast_iov) {
3167 iov_off = iter->iov - fast_iov;
3168 rw->iter.iov += iov_off;
3169 }
3170 if (rw->fast_iov != fast_iov)
3171 memcpy(rw->fast_iov + iov_off, fast_iov + iov_off,
Xiaoguang Wang45097da2020-04-08 22:29:58 +08003172 sizeof(struct iovec) * iter->nr_segs);
Pavel Begunkov99bc4c32020-02-07 22:04:45 +03003173 } else {
3174 req->flags |= REQ_F_NEED_CLEANUP;
Jens Axboef67676d2019-12-02 11:03:47 -07003175 }
3176}
3177
Pavel Begunkov6cb78682021-02-28 22:35:17 +00003178static inline int io_alloc_async_data(struct io_kiocb *req)
Xiaoguang Wang3d9932a2020-03-27 15:36:52 +08003179{
Jens Axboee8c2bc12020-08-15 18:44:09 -07003180 WARN_ON_ONCE(!io_op_defs[req->opcode].async_size);
3181 req->async_data = kmalloc(io_op_defs[req->opcode].async_size, GFP_KERNEL);
3182 return req->async_data == NULL;
Xiaoguang Wang3d9932a2020-03-27 15:36:52 +08003183}
3184
Jens Axboeff6165b2020-08-13 09:47:43 -06003185static int io_setup_async_rw(struct io_kiocb *req, const struct iovec *iovec,
3186 const struct iovec *fast_iov,
Jens Axboe227c0c92020-08-13 11:51:40 -06003187 struct iov_iter *iter, bool force)
Jens Axboeb7bb4f72019-12-15 22:13:43 -07003188{
Pavel Begunkov26f05052021-02-28 22:35:18 +00003189 if (!force && !io_op_defs[req->opcode].needs_async_setup)
Jens Axboe74566df2020-01-13 19:23:24 -07003190 return 0;
Jens Axboee8c2bc12020-08-15 18:44:09 -07003191 if (!req->async_data) {
Pavel Begunkov6cb78682021-02-28 22:35:17 +00003192 if (io_alloc_async_data(req)) {
Pavel Begunkov6bf985d2021-02-04 13:52:01 +00003193 kfree(iovec);
Jens Axboe5d204bc2020-01-31 12:06:52 -07003194 return -ENOMEM;
Pavel Begunkov6bf985d2021-02-04 13:52:01 +00003195 }
Jens Axboeb7bb4f72019-12-15 22:13:43 -07003196
Jens Axboeff6165b2020-08-13 09:47:43 -06003197 io_req_map_rw(req, iovec, fast_iov, iter);
Jens Axboe5d204bc2020-01-31 12:06:52 -07003198 }
Jens Axboeb7bb4f72019-12-15 22:13:43 -07003199 return 0;
Jens Axboef67676d2019-12-02 11:03:47 -07003200}
3201
Pavel Begunkov73debe62020-09-30 22:57:54 +03003202static inline int io_rw_prep_async(struct io_kiocb *req, int rw)
Pavel Begunkovc3e330a2020-07-13 22:59:19 +03003203{
Jens Axboee8c2bc12020-08-15 18:44:09 -07003204 struct io_async_rw *iorw = req->async_data;
Pavel Begunkovf4bff102020-09-06 00:45:45 +03003205 struct iovec *iov = iorw->fast_iov;
Pavel Begunkov847595d2021-02-04 13:52:06 +00003206 int ret;
Pavel Begunkovc3e330a2020-07-13 22:59:19 +03003207
Pavel Begunkov2846c482020-11-07 13:16:27 +00003208 ret = io_import_iovec(rw, req, &iov, &iorw->iter, false);
Pavel Begunkovc3e330a2020-07-13 22:59:19 +03003209 if (unlikely(ret < 0))
3210 return ret;
3211
Pavel Begunkovab0b1962020-09-06 00:45:47 +03003212 iorw->bytes_done = 0;
3213 iorw->free_iovec = iov;
3214 if (iov)
3215 req->flags |= REQ_F_NEED_CLEANUP;
Pavel Begunkovc3e330a2020-07-13 22:59:19 +03003216 return 0;
3217}
3218
Pavel Begunkov73debe62020-09-30 22:57:54 +03003219static int io_read_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboef67676d2019-12-02 11:03:47 -07003220{
Jens Axboe3529d8c2019-12-19 18:24:38 -07003221 if (unlikely(!(req->file->f_mode & FMODE_READ)))
3222 return -EBADF;
Pavel Begunkov93642ef2021-02-18 18:29:44 +00003223 return io_prep_rw(req, sqe);
Jens Axboef67676d2019-12-02 11:03:47 -07003224}
3225
Jens Axboec1dd91d2020-08-03 16:43:59 -06003226/*
3227 * This is our waitqueue callback handler, registered through lock_page_async()
3228 * when we initially tried to do the IO with the iocb armed our waitqueue.
3229 * This gets called when the page is unlocked, and we generally expect that to
3230 * happen when the page IO is completed and the page is now uptodate. This will
3231 * queue a task_work based retry of the operation, attempting to copy the data
3232 * again. If the latter fails because the page was NOT uptodate, then we will
3233 * do a thread based blocking retry of the operation. That's the unexpected
3234 * slow path.
3235 */
Jens Axboebcf5a062020-05-22 09:24:42 -06003236static int io_async_buf_func(struct wait_queue_entry *wait, unsigned mode,
3237 int sync, void *arg)
3238{
3239 struct wait_page_queue *wpq;
3240 struct io_kiocb *req = wait->private;
Jens Axboebcf5a062020-05-22 09:24:42 -06003241 struct wait_page_key *key = arg;
Jens Axboebcf5a062020-05-22 09:24:42 -06003242
3243 wpq = container_of(wait, struct wait_page_queue, wait);
3244
Linus Torvaldscdc8fcb2020-08-03 13:01:22 -07003245 if (!wake_page_match(wpq, key))
3246 return 0;
3247
Hao Xuc8d317a2020-09-29 20:00:45 +08003248 req->rw.kiocb.ki_flags &= ~IOCB_WAITQ;
Jens Axboebcf5a062020-05-22 09:24:42 -06003249 list_del_init(&wait->entry);
Pavel Begunkov921b9052021-02-12 03:23:53 +00003250 io_req_task_queue(req);
Jens Axboebcf5a062020-05-22 09:24:42 -06003251 return 1;
3252}
3253
Jens Axboec1dd91d2020-08-03 16:43:59 -06003254/*
3255 * This controls whether a given IO request should be armed for async page
3256 * based retry. If we return false here, the request is handed to the async
3257 * worker threads for retry. If we're doing buffered reads on a regular file,
3258 * we prepare a private wait_page_queue entry and retry the operation. This
3259 * will either succeed because the page is now uptodate and unlocked, or it
3260 * will register a callback when the page is unlocked at IO completion. Through
3261 * that callback, io_uring uses task_work to setup a retry of the operation.
3262 * That retry will attempt the buffered read again. The retry will generally
3263 * succeed, or in rare cases where it fails, we then fall back to using the
3264 * async worker threads for a blocking retry.
3265 */
Jens Axboe227c0c92020-08-13 11:51:40 -06003266static bool io_rw_should_retry(struct io_kiocb *req)
Jens Axboebcf5a062020-05-22 09:24:42 -06003267{
Jens Axboee8c2bc12020-08-15 18:44:09 -07003268 struct io_async_rw *rw = req->async_data;
3269 struct wait_page_queue *wait = &rw->wpq;
Jens Axboebcf5a062020-05-22 09:24:42 -06003270 struct kiocb *kiocb = &req->rw.kiocb;
Jens Axboebcf5a062020-05-22 09:24:42 -06003271
3272 /* never retry for NOWAIT, we just complete with -EAGAIN */
3273 if (req->flags & REQ_F_NOWAIT)
3274 return false;
3275
Jens Axboe227c0c92020-08-13 11:51:40 -06003276 /* Only for buffered IO */
Jens Axboe3b2a4432020-08-16 10:58:43 -07003277 if (kiocb->ki_flags & (IOCB_DIRECT | IOCB_HIPRI))
Jens Axboebcf5a062020-05-22 09:24:42 -06003278 return false;
Jens Axboe3b2a4432020-08-16 10:58:43 -07003279
Jens Axboebcf5a062020-05-22 09:24:42 -06003280 /*
3281 * just use poll if we can, and don't attempt if the fs doesn't
3282 * support callback based unlocks
3283 */
3284 if (file_can_poll(req->file) || !(req->file->f_mode & FMODE_BUF_RASYNC))
3285 return false;
3286
Jens Axboe3b2a4432020-08-16 10:58:43 -07003287 wait->wait.func = io_async_buf_func;
3288 wait->wait.private = req;
3289 wait->wait.flags = 0;
3290 INIT_LIST_HEAD(&wait->wait.entry);
3291 kiocb->ki_flags |= IOCB_WAITQ;
Hao Xuc8d317a2020-09-29 20:00:45 +08003292 kiocb->ki_flags &= ~IOCB_NOWAIT;
Jens Axboe3b2a4432020-08-16 10:58:43 -07003293 kiocb->ki_waitq = wait;
Jens Axboe3b2a4432020-08-16 10:58:43 -07003294 return true;
Jens Axboebcf5a062020-05-22 09:24:42 -06003295}
3296
Pavel Begunkovaeab9502021-06-14 02:36:24 +01003297static inline int io_iter_do_read(struct io_kiocb *req, struct iov_iter *iter)
Jens Axboebcf5a062020-05-22 09:24:42 -06003298{
3299 if (req->file->f_op->read_iter)
3300 return call_read_iter(req->file, &req->rw.kiocb, iter);
Guoyu Huang2dd21112020-08-05 03:53:50 -07003301 else if (req->file->f_op->read)
Jens Axboe4017eb92020-10-22 14:14:12 -06003302 return loop_rw_iter(READ, req, iter);
Guoyu Huang2dd21112020-08-05 03:53:50 -07003303 else
3304 return -EINVAL;
Jens Axboebcf5a062020-05-22 09:24:42 -06003305}
3306
Pavel Begunkov889fca72021-02-10 00:03:09 +00003307static int io_read(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe2b188cc2019-01-07 10:46:33 -07003308{
3309 struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
Jens Axboe9adbd452019-12-20 08:45:55 -07003310 struct kiocb *kiocb = &req->rw.kiocb;
Jens Axboeff6165b2020-08-13 09:47:43 -06003311 struct iov_iter __iter, *iter = &__iter;
Jens Axboee8c2bc12020-08-15 18:44:09 -07003312 struct io_async_rw *rw = req->async_data;
Jens Axboe227c0c92020-08-13 11:51:40 -06003313 ssize_t io_size, ret, ret2;
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003314 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003315
Pavel Begunkov2846c482020-11-07 13:16:27 +00003316 if (rw) {
Jens Axboee8c2bc12020-08-15 18:44:09 -07003317 iter = &rw->iter;
Pavel Begunkov2846c482020-11-07 13:16:27 +00003318 iovec = NULL;
3319 } else {
3320 ret = io_import_iovec(READ, req, &iovec, iter, !force_nonblock);
3321 if (ret < 0)
3322 return ret;
3323 }
Pavel Begunkov632546c2020-11-07 13:16:26 +00003324 io_size = iov_iter_count(iter);
Pavel Begunkovfa15baf2020-08-01 13:50:02 +03003325 req->result = io_size;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003326
Jens Axboefd6c2e42019-12-18 12:19:41 -07003327 /* Ensure we clear previously set non-block flag */
3328 if (!force_nonblock)
Jens Axboe29de5f62020-02-20 09:56:08 -07003329 kiocb->ki_flags &= ~IOCB_NOWAIT;
Pavel Begunkova88fc402020-09-30 22:57:53 +03003330 else
3331 kiocb->ki_flags |= IOCB_NOWAIT;
3332
Pavel Begunkov24c74672020-06-21 13:09:51 +03003333 /* If the file doesn't support async, just async punt */
Pavel Begunkovb191e2d2021-08-09 13:04:03 +01003334 if (force_nonblock && !io_file_supports_nowait(req, READ)) {
Pavel Begunkov6713e7a2021-02-04 13:51:59 +00003335 ret = io_setup_async_rw(req, iovec, inline_vecs, iter, true);
Pavel Begunkov6bf985d2021-02-04 13:52:01 +00003336 return ret ?: -EAGAIN;
Pavel Begunkov6713e7a2021-02-04 13:51:59 +00003337 }
Jens Axboe9e645e112019-05-10 16:07:28 -06003338
Pavel Begunkov632546c2020-11-07 13:16:26 +00003339 ret = rw_verify_area(READ, req->file, io_kiocb_ppos(kiocb), io_size);
Pavel Begunkov5ea5dd42021-02-04 13:52:03 +00003340 if (unlikely(ret)) {
3341 kfree(iovec);
3342 return ret;
3343 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07003344
Jens Axboe227c0c92020-08-13 11:51:40 -06003345 ret = io_iter_do_read(req, iter);
Jens Axboe32960612019-09-23 11:05:34 -06003346
Jens Axboe230d50d2021-04-01 20:41:15 -06003347 if (ret == -EAGAIN || (req->flags & REQ_F_REISSUE)) {
Pavel Begunkov6ad7f232021-04-08 01:54:39 +01003348 req->flags &= ~REQ_F_REISSUE;
Jens Axboeeefdf302020-08-27 16:40:19 -06003349 /* IOPOLL retry should happen for io-wq threads */
3350 if (!force_nonblock && !(req->ctx->flags & IORING_SETUP_IOPOLL))
Jens Axboef91daf52020-08-15 15:58:42 -07003351 goto done;
Pavel Begunkov75c668c2021-02-04 13:52:05 +00003352 /* no retry on NONBLOCK nor RWF_NOWAIT */
3353 if (req->flags & REQ_F_NOWAIT)
Jens Axboe355afae2020-09-02 09:30:31 -06003354 goto done;
Jens Axboe84216312020-08-24 11:45:26 -06003355 /* some cases will consume bytes even on error returns */
Pavel Begunkov632546c2020-11-07 13:16:26 +00003356 iov_iter_revert(iter, io_size - iov_iter_count(iter));
Jens Axboef38c7e32020-09-25 15:23:43 -06003357 ret = 0;
Jens Axboe230d50d2021-04-01 20:41:15 -06003358 } else if (ret == -EIOCBQUEUED) {
3359 goto out_free;
Pavel Begunkov7335e3b2021-02-04 13:52:02 +00003360 } else if (ret <= 0 || ret == io_size || !force_nonblock ||
Pavel Begunkov75c668c2021-02-04 13:52:05 +00003361 (req->flags & REQ_F_NOWAIT) || !(req->flags & REQ_F_ISREG)) {
Pavel Begunkov7335e3b2021-02-04 13:52:02 +00003362 /* read all, failed, already did sync or don't want to retry */
Jens Axboe00d23d52020-08-25 12:59:22 -06003363 goto done;
Jens Axboe227c0c92020-08-13 11:51:40 -06003364 }
3365
Jens Axboe227c0c92020-08-13 11:51:40 -06003366 ret2 = io_setup_async_rw(req, iovec, inline_vecs, iter, true);
Pavel Begunkov6bf985d2021-02-04 13:52:01 +00003367 if (ret2)
3368 return ret2;
3369
Pavel Begunkovfe1cdd52021-02-17 21:02:36 +00003370 iovec = NULL;
Jens Axboee8c2bc12020-08-15 18:44:09 -07003371 rw = req->async_data;
Jens Axboe227c0c92020-08-13 11:51:40 -06003372 /* now use our persistent iterator, if we aren't already */
Jens Axboee8c2bc12020-08-15 18:44:09 -07003373 iter = &rw->iter;
Jens Axboe227c0c92020-08-13 11:51:40 -06003374
Pavel Begunkovb23df912021-02-04 13:52:04 +00003375 do {
3376 io_size -= ret;
3377 rw->bytes_done += ret;
3378 /* if we can retry, do so with the callbacks armed */
3379 if (!io_rw_should_retry(req)) {
3380 kiocb->ki_flags &= ~IOCB_WAITQ;
3381 return -EAGAIN;
3382 }
3383
3384 /*
3385 * Now retry read with the IOCB_WAITQ parts set in the iocb. If
3386 * we get -EIOCBQUEUED, then we'll get a notification when the
3387 * desired page gets unlocked. We can also get a partial read
3388 * here, and if we do, then just retry at the new offset.
3389 */
3390 ret = io_iter_do_read(req, iter);
3391 if (ret == -EIOCBQUEUED)
3392 return 0;
Jens Axboe227c0c92020-08-13 11:51:40 -06003393 /* we got some bytes, but not all. retry. */
Jens Axboeb5b0ecb2021-03-04 21:02:58 -07003394 kiocb->ki_flags &= ~IOCB_WAITQ;
Pavel Begunkovb23df912021-02-04 13:52:04 +00003395 } while (ret > 0 && ret < io_size);
Jens Axboe227c0c92020-08-13 11:51:40 -06003396done:
Pavel Begunkov889fca72021-02-10 00:03:09 +00003397 kiocb_done(kiocb, ret, issue_flags);
Pavel Begunkovfe1cdd52021-02-17 21:02:36 +00003398out_free:
3399 /* it's faster to check here then delegate to kfree */
3400 if (iovec)
3401 kfree(iovec);
Pavel Begunkov5ea5dd42021-02-04 13:52:03 +00003402 return 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003403}
3404
Pavel Begunkov73debe62020-09-30 22:57:54 +03003405static int io_write_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboef67676d2019-12-02 11:03:47 -07003406{
Jens Axboe3529d8c2019-12-19 18:24:38 -07003407 if (unlikely(!(req->file->f_mode & FMODE_WRITE)))
3408 return -EBADF;
Pavel Begunkov93642ef2021-02-18 18:29:44 +00003409 return io_prep_rw(req, sqe);
Jens Axboef67676d2019-12-02 11:03:47 -07003410}
3411
Pavel Begunkov889fca72021-02-10 00:03:09 +00003412static int io_write(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe2b188cc2019-01-07 10:46:33 -07003413{
3414 struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
Jens Axboe9adbd452019-12-20 08:45:55 -07003415 struct kiocb *kiocb = &req->rw.kiocb;
Jens Axboeff6165b2020-08-13 09:47:43 -06003416 struct iov_iter __iter, *iter = &__iter;
Jens Axboee8c2bc12020-08-15 18:44:09 -07003417 struct io_async_rw *rw = req->async_data;
Pavel Begunkovfa15baf2020-08-01 13:50:02 +03003418 ssize_t ret, ret2, io_size;
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003419 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003420
Pavel Begunkov2846c482020-11-07 13:16:27 +00003421 if (rw) {
Jens Axboee8c2bc12020-08-15 18:44:09 -07003422 iter = &rw->iter;
Pavel Begunkov2846c482020-11-07 13:16:27 +00003423 iovec = NULL;
3424 } else {
3425 ret = io_import_iovec(WRITE, req, &iovec, iter, !force_nonblock);
3426 if (ret < 0)
3427 return ret;
3428 }
Pavel Begunkov632546c2020-11-07 13:16:26 +00003429 io_size = iov_iter_count(iter);
Pavel Begunkovfa15baf2020-08-01 13:50:02 +03003430 req->result = io_size;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003431
Jens Axboefd6c2e42019-12-18 12:19:41 -07003432 /* Ensure we clear previously set non-block flag */
3433 if (!force_nonblock)
Pavel Begunkova88fc402020-09-30 22:57:53 +03003434 kiocb->ki_flags &= ~IOCB_NOWAIT;
3435 else
3436 kiocb->ki_flags |= IOCB_NOWAIT;
Jens Axboefd6c2e42019-12-18 12:19:41 -07003437
Pavel Begunkov24c74672020-06-21 13:09:51 +03003438 /* If the file doesn't support async, just async punt */
Pavel Begunkovb191e2d2021-08-09 13:04:03 +01003439 if (force_nonblock && !io_file_supports_nowait(req, WRITE))
Jens Axboef67676d2019-12-02 11:03:47 -07003440 goto copy_iov;
Jens Axboef67676d2019-12-02 11:03:47 -07003441
Jens Axboe10d59342019-12-09 20:16:22 -07003442 /* file path doesn't support NOWAIT for non-direct_IO */
3443 if (force_nonblock && !(kiocb->ki_flags & IOCB_DIRECT) &&
3444 (req->flags & REQ_F_ISREG))
Jens Axboef67676d2019-12-02 11:03:47 -07003445 goto copy_iov;
Jens Axboe9e645e112019-05-10 16:07:28 -06003446
Pavel Begunkov632546c2020-11-07 13:16:26 +00003447 ret = rw_verify_area(WRITE, req->file, io_kiocb_ppos(kiocb), io_size);
Pavel Begunkovfa15baf2020-08-01 13:50:02 +03003448 if (unlikely(ret))
3449 goto out_free;
Roman Penyaev9bf79332019-03-25 20:09:24 +01003450
Pavel Begunkovfa15baf2020-08-01 13:50:02 +03003451 /*
3452 * Open-code file_start_write here to grab freeze protection,
3453 * which will be released by another thread in
3454 * io_complete_rw(). Fool lockdep by telling it the lock got
3455 * released so that it doesn't complain about the held lock when
3456 * we return to userspace.
3457 */
3458 if (req->flags & REQ_F_ISREG) {
Darrick J. Wong8a3c84b2020-11-10 16:50:21 -08003459 sb_start_write(file_inode(req->file)->i_sb);
Pavel Begunkovfa15baf2020-08-01 13:50:02 +03003460 __sb_writers_release(file_inode(req->file)->i_sb,
3461 SB_FREEZE_WRITE);
3462 }
3463 kiocb->ki_flags |= IOCB_WRITE;
Roman Penyaev9bf79332019-03-25 20:09:24 +01003464
Pavel Begunkovfa15baf2020-08-01 13:50:02 +03003465 if (req->file->f_op->write_iter)
Jens Axboeff6165b2020-08-13 09:47:43 -06003466 ret2 = call_write_iter(req->file, kiocb, iter);
Guoyu Huang2dd21112020-08-05 03:53:50 -07003467 else if (req->file->f_op->write)
Jens Axboe4017eb92020-10-22 14:14:12 -06003468 ret2 = loop_rw_iter(WRITE, req, iter);
Guoyu Huang2dd21112020-08-05 03:53:50 -07003469 else
3470 ret2 = -EINVAL;
Jens Axboe4ed734b2020-03-20 11:23:41 -06003471
Pavel Begunkov6ad7f232021-04-08 01:54:39 +01003472 if (req->flags & REQ_F_REISSUE) {
3473 req->flags &= ~REQ_F_REISSUE;
Jens Axboe230d50d2021-04-01 20:41:15 -06003474 ret2 = -EAGAIN;
Pavel Begunkov6ad7f232021-04-08 01:54:39 +01003475 }
Jens Axboe230d50d2021-04-01 20:41:15 -06003476
Pavel Begunkovfa15baf2020-08-01 13:50:02 +03003477 /*
3478 * Raw bdev writes will return -EOPNOTSUPP for IOCB_NOWAIT. Just
3479 * retry them without IOCB_NOWAIT.
3480 */
3481 if (ret2 == -EOPNOTSUPP && (kiocb->ki_flags & IOCB_NOWAIT))
3482 ret2 = -EAGAIN;
Pavel Begunkov75c668c2021-02-04 13:52:05 +00003483 /* no retry on NONBLOCK nor RWF_NOWAIT */
3484 if (ret2 == -EAGAIN && (req->flags & REQ_F_NOWAIT))
Jens Axboe355afae2020-09-02 09:30:31 -06003485 goto done;
Pavel Begunkovfa15baf2020-08-01 13:50:02 +03003486 if (!force_nonblock || ret2 != -EAGAIN) {
Jens Axboeeefdf302020-08-27 16:40:19 -06003487 /* IOPOLL retry should happen for io-wq threads */
3488 if ((req->ctx->flags & IORING_SETUP_IOPOLL) && ret2 == -EAGAIN)
3489 goto copy_iov;
Jens Axboe355afae2020-09-02 09:30:31 -06003490done:
Pavel Begunkov889fca72021-02-10 00:03:09 +00003491 kiocb_done(kiocb, ret2, issue_flags);
Pavel Begunkovfa15baf2020-08-01 13:50:02 +03003492 } else {
Jens Axboef67676d2019-12-02 11:03:47 -07003493copy_iov:
Jens Axboe84216312020-08-24 11:45:26 -06003494 /* some cases will consume bytes even on error returns */
Pavel Begunkov632546c2020-11-07 13:16:26 +00003495 iov_iter_revert(iter, io_size - iov_iter_count(iter));
Jens Axboe227c0c92020-08-13 11:51:40 -06003496 ret = io_setup_async_rw(req, iovec, inline_vecs, iter, false);
Pavel Begunkov6bf985d2021-02-04 13:52:01 +00003497 return ret ?: -EAGAIN;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003498 }
Jens Axboe31b51512019-01-18 22:56:34 -07003499out_free:
Pavel Begunkovf261c162020-08-20 11:34:10 +03003500 /* it's reportedly faster than delegating the null check to kfree() */
Pavel Begunkov252917c2020-07-13 22:59:20 +03003501 if (iovec)
Xiaoguang Wang6f2cc162020-06-18 15:01:56 +08003502 kfree(iovec);
Jens Axboe2b188cc2019-01-07 10:46:33 -07003503 return ret;
3504}
3505
Jens Axboe80a261f2020-09-28 14:23:58 -06003506static int io_renameat_prep(struct io_kiocb *req,
3507 const struct io_uring_sqe *sqe)
3508{
3509 struct io_rename *ren = &req->rename;
3510 const char __user *oldf, *newf;
3511
Jens Axboeed7eb252021-06-23 09:04:13 -06003512 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3513 return -EINVAL;
3514 if (sqe->ioprio || sqe->buf_index)
3515 return -EINVAL;
Jens Axboe80a261f2020-09-28 14:23:58 -06003516 if (unlikely(req->flags & REQ_F_FIXED_FILE))
3517 return -EBADF;
3518
3519 ren->old_dfd = READ_ONCE(sqe->fd);
3520 oldf = u64_to_user_ptr(READ_ONCE(sqe->addr));
3521 newf = u64_to_user_ptr(READ_ONCE(sqe->addr2));
3522 ren->new_dfd = READ_ONCE(sqe->len);
3523 ren->flags = READ_ONCE(sqe->rename_flags);
3524
3525 ren->oldpath = getname(oldf);
3526 if (IS_ERR(ren->oldpath))
3527 return PTR_ERR(ren->oldpath);
3528
3529 ren->newpath = getname(newf);
3530 if (IS_ERR(ren->newpath)) {
3531 putname(ren->oldpath);
3532 return PTR_ERR(ren->newpath);
3533 }
3534
3535 req->flags |= REQ_F_NEED_CLEANUP;
3536 return 0;
3537}
3538
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003539static int io_renameat(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe80a261f2020-09-28 14:23:58 -06003540{
3541 struct io_rename *ren = &req->rename;
3542 int ret;
3543
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003544 if (issue_flags & IO_URING_F_NONBLOCK)
Jens Axboe80a261f2020-09-28 14:23:58 -06003545 return -EAGAIN;
3546
3547 ret = do_renameat2(ren->old_dfd, ren->oldpath, ren->new_dfd,
3548 ren->newpath, ren->flags);
3549
3550 req->flags &= ~REQ_F_NEED_CLEANUP;
3551 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01003552 req_set_fail(req);
Jens Axboe80a261f2020-09-28 14:23:58 -06003553 io_req_complete(req, ret);
3554 return 0;
3555}
3556
Jens Axboe14a11432020-09-28 14:27:37 -06003557static int io_unlinkat_prep(struct io_kiocb *req,
3558 const struct io_uring_sqe *sqe)
3559{
3560 struct io_unlink *un = &req->unlink;
3561 const char __user *fname;
3562
Jens Axboe22634bc2021-06-23 09:07:45 -06003563 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3564 return -EINVAL;
3565 if (sqe->ioprio || sqe->off || sqe->len || sqe->buf_index)
3566 return -EINVAL;
Jens Axboe14a11432020-09-28 14:27:37 -06003567 if (unlikely(req->flags & REQ_F_FIXED_FILE))
3568 return -EBADF;
3569
3570 un->dfd = READ_ONCE(sqe->fd);
3571
3572 un->flags = READ_ONCE(sqe->unlink_flags);
3573 if (un->flags & ~AT_REMOVEDIR)
3574 return -EINVAL;
3575
3576 fname = u64_to_user_ptr(READ_ONCE(sqe->addr));
3577 un->filename = getname(fname);
3578 if (IS_ERR(un->filename))
3579 return PTR_ERR(un->filename);
3580
3581 req->flags |= REQ_F_NEED_CLEANUP;
3582 return 0;
3583}
3584
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003585static int io_unlinkat(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe14a11432020-09-28 14:27:37 -06003586{
3587 struct io_unlink *un = &req->unlink;
3588 int ret;
3589
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003590 if (issue_flags & IO_URING_F_NONBLOCK)
Jens Axboe14a11432020-09-28 14:27:37 -06003591 return -EAGAIN;
3592
3593 if (un->flags & AT_REMOVEDIR)
3594 ret = do_rmdir(un->dfd, un->filename);
3595 else
3596 ret = do_unlinkat(un->dfd, un->filename);
3597
3598 req->flags &= ~REQ_F_NEED_CLEANUP;
3599 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01003600 req_set_fail(req);
Jens Axboe14a11432020-09-28 14:27:37 -06003601 io_req_complete(req, ret);
3602 return 0;
3603}
3604
Jens Axboe36f4fa62020-09-05 11:14:22 -06003605static int io_shutdown_prep(struct io_kiocb *req,
3606 const struct io_uring_sqe *sqe)
3607{
3608#if defined(CONFIG_NET)
3609 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3610 return -EINVAL;
3611 if (sqe->ioprio || sqe->off || sqe->addr || sqe->rw_flags ||
3612 sqe->buf_index)
3613 return -EINVAL;
3614
3615 req->shutdown.how = READ_ONCE(sqe->len);
3616 return 0;
3617#else
3618 return -EOPNOTSUPP;
3619#endif
3620}
3621
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003622static int io_shutdown(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe36f4fa62020-09-05 11:14:22 -06003623{
3624#if defined(CONFIG_NET)
3625 struct socket *sock;
3626 int ret;
3627
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003628 if (issue_flags & IO_URING_F_NONBLOCK)
Jens Axboe36f4fa62020-09-05 11:14:22 -06003629 return -EAGAIN;
3630
Linus Torvalds48aba792020-12-16 12:44:05 -08003631 sock = sock_from_file(req->file);
Jens Axboe36f4fa62020-09-05 11:14:22 -06003632 if (unlikely(!sock))
Linus Torvalds48aba792020-12-16 12:44:05 -08003633 return -ENOTSOCK;
Jens Axboe36f4fa62020-09-05 11:14:22 -06003634
3635 ret = __sys_shutdown_sock(sock, req->shutdown.how);
Jens Axboea1464682020-12-14 20:57:27 -07003636 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01003637 req_set_fail(req);
Jens Axboe36f4fa62020-09-05 11:14:22 -06003638 io_req_complete(req, ret);
3639 return 0;
3640#else
3641 return -EOPNOTSUPP;
3642#endif
3643}
3644
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03003645static int __io_splice_prep(struct io_kiocb *req,
3646 const struct io_uring_sqe *sqe)
Pavel Begunkov7d67af22020-02-24 11:32:45 +03003647{
Pavel Begunkovfe7e3252021-06-24 15:09:57 +01003648 struct io_splice *sp = &req->splice;
Pavel Begunkov7d67af22020-02-24 11:32:45 +03003649 unsigned int valid_flags = SPLICE_F_FD_IN_FIXED | SPLICE_F_ALL;
Pavel Begunkov7d67af22020-02-24 11:32:45 +03003650
Pavel Begunkov3232dd02020-06-03 18:03:22 +03003651 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3652 return -EINVAL;
Pavel Begunkov7d67af22020-02-24 11:32:45 +03003653
3654 sp->file_in = NULL;
Pavel Begunkov7d67af22020-02-24 11:32:45 +03003655 sp->len = READ_ONCE(sqe->len);
3656 sp->flags = READ_ONCE(sqe->splice_flags);
3657
3658 if (unlikely(sp->flags & ~valid_flags))
3659 return -EINVAL;
3660
Pavel Begunkov62906e82021-08-10 14:52:47 +01003661 sp->file_in = io_file_get(req->ctx, req, READ_ONCE(sqe->splice_fd_in),
Pavel Begunkov8371adf2020-10-10 18:34:08 +01003662 (sp->flags & SPLICE_F_FD_IN_FIXED));
3663 if (!sp->file_in)
3664 return -EBADF;
Pavel Begunkov7d67af22020-02-24 11:32:45 +03003665 req->flags |= REQ_F_NEED_CLEANUP;
Pavel Begunkov7d67af22020-02-24 11:32:45 +03003666 return 0;
3667}
3668
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03003669static int io_tee_prep(struct io_kiocb *req,
3670 const struct io_uring_sqe *sqe)
3671{
3672 if (READ_ONCE(sqe->splice_off_in) || READ_ONCE(sqe->off))
3673 return -EINVAL;
3674 return __io_splice_prep(req, sqe);
3675}
3676
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003677static int io_tee(struct io_kiocb *req, unsigned int issue_flags)
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03003678{
3679 struct io_splice *sp = &req->splice;
3680 struct file *in = sp->file_in;
3681 struct file *out = sp->file_out;
3682 unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED;
3683 long ret = 0;
3684
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003685 if (issue_flags & IO_URING_F_NONBLOCK)
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03003686 return -EAGAIN;
3687 if (sp->len)
3688 ret = do_tee(in, out, sp->len, flags);
3689
Pavel Begunkove1d767f2021-03-19 17:22:43 +00003690 if (!(sp->flags & SPLICE_F_FD_IN_FIXED))
3691 io_put_file(in);
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03003692 req->flags &= ~REQ_F_NEED_CLEANUP;
3693
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03003694 if (ret != sp->len)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01003695 req_set_fail(req);
Jens Axboee1e16092020-06-22 09:17:17 -06003696 io_req_complete(req, ret);
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03003697 return 0;
3698}
3699
3700static int io_splice_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
3701{
Pavel Begunkovfe7e3252021-06-24 15:09:57 +01003702 struct io_splice *sp = &req->splice;
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03003703
3704 sp->off_in = READ_ONCE(sqe->splice_off_in);
3705 sp->off_out = READ_ONCE(sqe->off);
3706 return __io_splice_prep(req, sqe);
3707}
3708
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003709static int io_splice(struct io_kiocb *req, unsigned int issue_flags)
Pavel Begunkov7d67af22020-02-24 11:32:45 +03003710{
3711 struct io_splice *sp = &req->splice;
3712 struct file *in = sp->file_in;
3713 struct file *out = sp->file_out;
3714 unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED;
3715 loff_t *poff_in, *poff_out;
Pavel Begunkovc9687422020-05-04 23:00:54 +03003716 long ret = 0;
Pavel Begunkov7d67af22020-02-24 11:32:45 +03003717
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003718 if (issue_flags & IO_URING_F_NONBLOCK)
Pavel Begunkov2fb3e822020-05-01 17:09:38 +03003719 return -EAGAIN;
Pavel Begunkov7d67af22020-02-24 11:32:45 +03003720
3721 poff_in = (sp->off_in == -1) ? NULL : &sp->off_in;
3722 poff_out = (sp->off_out == -1) ? NULL : &sp->off_out;
Pavel Begunkovc9687422020-05-04 23:00:54 +03003723
Jens Axboe948a7742020-05-17 14:21:38 -06003724 if (sp->len)
Pavel Begunkovc9687422020-05-04 23:00:54 +03003725 ret = do_splice(in, poff_in, out, poff_out, sp->len, flags);
Pavel Begunkov7d67af22020-02-24 11:32:45 +03003726
Pavel Begunkove1d767f2021-03-19 17:22:43 +00003727 if (!(sp->flags & SPLICE_F_FD_IN_FIXED))
3728 io_put_file(in);
Pavel Begunkov7d67af22020-02-24 11:32:45 +03003729 req->flags &= ~REQ_F_NEED_CLEANUP;
3730
Pavel Begunkov7d67af22020-02-24 11:32:45 +03003731 if (ret != sp->len)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01003732 req_set_fail(req);
Jens Axboee1e16092020-06-22 09:17:17 -06003733 io_req_complete(req, ret);
Pavel Begunkov7d67af22020-02-24 11:32:45 +03003734 return 0;
3735}
3736
Jens Axboe2b188cc2019-01-07 10:46:33 -07003737/*
3738 * IORING_OP_NOP just posts a completion event, nothing else.
3739 */
Pavel Begunkov889fca72021-02-10 00:03:09 +00003740static int io_nop(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe2b188cc2019-01-07 10:46:33 -07003741{
3742 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe2b188cc2019-01-07 10:46:33 -07003743
Jens Axboedef596e2019-01-09 08:59:42 -07003744 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
3745 return -EINVAL;
3746
Pavel Begunkov889fca72021-02-10 00:03:09 +00003747 __io_req_complete(req, issue_flags, 0, 0);
Jens Axboe2b188cc2019-01-07 10:46:33 -07003748 return 0;
3749}
3750
Pavel Begunkov1155c762021-02-18 18:29:38 +00003751static int io_fsync_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Christoph Hellwigc992fe22019-01-11 09:43:02 -07003752{
Jens Axboe6b063142019-01-10 22:13:58 -07003753 struct io_ring_ctx *ctx = req->ctx;
Christoph Hellwigc992fe22019-01-11 09:43:02 -07003754
Jens Axboe09bb8392019-03-13 12:39:28 -06003755 if (!req->file)
3756 return -EBADF;
Christoph Hellwigc992fe22019-01-11 09:43:02 -07003757
Jens Axboe6b063142019-01-10 22:13:58 -07003758 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
Jens Axboedef596e2019-01-09 08:59:42 -07003759 return -EINVAL;
Jens Axboeedafcce2019-01-09 09:16:05 -07003760 if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index))
Christoph Hellwigc992fe22019-01-11 09:43:02 -07003761 return -EINVAL;
3762
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07003763 req->sync.flags = READ_ONCE(sqe->fsync_flags);
3764 if (unlikely(req->sync.flags & ~IORING_FSYNC_DATASYNC))
3765 return -EINVAL;
3766
3767 req->sync.off = READ_ONCE(sqe->off);
3768 req->sync.len = READ_ONCE(sqe->len);
Christoph Hellwigc992fe22019-01-11 09:43:02 -07003769 return 0;
3770}
3771
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003772static int io_fsync(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe78912932020-01-14 22:09:06 -07003773{
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07003774 loff_t end = req->sync.off + req->sync.len;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07003775 int ret;
3776
Pavel Begunkovac45abc2020-06-08 21:08:18 +03003777 /* fsync always requires a blocking context */
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003778 if (issue_flags & IO_URING_F_NONBLOCK)
Pavel Begunkovac45abc2020-06-08 21:08:18 +03003779 return -EAGAIN;
3780
Jens Axboe9adbd452019-12-20 08:45:55 -07003781 ret = vfs_fsync_range(req->file, req->sync.off,
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07003782 end > 0 ? end : LLONG_MAX,
3783 req->sync.flags & IORING_FSYNC_DATASYNC);
3784 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01003785 req_set_fail(req);
Jens Axboee1e16092020-06-22 09:17:17 -06003786 io_req_complete(req, ret);
Christoph Hellwigc992fe22019-01-11 09:43:02 -07003787 return 0;
3788}
3789
Jens Axboed63d1b52019-12-10 10:38:56 -07003790static int io_fallocate_prep(struct io_kiocb *req,
3791 const struct io_uring_sqe *sqe)
3792{
3793 if (sqe->ioprio || sqe->buf_index || sqe->rw_flags)
3794 return -EINVAL;
Pavel Begunkov3232dd02020-06-03 18:03:22 +03003795 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3796 return -EINVAL;
Jens Axboed63d1b52019-12-10 10:38:56 -07003797
3798 req->sync.off = READ_ONCE(sqe->off);
3799 req->sync.len = READ_ONCE(sqe->addr);
3800 req->sync.mode = READ_ONCE(sqe->len);
3801 return 0;
3802}
3803
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003804static int io_fallocate(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboed63d1b52019-12-10 10:38:56 -07003805{
Pavel Begunkovac45abc2020-06-08 21:08:18 +03003806 int ret;
Jens Axboed63d1b52019-12-10 10:38:56 -07003807
Pavel Begunkovac45abc2020-06-08 21:08:18 +03003808 /* fallocate always requiring blocking context */
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003809 if (issue_flags & IO_URING_F_NONBLOCK)
Pavel Begunkovac45abc2020-06-08 21:08:18 +03003810 return -EAGAIN;
Pavel Begunkovac45abc2020-06-08 21:08:18 +03003811 ret = vfs_fallocate(req->file, req->sync.mode, req->sync.off,
3812 req->sync.len);
Pavel Begunkovac45abc2020-06-08 21:08:18 +03003813 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01003814 req_set_fail(req);
Jens Axboee1e16092020-06-22 09:17:17 -06003815 io_req_complete(req, ret);
Jens Axboed63d1b52019-12-10 10:38:56 -07003816 return 0;
3817}
3818
Pavel Begunkovec65fea2020-06-03 18:03:24 +03003819static int __io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboe15b71ab2019-12-11 11:20:36 -07003820{
Jens Axboef8748882020-01-08 17:47:02 -07003821 const char __user *fname;
Jens Axboe15b71ab2019-12-11 11:20:36 -07003822 int ret;
3823
Pavel Begunkovd3fddf62021-08-09 13:04:16 +01003824 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
3825 return -EINVAL;
Pavel Begunkovec65fea2020-06-03 18:03:24 +03003826 if (unlikely(sqe->ioprio || sqe->buf_index))
Jens Axboe15b71ab2019-12-11 11:20:36 -07003827 return -EINVAL;
Pavel Begunkovec65fea2020-06-03 18:03:24 +03003828 if (unlikely(req->flags & REQ_F_FIXED_FILE))
Jens Axboecf3040c2020-02-06 21:31:40 -07003829 return -EBADF;
Jens Axboe15b71ab2019-12-11 11:20:36 -07003830
Pavel Begunkovec65fea2020-06-03 18:03:24 +03003831 /* open.how should be already initialised */
3832 if (!(req->open.how.flags & O_PATH) && force_o_largefile())
Jens Axboe08a1d26eb2020-04-08 09:20:54 -06003833 req->open.how.flags |= O_LARGEFILE;
Jens Axboe15b71ab2019-12-11 11:20:36 -07003834
Pavel Begunkov25e72d12020-06-03 18:03:23 +03003835 req->open.dfd = READ_ONCE(sqe->fd);
3836 fname = u64_to_user_ptr(READ_ONCE(sqe->addr));
Jens Axboef8748882020-01-08 17:47:02 -07003837 req->open.filename = getname(fname);
Jens Axboe15b71ab2019-12-11 11:20:36 -07003838 if (IS_ERR(req->open.filename)) {
3839 ret = PTR_ERR(req->open.filename);
3840 req->open.filename = NULL;
3841 return ret;
3842 }
Jens Axboe4022e7a2020-03-19 19:23:18 -06003843 req->open.nofile = rlimit(RLIMIT_NOFILE);
Pavel Begunkov8fef80b2020-02-07 23:59:53 +03003844 req->flags |= REQ_F_NEED_CLEANUP;
Jens Axboe15b71ab2019-12-11 11:20:36 -07003845 return 0;
3846}
3847
Pavel Begunkovec65fea2020-06-03 18:03:24 +03003848static int io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
3849{
Pavel Begunkovd3fddf62021-08-09 13:04:16 +01003850 u64 mode = READ_ONCE(sqe->len);
3851 u64 flags = READ_ONCE(sqe->open_flags);
Pavel Begunkovec65fea2020-06-03 18:03:24 +03003852
Pavel Begunkovec65fea2020-06-03 18:03:24 +03003853 req->open.how = build_open_how(flags, mode);
3854 return __io_openat_prep(req, sqe);
3855}
3856
Jens Axboecebdb982020-01-08 17:59:24 -07003857static int io_openat2_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
3858{
3859 struct open_how __user *how;
Jens Axboecebdb982020-01-08 17:59:24 -07003860 size_t len;
3861 int ret;
3862
Jens Axboecebdb982020-01-08 17:59:24 -07003863 how = u64_to_user_ptr(READ_ONCE(sqe->addr2));
3864 len = READ_ONCE(sqe->len);
Jens Axboecebdb982020-01-08 17:59:24 -07003865 if (len < OPEN_HOW_SIZE_VER0)
3866 return -EINVAL;
3867
3868 ret = copy_struct_from_user(&req->open.how, sizeof(req->open.how), how,
3869 len);
3870 if (ret)
3871 return ret;
3872
Pavel Begunkovec65fea2020-06-03 18:03:24 +03003873 return __io_openat_prep(req, sqe);
Jens Axboecebdb982020-01-08 17:59:24 -07003874}
3875
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003876static int io_openat2(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe15b71ab2019-12-11 11:20:36 -07003877{
3878 struct open_flags op;
Jens Axboe15b71ab2019-12-11 11:20:36 -07003879 struct file *file;
Jens Axboe3a81fd02020-12-10 12:25:36 -07003880 bool nonblock_set;
3881 bool resolve_nonblock;
Jens Axboe15b71ab2019-12-11 11:20:36 -07003882 int ret;
3883
Jens Axboecebdb982020-01-08 17:59:24 -07003884 ret = build_open_flags(&req->open.how, &op);
Jens Axboe15b71ab2019-12-11 11:20:36 -07003885 if (ret)
3886 goto err;
Jens Axboe3a81fd02020-12-10 12:25:36 -07003887 nonblock_set = op.open_flag & O_NONBLOCK;
3888 resolve_nonblock = req->open.how.resolve & RESOLVE_CACHED;
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003889 if (issue_flags & IO_URING_F_NONBLOCK) {
Jens Axboe3a81fd02020-12-10 12:25:36 -07003890 /*
3891 * Don't bother trying for O_TRUNC, O_CREAT, or O_TMPFILE open,
3892 * it'll always -EAGAIN
3893 */
3894 if (req->open.how.flags & (O_TRUNC | O_CREAT | O_TMPFILE))
3895 return -EAGAIN;
3896 op.lookup_flags |= LOOKUP_CACHED;
3897 op.open_flag |= O_NONBLOCK;
3898 }
Jens Axboe15b71ab2019-12-11 11:20:36 -07003899
Jens Axboe4022e7a2020-03-19 19:23:18 -06003900 ret = __get_unused_fd_flags(req->open.how.flags, req->open.nofile);
Jens Axboe15b71ab2019-12-11 11:20:36 -07003901 if (ret < 0)
3902 goto err;
3903
3904 file = do_filp_open(req->open.dfd, req->open.filename, &op);
Pavel Begunkov12dcb58a2021-06-24 15:10:00 +01003905 if (IS_ERR(file)) {
Jens Axboe3a81fd02020-12-10 12:25:36 -07003906 /*
Pavel Begunkov12dcb58a2021-06-24 15:10:00 +01003907 * We could hang on to this 'fd' on retrying, but seems like
3908 * marginal gain for something that is now known to be a slower
3909 * path. So just put it, and we'll get a new one when we retry.
Jens Axboe3a81fd02020-12-10 12:25:36 -07003910 */
3911 put_unused_fd(ret);
Pavel Begunkov12dcb58a2021-06-24 15:10:00 +01003912
3913 ret = PTR_ERR(file);
3914 /* only retry if RESOLVE_CACHED wasn't already set by application */
3915 if (ret == -EAGAIN &&
3916 (!resolve_nonblock && (issue_flags & IO_URING_F_NONBLOCK)))
3917 return -EAGAIN;
3918 goto err;
Jens Axboe3a81fd02020-12-10 12:25:36 -07003919 }
3920
Pavel Begunkov12dcb58a2021-06-24 15:10:00 +01003921 if ((issue_flags & IO_URING_F_NONBLOCK) && !nonblock_set)
3922 file->f_flags &= ~O_NONBLOCK;
3923 fsnotify_open(file);
3924 fd_install(ret, file);
Jens Axboe15b71ab2019-12-11 11:20:36 -07003925err:
3926 putname(req->open.filename);
Pavel Begunkov8fef80b2020-02-07 23:59:53 +03003927 req->flags &= ~REQ_F_NEED_CLEANUP;
Jens Axboe15b71ab2019-12-11 11:20:36 -07003928 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01003929 req_set_fail(req);
Pavel Begunkov0bdf3392021-04-11 01:46:29 +01003930 __io_req_complete(req, issue_flags, ret, 0);
Jens Axboe15b71ab2019-12-11 11:20:36 -07003931 return 0;
3932}
3933
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003934static int io_openat(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboecebdb982020-01-08 17:59:24 -07003935{
Pavel Begunkove45cff52021-02-28 22:35:14 +00003936 return io_openat2(req, issue_flags);
Jens Axboecebdb982020-01-08 17:59:24 -07003937}
3938
Jens Axboe067524e2020-03-02 16:32:28 -07003939static int io_remove_buffers_prep(struct io_kiocb *req,
3940 const struct io_uring_sqe *sqe)
3941{
3942 struct io_provide_buf *p = &req->pbuf;
3943 u64 tmp;
3944
3945 if (sqe->ioprio || sqe->rw_flags || sqe->addr || sqe->len || sqe->off)
3946 return -EINVAL;
3947
3948 tmp = READ_ONCE(sqe->fd);
3949 if (!tmp || tmp > USHRT_MAX)
3950 return -EINVAL;
3951
3952 memset(p, 0, sizeof(*p));
3953 p->nbufs = tmp;
3954 p->bgid = READ_ONCE(sqe->buf_group);
3955 return 0;
3956}
3957
3958static int __io_remove_buffers(struct io_ring_ctx *ctx, struct io_buffer *buf,
3959 int bgid, unsigned nbufs)
3960{
3961 unsigned i = 0;
3962
3963 /* shouldn't happen */
3964 if (!nbufs)
3965 return 0;
3966
3967 /* the head kbuf is the list itself */
3968 while (!list_empty(&buf->list)) {
3969 struct io_buffer *nxt;
3970
3971 nxt = list_first_entry(&buf->list, struct io_buffer, list);
3972 list_del(&nxt->list);
3973 kfree(nxt);
3974 if (++i == nbufs)
3975 return i;
3976 }
3977 i++;
3978 kfree(buf);
Jens Axboe9e15c3a2021-03-13 12:29:43 -07003979 xa_erase(&ctx->io_buffers, bgid);
Jens Axboe067524e2020-03-02 16:32:28 -07003980
3981 return i;
3982}
3983
Pavel Begunkov889fca72021-02-10 00:03:09 +00003984static int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe067524e2020-03-02 16:32:28 -07003985{
3986 struct io_provide_buf *p = &req->pbuf;
3987 struct io_ring_ctx *ctx = req->ctx;
3988 struct io_buffer *head;
3989 int ret = 0;
Pavel Begunkov45d189c2021-02-10 00:03:07 +00003990 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
Jens Axboe067524e2020-03-02 16:32:28 -07003991
3992 io_ring_submit_lock(ctx, !force_nonblock);
3993
3994 lockdep_assert_held(&ctx->uring_lock);
3995
3996 ret = -ENOENT;
Jens Axboe9e15c3a2021-03-13 12:29:43 -07003997 head = xa_load(&ctx->io_buffers, p->bgid);
Jens Axboe067524e2020-03-02 16:32:28 -07003998 if (head)
3999 ret = __io_remove_buffers(ctx, head, p->bgid, p->nbufs);
Jens Axboe067524e2020-03-02 16:32:28 -07004000 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004001 req_set_fail(req);
Pavel Begunkov31bff9a2020-12-06 22:22:43 +00004002
Pavel Begunkov9fb8cb42021-02-28 22:35:13 +00004003 /* complete before unlock, IOPOLL may need the lock */
4004 __io_req_complete(req, issue_flags, ret, 0);
4005 io_ring_submit_unlock(ctx, !force_nonblock);
Jens Axboe067524e2020-03-02 16:32:28 -07004006 return 0;
4007}
4008
Jens Axboeddf0322d2020-02-23 16:41:33 -07004009static int io_provide_buffers_prep(struct io_kiocb *req,
4010 const struct io_uring_sqe *sqe)
4011{
Pavel Begunkov38134ad2021-04-15 13:07:39 +01004012 unsigned long size, tmp_check;
Jens Axboeddf0322d2020-02-23 16:41:33 -07004013 struct io_provide_buf *p = &req->pbuf;
4014 u64 tmp;
4015
4016 if (sqe->ioprio || sqe->rw_flags)
4017 return -EINVAL;
4018
4019 tmp = READ_ONCE(sqe->fd);
4020 if (!tmp || tmp > USHRT_MAX)
4021 return -E2BIG;
4022 p->nbufs = tmp;
4023 p->addr = READ_ONCE(sqe->addr);
4024 p->len = READ_ONCE(sqe->len);
4025
Pavel Begunkov38134ad2021-04-15 13:07:39 +01004026 if (check_mul_overflow((unsigned long)p->len, (unsigned long)p->nbufs,
4027 &size))
4028 return -EOVERFLOW;
4029 if (check_add_overflow((unsigned long)p->addr, size, &tmp_check))
4030 return -EOVERFLOW;
4031
Pavel Begunkovd81269f2021-03-19 10:21:19 +00004032 size = (unsigned long)p->len * p->nbufs;
4033 if (!access_ok(u64_to_user_ptr(p->addr), size))
Jens Axboeddf0322d2020-02-23 16:41:33 -07004034 return -EFAULT;
4035
4036 p->bgid = READ_ONCE(sqe->buf_group);
4037 tmp = READ_ONCE(sqe->off);
4038 if (tmp > USHRT_MAX)
4039 return -E2BIG;
4040 p->bid = tmp;
4041 return 0;
4042}
4043
4044static int io_add_buffers(struct io_provide_buf *pbuf, struct io_buffer **head)
4045{
4046 struct io_buffer *buf;
4047 u64 addr = pbuf->addr;
4048 int i, bid = pbuf->bid;
4049
4050 for (i = 0; i < pbuf->nbufs; i++) {
4051 buf = kmalloc(sizeof(*buf), GFP_KERNEL);
4052 if (!buf)
4053 break;
4054
4055 buf->addr = addr;
Thadeu Lima de Souza Cascardod1f82802021-05-05 09:47:06 -03004056 buf->len = min_t(__u32, pbuf->len, MAX_RW_COUNT);
Jens Axboeddf0322d2020-02-23 16:41:33 -07004057 buf->bid = bid;
4058 addr += pbuf->len;
4059 bid++;
4060 if (!*head) {
4061 INIT_LIST_HEAD(&buf->list);
4062 *head = buf;
4063 } else {
4064 list_add_tail(&buf->list, &(*head)->list);
4065 }
4066 }
4067
4068 return i ? i : -ENOMEM;
4069}
4070
Pavel Begunkov889fca72021-02-10 00:03:09 +00004071static int io_provide_buffers(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboeddf0322d2020-02-23 16:41:33 -07004072{
4073 struct io_provide_buf *p = &req->pbuf;
4074 struct io_ring_ctx *ctx = req->ctx;
4075 struct io_buffer *head, *list;
4076 int ret = 0;
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004077 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
Jens Axboeddf0322d2020-02-23 16:41:33 -07004078
4079 io_ring_submit_lock(ctx, !force_nonblock);
4080
4081 lockdep_assert_held(&ctx->uring_lock);
4082
Jens Axboe9e15c3a2021-03-13 12:29:43 -07004083 list = head = xa_load(&ctx->io_buffers, p->bgid);
Jens Axboeddf0322d2020-02-23 16:41:33 -07004084
4085 ret = io_add_buffers(p, &head);
Jens Axboe9e15c3a2021-03-13 12:29:43 -07004086 if (ret >= 0 && !list) {
4087 ret = xa_insert(&ctx->io_buffers, p->bgid, head, GFP_KERNEL);
4088 if (ret < 0)
Jens Axboe067524e2020-03-02 16:32:28 -07004089 __io_remove_buffers(ctx, head, p->bgid, -1U);
Jens Axboeddf0322d2020-02-23 16:41:33 -07004090 }
Jens Axboeddf0322d2020-02-23 16:41:33 -07004091 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004092 req_set_fail(req);
Pavel Begunkov9fb8cb42021-02-28 22:35:13 +00004093 /* complete before unlock, IOPOLL may need the lock */
4094 __io_req_complete(req, issue_flags, ret, 0);
4095 io_ring_submit_unlock(ctx, !force_nonblock);
Jens Axboeddf0322d2020-02-23 16:41:33 -07004096 return 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07004097}
4098
Jens Axboe3e4827b2020-01-08 15:18:09 -07004099static int io_epoll_ctl_prep(struct io_kiocb *req,
4100 const struct io_uring_sqe *sqe)
4101{
4102#if defined(CONFIG_EPOLL)
4103 if (sqe->ioprio || sqe->buf_index)
4104 return -EINVAL;
Pavel Begunkov2d74d042021-05-14 12:05:46 +01004105 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
Pavel Begunkov3232dd02020-06-03 18:03:22 +03004106 return -EINVAL;
Jens Axboe3e4827b2020-01-08 15:18:09 -07004107
4108 req->epoll.epfd = READ_ONCE(sqe->fd);
4109 req->epoll.op = READ_ONCE(sqe->len);
4110 req->epoll.fd = READ_ONCE(sqe->off);
4111
4112 if (ep_op_has_event(req->epoll.op)) {
4113 struct epoll_event __user *ev;
4114
4115 ev = u64_to_user_ptr(READ_ONCE(sqe->addr));
4116 if (copy_from_user(&req->epoll.event, ev, sizeof(*ev)))
4117 return -EFAULT;
4118 }
4119
4120 return 0;
4121#else
4122 return -EOPNOTSUPP;
4123#endif
4124}
4125
Pavel Begunkov889fca72021-02-10 00:03:09 +00004126static int io_epoll_ctl(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe3e4827b2020-01-08 15:18:09 -07004127{
4128#if defined(CONFIG_EPOLL)
4129 struct io_epoll *ie = &req->epoll;
4130 int ret;
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004131 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
Jens Axboe3e4827b2020-01-08 15:18:09 -07004132
4133 ret = do_epoll_ctl(ie->epfd, ie->op, ie->fd, &ie->event, force_nonblock);
4134 if (force_nonblock && ret == -EAGAIN)
4135 return -EAGAIN;
4136
4137 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004138 req_set_fail(req);
Pavel Begunkov889fca72021-02-10 00:03:09 +00004139 __io_req_complete(req, issue_flags, ret, 0);
Jens Axboe3e4827b2020-01-08 15:18:09 -07004140 return 0;
4141#else
4142 return -EOPNOTSUPP;
4143#endif
4144}
4145
Jens Axboec1ca7572019-12-25 22:18:28 -07004146static int io_madvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4147{
4148#if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU)
4149 if (sqe->ioprio || sqe->buf_index || sqe->off)
4150 return -EINVAL;
Pavel Begunkov3232dd02020-06-03 18:03:22 +03004151 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4152 return -EINVAL;
Jens Axboec1ca7572019-12-25 22:18:28 -07004153
4154 req->madvise.addr = READ_ONCE(sqe->addr);
4155 req->madvise.len = READ_ONCE(sqe->len);
4156 req->madvise.advice = READ_ONCE(sqe->fadvise_advice);
4157 return 0;
4158#else
4159 return -EOPNOTSUPP;
4160#endif
4161}
4162
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004163static int io_madvise(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboec1ca7572019-12-25 22:18:28 -07004164{
4165#if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU)
4166 struct io_madvise *ma = &req->madvise;
4167 int ret;
4168
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004169 if (issue_flags & IO_URING_F_NONBLOCK)
Jens Axboec1ca7572019-12-25 22:18:28 -07004170 return -EAGAIN;
4171
Minchan Kim0726b012020-10-17 16:14:50 -07004172 ret = do_madvise(current->mm, ma->addr, ma->len, ma->advice);
Jens Axboec1ca7572019-12-25 22:18:28 -07004173 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004174 req_set_fail(req);
Jens Axboee1e16092020-06-22 09:17:17 -06004175 io_req_complete(req, ret);
Jens Axboec1ca7572019-12-25 22:18:28 -07004176 return 0;
4177#else
4178 return -EOPNOTSUPP;
4179#endif
4180}
4181
Jens Axboe4840e412019-12-25 22:03:45 -07004182static int io_fadvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4183{
4184 if (sqe->ioprio || sqe->buf_index || sqe->addr)
4185 return -EINVAL;
Pavel Begunkov3232dd02020-06-03 18:03:22 +03004186 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4187 return -EINVAL;
Jens Axboe4840e412019-12-25 22:03:45 -07004188
4189 req->fadvise.offset = READ_ONCE(sqe->off);
4190 req->fadvise.len = READ_ONCE(sqe->len);
4191 req->fadvise.advice = READ_ONCE(sqe->fadvise_advice);
4192 return 0;
4193}
4194
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004195static int io_fadvise(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe4840e412019-12-25 22:03:45 -07004196{
4197 struct io_fadvise *fa = &req->fadvise;
4198 int ret;
4199
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004200 if (issue_flags & IO_URING_F_NONBLOCK) {
Jens Axboe3e694262020-02-01 09:22:49 -07004201 switch (fa->advice) {
4202 case POSIX_FADV_NORMAL:
4203 case POSIX_FADV_RANDOM:
4204 case POSIX_FADV_SEQUENTIAL:
4205 break;
4206 default:
4207 return -EAGAIN;
4208 }
4209 }
Jens Axboe4840e412019-12-25 22:03:45 -07004210
4211 ret = vfs_fadvise(req->file, fa->offset, fa->len, fa->advice);
4212 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004213 req_set_fail(req);
Pavel Begunkov0bdf3392021-04-11 01:46:29 +01004214 __io_req_complete(req, issue_flags, ret, 0);
Jens Axboe4840e412019-12-25 22:03:45 -07004215 return 0;
4216}
4217
Jens Axboeeddc7ef2019-12-13 21:18:10 -07004218static int io_statx_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4219{
Pavel Begunkov2d74d042021-05-14 12:05:46 +01004220 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
Pavel Begunkov3232dd02020-06-03 18:03:22 +03004221 return -EINVAL;
Jens Axboeeddc7ef2019-12-13 21:18:10 -07004222 if (sqe->ioprio || sqe->buf_index)
4223 return -EINVAL;
Pavel Begunkov9c280f92020-04-08 08:58:46 +03004224 if (req->flags & REQ_F_FIXED_FILE)
Jens Axboecf3040c2020-02-06 21:31:40 -07004225 return -EBADF;
Jens Axboeeddc7ef2019-12-13 21:18:10 -07004226
Bijan Mottahedeh1d9e1282020-05-22 21:31:16 -07004227 req->statx.dfd = READ_ONCE(sqe->fd);
4228 req->statx.mask = READ_ONCE(sqe->len);
Bijan Mottahedehe62753e2020-05-22 21:31:18 -07004229 req->statx.filename = u64_to_user_ptr(READ_ONCE(sqe->addr));
Bijan Mottahedeh1d9e1282020-05-22 21:31:16 -07004230 req->statx.buffer = u64_to_user_ptr(READ_ONCE(sqe->addr2));
4231 req->statx.flags = READ_ONCE(sqe->statx_flags);
Jens Axboeeddc7ef2019-12-13 21:18:10 -07004232
Jens Axboeeddc7ef2019-12-13 21:18:10 -07004233 return 0;
4234}
4235
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004236static int io_statx(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboeeddc7ef2019-12-13 21:18:10 -07004237{
Bijan Mottahedeh1d9e1282020-05-22 21:31:16 -07004238 struct io_statx *ctx = &req->statx;
Jens Axboeeddc7ef2019-12-13 21:18:10 -07004239 int ret;
4240
Pavel Begunkov59d70012021-03-22 01:58:30 +00004241 if (issue_flags & IO_URING_F_NONBLOCK)
Jens Axboeeddc7ef2019-12-13 21:18:10 -07004242 return -EAGAIN;
4243
Bijan Mottahedehe62753e2020-05-22 21:31:18 -07004244 ret = do_statx(ctx->dfd, ctx->filename, ctx->flags, ctx->mask,
4245 ctx->buffer);
Jens Axboeeddc7ef2019-12-13 21:18:10 -07004246
Jens Axboeeddc7ef2019-12-13 21:18:10 -07004247 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004248 req_set_fail(req);
Jens Axboee1e16092020-06-22 09:17:17 -06004249 io_req_complete(req, ret);
Jens Axboeeddc7ef2019-12-13 21:18:10 -07004250 return 0;
4251}
4252
Jens Axboeb5dba592019-12-11 14:02:38 -07004253static int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4254{
Jens Axboe14587a462020-09-05 11:36:08 -06004255 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
Pavel Begunkov3232dd02020-06-03 18:03:22 +03004256 return -EINVAL;
Jens Axboeb5dba592019-12-11 14:02:38 -07004257 if (sqe->ioprio || sqe->off || sqe->addr || sqe->len ||
4258 sqe->rw_flags || sqe->buf_index)
4259 return -EINVAL;
Pavel Begunkov9c280f92020-04-08 08:58:46 +03004260 if (req->flags & REQ_F_FIXED_FILE)
Jens Axboecf3040c2020-02-06 21:31:40 -07004261 return -EBADF;
Jens Axboeb5dba592019-12-11 14:02:38 -07004262
4263 req->close.fd = READ_ONCE(sqe->fd);
Jens Axboeb5dba592019-12-11 14:02:38 -07004264 return 0;
4265}
4266
Pavel Begunkov889fca72021-02-10 00:03:09 +00004267static int io_close(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboeb5dba592019-12-11 14:02:38 -07004268{
Jens Axboe9eac1902021-01-19 15:50:37 -07004269 struct files_struct *files = current->files;
Pavel Begunkov3af73b22020-06-08 21:08:17 +03004270 struct io_close *close = &req->close;
Jens Axboe9eac1902021-01-19 15:50:37 -07004271 struct fdtable *fdt;
Pavel Begunkova1fde922021-04-11 01:46:28 +01004272 struct file *file = NULL;
4273 int ret = -EBADF;
Jens Axboeb5dba592019-12-11 14:02:38 -07004274
Jens Axboe9eac1902021-01-19 15:50:37 -07004275 spin_lock(&files->file_lock);
4276 fdt = files_fdtable(files);
4277 if (close->fd >= fdt->max_fds) {
4278 spin_unlock(&files->file_lock);
4279 goto err;
4280 }
4281 file = fdt->fd[close->fd];
Pavel Begunkova1fde922021-04-11 01:46:28 +01004282 if (!file || file->f_op == &io_uring_fops) {
Jens Axboe9eac1902021-01-19 15:50:37 -07004283 spin_unlock(&files->file_lock);
4284 file = NULL;
4285 goto err;
Pavel Begunkov3af73b22020-06-08 21:08:17 +03004286 }
Jens Axboeb5dba592019-12-11 14:02:38 -07004287
4288 /* if the file has a flush method, be safe and punt to async */
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004289 if (file->f_op->flush && (issue_flags & IO_URING_F_NONBLOCK)) {
Jens Axboe9eac1902021-01-19 15:50:37 -07004290 spin_unlock(&files->file_lock);
Pavel Begunkov0bf0eef2020-05-26 20:34:06 +03004291 return -EAGAIN;
Pavel Begunkova2100672020-03-02 23:45:16 +03004292 }
Jens Axboeb5dba592019-12-11 14:02:38 -07004293
Jens Axboe9eac1902021-01-19 15:50:37 -07004294 ret = __close_fd_get_file(close->fd, &file);
4295 spin_unlock(&files->file_lock);
4296 if (ret < 0) {
4297 if (ret == -ENOENT)
4298 ret = -EBADF;
4299 goto err;
4300 }
4301
Pavel Begunkov3af73b22020-06-08 21:08:17 +03004302 /* No ->flush() or already async, safely close from here */
Jens Axboe9eac1902021-01-19 15:50:37 -07004303 ret = filp_close(file, current->files);
4304err:
Pavel Begunkov3af73b22020-06-08 21:08:17 +03004305 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004306 req_set_fail(req);
Jens Axboe9eac1902021-01-19 15:50:37 -07004307 if (file)
4308 fput(file);
Pavel Begunkov889fca72021-02-10 00:03:09 +00004309 __io_req_complete(req, issue_flags, ret, 0);
Jens Axboe1a417f42020-01-31 17:16:48 -07004310 return 0;
Jens Axboeb5dba592019-12-11 14:02:38 -07004311}
4312
Pavel Begunkov1155c762021-02-18 18:29:38 +00004313static int io_sfr_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboe5d17b4a2019-04-09 14:56:44 -06004314{
4315 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe5d17b4a2019-04-09 14:56:44 -06004316
Jens Axboe5d17b4a2019-04-09 14:56:44 -06004317 if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
4318 return -EINVAL;
4319 if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index))
4320 return -EINVAL;
4321
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004322 req->sync.off = READ_ONCE(sqe->off);
4323 req->sync.len = READ_ONCE(sqe->len);
4324 req->sync.flags = READ_ONCE(sqe->sync_range_flags);
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004325 return 0;
4326}
4327
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004328static int io_sync_file_range(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004329{
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004330 int ret;
4331
Pavel Begunkovac45abc2020-06-08 21:08:18 +03004332 /* sync_file_range always requires a blocking context */
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004333 if (issue_flags & IO_URING_F_NONBLOCK)
Pavel Begunkovac45abc2020-06-08 21:08:18 +03004334 return -EAGAIN;
4335
Jens Axboe9adbd452019-12-20 08:45:55 -07004336 ret = sync_file_range(req->file, req->sync.off, req->sync.len,
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004337 req->sync.flags);
4338 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004339 req_set_fail(req);
Jens Axboee1e16092020-06-22 09:17:17 -06004340 io_req_complete(req, ret);
Jens Axboe5d17b4a2019-04-09 14:56:44 -06004341 return 0;
4342}
4343
YueHaibing469956e2020-03-04 15:53:52 +08004344#if defined(CONFIG_NET)
Pavel Begunkov02d27d82020-02-28 10:36:36 +03004345static int io_setup_async_msg(struct io_kiocb *req,
4346 struct io_async_msghdr *kmsg)
4347{
Jens Axboee8c2bc12020-08-15 18:44:09 -07004348 struct io_async_msghdr *async_msg = req->async_data;
4349
4350 if (async_msg)
Pavel Begunkov02d27d82020-02-28 10:36:36 +03004351 return -EAGAIN;
Jens Axboee8c2bc12020-08-15 18:44:09 -07004352 if (io_alloc_async_data(req)) {
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004353 kfree(kmsg->free_iov);
Pavel Begunkov02d27d82020-02-28 10:36:36 +03004354 return -ENOMEM;
4355 }
Jens Axboee8c2bc12020-08-15 18:44:09 -07004356 async_msg = req->async_data;
Pavel Begunkov02d27d82020-02-28 10:36:36 +03004357 req->flags |= REQ_F_NEED_CLEANUP;
Jens Axboee8c2bc12020-08-15 18:44:09 -07004358 memcpy(async_msg, kmsg, sizeof(*kmsg));
Pavel Begunkov2a780802021-02-05 00:57:58 +00004359 async_msg->msg.msg_name = &async_msg->addr;
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004360 /* if were using fast_iov, set it to the new one */
4361 if (!async_msg->free_iov)
4362 async_msg->msg.msg_iter.iov = async_msg->fast_iov;
4363
Pavel Begunkov02d27d82020-02-28 10:36:36 +03004364 return -EAGAIN;
4365}
4366
Pavel Begunkov2ae523e2020-07-12 20:41:06 +03004367static int io_sendmsg_copy_hdr(struct io_kiocb *req,
4368 struct io_async_msghdr *iomsg)
4369{
Pavel Begunkov2ae523e2020-07-12 20:41:06 +03004370 iomsg->msg.msg_name = &iomsg->addr;
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004371 iomsg->free_iov = iomsg->fast_iov;
Pavel Begunkov2ae523e2020-07-12 20:41:06 +03004372 return sendmsg_copy_msghdr(&iomsg->msg, req->sr_msg.umsg,
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004373 req->sr_msg.msg_flags, &iomsg->free_iov);
Pavel Begunkov2ae523e2020-07-12 20:41:06 +03004374}
4375
Pavel Begunkov93642ef2021-02-18 18:29:44 +00004376static int io_sendmsg_prep_async(struct io_kiocb *req)
4377{
4378 int ret;
4379
Pavel Begunkov93642ef2021-02-18 18:29:44 +00004380 ret = io_sendmsg_copy_hdr(req, req->async_data);
4381 if (!ret)
4382 req->flags |= REQ_F_NEED_CLEANUP;
4383 return ret;
4384}
4385
Jens Axboe3529d8c2019-12-19 18:24:38 -07004386static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboeaa1fa282019-04-19 13:38:09 -06004387{
Jens Axboee47293f2019-12-20 08:58:21 -07004388 struct io_sr_msg *sr = &req->sr_msg;
Jens Axboe03b12302019-12-02 18:50:25 -07004389
Pavel Begunkovd2b6f482020-06-03 18:03:25 +03004390 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4391 return -EINVAL;
4392
Pavel Begunkov270a5942020-07-12 20:41:04 +03004393 sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
Jens Axboefddafac2020-01-04 20:19:44 -07004394 sr->len = READ_ONCE(sqe->len);
Pavel Begunkov04411802021-04-01 15:44:00 +01004395 sr->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL;
4396 if (sr->msg_flags & MSG_DONTWAIT)
4397 req->flags |= REQ_F_NOWAIT;
Jens Axboe3529d8c2019-12-19 18:24:38 -07004398
Jens Axboed8768362020-02-27 14:17:49 -07004399#ifdef CONFIG_COMPAT
4400 if (req->ctx->compat)
4401 sr->msg_flags |= MSG_CMSG_COMPAT;
4402#endif
Pavel Begunkov93642ef2021-02-18 18:29:44 +00004403 return 0;
Jens Axboe03b12302019-12-02 18:50:25 -07004404}
4405
Pavel Begunkov889fca72021-02-10 00:03:09 +00004406static int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe03b12302019-12-02 18:50:25 -07004407{
Pavel Begunkov6b754c82020-07-16 23:28:00 +03004408 struct io_async_msghdr iomsg, *kmsg;
Jens Axboe03b12302019-12-02 18:50:25 -07004409 struct socket *sock;
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004410 unsigned flags;
Stefan Metzmacher00312752021-03-20 20:33:36 +01004411 int min_ret = 0;
Jens Axboe03b12302019-12-02 18:50:25 -07004412 int ret;
4413
Florent Revestdba4a922020-12-04 12:36:04 +01004414 sock = sock_from_file(req->file);
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004415 if (unlikely(!sock))
Florent Revestdba4a922020-12-04 12:36:04 +01004416 return -ENOTSOCK;
Jens Axboe03b12302019-12-02 18:50:25 -07004417
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004418 kmsg = req->async_data;
4419 if (!kmsg) {
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004420 ret = io_sendmsg_copy_hdr(req, &iomsg);
Jens Axboefddafac2020-01-04 20:19:44 -07004421 if (ret)
4422 return ret;
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004423 kmsg = &iomsg;
Jens Axboefddafac2020-01-04 20:19:44 -07004424 }
4425
Pavel Begunkov04411802021-04-01 15:44:00 +01004426 flags = req->sr_msg.msg_flags;
4427 if (issue_flags & IO_URING_F_NONBLOCK)
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004428 flags |= MSG_DONTWAIT;
Stefan Metzmacher00312752021-03-20 20:33:36 +01004429 if (flags & MSG_WAITALL)
4430 min_ret = iov_iter_count(&kmsg->msg.msg_iter);
4431
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004432 ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags);
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004433 if ((issue_flags & IO_URING_F_NONBLOCK) && ret == -EAGAIN)
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004434 return io_setup_async_msg(req, kmsg);
4435 if (ret == -ERESTARTSYS)
4436 ret = -EINTR;
4437
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004438 /* fast path, check for non-NULL to avoid function call */
4439 if (kmsg->free_iov)
4440 kfree(kmsg->free_iov);
Jens Axboe03b12302019-12-02 18:50:25 -07004441 req->flags &= ~REQ_F_NEED_CLEANUP;
Stefan Metzmacher00312752021-03-20 20:33:36 +01004442 if (ret < min_ret)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004443 req_set_fail(req);
Pavel Begunkov889fca72021-02-10 00:03:09 +00004444 __io_req_complete(req, issue_flags, ret, 0);
Jens Axboefddafac2020-01-04 20:19:44 -07004445 return 0;
Jens Axboefddafac2020-01-04 20:19:44 -07004446}
4447
Pavel Begunkov889fca72021-02-10 00:03:09 +00004448static int io_send(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe03b12302019-12-02 18:50:25 -07004449{
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004450 struct io_sr_msg *sr = &req->sr_msg;
4451 struct msghdr msg;
4452 struct iovec iov;
Jens Axboe03b12302019-12-02 18:50:25 -07004453 struct socket *sock;
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004454 unsigned flags;
Stefan Metzmacher00312752021-03-20 20:33:36 +01004455 int min_ret = 0;
Jens Axboe03b12302019-12-02 18:50:25 -07004456 int ret;
4457
Florent Revestdba4a922020-12-04 12:36:04 +01004458 sock = sock_from_file(req->file);
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004459 if (unlikely(!sock))
Florent Revestdba4a922020-12-04 12:36:04 +01004460 return -ENOTSOCK;
Jens Axboe03b12302019-12-02 18:50:25 -07004461
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004462 ret = import_single_range(WRITE, sr->buf, sr->len, &iov, &msg.msg_iter);
4463 if (unlikely(ret))
Zheng Bin14db8412020-09-09 20:12:37 +08004464 return ret;
Jens Axboe03b12302019-12-02 18:50:25 -07004465
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004466 msg.msg_name = NULL;
4467 msg.msg_control = NULL;
4468 msg.msg_controllen = 0;
4469 msg.msg_namelen = 0;
Jens Axboe03b12302019-12-02 18:50:25 -07004470
Pavel Begunkov04411802021-04-01 15:44:00 +01004471 flags = req->sr_msg.msg_flags;
4472 if (issue_flags & IO_URING_F_NONBLOCK)
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004473 flags |= MSG_DONTWAIT;
Stefan Metzmacher00312752021-03-20 20:33:36 +01004474 if (flags & MSG_WAITALL)
4475 min_ret = iov_iter_count(&msg.msg_iter);
4476
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004477 msg.msg_flags = flags;
4478 ret = sock_sendmsg(sock, &msg);
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004479 if ((issue_flags & IO_URING_F_NONBLOCK) && ret == -EAGAIN)
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004480 return -EAGAIN;
4481 if (ret == -ERESTARTSYS)
4482 ret = -EINTR;
Jens Axboe03b12302019-12-02 18:50:25 -07004483
Stefan Metzmacher00312752021-03-20 20:33:36 +01004484 if (ret < min_ret)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004485 req_set_fail(req);
Pavel Begunkov889fca72021-02-10 00:03:09 +00004486 __io_req_complete(req, issue_flags, ret, 0);
Jens Axboe03b12302019-12-02 18:50:25 -07004487 return 0;
Jens Axboe03b12302019-12-02 18:50:25 -07004488}
4489
Pavel Begunkov1400e692020-07-12 20:41:05 +03004490static int __io_recvmsg_copy_hdr(struct io_kiocb *req,
4491 struct io_async_msghdr *iomsg)
Jens Axboe52de1fe2020-02-27 10:15:42 -07004492{
4493 struct io_sr_msg *sr = &req->sr_msg;
4494 struct iovec __user *uiov;
4495 size_t iov_len;
4496 int ret;
4497
Pavel Begunkov1400e692020-07-12 20:41:05 +03004498 ret = __copy_msghdr_from_user(&iomsg->msg, sr->umsg,
4499 &iomsg->uaddr, &uiov, &iov_len);
Jens Axboe52de1fe2020-02-27 10:15:42 -07004500 if (ret)
4501 return ret;
4502
4503 if (req->flags & REQ_F_BUFFER_SELECT) {
4504 if (iov_len > 1)
4505 return -EINVAL;
Pavel Begunkov5476dfe2021-02-05 00:57:59 +00004506 if (copy_from_user(iomsg->fast_iov, uiov, sizeof(*uiov)))
Jens Axboe52de1fe2020-02-27 10:15:42 -07004507 return -EFAULT;
Pavel Begunkov5476dfe2021-02-05 00:57:59 +00004508 sr->len = iomsg->fast_iov[0].iov_len;
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004509 iomsg->free_iov = NULL;
Jens Axboe52de1fe2020-02-27 10:15:42 -07004510 } else {
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004511 iomsg->free_iov = iomsg->fast_iov;
Christoph Hellwig89cd35c2020-09-25 06:51:41 +02004512 ret = __import_iovec(READ, uiov, iov_len, UIO_FASTIOV,
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004513 &iomsg->free_iov, &iomsg->msg.msg_iter,
Christoph Hellwig89cd35c2020-09-25 06:51:41 +02004514 false);
Jens Axboe52de1fe2020-02-27 10:15:42 -07004515 if (ret > 0)
4516 ret = 0;
4517 }
4518
4519 return ret;
4520}
4521
4522#ifdef CONFIG_COMPAT
4523static int __io_compat_recvmsg_copy_hdr(struct io_kiocb *req,
Pavel Begunkov1400e692020-07-12 20:41:05 +03004524 struct io_async_msghdr *iomsg)
Jens Axboe52de1fe2020-02-27 10:15:42 -07004525{
Jens Axboe52de1fe2020-02-27 10:15:42 -07004526 struct io_sr_msg *sr = &req->sr_msg;
4527 struct compat_iovec __user *uiov;
4528 compat_uptr_t ptr;
4529 compat_size_t len;
4530 int ret;
4531
Pavel Begunkov4af34172021-04-11 01:46:30 +01004532 ret = __get_compat_msghdr(&iomsg->msg, sr->umsg_compat, &iomsg->uaddr,
4533 &ptr, &len);
Jens Axboe52de1fe2020-02-27 10:15:42 -07004534 if (ret)
4535 return ret;
4536
4537 uiov = compat_ptr(ptr);
4538 if (req->flags & REQ_F_BUFFER_SELECT) {
4539 compat_ssize_t clen;
4540
4541 if (len > 1)
4542 return -EINVAL;
4543 if (!access_ok(uiov, sizeof(*uiov)))
4544 return -EFAULT;
4545 if (__get_user(clen, &uiov->iov_len))
4546 return -EFAULT;
4547 if (clen < 0)
4548 return -EINVAL;
Pavel Begunkov2d280bc2020-11-29 18:33:32 +00004549 sr->len = clen;
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004550 iomsg->free_iov = NULL;
Jens Axboe52de1fe2020-02-27 10:15:42 -07004551 } else {
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004552 iomsg->free_iov = iomsg->fast_iov;
Christoph Hellwig89cd35c2020-09-25 06:51:41 +02004553 ret = __import_iovec(READ, (struct iovec __user *)uiov, len,
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004554 UIO_FASTIOV, &iomsg->free_iov,
Christoph Hellwig89cd35c2020-09-25 06:51:41 +02004555 &iomsg->msg.msg_iter, true);
Jens Axboe52de1fe2020-02-27 10:15:42 -07004556 if (ret < 0)
4557 return ret;
4558 }
4559
4560 return 0;
4561}
Jens Axboe03b12302019-12-02 18:50:25 -07004562#endif
Jens Axboe52de1fe2020-02-27 10:15:42 -07004563
Pavel Begunkov1400e692020-07-12 20:41:05 +03004564static int io_recvmsg_copy_hdr(struct io_kiocb *req,
4565 struct io_async_msghdr *iomsg)
Jens Axboe52de1fe2020-02-27 10:15:42 -07004566{
Pavel Begunkov1400e692020-07-12 20:41:05 +03004567 iomsg->msg.msg_name = &iomsg->addr;
Jens Axboe52de1fe2020-02-27 10:15:42 -07004568
4569#ifdef CONFIG_COMPAT
4570 if (req->ctx->compat)
Pavel Begunkov1400e692020-07-12 20:41:05 +03004571 return __io_compat_recvmsg_copy_hdr(req, iomsg);
Jens Axboe52de1fe2020-02-27 10:15:42 -07004572#endif
4573
Pavel Begunkov1400e692020-07-12 20:41:05 +03004574 return __io_recvmsg_copy_hdr(req, iomsg);
Jens Axboe52de1fe2020-02-27 10:15:42 -07004575}
4576
Jens Axboebcda7ba2020-02-23 16:42:51 -07004577static struct io_buffer *io_recv_buffer_select(struct io_kiocb *req,
Pavel Begunkov7fbb1b52020-07-16 23:28:05 +03004578 bool needs_lock)
Jens Axboebcda7ba2020-02-23 16:42:51 -07004579{
4580 struct io_sr_msg *sr = &req->sr_msg;
4581 struct io_buffer *kbuf;
4582
Jens Axboebcda7ba2020-02-23 16:42:51 -07004583 kbuf = io_buffer_select(req, &sr->len, sr->bgid, sr->kbuf, needs_lock);
4584 if (IS_ERR(kbuf))
4585 return kbuf;
4586
4587 sr->kbuf = kbuf;
4588 req->flags |= REQ_F_BUFFER_SELECTED;
Jens Axboebcda7ba2020-02-23 16:42:51 -07004589 return kbuf;
Jens Axboe03b12302019-12-02 18:50:25 -07004590}
4591
Pavel Begunkov7fbb1b52020-07-16 23:28:05 +03004592static inline unsigned int io_put_recv_kbuf(struct io_kiocb *req)
4593{
4594 return io_put_kbuf(req, req->sr_msg.kbuf);
4595}
4596
Pavel Begunkov93642ef2021-02-18 18:29:44 +00004597static int io_recvmsg_prep_async(struct io_kiocb *req)
Jens Axboe03b12302019-12-02 18:50:25 -07004598{
Pavel Begunkov99bc4c32020-02-07 22:04:45 +03004599 int ret;
Jens Axboe06b76d42019-12-19 14:44:26 -07004600
Pavel Begunkov93642ef2021-02-18 18:29:44 +00004601 ret = io_recvmsg_copy_hdr(req, req->async_data);
4602 if (!ret)
4603 req->flags |= REQ_F_NEED_CLEANUP;
4604 return ret;
4605}
4606
4607static int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
4608{
4609 struct io_sr_msg *sr = &req->sr_msg;
4610
Pavel Begunkovd2b6f482020-06-03 18:03:25 +03004611 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
4612 return -EINVAL;
4613
Pavel Begunkov270a5942020-07-12 20:41:04 +03004614 sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
Jens Axboe0b7b21e2020-01-31 08:34:59 -07004615 sr->len = READ_ONCE(sqe->len);
Jens Axboebcda7ba2020-02-23 16:42:51 -07004616 sr->bgid = READ_ONCE(sqe->buf_group);
Pavel Begunkov04411802021-04-01 15:44:00 +01004617 sr->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL;
4618 if (sr->msg_flags & MSG_DONTWAIT)
4619 req->flags |= REQ_F_NOWAIT;
Jens Axboe3529d8c2019-12-19 18:24:38 -07004620
Jens Axboed8768362020-02-27 14:17:49 -07004621#ifdef CONFIG_COMPAT
4622 if (req->ctx->compat)
4623 sr->msg_flags |= MSG_CMSG_COMPAT;
4624#endif
Pavel Begunkov93642ef2021-02-18 18:29:44 +00004625 return 0;
Jens Axboe03b12302019-12-02 18:50:25 -07004626}
4627
Pavel Begunkov889fca72021-02-10 00:03:09 +00004628static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe03b12302019-12-02 18:50:25 -07004629{
Pavel Begunkov6b754c82020-07-16 23:28:00 +03004630 struct io_async_msghdr iomsg, *kmsg;
Jens Axboe0fa03c62019-04-19 13:34:07 -06004631 struct socket *sock;
Pavel Begunkov7fbb1b52020-07-16 23:28:05 +03004632 struct io_buffer *kbuf;
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004633 unsigned flags;
Stefan Metzmacher00312752021-03-20 20:33:36 +01004634 int min_ret = 0;
Jens Axboe52de1fe2020-02-27 10:15:42 -07004635 int ret, cflags = 0;
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004636 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
Jens Axboe0fa03c62019-04-19 13:34:07 -06004637
Florent Revestdba4a922020-12-04 12:36:04 +01004638 sock = sock_from_file(req->file);
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004639 if (unlikely(!sock))
Florent Revestdba4a922020-12-04 12:36:04 +01004640 return -ENOTSOCK;
Jens Axboe0fa03c62019-04-19 13:34:07 -06004641
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004642 kmsg = req->async_data;
4643 if (!kmsg) {
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004644 ret = io_recvmsg_copy_hdr(req, &iomsg);
4645 if (ret)
Pavel Begunkov681fda82020-07-15 22:20:45 +03004646 return ret;
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004647 kmsg = &iomsg;
Jens Axboe0fa03c62019-04-19 13:34:07 -06004648 }
4649
Pavel Begunkovbc02ef32020-07-16 23:28:03 +03004650 if (req->flags & REQ_F_BUFFER_SELECT) {
Pavel Begunkov7fbb1b52020-07-16 23:28:05 +03004651 kbuf = io_recv_buffer_select(req, !force_nonblock);
Pavel Begunkovbc02ef32020-07-16 23:28:03 +03004652 if (IS_ERR(kbuf))
4653 return PTR_ERR(kbuf);
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004654 kmsg->fast_iov[0].iov_base = u64_to_user_ptr(kbuf->addr);
Pavel Begunkov5476dfe2021-02-05 00:57:59 +00004655 kmsg->fast_iov[0].iov_len = req->sr_msg.len;
4656 iov_iter_init(&kmsg->msg.msg_iter, READ, kmsg->fast_iov,
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004657 1, req->sr_msg.len);
4658 }
4659
Pavel Begunkov04411802021-04-01 15:44:00 +01004660 flags = req->sr_msg.msg_flags;
4661 if (force_nonblock)
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004662 flags |= MSG_DONTWAIT;
Stefan Metzmacher00312752021-03-20 20:33:36 +01004663 if (flags & MSG_WAITALL)
4664 min_ret = iov_iter_count(&kmsg->msg.msg_iter);
4665
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004666 ret = __sys_recvmsg_sock(sock, &kmsg->msg, req->sr_msg.umsg,
4667 kmsg->uaddr, flags);
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03004668 if (force_nonblock && ret == -EAGAIN)
4669 return io_setup_async_msg(req, kmsg);
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004670 if (ret == -ERESTARTSYS)
4671 ret = -EINTR;
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03004672
Pavel Begunkov7fbb1b52020-07-16 23:28:05 +03004673 if (req->flags & REQ_F_BUFFER_SELECTED)
4674 cflags = io_put_recv_kbuf(req);
Pavel Begunkov257e84a2021-02-05 00:58:00 +00004675 /* fast path, check for non-NULL to avoid function call */
4676 if (kmsg->free_iov)
4677 kfree(kmsg->free_iov);
Pavel Begunkov99bc4c32020-02-07 22:04:45 +03004678 req->flags &= ~REQ_F_NEED_CLEANUP;
Stefan Metzmacher00312752021-03-20 20:33:36 +01004679 if (ret < min_ret || ((flags & MSG_WAITALL) && (kmsg->msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))))
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004680 req_set_fail(req);
Pavel Begunkov889fca72021-02-10 00:03:09 +00004681 __io_req_complete(req, issue_flags, ret, cflags);
Jens Axboe0fa03c62019-04-19 13:34:07 -06004682 return 0;
Jens Axboe0fa03c62019-04-19 13:34:07 -06004683}
4684
Pavel Begunkov889fca72021-02-10 00:03:09 +00004685static int io_recv(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboefddafac2020-01-04 20:19:44 -07004686{
Pavel Begunkov6b754c82020-07-16 23:28:00 +03004687 struct io_buffer *kbuf;
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004688 struct io_sr_msg *sr = &req->sr_msg;
4689 struct msghdr msg;
4690 void __user *buf = sr->buf;
Jens Axboefddafac2020-01-04 20:19:44 -07004691 struct socket *sock;
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004692 struct iovec iov;
4693 unsigned flags;
Stefan Metzmacher00312752021-03-20 20:33:36 +01004694 int min_ret = 0;
Jens Axboebcda7ba2020-02-23 16:42:51 -07004695 int ret, cflags = 0;
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004696 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
Jens Axboefddafac2020-01-04 20:19:44 -07004697
Florent Revestdba4a922020-12-04 12:36:04 +01004698 sock = sock_from_file(req->file);
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004699 if (unlikely(!sock))
Florent Revestdba4a922020-12-04 12:36:04 +01004700 return -ENOTSOCK;
Jens Axboefddafac2020-01-04 20:19:44 -07004701
Pavel Begunkovbc02ef32020-07-16 23:28:03 +03004702 if (req->flags & REQ_F_BUFFER_SELECT) {
Pavel Begunkov7fbb1b52020-07-16 23:28:05 +03004703 kbuf = io_recv_buffer_select(req, !force_nonblock);
Jens Axboebcda7ba2020-02-23 16:42:51 -07004704 if (IS_ERR(kbuf))
4705 return PTR_ERR(kbuf);
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004706 buf = u64_to_user_ptr(kbuf->addr);
Jens Axboefddafac2020-01-04 20:19:44 -07004707 }
4708
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004709 ret = import_single_range(READ, buf, sr->len, &iov, &msg.msg_iter);
Pavel Begunkov14c32ee2020-07-16 23:28:01 +03004710 if (unlikely(ret))
4711 goto out_free;
Jens Axboefddafac2020-01-04 20:19:44 -07004712
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004713 msg.msg_name = NULL;
4714 msg.msg_control = NULL;
4715 msg.msg_controllen = 0;
4716 msg.msg_namelen = 0;
4717 msg.msg_iocb = NULL;
4718 msg.msg_flags = 0;
4719
Pavel Begunkov04411802021-04-01 15:44:00 +01004720 flags = req->sr_msg.msg_flags;
4721 if (force_nonblock)
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004722 flags |= MSG_DONTWAIT;
Stefan Metzmacher00312752021-03-20 20:33:36 +01004723 if (flags & MSG_WAITALL)
4724 min_ret = iov_iter_count(&msg.msg_iter);
4725
Pavel Begunkov7a7cacb2020-07-16 23:27:59 +03004726 ret = sock_recvmsg(sock, &msg, flags);
4727 if (force_nonblock && ret == -EAGAIN)
4728 return -EAGAIN;
4729 if (ret == -ERESTARTSYS)
4730 ret = -EINTR;
Pavel Begunkov14c32ee2020-07-16 23:28:01 +03004731out_free:
Pavel Begunkov7fbb1b52020-07-16 23:28:05 +03004732 if (req->flags & REQ_F_BUFFER_SELECTED)
4733 cflags = io_put_recv_kbuf(req);
Stefan Metzmacher00312752021-03-20 20:33:36 +01004734 if (ret < min_ret || ((flags & MSG_WAITALL) && (msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))))
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004735 req_set_fail(req);
Pavel Begunkov889fca72021-02-10 00:03:09 +00004736 __io_req_complete(req, issue_flags, ret, cflags);
Jens Axboefddafac2020-01-04 20:19:44 -07004737 return 0;
Jens Axboefddafac2020-01-04 20:19:44 -07004738}
4739
Jens Axboe3529d8c2019-12-19 18:24:38 -07004740static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboe17f2fe32019-10-17 14:42:58 -06004741{
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004742 struct io_accept *accept = &req->accept;
4743
Jens Axboe14587a462020-09-05 11:36:08 -06004744 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
Jens Axboe17f2fe32019-10-17 14:42:58 -06004745 return -EINVAL;
Hrvoje Zeba8042d6c2019-11-25 14:40:22 -05004746 if (sqe->ioprio || sqe->len || sqe->buf_index)
Jens Axboe17f2fe32019-10-17 14:42:58 -06004747 return -EINVAL;
4748
Jens Axboed55e5f52019-12-11 16:12:15 -07004749 accept->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));
4750 accept->addr_len = u64_to_user_ptr(READ_ONCE(sqe->addr2));
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004751 accept->flags = READ_ONCE(sqe->accept_flags);
Jens Axboe09952e32020-03-19 20:16:56 -06004752 accept->nofile = rlimit(RLIMIT_NOFILE);
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004753 return 0;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004754}
Jens Axboe17f2fe32019-10-17 14:42:58 -06004755
Pavel Begunkov889fca72021-02-10 00:03:09 +00004756static int io_accept(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004757{
4758 struct io_accept *accept = &req->accept;
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004759 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
Pavel Begunkovac45abc2020-06-08 21:08:18 +03004760 unsigned int file_flags = force_nonblock ? O_NONBLOCK : 0;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004761 int ret;
4762
Jiufei Xuee697dee2020-06-10 13:41:59 +08004763 if (req->file->f_flags & O_NONBLOCK)
4764 req->flags |= REQ_F_NOWAIT;
4765
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004766 ret = __sys_accept4_file(req->file, file_flags, accept->addr,
Jens Axboe09952e32020-03-19 20:16:56 -06004767 accept->addr_len, accept->flags,
4768 accept->nofile);
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004769 if (ret == -EAGAIN && force_nonblock)
Jens Axboe17f2fe32019-10-17 14:42:58 -06004770 return -EAGAIN;
Pavel Begunkovac45abc2020-06-08 21:08:18 +03004771 if (ret < 0) {
4772 if (ret == -ERESTARTSYS)
4773 ret = -EINTR;
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004774 req_set_fail(req);
Pavel Begunkovac45abc2020-06-08 21:08:18 +03004775 }
Pavel Begunkov889fca72021-02-10 00:03:09 +00004776 __io_req_complete(req, issue_flags, ret, 0);
Jens Axboe17f2fe32019-10-17 14:42:58 -06004777 return 0;
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07004778}
4779
Pavel Begunkov93642ef2021-02-18 18:29:44 +00004780static int io_connect_prep_async(struct io_kiocb *req)
4781{
4782 struct io_async_connect *io = req->async_data;
4783 struct io_connect *conn = &req->connect;
4784
4785 return move_addr_to_kernel(conn->addr, conn->addr_len, &io->address);
4786}
4787
Jens Axboe3529d8c2019-12-19 18:24:38 -07004788static int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboef499a022019-12-02 16:28:46 -07004789{
Jens Axboe3529d8c2019-12-19 18:24:38 -07004790 struct io_connect *conn = &req->connect;
Jens Axboef499a022019-12-02 16:28:46 -07004791
Jens Axboe14587a462020-09-05 11:36:08 -06004792 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
Jens Axboe3fbb51c2019-12-20 08:51:52 -07004793 return -EINVAL;
4794 if (sqe->ioprio || sqe->len || sqe->buf_index || sqe->rw_flags)
4795 return -EINVAL;
4796
Jens Axboe3529d8c2019-12-19 18:24:38 -07004797 conn->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));
4798 conn->addr_len = READ_ONCE(sqe->addr2);
Pavel Begunkov93642ef2021-02-18 18:29:44 +00004799 return 0;
Jens Axboef499a022019-12-02 16:28:46 -07004800}
4801
Pavel Begunkov889fca72021-02-10 00:03:09 +00004802static int io_connect(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboef8e85cf2019-11-23 14:24:24 -07004803{
Jens Axboee8c2bc12020-08-15 18:44:09 -07004804 struct io_async_connect __io, *io;
Jens Axboef8e85cf2019-11-23 14:24:24 -07004805 unsigned file_flags;
Jens Axboe3fbb51c2019-12-20 08:51:52 -07004806 int ret;
Pavel Begunkov45d189c2021-02-10 00:03:07 +00004807 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
Jens Axboef8e85cf2019-11-23 14:24:24 -07004808
Jens Axboee8c2bc12020-08-15 18:44:09 -07004809 if (req->async_data) {
4810 io = req->async_data;
Jens Axboef499a022019-12-02 16:28:46 -07004811 } else {
Jens Axboe3529d8c2019-12-19 18:24:38 -07004812 ret = move_addr_to_kernel(req->connect.addr,
4813 req->connect.addr_len,
Jens Axboee8c2bc12020-08-15 18:44:09 -07004814 &__io.address);
Jens Axboef499a022019-12-02 16:28:46 -07004815 if (ret)
4816 goto out;
4817 io = &__io;
4818 }
4819
Jens Axboe3fbb51c2019-12-20 08:51:52 -07004820 file_flags = force_nonblock ? O_NONBLOCK : 0;
4821
Jens Axboee8c2bc12020-08-15 18:44:09 -07004822 ret = __sys_connect_file(req->file, &io->address,
Jens Axboe3fbb51c2019-12-20 08:51:52 -07004823 req->connect.addr_len, file_flags);
Jens Axboe87f80d62019-12-03 11:23:54 -07004824 if ((ret == -EAGAIN || ret == -EINPROGRESS) && force_nonblock) {
Jens Axboee8c2bc12020-08-15 18:44:09 -07004825 if (req->async_data)
Jens Axboeb7bb4f72019-12-15 22:13:43 -07004826 return -EAGAIN;
Jens Axboee8c2bc12020-08-15 18:44:09 -07004827 if (io_alloc_async_data(req)) {
Jens Axboef499a022019-12-02 16:28:46 -07004828 ret = -ENOMEM;
4829 goto out;
4830 }
Jens Axboee8c2bc12020-08-15 18:44:09 -07004831 memcpy(req->async_data, &__io, sizeof(__io));
Jens Axboef8e85cf2019-11-23 14:24:24 -07004832 return -EAGAIN;
Jens Axboef499a022019-12-02 16:28:46 -07004833 }
Jens Axboef8e85cf2019-11-23 14:24:24 -07004834 if (ret == -ERESTARTSYS)
4835 ret = -EINTR;
Jens Axboef499a022019-12-02 16:28:46 -07004836out:
Jens Axboe4e88d6e2019-12-07 20:59:47 -07004837 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01004838 req_set_fail(req);
Pavel Begunkov889fca72021-02-10 00:03:09 +00004839 __io_req_complete(req, issue_flags, ret, 0);
Jens Axboef8e85cf2019-11-23 14:24:24 -07004840 return 0;
Jens Axboef8e85cf2019-11-23 14:24:24 -07004841}
YueHaibing469956e2020-03-04 15:53:52 +08004842#else /* !CONFIG_NET */
Jens Axboe99a10082021-02-19 09:35:19 -07004843#define IO_NETOP_FN(op) \
4844static int io_##op(struct io_kiocb *req, unsigned int issue_flags) \
4845{ \
4846 return -EOPNOTSUPP; \
Jens Axboef8e85cf2019-11-23 14:24:24 -07004847}
4848
Jens Axboe99a10082021-02-19 09:35:19 -07004849#define IO_NETOP_PREP(op) \
4850IO_NETOP_FN(op) \
4851static int io_##op##_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) \
4852{ \
4853 return -EOPNOTSUPP; \
4854} \
4855
4856#define IO_NETOP_PREP_ASYNC(op) \
4857IO_NETOP_PREP(op) \
4858static int io_##op##_prep_async(struct io_kiocb *req) \
4859{ \
4860 return -EOPNOTSUPP; \
YueHaibing469956e2020-03-04 15:53:52 +08004861}
4862
Jens Axboe99a10082021-02-19 09:35:19 -07004863IO_NETOP_PREP_ASYNC(sendmsg);
4864IO_NETOP_PREP_ASYNC(recvmsg);
4865IO_NETOP_PREP_ASYNC(connect);
4866IO_NETOP_PREP(accept);
4867IO_NETOP_FN(send);
4868IO_NETOP_FN(recv);
YueHaibing469956e2020-03-04 15:53:52 +08004869#endif /* CONFIG_NET */
Jens Axboe17f2fe32019-10-17 14:42:58 -06004870
Jens Axboed7718a92020-02-14 22:23:12 -07004871struct io_poll_table {
4872 struct poll_table_struct pt;
4873 struct io_kiocb *req;
Pavel Begunkov68b11e82021-07-20 10:50:43 +01004874 int nr_entries;
Jens Axboed7718a92020-02-14 22:23:12 -07004875 int error;
4876};
4877
Jens Axboed7718a92020-02-14 22:23:12 -07004878static int __io_async_wake(struct io_kiocb *req, struct io_poll_iocb *poll,
Pavel Begunkov5b0a6ac2021-06-30 21:54:04 +01004879 __poll_t mask, io_req_tw_func_t func)
Jens Axboed7718a92020-02-14 22:23:12 -07004880{
Jens Axboed7718a92020-02-14 22:23:12 -07004881 /* for instances that support it check for an event match first: */
4882 if (mask && !(mask & poll->events))
4883 return 0;
4884
4885 trace_io_uring_task_add(req->ctx, req->opcode, req->user_data, mask);
4886
4887 list_del_init(&poll->wait.entry);
4888
Jens Axboed7718a92020-02-14 22:23:12 -07004889 req->result = mask;
Pavel Begunkov5b0a6ac2021-06-30 21:54:04 +01004890 req->io_task_work.func = func;
Jens Axboe6d816e02020-08-11 08:04:14 -06004891
Jens Axboed7718a92020-02-14 22:23:12 -07004892 /*
Jens Axboee3aabf92020-05-18 11:04:17 -06004893 * If this fails, then the task is exiting. When a task exits, the
4894 * work gets canceled, so just cancel this request as well instead
4895 * of executing it. We can't safely execute it anyway, as we may not
4896 * have the needed state needed for it anyway.
Jens Axboed7718a92020-02-14 22:23:12 -07004897 */
Pavel Begunkove09ee512021-07-01 13:26:05 +01004898 io_req_task_work_add(req);
Jens Axboed7718a92020-02-14 22:23:12 -07004899 return 1;
4900}
4901
Jens Axboe74ce6ce2020-04-13 11:09:12 -06004902static bool io_poll_rewait(struct io_kiocb *req, struct io_poll_iocb *poll)
4903 __acquires(&req->ctx->completion_lock)
4904{
4905 struct io_ring_ctx *ctx = req->ctx;
4906
Jens Axboe316319e2021-08-19 09:41:42 -06004907 /* req->task == current here, checking PF_EXITING is safe */
Pavel Begunkove09ee512021-07-01 13:26:05 +01004908 if (unlikely(req->task->flags & PF_EXITING))
4909 WRITE_ONCE(poll->canceled, true);
4910
Jens Axboe74ce6ce2020-04-13 11:09:12 -06004911 if (!req->result && !READ_ONCE(poll->canceled)) {
4912 struct poll_table_struct pt = { ._key = poll->events };
4913
4914 req->result = vfs_poll(req->file, &pt) & poll->events;
4915 }
4916
Jens Axboe79ebeae2021-08-10 15:18:27 -06004917 spin_lock(&ctx->completion_lock);
Jens Axboe74ce6ce2020-04-13 11:09:12 -06004918 if (!req->result && !READ_ONCE(poll->canceled)) {
4919 add_wait_queue(poll->head, &poll->wait);
4920 return true;
4921 }
4922
4923 return false;
4924}
4925
Jens Axboed4e7cd32020-08-15 11:44:50 -07004926static struct io_poll_iocb *io_poll_get_double(struct io_kiocb *req)
Jens Axboe18bceab2020-05-15 11:56:54 -06004927{
Jens Axboee8c2bc12020-08-15 18:44:09 -07004928 /* pure poll stashes this in ->async_data, poll driven retry elsewhere */
Jens Axboed4e7cd32020-08-15 11:44:50 -07004929 if (req->opcode == IORING_OP_POLL_ADD)
Jens Axboee8c2bc12020-08-15 18:44:09 -07004930 return req->async_data;
Jens Axboed4e7cd32020-08-15 11:44:50 -07004931 return req->apoll->double_poll;
4932}
4933
4934static struct io_poll_iocb *io_poll_get_single(struct io_kiocb *req)
4935{
4936 if (req->opcode == IORING_OP_POLL_ADD)
4937 return &req->poll;
4938 return &req->apoll->poll;
4939}
4940
4941static void io_poll_remove_double(struct io_kiocb *req)
Pavel Begunkove07785b2021-04-01 15:43:57 +01004942 __must_hold(&req->ctx->completion_lock)
Jens Axboed4e7cd32020-08-15 11:44:50 -07004943{
4944 struct io_poll_iocb *poll = io_poll_get_double(req);
Jens Axboe18bceab2020-05-15 11:56:54 -06004945
4946 lockdep_assert_held(&req->ctx->completion_lock);
4947
4948 if (poll && poll->head) {
4949 struct wait_queue_head *head = poll->head;
4950
Jens Axboe79ebeae2021-08-10 15:18:27 -06004951 spin_lock_irq(&head->lock);
Jens Axboe18bceab2020-05-15 11:56:54 -06004952 list_del_init(&poll->wait.entry);
4953 if (poll->wait.private)
Jens Axboede9b4cc2021-02-24 13:28:27 -07004954 req_ref_put(req);
Jens Axboe18bceab2020-05-15 11:56:54 -06004955 poll->head = NULL;
Jens Axboe79ebeae2021-08-10 15:18:27 -06004956 spin_unlock_irq(&head->lock);
Jens Axboe18bceab2020-05-15 11:56:54 -06004957 }
4958}
4959
Pavel Begunkove27414b2021-04-09 09:13:20 +01004960static bool io_poll_complete(struct io_kiocb *req, __poll_t mask)
Pavel Begunkove07785b2021-04-01 15:43:57 +01004961 __must_hold(&req->ctx->completion_lock)
Jens Axboe18bceab2020-05-15 11:56:54 -06004962{
4963 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe88e41cf2021-02-22 22:08:01 -07004964 unsigned flags = IORING_CQE_F_MORE;
Pavel Begunkove27414b2021-04-09 09:13:20 +01004965 int error;
Jens Axboe18bceab2020-05-15 11:56:54 -06004966
Pavel Begunkove27414b2021-04-09 09:13:20 +01004967 if (READ_ONCE(req->poll.canceled)) {
Jens Axboe45ab03b2021-02-23 08:19:33 -07004968 error = -ECANCELED;
Jens Axboe88e41cf2021-02-22 22:08:01 -07004969 req->poll.events |= EPOLLONESHOT;
Pavel Begunkove27414b2021-04-09 09:13:20 +01004970 } else {
Jens Axboe50826202021-02-23 09:02:26 -07004971 error = mangle_poll(mask);
Pavel Begunkove27414b2021-04-09 09:13:20 +01004972 }
Jens Axboeb69de282021-03-17 08:37:41 -06004973 if (req->poll.events & EPOLLONESHOT)
4974 flags = 0;
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01004975 if (!io_cqring_fill_event(ctx, req->user_data, error, flags)) {
Jens Axboe88e41cf2021-02-22 22:08:01 -07004976 req->poll.done = true;
4977 flags = 0;
4978 }
Hao Xu7b289c32021-04-13 15:20:39 +08004979 if (flags & IORING_CQE_F_MORE)
4980 ctx->cq_extra++;
4981
Jens Axboe18bceab2020-05-15 11:56:54 -06004982 io_commit_cqring(ctx);
Jens Axboe88e41cf2021-02-22 22:08:01 -07004983 return !(flags & IORING_CQE_F_MORE);
Jens Axboe18bceab2020-05-15 11:56:54 -06004984}
4985
Pavel Begunkov5b0a6ac2021-06-30 21:54:04 +01004986static void io_poll_task_func(struct io_kiocb *req)
Jens Axboe18bceab2020-05-15 11:56:54 -06004987{
Jens Axboe6d816e02020-08-11 08:04:14 -06004988 struct io_ring_ctx *ctx = req->ctx;
Pavel Begunkovdd221f462020-10-18 10:17:42 +01004989 struct io_kiocb *nxt;
Jens Axboe18bceab2020-05-15 11:56:54 -06004990
Pavel Begunkovdd221f462020-10-18 10:17:42 +01004991 if (io_poll_rewait(req, &req->poll)) {
Jens Axboe79ebeae2021-08-10 15:18:27 -06004992 spin_unlock(&ctx->completion_lock);
Pavel Begunkovdd221f462020-10-18 10:17:42 +01004993 } else {
Pavel Begunkovf40b9642021-04-09 09:13:19 +01004994 bool done;
Jens Axboe88e41cf2021-02-22 22:08:01 -07004995
Pavel Begunkove27414b2021-04-09 09:13:20 +01004996 done = io_poll_complete(req, req->result);
Jens Axboe88e41cf2021-02-22 22:08:01 -07004997 if (done) {
Hao Xua890d012021-07-28 11:03:22 +08004998 io_poll_remove_double(req);
Jens Axboe88e41cf2021-02-22 22:08:01 -07004999 hash_del(&req->hash_node);
Pavel Begunkovf40b9642021-04-09 09:13:19 +01005000 } else {
Jens Axboe88e41cf2021-02-22 22:08:01 -07005001 req->result = 0;
5002 add_wait_queue(req->poll.head, &req->poll.wait);
5003 }
Jens Axboe79ebeae2021-08-10 15:18:27 -06005004 spin_unlock(&ctx->completion_lock);
Pavel Begunkovdd221f462020-10-18 10:17:42 +01005005 io_cqring_ev_posted(ctx);
Pavel Begunkovdd221f462020-10-18 10:17:42 +01005006
Jens Axboe88e41cf2021-02-22 22:08:01 -07005007 if (done) {
5008 nxt = io_put_req_find_next(req);
5009 if (nxt)
Pavel Begunkov5b0a6ac2021-06-30 21:54:04 +01005010 io_req_task_submit(nxt);
Jens Axboe88e41cf2021-02-22 22:08:01 -07005011 }
Pavel Begunkovea1164e2020-06-30 15:20:41 +03005012 }
Jens Axboe18bceab2020-05-15 11:56:54 -06005013}
5014
5015static int io_poll_double_wake(struct wait_queue_entry *wait, unsigned mode,
5016 int sync, void *key)
5017{
5018 struct io_kiocb *req = wait->private;
Jens Axboed4e7cd32020-08-15 11:44:50 -07005019 struct io_poll_iocb *poll = io_poll_get_single(req);
Jens Axboe18bceab2020-05-15 11:56:54 -06005020 __poll_t mask = key_to_poll(key);
Jens Axboe79ebeae2021-08-10 15:18:27 -06005021 unsigned long flags;
Jens Axboe18bceab2020-05-15 11:56:54 -06005022
5023 /* for instances that support it check for an event match first: */
5024 if (mask && !(mask & poll->events))
5025 return 0;
Jens Axboe88e41cf2021-02-22 22:08:01 -07005026 if (!(poll->events & EPOLLONESHOT))
5027 return poll->wait.func(&poll->wait, mode, sync, key);
Jens Axboe18bceab2020-05-15 11:56:54 -06005028
Jens Axboe8706e042020-09-28 08:38:54 -06005029 list_del_init(&wait->entry);
5030
Jens Axboe9ce85ef2021-07-09 08:20:28 -06005031 if (poll->head) {
Jens Axboe18bceab2020-05-15 11:56:54 -06005032 bool done;
5033
Jens Axboe79ebeae2021-08-10 15:18:27 -06005034 spin_lock_irqsave(&poll->head->lock, flags);
Jens Axboe807abcb2020-07-17 17:09:27 -06005035 done = list_empty(&poll->wait.entry);
Jens Axboe18bceab2020-05-15 11:56:54 -06005036 if (!done)
Jens Axboe807abcb2020-07-17 17:09:27 -06005037 list_del_init(&poll->wait.entry);
Jens Axboed4e7cd32020-08-15 11:44:50 -07005038 /* make sure double remove sees this as being gone */
5039 wait->private = NULL;
Jens Axboe79ebeae2021-08-10 15:18:27 -06005040 spin_unlock_irqrestore(&poll->head->lock, flags);
Jens Axboec8b5e262020-10-25 13:53:26 -06005041 if (!done) {
5042 /* use wait func handler, so it matches the rq type */
5043 poll->wait.func(&poll->wait, mode, sync, key);
5044 }
Jens Axboe18bceab2020-05-15 11:56:54 -06005045 }
Jens Axboede9b4cc2021-02-24 13:28:27 -07005046 req_ref_put(req);
Jens Axboe18bceab2020-05-15 11:56:54 -06005047 return 1;
5048}
5049
5050static void io_init_poll_iocb(struct io_poll_iocb *poll, __poll_t events,
5051 wait_queue_func_t wake_func)
5052{
5053 poll->head = NULL;
5054 poll->done = false;
5055 poll->canceled = false;
Jens Axboe464dca62021-03-19 14:06:24 -06005056#define IO_POLL_UNMASK (EPOLLERR|EPOLLHUP|EPOLLNVAL|EPOLLRDHUP)
5057 /* mask in events that we always want/need */
5058 poll->events = events | IO_POLL_UNMASK;
Jens Axboe18bceab2020-05-15 11:56:54 -06005059 INIT_LIST_HEAD(&poll->wait.entry);
5060 init_waitqueue_func_entry(&poll->wait, wake_func);
5061}
5062
5063static void __io_queue_proc(struct io_poll_iocb *poll, struct io_poll_table *pt,
Jens Axboe807abcb2020-07-17 17:09:27 -06005064 struct wait_queue_head *head,
5065 struct io_poll_iocb **poll_ptr)
Jens Axboe18bceab2020-05-15 11:56:54 -06005066{
5067 struct io_kiocb *req = pt->req;
5068
5069 /*
Pavel Begunkov68b11e82021-07-20 10:50:43 +01005070 * The file being polled uses multiple waitqueues for poll handling
5071 * (e.g. one for read, one for write). Setup a separate io_poll_iocb
5072 * if this happens.
Jens Axboe18bceab2020-05-15 11:56:54 -06005073 */
Pavel Begunkov68b11e82021-07-20 10:50:43 +01005074 if (unlikely(pt->nr_entries)) {
Pavel Begunkov58852d42020-10-16 20:55:56 +01005075 struct io_poll_iocb *poll_one = poll;
5076
Pavel Begunkov23a65db2021-08-17 20:28:11 +01005077 /* double add on the same waitqueue head, ignore */
5078 if (poll_one->head == head)
5079 return;
Jens Axboe18bceab2020-05-15 11:56:54 -06005080 /* already have a 2nd entry, fail a third attempt */
Jens Axboe807abcb2020-07-17 17:09:27 -06005081 if (*poll_ptr) {
Pavel Begunkov23a65db2021-08-17 20:28:11 +01005082 if ((*poll_ptr)->head == head)
5083 return;
Jens Axboe18bceab2020-05-15 11:56:54 -06005084 pt->error = -EINVAL;
5085 return;
5086 }
Jens Axboeea6a693d2021-04-15 09:47:13 -06005087 /*
5088 * Can't handle multishot for double wait for now, turn it
5089 * into one-shot mode.
5090 */
Pavel Begunkov7a274722021-05-17 12:43:34 +01005091 if (!(poll_one->events & EPOLLONESHOT))
5092 poll_one->events |= EPOLLONESHOT;
Jens Axboe18bceab2020-05-15 11:56:54 -06005093 poll = kmalloc(sizeof(*poll), GFP_ATOMIC);
5094 if (!poll) {
5095 pt->error = -ENOMEM;
5096 return;
5097 }
Pavel Begunkov58852d42020-10-16 20:55:56 +01005098 io_init_poll_iocb(poll, poll_one->events, io_poll_double_wake);
Jens Axboede9b4cc2021-02-24 13:28:27 -07005099 req_ref_get(req);
Jens Axboe18bceab2020-05-15 11:56:54 -06005100 poll->wait.private = req;
Jens Axboe807abcb2020-07-17 17:09:27 -06005101 *poll_ptr = poll;
Jens Axboe18bceab2020-05-15 11:56:54 -06005102 }
5103
Pavel Begunkov68b11e82021-07-20 10:50:43 +01005104 pt->nr_entries++;
Jens Axboe18bceab2020-05-15 11:56:54 -06005105 poll->head = head;
Jiufei Xuea31eb4a2020-06-17 17:53:56 +08005106
5107 if (poll->events & EPOLLEXCLUSIVE)
5108 add_wait_queue_exclusive(head, &poll->wait);
5109 else
5110 add_wait_queue(head, &poll->wait);
Jens Axboe18bceab2020-05-15 11:56:54 -06005111}
5112
5113static void io_async_queue_proc(struct file *file, struct wait_queue_head *head,
5114 struct poll_table_struct *p)
5115{
5116 struct io_poll_table *pt = container_of(p, struct io_poll_table, pt);
Jens Axboe807abcb2020-07-17 17:09:27 -06005117 struct async_poll *apoll = pt->req->apoll;
Jens Axboe18bceab2020-05-15 11:56:54 -06005118
Jens Axboe807abcb2020-07-17 17:09:27 -06005119 __io_queue_proc(&apoll->poll, pt, head, &apoll->double_poll);
Jens Axboe18bceab2020-05-15 11:56:54 -06005120}
5121
Pavel Begunkov5b0a6ac2021-06-30 21:54:04 +01005122static void io_async_task_func(struct io_kiocb *req)
Jens Axboed7718a92020-02-14 22:23:12 -07005123{
Jens Axboed7718a92020-02-14 22:23:12 -07005124 struct async_poll *apoll = req->apoll;
5125 struct io_ring_ctx *ctx = req->ctx;
5126
Olivier Langlois236daeae2021-05-31 02:36:37 -04005127 trace_io_uring_task_run(req->ctx, req, req->opcode, req->user_data);
Jens Axboed7718a92020-02-14 22:23:12 -07005128
Jens Axboe74ce6ce2020-04-13 11:09:12 -06005129 if (io_poll_rewait(req, &apoll->poll)) {
Jens Axboe79ebeae2021-08-10 15:18:27 -06005130 spin_unlock(&ctx->completion_lock);
Jens Axboe74ce6ce2020-04-13 11:09:12 -06005131 return;
Jens Axboed7718a92020-02-14 22:23:12 -07005132 }
5133
Pavel Begunkov0ea13b42021-04-09 09:13:21 +01005134 hash_del(&req->hash_node);
Jens Axboed4e7cd32020-08-15 11:44:50 -07005135 io_poll_remove_double(req);
Jens Axboe79ebeae2021-08-10 15:18:27 -06005136 spin_unlock(&ctx->completion_lock);
Jens Axboe74ce6ce2020-04-13 11:09:12 -06005137
Pavel Begunkov0be0b0e2020-06-30 15:20:42 +03005138 if (!READ_ONCE(apoll->poll.canceled))
Pavel Begunkov5b0a6ac2021-06-30 21:54:04 +01005139 io_req_task_submit(req);
Pavel Begunkov0be0b0e2020-06-30 15:20:42 +03005140 else
Pavel Begunkov25935532021-03-19 17:22:40 +00005141 io_req_complete_failed(req, -ECANCELED);
Jens Axboed7718a92020-02-14 22:23:12 -07005142}
5143
5144static int io_async_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
5145 void *key)
5146{
5147 struct io_kiocb *req = wait->private;
5148 struct io_poll_iocb *poll = &req->apoll->poll;
5149
5150 trace_io_uring_poll_wake(req->ctx, req->opcode, req->user_data,
5151 key_to_poll(key));
5152
5153 return __io_async_wake(req, poll, key_to_poll(key), io_async_task_func);
5154}
5155
5156static void io_poll_req_insert(struct io_kiocb *req)
5157{
5158 struct io_ring_ctx *ctx = req->ctx;
5159 struct hlist_head *list;
5160
5161 list = &ctx->cancel_hash[hash_long(req->user_data, ctx->cancel_hash_bits)];
5162 hlist_add_head(&req->hash_node, list);
5163}
5164
5165static __poll_t __io_arm_poll_handler(struct io_kiocb *req,
5166 struct io_poll_iocb *poll,
5167 struct io_poll_table *ipt, __poll_t mask,
5168 wait_queue_func_t wake_func)
5169 __acquires(&ctx->completion_lock)
5170{
5171 struct io_ring_ctx *ctx = req->ctx;
5172 bool cancel = false;
5173
Pavel Begunkov4d52f332020-10-18 10:17:43 +01005174 INIT_HLIST_NODE(&req->hash_node);
Jens Axboe18bceab2020-05-15 11:56:54 -06005175 io_init_poll_iocb(poll, mask, wake_func);
Pavel Begunkovb90cd192020-06-21 13:09:52 +03005176 poll->file = req->file;
Jens Axboe18bceab2020-05-15 11:56:54 -06005177 poll->wait.private = req;
Jens Axboed7718a92020-02-14 22:23:12 -07005178
5179 ipt->pt._key = mask;
5180 ipt->req = req;
Pavel Begunkov68b11e82021-07-20 10:50:43 +01005181 ipt->error = 0;
5182 ipt->nr_entries = 0;
Jens Axboed7718a92020-02-14 22:23:12 -07005183
Jens Axboed7718a92020-02-14 22:23:12 -07005184 mask = vfs_poll(req->file, &ipt->pt) & poll->events;
Pavel Begunkov68b11e82021-07-20 10:50:43 +01005185 if (unlikely(!ipt->nr_entries) && !ipt->error)
5186 ipt->error = -EINVAL;
Jens Axboed7718a92020-02-14 22:23:12 -07005187
Jens Axboe79ebeae2021-08-10 15:18:27 -06005188 spin_lock(&ctx->completion_lock);
Hao Xua890d012021-07-28 11:03:22 +08005189 if (ipt->error || (mask && (poll->events & EPOLLONESHOT)))
Pavel Begunkov46fee9a2021-07-20 10:50:44 +01005190 io_poll_remove_double(req);
Jens Axboed7718a92020-02-14 22:23:12 -07005191 if (likely(poll->head)) {
Jens Axboe79ebeae2021-08-10 15:18:27 -06005192 spin_lock_irq(&poll->head->lock);
Jens Axboed7718a92020-02-14 22:23:12 -07005193 if (unlikely(list_empty(&poll->wait.entry))) {
5194 if (ipt->error)
5195 cancel = true;
5196 ipt->error = 0;
5197 mask = 0;
5198 }
Jens Axboe88e41cf2021-02-22 22:08:01 -07005199 if ((mask && (poll->events & EPOLLONESHOT)) || ipt->error)
Jens Axboed7718a92020-02-14 22:23:12 -07005200 list_del_init(&poll->wait.entry);
5201 else if (cancel)
5202 WRITE_ONCE(poll->canceled, true);
5203 else if (!poll->done) /* actually waiting for an event */
5204 io_poll_req_insert(req);
Jens Axboe79ebeae2021-08-10 15:18:27 -06005205 spin_unlock_irq(&poll->head->lock);
Jens Axboed7718a92020-02-14 22:23:12 -07005206 }
5207
5208 return mask;
5209}
5210
Olivier Langlois59b735a2021-06-22 05:17:39 -07005211enum {
5212 IO_APOLL_OK,
5213 IO_APOLL_ABORTED,
5214 IO_APOLL_READY
5215};
5216
5217static int io_arm_poll_handler(struct io_kiocb *req)
Jens Axboed7718a92020-02-14 22:23:12 -07005218{
5219 const struct io_op_def *def = &io_op_defs[req->opcode];
5220 struct io_ring_ctx *ctx = req->ctx;
5221 struct async_poll *apoll;
5222 struct io_poll_table ipt;
Pavel Begunkovb2d9c3d2021-06-26 21:40:44 +01005223 __poll_t ret, mask = EPOLLONESHOT | POLLERR | POLLPRI;
Jens Axboe9dab14b2020-08-25 12:27:50 -06005224 int rw;
Jens Axboed7718a92020-02-14 22:23:12 -07005225
5226 if (!req->file || !file_can_poll(req->file))
Olivier Langlois59b735a2021-06-22 05:17:39 -07005227 return IO_APOLL_ABORTED;
Pavel Begunkov24c74672020-06-21 13:09:51 +03005228 if (req->flags & REQ_F_POLLED)
Olivier Langlois59b735a2021-06-22 05:17:39 -07005229 return IO_APOLL_ABORTED;
Pavel Begunkovb2d9c3d2021-06-26 21:40:44 +01005230 if (!def->pollin && !def->pollout)
Olivier Langlois59b735a2021-06-22 05:17:39 -07005231 return IO_APOLL_ABORTED;
Pavel Begunkovb2d9c3d2021-06-26 21:40:44 +01005232
5233 if (def->pollin) {
5234 rw = READ;
5235 mask |= POLLIN | POLLRDNORM;
5236
5237 /* If reading from MSG_ERRQUEUE using recvmsg, ignore POLLIN */
5238 if ((req->opcode == IORING_OP_RECVMSG) &&
5239 (req->sr_msg.msg_flags & MSG_ERRQUEUE))
5240 mask &= ~POLLIN;
5241 } else {
5242 rw = WRITE;
5243 mask |= POLLOUT | POLLWRNORM;
5244 }
5245
Jens Axboe9dab14b2020-08-25 12:27:50 -06005246 /* if we can't nonblock try, then no point in arming a poll handler */
Pavel Begunkovb191e2d2021-08-09 13:04:03 +01005247 if (!io_file_supports_nowait(req, rw))
Olivier Langlois59b735a2021-06-22 05:17:39 -07005248 return IO_APOLL_ABORTED;
Jens Axboed7718a92020-02-14 22:23:12 -07005249
5250 apoll = kmalloc(sizeof(*apoll), GFP_ATOMIC);
5251 if (unlikely(!apoll))
Olivier Langlois59b735a2021-06-22 05:17:39 -07005252 return IO_APOLL_ABORTED;
Jens Axboe807abcb2020-07-17 17:09:27 -06005253 apoll->double_poll = NULL;
Jens Axboed7718a92020-02-14 22:23:12 -07005254 req->apoll = apoll;
Pavel Begunkovb2d9c3d2021-06-26 21:40:44 +01005255 req->flags |= REQ_F_POLLED;
Jens Axboed7718a92020-02-14 22:23:12 -07005256 ipt.pt._qproc = io_async_queue_proc;
Pavel Begunkov48dcd382021-08-15 10:40:18 +01005257 io_req_set_refcount(req);
Jens Axboed7718a92020-02-14 22:23:12 -07005258
5259 ret = __io_arm_poll_handler(req, &apoll->poll, &ipt, mask,
5260 io_async_wake);
Jens Axboe79ebeae2021-08-10 15:18:27 -06005261 spin_unlock(&ctx->completion_lock);
Hao Xu41a51692021-08-12 15:47:02 +08005262 if (ret || ipt.error)
5263 return ret ? IO_APOLL_READY : IO_APOLL_ABORTED;
5264
Olivier Langlois236daeae2021-05-31 02:36:37 -04005265 trace_io_uring_poll_arm(ctx, req, req->opcode, req->user_data,
5266 mask, apoll->poll.events);
Olivier Langlois59b735a2021-06-22 05:17:39 -07005267 return IO_APOLL_OK;
Jens Axboed7718a92020-02-14 22:23:12 -07005268}
5269
5270static bool __io_poll_remove_one(struct io_kiocb *req,
Jens Axboeb2e720a2021-03-31 09:03:03 -06005271 struct io_poll_iocb *poll, bool do_cancel)
Pavel Begunkove07785b2021-04-01 15:43:57 +01005272 __must_hold(&req->ctx->completion_lock)
Jens Axboed7718a92020-02-14 22:23:12 -07005273{
Jens Axboeb41e9852020-02-17 09:52:41 -07005274 bool do_complete = false;
Jens Axboe221c5eb2019-01-17 09:41:58 -07005275
Jens Axboe50826202021-02-23 09:02:26 -07005276 if (!poll->head)
5277 return false;
Jens Axboe79ebeae2021-08-10 15:18:27 -06005278 spin_lock_irq(&poll->head->lock);
Jens Axboeb2e720a2021-03-31 09:03:03 -06005279 if (do_cancel)
5280 WRITE_ONCE(poll->canceled, true);
Jens Axboe392edb42019-12-09 17:52:20 -07005281 if (!list_empty(&poll->wait.entry)) {
5282 list_del_init(&poll->wait.entry);
Jens Axboeb41e9852020-02-17 09:52:41 -07005283 do_complete = true;
Jens Axboe221c5eb2019-01-17 09:41:58 -07005284 }
Jens Axboe79ebeae2021-08-10 15:18:27 -06005285 spin_unlock_irq(&poll->head->lock);
Jens Axboe3bfa5bc2020-05-17 13:54:12 -06005286 hash_del(&req->hash_node);
Jens Axboed7718a92020-02-14 22:23:12 -07005287 return do_complete;
5288}
5289
Pavel Begunkov5d709042021-08-09 20:18:13 +01005290static bool io_poll_remove_one(struct io_kiocb *req)
Pavel Begunkove07785b2021-04-01 15:43:57 +01005291 __must_hold(&req->ctx->completion_lock)
Jens Axboed7718a92020-02-14 22:23:12 -07005292{
5293 bool do_complete;
5294
Jens Axboed4e7cd32020-08-15 11:44:50 -07005295 io_poll_remove_double(req);
Pavel Begunkove31001a2021-04-13 02:58:43 +01005296 do_complete = __io_poll_remove_one(req, io_poll_get_single(req), true);
Jens Axboed4e7cd32020-08-15 11:44:50 -07005297
Jens Axboeb41e9852020-02-17 09:52:41 -07005298 if (do_complete) {
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01005299 io_cqring_fill_event(req->ctx, req->user_data, -ECANCELED, 0);
Jens Axboeb41e9852020-02-17 09:52:41 -07005300 io_commit_cqring(req->ctx);
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01005301 req_set_fail(req);
Pavel Begunkov91c2f692021-08-11 19:28:28 +01005302 io_put_req_deferred(req);
Pavel Begunkov5d709042021-08-09 20:18:13 +01005303 }
Jens Axboeb41e9852020-02-17 09:52:41 -07005304 return do_complete;
Jens Axboe221c5eb2019-01-17 09:41:58 -07005305}
5306
Jens Axboe76e1b642020-09-26 15:05:03 -06005307/*
5308 * Returns true if we found and killed one or more poll requests
5309 */
Pavel Begunkov6b819282020-11-06 13:00:25 +00005310static bool io_poll_remove_all(struct io_ring_ctx *ctx, struct task_struct *tsk,
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01005311 bool cancel_all)
Jens Axboe221c5eb2019-01-17 09:41:58 -07005312{
Jens Axboe78076bb2019-12-04 19:56:40 -07005313 struct hlist_node *tmp;
Jens Axboe221c5eb2019-01-17 09:41:58 -07005314 struct io_kiocb *req;
Jens Axboe8e2e1fa2020-04-13 17:05:14 -06005315 int posted = 0, i;
Jens Axboe221c5eb2019-01-17 09:41:58 -07005316
Jens Axboe79ebeae2021-08-10 15:18:27 -06005317 spin_lock(&ctx->completion_lock);
Jens Axboe78076bb2019-12-04 19:56:40 -07005318 for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) {
5319 struct hlist_head *list;
5320
5321 list = &ctx->cancel_hash[i];
Jens Axboef3606e32020-09-22 08:18:24 -06005322 hlist_for_each_entry_safe(req, tmp, list, hash_node) {
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01005323 if (io_match_task(req, tsk, cancel_all))
Jens Axboef3606e32020-09-22 08:18:24 -06005324 posted += io_poll_remove_one(req);
5325 }
Jens Axboe221c5eb2019-01-17 09:41:58 -07005326 }
Jens Axboe79ebeae2021-08-10 15:18:27 -06005327 spin_unlock(&ctx->completion_lock);
Jens Axboeb41e9852020-02-17 09:52:41 -07005328
Jens Axboe8e2e1fa2020-04-13 17:05:14 -06005329 if (posted)
5330 io_cqring_ev_posted(ctx);
Jens Axboe76e1b642020-09-26 15:05:03 -06005331
5332 return posted != 0;
Jens Axboe221c5eb2019-01-17 09:41:58 -07005333}
5334
Pavel Begunkov9ba5fac2021-04-14 13:38:35 +01005335static struct io_kiocb *io_poll_find(struct io_ring_ctx *ctx, __u64 sqe_addr,
5336 bool poll_only)
Pavel Begunkove07785b2021-04-01 15:43:57 +01005337 __must_hold(&ctx->completion_lock)
Jens Axboe47f46762019-11-09 17:43:02 -07005338{
Jens Axboe78076bb2019-12-04 19:56:40 -07005339 struct hlist_head *list;
Jens Axboe47f46762019-11-09 17:43:02 -07005340 struct io_kiocb *req;
5341
Jens Axboe78076bb2019-12-04 19:56:40 -07005342 list = &ctx->cancel_hash[hash_long(sqe_addr, ctx->cancel_hash_bits)];
5343 hlist_for_each_entry(req, list, hash_node) {
Jens Axboeb41e9852020-02-17 09:52:41 -07005344 if (sqe_addr != req->user_data)
5345 continue;
Pavel Begunkov9ba5fac2021-04-14 13:38:35 +01005346 if (poll_only && req->opcode != IORING_OP_POLL_ADD)
5347 continue;
Jens Axboeb2cb8052021-03-17 08:17:19 -06005348 return req;
Jens Axboe47f46762019-11-09 17:43:02 -07005349 }
Jens Axboeb2cb8052021-03-17 08:17:19 -06005350 return NULL;
Jens Axboe47f46762019-11-09 17:43:02 -07005351}
5352
Pavel Begunkov9ba5fac2021-04-14 13:38:35 +01005353static int io_poll_cancel(struct io_ring_ctx *ctx, __u64 sqe_addr,
5354 bool poll_only)
Pavel Begunkove07785b2021-04-01 15:43:57 +01005355 __must_hold(&ctx->completion_lock)
Jens Axboeb2cb8052021-03-17 08:17:19 -06005356{
5357 struct io_kiocb *req;
5358
Pavel Begunkov9ba5fac2021-04-14 13:38:35 +01005359 req = io_poll_find(ctx, sqe_addr, poll_only);
Jens Axboeb2cb8052021-03-17 08:17:19 -06005360 if (!req)
5361 return -ENOENT;
5362 if (io_poll_remove_one(req))
5363 return 0;
5364
5365 return -EALREADY;
Jens Axboe221c5eb2019-01-17 09:41:58 -07005366}
5367
Pavel Begunkov9096af32021-04-14 13:38:36 +01005368static __poll_t io_poll_parse_events(const struct io_uring_sqe *sqe,
5369 unsigned int flags)
5370{
5371 u32 events;
5372
5373 events = READ_ONCE(sqe->poll32_events);
5374#ifdef __BIG_ENDIAN
5375 events = swahw32(events);
5376#endif
5377 if (!(flags & IORING_POLL_ADD_MULTI))
5378 events |= EPOLLONESHOT;
5379 return demangle_poll(events) | (events & (EPOLLEXCLUSIVE|EPOLLONESHOT));
5380}
5381
Pavel Begunkovc5de0032021-04-14 13:38:37 +01005382static int io_poll_update_prep(struct io_kiocb *req,
Jens Axboe3529d8c2019-12-19 18:24:38 -07005383 const struct io_uring_sqe *sqe)
Jens Axboe221c5eb2019-01-17 09:41:58 -07005384{
Pavel Begunkovc5de0032021-04-14 13:38:37 +01005385 struct io_poll_update *upd = &req->poll_update;
5386 u32 flags;
5387
Jens Axboe221c5eb2019-01-17 09:41:58 -07005388 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
5389 return -EINVAL;
Pavel Begunkovc5de0032021-04-14 13:38:37 +01005390 if (sqe->ioprio || sqe->buf_index)
5391 return -EINVAL;
5392 flags = READ_ONCE(sqe->len);
5393 if (flags & ~(IORING_POLL_UPDATE_EVENTS | IORING_POLL_UPDATE_USER_DATA |
5394 IORING_POLL_ADD_MULTI))
5395 return -EINVAL;
5396 /* meaningless without update */
5397 if (flags == IORING_POLL_ADD_MULTI)
Jens Axboe221c5eb2019-01-17 09:41:58 -07005398 return -EINVAL;
5399
Pavel Begunkovc5de0032021-04-14 13:38:37 +01005400 upd->old_user_data = READ_ONCE(sqe->addr);
5401 upd->update_events = flags & IORING_POLL_UPDATE_EVENTS;
5402 upd->update_user_data = flags & IORING_POLL_UPDATE_USER_DATA;
Jens Axboe0969e782019-12-17 18:40:57 -07005403
Pavel Begunkovc5de0032021-04-14 13:38:37 +01005404 upd->new_user_data = READ_ONCE(sqe->off);
5405 if (!upd->update_user_data && upd->new_user_data)
5406 return -EINVAL;
5407 if (upd->update_events)
5408 upd->events = io_poll_parse_events(sqe, flags);
5409 else if (sqe->poll32_events)
5410 return -EINVAL;
Jens Axboe0969e782019-12-17 18:40:57 -07005411
Jens Axboe221c5eb2019-01-17 09:41:58 -07005412 return 0;
5413}
5414
Jens Axboe221c5eb2019-01-17 09:41:58 -07005415static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
5416 void *key)
5417{
Jens Axboec2f2eb72020-02-10 09:07:05 -07005418 struct io_kiocb *req = wait->private;
5419 struct io_poll_iocb *poll = &req->poll;
Jens Axboe221c5eb2019-01-17 09:41:58 -07005420
Jens Axboed7718a92020-02-14 22:23:12 -07005421 return __io_async_wake(req, poll, key_to_poll(key), io_poll_task_func);
Jens Axboe221c5eb2019-01-17 09:41:58 -07005422}
5423
Jens Axboe221c5eb2019-01-17 09:41:58 -07005424static void io_poll_queue_proc(struct file *file, struct wait_queue_head *head,
5425 struct poll_table_struct *p)
5426{
5427 struct io_poll_table *pt = container_of(p, struct io_poll_table, pt);
5428
Jens Axboee8c2bc12020-08-15 18:44:09 -07005429 __io_queue_proc(&pt->req->poll, pt, head, (struct io_poll_iocb **) &pt->req->async_data);
Jens Axboeeac406c2019-11-14 12:09:58 -07005430}
5431
Jens Axboe3529d8c2019-12-19 18:24:38 -07005432static int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboe221c5eb2019-01-17 09:41:58 -07005433{
5434 struct io_poll_iocb *poll = &req->poll;
Pavel Begunkovc5de0032021-04-14 13:38:37 +01005435 u32 flags;
Jens Axboe221c5eb2019-01-17 09:41:58 -07005436
5437 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
5438 return -EINVAL;
Pavel Begunkovc5de0032021-04-14 13:38:37 +01005439 if (sqe->ioprio || sqe->buf_index || sqe->off || sqe->addr)
Jens Axboe88e41cf2021-02-22 22:08:01 -07005440 return -EINVAL;
5441 flags = READ_ONCE(sqe->len);
Pavel Begunkovc5de0032021-04-14 13:38:37 +01005442 if (flags & ~IORING_POLL_ADD_MULTI)
Jens Axboe221c5eb2019-01-17 09:41:58 -07005443 return -EINVAL;
5444
Pavel Begunkov48dcd382021-08-15 10:40:18 +01005445 io_req_set_refcount(req);
Pavel Begunkovc5de0032021-04-14 13:38:37 +01005446 poll->events = io_poll_parse_events(sqe, flags);
Jens Axboe0969e782019-12-17 18:40:57 -07005447 return 0;
5448}
5449
Pavel Begunkov61e98202021-02-10 00:03:08 +00005450static int io_poll_add(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe0969e782019-12-17 18:40:57 -07005451{
5452 struct io_poll_iocb *poll = &req->poll;
5453 struct io_ring_ctx *ctx = req->ctx;
5454 struct io_poll_table ipt;
Jens Axboe0969e782019-12-17 18:40:57 -07005455 __poll_t mask;
Jens Axboe0969e782019-12-17 18:40:57 -07005456
Jens Axboed7718a92020-02-14 22:23:12 -07005457 ipt.pt._qproc = io_poll_queue_proc;
Jens Axboe36703242019-07-25 10:20:18 -06005458
Jens Axboed7718a92020-02-14 22:23:12 -07005459 mask = __io_arm_poll_handler(req, &req->poll, &ipt, poll->events,
5460 io_poll_wake);
Jens Axboe221c5eb2019-01-17 09:41:58 -07005461
Jens Axboe8c838782019-03-12 15:48:16 -06005462 if (mask) { /* no async, we'd stolen it */
Jens Axboe8c838782019-03-12 15:48:16 -06005463 ipt.error = 0;
Pavel Begunkove27414b2021-04-09 09:13:20 +01005464 io_poll_complete(req, mask);
Jens Axboe8c838782019-03-12 15:48:16 -06005465 }
Jens Axboe79ebeae2021-08-10 15:18:27 -06005466 spin_unlock(&ctx->completion_lock);
Jens Axboe221c5eb2019-01-17 09:41:58 -07005467
Jens Axboe8c838782019-03-12 15:48:16 -06005468 if (mask) {
5469 io_cqring_ev_posted(ctx);
Jens Axboe88e41cf2021-02-22 22:08:01 -07005470 if (poll->events & EPOLLONESHOT)
5471 io_put_req(req);
Jens Axboe221c5eb2019-01-17 09:41:58 -07005472 }
Jens Axboe8c838782019-03-12 15:48:16 -06005473 return ipt.error;
Jens Axboe221c5eb2019-01-17 09:41:58 -07005474}
5475
Pavel Begunkovc5de0032021-04-14 13:38:37 +01005476static int io_poll_update(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboeb69de282021-03-17 08:37:41 -06005477{
5478 struct io_ring_ctx *ctx = req->ctx;
5479 struct io_kiocb *preq;
Jens Axboecb3b200e2021-04-06 09:49:31 -06005480 bool completing;
Jens Axboeb69de282021-03-17 08:37:41 -06005481 int ret;
5482
Jens Axboe79ebeae2021-08-10 15:18:27 -06005483 spin_lock(&ctx->completion_lock);
Pavel Begunkov9ba5fac2021-04-14 13:38:35 +01005484 preq = io_poll_find(ctx, req->poll_update.old_user_data, true);
Jens Axboeb69de282021-03-17 08:37:41 -06005485 if (!preq) {
5486 ret = -ENOENT;
5487 goto err;
Jens Axboeb69de282021-03-17 08:37:41 -06005488 }
Jens Axboecb3b200e2021-04-06 09:49:31 -06005489
Pavel Begunkovc5de0032021-04-14 13:38:37 +01005490 if (!req->poll_update.update_events && !req->poll_update.update_user_data) {
5491 completing = true;
5492 ret = io_poll_remove_one(preq) ? 0 : -EALREADY;
5493 goto err;
5494 }
5495
Jens Axboecb3b200e2021-04-06 09:49:31 -06005496 /*
5497 * Don't allow racy completion with singleshot, as we cannot safely
5498 * update those. For multishot, if we're racing with completion, just
5499 * let completion re-add it.
5500 */
5501 completing = !__io_poll_remove_one(preq, &preq->poll, false);
5502 if (completing && (preq->poll.events & EPOLLONESHOT)) {
5503 ret = -EALREADY;
5504 goto err;
Jens Axboeb69de282021-03-17 08:37:41 -06005505 }
5506 /* we now have a detached poll request. reissue. */
5507 ret = 0;
5508err:
Jens Axboeb69de282021-03-17 08:37:41 -06005509 if (ret < 0) {
Jens Axboe79ebeae2021-08-10 15:18:27 -06005510 spin_unlock(&ctx->completion_lock);
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01005511 req_set_fail(req);
Jens Axboeb69de282021-03-17 08:37:41 -06005512 io_req_complete(req, ret);
5513 return 0;
5514 }
5515 /* only mask one event flags, keep behavior flags */
Pavel Begunkov9d805892021-04-13 02:58:40 +01005516 if (req->poll_update.update_events) {
Jens Axboeb69de282021-03-17 08:37:41 -06005517 preq->poll.events &= ~0xffff;
Pavel Begunkov9d805892021-04-13 02:58:40 +01005518 preq->poll.events |= req->poll_update.events & 0xffff;
Jens Axboeb69de282021-03-17 08:37:41 -06005519 preq->poll.events |= IO_POLL_UNMASK;
5520 }
Pavel Begunkov9d805892021-04-13 02:58:40 +01005521 if (req->poll_update.update_user_data)
5522 preq->user_data = req->poll_update.new_user_data;
Jens Axboe79ebeae2021-08-10 15:18:27 -06005523 spin_unlock(&ctx->completion_lock);
Jens Axboecb3b200e2021-04-06 09:49:31 -06005524
Jens Axboeb69de282021-03-17 08:37:41 -06005525 /* complete update request, we're done with it */
5526 io_req_complete(req, ret);
5527
Jens Axboecb3b200e2021-04-06 09:49:31 -06005528 if (!completing) {
Pavel Begunkovc5de0032021-04-14 13:38:37 +01005529 ret = io_poll_add(preq, issue_flags);
Jens Axboecb3b200e2021-04-06 09:49:31 -06005530 if (ret < 0) {
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01005531 req_set_fail(preq);
Jens Axboecb3b200e2021-04-06 09:49:31 -06005532 io_req_complete(preq, ret);
5533 }
Jens Axboeb69de282021-03-17 08:37:41 -06005534 }
5535 return 0;
5536}
5537
Jens Axboe89850fc2021-08-10 15:11:51 -06005538static void io_req_task_timeout(struct io_kiocb *req)
5539{
Jens Axboe89850fc2021-08-10 15:11:51 -06005540 req_set_fail(req);
Pavel Begunkov505657b2021-08-17 20:28:09 +01005541 io_req_complete_post(req, -ETIME, 0);
Jens Axboe89850fc2021-08-10 15:11:51 -06005542}
5543
Jens Axboe5262f562019-09-17 12:26:57 -06005544static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer)
5545{
Jens Axboead8a48a2019-11-15 08:49:11 -07005546 struct io_timeout_data *data = container_of(timer,
5547 struct io_timeout_data, timer);
5548 struct io_kiocb *req = data->req;
5549 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe5262f562019-09-17 12:26:57 -06005550 unsigned long flags;
5551
Jens Axboe89850fc2021-08-10 15:11:51 -06005552 spin_lock_irqsave(&ctx->timeout_lock, flags);
Pavel Begunkova71976f2020-10-10 18:34:11 +01005553 list_del_init(&req->timeout.list);
Pavel Begunkov01cec8c2020-07-30 18:43:50 +03005554 atomic_set(&req->ctx->cq_timeouts,
5555 atomic_read(&req->ctx->cq_timeouts) + 1);
Jens Axboe89850fc2021-08-10 15:11:51 -06005556 spin_unlock_irqrestore(&ctx->timeout_lock, flags);
Pavel Begunkov01cec8c2020-07-30 18:43:50 +03005557
Jens Axboe89850fc2021-08-10 15:11:51 -06005558 req->io_task_work.func = io_req_task_timeout;
5559 io_req_task_work_add(req);
Jens Axboe5262f562019-09-17 12:26:57 -06005560 return HRTIMER_NORESTART;
5561}
5562
Pavel Begunkovfbd15842020-11-30 19:11:15 +00005563static struct io_kiocb *io_timeout_extract(struct io_ring_ctx *ctx,
5564 __u64 user_data)
Jens Axboe89850fc2021-08-10 15:11:51 -06005565 __must_hold(&ctx->timeout_lock)
Jens Axboe47f46762019-11-09 17:43:02 -07005566{
Pavel Begunkovfbd15842020-11-30 19:11:15 +00005567 struct io_timeout_data *io;
Jens Axboef254ac02020-08-12 17:33:30 -06005568 struct io_kiocb *req;
Pavel Begunkovfd9c7bc2021-04-13 02:58:42 +01005569 bool found = false;
Jens Axboef254ac02020-08-12 17:33:30 -06005570
5571 list_for_each_entry(req, &ctx->timeout_list, timeout.list) {
Pavel Begunkovfd9c7bc2021-04-13 02:58:42 +01005572 found = user_data == req->user_data;
5573 if (found)
Jens Axboef254ac02020-08-12 17:33:30 -06005574 break;
Jens Axboef254ac02020-08-12 17:33:30 -06005575 }
Pavel Begunkovfd9c7bc2021-04-13 02:58:42 +01005576 if (!found)
5577 return ERR_PTR(-ENOENT);
Jens Axboef254ac02020-08-12 17:33:30 -06005578
Pavel Begunkovfbd15842020-11-30 19:11:15 +00005579 io = req->async_data;
Pavel Begunkovfd9c7bc2021-04-13 02:58:42 +01005580 if (hrtimer_try_to_cancel(&io->timer) == -1)
Pavel Begunkovfbd15842020-11-30 19:11:15 +00005581 return ERR_PTR(-EALREADY);
5582 list_del_init(&req->timeout.list);
5583 return req;
5584}
5585
5586static int io_timeout_cancel(struct io_ring_ctx *ctx, __u64 user_data)
Pavel Begunkovec3c3d02021-08-18 10:50:52 +01005587 __must_hold(&ctx->completion_lock)
Jens Axboe89850fc2021-08-10 15:11:51 -06005588 __must_hold(&ctx->timeout_lock)
Pavel Begunkovfbd15842020-11-30 19:11:15 +00005589{
5590 struct io_kiocb *req = io_timeout_extract(ctx, user_data);
5591
5592 if (IS_ERR(req))
5593 return PTR_ERR(req);
5594
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01005595 req_set_fail(req);
Pavel Begunkovd4d19c12021-04-25 14:32:17 +01005596 io_cqring_fill_event(ctx, req->user_data, -ECANCELED, 0);
Pavel Begunkov91c2f692021-08-11 19:28:28 +01005597 io_put_req_deferred(req);
Pavel Begunkovfbd15842020-11-30 19:11:15 +00005598 return 0;
Jens Axboef254ac02020-08-12 17:33:30 -06005599}
5600
Pavel Begunkov9c8e11b2020-11-30 19:11:16 +00005601static int io_timeout_update(struct io_ring_ctx *ctx, __u64 user_data,
5602 struct timespec64 *ts, enum hrtimer_mode mode)
Jens Axboe89850fc2021-08-10 15:11:51 -06005603 __must_hold(&ctx->timeout_lock)
Pavel Begunkov9c8e11b2020-11-30 19:11:16 +00005604{
5605 struct io_kiocb *req = io_timeout_extract(ctx, user_data);
5606 struct io_timeout_data *data;
5607
5608 if (IS_ERR(req))
5609 return PTR_ERR(req);
5610
5611 req->timeout.off = 0; /* noseq */
5612 data = req->async_data;
5613 list_add_tail(&req->timeout.list, &ctx->timeout_list);
5614 hrtimer_init(&data->timer, CLOCK_MONOTONIC, mode);
5615 data->timer.function = io_timeout_fn;
5616 hrtimer_start(&data->timer, timespec64_to_ktime(*ts), mode);
5617 return 0;
Jens Axboe47f46762019-11-09 17:43:02 -07005618}
5619
Jens Axboe3529d8c2019-12-19 18:24:38 -07005620static int io_timeout_remove_prep(struct io_kiocb *req,
5621 const struct io_uring_sqe *sqe)
Jens Axboeb29472e2019-12-17 18:50:29 -07005622{
Pavel Begunkov9c8e11b2020-11-30 19:11:16 +00005623 struct io_timeout_rem *tr = &req->timeout_rem;
5624
Jens Axboeb29472e2019-12-17 18:50:29 -07005625 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
5626 return -EINVAL;
Daniele Albano61710e42020-07-18 14:15:16 -06005627 if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT)))
5628 return -EINVAL;
Pavel Begunkov9c8e11b2020-11-30 19:11:16 +00005629 if (sqe->ioprio || sqe->buf_index || sqe->len)
Jens Axboeb29472e2019-12-17 18:50:29 -07005630 return -EINVAL;
5631
Pavel Begunkov9c8e11b2020-11-30 19:11:16 +00005632 tr->addr = READ_ONCE(sqe->addr);
5633 tr->flags = READ_ONCE(sqe->timeout_flags);
5634 if (tr->flags & IORING_TIMEOUT_UPDATE) {
5635 if (tr->flags & ~(IORING_TIMEOUT_UPDATE|IORING_TIMEOUT_ABS))
5636 return -EINVAL;
5637 if (get_timespec64(&tr->ts, u64_to_user_ptr(sqe->addr2)))
5638 return -EFAULT;
5639 } else if (tr->flags) {
5640 /* timeout removal doesn't support flags */
5641 return -EINVAL;
5642 }
5643
Jens Axboeb29472e2019-12-17 18:50:29 -07005644 return 0;
5645}
5646
Pavel Begunkov8662dae2021-01-19 13:32:44 +00005647static inline enum hrtimer_mode io_translate_timeout_mode(unsigned int flags)
5648{
5649 return (flags & IORING_TIMEOUT_ABS) ? HRTIMER_MODE_ABS
5650 : HRTIMER_MODE_REL;
5651}
5652
Jens Axboe11365042019-10-16 09:08:32 -06005653/*
5654 * Remove or update an existing timeout command
5655 */
Pavel Begunkov61e98202021-02-10 00:03:08 +00005656static int io_timeout_remove(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe11365042019-10-16 09:08:32 -06005657{
Pavel Begunkov9c8e11b2020-11-30 19:11:16 +00005658 struct io_timeout_rem *tr = &req->timeout_rem;
Jens Axboe11365042019-10-16 09:08:32 -06005659 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe47f46762019-11-09 17:43:02 -07005660 int ret;
Jens Axboe11365042019-10-16 09:08:32 -06005661
Pavel Begunkovec3c3d02021-08-18 10:50:52 +01005662 if (!(req->timeout_rem.flags & IORING_TIMEOUT_UPDATE)) {
5663 spin_lock(&ctx->completion_lock);
5664 spin_lock_irq(&ctx->timeout_lock);
Pavel Begunkov9c8e11b2020-11-30 19:11:16 +00005665 ret = io_timeout_cancel(ctx, tr->addr);
Pavel Begunkovec3c3d02021-08-18 10:50:52 +01005666 spin_unlock_irq(&ctx->timeout_lock);
5667 spin_unlock(&ctx->completion_lock);
5668 } else {
5669 spin_lock_irq(&ctx->timeout_lock);
Pavel Begunkov8662dae2021-01-19 13:32:44 +00005670 ret = io_timeout_update(ctx, tr->addr, &tr->ts,
5671 io_translate_timeout_mode(tr->flags));
Pavel Begunkovec3c3d02021-08-18 10:50:52 +01005672 spin_unlock_irq(&ctx->timeout_lock);
5673 }
Jens Axboe11365042019-10-16 09:08:32 -06005674
Jens Axboe4e88d6e2019-12-07 20:59:47 -07005675 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01005676 req_set_fail(req);
Pavel Begunkov505657b2021-08-17 20:28:09 +01005677 io_req_complete_post(req, ret, 0);
Jens Axboe11365042019-10-16 09:08:32 -06005678 return 0;
Jens Axboe5262f562019-09-17 12:26:57 -06005679}
5680
Jens Axboe3529d8c2019-12-19 18:24:38 -07005681static int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe,
Jens Axboe2d283902019-12-04 11:08:05 -07005682 bool is_timeout_link)
Jens Axboe5262f562019-09-17 12:26:57 -06005683{
Jens Axboead8a48a2019-11-15 08:49:11 -07005684 struct io_timeout_data *data;
Jens Axboea41525a2019-10-15 16:48:15 -06005685 unsigned flags;
Pavel Begunkov56080b02020-05-26 20:34:04 +03005686 u32 off = READ_ONCE(sqe->off);
Jens Axboe5262f562019-09-17 12:26:57 -06005687
Jens Axboead8a48a2019-11-15 08:49:11 -07005688 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
Jens Axboe5262f562019-09-17 12:26:57 -06005689 return -EINVAL;
Jens Axboead8a48a2019-11-15 08:49:11 -07005690 if (sqe->ioprio || sqe->buf_index || sqe->len != 1)
Jens Axboea41525a2019-10-15 16:48:15 -06005691 return -EINVAL;
Pavel Begunkov56080b02020-05-26 20:34:04 +03005692 if (off && is_timeout_link)
Jens Axboe2d283902019-12-04 11:08:05 -07005693 return -EINVAL;
Jens Axboea41525a2019-10-15 16:48:15 -06005694 flags = READ_ONCE(sqe->timeout_flags);
5695 if (flags & ~IORING_TIMEOUT_ABS)
Jens Axboe5262f562019-09-17 12:26:57 -06005696 return -EINVAL;
Arnd Bergmannbdf20072019-10-01 09:53:29 -06005697
Pavel Begunkovbfe68a22020-05-30 14:54:18 +03005698 req->timeout.off = off;
Pavel Begunkovf18ee4c2021-06-14 23:37:25 +01005699 if (unlikely(off && !req->ctx->off_timeout_used))
5700 req->ctx->off_timeout_used = true;
Jens Axboe26a61672019-12-20 09:02:01 -07005701
Jens Axboee8c2bc12020-08-15 18:44:09 -07005702 if (!req->async_data && io_alloc_async_data(req))
Jens Axboe26a61672019-12-20 09:02:01 -07005703 return -ENOMEM;
5704
Jens Axboee8c2bc12020-08-15 18:44:09 -07005705 data = req->async_data;
Jens Axboead8a48a2019-11-15 08:49:11 -07005706 data->req = req;
Jens Axboead8a48a2019-11-15 08:49:11 -07005707
5708 if (get_timespec64(&data->ts, u64_to_user_ptr(sqe->addr)))
Jens Axboe5262f562019-09-17 12:26:57 -06005709 return -EFAULT;
5710
Pavel Begunkov8662dae2021-01-19 13:32:44 +00005711 data->mode = io_translate_timeout_mode(flags);
Jens Axboead8a48a2019-11-15 08:49:11 -07005712 hrtimer_init(&data->timer, CLOCK_MONOTONIC, data->mode);
Pavel Begunkovb97e7362021-08-15 10:40:23 +01005713
5714 if (is_timeout_link) {
5715 struct io_submit_link *link = &req->ctx->submit_state.link;
5716
5717 if (!link->head)
5718 return -EINVAL;
5719 if (link->last->opcode == IORING_OP_LINK_TIMEOUT)
5720 return -EINVAL;
Pavel Begunkov4d13d1a2021-08-15 10:40:24 +01005721 req->timeout.head = link->last;
5722 link->last->flags |= REQ_F_ARM_LTIMEOUT;
Pavel Begunkovb97e7362021-08-15 10:40:23 +01005723 }
Jens Axboead8a48a2019-11-15 08:49:11 -07005724 return 0;
5725}
5726
Pavel Begunkov61e98202021-02-10 00:03:08 +00005727static int io_timeout(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboead8a48a2019-11-15 08:49:11 -07005728{
Jens Axboead8a48a2019-11-15 08:49:11 -07005729 struct io_ring_ctx *ctx = req->ctx;
Jens Axboee8c2bc12020-08-15 18:44:09 -07005730 struct io_timeout_data *data = req->async_data;
Jens Axboead8a48a2019-11-15 08:49:11 -07005731 struct list_head *entry;
Pavel Begunkovbfe68a22020-05-30 14:54:18 +03005732 u32 tail, off = req->timeout.off;
Jens Axboead8a48a2019-11-15 08:49:11 -07005733
Jens Axboe89850fc2021-08-10 15:11:51 -06005734 spin_lock_irq(&ctx->timeout_lock);
Jens Axboe93bd25b2019-11-11 23:34:31 -07005735
Jens Axboe5262f562019-09-17 12:26:57 -06005736 /*
5737 * sqe->off holds how many events that need to occur for this
Jens Axboe93bd25b2019-11-11 23:34:31 -07005738 * timeout event to be satisfied. If it isn't set, then this is
5739 * a pure timeout request, sequence isn't used.
Jens Axboe5262f562019-09-17 12:26:57 -06005740 */
Pavel Begunkov8eb7e2d2020-06-29 13:13:02 +03005741 if (io_is_timeout_noseq(req)) {
Jens Axboe93bd25b2019-11-11 23:34:31 -07005742 entry = ctx->timeout_list.prev;
5743 goto add;
5744 }
Jens Axboe5262f562019-09-17 12:26:57 -06005745
Pavel Begunkovbfe68a22020-05-30 14:54:18 +03005746 tail = ctx->cached_cq_tail - atomic_read(&ctx->cq_timeouts);
5747 req->timeout.target_seq = tail + off;
Jens Axboe5262f562019-09-17 12:26:57 -06005748
Marcelo Diop-Gonzalezf0105052021-01-15 11:54:40 -05005749 /* Update the last seq here in case io_flush_timeouts() hasn't.
5750 * This is safe because ->completion_lock is held, and submissions
5751 * and completions are never mixed in the same ->completion_lock section.
5752 */
5753 ctx->cq_last_tm_flush = tail;
5754
Jens Axboe5262f562019-09-17 12:26:57 -06005755 /*
5756 * Insertion sort, ensuring the first entry in the list is always
5757 * the one we need first.
5758 */
Jens Axboe5262f562019-09-17 12:26:57 -06005759 list_for_each_prev(entry, &ctx->timeout_list) {
Pavel Begunkov135fcde2020-07-13 23:37:12 +03005760 struct io_kiocb *nxt = list_entry(entry, struct io_kiocb,
5761 timeout.list);
Jens Axboe5262f562019-09-17 12:26:57 -06005762
Pavel Begunkov8eb7e2d2020-06-29 13:13:02 +03005763 if (io_is_timeout_noseq(nxt))
Jens Axboe93bd25b2019-11-11 23:34:31 -07005764 continue;
Pavel Begunkovbfe68a22020-05-30 14:54:18 +03005765 /* nxt.seq is behind @tail, otherwise would've been completed */
5766 if (off >= nxt->timeout.target_seq - tail)
Jens Axboe5262f562019-09-17 12:26:57 -06005767 break;
5768 }
Jens Axboe93bd25b2019-11-11 23:34:31 -07005769add:
Pavel Begunkov135fcde2020-07-13 23:37:12 +03005770 list_add(&req->timeout.list, entry);
Jens Axboead8a48a2019-11-15 08:49:11 -07005771 data->timer.function = io_timeout_fn;
5772 hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), data->mode);
Jens Axboe89850fc2021-08-10 15:11:51 -06005773 spin_unlock_irq(&ctx->timeout_lock);
Jens Axboe5262f562019-09-17 12:26:57 -06005774 return 0;
5775}
5776
Pavel Begunkovf458dd842021-03-08 12:14:14 +00005777struct io_cancel_data {
5778 struct io_ring_ctx *ctx;
5779 u64 user_data;
5780};
5781
Jens Axboe62755e32019-10-28 21:49:21 -06005782static bool io_cancel_cb(struct io_wq_work *work, void *data)
Jens Axboede0617e2019-04-06 21:51:27 -06005783{
Jens Axboe62755e32019-10-28 21:49:21 -06005784 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
Pavel Begunkovf458dd842021-03-08 12:14:14 +00005785 struct io_cancel_data *cd = data;
Jens Axboede0617e2019-04-06 21:51:27 -06005786
Pavel Begunkovf458dd842021-03-08 12:14:14 +00005787 return req->ctx == cd->ctx && req->user_data == cd->user_data;
Jens Axboe62755e32019-10-28 21:49:21 -06005788}
5789
Pavel Begunkovf458dd842021-03-08 12:14:14 +00005790static int io_async_cancel_one(struct io_uring_task *tctx, u64 user_data,
5791 struct io_ring_ctx *ctx)
Jens Axboe62755e32019-10-28 21:49:21 -06005792{
Pavel Begunkovf458dd842021-03-08 12:14:14 +00005793 struct io_cancel_data data = { .ctx = ctx, .user_data = user_data, };
Jens Axboe62755e32019-10-28 21:49:21 -06005794 enum io_wq_cancel cancel_ret;
Jens Axboe62755e32019-10-28 21:49:21 -06005795 int ret = 0;
5796
Pavel Begunkovf458dd842021-03-08 12:14:14 +00005797 if (!tctx || !tctx->io_wq)
Jens Axboe5aa75ed2021-02-16 12:56:50 -07005798 return -ENOENT;
5799
Pavel Begunkovf458dd842021-03-08 12:14:14 +00005800 cancel_ret = io_wq_cancel_cb(tctx->io_wq, io_cancel_cb, &data, false);
Jens Axboe62755e32019-10-28 21:49:21 -06005801 switch (cancel_ret) {
5802 case IO_WQ_CANCEL_OK:
5803 ret = 0;
5804 break;
5805 case IO_WQ_CANCEL_RUNNING:
5806 ret = -EALREADY;
5807 break;
5808 case IO_WQ_CANCEL_NOTFOUND:
5809 ret = -ENOENT;
5810 break;
5811 }
5812
Jens Axboee977d6d2019-11-05 12:39:45 -07005813 return ret;
5814}
5815
Pavel Begunkov8cb01fa2021-08-15 10:40:22 +01005816static int io_try_cancel_userdata(struct io_kiocb *req, u64 sqe_addr)
Jens Axboe47f46762019-11-09 17:43:02 -07005817{
Pavel Begunkov8cb01fa2021-08-15 10:40:22 +01005818 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe47f46762019-11-09 17:43:02 -07005819 int ret;
5820
Pavel Begunkov8cb01fa2021-08-15 10:40:22 +01005821 WARN_ON_ONCE(req->task != current);
5822
Pavel Begunkovf458dd842021-03-08 12:14:14 +00005823 ret = io_async_cancel_one(req->task->io_uring, sqe_addr, ctx);
Pavel Begunkovdf9727a2021-04-01 15:43:59 +01005824 if (ret != -ENOENT)
Pavel Begunkov8cb01fa2021-08-15 10:40:22 +01005825 return ret;
Pavel Begunkov505657b2021-08-17 20:28:09 +01005826
5827 spin_lock(&ctx->completion_lock);
Jens Axboe79ebeae2021-08-10 15:18:27 -06005828 spin_lock_irq(&ctx->timeout_lock);
Jens Axboe47f46762019-11-09 17:43:02 -07005829 ret = io_timeout_cancel(ctx, sqe_addr);
Jens Axboe79ebeae2021-08-10 15:18:27 -06005830 spin_unlock_irq(&ctx->timeout_lock);
Jens Axboe47f46762019-11-09 17:43:02 -07005831 if (ret != -ENOENT)
Pavel Begunkov505657b2021-08-17 20:28:09 +01005832 goto out;
5833 ret = io_poll_cancel(ctx, sqe_addr, false);
5834out:
5835 spin_unlock(&ctx->completion_lock);
5836 return ret;
Jens Axboe47f46762019-11-09 17:43:02 -07005837}
5838
Jens Axboe3529d8c2019-12-19 18:24:38 -07005839static int io_async_cancel_prep(struct io_kiocb *req,
5840 const struct io_uring_sqe *sqe)
Jens Axboee977d6d2019-11-05 12:39:45 -07005841{
Jens Axboefbf23842019-12-17 18:45:56 -07005842 if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
Jens Axboee977d6d2019-11-05 12:39:45 -07005843 return -EINVAL;
Daniele Albano61710e42020-07-18 14:15:16 -06005844 if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT)))
5845 return -EINVAL;
5846 if (sqe->ioprio || sqe->off || sqe->len || sqe->cancel_flags)
Jens Axboee977d6d2019-11-05 12:39:45 -07005847 return -EINVAL;
5848
Jens Axboefbf23842019-12-17 18:45:56 -07005849 req->cancel.addr = READ_ONCE(sqe->addr);
5850 return 0;
5851}
5852
Pavel Begunkov61e98202021-02-10 00:03:08 +00005853static int io_async_cancel(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboefbf23842019-12-17 18:45:56 -07005854{
5855 struct io_ring_ctx *ctx = req->ctx;
Pavel Begunkov58f99372021-03-12 16:25:55 +00005856 u64 sqe_addr = req->cancel.addr;
5857 struct io_tctx_node *node;
5858 int ret;
Jens Axboefbf23842019-12-17 18:45:56 -07005859
Pavel Begunkov8cb01fa2021-08-15 10:40:22 +01005860 ret = io_try_cancel_userdata(req, sqe_addr);
Pavel Begunkov58f99372021-03-12 16:25:55 +00005861 if (ret != -ENOENT)
5862 goto done;
Pavel Begunkov58f99372021-03-12 16:25:55 +00005863
5864 /* slow path, try all io-wq's */
5865 io_ring_submit_lock(ctx, !(issue_flags & IO_URING_F_NONBLOCK));
5866 ret = -ENOENT;
5867 list_for_each_entry(node, &ctx->tctx_list, ctx_node) {
5868 struct io_uring_task *tctx = node->task->io_uring;
5869
Pavel Begunkov58f99372021-03-12 16:25:55 +00005870 ret = io_async_cancel_one(tctx, req->cancel.addr, ctx);
5871 if (ret != -ENOENT)
5872 break;
5873 }
5874 io_ring_submit_unlock(ctx, !(issue_flags & IO_URING_F_NONBLOCK));
Pavel Begunkov58f99372021-03-12 16:25:55 +00005875done:
Pavel Begunkov58f99372021-03-12 16:25:55 +00005876 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01005877 req_set_fail(req);
Pavel Begunkov505657b2021-08-17 20:28:09 +01005878 io_req_complete_post(req, ret, 0);
Jens Axboe62755e32019-10-28 21:49:21 -06005879 return 0;
5880}
5881
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00005882static int io_rsrc_update_prep(struct io_kiocb *req,
Jens Axboe05f3fb32019-12-09 11:22:50 -07005883 const struct io_uring_sqe *sqe)
5884{
Daniele Albano61710e42020-07-18 14:15:16 -06005885 if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT)))
5886 return -EINVAL;
5887 if (sqe->ioprio || sqe->rw_flags)
Jens Axboe05f3fb32019-12-09 11:22:50 -07005888 return -EINVAL;
5889
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00005890 req->rsrc_update.offset = READ_ONCE(sqe->off);
5891 req->rsrc_update.nr_args = READ_ONCE(sqe->len);
5892 if (!req->rsrc_update.nr_args)
Jens Axboe05f3fb32019-12-09 11:22:50 -07005893 return -EINVAL;
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00005894 req->rsrc_update.arg = READ_ONCE(sqe->addr);
Jens Axboe05f3fb32019-12-09 11:22:50 -07005895 return 0;
5896}
5897
Pavel Begunkov889fca72021-02-10 00:03:09 +00005898static int io_files_update(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboe05f3fb32019-12-09 11:22:50 -07005899{
5900 struct io_ring_ctx *ctx = req->ctx;
Pavel Begunkovc3bdad02021-04-25 14:32:22 +01005901 struct io_uring_rsrc_update2 up;
Jens Axboe05f3fb32019-12-09 11:22:50 -07005902 int ret;
5903
Pavel Begunkov45d189c2021-02-10 00:03:07 +00005904 if (issue_flags & IO_URING_F_NONBLOCK)
Jens Axboe05f3fb32019-12-09 11:22:50 -07005905 return -EAGAIN;
Jens Axboe05f3fb32019-12-09 11:22:50 -07005906
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00005907 up.offset = req->rsrc_update.offset;
5908 up.data = req->rsrc_update.arg;
Pavel Begunkovc3bdad02021-04-25 14:32:22 +01005909 up.nr = 0;
5910 up.tags = 0;
Colin Ian King615cee42021-04-26 10:47:35 +01005911 up.resv = 0;
Jens Axboe05f3fb32019-12-09 11:22:50 -07005912
5913 mutex_lock(&ctx->uring_lock);
Pavel Begunkovfdecb662021-04-25 14:32:20 +01005914 ret = __io_register_rsrc_update(ctx, IORING_RSRC_FILE,
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +01005915 &up, req->rsrc_update.nr_args);
Jens Axboe05f3fb32019-12-09 11:22:50 -07005916 mutex_unlock(&ctx->uring_lock);
5917
5918 if (ret < 0)
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01005919 req_set_fail(req);
Pavel Begunkov889fca72021-02-10 00:03:09 +00005920 __io_req_complete(req, issue_flags, ret, 0);
Jens Axboe05f3fb32019-12-09 11:22:50 -07005921 return 0;
5922}
5923
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005924static int io_req_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
Jens Axboef67676d2019-12-02 11:03:47 -07005925{
Jens Axboed625c6e2019-12-17 19:53:05 -07005926 switch (req->opcode) {
Jens Axboee7815732019-12-17 19:45:06 -07005927 case IORING_OP_NOP:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005928 return 0;
Jens Axboef67676d2019-12-02 11:03:47 -07005929 case IORING_OP_READV:
5930 case IORING_OP_READ_FIXED:
Jens Axboe3a6820f2019-12-22 15:19:35 -07005931 case IORING_OP_READ:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005932 return io_read_prep(req, sqe);
Jens Axboef67676d2019-12-02 11:03:47 -07005933 case IORING_OP_WRITEV:
5934 case IORING_OP_WRITE_FIXED:
Jens Axboe3a6820f2019-12-22 15:19:35 -07005935 case IORING_OP_WRITE:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005936 return io_write_prep(req, sqe);
Jens Axboe0969e782019-12-17 18:40:57 -07005937 case IORING_OP_POLL_ADD:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005938 return io_poll_add_prep(req, sqe);
Jens Axboe0969e782019-12-17 18:40:57 -07005939 case IORING_OP_POLL_REMOVE:
Pavel Begunkovc5de0032021-04-14 13:38:37 +01005940 return io_poll_update_prep(req, sqe);
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07005941 case IORING_OP_FSYNC:
Pavel Begunkov1155c762021-02-18 18:29:38 +00005942 return io_fsync_prep(req, sqe);
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07005943 case IORING_OP_SYNC_FILE_RANGE:
Pavel Begunkov1155c762021-02-18 18:29:38 +00005944 return io_sfr_prep(req, sqe);
Jens Axboe03b12302019-12-02 18:50:25 -07005945 case IORING_OP_SENDMSG:
Jens Axboefddafac2020-01-04 20:19:44 -07005946 case IORING_OP_SEND:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005947 return io_sendmsg_prep(req, sqe);
Jens Axboe03b12302019-12-02 18:50:25 -07005948 case IORING_OP_RECVMSG:
Jens Axboefddafac2020-01-04 20:19:44 -07005949 case IORING_OP_RECV:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005950 return io_recvmsg_prep(req, sqe);
Jens Axboef499a022019-12-02 16:28:46 -07005951 case IORING_OP_CONNECT:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005952 return io_connect_prep(req, sqe);
Jens Axboe2d283902019-12-04 11:08:05 -07005953 case IORING_OP_TIMEOUT:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005954 return io_timeout_prep(req, sqe, false);
Jens Axboeb29472e2019-12-17 18:50:29 -07005955 case IORING_OP_TIMEOUT_REMOVE:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005956 return io_timeout_remove_prep(req, sqe);
Jens Axboefbf23842019-12-17 18:45:56 -07005957 case IORING_OP_ASYNC_CANCEL:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005958 return io_async_cancel_prep(req, sqe);
Jens Axboe2d283902019-12-04 11:08:05 -07005959 case IORING_OP_LINK_TIMEOUT:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005960 return io_timeout_prep(req, sqe, true);
Jens Axboe8ed8d3c2019-12-16 11:55:28 -07005961 case IORING_OP_ACCEPT:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005962 return io_accept_prep(req, sqe);
Jens Axboed63d1b52019-12-10 10:38:56 -07005963 case IORING_OP_FALLOCATE:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005964 return io_fallocate_prep(req, sqe);
Jens Axboe15b71ab2019-12-11 11:20:36 -07005965 case IORING_OP_OPENAT:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005966 return io_openat_prep(req, sqe);
Jens Axboeb5dba592019-12-11 14:02:38 -07005967 case IORING_OP_CLOSE:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005968 return io_close_prep(req, sqe);
Jens Axboe05f3fb32019-12-09 11:22:50 -07005969 case IORING_OP_FILES_UPDATE:
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00005970 return io_rsrc_update_prep(req, sqe);
Jens Axboeeddc7ef2019-12-13 21:18:10 -07005971 case IORING_OP_STATX:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005972 return io_statx_prep(req, sqe);
Jens Axboe4840e412019-12-25 22:03:45 -07005973 case IORING_OP_FADVISE:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005974 return io_fadvise_prep(req, sqe);
Jens Axboec1ca7572019-12-25 22:18:28 -07005975 case IORING_OP_MADVISE:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005976 return io_madvise_prep(req, sqe);
Jens Axboecebdb982020-01-08 17:59:24 -07005977 case IORING_OP_OPENAT2:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005978 return io_openat2_prep(req, sqe);
Jens Axboe3e4827b2020-01-08 15:18:09 -07005979 case IORING_OP_EPOLL_CTL:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005980 return io_epoll_ctl_prep(req, sqe);
Pavel Begunkov7d67af22020-02-24 11:32:45 +03005981 case IORING_OP_SPLICE:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005982 return io_splice_prep(req, sqe);
Jens Axboeddf0322d2020-02-23 16:41:33 -07005983 case IORING_OP_PROVIDE_BUFFERS:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005984 return io_provide_buffers_prep(req, sqe);
Jens Axboe067524e2020-03-02 16:32:28 -07005985 case IORING_OP_REMOVE_BUFFERS:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005986 return io_remove_buffers_prep(req, sqe);
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03005987 case IORING_OP_TEE:
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005988 return io_tee_prep(req, sqe);
Jens Axboe36f4fa62020-09-05 11:14:22 -06005989 case IORING_OP_SHUTDOWN:
5990 return io_shutdown_prep(req, sqe);
Jens Axboe80a261f2020-09-28 14:23:58 -06005991 case IORING_OP_RENAMEAT:
5992 return io_renameat_prep(req, sqe);
Jens Axboe14a11432020-09-28 14:27:37 -06005993 case IORING_OP_UNLINKAT:
5994 return io_unlinkat_prep(req, sqe);
Jens Axboef67676d2019-12-02 11:03:47 -07005995 }
5996
Pavel Begunkovbfe76552020-09-30 22:57:55 +03005997 printk_once(KERN_WARNING "io_uring: unhandled opcode %d\n",
5998 req->opcode);
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01005999 return -EINVAL;
Pavel Begunkovbfe76552020-09-30 22:57:55 +03006000}
6001
Pavel Begunkov93642ef2021-02-18 18:29:44 +00006002static int io_req_prep_async(struct io_kiocb *req)
Jens Axboedef596e2019-01-09 08:59:42 -07006003{
Pavel Begunkovb7e298d2021-02-28 22:35:19 +00006004 if (!io_op_defs[req->opcode].needs_async_setup)
6005 return 0;
6006 if (WARN_ON_ONCE(req->async_data))
6007 return -EFAULT;
6008 if (io_alloc_async_data(req))
6009 return -EAGAIN;
6010
Pavel Begunkov93642ef2021-02-18 18:29:44 +00006011 switch (req->opcode) {
6012 case IORING_OP_READV:
Pavel Begunkov93642ef2021-02-18 18:29:44 +00006013 return io_rw_prep_async(req, READ);
6014 case IORING_OP_WRITEV:
Pavel Begunkov93642ef2021-02-18 18:29:44 +00006015 return io_rw_prep_async(req, WRITE);
6016 case IORING_OP_SENDMSG:
Pavel Begunkov93642ef2021-02-18 18:29:44 +00006017 return io_sendmsg_prep_async(req);
6018 case IORING_OP_RECVMSG:
Pavel Begunkov93642ef2021-02-18 18:29:44 +00006019 return io_recvmsg_prep_async(req);
6020 case IORING_OP_CONNECT:
6021 return io_connect_prep_async(req);
6022 }
Pavel Begunkovb7e298d2021-02-28 22:35:19 +00006023 printk_once(KERN_WARNING "io_uring: prep_async() bad opcode %d\n",
6024 req->opcode);
6025 return -EFAULT;
Jens Axboedef596e2019-01-09 08:59:42 -07006026}
6027
Pavel Begunkov9cf7c102020-07-13 23:37:15 +03006028static u32 io_get_sequence(struct io_kiocb *req)
6029{
Pavel Begunkova3dbdf52021-06-17 18:14:05 +01006030 u32 seq = req->ctx->cached_sq_head;
Pavel Begunkov9cf7c102020-07-13 23:37:15 +03006031
Pavel Begunkova3dbdf52021-06-17 18:14:05 +01006032 /* need original cached_sq_head, but it was increased for each req */
6033 io_for_each_link(req, req)
6034 seq--;
6035 return seq;
Pavel Begunkov9cf7c102020-07-13 23:37:15 +03006036}
6037
Pavel Begunkov76cc33d2021-06-14 23:37:30 +01006038static bool io_drain_req(struct io_kiocb *req)
Jens Axboedef596e2019-01-09 08:59:42 -07006039{
Pavel Begunkov3c199662021-06-15 16:47:57 +01006040 struct io_kiocb *pos;
Jens Axboedef596e2019-01-09 08:59:42 -07006041 struct io_ring_ctx *ctx = req->ctx;
Pavel Begunkov27dc8332020-07-13 23:37:14 +03006042 struct io_defer_entry *de;
Jens Axboedef596e2019-01-09 08:59:42 -07006043 int ret;
Pavel Begunkov9cf7c102020-07-13 23:37:15 +03006044 u32 seq;
Jens Axboedef596e2019-01-09 08:59:42 -07006045
Pavel Begunkov3c199662021-06-15 16:47:57 +01006046 /*
6047 * If we need to drain a request in the middle of a link, drain the
6048 * head request and the next request/link after the current link.
6049 * Considering sequential execution of links, IOSQE_IO_DRAIN will be
6050 * maintained for every request of our link.
6051 */
6052 if (ctx->drain_next) {
6053 req->flags |= REQ_F_IO_DRAIN;
6054 ctx->drain_next = false;
6055 }
6056 /* not interested in head, start from the first linked */
6057 io_for_each_link(pos, req->link) {
6058 if (pos->flags & REQ_F_IO_DRAIN) {
6059 ctx->drain_next = true;
6060 req->flags |= REQ_F_IO_DRAIN;
6061 break;
6062 }
6063 }
6064
Jens Axboedef596e2019-01-09 08:59:42 -07006065 /* Still need defer if there is pending req in defer list. */
Pavel Begunkov9cf7c102020-07-13 23:37:15 +03006066 if (likely(list_empty_careful(&ctx->defer_list) &&
Pavel Begunkov10c66902021-06-15 16:47:56 +01006067 !(req->flags & REQ_F_IO_DRAIN))) {
6068 ctx->drain_active = false;
Pavel Begunkov76cc33d2021-06-14 23:37:30 +01006069 return false;
Pavel Begunkov10c66902021-06-15 16:47:56 +01006070 }
Pavel Begunkov9cf7c102020-07-13 23:37:15 +03006071
6072 seq = io_get_sequence(req);
6073 /* Still a chance to pass the sequence check */
6074 if (!req_need_defer(req, seq) && list_empty_careful(&ctx->defer_list))
Pavel Begunkov76cc33d2021-06-14 23:37:30 +01006075 return false;
Jens Axboedef596e2019-01-09 08:59:42 -07006076
Pavel Begunkovb7e298d2021-02-28 22:35:19 +00006077 ret = io_req_prep_async(req);
Pavel Begunkovbe7053b2021-02-18 18:29:45 +00006078 if (ret)
Pavel Begunkov1b487732021-07-11 22:41:13 +01006079 goto fail;
Pavel Begunkovcbdcb432020-06-29 19:18:43 +03006080 io_prep_async_link(req);
Pavel Begunkov27dc8332020-07-13 23:37:14 +03006081 de = kmalloc(sizeof(*de), GFP_KERNEL);
Pavel Begunkov76cc33d2021-06-14 23:37:30 +01006082 if (!de) {
Pavel Begunkov1b487732021-07-11 22:41:13 +01006083 ret = -ENOMEM;
6084fail:
6085 io_req_complete_failed(req, ret);
Pavel Begunkov76cc33d2021-06-14 23:37:30 +01006086 return true;
6087 }
Jens Axboe31b51512019-01-18 22:56:34 -07006088
Jens Axboe79ebeae2021-08-10 15:18:27 -06006089 spin_lock(&ctx->completion_lock);
Pavel Begunkov9cf7c102020-07-13 23:37:15 +03006090 if (!req_need_defer(req, seq) && list_empty(&ctx->defer_list)) {
Jens Axboe79ebeae2021-08-10 15:18:27 -06006091 spin_unlock(&ctx->completion_lock);
Pavel Begunkov27dc8332020-07-13 23:37:14 +03006092 kfree(de);
Pavel Begunkovae348172020-07-23 20:25:20 +03006093 io_queue_async_work(req);
Pavel Begunkov76cc33d2021-06-14 23:37:30 +01006094 return true;
Jens Axboe31b51512019-01-18 22:56:34 -07006095 }
6096
6097 trace_io_uring_defer(ctx, req, req->user_data);
Pavel Begunkov27dc8332020-07-13 23:37:14 +03006098 de->req = req;
Pavel Begunkov9cf7c102020-07-13 23:37:15 +03006099 de->seq = seq;
Pavel Begunkov27dc8332020-07-13 23:37:14 +03006100 list_add_tail(&de->list, &ctx->defer_list);
Jens Axboe79ebeae2021-08-10 15:18:27 -06006101 spin_unlock(&ctx->completion_lock);
Pavel Begunkov76cc33d2021-06-14 23:37:30 +01006102 return true;
Jens Axboe31b51512019-01-18 22:56:34 -07006103}
6104
Pavel Begunkov68fb8972021-03-19 17:22:41 +00006105static void io_clean_op(struct io_kiocb *req)
Pavel Begunkov99bc4c32020-02-07 22:04:45 +03006106{
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03006107 if (req->flags & REQ_F_BUFFER_SELECTED) {
6108 switch (req->opcode) {
6109 case IORING_OP_READV:
6110 case IORING_OP_READ_FIXED:
6111 case IORING_OP_READ:
Jens Axboebcda7ba2020-02-23 16:42:51 -07006112 kfree((void *)(unsigned long)req->rw.addr);
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03006113 break;
6114 case IORING_OP_RECVMSG:
6115 case IORING_OP_RECV:
Jens Axboe52de1fe2020-02-27 10:15:42 -07006116 kfree(req->sr_msg.kbuf);
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03006117 break;
6118 }
Pavel Begunkov99bc4c32020-02-07 22:04:45 +03006119 }
6120
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03006121 if (req->flags & REQ_F_NEED_CLEANUP) {
6122 switch (req->opcode) {
6123 case IORING_OP_READV:
6124 case IORING_OP_READ_FIXED:
6125 case IORING_OP_READ:
6126 case IORING_OP_WRITEV:
6127 case IORING_OP_WRITE_FIXED:
Jens Axboee8c2bc12020-08-15 18:44:09 -07006128 case IORING_OP_WRITE: {
6129 struct io_async_rw *io = req->async_data;
Pavel Begunkov1dacb4d2021-06-17 18:14:03 +01006130
6131 kfree(io->free_iovec);
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03006132 break;
Jens Axboee8c2bc12020-08-15 18:44:09 -07006133 }
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03006134 case IORING_OP_RECVMSG:
Jens Axboee8c2bc12020-08-15 18:44:09 -07006135 case IORING_OP_SENDMSG: {
6136 struct io_async_msghdr *io = req->async_data;
Pavel Begunkov257e84a2021-02-05 00:58:00 +00006137
6138 kfree(io->free_iov);
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03006139 break;
Jens Axboee8c2bc12020-08-15 18:44:09 -07006140 }
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03006141 case IORING_OP_SPLICE:
6142 case IORING_OP_TEE:
Pavel Begunkove1d767f2021-03-19 17:22:43 +00006143 if (!(req->splice.flags & SPLICE_F_FD_IN_FIXED))
6144 io_put_file(req->splice.file_in);
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03006145 break;
Jens Axboef3cd48502020-09-24 14:55:54 -06006146 case IORING_OP_OPENAT:
6147 case IORING_OP_OPENAT2:
6148 if (req->open.filename)
6149 putname(req->open.filename);
6150 break;
Jens Axboe80a261f2020-09-28 14:23:58 -06006151 case IORING_OP_RENAMEAT:
6152 putname(req->rename.oldpath);
6153 putname(req->rename.newpath);
6154 break;
Jens Axboe14a11432020-09-28 14:27:37 -06006155 case IORING_OP_UNLINKAT:
6156 putname(req->unlink.filename);
6157 break;
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03006158 }
Pavel Begunkov0e1b6fe32020-07-16 23:28:02 +03006159 }
Jens Axboe75652a302021-04-15 09:52:40 -06006160 if ((req->flags & REQ_F_POLLED) && req->apoll) {
6161 kfree(req->apoll->double_poll);
6162 kfree(req->apoll);
6163 req->apoll = NULL;
6164 }
Pavel Begunkov3a0a6902021-04-20 12:03:31 +01006165 if (req->flags & REQ_F_INFLIGHT) {
6166 struct io_uring_task *tctx = req->task->io_uring;
6167
6168 atomic_dec(&tctx->inflight_tracked);
Pavel Begunkov3a0a6902021-04-20 12:03:31 +01006169 }
Pavel Begunkovc8543572021-06-17 18:14:04 +01006170 if (req->flags & REQ_F_CREDS)
Pavel Begunkovb8e64b52021-06-17 18:14:02 +01006171 put_cred(req->creds);
Pavel Begunkovc8543572021-06-17 18:14:04 +01006172
6173 req->flags &= ~IO_REQ_CLEAN_FLAGS;
Pavel Begunkov99bc4c32020-02-07 22:04:45 +03006174}
6175
Pavel Begunkov889fca72021-02-10 00:03:09 +00006176static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags)
Jens Axboeedafcce2019-01-09 09:16:05 -07006177{
Jens Axboeedafcce2019-01-09 09:16:05 -07006178 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe5730b272021-02-27 15:57:30 -07006179 const struct cred *creds = NULL;
Jens Axboed625c6e2019-12-17 19:53:05 -07006180 int ret;
Jens Axboeedafcce2019-01-09 09:16:05 -07006181
Pavel Begunkovb8e64b52021-06-17 18:14:02 +01006182 if ((req->flags & REQ_F_CREDS) && req->creds != current_cred())
Pavel Begunkovc10d1f92021-06-17 18:14:01 +01006183 creds = override_creds(req->creds);
Jens Axboe5730b272021-02-27 15:57:30 -07006184
Jens Axboed625c6e2019-12-17 19:53:05 -07006185 switch (req->opcode) {
Jens Axboe2b188cc2019-01-07 10:46:33 -07006186 case IORING_OP_NOP:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006187 ret = io_nop(req, issue_flags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006188 break;
Jens Axboe2b188cc2019-01-07 10:46:33 -07006189 case IORING_OP_READV:
Jens Axboe3529d8c2019-12-19 18:24:38 -07006190 case IORING_OP_READ_FIXED:
Jens Axboe3a6820f2019-12-22 15:19:35 -07006191 case IORING_OP_READ:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006192 ret = io_read(req, issue_flags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006193 break;
6194 case IORING_OP_WRITEV:
Jens Axboe2b188cc2019-01-07 10:46:33 -07006195 case IORING_OP_WRITE_FIXED:
Jens Axboe3a6820f2019-12-22 15:19:35 -07006196 case IORING_OP_WRITE:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006197 ret = io_write(req, issue_flags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006198 break;
6199 case IORING_OP_FSYNC:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006200 ret = io_fsync(req, issue_flags);
Jackie Liuba5290c2019-10-09 09:19:59 +08006201 break;
Jens Axboe2b188cc2019-01-07 10:46:33 -07006202 case IORING_OP_POLL_ADD:
Pavel Begunkov61e98202021-02-10 00:03:08 +00006203 ret = io_poll_add(req, issue_flags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006204 break;
6205 case IORING_OP_POLL_REMOVE:
Pavel Begunkovc5de0032021-04-14 13:38:37 +01006206 ret = io_poll_update(req, issue_flags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006207 break;
Jens Axboeb76da702019-11-20 13:05:32 -07006208 case IORING_OP_SYNC_FILE_RANGE:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006209 ret = io_sync_file_range(req, issue_flags);
Jens Axboeb76da702019-11-20 13:05:32 -07006210 break;
6211 case IORING_OP_SENDMSG:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006212 ret = io_sendmsg(req, issue_flags);
Pavel Begunkov062d04d2020-10-10 18:34:12 +01006213 break;
Jens Axboefddafac2020-01-04 20:19:44 -07006214 case IORING_OP_SEND:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006215 ret = io_send(req, issue_flags);
Jens Axboeb76da702019-11-20 13:05:32 -07006216 break;
6217 case IORING_OP_RECVMSG:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006218 ret = io_recvmsg(req, issue_flags);
Pavel Begunkov062d04d2020-10-10 18:34:12 +01006219 break;
Jens Axboefddafac2020-01-04 20:19:44 -07006220 case IORING_OP_RECV:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006221 ret = io_recv(req, issue_flags);
Jens Axboeb76da702019-11-20 13:05:32 -07006222 break;
Jens Axboe561fb042019-10-24 07:25:42 -06006223 case IORING_OP_TIMEOUT:
Pavel Begunkov61e98202021-02-10 00:03:08 +00006224 ret = io_timeout(req, issue_flags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006225 break;
6226 case IORING_OP_TIMEOUT_REMOVE:
Pavel Begunkov61e98202021-02-10 00:03:08 +00006227 ret = io_timeout_remove(req, issue_flags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006228 break;
6229 case IORING_OP_ACCEPT:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006230 ret = io_accept(req, issue_flags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006231 break;
6232 case IORING_OP_CONNECT:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006233 ret = io_connect(req, issue_flags);
Jens Axboe31b51512019-01-18 22:56:34 -07006234 break;
6235 case IORING_OP_ASYNC_CANCEL:
Pavel Begunkov61e98202021-02-10 00:03:08 +00006236 ret = io_async_cancel(req, issue_flags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006237 break;
Jens Axboed63d1b52019-12-10 10:38:56 -07006238 case IORING_OP_FALLOCATE:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006239 ret = io_fallocate(req, issue_flags);
Jens Axboed63d1b52019-12-10 10:38:56 -07006240 break;
Jens Axboe15b71ab2019-12-11 11:20:36 -07006241 case IORING_OP_OPENAT:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006242 ret = io_openat(req, issue_flags);
Jens Axboe15b71ab2019-12-11 11:20:36 -07006243 break;
Jens Axboeb5dba592019-12-11 14:02:38 -07006244 case IORING_OP_CLOSE:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006245 ret = io_close(req, issue_flags);
Jens Axboeb5dba592019-12-11 14:02:38 -07006246 break;
Jens Axboe05f3fb32019-12-09 11:22:50 -07006247 case IORING_OP_FILES_UPDATE:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006248 ret = io_files_update(req, issue_flags);
Jens Axboe05f3fb32019-12-09 11:22:50 -07006249 break;
Jens Axboeeddc7ef2019-12-13 21:18:10 -07006250 case IORING_OP_STATX:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006251 ret = io_statx(req, issue_flags);
Jens Axboeeddc7ef2019-12-13 21:18:10 -07006252 break;
Jens Axboe4840e412019-12-25 22:03:45 -07006253 case IORING_OP_FADVISE:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006254 ret = io_fadvise(req, issue_flags);
Jens Axboe4840e412019-12-25 22:03:45 -07006255 break;
Jens Axboec1ca7572019-12-25 22:18:28 -07006256 case IORING_OP_MADVISE:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006257 ret = io_madvise(req, issue_flags);
Jens Axboec1ca7572019-12-25 22:18:28 -07006258 break;
Jens Axboecebdb982020-01-08 17:59:24 -07006259 case IORING_OP_OPENAT2:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006260 ret = io_openat2(req, issue_flags);
Jens Axboecebdb982020-01-08 17:59:24 -07006261 break;
Jens Axboe3e4827b2020-01-08 15:18:09 -07006262 case IORING_OP_EPOLL_CTL:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006263 ret = io_epoll_ctl(req, issue_flags);
Jens Axboe3e4827b2020-01-08 15:18:09 -07006264 break;
Pavel Begunkov7d67af22020-02-24 11:32:45 +03006265 case IORING_OP_SPLICE:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006266 ret = io_splice(req, issue_flags);
Pavel Begunkov7d67af22020-02-24 11:32:45 +03006267 break;
Jens Axboeddf0322d2020-02-23 16:41:33 -07006268 case IORING_OP_PROVIDE_BUFFERS:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006269 ret = io_provide_buffers(req, issue_flags);
Jens Axboeddf0322d2020-02-23 16:41:33 -07006270 break;
Jens Axboe067524e2020-03-02 16:32:28 -07006271 case IORING_OP_REMOVE_BUFFERS:
Pavel Begunkov889fca72021-02-10 00:03:09 +00006272 ret = io_remove_buffers(req, issue_flags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006273 break;
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03006274 case IORING_OP_TEE:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006275 ret = io_tee(req, issue_flags);
Pavel Begunkovf2a8d5c2020-05-17 14:18:06 +03006276 break;
Jens Axboe36f4fa62020-09-05 11:14:22 -06006277 case IORING_OP_SHUTDOWN:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006278 ret = io_shutdown(req, issue_flags);
Jens Axboe36f4fa62020-09-05 11:14:22 -06006279 break;
Jens Axboe80a261f2020-09-28 14:23:58 -06006280 case IORING_OP_RENAMEAT:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006281 ret = io_renameat(req, issue_flags);
Jens Axboe80a261f2020-09-28 14:23:58 -06006282 break;
Jens Axboe14a11432020-09-28 14:27:37 -06006283 case IORING_OP_UNLINKAT:
Pavel Begunkov45d189c2021-02-10 00:03:07 +00006284 ret = io_unlinkat(req, issue_flags);
Jens Axboe14a11432020-09-28 14:27:37 -06006285 break;
Jens Axboe2b188cc2019-01-07 10:46:33 -07006286 default:
6287 ret = -EINVAL;
6288 break;
6289 }
Jens Axboe31b51512019-01-18 22:56:34 -07006290
Jens Axboe5730b272021-02-27 15:57:30 -07006291 if (creds)
6292 revert_creds(creds);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006293 if (ret)
6294 return ret;
Jens Axboeb5325762020-05-19 21:20:27 -06006295 /* If the op doesn't have a file, we're not polling for it */
Pavel Begunkovcb3d8972021-06-14 02:36:14 +01006296 if ((ctx->flags & IORING_SETUP_IOPOLL) && req->file)
6297 io_iopoll_req_issued(req);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006298
6299 return 0;
6300}
6301
Pavel Begunkovebc11b62021-08-09 13:04:05 +01006302static struct io_wq_work *io_wq_free_work(struct io_wq_work *work)
6303{
6304 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
6305
6306 req = io_put_req_find_next(req);
6307 return req ? &req->work : NULL;
6308}
6309
Pavel Begunkov5280f7e2021-02-04 13:52:08 +00006310static void io_wq_submit_work(struct io_wq_work *work)
Pavel Begunkovd4c81f32020-06-08 21:08:19 +03006311{
Jens Axboe2b188cc2019-01-07 10:46:33 -07006312 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
Pavel Begunkov6df1db62020-07-03 22:15:06 +03006313 struct io_kiocb *timeout;
Jens Axboe561fb042019-10-24 07:25:42 -06006314 int ret = 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07006315
Pavel Begunkov48dcd382021-08-15 10:40:18 +01006316 /* one will be dropped by ->io_free_work() after returning to io-wq */
6317 if (!(req->flags & REQ_F_REFCOUNT))
6318 __io_req_set_refcount(req, 2);
6319 else
6320 req_ref_get(req);
Pavel Begunkov5d5901a2021-08-11 19:28:29 +01006321
Pavel Begunkov6df1db62020-07-03 22:15:06 +03006322 timeout = io_prep_linked_timeout(req);
6323 if (timeout)
6324 io_queue_linked_timeout(timeout);
Pavel Begunkovd4c81f32020-06-08 21:08:19 +03006325
Jens Axboe4014d942021-01-19 15:53:54 -07006326 if (work->flags & IO_WQ_WORK_CANCEL)
Jens Axboe561fb042019-10-24 07:25:42 -06006327 ret = -ECANCELED;
Jens Axboe31b51512019-01-18 22:56:34 -07006328
Jens Axboe561fb042019-10-24 07:25:42 -06006329 if (!ret) {
Jens Axboe561fb042019-10-24 07:25:42 -06006330 do {
Pavel Begunkov889fca72021-02-10 00:03:09 +00006331 ret = io_issue_sqe(req, 0);
Jens Axboe561fb042019-10-24 07:25:42 -06006332 /*
6333 * We can get EAGAIN for polled IO even though we're
6334 * forcing a sync submission from here, since we can't
6335 * wait for request slots on the block side.
6336 */
6337 if (ret != -EAGAIN)
6338 break;
6339 cond_resched();
6340 } while (1);
6341 }
Jens Axboe31b51512019-01-18 22:56:34 -07006342
Pavel Begunkova3df76982021-02-18 22:32:52 +00006343 /* avoid locking problems by failing it from a clean context */
Pavel Begunkov5d5901a2021-08-11 19:28:29 +01006344 if (ret)
Pavel Begunkova3df76982021-02-18 22:32:52 +00006345 io_req_task_queue_fail(req, ret);
Jens Axboe31b51512019-01-18 22:56:34 -07006346}
Jens Axboe2b188cc2019-01-07 10:46:33 -07006347
Pavel Begunkovaeca2412021-04-11 01:46:37 +01006348static inline struct io_fixed_file *io_fixed_file_slot(struct io_file_table *table,
Pavel Begunkov042b0d82021-08-09 13:04:01 +01006349 unsigned i)
Jens Axboe09bb8392019-03-13 12:39:28 -06006350{
Pavel Begunkov042b0d82021-08-09 13:04:01 +01006351 return &table->files[i];
Pavel Begunkovdafecf12021-02-28 22:35:11 +00006352}
6353
Jens Axboe09bb8392019-03-13 12:39:28 -06006354static inline struct file *io_file_from_index(struct io_ring_ctx *ctx,
6355 int index)
6356{
Pavel Begunkovaeca2412021-04-11 01:46:37 +01006357 struct io_fixed_file *slot = io_fixed_file_slot(&ctx->file_table, index);
Jens Axboe65e19f52019-10-26 07:20:21 -06006358
Pavel Begunkova04b0ac2021-04-01 15:44:04 +01006359 return (struct file *) (slot->file_ptr & FFS_MASK);
Jens Axboe65e19f52019-10-26 07:20:21 -06006360}
6361
Pavel Begunkova04b0ac2021-04-01 15:44:04 +01006362static void io_fixed_file_set(struct io_fixed_file *file_slot, struct file *file)
Pavel Begunkov9a321c92021-04-01 15:44:01 +01006363{
6364 unsigned long file_ptr = (unsigned long) file;
6365
Pavel Begunkovb191e2d2021-08-09 13:04:03 +01006366 if (__io_file_supports_nowait(file, READ))
Pavel Begunkov9a321c92021-04-01 15:44:01 +01006367 file_ptr |= FFS_ASYNC_READ;
Pavel Begunkovb191e2d2021-08-09 13:04:03 +01006368 if (__io_file_supports_nowait(file, WRITE))
Pavel Begunkov9a321c92021-04-01 15:44:01 +01006369 file_ptr |= FFS_ASYNC_WRITE;
6370 if (S_ISREG(file_inode(file)->i_mode))
6371 file_ptr |= FFS_ISREG;
Pavel Begunkova04b0ac2021-04-01 15:44:04 +01006372 file_slot->file_ptr = file_ptr;
Jens Axboe09bb8392019-03-13 12:39:28 -06006373}
6374
Pavel Begunkovac177052021-08-09 13:04:02 +01006375static inline struct file *io_file_get_fixed(struct io_ring_ctx *ctx,
6376 struct io_kiocb *req, int fd)
Pavel Begunkov8da11c12020-02-24 11:32:44 +03006377{
Pavel Begunkov8da11c12020-02-24 11:32:44 +03006378 struct file *file;
Pavel Begunkovac177052021-08-09 13:04:02 +01006379 unsigned long file_ptr;
Pavel Begunkov8da11c12020-02-24 11:32:44 +03006380
Pavel Begunkovac177052021-08-09 13:04:02 +01006381 if (unlikely((unsigned int)fd >= ctx->nr_user_files))
6382 return NULL;
6383 fd = array_index_nospec(fd, ctx->nr_user_files);
6384 file_ptr = io_fixed_file_slot(&ctx->file_table, fd)->file_ptr;
6385 file = (struct file *) (file_ptr & FFS_MASK);
6386 file_ptr &= ~FFS_MASK;
6387 /* mask in overlapping REQ_F and FFS bits */
Pavel Begunkovb191e2d2021-08-09 13:04:03 +01006388 req->flags |= (file_ptr << REQ_F_NOWAIT_READ_BIT);
Pavel Begunkovac177052021-08-09 13:04:02 +01006389 io_req_set_rsrc_node(req);
Pavel Begunkov8371adf2020-10-10 18:34:08 +01006390 return file;
Pavel Begunkov8da11c12020-02-24 11:32:44 +03006391}
6392
Pavel Begunkovac177052021-08-09 13:04:02 +01006393static struct file *io_file_get_normal(struct io_ring_ctx *ctx,
Pavel Begunkovac177052021-08-09 13:04:02 +01006394 struct io_kiocb *req, int fd)
6395{
Pavel Begunkov62906e82021-08-10 14:52:47 +01006396 struct file *file = fget(fd);
Pavel Begunkovac177052021-08-09 13:04:02 +01006397
6398 trace_io_uring_file_get(ctx, fd);
6399
6400 /* we don't allow fixed io_uring files */
6401 if (file && unlikely(file->f_op == &io_uring_fops))
6402 io_req_track_inflight(req);
6403 return file;
6404}
6405
6406static inline struct file *io_file_get(struct io_ring_ctx *ctx,
Pavel Begunkovac177052021-08-09 13:04:02 +01006407 struct io_kiocb *req, int fd, bool fixed)
6408{
6409 if (fixed)
6410 return io_file_get_fixed(ctx, req, fd);
6411 else
Pavel Begunkov62906e82021-08-10 14:52:47 +01006412 return io_file_get_normal(ctx, req, fd);
Pavel Begunkovac177052021-08-09 13:04:02 +01006413}
6414
Jens Axboe89b263f2021-08-10 15:14:18 -06006415static void io_req_task_link_timeout(struct io_kiocb *req)
6416{
6417 struct io_kiocb *prev = req->timeout.prev;
Pavel Begunkov8cb01fa2021-08-15 10:40:22 +01006418 int ret;
Jens Axboe89b263f2021-08-10 15:14:18 -06006419
6420 if (prev) {
Pavel Begunkov8cb01fa2021-08-15 10:40:22 +01006421 ret = io_try_cancel_userdata(req, prev->user_data);
Pavel Begunkov505657b2021-08-17 20:28:09 +01006422 io_req_complete_post(req, ret ?: -ETIME, 0);
Jens Axboe89b263f2021-08-10 15:14:18 -06006423 io_put_req(prev);
Jens Axboe89b263f2021-08-10 15:14:18 -06006424 } else {
6425 io_req_complete_post(req, -ETIME, 0);
6426 }
6427}
6428
Jens Axboe2665abf2019-11-05 12:40:47 -07006429static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer)
6430{
Jens Axboead8a48a2019-11-15 08:49:11 -07006431 struct io_timeout_data *data = container_of(timer,
6432 struct io_timeout_data, timer);
Pavel Begunkov90cd7e42020-10-27 23:25:36 +00006433 struct io_kiocb *prev, *req = data->req;
Jens Axboe2665abf2019-11-05 12:40:47 -07006434 struct io_ring_ctx *ctx = req->ctx;
Jens Axboe2665abf2019-11-05 12:40:47 -07006435 unsigned long flags;
Jens Axboe2665abf2019-11-05 12:40:47 -07006436
Jens Axboe89b263f2021-08-10 15:14:18 -06006437 spin_lock_irqsave(&ctx->timeout_lock, flags);
Pavel Begunkov90cd7e42020-10-27 23:25:36 +00006438 prev = req->timeout.head;
6439 req->timeout.head = NULL;
Jens Axboe2665abf2019-11-05 12:40:47 -07006440
6441 /*
6442 * We don't expect the list to be empty, that will only happen if we
6443 * race with the completion of the linked work.
6444 */
Pavel Begunkov447c19f2021-05-14 12:02:50 +01006445 if (prev) {
Pavel Begunkovf2f87372020-10-27 23:25:37 +00006446 io_remove_next_linked(prev);
Pavel Begunkov447c19f2021-05-14 12:02:50 +01006447 if (!req_ref_inc_not_zero(prev))
6448 prev = NULL;
6449 }
Jens Axboe89b263f2021-08-10 15:14:18 -06006450 req->timeout.prev = prev;
6451 spin_unlock_irqrestore(&ctx->timeout_lock, flags);
Jens Axboe2665abf2019-11-05 12:40:47 -07006452
Jens Axboe89b263f2021-08-10 15:14:18 -06006453 req->io_task_work.func = io_req_task_link_timeout;
6454 io_req_task_work_add(req);
Jens Axboe2665abf2019-11-05 12:40:47 -07006455 return HRTIMER_NORESTART;
6456}
6457
Pavel Begunkovde968c12021-03-19 17:22:33 +00006458static void io_queue_linked_timeout(struct io_kiocb *req)
Jens Axboe2665abf2019-11-05 12:40:47 -07006459{
Pavel Begunkovde968c12021-03-19 17:22:33 +00006460 struct io_ring_ctx *ctx = req->ctx;
6461
Jens Axboe89b263f2021-08-10 15:14:18 -06006462 spin_lock_irq(&ctx->timeout_lock);
Jens Axboe76a46e02019-11-10 23:34:16 -07006463 /*
Pavel Begunkovf2f87372020-10-27 23:25:37 +00006464 * If the back reference is NULL, then our linked request finished
6465 * before we got a chance to setup the timer
Jens Axboe76a46e02019-11-10 23:34:16 -07006466 */
Pavel Begunkov90cd7e42020-10-27 23:25:36 +00006467 if (req->timeout.head) {
Jens Axboee8c2bc12020-08-15 18:44:09 -07006468 struct io_timeout_data *data = req->async_data;
Jens Axboe94ae5e72019-11-14 19:39:52 -07006469
Jens Axboead8a48a2019-11-15 08:49:11 -07006470 data->timer.function = io_link_timeout_fn;
6471 hrtimer_start(&data->timer, timespec64_to_ktime(data->ts),
6472 data->mode);
Jens Axboe2665abf2019-11-05 12:40:47 -07006473 }
Jens Axboe89b263f2021-08-10 15:14:18 -06006474 spin_unlock_irq(&ctx->timeout_lock);
Jens Axboe2665abf2019-11-05 12:40:47 -07006475 /* drop submission reference */
Jens Axboe76a46e02019-11-10 23:34:16 -07006476 io_put_req(req);
Jens Axboe2665abf2019-11-05 12:40:47 -07006477}
6478
Pavel Begunkovc5eef2b2021-02-10 00:03:22 +00006479static void __io_queue_sqe(struct io_kiocb *req)
Pavel Begunkov282cdc82021-08-09 13:04:10 +01006480 __must_hold(&req->ctx->uring_lock)
Jens Axboe2b188cc2019-01-07 10:46:33 -07006481{
Pavel Begunkov906c6ca2021-08-15 10:40:26 +01006482 struct io_kiocb *linked_timeout;
Jens Axboee0c5c572019-03-12 10:18:47 -06006483 int ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07006484
Olivier Langlois59b735a2021-06-22 05:17:39 -07006485issue_sqe:
Pavel Begunkovc5eef2b2021-02-10 00:03:22 +00006486 ret = io_issue_sqe(req, IO_URING_F_NONBLOCK|IO_URING_F_COMPLETE_DEFER);
Jens Axboe491381ce2019-10-17 09:20:46 -06006487
6488 /*
6489 * We async punt it if the file wasn't marked NOWAIT, or if the file
6490 * doesn't support non-blocking read/write attempts
6491 */
Pavel Begunkov18400382021-03-19 17:22:34 +00006492 if (likely(!ret)) {
Pavel Begunkove342c802021-01-19 13:32:47 +00006493 if (req->flags & REQ_F_COMPLETE_INLINE) {
Pavel Begunkovc5eef2b2021-02-10 00:03:22 +00006494 struct io_ring_ctx *ctx = req->ctx;
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01006495 struct io_submit_state *state = &ctx->submit_state;
Jens Axboee65ef562019-03-12 10:16:44 -06006496
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01006497 state->compl_reqs[state->compl_nr++] = req;
6498 if (state->compl_nr == ARRAY_SIZE(state->compl_reqs))
Pavel Begunkov2a2758f2021-06-17 18:14:00 +01006499 io_submit_flush_completions(ctx);
Pavel Begunkov906c6ca2021-08-15 10:40:26 +01006500 return;
Pavel Begunkov0d63c142020-10-22 16:47:18 +01006501 }
Pavel Begunkov906c6ca2021-08-15 10:40:26 +01006502
6503 linked_timeout = io_prep_linked_timeout(req);
6504 if (linked_timeout)
6505 io_queue_linked_timeout(linked_timeout);
Pavel Begunkov18400382021-03-19 17:22:34 +00006506 } else if (ret == -EAGAIN && !(req->flags & REQ_F_NOWAIT)) {
Pavel Begunkov906c6ca2021-08-15 10:40:26 +01006507 linked_timeout = io_prep_linked_timeout(req);
6508
Olivier Langlois59b735a2021-06-22 05:17:39 -07006509 switch (io_arm_poll_handler(req)) {
6510 case IO_APOLL_READY:
Pavel Begunkov906c6ca2021-08-15 10:40:26 +01006511 if (linked_timeout)
6512 io_unprep_linked_timeout(req);
Olivier Langlois59b735a2021-06-22 05:17:39 -07006513 goto issue_sqe;
6514 case IO_APOLL_ABORTED:
Pavel Begunkov18400382021-03-19 17:22:34 +00006515 /*
6516 * Queued up for async execution, worker will release
6517 * submit reference when the iocb is actually submitted.
6518 */
6519 io_queue_async_work(req);
Olivier Langlois59b735a2021-06-22 05:17:39 -07006520 break;
Pavel Begunkov18400382021-03-19 17:22:34 +00006521 }
Pavel Begunkov906c6ca2021-08-15 10:40:26 +01006522
6523 if (linked_timeout)
6524 io_queue_linked_timeout(linked_timeout);
Pavel Begunkov0d63c142020-10-22 16:47:18 +01006525 } else {
Pavel Begunkovf41db2732021-02-28 22:35:12 +00006526 io_req_complete_failed(req, ret);
Jens Axboe9e645e112019-05-10 16:07:28 -06006527 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07006528}
6529
Pavel Begunkov441b8a72021-06-14 23:37:31 +01006530static inline void io_queue_sqe(struct io_kiocb *req)
Pavel Begunkov282cdc82021-08-09 13:04:10 +01006531 __must_hold(&req->ctx->uring_lock)
Jackie Liu4fe2c962019-09-09 20:50:40 +08006532{
Pavel Begunkov10c66902021-06-15 16:47:56 +01006533 if (unlikely(req->ctx->drain_active) && io_drain_req(req))
Pavel Begunkov76cc33d2021-06-14 23:37:30 +01006534 return;
Jackie Liu4fe2c962019-09-09 20:50:40 +08006535
Pavel Begunkov76cc33d2021-06-14 23:37:30 +01006536 if (likely(!(req->flags & REQ_F_FORCE_ASYNC))) {
Pavel Begunkovc5eef2b2021-02-10 00:03:22 +00006537 __io_queue_sqe(req);
Pavel Begunkov76cc33d2021-06-14 23:37:30 +01006538 } else {
6539 int ret = io_req_prep_async(req);
6540
6541 if (unlikely(ret))
6542 io_req_complete_failed(req, ret);
6543 else
6544 io_queue_async_work(req);
Jens Axboece35a472019-12-17 08:04:44 -07006545 }
Jackie Liu4fe2c962019-09-09 20:50:40 +08006546}
6547
Stefano Garzarella21b55db2020-08-27 16:58:30 +02006548/*
6549 * Check SQE restrictions (opcode and flags).
6550 *
6551 * Returns 'true' if SQE is allowed, 'false' otherwise.
6552 */
6553static inline bool io_check_restriction(struct io_ring_ctx *ctx,
6554 struct io_kiocb *req,
6555 unsigned int sqe_flags)
6556{
Pavel Begunkov4cfb25b2021-06-26 21:40:47 +01006557 if (likely(!ctx->restricted))
Stefano Garzarella21b55db2020-08-27 16:58:30 +02006558 return true;
6559
6560 if (!test_bit(req->opcode, ctx->restrictions.sqe_op))
6561 return false;
6562
6563 if ((sqe_flags & ctx->restrictions.sqe_flags_required) !=
6564 ctx->restrictions.sqe_flags_required)
6565 return false;
6566
6567 if (sqe_flags & ~(ctx->restrictions.sqe_flags_allowed |
6568 ctx->restrictions.sqe_flags_required))
6569 return false;
6570
6571 return true;
6572}
6573
Pavel Begunkovef4ff582020-04-12 02:05:05 +03006574static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,
Pavel Begunkov258b29a2021-02-10 00:03:10 +00006575 const struct io_uring_sqe *sqe)
Pavel Begunkov282cdc82021-08-09 13:04:10 +01006576 __must_hold(&ctx->uring_lock)
Pavel Begunkov0553b8b2020-04-08 08:58:45 +03006577{
Pavel Begunkov258b29a2021-02-10 00:03:10 +00006578 struct io_submit_state *state;
Pavel Begunkovef4ff582020-04-12 02:05:05 +03006579 unsigned int sqe_flags;
Jens Axboe003e8dc2021-03-06 09:22:27 -07006580 int personality, ret = 0;
Pavel Begunkovef4ff582020-04-12 02:05:05 +03006581
Pavel Begunkov864ea922021-08-09 13:04:08 +01006582 /* req is partially pre-initialised, see io_preinit_req() */
Pavel Begunkov0553b8b2020-04-08 08:58:45 +03006583 req->opcode = READ_ONCE(sqe->opcode);
Pavel Begunkov5be9ad12021-02-12 18:41:17 +00006584 /* same numerical values with corresponding REQ_F_*, safe to copy */
6585 req->flags = sqe_flags = READ_ONCE(sqe->flags);
Pavel Begunkov0553b8b2020-04-08 08:58:45 +03006586 req->user_data = READ_ONCE(sqe->user_data);
Pavel Begunkov0553b8b2020-04-08 08:58:45 +03006587 req->file = NULL;
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00006588 req->fixed_rsrc_refs = NULL;
Pavel Begunkov4dd28242020-06-15 10:33:13 +03006589 req->task = current;
Pavel Begunkovef4ff582020-04-12 02:05:05 +03006590
Pavel Begunkov5be9ad12021-02-12 18:41:17 +00006591 /* enforce forwards compatibility on users */
Pavel Begunkovdddca222021-04-27 16:13:52 +01006592 if (unlikely(sqe_flags & ~SQE_VALID_FLAGS))
Pavel Begunkov5be9ad12021-02-12 18:41:17 +00006593 return -EINVAL;
Pavel Begunkovef4ff582020-04-12 02:05:05 +03006594 if (unlikely(req->opcode >= IORING_OP_LAST))
6595 return -EINVAL;
Pavel Begunkov4cfb25b2021-06-26 21:40:47 +01006596 if (!io_check_restriction(ctx, req, sqe_flags))
Stefano Garzarella21b55db2020-08-27 16:58:30 +02006597 return -EACCES;
6598
Pavel Begunkovef4ff582020-04-12 02:05:05 +03006599 if ((sqe_flags & IOSQE_BUFFER_SELECT) &&
6600 !io_op_defs[req->opcode].buffer_select)
6601 return -EOPNOTSUPP;
Pavel Begunkov3c199662021-06-15 16:47:57 +01006602 if (unlikely(sqe_flags & IOSQE_IO_DRAIN))
6603 ctx->drain_active = true;
Pavel Begunkovef4ff582020-04-12 02:05:05 +03006604
Jens Axboe003e8dc2021-03-06 09:22:27 -07006605 personality = READ_ONCE(sqe->personality);
6606 if (personality) {
Pavel Begunkovc10d1f92021-06-17 18:14:01 +01006607 req->creds = xa_load(&ctx->personalities, personality);
6608 if (!req->creds)
Jens Axboe003e8dc2021-03-06 09:22:27 -07006609 return -EINVAL;
Pavel Begunkovc10d1f92021-06-17 18:14:01 +01006610 get_cred(req->creds);
Pavel Begunkovb8e64b52021-06-17 18:14:02 +01006611 req->flags |= REQ_F_CREDS;
Jens Axboe003e8dc2021-03-06 09:22:27 -07006612 }
Pavel Begunkov258b29a2021-02-10 00:03:10 +00006613 state = &ctx->submit_state;
Pavel Begunkovef4ff582020-04-12 02:05:05 +03006614
Jens Axboe27926b62020-10-28 09:33:23 -06006615 /*
6616 * Plug now if we have more than 1 IO left after this, and the target
6617 * is potentially a read/write to block based storage.
6618 */
6619 if (!state->plug_started && state->ios_left > 1 &&
6620 io_op_defs[req->opcode].plug) {
6621 blk_start_plug(&state->plug);
6622 state->plug_started = true;
6623 }
Jens Axboe63ff8222020-05-07 14:56:15 -06006624
Pavel Begunkovbd5bbda2020-11-20 15:50:51 +00006625 if (io_op_defs[req->opcode].needs_file) {
Pavel Begunkov62906e82021-08-10 14:52:47 +01006626 req->file = io_file_get(ctx, req, READ_ONCE(sqe->fd),
Pavel Begunkovac177052021-08-09 13:04:02 +01006627 (sqe_flags & IOSQE_FIXED_FILE));
Pavel Begunkovba13e232021-02-01 18:59:52 +00006628 if (unlikely(!req->file))
Pavel Begunkovbd5bbda2020-11-20 15:50:51 +00006629 ret = -EBADF;
6630 }
6631
Pavel Begunkov71b547c2020-10-10 18:34:09 +01006632 state->ios_left--;
6633 return ret;
Pavel Begunkov0553b8b2020-04-08 08:58:45 +03006634}
6635
Pavel Begunkova6b8cadc2021-02-18 18:29:41 +00006636static int io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,
Pavel Begunkova1ab7b32021-02-18 18:29:42 +00006637 const struct io_uring_sqe *sqe)
Pavel Begunkov282cdc82021-08-09 13:04:10 +01006638 __must_hold(&ctx->uring_lock)
Jens Axboe6c271ce2019-01-10 11:22:30 -07006639{
Pavel Begunkova1ab7b32021-02-18 18:29:42 +00006640 struct io_submit_link *link = &ctx->submit_state.link;
Jens Axboe6c271ce2019-01-10 11:22:30 -07006641 int ret;
6642
Pavel Begunkova6b8cadc2021-02-18 18:29:41 +00006643 ret = io_init_req(ctx, req, sqe);
6644 if (unlikely(ret)) {
6645fail_req:
Pavel Begunkovde59bc12021-02-18 18:29:47 +00006646 if (link->head) {
6647 /* fail even hard links since we don't submit */
Pavel Begunkov93d2bcd2021-05-16 22:58:05 +01006648 req_set_fail(link->head);
Pavel Begunkovf41db2732021-02-28 22:35:12 +00006649 io_req_complete_failed(link->head, -ECANCELED);
Pavel Begunkovde59bc12021-02-18 18:29:47 +00006650 link->head = NULL;
6651 }
Pavel Begunkovf41db2732021-02-28 22:35:12 +00006652 io_req_complete_failed(req, ret);
Pavel Begunkova6b8cadc2021-02-18 18:29:41 +00006653 return ret;
6654 }
Pavel Begunkov441b8a72021-06-14 23:37:31 +01006655
Pavel Begunkovbe7053b2021-02-18 18:29:45 +00006656 ret = io_req_prep(req, sqe);
6657 if (unlikely(ret))
6658 goto fail_req;
Pavel Begunkova6b8cadc2021-02-18 18:29:41 +00006659
Pavel Begunkovbe7053b2021-02-18 18:29:45 +00006660 /* don't need @sqe from now on */
Olivier Langlois236daeae2021-05-31 02:36:37 -04006661 trace_io_uring_submit_sqe(ctx, req, req->opcode, req->user_data,
6662 req->flags, true,
6663 ctx->flags & IORING_SETUP_SQPOLL);
Pavel Begunkova6b8cadc2021-02-18 18:29:41 +00006664
Jens Axboe6c271ce2019-01-10 11:22:30 -07006665 /*
6666 * If we already have a head request, queue this one for async
6667 * submittal once the head completes. If we don't have a head but
6668 * IOSQE_IO_LINK is set in the sqe, start a new head. This one will be
6669 * submitted sync once the chain is complete. If none of those
6670 * conditions are true (normal request), then just queue it.
6671 */
6672 if (link->head) {
6673 struct io_kiocb *head = link->head;
6674
Pavel Begunkovb7e298d2021-02-28 22:35:19 +00006675 ret = io_req_prep_async(req);
Pavel Begunkovcf109602021-02-18 18:29:43 +00006676 if (unlikely(ret))
Pavel Begunkova6b8cadc2021-02-18 18:29:41 +00006677 goto fail_req;
Jens Axboe6c271ce2019-01-10 11:22:30 -07006678 trace_io_uring_link(ctx, req, head);
6679 link->last->link = req;
6680 link->last = req;
6681
6682 /* last request of a link, enqueue the link */
6683 if (!(req->flags & (REQ_F_LINK | REQ_F_HARDLINK))) {
6684 link->head = NULL;
Pavel Begunkov5e159202021-06-14 23:37:26 +01006685 io_queue_sqe(head);
Jens Axboe2b188cc2019-01-07 10:46:33 -07006686 }
Jackie Liu4fe2c962019-09-09 20:50:40 +08006687 } else {
Jens Axboe2b188cc2019-01-07 10:46:33 -07006688 if (req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) {
Jackie Liu4fe2c962019-09-09 20:50:40 +08006689 link->head = req;
6690 link->last = req;
6691 } else {
Pavel Begunkovbe7053b2021-02-18 18:29:45 +00006692 io_queue_sqe(req);
Jackie Liu4fe2c962019-09-09 20:50:40 +08006693 }
6694 }
6695
6696 return 0;
6697}
6698
6699/*
6700 * Batched submission is done, ensure local IO is flushed out.
6701 */
6702static void io_submit_state_end(struct io_submit_state *state,
6703 struct io_ring_ctx *ctx)
Pavel Begunkov1b4a51b2019-11-21 11:54:28 +03006704{
Pavel Begunkova1ab7b32021-02-18 18:29:42 +00006705 if (state->link.head)
Pavel Begunkovde59bc12021-02-18 18:29:47 +00006706 io_queue_sqe(state->link.head);
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01006707 if (state->compl_nr)
Pavel Begunkov2a2758f2021-06-17 18:14:00 +01006708 io_submit_flush_completions(ctx);
Jackie Liua197f662019-11-08 08:09:12 -07006709 if (state->plug_started)
Pavel Begunkov32fe5252019-12-17 22:26:58 +03006710 blk_finish_plug(&state->plug);
Jens Axboe9e645e112019-05-10 16:07:28 -06006711}
Pavel Begunkov32fe5252019-12-17 22:26:58 +03006712
Jens Axboe9e645e112019-05-10 16:07:28 -06006713/*
6714 * Start submission side cache.
Pavel Begunkov32fe5252019-12-17 22:26:58 +03006715 */
Jens Axboe9e645e112019-05-10 16:07:28 -06006716static void io_submit_state_start(struct io_submit_state *state,
Pavel Begunkov196be952019-11-07 01:41:06 +03006717 unsigned int max_ios)
Jens Axboe9e645e112019-05-10 16:07:28 -06006718{
6719 state->plug_started = false;
Jens Axboebcda7ba2020-02-23 16:42:51 -07006720 state->ios_left = max_ios;
Pavel Begunkova1ab7b32021-02-18 18:29:42 +00006721 /* set only head, no need to init link_last in advance */
6722 state->link.head = NULL;
Jens Axboe75c6a032020-01-28 10:15:23 -07006723}
6724
Jens Axboe193155c2020-02-22 23:22:19 -07006725static void io_commit_sqring(struct io_ring_ctx *ctx)
6726{
Jens Axboe75c6a032020-01-28 10:15:23 -07006727 struct io_rings *rings = ctx->rings;
6728
6729 /*
Jens Axboe193155c2020-02-22 23:22:19 -07006730 * Ensure any loads from the SQEs are done at this point,
Jens Axboe75c6a032020-01-28 10:15:23 -07006731 * since once we write the new head, the application could
6732 * write new data to them.
Pavel Begunkov6b47ee62020-01-18 20:22:41 +03006733 */
Pavel Begunkov8da11c12020-02-24 11:32:44 +03006734 smp_store_release(&rings->sq.head, ctx->cached_sq_head);
Jens Axboebcda7ba2020-02-23 16:42:51 -07006735}
6736
Jens Axboe9e645e112019-05-10 16:07:28 -06006737/*
Fam Zhengdd9ae8a2021-06-04 17:42:56 +01006738 * Fetch an sqe, if one is available. Note this returns a pointer to memory
Jens Axboe9e645e112019-05-10 16:07:28 -06006739 * that is mapped by userspace. This means that care needs to be taken to
6740 * ensure that reads are stable, as we cannot rely on userspace always
Jens Axboe78e19bb2019-11-06 15:21:34 -07006741 * being a good citizen. If members of the sqe are validated and then later
6742 * used, it's important that those reads are done through READ_ONCE() to
Pavel Begunkov2e6e1fd2019-12-05 16:15:45 +03006743 * prevent a re-load down the line.
Jens Axboe9e645e112019-05-10 16:07:28 -06006744 */
6745static const struct io_uring_sqe *io_get_sqe(struct io_ring_ctx *ctx)
Jens Axboe9e645e112019-05-10 16:07:28 -06006746{
Pavel Begunkovea5ab3b2021-05-16 22:58:09 +01006747 unsigned head, mask = ctx->sq_entries - 1;
Pavel Begunkov17d3aeb2021-06-14 23:37:23 +01006748 unsigned sq_idx = ctx->cached_sq_head++ & mask;
Jens Axboe9e645e112019-05-10 16:07:28 -06006749
6750 /*
6751 * The cached sq head (or cq tail) serves two purposes:
6752 *
6753 * 1) allows us to batch the cost of updating the user visible
Pavel Begunkov9d763772019-12-17 02:22:07 +03006754 * head updates.
Jens Axboe9e645e112019-05-10 16:07:28 -06006755 * 2) allows the kernel side to track the head on its own, even
Pavel Begunkov8cdf2192020-01-25 00:40:24 +03006756 * though the application is the one updating it.
6757 */
Pavel Begunkov17d3aeb2021-06-14 23:37:23 +01006758 head = READ_ONCE(ctx->sq_array[sq_idx]);
Pavel Begunkov8cdf2192020-01-25 00:40:24 +03006759 if (likely(head < ctx->sq_entries))
6760 return &ctx->sq_sqes[head];
6761
6762 /* drop invalid entries */
Pavel Begunkov15641e42021-06-14 23:37:24 +01006763 ctx->cq_extra--;
6764 WRITE_ONCE(ctx->rings->sq_dropped,
6765 READ_ONCE(ctx->rings->sq_dropped) + 1);
Pavel Begunkov711be032020-01-17 03:57:59 +03006766 return NULL;
6767}
Jens Axboeb7bb4f72019-12-15 22:13:43 -07006768
Jens Axboe0f212202020-09-13 13:09:39 -06006769static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr)
Pavel Begunkov282cdc82021-08-09 13:04:10 +01006770 __must_hold(&ctx->uring_lock)
Jens Axboe6c271ce2019-01-10 11:22:30 -07006771{
Pavel Begunkov09899b12021-06-14 02:36:22 +01006772 struct io_uring_task *tctx;
Pavel Begunkov46c4e162021-02-18 18:29:37 +00006773 int submitted = 0;
Jens Axboe6c271ce2019-01-10 11:22:30 -07006774
Pavel Begunkovee7d46d2019-12-30 21:24:45 +03006775 /* make sure SQ entry isn't read before tail */
6776 nr = min3(nr, ctx->sq_entries, io_sqring_entries(ctx));
Pavel Begunkov2b85edf2019-12-28 14:13:03 +03006777 if (!percpu_ref_tryget_many(&ctx->refs, nr))
6778 return -EAGAIN;
Jens Axboe6c271ce2019-01-10 11:22:30 -07006779
Pavel Begunkov09899b12021-06-14 02:36:22 +01006780 tctx = current->io_uring;
6781 tctx->cached_refs -= nr;
6782 if (unlikely(tctx->cached_refs < 0)) {
6783 unsigned int refill = -tctx->cached_refs + IO_TCTX_REFS_CACHE_NR;
6784
6785 percpu_counter_add(&tctx->inflight, refill);
6786 refcount_add(refill, &current->usage);
6787 tctx->cached_refs += refill;
6788 }
Pavel Begunkovba88ff12021-02-10 00:03:11 +00006789 io_submit_state_start(&ctx->submit_state, nr);
Pavel Begunkovb14cca02020-01-17 04:45:59 +03006790
Pavel Begunkov46c4e162021-02-18 18:29:37 +00006791 while (submitted < nr) {
Jens Axboe3529d8c2019-12-19 18:24:38 -07006792 const struct io_uring_sqe *sqe;
Pavel Begunkov196be952019-11-07 01:41:06 +03006793 struct io_kiocb *req;
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03006794
Pavel Begunkov258b29a2021-02-10 00:03:10 +00006795 req = io_alloc_req(ctx);
Pavel Begunkov196be952019-11-07 01:41:06 +03006796 if (unlikely(!req)) {
6797 if (!submitted)
6798 submitted = -EAGAIN;
Pavel Begunkovfb5ccc92019-10-25 12:31:30 +03006799 break;
Jens Axboe9e645e112019-05-10 16:07:28 -06006800 }
Pavel Begunkov4fccfcb2021-02-12 11:55:17 +00006801 sqe = io_get_sqe(ctx);
6802 if (unlikely(!sqe)) {
6803 kmem_cache_free(req_cachep, req);
6804 break;
6805 }
Jens Axboed3656342019-12-18 09:50:26 -07006806 /* will complete beyond this point, count as submitted */
6807 submitted++;
Pavel Begunkova1ab7b32021-02-18 18:29:42 +00006808 if (io_submit_sqe(ctx, req, sqe))
Jens Axboed3656342019-12-18 09:50:26 -07006809 break;
Jens Axboe6c271ce2019-01-10 11:22:30 -07006810 }
6811
Pavel Begunkov9466f432020-01-25 22:34:01 +03006812 if (unlikely(submitted != nr)) {
6813 int ref_used = (submitted == -EAGAIN) ? 0 : submitted;
Jens Axboed8a6df12020-10-15 16:24:45 -06006814 int unused = nr - ref_used;
Pavel Begunkov9466f432020-01-25 22:34:01 +03006815
Pavel Begunkov09899b12021-06-14 02:36:22 +01006816 current->io_uring->cached_refs += unused;
Jens Axboed8a6df12020-10-15 16:24:45 -06006817 percpu_ref_put_many(&ctx->refs, unused);
Pavel Begunkov9466f432020-01-25 22:34:01 +03006818 }
Jens Axboe6c271ce2019-01-10 11:22:30 -07006819
Pavel Begunkova1ab7b32021-02-18 18:29:42 +00006820 io_submit_state_end(&ctx->submit_state, ctx);
Pavel Begunkovae9428c2019-11-06 00:22:14 +03006821 /* Commit SQ ring head once we've consumed and submitted all SQEs */
6822 io_commit_sqring(ctx);
6823
Jens Axboe6c271ce2019-01-10 11:22:30 -07006824 return submitted;
6825}
6826
Pavel Begunkove4b6d902021-05-16 22:58:00 +01006827static inline bool io_sqd_events_pending(struct io_sq_data *sqd)
6828{
6829 return READ_ONCE(sqd->state);
6830}
6831
Xiaoguang Wang23b36282020-07-23 20:57:24 +08006832static inline void io_ring_set_wakeup_flag(struct io_ring_ctx *ctx)
6833{
6834 /* Tell userspace we may need a wakeup call */
Jens Axboe79ebeae2021-08-10 15:18:27 -06006835 spin_lock(&ctx->completion_lock);
Nadav Amit20c0b382021-08-07 17:13:42 -07006836 WRITE_ONCE(ctx->rings->sq_flags,
6837 ctx->rings->sq_flags | IORING_SQ_NEED_WAKEUP);
Jens Axboe79ebeae2021-08-10 15:18:27 -06006838 spin_unlock(&ctx->completion_lock);
Xiaoguang Wang23b36282020-07-23 20:57:24 +08006839}
6840
6841static inline void io_ring_clear_wakeup_flag(struct io_ring_ctx *ctx)
6842{
Jens Axboe79ebeae2021-08-10 15:18:27 -06006843 spin_lock(&ctx->completion_lock);
Nadav Amit20c0b382021-08-07 17:13:42 -07006844 WRITE_ONCE(ctx->rings->sq_flags,
6845 ctx->rings->sq_flags & ~IORING_SQ_NEED_WAKEUP);
Jens Axboe79ebeae2021-08-10 15:18:27 -06006846 spin_unlock(&ctx->completion_lock);
Xiaoguang Wang23b36282020-07-23 20:57:24 +08006847}
6848
Xiaoguang Wang08369242020-11-03 14:15:59 +08006849static int __io_sq_thread(struct io_ring_ctx *ctx, bool cap_entries)
Jens Axboe6c271ce2019-01-10 11:22:30 -07006850{
Jens Axboec8d1ba52020-09-14 11:07:26 -06006851 unsigned int to_submit;
Xiaoguang Wangbdcd3ea2020-02-25 22:12:08 +08006852 int ret = 0;
Jens Axboe6c271ce2019-01-10 11:22:30 -07006853
Jens Axboec8d1ba52020-09-14 11:07:26 -06006854 to_submit = io_sqring_entries(ctx);
Jens Axboee95eee22020-09-08 09:11:32 -06006855 /* if we're handling multiple rings, cap submit size for fairness */
Olivier Langlois4ce8ad92021-06-23 11:50:18 -07006856 if (cap_entries && to_submit > IORING_SQPOLL_CAP_ENTRIES_VALUE)
6857 to_submit = IORING_SQPOLL_CAP_ENTRIES_VALUE;
Jens Axboee95eee22020-09-08 09:11:32 -06006858
Xiaoguang Wang906a3c62020-11-12 14:56:00 +08006859 if (!list_empty(&ctx->iopoll_list) || to_submit) {
6860 unsigned nr_events = 0;
Pavel Begunkov948e1942021-06-24 15:09:55 +01006861 const struct cred *creds = NULL;
6862
6863 if (ctx->sq_creds != current_cred())
6864 creds = override_creds(ctx->sq_creds);
Xiaoguang Wang906a3c62020-11-12 14:56:00 +08006865
Xiaoguang Wang08369242020-11-03 14:15:59 +08006866 mutex_lock(&ctx->uring_lock);
Xiaoguang Wang906a3c62020-11-12 14:56:00 +08006867 if (!list_empty(&ctx->iopoll_list))
Pavel Begunkova8576af2021-08-15 10:40:21 +01006868 io_do_iopoll(ctx, &nr_events, 0);
Xiaoguang Wang906a3c62020-11-12 14:56:00 +08006869
Pavel Begunkov3b763ba2021-04-18 14:52:08 +01006870 /*
6871 * Don't submit if refs are dying, good for io_uring_register(),
6872 * but also it is relied upon by io_ring_exit_work()
6873 */
Pavel Begunkov0298ef92021-03-08 13:20:57 +00006874 if (to_submit && likely(!percpu_ref_is_dying(&ctx->refs)) &&
6875 !(ctx->flags & IORING_SETUP_R_DISABLED))
Xiaoguang Wang08369242020-11-03 14:15:59 +08006876 ret = io_submit_sqes(ctx, to_submit);
6877 mutex_unlock(&ctx->uring_lock);
Jens Axboe90554202020-09-03 12:12:41 -06006878
Pavel Begunkovacfb3812021-05-16 22:58:03 +01006879 if (to_submit && wq_has_sleeper(&ctx->sqo_sq_wait))
6880 wake_up(&ctx->sqo_sq_wait);
Pavel Begunkov948e1942021-06-24 15:09:55 +01006881 if (creds)
6882 revert_creds(creds);
Pavel Begunkovacfb3812021-05-16 22:58:03 +01006883 }
Jens Axboe90554202020-09-03 12:12:41 -06006884
Xiaoguang Wang08369242020-11-03 14:15:59 +08006885 return ret;
6886}
6887
6888static void io_sqd_update_thread_idle(struct io_sq_data *sqd)
6889{
6890 struct io_ring_ctx *ctx;
6891 unsigned sq_thread_idle = 0;
6892
Pavel Begunkovc9dca272021-03-10 13:13:55 +00006893 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list)
6894 sq_thread_idle = max(sq_thread_idle, ctx->sq_thread_idle);
Xiaoguang Wang08369242020-11-03 14:15:59 +08006895 sqd->sq_thread_idle = sq_thread_idle;
Jens Axboec8d1ba52020-09-14 11:07:26 -06006896}
6897
Pavel Begunkove4b6d902021-05-16 22:58:00 +01006898static bool io_sqd_handle_event(struct io_sq_data *sqd)
6899{
6900 bool did_sig = false;
6901 struct ksignal ksig;
6902
6903 if (test_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state) ||
6904 signal_pending(current)) {
6905 mutex_unlock(&sqd->lock);
6906 if (signal_pending(current))
6907 did_sig = get_signal(&ksig);
6908 cond_resched();
6909 mutex_lock(&sqd->lock);
6910 }
Pavel Begunkove4b6d902021-05-16 22:58:00 +01006911 return did_sig || test_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state);
6912}
6913
Jens Axboe6c271ce2019-01-10 11:22:30 -07006914static int io_sq_thread(void *data)
6915{
Jens Axboe69fb2132020-09-14 11:16:23 -06006916 struct io_sq_data *sqd = data;
6917 struct io_ring_ctx *ctx;
Xiaoguang Wanga0d92052020-11-12 14:55:59 +08006918 unsigned long timeout = 0;
Jens Axboe37d1e2e2021-02-17 21:03:43 -07006919 char buf[TASK_COMM_LEN];
Xiaoguang Wang08369242020-11-03 14:15:59 +08006920 DEFINE_WAIT(wait);
Jens Axboe6c271ce2019-01-10 11:22:30 -07006921
Pavel Begunkov696ee882021-04-01 09:55:04 +01006922 snprintf(buf, sizeof(buf), "iou-sqp-%d", sqd->task_pid);
Jens Axboe37d1e2e2021-02-17 21:03:43 -07006923 set_task_comm(current, buf);
Jens Axboe28cea78a2020-09-14 10:51:17 -06006924
Jens Axboe37d1e2e2021-02-17 21:03:43 -07006925 if (sqd->sq_cpu != -1)
6926 set_cpus_allowed_ptr(current, cpumask_of(sqd->sq_cpu));
6927 else
6928 set_cpus_allowed_ptr(current, cpu_online_mask);
6929 current->flags |= PF_NO_SETAFFINITY;
6930
Pavel Begunkov09a6f4e2021-03-14 20:57:10 +00006931 mutex_lock(&sqd->lock);
Pavel Begunkove4b6d902021-05-16 22:58:00 +01006932 while (1) {
Pavel Begunkov1a924a82021-06-24 15:09:56 +01006933 bool cap_entries, sqt_spin = false;
Jens Axboec1edbf52019-11-10 16:56:04 -07006934
Pavel Begunkove4b6d902021-05-16 22:58:00 +01006935 if (io_sqd_events_pending(sqd) || signal_pending(current)) {
6936 if (io_sqd_handle_event(sqd))
Pavel Begunkovc7d95612021-04-13 11:43:00 +01006937 break;
Xiaoguang Wang08369242020-11-03 14:15:59 +08006938 timeout = jiffies + sqd->sq_thread_idle;
6939 }
Pavel Begunkove4b6d902021-05-16 22:58:00 +01006940
Jens Axboee95eee22020-09-08 09:11:32 -06006941 cap_entries = !list_is_singular(&sqd->ctx_list);
Jens Axboe69fb2132020-09-14 11:16:23 -06006942 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) {
Pavel Begunkov948e1942021-06-24 15:09:55 +01006943 int ret = __io_sq_thread(ctx, cap_entries);
Stefan Metzmacher7c30f36a2021-03-07 11:54:28 +01006944
Xiaoguang Wang08369242020-11-03 14:15:59 +08006945 if (!sqt_spin && (ret > 0 || !list_empty(&ctx->iopoll_list)))
6946 sqt_spin = true;
Jens Axboe6c271ce2019-01-10 11:22:30 -07006947 }
Pavel Begunkovdd432ea52021-06-26 21:40:45 +01006948 if (io_run_task_work())
6949 sqt_spin = true;
Jens Axboe6c271ce2019-01-10 11:22:30 -07006950
Xiaoguang Wang08369242020-11-03 14:15:59 +08006951 if (sqt_spin || !time_after(jiffies, timeout)) {
Jens Axboec8d1ba52020-09-14 11:07:26 -06006952 cond_resched();
Xiaoguang Wang08369242020-11-03 14:15:59 +08006953 if (sqt_spin)
6954 timeout = jiffies + sqd->sq_thread_idle;
6955 continue;
6956 }
6957
Xiaoguang Wang08369242020-11-03 14:15:59 +08006958 prepare_to_wait(&sqd->wait, &wait, TASK_INTERRUPTIBLE);
Pavel Begunkovdd432ea52021-06-26 21:40:45 +01006959 if (!io_sqd_events_pending(sqd) && !current->task_works) {
Pavel Begunkov1a924a82021-06-24 15:09:56 +01006960 bool needs_sched = true;
6961
Hao Xu724cb4f2021-04-21 23:19:11 +08006962 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list) {
Pavel Begunkovaaa9f0f2021-05-16 22:58:01 +01006963 io_ring_set_wakeup_flag(ctx);
6964
Hao Xu724cb4f2021-04-21 23:19:11 +08006965 if ((ctx->flags & IORING_SETUP_IOPOLL) &&
6966 !list_empty_careful(&ctx->iopoll_list)) {
6967 needs_sched = false;
6968 break;
6969 }
6970 if (io_sqring_entries(ctx)) {
6971 needs_sched = false;
6972 break;
6973 }
6974 }
6975
6976 if (needs_sched) {
6977 mutex_unlock(&sqd->lock);
6978 schedule();
6979 mutex_lock(&sqd->lock);
6980 }
Jens Axboe69fb2132020-09-14 11:16:23 -06006981 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list)
6982 io_ring_clear_wakeup_flag(ctx);
Jens Axboe6c271ce2019-01-10 11:22:30 -07006983 }
Xiaoguang Wang08369242020-11-03 14:15:59 +08006984
6985 finish_wait(&sqd->wait, &wait);
6986 timeout = jiffies + sqd->sq_thread_idle;
Jens Axboe6c271ce2019-01-10 11:22:30 -07006987 }
6988
Pavel Begunkov78cc6872021-06-14 02:36:23 +01006989 io_uring_cancel_generic(true, sqd);
Jens Axboe37d1e2e2021-02-17 21:03:43 -07006990 sqd->thread = NULL;
Jens Axboe05962f92021-03-06 13:58:48 -07006991 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list)
Jens Axboe5f3f26f2021-02-25 10:17:46 -07006992 io_ring_set_wakeup_flag(ctx);
Pavel Begunkov521d6a72021-03-11 23:29:38 +00006993 io_run_task_work();
Pavel Begunkov734551d2021-04-18 14:52:09 +01006994 mutex_unlock(&sqd->lock);
6995
Jens Axboe37d1e2e2021-02-17 21:03:43 -07006996 complete(&sqd->exited);
6997 do_exit(0);
Jens Axboe6c271ce2019-01-10 11:22:30 -07006998}
6999
Jens Axboebda52162019-09-24 13:47:15 -06007000struct io_wait_queue {
7001 struct wait_queue_entry wq;
7002 struct io_ring_ctx *ctx;
Jens Axboe5fd46172021-08-06 14:04:31 -06007003 unsigned cq_tail;
Jens Axboebda52162019-09-24 13:47:15 -06007004 unsigned nr_timeouts;
7005};
7006
Pavel Begunkov6c503152021-01-04 20:36:36 +00007007static inline bool io_should_wake(struct io_wait_queue *iowq)
Jens Axboebda52162019-09-24 13:47:15 -06007008{
7009 struct io_ring_ctx *ctx = iowq->ctx;
Jens Axboe5fd46172021-08-06 14:04:31 -06007010 int dist = ctx->cached_cq_tail - (int) iowq->cq_tail;
Jens Axboebda52162019-09-24 13:47:15 -06007011
7012 /*
Brian Gianforcarod195a662019-12-13 03:09:50 -08007013 * Wake up if we have enough events, or if a timeout occurred since we
Jens Axboebda52162019-09-24 13:47:15 -06007014 * started waiting. For timeouts, we always want to return to userspace,
7015 * regardless of event count.
7016 */
Jens Axboe5fd46172021-08-06 14:04:31 -06007017 return dist >= 0 || atomic_read(&ctx->cq_timeouts) != iowq->nr_timeouts;
Jens Axboebda52162019-09-24 13:47:15 -06007018}
7019
7020static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode,
7021 int wake_flags, void *key)
7022{
7023 struct io_wait_queue *iowq = container_of(curr, struct io_wait_queue,
7024 wq);
7025
Pavel Begunkov6c503152021-01-04 20:36:36 +00007026 /*
7027 * Cannot safely flush overflowed CQEs from here, ensure we wake up
7028 * the task, and the next invocation will do it.
7029 */
Pavel Begunkov5ed7a372021-06-14 23:37:27 +01007030 if (io_should_wake(iowq) || test_bit(0, &iowq->ctx->check_cq_overflow))
Pavel Begunkov6c503152021-01-04 20:36:36 +00007031 return autoremove_wake_function(curr, mode, wake_flags, key);
7032 return -1;
Jens Axboebda52162019-09-24 13:47:15 -06007033}
7034
Jens Axboeaf9c1a42020-09-24 13:32:18 -06007035static int io_run_task_work_sig(void)
7036{
7037 if (io_run_task_work())
7038 return 1;
7039 if (!signal_pending(current))
7040 return 0;
Jens Axboe0b8cfa92021-03-21 14:16:08 -06007041 if (test_thread_flag(TIF_NOTIFY_SIGNAL))
Jens Axboe792ee0f62020-10-22 20:17:18 -06007042 return -ERESTARTSYS;
Jens Axboeaf9c1a42020-09-24 13:32:18 -06007043 return -EINTR;
7044}
7045
Pavel Begunkoveeb60b92021-02-04 13:51:58 +00007046/* when returns >0, the caller should retry */
7047static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx,
7048 struct io_wait_queue *iowq,
7049 signed long *timeout)
7050{
7051 int ret;
7052
7053 /* make sure we run task_work before checking for signals */
7054 ret = io_run_task_work_sig();
7055 if (ret || io_should_wake(iowq))
7056 return ret;
7057 /* let the caller flush overflows, retry */
Pavel Begunkov5ed7a372021-06-14 23:37:27 +01007058 if (test_bit(0, &ctx->check_cq_overflow))
Pavel Begunkoveeb60b92021-02-04 13:51:58 +00007059 return 1;
7060
7061 *timeout = schedule_timeout(*timeout);
7062 return !*timeout ? -ETIME : 1;
7063}
7064
Jens Axboe2b188cc2019-01-07 10:46:33 -07007065/*
7066 * Wait until events become available, if we don't already have some. The
7067 * application must reap them itself, as they reside on the shared cq ring.
7068 */
7069static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events,
Hao Xuc73ebb62020-11-03 10:54:37 +08007070 const sigset_t __user *sig, size_t sigsz,
7071 struct __kernel_timespec __user *uts)
Jens Axboe2b188cc2019-01-07 10:46:33 -07007072{
Pavel Begunkov902910992021-08-09 09:07:32 -06007073 struct io_wait_queue iowq;
Hristo Venev75b28af2019-08-26 17:23:46 +00007074 struct io_rings *rings = ctx->rings;
Pavel Begunkovc1d5a222021-02-04 13:51:57 +00007075 signed long timeout = MAX_SCHEDULE_TIMEOUT;
7076 int ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07007077
Jens Axboeb41e9852020-02-17 09:52:41 -07007078 do {
Pavel Begunkov90f67362021-08-09 20:18:12 +01007079 io_cqring_overflow_flush(ctx);
Pavel Begunkov6c503152021-01-04 20:36:36 +00007080 if (io_cqring_events(ctx) >= min_events)
Jens Axboeb41e9852020-02-17 09:52:41 -07007081 return 0;
Jens Axboe4c6e2772020-07-01 11:29:10 -06007082 if (!io_run_task_work())
Jens Axboeb41e9852020-02-17 09:52:41 -07007083 break;
Jens Axboeb41e9852020-02-17 09:52:41 -07007084 } while (1);
Jens Axboe2b188cc2019-01-07 10:46:33 -07007085
7086 if (sig) {
Arnd Bergmann9e75ad52019-03-25 15:34:53 +01007087#ifdef CONFIG_COMPAT
7088 if (in_compat_syscall())
7089 ret = set_compat_user_sigmask((const compat_sigset_t __user *)sig,
Oleg Nesterovb7724342019-07-16 16:29:53 -07007090 sigsz);
Arnd Bergmann9e75ad52019-03-25 15:34:53 +01007091 else
7092#endif
Oleg Nesterovb7724342019-07-16 16:29:53 -07007093 ret = set_user_sigmask(sig, sigsz);
Arnd Bergmann9e75ad52019-03-25 15:34:53 +01007094
Jens Axboe2b188cc2019-01-07 10:46:33 -07007095 if (ret)
7096 return ret;
7097 }
7098
Hao Xuc73ebb62020-11-03 10:54:37 +08007099 if (uts) {
Pavel Begunkovc1d5a222021-02-04 13:51:57 +00007100 struct timespec64 ts;
7101
Hao Xuc73ebb62020-11-03 10:54:37 +08007102 if (get_timespec64(&ts, uts))
7103 return -EFAULT;
7104 timeout = timespec64_to_jiffies(&ts);
7105 }
7106
Pavel Begunkov902910992021-08-09 09:07:32 -06007107 init_waitqueue_func_entry(&iowq.wq, io_wake_function);
7108 iowq.wq.private = current;
7109 INIT_LIST_HEAD(&iowq.wq.entry);
7110 iowq.ctx = ctx;
Jens Axboebda52162019-09-24 13:47:15 -06007111 iowq.nr_timeouts = atomic_read(&ctx->cq_timeouts);
Jens Axboe5fd46172021-08-06 14:04:31 -06007112 iowq.cq_tail = READ_ONCE(ctx->rings->cq.head) + min_events;
Pavel Begunkov902910992021-08-09 09:07:32 -06007113
Dmitrii Dolgovc826bd72019-10-15 19:02:01 +02007114 trace_io_uring_cqring_wait(ctx, min_events);
Jens Axboebda52162019-09-24 13:47:15 -06007115 do {
Jens Axboeca0a2652021-03-04 17:15:48 -07007116 /* if we can't even flush overflow, don't wait for more */
Pavel Begunkov90f67362021-08-09 20:18:12 +01007117 if (!io_cqring_overflow_flush(ctx)) {
Jens Axboeca0a2652021-03-04 17:15:48 -07007118 ret = -EBUSY;
7119 break;
7120 }
Pavel Begunkov311997b2021-06-14 23:37:28 +01007121 prepare_to_wait_exclusive(&ctx->cq_wait, &iowq.wq,
Jens Axboebda52162019-09-24 13:47:15 -06007122 TASK_INTERRUPTIBLE);
Pavel Begunkoveeb60b92021-02-04 13:51:58 +00007123 ret = io_cqring_wait_schedule(ctx, &iowq, &timeout);
Pavel Begunkov311997b2021-06-14 23:37:28 +01007124 finish_wait(&ctx->cq_wait, &iowq.wq);
Jens Axboeca0a2652021-03-04 17:15:48 -07007125 cond_resched();
Pavel Begunkoveeb60b92021-02-04 13:51:58 +00007126 } while (ret > 0);
Jens Axboebda52162019-09-24 13:47:15 -06007127
Jens Axboeb7db41c2020-07-04 08:55:50 -06007128 restore_saved_sigmask_unless(ret == -EINTR);
Jens Axboe2b188cc2019-01-07 10:46:33 -07007129
Hristo Venev75b28af2019-08-26 17:23:46 +00007130 return READ_ONCE(rings->cq.head) == READ_ONCE(rings->cq.tail) ? ret : 0;
Jens Axboe2b188cc2019-01-07 10:46:33 -07007131}
7132
Pavel Begunkov9123c8f2021-06-14 02:36:20 +01007133static void io_free_page_table(void **table, size_t size)
Pavel Begunkov846a4ef2021-04-01 15:44:03 +01007134{
Pavel Begunkov9123c8f2021-06-14 02:36:20 +01007135 unsigned i, nr_tables = DIV_ROUND_UP(size, PAGE_SIZE);
Pavel Begunkov846a4ef2021-04-01 15:44:03 +01007136
7137 for (i = 0; i < nr_tables; i++)
Pavel Begunkov9123c8f2021-06-14 02:36:20 +01007138 kfree(table[i]);
7139 kfree(table);
7140}
7141
7142static void **io_alloc_page_table(size_t size)
7143{
7144 unsigned i, nr_tables = DIV_ROUND_UP(size, PAGE_SIZE);
7145 size_t init_size = size;
7146 void **table;
7147
Pavel Begunkov0bea96f2021-08-20 10:36:36 +01007148 table = kcalloc(nr_tables, sizeof(*table), GFP_KERNEL_ACCOUNT);
Pavel Begunkov9123c8f2021-06-14 02:36:20 +01007149 if (!table)
7150 return NULL;
7151
7152 for (i = 0; i < nr_tables; i++) {
Pavel Begunkov27f6b312021-06-15 13:20:13 +01007153 unsigned int this_size = min_t(size_t, size, PAGE_SIZE);
Pavel Begunkov9123c8f2021-06-14 02:36:20 +01007154
Pavel Begunkov0bea96f2021-08-20 10:36:36 +01007155 table[i] = kzalloc(this_size, GFP_KERNEL_ACCOUNT);
Pavel Begunkov9123c8f2021-06-14 02:36:20 +01007156 if (!table[i]) {
7157 io_free_page_table(table, init_size);
7158 return NULL;
7159 }
7160 size -= this_size;
7161 }
7162 return table;
Pavel Begunkov846a4ef2021-04-01 15:44:03 +01007163}
7164
Pavel Begunkov28a9fe22021-04-01 15:43:47 +01007165static void io_rsrc_node_destroy(struct io_rsrc_node *ref_node)
7166{
7167 percpu_ref_exit(&ref_node->refs);
7168 kfree(ref_node);
7169}
7170
Pavel Begunkovb9bd2be2021-08-09 09:09:47 -06007171static void io_rsrc_node_ref_zero(struct percpu_ref *ref)
7172{
7173 struct io_rsrc_node *node = container_of(ref, struct io_rsrc_node, refs);
7174 struct io_ring_ctx *ctx = node->rsrc_data->ctx;
7175 unsigned long flags;
7176 bool first_add = false;
7177
7178 spin_lock_irqsave(&ctx->rsrc_ref_lock, flags);
7179 node->done = true;
7180
7181 while (!list_empty(&ctx->rsrc_ref_list)) {
7182 node = list_first_entry(&ctx->rsrc_ref_list,
7183 struct io_rsrc_node, node);
7184 /* recycle ref nodes in order */
7185 if (!node->done)
7186 break;
7187 list_del(&node->node);
7188 first_add |= llist_add(&node->llist, &ctx->rsrc_put_llist);
7189 }
7190 spin_unlock_irqrestore(&ctx->rsrc_ref_lock, flags);
7191
7192 if (first_add)
7193 mod_delayed_work(system_wq, &ctx->rsrc_put_work, HZ);
7194}
7195
7196static struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx)
7197{
7198 struct io_rsrc_node *ref_node;
7199
7200 ref_node = kzalloc(sizeof(*ref_node), GFP_KERNEL);
7201 if (!ref_node)
7202 return NULL;
7203
7204 if (percpu_ref_init(&ref_node->refs, io_rsrc_node_ref_zero,
7205 0, GFP_KERNEL)) {
7206 kfree(ref_node);
7207 return NULL;
7208 }
7209 INIT_LIST_HEAD(&ref_node->node);
7210 INIT_LIST_HEAD(&ref_node->rsrc_list);
7211 ref_node->done = false;
7212 return ref_node;
7213}
7214
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01007215static void io_rsrc_node_switch(struct io_ring_ctx *ctx,
7216 struct io_rsrc_data *data_to_kill)
Pavel Begunkov1642b442020-12-30 21:34:14 +00007217{
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01007218 WARN_ON_ONCE(!ctx->rsrc_backup_node);
7219 WARN_ON_ONCE(data_to_kill && !ctx->rsrc_node);
Pavel Begunkov82fbcfa2021-04-01 15:43:43 +01007220
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01007221 if (data_to_kill) {
7222 struct io_rsrc_node *rsrc_node = ctx->rsrc_node;
Pavel Begunkov82fbcfa2021-04-01 15:43:43 +01007223
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01007224 rsrc_node->rsrc_data = data_to_kill;
Jens Axboe4956b9e2021-08-09 07:49:41 -06007225 spin_lock_irq(&ctx->rsrc_ref_lock);
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01007226 list_add_tail(&rsrc_node->node, &ctx->rsrc_ref_list);
Jens Axboe4956b9e2021-08-09 07:49:41 -06007227 spin_unlock_irq(&ctx->rsrc_ref_lock);
Pavel Begunkov82fbcfa2021-04-01 15:43:43 +01007228
Pavel Begunkov3e942492021-04-11 01:46:34 +01007229 atomic_inc(&data_to_kill->refs);
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01007230 percpu_ref_kill(&rsrc_node->refs);
7231 ctx->rsrc_node = NULL;
7232 }
7233
7234 if (!ctx->rsrc_node) {
7235 ctx->rsrc_node = ctx->rsrc_backup_node;
7236 ctx->rsrc_backup_node = NULL;
7237 }
Pavel Begunkov1642b442020-12-30 21:34:14 +00007238}
7239
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01007240static int io_rsrc_node_switch_start(struct io_ring_ctx *ctx)
Pavel Begunkov8dd03af2021-03-19 17:22:36 +00007241{
7242 if (ctx->rsrc_backup_node)
7243 return 0;
Pavel Begunkovb895c9a2021-04-01 15:43:40 +01007244 ctx->rsrc_backup_node = io_rsrc_node_alloc(ctx);
Pavel Begunkov8dd03af2021-03-19 17:22:36 +00007245 return ctx->rsrc_backup_node ? 0 : -ENOMEM;
7246}
7247
Pavel Begunkov40ae0ff2021-04-01 15:43:44 +01007248static int io_rsrc_ref_quiesce(struct io_rsrc_data *data, struct io_ring_ctx *ctx)
Hao Xu8bad28d2021-02-19 17:19:36 +08007249{
7250 int ret;
Xiaoguang Wang05589552020-03-31 14:05:18 +08007251
Pavel Begunkov215c3902021-04-01 15:43:48 +01007252 /* As we may drop ->uring_lock, other task may have started quiesce */
Hao Xu8bad28d2021-02-19 17:19:36 +08007253 if (data->quiesce)
7254 return -ENXIO;
7255
7256 data->quiesce = true;
Pavel Begunkov1ffc5422020-12-30 21:34:15 +00007257 do {
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01007258 ret = io_rsrc_node_switch_start(ctx);
Pavel Begunkov8dd03af2021-03-19 17:22:36 +00007259 if (ret)
Pavel Begunkovf2303b12021-02-20 18:03:49 +00007260 break;
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01007261 io_rsrc_node_switch(ctx, data);
7262
Pavel Begunkov3e942492021-04-11 01:46:34 +01007263 /* kill initial ref, already quiesced if zero */
7264 if (atomic_dec_and_test(&data->refs))
7265 break;
Jens Axboec018db42021-08-09 08:15:50 -06007266 mutex_unlock(&ctx->uring_lock);
Hao Xu8bad28d2021-02-19 17:19:36 +08007267 flush_delayed_work(&ctx->rsrc_put_work);
Pavel Begunkov1ffc5422020-12-30 21:34:15 +00007268 ret = wait_for_completion_interruptible(&data->done);
Jens Axboec018db42021-08-09 08:15:50 -06007269 if (!ret) {
7270 mutex_lock(&ctx->uring_lock);
Pavel Begunkov1ffc5422020-12-30 21:34:15 +00007271 break;
Jens Axboec018db42021-08-09 08:15:50 -06007272 }
Jens Axboe05f3fb32019-12-09 11:22:50 -07007273
Pavel Begunkov3e942492021-04-11 01:46:34 +01007274 atomic_inc(&data->refs);
7275 /* wait for all works potentially completing data->done */
7276 flush_delayed_work(&ctx->rsrc_put_work);
Jens Axboecb5e1b82021-02-25 07:37:35 -07007277 reinit_completion(&data->done);
Pavel Begunkov8dd03af2021-03-19 17:22:36 +00007278
Hao Xu8bad28d2021-02-19 17:19:36 +08007279 ret = io_run_task_work_sig();
7280 mutex_lock(&ctx->uring_lock);
Pavel Begunkovf2303b12021-02-20 18:03:49 +00007281 } while (ret >= 0);
Hao Xu8bad28d2021-02-19 17:19:36 +08007282 data->quiesce = false;
7283
Hao Xu8bad28d2021-02-19 17:19:36 +08007284 return ret;
Bijan Mottahedehd7954b22021-01-15 17:37:50 +00007285}
7286
Pavel Begunkov2d091d62021-06-14 02:36:21 +01007287static u64 *io_get_tag_slot(struct io_rsrc_data *data, unsigned int idx)
7288{
7289 unsigned int off = idx & IO_RSRC_TAG_TABLE_MASK;
7290 unsigned int table_idx = idx >> IO_RSRC_TAG_TABLE_SHIFT;
7291
7292 return &data->tags[table_idx][off];
7293}
7294
Pavel Begunkov44b31f22021-04-25 14:32:16 +01007295static void io_rsrc_data_free(struct io_rsrc_data *data)
7296{
Pavel Begunkov2d091d62021-06-14 02:36:21 +01007297 size_t size = data->nr * sizeof(data->tags[0][0]);
7298
7299 if (data->tags)
7300 io_free_page_table((void **)data->tags, size);
Pavel Begunkov44b31f22021-04-25 14:32:16 +01007301 kfree(data);
7302}
7303
Pavel Begunkovd878c812021-06-14 02:36:18 +01007304static int io_rsrc_data_alloc(struct io_ring_ctx *ctx, rsrc_put_fn *do_put,
7305 u64 __user *utags, unsigned nr,
7306 struct io_rsrc_data **pdata)
Bijan Mottahedeh1ad555c2021-01-15 17:37:51 +00007307{
Pavel Begunkovb895c9a2021-04-01 15:43:40 +01007308 struct io_rsrc_data *data;
Pavel Begunkov2d091d62021-06-14 02:36:21 +01007309 int ret = -ENOMEM;
Pavel Begunkovd878c812021-06-14 02:36:18 +01007310 unsigned i;
Bijan Mottahedeh1ad555c2021-01-15 17:37:51 +00007311
7312 data = kzalloc(sizeof(*data), GFP_KERNEL);
7313 if (!data)
Pavel Begunkovd878c812021-06-14 02:36:18 +01007314 return -ENOMEM;
Pavel Begunkov2d091d62021-06-14 02:36:21 +01007315 data->tags = (u64 **)io_alloc_page_table(nr * sizeof(data->tags[0][0]));
Pavel Begunkovb60c8dc2021-04-25 14:32:18 +01007316 if (!data->tags) {
7317 kfree(data);
Pavel Begunkovd878c812021-06-14 02:36:18 +01007318 return -ENOMEM;
7319 }
Pavel Begunkov2d091d62021-06-14 02:36:21 +01007320
7321 data->nr = nr;
7322 data->ctx = ctx;
7323 data->do_put = do_put;
Pavel Begunkovd878c812021-06-14 02:36:18 +01007324 if (utags) {
Pavel Begunkov2d091d62021-06-14 02:36:21 +01007325 ret = -EFAULT;
Pavel Begunkovd878c812021-06-14 02:36:18 +01007326 for (i = 0; i < nr; i++) {
Colin Ian Kingfdd1dc32021-06-15 14:00:11 +01007327 u64 *tag_slot = io_get_tag_slot(data, i);
7328
7329 if (copy_from_user(tag_slot, &utags[i],
7330 sizeof(*tag_slot)))
Pavel Begunkov2d091d62021-06-14 02:36:21 +01007331 goto fail;
Pavel Begunkovd878c812021-06-14 02:36:18 +01007332 }
Pavel Begunkovb60c8dc2021-04-25 14:32:18 +01007333 }
7334
Pavel Begunkov3e942492021-04-11 01:46:34 +01007335 atomic_set(&data->refs, 1);
Bijan Mottahedeh1ad555c2021-01-15 17:37:51 +00007336 init_completion(&data->done);
Pavel Begunkovd878c812021-06-14 02:36:18 +01007337 *pdata = data;
7338 return 0;
Pavel Begunkov2d091d62021-06-14 02:36:21 +01007339fail:
7340 io_rsrc_data_free(data);
7341 return ret;
Bijan Mottahedeh1ad555c2021-01-15 17:37:51 +00007342}
7343
Pavel Begunkov9123c8f2021-06-14 02:36:20 +01007344static bool io_alloc_file_tables(struct io_file_table *table, unsigned nr_files)
7345{
Pavel Begunkov0bea96f2021-08-20 10:36:36 +01007346 table->files = kvcalloc(nr_files, sizeof(table->files[0]),
7347 GFP_KERNEL_ACCOUNT);
Pavel Begunkov9123c8f2021-06-14 02:36:20 +01007348 return !!table->files;
7349}
7350
Pavel Begunkov042b0d82021-08-09 13:04:01 +01007351static void io_free_file_tables(struct io_file_table *table)
Pavel Begunkov9123c8f2021-06-14 02:36:20 +01007352{
Pavel Begunkov042b0d82021-08-09 13:04:01 +01007353 kvfree(table->files);
Pavel Begunkov9123c8f2021-06-14 02:36:20 +01007354 table->files = NULL;
7355}
7356
Jens Axboe2b188cc2019-01-07 10:46:33 -07007357static void __io_sqe_files_unregister(struct io_ring_ctx *ctx)
7358{
7359#if defined(CONFIG_UNIX)
7360 if (ctx->ring_sock) {
7361 struct sock *sock = ctx->ring_sock->sk;
7362 struct sk_buff *skb;
7363
7364 while ((skb = skb_dequeue(&sock->sk_receive_queue)) != NULL)
7365 kfree_skb(skb);
7366 }
7367#else
7368 int i;
7369
7370 for (i = 0; i < ctx->nr_user_files; i++) {
7371 struct file *file;
7372
7373 file = io_file_from_index(ctx, i);
7374 if (file)
7375 fput(file);
7376 }
7377#endif
Pavel Begunkov042b0d82021-08-09 13:04:01 +01007378 io_free_file_tables(&ctx->file_table);
Pavel Begunkov44b31f22021-04-25 14:32:16 +01007379 io_rsrc_data_free(ctx->file_data);
Pavel Begunkovfff4db72021-04-25 14:32:15 +01007380 ctx->file_data = NULL;
7381 ctx->nr_user_files = 0;
Bijan Mottahedeh1ad555c2021-01-15 17:37:51 +00007382}
7383
Bijan Mottahedehd7954b22021-01-15 17:37:50 +00007384static int io_sqe_files_unregister(struct io_ring_ctx *ctx)
7385{
Bijan Mottahedehd7954b22021-01-15 17:37:50 +00007386 int ret;
7387
Pavel Begunkov08480402021-04-13 02:58:38 +01007388 if (!ctx->file_data)
Bijan Mottahedehd7954b22021-01-15 17:37:50 +00007389 return -ENXIO;
Pavel Begunkov08480402021-04-13 02:58:38 +01007390 ret = io_rsrc_ref_quiesce(ctx->file_data, ctx);
7391 if (!ret)
7392 __io_sqe_files_unregister(ctx);
7393 return ret;
Jens Axboe6b063142019-01-10 22:13:58 -07007394}
7395
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007396static void io_sq_thread_unpark(struct io_sq_data *sqd)
Pavel Begunkov09a6f4e2021-03-14 20:57:10 +00007397 __releases(&sqd->lock)
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007398{
Pavel Begunkov521d6a72021-03-11 23:29:38 +00007399 WARN_ON_ONCE(sqd->thread == current);
7400
Pavel Begunkov9e138a42021-03-14 20:57:12 +00007401 /*
7402 * Do the dance but not conditional clear_bit() because it'd race with
7403 * other threads incrementing park_pending and setting the bit.
7404 */
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007405 clear_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state);
Pavel Begunkov9e138a42021-03-14 20:57:12 +00007406 if (atomic_dec_return(&sqd->park_pending))
7407 set_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state);
Pavel Begunkov09a6f4e2021-03-14 20:57:10 +00007408 mutex_unlock(&sqd->lock);
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007409}
7410
Jens Axboe86e0d672021-03-05 08:44:39 -07007411static void io_sq_thread_park(struct io_sq_data *sqd)
Pavel Begunkov09a6f4e2021-03-14 20:57:10 +00007412 __acquires(&sqd->lock)
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007413{
Pavel Begunkov521d6a72021-03-11 23:29:38 +00007414 WARN_ON_ONCE(sqd->thread == current);
7415
Pavel Begunkov9e138a42021-03-14 20:57:12 +00007416 atomic_inc(&sqd->park_pending);
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007417 set_bit(IO_SQ_THREAD_SHOULD_PARK, &sqd->state);
Pavel Begunkov09a6f4e2021-03-14 20:57:10 +00007418 mutex_lock(&sqd->lock);
Jens Axboe05962f92021-03-06 13:58:48 -07007419 if (sqd->thread)
Jens Axboe86e0d672021-03-05 08:44:39 -07007420 wake_up_process(sqd->thread);
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007421}
7422
7423static void io_sq_thread_stop(struct io_sq_data *sqd)
7424{
Pavel Begunkov521d6a72021-03-11 23:29:38 +00007425 WARN_ON_ONCE(sqd->thread == current);
Pavel Begunkov88885f62021-04-11 01:46:38 +01007426 WARN_ON_ONCE(test_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state));
Pavel Begunkov521d6a72021-03-11 23:29:38 +00007427
Jens Axboe05962f92021-03-06 13:58:48 -07007428 set_bit(IO_SQ_THREAD_SHOULD_STOP, &sqd->state);
Pavel Begunkov88885f62021-04-11 01:46:38 +01007429 mutex_lock(&sqd->lock);
Jens Axboee8f98f242021-03-09 16:32:13 -07007430 if (sqd->thread)
7431 wake_up_process(sqd->thread);
Pavel Begunkov09a6f4e2021-03-14 20:57:10 +00007432 mutex_unlock(&sqd->lock);
Jens Axboe05962f92021-03-06 13:58:48 -07007433 wait_for_completion(&sqd->exited);
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007434}
7435
Jens Axboe534ca6d2020-09-02 13:52:19 -06007436static void io_put_sq_data(struct io_sq_data *sqd)
Jens Axboe6c271ce2019-01-10 11:22:30 -07007437{
Jens Axboe534ca6d2020-09-02 13:52:19 -06007438 if (refcount_dec_and_test(&sqd->refs)) {
Pavel Begunkov9e138a42021-03-14 20:57:12 +00007439 WARN_ON_ONCE(atomic_read(&sqd->park_pending));
7440
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007441 io_sq_thread_stop(sqd);
7442 kfree(sqd);
7443 }
7444}
7445
7446static void io_sq_thread_finish(struct io_ring_ctx *ctx)
7447{
7448 struct io_sq_data *sqd = ctx->sq_data;
7449
7450 if (sqd) {
Jens Axboe05962f92021-03-06 13:58:48 -07007451 io_sq_thread_park(sqd);
Pavel Begunkov521d6a72021-03-11 23:29:38 +00007452 list_del_init(&ctx->sqd_list);
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007453 io_sqd_update_thread_idle(sqd);
Jens Axboe05962f92021-03-06 13:58:48 -07007454 io_sq_thread_unpark(sqd);
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007455
7456 io_put_sq_data(sqd);
7457 ctx->sq_data = NULL;
Jens Axboe534ca6d2020-09-02 13:52:19 -06007458 }
7459}
7460
Jens Axboeaa061652020-09-02 14:50:27 -06007461static struct io_sq_data *io_attach_sq_data(struct io_uring_params *p)
7462{
7463 struct io_ring_ctx *ctx_attach;
7464 struct io_sq_data *sqd;
7465 struct fd f;
7466
7467 f = fdget(p->wq_fd);
7468 if (!f.file)
7469 return ERR_PTR(-ENXIO);
7470 if (f.file->f_op != &io_uring_fops) {
7471 fdput(f);
7472 return ERR_PTR(-EINVAL);
7473 }
7474
7475 ctx_attach = f.file->private_data;
7476 sqd = ctx_attach->sq_data;
7477 if (!sqd) {
7478 fdput(f);
7479 return ERR_PTR(-EINVAL);
7480 }
Jens Axboe5c2469e2021-03-11 10:17:56 -07007481 if (sqd->task_tgid != current->tgid) {
7482 fdput(f);
7483 return ERR_PTR(-EPERM);
7484 }
Jens Axboeaa061652020-09-02 14:50:27 -06007485
7486 refcount_inc(&sqd->refs);
7487 fdput(f);
7488 return sqd;
7489}
7490
Pavel Begunkov26984fb2021-03-11 23:29:37 +00007491static struct io_sq_data *io_get_sq_data(struct io_uring_params *p,
7492 bool *attached)
Jens Axboe534ca6d2020-09-02 13:52:19 -06007493{
7494 struct io_sq_data *sqd;
7495
Pavel Begunkov26984fb2021-03-11 23:29:37 +00007496 *attached = false;
Jens Axboe5c2469e2021-03-11 10:17:56 -07007497 if (p->flags & IORING_SETUP_ATTACH_WQ) {
7498 sqd = io_attach_sq_data(p);
Pavel Begunkov26984fb2021-03-11 23:29:37 +00007499 if (!IS_ERR(sqd)) {
7500 *attached = true;
Jens Axboe5c2469e2021-03-11 10:17:56 -07007501 return sqd;
Pavel Begunkov26984fb2021-03-11 23:29:37 +00007502 }
Jens Axboe5c2469e2021-03-11 10:17:56 -07007503 /* fall through for EPERM case, setup new sqd/task */
7504 if (PTR_ERR(sqd) != -EPERM)
7505 return sqd;
7506 }
Jens Axboeaa061652020-09-02 14:50:27 -06007507
Jens Axboe534ca6d2020-09-02 13:52:19 -06007508 sqd = kzalloc(sizeof(*sqd), GFP_KERNEL);
7509 if (!sqd)
7510 return ERR_PTR(-ENOMEM);
7511
Pavel Begunkov9e138a42021-03-14 20:57:12 +00007512 atomic_set(&sqd->park_pending, 0);
Jens Axboe534ca6d2020-09-02 13:52:19 -06007513 refcount_set(&sqd->refs, 1);
Jens Axboe69fb2132020-09-14 11:16:23 -06007514 INIT_LIST_HEAD(&sqd->ctx_list);
Pavel Begunkov09a6f4e2021-03-14 20:57:10 +00007515 mutex_init(&sqd->lock);
Jens Axboe534ca6d2020-09-02 13:52:19 -06007516 init_waitqueue_head(&sqd->wait);
Jens Axboe37d1e2e2021-02-17 21:03:43 -07007517 init_completion(&sqd->exited);
Jens Axboe534ca6d2020-09-02 13:52:19 -06007518 return sqd;
7519}
7520
Jens Axboe6b063142019-01-10 22:13:58 -07007521#if defined(CONFIG_UNIX)
Jens Axboe6b063142019-01-10 22:13:58 -07007522/*
7523 * Ensure the UNIX gc is aware of our file set, so we are certain that
7524 * the io_uring can be safely unregistered on process exit, even if we have
7525 * loops in the file referencing.
7526 */
7527static int __io_sqe_files_scm(struct io_ring_ctx *ctx, int nr, int offset)
7528{
7529 struct sock *sk = ctx->ring_sock->sk;
7530 struct scm_fp_list *fpl;
7531 struct sk_buff *skb;
Jens Axboe08a45172019-10-03 08:11:03 -06007532 int i, nr_files;
Jens Axboe6b063142019-01-10 22:13:58 -07007533
Jens Axboe6b063142019-01-10 22:13:58 -07007534 fpl = kzalloc(sizeof(*fpl), GFP_KERNEL);
7535 if (!fpl)
7536 return -ENOMEM;
7537
7538 skb = alloc_skb(0, GFP_KERNEL);
7539 if (!skb) {
7540 kfree(fpl);
7541 return -ENOMEM;
7542 }
7543
7544 skb->sk = sk;
Jens Axboe6b063142019-01-10 22:13:58 -07007545
Jens Axboe08a45172019-10-03 08:11:03 -06007546 nr_files = 0;
Jens Axboe62e398b2021-02-21 16:19:37 -07007547 fpl->user = get_uid(current_user());
Jens Axboe6b063142019-01-10 22:13:58 -07007548 for (i = 0; i < nr; i++) {
Jens Axboe65e19f52019-10-26 07:20:21 -06007549 struct file *file = io_file_from_index(ctx, i + offset);
7550
7551 if (!file)
Jens Axboe08a45172019-10-03 08:11:03 -06007552 continue;
Jens Axboe65e19f52019-10-26 07:20:21 -06007553 fpl->fp[nr_files] = get_file(file);
Jens Axboe08a45172019-10-03 08:11:03 -06007554 unix_inflight(fpl->user, fpl->fp[nr_files]);
7555 nr_files++;
Jens Axboe6b063142019-01-10 22:13:58 -07007556 }
7557
Jens Axboe08a45172019-10-03 08:11:03 -06007558 if (nr_files) {
7559 fpl->max = SCM_MAX_FD;
7560 fpl->count = nr_files;
7561 UNIXCB(skb).fp = fpl;
Jens Axboe05f3fb32019-12-09 11:22:50 -07007562 skb->destructor = unix_destruct_scm;
Jens Axboe08a45172019-10-03 08:11:03 -06007563 refcount_add(skb->truesize, &sk->sk_wmem_alloc);
7564 skb_queue_head(&sk->sk_receive_queue, skb);
Jens Axboe6b063142019-01-10 22:13:58 -07007565
Jens Axboe08a45172019-10-03 08:11:03 -06007566 for (i = 0; i < nr_files; i++)
7567 fput(fpl->fp[i]);
7568 } else {
7569 kfree_skb(skb);
7570 kfree(fpl);
7571 }
Jens Axboe6b063142019-01-10 22:13:58 -07007572
7573 return 0;
7574}
7575
7576/*
7577 * If UNIX sockets are enabled, fd passing can cause a reference cycle which
7578 * causes regular reference counting to break down. We rely on the UNIX
7579 * garbage collection to take care of this problem for us.
7580 */
7581static int io_sqe_files_scm(struct io_ring_ctx *ctx)
7582{
7583 unsigned left, total;
7584 int ret = 0;
7585
7586 total = 0;
7587 left = ctx->nr_user_files;
7588 while (left) {
7589 unsigned this_files = min_t(unsigned, left, SCM_MAX_FD);
Jens Axboe6b063142019-01-10 22:13:58 -07007590
7591 ret = __io_sqe_files_scm(ctx, this_files, total);
7592 if (ret)
7593 break;
7594 left -= this_files;
7595 total += this_files;
7596 }
7597
7598 if (!ret)
7599 return 0;
7600
7601 while (total < ctx->nr_user_files) {
Jens Axboe65e19f52019-10-26 07:20:21 -06007602 struct file *file = io_file_from_index(ctx, total);
7603
7604 if (file)
7605 fput(file);
Jens Axboe6b063142019-01-10 22:13:58 -07007606 total++;
7607 }
7608
7609 return ret;
7610}
7611#else
7612static int io_sqe_files_scm(struct io_ring_ctx *ctx)
7613{
7614 return 0;
7615}
7616#endif
7617
Pavel Begunkov47e90392021-04-01 15:43:56 +01007618static void io_rsrc_file_put(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc)
Jens Axboec3a31e62019-10-03 13:59:56 -06007619{
Bijan Mottahedeh50238532021-01-15 17:37:45 +00007620 struct file *file = prsrc->file;
Jens Axboec3a31e62019-10-03 13:59:56 -06007621#if defined(CONFIG_UNIX)
Jens Axboec3a31e62019-10-03 13:59:56 -06007622 struct sock *sock = ctx->ring_sock->sk;
7623 struct sk_buff_head list, *head = &sock->sk_receive_queue;
7624 struct sk_buff *skb;
7625 int i;
7626
7627 __skb_queue_head_init(&list);
7628
7629 /*
7630 * Find the skb that holds this file in its SCM_RIGHTS. When found,
7631 * remove this entry and rearrange the file array.
7632 */
7633 skb = skb_dequeue(head);
7634 while (skb) {
7635 struct scm_fp_list *fp;
7636
7637 fp = UNIXCB(skb).fp;
7638 for (i = 0; i < fp->count; i++) {
7639 int left;
7640
7641 if (fp->fp[i] != file)
7642 continue;
7643
7644 unix_notinflight(fp->user, fp->fp[i]);
7645 left = fp->count - 1 - i;
7646 if (left) {
7647 memmove(&fp->fp[i], &fp->fp[i + 1],
7648 left * sizeof(struct file *));
7649 }
7650 fp->count--;
7651 if (!fp->count) {
7652 kfree_skb(skb);
7653 skb = NULL;
7654 } else {
7655 __skb_queue_tail(&list, skb);
7656 }
7657 fput(file);
7658 file = NULL;
7659 break;
7660 }
7661
7662 if (!file)
7663 break;
7664
7665 __skb_queue_tail(&list, skb);
7666
7667 skb = skb_dequeue(head);
7668 }
7669
7670 if (skb_peek(&list)) {
7671 spin_lock_irq(&head->lock);
7672 while ((skb = __skb_dequeue(&list)) != NULL)
7673 __skb_queue_tail(head, skb);
7674 spin_unlock_irq(&head->lock);
7675 }
7676#else
Jens Axboe05f3fb32019-12-09 11:22:50 -07007677 fput(file);
Jens Axboec3a31e62019-10-03 13:59:56 -06007678#endif
7679}
7680
Pavel Begunkovb895c9a2021-04-01 15:43:40 +01007681static void __io_rsrc_put_work(struct io_rsrc_node *ref_node)
Jens Axboe05f3fb32019-12-09 11:22:50 -07007682{
Pavel Begunkovb895c9a2021-04-01 15:43:40 +01007683 struct io_rsrc_data *rsrc_data = ref_node->rsrc_data;
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00007684 struct io_ring_ctx *ctx = rsrc_data->ctx;
7685 struct io_rsrc_put *prsrc, *tmp;
Xiaoguang Wang05589552020-03-31 14:05:18 +08007686
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00007687 list_for_each_entry_safe(prsrc, tmp, &ref_node->rsrc_list, list) {
7688 list_del(&prsrc->list);
Pavel Begunkovb60c8dc2021-04-25 14:32:18 +01007689
7690 if (prsrc->tag) {
7691 bool lock_ring = ctx->flags & IORING_SETUP_IOPOLL;
Pavel Begunkovb60c8dc2021-04-25 14:32:18 +01007692
7693 io_ring_submit_lock(ctx, lock_ring);
Jens Axboe79ebeae2021-08-10 15:18:27 -06007694 spin_lock(&ctx->completion_lock);
Pavel Begunkovb60c8dc2021-04-25 14:32:18 +01007695 io_cqring_fill_event(ctx, prsrc->tag, 0, 0);
Pavel Begunkov2840f712021-04-27 16:13:51 +01007696 ctx->cq_extra++;
Pavel Begunkovb60c8dc2021-04-25 14:32:18 +01007697 io_commit_cqring(ctx);
Jens Axboe79ebeae2021-08-10 15:18:27 -06007698 spin_unlock(&ctx->completion_lock);
Pavel Begunkovb60c8dc2021-04-25 14:32:18 +01007699 io_cqring_ev_posted(ctx);
7700 io_ring_submit_unlock(ctx, lock_ring);
7701 }
7702
Pavel Begunkov40ae0ff2021-04-01 15:43:44 +01007703 rsrc_data->do_put(ctx, prsrc);
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00007704 kfree(prsrc);
Jens Axboe05f3fb32019-12-09 11:22:50 -07007705 }
7706
Pavel Begunkov28a9fe22021-04-01 15:43:47 +01007707 io_rsrc_node_destroy(ref_node);
Pavel Begunkov3e942492021-04-11 01:46:34 +01007708 if (atomic_dec_and_test(&rsrc_data->refs))
7709 complete(&rsrc_data->done);
Jens Axboe05f3fb32019-12-09 11:22:50 -07007710}
7711
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00007712static void io_rsrc_put_work(struct work_struct *work)
Jens Axboe4a38aed22020-05-14 17:21:15 -06007713{
7714 struct io_ring_ctx *ctx;
7715 struct llist_node *node;
7716
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00007717 ctx = container_of(work, struct io_ring_ctx, rsrc_put_work.work);
7718 node = llist_del_all(&ctx->rsrc_put_llist);
Jens Axboe4a38aed22020-05-14 17:21:15 -06007719
7720 while (node) {
Pavel Begunkovb895c9a2021-04-01 15:43:40 +01007721 struct io_rsrc_node *ref_node;
Jens Axboe4a38aed22020-05-14 17:21:15 -06007722 struct llist_node *next = node->next;
7723
Pavel Begunkovb895c9a2021-04-01 15:43:40 +01007724 ref_node = llist_entry(node, struct io_rsrc_node, llist);
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00007725 __io_rsrc_put_work(ref_node);
Jens Axboe4a38aed22020-05-14 17:21:15 -06007726 node = next;
7727 }
7728}
7729
Jens Axboe05f3fb32019-12-09 11:22:50 -07007730static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg,
Pavel Begunkov792e3582021-04-25 14:32:21 +01007731 unsigned nr_args, u64 __user *tags)
Jens Axboe05f3fb32019-12-09 11:22:50 -07007732{
7733 __s32 __user *fds = (__s32 __user *) arg;
Jens Axboe05f3fb32019-12-09 11:22:50 -07007734 struct file *file;
Pavel Begunkovf3baed32021-04-01 15:43:42 +01007735 int fd, ret;
Pavel Begunkov846a4ef2021-04-01 15:44:03 +01007736 unsigned i;
Jens Axboe05f3fb32019-12-09 11:22:50 -07007737
7738 if (ctx->file_data)
7739 return -EBUSY;
7740 if (!nr_args)
7741 return -EINVAL;
7742 if (nr_args > IORING_MAX_FIXED_FILES)
7743 return -EMFILE;
Pavel Begunkov3a1b8a42021-08-20 10:36:35 +01007744 if (nr_args > rlimit(RLIMIT_NOFILE))
7745 return -EMFILE;
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01007746 ret = io_rsrc_node_switch_start(ctx);
Pavel Begunkovf3baed32021-04-01 15:43:42 +01007747 if (ret)
7748 return ret;
Pavel Begunkovd878c812021-06-14 02:36:18 +01007749 ret = io_rsrc_data_alloc(ctx, io_rsrc_file_put, tags, nr_args,
7750 &ctx->file_data);
7751 if (ret)
7752 return ret;
Jens Axboe05f3fb32019-12-09 11:22:50 -07007753
Pavel Begunkovf3baed32021-04-01 15:43:42 +01007754 ret = -ENOMEM;
Pavel Begunkovaeca2412021-04-11 01:46:37 +01007755 if (!io_alloc_file_tables(&ctx->file_table, nr_args))
Pavel Begunkov600cf3f2020-10-10 18:34:15 +01007756 goto out_free;
Jens Axboe05f3fb32019-12-09 11:22:50 -07007757
Jens Axboe05f3fb32019-12-09 11:22:50 -07007758 for (i = 0; i < nr_args; i++, ctx->nr_user_files++) {
Pavel Begunkovd878c812021-06-14 02:36:18 +01007759 if (copy_from_user(&fd, &fds[i], sizeof(fd))) {
Pavel Begunkov600cf3f2020-10-10 18:34:15 +01007760 ret = -EFAULT;
7761 goto out_fput;
Jens Axboe05f3fb32019-12-09 11:22:50 -07007762 }
Pavel Begunkov600cf3f2020-10-10 18:34:15 +01007763 /* allow sparse sets */
Pavel Begunkov792e3582021-04-25 14:32:21 +01007764 if (fd == -1) {
7765 ret = -EINVAL;
Pavel Begunkov2d091d62021-06-14 02:36:21 +01007766 if (unlikely(*io_get_tag_slot(ctx->file_data, i)))
Pavel Begunkov792e3582021-04-25 14:32:21 +01007767 goto out_fput;
Pavel Begunkov600cf3f2020-10-10 18:34:15 +01007768 continue;
Pavel Begunkov792e3582021-04-25 14:32:21 +01007769 }
Jens Axboe05f3fb32019-12-09 11:22:50 -07007770
Jens Axboe05f3fb32019-12-09 11:22:50 -07007771 file = fget(fd);
Jens Axboe05f3fb32019-12-09 11:22:50 -07007772 ret = -EBADF;
Pavel Begunkov792e3582021-04-25 14:32:21 +01007773 if (unlikely(!file))
Pavel Begunkov600cf3f2020-10-10 18:34:15 +01007774 goto out_fput;
Jens Axboe05f3fb32019-12-09 11:22:50 -07007775
7776 /*
7777 * Don't allow io_uring instances to be registered. If UNIX
7778 * isn't enabled, then this causes a reference cycle and this
7779 * instance can never get freed. If UNIX is enabled we'll
7780 * handle it just fine, but there's still no point in allowing
7781 * a ring fd as it doesn't support regular read/write anyway.
7782 */
7783 if (file->f_op == &io_uring_fops) {
7784 fput(file);
Pavel Begunkov600cf3f2020-10-10 18:34:15 +01007785 goto out_fput;
Jens Axboe05f3fb32019-12-09 11:22:50 -07007786 }
Pavel Begunkovaeca2412021-04-11 01:46:37 +01007787 io_fixed_file_set(io_fixed_file_slot(&ctx->file_table, i), file);
Jens Axboe05f3fb32019-12-09 11:22:50 -07007788 }
7789
Jens Axboe05f3fb32019-12-09 11:22:50 -07007790 ret = io_sqe_files_scm(ctx);
Xiaoguang Wang05589552020-03-31 14:05:18 +08007791 if (ret) {
Pavel Begunkov08480402021-04-13 02:58:38 +01007792 __io_sqe_files_unregister(ctx);
Xiaoguang Wang05589552020-03-31 14:05:18 +08007793 return ret;
7794 }
Jens Axboe05f3fb32019-12-09 11:22:50 -07007795
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01007796 io_rsrc_node_switch(ctx, NULL);
Jens Axboe05f3fb32019-12-09 11:22:50 -07007797 return ret;
Pavel Begunkov600cf3f2020-10-10 18:34:15 +01007798out_fput:
7799 for (i = 0; i < ctx->nr_user_files; i++) {
7800 file = io_file_from_index(ctx, i);
7801 if (file)
7802 fput(file);
7803 }
Pavel Begunkov042b0d82021-08-09 13:04:01 +01007804 io_free_file_tables(&ctx->file_table);
Pavel Begunkov600cf3f2020-10-10 18:34:15 +01007805 ctx->nr_user_files = 0;
Pavel Begunkov600cf3f2020-10-10 18:34:15 +01007806out_free:
Pavel Begunkov44b31f22021-04-25 14:32:16 +01007807 io_rsrc_data_free(ctx->file_data);
Jens Axboe55cbc252020-10-14 07:35:57 -06007808 ctx->file_data = NULL;
Jens Axboe05f3fb32019-12-09 11:22:50 -07007809 return ret;
7810}
7811
Jens Axboec3a31e62019-10-03 13:59:56 -06007812static int io_sqe_file_register(struct io_ring_ctx *ctx, struct file *file,
7813 int index)
7814{
7815#if defined(CONFIG_UNIX)
7816 struct sock *sock = ctx->ring_sock->sk;
7817 struct sk_buff_head *head = &sock->sk_receive_queue;
7818 struct sk_buff *skb;
7819
7820 /*
7821 * See if we can merge this file into an existing skb SCM_RIGHTS
7822 * file set. If there's no room, fall back to allocating a new skb
7823 * and filling it in.
7824 */
7825 spin_lock_irq(&head->lock);
7826 skb = skb_peek(head);
7827 if (skb) {
7828 struct scm_fp_list *fpl = UNIXCB(skb).fp;
7829
7830 if (fpl->count < SCM_MAX_FD) {
7831 __skb_unlink(skb, head);
7832 spin_unlock_irq(&head->lock);
7833 fpl->fp[fpl->count] = get_file(file);
7834 unix_inflight(fpl->user, fpl->fp[fpl->count]);
7835 fpl->count++;
7836 spin_lock_irq(&head->lock);
7837 __skb_queue_head(head, skb);
7838 } else {
7839 skb = NULL;
7840 }
7841 }
7842 spin_unlock_irq(&head->lock);
7843
7844 if (skb) {
7845 fput(file);
7846 return 0;
7847 }
7848
7849 return __io_sqe_files_scm(ctx, 1, index);
7850#else
7851 return 0;
7852#endif
7853}
7854
Pavel Begunkovb60c8dc2021-04-25 14:32:18 +01007855static int io_queue_rsrc_removal(struct io_rsrc_data *data, unsigned idx,
Pavel Begunkove7c78372021-04-01 15:43:45 +01007856 struct io_rsrc_node *node, void *rsrc)
Jens Axboe05f3fb32019-12-09 11:22:50 -07007857{
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00007858 struct io_rsrc_put *prsrc;
Jens Axboe05f3fb32019-12-09 11:22:50 -07007859
Bijan Mottahedeh269bbe52021-01-15 17:37:44 +00007860 prsrc = kzalloc(sizeof(*prsrc), GFP_KERNEL);
7861 if (!prsrc)
Hillf Dantona5318d32020-03-23 17:47:15 +08007862 return -ENOMEM;
Jens Axboe05f3fb32019-12-09 11:22:50 -07007863
Pavel Begunkov2d091d62021-06-14 02:36:21 +01007864 prsrc->tag = *io_get_tag_slot(data, idx);
Bijan Mottahedeh50238532021-01-15 17:37:45 +00007865 prsrc->rsrc = rsrc;
Pavel Begunkove7c78372021-04-01 15:43:45 +01007866 list_add(&prsrc->list, &node->rsrc_list);
Hillf Dantona5318d32020-03-23 17:47:15 +08007867 return 0;
Jens Axboe05f3fb32019-12-09 11:22:50 -07007868}
7869
7870static int __io_sqe_files_update(struct io_ring_ctx *ctx,
Pavel Begunkovc3bdad02021-04-25 14:32:22 +01007871 struct io_uring_rsrc_update2 *up,
Jens Axboe05f3fb32019-12-09 11:22:50 -07007872 unsigned nr_args)
7873{
Pavel Begunkovc3bdad02021-04-25 14:32:22 +01007874 u64 __user *tags = u64_to_user_ptr(up->tags);
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +01007875 __s32 __user *fds = u64_to_user_ptr(up->data);
Pavel Begunkovb895c9a2021-04-01 15:43:40 +01007876 struct io_rsrc_data *data = ctx->file_data;
Pavel Begunkova04b0ac2021-04-01 15:44:04 +01007877 struct io_fixed_file *file_slot;
7878 struct file *file;
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +01007879 int fd, i, err = 0;
7880 unsigned int done;
Xiaoguang Wang05589552020-03-31 14:05:18 +08007881 bool needs_switch = false;
Jens Axboec3a31e62019-10-03 13:59:56 -06007882
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +01007883 if (!ctx->file_data)
7884 return -ENXIO;
7885 if (up->offset + nr_args > ctx->nr_user_files)
Jens Axboec3a31e62019-10-03 13:59:56 -06007886 return -EINVAL;
7887
Pavel Begunkov67973b92021-01-26 13:51:09 +00007888 for (done = 0; done < nr_args; done++) {
Pavel Begunkovc3bdad02021-04-25 14:32:22 +01007889 u64 tag = 0;
7890
7891 if ((tags && copy_from_user(&tag, &tags[done], sizeof(tag))) ||
7892 copy_from_user(&fd, &fds[done], sizeof(fd))) {
Jens Axboec3a31e62019-10-03 13:59:56 -06007893 err = -EFAULT;
7894 break;
7895 }
Pavel Begunkovc3bdad02021-04-25 14:32:22 +01007896 if ((fd == IORING_REGISTER_FILES_SKIP || fd == -1) && tag) {
7897 err = -EINVAL;
7898 break;
7899 }
noah4e0377a2021-01-26 15:23:28 -05007900 if (fd == IORING_REGISTER_FILES_SKIP)
7901 continue;
7902
Pavel Begunkov67973b92021-01-26 13:51:09 +00007903 i = array_index_nospec(up->offset + done, ctx->nr_user_files);
Pavel Begunkovaeca2412021-04-11 01:46:37 +01007904 file_slot = io_fixed_file_slot(&ctx->file_table, i);
Pavel Begunkovea64ec022021-02-04 13:52:07 +00007905
Pavel Begunkova04b0ac2021-04-01 15:44:04 +01007906 if (file_slot->file_ptr) {
7907 file = (struct file *)(file_slot->file_ptr & FFS_MASK);
Pavel Begunkovb60c8dc2021-04-25 14:32:18 +01007908 err = io_queue_rsrc_removal(data, up->offset + done,
7909 ctx->rsrc_node, file);
Hillf Dantona5318d32020-03-23 17:47:15 +08007910 if (err)
7911 break;
Pavel Begunkova04b0ac2021-04-01 15:44:04 +01007912 file_slot->file_ptr = 0;
Xiaoguang Wang05589552020-03-31 14:05:18 +08007913 needs_switch = true;
Jens Axboec3a31e62019-10-03 13:59:56 -06007914 }
7915 if (fd != -1) {
Jens Axboec3a31e62019-10-03 13:59:56 -06007916 file = fget(fd);
7917 if (!file) {
7918 err = -EBADF;
7919 break;
7920 }
7921 /*
7922 * Don't allow io_uring instances to be registered. If
7923 * UNIX isn't enabled, then this causes a reference
7924 * cycle and this instance can never get freed. If UNIX
7925 * is enabled we'll handle it just fine, but there's
7926 * still no point in allowing a ring fd as it doesn't
7927 * support regular read/write anyway.
7928 */
7929 if (file->f_op == &io_uring_fops) {
7930 fput(file);
7931 err = -EBADF;
7932 break;
7933 }
Pavel Begunkov2d091d62021-06-14 02:36:21 +01007934 *io_get_tag_slot(data, up->offset + done) = tag;
Pavel Begunkov9a321c92021-04-01 15:44:01 +01007935 io_fixed_file_set(file_slot, file);
Jens Axboec3a31e62019-10-03 13:59:56 -06007936 err = io_sqe_file_register(ctx, file, i);
Yang Yingliangf3bd9da2020-07-09 10:11:41 +00007937 if (err) {
Pavel Begunkova04b0ac2021-04-01 15:44:04 +01007938 file_slot->file_ptr = 0;
Yang Yingliangf3bd9da2020-07-09 10:11:41 +00007939 fput(file);
Jens Axboec3a31e62019-10-03 13:59:56 -06007940 break;
Yang Yingliangf3bd9da2020-07-09 10:11:41 +00007941 }
Jens Axboec3a31e62019-10-03 13:59:56 -06007942 }
Jens Axboe05f3fb32019-12-09 11:22:50 -07007943 }
7944
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01007945 if (needs_switch)
7946 io_rsrc_node_switch(ctx, data);
Jens Axboec3a31e62019-10-03 13:59:56 -06007947 return done ? done : err;
7948}
Xiaoguang Wang05589552020-03-31 14:05:18 +08007949
Jens Axboe685fe7f2021-03-08 09:37:51 -07007950static struct io_wq *io_init_wq_offload(struct io_ring_ctx *ctx,
7951 struct task_struct *task)
Pavel Begunkov24369c22020-01-28 03:15:48 +03007952{
Jens Axboee9418942021-02-19 12:33:30 -07007953 struct io_wq_hash *hash;
Pavel Begunkov24369c22020-01-28 03:15:48 +03007954 struct io_wq_data data;
Pavel Begunkov24369c22020-01-28 03:15:48 +03007955 unsigned int concurrency;
Pavel Begunkov24369c22020-01-28 03:15:48 +03007956
Yang Yingliang362a9e62021-07-20 16:38:05 +08007957 mutex_lock(&ctx->uring_lock);
Jens Axboee9418942021-02-19 12:33:30 -07007958 hash = ctx->hash_map;
7959 if (!hash) {
7960 hash = kzalloc(sizeof(*hash), GFP_KERNEL);
Yang Yingliang362a9e62021-07-20 16:38:05 +08007961 if (!hash) {
7962 mutex_unlock(&ctx->uring_lock);
Jens Axboee9418942021-02-19 12:33:30 -07007963 return ERR_PTR(-ENOMEM);
Yang Yingliang362a9e62021-07-20 16:38:05 +08007964 }
Jens Axboee9418942021-02-19 12:33:30 -07007965 refcount_set(&hash->refs, 1);
7966 init_waitqueue_head(&hash->wait);
7967 ctx->hash_map = hash;
7968 }
Yang Yingliang362a9e62021-07-20 16:38:05 +08007969 mutex_unlock(&ctx->uring_lock);
Jens Axboee9418942021-02-19 12:33:30 -07007970
7971 data.hash = hash;
Jens Axboe685fe7f2021-03-08 09:37:51 -07007972 data.task = task;
Pavel Begunkovebc11b62021-08-09 13:04:05 +01007973 data.free_work = io_wq_free_work;
Pavel Begunkovf5fa38c2020-06-08 21:08:20 +03007974 data.do_work = io_wq_submit_work;
Pavel Begunkov24369c22020-01-28 03:15:48 +03007975
Jens Axboed25e3a32021-02-16 11:41:41 -07007976 /* Do QD, or 4 * CPUS, whatever is smallest */
7977 concurrency = min(ctx->sq_entries, 4 * num_online_cpus());
Pavel Begunkov24369c22020-01-28 03:15:48 +03007978
Jens Axboe5aa75ed2021-02-16 12:56:50 -07007979 return io_wq_create(concurrency, &data);
Pavel Begunkov24369c22020-01-28 03:15:48 +03007980}
7981
Jens Axboe5aa75ed2021-02-16 12:56:50 -07007982static int io_uring_alloc_task_context(struct task_struct *task,
7983 struct io_ring_ctx *ctx)
Jens Axboe0f212202020-09-13 13:09:39 -06007984{
7985 struct io_uring_task *tctx;
Jens Axboed8a6df12020-10-15 16:24:45 -06007986 int ret;
Jens Axboe0f212202020-09-13 13:09:39 -06007987
Pavel Begunkov09899b12021-06-14 02:36:22 +01007988 tctx = kzalloc(sizeof(*tctx), GFP_KERNEL);
Jens Axboe0f212202020-09-13 13:09:39 -06007989 if (unlikely(!tctx))
7990 return -ENOMEM;
7991
Jens Axboed8a6df12020-10-15 16:24:45 -06007992 ret = percpu_counter_init(&tctx->inflight, 0, GFP_KERNEL);
7993 if (unlikely(ret)) {
7994 kfree(tctx);
7995 return ret;
7996 }
7997
Jens Axboe685fe7f2021-03-08 09:37:51 -07007998 tctx->io_wq = io_init_wq_offload(ctx, task);
Jens Axboe5aa75ed2021-02-16 12:56:50 -07007999 if (IS_ERR(tctx->io_wq)) {
8000 ret = PTR_ERR(tctx->io_wq);
8001 percpu_counter_destroy(&tctx->inflight);
8002 kfree(tctx);
8003 return ret;
8004 }
8005
Jens Axboe0f212202020-09-13 13:09:39 -06008006 xa_init(&tctx->xa);
8007 init_waitqueue_head(&tctx->wait);
Jens Axboefdaf0832020-10-30 09:37:30 -06008008 atomic_set(&tctx->in_idle, 0);
Pavel Begunkovb303fe22021-04-11 01:46:26 +01008009 atomic_set(&tctx->inflight_tracked, 0);
Jens Axboe0f212202020-09-13 13:09:39 -06008010 task->io_uring = tctx;
Jens Axboe7cbf1722021-02-10 00:03:20 +00008011 spin_lock_init(&tctx->task_lock);
8012 INIT_WQ_LIST(&tctx->task_list);
Jens Axboe7cbf1722021-02-10 00:03:20 +00008013 init_task_work(&tctx->task_work, tctx_task_work);
Jens Axboe0f212202020-09-13 13:09:39 -06008014 return 0;
8015}
8016
8017void __io_uring_free(struct task_struct *tsk)
8018{
8019 struct io_uring_task *tctx = tsk->io_uring;
8020
8021 WARN_ON_ONCE(!xa_empty(&tctx->xa));
Pavel Begunkovef8eaa42021-02-27 11:16:45 +00008022 WARN_ON_ONCE(tctx->io_wq);
Pavel Begunkov09899b12021-06-14 02:36:22 +01008023 WARN_ON_ONCE(tctx->cached_refs);
Pavel Begunkovef8eaa42021-02-27 11:16:45 +00008024
Jens Axboed8a6df12020-10-15 16:24:45 -06008025 percpu_counter_destroy(&tctx->inflight);
Jens Axboe0f212202020-09-13 13:09:39 -06008026 kfree(tctx);
8027 tsk->io_uring = NULL;
8028}
8029
Stefano Garzarella7e84e1c2020-08-27 16:58:31 +02008030static int io_sq_offload_create(struct io_ring_ctx *ctx,
8031 struct io_uring_params *p)
Jens Axboe2b188cc2019-01-07 10:46:33 -07008032{
8033 int ret;
8034
Jens Axboed25e3a32021-02-16 11:41:41 -07008035 /* Retain compatibility with failing for an invalid attach attempt */
8036 if ((ctx->flags & (IORING_SETUP_ATTACH_WQ | IORING_SETUP_SQPOLL)) ==
8037 IORING_SETUP_ATTACH_WQ) {
8038 struct fd f;
8039
8040 f = fdget(p->wq_fd);
8041 if (!f.file)
8042 return -ENXIO;
Jens Axboe0cc936f2021-07-22 17:08:07 -06008043 if (f.file->f_op != &io_uring_fops) {
8044 fdput(f);
Pavel Begunkovf2a48dd2021-04-20 12:03:33 +01008045 return -EINVAL;
Jens Axboe0cc936f2021-07-22 17:08:07 -06008046 }
8047 fdput(f);
Jens Axboed25e3a32021-02-16 11:41:41 -07008048 }
Jens Axboe6c271ce2019-01-10 11:22:30 -07008049 if (ctx->flags & IORING_SETUP_SQPOLL) {
Jens Axboe46fe18b2021-03-04 12:39:36 -07008050 struct task_struct *tsk;
Jens Axboe534ca6d2020-09-02 13:52:19 -06008051 struct io_sq_data *sqd;
Pavel Begunkov26984fb2021-03-11 23:29:37 +00008052 bool attached;
Jens Axboe534ca6d2020-09-02 13:52:19 -06008053
Pavel Begunkov26984fb2021-03-11 23:29:37 +00008054 sqd = io_get_sq_data(p, &attached);
Jens Axboe534ca6d2020-09-02 13:52:19 -06008055 if (IS_ERR(sqd)) {
8056 ret = PTR_ERR(sqd);
8057 goto err;
8058 }
Jens Axboe69fb2132020-09-14 11:16:23 -06008059
Stefan Metzmacher7c30f36a2021-03-07 11:54:28 +01008060 ctx->sq_creds = get_current_cred();
Jens Axboe534ca6d2020-09-02 13:52:19 -06008061 ctx->sq_data = sqd;
Jens Axboe917257d2019-04-13 09:28:55 -06008062 ctx->sq_thread_idle = msecs_to_jiffies(p->sq_thread_idle);
8063 if (!ctx->sq_thread_idle)
8064 ctx->sq_thread_idle = HZ;
8065
Pavel Begunkov78d7f6b2021-03-10 13:13:53 +00008066 io_sq_thread_park(sqd);
Pavel Begunkovde75a3d2021-03-18 11:54:35 +00008067 list_add(&ctx->sqd_list, &sqd->ctx_list);
8068 io_sqd_update_thread_idle(sqd);
Pavel Begunkov26984fb2021-03-11 23:29:37 +00008069 /* don't attach to a dying SQPOLL thread, would be racy */
Pavel Begunkovf2a48dd2021-04-20 12:03:33 +01008070 ret = (attached && !sqd->thread) ? -ENXIO : 0;
Pavel Begunkov78d7f6b2021-03-10 13:13:53 +00008071 io_sq_thread_unpark(sqd);
8072
Pavel Begunkovde75a3d2021-03-18 11:54:35 +00008073 if (ret < 0)
8074 goto err;
8075 if (attached)
Jens Axboe5aa75ed2021-02-16 12:56:50 -07008076 return 0;
Jens Axboeaa061652020-09-02 14:50:27 -06008077
Jens Axboe6c271ce2019-01-10 11:22:30 -07008078 if (p->flags & IORING_SETUP_SQ_AFF) {
Jens Axboe44a9bd12019-05-14 20:00:30 -06008079 int cpu = p->sq_thread_cpu;
Jens Axboe6c271ce2019-01-10 11:22:30 -07008080
Jens Axboe917257d2019-04-13 09:28:55 -06008081 ret = -EINVAL;
Pavel Begunkovf2a48dd2021-04-20 12:03:33 +01008082 if (cpu >= nr_cpu_ids || !cpu_online(cpu))
Jens Axboee8f98f242021-03-09 16:32:13 -07008083 goto err_sqpoll;
Jens Axboe37d1e2e2021-02-17 21:03:43 -07008084 sqd->sq_cpu = cpu;
Jens Axboe6c271ce2019-01-10 11:22:30 -07008085 } else {
Jens Axboe37d1e2e2021-02-17 21:03:43 -07008086 sqd->sq_cpu = -1;
Jens Axboe6c271ce2019-01-10 11:22:30 -07008087 }
Jens Axboe37d1e2e2021-02-17 21:03:43 -07008088
8089 sqd->task_pid = current->pid;
Jens Axboe5c2469e2021-03-11 10:17:56 -07008090 sqd->task_tgid = current->tgid;
Jens Axboe46fe18b2021-03-04 12:39:36 -07008091 tsk = create_io_thread(io_sq_thread, sqd, NUMA_NO_NODE);
8092 if (IS_ERR(tsk)) {
8093 ret = PTR_ERR(tsk);
Jens Axboee8f98f242021-03-09 16:32:13 -07008094 goto err_sqpoll;
Jens Axboe6c271ce2019-01-10 11:22:30 -07008095 }
Pavel Begunkov97a73a02021-03-08 17:30:54 +00008096
Jens Axboe46fe18b2021-03-04 12:39:36 -07008097 sqd->thread = tsk;
Pavel Begunkov97a73a02021-03-08 17:30:54 +00008098 ret = io_uring_alloc_task_context(tsk, ctx);
Jens Axboe46fe18b2021-03-04 12:39:36 -07008099 wake_up_new_task(tsk);
Jens Axboe0f212202020-09-13 13:09:39 -06008100 if (ret)
8101 goto err;
Jens Axboe6c271ce2019-01-10 11:22:30 -07008102 } else if (p->flags & IORING_SETUP_SQ_AFF) {
8103 /* Can't have SQ_AFF without SQPOLL */
8104 ret = -EINVAL;
8105 goto err;
8106 }
8107
Jens Axboe2b188cc2019-01-07 10:46:33 -07008108 return 0;
Pavel Begunkovf2a48dd2021-04-20 12:03:33 +01008109err_sqpoll:
8110 complete(&ctx->sq_data->exited);
Jens Axboe2b188cc2019-01-07 10:46:33 -07008111err:
Jens Axboe37d1e2e2021-02-17 21:03:43 -07008112 io_sq_thread_finish(ctx);
Jens Axboe2b188cc2019-01-07 10:46:33 -07008113 return ret;
8114}
8115
Bijan Mottahedeha087e2b2020-06-16 16:36:07 -07008116static inline void __io_unaccount_mem(struct user_struct *user,
8117 unsigned long nr_pages)
Jens Axboe2b188cc2019-01-07 10:46:33 -07008118{
8119 atomic_long_sub(nr_pages, &user->locked_vm);
8120}
8121
Bijan Mottahedeha087e2b2020-06-16 16:36:07 -07008122static inline int __io_account_mem(struct user_struct *user,
8123 unsigned long nr_pages)
Jens Axboe2b188cc2019-01-07 10:46:33 -07008124{
8125 unsigned long page_limit, cur_pages, new_pages;
8126
8127 /* Don't allow more pages than we can safely lock */
8128 page_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
8129
8130 do {
8131 cur_pages = atomic_long_read(&user->locked_vm);
8132 new_pages = cur_pages + nr_pages;
8133 if (new_pages > page_limit)
8134 return -ENOMEM;
8135 } while (atomic_long_cmpxchg(&user->locked_vm, cur_pages,
8136 new_pages) != cur_pages);
8137
8138 return 0;
8139}
8140
Jens Axboe26bfa89e2021-02-09 20:14:12 -07008141static void io_unaccount_mem(struct io_ring_ctx *ctx, unsigned long nr_pages)
Bijan Mottahedeha087e2b2020-06-16 16:36:07 -07008142{
Jens Axboe62e398b2021-02-21 16:19:37 -07008143 if (ctx->user)
Bijan Mottahedeha087e2b2020-06-16 16:36:07 -07008144 __io_unaccount_mem(ctx->user, nr_pages);
Bijan Mottahedeh30975822020-06-16 16:36:09 -07008145
Jens Axboe26bfa89e2021-02-09 20:14:12 -07008146 if (ctx->mm_account)
8147 atomic64_sub(nr_pages, &ctx->mm_account->pinned_vm);
Bijan Mottahedeha087e2b2020-06-16 16:36:07 -07008148}
8149
Jens Axboe26bfa89e2021-02-09 20:14:12 -07008150static int io_account_mem(struct io_ring_ctx *ctx, unsigned long nr_pages)
Bijan Mottahedeha087e2b2020-06-16 16:36:07 -07008151{
Bijan Mottahedeh30975822020-06-16 16:36:09 -07008152 int ret;
8153
Jens Axboe62e398b2021-02-21 16:19:37 -07008154 if (ctx->user) {
Bijan Mottahedeh30975822020-06-16 16:36:09 -07008155 ret = __io_account_mem(ctx->user, nr_pages);
8156 if (ret)
8157 return ret;
8158 }
8159
Jens Axboe26bfa89e2021-02-09 20:14:12 -07008160 if (ctx->mm_account)
8161 atomic64_add(nr_pages, &ctx->mm_account->pinned_vm);
Bijan Mottahedeha087e2b2020-06-16 16:36:07 -07008162
8163 return 0;
8164}
8165
Jens Axboe2b188cc2019-01-07 10:46:33 -07008166static void io_mem_free(void *ptr)
8167{
Mark Rutland52e04ef2019-04-30 17:30:21 +01008168 struct page *page;
Jens Axboe2b188cc2019-01-07 10:46:33 -07008169
Mark Rutland52e04ef2019-04-30 17:30:21 +01008170 if (!ptr)
8171 return;
8172
8173 page = virt_to_head_page(ptr);
Jens Axboe2b188cc2019-01-07 10:46:33 -07008174 if (put_page_testzero(page))
8175 free_compound_page(page);
8176}
8177
8178static void *io_mem_alloc(size_t size)
8179{
8180 gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN | __GFP_COMP |
Jens Axboe26bfa89e2021-02-09 20:14:12 -07008181 __GFP_NORETRY | __GFP_ACCOUNT;
Jens Axboe2b188cc2019-01-07 10:46:33 -07008182
8183 return (void *) __get_free_pages(gfp_flags, get_order(size));
8184}
8185
Hristo Venev75b28af2019-08-26 17:23:46 +00008186static unsigned long rings_size(unsigned sq_entries, unsigned cq_entries,
8187 size_t *sq_offset)
8188{
8189 struct io_rings *rings;
8190 size_t off, sq_array_size;
8191
8192 off = struct_size(rings, cqes, cq_entries);
8193 if (off == SIZE_MAX)
8194 return SIZE_MAX;
8195
8196#ifdef CONFIG_SMP
8197 off = ALIGN(off, SMP_CACHE_BYTES);
8198 if (off == 0)
8199 return SIZE_MAX;
8200#endif
8201
Dmitry Vyukovb36200f2020-07-11 11:31:11 +02008202 if (sq_offset)
8203 *sq_offset = off;
8204
Hristo Venev75b28af2019-08-26 17:23:46 +00008205 sq_array_size = array_size(sizeof(u32), sq_entries);
8206 if (sq_array_size == SIZE_MAX)
8207 return SIZE_MAX;
8208
8209 if (check_add_overflow(off, sq_array_size, &off))
8210 return SIZE_MAX;
8211
Hristo Venev75b28af2019-08-26 17:23:46 +00008212 return off;
8213}
8214
Pavel Begunkov41edf1a2021-04-25 14:32:23 +01008215static void io_buffer_unmap(struct io_ring_ctx *ctx, struct io_mapped_ubuf **slot)
Pavel Begunkov7f61a1e2021-04-11 01:46:35 +01008216{
Pavel Begunkov41edf1a2021-04-25 14:32:23 +01008217 struct io_mapped_ubuf *imu = *slot;
Pavel Begunkov7f61a1e2021-04-11 01:46:35 +01008218 unsigned int i;
8219
Pavel Begunkov62248432021-04-28 13:11:29 +01008220 if (imu != ctx->dummy_ubuf) {
8221 for (i = 0; i < imu->nr_bvecs; i++)
8222 unpin_user_page(imu->bvec[i].bv_page);
8223 if (imu->acct_pages)
8224 io_unaccount_mem(ctx, imu->acct_pages);
8225 kvfree(imu);
8226 }
Pavel Begunkov41edf1a2021-04-25 14:32:23 +01008227 *slot = NULL;
Pavel Begunkov7f61a1e2021-04-11 01:46:35 +01008228}
8229
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01008230static void io_rsrc_buf_put(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc)
8231{
Pavel Begunkov634d00d2021-04-25 14:32:26 +01008232 io_buffer_unmap(ctx, &prsrc->buf);
8233 prsrc->buf = NULL;
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01008234}
8235
8236static void __io_sqe_buffers_unregister(struct io_ring_ctx *ctx)
Jens Axboeedafcce2019-01-09 09:16:05 -07008237{
Pavel Begunkov7f61a1e2021-04-11 01:46:35 +01008238 unsigned int i;
Jens Axboeedafcce2019-01-09 09:16:05 -07008239
Pavel Begunkov7f61a1e2021-04-11 01:46:35 +01008240 for (i = 0; i < ctx->nr_user_bufs; i++)
8241 io_buffer_unmap(ctx, &ctx->user_bufs[i]);
Jens Axboeedafcce2019-01-09 09:16:05 -07008242 kfree(ctx->user_bufs);
Zqiangbb6659c2021-04-30 16:25:15 +08008243 io_rsrc_data_free(ctx->buf_data);
Jens Axboeedafcce2019-01-09 09:16:05 -07008244 ctx->user_bufs = NULL;
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01008245 ctx->buf_data = NULL;
Jens Axboeedafcce2019-01-09 09:16:05 -07008246 ctx->nr_user_bufs = 0;
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01008247}
8248
Jens Axboeedafcce2019-01-09 09:16:05 -07008249static int io_sqe_buffers_unregister(struct io_ring_ctx *ctx)
8250{
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01008251 int ret;
Jens Axboeedafcce2019-01-09 09:16:05 -07008252
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01008253 if (!ctx->buf_data)
Jens Axboeedafcce2019-01-09 09:16:05 -07008254 return -ENXIO;
8255
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01008256 ret = io_rsrc_ref_quiesce(ctx->buf_data, ctx);
8257 if (!ret)
8258 __io_sqe_buffers_unregister(ctx);
8259 return ret;
Jens Axboeedafcce2019-01-09 09:16:05 -07008260}
8261
8262static int io_copy_iov(struct io_ring_ctx *ctx, struct iovec *dst,
8263 void __user *arg, unsigned index)
8264{
8265 struct iovec __user *src;
8266
8267#ifdef CONFIG_COMPAT
8268 if (ctx->compat) {
8269 struct compat_iovec __user *ciovs;
8270 struct compat_iovec ciov;
8271
8272 ciovs = (struct compat_iovec __user *) arg;
8273 if (copy_from_user(&ciov, &ciovs[index], sizeof(ciov)))
8274 return -EFAULT;
8275
Jens Axboed55e5f52019-12-11 16:12:15 -07008276 dst->iov_base = u64_to_user_ptr((u64)ciov.iov_base);
Jens Axboeedafcce2019-01-09 09:16:05 -07008277 dst->iov_len = ciov.iov_len;
8278 return 0;
8279 }
8280#endif
8281 src = (struct iovec __user *) arg;
8282 if (copy_from_user(dst, &src[index], sizeof(*dst)))
8283 return -EFAULT;
8284 return 0;
8285}
8286
Jens Axboede293932020-09-17 16:19:16 -06008287/*
8288 * Not super efficient, but this is just a registration time. And we do cache
8289 * the last compound head, so generally we'll only do a full search if we don't
8290 * match that one.
8291 *
8292 * We check if the given compound head page has already been accounted, to
8293 * avoid double accounting it. This allows us to account the full size of the
8294 * page, not just the constituent pages of a huge page.
8295 */
8296static bool headpage_already_acct(struct io_ring_ctx *ctx, struct page **pages,
8297 int nr_pages, struct page *hpage)
8298{
8299 int i, j;
8300
8301 /* check current page array */
8302 for (i = 0; i < nr_pages; i++) {
8303 if (!PageCompound(pages[i]))
8304 continue;
8305 if (compound_head(pages[i]) == hpage)
8306 return true;
8307 }
8308
8309 /* check previously registered pages */
8310 for (i = 0; i < ctx->nr_user_bufs; i++) {
Pavel Begunkov41edf1a2021-04-25 14:32:23 +01008311 struct io_mapped_ubuf *imu = ctx->user_bufs[i];
Jens Axboede293932020-09-17 16:19:16 -06008312
8313 for (j = 0; j < imu->nr_bvecs; j++) {
8314 if (!PageCompound(imu->bvec[j].bv_page))
8315 continue;
8316 if (compound_head(imu->bvec[j].bv_page) == hpage)
8317 return true;
8318 }
8319 }
8320
8321 return false;
8322}
8323
8324static int io_buffer_account_pin(struct io_ring_ctx *ctx, struct page **pages,
8325 int nr_pages, struct io_mapped_ubuf *imu,
8326 struct page **last_hpage)
8327{
8328 int i, ret;
8329
Pavel Begunkov216e5832021-05-29 12:01:02 +01008330 imu->acct_pages = 0;
Jens Axboede293932020-09-17 16:19:16 -06008331 for (i = 0; i < nr_pages; i++) {
8332 if (!PageCompound(pages[i])) {
8333 imu->acct_pages++;
8334 } else {
8335 struct page *hpage;
8336
8337 hpage = compound_head(pages[i]);
8338 if (hpage == *last_hpage)
8339 continue;
8340 *last_hpage = hpage;
8341 if (headpage_already_acct(ctx, pages, i, hpage))
8342 continue;
8343 imu->acct_pages += page_size(hpage) >> PAGE_SHIFT;
8344 }
8345 }
8346
8347 if (!imu->acct_pages)
8348 return 0;
8349
Jens Axboe26bfa89e2021-02-09 20:14:12 -07008350 ret = io_account_mem(ctx, imu->acct_pages);
Jens Axboede293932020-09-17 16:19:16 -06008351 if (ret)
8352 imu->acct_pages = 0;
8353 return ret;
8354}
8355
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08008356static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov,
Pavel Begunkov41edf1a2021-04-25 14:32:23 +01008357 struct io_mapped_ubuf **pimu,
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08008358 struct page **last_hpage)
Jens Axboeedafcce2019-01-09 09:16:05 -07008359{
Pavel Begunkov41edf1a2021-04-25 14:32:23 +01008360 struct io_mapped_ubuf *imu = NULL;
Jens Axboeedafcce2019-01-09 09:16:05 -07008361 struct vm_area_struct **vmas = NULL;
8362 struct page **pages = NULL;
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08008363 unsigned long off, start, end, ubuf;
8364 size_t size;
8365 int ret, pret, nr_pages, i;
Jens Axboeedafcce2019-01-09 09:16:05 -07008366
Pavel Begunkov62248432021-04-28 13:11:29 +01008367 if (!iov->iov_base) {
8368 *pimu = ctx->dummy_ubuf;
8369 return 0;
8370 }
8371
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08008372 ubuf = (unsigned long) iov->iov_base;
8373 end = (ubuf + iov->iov_len + PAGE_SIZE - 1) >> PAGE_SHIFT;
8374 start = ubuf >> PAGE_SHIFT;
8375 nr_pages = end - start;
8376
Pavel Begunkov41edf1a2021-04-25 14:32:23 +01008377 *pimu = NULL;
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08008378 ret = -ENOMEM;
8379
8380 pages = kvmalloc_array(nr_pages, sizeof(struct page *), GFP_KERNEL);
8381 if (!pages)
8382 goto done;
8383
8384 vmas = kvmalloc_array(nr_pages, sizeof(struct vm_area_struct *),
8385 GFP_KERNEL);
8386 if (!vmas)
8387 goto done;
8388
Pavel Begunkov41edf1a2021-04-25 14:32:23 +01008389 imu = kvmalloc(struct_size(imu, bvec, nr_pages), GFP_KERNEL);
Pavel Begunkova2b41982021-04-26 00:16:31 +01008390 if (!imu)
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08008391 goto done;
8392
8393 ret = 0;
8394 mmap_read_lock(current->mm);
8395 pret = pin_user_pages(ubuf, nr_pages, FOLL_WRITE | FOLL_LONGTERM,
8396 pages, vmas);
8397 if (pret == nr_pages) {
8398 /* don't support file backed memory */
8399 for (i = 0; i < nr_pages; i++) {
8400 struct vm_area_struct *vma = vmas[i];
8401
Pavel Begunkov40dad762021-06-09 15:26:54 +01008402 if (vma_is_shmem(vma))
8403 continue;
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08008404 if (vma->vm_file &&
8405 !is_file_hugepages(vma->vm_file)) {
8406 ret = -EOPNOTSUPP;
8407 break;
8408 }
8409 }
8410 } else {
8411 ret = pret < 0 ? pret : -EFAULT;
8412 }
8413 mmap_read_unlock(current->mm);
8414 if (ret) {
8415 /*
8416 * if we did partial map, or found file backed vmas,
8417 * release any pages we did get
8418 */
8419 if (pret > 0)
8420 unpin_user_pages(pages, pret);
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08008421 goto done;
8422 }
8423
8424 ret = io_buffer_account_pin(ctx, pages, pret, imu, last_hpage);
8425 if (ret) {
8426 unpin_user_pages(pages, pret);
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08008427 goto done;
8428 }
8429
8430 off = ubuf & ~PAGE_MASK;
8431 size = iov->iov_len;
8432 for (i = 0; i < nr_pages; i++) {
8433 size_t vec_len;
8434
8435 vec_len = min_t(size_t, size, PAGE_SIZE - off);
8436 imu->bvec[i].bv_page = pages[i];
8437 imu->bvec[i].bv_len = vec_len;
8438 imu->bvec[i].bv_offset = off;
8439 off = 0;
8440 size -= vec_len;
8441 }
8442 /* store original address for later verification */
8443 imu->ubuf = ubuf;
Pavel Begunkov4751f532021-04-01 15:43:55 +01008444 imu->ubuf_end = ubuf + iov->iov_len;
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08008445 imu->nr_bvecs = nr_pages;
Pavel Begunkov41edf1a2021-04-25 14:32:23 +01008446 *pimu = imu;
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08008447 ret = 0;
8448done:
Pavel Begunkov41edf1a2021-04-25 14:32:23 +01008449 if (ret)
8450 kvfree(imu);
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08008451 kvfree(pages);
8452 kvfree(vmas);
8453 return ret;
8454}
8455
Bijan Mottahedeh2b358602021-01-06 12:39:11 -08008456static int io_buffers_map_alloc(struct io_ring_ctx *ctx, unsigned int nr_args)
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08008457{
Pavel Begunkov87094462021-04-11 01:46:36 +01008458 ctx->user_bufs = kcalloc(nr_args, sizeof(*ctx->user_bufs), GFP_KERNEL);
8459 return ctx->user_bufs ? 0 : -ENOMEM;
Bijan Mottahedeh2b358602021-01-06 12:39:11 -08008460}
8461
8462static int io_buffer_validate(struct iovec *iov)
8463{
Pavel Begunkov50e96982021-03-24 22:59:01 +00008464 unsigned long tmp, acct_len = iov->iov_len + (PAGE_SIZE - 1);
8465
Bijan Mottahedeh2b358602021-01-06 12:39:11 -08008466 /*
8467 * Don't impose further limits on the size and buffer
8468 * constraints here, we'll -EINVAL later when IO is
8469 * submitted if they are wrong.
8470 */
Pavel Begunkov62248432021-04-28 13:11:29 +01008471 if (!iov->iov_base)
8472 return iov->iov_len ? -EFAULT : 0;
8473 if (!iov->iov_len)
Bijan Mottahedeh2b358602021-01-06 12:39:11 -08008474 return -EFAULT;
8475
8476 /* arbitrary limit, but we need something */
8477 if (iov->iov_len > SZ_1G)
8478 return -EFAULT;
8479
Pavel Begunkov50e96982021-03-24 22:59:01 +00008480 if (check_add_overflow((unsigned long)iov->iov_base, acct_len, &tmp))
8481 return -EOVERFLOW;
8482
Bijan Mottahedeh2b358602021-01-06 12:39:11 -08008483 return 0;
8484}
8485
8486static int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg,
Pavel Begunkov634d00d2021-04-25 14:32:26 +01008487 unsigned int nr_args, u64 __user *tags)
Bijan Mottahedeh2b358602021-01-06 12:39:11 -08008488{
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01008489 struct page *last_hpage = NULL;
8490 struct io_rsrc_data *data;
Bijan Mottahedeh2b358602021-01-06 12:39:11 -08008491 int i, ret;
8492 struct iovec iov;
Bijan Mottahedeh2b358602021-01-06 12:39:11 -08008493
Pavel Begunkov87094462021-04-11 01:46:36 +01008494 if (ctx->user_bufs)
8495 return -EBUSY;
Pavel Begunkov489809e2021-05-14 12:06:44 +01008496 if (!nr_args || nr_args > IORING_MAX_REG_BUFFERS)
Pavel Begunkov87094462021-04-11 01:46:36 +01008497 return -EINVAL;
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01008498 ret = io_rsrc_node_switch_start(ctx);
Bijan Mottahedeh2b358602021-01-06 12:39:11 -08008499 if (ret)
8500 return ret;
Pavel Begunkovd878c812021-06-14 02:36:18 +01008501 ret = io_rsrc_data_alloc(ctx, io_rsrc_buf_put, tags, nr_args, &data);
8502 if (ret)
8503 return ret;
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01008504 ret = io_buffers_map_alloc(ctx, nr_args);
8505 if (ret) {
Zqiangbb6659c2021-04-30 16:25:15 +08008506 io_rsrc_data_free(data);
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01008507 return ret;
8508 }
Bijan Mottahedeh2b358602021-01-06 12:39:11 -08008509
Pavel Begunkov87094462021-04-11 01:46:36 +01008510 for (i = 0; i < nr_args; i++, ctx->nr_user_bufs++) {
Jens Axboeedafcce2019-01-09 09:16:05 -07008511 ret = io_copy_iov(ctx, &iov, arg, i);
8512 if (ret)
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08008513 break;
Bijan Mottahedeh2b358602021-01-06 12:39:11 -08008514 ret = io_buffer_validate(&iov);
8515 if (ret)
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08008516 break;
Pavel Begunkov2d091d62021-06-14 02:36:21 +01008517 if (!iov.iov_base && *io_get_tag_slot(data, i)) {
Colin Ian Kingcf3770e2021-04-29 11:46:02 +01008518 ret = -EINVAL;
8519 break;
8520 }
Jens Axboeedafcce2019-01-09 09:16:05 -07008521
Pavel Begunkov41edf1a2021-04-25 14:32:23 +01008522 ret = io_sqe_buffer_register(ctx, &iov, &ctx->user_bufs[i],
8523 &last_hpage);
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08008524 if (ret)
8525 break;
Jens Axboeedafcce2019-01-09 09:16:05 -07008526 }
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08008527
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01008528 WARN_ON_ONCE(ctx->buf_data);
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -08008529
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01008530 ctx->buf_data = data;
8531 if (ret)
8532 __io_sqe_buffers_unregister(ctx);
8533 else
8534 io_rsrc_node_switch(ctx, NULL);
Jens Axboeedafcce2019-01-09 09:16:05 -07008535 return ret;
8536}
8537
Pavel Begunkov634d00d2021-04-25 14:32:26 +01008538static int __io_sqe_buffers_update(struct io_ring_ctx *ctx,
8539 struct io_uring_rsrc_update2 *up,
8540 unsigned int nr_args)
8541{
8542 u64 __user *tags = u64_to_user_ptr(up->tags);
8543 struct iovec iov, __user *iovs = u64_to_user_ptr(up->data);
Pavel Begunkov634d00d2021-04-25 14:32:26 +01008544 struct page *last_hpage = NULL;
8545 bool needs_switch = false;
8546 __u32 done;
8547 int i, err;
8548
8549 if (!ctx->buf_data)
8550 return -ENXIO;
8551 if (up->offset + nr_args > ctx->nr_user_bufs)
8552 return -EINVAL;
8553
8554 for (done = 0; done < nr_args; done++) {
Pavel Begunkov0b8c0e72021-04-26 15:17:38 +01008555 struct io_mapped_ubuf *imu;
8556 int offset = up->offset + done;
Pavel Begunkov634d00d2021-04-25 14:32:26 +01008557 u64 tag = 0;
8558
8559 err = io_copy_iov(ctx, &iov, iovs, done);
8560 if (err)
8561 break;
8562 if (tags && copy_from_user(&tag, &tags[done], sizeof(tag))) {
8563 err = -EFAULT;
8564 break;
8565 }
Pavel Begunkov0b8c0e72021-04-26 15:17:38 +01008566 err = io_buffer_validate(&iov);
8567 if (err)
8568 break;
Colin Ian Kingcf3770e2021-04-29 11:46:02 +01008569 if (!iov.iov_base && tag) {
8570 err = -EINVAL;
8571 break;
8572 }
Pavel Begunkov0b8c0e72021-04-26 15:17:38 +01008573 err = io_sqe_buffer_register(ctx, &iov, &imu, &last_hpage);
8574 if (err)
8575 break;
Pavel Begunkov634d00d2021-04-25 14:32:26 +01008576
Pavel Begunkov0b8c0e72021-04-26 15:17:38 +01008577 i = array_index_nospec(offset, ctx->nr_user_bufs);
Pavel Begunkov62248432021-04-28 13:11:29 +01008578 if (ctx->user_bufs[i] != ctx->dummy_ubuf) {
Pavel Begunkov0b8c0e72021-04-26 15:17:38 +01008579 err = io_queue_rsrc_removal(ctx->buf_data, offset,
8580 ctx->rsrc_node, ctx->user_bufs[i]);
8581 if (unlikely(err)) {
8582 io_buffer_unmap(ctx, &imu);
Pavel Begunkov634d00d2021-04-25 14:32:26 +01008583 break;
Pavel Begunkov0b8c0e72021-04-26 15:17:38 +01008584 }
Pavel Begunkov634d00d2021-04-25 14:32:26 +01008585 ctx->user_bufs[i] = NULL;
8586 needs_switch = true;
8587 }
8588
Pavel Begunkov0b8c0e72021-04-26 15:17:38 +01008589 ctx->user_bufs[i] = imu;
Pavel Begunkov2d091d62021-06-14 02:36:21 +01008590 *io_get_tag_slot(ctx->buf_data, offset) = tag;
Pavel Begunkov634d00d2021-04-25 14:32:26 +01008591 }
8592
8593 if (needs_switch)
8594 io_rsrc_node_switch(ctx, ctx->buf_data);
8595 return done ? done : err;
8596}
8597
Jens Axboe9b402842019-04-11 11:45:41 -06008598static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg)
8599{
8600 __s32 __user *fds = arg;
8601 int fd;
8602
8603 if (ctx->cq_ev_fd)
8604 return -EBUSY;
8605
8606 if (copy_from_user(&fd, fds, sizeof(*fds)))
8607 return -EFAULT;
8608
8609 ctx->cq_ev_fd = eventfd_ctx_fdget(fd);
8610 if (IS_ERR(ctx->cq_ev_fd)) {
8611 int ret = PTR_ERR(ctx->cq_ev_fd);
Pavel Begunkovfe7e3252021-06-24 15:09:57 +01008612
Jens Axboe9b402842019-04-11 11:45:41 -06008613 ctx->cq_ev_fd = NULL;
8614 return ret;
8615 }
8616
8617 return 0;
8618}
8619
8620static int io_eventfd_unregister(struct io_ring_ctx *ctx)
8621{
8622 if (ctx->cq_ev_fd) {
8623 eventfd_ctx_put(ctx->cq_ev_fd);
8624 ctx->cq_ev_fd = NULL;
8625 return 0;
8626 }
8627
8628 return -ENXIO;
8629}
8630
Jens Axboe5a2e7452020-02-23 16:23:11 -07008631static void io_destroy_buffers(struct io_ring_ctx *ctx)
8632{
Jens Axboe9e15c3a2021-03-13 12:29:43 -07008633 struct io_buffer *buf;
8634 unsigned long index;
8635
8636 xa_for_each(&ctx->io_buffers, index, buf)
8637 __io_remove_buffers(ctx, buf, index, -1U);
Jens Axboe5a2e7452020-02-23 16:23:11 -07008638}
8639
Pavel Begunkov72558342021-08-09 20:18:09 +01008640static void io_req_cache_free(struct list_head *list)
Jens Axboe1b4c3512021-02-10 00:03:19 +00008641{
Jens Axboe68e68ee2021-02-13 09:00:02 -07008642 struct io_kiocb *req, *nxt;
Jens Axboe1b4c3512021-02-10 00:03:19 +00008643
Pavel Begunkovbb943b82021-08-09 20:18:10 +01008644 list_for_each_entry_safe(req, nxt, list, inflight_entry) {
8645 list_del(&req->inflight_entry);
Jens Axboe1b4c3512021-02-10 00:03:19 +00008646 kmem_cache_free(req_cachep, req);
8647 }
8648}
8649
Jens Axboe4010fec2021-02-27 15:04:18 -07008650static void io_req_caches_free(struct io_ring_ctx *ctx)
Jens Axboe2b188cc2019-01-07 10:46:33 -07008651{
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01008652 struct io_submit_state *state = &ctx->submit_state;
Pavel Begunkovbf019da2021-02-10 00:03:17 +00008653
Jens Axboe9a4fdbd2021-02-13 09:09:44 -07008654 mutex_lock(&ctx->uring_lock);
8655
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01008656 if (state->free_reqs) {
8657 kmem_cache_free_bulk(req_cachep, state->free_reqs, state->reqs);
8658 state->free_reqs = 0;
Pavel Begunkov8e5c66c2021-02-22 11:45:55 +00008659 }
Jens Axboe9a4fdbd2021-02-13 09:09:44 -07008660
Pavel Begunkovcd0ca2e2021-08-09 20:18:11 +01008661 io_flush_cached_locked_reqs(ctx, state);
8662 io_req_cache_free(&state->free_list);
Jens Axboe9a4fdbd2021-02-13 09:09:44 -07008663 mutex_unlock(&ctx->uring_lock);
8664}
8665
Pavel Begunkov43597aa2021-08-10 02:44:23 +01008666static void io_wait_rsrc_data(struct io_rsrc_data *data)
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01008667{
Pavel Begunkov43597aa2021-08-10 02:44:23 +01008668 if (data && !atomic_dec_and_test(&data->refs))
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01008669 wait_for_completion(&data->done);
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01008670}
8671
Jens Axboe2b188cc2019-01-07 10:46:33 -07008672static void io_ring_ctx_free(struct io_ring_ctx *ctx)
8673{
Jens Axboe37d1e2e2021-02-17 21:03:43 -07008674 io_sq_thread_finish(ctx);
Jens Axboe2aede0e2020-09-14 10:45:53 -06008675
Jens Axboe37d1e2e2021-02-17 21:03:43 -07008676 if (ctx->mm_account) {
Jens Axboe2aede0e2020-09-14 10:45:53 -06008677 mmdrop(ctx->mm_account);
8678 ctx->mm_account = NULL;
Bijan Mottahedeh30975822020-06-16 16:36:09 -07008679 }
Jens Axboedef596e2019-01-09 08:59:42 -07008680
Pavel Begunkov43597aa2021-08-10 02:44:23 +01008681 /* __io_rsrc_put_work() may need uring_lock to progress, wait w/o it */
8682 io_wait_rsrc_data(ctx->buf_data);
8683 io_wait_rsrc_data(ctx->file_data);
8684
Hao Xu8bad28d2021-02-19 17:19:36 +08008685 mutex_lock(&ctx->uring_lock);
Pavel Begunkov43597aa2021-08-10 02:44:23 +01008686 if (ctx->buf_data)
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +01008687 __io_sqe_buffers_unregister(ctx);
Pavel Begunkov43597aa2021-08-10 02:44:23 +01008688 if (ctx->file_data)
Pavel Begunkov08480402021-04-13 02:58:38 +01008689 __io_sqe_files_unregister(ctx);
Pavel Begunkovc4ea0602021-04-01 15:43:58 +01008690 if (ctx->rings)
8691 __io_cqring_overflow_flush(ctx, true);
Hao Xu8bad28d2021-02-19 17:19:36 +08008692 mutex_unlock(&ctx->uring_lock);
Jens Axboe9b402842019-04-11 11:45:41 -06008693 io_eventfd_unregister(ctx);
Jens Axboe5a2e7452020-02-23 16:23:11 -07008694 io_destroy_buffers(ctx);
Pavel Begunkov07db2982021-04-20 12:03:32 +01008695 if (ctx->sq_creds)
8696 put_cred(ctx->sq_creds);
Jens Axboedef596e2019-01-09 08:59:42 -07008697
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01008698 /* there are no registered resources left, nobody uses it */
8699 if (ctx->rsrc_node)
8700 io_rsrc_node_destroy(ctx->rsrc_node);
Pavel Begunkov8dd03af2021-03-19 17:22:36 +00008701 if (ctx->rsrc_backup_node)
Pavel Begunkovb895c9a2021-04-01 15:43:40 +01008702 io_rsrc_node_destroy(ctx->rsrc_backup_node);
Pavel Begunkova7f0ed52021-04-01 15:43:46 +01008703 flush_delayed_work(&ctx->rsrc_put_work);
8704
8705 WARN_ON_ONCE(!list_empty(&ctx->rsrc_ref_list));
8706 WARN_ON_ONCE(!llist_empty(&ctx->rsrc_put_llist));
Jens Axboe2b188cc2019-01-07 10:46:33 -07008707
8708#if defined(CONFIG_UNIX)
Eric Biggers355e8d22019-06-12 14:58:43 -07008709 if (ctx->ring_sock) {
8710 ctx->ring_sock->file = NULL; /* so that iput() is called */
Jens Axboe2b188cc2019-01-07 10:46:33 -07008711 sock_release(ctx->ring_sock);
Eric Biggers355e8d22019-06-12 14:58:43 -07008712 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07008713#endif
8714
Hristo Venev75b28af2019-08-26 17:23:46 +00008715 io_mem_free(ctx->rings);
Jens Axboe2b188cc2019-01-07 10:46:33 -07008716 io_mem_free(ctx->sq_sqes);
Jens Axboe2b188cc2019-01-07 10:46:33 -07008717
8718 percpu_ref_exit(&ctx->refs);
Jens Axboe2b188cc2019-01-07 10:46:33 -07008719 free_uid(ctx->user);
Jens Axboe4010fec2021-02-27 15:04:18 -07008720 io_req_caches_free(ctx);
Jens Axboee9418942021-02-19 12:33:30 -07008721 if (ctx->hash_map)
8722 io_wq_put_hash(ctx->hash_map);
Jens Axboe78076bb2019-12-04 19:56:40 -07008723 kfree(ctx->cancel_hash);
Pavel Begunkov62248432021-04-28 13:11:29 +01008724 kfree(ctx->dummy_ubuf);
Jens Axboe2b188cc2019-01-07 10:46:33 -07008725 kfree(ctx);
8726}
8727
8728static __poll_t io_uring_poll(struct file *file, poll_table *wait)
8729{
8730 struct io_ring_ctx *ctx = file->private_data;
8731 __poll_t mask = 0;
8732
Pavel Begunkov311997b2021-06-14 23:37:28 +01008733 poll_wait(file, &ctx->poll_wait, wait);
Stefan Bühler4f7067c2019-04-24 23:54:17 +02008734 /*
8735 * synchronizes with barrier from wq_has_sleeper call in
8736 * io_commit_cqring
8737 */
Jens Axboe2b188cc2019-01-07 10:46:33 -07008738 smp_rmb();
Jens Axboe90554202020-09-03 12:12:41 -06008739 if (!io_sqring_full(ctx))
Jens Axboe2b188cc2019-01-07 10:46:33 -07008740 mask |= EPOLLOUT | EPOLLWRNORM;
Hao Xued670c32021-02-05 16:34:21 +08008741
8742 /*
8743 * Don't flush cqring overflow list here, just do a simple check.
8744 * Otherwise there could possible be ABBA deadlock:
8745 * CPU0 CPU1
8746 * ---- ----
8747 * lock(&ctx->uring_lock);
8748 * lock(&ep->mtx);
8749 * lock(&ctx->uring_lock);
8750 * lock(&ep->mtx);
8751 *
8752 * Users may get EPOLLIN meanwhile seeing nothing in cqring, this
8753 * pushs them to do the flush.
8754 */
Pavel Begunkov5ed7a372021-06-14 23:37:27 +01008755 if (io_cqring_events(ctx) || test_bit(0, &ctx->check_cq_overflow))
Jens Axboe2b188cc2019-01-07 10:46:33 -07008756 mask |= EPOLLIN | EPOLLRDNORM;
8757
8758 return mask;
8759}
8760
8761static int io_uring_fasync(int fd, struct file *file, int on)
8762{
8763 struct io_ring_ctx *ctx = file->private_data;
8764
8765 return fasync_helper(fd, file, on, &ctx->cq_fasync);
8766}
8767
Yejune Deng0bead8c2020-12-24 11:02:20 +08008768static int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id)
Jens Axboe071698e2020-01-28 10:04:42 -07008769{
Jens Axboe4379bf82021-02-15 13:40:22 -07008770 const struct cred *creds;
Jens Axboe071698e2020-01-28 10:04:42 -07008771
Matthew Wilcox (Oracle)61cf9372021-03-08 14:16:16 +00008772 creds = xa_erase(&ctx->personalities, id);
Jens Axboe4379bf82021-02-15 13:40:22 -07008773 if (creds) {
8774 put_cred(creds);
Yejune Deng0bead8c2020-12-24 11:02:20 +08008775 return 0;
Jens Axboe1e6fa522020-10-15 08:46:24 -06008776 }
Yejune Deng0bead8c2020-12-24 11:02:20 +08008777
8778 return -EINVAL;
8779}
8780
Pavel Begunkovd56d9382021-03-06 11:02:13 +00008781struct io_tctx_exit {
8782 struct callback_head task_work;
8783 struct completion completion;
Pavel Begunkovbaf186c2021-03-06 11:02:15 +00008784 struct io_ring_ctx *ctx;
Pavel Begunkovd56d9382021-03-06 11:02:13 +00008785};
8786
8787static void io_tctx_exit_cb(struct callback_head *cb)
8788{
8789 struct io_uring_task *tctx = current->io_uring;
8790 struct io_tctx_exit *work;
8791
8792 work = container_of(cb, struct io_tctx_exit, task_work);
8793 /*
8794 * When @in_idle, we're in cancellation and it's racy to remove the
8795 * node. It'll be removed by the end of cancellation, just ignore it.
8796 */
8797 if (!atomic_read(&tctx->in_idle))
Pavel Begunkoveef51da2021-06-14 02:36:15 +01008798 io_uring_del_tctx_node((unsigned long)work->ctx);
Pavel Begunkovd56d9382021-03-06 11:02:13 +00008799 complete(&work->completion);
8800}
8801
Pavel Begunkov28090c12021-04-25 23:34:45 +01008802static bool io_cancel_ctx_cb(struct io_wq_work *work, void *data)
8803{
8804 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
8805
8806 return req->ctx == data;
8807}
8808
Jens Axboe85faa7b2020-04-09 18:14:00 -06008809static void io_ring_exit_work(struct work_struct *work)
8810{
Pavel Begunkovd56d9382021-03-06 11:02:13 +00008811 struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx, exit_work);
Pavel Begunkovb5bb3a22021-03-06 11:02:16 +00008812 unsigned long timeout = jiffies + HZ * 60 * 5;
Pavel Begunkov58d3be22021-08-09 13:04:17 +01008813 unsigned long interval = HZ / 20;
Pavel Begunkovd56d9382021-03-06 11:02:13 +00008814 struct io_tctx_exit exit;
8815 struct io_tctx_node *node;
8816 int ret;
Jens Axboe85faa7b2020-04-09 18:14:00 -06008817
Jens Axboe56952e92020-06-17 15:00:04 -06008818 /*
8819 * If we're doing polled IO and end up having requests being
8820 * submitted async (out-of-line), then completions can come in while
8821 * we're waiting for refs to drop. We need to reap these manually,
8822 * as nobody else will be looking for them.
8823 */
Pavel Begunkovb2edc0a2020-07-07 16:36:22 +03008824 do {
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01008825 io_uring_try_cancel_requests(ctx, NULL, true);
Pavel Begunkov28090c12021-04-25 23:34:45 +01008826 if (ctx->sq_data) {
8827 struct io_sq_data *sqd = ctx->sq_data;
8828 struct task_struct *tsk;
8829
8830 io_sq_thread_park(sqd);
8831 tsk = sqd->thread;
8832 if (tsk && tsk->io_uring && tsk->io_uring->io_wq)
8833 io_wq_cancel_cb(tsk->io_uring->io_wq,
8834 io_cancel_ctx_cb, ctx, true);
8835 io_sq_thread_unpark(sqd);
8836 }
Pavel Begunkovb5bb3a22021-03-06 11:02:16 +00008837
Pavel Begunkov58d3be22021-08-09 13:04:17 +01008838 if (WARN_ON_ONCE(time_after(jiffies, timeout))) {
8839 /* there is little hope left, don't run it too often */
8840 interval = HZ * 60;
8841 }
8842 } while (!wait_for_completion_timeout(&ctx->ref_comp, interval));
Pavel Begunkovd56d9382021-03-06 11:02:13 +00008843
Pavel Begunkov7f006512021-04-14 13:38:34 +01008844 init_completion(&exit.completion);
8845 init_task_work(&exit.task_work, io_tctx_exit_cb);
8846 exit.ctx = ctx;
Pavel Begunkov89b50662021-04-01 15:43:50 +01008847 /*
8848 * Some may use context even when all refs and requests have been put,
8849 * and they are free to do so while still holding uring_lock or
Pavel Begunkov5b0a6ac2021-06-30 21:54:04 +01008850 * completion_lock, see io_req_task_submit(). Apart from other work,
Pavel Begunkov89b50662021-04-01 15:43:50 +01008851 * this lock/unlock section also waits them to finish.
8852 */
Pavel Begunkovd56d9382021-03-06 11:02:13 +00008853 mutex_lock(&ctx->uring_lock);
8854 while (!list_empty(&ctx->tctx_list)) {
Pavel Begunkovb5bb3a22021-03-06 11:02:16 +00008855 WARN_ON_ONCE(time_after(jiffies, timeout));
8856
Pavel Begunkovd56d9382021-03-06 11:02:13 +00008857 node = list_first_entry(&ctx->tctx_list, struct io_tctx_node,
8858 ctx_node);
Pavel Begunkov7f006512021-04-14 13:38:34 +01008859 /* don't spin on a single task if cancellation failed */
8860 list_rotate_left(&ctx->tctx_list);
Pavel Begunkovd56d9382021-03-06 11:02:13 +00008861 ret = task_work_add(node->task, &exit.task_work, TWA_SIGNAL);
8862 if (WARN_ON_ONCE(ret))
8863 continue;
8864 wake_up_process(node->task);
8865
8866 mutex_unlock(&ctx->uring_lock);
8867 wait_for_completion(&exit.completion);
Pavel Begunkovd56d9382021-03-06 11:02:13 +00008868 mutex_lock(&ctx->uring_lock);
8869 }
8870 mutex_unlock(&ctx->uring_lock);
Jens Axboe79ebeae2021-08-10 15:18:27 -06008871 spin_lock(&ctx->completion_lock);
8872 spin_unlock(&ctx->completion_lock);
Pavel Begunkovd56d9382021-03-06 11:02:13 +00008873
Jens Axboe85faa7b2020-04-09 18:14:00 -06008874 io_ring_ctx_free(ctx);
8875}
8876
Pavel Begunkov80c4cbd2021-03-25 18:32:43 +00008877/* Returns true if we found and killed one or more timeouts */
8878static bool io_kill_timeouts(struct io_ring_ctx *ctx, struct task_struct *tsk,
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01008879 bool cancel_all)
Pavel Begunkov80c4cbd2021-03-25 18:32:43 +00008880{
8881 struct io_kiocb *req, *tmp;
8882 int canceled = 0;
8883
Jens Axboe79ebeae2021-08-10 15:18:27 -06008884 spin_lock(&ctx->completion_lock);
8885 spin_lock_irq(&ctx->timeout_lock);
Pavel Begunkov80c4cbd2021-03-25 18:32:43 +00008886 list_for_each_entry_safe(req, tmp, &ctx->timeout_list, timeout.list) {
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01008887 if (io_match_task(req, tsk, cancel_all)) {
Pavel Begunkov80c4cbd2021-03-25 18:32:43 +00008888 io_kill_timeout(req, -ECANCELED);
8889 canceled++;
8890 }
8891 }
Jens Axboe79ebeae2021-08-10 15:18:27 -06008892 spin_unlock_irq(&ctx->timeout_lock);
Pavel Begunkov51520422021-03-29 11:39:29 +01008893 if (canceled != 0)
8894 io_commit_cqring(ctx);
Jens Axboe79ebeae2021-08-10 15:18:27 -06008895 spin_unlock(&ctx->completion_lock);
Pavel Begunkov80c4cbd2021-03-25 18:32:43 +00008896 if (canceled != 0)
8897 io_cqring_ev_posted(ctx);
8898 return canceled != 0;
8899}
8900
Jens Axboe2b188cc2019-01-07 10:46:33 -07008901static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx)
8902{
Matthew Wilcox (Oracle)61cf9372021-03-08 14:16:16 +00008903 unsigned long index;
8904 struct creds *creds;
8905
Jens Axboe2b188cc2019-01-07 10:46:33 -07008906 mutex_lock(&ctx->uring_lock);
8907 percpu_ref_kill(&ctx->refs);
Pavel Begunkov634578f2020-12-06 22:22:44 +00008908 if (ctx->rings)
Pavel Begunkov6c2450a2021-02-23 12:40:22 +00008909 __io_cqring_overflow_flush(ctx, true);
Matthew Wilcox (Oracle)61cf9372021-03-08 14:16:16 +00008910 xa_for_each(&ctx->personalities, index, creds)
8911 io_unregister_personality(ctx, index);
Jens Axboe2b188cc2019-01-07 10:46:33 -07008912 mutex_unlock(&ctx->uring_lock);
8913
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01008914 io_kill_timeouts(ctx, NULL, true);
8915 io_poll_remove_all(ctx, NULL, true);
Jens Axboe561fb042019-10-24 07:25:42 -06008916
Jens Axboe15dff282019-11-13 09:09:23 -07008917 /* if we failed setting up the ctx, we might not have any rings */
Pavel Begunkovb2edc0a2020-07-07 16:36:22 +03008918 io_iopoll_try_reap_events(ctx);
Jens Axboe309fc032020-07-10 09:13:34 -06008919
Jens Axboe85faa7b2020-04-09 18:14:00 -06008920 INIT_WORK(&ctx->exit_work, io_ring_exit_work);
Jens Axboefc666772020-08-19 11:10:51 -06008921 /*
8922 * Use system_unbound_wq to avoid spawning tons of event kworkers
8923 * if we're exiting a ton of rings at the same time. It just adds
8924 * noise and overhead, there's no discernable change in runtime
8925 * over using system_wq.
8926 */
8927 queue_work(system_unbound_wq, &ctx->exit_work);
Jens Axboe2b188cc2019-01-07 10:46:33 -07008928}
8929
8930static int io_uring_release(struct inode *inode, struct file *file)
8931{
8932 struct io_ring_ctx *ctx = file->private_data;
8933
8934 file->private_data = NULL;
8935 io_ring_ctx_wait_and_kill(ctx);
8936 return 0;
8937}
8938
Pavel Begunkovf6edbab2020-11-06 13:00:26 +00008939struct io_task_cancel {
8940 struct task_struct *task;
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01008941 bool all;
Pavel Begunkovf6edbab2020-11-06 13:00:26 +00008942};
Pavel Begunkov67c4d9e2020-06-15 10:24:05 +03008943
Pavel Begunkovf6edbab2020-11-06 13:00:26 +00008944static bool io_cancel_task_cb(struct io_wq_work *work, void *data)
Jens Axboeb711d4e2020-08-16 08:23:05 -07008945{
Pavel Begunkov9a472ef2020-11-05 22:31:37 +00008946 struct io_kiocb *req = container_of(work, struct io_kiocb, work);
Pavel Begunkovf6edbab2020-11-06 13:00:26 +00008947 struct io_task_cancel *cancel = data;
Pavel Begunkov9a472ef2020-11-05 22:31:37 +00008948 bool ret;
8949
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01008950 if (!cancel->all && (req->flags & REQ_F_LINK_TIMEOUT)) {
Pavel Begunkov9a472ef2020-11-05 22:31:37 +00008951 struct io_ring_ctx *ctx = req->ctx;
8952
8953 /* protect against races with linked timeouts */
Jens Axboe79ebeae2021-08-10 15:18:27 -06008954 spin_lock(&ctx->completion_lock);
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01008955 ret = io_match_task(req, cancel->task, cancel->all);
Jens Axboe79ebeae2021-08-10 15:18:27 -06008956 spin_unlock(&ctx->completion_lock);
Pavel Begunkov9a472ef2020-11-05 22:31:37 +00008957 } else {
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01008958 ret = io_match_task(req, cancel->task, cancel->all);
Pavel Begunkov9a472ef2020-11-05 22:31:37 +00008959 }
8960 return ret;
Jens Axboeb711d4e2020-08-16 08:23:05 -07008961}
8962
Pavel Begunkove1915f72021-03-11 23:29:35 +00008963static bool io_cancel_defer_files(struct io_ring_ctx *ctx,
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01008964 struct task_struct *task, bool cancel_all)
Pavel Begunkovb7ddce32020-09-06 00:45:14 +03008965{
Pavel Begunkove1915f72021-03-11 23:29:35 +00008966 struct io_defer_entry *de;
Pavel Begunkovb7ddce32020-09-06 00:45:14 +03008967 LIST_HEAD(list);
8968
Jens Axboe79ebeae2021-08-10 15:18:27 -06008969 spin_lock(&ctx->completion_lock);
Pavel Begunkovb7ddce32020-09-06 00:45:14 +03008970 list_for_each_entry_reverse(de, &ctx->defer_list, list) {
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01008971 if (io_match_task(de->req, task, cancel_all)) {
Pavel Begunkovb7ddce32020-09-06 00:45:14 +03008972 list_cut_position(&list, &ctx->defer_list, &de->list);
8973 break;
8974 }
8975 }
Jens Axboe79ebeae2021-08-10 15:18:27 -06008976 spin_unlock(&ctx->completion_lock);
Pavel Begunkove1915f72021-03-11 23:29:35 +00008977 if (list_empty(&list))
8978 return false;
Pavel Begunkovb7ddce32020-09-06 00:45:14 +03008979
8980 while (!list_empty(&list)) {
8981 de = list_first_entry(&list, struct io_defer_entry, list);
8982 list_del_init(&de->list);
Pavel Begunkovf41db2732021-02-28 22:35:12 +00008983 io_req_complete_failed(de->req, -ECANCELED);
Pavel Begunkovb7ddce32020-09-06 00:45:14 +03008984 kfree(de);
8985 }
Pavel Begunkove1915f72021-03-11 23:29:35 +00008986 return true;
Pavel Begunkovb7ddce32020-09-06 00:45:14 +03008987}
8988
Pavel Begunkov1b007642021-03-06 11:02:17 +00008989static bool io_uring_try_cancel_iowq(struct io_ring_ctx *ctx)
8990{
8991 struct io_tctx_node *node;
8992 enum io_wq_cancel cret;
8993 bool ret = false;
8994
8995 mutex_lock(&ctx->uring_lock);
8996 list_for_each_entry(node, &ctx->tctx_list, ctx_node) {
8997 struct io_uring_task *tctx = node->task->io_uring;
8998
8999 /*
9000 * io_wq will stay alive while we hold uring_lock, because it's
9001 * killed after ctx nodes, which requires to take the lock.
9002 */
9003 if (!tctx || !tctx->io_wq)
9004 continue;
9005 cret = io_wq_cancel_cb(tctx->io_wq, io_cancel_ctx_cb, ctx, true);
9006 ret |= (cret != IO_WQ_CANCEL_NOTFOUND);
9007 }
9008 mutex_unlock(&ctx->uring_lock);
9009
9010 return ret;
9011}
9012
Pavel Begunkov9936c7c2021-02-04 13:51:56 +00009013static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
9014 struct task_struct *task,
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01009015 bool cancel_all)
Pavel Begunkov9936c7c2021-02-04 13:51:56 +00009016{
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01009017 struct io_task_cancel cancel = { .task = task, .all = cancel_all, };
Pavel Begunkov1b007642021-03-06 11:02:17 +00009018 struct io_uring_task *tctx = task ? task->io_uring : NULL;
Pavel Begunkov9936c7c2021-02-04 13:51:56 +00009019
9020 while (1) {
9021 enum io_wq_cancel cret;
9022 bool ret = false;
9023
Pavel Begunkov1b007642021-03-06 11:02:17 +00009024 if (!task) {
9025 ret |= io_uring_try_cancel_iowq(ctx);
9026 } else if (tctx && tctx->io_wq) {
9027 /*
9028 * Cancels requests of all rings, not only @ctx, but
9029 * it's fine as the task is in exit/exec.
9030 */
Jens Axboe5aa75ed2021-02-16 12:56:50 -07009031 cret = io_wq_cancel_cb(tctx->io_wq, io_cancel_task_cb,
Pavel Begunkov9936c7c2021-02-04 13:51:56 +00009032 &cancel, true);
9033 ret |= (cret != IO_WQ_CANCEL_NOTFOUND);
9034 }
9035
9036 /* SQPOLL thread does its own polling */
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01009037 if ((!(ctx->flags & IORING_SETUP_SQPOLL) && cancel_all) ||
Jens Axboed052d1d2021-03-11 10:49:20 -07009038 (ctx->sq_data && ctx->sq_data->thread == current)) {
Pavel Begunkov9936c7c2021-02-04 13:51:56 +00009039 while (!list_empty_careful(&ctx->iopoll_list)) {
9040 io_iopoll_try_reap_events(ctx);
9041 ret = true;
9042 }
9043 }
9044
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01009045 ret |= io_cancel_defer_files(ctx, task, cancel_all);
9046 ret |= io_poll_remove_all(ctx, task, cancel_all);
9047 ret |= io_kill_timeouts(ctx, task, cancel_all);
Pavel Begunkove5dc4802021-06-26 21:40:46 +01009048 if (task)
9049 ret |= io_run_task_work();
Pavel Begunkov9936c7c2021-02-04 13:51:56 +00009050 if (!ret)
9051 break;
9052 cond_resched();
9053 }
9054}
9055
Pavel Begunkoveef51da2021-06-14 02:36:15 +01009056static int __io_uring_add_tctx_node(struct io_ring_ctx *ctx)
Jens Axboe0f212202020-09-13 13:09:39 -06009057{
Matthew Wilcox (Oracle)236434c2020-10-09 13:49:52 +01009058 struct io_uring_task *tctx = current->io_uring;
Pavel Begunkov13bf43f2021-03-06 11:02:12 +00009059 struct io_tctx_node *node;
Pavel Begunkova528b042020-12-21 18:34:04 +00009060 int ret;
Matthew Wilcox (Oracle)236434c2020-10-09 13:49:52 +01009061
9062 if (unlikely(!tctx)) {
Jens Axboe5aa75ed2021-02-16 12:56:50 -07009063 ret = io_uring_alloc_task_context(current, ctx);
Jens Axboe0f212202020-09-13 13:09:39 -06009064 if (unlikely(ret))
9065 return ret;
Matthew Wilcox (Oracle)236434c2020-10-09 13:49:52 +01009066 tctx = current->io_uring;
Jens Axboe0f212202020-09-13 13:09:39 -06009067 }
Pavel Begunkovcf27f3b2021-03-19 17:22:31 +00009068 if (!xa_load(&tctx->xa, (unsigned long)ctx)) {
9069 node = kmalloc(sizeof(*node), GFP_KERNEL);
9070 if (!node)
9071 return -ENOMEM;
9072 node->ctx = ctx;
9073 node->task = current;
Jens Axboe0f212202020-09-13 13:09:39 -06009074
Pavel Begunkovcf27f3b2021-03-19 17:22:31 +00009075 ret = xa_err(xa_store(&tctx->xa, (unsigned long)ctx,
9076 node, GFP_KERNEL));
9077 if (ret) {
9078 kfree(node);
9079 return ret;
Jens Axboe0f212202020-09-13 13:09:39 -06009080 }
Pavel Begunkovcf27f3b2021-03-19 17:22:31 +00009081
9082 mutex_lock(&ctx->uring_lock);
9083 list_add(&node->ctx_node, &ctx->tctx_list);
9084 mutex_unlock(&ctx->uring_lock);
Jens Axboe0f212202020-09-13 13:09:39 -06009085 }
Pavel Begunkovcf27f3b2021-03-19 17:22:31 +00009086 tctx->last = ctx;
Jens Axboe0f212202020-09-13 13:09:39 -06009087 return 0;
9088}
9089
9090/*
Pavel Begunkovcf27f3b2021-03-19 17:22:31 +00009091 * Note that this task has used io_uring. We use it for cancelation purposes.
9092 */
Pavel Begunkoveef51da2021-06-14 02:36:15 +01009093static inline int io_uring_add_tctx_node(struct io_ring_ctx *ctx)
Pavel Begunkovcf27f3b2021-03-19 17:22:31 +00009094{
9095 struct io_uring_task *tctx = current->io_uring;
9096
9097 if (likely(tctx && tctx->last == ctx))
9098 return 0;
Pavel Begunkoveef51da2021-06-14 02:36:15 +01009099 return __io_uring_add_tctx_node(ctx);
Pavel Begunkovcf27f3b2021-03-19 17:22:31 +00009100}
9101
9102/*
Jens Axboe0f212202020-09-13 13:09:39 -06009103 * Remove this io_uring_file -> task mapping.
9104 */
Pavel Begunkoveef51da2021-06-14 02:36:15 +01009105static void io_uring_del_tctx_node(unsigned long index)
Jens Axboe0f212202020-09-13 13:09:39 -06009106{
9107 struct io_uring_task *tctx = current->io_uring;
Pavel Begunkov13bf43f2021-03-06 11:02:12 +00009108 struct io_tctx_node *node;
Pavel Begunkov29412672021-03-06 11:02:11 +00009109
Pavel Begunkoveebd2e32021-03-06 11:02:14 +00009110 if (!tctx)
9111 return;
Pavel Begunkov13bf43f2021-03-06 11:02:12 +00009112 node = xa_erase(&tctx->xa, index);
9113 if (!node)
Pavel Begunkov29412672021-03-06 11:02:11 +00009114 return;
Jens Axboe0f212202020-09-13 13:09:39 -06009115
Pavel Begunkov13bf43f2021-03-06 11:02:12 +00009116 WARN_ON_ONCE(current != node->task);
9117 WARN_ON_ONCE(list_empty(&node->ctx_node));
9118
9119 mutex_lock(&node->ctx->uring_lock);
9120 list_del(&node->ctx_node);
9121 mutex_unlock(&node->ctx->uring_lock);
9122
Pavel Begunkovbaf186c2021-03-06 11:02:15 +00009123 if (tctx->last == node->ctx)
Jens Axboe0f212202020-09-13 13:09:39 -06009124 tctx->last = NULL;
Pavel Begunkov13bf43f2021-03-06 11:02:12 +00009125 kfree(node);
Jens Axboe0f212202020-09-13 13:09:39 -06009126}
9127
Pavel Begunkov8452d4a2021-02-27 11:16:46 +00009128static void io_uring_clean_tctx(struct io_uring_task *tctx)
Pavel Begunkovde7f1d92021-01-04 20:43:29 +00009129{
Pavel Begunkovba5ef6d2021-05-20 13:21:20 +01009130 struct io_wq *wq = tctx->io_wq;
Pavel Begunkov13bf43f2021-03-06 11:02:12 +00009131 struct io_tctx_node *node;
Pavel Begunkovde7f1d92021-01-04 20:43:29 +00009132 unsigned long index;
9133
Pavel Begunkov13bf43f2021-03-06 11:02:12 +00009134 xa_for_each(&tctx->xa, index, node)
Pavel Begunkoveef51da2021-06-14 02:36:15 +01009135 io_uring_del_tctx_node(index);
Marco Elverb16ef422021-05-27 11:25:48 +02009136 if (wq) {
9137 /*
9138 * Must be after io_uring_del_task_file() (removes nodes under
9139 * uring_lock) to avoid race with io_uring_try_cancel_iowq().
9140 */
9141 tctx->io_wq = NULL;
Pavel Begunkovba5ef6d2021-05-20 13:21:20 +01009142 io_wq_put_and_exit(wq);
Marco Elverb16ef422021-05-27 11:25:48 +02009143 }
Pavel Begunkovde7f1d92021-01-04 20:43:29 +00009144}
9145
Pavel Begunkov3f48cf12021-04-11 01:46:27 +01009146static s64 tctx_inflight(struct io_uring_task *tctx, bool tracked)
Pavel Begunkov521d6a72021-03-11 23:29:38 +00009147{
Pavel Begunkov3f48cf12021-04-11 01:46:27 +01009148 if (tracked)
9149 return atomic_read(&tctx->inflight_tracked);
Pavel Begunkov521d6a72021-03-11 23:29:38 +00009150 return percpu_counter_sum(&tctx->inflight);
9151}
9152
Pavel Begunkov09899b12021-06-14 02:36:22 +01009153static void io_uring_drop_tctx_refs(struct task_struct *task)
9154{
9155 struct io_uring_task *tctx = task->io_uring;
9156 unsigned int refs = tctx->cached_refs;
9157
Pavel Begunkove9dbe222021-08-09 13:04:20 +01009158 if (refs) {
9159 tctx->cached_refs = 0;
9160 percpu_counter_sub(&tctx->inflight, refs);
9161 put_task_struct_many(task, refs);
9162 }
Pavel Begunkov09899b12021-06-14 02:36:22 +01009163}
9164
Pavel Begunkov78cc6872021-06-14 02:36:23 +01009165/*
9166 * Find any io_uring ctx that this task has registered or done IO on, and cancel
9167 * requests. @sqd should be not-null IIF it's an SQPOLL thread cancellation.
9168 */
9169static void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd)
Pavel Begunkov0e9ddb32021-02-07 22:34:26 +00009170{
Pavel Begunkov521d6a72021-03-11 23:29:38 +00009171 struct io_uring_task *tctx = current->io_uring;
Pavel Begunkov734551d2021-04-18 14:52:09 +01009172 struct io_ring_ctx *ctx;
Jens Axboefdaf0832020-10-30 09:37:30 -06009173 s64 inflight;
Pavel Begunkov0e9ddb32021-02-07 22:34:26 +00009174 DEFINE_WAIT(wait);
Jens Axboefdaf0832020-10-30 09:37:30 -06009175
Pavel Begunkov78cc6872021-06-14 02:36:23 +01009176 WARN_ON_ONCE(sqd && sqd->thread != current);
9177
Palash Oswal6d042ff2021-04-27 18:21:49 +05309178 if (!current->io_uring)
9179 return;
Pavel Begunkov17a91052021-05-23 15:48:39 +01009180 if (tctx->io_wq)
9181 io_wq_exit_start(tctx->io_wq);
9182
Jens Axboefdaf0832020-10-30 09:37:30 -06009183 atomic_inc(&tctx->in_idle);
Jens Axboed8a6df12020-10-15 16:24:45 -06009184 do {
Pavel Begunkove9dbe222021-08-09 13:04:20 +01009185 io_uring_drop_tctx_refs(current);
Jens Axboe0f212202020-09-13 13:09:39 -06009186 /* read completions before cancelations */
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01009187 inflight = tctx_inflight(tctx, !cancel_all);
Jens Axboed8a6df12020-10-15 16:24:45 -06009188 if (!inflight)
9189 break;
Jens Axboe0f212202020-09-13 13:09:39 -06009190
Pavel Begunkov78cc6872021-06-14 02:36:23 +01009191 if (!sqd) {
9192 struct io_tctx_node *node;
9193 unsigned long index;
9194
9195 xa_for_each(&tctx->xa, index, node) {
9196 /* sqpoll task will cancel all its requests */
9197 if (node->ctx->sq_data)
9198 continue;
9199 io_uring_try_cancel_requests(node->ctx, current,
9200 cancel_all);
9201 }
9202 } else {
9203 list_for_each_entry(ctx, &sqd->ctx_list, sqd_list)
9204 io_uring_try_cancel_requests(ctx, current,
9205 cancel_all);
9206 }
9207
9208 prepare_to_wait(&tctx->wait, &wait, TASK_UNINTERRUPTIBLE);
Pavel Begunkove9dbe222021-08-09 13:04:20 +01009209 io_uring_drop_tctx_refs(current);
Jens Axboe0f212202020-09-13 13:09:39 -06009210 /*
Pavel Begunkova1bb3cd2021-01-26 15:28:26 +00009211 * If we've seen completions, retry without waiting. This
9212 * avoids a race where a completion comes in before we did
9213 * prepare_to_wait().
Jens Axboe0f212202020-09-13 13:09:39 -06009214 */
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01009215 if (inflight == tctx_inflight(tctx, !cancel_all))
Pavel Begunkova1bb3cd2021-01-26 15:28:26 +00009216 schedule();
Pavel Begunkovf57555e2020-12-20 13:21:44 +00009217 finish_wait(&tctx->wait, &wait);
Jens Axboed8a6df12020-10-15 16:24:45 -06009218 } while (1);
Jens Axboefdaf0832020-10-30 09:37:30 -06009219 atomic_dec(&tctx->in_idle);
Pavel Begunkovde7f1d92021-01-04 20:43:29 +00009220
Pavel Begunkov8452d4a2021-02-27 11:16:46 +00009221 io_uring_clean_tctx(tctx);
Pavel Begunkov3dd0c972021-05-16 22:58:04 +01009222 if (cancel_all) {
Pavel Begunkov3f48cf12021-04-11 01:46:27 +01009223 /* for exec all current's requests should be gone, kill tctx */
9224 __io_uring_free(current);
9225 }
Pavel Begunkov44e728b2020-06-15 10:24:04 +03009226}
9227
Hao Xuf552a272021-08-12 12:14:35 +08009228void __io_uring_cancel(bool cancel_all)
Pavel Begunkov78cc6872021-06-14 02:36:23 +01009229{
Hao Xuf552a272021-08-12 12:14:35 +08009230 io_uring_cancel_generic(cancel_all, NULL);
Pavel Begunkov78cc6872021-06-14 02:36:23 +01009231}
9232
Roman Penyaev6c5c2402019-11-28 12:53:22 +01009233static void *io_uring_validate_mmap_request(struct file *file,
9234 loff_t pgoff, size_t sz)
Jens Axboe2b188cc2019-01-07 10:46:33 -07009235{
Jens Axboe2b188cc2019-01-07 10:46:33 -07009236 struct io_ring_ctx *ctx = file->private_data;
Roman Penyaev6c5c2402019-11-28 12:53:22 +01009237 loff_t offset = pgoff << PAGE_SHIFT;
Jens Axboe2b188cc2019-01-07 10:46:33 -07009238 struct page *page;
9239 void *ptr;
9240
9241 switch (offset) {
9242 case IORING_OFF_SQ_RING:
Hristo Venev75b28af2019-08-26 17:23:46 +00009243 case IORING_OFF_CQ_RING:
9244 ptr = ctx->rings;
Jens Axboe2b188cc2019-01-07 10:46:33 -07009245 break;
9246 case IORING_OFF_SQES:
9247 ptr = ctx->sq_sqes;
9248 break;
Jens Axboe2b188cc2019-01-07 10:46:33 -07009249 default:
Roman Penyaev6c5c2402019-11-28 12:53:22 +01009250 return ERR_PTR(-EINVAL);
Jens Axboe2b188cc2019-01-07 10:46:33 -07009251 }
9252
9253 page = virt_to_head_page(ptr);
Matthew Wilcox (Oracle)a50b8542019-09-23 15:34:25 -07009254 if (sz > page_size(page))
Roman Penyaev6c5c2402019-11-28 12:53:22 +01009255 return ERR_PTR(-EINVAL);
9256
9257 return ptr;
9258}
9259
9260#ifdef CONFIG_MMU
9261
9262static int io_uring_mmap(struct file *file, struct vm_area_struct *vma)
9263{
9264 size_t sz = vma->vm_end - vma->vm_start;
9265 unsigned long pfn;
9266 void *ptr;
9267
9268 ptr = io_uring_validate_mmap_request(file, vma->vm_pgoff, sz);
9269 if (IS_ERR(ptr))
9270 return PTR_ERR(ptr);
Jens Axboe2b188cc2019-01-07 10:46:33 -07009271
9272 pfn = virt_to_phys(ptr) >> PAGE_SHIFT;
9273 return remap_pfn_range(vma, vma->vm_start, pfn, sz, vma->vm_page_prot);
9274}
9275
Roman Penyaev6c5c2402019-11-28 12:53:22 +01009276#else /* !CONFIG_MMU */
9277
9278static int io_uring_mmap(struct file *file, struct vm_area_struct *vma)
9279{
9280 return vma->vm_flags & (VM_SHARED | VM_MAYSHARE) ? 0 : -EINVAL;
9281}
9282
9283static unsigned int io_uring_nommu_mmap_capabilities(struct file *file)
9284{
9285 return NOMMU_MAP_DIRECT | NOMMU_MAP_READ | NOMMU_MAP_WRITE;
9286}
9287
9288static unsigned long io_uring_nommu_get_unmapped_area(struct file *file,
9289 unsigned long addr, unsigned long len,
9290 unsigned long pgoff, unsigned long flags)
9291{
9292 void *ptr;
9293
9294 ptr = io_uring_validate_mmap_request(file, pgoff, len);
9295 if (IS_ERR(ptr))
9296 return PTR_ERR(ptr);
9297
9298 return (unsigned long) ptr;
9299}
9300
9301#endif /* !CONFIG_MMU */
9302
Pavel Begunkovd9d05212021-01-08 20:57:25 +00009303static int io_sqpoll_wait_sq(struct io_ring_ctx *ctx)
Jens Axboe90554202020-09-03 12:12:41 -06009304{
9305 DEFINE_WAIT(wait);
9306
9307 do {
9308 if (!io_sqring_full(ctx))
9309 break;
Jens Axboe90554202020-09-03 12:12:41 -06009310 prepare_to_wait(&ctx->sqo_sq_wait, &wait, TASK_INTERRUPTIBLE);
9311
9312 if (!io_sqring_full(ctx))
9313 break;
Jens Axboe90554202020-09-03 12:12:41 -06009314 schedule();
9315 } while (!signal_pending(current));
9316
9317 finish_wait(&ctx->sqo_sq_wait, &wait);
Yang Li51993282021-03-09 14:30:41 +08009318 return 0;
Jens Axboe90554202020-09-03 12:12:41 -06009319}
9320
Hao Xuc73ebb62020-11-03 10:54:37 +08009321static int io_get_ext_arg(unsigned flags, const void __user *argp, size_t *argsz,
9322 struct __kernel_timespec __user **ts,
9323 const sigset_t __user **sig)
9324{
9325 struct io_uring_getevents_arg arg;
9326
9327 /*
9328 * If EXT_ARG isn't set, then we have no timespec and the argp pointer
9329 * is just a pointer to the sigset_t.
9330 */
9331 if (!(flags & IORING_ENTER_EXT_ARG)) {
9332 *sig = (const sigset_t __user *) argp;
9333 *ts = NULL;
9334 return 0;
9335 }
9336
9337 /*
9338 * EXT_ARG is set - ensure we agree on the size of it and copy in our
9339 * timespec and sigset_t pointers if good.
9340 */
9341 if (*argsz != sizeof(arg))
9342 return -EINVAL;
9343 if (copy_from_user(&arg, argp, sizeof(arg)))
9344 return -EFAULT;
9345 *sig = u64_to_user_ptr(arg.sigmask);
9346 *argsz = arg.sigmask_sz;
9347 *ts = u64_to_user_ptr(arg.ts);
9348 return 0;
9349}
9350
Jens Axboe2b188cc2019-01-07 10:46:33 -07009351SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
Hao Xuc73ebb62020-11-03 10:54:37 +08009352 u32, min_complete, u32, flags, const void __user *, argp,
9353 size_t, argsz)
Jens Axboe2b188cc2019-01-07 10:46:33 -07009354{
9355 struct io_ring_ctx *ctx;
Jens Axboe2b188cc2019-01-07 10:46:33 -07009356 int submitted = 0;
9357 struct fd f;
Pavel Begunkov33f993d2021-03-19 17:22:30 +00009358 long ret;
Jens Axboe2b188cc2019-01-07 10:46:33 -07009359
Jens Axboe4c6e2772020-07-01 11:29:10 -06009360 io_run_task_work();
Jens Axboeb41e9852020-02-17 09:52:41 -07009361
Pavel Begunkov33f993d2021-03-19 17:22:30 +00009362 if (unlikely(flags & ~(IORING_ENTER_GETEVENTS | IORING_ENTER_SQ_WAKEUP |
9363 IORING_ENTER_SQ_WAIT | IORING_ENTER_EXT_ARG)))
Jens Axboe2b188cc2019-01-07 10:46:33 -07009364 return -EINVAL;
9365
9366 f = fdget(fd);
Pavel Begunkov33f993d2021-03-19 17:22:30 +00009367 if (unlikely(!f.file))
Jens Axboe2b188cc2019-01-07 10:46:33 -07009368 return -EBADF;
9369
9370 ret = -EOPNOTSUPP;
Pavel Begunkov33f993d2021-03-19 17:22:30 +00009371 if (unlikely(f.file->f_op != &io_uring_fops))
Jens Axboe2b188cc2019-01-07 10:46:33 -07009372 goto out_fput;
9373
9374 ret = -ENXIO;
9375 ctx = f.file->private_data;
Pavel Begunkov33f993d2021-03-19 17:22:30 +00009376 if (unlikely(!percpu_ref_tryget(&ctx->refs)))
Jens Axboe2b188cc2019-01-07 10:46:33 -07009377 goto out_fput;
9378
Stefano Garzarella7e84e1c2020-08-27 16:58:31 +02009379 ret = -EBADFD;
Pavel Begunkov33f993d2021-03-19 17:22:30 +00009380 if (unlikely(ctx->flags & IORING_SETUP_R_DISABLED))
Stefano Garzarella7e84e1c2020-08-27 16:58:31 +02009381 goto out;
9382
Jens Axboe6c271ce2019-01-10 11:22:30 -07009383 /*
9384 * For SQ polling, the thread will do all submissions and completions.
9385 * Just return the requested submit count, and wake the thread if
9386 * we were asked to.
9387 */
Jens Axboeb2a9ead2019-09-12 14:19:16 -06009388 ret = 0;
Jens Axboe6c271ce2019-01-10 11:22:30 -07009389 if (ctx->flags & IORING_SETUP_SQPOLL) {
Pavel Begunkov90f67362021-08-09 20:18:12 +01009390 io_cqring_overflow_flush(ctx);
Pavel Begunkov89448c42020-12-17 00:24:39 +00009391
Jens Axboe21f96522021-08-14 09:04:40 -06009392 if (unlikely(ctx->sq_data->thread == NULL)) {
9393 ret = -EOWNERDEAD;
Stefan Metzmacher04147482021-03-07 11:54:29 +01009394 goto out;
Jens Axboe21f96522021-08-14 09:04:40 -06009395 }
Jens Axboe6c271ce2019-01-10 11:22:30 -07009396 if (flags & IORING_ENTER_SQ_WAKEUP)
Jens Axboe534ca6d2020-09-02 13:52:19 -06009397 wake_up(&ctx->sq_data->wait);
Pavel Begunkovd9d05212021-01-08 20:57:25 +00009398 if (flags & IORING_ENTER_SQ_WAIT) {
9399 ret = io_sqpoll_wait_sq(ctx);
9400 if (ret)
9401 goto out;
9402 }
Jens Axboe6c271ce2019-01-10 11:22:30 -07009403 submitted = to_submit;
Jens Axboeb2a9ead2019-09-12 14:19:16 -06009404 } else if (to_submit) {
Pavel Begunkoveef51da2021-06-14 02:36:15 +01009405 ret = io_uring_add_tctx_node(ctx);
Jens Axboe0f212202020-09-13 13:09:39 -06009406 if (unlikely(ret))
9407 goto out;
Jens Axboe2b188cc2019-01-07 10:46:33 -07009408 mutex_lock(&ctx->uring_lock);
Jens Axboe0f212202020-09-13 13:09:39 -06009409 submitted = io_submit_sqes(ctx, to_submit);
Jens Axboe2b188cc2019-01-07 10:46:33 -07009410 mutex_unlock(&ctx->uring_lock);
Pavel Begunkov7c504e652019-12-18 19:53:45 +03009411
9412 if (submitted != to_submit)
9413 goto out;
Jens Axboe2b188cc2019-01-07 10:46:33 -07009414 }
9415 if (flags & IORING_ENTER_GETEVENTS) {
Hao Xuc73ebb62020-11-03 10:54:37 +08009416 const sigset_t __user *sig;
9417 struct __kernel_timespec __user *ts;
9418
9419 ret = io_get_ext_arg(flags, argp, &argsz, &ts, &sig);
9420 if (unlikely(ret))
9421 goto out;
9422
Jens Axboe2b188cc2019-01-07 10:46:33 -07009423 min_complete = min(min_complete, ctx->cq_entries);
9424
Xiaoguang Wang32b22442020-03-11 09:26:09 +08009425 /*
9426 * When SETUP_IOPOLL and SETUP_SQPOLL are both enabled, user
9427 * space applications don't need to do io completion events
9428 * polling again, they can rely on io_sq_thread to do polling
9429 * work, which can reduce cpu usage and uring_lock contention.
9430 */
9431 if (ctx->flags & IORING_SETUP_IOPOLL &&
9432 !(ctx->flags & IORING_SETUP_SQPOLL)) {
Pavel Begunkov7668b922020-07-07 16:36:21 +03009433 ret = io_iopoll_check(ctx, min_complete);
Jens Axboedef596e2019-01-09 08:59:42 -07009434 } else {
Hao Xuc73ebb62020-11-03 10:54:37 +08009435 ret = io_cqring_wait(ctx, min_complete, sig, argsz, ts);
Jens Axboedef596e2019-01-09 08:59:42 -07009436 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07009437 }
9438
Pavel Begunkov7c504e652019-12-18 19:53:45 +03009439out:
Pavel Begunkov6805b322019-10-08 02:18:42 +03009440 percpu_ref_put(&ctx->refs);
Jens Axboe2b188cc2019-01-07 10:46:33 -07009441out_fput:
9442 fdput(f);
9443 return submitted ? submitted : ret;
9444}
9445
Tobias Klauserbebdb652020-02-26 18:38:32 +01009446#ifdef CONFIG_PROC_FS
Matthew Wilcox (Oracle)61cf9372021-03-08 14:16:16 +00009447static int io_uring_show_cred(struct seq_file *m, unsigned int id,
9448 const struct cred *cred)
Jens Axboe87ce9552020-01-30 08:25:34 -07009449{
Jens Axboe87ce9552020-01-30 08:25:34 -07009450 struct user_namespace *uns = seq_user_ns(m);
9451 struct group_info *gi;
9452 kernel_cap_t cap;
9453 unsigned __capi;
9454 int g;
9455
9456 seq_printf(m, "%5d\n", id);
9457 seq_put_decimal_ull(m, "\tUid:\t", from_kuid_munged(uns, cred->uid));
9458 seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->euid));
9459 seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->suid));
9460 seq_put_decimal_ull(m, "\t\t", from_kuid_munged(uns, cred->fsuid));
9461 seq_put_decimal_ull(m, "\n\tGid:\t", from_kgid_munged(uns, cred->gid));
9462 seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->egid));
9463 seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->sgid));
9464 seq_put_decimal_ull(m, "\t\t", from_kgid_munged(uns, cred->fsgid));
9465 seq_puts(m, "\n\tGroups:\t");
9466 gi = cred->group_info;
9467 for (g = 0; g < gi->ngroups; g++) {
9468 seq_put_decimal_ull(m, g ? " " : "",
9469 from_kgid_munged(uns, gi->gid[g]));
9470 }
9471 seq_puts(m, "\n\tCapEff:\t");
9472 cap = cred->cap_effective;
9473 CAP_FOR_EACH_U32(__capi)
9474 seq_put_hex_ll(m, NULL, cap.cap[CAP_LAST_U32 - __capi], 8);
9475 seq_putc(m, '\n');
9476 return 0;
9477}
9478
9479static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m)
9480{
Joseph Qidbbe9c62020-09-29 09:01:22 -06009481 struct io_sq_data *sq = NULL;
Jens Axboefad8e0d2020-09-28 08:57:48 -06009482 bool has_lock;
Jens Axboe87ce9552020-01-30 08:25:34 -07009483 int i;
9484
Jens Axboefad8e0d2020-09-28 08:57:48 -06009485 /*
9486 * Avoid ABBA deadlock between the seq lock and the io_uring mutex,
9487 * since fdinfo case grabs it in the opposite direction of normal use
9488 * cases. If we fail to get the lock, we just don't iterate any
9489 * structures that could be going away outside the io_uring mutex.
9490 */
9491 has_lock = mutex_trylock(&ctx->uring_lock);
9492
Jens Axboe5f3f26f2021-02-25 10:17:46 -07009493 if (has_lock && (ctx->flags & IORING_SETUP_SQPOLL)) {
Joseph Qidbbe9c62020-09-29 09:01:22 -06009494 sq = ctx->sq_data;
Jens Axboe5f3f26f2021-02-25 10:17:46 -07009495 if (!sq->thread)
9496 sq = NULL;
9497 }
Joseph Qidbbe9c62020-09-29 09:01:22 -06009498
9499 seq_printf(m, "SqThread:\t%d\n", sq ? task_pid_nr(sq->thread) : -1);
9500 seq_printf(m, "SqThreadCpu:\t%d\n", sq ? task_cpu(sq->thread) : -1);
Jens Axboe87ce9552020-01-30 08:25:34 -07009501 seq_printf(m, "UserFiles:\t%u\n", ctx->nr_user_files);
Jens Axboefad8e0d2020-09-28 08:57:48 -06009502 for (i = 0; has_lock && i < ctx->nr_user_files; i++) {
Jens Axboe7b29f922021-03-12 08:30:14 -07009503 struct file *f = io_file_from_index(ctx, i);
Jens Axboe87ce9552020-01-30 08:25:34 -07009504
Jens Axboe87ce9552020-01-30 08:25:34 -07009505 if (f)
9506 seq_printf(m, "%5u: %s\n", i, file_dentry(f)->d_iname);
9507 else
9508 seq_printf(m, "%5u: <none>\n", i);
9509 }
9510 seq_printf(m, "UserBufs:\t%u\n", ctx->nr_user_bufs);
Jens Axboefad8e0d2020-09-28 08:57:48 -06009511 for (i = 0; has_lock && i < ctx->nr_user_bufs; i++) {
Pavel Begunkov41edf1a2021-04-25 14:32:23 +01009512 struct io_mapped_ubuf *buf = ctx->user_bufs[i];
Pavel Begunkov4751f532021-04-01 15:43:55 +01009513 unsigned int len = buf->ubuf_end - buf->ubuf;
Jens Axboe87ce9552020-01-30 08:25:34 -07009514
Pavel Begunkov4751f532021-04-01 15:43:55 +01009515 seq_printf(m, "%5u: 0x%llx/%u\n", i, buf->ubuf, len);
Jens Axboe87ce9552020-01-30 08:25:34 -07009516 }
Matthew Wilcox (Oracle)61cf9372021-03-08 14:16:16 +00009517 if (has_lock && !xa_empty(&ctx->personalities)) {
9518 unsigned long index;
9519 const struct cred *cred;
9520
Jens Axboe87ce9552020-01-30 08:25:34 -07009521 seq_printf(m, "Personalities:\n");
Matthew Wilcox (Oracle)61cf9372021-03-08 14:16:16 +00009522 xa_for_each(&ctx->personalities, index, cred)
9523 io_uring_show_cred(m, index, cred);
Jens Axboe87ce9552020-01-30 08:25:34 -07009524 }
Jens Axboed7718a92020-02-14 22:23:12 -07009525 seq_printf(m, "PollList:\n");
Jens Axboe79ebeae2021-08-10 15:18:27 -06009526 spin_lock(&ctx->completion_lock);
Jens Axboed7718a92020-02-14 22:23:12 -07009527 for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) {
9528 struct hlist_head *list = &ctx->cancel_hash[i];
9529 struct io_kiocb *req;
9530
9531 hlist_for_each_entry(req, list, hash_node)
9532 seq_printf(m, " op=%d, task_works=%d\n", req->opcode,
9533 req->task->task_works != NULL);
9534 }
Jens Axboe79ebeae2021-08-10 15:18:27 -06009535 spin_unlock(&ctx->completion_lock);
Jens Axboefad8e0d2020-09-28 08:57:48 -06009536 if (has_lock)
9537 mutex_unlock(&ctx->uring_lock);
Jens Axboe87ce9552020-01-30 08:25:34 -07009538}
9539
9540static void io_uring_show_fdinfo(struct seq_file *m, struct file *f)
9541{
9542 struct io_ring_ctx *ctx = f->private_data;
9543
9544 if (percpu_ref_tryget(&ctx->refs)) {
9545 __io_uring_show_fdinfo(ctx, m);
9546 percpu_ref_put(&ctx->refs);
9547 }
9548}
Tobias Klauserbebdb652020-02-26 18:38:32 +01009549#endif
Jens Axboe87ce9552020-01-30 08:25:34 -07009550
Jens Axboe2b188cc2019-01-07 10:46:33 -07009551static const struct file_operations io_uring_fops = {
9552 .release = io_uring_release,
9553 .mmap = io_uring_mmap,
Roman Penyaev6c5c2402019-11-28 12:53:22 +01009554#ifndef CONFIG_MMU
9555 .get_unmapped_area = io_uring_nommu_get_unmapped_area,
9556 .mmap_capabilities = io_uring_nommu_mmap_capabilities,
9557#endif
Jens Axboe2b188cc2019-01-07 10:46:33 -07009558 .poll = io_uring_poll,
9559 .fasync = io_uring_fasync,
Tobias Klauserbebdb652020-02-26 18:38:32 +01009560#ifdef CONFIG_PROC_FS
Jens Axboe87ce9552020-01-30 08:25:34 -07009561 .show_fdinfo = io_uring_show_fdinfo,
Tobias Klauserbebdb652020-02-26 18:38:32 +01009562#endif
Jens Axboe2b188cc2019-01-07 10:46:33 -07009563};
9564
9565static int io_allocate_scq_urings(struct io_ring_ctx *ctx,
9566 struct io_uring_params *p)
9567{
Hristo Venev75b28af2019-08-26 17:23:46 +00009568 struct io_rings *rings;
9569 size_t size, sq_array_offset;
Jens Axboe2b188cc2019-01-07 10:46:33 -07009570
Jens Axboebd740482020-08-05 12:58:23 -06009571 /* make sure these are sane, as we already accounted them */
9572 ctx->sq_entries = p->sq_entries;
9573 ctx->cq_entries = p->cq_entries;
9574
Hristo Venev75b28af2019-08-26 17:23:46 +00009575 size = rings_size(p->sq_entries, p->cq_entries, &sq_array_offset);
9576 if (size == SIZE_MAX)
9577 return -EOVERFLOW;
9578
9579 rings = io_mem_alloc(size);
9580 if (!rings)
Jens Axboe2b188cc2019-01-07 10:46:33 -07009581 return -ENOMEM;
9582
Hristo Venev75b28af2019-08-26 17:23:46 +00009583 ctx->rings = rings;
9584 ctx->sq_array = (u32 *)((char *)rings + sq_array_offset);
9585 rings->sq_ring_mask = p->sq_entries - 1;
9586 rings->cq_ring_mask = p->cq_entries - 1;
9587 rings->sq_ring_entries = p->sq_entries;
9588 rings->cq_ring_entries = p->cq_entries;
Jens Axboe2b188cc2019-01-07 10:46:33 -07009589
9590 size = array_size(sizeof(struct io_uring_sqe), p->sq_entries);
Jens Axboeeb065d32019-11-20 09:26:29 -07009591 if (size == SIZE_MAX) {
9592 io_mem_free(ctx->rings);
9593 ctx->rings = NULL;
Jens Axboe2b188cc2019-01-07 10:46:33 -07009594 return -EOVERFLOW;
Jens Axboeeb065d32019-11-20 09:26:29 -07009595 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07009596
9597 ctx->sq_sqes = io_mem_alloc(size);
Jens Axboeeb065d32019-11-20 09:26:29 -07009598 if (!ctx->sq_sqes) {
9599 io_mem_free(ctx->rings);
9600 ctx->rings = NULL;
Jens Axboe2b188cc2019-01-07 10:46:33 -07009601 return -ENOMEM;
Jens Axboeeb065d32019-11-20 09:26:29 -07009602 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07009603
Jens Axboe2b188cc2019-01-07 10:46:33 -07009604 return 0;
9605}
9606
Pavel Begunkov9faadcc2020-12-21 18:34:05 +00009607static int io_uring_install_fd(struct io_ring_ctx *ctx, struct file *file)
9608{
9609 int ret, fd;
9610
9611 fd = get_unused_fd_flags(O_RDWR | O_CLOEXEC);
9612 if (fd < 0)
9613 return fd;
9614
Pavel Begunkoveef51da2021-06-14 02:36:15 +01009615 ret = io_uring_add_tctx_node(ctx);
Pavel Begunkov9faadcc2020-12-21 18:34:05 +00009616 if (ret) {
9617 put_unused_fd(fd);
9618 return ret;
9619 }
9620 fd_install(fd, file);
9621 return fd;
9622}
9623
Jens Axboe2b188cc2019-01-07 10:46:33 -07009624/*
9625 * Allocate an anonymous fd, this is what constitutes the application
9626 * visible backing of an io_uring instance. The application mmaps this
9627 * fd to gain access to the SQ/CQ ring details. If UNIX sockets are enabled,
9628 * we have to tie this fd to a socket for file garbage collection purposes.
9629 */
Pavel Begunkov9faadcc2020-12-21 18:34:05 +00009630static struct file *io_uring_get_file(struct io_ring_ctx *ctx)
Jens Axboe2b188cc2019-01-07 10:46:33 -07009631{
9632 struct file *file;
Jens Axboe2b188cc2019-01-07 10:46:33 -07009633#if defined(CONFIG_UNIX)
Pavel Begunkov9faadcc2020-12-21 18:34:05 +00009634 int ret;
9635
Jens Axboe2b188cc2019-01-07 10:46:33 -07009636 ret = sock_create_kern(&init_net, PF_UNIX, SOCK_RAW, IPPROTO_IP,
9637 &ctx->ring_sock);
9638 if (ret)
Pavel Begunkov9faadcc2020-12-21 18:34:05 +00009639 return ERR_PTR(ret);
Jens Axboe2b188cc2019-01-07 10:46:33 -07009640#endif
9641
Jens Axboe2b188cc2019-01-07 10:46:33 -07009642 file = anon_inode_getfile("[io_uring]", &io_uring_fops, ctx,
9643 O_RDWR | O_CLOEXEC);
Pavel Begunkov9faadcc2020-12-21 18:34:05 +00009644#if defined(CONFIG_UNIX)
Jens Axboe2b188cc2019-01-07 10:46:33 -07009645 if (IS_ERR(file)) {
Pavel Begunkov9faadcc2020-12-21 18:34:05 +00009646 sock_release(ctx->ring_sock);
9647 ctx->ring_sock = NULL;
9648 } else {
9649 ctx->ring_sock->file = file;
Jens Axboe2b188cc2019-01-07 10:46:33 -07009650 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07009651#endif
Pavel Begunkov9faadcc2020-12-21 18:34:05 +00009652 return file;
Jens Axboe2b188cc2019-01-07 10:46:33 -07009653}
9654
Xiaoguang Wang7f136572020-05-05 16:28:53 +08009655static int io_uring_create(unsigned entries, struct io_uring_params *p,
9656 struct io_uring_params __user *params)
Jens Axboe2b188cc2019-01-07 10:46:33 -07009657{
Jens Axboe2b188cc2019-01-07 10:46:33 -07009658 struct io_ring_ctx *ctx;
Pavel Begunkov9faadcc2020-12-21 18:34:05 +00009659 struct file *file;
Jens Axboe2b188cc2019-01-07 10:46:33 -07009660 int ret;
9661
Jens Axboe8110c1a2019-12-28 15:39:54 -07009662 if (!entries)
Jens Axboe2b188cc2019-01-07 10:46:33 -07009663 return -EINVAL;
Jens Axboe8110c1a2019-12-28 15:39:54 -07009664 if (entries > IORING_MAX_ENTRIES) {
9665 if (!(p->flags & IORING_SETUP_CLAMP))
9666 return -EINVAL;
9667 entries = IORING_MAX_ENTRIES;
9668 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07009669
9670 /*
9671 * Use twice as many entries for the CQ ring. It's possible for the
9672 * application to drive a higher depth than the size of the SQ ring,
9673 * since the sqes are only used at submission time. This allows for
Jens Axboe33a107f2019-10-04 12:10:03 -06009674 * some flexibility in overcommitting a bit. If the application has
9675 * set IORING_SETUP_CQSIZE, it will have passed in the desired number
9676 * of CQ ring entries manually.
Jens Axboe2b188cc2019-01-07 10:46:33 -07009677 */
9678 p->sq_entries = roundup_pow_of_two(entries);
Jens Axboe33a107f2019-10-04 12:10:03 -06009679 if (p->flags & IORING_SETUP_CQSIZE) {
9680 /*
9681 * If IORING_SETUP_CQSIZE is set, we do the same roundup
9682 * to a power-of-two, if it isn't already. We do NOT impose
9683 * any cq vs sq ring sizing.
9684 */
Joseph Qieb2667b32020-11-24 15:03:03 +08009685 if (!p->cq_entries)
Jens Axboe33a107f2019-10-04 12:10:03 -06009686 return -EINVAL;
Jens Axboe8110c1a2019-12-28 15:39:54 -07009687 if (p->cq_entries > IORING_MAX_CQ_ENTRIES) {
9688 if (!(p->flags & IORING_SETUP_CLAMP))
9689 return -EINVAL;
9690 p->cq_entries = IORING_MAX_CQ_ENTRIES;
9691 }
Joseph Qieb2667b32020-11-24 15:03:03 +08009692 p->cq_entries = roundup_pow_of_two(p->cq_entries);
9693 if (p->cq_entries < p->sq_entries)
9694 return -EINVAL;
Jens Axboe33a107f2019-10-04 12:10:03 -06009695 } else {
9696 p->cq_entries = 2 * p->sq_entries;
9697 }
Jens Axboe2b188cc2019-01-07 10:46:33 -07009698
Jens Axboe2b188cc2019-01-07 10:46:33 -07009699 ctx = io_ring_ctx_alloc(p);
Jens Axboe62e398b2021-02-21 16:19:37 -07009700 if (!ctx)
Jens Axboe2b188cc2019-01-07 10:46:33 -07009701 return -ENOMEM;
Jens Axboe2b188cc2019-01-07 10:46:33 -07009702 ctx->compat = in_compat_syscall();
Jens Axboe62e398b2021-02-21 16:19:37 -07009703 if (!capable(CAP_IPC_LOCK))
9704 ctx->user = get_uid(current_user());
Jens Axboe2aede0e2020-09-14 10:45:53 -06009705
9706 /*
9707 * This is just grabbed for accounting purposes. When a process exits,
9708 * the mm is exited and dropped before the files, hence we need to hang
9709 * on to this mm purely for the purposes of being able to unaccount
9710 * memory (locked/pinned vm). It's not used for anything else.
9711 */
Jens Axboe6b7898e2020-08-25 07:58:00 -06009712 mmgrab(current->mm);
Jens Axboe2aede0e2020-09-14 10:45:53 -06009713 ctx->mm_account = current->mm;
Jens Axboe6b7898e2020-08-25 07:58:00 -06009714
Jens Axboe2b188cc2019-01-07 10:46:33 -07009715 ret = io_allocate_scq_urings(ctx, p);
9716 if (ret)
9717 goto err;
9718
Stefano Garzarella7e84e1c2020-08-27 16:58:31 +02009719 ret = io_sq_offload_create(ctx, p);
Jens Axboe2b188cc2019-01-07 10:46:33 -07009720 if (ret)
9721 goto err;
Pavel Begunkoveae071c2021-04-25 14:32:24 +01009722 /* always set a rsrc node */
Pavel Begunkov47b228c2021-04-29 11:46:48 +01009723 ret = io_rsrc_node_switch_start(ctx);
9724 if (ret)
9725 goto err;
Pavel Begunkoveae071c2021-04-25 14:32:24 +01009726 io_rsrc_node_switch(ctx, NULL);
Jens Axboe2b188cc2019-01-07 10:46:33 -07009727
Jens Axboe2b188cc2019-01-07 10:46:33 -07009728 memset(&p->sq_off, 0, sizeof(p->sq_off));
Hristo Venev75b28af2019-08-26 17:23:46 +00009729 p->sq_off.head = offsetof(struct io_rings, sq.head);
9730 p->sq_off.tail = offsetof(struct io_rings, sq.tail);
9731 p->sq_off.ring_mask = offsetof(struct io_rings, sq_ring_mask);
9732 p->sq_off.ring_entries = offsetof(struct io_rings, sq_ring_entries);
9733 p->sq_off.flags = offsetof(struct io_rings, sq_flags);
9734 p->sq_off.dropped = offsetof(struct io_rings, sq_dropped);
9735 p->sq_off.array = (char *)ctx->sq_array - (char *)ctx->rings;
Jens Axboe2b188cc2019-01-07 10:46:33 -07009736
9737 memset(&p->cq_off, 0, sizeof(p->cq_off));
Hristo Venev75b28af2019-08-26 17:23:46 +00009738 p->cq_off.head = offsetof(struct io_rings, cq.head);
9739 p->cq_off.tail = offsetof(struct io_rings, cq.tail);
9740 p->cq_off.ring_mask = offsetof(struct io_rings, cq_ring_mask);
9741 p->cq_off.ring_entries = offsetof(struct io_rings, cq_ring_entries);
9742 p->cq_off.overflow = offsetof(struct io_rings, cq_overflow);
9743 p->cq_off.cqes = offsetof(struct io_rings, cqes);
Stefano Garzarella0d9b5b32020-05-15 18:38:04 +02009744 p->cq_off.flags = offsetof(struct io_rings, cq_flags);
Jens Axboeac90f242019-09-06 10:26:21 -06009745
Xiaoguang Wang7f136572020-05-05 16:28:53 +08009746 p->features = IORING_FEAT_SINGLE_MMAP | IORING_FEAT_NODROP |
9747 IORING_FEAT_SUBMIT_STABLE | IORING_FEAT_RW_CUR_POS |
Jiufei Xue5769a352020-06-17 17:53:55 +08009748 IORING_FEAT_CUR_PERSONALITY | IORING_FEAT_FAST_POLL |
Hao Xuc73ebb62020-11-03 10:54:37 +08009749 IORING_FEAT_POLL_32BITS | IORING_FEAT_SQPOLL_NONFIXED |
Pavel Begunkov96905572021-06-10 16:37:38 +01009750 IORING_FEAT_EXT_ARG | IORING_FEAT_NATIVE_WORKERS |
9751 IORING_FEAT_RSRC_TAGS;
Xiaoguang Wang7f136572020-05-05 16:28:53 +08009752
9753 if (copy_to_user(params, p, sizeof(*p))) {
9754 ret = -EFAULT;
9755 goto err;
9756 }
Jens Axboed1719f72020-07-30 13:43:53 -06009757
Pavel Begunkov9faadcc2020-12-21 18:34:05 +00009758 file = io_uring_get_file(ctx);
9759 if (IS_ERR(file)) {
9760 ret = PTR_ERR(file);
9761 goto err;
9762 }
9763
Jens Axboed1719f72020-07-30 13:43:53 -06009764 /*
Jens Axboe044c1ab2019-10-28 09:15:33 -06009765 * Install ring fd as the very last thing, so we don't risk someone
9766 * having closed it before we finish setup
9767 */
Pavel Begunkov9faadcc2020-12-21 18:34:05 +00009768 ret = io_uring_install_fd(ctx, file);
9769 if (ret < 0) {
9770 /* fput will clean it up */
9771 fput(file);
9772 return ret;
9773 }
Jens Axboe044c1ab2019-10-28 09:15:33 -06009774
Dmitrii Dolgovc826bd72019-10-15 19:02:01 +02009775 trace_io_uring_create(ret, ctx, p->sq_entries, p->cq_entries, p->flags);
Jens Axboe2b188cc2019-01-07 10:46:33 -07009776 return ret;
9777err:
9778 io_ring_ctx_wait_and_kill(ctx);
9779 return ret;
9780}
9781
9782/*
9783 * Sets up an aio uring context, and returns the fd. Applications asks for a
9784 * ring size, we return the actual sq/cq ring sizes (among other things) in the
9785 * params structure passed in.
9786 */
9787static long io_uring_setup(u32 entries, struct io_uring_params __user *params)
9788{
9789 struct io_uring_params p;
Jens Axboe2b188cc2019-01-07 10:46:33 -07009790 int i;
9791
9792 if (copy_from_user(&p, params, sizeof(p)))
9793 return -EFAULT;
9794 for (i = 0; i < ARRAY_SIZE(p.resv); i++) {
9795 if (p.resv[i])
9796 return -EINVAL;
9797 }
9798
Jens Axboe6c271ce2019-01-10 11:22:30 -07009799 if (p.flags & ~(IORING_SETUP_IOPOLL | IORING_SETUP_SQPOLL |
Jens Axboe8110c1a2019-12-28 15:39:54 -07009800 IORING_SETUP_SQ_AFF | IORING_SETUP_CQSIZE |
Stefano Garzarella7e84e1c2020-08-27 16:58:31 +02009801 IORING_SETUP_CLAMP | IORING_SETUP_ATTACH_WQ |
9802 IORING_SETUP_R_DISABLED))
Jens Axboe2b188cc2019-01-07 10:46:33 -07009803 return -EINVAL;
9804
Xiaoguang Wang7f136572020-05-05 16:28:53 +08009805 return io_uring_create(entries, &p, params);
Jens Axboe2b188cc2019-01-07 10:46:33 -07009806}
9807
9808SYSCALL_DEFINE2(io_uring_setup, u32, entries,
9809 struct io_uring_params __user *, params)
9810{
9811 return io_uring_setup(entries, params);
9812}
9813
Jens Axboe66f4af92020-01-16 15:36:52 -07009814static int io_probe(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args)
9815{
9816 struct io_uring_probe *p;
9817 size_t size;
9818 int i, ret;
9819
9820 size = struct_size(p, ops, nr_args);
9821 if (size == SIZE_MAX)
9822 return -EOVERFLOW;
9823 p = kzalloc(size, GFP_KERNEL);
9824 if (!p)
9825 return -ENOMEM;
9826
9827 ret = -EFAULT;
9828 if (copy_from_user(p, arg, size))
9829 goto out;
9830 ret = -EINVAL;
9831 if (memchr_inv(p, 0, size))
9832 goto out;
9833
9834 p->last_op = IORING_OP_LAST - 1;
9835 if (nr_args > IORING_OP_LAST)
9836 nr_args = IORING_OP_LAST;
9837
9838 for (i = 0; i < nr_args; i++) {
9839 p->ops[i].op = i;
9840 if (!io_op_defs[i].not_supported)
9841 p->ops[i].flags = IO_URING_OP_SUPPORTED;
9842 }
9843 p->ops_len = i;
9844
9845 ret = 0;
9846 if (copy_to_user(arg, p, size))
9847 ret = -EFAULT;
9848out:
9849 kfree(p);
9850 return ret;
9851}
9852
Jens Axboe071698e2020-01-28 10:04:42 -07009853static int io_register_personality(struct io_ring_ctx *ctx)
9854{
Jens Axboe4379bf82021-02-15 13:40:22 -07009855 const struct cred *creds;
Matthew Wilcox (Oracle)61cf9372021-03-08 14:16:16 +00009856 u32 id;
Jens Axboe1e6fa522020-10-15 08:46:24 -06009857 int ret;
Jens Axboe071698e2020-01-28 10:04:42 -07009858
Jens Axboe4379bf82021-02-15 13:40:22 -07009859 creds = get_current_cred();
Jens Axboe1e6fa522020-10-15 08:46:24 -06009860
Matthew Wilcox (Oracle)61cf9372021-03-08 14:16:16 +00009861 ret = xa_alloc_cyclic(&ctx->personalities, &id, (void *)creds,
9862 XA_LIMIT(0, USHRT_MAX), &ctx->pers_next, GFP_KERNEL);
Jens Axboea30f8952021-08-20 14:53:59 -06009863 if (ret < 0) {
9864 put_cred(creds);
9865 return ret;
9866 }
9867 return id;
Jens Axboe071698e2020-01-28 10:04:42 -07009868}
9869
Stefano Garzarella21b55db2020-08-27 16:58:30 +02009870static int io_register_restrictions(struct io_ring_ctx *ctx, void __user *arg,
9871 unsigned int nr_args)
9872{
9873 struct io_uring_restriction *res;
9874 size_t size;
9875 int i, ret;
9876
Stefano Garzarella7e84e1c2020-08-27 16:58:31 +02009877 /* Restrictions allowed only if rings started disabled */
9878 if (!(ctx->flags & IORING_SETUP_R_DISABLED))
9879 return -EBADFD;
9880
Stefano Garzarella21b55db2020-08-27 16:58:30 +02009881 /* We allow only a single restrictions registration */
Stefano Garzarella7e84e1c2020-08-27 16:58:31 +02009882 if (ctx->restrictions.registered)
Stefano Garzarella21b55db2020-08-27 16:58:30 +02009883 return -EBUSY;
9884
9885 if (!arg || nr_args > IORING_MAX_RESTRICTIONS)
9886 return -EINVAL;
9887
9888 size = array_size(nr_args, sizeof(*res));
9889 if (size == SIZE_MAX)
9890 return -EOVERFLOW;
9891
9892 res = memdup_user(arg, size);
9893 if (IS_ERR(res))
9894 return PTR_ERR(res);
9895
9896 ret = 0;
9897
9898 for (i = 0; i < nr_args; i++) {
9899 switch (res[i].opcode) {
9900 case IORING_RESTRICTION_REGISTER_OP:
9901 if (res[i].register_op >= IORING_REGISTER_LAST) {
9902 ret = -EINVAL;
9903 goto out;
9904 }
9905
9906 __set_bit(res[i].register_op,
9907 ctx->restrictions.register_op);
9908 break;
9909 case IORING_RESTRICTION_SQE_OP:
9910 if (res[i].sqe_op >= IORING_OP_LAST) {
9911 ret = -EINVAL;
9912 goto out;
9913 }
9914
9915 __set_bit(res[i].sqe_op, ctx->restrictions.sqe_op);
9916 break;
9917 case IORING_RESTRICTION_SQE_FLAGS_ALLOWED:
9918 ctx->restrictions.sqe_flags_allowed = res[i].sqe_flags;
9919 break;
9920 case IORING_RESTRICTION_SQE_FLAGS_REQUIRED:
9921 ctx->restrictions.sqe_flags_required = res[i].sqe_flags;
9922 break;
9923 default:
9924 ret = -EINVAL;
9925 goto out;
9926 }
9927 }
9928
9929out:
9930 /* Reset all restrictions if an error happened */
9931 if (ret != 0)
9932 memset(&ctx->restrictions, 0, sizeof(ctx->restrictions));
9933 else
Stefano Garzarella7e84e1c2020-08-27 16:58:31 +02009934 ctx->restrictions.registered = true;
Stefano Garzarella21b55db2020-08-27 16:58:30 +02009935
9936 kfree(res);
9937 return ret;
9938}
9939
Stefano Garzarella7e84e1c2020-08-27 16:58:31 +02009940static int io_register_enable_rings(struct io_ring_ctx *ctx)
9941{
9942 if (!(ctx->flags & IORING_SETUP_R_DISABLED))
9943 return -EBADFD;
9944
9945 if (ctx->restrictions.registered)
9946 ctx->restricted = 1;
9947
Pavel Begunkov0298ef92021-03-08 13:20:57 +00009948 ctx->flags &= ~IORING_SETUP_R_DISABLED;
9949 if (ctx->sq_data && wq_has_sleeper(&ctx->sq_data->wait))
9950 wake_up(&ctx->sq_data->wait);
Stefano Garzarella7e84e1c2020-08-27 16:58:31 +02009951 return 0;
9952}
9953
Pavel Begunkovfdecb662021-04-25 14:32:20 +01009954static int __io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned type,
Pavel Begunkovc3bdad02021-04-25 14:32:22 +01009955 struct io_uring_rsrc_update2 *up,
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +01009956 unsigned nr_args)
9957{
9958 __u32 tmp;
9959 int err;
9960
Pavel Begunkovc3bdad02021-04-25 14:32:22 +01009961 if (up->resv)
9962 return -EINVAL;
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +01009963 if (check_add_overflow(up->offset, nr_args, &tmp))
9964 return -EOVERFLOW;
9965 err = io_rsrc_node_switch_start(ctx);
9966 if (err)
9967 return err;
9968
Pavel Begunkovfdecb662021-04-25 14:32:20 +01009969 switch (type) {
9970 case IORING_RSRC_FILE:
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +01009971 return __io_sqe_files_update(ctx, up, nr_args);
Pavel Begunkov634d00d2021-04-25 14:32:26 +01009972 case IORING_RSRC_BUFFER:
9973 return __io_sqe_buffers_update(ctx, up, nr_args);
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +01009974 }
9975 return -EINVAL;
9976}
9977
Pavel Begunkovc3bdad02021-04-25 14:32:22 +01009978static int io_register_files_update(struct io_ring_ctx *ctx, void __user *arg,
9979 unsigned nr_args)
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +01009980{
Pavel Begunkovc3bdad02021-04-25 14:32:22 +01009981 struct io_uring_rsrc_update2 up;
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +01009982
9983 if (!nr_args)
9984 return -EINVAL;
Pavel Begunkovc3bdad02021-04-25 14:32:22 +01009985 memset(&up, 0, sizeof(up));
9986 if (copy_from_user(&up, arg, sizeof(struct io_uring_rsrc_update)))
9987 return -EFAULT;
9988 return __io_register_rsrc_update(ctx, IORING_RSRC_FILE, &up, nr_args);
9989}
9990
9991static int io_register_rsrc_update(struct io_ring_ctx *ctx, void __user *arg,
Pavel Begunkov992da012021-06-10 16:37:37 +01009992 unsigned size, unsigned type)
Pavel Begunkovc3bdad02021-04-25 14:32:22 +01009993{
9994 struct io_uring_rsrc_update2 up;
9995
9996 if (size != sizeof(up))
9997 return -EINVAL;
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +01009998 if (copy_from_user(&up, arg, sizeof(up)))
9999 return -EFAULT;
Pavel Begunkov992da012021-06-10 16:37:37 +010010000 if (!up.nr || up.resv)
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +010010001 return -EINVAL;
Pavel Begunkov992da012021-06-10 16:37:37 +010010002 return __io_register_rsrc_update(ctx, type, &up, up.nr);
Pavel Begunkov98f0b3b2021-04-25 14:32:19 +010010003}
10004
Pavel Begunkov792e3582021-04-25 14:32:21 +010010005static int io_register_rsrc(struct io_ring_ctx *ctx, void __user *arg,
Pavel Begunkov992da012021-06-10 16:37:37 +010010006 unsigned int size, unsigned int type)
Pavel Begunkov792e3582021-04-25 14:32:21 +010010007{
10008 struct io_uring_rsrc_register rr;
10009
10010 /* keep it extendible */
10011 if (size != sizeof(rr))
10012 return -EINVAL;
10013
10014 memset(&rr, 0, sizeof(rr));
10015 if (copy_from_user(&rr, arg, size))
10016 return -EFAULT;
Pavel Begunkov992da012021-06-10 16:37:37 +010010017 if (!rr.nr || rr.resv || rr.resv2)
Pavel Begunkov792e3582021-04-25 14:32:21 +010010018 return -EINVAL;
10019
Pavel Begunkov992da012021-06-10 16:37:37 +010010020 switch (type) {
Pavel Begunkov792e3582021-04-25 14:32:21 +010010021 case IORING_RSRC_FILE:
10022 return io_sqe_files_register(ctx, u64_to_user_ptr(rr.data),
10023 rr.nr, u64_to_user_ptr(rr.tags));
Pavel Begunkov634d00d2021-04-25 14:32:26 +010010024 case IORING_RSRC_BUFFER:
10025 return io_sqe_buffers_register(ctx, u64_to_user_ptr(rr.data),
10026 rr.nr, u64_to_user_ptr(rr.tags));
Pavel Begunkov792e3582021-04-25 14:32:21 +010010027 }
10028 return -EINVAL;
10029}
10030
Jens Axboefe764212021-06-17 10:19:54 -060010031static int io_register_iowq_aff(struct io_ring_ctx *ctx, void __user *arg,
10032 unsigned len)
10033{
10034 struct io_uring_task *tctx = current->io_uring;
10035 cpumask_var_t new_mask;
10036 int ret;
10037
10038 if (!tctx || !tctx->io_wq)
10039 return -EINVAL;
10040
10041 if (!alloc_cpumask_var(&new_mask, GFP_KERNEL))
10042 return -ENOMEM;
10043
10044 cpumask_clear(new_mask);
10045 if (len > cpumask_size())
10046 len = cpumask_size();
10047
10048 if (copy_from_user(new_mask, arg, len)) {
10049 free_cpumask_var(new_mask);
10050 return -EFAULT;
10051 }
10052
10053 ret = io_wq_cpu_affinity(tctx->io_wq, new_mask);
10054 free_cpumask_var(new_mask);
10055 return ret;
10056}
10057
10058static int io_unregister_iowq_aff(struct io_ring_ctx *ctx)
10059{
10060 struct io_uring_task *tctx = current->io_uring;
10061
10062 if (!tctx || !tctx->io_wq)
10063 return -EINVAL;
10064
10065 return io_wq_cpu_affinity(tctx->io_wq, NULL);
10066}
10067
Jens Axboe071698e2020-01-28 10:04:42 -070010068static bool io_register_op_must_quiesce(int op)
10069{
10070 switch (op) {
Bijan Mottahedehbd54b6f2021-04-25 14:32:25 +010010071 case IORING_REGISTER_BUFFERS:
10072 case IORING_UNREGISTER_BUFFERS:
Pavel Begunkovf4f7d212021-04-01 15:44:02 +010010073 case IORING_REGISTER_FILES:
Jens Axboe071698e2020-01-28 10:04:42 -070010074 case IORING_UNREGISTER_FILES:
10075 case IORING_REGISTER_FILES_UPDATE:
10076 case IORING_REGISTER_PROBE:
10077 case IORING_REGISTER_PERSONALITY:
10078 case IORING_UNREGISTER_PERSONALITY:
Pavel Begunkov992da012021-06-10 16:37:37 +010010079 case IORING_REGISTER_FILES2:
10080 case IORING_REGISTER_FILES_UPDATE2:
10081 case IORING_REGISTER_BUFFERS2:
10082 case IORING_REGISTER_BUFFERS_UPDATE:
Jens Axboefe764212021-06-17 10:19:54 -060010083 case IORING_REGISTER_IOWQ_AFF:
10084 case IORING_UNREGISTER_IOWQ_AFF:
Jens Axboe071698e2020-01-28 10:04:42 -070010085 return false;
10086 default:
10087 return true;
10088 }
10089}
10090
Pavel Begunkove73c5c72021-08-09 13:04:12 +010010091static int io_ctx_quiesce(struct io_ring_ctx *ctx)
10092{
10093 long ret;
10094
10095 percpu_ref_kill(&ctx->refs);
10096
10097 /*
10098 * Drop uring mutex before waiting for references to exit. If another
10099 * thread is currently inside io_uring_enter() it might need to grab the
10100 * uring_lock to make progress. If we hold it here across the drain
10101 * wait, then we can deadlock. It's safe to drop the mutex here, since
10102 * no new references will come in after we've killed the percpu ref.
10103 */
10104 mutex_unlock(&ctx->uring_lock);
10105 do {
10106 ret = wait_for_completion_interruptible(&ctx->ref_comp);
10107 if (!ret)
10108 break;
10109 ret = io_run_task_work_sig();
10110 } while (ret >= 0);
10111 mutex_lock(&ctx->uring_lock);
10112
10113 if (ret)
10114 io_refs_resurrect(&ctx->refs, &ctx->ref_comp);
10115 return ret;
10116}
10117
Jens Axboeedafcce2019-01-09 09:16:05 -070010118static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,
10119 void __user *arg, unsigned nr_args)
Jens Axboeb19062a2019-04-15 10:49:38 -060010120 __releases(ctx->uring_lock)
10121 __acquires(ctx->uring_lock)
Jens Axboeedafcce2019-01-09 09:16:05 -070010122{
10123 int ret;
10124
Jens Axboe35fa71a2019-04-22 10:23:23 -060010125 /*
10126 * We're inside the ring mutex, if the ref is already dying, then
10127 * someone else killed the ctx or is already going through
10128 * io_uring_register().
10129 */
10130 if (percpu_ref_is_dying(&ctx->refs))
10131 return -ENXIO;
10132
Pavel Begunkov75c40212021-04-15 13:07:40 +010010133 if (ctx->restricted) {
10134 if (opcode >= IORING_REGISTER_LAST)
10135 return -EINVAL;
10136 opcode = array_index_nospec(opcode, IORING_REGISTER_LAST);
10137 if (!test_bit(opcode, ctx->restrictions.register_op))
10138 return -EACCES;
10139 }
10140
Jens Axboe071698e2020-01-28 10:04:42 -070010141 if (io_register_op_must_quiesce(opcode)) {
Pavel Begunkove73c5c72021-08-09 13:04:12 +010010142 ret = io_ctx_quiesce(ctx);
10143 if (ret)
Pavel Begunkovf70865d2021-04-11 01:46:40 +010010144 return ret;
Jens Axboe05f3fb32019-12-09 11:22:50 -070010145 }
Jens Axboeedafcce2019-01-09 09:16:05 -070010146
10147 switch (opcode) {
10148 case IORING_REGISTER_BUFFERS:
Pavel Begunkov634d00d2021-04-25 14:32:26 +010010149 ret = io_sqe_buffers_register(ctx, arg, nr_args, NULL);
Jens Axboeedafcce2019-01-09 09:16:05 -070010150 break;
10151 case IORING_UNREGISTER_BUFFERS:
10152 ret = -EINVAL;
10153 if (arg || nr_args)
10154 break;
Bijan Mottahedeh0a96bbe2021-01-06 12:39:10 -080010155 ret = io_sqe_buffers_unregister(ctx);
Jens Axboeedafcce2019-01-09 09:16:05 -070010156 break;
Jens Axboe6b063142019-01-10 22:13:58 -070010157 case IORING_REGISTER_FILES:
Pavel Begunkov792e3582021-04-25 14:32:21 +010010158 ret = io_sqe_files_register(ctx, arg, nr_args, NULL);
Jens Axboe6b063142019-01-10 22:13:58 -070010159 break;
10160 case IORING_UNREGISTER_FILES:
10161 ret = -EINVAL;
10162 if (arg || nr_args)
10163 break;
10164 ret = io_sqe_files_unregister(ctx);
10165 break;
Jens Axboec3a31e62019-10-03 13:59:56 -060010166 case IORING_REGISTER_FILES_UPDATE:
Pavel Begunkovc3bdad02021-04-25 14:32:22 +010010167 ret = io_register_files_update(ctx, arg, nr_args);
Jens Axboec3a31e62019-10-03 13:59:56 -060010168 break;
Jens Axboe9b402842019-04-11 11:45:41 -060010169 case IORING_REGISTER_EVENTFD:
Jens Axboef2842ab2020-01-08 11:04:00 -070010170 case IORING_REGISTER_EVENTFD_ASYNC:
Jens Axboe9b402842019-04-11 11:45:41 -060010171 ret = -EINVAL;
10172 if (nr_args != 1)
10173 break;
10174 ret = io_eventfd_register(ctx, arg);
Jens Axboef2842ab2020-01-08 11:04:00 -070010175 if (ret)
10176 break;
10177 if (opcode == IORING_REGISTER_EVENTFD_ASYNC)
10178 ctx->eventfd_async = 1;
10179 else
10180 ctx->eventfd_async = 0;
Jens Axboe9b402842019-04-11 11:45:41 -060010181 break;
10182 case IORING_UNREGISTER_EVENTFD:
10183 ret = -EINVAL;
10184 if (arg || nr_args)
10185 break;
10186 ret = io_eventfd_unregister(ctx);
10187 break;
Jens Axboe66f4af92020-01-16 15:36:52 -070010188 case IORING_REGISTER_PROBE:
10189 ret = -EINVAL;
10190 if (!arg || nr_args > 256)
10191 break;
10192 ret = io_probe(ctx, arg, nr_args);
10193 break;
Jens Axboe071698e2020-01-28 10:04:42 -070010194 case IORING_REGISTER_PERSONALITY:
10195 ret = -EINVAL;
10196 if (arg || nr_args)
10197 break;
10198 ret = io_register_personality(ctx);
10199 break;
10200 case IORING_UNREGISTER_PERSONALITY:
10201 ret = -EINVAL;
10202 if (arg)
10203 break;
10204 ret = io_unregister_personality(ctx, nr_args);
10205 break;
Stefano Garzarella7e84e1c2020-08-27 16:58:31 +020010206 case IORING_REGISTER_ENABLE_RINGS:
10207 ret = -EINVAL;
10208 if (arg || nr_args)
10209 break;
10210 ret = io_register_enable_rings(ctx);
10211 break;
Stefano Garzarella21b55db2020-08-27 16:58:30 +020010212 case IORING_REGISTER_RESTRICTIONS:
10213 ret = io_register_restrictions(ctx, arg, nr_args);
10214 break;
Pavel Begunkov992da012021-06-10 16:37:37 +010010215 case IORING_REGISTER_FILES2:
10216 ret = io_register_rsrc(ctx, arg, nr_args, IORING_RSRC_FILE);
Pavel Begunkov792e3582021-04-25 14:32:21 +010010217 break;
Pavel Begunkov992da012021-06-10 16:37:37 +010010218 case IORING_REGISTER_FILES_UPDATE2:
10219 ret = io_register_rsrc_update(ctx, arg, nr_args,
10220 IORING_RSRC_FILE);
10221 break;
10222 case IORING_REGISTER_BUFFERS2:
10223 ret = io_register_rsrc(ctx, arg, nr_args, IORING_RSRC_BUFFER);
10224 break;
10225 case IORING_REGISTER_BUFFERS_UPDATE:
10226 ret = io_register_rsrc_update(ctx, arg, nr_args,
10227 IORING_RSRC_BUFFER);
Pavel Begunkovc3bdad02021-04-25 14:32:22 +010010228 break;
Jens Axboefe764212021-06-17 10:19:54 -060010229 case IORING_REGISTER_IOWQ_AFF:
10230 ret = -EINVAL;
10231 if (!arg || !nr_args)
10232 break;
10233 ret = io_register_iowq_aff(ctx, arg, nr_args);
10234 break;
10235 case IORING_UNREGISTER_IOWQ_AFF:
10236 ret = -EINVAL;
10237 if (arg || nr_args)
10238 break;
10239 ret = io_unregister_iowq_aff(ctx);
10240 break;
Jens Axboeedafcce2019-01-09 09:16:05 -070010241 default:
10242 ret = -EINVAL;
10243 break;
10244 }
10245
Jens Axboe071698e2020-01-28 10:04:42 -070010246 if (io_register_op_must_quiesce(opcode)) {
Jens Axboe05f3fb32019-12-09 11:22:50 -070010247 /* bring the ctx back to life */
Jens Axboe05f3fb32019-12-09 11:22:50 -070010248 percpu_ref_reinit(&ctx->refs);
Jens Axboe0f158b42020-05-14 17:18:39 -060010249 reinit_completion(&ctx->ref_comp);
Jens Axboe05f3fb32019-12-09 11:22:50 -070010250 }
Jens Axboeedafcce2019-01-09 09:16:05 -070010251 return ret;
10252}
10253
10254SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode,
10255 void __user *, arg, unsigned int, nr_args)
10256{
10257 struct io_ring_ctx *ctx;
10258 long ret = -EBADF;
10259 struct fd f;
10260
10261 f = fdget(fd);
10262 if (!f.file)
10263 return -EBADF;
10264
10265 ret = -EOPNOTSUPP;
10266 if (f.file->f_op != &io_uring_fops)
10267 goto out_fput;
10268
10269 ctx = f.file->private_data;
10270
Pavel Begunkovb6c23dd2021-02-20 15:17:18 +000010271 io_run_task_work();
10272
Jens Axboeedafcce2019-01-09 09:16:05 -070010273 mutex_lock(&ctx->uring_lock);
10274 ret = __io_uring_register(ctx, opcode, arg, nr_args);
10275 mutex_unlock(&ctx->uring_lock);
Dmitrii Dolgovc826bd72019-10-15 19:02:01 +020010276 trace_io_uring_register(ctx, opcode, ctx->nr_user_files, ctx->nr_user_bufs,
10277 ctx->cq_ev_fd != NULL, ret);
Jens Axboeedafcce2019-01-09 09:16:05 -070010278out_fput:
10279 fdput(f);
10280 return ret;
10281}
10282
Jens Axboe2b188cc2019-01-07 10:46:33 -070010283static int __init io_uring_init(void)
10284{
Stefan Metzmacherd7f62e82020-01-29 14:39:41 +010010285#define __BUILD_BUG_VERIFY_ELEMENT(stype, eoffset, etype, ename) do { \
10286 BUILD_BUG_ON(offsetof(stype, ename) != eoffset); \
10287 BUILD_BUG_ON(sizeof(etype) != sizeof_field(stype, ename)); \
10288} while (0)
10289
10290#define BUILD_BUG_SQE_ELEM(eoffset, etype, ename) \
10291 __BUILD_BUG_VERIFY_ELEMENT(struct io_uring_sqe, eoffset, etype, ename)
10292 BUILD_BUG_ON(sizeof(struct io_uring_sqe) != 64);
10293 BUILD_BUG_SQE_ELEM(0, __u8, opcode);
10294 BUILD_BUG_SQE_ELEM(1, __u8, flags);
10295 BUILD_BUG_SQE_ELEM(2, __u16, ioprio);
10296 BUILD_BUG_SQE_ELEM(4, __s32, fd);
10297 BUILD_BUG_SQE_ELEM(8, __u64, off);
10298 BUILD_BUG_SQE_ELEM(8, __u64, addr2);
10299 BUILD_BUG_SQE_ELEM(16, __u64, addr);
Pavel Begunkov7d67af22020-02-24 11:32:45 +030010300 BUILD_BUG_SQE_ELEM(16, __u64, splice_off_in);
Stefan Metzmacherd7f62e82020-01-29 14:39:41 +010010301 BUILD_BUG_SQE_ELEM(24, __u32, len);
10302 BUILD_BUG_SQE_ELEM(28, __kernel_rwf_t, rw_flags);
10303 BUILD_BUG_SQE_ELEM(28, /* compat */ int, rw_flags);
10304 BUILD_BUG_SQE_ELEM(28, /* compat */ __u32, rw_flags);
10305 BUILD_BUG_SQE_ELEM(28, __u32, fsync_flags);
Jiufei Xue5769a352020-06-17 17:53:55 +080010306 BUILD_BUG_SQE_ELEM(28, /* compat */ __u16, poll_events);
10307 BUILD_BUG_SQE_ELEM(28, __u32, poll32_events);
Stefan Metzmacherd7f62e82020-01-29 14:39:41 +010010308 BUILD_BUG_SQE_ELEM(28, __u32, sync_range_flags);
10309 BUILD_BUG_SQE_ELEM(28, __u32, msg_flags);
10310 BUILD_BUG_SQE_ELEM(28, __u32, timeout_flags);
10311 BUILD_BUG_SQE_ELEM(28, __u32, accept_flags);
10312 BUILD_BUG_SQE_ELEM(28, __u32, cancel_flags);
10313 BUILD_BUG_SQE_ELEM(28, __u32, open_flags);
10314 BUILD_BUG_SQE_ELEM(28, __u32, statx_flags);
10315 BUILD_BUG_SQE_ELEM(28, __u32, fadvise_advice);
Pavel Begunkov7d67af22020-02-24 11:32:45 +030010316 BUILD_BUG_SQE_ELEM(28, __u32, splice_flags);
Stefan Metzmacherd7f62e82020-01-29 14:39:41 +010010317 BUILD_BUG_SQE_ELEM(32, __u64, user_data);
10318 BUILD_BUG_SQE_ELEM(40, __u16, buf_index);
Pavel Begunkov16340ea2021-06-24 15:09:58 +010010319 BUILD_BUG_SQE_ELEM(40, __u16, buf_group);
Stefan Metzmacherd7f62e82020-01-29 14:39:41 +010010320 BUILD_BUG_SQE_ELEM(42, __u16, personality);
Pavel Begunkov7d67af22020-02-24 11:32:45 +030010321 BUILD_BUG_SQE_ELEM(44, __s32, splice_fd_in);
Stefan Metzmacherd7f62e82020-01-29 14:39:41 +010010322
Pavel Begunkovb0d658ec2021-04-27 16:13:53 +010010323 BUILD_BUG_ON(sizeof(struct io_uring_files_update) !=
10324 sizeof(struct io_uring_rsrc_update));
10325 BUILD_BUG_ON(sizeof(struct io_uring_rsrc_update) >
10326 sizeof(struct io_uring_rsrc_update2));
10327 /* should fit into one byte */
10328 BUILD_BUG_ON(SQE_VALID_FLAGS >= (1 << 8));
10329
Jens Axboed3656342019-12-18 09:50:26 -070010330 BUILD_BUG_ON(ARRAY_SIZE(io_op_defs) != IORING_OP_LAST);
Jens Axboe84557872020-03-03 15:28:17 -070010331 BUILD_BUG_ON(__REQ_F_LAST_BIT >= 8 * sizeof(int));
Pavel Begunkov16340ea2021-06-24 15:09:58 +010010332
Jens Axboe91f245d2021-02-09 13:48:50 -070010333 req_cachep = KMEM_CACHE(io_kiocb, SLAB_HWCACHE_ALIGN | SLAB_PANIC |
10334 SLAB_ACCOUNT);
Jens Axboe2b188cc2019-01-07 10:46:33 -070010335 return 0;
10336};
10337__initcall(io_uring_init);