blob: a47a1edb940493acd8f951ba887ba495d43888b7 [file] [log] [blame]
Greg Kroah-Hartmanb2441312017-11-01 15:07:57 +01001// SPDX-License-Identifier: GPL-2.0
Linus Torvalds1da177e2005-04-16 15:20:36 -07002/*
3 * linux/fs/file.c
4 *
5 * Copyright (C) 1998-1999, Stephen Tweedie and Bill Hawes
6 *
7 * Manage the dynamic fd arrays in the process files_struct.
8 */
9
Al Virofe17f222012-08-21 11:48:11 -040010#include <linux/syscalls.h>
Paul Gortmaker630d9c42011-11-16 23:57:37 -050011#include <linux/export.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070012#include <linux/fs.h>
Christian Brauner278a5fb2019-05-24 11:30:34 +020013#include <linux/kernel.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070014#include <linux/mm.h>
Ingo Molnar3f07c012017-02-08 18:51:30 +010015#include <linux/sched/signal.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070016#include <linux/slab.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070017#include <linux/file.h>
Al Viro9f3acc32008-04-24 07:44:08 -040018#include <linux/fdtable.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070019#include <linux/bitops.h>
Dipankar Sarmaab2af1f2005-09-09 13:04:13 -070020#include <linux/spinlock.h>
21#include <linux/rcupdate.h>
Christian Brauner60997c32020-06-03 21:48:55 +020022#include <linux/close_range.h>
Kees Cook66590612020-06-10 08:20:05 -070023#include <net/sock.h>
Dipankar Sarmaab2af1f2005-09-09 13:04:13 -070024
Alexey Dobriyan9b80a182016-09-02 00:38:52 +030025unsigned int sysctl_nr_open __read_mostly = 1024*1024;
26unsigned int sysctl_nr_open_min = BITS_PER_LONG;
Rasmus Villemoes752343b2015-10-29 12:01:41 +010027/* our min() is unusable in constant expressions ;-/ */
28#define __const_min(x, y) ((x) < (y) ? (x) : (y))
Alexey Dobriyan9b80a182016-09-02 00:38:52 +030029unsigned int sysctl_nr_open_max =
30 __const_min(INT_MAX, ~(size_t)0/sizeof(void *)) & -BITS_PER_LONG;
Eric Dumazet9cfe0152008-02-06 01:37:16 -080031
Changli Gaoa892e2d2010-08-10 18:01:35 -070032static void __free_fdtable(struct fdtable *fdt)
Vadim Lobanov5466b452006-12-10 02:21:22 -080033{
Al Virof6c0a192014-04-23 10:18:46 -040034 kvfree(fdt->fd);
35 kvfree(fdt->open_fds);
Changli Gaoa892e2d2010-08-10 18:01:35 -070036 kfree(fdt);
Dipankar Sarmaab2af1f2005-09-09 13:04:13 -070037}
38
Al Viro7cf4dc32012-08-15 19:56:12 -040039static void free_fdtable_rcu(struct rcu_head *rcu)
Dipankar Sarmaab2af1f2005-09-09 13:04:13 -070040{
Al Viroac3e3c52013-04-28 21:42:33 -040041 __free_fdtable(container_of(rcu, struct fdtable, rcu));
Dipankar Sarmaab2af1f2005-09-09 13:04:13 -070042}
43
Linus Torvaldsf3f86e32015-10-30 16:53:57 -070044#define BITBIT_NR(nr) BITS_TO_LONGS(BITS_TO_LONGS(nr))
45#define BITBIT_SIZE(nr) (BITBIT_NR(nr) * sizeof(long))
46
Dipankar Sarmaab2af1f2005-09-09 13:04:13 -070047/*
Eric Biggersea5c58e2015-11-06 00:32:04 -060048 * Copy 'count' fd bits from the old table to the new table and clear the extra
49 * space if any. This does not copy the file pointers. Called with the files
50 * spinlock held for write.
51 */
52static void copy_fd_bitmaps(struct fdtable *nfdt, struct fdtable *ofdt,
53 unsigned int count)
54{
55 unsigned int cpy, set;
56
57 cpy = count / BITS_PER_BYTE;
58 set = (nfdt->max_fds - count) / BITS_PER_BYTE;
59 memcpy(nfdt->open_fds, ofdt->open_fds, cpy);
60 memset((char *)nfdt->open_fds + cpy, 0, set);
61 memcpy(nfdt->close_on_exec, ofdt->close_on_exec, cpy);
62 memset((char *)nfdt->close_on_exec + cpy, 0, set);
63
64 cpy = BITBIT_SIZE(count);
65 set = BITBIT_SIZE(nfdt->max_fds) - cpy;
66 memcpy(nfdt->full_fds_bits, ofdt->full_fds_bits, cpy);
67 memset((char *)nfdt->full_fds_bits + cpy, 0, set);
68}
69
70/*
71 * Copy all file descriptors from the old table to the new, expanded table and
72 * clear the extra space. Called with the files spinlock held for write.
Dipankar Sarmaab2af1f2005-09-09 13:04:13 -070073 */
Vadim Lobanov5466b452006-12-10 02:21:22 -080074static void copy_fdtable(struct fdtable *nfdt, struct fdtable *ofdt)
Dipankar Sarmaab2af1f2005-09-09 13:04:13 -070075{
Al Viro4e89b722020-05-19 17:48:52 -040076 size_t cpy, set;
Dipankar Sarmaab2af1f2005-09-09 13:04:13 -070077
Vadim Lobanov5466b452006-12-10 02:21:22 -080078 BUG_ON(nfdt->max_fds < ofdt->max_fds);
Vadim Lobanov5466b452006-12-10 02:21:22 -080079
80 cpy = ofdt->max_fds * sizeof(struct file *);
81 set = (nfdt->max_fds - ofdt->max_fds) * sizeof(struct file *);
82 memcpy(nfdt->fd, ofdt->fd, cpy);
Eric Biggersea5c58e2015-11-06 00:32:04 -060083 memset((char *)nfdt->fd + cpy, 0, set);
Vadim Lobanov5466b452006-12-10 02:21:22 -080084
Eric Biggersea5c58e2015-11-06 00:32:04 -060085 copy_fd_bitmaps(nfdt, ofdt, ofdt->max_fds);
Linus Torvalds1da177e2005-04-16 15:20:36 -070086}
87
Linus Torvalds00d2b9f2022-03-29 15:06:39 -070088/*
89 * Note how the fdtable bitmap allocations very much have to be a multiple of
90 * BITS_PER_LONG. This is not only because we walk those things in chunks of
91 * 'unsigned long' in some places, but simply because that is how the Linux
92 * kernel bitmaps are defined to work: they are not "bits in an array of bytes",
93 * they are very much "bits in an array of unsigned long".
94 *
95 * The ALIGN(nr, BITS_PER_LONG) here is for clarity: since we just multiplied
96 * by that "1024/sizeof(ptr)" before, we already know there are sufficient
97 * clear low bits. Clang seems to realize that, gcc ends up being confused.
98 *
99 * On a 128-bit machine, the ALIGN() would actually matter. In the meantime,
100 * let's consider it documentation (and maybe a test-case for gcc to improve
101 * its code generation ;)
102 */
Vadim Lobanov5466b452006-12-10 02:21:22 -0800103static struct fdtable * alloc_fdtable(unsigned int nr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700104{
Vadim Lobanov5466b452006-12-10 02:21:22 -0800105 struct fdtable *fdt;
David Howells1fd36ad2012-02-16 17:49:54 +0000106 void *data;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700107
Dipankar Sarmaab2af1f2005-09-09 13:04:13 -0700108 /*
Vadim Lobanov5466b452006-12-10 02:21:22 -0800109 * Figure out how many fds we actually want to support in this fdtable.
110 * Allocation steps are keyed to the size of the fdarray, since it
111 * grows far faster than any of the other dynamic data. We try to fit
112 * the fdarray into comfortable page-tuned chunks: starting at 1024B
113 * and growing in powers of two from there on.
Dipankar Sarmaab2af1f2005-09-09 13:04:13 -0700114 */
Vadim Lobanov5466b452006-12-10 02:21:22 -0800115 nr /= (1024 / sizeof(struct file *));
116 nr = roundup_pow_of_two(nr + 1);
117 nr *= (1024 / sizeof(struct file *));
Linus Torvalds00d2b9f2022-03-29 15:06:39 -0700118 nr = ALIGN(nr, BITS_PER_LONG);
Al Viro5c598b32008-04-27 20:04:15 -0400119 /*
120 * Note that this can drive nr *below* what we had passed if sysctl_nr_open
121 * had been set lower between the check in expand_files() and here. Deal
122 * with that in caller, it's cheaper that way.
123 *
124 * We make sure that nr remains a multiple of BITS_PER_LONG - otherwise
125 * bitmaps handling below becomes unpleasant, to put it mildly...
126 */
127 if (unlikely(nr > sysctl_nr_open))
128 nr = ((sysctl_nr_open - 1) | (BITS_PER_LONG - 1)) + 1;
Vadim Lobanovbbea9f62006-12-10 02:21:12 -0800129
Vladimir Davydov5d097052016-01-14 15:18:21 -0800130 fdt = kmalloc(sizeof(struct fdtable), GFP_KERNEL_ACCOUNT);
Vadim Lobanov5466b452006-12-10 02:21:22 -0800131 if (!fdt)
Vadim Lobanovbbea9f62006-12-10 02:21:12 -0800132 goto out;
Vadim Lobanov5466b452006-12-10 02:21:22 -0800133 fdt->max_fds = nr;
Michal Hockoc823bd92017-07-06 15:36:19 -0700134 data = kvmalloc_array(nr, sizeof(struct file *), GFP_KERNEL_ACCOUNT);
Vadim Lobanov5466b452006-12-10 02:21:22 -0800135 if (!data)
136 goto out_fdt;
David Howells1fd36ad2012-02-16 17:49:54 +0000137 fdt->fd = data;
138
Michal Hockoc823bd92017-07-06 15:36:19 -0700139 data = kvmalloc(max_t(size_t,
140 2 * nr / BITS_PER_BYTE + BITBIT_SIZE(nr), L1_CACHE_BYTES),
141 GFP_KERNEL_ACCOUNT);
Vadim Lobanov5466b452006-12-10 02:21:22 -0800142 if (!data)
143 goto out_arr;
David Howells1fd36ad2012-02-16 17:49:54 +0000144 fdt->open_fds = data;
Vadim Lobanov5466b452006-12-10 02:21:22 -0800145 data += nr / BITS_PER_BYTE;
David Howells1fd36ad2012-02-16 17:49:54 +0000146 fdt->close_on_exec = data;
Linus Torvaldsf3f86e32015-10-30 16:53:57 -0700147 data += nr / BITS_PER_BYTE;
148 fdt->full_fds_bits = data;
Vadim Lobanov5466b452006-12-10 02:21:22 -0800149
Dipankar Sarmaab2af1f2005-09-09 13:04:13 -0700150 return fdt;
Vadim Lobanov5466b452006-12-10 02:21:22 -0800151
152out_arr:
Al Virof6c0a192014-04-23 10:18:46 -0400153 kvfree(fdt->fd);
Vadim Lobanov5466b452006-12-10 02:21:22 -0800154out_fdt:
Dipankar Sarmaab2af1f2005-09-09 13:04:13 -0700155 kfree(fdt);
Vadim Lobanov5466b452006-12-10 02:21:22 -0800156out:
Dipankar Sarmaab2af1f2005-09-09 13:04:13 -0700157 return NULL;
158}
159
160/*
Vadim Lobanov74d392a2006-09-29 02:01:43 -0700161 * Expand the file descriptor table.
162 * This function will allocate a new fdtable and both fd array and fdset, of
163 * the given size.
164 * Return <0 error code on error; 1 on successful completion.
165 * The files->file_lock should be held on entry, and will be held on exit.
Dipankar Sarmaab2af1f2005-09-09 13:04:13 -0700166 */
Alexey Dobriyan9b80a182016-09-02 00:38:52 +0300167static int expand_fdtable(struct files_struct *files, unsigned int nr)
Dipankar Sarmaab2af1f2005-09-09 13:04:13 -0700168 __releases(files->file_lock)
169 __acquires(files->file_lock)
170{
Vadim Lobanov74d392a2006-09-29 02:01:43 -0700171 struct fdtable *new_fdt, *cur_fdt;
Dipankar Sarmaab2af1f2005-09-09 13:04:13 -0700172
Linus Torvalds1da177e2005-04-16 15:20:36 -0700173 spin_unlock(&files->file_lock);
Vadim Lobanov74d392a2006-09-29 02:01:43 -0700174 new_fdt = alloc_fdtable(nr);
Eric Dumazet8a81252b2015-06-30 15:54:08 +0200175
176 /* make sure all __fd_install() have seen resize_in_progress
177 * or have finished their rcu_read_lock_sched() section.
178 */
179 if (atomic_read(&files->count) > 1)
Paul E. McKenneyc93ffc12018-11-05 17:31:31 -0800180 synchronize_rcu();
Eric Dumazet8a81252b2015-06-30 15:54:08 +0200181
Linus Torvalds1da177e2005-04-16 15:20:36 -0700182 spin_lock(&files->file_lock);
Vadim Lobanov74d392a2006-09-29 02:01:43 -0700183 if (!new_fdt)
184 return -ENOMEM;
Dipankar Sarmaab2af1f2005-09-09 13:04:13 -0700185 /*
Al Viro5c598b32008-04-27 20:04:15 -0400186 * extremely unlikely race - sysctl_nr_open decreased between the check in
187 * caller and alloc_fdtable(). Cheaper to catch it here...
188 */
189 if (unlikely(new_fdt->max_fds <= nr)) {
Changli Gaoa892e2d2010-08-10 18:01:35 -0700190 __free_fdtable(new_fdt);
Al Viro5c598b32008-04-27 20:04:15 -0400191 return -EMFILE;
192 }
Vadim Lobanov74d392a2006-09-29 02:01:43 -0700193 cur_fdt = files_fdtable(files);
Eric Dumazet8a81252b2015-06-30 15:54:08 +0200194 BUG_ON(nr < cur_fdt->max_fds);
195 copy_fdtable(new_fdt, cur_fdt);
196 rcu_assign_pointer(files->fdt, new_fdt);
197 if (cur_fdt != &files->fdtab)
198 call_rcu(&cur_fdt->rcu, free_fdtable_rcu);
199 /* coupled with smp_rmb() in __fd_install() */
200 smp_wmb();
Vadim Lobanov74d392a2006-09-29 02:01:43 -0700201 return 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700202}
203
204/*
205 * Expand files.
Vadim Lobanov74d392a2006-09-29 02:01:43 -0700206 * This function will expand the file structures, if the requested size exceeds
207 * the current capacity and there is room for expansion.
208 * Return <0 error code on error; 0 when nothing done; 1 when files were
209 * expanded and execution may have blocked.
210 * The files->file_lock should be held on entry, and will be held on exit.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700211 */
Alexey Dobriyan9b80a182016-09-02 00:38:52 +0300212static int expand_files(struct files_struct *files, unsigned int nr)
Eric Dumazet8a81252b2015-06-30 15:54:08 +0200213 __releases(files->file_lock)
214 __acquires(files->file_lock)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700215{
Dipankar Sarmabadf1662005-09-09 13:04:10 -0700216 struct fdtable *fdt;
Eric Dumazet8a81252b2015-06-30 15:54:08 +0200217 int expanded = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700218
Eric Dumazet8a81252b2015-06-30 15:54:08 +0200219repeat:
Dipankar Sarmabadf1662005-09-09 13:04:10 -0700220 fdt = files_fdtable(files);
Al Viro4e1e0182008-07-26 16:01:20 -0400221
Vadim Lobanov74d392a2006-09-29 02:01:43 -0700222 /* Do we need to expand? */
Vadim Lobanovbbea9f62006-12-10 02:21:12 -0800223 if (nr < fdt->max_fds)
Eric Dumazet8a81252b2015-06-30 15:54:08 +0200224 return expanded;
Al Viro4e1e0182008-07-26 16:01:20 -0400225
Vadim Lobanov74d392a2006-09-29 02:01:43 -0700226 /* Can we expand? */
Eric Dumazet9cfe0152008-02-06 01:37:16 -0800227 if (nr >= sysctl_nr_open)
Vadim Lobanov74d392a2006-09-29 02:01:43 -0700228 return -EMFILE;
229
Eric Dumazet8a81252b2015-06-30 15:54:08 +0200230 if (unlikely(files->resize_in_progress)) {
231 spin_unlock(&files->file_lock);
232 expanded = 1;
233 wait_event(files->resize_wait, !files->resize_in_progress);
234 spin_lock(&files->file_lock);
235 goto repeat;
236 }
237
Vadim Lobanov74d392a2006-09-29 02:01:43 -0700238 /* All good, so we try */
Eric Dumazet8a81252b2015-06-30 15:54:08 +0200239 files->resize_in_progress = true;
240 expanded = expand_fdtable(files, nr);
241 files->resize_in_progress = false;
242
243 wake_up_all(&files->resize_wait);
244 return expanded;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700245}
Dipankar Sarmaab2af1f2005-09-09 13:04:13 -0700246
Alexey Dobriyan9b80a182016-09-02 00:38:52 +0300247static inline void __set_close_on_exec(unsigned int fd, struct fdtable *fdt)
Al Virob8318b02012-08-21 20:09:42 -0400248{
249 __set_bit(fd, fdt->close_on_exec);
250}
251
Alexey Dobriyan9b80a182016-09-02 00:38:52 +0300252static inline void __clear_close_on_exec(unsigned int fd, struct fdtable *fdt)
Al Virob8318b02012-08-21 20:09:42 -0400253{
Linus Torvaldsfc908882015-10-31 16:06:40 -0700254 if (test_bit(fd, fdt->close_on_exec))
255 __clear_bit(fd, fdt->close_on_exec);
Al Virob8318b02012-08-21 20:09:42 -0400256}
257
Linus Torvaldsf3f86e32015-10-30 16:53:57 -0700258static inline void __set_open_fd(unsigned int fd, struct fdtable *fdt)
Al Virob8318b02012-08-21 20:09:42 -0400259{
260 __set_bit(fd, fdt->open_fds);
Linus Torvaldsf3f86e32015-10-30 16:53:57 -0700261 fd /= BITS_PER_LONG;
262 if (!~fdt->open_fds[fd])
263 __set_bit(fd, fdt->full_fds_bits);
Al Virob8318b02012-08-21 20:09:42 -0400264}
265
Linus Torvaldsf3f86e32015-10-30 16:53:57 -0700266static inline void __clear_open_fd(unsigned int fd, struct fdtable *fdt)
Al Virob8318b02012-08-21 20:09:42 -0400267{
268 __clear_bit(fd, fdt->open_fds);
Linus Torvaldsf3f86e32015-10-30 16:53:57 -0700269 __clear_bit(fd / BITS_PER_LONG, fdt->full_fds_bits);
Al Virob8318b02012-08-21 20:09:42 -0400270}
271
Alexey Dobriyan9b80a182016-09-02 00:38:52 +0300272static unsigned int count_open_files(struct fdtable *fdt)
Al Viro02afc6262008-05-08 19:42:56 -0400273{
Alexey Dobriyan9b80a182016-09-02 00:38:52 +0300274 unsigned int size = fdt->max_fds;
275 unsigned int i;
Al Viro02afc6262008-05-08 19:42:56 -0400276
277 /* Find the last open fd */
David Howells1fd36ad2012-02-16 17:49:54 +0000278 for (i = size / BITS_PER_LONG; i > 0; ) {
279 if (fdt->open_fds[--i])
Al Viro02afc6262008-05-08 19:42:56 -0400280 break;
281 }
David Howells1fd36ad2012-02-16 17:49:54 +0000282 i = (i + 1) * BITS_PER_LONG;
Al Viro02afc6262008-05-08 19:42:56 -0400283 return i;
284}
285
Linus Torvalds00d2b9f2022-03-29 15:06:39 -0700286/*
287 * Note that a sane fdtable size always has to be a multiple of
288 * BITS_PER_LONG, since we have bitmaps that are sized by this.
289 *
290 * 'max_fds' will normally already be properly aligned, but it
291 * turns out that in the close_range() -> __close_range() ->
292 * unshare_fd() -> dup_fd() -> sane_fdtable_size() we can end
293 * up having a 'max_fds' value that isn't already aligned.
294 *
295 * Rather than make close_range() have to worry about this,
296 * just make that BITS_PER_LONG alignment be part of a sane
297 * fdtable size. Becuase that's really what it is.
298 */
Christian Brauner60997c32020-06-03 21:48:55 +0200299static unsigned int sane_fdtable_size(struct fdtable *fdt, unsigned int max_fds)
300{
301 unsigned int count;
302
303 count = count_open_files(fdt);
Linus Torvalds00d2b9f2022-03-29 15:06:39 -0700304 max_fds = ALIGN(max_fds, BITS_PER_LONG);
Christian Brauner60997c32020-06-03 21:48:55 +0200305 if (max_fds < NR_OPEN_DEFAULT)
306 max_fds = NR_OPEN_DEFAULT;
307 return min(count, max_fds);
308}
309
Al Viro02afc6262008-05-08 19:42:56 -0400310/*
311 * Allocate a new files structure and copy contents from the
312 * passed in files structure.
313 * errorp will be valid only when the returned files_struct is NULL.
314 */
Christian Brauner60997c32020-06-03 21:48:55 +0200315struct files_struct *dup_fd(struct files_struct *oldf, unsigned int max_fds, int *errorp)
Al Viro02afc6262008-05-08 19:42:56 -0400316{
317 struct files_struct *newf;
318 struct file **old_fds, **new_fds;
Alexey Dobriyan9b80a182016-09-02 00:38:52 +0300319 unsigned int open_files, i;
Al Viro02afc6262008-05-08 19:42:56 -0400320 struct fdtable *old_fdt, *new_fdt;
321
322 *errorp = -ENOMEM;
Al Viroafbec7f2008-05-08 21:11:17 -0400323 newf = kmem_cache_alloc(files_cachep, GFP_KERNEL);
Al Viro02afc6262008-05-08 19:42:56 -0400324 if (!newf)
325 goto out;
326
Al Viroafbec7f2008-05-08 21:11:17 -0400327 atomic_set(&newf->count, 1);
328
329 spin_lock_init(&newf->file_lock);
Eric Dumazet8a81252b2015-06-30 15:54:08 +0200330 newf->resize_in_progress = false;
331 init_waitqueue_head(&newf->resize_wait);
Al Viroafbec7f2008-05-08 21:11:17 -0400332 newf->next_fd = 0;
333 new_fdt = &newf->fdtab;
334 new_fdt->max_fds = NR_OPEN_DEFAULT;
David Howells1fd36ad2012-02-16 17:49:54 +0000335 new_fdt->close_on_exec = newf->close_on_exec_init;
336 new_fdt->open_fds = newf->open_fds_init;
Linus Torvaldsf3f86e32015-10-30 16:53:57 -0700337 new_fdt->full_fds_bits = newf->full_fds_bits_init;
Al Viroafbec7f2008-05-08 21:11:17 -0400338 new_fdt->fd = &newf->fd_array[0];
Al Viroafbec7f2008-05-08 21:11:17 -0400339
Al Viro02afc6262008-05-08 19:42:56 -0400340 spin_lock(&oldf->file_lock);
341 old_fdt = files_fdtable(oldf);
Christian Brauner60997c32020-06-03 21:48:55 +0200342 open_files = sane_fdtable_size(old_fdt, max_fds);
Al Viro02afc6262008-05-08 19:42:56 -0400343
344 /*
345 * Check whether we need to allocate a larger fd array and fd set.
Al Viro02afc6262008-05-08 19:42:56 -0400346 */
Al Viroadbecb12008-05-08 21:19:42 -0400347 while (unlikely(open_files > new_fdt->max_fds)) {
Al Viro02afc6262008-05-08 19:42:56 -0400348 spin_unlock(&oldf->file_lock);
Al Viro9dec3c42008-05-08 21:02:45 -0400349
Changli Gaoa892e2d2010-08-10 18:01:35 -0700350 if (new_fdt != &newf->fdtab)
351 __free_fdtable(new_fdt);
Al Viroadbecb12008-05-08 21:19:42 -0400352
Al Viro9dec3c42008-05-08 21:02:45 -0400353 new_fdt = alloc_fdtable(open_files - 1);
354 if (!new_fdt) {
355 *errorp = -ENOMEM;
Al Viro02afc6262008-05-08 19:42:56 -0400356 goto out_release;
Al Viro9dec3c42008-05-08 21:02:45 -0400357 }
358
359 /* beyond sysctl_nr_open; nothing to do */
360 if (unlikely(new_fdt->max_fds < open_files)) {
Changli Gaoa892e2d2010-08-10 18:01:35 -0700361 __free_fdtable(new_fdt);
Al Viro9dec3c42008-05-08 21:02:45 -0400362 *errorp = -EMFILE;
363 goto out_release;
364 }
Al Viro9dec3c42008-05-08 21:02:45 -0400365
Al Viro02afc6262008-05-08 19:42:56 -0400366 /*
367 * Reacquire the oldf lock and a pointer to its fd table
368 * who knows it may have a new bigger fd table. We need
369 * the latest pointer.
370 */
371 spin_lock(&oldf->file_lock);
372 old_fdt = files_fdtable(oldf);
Christian Brauner60997c32020-06-03 21:48:55 +0200373 open_files = sane_fdtable_size(old_fdt, max_fds);
Al Viro02afc6262008-05-08 19:42:56 -0400374 }
375
Eric Biggersea5c58e2015-11-06 00:32:04 -0600376 copy_fd_bitmaps(new_fdt, old_fdt, open_files);
377
Al Viro02afc6262008-05-08 19:42:56 -0400378 old_fds = old_fdt->fd;
379 new_fds = new_fdt->fd;
380
Al Viro02afc6262008-05-08 19:42:56 -0400381 for (i = open_files; i != 0; i--) {
382 struct file *f = *old_fds++;
383 if (f) {
384 get_file(f);
385 } else {
386 /*
387 * The fd may be claimed in the fd bitmap but not yet
388 * instantiated in the files array if a sibling thread
389 * is partway through open(). So make sure that this
390 * fd is available to the new process.
391 */
David Howells1dce27c2012-02-16 17:49:42 +0000392 __clear_open_fd(open_files - i, new_fdt);
Al Viro02afc6262008-05-08 19:42:56 -0400393 }
394 rcu_assign_pointer(*new_fds++, f);
395 }
396 spin_unlock(&oldf->file_lock);
397
Eric Biggersea5c58e2015-11-06 00:32:04 -0600398 /* clear the remainder */
399 memset(new_fds, 0, (new_fdt->max_fds - open_files) * sizeof(struct file *));
Al Viro02afc6262008-05-08 19:42:56 -0400400
Al Viroafbec7f2008-05-08 21:11:17 -0400401 rcu_assign_pointer(newf->fdt, new_fdt);
402
Al Viro02afc6262008-05-08 19:42:56 -0400403 return newf;
404
405out_release:
406 kmem_cache_free(files_cachep, newf);
407out:
408 return NULL;
409}
410
Oleg Nesterovce08b622014-01-11 19:19:53 +0100411static struct fdtable *close_files(struct files_struct * files)
Al Viro7cf4dc32012-08-15 19:56:12 -0400412{
Al Viro7cf4dc32012-08-15 19:56:12 -0400413 /*
414 * It is safe to dereference the fd table without RCU or
415 * ->file_lock because this is the last reference to the
Oleg Nesterovce08b622014-01-11 19:19:53 +0100416 * files structure.
Al Viro7cf4dc32012-08-15 19:56:12 -0400417 */
Oleg Nesterovce08b622014-01-11 19:19:53 +0100418 struct fdtable *fdt = rcu_dereference_raw(files->fdt);
Alexey Dobriyan9b80a182016-09-02 00:38:52 +0300419 unsigned int i, j = 0;
Oleg Nesterovce08b622014-01-11 19:19:53 +0100420
Al Viro7cf4dc32012-08-15 19:56:12 -0400421 for (;;) {
422 unsigned long set;
423 i = j * BITS_PER_LONG;
424 if (i >= fdt->max_fds)
425 break;
426 set = fdt->open_fds[j++];
427 while (set) {
428 if (set & 1) {
429 struct file * file = xchg(&fdt->fd[i], NULL);
430 if (file) {
431 filp_close(file, files);
Paul E. McKenney388a4c82017-10-24 08:39:34 -0700432 cond_resched();
Al Viro7cf4dc32012-08-15 19:56:12 -0400433 }
434 }
435 i++;
436 set >>= 1;
437 }
438 }
Oleg Nesterovce08b622014-01-11 19:19:53 +0100439
440 return fdt;
Al Viro7cf4dc32012-08-15 19:56:12 -0400441}
442
443struct files_struct *get_files_struct(struct task_struct *task)
444{
445 struct files_struct *files;
446
447 task_lock(task);
448 files = task->files;
449 if (files)
450 atomic_inc(&files->count);
451 task_unlock(task);
452
453 return files;
454}
455
456void put_files_struct(struct files_struct *files)
457{
Al Viro7cf4dc32012-08-15 19:56:12 -0400458 if (atomic_dec_and_test(&files->count)) {
Oleg Nesterovce08b622014-01-11 19:19:53 +0100459 struct fdtable *fdt = close_files(files);
460
Al Virob9e02af2012-08-15 20:00:58 -0400461 /* free the arrays if they are not embedded */
462 if (fdt != &files->fdtab)
463 __free_fdtable(fdt);
464 kmem_cache_free(files_cachep, files);
Al Viro7cf4dc32012-08-15 19:56:12 -0400465 }
466}
467
468void reset_files_struct(struct files_struct *files)
469{
470 struct task_struct *tsk = current;
471 struct files_struct *old;
472
473 old = tsk->files;
474 task_lock(tsk);
475 tsk->files = files;
476 task_unlock(tsk);
477 put_files_struct(old);
478}
479
480void exit_files(struct task_struct *tsk)
481{
482 struct files_struct * files = tsk->files;
483
484 if (files) {
485 task_lock(tsk);
486 tsk->files = NULL;
487 task_unlock(tsk);
488 put_files_struct(files);
489 }
490}
491
Al Virof52111b2008-05-08 18:19:16 -0400492struct files_struct init_files = {
493 .count = ATOMIC_INIT(1),
494 .fdt = &init_files.fdtab,
495 .fdtab = {
496 .max_fds = NR_OPEN_DEFAULT,
497 .fd = &init_files.fd_array[0],
David Howells1fd36ad2012-02-16 17:49:54 +0000498 .close_on_exec = init_files.close_on_exec_init,
499 .open_fds = init_files.open_fds_init,
Linus Torvaldsf3f86e32015-10-30 16:53:57 -0700500 .full_fds_bits = init_files.full_fds_bits_init,
Al Virof52111b2008-05-08 18:19:16 -0400501 },
Thomas Gleixnereece09e2011-07-17 21:25:03 +0200502 .file_lock = __SPIN_LOCK_UNLOCKED(init_files.file_lock),
Shuriyc Chu5704a062019-03-05 15:41:56 -0800503 .resize_wait = __WAIT_QUEUE_HEAD_INITIALIZER(init_files.resize_wait),
Al Virof52111b2008-05-08 18:19:16 -0400504};
Al Viro1027abe2008-07-30 04:13:04 -0400505
Alexey Dobriyan9b80a182016-09-02 00:38:52 +0300506static unsigned int find_next_fd(struct fdtable *fdt, unsigned int start)
Linus Torvaldsf3f86e32015-10-30 16:53:57 -0700507{
Alexey Dobriyan9b80a182016-09-02 00:38:52 +0300508 unsigned int maxfd = fdt->max_fds;
509 unsigned int maxbit = maxfd / BITS_PER_LONG;
510 unsigned int bitbit = start / BITS_PER_LONG;
Linus Torvaldsf3f86e32015-10-30 16:53:57 -0700511
512 bitbit = find_next_zero_bit(fdt->full_fds_bits, maxbit, bitbit) * BITS_PER_LONG;
513 if (bitbit > maxfd)
514 return maxfd;
515 if (bitbit > start)
516 start = bitbit;
517 return find_next_zero_bit(fdt->open_fds, maxfd, start);
518}
519
Al Viro1027abe2008-07-30 04:13:04 -0400520/*
521 * allocate a file descriptor, mark it busy.
522 */
Al Virodcfadfa2012-08-12 17:27:30 -0400523int __alloc_fd(struct files_struct *files,
524 unsigned start, unsigned end, unsigned flags)
Al Viro1027abe2008-07-30 04:13:04 -0400525{
Al Viro1027abe2008-07-30 04:13:04 -0400526 unsigned int fd;
527 int error;
528 struct fdtable *fdt;
529
530 spin_lock(&files->file_lock);
531repeat:
532 fdt = files_fdtable(files);
533 fd = start;
534 if (fd < files->next_fd)
535 fd = files->next_fd;
536
537 if (fd < fdt->max_fds)
Linus Torvaldsf3f86e32015-10-30 16:53:57 -0700538 fd = find_next_fd(fdt, fd);
Al Viro1027abe2008-07-30 04:13:04 -0400539
Al Virof33ff992012-08-12 16:17:59 -0400540 /*
541 * N.B. For clone tasks sharing a files structure, this test
542 * will limit the total number of files that can be opened.
543 */
544 error = -EMFILE;
545 if (fd >= end)
546 goto out;
547
Al Viro1027abe2008-07-30 04:13:04 -0400548 error = expand_files(files, fd);
549 if (error < 0)
550 goto out;
551
552 /*
553 * If we needed to expand the fs array we
554 * might have blocked - try again.
555 */
556 if (error)
557 goto repeat;
558
559 if (start <= files->next_fd)
560 files->next_fd = fd + 1;
561
David Howells1dce27c2012-02-16 17:49:42 +0000562 __set_open_fd(fd, fdt);
Al Viro1027abe2008-07-30 04:13:04 -0400563 if (flags & O_CLOEXEC)
David Howells1dce27c2012-02-16 17:49:42 +0000564 __set_close_on_exec(fd, fdt);
Al Viro1027abe2008-07-30 04:13:04 -0400565 else
David Howells1dce27c2012-02-16 17:49:42 +0000566 __clear_close_on_exec(fd, fdt);
Al Viro1027abe2008-07-30 04:13:04 -0400567 error = fd;
568#if 1
569 /* Sanity check */
Paul E. McKenneyadd1f092014-02-12 12:51:09 -0800570 if (rcu_access_pointer(fdt->fd[fd]) != NULL) {
Al Viro1027abe2008-07-30 04:13:04 -0400571 printk(KERN_WARNING "alloc_fd: slot %d not NULL!\n", fd);
572 rcu_assign_pointer(fdt->fd[fd], NULL);
573 }
574#endif
575
576out:
577 spin_unlock(&files->file_lock);
578 return error;
579}
580
Al Viroad47bd72012-08-21 20:11:34 -0400581static int alloc_fd(unsigned start, unsigned flags)
Al Virodcfadfa2012-08-12 17:27:30 -0400582{
583 return __alloc_fd(current->files, start, rlimit(RLIMIT_NOFILE), flags);
584}
585
Jens Axboe4022e7a2020-03-19 19:23:18 -0600586int __get_unused_fd_flags(unsigned flags, unsigned long nofile)
587{
588 return __alloc_fd(current->files, 0, nofile, flags);
589}
590
Al Viro1a7bd222012-08-12 17:18:05 -0400591int get_unused_fd_flags(unsigned flags)
Al Viro1027abe2008-07-30 04:13:04 -0400592{
Jens Axboe4022e7a2020-03-19 19:23:18 -0600593 return __get_unused_fd_flags(flags, rlimit(RLIMIT_NOFILE));
Al Viro1027abe2008-07-30 04:13:04 -0400594}
Al Viro1a7bd222012-08-12 17:18:05 -0400595EXPORT_SYMBOL(get_unused_fd_flags);
Al Viro56007ca2012-08-15 21:03:26 -0400596
597static void __put_unused_fd(struct files_struct *files, unsigned int fd)
598{
599 struct fdtable *fdt = files_fdtable(files);
600 __clear_open_fd(fd, fdt);
601 if (fd < files->next_fd)
602 files->next_fd = fd;
603}
604
605void put_unused_fd(unsigned int fd)
606{
607 struct files_struct *files = current->files;
608 spin_lock(&files->file_lock);
609 __put_unused_fd(files, fd);
610 spin_unlock(&files->file_lock);
611}
612
613EXPORT_SYMBOL(put_unused_fd);
614
615/*
616 * Install a file pointer in the fd array.
617 *
618 * The VFS is full of places where we drop the files lock between
619 * setting the open_fds bitmap and installing the file in the file
620 * array. At any such point, we are vulnerable to a dup2() race
621 * installing a file in the array before us. We need to detect this and
622 * fput() the struct file we are about to overwrite in this case.
623 *
624 * It should never happen - if we allow dup2() do it, _really_ bad things
625 * will follow.
Al Virof869e8a2012-08-15 21:06:33 -0400626 *
627 * NOTE: __fd_install() variant is really, really low-level; don't
628 * use it unless you are forced to by truly lousy API shoved down
629 * your throat. 'files' *MUST* be either current->files or obtained
630 * by get_files_struct(current) done by whoever had given it to you,
631 * or really bad things will happen. Normally you want to use
632 * fd_install() instead.
Al Viro56007ca2012-08-15 21:03:26 -0400633 */
634
Al Virof869e8a2012-08-15 21:06:33 -0400635void __fd_install(struct files_struct *files, unsigned int fd,
636 struct file *file)
Al Viro56007ca2012-08-15 21:03:26 -0400637{
Al Viro56007ca2012-08-15 21:03:26 -0400638 struct fdtable *fdt;
Eric Dumazet8a81252b2015-06-30 15:54:08 +0200639
Eric Dumazet8a81252b2015-06-30 15:54:08 +0200640 rcu_read_lock_sched();
641
Mateusz Guzikc02b1a92017-10-03 12:58:15 +0200642 if (unlikely(files->resize_in_progress)) {
Eric Dumazet8a81252b2015-06-30 15:54:08 +0200643 rcu_read_unlock_sched();
Mateusz Guzikc02b1a92017-10-03 12:58:15 +0200644 spin_lock(&files->file_lock);
645 fdt = files_fdtable(files);
646 BUG_ON(fdt->fd[fd] != NULL);
647 rcu_assign_pointer(fdt->fd[fd], file);
648 spin_unlock(&files->file_lock);
649 return;
Eric Dumazet8a81252b2015-06-30 15:54:08 +0200650 }
651 /* coupled with smp_wmb() in expand_fdtable() */
652 smp_rmb();
653 fdt = rcu_dereference_sched(files->fdt);
Al Viro56007ca2012-08-15 21:03:26 -0400654 BUG_ON(fdt->fd[fd] != NULL);
655 rcu_assign_pointer(fdt->fd[fd], file);
Eric Dumazet8a81252b2015-06-30 15:54:08 +0200656 rcu_read_unlock_sched();
Al Viro56007ca2012-08-15 21:03:26 -0400657}
658
Kees Cook66590612020-06-10 08:20:05 -0700659/*
660 * This consumes the "file" refcount, so callers should treat it
661 * as if they had called fput(file).
662 */
Al Virof869e8a2012-08-15 21:06:33 -0400663void fd_install(unsigned int fd, struct file *file)
664{
665 __fd_install(current->files, fd, file);
666}
667
Al Viro56007ca2012-08-15 21:03:26 -0400668EXPORT_SYMBOL(fd_install);
Al Viro0ee8cdf2012-08-15 21:12:10 -0400669
Christian Brauner278a5fb2019-05-24 11:30:34 +0200670static struct file *pick_file(struct files_struct *files, unsigned fd)
Al Viro483ce1d2012-08-19 12:04:24 -0400671{
Christian Brauner278a5fb2019-05-24 11:30:34 +0200672 struct file *file = NULL;
Al Viro483ce1d2012-08-19 12:04:24 -0400673 struct fdtable *fdt;
674
675 spin_lock(&files->file_lock);
676 fdt = files_fdtable(files);
677 if (fd >= fdt->max_fds)
678 goto out_unlock;
679 file = fdt->fd[fd];
680 if (!file)
681 goto out_unlock;
682 rcu_assign_pointer(fdt->fd[fd], NULL);
Al Viro483ce1d2012-08-19 12:04:24 -0400683 __put_unused_fd(files, fd);
Al Viro483ce1d2012-08-19 12:04:24 -0400684
685out_unlock:
686 spin_unlock(&files->file_lock);
Christian Brauner278a5fb2019-05-24 11:30:34 +0200687 return file;
688}
689
690/*
691 * The same warnings as for __alloc_fd()/__fd_install() apply here...
692 */
693int __close_fd(struct files_struct *files, unsigned fd)
694{
695 struct file *file;
696
697 file = pick_file(files, fd);
698 if (!file)
699 return -EBADF;
700
701 return filp_close(file, files);
Al Viro483ce1d2012-08-19 12:04:24 -0400702}
Dominik Brodowski2ca2a09d62018-03-11 11:34:55 +0100703EXPORT_SYMBOL(__close_fd); /* for ksys_close() */
Al Viro483ce1d2012-08-19 12:04:24 -0400704
Christian Brauner278a5fb2019-05-24 11:30:34 +0200705/**
706 * __close_range() - Close all file descriptors in a given range.
707 *
708 * @fd: starting file descriptor to close
709 * @max_fd: last file descriptor to close
710 *
711 * This closes a range of file descriptors. All file descriptors
712 * from @fd up to and including @max_fd are closed.
713 */
Christian Brauner60997c32020-06-03 21:48:55 +0200714int __close_range(unsigned fd, unsigned max_fd, unsigned int flags)
Christian Brauner278a5fb2019-05-24 11:30:34 +0200715{
716 unsigned int cur_max;
Christian Brauner60997c32020-06-03 21:48:55 +0200717 struct task_struct *me = current;
718 struct files_struct *cur_fds = me->files, *fds = NULL;
719
720 if (flags & ~CLOSE_RANGE_UNSHARE)
721 return -EINVAL;
Christian Brauner278a5fb2019-05-24 11:30:34 +0200722
723 if (fd > max_fd)
724 return -EINVAL;
725
726 rcu_read_lock();
Christian Brauner60997c32020-06-03 21:48:55 +0200727 cur_max = files_fdtable(cur_fds)->max_fds;
Christian Brauner278a5fb2019-05-24 11:30:34 +0200728 rcu_read_unlock();
729
730 /* cap to last valid index into fdtable */
731 cur_max--;
732
Christian Brauner60997c32020-06-03 21:48:55 +0200733 if (flags & CLOSE_RANGE_UNSHARE) {
734 int ret;
735 unsigned int max_unshare_fds = NR_OPEN_MAX;
736
737 /*
738 * If the requested range is greater than the current maximum,
739 * we're closing everything so only copy all file descriptors
740 * beneath the lowest file descriptor.
741 */
742 if (max_fd >= cur_max)
743 max_unshare_fds = fd;
744
745 ret = unshare_fd(CLONE_FILES, max_unshare_fds, &fds);
746 if (ret)
747 return ret;
748
749 /*
750 * We used to share our file descriptor table, and have now
751 * created a private one, make sure we're using it below.
752 */
753 if (fds)
754 swap(cur_fds, fds);
755 }
756
Christian Brauner278a5fb2019-05-24 11:30:34 +0200757 max_fd = min(max_fd, cur_max);
758 while (fd <= max_fd) {
759 struct file *file;
760
Christian Brauner60997c32020-06-03 21:48:55 +0200761 file = pick_file(cur_fds, fd++);
Christian Brauner278a5fb2019-05-24 11:30:34 +0200762 if (!file)
763 continue;
764
Christian Brauner60997c32020-06-03 21:48:55 +0200765 filp_close(file, cur_fds);
Christian Brauner278a5fb2019-05-24 11:30:34 +0200766 cond_resched();
767 }
768
Christian Brauner60997c32020-06-03 21:48:55 +0200769 if (fds) {
770 /*
771 * We're done closing the files we were supposed to. Time to install
772 * the new file descriptor table and drop the old one.
773 */
774 task_lock(me);
775 me->files = cur_fds;
776 task_unlock(me);
777 put_files_struct(fds);
778 }
779
Christian Brauner278a5fb2019-05-24 11:30:34 +0200780 return 0;
781}
782
Todd Kjos80cd7952018-12-14 15:58:21 -0800783/*
Jens Axboe6e802a42019-12-11 14:10:35 -0700784 * variant of __close_fd that gets a ref on the file for later fput.
785 * The caller must ensure that filp_close() called on the file, and then
786 * an fput().
Todd Kjos80cd7952018-12-14 15:58:21 -0800787 */
788int __close_fd_get_file(unsigned int fd, struct file **res)
789{
790 struct files_struct *files = current->files;
791 struct file *file;
792 struct fdtable *fdt;
793
794 spin_lock(&files->file_lock);
795 fdt = files_fdtable(files);
796 if (fd >= fdt->max_fds)
797 goto out_unlock;
798 file = fdt->fd[fd];
799 if (!file)
800 goto out_unlock;
801 rcu_assign_pointer(fdt->fd[fd], NULL);
802 __put_unused_fd(files, fd);
803 spin_unlock(&files->file_lock);
804 get_file(file);
805 *res = file;
Jens Axboe6e802a42019-12-11 14:10:35 -0700806 return 0;
Todd Kjos80cd7952018-12-14 15:58:21 -0800807
808out_unlock:
809 spin_unlock(&files->file_lock);
810 *res = NULL;
811 return -ENOENT;
812}
813
Al Viro6a6d27d2012-08-21 09:56:33 -0400814void do_close_on_exec(struct files_struct *files)
815{
816 unsigned i;
817 struct fdtable *fdt;
818
819 /* exec unshares first */
Al Viro6a6d27d2012-08-21 09:56:33 -0400820 spin_lock(&files->file_lock);
821 for (i = 0; ; i++) {
822 unsigned long set;
823 unsigned fd = i * BITS_PER_LONG;
824 fdt = files_fdtable(files);
825 if (fd >= fdt->max_fds)
826 break;
827 set = fdt->close_on_exec[i];
828 if (!set)
829 continue;
830 fdt->close_on_exec[i] = 0;
831 for ( ; set ; fd++, set >>= 1) {
832 struct file *file;
833 if (!(set & 1))
834 continue;
835 file = fdt->fd[fd];
836 if (!file)
837 continue;
838 rcu_assign_pointer(fdt->fd[fd], NULL);
839 __put_unused_fd(files, fd);
840 spin_unlock(&files->file_lock);
841 filp_close(file, files);
842 cond_resched();
843 spin_lock(&files->file_lock);
844 }
845
846 }
847 spin_unlock(&files->file_lock);
848}
849
Linus Torvalds0849f832021-12-10 14:00:15 -0800850static inline struct file *__fget_files_rcu(struct files_struct *files,
851 unsigned int fd, fmode_t mask, unsigned int refs)
852{
853 for (;;) {
854 struct file *file;
855 struct fdtable *fdt = rcu_dereference_raw(files->fdt);
856 struct file __rcu **fdentry;
857
858 if (unlikely(fd >= fdt->max_fds))
859 return NULL;
860
861 fdentry = fdt->fd + array_index_nospec(fd, fdt->max_fds);
862 file = rcu_dereference_raw(*fdentry);
863 if (unlikely(!file))
864 return NULL;
865
866 if (unlikely(file->f_mode & mask))
867 return NULL;
868
869 /*
870 * Ok, we have a file pointer. However, because we do
871 * this all locklessly under RCU, we may be racing with
872 * that file being closed.
873 *
874 * Such a race can take two forms:
875 *
876 * (a) the file ref already went down to zero,
877 * and get_file_rcu_many() fails. Just try
878 * again:
879 */
880 if (unlikely(!get_file_rcu_many(file, refs)))
881 continue;
882
883 /*
884 * (b) the file table entry has changed under us.
885 * Note that we don't need to re-check the 'fdt->fd'
886 * pointer having changed, because it always goes
887 * hand-in-hand with 'fdt'.
888 *
889 * If so, we need to put our refs and try again.
890 */
891 if (unlikely(rcu_dereference_raw(files->fdt) != fdt) ||
892 unlikely(rcu_dereference_raw(*fdentry) != file)) {
893 fput_many(file, refs);
894 continue;
895 }
896
897 /*
898 * Ok, we have a ref to the file, and checked that it
899 * still exists.
900 */
901 return file;
902 }
903}
904
Sargun Dhillon5e876fb2020-01-07 09:59:24 -0800905static struct file *__fget_files(struct files_struct *files, unsigned int fd,
906 fmode_t mask, unsigned int refs)
Al Viro0ee8cdf2012-08-15 21:12:10 -0400907{
Oleg Nesterov1deb46e2014-01-13 16:48:19 +0100908 struct file *file;
Al Viro0ee8cdf2012-08-15 21:12:10 -0400909
910 rcu_read_lock();
Linus Torvalds0849f832021-12-10 14:00:15 -0800911 file = __fget_files_rcu(files, fd, mask, refs);
Al Viro0ee8cdf2012-08-15 21:12:10 -0400912 rcu_read_unlock();
913
914 return file;
915}
916
Sargun Dhillon5e876fb2020-01-07 09:59:24 -0800917static inline struct file *__fget(unsigned int fd, fmode_t mask,
918 unsigned int refs)
919{
920 return __fget_files(current->files, fd, mask, refs);
921}
922
Jens Axboe091141a2018-11-21 10:32:39 -0700923struct file *fget_many(unsigned int fd, unsigned int refs)
924{
925 return __fget(fd, FMODE_PATH, refs);
926}
927
Oleg Nesterov1deb46e2014-01-13 16:48:19 +0100928struct file *fget(unsigned int fd)
929{
Jens Axboe091141a2018-11-21 10:32:39 -0700930 return __fget(fd, FMODE_PATH, 1);
Oleg Nesterov1deb46e2014-01-13 16:48:19 +0100931}
Al Viro0ee8cdf2012-08-15 21:12:10 -0400932EXPORT_SYMBOL(fget);
933
934struct file *fget_raw(unsigned int fd)
935{
Jens Axboe091141a2018-11-21 10:32:39 -0700936 return __fget(fd, 0, 1);
Al Viro0ee8cdf2012-08-15 21:12:10 -0400937}
Al Viro0ee8cdf2012-08-15 21:12:10 -0400938EXPORT_SYMBOL(fget_raw);
939
Sargun Dhillon5e876fb2020-01-07 09:59:24 -0800940struct file *fget_task(struct task_struct *task, unsigned int fd)
941{
942 struct file *file = NULL;
943
944 task_lock(task);
945 if (task->files)
946 file = __fget_files(task->files, fd, 0, 1);
947 task_unlock(task);
948
949 return file;
950}
951
Al Viro0ee8cdf2012-08-15 21:12:10 -0400952/*
953 * Lightweight file lookup - no refcnt increment if fd table isn't shared.
954 *
955 * You can use this instead of fget if you satisfy all of the following
956 * conditions:
957 * 1) You must call fput_light before exiting the syscall and returning control
958 * to userspace (i.e. you cannot remember the returned struct file * after
959 * returning to userspace).
960 * 2) You must not call filp_close on the returned struct file * in between
961 * calls to fget_light and fput_light.
962 * 3) You must not clone the current task in between the calls to fget_light
963 * and fput_light.
964 *
965 * The fput_needed flag returned by fget_light should be passed to the
966 * corresponding fput_light.
967 */
Al Virobd2a31d2014-03-04 14:54:22 -0500968static unsigned long __fget_light(unsigned int fd, fmode_t mask)
Al Viro0ee8cdf2012-08-15 21:12:10 -0400969{
Al Viro0ee8cdf2012-08-15 21:12:10 -0400970 struct files_struct *files = current->files;
Oleg Nesterovad461832014-01-13 16:48:40 +0100971 struct file *file;
Al Viro0ee8cdf2012-08-15 21:12:10 -0400972
Al Viro0ee8cdf2012-08-15 21:12:10 -0400973 if (atomic_read(&files->count) == 1) {
Oleg Nesterova8d4b832014-01-11 19:19:32 +0100974 file = __fcheck_files(files, fd);
Al Virobd2a31d2014-03-04 14:54:22 -0500975 if (!file || unlikely(file->f_mode & mask))
976 return 0;
977 return (unsigned long)file;
Al Viro0ee8cdf2012-08-15 21:12:10 -0400978 } else {
Jens Axboe091141a2018-11-21 10:32:39 -0700979 file = __fget(fd, mask, 1);
Al Virobd2a31d2014-03-04 14:54:22 -0500980 if (!file)
981 return 0;
982 return FDPUT_FPUT | (unsigned long)file;
Al Viro0ee8cdf2012-08-15 21:12:10 -0400983 }
Al Viro0ee8cdf2012-08-15 21:12:10 -0400984}
Al Virobd2a31d2014-03-04 14:54:22 -0500985unsigned long __fdget(unsigned int fd)
Oleg Nesterovad461832014-01-13 16:48:40 +0100986{
Al Virobd2a31d2014-03-04 14:54:22 -0500987 return __fget_light(fd, FMODE_PATH);
Oleg Nesterovad461832014-01-13 16:48:40 +0100988}
Al Virobd2a31d2014-03-04 14:54:22 -0500989EXPORT_SYMBOL(__fdget);
Al Viro0ee8cdf2012-08-15 21:12:10 -0400990
Al Virobd2a31d2014-03-04 14:54:22 -0500991unsigned long __fdget_raw(unsigned int fd)
Al Viro0ee8cdf2012-08-15 21:12:10 -0400992{
Al Virobd2a31d2014-03-04 14:54:22 -0500993 return __fget_light(fd, 0);
Al Viro0ee8cdf2012-08-15 21:12:10 -0400994}
Al Virofe17f222012-08-21 11:48:11 -0400995
Al Virobd2a31d2014-03-04 14:54:22 -0500996unsigned long __fdget_pos(unsigned int fd)
997{
Eric Biggers99aea682014-03-16 15:47:48 -0500998 unsigned long v = __fdget(fd);
999 struct file *file = (struct file *)(v & ~3);
Al Virobd2a31d2014-03-04 14:54:22 -05001000
Linus Torvalds2be7d342019-11-26 11:34:06 -08001001 if (file && (file->f_mode & FMODE_ATOMIC_POS)) {
Al Virobd2a31d2014-03-04 14:54:22 -05001002 if (file_count(file) > 1) {
1003 v |= FDPUT_POS_UNLOCK;
1004 mutex_lock(&file->f_pos_lock);
1005 }
1006 }
Eric Biggers99aea682014-03-16 15:47:48 -05001007 return v;
Al Virobd2a31d2014-03-04 14:54:22 -05001008}
1009
Al Viro63b6df12016-04-20 17:08:21 -04001010void __f_unlock_pos(struct file *f)
1011{
1012 mutex_unlock(&f->f_pos_lock);
1013}
1014
Al Virobd2a31d2014-03-04 14:54:22 -05001015/*
1016 * We only lock f_pos if we have threads or if the file might be
1017 * shared with another process. In both cases we'll have an elevated
1018 * file count (done either by fdget() or by fork()).
1019 */
1020
Al Virofe17f222012-08-21 11:48:11 -04001021void set_close_on_exec(unsigned int fd, int flag)
1022{
1023 struct files_struct *files = current->files;
1024 struct fdtable *fdt;
1025 spin_lock(&files->file_lock);
1026 fdt = files_fdtable(files);
1027 if (flag)
1028 __set_close_on_exec(fd, fdt);
1029 else
1030 __clear_close_on_exec(fd, fdt);
1031 spin_unlock(&files->file_lock);
1032}
1033
1034bool get_close_on_exec(unsigned int fd)
1035{
1036 struct files_struct *files = current->files;
1037 struct fdtable *fdt;
1038 bool res;
1039 rcu_read_lock();
1040 fdt = files_fdtable(files);
1041 res = close_on_exec(fd, fdt);
1042 rcu_read_unlock();
1043 return res;
1044}
1045
Al Viro8280d162012-08-21 12:11:46 -04001046static int do_dup2(struct files_struct *files,
1047 struct file *file, unsigned fd, unsigned flags)
Al Viroe9830942014-08-31 14:12:09 -04001048__releases(&files->file_lock)
Al Viro8280d162012-08-21 12:11:46 -04001049{
1050 struct file *tofree;
1051 struct fdtable *fdt;
1052
1053 /*
1054 * We need to detect attempts to do dup2() over allocated but still
1055 * not finished descriptor. NB: OpenBSD avoids that at the price of
1056 * extra work in their equivalent of fget() - they insert struct
1057 * file immediately after grabbing descriptor, mark it larval if
1058 * more work (e.g. actual opening) is needed and make sure that
1059 * fget() treats larval files as absent. Potentially interesting,
1060 * but while extra work in fget() is trivial, locking implications
1061 * and amount of surgery on open()-related paths in VFS are not.
1062 * FreeBSD fails with -EBADF in the same situation, NetBSD "solution"
1063 * deadlocks in rather amusing ways, AFAICS. All of that is out of
1064 * scope of POSIX or SUS, since neither considers shared descriptor
1065 * tables and this condition does not arise without those.
1066 */
1067 fdt = files_fdtable(files);
1068 tofree = fdt->fd[fd];
1069 if (!tofree && fd_is_open(fd, fdt))
1070 goto Ebusy;
1071 get_file(file);
1072 rcu_assign_pointer(fdt->fd[fd], file);
1073 __set_open_fd(fd, fdt);
1074 if (flags & O_CLOEXEC)
1075 __set_close_on_exec(fd, fdt);
1076 else
1077 __clear_close_on_exec(fd, fdt);
1078 spin_unlock(&files->file_lock);
1079
1080 if (tofree)
1081 filp_close(tofree, files);
1082
1083 return fd;
1084
1085Ebusy:
1086 spin_unlock(&files->file_lock);
1087 return -EBUSY;
1088}
1089
1090int replace_fd(unsigned fd, struct file *file, unsigned flags)
1091{
1092 int err;
1093 struct files_struct *files = current->files;
1094
1095 if (!file)
1096 return __close_fd(files, fd);
1097
1098 if (fd >= rlimit(RLIMIT_NOFILE))
Al Viro08f05c42012-10-31 03:37:48 +00001099 return -EBADF;
Al Viro8280d162012-08-21 12:11:46 -04001100
1101 spin_lock(&files->file_lock);
1102 err = expand_files(files, fd);
1103 if (unlikely(err < 0))
1104 goto out_unlock;
1105 return do_dup2(files, file, fd, flags);
1106
1107out_unlock:
1108 spin_unlock(&files->file_lock);
1109 return err;
1110}
1111
Kees Cook66590612020-06-10 08:20:05 -07001112/**
1113 * __receive_fd() - Install received file into file descriptor table
1114 *
Kees Cook173817152020-06-10 08:46:58 -07001115 * @fd: fd to install into (if negative, a new fd will be allocated)
Kees Cook66590612020-06-10 08:20:05 -07001116 * @file: struct file that was received from another process
1117 * @ufd: __user pointer to write new fd number to
1118 * @o_flags: the O_* flags to apply to the new fd entry
1119 *
1120 * Installs a received file into the file descriptor table, with appropriate
Kees Cookdeefa7f2020-06-10 20:47:45 -07001121 * checks and count updates. Optionally writes the fd number to userspace, if
1122 * @ufd is non-NULL.
Kees Cook66590612020-06-10 08:20:05 -07001123 *
1124 * This helper handles its own reference counting of the incoming
1125 * struct file.
1126 *
Kees Cookdeefa7f2020-06-10 20:47:45 -07001127 * Returns newly install fd or -ve on error.
Kees Cook66590612020-06-10 08:20:05 -07001128 */
Kees Cook173817152020-06-10 08:46:58 -07001129int __receive_fd(int fd, struct file *file, int __user *ufd, unsigned int o_flags)
Kees Cook66590612020-06-10 08:20:05 -07001130{
1131 int new_fd;
1132 int error;
1133
1134 error = security_file_receive(file);
1135 if (error)
1136 return error;
1137
Kees Cook173817152020-06-10 08:46:58 -07001138 if (fd < 0) {
1139 new_fd = get_unused_fd_flags(o_flags);
1140 if (new_fd < 0)
1141 return new_fd;
1142 } else {
1143 new_fd = fd;
1144 }
Kees Cook66590612020-06-10 08:20:05 -07001145
Kees Cookdeefa7f2020-06-10 20:47:45 -07001146 if (ufd) {
1147 error = put_user(new_fd, ufd);
1148 if (error) {
Kees Cook173817152020-06-10 08:46:58 -07001149 if (fd < 0)
1150 put_unused_fd(new_fd);
Kees Cookdeefa7f2020-06-10 20:47:45 -07001151 return error;
1152 }
Kees Cook66590612020-06-10 08:20:05 -07001153 }
1154
Kees Cook173817152020-06-10 08:46:58 -07001155 if (fd < 0) {
1156 fd_install(new_fd, get_file(file));
1157 } else {
1158 error = replace_fd(new_fd, file, o_flags);
1159 if (error)
1160 return error;
1161 }
1162
Kees Cook66590612020-06-10 08:20:05 -07001163 /* Bump the sock usage counts, if any. */
1164 __receive_sock(file);
Kees Cookdeefa7f2020-06-10 20:47:45 -07001165 return new_fd;
Kees Cook66590612020-06-10 08:20:05 -07001166}
1167
Dominik Brodowskic7248322018-03-11 11:34:40 +01001168static int ksys_dup3(unsigned int oldfd, unsigned int newfd, int flags)
Al Virofe17f222012-08-21 11:48:11 -04001169{
1170 int err = -EBADF;
Al Viro8280d162012-08-21 12:11:46 -04001171 struct file *file;
1172 struct files_struct *files = current->files;
Al Virofe17f222012-08-21 11:48:11 -04001173
1174 if ((flags & ~O_CLOEXEC) != 0)
1175 return -EINVAL;
1176
Richard W.M. Jonesaed97642012-10-09 15:27:43 +01001177 if (unlikely(oldfd == newfd))
1178 return -EINVAL;
1179
Al Virofe17f222012-08-21 11:48:11 -04001180 if (newfd >= rlimit(RLIMIT_NOFILE))
Al Viro08f05c42012-10-31 03:37:48 +00001181 return -EBADF;
Al Virofe17f222012-08-21 11:48:11 -04001182
1183 spin_lock(&files->file_lock);
1184 err = expand_files(files, newfd);
1185 file = fcheck(oldfd);
1186 if (unlikely(!file))
1187 goto Ebadf;
1188 if (unlikely(err < 0)) {
1189 if (err == -EMFILE)
1190 goto Ebadf;
1191 goto out_unlock;
1192 }
Al Viro8280d162012-08-21 12:11:46 -04001193 return do_dup2(files, file, newfd, flags);
Al Virofe17f222012-08-21 11:48:11 -04001194
1195Ebadf:
1196 err = -EBADF;
1197out_unlock:
1198 spin_unlock(&files->file_lock);
1199 return err;
1200}
1201
Dominik Brodowskic7248322018-03-11 11:34:40 +01001202SYSCALL_DEFINE3(dup3, unsigned int, oldfd, unsigned int, newfd, int, flags)
1203{
1204 return ksys_dup3(oldfd, newfd, flags);
1205}
1206
Al Virofe17f222012-08-21 11:48:11 -04001207SYSCALL_DEFINE2(dup2, unsigned int, oldfd, unsigned int, newfd)
1208{
1209 if (unlikely(newfd == oldfd)) { /* corner case */
1210 struct files_struct *files = current->files;
1211 int retval = oldfd;
1212
1213 rcu_read_lock();
1214 if (!fcheck_files(files, oldfd))
1215 retval = -EBADF;
1216 rcu_read_unlock();
1217 return retval;
1218 }
Dominik Brodowskic7248322018-03-11 11:34:40 +01001219 return ksys_dup3(oldfd, newfd, 0);
Al Virofe17f222012-08-21 11:48:11 -04001220}
1221
Christoph Hellwigbc1cd992020-07-14 08:58:49 +02001222SYSCALL_DEFINE1(dup, unsigned int, fildes)
Al Virofe17f222012-08-21 11:48:11 -04001223{
1224 int ret = -EBADF;
1225 struct file *file = fget_raw(fildes);
1226
1227 if (file) {
Yann Droneaud8d10a032014-12-10 15:45:44 -08001228 ret = get_unused_fd_flags(0);
Al Virofe17f222012-08-21 11:48:11 -04001229 if (ret >= 0)
1230 fd_install(ret, file);
1231 else
1232 fput(file);
1233 }
1234 return ret;
1235}
1236
1237int f_dupfd(unsigned int from, struct file *file, unsigned flags)
1238{
1239 int err;
1240 if (from >= rlimit(RLIMIT_NOFILE))
1241 return -EINVAL;
1242 err = alloc_fd(from, flags);
1243 if (err >= 0) {
1244 get_file(file);
1245 fd_install(err, file);
1246 }
1247 return err;
1248}
Al Viroc3c073f2012-08-21 22:32:06 -04001249
1250int iterate_fd(struct files_struct *files, unsigned n,
1251 int (*f)(const void *, struct file *, unsigned),
1252 const void *p)
1253{
1254 struct fdtable *fdt;
Al Viroc3c073f2012-08-21 22:32:06 -04001255 int res = 0;
1256 if (!files)
1257 return 0;
1258 spin_lock(&files->file_lock);
Al Viroa77cfcb2012-11-29 22:57:33 -05001259 for (fdt = files_fdtable(files); n < fdt->max_fds; n++) {
1260 struct file *file;
1261 file = rcu_dereference_check_fdtable(files, fdt->fd[n]);
1262 if (!file)
1263 continue;
1264 res = f(p, file, n);
1265 if (res)
1266 break;
Al Viroc3c073f2012-08-21 22:32:06 -04001267 }
1268 spin_unlock(&files->file_lock);
1269 return res;
1270}
1271EXPORT_SYMBOL(iterate_fd);