blob: c01c29417ae6ce303d5dccd730e9ade12c42596d [file] [log] [blame]
Greg Kroah-Hartmanb2441312017-11-01 15:07:57 +01001// SPDX-License-Identifier: GPL-2.0
Linus Torvalds1da177e2005-04-16 15:20:36 -07002/*
3 * linux/fs/file.c
4 *
5 * Copyright (C) 1998-1999, Stephen Tweedie and Bill Hawes
6 *
7 * Manage the dynamic fd arrays in the process files_struct.
8 */
9
Al Virofe17f222012-08-21 11:48:11 -040010#include <linux/syscalls.h>
Paul Gortmaker630d9c42011-11-16 23:57:37 -050011#include <linux/export.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070012#include <linux/fs.h>
Christian Brauner278a5fb2019-05-24 11:30:34 +020013#include <linux/kernel.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070014#include <linux/mm.h>
Ingo Molnar3f07c012017-02-08 18:51:30 +010015#include <linux/sched/signal.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070016#include <linux/slab.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070017#include <linux/file.h>
Al Viro9f3acc32008-04-24 07:44:08 -040018#include <linux/fdtable.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070019#include <linux/bitops.h>
Dipankar Sarmaab2af1f2005-09-09 13:04:13 -070020#include <linux/spinlock.h>
21#include <linux/rcupdate.h>
Christian Brauner60997c32020-06-03 21:48:55 +020022#include <linux/close_range.h>
Kees Cook66590612020-06-10 08:20:05 -070023#include <net/sock.h>
Dipankar Sarmaab2af1f2005-09-09 13:04:13 -070024
Jens Axboe53dec2e2021-01-19 15:41:52 -070025#include "internal.h"
26
Alexey Dobriyan9b80a182016-09-02 00:38:52 +030027unsigned int sysctl_nr_open __read_mostly = 1024*1024;
28unsigned int sysctl_nr_open_min = BITS_PER_LONG;
Rasmus Villemoes752343b2015-10-29 12:01:41 +010029/* our min() is unusable in constant expressions ;-/ */
30#define __const_min(x, y) ((x) < (y) ? (x) : (y))
Alexey Dobriyan9b80a182016-09-02 00:38:52 +030031unsigned int sysctl_nr_open_max =
32 __const_min(INT_MAX, ~(size_t)0/sizeof(void *)) & -BITS_PER_LONG;
Eric Dumazet9cfe0152008-02-06 01:37:16 -080033
Changli Gaoa892e2d2010-08-10 18:01:35 -070034static void __free_fdtable(struct fdtable *fdt)
Vadim Lobanov5466b452006-12-10 02:21:22 -080035{
Al Virof6c0a192014-04-23 10:18:46 -040036 kvfree(fdt->fd);
37 kvfree(fdt->open_fds);
Changli Gaoa892e2d2010-08-10 18:01:35 -070038 kfree(fdt);
Dipankar Sarmaab2af1f2005-09-09 13:04:13 -070039}
40
Al Viro7cf4dc32012-08-15 19:56:12 -040041static void free_fdtable_rcu(struct rcu_head *rcu)
Dipankar Sarmaab2af1f2005-09-09 13:04:13 -070042{
Al Viroac3e3c52013-04-28 21:42:33 -040043 __free_fdtable(container_of(rcu, struct fdtable, rcu));
Dipankar Sarmaab2af1f2005-09-09 13:04:13 -070044}
45
Linus Torvaldsf3f86e32015-10-30 16:53:57 -070046#define BITBIT_NR(nr) BITS_TO_LONGS(BITS_TO_LONGS(nr))
47#define BITBIT_SIZE(nr) (BITBIT_NR(nr) * sizeof(long))
48
Dipankar Sarmaab2af1f2005-09-09 13:04:13 -070049/*
Eric Biggersea5c58e2015-11-06 00:32:04 -060050 * Copy 'count' fd bits from the old table to the new table and clear the extra
51 * space if any. This does not copy the file pointers. Called with the files
52 * spinlock held for write.
53 */
54static void copy_fd_bitmaps(struct fdtable *nfdt, struct fdtable *ofdt,
55 unsigned int count)
56{
57 unsigned int cpy, set;
58
59 cpy = count / BITS_PER_BYTE;
60 set = (nfdt->max_fds - count) / BITS_PER_BYTE;
61 memcpy(nfdt->open_fds, ofdt->open_fds, cpy);
62 memset((char *)nfdt->open_fds + cpy, 0, set);
63 memcpy(nfdt->close_on_exec, ofdt->close_on_exec, cpy);
64 memset((char *)nfdt->close_on_exec + cpy, 0, set);
65
66 cpy = BITBIT_SIZE(count);
67 set = BITBIT_SIZE(nfdt->max_fds) - cpy;
68 memcpy(nfdt->full_fds_bits, ofdt->full_fds_bits, cpy);
69 memset((char *)nfdt->full_fds_bits + cpy, 0, set);
70}
71
72/*
73 * Copy all file descriptors from the old table to the new, expanded table and
74 * clear the extra space. Called with the files spinlock held for write.
Dipankar Sarmaab2af1f2005-09-09 13:04:13 -070075 */
Vadim Lobanov5466b452006-12-10 02:21:22 -080076static void copy_fdtable(struct fdtable *nfdt, struct fdtable *ofdt)
Dipankar Sarmaab2af1f2005-09-09 13:04:13 -070077{
Al Viro4e89b722020-05-19 17:48:52 -040078 size_t cpy, set;
Dipankar Sarmaab2af1f2005-09-09 13:04:13 -070079
Vadim Lobanov5466b452006-12-10 02:21:22 -080080 BUG_ON(nfdt->max_fds < ofdt->max_fds);
Vadim Lobanov5466b452006-12-10 02:21:22 -080081
82 cpy = ofdt->max_fds * sizeof(struct file *);
83 set = (nfdt->max_fds - ofdt->max_fds) * sizeof(struct file *);
84 memcpy(nfdt->fd, ofdt->fd, cpy);
Eric Biggersea5c58e2015-11-06 00:32:04 -060085 memset((char *)nfdt->fd + cpy, 0, set);
Vadim Lobanov5466b452006-12-10 02:21:22 -080086
Eric Biggersea5c58e2015-11-06 00:32:04 -060087 copy_fd_bitmaps(nfdt, ofdt, ofdt->max_fds);
Linus Torvalds1da177e2005-04-16 15:20:36 -070088}
89
Linus Torvalds136736a2022-03-29 15:06:39 -070090/*
91 * Note how the fdtable bitmap allocations very much have to be a multiple of
92 * BITS_PER_LONG. This is not only because we walk those things in chunks of
93 * 'unsigned long' in some places, but simply because that is how the Linux
94 * kernel bitmaps are defined to work: they are not "bits in an array of bytes",
95 * they are very much "bits in an array of unsigned long".
96 *
97 * The ALIGN(nr, BITS_PER_LONG) here is for clarity: since we just multiplied
98 * by that "1024/sizeof(ptr)" before, we already know there are sufficient
99 * clear low bits. Clang seems to realize that, gcc ends up being confused.
100 *
101 * On a 128-bit machine, the ALIGN() would actually matter. In the meantime,
102 * let's consider it documentation (and maybe a test-case for gcc to improve
103 * its code generation ;)
104 */
Vadim Lobanov5466b452006-12-10 02:21:22 -0800105static struct fdtable * alloc_fdtable(unsigned int nr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700106{
Vadim Lobanov5466b452006-12-10 02:21:22 -0800107 struct fdtable *fdt;
David Howells1fd36ad2012-02-16 17:49:54 +0000108 void *data;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700109
Dipankar Sarmaab2af1f2005-09-09 13:04:13 -0700110 /*
Vadim Lobanov5466b452006-12-10 02:21:22 -0800111 * Figure out how many fds we actually want to support in this fdtable.
112 * Allocation steps are keyed to the size of the fdarray, since it
113 * grows far faster than any of the other dynamic data. We try to fit
114 * the fdarray into comfortable page-tuned chunks: starting at 1024B
115 * and growing in powers of two from there on.
Dipankar Sarmaab2af1f2005-09-09 13:04:13 -0700116 */
Vadim Lobanov5466b452006-12-10 02:21:22 -0800117 nr /= (1024 / sizeof(struct file *));
118 nr = roundup_pow_of_two(nr + 1);
119 nr *= (1024 / sizeof(struct file *));
Linus Torvalds136736a2022-03-29 15:06:39 -0700120 nr = ALIGN(nr, BITS_PER_LONG);
Al Viro5c598b32008-04-27 20:04:15 -0400121 /*
122 * Note that this can drive nr *below* what we had passed if sysctl_nr_open
123 * had been set lower between the check in expand_files() and here. Deal
124 * with that in caller, it's cheaper that way.
125 *
126 * We make sure that nr remains a multiple of BITS_PER_LONG - otherwise
127 * bitmaps handling below becomes unpleasant, to put it mildly...
128 */
129 if (unlikely(nr > sysctl_nr_open))
130 nr = ((sysctl_nr_open - 1) | (BITS_PER_LONG - 1)) + 1;
Vadim Lobanovbbea9f62006-12-10 02:21:12 -0800131
Vladimir Davydov5d097052016-01-14 15:18:21 -0800132 fdt = kmalloc(sizeof(struct fdtable), GFP_KERNEL_ACCOUNT);
Vadim Lobanov5466b452006-12-10 02:21:22 -0800133 if (!fdt)
Vadim Lobanovbbea9f62006-12-10 02:21:12 -0800134 goto out;
Vadim Lobanov5466b452006-12-10 02:21:22 -0800135 fdt->max_fds = nr;
Michal Hockoc823bd92017-07-06 15:36:19 -0700136 data = kvmalloc_array(nr, sizeof(struct file *), GFP_KERNEL_ACCOUNT);
Vadim Lobanov5466b452006-12-10 02:21:22 -0800137 if (!data)
138 goto out_fdt;
David Howells1fd36ad2012-02-16 17:49:54 +0000139 fdt->fd = data;
140
Michal Hockoc823bd92017-07-06 15:36:19 -0700141 data = kvmalloc(max_t(size_t,
142 2 * nr / BITS_PER_BYTE + BITBIT_SIZE(nr), L1_CACHE_BYTES),
143 GFP_KERNEL_ACCOUNT);
Vadim Lobanov5466b452006-12-10 02:21:22 -0800144 if (!data)
145 goto out_arr;
David Howells1fd36ad2012-02-16 17:49:54 +0000146 fdt->open_fds = data;
Vadim Lobanov5466b452006-12-10 02:21:22 -0800147 data += nr / BITS_PER_BYTE;
David Howells1fd36ad2012-02-16 17:49:54 +0000148 fdt->close_on_exec = data;
Linus Torvaldsf3f86e32015-10-30 16:53:57 -0700149 data += nr / BITS_PER_BYTE;
150 fdt->full_fds_bits = data;
Vadim Lobanov5466b452006-12-10 02:21:22 -0800151
Dipankar Sarmaab2af1f2005-09-09 13:04:13 -0700152 return fdt;
Vadim Lobanov5466b452006-12-10 02:21:22 -0800153
154out_arr:
Al Virof6c0a192014-04-23 10:18:46 -0400155 kvfree(fdt->fd);
Vadim Lobanov5466b452006-12-10 02:21:22 -0800156out_fdt:
Dipankar Sarmaab2af1f2005-09-09 13:04:13 -0700157 kfree(fdt);
Vadim Lobanov5466b452006-12-10 02:21:22 -0800158out:
Dipankar Sarmaab2af1f2005-09-09 13:04:13 -0700159 return NULL;
160}
161
162/*
Vadim Lobanov74d392a2006-09-29 02:01:43 -0700163 * Expand the file descriptor table.
164 * This function will allocate a new fdtable and both fd array and fdset, of
165 * the given size.
166 * Return <0 error code on error; 1 on successful completion.
167 * The files->file_lock should be held on entry, and will be held on exit.
Dipankar Sarmaab2af1f2005-09-09 13:04:13 -0700168 */
Alexey Dobriyan9b80a182016-09-02 00:38:52 +0300169static int expand_fdtable(struct files_struct *files, unsigned int nr)
Dipankar Sarmaab2af1f2005-09-09 13:04:13 -0700170 __releases(files->file_lock)
171 __acquires(files->file_lock)
172{
Vadim Lobanov74d392a2006-09-29 02:01:43 -0700173 struct fdtable *new_fdt, *cur_fdt;
Dipankar Sarmaab2af1f2005-09-09 13:04:13 -0700174
Linus Torvalds1da177e2005-04-16 15:20:36 -0700175 spin_unlock(&files->file_lock);
Vadim Lobanov74d392a2006-09-29 02:01:43 -0700176 new_fdt = alloc_fdtable(nr);
Eric Dumazet8a81252b2015-06-30 15:54:08 +0200177
Eric W. Biedermand74ba042020-11-20 17:14:35 -0600178 /* make sure all fd_install() have seen resize_in_progress
Eric Dumazet8a81252b2015-06-30 15:54:08 +0200179 * or have finished their rcu_read_lock_sched() section.
180 */
181 if (atomic_read(&files->count) > 1)
Paul E. McKenneyc93ffc12018-11-05 17:31:31 -0800182 synchronize_rcu();
Eric Dumazet8a81252b2015-06-30 15:54:08 +0200183
Linus Torvalds1da177e2005-04-16 15:20:36 -0700184 spin_lock(&files->file_lock);
Vadim Lobanov74d392a2006-09-29 02:01:43 -0700185 if (!new_fdt)
186 return -ENOMEM;
Dipankar Sarmaab2af1f2005-09-09 13:04:13 -0700187 /*
Al Viro5c598b32008-04-27 20:04:15 -0400188 * extremely unlikely race - sysctl_nr_open decreased between the check in
189 * caller and alloc_fdtable(). Cheaper to catch it here...
190 */
191 if (unlikely(new_fdt->max_fds <= nr)) {
Changli Gaoa892e2d2010-08-10 18:01:35 -0700192 __free_fdtable(new_fdt);
Al Viro5c598b32008-04-27 20:04:15 -0400193 return -EMFILE;
194 }
Vadim Lobanov74d392a2006-09-29 02:01:43 -0700195 cur_fdt = files_fdtable(files);
Eric Dumazet8a81252b2015-06-30 15:54:08 +0200196 BUG_ON(nr < cur_fdt->max_fds);
197 copy_fdtable(new_fdt, cur_fdt);
198 rcu_assign_pointer(files->fdt, new_fdt);
199 if (cur_fdt != &files->fdtab)
200 call_rcu(&cur_fdt->rcu, free_fdtable_rcu);
Eric W. Biedermand74ba042020-11-20 17:14:35 -0600201 /* coupled with smp_rmb() in fd_install() */
Eric Dumazet8a81252b2015-06-30 15:54:08 +0200202 smp_wmb();
Vadim Lobanov74d392a2006-09-29 02:01:43 -0700203 return 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700204}
205
206/*
207 * Expand files.
Vadim Lobanov74d392a2006-09-29 02:01:43 -0700208 * This function will expand the file structures, if the requested size exceeds
209 * the current capacity and there is room for expansion.
210 * Return <0 error code on error; 0 when nothing done; 1 when files were
211 * expanded and execution may have blocked.
212 * The files->file_lock should be held on entry, and will be held on exit.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700213 */
Alexey Dobriyan9b80a182016-09-02 00:38:52 +0300214static int expand_files(struct files_struct *files, unsigned int nr)
Eric Dumazet8a81252b2015-06-30 15:54:08 +0200215 __releases(files->file_lock)
216 __acquires(files->file_lock)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700217{
Dipankar Sarmabadf1662005-09-09 13:04:10 -0700218 struct fdtable *fdt;
Eric Dumazet8a81252b2015-06-30 15:54:08 +0200219 int expanded = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700220
Eric Dumazet8a81252b2015-06-30 15:54:08 +0200221repeat:
Dipankar Sarmabadf1662005-09-09 13:04:10 -0700222 fdt = files_fdtable(files);
Al Viro4e1e0182008-07-26 16:01:20 -0400223
Vadim Lobanov74d392a2006-09-29 02:01:43 -0700224 /* Do we need to expand? */
Vadim Lobanovbbea9f62006-12-10 02:21:12 -0800225 if (nr < fdt->max_fds)
Eric Dumazet8a81252b2015-06-30 15:54:08 +0200226 return expanded;
Al Viro4e1e0182008-07-26 16:01:20 -0400227
Vadim Lobanov74d392a2006-09-29 02:01:43 -0700228 /* Can we expand? */
Eric Dumazet9cfe0152008-02-06 01:37:16 -0800229 if (nr >= sysctl_nr_open)
Vadim Lobanov74d392a2006-09-29 02:01:43 -0700230 return -EMFILE;
231
Eric Dumazet8a81252b2015-06-30 15:54:08 +0200232 if (unlikely(files->resize_in_progress)) {
233 spin_unlock(&files->file_lock);
234 expanded = 1;
235 wait_event(files->resize_wait, !files->resize_in_progress);
236 spin_lock(&files->file_lock);
237 goto repeat;
238 }
239
Vadim Lobanov74d392a2006-09-29 02:01:43 -0700240 /* All good, so we try */
Eric Dumazet8a81252b2015-06-30 15:54:08 +0200241 files->resize_in_progress = true;
242 expanded = expand_fdtable(files, nr);
243 files->resize_in_progress = false;
244
245 wake_up_all(&files->resize_wait);
246 return expanded;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700247}
Dipankar Sarmaab2af1f2005-09-09 13:04:13 -0700248
Alexey Dobriyan9b80a182016-09-02 00:38:52 +0300249static inline void __set_close_on_exec(unsigned int fd, struct fdtable *fdt)
Al Virob8318b02012-08-21 20:09:42 -0400250{
251 __set_bit(fd, fdt->close_on_exec);
252}
253
Alexey Dobriyan9b80a182016-09-02 00:38:52 +0300254static inline void __clear_close_on_exec(unsigned int fd, struct fdtable *fdt)
Al Virob8318b02012-08-21 20:09:42 -0400255{
Linus Torvaldsfc908882015-10-31 16:06:40 -0700256 if (test_bit(fd, fdt->close_on_exec))
257 __clear_bit(fd, fdt->close_on_exec);
Al Virob8318b02012-08-21 20:09:42 -0400258}
259
Linus Torvaldsf3f86e32015-10-30 16:53:57 -0700260static inline void __set_open_fd(unsigned int fd, struct fdtable *fdt)
Al Virob8318b02012-08-21 20:09:42 -0400261{
262 __set_bit(fd, fdt->open_fds);
Linus Torvaldsf3f86e32015-10-30 16:53:57 -0700263 fd /= BITS_PER_LONG;
264 if (!~fdt->open_fds[fd])
265 __set_bit(fd, fdt->full_fds_bits);
Al Virob8318b02012-08-21 20:09:42 -0400266}
267
Linus Torvaldsf3f86e32015-10-30 16:53:57 -0700268static inline void __clear_open_fd(unsigned int fd, struct fdtable *fdt)
Al Virob8318b02012-08-21 20:09:42 -0400269{
270 __clear_bit(fd, fdt->open_fds);
Linus Torvaldsf3f86e32015-10-30 16:53:57 -0700271 __clear_bit(fd / BITS_PER_LONG, fdt->full_fds_bits);
Al Virob8318b02012-08-21 20:09:42 -0400272}
273
Alexey Dobriyan9b80a182016-09-02 00:38:52 +0300274static unsigned int count_open_files(struct fdtable *fdt)
Al Viro02afc6262008-05-08 19:42:56 -0400275{
Alexey Dobriyan9b80a182016-09-02 00:38:52 +0300276 unsigned int size = fdt->max_fds;
277 unsigned int i;
Al Viro02afc6262008-05-08 19:42:56 -0400278
279 /* Find the last open fd */
David Howells1fd36ad2012-02-16 17:49:54 +0000280 for (i = size / BITS_PER_LONG; i > 0; ) {
281 if (fdt->open_fds[--i])
Al Viro02afc6262008-05-08 19:42:56 -0400282 break;
283 }
David Howells1fd36ad2012-02-16 17:49:54 +0000284 i = (i + 1) * BITS_PER_LONG;
Al Viro02afc6262008-05-08 19:42:56 -0400285 return i;
286}
287
Linus Torvalds136736a2022-03-29 15:06:39 -0700288/*
289 * Note that a sane fdtable size always has to be a multiple of
290 * BITS_PER_LONG, since we have bitmaps that are sized by this.
291 *
292 * 'max_fds' will normally already be properly aligned, but it
293 * turns out that in the close_range() -> __close_range() ->
294 * unshare_fd() -> dup_fd() -> sane_fdtable_size() we can end
295 * up having a 'max_fds' value that isn't already aligned.
296 *
297 * Rather than make close_range() have to worry about this,
298 * just make that BITS_PER_LONG alignment be part of a sane
299 * fdtable size. Becuase that's really what it is.
300 */
Christian Brauner60997c32020-06-03 21:48:55 +0200301static unsigned int sane_fdtable_size(struct fdtable *fdt, unsigned int max_fds)
302{
303 unsigned int count;
304
305 count = count_open_files(fdt);
Linus Torvalds136736a2022-03-29 15:06:39 -0700306 max_fds = ALIGN(max_fds, BITS_PER_LONG);
Christian Brauner60997c32020-06-03 21:48:55 +0200307 if (max_fds < NR_OPEN_DEFAULT)
308 max_fds = NR_OPEN_DEFAULT;
309 return min(count, max_fds);
310}
311
Al Viro02afc6262008-05-08 19:42:56 -0400312/*
313 * Allocate a new files structure and copy contents from the
314 * passed in files structure.
315 * errorp will be valid only when the returned files_struct is NULL.
316 */
Christian Brauner60997c32020-06-03 21:48:55 +0200317struct files_struct *dup_fd(struct files_struct *oldf, unsigned int max_fds, int *errorp)
Al Viro02afc6262008-05-08 19:42:56 -0400318{
319 struct files_struct *newf;
320 struct file **old_fds, **new_fds;
Alexey Dobriyan9b80a182016-09-02 00:38:52 +0300321 unsigned int open_files, i;
Al Viro02afc6262008-05-08 19:42:56 -0400322 struct fdtable *old_fdt, *new_fdt;
323
324 *errorp = -ENOMEM;
Al Viroafbec7f2008-05-08 21:11:17 -0400325 newf = kmem_cache_alloc(files_cachep, GFP_KERNEL);
Al Viro02afc6262008-05-08 19:42:56 -0400326 if (!newf)
327 goto out;
328
Al Viroafbec7f2008-05-08 21:11:17 -0400329 atomic_set(&newf->count, 1);
330
331 spin_lock_init(&newf->file_lock);
Eric Dumazet8a81252b2015-06-30 15:54:08 +0200332 newf->resize_in_progress = false;
333 init_waitqueue_head(&newf->resize_wait);
Al Viroafbec7f2008-05-08 21:11:17 -0400334 newf->next_fd = 0;
335 new_fdt = &newf->fdtab;
336 new_fdt->max_fds = NR_OPEN_DEFAULT;
David Howells1fd36ad2012-02-16 17:49:54 +0000337 new_fdt->close_on_exec = newf->close_on_exec_init;
338 new_fdt->open_fds = newf->open_fds_init;
Linus Torvaldsf3f86e32015-10-30 16:53:57 -0700339 new_fdt->full_fds_bits = newf->full_fds_bits_init;
Al Viroafbec7f2008-05-08 21:11:17 -0400340 new_fdt->fd = &newf->fd_array[0];
Al Viroafbec7f2008-05-08 21:11:17 -0400341
Al Viro02afc6262008-05-08 19:42:56 -0400342 spin_lock(&oldf->file_lock);
343 old_fdt = files_fdtable(oldf);
Christian Brauner60997c32020-06-03 21:48:55 +0200344 open_files = sane_fdtable_size(old_fdt, max_fds);
Al Viro02afc6262008-05-08 19:42:56 -0400345
346 /*
347 * Check whether we need to allocate a larger fd array and fd set.
Al Viro02afc6262008-05-08 19:42:56 -0400348 */
Al Viroadbecb12008-05-08 21:19:42 -0400349 while (unlikely(open_files > new_fdt->max_fds)) {
Al Viro02afc6262008-05-08 19:42:56 -0400350 spin_unlock(&oldf->file_lock);
Al Viro9dec3c42008-05-08 21:02:45 -0400351
Changli Gaoa892e2d2010-08-10 18:01:35 -0700352 if (new_fdt != &newf->fdtab)
353 __free_fdtable(new_fdt);
Al Viroadbecb12008-05-08 21:19:42 -0400354
Al Viro9dec3c42008-05-08 21:02:45 -0400355 new_fdt = alloc_fdtable(open_files - 1);
356 if (!new_fdt) {
357 *errorp = -ENOMEM;
Al Viro02afc6262008-05-08 19:42:56 -0400358 goto out_release;
Al Viro9dec3c42008-05-08 21:02:45 -0400359 }
360
361 /* beyond sysctl_nr_open; nothing to do */
362 if (unlikely(new_fdt->max_fds < open_files)) {
Changli Gaoa892e2d2010-08-10 18:01:35 -0700363 __free_fdtable(new_fdt);
Al Viro9dec3c42008-05-08 21:02:45 -0400364 *errorp = -EMFILE;
365 goto out_release;
366 }
Al Viro9dec3c42008-05-08 21:02:45 -0400367
Al Viro02afc6262008-05-08 19:42:56 -0400368 /*
369 * Reacquire the oldf lock and a pointer to its fd table
370 * who knows it may have a new bigger fd table. We need
371 * the latest pointer.
372 */
373 spin_lock(&oldf->file_lock);
374 old_fdt = files_fdtable(oldf);
Christian Brauner60997c32020-06-03 21:48:55 +0200375 open_files = sane_fdtable_size(old_fdt, max_fds);
Al Viro02afc6262008-05-08 19:42:56 -0400376 }
377
Eric Biggersea5c58e2015-11-06 00:32:04 -0600378 copy_fd_bitmaps(new_fdt, old_fdt, open_files);
379
Al Viro02afc6262008-05-08 19:42:56 -0400380 old_fds = old_fdt->fd;
381 new_fds = new_fdt->fd;
382
Al Viro02afc6262008-05-08 19:42:56 -0400383 for (i = open_files; i != 0; i--) {
384 struct file *f = *old_fds++;
385 if (f) {
386 get_file(f);
387 } else {
388 /*
389 * The fd may be claimed in the fd bitmap but not yet
390 * instantiated in the files array if a sibling thread
391 * is partway through open(). So make sure that this
392 * fd is available to the new process.
393 */
David Howells1dce27c2012-02-16 17:49:42 +0000394 __clear_open_fd(open_files - i, new_fdt);
Al Viro02afc6262008-05-08 19:42:56 -0400395 }
396 rcu_assign_pointer(*new_fds++, f);
397 }
398 spin_unlock(&oldf->file_lock);
399
Eric Biggersea5c58e2015-11-06 00:32:04 -0600400 /* clear the remainder */
401 memset(new_fds, 0, (new_fdt->max_fds - open_files) * sizeof(struct file *));
Al Viro02afc6262008-05-08 19:42:56 -0400402
Al Viroafbec7f2008-05-08 21:11:17 -0400403 rcu_assign_pointer(newf->fdt, new_fdt);
404
Al Viro02afc6262008-05-08 19:42:56 -0400405 return newf;
406
407out_release:
408 kmem_cache_free(files_cachep, newf);
409out:
410 return NULL;
411}
412
Oleg Nesterovce08b622014-01-11 19:19:53 +0100413static struct fdtable *close_files(struct files_struct * files)
Al Viro7cf4dc32012-08-15 19:56:12 -0400414{
Al Viro7cf4dc32012-08-15 19:56:12 -0400415 /*
416 * It is safe to dereference the fd table without RCU or
417 * ->file_lock because this is the last reference to the
Oleg Nesterovce08b622014-01-11 19:19:53 +0100418 * files structure.
Al Viro7cf4dc32012-08-15 19:56:12 -0400419 */
Oleg Nesterovce08b622014-01-11 19:19:53 +0100420 struct fdtable *fdt = rcu_dereference_raw(files->fdt);
Alexey Dobriyan9b80a182016-09-02 00:38:52 +0300421 unsigned int i, j = 0;
Oleg Nesterovce08b622014-01-11 19:19:53 +0100422
Al Viro7cf4dc32012-08-15 19:56:12 -0400423 for (;;) {
424 unsigned long set;
425 i = j * BITS_PER_LONG;
426 if (i >= fdt->max_fds)
427 break;
428 set = fdt->open_fds[j++];
429 while (set) {
430 if (set & 1) {
431 struct file * file = xchg(&fdt->fd[i], NULL);
432 if (file) {
433 filp_close(file, files);
Paul E. McKenney388a4c82017-10-24 08:39:34 -0700434 cond_resched();
Al Viro7cf4dc32012-08-15 19:56:12 -0400435 }
436 }
437 i++;
438 set >>= 1;
439 }
440 }
Oleg Nesterovce08b622014-01-11 19:19:53 +0100441
442 return fdt;
Al Viro7cf4dc32012-08-15 19:56:12 -0400443}
444
Al Viro7cf4dc32012-08-15 19:56:12 -0400445void put_files_struct(struct files_struct *files)
446{
Al Viro7cf4dc32012-08-15 19:56:12 -0400447 if (atomic_dec_and_test(&files->count)) {
Oleg Nesterovce08b622014-01-11 19:19:53 +0100448 struct fdtable *fdt = close_files(files);
449
Al Virob9e02af2012-08-15 20:00:58 -0400450 /* free the arrays if they are not embedded */
451 if (fdt != &files->fdtab)
452 __free_fdtable(fdt);
453 kmem_cache_free(files_cachep, files);
Al Viro7cf4dc32012-08-15 19:56:12 -0400454 }
455}
456
Al Viro7cf4dc32012-08-15 19:56:12 -0400457void exit_files(struct task_struct *tsk)
458{
459 struct files_struct * files = tsk->files;
460
461 if (files) {
462 task_lock(tsk);
463 tsk->files = NULL;
464 task_unlock(tsk);
465 put_files_struct(files);
466 }
467}
468
Al Virof52111b2008-05-08 18:19:16 -0400469struct files_struct init_files = {
470 .count = ATOMIC_INIT(1),
471 .fdt = &init_files.fdtab,
472 .fdtab = {
473 .max_fds = NR_OPEN_DEFAULT,
474 .fd = &init_files.fd_array[0],
David Howells1fd36ad2012-02-16 17:49:54 +0000475 .close_on_exec = init_files.close_on_exec_init,
476 .open_fds = init_files.open_fds_init,
Linus Torvaldsf3f86e32015-10-30 16:53:57 -0700477 .full_fds_bits = init_files.full_fds_bits_init,
Al Virof52111b2008-05-08 18:19:16 -0400478 },
Thomas Gleixnereece09e2011-07-17 21:25:03 +0200479 .file_lock = __SPIN_LOCK_UNLOCKED(init_files.file_lock),
Shuriyc Chu5704a062019-03-05 15:41:56 -0800480 .resize_wait = __WAIT_QUEUE_HEAD_INITIALIZER(init_files.resize_wait),
Al Virof52111b2008-05-08 18:19:16 -0400481};
Al Viro1027abe2008-07-30 04:13:04 -0400482
Alexey Dobriyan9b80a182016-09-02 00:38:52 +0300483static unsigned int find_next_fd(struct fdtable *fdt, unsigned int start)
Linus Torvaldsf3f86e32015-10-30 16:53:57 -0700484{
Alexey Dobriyan9b80a182016-09-02 00:38:52 +0300485 unsigned int maxfd = fdt->max_fds;
486 unsigned int maxbit = maxfd / BITS_PER_LONG;
487 unsigned int bitbit = start / BITS_PER_LONG;
Linus Torvaldsf3f86e32015-10-30 16:53:57 -0700488
489 bitbit = find_next_zero_bit(fdt->full_fds_bits, maxbit, bitbit) * BITS_PER_LONG;
490 if (bitbit > maxfd)
491 return maxfd;
492 if (bitbit > start)
493 start = bitbit;
494 return find_next_zero_bit(fdt->open_fds, maxfd, start);
495}
496
Al Viro1027abe2008-07-30 04:13:04 -0400497/*
498 * allocate a file descriptor, mark it busy.
499 */
Eric W. Biedermanaa384d12020-11-20 17:14:37 -0600500static int alloc_fd(unsigned start, unsigned end, unsigned flags)
Al Viro1027abe2008-07-30 04:13:04 -0400501{
Eric W. Biedermanaa384d12020-11-20 17:14:37 -0600502 struct files_struct *files = current->files;
Al Viro1027abe2008-07-30 04:13:04 -0400503 unsigned int fd;
504 int error;
505 struct fdtable *fdt;
506
507 spin_lock(&files->file_lock);
508repeat:
509 fdt = files_fdtable(files);
510 fd = start;
511 if (fd < files->next_fd)
512 fd = files->next_fd;
513
514 if (fd < fdt->max_fds)
Linus Torvaldsf3f86e32015-10-30 16:53:57 -0700515 fd = find_next_fd(fdt, fd);
Al Viro1027abe2008-07-30 04:13:04 -0400516
Al Virof33ff992012-08-12 16:17:59 -0400517 /*
518 * N.B. For clone tasks sharing a files structure, this test
519 * will limit the total number of files that can be opened.
520 */
521 error = -EMFILE;
522 if (fd >= end)
523 goto out;
524
Al Viro1027abe2008-07-30 04:13:04 -0400525 error = expand_files(files, fd);
526 if (error < 0)
527 goto out;
528
529 /*
530 * If we needed to expand the fs array we
531 * might have blocked - try again.
532 */
533 if (error)
534 goto repeat;
535
536 if (start <= files->next_fd)
537 files->next_fd = fd + 1;
538
David Howells1dce27c2012-02-16 17:49:42 +0000539 __set_open_fd(fd, fdt);
Al Viro1027abe2008-07-30 04:13:04 -0400540 if (flags & O_CLOEXEC)
David Howells1dce27c2012-02-16 17:49:42 +0000541 __set_close_on_exec(fd, fdt);
Al Viro1027abe2008-07-30 04:13:04 -0400542 else
David Howells1dce27c2012-02-16 17:49:42 +0000543 __clear_close_on_exec(fd, fdt);
Al Viro1027abe2008-07-30 04:13:04 -0400544 error = fd;
545#if 1
546 /* Sanity check */
Paul E. McKenneyadd1f092014-02-12 12:51:09 -0800547 if (rcu_access_pointer(fdt->fd[fd]) != NULL) {
Al Viro1027abe2008-07-30 04:13:04 -0400548 printk(KERN_WARNING "alloc_fd: slot %d not NULL!\n", fd);
549 rcu_assign_pointer(fdt->fd[fd], NULL);
550 }
551#endif
552
553out:
554 spin_unlock(&files->file_lock);
555 return error;
556}
557
Jens Axboe4022e7a2020-03-19 19:23:18 -0600558int __get_unused_fd_flags(unsigned flags, unsigned long nofile)
559{
Eric W. Biedermanaa384d12020-11-20 17:14:37 -0600560 return alloc_fd(0, nofile, flags);
Jens Axboe4022e7a2020-03-19 19:23:18 -0600561}
562
Al Viro1a7bd222012-08-12 17:18:05 -0400563int get_unused_fd_flags(unsigned flags)
Al Viro1027abe2008-07-30 04:13:04 -0400564{
Jens Axboe4022e7a2020-03-19 19:23:18 -0600565 return __get_unused_fd_flags(flags, rlimit(RLIMIT_NOFILE));
Al Viro1027abe2008-07-30 04:13:04 -0400566}
Al Viro1a7bd222012-08-12 17:18:05 -0400567EXPORT_SYMBOL(get_unused_fd_flags);
Al Viro56007ca2012-08-15 21:03:26 -0400568
569static void __put_unused_fd(struct files_struct *files, unsigned int fd)
570{
571 struct fdtable *fdt = files_fdtable(files);
572 __clear_open_fd(fd, fdt);
573 if (fd < files->next_fd)
574 files->next_fd = fd;
575}
576
577void put_unused_fd(unsigned int fd)
578{
579 struct files_struct *files = current->files;
580 spin_lock(&files->file_lock);
581 __put_unused_fd(files, fd);
582 spin_unlock(&files->file_lock);
583}
584
585EXPORT_SYMBOL(put_unused_fd);
586
587/*
588 * Install a file pointer in the fd array.
589 *
590 * The VFS is full of places where we drop the files lock between
591 * setting the open_fds bitmap and installing the file in the file
592 * array. At any such point, we are vulnerable to a dup2() race
593 * installing a file in the array before us. We need to detect this and
594 * fput() the struct file we are about to overwrite in this case.
595 *
596 * It should never happen - if we allow dup2() do it, _really_ bad things
597 * will follow.
Al Virof869e8a2012-08-15 21:06:33 -0400598 *
Eric W. Biedermand74ba042020-11-20 17:14:35 -0600599 * This consumes the "file" refcount, so callers should treat it
600 * as if they had called fput(file).
Al Viro56007ca2012-08-15 21:03:26 -0400601 */
602
Eric W. Biedermand74ba042020-11-20 17:14:35 -0600603void fd_install(unsigned int fd, struct file *file)
Al Viro56007ca2012-08-15 21:03:26 -0400604{
Eric W. Biedermand74ba042020-11-20 17:14:35 -0600605 struct files_struct *files = current->files;
Al Viro56007ca2012-08-15 21:03:26 -0400606 struct fdtable *fdt;
Eric Dumazet8a81252b2015-06-30 15:54:08 +0200607
Eric Dumazet8a81252b2015-06-30 15:54:08 +0200608 rcu_read_lock_sched();
609
Mateusz Guzikc02b1a92017-10-03 12:58:15 +0200610 if (unlikely(files->resize_in_progress)) {
Eric Dumazet8a81252b2015-06-30 15:54:08 +0200611 rcu_read_unlock_sched();
Mateusz Guzikc02b1a92017-10-03 12:58:15 +0200612 spin_lock(&files->file_lock);
613 fdt = files_fdtable(files);
614 BUG_ON(fdt->fd[fd] != NULL);
615 rcu_assign_pointer(fdt->fd[fd], file);
616 spin_unlock(&files->file_lock);
617 return;
Eric Dumazet8a81252b2015-06-30 15:54:08 +0200618 }
619 /* coupled with smp_wmb() in expand_fdtable() */
620 smp_rmb();
621 fdt = rcu_dereference_sched(files->fdt);
Al Viro56007ca2012-08-15 21:03:26 -0400622 BUG_ON(fdt->fd[fd] != NULL);
623 rcu_assign_pointer(fdt->fd[fd], file);
Eric Dumazet8a81252b2015-06-30 15:54:08 +0200624 rcu_read_unlock_sched();
Al Viro56007ca2012-08-15 21:03:26 -0400625}
626
627EXPORT_SYMBOL(fd_install);
Al Viro0ee8cdf2012-08-15 21:12:10 -0400628
Christian Braunerf49fd6d2021-04-02 10:32:21 +0200629/**
630 * pick_file - return file associatd with fd
631 * @files: file struct to retrieve file from
632 * @fd: file descriptor to retrieve file for
633 *
634 * If this functions returns an EINVAL error pointer the fd was beyond the
635 * current maximum number of file descriptors for that fdtable.
636 *
637 * Returns: The file associated with @fd, on error returns an error pointer.
638 */
Christian Brauner278a5fb2019-05-24 11:30:34 +0200639static struct file *pick_file(struct files_struct *files, unsigned fd)
Al Viro483ce1d2012-08-19 12:04:24 -0400640{
Christian Braunerf49fd6d2021-04-02 10:32:21 +0200641 struct file *file;
Al Viro483ce1d2012-08-19 12:04:24 -0400642 struct fdtable *fdt;
643
644 spin_lock(&files->file_lock);
645 fdt = files_fdtable(files);
Christian Braunerf49fd6d2021-04-02 10:32:21 +0200646 if (fd >= fdt->max_fds) {
647 file = ERR_PTR(-EINVAL);
Al Viro483ce1d2012-08-19 12:04:24 -0400648 goto out_unlock;
Christian Braunerf49fd6d2021-04-02 10:32:21 +0200649 }
Al Viro483ce1d2012-08-19 12:04:24 -0400650 file = fdt->fd[fd];
Christian Braunerf49fd6d2021-04-02 10:32:21 +0200651 if (!file) {
652 file = ERR_PTR(-EBADF);
Al Viro483ce1d2012-08-19 12:04:24 -0400653 goto out_unlock;
Christian Braunerf49fd6d2021-04-02 10:32:21 +0200654 }
Al Viro483ce1d2012-08-19 12:04:24 -0400655 rcu_assign_pointer(fdt->fd[fd], NULL);
Al Viro483ce1d2012-08-19 12:04:24 -0400656 __put_unused_fd(files, fd);
Al Viro483ce1d2012-08-19 12:04:24 -0400657
658out_unlock:
659 spin_unlock(&files->file_lock);
Christian Brauner278a5fb2019-05-24 11:30:34 +0200660 return file;
661}
662
Eric W. Biederman8760c902020-11-20 17:14:38 -0600663int close_fd(unsigned fd)
Christian Brauner278a5fb2019-05-24 11:30:34 +0200664{
Eric W. Biederman8760c902020-11-20 17:14:38 -0600665 struct files_struct *files = current->files;
Christian Brauner278a5fb2019-05-24 11:30:34 +0200666 struct file *file;
667
668 file = pick_file(files, fd);
Christian Braunerf49fd6d2021-04-02 10:32:21 +0200669 if (IS_ERR(file))
Christian Brauner278a5fb2019-05-24 11:30:34 +0200670 return -EBADF;
671
672 return filp_close(file, files);
Al Viro483ce1d2012-08-19 12:04:24 -0400673}
Eric W. Biederman8760c902020-11-20 17:14:38 -0600674EXPORT_SYMBOL(close_fd); /* for ksys_close() */
Al Viro483ce1d2012-08-19 12:04:24 -0400675
Christian Brauner9b5b87222021-04-02 10:29:36 +0200676/**
677 * last_fd - return last valid index into fd table
678 * @cur_fds: files struct
679 *
680 * Context: Either rcu read lock or files_lock must be held.
681 *
682 * Returns: Last valid index into fdtable.
683 */
684static inline unsigned last_fd(struct fdtable *fdt)
685{
686 return fdt->max_fds - 1;
687}
688
Giuseppe Scrivano582f1fb2020-11-18 11:47:45 +0100689static inline void __range_cloexec(struct files_struct *cur_fds,
690 unsigned int fd, unsigned int max_fd)
691{
692 struct fdtable *fdt;
693
Christian Brauner9b5b87222021-04-02 10:29:36 +0200694 /* make sure we're using the correct maximum value */
Giuseppe Scrivano582f1fb2020-11-18 11:47:45 +0100695 spin_lock(&cur_fds->file_lock);
696 fdt = files_fdtable(cur_fds);
Christian Brauner9b5b87222021-04-02 10:29:36 +0200697 max_fd = min(last_fd(fdt), max_fd);
698 if (fd <= max_fd)
699 bitmap_set(fdt->close_on_exec, fd, max_fd - fd + 1);
Giuseppe Scrivano582f1fb2020-11-18 11:47:45 +0100700 spin_unlock(&cur_fds->file_lock);
701}
702
703static inline void __range_close(struct files_struct *cur_fds, unsigned int fd,
704 unsigned int max_fd)
705{
706 while (fd <= max_fd) {
707 struct file *file;
708
709 file = pick_file(cur_fds, fd++);
Christian Braunerf49fd6d2021-04-02 10:32:21 +0200710 if (!IS_ERR(file)) {
711 /* found a valid file to close */
712 filp_close(file, cur_fds);
713 cond_resched();
Giuseppe Scrivano582f1fb2020-11-18 11:47:45 +0100714 continue;
Christian Braunerf49fd6d2021-04-02 10:32:21 +0200715 }
Giuseppe Scrivano582f1fb2020-11-18 11:47:45 +0100716
Christian Braunerf49fd6d2021-04-02 10:32:21 +0200717 /* beyond the last fd in that table */
718 if (PTR_ERR(file) == -EINVAL)
719 return;
Giuseppe Scrivano582f1fb2020-11-18 11:47:45 +0100720 }
721}
722
Christian Brauner278a5fb2019-05-24 11:30:34 +0200723/**
724 * __close_range() - Close all file descriptors in a given range.
725 *
726 * @fd: starting file descriptor to close
727 * @max_fd: last file descriptor to close
728 *
729 * This closes a range of file descriptors. All file descriptors
730 * from @fd up to and including @max_fd are closed.
731 */
Christian Brauner60997c32020-06-03 21:48:55 +0200732int __close_range(unsigned fd, unsigned max_fd, unsigned int flags)
Christian Brauner278a5fb2019-05-24 11:30:34 +0200733{
Christian Brauner60997c32020-06-03 21:48:55 +0200734 struct task_struct *me = current;
735 struct files_struct *cur_fds = me->files, *fds = NULL;
736
Giuseppe Scrivano582f1fb2020-11-18 11:47:45 +0100737 if (flags & ~(CLOSE_RANGE_UNSHARE | CLOSE_RANGE_CLOEXEC))
Christian Brauner60997c32020-06-03 21:48:55 +0200738 return -EINVAL;
Christian Brauner278a5fb2019-05-24 11:30:34 +0200739
740 if (fd > max_fd)
741 return -EINVAL;
742
Christian Brauner60997c32020-06-03 21:48:55 +0200743 if (flags & CLOSE_RANGE_UNSHARE) {
744 int ret;
745 unsigned int max_unshare_fds = NR_OPEN_MAX;
746
747 /*
Christian Brauner03ba0fe2021-04-02 10:38:09 +0200748 * If the caller requested all fds to be made cloexec we always
749 * copy all of the file descriptors since they still want to
750 * use them.
Christian Brauner60997c32020-06-03 21:48:55 +0200751 */
Christian Brauner03ba0fe2021-04-02 10:38:09 +0200752 if (!(flags & CLOSE_RANGE_CLOEXEC)) {
753 /*
754 * If the requested range is greater than the current
755 * maximum, we're closing everything so only copy all
756 * file descriptors beneath the lowest file descriptor.
757 */
758 rcu_read_lock();
759 if (max_fd >= last_fd(files_fdtable(cur_fds)))
760 max_unshare_fds = fd;
761 rcu_read_unlock();
762 }
Christian Brauner60997c32020-06-03 21:48:55 +0200763
764 ret = unshare_fd(CLONE_FILES, max_unshare_fds, &fds);
765 if (ret)
766 return ret;
767
768 /*
769 * We used to share our file descriptor table, and have now
770 * created a private one, make sure we're using it below.
771 */
772 if (fds)
773 swap(cur_fds, fds);
774 }
775
Giuseppe Scrivano582f1fb2020-11-18 11:47:45 +0100776 if (flags & CLOSE_RANGE_CLOEXEC)
777 __range_cloexec(cur_fds, fd, max_fd);
778 else
779 __range_close(cur_fds, fd, max_fd);
Christian Brauner278a5fb2019-05-24 11:30:34 +0200780
Christian Brauner60997c32020-06-03 21:48:55 +0200781 if (fds) {
782 /*
783 * We're done closing the files we were supposed to. Time to install
784 * the new file descriptor table and drop the old one.
785 */
786 task_lock(me);
787 me->files = cur_fds;
788 task_unlock(me);
789 put_files_struct(fds);
790 }
791
Christian Brauner278a5fb2019-05-24 11:30:34 +0200792 return 0;
793}
794
Todd Kjos80cd7952018-12-14 15:58:21 -0800795/*
Jens Axboe53dec2e2021-01-19 15:41:52 -0700796 * See close_fd_get_file() below, this variant assumes current->files->file_lock
797 * is held.
798 */
799int __close_fd_get_file(unsigned int fd, struct file **res)
800{
801 struct files_struct *files = current->files;
802 struct file *file;
803 struct fdtable *fdt;
804
805 fdt = files_fdtable(files);
806 if (fd >= fdt->max_fds)
807 goto out_err;
808 file = fdt->fd[fd];
809 if (!file)
810 goto out_err;
811 rcu_assign_pointer(fdt->fd[fd], NULL);
812 __put_unused_fd(files, fd);
813 get_file(file);
814 *res = file;
815 return 0;
816out_err:
817 *res = NULL;
818 return -ENOENT;
819}
820
821/*
Eric W. Biederman9fe83c42020-11-20 17:14:40 -0600822 * variant of close_fd that gets a ref on the file for later fput.
Jens Axboe6e802a42019-12-11 14:10:35 -0700823 * The caller must ensure that filp_close() called on the file, and then
824 * an fput().
Todd Kjos80cd7952018-12-14 15:58:21 -0800825 */
Eric W. Biederman9fe83c42020-11-20 17:14:40 -0600826int close_fd_get_file(unsigned int fd, struct file **res)
Todd Kjos80cd7952018-12-14 15:58:21 -0800827{
828 struct files_struct *files = current->files;
Jens Axboe53dec2e2021-01-19 15:41:52 -0700829 int ret;
Todd Kjos80cd7952018-12-14 15:58:21 -0800830
831 spin_lock(&files->file_lock);
Jens Axboe53dec2e2021-01-19 15:41:52 -0700832 ret = __close_fd_get_file(fd, res);
Todd Kjos80cd7952018-12-14 15:58:21 -0800833 spin_unlock(&files->file_lock);
Todd Kjos80cd7952018-12-14 15:58:21 -0800834
Jens Axboe53dec2e2021-01-19 15:41:52 -0700835 return ret;
Todd Kjos80cd7952018-12-14 15:58:21 -0800836}
837
Al Viro6a6d27d2012-08-21 09:56:33 -0400838void do_close_on_exec(struct files_struct *files)
839{
840 unsigned i;
841 struct fdtable *fdt;
842
843 /* exec unshares first */
Al Viro6a6d27d2012-08-21 09:56:33 -0400844 spin_lock(&files->file_lock);
845 for (i = 0; ; i++) {
846 unsigned long set;
847 unsigned fd = i * BITS_PER_LONG;
848 fdt = files_fdtable(files);
849 if (fd >= fdt->max_fds)
850 break;
851 set = fdt->close_on_exec[i];
852 if (!set)
853 continue;
854 fdt->close_on_exec[i] = 0;
855 for ( ; set ; fd++, set >>= 1) {
856 struct file *file;
857 if (!(set & 1))
858 continue;
859 file = fdt->fd[fd];
860 if (!file)
861 continue;
862 rcu_assign_pointer(fdt->fd[fd], NULL);
863 __put_unused_fd(files, fd);
864 spin_unlock(&files->file_lock);
865 filp_close(file, files);
866 cond_resched();
867 spin_lock(&files->file_lock);
868 }
869
870 }
871 spin_unlock(&files->file_lock);
872}
873
Linus Torvaldsf39825f2021-12-10 14:00:15 -0800874static inline struct file *__fget_files_rcu(struct files_struct *files,
875 unsigned int fd, fmode_t mask, unsigned int refs)
876{
877 for (;;) {
878 struct file *file;
879 struct fdtable *fdt = rcu_dereference_raw(files->fdt);
880 struct file __rcu **fdentry;
881
882 if (unlikely(fd >= fdt->max_fds))
883 return NULL;
884
885 fdentry = fdt->fd + array_index_nospec(fd, fdt->max_fds);
886 file = rcu_dereference_raw(*fdentry);
887 if (unlikely(!file))
888 return NULL;
889
890 if (unlikely(file->f_mode & mask))
891 return NULL;
892
893 /*
894 * Ok, we have a file pointer. However, because we do
895 * this all locklessly under RCU, we may be racing with
896 * that file being closed.
897 *
898 * Such a race can take two forms:
899 *
900 * (a) the file ref already went down to zero,
901 * and get_file_rcu_many() fails. Just try
902 * again:
903 */
904 if (unlikely(!get_file_rcu_many(file, refs)))
905 continue;
906
907 /*
908 * (b) the file table entry has changed under us.
909 * Note that we don't need to re-check the 'fdt->fd'
910 * pointer having changed, because it always goes
911 * hand-in-hand with 'fdt'.
912 *
913 * If so, we need to put our refs and try again.
914 */
915 if (unlikely(rcu_dereference_raw(files->fdt) != fdt) ||
916 unlikely(rcu_dereference_raw(*fdentry) != file)) {
917 fput_many(file, refs);
918 continue;
919 }
920
921 /*
922 * Ok, we have a ref to the file, and checked that it
923 * still exists.
924 */
925 return file;
926 }
927}
928
Sargun Dhillon5e876fb2020-01-07 09:59:24 -0800929static struct file *__fget_files(struct files_struct *files, unsigned int fd,
930 fmode_t mask, unsigned int refs)
Al Viro0ee8cdf2012-08-15 21:12:10 -0400931{
Oleg Nesterov1deb46e2014-01-13 16:48:19 +0100932 struct file *file;
Al Viro0ee8cdf2012-08-15 21:12:10 -0400933
934 rcu_read_lock();
Linus Torvaldsf39825f2021-12-10 14:00:15 -0800935 file = __fget_files_rcu(files, fd, mask, refs);
Al Viro0ee8cdf2012-08-15 21:12:10 -0400936 rcu_read_unlock();
937
938 return file;
939}
940
Sargun Dhillon5e876fb2020-01-07 09:59:24 -0800941static inline struct file *__fget(unsigned int fd, fmode_t mask,
942 unsigned int refs)
943{
944 return __fget_files(current->files, fd, mask, refs);
945}
946
Jens Axboe091141a2018-11-21 10:32:39 -0700947struct file *fget_many(unsigned int fd, unsigned int refs)
948{
949 return __fget(fd, FMODE_PATH, refs);
950}
951
Oleg Nesterov1deb46e2014-01-13 16:48:19 +0100952struct file *fget(unsigned int fd)
953{
Jens Axboe091141a2018-11-21 10:32:39 -0700954 return __fget(fd, FMODE_PATH, 1);
Oleg Nesterov1deb46e2014-01-13 16:48:19 +0100955}
Al Viro0ee8cdf2012-08-15 21:12:10 -0400956EXPORT_SYMBOL(fget);
957
958struct file *fget_raw(unsigned int fd)
959{
Jens Axboe091141a2018-11-21 10:32:39 -0700960 return __fget(fd, 0, 1);
Al Viro0ee8cdf2012-08-15 21:12:10 -0400961}
Al Viro0ee8cdf2012-08-15 21:12:10 -0400962EXPORT_SYMBOL(fget_raw);
963
Sargun Dhillon5e876fb2020-01-07 09:59:24 -0800964struct file *fget_task(struct task_struct *task, unsigned int fd)
965{
966 struct file *file = NULL;
967
968 task_lock(task);
969 if (task->files)
970 file = __fget_files(task->files, fd, 0, 1);
971 task_unlock(task);
972
973 return file;
974}
975
Eric W. Biederman3a879fb2020-11-20 17:14:28 -0600976struct file *task_lookup_fd_rcu(struct task_struct *task, unsigned int fd)
977{
978 /* Must be called with rcu_read_lock held */
979 struct files_struct *files;
980 struct file *file = NULL;
981
982 task_lock(task);
983 files = task->files;
984 if (files)
985 file = files_lookup_fd_rcu(files, fd);
986 task_unlock(task);
987
988 return file;
989}
990
Eric W. Biedermane9a53ae2020-11-20 17:14:31 -0600991struct file *task_lookup_next_fd_rcu(struct task_struct *task, unsigned int *ret_fd)
992{
993 /* Must be called with rcu_read_lock held */
994 struct files_struct *files;
995 unsigned int fd = *ret_fd;
996 struct file *file = NULL;
997
998 task_lock(task);
999 files = task->files;
1000 if (files) {
1001 for (; fd < files_fdtable(files)->max_fds; fd++) {
1002 file = files_lookup_fd_rcu(files, fd);
1003 if (file)
1004 break;
1005 }
1006 }
1007 task_unlock(task);
1008 *ret_fd = fd;
1009 return file;
1010}
1011
Al Viro0ee8cdf2012-08-15 21:12:10 -04001012/*
1013 * Lightweight file lookup - no refcnt increment if fd table isn't shared.
1014 *
1015 * You can use this instead of fget if you satisfy all of the following
1016 * conditions:
1017 * 1) You must call fput_light before exiting the syscall and returning control
1018 * to userspace (i.e. you cannot remember the returned struct file * after
1019 * returning to userspace).
1020 * 2) You must not call filp_close on the returned struct file * in between
1021 * calls to fget_light and fput_light.
1022 * 3) You must not clone the current task in between the calls to fget_light
1023 * and fput_light.
1024 *
1025 * The fput_needed flag returned by fget_light should be passed to the
1026 * corresponding fput_light.
1027 */
Al Virobd2a31d2014-03-04 14:54:22 -05001028static unsigned long __fget_light(unsigned int fd, fmode_t mask)
Al Viro0ee8cdf2012-08-15 21:12:10 -04001029{
Al Viro0ee8cdf2012-08-15 21:12:10 -04001030 struct files_struct *files = current->files;
Oleg Nesterovad461832014-01-13 16:48:40 +01001031 struct file *file;
Al Viro0ee8cdf2012-08-15 21:12:10 -04001032
Al Viro0ee8cdf2012-08-15 21:12:10 -04001033 if (atomic_read(&files->count) == 1) {
Eric W. Biedermanbebf6842020-11-20 17:14:24 -06001034 file = files_lookup_fd_raw(files, fd);
Al Virobd2a31d2014-03-04 14:54:22 -05001035 if (!file || unlikely(file->f_mode & mask))
1036 return 0;
1037 return (unsigned long)file;
Al Viro0ee8cdf2012-08-15 21:12:10 -04001038 } else {
Jens Axboe091141a2018-11-21 10:32:39 -07001039 file = __fget(fd, mask, 1);
Al Virobd2a31d2014-03-04 14:54:22 -05001040 if (!file)
1041 return 0;
1042 return FDPUT_FPUT | (unsigned long)file;
Al Viro0ee8cdf2012-08-15 21:12:10 -04001043 }
Al Viro0ee8cdf2012-08-15 21:12:10 -04001044}
Al Virobd2a31d2014-03-04 14:54:22 -05001045unsigned long __fdget(unsigned int fd)
Oleg Nesterovad461832014-01-13 16:48:40 +01001046{
Al Virobd2a31d2014-03-04 14:54:22 -05001047 return __fget_light(fd, FMODE_PATH);
Oleg Nesterovad461832014-01-13 16:48:40 +01001048}
Al Virobd2a31d2014-03-04 14:54:22 -05001049EXPORT_SYMBOL(__fdget);
Al Viro0ee8cdf2012-08-15 21:12:10 -04001050
Al Virobd2a31d2014-03-04 14:54:22 -05001051unsigned long __fdget_raw(unsigned int fd)
Al Viro0ee8cdf2012-08-15 21:12:10 -04001052{
Al Virobd2a31d2014-03-04 14:54:22 -05001053 return __fget_light(fd, 0);
Al Viro0ee8cdf2012-08-15 21:12:10 -04001054}
Al Virofe17f222012-08-21 11:48:11 -04001055
Al Virobd2a31d2014-03-04 14:54:22 -05001056unsigned long __fdget_pos(unsigned int fd)
1057{
Eric Biggers99aea682014-03-16 15:47:48 -05001058 unsigned long v = __fdget(fd);
1059 struct file *file = (struct file *)(v & ~3);
Al Virobd2a31d2014-03-04 14:54:22 -05001060
Linus Torvalds2be7d342019-11-26 11:34:06 -08001061 if (file && (file->f_mode & FMODE_ATOMIC_POS)) {
Al Virobd2a31d2014-03-04 14:54:22 -05001062 if (file_count(file) > 1) {
1063 v |= FDPUT_POS_UNLOCK;
1064 mutex_lock(&file->f_pos_lock);
1065 }
1066 }
Eric Biggers99aea682014-03-16 15:47:48 -05001067 return v;
Al Virobd2a31d2014-03-04 14:54:22 -05001068}
1069
Al Viro63b6df12016-04-20 17:08:21 -04001070void __f_unlock_pos(struct file *f)
1071{
1072 mutex_unlock(&f->f_pos_lock);
1073}
1074
Al Virobd2a31d2014-03-04 14:54:22 -05001075/*
1076 * We only lock f_pos if we have threads or if the file might be
1077 * shared with another process. In both cases we'll have an elevated
1078 * file count (done either by fdget() or by fork()).
1079 */
1080
Al Virofe17f222012-08-21 11:48:11 -04001081void set_close_on_exec(unsigned int fd, int flag)
1082{
1083 struct files_struct *files = current->files;
1084 struct fdtable *fdt;
1085 spin_lock(&files->file_lock);
1086 fdt = files_fdtable(files);
1087 if (flag)
1088 __set_close_on_exec(fd, fdt);
1089 else
1090 __clear_close_on_exec(fd, fdt);
1091 spin_unlock(&files->file_lock);
1092}
1093
1094bool get_close_on_exec(unsigned int fd)
1095{
1096 struct files_struct *files = current->files;
1097 struct fdtable *fdt;
1098 bool res;
1099 rcu_read_lock();
1100 fdt = files_fdtable(files);
1101 res = close_on_exec(fd, fdt);
1102 rcu_read_unlock();
1103 return res;
1104}
1105
Al Viro8280d162012-08-21 12:11:46 -04001106static int do_dup2(struct files_struct *files,
1107 struct file *file, unsigned fd, unsigned flags)
Al Viroe9830942014-08-31 14:12:09 -04001108__releases(&files->file_lock)
Al Viro8280d162012-08-21 12:11:46 -04001109{
1110 struct file *tofree;
1111 struct fdtable *fdt;
1112
1113 /*
1114 * We need to detect attempts to do dup2() over allocated but still
1115 * not finished descriptor. NB: OpenBSD avoids that at the price of
1116 * extra work in their equivalent of fget() - they insert struct
1117 * file immediately after grabbing descriptor, mark it larval if
1118 * more work (e.g. actual opening) is needed and make sure that
1119 * fget() treats larval files as absent. Potentially interesting,
1120 * but while extra work in fget() is trivial, locking implications
1121 * and amount of surgery on open()-related paths in VFS are not.
1122 * FreeBSD fails with -EBADF in the same situation, NetBSD "solution"
1123 * deadlocks in rather amusing ways, AFAICS. All of that is out of
1124 * scope of POSIX or SUS, since neither considers shared descriptor
1125 * tables and this condition does not arise without those.
1126 */
1127 fdt = files_fdtable(files);
1128 tofree = fdt->fd[fd];
1129 if (!tofree && fd_is_open(fd, fdt))
1130 goto Ebusy;
1131 get_file(file);
1132 rcu_assign_pointer(fdt->fd[fd], file);
1133 __set_open_fd(fd, fdt);
1134 if (flags & O_CLOEXEC)
1135 __set_close_on_exec(fd, fdt);
1136 else
1137 __clear_close_on_exec(fd, fdt);
1138 spin_unlock(&files->file_lock);
1139
1140 if (tofree)
1141 filp_close(tofree, files);
1142
1143 return fd;
1144
1145Ebusy:
1146 spin_unlock(&files->file_lock);
1147 return -EBUSY;
1148}
1149
1150int replace_fd(unsigned fd, struct file *file, unsigned flags)
1151{
1152 int err;
1153 struct files_struct *files = current->files;
1154
1155 if (!file)
Eric W. Biederman8760c902020-11-20 17:14:38 -06001156 return close_fd(fd);
Al Viro8280d162012-08-21 12:11:46 -04001157
1158 if (fd >= rlimit(RLIMIT_NOFILE))
Al Viro08f05c42012-10-31 03:37:48 +00001159 return -EBADF;
Al Viro8280d162012-08-21 12:11:46 -04001160
1161 spin_lock(&files->file_lock);
1162 err = expand_files(files, fd);
1163 if (unlikely(err < 0))
1164 goto out_unlock;
1165 return do_dup2(files, file, fd, flags);
1166
1167out_unlock:
1168 spin_unlock(&files->file_lock);
1169 return err;
1170}
1171
Kees Cook66590612020-06-10 08:20:05 -07001172/**
1173 * __receive_fd() - Install received file into file descriptor table
Kees Cook66590612020-06-10 08:20:05 -07001174 * @file: struct file that was received from another process
1175 * @ufd: __user pointer to write new fd number to
1176 * @o_flags: the O_* flags to apply to the new fd entry
1177 *
1178 * Installs a received file into the file descriptor table, with appropriate
Kees Cookdeefa7f2020-06-10 20:47:45 -07001179 * checks and count updates. Optionally writes the fd number to userspace, if
1180 * @ufd is non-NULL.
Kees Cook66590612020-06-10 08:20:05 -07001181 *
1182 * This helper handles its own reference counting of the incoming
1183 * struct file.
1184 *
Kees Cookdeefa7f2020-06-10 20:47:45 -07001185 * Returns newly install fd or -ve on error.
Kees Cook66590612020-06-10 08:20:05 -07001186 */
Christoph Hellwig42eb0d52021-03-25 09:22:09 +01001187int __receive_fd(struct file *file, int __user *ufd, unsigned int o_flags)
Kees Cook66590612020-06-10 08:20:05 -07001188{
1189 int new_fd;
1190 int error;
1191
1192 error = security_file_receive(file);
1193 if (error)
1194 return error;
1195
Christoph Hellwig42eb0d52021-03-25 09:22:09 +01001196 new_fd = get_unused_fd_flags(o_flags);
1197 if (new_fd < 0)
1198 return new_fd;
Kees Cook66590612020-06-10 08:20:05 -07001199
Kees Cookdeefa7f2020-06-10 20:47:45 -07001200 if (ufd) {
1201 error = put_user(new_fd, ufd);
1202 if (error) {
Christoph Hellwig42eb0d52021-03-25 09:22:09 +01001203 put_unused_fd(new_fd);
Kees Cookdeefa7f2020-06-10 20:47:45 -07001204 return error;
1205 }
Kees Cook66590612020-06-10 08:20:05 -07001206 }
1207
Christoph Hellwig42eb0d52021-03-25 09:22:09 +01001208 fd_install(new_fd, get_file(file));
1209 __receive_sock(file);
1210 return new_fd;
1211}
Kees Cook173817152020-06-10 08:46:58 -07001212
Christoph Hellwig42eb0d52021-03-25 09:22:09 +01001213int receive_fd_replace(int new_fd, struct file *file, unsigned int o_flags)
1214{
1215 int error;
1216
1217 error = security_file_receive(file);
1218 if (error)
1219 return error;
1220 error = replace_fd(new_fd, file, o_flags);
1221 if (error)
1222 return error;
Kees Cook66590612020-06-10 08:20:05 -07001223 __receive_sock(file);
Kees Cookdeefa7f2020-06-10 20:47:45 -07001224 return new_fd;
Kees Cook66590612020-06-10 08:20:05 -07001225}
1226
Xie Yongji9c930052021-08-31 18:36:24 +08001227int receive_fd(struct file *file, unsigned int o_flags)
1228{
1229 return __receive_fd(file, NULL, o_flags);
1230}
1231EXPORT_SYMBOL_GPL(receive_fd);
1232
Dominik Brodowskic7248322018-03-11 11:34:40 +01001233static int ksys_dup3(unsigned int oldfd, unsigned int newfd, int flags)
Al Virofe17f222012-08-21 11:48:11 -04001234{
1235 int err = -EBADF;
Al Viro8280d162012-08-21 12:11:46 -04001236 struct file *file;
1237 struct files_struct *files = current->files;
Al Virofe17f222012-08-21 11:48:11 -04001238
1239 if ((flags & ~O_CLOEXEC) != 0)
1240 return -EINVAL;
1241
Richard W.M. Jonesaed97642012-10-09 15:27:43 +01001242 if (unlikely(oldfd == newfd))
1243 return -EINVAL;
1244
Al Virofe17f222012-08-21 11:48:11 -04001245 if (newfd >= rlimit(RLIMIT_NOFILE))
Al Viro08f05c42012-10-31 03:37:48 +00001246 return -EBADF;
Al Virofe17f222012-08-21 11:48:11 -04001247
1248 spin_lock(&files->file_lock);
1249 err = expand_files(files, newfd);
Eric W. Biederman120ce2b2020-11-20 17:14:25 -06001250 file = files_lookup_fd_locked(files, oldfd);
Al Virofe17f222012-08-21 11:48:11 -04001251 if (unlikely(!file))
1252 goto Ebadf;
1253 if (unlikely(err < 0)) {
1254 if (err == -EMFILE)
1255 goto Ebadf;
1256 goto out_unlock;
1257 }
Al Viro8280d162012-08-21 12:11:46 -04001258 return do_dup2(files, file, newfd, flags);
Al Virofe17f222012-08-21 11:48:11 -04001259
1260Ebadf:
1261 err = -EBADF;
1262out_unlock:
1263 spin_unlock(&files->file_lock);
1264 return err;
1265}
1266
Dominik Brodowskic7248322018-03-11 11:34:40 +01001267SYSCALL_DEFINE3(dup3, unsigned int, oldfd, unsigned int, newfd, int, flags)
1268{
1269 return ksys_dup3(oldfd, newfd, flags);
1270}
1271
Al Virofe17f222012-08-21 11:48:11 -04001272SYSCALL_DEFINE2(dup2, unsigned int, oldfd, unsigned int, newfd)
1273{
1274 if (unlikely(newfd == oldfd)) { /* corner case */
1275 struct files_struct *files = current->files;
1276 int retval = oldfd;
1277
1278 rcu_read_lock();
Eric W. Biedermanf36c2942020-11-20 17:14:26 -06001279 if (!files_lookup_fd_rcu(files, oldfd))
Al Virofe17f222012-08-21 11:48:11 -04001280 retval = -EBADF;
1281 rcu_read_unlock();
1282 return retval;
1283 }
Dominik Brodowskic7248322018-03-11 11:34:40 +01001284 return ksys_dup3(oldfd, newfd, 0);
Al Virofe17f222012-08-21 11:48:11 -04001285}
1286
Christoph Hellwigbc1cd992020-07-14 08:58:49 +02001287SYSCALL_DEFINE1(dup, unsigned int, fildes)
Al Virofe17f222012-08-21 11:48:11 -04001288{
1289 int ret = -EBADF;
1290 struct file *file = fget_raw(fildes);
1291
1292 if (file) {
Yann Droneaud8d10a032014-12-10 15:45:44 -08001293 ret = get_unused_fd_flags(0);
Al Virofe17f222012-08-21 11:48:11 -04001294 if (ret >= 0)
1295 fd_install(ret, file);
1296 else
1297 fput(file);
1298 }
1299 return ret;
1300}
1301
1302int f_dupfd(unsigned int from, struct file *file, unsigned flags)
1303{
Eric W. Biedermane06b53c2020-11-20 17:14:36 -06001304 unsigned long nofile = rlimit(RLIMIT_NOFILE);
Al Virofe17f222012-08-21 11:48:11 -04001305 int err;
Eric W. Biedermane06b53c2020-11-20 17:14:36 -06001306 if (from >= nofile)
Al Virofe17f222012-08-21 11:48:11 -04001307 return -EINVAL;
Eric W. Biedermane06b53c2020-11-20 17:14:36 -06001308 err = alloc_fd(from, nofile, flags);
Al Virofe17f222012-08-21 11:48:11 -04001309 if (err >= 0) {
1310 get_file(file);
1311 fd_install(err, file);
1312 }
1313 return err;
1314}
Al Viroc3c073f2012-08-21 22:32:06 -04001315
1316int iterate_fd(struct files_struct *files, unsigned n,
1317 int (*f)(const void *, struct file *, unsigned),
1318 const void *p)
1319{
1320 struct fdtable *fdt;
Al Viroc3c073f2012-08-21 22:32:06 -04001321 int res = 0;
1322 if (!files)
1323 return 0;
1324 spin_lock(&files->file_lock);
Al Viroa77cfcb2012-11-29 22:57:33 -05001325 for (fdt = files_fdtable(files); n < fdt->max_fds; n++) {
1326 struct file *file;
1327 file = rcu_dereference_check_fdtable(files, fdt->fd[n]);
1328 if (!file)
1329 continue;
1330 res = f(p, file, n);
1331 if (res)
1332 break;
Al Viroc3c073f2012-08-21 22:32:06 -04001333 }
1334 spin_unlock(&files->file_lock);
1335 return res;
1336}
1337EXPORT_SYMBOL(iterate_fd);