blob: 9e2b223ebde30775fe8e24b9850b6d23c26bb769 [file] [log] [blame]
Thomas Gleixner457c8992019-05-19 13:08:55 +01001// SPDX-License-Identifier: GPL-2.0-only
Andrew Morton16d69262008-07-25 19:44:36 -07002#include <linux/mm.h>
Matt Mackall30992c92006-01-08 01:01:43 -08003#include <linux/slab.h>
4#include <linux/string.h>
Gideon Israel Dsouza3b321232014-04-07 15:37:26 -07005#include <linux/compiler.h>
Paul Gortmakerb95f1b312011-10-16 02:01:52 -04006#include <linux/export.h>
Davi Arnaut96840aa2006-03-24 03:18:42 -08007#include <linux/err.h>
Adrian Bunk3b8f14b2008-07-26 15:22:28 -07008#include <linux/sched.h>
Ingo Molnar6e84f312017-02-08 18:51:29 +01009#include <linux/sched/mm.h>
Daniel Jordan79eb5972019-07-16 16:30:54 -070010#include <linux/sched/signal.h>
Ingo Molnar68db0cf2017-02-08 18:51:37 +010011#include <linux/sched/task_stack.h>
Al Viroeb36c582012-05-30 20:17:35 -040012#include <linux/security.h>
Shaohua Li98003392013-02-22 16:34:35 -080013#include <linux/swap.h>
Shaohua Li33806f02013-02-22 16:34:37 -080014#include <linux/swapops.h>
Jerome Marchand00619bc2013-11-12 15:08:31 -080015#include <linux/mman.h>
16#include <linux/hugetlb.h>
Al Viro39f1f782014-05-06 14:02:53 -040017#include <linux/vmalloc.h>
Mike Rapoport897ab3e2017-02-24 14:58:22 -080018#include <linux/userfaultfd_k.h>
Alexandre Ghiti649775b2019-09-23 15:38:37 -070019#include <linux/elf.h>
Alexandre Ghiti67f39772019-09-23 15:38:47 -070020#include <linux/elf-randomize.h>
21#include <linux/personality.h>
Alexandre Ghiti649775b2019-09-23 15:38:37 -070022#include <linux/random.h>
Alexandre Ghiti67f39772019-09-23 15:38:47 -070023#include <linux/processor.h>
24#include <linux/sizes.h>
25#include <linux/compat.h>
Jerome Marchand00619bc2013-11-12 15:08:31 -080026
Linus Torvalds7c0f6ba2016-12-24 11:46:01 -080027#include <linux/uaccess.h>
Matt Mackall30992c92006-01-08 01:01:43 -080028
Namhyung Kim6038def2011-05-24 17:11:22 -070029#include "internal.h"
Kuan-Ying Leea5543c92021-06-18 13:18:59 +080030#ifndef __GENKSYMS__
31#include <trace/hooks/syscall_check.h>
32#endif
Namhyung Kim6038def2011-05-24 17:11:22 -070033
Andrzej Hajdaa4bb1e42015-02-13 14:36:24 -080034/**
35 * kfree_const - conditionally free memory
36 * @x: pointer to the memory
37 *
38 * Function calls kfree only if @x is not in .rodata section.
39 */
40void kfree_const(const void *x)
41{
42 if (!is_kernel_rodata((unsigned long)x))
43 kfree(x);
44}
45EXPORT_SYMBOL(kfree_const);
46
Matt Mackall30992c92006-01-08 01:01:43 -080047/**
Matt Mackall30992c92006-01-08 01:01:43 -080048 * kstrdup - allocate space for and copy an existing string
Matt Mackall30992c92006-01-08 01:01:43 -080049 * @s: the string to duplicate
50 * @gfp: the GFP mask used in the kmalloc() call when allocating memory
Mike Rapoporta862f682019-03-05 15:48:42 -080051 *
52 * Return: newly allocated copy of @s or %NULL in case of error
Matt Mackall30992c92006-01-08 01:01:43 -080053 */
54char *kstrdup(const char *s, gfp_t gfp)
55{
56 size_t len;
57 char *buf;
58
59 if (!s)
60 return NULL;
61
62 len = strlen(s) + 1;
Christoph Hellwig1d2c8ee2006-10-04 02:15:25 -070063 buf = kmalloc_track_caller(len, gfp);
Matt Mackall30992c92006-01-08 01:01:43 -080064 if (buf)
65 memcpy(buf, s, len);
66 return buf;
67}
68EXPORT_SYMBOL(kstrdup);
Davi Arnaut96840aa2006-03-24 03:18:42 -080069
Alexey Dobriyan1a2f67b2006-09-30 23:27:20 -070070/**
Andrzej Hajdaa4bb1e42015-02-13 14:36:24 -080071 * kstrdup_const - conditionally duplicate an existing const string
72 * @s: the string to duplicate
73 * @gfp: the GFP mask used in the kmalloc() call when allocating memory
74 *
Bartosz Golaszewski295a1732020-10-15 20:07:39 -070075 * Note: Strings allocated by kstrdup_const should be freed by kfree_const and
76 * must not be passed to krealloc().
Mike Rapoporta862f682019-03-05 15:48:42 -080077 *
78 * Return: source string if it is in .rodata section otherwise
79 * fallback to kstrdup.
Andrzej Hajdaa4bb1e42015-02-13 14:36:24 -080080 */
81const char *kstrdup_const(const char *s, gfp_t gfp)
82{
83 if (is_kernel_rodata((unsigned long)s))
84 return s;
85
86 return kstrdup(s, gfp);
87}
88EXPORT_SYMBOL(kstrdup_const);
89
90/**
Jeremy Fitzhardinge1e66df32007-07-17 18:37:02 -070091 * kstrndup - allocate space for and copy an existing string
92 * @s: the string to duplicate
93 * @max: read at most @max chars from @s
94 * @gfp: the GFP mask used in the kmalloc() call when allocating memory
David Howellsf3515742017-07-04 17:25:02 +010095 *
96 * Note: Use kmemdup_nul() instead if the size is known exactly.
Mike Rapoporta862f682019-03-05 15:48:42 -080097 *
98 * Return: newly allocated copy of @s or %NULL in case of error
Jeremy Fitzhardinge1e66df32007-07-17 18:37:02 -070099 */
100char *kstrndup(const char *s, size_t max, gfp_t gfp)
101{
102 size_t len;
103 char *buf;
104
105 if (!s)
106 return NULL;
107
108 len = strnlen(s, max);
109 buf = kmalloc_track_caller(len+1, gfp);
110 if (buf) {
111 memcpy(buf, s, len);
112 buf[len] = '\0';
113 }
114 return buf;
115}
116EXPORT_SYMBOL(kstrndup);
117
118/**
Alexey Dobriyan1a2f67b2006-09-30 23:27:20 -0700119 * kmemdup - duplicate region of memory
120 *
121 * @src: memory region to duplicate
122 * @len: memory region length
123 * @gfp: GFP mask to use
Mike Rapoporta862f682019-03-05 15:48:42 -0800124 *
125 * Return: newly allocated copy of @src or %NULL in case of error
Alexey Dobriyan1a2f67b2006-09-30 23:27:20 -0700126 */
127void *kmemdup(const void *src, size_t len, gfp_t gfp)
128{
129 void *p;
130
Christoph Hellwig1d2c8ee2006-10-04 02:15:25 -0700131 p = kmalloc_track_caller(len, gfp);
Alexey Dobriyan1a2f67b2006-09-30 23:27:20 -0700132 if (p)
133 memcpy(p, src, len);
134 return p;
135}
136EXPORT_SYMBOL(kmemdup);
137
Christoph Lameteref2ad802007-07-17 04:03:21 -0700138/**
David Howellsf3515742017-07-04 17:25:02 +0100139 * kmemdup_nul - Create a NUL-terminated string from unterminated data
140 * @s: The data to stringify
141 * @len: The size of the data
142 * @gfp: the GFP mask used in the kmalloc() call when allocating memory
Mike Rapoporta862f682019-03-05 15:48:42 -0800143 *
144 * Return: newly allocated copy of @s with NUL-termination or %NULL in
145 * case of error
David Howellsf3515742017-07-04 17:25:02 +0100146 */
147char *kmemdup_nul(const char *s, size_t len, gfp_t gfp)
148{
149 char *buf;
150
151 if (!s)
152 return NULL;
153
154 buf = kmalloc_track_caller(len + 1, gfp);
155 if (buf) {
156 memcpy(buf, s, len);
157 buf[len] = '\0';
158 }
159 return buf;
160}
161EXPORT_SYMBOL(kmemdup_nul);
162
163/**
Li Zefan610a77e2009-03-31 15:23:16 -0700164 * memdup_user - duplicate memory region from user space
165 *
166 * @src: source address in user space
167 * @len: number of bytes to copy
168 *
Mike Rapoporta862f682019-03-05 15:48:42 -0800169 * Return: an ERR_PTR() on failure. Result is physically
Al Viro50fd2f22018-01-07 13:06:15 -0500170 * contiguous, to be freed by kfree().
Li Zefan610a77e2009-03-31 15:23:16 -0700171 */
172void *memdup_user(const void __user *src, size_t len)
173{
174 void *p;
175
Daniel Vetter6c8fcc02019-02-20 22:20:42 -0800176 p = kmalloc_track_caller(len, GFP_USER | __GFP_NOWARN);
Li Zefan610a77e2009-03-31 15:23:16 -0700177 if (!p)
178 return ERR_PTR(-ENOMEM);
179
180 if (copy_from_user(p, src, len)) {
181 kfree(p);
182 return ERR_PTR(-EFAULT);
183 }
184
185 return p;
186}
187EXPORT_SYMBOL(memdup_user);
188
Al Viro50fd2f22018-01-07 13:06:15 -0500189/**
190 * vmemdup_user - duplicate memory region from user space
191 *
192 * @src: source address in user space
193 * @len: number of bytes to copy
194 *
Mike Rapoporta862f682019-03-05 15:48:42 -0800195 * Return: an ERR_PTR() on failure. Result may be not
Al Viro50fd2f22018-01-07 13:06:15 -0500196 * physically contiguous. Use kvfree() to free.
197 */
198void *vmemdup_user(const void __user *src, size_t len)
199{
200 void *p;
201
202 p = kvmalloc(len, GFP_USER);
203 if (!p)
204 return ERR_PTR(-ENOMEM);
205
206 if (copy_from_user(p, src, len)) {
207 kvfree(p);
208 return ERR_PTR(-EFAULT);
209 }
210
211 return p;
212}
213EXPORT_SYMBOL(vmemdup_user);
214
Mike Rapoportb86181f2018-08-23 17:00:59 -0700215/**
Davi Arnaut96840aa2006-03-24 03:18:42 -0800216 * strndup_user - duplicate an existing string from user space
Davi Arnaut96840aa2006-03-24 03:18:42 -0800217 * @s: The string to duplicate
218 * @n: Maximum number of bytes to copy, including the trailing NUL.
Mike Rapoporta862f682019-03-05 15:48:42 -0800219 *
Andrew Mortone9145522019-04-05 18:39:34 -0700220 * Return: newly allocated copy of @s or an ERR_PTR() in case of error
Davi Arnaut96840aa2006-03-24 03:18:42 -0800221 */
222char *strndup_user(const char __user *s, long n)
223{
224 char *p;
225 long length;
226
227 length = strnlen_user(s, n);
228
229 if (!length)
230 return ERR_PTR(-EFAULT);
231
232 if (length > n)
233 return ERR_PTR(-EINVAL);
234
Julia Lawall90d74042010-08-09 17:18:26 -0700235 p = memdup_user(s, length);
Davi Arnaut96840aa2006-03-24 03:18:42 -0800236
Julia Lawall90d74042010-08-09 17:18:26 -0700237 if (IS_ERR(p))
238 return p;
Davi Arnaut96840aa2006-03-24 03:18:42 -0800239
240 p[length - 1] = '\0';
241
242 return p;
243}
244EXPORT_SYMBOL(strndup_user);
Andrew Morton16d69262008-07-25 19:44:36 -0700245
Al Viroe9d408e2015-12-24 00:06:05 -0500246/**
247 * memdup_user_nul - duplicate memory region from user space and NUL-terminate
248 *
249 * @src: source address in user space
250 * @len: number of bytes to copy
251 *
Mike Rapoporta862f682019-03-05 15:48:42 -0800252 * Return: an ERR_PTR() on failure.
Al Viroe9d408e2015-12-24 00:06:05 -0500253 */
254void *memdup_user_nul(const void __user *src, size_t len)
255{
256 char *p;
257
258 /*
259 * Always use GFP_KERNEL, since copy_from_user() can sleep and
260 * cause pagefault, which makes it pointless to use GFP_NOFS
261 * or GFP_ATOMIC.
262 */
263 p = kmalloc_track_caller(len + 1, GFP_KERNEL);
264 if (!p)
265 return ERR_PTR(-ENOMEM);
266
267 if (copy_from_user(p, src, len)) {
268 kfree(p);
269 return ERR_PTR(-EFAULT);
270 }
271 p[len] = '\0';
272
273 return p;
274}
275EXPORT_SYMBOL(memdup_user_nul);
276
Namhyung Kim6038def2011-05-24 17:11:22 -0700277void __vma_link_list(struct mm_struct *mm, struct vm_area_struct *vma,
Wei Yangaba6dfb2019-11-30 17:50:53 -0800278 struct vm_area_struct *prev)
Namhyung Kim6038def2011-05-24 17:11:22 -0700279{
280 struct vm_area_struct *next;
281
282 vma->vm_prev = prev;
283 if (prev) {
284 next = prev->vm_next;
285 prev->vm_next = vma;
286 } else {
Wei Yangaba6dfb2019-11-30 17:50:53 -0800287 next = mm->mmap;
Namhyung Kim6038def2011-05-24 17:11:22 -0700288 mm->mmap = vma;
Namhyung Kim6038def2011-05-24 17:11:22 -0700289 }
290 vma->vm_next = next;
291 if (next)
292 next->vm_prev = vma;
293}
294
Wei Yang1b9fc5b22019-11-30 17:50:49 -0800295void __vma_unlink_list(struct mm_struct *mm, struct vm_area_struct *vma)
296{
297 struct vm_area_struct *prev, *next;
298
299 next = vma->vm_next;
300 prev = vma->vm_prev;
301 if (prev)
302 prev->vm_next = next;
303 else
304 mm->mmap = next;
305 if (next)
306 next->vm_prev = prev;
307}
308
Siddhesh Poyarekarb7643752012-03-21 16:34:04 -0700309/* Check if the vma is being used as a stack by this task */
Andy Lutomirskid17af502016-09-30 10:58:58 -0700310int vma_is_stack_for_current(struct vm_area_struct *vma)
Siddhesh Poyarekarb7643752012-03-21 16:34:04 -0700311{
Andy Lutomirskid17af502016-09-30 10:58:58 -0700312 struct task_struct * __maybe_unused t = current;
313
Siddhesh Poyarekarb7643752012-03-21 16:34:04 -0700314 return (vma->vm_start <= KSTK_ESP(t) && vma->vm_end >= KSTK_ESP(t));
315}
316
Alexandre Ghiti649775b2019-09-23 15:38:37 -0700317#ifndef STACK_RND_MASK
318#define STACK_RND_MASK (0x7ff >> (PAGE_SHIFT - 12)) /* 8MB of VA */
319#endif
320
321unsigned long randomize_stack_top(unsigned long stack_top)
322{
323 unsigned long random_variable = 0;
324
325 if (current->flags & PF_RANDOMIZE) {
326 random_variable = get_random_long();
327 random_variable &= STACK_RND_MASK;
328 random_variable <<= PAGE_SHIFT;
329 }
330#ifdef CONFIG_STACK_GROWSUP
331 return PAGE_ALIGN(stack_top) + random_variable;
332#else
333 return PAGE_ALIGN(stack_top) - random_variable;
334#endif
335}
336
Jason A. Donenfeld552ae8e2022-05-14 13:59:30 +0200337/**
338 * randomize_page - Generate a random, page aligned address
339 * @start: The smallest acceptable address the caller will take.
340 * @range: The size of the area, starting at @start, within which the
341 * random address must fall.
342 *
343 * If @start + @range would overflow, @range is capped.
344 *
345 * NOTE: Historical use of randomize_range, which this replaces, presumed that
346 * @start was already page aligned. We now align it regardless.
347 *
348 * Return: A page aligned address within [start, start + range). On error,
349 * @start is returned.
350 */
351unsigned long randomize_page(unsigned long start, unsigned long range)
352{
353 if (!PAGE_ALIGNED(start)) {
354 range -= PAGE_ALIGN(start) - start;
355 start = PAGE_ALIGN(start);
356 }
357
358 if (start > ULONG_MAX - range)
359 range = ULONG_MAX - start;
360
361 range >>= PAGE_SHIFT;
362
363 if (range == 0)
364 return start;
365
366 return start + (get_random_long() % range << PAGE_SHIFT);
367}
368
Alexandre Ghiti67f39772019-09-23 15:38:47 -0700369#ifdef CONFIG_ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT
Alexandre Ghitie7142bf2019-09-23 15:38:50 -0700370unsigned long arch_randomize_brk(struct mm_struct *mm)
371{
372 /* Is the current task 32bit ? */
373 if (!IS_ENABLED(CONFIG_64BIT) || is_compat_task())
374 return randomize_page(mm->brk, SZ_32M);
375
376 return randomize_page(mm->brk, SZ_1G);
377}
378
Alexandre Ghiti67f39772019-09-23 15:38:47 -0700379unsigned long arch_mmap_rnd(void)
380{
381 unsigned long rnd;
382
383#ifdef CONFIG_HAVE_ARCH_MMAP_RND_COMPAT_BITS
384 if (is_compat_task())
385 rnd = get_random_long() & ((1UL << mmap_rnd_compat_bits) - 1);
386 else
387#endif /* CONFIG_HAVE_ARCH_MMAP_RND_COMPAT_BITS */
388 rnd = get_random_long() & ((1UL << mmap_rnd_bits) - 1);
389
390 return rnd << PAGE_SHIFT;
391}
xieliujied80c70d2021-07-03 16:59:16 +0800392EXPORT_SYMBOL_GPL(arch_mmap_rnd);
Alexandre Ghiti67f39772019-09-23 15:38:47 -0700393
394static int mmap_is_legacy(struct rlimit *rlim_stack)
395{
396 if (current->personality & ADDR_COMPAT_LAYOUT)
397 return 1;
398
399 if (rlim_stack->rlim_cur == RLIM_INFINITY)
400 return 1;
401
402 return sysctl_legacy_va_layout;
403}
404
405/*
406 * Leave enough space between the mmap area and the stack to honour ulimit in
407 * the face of randomisation.
408 */
409#define MIN_GAP (SZ_128M)
410#define MAX_GAP (STACK_TOP / 6 * 5)
411
412static unsigned long mmap_base(unsigned long rnd, struct rlimit *rlim_stack)
413{
414 unsigned long gap = rlim_stack->rlim_cur;
415 unsigned long pad = stack_guard_gap;
416
417 /* Account for stack randomization if necessary */
418 if (current->flags & PF_RANDOMIZE)
419 pad += (STACK_RND_MASK << PAGE_SHIFT);
420
421 /* Values close to RLIM_INFINITY can overflow. */
422 if (gap + pad > gap)
423 gap += pad;
424
425 if (gap < MIN_GAP)
426 gap = MIN_GAP;
427 else if (gap > MAX_GAP)
428 gap = MAX_GAP;
429
430 return PAGE_ALIGN(STACK_TOP - gap - rnd);
431}
432
433void arch_pick_mmap_layout(struct mm_struct *mm, struct rlimit *rlim_stack)
434{
435 unsigned long random_factor = 0UL;
436
437 if (current->flags & PF_RANDOMIZE)
438 random_factor = arch_mmap_rnd();
439
440 if (mmap_is_legacy(rlim_stack)) {
441 mm->mmap_base = TASK_UNMAPPED_BASE + random_factor;
442 mm->get_unmapped_area = arch_get_unmapped_area;
443 } else {
444 mm->mmap_base = mmap_base(random_factor, rlim_stack);
445 mm->get_unmapped_area = arch_get_unmapped_area_topdown;
446 }
447}
448#elif defined(CONFIG_MMU) && !defined(HAVE_ARCH_PICK_MMAP_LAYOUT)
Kees Cook8f2af152018-04-10 16:34:53 -0700449void arch_pick_mmap_layout(struct mm_struct *mm, struct rlimit *rlim_stack)
Andrew Morton16d69262008-07-25 19:44:36 -0700450{
451 mm->mmap_base = TASK_UNMAPPED_BASE;
452 mm->get_unmapped_area = arch_get_unmapped_area;
Andrew Morton16d69262008-07-25 19:44:36 -0700453}
454#endif
Rusty Russell912985d2008-08-12 17:52:52 -0500455
Daniel Jordan79eb5972019-07-16 16:30:54 -0700456/**
457 * __account_locked_vm - account locked pages to an mm's locked_vm
458 * @mm: mm to account against
459 * @pages: number of pages to account
460 * @inc: %true if @pages should be considered positive, %false if not
461 * @task: task used to check RLIMIT_MEMLOCK
462 * @bypass_rlim: %true if checking RLIMIT_MEMLOCK should be skipped
463 *
464 * Assumes @task and @mm are valid (i.e. at least one reference on each), and
Michel Lespinassec1e8d7c2020-06-08 21:33:54 -0700465 * that mmap_lock is held as writer.
Daniel Jordan79eb5972019-07-16 16:30:54 -0700466 *
467 * Return:
468 * * 0 on success
469 * * -ENOMEM if RLIMIT_MEMLOCK would be exceeded.
470 */
471int __account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc,
472 struct task_struct *task, bool bypass_rlim)
473{
474 unsigned long locked_vm, limit;
475 int ret = 0;
476
Michel Lespinasse42fc5412020-06-08 21:33:44 -0700477 mmap_assert_write_locked(mm);
Daniel Jordan79eb5972019-07-16 16:30:54 -0700478
479 locked_vm = mm->locked_vm;
480 if (inc) {
481 if (!bypass_rlim) {
482 limit = task_rlimit(task, RLIMIT_MEMLOCK) >> PAGE_SHIFT;
483 if (locked_vm + pages > limit)
484 ret = -ENOMEM;
485 }
486 if (!ret)
487 mm->locked_vm = locked_vm + pages;
488 } else {
489 WARN_ON_ONCE(pages > locked_vm);
490 mm->locked_vm = locked_vm - pages;
491 }
492
493 pr_debug("%s: [%d] caller %ps %c%lu %lu/%lu%s\n", __func__, task->pid,
494 (void *)_RET_IP_, (inc) ? '+' : '-', pages << PAGE_SHIFT,
495 locked_vm << PAGE_SHIFT, task_rlimit(task, RLIMIT_MEMLOCK),
496 ret ? " - exceeded" : "");
497
498 return ret;
499}
500EXPORT_SYMBOL_GPL(__account_locked_vm);
501
502/**
503 * account_locked_vm - account locked pages to an mm's locked_vm
504 * @mm: mm to account against, may be NULL
505 * @pages: number of pages to account
506 * @inc: %true if @pages should be considered positive, %false if not
507 *
508 * Assumes a non-NULL @mm is valid (i.e. at least one reference on it).
509 *
510 * Return:
511 * * 0 on success, or if mm is NULL
512 * * -ENOMEM if RLIMIT_MEMLOCK would be exceeded.
513 */
514int account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc)
515{
516 int ret;
517
518 if (pages == 0 || !mm)
519 return 0;
520
Michel Lespinassed8ed45c2020-06-08 21:33:25 -0700521 mmap_write_lock(mm);
Daniel Jordan79eb5972019-07-16 16:30:54 -0700522 ret = __account_locked_vm(mm, pages, inc, current,
523 capable(CAP_IPC_LOCK));
Michel Lespinassed8ed45c2020-06-08 21:33:25 -0700524 mmap_write_unlock(mm);
Daniel Jordan79eb5972019-07-16 16:30:54 -0700525
526 return ret;
527}
528EXPORT_SYMBOL_GPL(account_locked_vm);
529
Al Viroeb36c582012-05-30 20:17:35 -0400530unsigned long vm_mmap_pgoff(struct file *file, unsigned long addr,
531 unsigned long len, unsigned long prot,
Michal Hocko9fbeb5a2016-05-23 16:25:30 -0700532 unsigned long flag, unsigned long pgoff)
Al Viroeb36c582012-05-30 20:17:35 -0400533{
534 unsigned long ret;
535 struct mm_struct *mm = current->mm;
Michel Lespinasse41badc12013-02-22 16:32:47 -0800536 unsigned long populate;
Mike Rapoport897ab3e2017-02-24 14:58:22 -0800537 LIST_HEAD(uf);
Al Viroeb36c582012-05-30 20:17:35 -0400538
539 ret = security_mmap_file(file, prot, flag);
540 if (!ret) {
Michel Lespinassed8ed45c2020-06-08 21:33:25 -0700541 if (mmap_write_lock_killable(mm))
Michal Hocko9fbeb5a2016-05-23 16:25:30 -0700542 return -EINTR;
Peter Collingbourne45e55302020-08-06 23:23:37 -0700543 ret = do_mmap(file, addr, len, prot, flag, pgoff, &populate,
544 &uf);
Michel Lespinassed8ed45c2020-06-08 21:33:25 -0700545 mmap_write_unlock(mm);
Mike Rapoport897ab3e2017-02-24 14:58:22 -0800546 userfaultfd_unmap_complete(mm, &uf);
Michel Lespinasse41badc12013-02-22 16:32:47 -0800547 if (populate)
548 mm_populate(ret, populate);
Al Viroeb36c582012-05-30 20:17:35 -0400549 }
Kuan-Ying Leea5543c92021-06-18 13:18:59 +0800550 trace_android_vh_check_mmap_file(file, prot, flag, ret);
Al Viroeb36c582012-05-30 20:17:35 -0400551 return ret;
552}
553
554unsigned long vm_mmap(struct file *file, unsigned long addr,
555 unsigned long len, unsigned long prot,
556 unsigned long flag, unsigned long offset)
557{
558 if (unlikely(offset + PAGE_ALIGN(len) < offset))
559 return -EINVAL;
Alexander Kuleshovea53cde2015-11-05 18:46:46 -0800560 if (unlikely(offset_in_page(offset)))
Al Viroeb36c582012-05-30 20:17:35 -0400561 return -EINVAL;
562
Michal Hocko9fbeb5a2016-05-23 16:25:30 -0700563 return vm_mmap_pgoff(file, addr, len, prot, flag, offset >> PAGE_SHIFT);
Al Viroeb36c582012-05-30 20:17:35 -0400564}
565EXPORT_SYMBOL(vm_mmap);
566
Michal Hockoa7c3e902017-05-08 15:57:09 -0700567/**
568 * kvmalloc_node - attempt to allocate physically contiguous memory, but upon
569 * failure, fall back to non-contiguous (vmalloc) allocation.
570 * @size: size of the request.
571 * @flags: gfp mask for the allocation - must be compatible (superset) with GFP_KERNEL.
572 * @node: numa node to allocate from
573 *
574 * Uses kmalloc to get the memory but if the allocation fails then falls back
575 * to the vmalloc allocator. Use kvfree for freeing the memory.
576 *
Michal Hockocc965a22017-07-12 14:36:52 -0700577 * Reclaim modifiers - __GFP_NORETRY and __GFP_NOFAIL are not supported.
578 * __GFP_RETRY_MAYFAIL is supported, and it should be used only if kmalloc is
579 * preferable to the vmalloc fallback, due to visible performance drawbacks.
Michal Hockoa7c3e902017-05-08 15:57:09 -0700580 *
Michal Hockoce91f6e2018-06-07 17:09:40 -0700581 * Please note that any use of gfp flags outside of GFP_KERNEL is careful to not
582 * fall back to vmalloc.
Mike Rapoporta862f682019-03-05 15:48:42 -0800583 *
584 * Return: pointer to the allocated memory of %NULL in case of failure
Michal Hockoa7c3e902017-05-08 15:57:09 -0700585 */
586void *kvmalloc_node(size_t size, gfp_t flags, int node)
587{
588 gfp_t kmalloc_flags = flags;
589 void *ret;
590
591 /*
592 * vmalloc uses GFP_KERNEL for some internal allocations (e.g page tables)
593 * so the given set of flags has to be compatible.
594 */
Michal Hockoce91f6e2018-06-07 17:09:40 -0700595 if ((flags & GFP_KERNEL) != GFP_KERNEL)
596 return kmalloc_node(size, flags, node);
Michal Hockoa7c3e902017-05-08 15:57:09 -0700597
598 /*
Michal Hocko4f4f2ba2017-06-02 14:46:19 -0700599 * We want to attempt a large physically contiguous block first because
600 * it is less likely to fragment multiple larger blocks and therefore
601 * contribute to a long term fragmentation less than vmalloc fallback.
602 * However make sure that larger requests are not too disruptive - no
603 * OOM killer and no allocation failure warnings as we have a fallback.
Michal Hockoa7c3e902017-05-08 15:57:09 -0700604 */
Michal Hocko6c5ab652017-05-08 15:57:15 -0700605 if (size > PAGE_SIZE) {
606 kmalloc_flags |= __GFP_NOWARN;
607
Michal Hockocc965a22017-07-12 14:36:52 -0700608 if (!(kmalloc_flags & __GFP_RETRY_MAYFAIL))
Michal Hocko6c5ab652017-05-08 15:57:15 -0700609 kmalloc_flags |= __GFP_NORETRY;
610 }
Michal Hockoa7c3e902017-05-08 15:57:09 -0700611
612 ret = kmalloc_node(size, kmalloc_flags, node);
613
614 /*
615 * It doesn't really make sense to fallback to vmalloc for sub page
616 * requests
617 */
618 if (ret || size <= PAGE_SIZE)
619 return ret;
620
Linus Torvalds57a269a2021-07-14 09:45:49 -0700621 /* Don't even allow crazy sizes */
Daniel Borkmanne93f2be2022-03-04 15:26:32 +0100622 if (unlikely(size > INT_MAX)) {
623 WARN_ON_ONCE(!(flags & __GFP_NOWARN));
Linus Torvalds57a269a2021-07-14 09:45:49 -0700624 return NULL;
Daniel Borkmanne93f2be2022-03-04 15:26:32 +0100625 }
Linus Torvalds57a269a2021-07-14 09:45:49 -0700626
Christoph Hellwig2b905942020-06-01 21:51:53 -0700627 return __vmalloc_node(size, 1, flags, node,
Michal Hocko8594a212017-05-12 15:46:41 -0700628 __builtin_return_address(0));
Michal Hockoa7c3e902017-05-08 15:57:09 -0700629}
630EXPORT_SYMBOL(kvmalloc_node);
631
Mike Rapoportff4dc772018-08-23 17:01:02 -0700632/**
Andrew Morton04b8e942018-09-04 15:45:55 -0700633 * kvfree() - Free memory.
634 * @addr: Pointer to allocated memory.
Mike Rapoportff4dc772018-08-23 17:01:02 -0700635 *
Andrew Morton04b8e942018-09-04 15:45:55 -0700636 * kvfree frees memory allocated by any of vmalloc(), kmalloc() or kvmalloc().
637 * It is slightly more efficient to use kfree() or vfree() if you are certain
638 * that you know which one to use.
639 *
Andrey Ryabinin52414d32018-10-26 15:07:00 -0700640 * Context: Either preemptible task context or not-NMI interrupt.
Mike Rapoportff4dc772018-08-23 17:01:02 -0700641 */
Al Viro39f1f782014-05-06 14:02:53 -0400642void kvfree(const void *addr)
643{
644 if (is_vmalloc_addr(addr))
645 vfree(addr);
646 else
647 kfree(addr);
648}
649EXPORT_SYMBOL(kvfree);
650
Waiman Longd4eaa282020-06-04 16:48:21 -0700651/**
652 * kvfree_sensitive - Free a data object containing sensitive information.
653 * @addr: address of the data object to be freed.
654 * @len: length of the data object.
655 *
656 * Use the special memzero_explicit() function to clear the content of a
657 * kvmalloc'ed object containing sensitive data to make sure that the
658 * compiler won't optimize out the data clearing.
659 */
660void kvfree_sensitive(const void *addr, size_t len)
661{
662 if (likely(!ZERO_OR_NULL_PTR(addr))) {
663 memzero_explicit((void *)addr, len);
664 kvfree(addr);
665 }
666}
667EXPORT_SYMBOL(kvfree_sensitive);
668
Dave Chinnerf5f3e542022-08-10 16:15:50 +0200669void *kvrealloc(const void *p, size_t oldsize, size_t newsize, gfp_t flags)
670{
671 void *newp;
672
673 if (oldsize >= newsize)
674 return (void *)p;
675 newp = kvmalloc(newsize, flags);
676 if (!newp)
677 return NULL;
678 memcpy(newp, p, oldsize);
679 kvfree(p);
680 return newp;
681}
682EXPORT_SYMBOL(kvrealloc);
683
Kirill A. Shutemove39155e2015-04-15 16:14:53 -0700684static inline void *__page_rmapping(struct page *page)
685{
686 unsigned long mapping;
687
688 mapping = (unsigned long)page->mapping;
689 mapping &= ~PAGE_MAPPING_FLAGS;
690
691 return (void *)mapping;
692}
693
694/* Neutral page->mapping pointer to address_space or anon_vma or other */
695void *page_rmapping(struct page *page)
696{
697 page = compound_head(page);
698 return __page_rmapping(page);
699}
700
Andrew Morton1aa8aea2016-05-19 17:12:00 -0700701/*
702 * Return true if this page is mapped into pagetables.
703 * For compound page it returns true if any subpage of compound page is mapped.
704 */
705bool page_mapped(struct page *page)
706{
707 int i;
708
709 if (likely(!PageCompound(page)))
710 return atomic_read(&page->_mapcount) >= 0;
711 page = compound_head(page);
712 if (atomic_read(compound_mapcount_ptr(page)) >= 0)
713 return true;
714 if (PageHuge(page))
715 return false;
Matthew Wilcox (Oracle)d8c65462019-09-23 15:34:30 -0700716 for (i = 0; i < compound_nr(page); i++) {
Andrew Morton1aa8aea2016-05-19 17:12:00 -0700717 if (atomic_read(&page[i]._mapcount) >= 0)
718 return true;
719 }
720 return false;
721}
722EXPORT_SYMBOL(page_mapped);
723
Kirill A. Shutemove39155e2015-04-15 16:14:53 -0700724struct anon_vma *page_anon_vma(struct page *page)
725{
726 unsigned long mapping;
727
728 page = compound_head(page);
729 mapping = (unsigned long)page->mapping;
730 if ((mapping & PAGE_MAPPING_FLAGS) != PAGE_MAPPING_ANON)
731 return NULL;
732 return __page_rmapping(page);
733}
734
Shaohua Li98003392013-02-22 16:34:35 -0800735struct address_space *page_mapping(struct page *page)
736{
Kirill A. Shutemov1c290f62016-01-15 16:52:07 -0800737 struct address_space *mapping;
738
739 page = compound_head(page);
Shaohua Li98003392013-02-22 16:34:35 -0800740
Mikulas Patocka03e5ac22014-01-14 17:56:40 -0800741 /* This happens if someone calls flush_dcache_page on slab page */
742 if (unlikely(PageSlab(page)))
743 return NULL;
744
Shaohua Li33806f02013-02-22 16:34:37 -0800745 if (unlikely(PageSwapCache(page))) {
746 swp_entry_t entry;
747
748 entry.val = page_private(page);
Kirill A. Shutemove39155e2015-04-15 16:14:53 -0700749 return swap_address_space(entry);
750 }
751
Kirill A. Shutemov1c290f62016-01-15 16:52:07 -0800752 mapping = page->mapping;
Minchan Kimbda807d2016-07-26 15:23:05 -0700753 if ((unsigned long)mapping & PAGE_MAPPING_ANON)
Kirill A. Shutemove39155e2015-04-15 16:14:53 -0700754 return NULL;
Minchan Kimbda807d2016-07-26 15:23:05 -0700755
756 return (void *)((unsigned long)mapping & ~PAGE_MAPPING_FLAGS);
Shaohua Li98003392013-02-22 16:34:35 -0800757}
Minchan Kimbda807d2016-07-26 15:23:05 -0700758EXPORT_SYMBOL(page_mapping);
Shaohua Li98003392013-02-22 16:34:35 -0800759
Huang Yingcb9f7532018-04-05 16:24:39 -0700760/*
761 * For file cache pages, return the address_space, otherwise return NULL
762 */
763struct address_space *page_mapping_file(struct page *page)
764{
765 if (unlikely(PageSwapCache(page)))
766 return NULL;
767 return page_mapping(page);
768}
769
Kirill A. Shutemovb20ce5e2016-01-15 16:54:37 -0800770/* Slow path of page_mapcount() for compound pages */
771int __page_mapcount(struct page *page)
772{
773 int ret;
774
775 ret = atomic_read(&page->_mapcount) + 1;
Kirill A. Shutemovdd78fed2016-07-26 15:25:26 -0700776 /*
777 * For file THP page->_mapcount contains total number of mapping
778 * of the page: no need to look into compound_mapcount.
779 */
780 if (!PageAnon(page) && !PageHuge(page))
781 return ret;
Kirill A. Shutemovb20ce5e2016-01-15 16:54:37 -0800782 page = compound_head(page);
783 ret += atomic_read(compound_mapcount_ptr(page)) + 1;
784 if (PageDoubleMap(page))
785 ret--;
786 return ret;
787}
788EXPORT_SYMBOL_GPL(__page_mapcount);
789
Andrey Ryabinin39a1aa82016-03-17 14:18:50 -0700790int sysctl_overcommit_memory __read_mostly = OVERCOMMIT_GUESS;
791int sysctl_overcommit_ratio __read_mostly = 50;
792unsigned long sysctl_overcommit_kbytes __read_mostly;
793int sysctl_max_map_count __read_mostly = DEFAULT_MAX_MAP_COUNT;
794unsigned long sysctl_user_reserve_kbytes __read_mostly = 1UL << 17; /* 128MB */
795unsigned long sysctl_admin_reserve_kbytes __read_mostly = 1UL << 13; /* 8MB */
796
Christoph Hellwig32927392020-04-24 08:43:38 +0200797int overcommit_ratio_handler(struct ctl_table *table, int write, void *buffer,
798 size_t *lenp, loff_t *ppos)
Jerome Marchand49f0ce52014-01-21 15:49:14 -0800799{
800 int ret;
801
802 ret = proc_dointvec(table, write, buffer, lenp, ppos);
803 if (ret == 0 && write)
804 sysctl_overcommit_kbytes = 0;
805 return ret;
806}
807
Feng Tang56f35472020-08-06 23:23:15 -0700808static void sync_overcommit_as(struct work_struct *dummy)
809{
810 percpu_counter_sync(&vm_committed_as);
811}
812
813int overcommit_policy_handler(struct ctl_table *table, int write, void *buffer,
814 size_t *lenp, loff_t *ppos)
815{
816 struct ctl_table t;
Chen Jun83829722021-09-24 15:44:06 -0700817 int new_policy = -1;
Feng Tang56f35472020-08-06 23:23:15 -0700818 int ret;
819
820 /*
821 * The deviation of sync_overcommit_as could be big with loose policy
822 * like OVERCOMMIT_ALWAYS/OVERCOMMIT_GUESS. When changing policy to
823 * strict OVERCOMMIT_NEVER, we need to reduce the deviation to comply
824 * with the strict "NEVER", and to avoid possible race condtion (even
825 * though user usually won't too frequently do the switching to policy
826 * OVERCOMMIT_NEVER), the switch is done in the following order:
827 * 1. changing the batch
828 * 2. sync percpu count on each CPU
829 * 3. switch the policy
830 */
831 if (write) {
832 t = *table;
833 t.data = &new_policy;
834 ret = proc_dointvec_minmax(&t, write, buffer, lenp, ppos);
Chen Jun83829722021-09-24 15:44:06 -0700835 if (ret || new_policy == -1)
Feng Tang56f35472020-08-06 23:23:15 -0700836 return ret;
837
838 mm_compute_batch(new_policy);
839 if (new_policy == OVERCOMMIT_NEVER)
840 schedule_on_each_cpu(sync_overcommit_as);
841 sysctl_overcommit_memory = new_policy;
842 } else {
843 ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
844 }
845
846 return ret;
847}
848
Christoph Hellwig32927392020-04-24 08:43:38 +0200849int overcommit_kbytes_handler(struct ctl_table *table, int write, void *buffer,
850 size_t *lenp, loff_t *ppos)
Jerome Marchand49f0ce52014-01-21 15:49:14 -0800851{
852 int ret;
853
854 ret = proc_doulongvec_minmax(table, write, buffer, lenp, ppos);
855 if (ret == 0 && write)
856 sysctl_overcommit_ratio = 0;
857 return ret;
858}
859
Jerome Marchand00619bc2013-11-12 15:08:31 -0800860/*
861 * Committed memory limit enforced when OVERCOMMIT_NEVER policy is used
862 */
863unsigned long vm_commit_limit(void)
864{
Jerome Marchand49f0ce52014-01-21 15:49:14 -0800865 unsigned long allowed;
866
867 if (sysctl_overcommit_kbytes)
868 allowed = sysctl_overcommit_kbytes >> (PAGE_SHIFT - 10);
869 else
Arun KSca79b0c2018-12-28 00:34:29 -0800870 allowed = ((totalram_pages() - hugetlb_total_pages())
Jerome Marchand49f0ce52014-01-21 15:49:14 -0800871 * sysctl_overcommit_ratio / 100);
872 allowed += total_swap_pages;
873
874 return allowed;
Jerome Marchand00619bc2013-11-12 15:08:31 -0800875}
876
Andrey Ryabinin39a1aa82016-03-17 14:18:50 -0700877/*
878 * Make sure vm_committed_as in one cacheline and not cacheline shared with
879 * other variables. It can be updated by several CPUs frequently.
880 */
881struct percpu_counter vm_committed_as ____cacheline_aligned_in_smp;
882
883/*
884 * The global memory commitment made in the system can be a metric
885 * that can be used to drive ballooning decisions when Linux is hosted
886 * as a guest. On Hyper-V, the host implements a policy engine for dynamically
887 * balancing memory across competing virtual machines that are hosted.
888 * Several metrics drive this policy engine including the guest reported
889 * memory commitment.
Feng Tang4e2ee512020-08-06 23:23:07 -0700890 *
891 * The time cost of this is very low for small platforms, and for big
892 * platform like a 2S/36C/72T Skylake server, in worst case where
893 * vm_committed_as's spinlock is under severe contention, the time cost
894 * could be about 30~40 microseconds.
Andrey Ryabinin39a1aa82016-03-17 14:18:50 -0700895 */
896unsigned long vm_memory_committed(void)
897{
Feng Tang4e2ee512020-08-06 23:23:07 -0700898 return percpu_counter_sum_positive(&vm_committed_as);
Andrey Ryabinin39a1aa82016-03-17 14:18:50 -0700899}
900EXPORT_SYMBOL_GPL(vm_memory_committed);
901
902/*
903 * Check that a process has enough memory to allocate a new virtual
904 * mapping. 0 means there is enough memory for the allocation to
905 * succeed and -ENOMEM implies there is not.
906 *
907 * We currently support three overcommit policies, which are set via the
Mike Rapoportad56b732018-03-21 21:22:47 +0200908 * vm.overcommit_memory sysctl. See Documentation/vm/overcommit-accounting.rst
Andrey Ryabinin39a1aa82016-03-17 14:18:50 -0700909 *
910 * Strict overcommit modes added 2002 Feb 26 by Alan Cox.
911 * Additional code 2002 Jul 20 by Robert Love.
912 *
913 * cap_sys_admin is 1 if the process has admin privileges, 0 otherwise.
914 *
915 * Note this is a helper function intended to be used by LSMs which
916 * wish to use this logic.
917 */
918int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin)
919{
Johannes Weiner8c7829b2019-05-13 17:21:50 -0700920 long allowed;
Andrey Ryabinin39a1aa82016-03-17 14:18:50 -0700921
Andrey Ryabinin39a1aa82016-03-17 14:18:50 -0700922 vm_acct_memory(pages);
923
924 /*
925 * Sometimes we want to use more memory than we have
926 */
927 if (sysctl_overcommit_memory == OVERCOMMIT_ALWAYS)
928 return 0;
929
930 if (sysctl_overcommit_memory == OVERCOMMIT_GUESS) {
Johannes Weiner8c7829b2019-05-13 17:21:50 -0700931 if (pages > totalram_pages() + total_swap_pages)
Andrey Ryabinin39a1aa82016-03-17 14:18:50 -0700932 goto error;
Johannes Weiner8c7829b2019-05-13 17:21:50 -0700933 return 0;
Andrey Ryabinin39a1aa82016-03-17 14:18:50 -0700934 }
935
936 allowed = vm_commit_limit();
937 /*
938 * Reserve some for root
939 */
940 if (!cap_sys_admin)
941 allowed -= sysctl_admin_reserve_kbytes >> (PAGE_SHIFT - 10);
942
943 /*
944 * Don't let a single process grow so big a user can't recover
945 */
946 if (mm) {
Johannes Weiner8c7829b2019-05-13 17:21:50 -0700947 long reserve = sysctl_user_reserve_kbytes >> (PAGE_SHIFT - 10);
948
Andrey Ryabinin39a1aa82016-03-17 14:18:50 -0700949 allowed -= min_t(long, mm->total_vm / 32, reserve);
950 }
951
952 if (percpu_counter_read_positive(&vm_committed_as) < allowed)
953 return 0;
954error:
955 vm_unacct_memory(pages);
956
957 return -ENOMEM;
958}
959
William Robertsa9090252014-02-11 10:11:59 -0800960/**
961 * get_cmdline() - copy the cmdline value to a buffer.
962 * @task: the task whose cmdline value to copy.
963 * @buffer: the buffer to copy to.
964 * @buflen: the length of the buffer. Larger cmdline values are truncated
965 * to this length.
Mike Rapoporta862f682019-03-05 15:48:42 -0800966 *
967 * Return: the size of the cmdline field copied. Note that the copy does
William Robertsa9090252014-02-11 10:11:59 -0800968 * not guarantee an ending NULL byte.
969 */
970int get_cmdline(struct task_struct *task, char *buffer, int buflen)
971{
972 int res = 0;
973 unsigned int len;
974 struct mm_struct *mm = get_task_mm(task);
Mateusz Guzika3b609e2016-01-20 15:01:05 -0800975 unsigned long arg_start, arg_end, env_start, env_end;
William Robertsa9090252014-02-11 10:11:59 -0800976 if (!mm)
977 goto out;
978 if (!mm->arg_end)
979 goto out_mm; /* Shh! No looking before we're done */
980
Michal Koutnýbc814262019-05-31 22:30:19 -0700981 spin_lock(&mm->arg_lock);
Mateusz Guzika3b609e2016-01-20 15:01:05 -0800982 arg_start = mm->arg_start;
983 arg_end = mm->arg_end;
984 env_start = mm->env_start;
985 env_end = mm->env_end;
Michal Koutnýbc814262019-05-31 22:30:19 -0700986 spin_unlock(&mm->arg_lock);
Mateusz Guzika3b609e2016-01-20 15:01:05 -0800987
988 len = arg_end - arg_start;
William Robertsa9090252014-02-11 10:11:59 -0800989
990 if (len > buflen)
991 len = buflen;
992
Lorenzo Stoakesf307ab62016-10-13 01:20:20 +0100993 res = access_process_vm(task, arg_start, buffer, len, FOLL_FORCE);
William Robertsa9090252014-02-11 10:11:59 -0800994
995 /*
996 * If the nul at the end of args has been overwritten, then
997 * assume application is using setproctitle(3).
998 */
999 if (res > 0 && buffer[res-1] != '\0' && len < buflen) {
1000 len = strnlen(buffer, res);
1001 if (len < res) {
1002 res = len;
1003 } else {
Mateusz Guzika3b609e2016-01-20 15:01:05 -08001004 len = env_end - env_start;
William Robertsa9090252014-02-11 10:11:59 -08001005 if (len > buflen - res)
1006 len = buflen - res;
Mateusz Guzika3b609e2016-01-20 15:01:05 -08001007 res += access_process_vm(task, env_start,
Lorenzo Stoakesf307ab62016-10-13 01:20:20 +01001008 buffer+res, len,
1009 FOLL_FORCE);
William Robertsa9090252014-02-11 10:11:59 -08001010 res = strnlen(buffer, res);
1011 }
1012 }
1013out_mm:
1014 mmput(mm);
1015out:
1016 return res;
1017}
Song Liu010c1642019-09-23 15:38:19 -07001018
Catalin Marinas4d1a8a22019-11-27 09:53:44 +00001019int __weak memcmp_pages(struct page *page1, struct page *page2)
Song Liu010c1642019-09-23 15:38:19 -07001020{
1021 char *addr1, *addr2;
1022 int ret;
1023
1024 addr1 = kmap_atomic(page1);
1025 addr2 = kmap_atomic(page2);
1026 ret = memcmp(addr1, addr2, PAGE_SIZE);
1027 kunmap_atomic(addr2);
1028 kunmap_atomic(addr1);
1029 return ret;
1030}