blob: 32f783ddb5c3a655b8aa124f0667520df5fdb039 [file] [log] [blame]
Thomas Gleixner457c8992019-05-19 13:08:55 +01001// SPDX-License-Identifier: GPL-2.0-only
Linus Torvalds1da177e2005-04-16 15:20:36 -07002/*
3 * linux/mm/page_alloc.c
4 *
5 * Manages the free list, the system allocates free pages here.
6 * Note that kmalloc() lives in slab.c
7 *
8 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
9 * Swap reorganised 29.12.95, Stephen Tweedie
10 * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
11 * Reshaped it to be a zoned allocator, Ingo Molnar, Red Hat, 1999
12 * Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999
13 * Zone balancing, Kanoj Sarcar, SGI, Jan 2000
14 * Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002
15 * (lots of bits borrowed from Ingo Molnar & Andrew Morton)
16 */
17
Linus Torvalds1da177e2005-04-16 15:20:36 -070018#include <linux/stddef.h>
19#include <linux/mm.h>
Arun KSca79b0c2018-12-28 00:34:29 -080020#include <linux/highmem.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070021#include <linux/swap.h>
22#include <linux/interrupt.h>
23#include <linux/pagemap.h>
KOSAKI Motohiro10ed2732008-03-04 14:28:32 -080024#include <linux/jiffies.h>
Yinghai Luedbe7d22010-08-25 13:39:16 -070025#include <linux/memblock.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070026#include <linux/compiler.h>
Randy Dunlap9f158332005-09-13 01:25:16 -070027#include <linux/kernel.h>
Andrey Ryabininb8c73fc2015-02-13 14:39:28 -080028#include <linux/kasan.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070029#include <linux/module.h>
30#include <linux/suspend.h>
31#include <linux/pagevec.h>
32#include <linux/blkdev.h>
33#include <linux/slab.h>
Dave Hansena238ab52011-05-24 17:12:16 -070034#include <linux/ratelimit.h>
David Rientjes5a3135c22007-10-16 23:25:53 -070035#include <linux/oom.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070036#include <linux/topology.h>
37#include <linux/sysctl.h>
38#include <linux/cpu.h>
39#include <linux/cpuset.h>
Dave Hansenbdc8cb92005-10-29 18:16:53 -070040#include <linux/memory_hotplug.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070041#include <linux/nodemask.h>
42#include <linux/vmalloc.h>
KOSAKI Motohiroa6cccdc2011-05-24 17:11:33 -070043#include <linux/vmstat.h>
Christoph Lameter4be38e32006-01-06 00:11:17 -080044#include <linux/mempolicy.h>
Dan Williams4b94ffd2016-01-15 16:56:22 -080045#include <linux/memremap.h>
Yasunori Goto68113782006-06-23 02:03:11 -070046#include <linux/stop_machine.h>
Dan Williams97500a42019-05-14 15:41:35 -070047#include <linux/random.h>
Mel Gormanc7132162006-09-27 01:49:43 -070048#include <linux/sort.h>
49#include <linux/pfn.h>
Andrew Morton3fcfab12006-10-19 23:28:16 -070050#include <linux/backing-dev.h>
Akinobu Mita933e3122006-12-08 02:39:45 -080051#include <linux/fault-inject.h>
KAMEZAWA Hiroyukia5d76b542007-10-16 01:26:11 -070052#include <linux/page-isolation.h>
Thomas Gleixner3ac7fe52008-04-30 00:55:01 -070053#include <linux/debugobjects.h>
Catalin Marinasdbb1f812009-06-11 13:23:19 +010054#include <linux/kmemleak.h>
Mel Gorman56de7262010-05-24 14:32:30 -070055#include <linux/compaction.h>
Mel Gorman0d3d0622009-09-21 17:02:44 -070056#include <trace/events/kmem.h>
Michal Hockod379f012017-02-22 15:42:00 -080057#include <trace/events/oom.h>
Linus Torvalds268bb0c2011-05-20 12:50:29 -070058#include <linux/prefetch.h>
Lisa Du6e543d52013-09-11 14:22:36 -070059#include <linux/mm_inline.h>
Michal Nazarewicz041d3a82011-12-29 13:09:50 +010060#include <linux/migrate.h>
David Rientjes949f7ec2013-04-29 15:07:48 -070061#include <linux/hugetlb.h>
Clark Williams8bd75c72013-02-07 09:47:07 -060062#include <linux/sched/rt.h>
Ingo Molnar5b3cc152017-02-02 20:43:54 +010063#include <linux/sched/mm.h>
Joonsoo Kim48c96a32014-12-12 16:56:01 -080064#include <linux/page_owner.h>
Mel Gorman0e1cc952015-06-30 14:57:27 -070065#include <linux/kthread.h>
Vladimir Davydov49491482016-07-26 15:24:24 -070066#include <linux/memcontrol.h>
Steven Rostedt (VMware)42c269c2017-03-03 16:15:39 -050067#include <linux/ftrace.h>
Peter Zijlstrad92a8cf2017-03-03 10:13:38 +010068#include <linux/lockdep.h>
Chen Yu556b9692017-08-25 15:55:30 -070069#include <linux/nmi.h>
Johannes Weinereb414682018-10-26 15:06:27 -070070#include <linux/psi.h>
Daniel Jordane4443142020-06-03 15:59:51 -070071#include <linux/padata.h>
Vijay Balakrishna4aab2be2020-10-10 23:16:40 -070072#include <linux/khugepaged.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070073
Jiang Liu7ee3d4e2013-07-03 15:03:41 -070074#include <asm/sections.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070075#include <asm/tlbflush.h>
Andrew Mortonac924c62006-05-15 09:43:59 -070076#include <asm/div64.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070077#include "internal.h"
Dan Williamse900a912019-05-14 15:41:28 -070078#include "shuffle.h"
Alexander Duyck36e66c52020-04-06 20:04:56 -070079#include "page_reporting.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070080
David Hildenbrandf04a5d52020-10-15 20:09:20 -070081/* Free Page Internal flags: for internal, non-pcp variants of free_pages(). */
82typedef int __bitwise fpi_t;
83
84/* No special request */
85#define FPI_NONE ((__force fpi_t)0)
86
87/*
88 * Skip free page reporting notification for the (possibly merged) page.
89 * This does not hinder free page reporting from grabbing the page,
90 * reporting it and marking it "reported" - it only skips notifying
91 * the free page reporting infrastructure about a newly freed page. For
92 * example, used when temporarily pulling a page from a freelist and
93 * putting it back unmodified.
94 */
95#define FPI_SKIP_REPORT_NOTIFY ((__force fpi_t)BIT(0))
96
David Hildenbrand47b6a24a22020-10-15 20:09:26 -070097/*
98 * Place the (possibly merged) page to the tail of the freelist. Will ignore
99 * page shuffling (relevant code - e.g., memory onlining - is expected to
100 * shuffle the whole zone).
101 *
102 * Note: No code should rely on this flag for correctness - it's purely
103 * to allow for optimizations when handing back either fresh pages
104 * (memory onlining) or untouched pages (page isolation, free page
105 * reporting).
106 */
107#define FPI_TO_TAIL ((__force fpi_t)BIT(1))
108
Cody P Schaferc8e251f2013-07-03 15:01:29 -0700109/* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */
110static DEFINE_MUTEX(pcp_batch_high_lock);
David Rientjes7cd2b0a2014-06-23 13:22:04 -0700111#define MIN_PERCPU_PAGELIST_FRACTION (8)
Cody P Schaferc8e251f2013-07-03 15:01:29 -0700112
Lee Schermerhorn72812012010-05-26 14:44:56 -0700113#ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID
114DEFINE_PER_CPU(int, numa_node);
115EXPORT_PER_CPU_SYMBOL(numa_node);
116#endif
117
Kemi Wang45180852017-11-15 17:38:22 -0800118DEFINE_STATIC_KEY_TRUE(vm_numa_stat_key);
119
Lee Schermerhorn7aac7892010-05-26 14:45:00 -0700120#ifdef CONFIG_HAVE_MEMORYLESS_NODES
121/*
122 * N.B., Do NOT reference the '_numa_mem_' per cpu variable directly.
123 * It will not be defined when CONFIG_HAVE_MEMORYLESS_NODES is not defined.
124 * Use the accessor functions set_numa_mem(), numa_mem_id() and cpu_to_mem()
125 * defined in <linux/topology.h>.
126 */
127DEFINE_PER_CPU(int, _numa_mem_); /* Kernel "local memory" node */
128EXPORT_PER_CPU_SYMBOL(_numa_mem_);
129#endif
130
Mel Gormanbd233f52017-02-24 14:56:56 -0800131/* work_structs for global per-cpu drains */
Wei Yangd9367bd2018-12-28 00:38:58 -0800132struct pcpu_drain {
133 struct zone *zone;
134 struct work_struct work;
135};
Jason Yan8b885f52020-04-10 14:32:32 -0700136static DEFINE_MUTEX(pcpu_drain_mutex);
137static DEFINE_PER_CPU(struct pcpu_drain, pcpu_drain);
Mel Gormanbd233f52017-02-24 14:56:56 -0800138
Emese Revfy38addce2016-06-20 20:41:19 +0200139#ifdef CONFIG_GCC_PLUGIN_LATENT_ENTROPY
Kees Cook58bea412016-10-19 00:08:04 +0200140volatile unsigned long latent_entropy __latent_entropy;
Emese Revfy38addce2016-06-20 20:41:19 +0200141EXPORT_SYMBOL(latent_entropy);
142#endif
143
Linus Torvalds1da177e2005-04-16 15:20:36 -0700144/*
Christoph Lameter13808912007-10-16 01:25:27 -0700145 * Array of node states.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700146 */
Christoph Lameter13808912007-10-16 01:25:27 -0700147nodemask_t node_states[NR_NODE_STATES] __read_mostly = {
148 [N_POSSIBLE] = NODE_MASK_ALL,
149 [N_ONLINE] = { { [0] = 1UL } },
150#ifndef CONFIG_NUMA
151 [N_NORMAL_MEMORY] = { { [0] = 1UL } },
152#ifdef CONFIG_HIGHMEM
153 [N_HIGH_MEMORY] = { { [0] = 1UL } },
154#endif
Lai Jiangshan20b2f522012-12-12 13:52:00 -0800155 [N_MEMORY] = { { [0] = 1UL } },
Christoph Lameter13808912007-10-16 01:25:27 -0700156 [N_CPU] = { { [0] = 1UL } },
157#endif /* NUMA */
158};
159EXPORT_SYMBOL(node_states);
160
Arun KSca79b0c2018-12-28 00:34:29 -0800161atomic_long_t _totalram_pages __read_mostly;
162EXPORT_SYMBOL(_totalram_pages);
Hideo AOKIcb45b0e2006-04-10 22:52:59 -0700163unsigned long totalreserve_pages __read_mostly;
Pintu Kumare48322a2014-12-18 16:17:15 -0800164unsigned long totalcma_pages __read_mostly;
Johannes Weinerab8fabd2012-01-10 15:07:42 -0800165
Hugh Dickins1b76b022012-05-11 01:00:07 -0700166int percpu_pagelist_fraction;
Benjamin Herrenschmidtdcce2842009-06-18 13:24:12 +1000167gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK;
Alexander Potapenko64713842019-07-11 20:59:19 -0700168#ifdef CONFIG_INIT_ON_ALLOC_DEFAULT_ON
169DEFINE_STATIC_KEY_TRUE(init_on_alloc);
170#else
171DEFINE_STATIC_KEY_FALSE(init_on_alloc);
172#endif
173EXPORT_SYMBOL(init_on_alloc);
174
175#ifdef CONFIG_INIT_ON_FREE_DEFAULT_ON
176DEFINE_STATIC_KEY_TRUE(init_on_free);
177#else
178DEFINE_STATIC_KEY_FALSE(init_on_free);
179#endif
180EXPORT_SYMBOL(init_on_free);
181
182static int __init early_init_on_alloc(char *buf)
183{
184 int ret;
185 bool bool_result;
186
Alexander Potapenko64713842019-07-11 20:59:19 -0700187 ret = kstrtobool(buf, &bool_result);
Mateusz Nosekfdd4fa12020-10-13 16:55:48 -0700188 if (ret)
189 return ret;
Alexander Potapenko64713842019-07-11 20:59:19 -0700190 if (bool_result && page_poisoning_enabled())
191 pr_info("mem auto-init: CONFIG_PAGE_POISONING is on, will take precedence over init_on_alloc\n");
192 if (bool_result)
193 static_branch_enable(&init_on_alloc);
194 else
195 static_branch_disable(&init_on_alloc);
Mateusz Nosekfdd4fa12020-10-13 16:55:48 -0700196 return 0;
Alexander Potapenko64713842019-07-11 20:59:19 -0700197}
198early_param("init_on_alloc", early_init_on_alloc);
199
200static int __init early_init_on_free(char *buf)
201{
202 int ret;
203 bool bool_result;
204
Alexander Potapenko64713842019-07-11 20:59:19 -0700205 ret = kstrtobool(buf, &bool_result);
Mateusz Nosekfdd4fa12020-10-13 16:55:48 -0700206 if (ret)
207 return ret;
Alexander Potapenko64713842019-07-11 20:59:19 -0700208 if (bool_result && page_poisoning_enabled())
209 pr_info("mem auto-init: CONFIG_PAGE_POISONING is on, will take precedence over init_on_free\n");
210 if (bool_result)
211 static_branch_enable(&init_on_free);
212 else
213 static_branch_disable(&init_on_free);
Mateusz Nosekfdd4fa12020-10-13 16:55:48 -0700214 return 0;
Alexander Potapenko64713842019-07-11 20:59:19 -0700215}
216early_param("init_on_free", early_init_on_free);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700217
Vlastimil Babkabb14c2c2015-09-08 15:01:25 -0700218/*
219 * A cached value of the page's pageblock's migratetype, used when the page is
220 * put on a pcplist. Used to avoid the pageblock migratetype lookup when
221 * freeing from pcplists in most cases, at the cost of possibly becoming stale.
222 * Also the migratetype set in the page does not necessarily match the pcplist
223 * index, e.g. page might have MIGRATE_CMA set but be on a pcplist with any
224 * other index - this ensures that it will be put on the correct CMA freelist.
225 */
226static inline int get_pcppage_migratetype(struct page *page)
227{
228 return page->index;
229}
230
231static inline void set_pcppage_migratetype(struct page *page, int migratetype)
232{
233 page->index = migratetype;
234}
235
Rafael J. Wysocki452aa692010-03-05 13:42:13 -0800236#ifdef CONFIG_PM_SLEEP
237/*
238 * The following functions are used by the suspend/hibernate code to temporarily
239 * change gfp_allowed_mask in order to avoid using I/O during memory allocations
240 * while devices are suspended. To avoid races with the suspend/hibernate code,
Pingfan Liu55f25032018-07-31 16:51:32 +0800241 * they should always be called with system_transition_mutex held
242 * (gfp_allowed_mask also should only be modified with system_transition_mutex
243 * held, unless the suspend/hibernate code is guaranteed not to run in parallel
244 * with that modification).
Rafael J. Wysocki452aa692010-03-05 13:42:13 -0800245 */
Rafael J. Wysockic9e664f2010-12-03 22:57:45 +0100246
247static gfp_t saved_gfp_mask;
248
249void pm_restore_gfp_mask(void)
Rafael J. Wysocki452aa692010-03-05 13:42:13 -0800250{
Pingfan Liu55f25032018-07-31 16:51:32 +0800251 WARN_ON(!mutex_is_locked(&system_transition_mutex));
Rafael J. Wysockic9e664f2010-12-03 22:57:45 +0100252 if (saved_gfp_mask) {
253 gfp_allowed_mask = saved_gfp_mask;
254 saved_gfp_mask = 0;
255 }
Rafael J. Wysocki452aa692010-03-05 13:42:13 -0800256}
257
Rafael J. Wysockic9e664f2010-12-03 22:57:45 +0100258void pm_restrict_gfp_mask(void)
Rafael J. Wysocki452aa692010-03-05 13:42:13 -0800259{
Pingfan Liu55f25032018-07-31 16:51:32 +0800260 WARN_ON(!mutex_is_locked(&system_transition_mutex));
Rafael J. Wysockic9e664f2010-12-03 22:57:45 +0100261 WARN_ON(saved_gfp_mask);
262 saved_gfp_mask = gfp_allowed_mask;
Mel Gormand0164ad2015-11-06 16:28:21 -0800263 gfp_allowed_mask &= ~(__GFP_IO | __GFP_FS);
Rafael J. Wysocki452aa692010-03-05 13:42:13 -0800264}
Mel Gormanf90ac392012-01-10 15:07:15 -0800265
266bool pm_suspended_storage(void)
267{
Mel Gormand0164ad2015-11-06 16:28:21 -0800268 if ((gfp_allowed_mask & (__GFP_IO | __GFP_FS)) == (__GFP_IO | __GFP_FS))
Mel Gormanf90ac392012-01-10 15:07:15 -0800269 return false;
270 return true;
271}
Rafael J. Wysocki452aa692010-03-05 13:42:13 -0800272#endif /* CONFIG_PM_SLEEP */
273
Mel Gormand9c23402007-10-16 01:26:01 -0700274#ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
Kirill A. Shutemovd00181b2015-11-06 16:29:57 -0800275unsigned int pageblock_order __read_mostly;
Mel Gormand9c23402007-10-16 01:26:01 -0700276#endif
277
David Hildenbrand7fef4312020-10-15 20:09:35 -0700278static void __free_pages_ok(struct page *page, unsigned int order,
279 fpi_t fpi_flags);
David Howellsa226f6c2006-01-06 00:11:08 -0800280
Linus Torvalds1da177e2005-04-16 15:20:36 -0700281/*
282 * results with 256, 32 in the lowmem_reserve sysctl:
283 * 1G machine -> (16M dma, 800M-16M normal, 1G-800M high)
284 * 1G machine -> (16M dma, 784M normal, 224M high)
285 * NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA
286 * HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL
Yaowei Bai84109e12015-02-12 15:00:22 -0800287 * HIGHMEM allocation will leave (224M+784M)/256 of ram reserved in ZONE_DMA
Andi Kleena2f1b422005-11-05 17:25:53 +0100288 *
289 * TBD: should special case ZONE_DMA32 machines here - in those we normally
290 * don't need any ZONE_NORMAL reservation
Linus Torvalds1da177e2005-04-16 15:20:36 -0700291 */
Joonsoo Kimd3cda232018-04-10 16:30:11 -0700292int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES] = {
Christoph Lameter4b51d662007-02-10 01:43:10 -0800293#ifdef CONFIG_ZONE_DMA
Joonsoo Kimd3cda232018-04-10 16:30:11 -0700294 [ZONE_DMA] = 256,
Christoph Lameter4b51d662007-02-10 01:43:10 -0800295#endif
Christoph Lameterfb0e7942006-09-25 23:31:13 -0700296#ifdef CONFIG_ZONE_DMA32
Joonsoo Kimd3cda232018-04-10 16:30:11 -0700297 [ZONE_DMA32] = 256,
Christoph Lameterfb0e7942006-09-25 23:31:13 -0700298#endif
Joonsoo Kimd3cda232018-04-10 16:30:11 -0700299 [ZONE_NORMAL] = 32,
Christoph Lametere53ef382006-09-25 23:31:14 -0700300#ifdef CONFIG_HIGHMEM
Joonsoo Kimd3cda232018-04-10 16:30:11 -0700301 [ZONE_HIGHMEM] = 0,
Christoph Lametere53ef382006-09-25 23:31:14 -0700302#endif
Joonsoo Kimd3cda232018-04-10 16:30:11 -0700303 [ZONE_MOVABLE] = 0,
Christoph Lameter2f1b6242006-09-25 23:31:13 -0700304};
Linus Torvalds1da177e2005-04-16 15:20:36 -0700305
Helge Deller15ad7cd2006-12-06 20:40:36 -0800306static char * const zone_names[MAX_NR_ZONES] = {
Christoph Lameter4b51d662007-02-10 01:43:10 -0800307#ifdef CONFIG_ZONE_DMA
Christoph Lameter2f1b6242006-09-25 23:31:13 -0700308 "DMA",
Christoph Lameter4b51d662007-02-10 01:43:10 -0800309#endif
Christoph Lameterfb0e7942006-09-25 23:31:13 -0700310#ifdef CONFIG_ZONE_DMA32
Christoph Lameter2f1b6242006-09-25 23:31:13 -0700311 "DMA32",
Christoph Lameterfb0e7942006-09-25 23:31:13 -0700312#endif
Christoph Lameter2f1b6242006-09-25 23:31:13 -0700313 "Normal",
Christoph Lametere53ef382006-09-25 23:31:14 -0700314#ifdef CONFIG_HIGHMEM
Mel Gorman2a1e2742007-07-17 04:03:12 -0700315 "HighMem",
Christoph Lametere53ef382006-09-25 23:31:14 -0700316#endif
Mel Gorman2a1e2742007-07-17 04:03:12 -0700317 "Movable",
Dan Williams033fbae2015-08-09 15:29:06 -0400318#ifdef CONFIG_ZONE_DEVICE
319 "Device",
320#endif
Christoph Lameter2f1b6242006-09-25 23:31:13 -0700321};
322
Alexey Dobriyanc999fbd2018-12-28 00:35:55 -0800323const char * const migratetype_names[MIGRATE_TYPES] = {
Vlastimil Babka60f30352016-03-15 14:56:08 -0700324 "Unmovable",
325 "Movable",
326 "Reclaimable",
327 "HighAtomic",
328#ifdef CONFIG_CMA
329 "CMA",
330#endif
331#ifdef CONFIG_MEMORY_ISOLATION
332 "Isolate",
333#endif
334};
335
Anshuman Khandualae70edd2020-06-03 15:59:17 -0700336compound_page_dtor * const compound_page_dtors[NR_COMPOUND_DTORS] = {
337 [NULL_COMPOUND_DTOR] = NULL,
338 [COMPOUND_PAGE_DTOR] = free_compound_page,
Kirill A. Shutemovf1e61552015-11-06 16:29:50 -0800339#ifdef CONFIG_HUGETLB_PAGE
Anshuman Khandualae70edd2020-06-03 15:59:17 -0700340 [HUGETLB_PAGE_DTOR] = free_huge_page,
Kirill A. Shutemovf1e61552015-11-06 16:29:50 -0800341#endif
Kirill A. Shutemov9a982252016-01-15 16:54:17 -0800342#ifdef CONFIG_TRANSPARENT_HUGEPAGE
Anshuman Khandualae70edd2020-06-03 15:59:17 -0700343 [TRANSHUGE_PAGE_DTOR] = free_transhuge_page,
Kirill A. Shutemov9a982252016-01-15 16:54:17 -0800344#endif
Kirill A. Shutemovf1e61552015-11-06 16:29:50 -0800345};
346
Linus Torvalds1da177e2005-04-16 15:20:36 -0700347int min_free_kbytes = 1024;
Han Pingtian42aa83c2014-01-23 15:53:28 -0800348int user_min_free_kbytes = -1;
Mel Gorman24512228b2019-04-25 22:23:51 -0700349#ifdef CONFIG_DISCONTIGMEM
350/*
351 * DiscontigMem defines memory ranges as separate pg_data_t even if the ranges
352 * are not on separate NUMA nodes. Functionally this works but with
353 * watermark_boost_factor, it can reclaim prematurely as the ranges can be
354 * quite small. By default, do not boost watermarks on discontigmem as in
355 * many cases very high-order allocations like THP are likely to be
356 * unsupported and the premature reclaim offsets the advantage of long-term
357 * fragmentation avoidance.
358 */
359int watermark_boost_factor __read_mostly;
360#else
Mel Gorman1c308442018-12-28 00:35:52 -0800361int watermark_boost_factor __read_mostly = 15000;
Mel Gorman24512228b2019-04-25 22:23:51 -0700362#endif
Johannes Weiner795ae7a2016-03-17 14:19:14 -0700363int watermark_scale_factor = 10;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700364
Oscar Salvadorbbe5d992018-12-28 00:37:24 -0800365static unsigned long nr_kernel_pages __initdata;
366static unsigned long nr_all_pages __initdata;
367static unsigned long dma_reserve __initdata;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700368
Oscar Salvadorbbe5d992018-12-28 00:37:24 -0800369static unsigned long arch_zone_lowest_possible_pfn[MAX_NR_ZONES] __initdata;
370static unsigned long arch_zone_highest_possible_pfn[MAX_NR_ZONES] __initdata;
David Rientjes7f16f912018-04-05 16:23:12 -0700371static unsigned long required_kernelcore __initdata;
David Rientjesa5c6d652018-04-05 16:23:09 -0700372static unsigned long required_kernelcore_percent __initdata;
David Rientjes7f16f912018-04-05 16:23:12 -0700373static unsigned long required_movablecore __initdata;
David Rientjesa5c6d652018-04-05 16:23:09 -0700374static unsigned long required_movablecore_percent __initdata;
Oscar Salvadorbbe5d992018-12-28 00:37:24 -0800375static unsigned long zone_movable_pfn[MAX_NUMNODES] __initdata;
David Rientjes7f16f912018-04-05 16:23:12 -0700376static bool mirrored_kernelcore __meminitdata;
Mel Gormanc7132162006-09-27 01:49:43 -0700377
Tejun Heo0ee332c2011-12-08 10:22:09 -0800378/* movable_zone is the "real" zone pages in ZONE_MOVABLE are taken from */
379int movable_zone;
380EXPORT_SYMBOL(movable_zone);
Mel Gormanc7132162006-09-27 01:49:43 -0700381
Miklos Szeredi418508c2007-05-23 13:57:55 -0700382#if MAX_NUMNODES > 1
Alexey Dobriyanb9726c22019-03-05 15:48:26 -0800383unsigned int nr_node_ids __read_mostly = MAX_NUMNODES;
Alexey Dobriyance0725f2019-03-05 15:48:29 -0800384unsigned int nr_online_nodes __read_mostly = 1;
Miklos Szeredi418508c2007-05-23 13:57:55 -0700385EXPORT_SYMBOL(nr_node_ids);
Christoph Lameter62bc62a2009-06-16 15:32:15 -0700386EXPORT_SYMBOL(nr_online_nodes);
Miklos Szeredi418508c2007-05-23 13:57:55 -0700387#endif
388
Mel Gorman9ef9acb2007-10-16 01:25:54 -0700389int page_group_by_mobility_disabled __read_mostly;
390
Mel Gorman3a80a7f2015-06-30 14:57:02 -0700391#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
Waiman Long3c0c12c2018-12-28 00:38:51 -0800392/*
393 * During boot we initialize deferred pages on-demand, as needed, but once
394 * page_alloc_init_late() has finished, the deferred pages are all initialized,
395 * and we can permanently disable that path.
396 */
397static DEFINE_STATIC_KEY_TRUE(deferred_pages);
398
399/*
400 * Calling kasan_free_pages() only after deferred memory initialization
401 * has completed. Poisoning pages during deferred memory init will greatly
402 * lengthen the process and cause problem in large memory systems as the
403 * deferred pages initialization is done with interrupt disabled.
404 *
405 * Assuming that there will be no reference to those newly initialized
406 * pages before they are ever allocated, this should have no effect on
407 * KASAN memory tracking as the poison will be properly inserted at page
408 * allocation time. The only corner case is when pages are allocated by
409 * on-demand allocation and then freed again before the deferred pages
410 * initialization is done, but this is not likely to happen.
411 */
412static inline void kasan_free_nondeferred_pages(struct page *page, int order)
413{
414 if (!static_branch_unlikely(&deferred_pages))
415 kasan_free_pages(page, order);
416}
417
Mel Gorman3a80a7f2015-06-30 14:57:02 -0700418/* Returns true if the struct page for the pfn is uninitialised */
Mel Gorman0e1cc952015-06-30 14:57:27 -0700419static inline bool __meminit early_page_uninitialised(unsigned long pfn)
Mel Gorman3a80a7f2015-06-30 14:57:02 -0700420{
Mel Gormanef70b6f2016-07-14 12:07:23 -0700421 int nid = early_pfn_to_nid(pfn);
422
423 if (node_online(nid) && pfn >= NODE_DATA(nid)->first_deferred_pfn)
Mel Gorman3a80a7f2015-06-30 14:57:02 -0700424 return true;
425
426 return false;
427}
428
429/*
Pavel Tatashind3035be2018-10-26 15:09:37 -0700430 * Returns true when the remaining initialisation should be deferred until
Mel Gorman3a80a7f2015-06-30 14:57:02 -0700431 * later in the boot cycle when it can be parallelised.
432 */
Pavel Tatashind3035be2018-10-26 15:09:37 -0700433static bool __meminit
434defer_init(int nid, unsigned long pfn, unsigned long end_pfn)
Mel Gorman3a80a7f2015-06-30 14:57:02 -0700435{
Pavel Tatashind3035be2018-10-26 15:09:37 -0700436 static unsigned long prev_end_pfn, nr_initialised;
437
438 /*
439 * prev_end_pfn static that contains the end of previous zone
440 * No need to protect because called very early in boot before smp_init.
441 */
442 if (prev_end_pfn != end_pfn) {
443 prev_end_pfn = end_pfn;
444 nr_initialised = 0;
Mel Gorman3a80a7f2015-06-30 14:57:02 -0700445 }
446
Pavel Tatashind3035be2018-10-26 15:09:37 -0700447 /* Always populate low zones for address-constrained allocations */
448 if (end_pfn < pgdat_end_pfn(NODE_DATA(nid)))
449 return false;
Wei Yang23b68cf2018-12-28 00:36:18 -0800450
451 /*
452 * We start only with one section of pages, more pages are added as
453 * needed until the rest of deferred pages are initialized.
454 */
Pavel Tatashind3035be2018-10-26 15:09:37 -0700455 nr_initialised++;
Wei Yang23b68cf2018-12-28 00:36:18 -0800456 if ((nr_initialised > PAGES_PER_SECTION) &&
Pavel Tatashind3035be2018-10-26 15:09:37 -0700457 (pfn & (PAGES_PER_SECTION - 1)) == 0) {
458 NODE_DATA(nid)->first_deferred_pfn = pfn;
459 return true;
460 }
461 return false;
Mel Gorman3a80a7f2015-06-30 14:57:02 -0700462}
463#else
Waiman Long3c0c12c2018-12-28 00:38:51 -0800464#define kasan_free_nondeferred_pages(p, o) kasan_free_pages(p, o)
465
Mel Gorman3a80a7f2015-06-30 14:57:02 -0700466static inline bool early_page_uninitialised(unsigned long pfn)
467{
468 return false;
469}
470
Pavel Tatashind3035be2018-10-26 15:09:37 -0700471static inline bool defer_init(int nid, unsigned long pfn, unsigned long end_pfn)
Mel Gorman3a80a7f2015-06-30 14:57:02 -0700472{
Pavel Tatashind3035be2018-10-26 15:09:37 -0700473 return false;
Mel Gorman3a80a7f2015-06-30 14:57:02 -0700474}
475#endif
476
Mel Gorman0b423ca2016-05-19 17:14:27 -0700477/* Return a pointer to the bitmap storing bits affecting a block of pages */
478static inline unsigned long *get_pageblock_bitmap(struct page *page,
479 unsigned long pfn)
480{
481#ifdef CONFIG_SPARSEMEM
Dan Williamsf1eca352019-07-18 15:57:57 -0700482 return section_to_usemap(__pfn_to_section(pfn));
Mel Gorman0b423ca2016-05-19 17:14:27 -0700483#else
484 return page_zone(page)->pageblock_flags;
485#endif /* CONFIG_SPARSEMEM */
486}
487
488static inline int pfn_to_bitidx(struct page *page, unsigned long pfn)
489{
490#ifdef CONFIG_SPARSEMEM
491 pfn &= (PAGES_PER_SECTION-1);
Mel Gorman0b423ca2016-05-19 17:14:27 -0700492#else
493 pfn = pfn - round_down(page_zone(page)->zone_start_pfn, pageblock_nr_pages);
Mel Gorman0b423ca2016-05-19 17:14:27 -0700494#endif /* CONFIG_SPARSEMEM */
Wei Yang399b7952020-08-06 23:25:44 -0700495 return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
Mel Gorman0b423ca2016-05-19 17:14:27 -0700496}
497
498/**
499 * get_pfnblock_flags_mask - Return the requested group of flags for the pageblock_nr_pages block of pages
500 * @page: The page within the block of interest
501 * @pfn: The target page frame number
Mel Gorman0b423ca2016-05-19 17:14:27 -0700502 * @mask: mask of bits that the caller is interested in
503 *
504 * Return: pageblock_bits flags
505 */
Wei Yang535b81e2020-08-06 23:25:51 -0700506static __always_inline
507unsigned long __get_pfnblock_flags_mask(struct page *page,
Mel Gorman0b423ca2016-05-19 17:14:27 -0700508 unsigned long pfn,
Mel Gorman0b423ca2016-05-19 17:14:27 -0700509 unsigned long mask)
510{
511 unsigned long *bitmap;
512 unsigned long bitidx, word_bitidx;
513 unsigned long word;
514
515 bitmap = get_pageblock_bitmap(page, pfn);
516 bitidx = pfn_to_bitidx(page, pfn);
517 word_bitidx = bitidx / BITS_PER_LONG;
518 bitidx &= (BITS_PER_LONG-1);
519
520 word = bitmap[word_bitidx];
Wei Yangd93d5ab2020-08-06 23:25:48 -0700521 return (word >> bitidx) & mask;
Mel Gorman0b423ca2016-05-19 17:14:27 -0700522}
523
524unsigned long get_pfnblock_flags_mask(struct page *page, unsigned long pfn,
Mel Gorman0b423ca2016-05-19 17:14:27 -0700525 unsigned long mask)
526{
Wei Yang535b81e2020-08-06 23:25:51 -0700527 return __get_pfnblock_flags_mask(page, pfn, mask);
Mel Gorman0b423ca2016-05-19 17:14:27 -0700528}
529
530static __always_inline int get_pfnblock_migratetype(struct page *page, unsigned long pfn)
531{
Wei Yang535b81e2020-08-06 23:25:51 -0700532 return __get_pfnblock_flags_mask(page, pfn, MIGRATETYPE_MASK);
Mel Gorman0b423ca2016-05-19 17:14:27 -0700533}
534
535/**
536 * set_pfnblock_flags_mask - Set the requested group of flags for a pageblock_nr_pages block of pages
537 * @page: The page within the block of interest
538 * @flags: The flags to set
539 * @pfn: The target page frame number
Mel Gorman0b423ca2016-05-19 17:14:27 -0700540 * @mask: mask of bits that the caller is interested in
541 */
542void set_pfnblock_flags_mask(struct page *page, unsigned long flags,
543 unsigned long pfn,
Mel Gorman0b423ca2016-05-19 17:14:27 -0700544 unsigned long mask)
545{
546 unsigned long *bitmap;
547 unsigned long bitidx, word_bitidx;
548 unsigned long old_word, word;
549
550 BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 4);
Pingfan Liu125b8602018-12-28 00:38:43 -0800551 BUILD_BUG_ON(MIGRATE_TYPES > (1 << PB_migratetype_bits));
Mel Gorman0b423ca2016-05-19 17:14:27 -0700552
553 bitmap = get_pageblock_bitmap(page, pfn);
554 bitidx = pfn_to_bitidx(page, pfn);
555 word_bitidx = bitidx / BITS_PER_LONG;
556 bitidx &= (BITS_PER_LONG-1);
557
558 VM_BUG_ON_PAGE(!zone_spans_pfn(page_zone(page), pfn), page);
559
Wei Yangd93d5ab2020-08-06 23:25:48 -0700560 mask <<= bitidx;
561 flags <<= bitidx;
Mel Gorman0b423ca2016-05-19 17:14:27 -0700562
563 word = READ_ONCE(bitmap[word_bitidx]);
564 for (;;) {
565 old_word = cmpxchg(&bitmap[word_bitidx], word, (word & ~mask) | flags);
566 if (word == old_word)
567 break;
568 word = old_word;
569 }
570}
Mel Gorman3a80a7f2015-06-30 14:57:02 -0700571
Minchan Kimee6f5092012-07-31 16:43:50 -0700572void set_pageblock_migratetype(struct page *page, int migratetype)
Mel Gormanb2a0ac82007-10-16 01:25:48 -0700573{
KOSAKI Motohiro5d0f3f72013-11-12 15:08:18 -0800574 if (unlikely(page_group_by_mobility_disabled &&
575 migratetype < MIGRATE_PCPTYPES))
Mel Gorman49255c62009-06-16 15:31:58 -0700576 migratetype = MIGRATE_UNMOVABLE;
577
Wei Yangd93d5ab2020-08-06 23:25:48 -0700578 set_pfnblock_flags_mask(page, (unsigned long)migratetype,
Wei Yang535b81e2020-08-06 23:25:51 -0700579 page_to_pfn(page), MIGRATETYPE_MASK);
Mel Gormanb2a0ac82007-10-16 01:25:48 -0700580}
581
Nick Piggin13e74442006-01-06 00:10:58 -0800582#ifdef CONFIG_DEBUG_VM
Dave Hansenc6a57e12005-10-29 18:16:52 -0700583static int page_outside_zone_boundaries(struct zone *zone, struct page *page)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700584{
Dave Hansenbdc8cb92005-10-29 18:16:53 -0700585 int ret = 0;
586 unsigned seq;
587 unsigned long pfn = page_to_pfn(page);
Cody P Schaferb5e6a5a2013-02-22 16:35:28 -0800588 unsigned long sp, start_pfn;
Dave Hansenc6a57e12005-10-29 18:16:52 -0700589
Dave Hansenbdc8cb92005-10-29 18:16:53 -0700590 do {
591 seq = zone_span_seqbegin(zone);
Cody P Schaferb5e6a5a2013-02-22 16:35:28 -0800592 start_pfn = zone->zone_start_pfn;
593 sp = zone->spanned_pages;
Cody P Schafer108bcc92013-02-22 16:35:23 -0800594 if (!zone_spans_pfn(zone, pfn))
Dave Hansenbdc8cb92005-10-29 18:16:53 -0700595 ret = 1;
596 } while (zone_span_seqretry(zone, seq));
597
Cody P Schaferb5e6a5a2013-02-22 16:35:28 -0800598 if (ret)
Dave Hansen613813e2014-06-04 16:07:27 -0700599 pr_err("page 0x%lx outside node %d zone %s [ 0x%lx - 0x%lx ]\n",
600 pfn, zone_to_nid(zone), zone->name,
601 start_pfn, start_pfn + sp);
Cody P Schaferb5e6a5a2013-02-22 16:35:28 -0800602
Dave Hansenbdc8cb92005-10-29 18:16:53 -0700603 return ret;
Dave Hansenc6a57e12005-10-29 18:16:52 -0700604}
605
606static int page_is_consistent(struct zone *zone, struct page *page)
607{
Andy Whitcroft14e07292007-05-06 14:49:14 -0700608 if (!pfn_valid_within(page_to_pfn(page)))
Dave Hansenc6a57e12005-10-29 18:16:52 -0700609 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700610 if (zone != page_zone(page))
Dave Hansenc6a57e12005-10-29 18:16:52 -0700611 return 0;
612
613 return 1;
614}
615/*
616 * Temporary debugging check for pages not lying within a given zone.
617 */
Matthias Kaehlcked73d3c92017-07-06 15:39:23 -0700618static int __maybe_unused bad_range(struct zone *zone, struct page *page)
Dave Hansenc6a57e12005-10-29 18:16:52 -0700619{
620 if (page_outside_zone_boundaries(zone, page))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700621 return 1;
Dave Hansenc6a57e12005-10-29 18:16:52 -0700622 if (!page_is_consistent(zone, page))
623 return 1;
624
Linus Torvalds1da177e2005-04-16 15:20:36 -0700625 return 0;
626}
Nick Piggin13e74442006-01-06 00:10:58 -0800627#else
Matthias Kaehlcked73d3c92017-07-06 15:39:23 -0700628static inline int __maybe_unused bad_range(struct zone *zone, struct page *page)
Nick Piggin13e74442006-01-06 00:10:58 -0800629{
630 return 0;
631}
632#endif
633
Wei Yang82a32412020-06-03 15:58:29 -0700634static void bad_page(struct page *page, const char *reason)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700635{
Hugh Dickinsd936cf92009-01-06 14:40:12 -0800636 static unsigned long resume;
637 static unsigned long nr_shown;
638 static unsigned long nr_unshown;
639
640 /*
641 * Allow a burst of 60 reports, then keep quiet for that minute;
642 * or allow a steady drip of one report per second.
643 */
644 if (nr_shown == 60) {
645 if (time_before(jiffies, resume)) {
646 nr_unshown++;
647 goto out;
648 }
649 if (nr_unshown) {
Vlastimil Babkaff8e8112016-03-15 14:56:24 -0700650 pr_alert(
Hugh Dickins1e9e6362009-01-06 14:40:13 -0800651 "BUG: Bad page state: %lu messages suppressed\n",
Hugh Dickinsd936cf92009-01-06 14:40:12 -0800652 nr_unshown);
653 nr_unshown = 0;
654 }
655 nr_shown = 0;
656 }
657 if (nr_shown++ == 0)
658 resume = jiffies + 60 * HZ;
659
Vlastimil Babkaff8e8112016-03-15 14:56:24 -0700660 pr_alert("BUG: Bad page state in process %s pfn:%05lx\n",
Hugh Dickins3dc14742009-01-06 14:40:08 -0800661 current->comm, page_to_pfn(page));
Vlastimil Babkaff8e8112016-03-15 14:56:24 -0700662 __dump_page(page, reason);
Vlastimil Babka4e462112016-03-15 14:56:21 -0700663 dump_page_owner(page);
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -0700664
Dave Jones4f318882011-10-31 17:07:24 -0700665 print_modules();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700666 dump_stack();
Hugh Dickinsd936cf92009-01-06 14:40:12 -0800667out:
Hugh Dickins8cc3b392009-01-06 14:40:06 -0800668 /* Leave bad fields for debug, except PageBuddy could make trouble */
Mel Gorman22b751c2013-02-22 16:34:59 -0800669 page_mapcount_reset(page); /* remove PageBuddy */
Rusty Russell373d4d02013-01-21 17:17:39 +1030670 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700671}
672
Linus Torvalds1da177e2005-04-16 15:20:36 -0700673/*
674 * Higher-order pages are called "compound pages". They are structured thusly:
675 *
Kirill A. Shutemov1d798ca2015-11-06 16:29:54 -0800676 * The first PAGE_SIZE page is called the "head page" and have PG_head set.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700677 *
Kirill A. Shutemov1d798ca2015-11-06 16:29:54 -0800678 * The remaining PAGE_SIZE pages are called "tail pages". PageTail() is encoded
679 * in bit 0 of page->compound_head. The rest of bits is pointer to head page.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700680 *
Kirill A. Shutemov1d798ca2015-11-06 16:29:54 -0800681 * The first tail page's ->compound_dtor holds the offset in array of compound
682 * page destructors. See compound_page_dtors.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700683 *
Kirill A. Shutemov1d798ca2015-11-06 16:29:54 -0800684 * The first tail page's ->compound_order holds the order of allocation.
Hugh Dickins41d78ba2006-02-14 13:52:58 -0800685 * This usage means that zero-order pages may not be compound.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700686 */
Hugh Dickinsd98c7a02006-02-14 13:52:59 -0800687
Kirill A. Shutemov9a982252016-01-15 16:54:17 -0800688void free_compound_page(struct page *page)
Hugh Dickinsd98c7a02006-02-14 13:52:59 -0800689{
Yang Shi7ae88532019-09-23 15:38:09 -0700690 mem_cgroup_uncharge(page);
David Hildenbrand7fef4312020-10-15 20:09:35 -0700691 __free_pages_ok(page, compound_order(page), FPI_NONE);
Hugh Dickinsd98c7a02006-02-14 13:52:59 -0800692}
693
Kirill A. Shutemovd00181b2015-11-06 16:29:57 -0800694void prep_compound_page(struct page *page, unsigned int order)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700695{
696 int i;
697 int nr_pages = 1 << order;
698
Christoph Lameter6d777952007-05-06 14:49:40 -0700699 __SetPageHead(page);
Andy Whitcroft18229df2008-11-06 12:53:27 -0800700 for (i = 1; i < nr_pages; i++) {
701 struct page *p = page + i;
Youquan Song58a84aa2011-12-08 14:34:18 -0800702 set_page_count(p, 0);
Kirill A. Shutemov1c290f62016-01-15 16:52:07 -0800703 p->mapping = TAIL_MAPPING;
Kirill A. Shutemov1d798ca2015-11-06 16:29:54 -0800704 set_compound_head(p, page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700705 }
Matthew Wilcox (Oracle)1378a5e2020-08-14 17:30:23 -0700706
707 set_compound_page_dtor(page, COMPOUND_PAGE_DTOR);
708 set_compound_order(page, order);
Kirill A. Shutemov53f92632016-01-15 16:53:42 -0800709 atomic_set(compound_mapcount_ptr(page), -1);
John Hubbard47e29d32020-04-01 21:05:33 -0700710 if (hpage_pincount_available(page))
711 atomic_set(compound_pincount_ptr(page), 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700712}
713
Stanislaw Gruszkac0a32fc2012-01-10 15:07:28 -0800714#ifdef CONFIG_DEBUG_PAGEALLOC
715unsigned int _debug_guardpage_minorder;
Vlastimil Babka96a2b032019-07-11 20:55:06 -0700716
Vlastimil Babka8e57f8a2020-01-13 16:29:20 -0800717bool _debug_pagealloc_enabled_early __read_mostly
718 = IS_ENABLED(CONFIG_DEBUG_PAGEALLOC_ENABLE_DEFAULT);
719EXPORT_SYMBOL(_debug_pagealloc_enabled_early);
Vlastimil Babka96a2b032019-07-11 20:55:06 -0700720DEFINE_STATIC_KEY_FALSE(_debug_pagealloc_enabled);
Joonsoo Kim505f6d22016-03-17 14:17:56 -0700721EXPORT_SYMBOL(_debug_pagealloc_enabled);
Vlastimil Babka96a2b032019-07-11 20:55:06 -0700722
723DEFINE_STATIC_KEY_FALSE(_debug_guardpage_enabled);
Joonsoo Kime30825f2014-12-12 16:55:49 -0800724
Joonsoo Kim031bc572014-12-12 16:55:52 -0800725static int __init early_debug_pagealloc(char *buf)
726{
Vlastimil Babka8e57f8a2020-01-13 16:29:20 -0800727 return kstrtobool(buf, &_debug_pagealloc_enabled_early);
Joonsoo Kim031bc572014-12-12 16:55:52 -0800728}
729early_param("debug_pagealloc", early_debug_pagealloc);
730
Vlastimil Babka8e57f8a2020-01-13 16:29:20 -0800731void init_debug_pagealloc(void)
Joonsoo Kime30825f2014-12-12 16:55:49 -0800732{
Joonsoo Kim031bc572014-12-12 16:55:52 -0800733 if (!debug_pagealloc_enabled())
734 return;
735
Vlastimil Babka8e57f8a2020-01-13 16:29:20 -0800736 static_branch_enable(&_debug_pagealloc_enabled);
737
Joonsoo Kimf1c1e9f2016-10-07 16:58:18 -0700738 if (!debug_guardpage_minorder())
739 return;
740
Vlastimil Babka96a2b032019-07-11 20:55:06 -0700741 static_branch_enable(&_debug_guardpage_enabled);
Joonsoo Kime30825f2014-12-12 16:55:49 -0800742}
743
Stanislaw Gruszkac0a32fc2012-01-10 15:07:28 -0800744static int __init debug_guardpage_minorder_setup(char *buf)
745{
746 unsigned long res;
747
748 if (kstrtoul(buf, 10, &res) < 0 || res > MAX_ORDER / 2) {
Joe Perches11705322016-03-17 14:19:50 -0700749 pr_err("Bad debug_guardpage_minorder value\n");
Stanislaw Gruszkac0a32fc2012-01-10 15:07:28 -0800750 return 0;
751 }
752 _debug_guardpage_minorder = res;
Joe Perches11705322016-03-17 14:19:50 -0700753 pr_info("Setting debug_guardpage_minorder to %lu\n", res);
Stanislaw Gruszkac0a32fc2012-01-10 15:07:28 -0800754 return 0;
755}
Joonsoo Kimf1c1e9f2016-10-07 16:58:18 -0700756early_param("debug_guardpage_minorder", debug_guardpage_minorder_setup);
Stanislaw Gruszkac0a32fc2012-01-10 15:07:28 -0800757
Joonsoo Kimacbc15a2016-10-07 16:58:15 -0700758static inline bool set_page_guard(struct zone *zone, struct page *page,
Joonsoo Kim2847cf92014-12-12 16:55:01 -0800759 unsigned int order, int migratetype)
Stanislaw Gruszkac0a32fc2012-01-10 15:07:28 -0800760{
Joonsoo Kime30825f2014-12-12 16:55:49 -0800761 if (!debug_guardpage_enabled())
Joonsoo Kimacbc15a2016-10-07 16:58:15 -0700762 return false;
763
764 if (order >= debug_guardpage_minorder())
765 return false;
Joonsoo Kime30825f2014-12-12 16:55:49 -0800766
Vlastimil Babka3972f6b2019-07-11 20:55:13 -0700767 __SetPageGuard(page);
Joonsoo Kim2847cf92014-12-12 16:55:01 -0800768 INIT_LIST_HEAD(&page->lru);
769 set_page_private(page, order);
770 /* Guard pages are not available for any usage */
771 __mod_zone_freepage_state(zone, -(1 << order), migratetype);
Joonsoo Kimacbc15a2016-10-07 16:58:15 -0700772
773 return true;
Stanislaw Gruszkac0a32fc2012-01-10 15:07:28 -0800774}
775
Joonsoo Kim2847cf92014-12-12 16:55:01 -0800776static inline void clear_page_guard(struct zone *zone, struct page *page,
777 unsigned int order, int migratetype)
Stanislaw Gruszkac0a32fc2012-01-10 15:07:28 -0800778{
Joonsoo Kime30825f2014-12-12 16:55:49 -0800779 if (!debug_guardpage_enabled())
780 return;
781
Vlastimil Babka3972f6b2019-07-11 20:55:13 -0700782 __ClearPageGuard(page);
Joonsoo Kime30825f2014-12-12 16:55:49 -0800783
Joonsoo Kim2847cf92014-12-12 16:55:01 -0800784 set_page_private(page, 0);
785 if (!is_migrate_isolate(migratetype))
786 __mod_zone_freepage_state(zone, (1 << order), migratetype);
Stanislaw Gruszkac0a32fc2012-01-10 15:07:28 -0800787}
788#else
Joonsoo Kimacbc15a2016-10-07 16:58:15 -0700789static inline bool set_page_guard(struct zone *zone, struct page *page,
790 unsigned int order, int migratetype) { return false; }
Joonsoo Kim2847cf92014-12-12 16:55:01 -0800791static inline void clear_page_guard(struct zone *zone, struct page *page,
792 unsigned int order, int migratetype) {}
Stanislaw Gruszkac0a32fc2012-01-10 15:07:28 -0800793#endif
794
Matthew Wilcox (Oracle)ab130f912020-10-15 20:10:15 -0700795static inline void set_buddy_order(struct page *page, unsigned int order)
Andrew Morton6aa3001b22006-04-18 22:20:52 -0700796{
Hugh Dickins4c21e2f2005-10-29 18:16:40 -0700797 set_page_private(page, order);
Nick Piggin676165a2006-04-10 11:21:48 +1000798 __SetPageBuddy(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700799}
800
Linus Torvalds1da177e2005-04-16 15:20:36 -0700801/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700802 * This function checks whether a page is free && is the buddy
Matthew Wilcox6e292b92018-06-07 17:08:18 -0700803 * we can coalesce a page and its buddy if
Vlastimil Babka13ad59d2017-02-22 15:41:51 -0800804 * (a) the buddy is not in a hole (check before calling!) &&
Nick Piggin676165a2006-04-10 11:21:48 +1000805 * (b) the buddy is in the buddy system &&
Andy Whitcroftcb2b95e2006-06-23 02:03:01 -0700806 * (c) a page and its buddy have the same order &&
807 * (d) a page and its buddy are in the same zone.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700808 *
Matthew Wilcox6e292b92018-06-07 17:08:18 -0700809 * For recording whether a page is in the buddy system, we set PageBuddy.
810 * Setting, clearing, and testing PageBuddy is serialized by zone->lock.
Nick Piggin676165a2006-04-10 11:21:48 +1000811 *
812 * For recording page's order, we use page_private(page).
Linus Torvalds1da177e2005-04-16 15:20:36 -0700813 */
chenqiwufe925c02020-04-01 21:09:56 -0700814static inline bool page_is_buddy(struct page *page, struct page *buddy,
Mel Gorman7aeb09f2014-06-04 16:10:21 -0700815 unsigned int order)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700816{
chenqiwufe925c02020-04-01 21:09:56 -0700817 if (!page_is_guard(buddy) && !PageBuddy(buddy))
818 return false;
Mel Gormand34c5fa2014-06-04 16:10:10 -0700819
Matthew Wilcox (Oracle)ab130f912020-10-15 20:10:15 -0700820 if (buddy_order(buddy) != order)
chenqiwufe925c02020-04-01 21:09:56 -0700821 return false;
Weijie Yang4c5018c2015-02-10 14:11:39 -0800822
chenqiwufe925c02020-04-01 21:09:56 -0700823 /*
824 * zone check is done late to avoid uselessly calculating
825 * zone/node ids for pages that could never merge.
826 */
827 if (page_zone_id(page) != page_zone_id(buddy))
828 return false;
Stanislaw Gruszkac0a32fc2012-01-10 15:07:28 -0800829
chenqiwufe925c02020-04-01 21:09:56 -0700830 VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy);
Mel Gormand34c5fa2014-06-04 16:10:10 -0700831
chenqiwufe925c02020-04-01 21:09:56 -0700832 return true;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700833}
834
Mel Gorman5e1f0f02019-03-05 15:45:41 -0800835#ifdef CONFIG_COMPACTION
836static inline struct capture_control *task_capc(struct zone *zone)
837{
838 struct capture_control *capc = current->capture_control;
839
Vlastimil Babkadeba0482020-08-06 23:25:16 -0700840 return unlikely(capc) &&
Mel Gorman5e1f0f02019-03-05 15:45:41 -0800841 !(current->flags & PF_KTHREAD) &&
842 !capc->page &&
Vlastimil Babkadeba0482020-08-06 23:25:16 -0700843 capc->cc->zone == zone ? capc : NULL;
Mel Gorman5e1f0f02019-03-05 15:45:41 -0800844}
845
846static inline bool
847compaction_capture(struct capture_control *capc, struct page *page,
848 int order, int migratetype)
849{
850 if (!capc || order != capc->cc->order)
851 return false;
852
853 /* Do not accidentally pollute CMA or isolated regions*/
854 if (is_migrate_cma(migratetype) ||
855 is_migrate_isolate(migratetype))
856 return false;
857
858 /*
859 * Do not let lower order allocations polluate a movable pageblock.
860 * This might let an unmovable request use a reclaimable pageblock
861 * and vice-versa but no more than normal fallback logic which can
862 * have trouble finding a high-order free page.
863 */
864 if (order < pageblock_order && migratetype == MIGRATE_MOVABLE)
865 return false;
866
867 capc->page = page;
868 return true;
869}
870
871#else
872static inline struct capture_control *task_capc(struct zone *zone)
873{
874 return NULL;
875}
876
877static inline bool
878compaction_capture(struct capture_control *capc, struct page *page,
879 int order, int migratetype)
880{
881 return false;
882}
883#endif /* CONFIG_COMPACTION */
884
Alexander Duyck6ab01362020-04-06 20:04:49 -0700885/* Used for pages not on another list */
886static inline void add_to_free_list(struct page *page, struct zone *zone,
887 unsigned int order, int migratetype)
888{
889 struct free_area *area = &zone->free_area[order];
890
891 list_add(&page->lru, &area->free_list[migratetype]);
892 area->nr_free++;
893}
894
895/* Used for pages not on another list */
896static inline void add_to_free_list_tail(struct page *page, struct zone *zone,
897 unsigned int order, int migratetype)
898{
899 struct free_area *area = &zone->free_area[order];
900
901 list_add_tail(&page->lru, &area->free_list[migratetype]);
902 area->nr_free++;
903}
904
David Hildenbrand293ffa52020-10-15 20:09:30 -0700905/*
906 * Used for pages which are on another list. Move the pages to the tail
907 * of the list - so the moved pages won't immediately be considered for
908 * allocation again (e.g., optimization for memory onlining).
909 */
Alexander Duyck6ab01362020-04-06 20:04:49 -0700910static inline void move_to_free_list(struct page *page, struct zone *zone,
911 unsigned int order, int migratetype)
912{
913 struct free_area *area = &zone->free_area[order];
914
David Hildenbrand293ffa52020-10-15 20:09:30 -0700915 list_move_tail(&page->lru, &area->free_list[migratetype]);
Alexander Duyck6ab01362020-04-06 20:04:49 -0700916}
917
918static inline void del_page_from_free_list(struct page *page, struct zone *zone,
919 unsigned int order)
920{
Alexander Duyck36e66c52020-04-06 20:04:56 -0700921 /* clear reported state and update reported page count */
922 if (page_reported(page))
923 __ClearPageReported(page);
924
Alexander Duyck6ab01362020-04-06 20:04:49 -0700925 list_del(&page->lru);
926 __ClearPageBuddy(page);
927 set_page_private(page, 0);
928 zone->free_area[order].nr_free--;
929}
930
Linus Torvalds1da177e2005-04-16 15:20:36 -0700931/*
Alexander Duycka2129f22020-04-06 20:04:45 -0700932 * If this is not the largest possible page, check if the buddy
933 * of the next-highest order is free. If it is, it's possible
934 * that pages are being freed that will coalesce soon. In case,
935 * that is happening, add the free page to the tail of the list
936 * so it's less likely to be used soon and more likely to be merged
937 * as a higher order page
938 */
939static inline bool
940buddy_merge_likely(unsigned long pfn, unsigned long buddy_pfn,
941 struct page *page, unsigned int order)
942{
943 struct page *higher_page, *higher_buddy;
944 unsigned long combined_pfn;
945
946 if (order >= MAX_ORDER - 2)
947 return false;
948
949 if (!pfn_valid_within(buddy_pfn))
950 return false;
951
952 combined_pfn = buddy_pfn & pfn;
953 higher_page = page + (combined_pfn - pfn);
954 buddy_pfn = __find_buddy_pfn(combined_pfn, order + 1);
955 higher_buddy = higher_page + (buddy_pfn - combined_pfn);
956
957 return pfn_valid_within(buddy_pfn) &&
958 page_is_buddy(higher_page, higher_buddy, order + 1);
959}
960
961/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700962 * Freeing function for a buddy system allocator.
963 *
964 * The concept of a buddy system is to maintain direct-mapped table
965 * (containing bit values) for memory blocks of various "orders".
966 * The bottom level table contains the map for the smallest allocatable
967 * units of memory (here, pages), and each level above it describes
968 * pairs of units from the levels below, hence, "buddies".
969 * At a high level, all that happens here is marking the table entry
970 * at the bottom level available, and propagating the changes upward
971 * as necessary, plus some accounting needed to play nicely with other
972 * parts of the VM system.
973 * At each level, we keep a list of pages, which are heads of continuous
Matthew Wilcox6e292b92018-06-07 17:08:18 -0700974 * free pages of length of (1 << order) and marked with PageBuddy.
975 * Page's order is recorded in page_private(page) field.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700976 * So when we are allocating or freeing one, we can derive the state of the
Michal Nazarewicz5f63b722012-01-11 15:16:11 +0100977 * other. That is, if we allocate a small block, and both were
978 * free, the remainder of the region must be split into blocks.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700979 * If a block is freed, and its buddy is also free, then this
Michal Nazarewicz5f63b722012-01-11 15:16:11 +0100980 * triggers coalescing into a block of larger size.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700981 *
Nadia Yvette Chambers6d49e352012-12-06 10:39:54 +0100982 * -- nyc
Linus Torvalds1da177e2005-04-16 15:20:36 -0700983 */
984
Nick Piggin48db57f2006-01-08 01:00:42 -0800985static inline void __free_one_page(struct page *page,
Mel Gormandc4b0ca2014-06-04 16:10:17 -0700986 unsigned long pfn,
Mel Gormaned0ae212009-06-16 15:32:07 -0700987 struct zone *zone, unsigned int order,
David Hildenbrandf04a5d52020-10-15 20:09:20 -0700988 int migratetype, fpi_t fpi_flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700989{
Mel Gorman5e1f0f02019-03-05 15:45:41 -0800990 struct capture_control *capc = task_capc(zone);
Kees Cook3f649ab2020-06-03 13:09:38 -0700991 unsigned long buddy_pfn;
Alexander Duycka2129f22020-04-06 20:04:45 -0700992 unsigned long combined_pfn;
Alexander Duycka2129f22020-04-06 20:04:45 -0700993 unsigned int max_order;
994 struct page *buddy;
995 bool to_tail;
Vlastimil Babkad9dddbf2016-03-25 14:21:50 -0700996
997 max_order = min_t(unsigned int, MAX_ORDER, pageblock_order + 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700998
Cody P Schaferd29bb972013-02-22 16:35:25 -0800999 VM_BUG_ON(!zone_is_initialized(zone));
Kirill A. Shutemov6e9f0d52015-02-11 15:25:50 -08001000 VM_BUG_ON_PAGE(page->flags & PAGE_FLAGS_CHECK_AT_PREP, page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001001
Mel Gormaned0ae212009-06-16 15:32:07 -07001002 VM_BUG_ON(migratetype == -1);
Vlastimil Babkad9dddbf2016-03-25 14:21:50 -07001003 if (likely(!is_migrate_isolate(migratetype)))
Joonsoo Kim8f82b552014-11-13 15:19:18 -08001004 __mod_zone_freepage_state(zone, 1 << order, migratetype);
Mel Gormaned0ae212009-06-16 15:32:07 -07001005
Vlastimil Babka76741e72017-02-22 15:41:48 -08001006 VM_BUG_ON_PAGE(pfn & ((1 << order) - 1), page);
Sasha Levin309381fea2014-01-23 15:52:54 -08001007 VM_BUG_ON_PAGE(bad_range(zone, page), page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001008
Vlastimil Babkad9dddbf2016-03-25 14:21:50 -07001009continue_merging:
Joonsoo Kim3c605092014-11-13 15:19:21 -08001010 while (order < max_order - 1) {
Mel Gorman5e1f0f02019-03-05 15:45:41 -08001011 if (compaction_capture(capc, page, order, migratetype)) {
1012 __mod_zone_freepage_state(zone, -(1 << order),
1013 migratetype);
1014 return;
1015 }
Vlastimil Babka76741e72017-02-22 15:41:48 -08001016 buddy_pfn = __find_buddy_pfn(pfn, order);
1017 buddy = page + (buddy_pfn - pfn);
Vlastimil Babka13ad59d2017-02-22 15:41:51 -08001018
1019 if (!pfn_valid_within(buddy_pfn))
1020 goto done_merging;
Andy Whitcroftcb2b95e2006-06-23 02:03:01 -07001021 if (!page_is_buddy(page, buddy, order))
Vlastimil Babkad9dddbf2016-03-25 14:21:50 -07001022 goto done_merging;
Stanislaw Gruszkac0a32fc2012-01-10 15:07:28 -08001023 /*
1024 * Our buddy is free or it is CONFIG_DEBUG_PAGEALLOC guard page,
1025 * merge with it and move up one order.
1026 */
Dan Williamsb03641a2019-05-14 15:41:32 -07001027 if (page_is_guard(buddy))
Joonsoo Kim2847cf92014-12-12 16:55:01 -08001028 clear_page_guard(zone, buddy, order, migratetype);
Dan Williamsb03641a2019-05-14 15:41:32 -07001029 else
Alexander Duyck6ab01362020-04-06 20:04:49 -07001030 del_page_from_free_list(buddy, zone, order);
Vlastimil Babka76741e72017-02-22 15:41:48 -08001031 combined_pfn = buddy_pfn & pfn;
1032 page = page + (combined_pfn - pfn);
1033 pfn = combined_pfn;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001034 order++;
1035 }
Vlastimil Babkad9dddbf2016-03-25 14:21:50 -07001036 if (max_order < MAX_ORDER) {
1037 /* If we are here, it means order is >= pageblock_order.
1038 * We want to prevent merge between freepages on isolate
1039 * pageblock and normal pageblock. Without this, pageblock
1040 * isolation could cause incorrect freepage or CMA accounting.
1041 *
1042 * We don't want to hit this code for the more frequent
1043 * low-order merging.
1044 */
1045 if (unlikely(has_isolate_pageblock(zone))) {
1046 int buddy_mt;
1047
Vlastimil Babka76741e72017-02-22 15:41:48 -08001048 buddy_pfn = __find_buddy_pfn(pfn, order);
1049 buddy = page + (buddy_pfn - pfn);
Vlastimil Babkad9dddbf2016-03-25 14:21:50 -07001050 buddy_mt = get_pageblock_migratetype(buddy);
1051
1052 if (migratetype != buddy_mt
1053 && (is_migrate_isolate(migratetype) ||
1054 is_migrate_isolate(buddy_mt)))
1055 goto done_merging;
1056 }
1057 max_order++;
1058 goto continue_merging;
1059 }
1060
1061done_merging:
Matthew Wilcox (Oracle)ab130f912020-10-15 20:10:15 -07001062 set_buddy_order(page, order);
Corrado Zoccolo6dda9d52010-05-24 14:31:54 -07001063
David Hildenbrand47b6a24a22020-10-15 20:09:26 -07001064 if (fpi_flags & FPI_TO_TAIL)
1065 to_tail = true;
1066 else if (is_shuffle_order(order))
Alexander Duycka2129f22020-04-06 20:04:45 -07001067 to_tail = shuffle_pick_tail();
Dan Williams97500a42019-05-14 15:41:35 -07001068 else
Alexander Duycka2129f22020-04-06 20:04:45 -07001069 to_tail = buddy_merge_likely(pfn, buddy_pfn, page, order);
Dan Williams97500a42019-05-14 15:41:35 -07001070
Alexander Duycka2129f22020-04-06 20:04:45 -07001071 if (to_tail)
Alexander Duyck6ab01362020-04-06 20:04:49 -07001072 add_to_free_list_tail(page, zone, order, migratetype);
Alexander Duycka2129f22020-04-06 20:04:45 -07001073 else
Alexander Duyck6ab01362020-04-06 20:04:49 -07001074 add_to_free_list(page, zone, order, migratetype);
Alexander Duyck36e66c52020-04-06 20:04:56 -07001075
1076 /* Notify page reporting subsystem of freed page */
David Hildenbrandf04a5d52020-10-15 20:09:20 -07001077 if (!(fpi_flags & FPI_SKIP_REPORT_NOTIFY))
Alexander Duyck36e66c52020-04-06 20:04:56 -07001078 page_reporting_notify_free(order);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001079}
1080
Mel Gorman7bfec6f2016-05-19 17:14:15 -07001081/*
1082 * A bad page could be due to a number of fields. Instead of multiple branches,
1083 * try and check multiple fields with one check. The caller must do a detailed
1084 * check if necessary.
1085 */
1086static inline bool page_expected_state(struct page *page,
1087 unsigned long check_flags)
1088{
1089 if (unlikely(atomic_read(&page->_mapcount) != -1))
1090 return false;
1091
1092 if (unlikely((unsigned long)page->mapping |
1093 page_ref_count(page) |
1094#ifdef CONFIG_MEMCG
1095 (unsigned long)page->mem_cgroup |
1096#endif
1097 (page->flags & check_flags)))
1098 return false;
1099
1100 return true;
1101}
1102
Wei Yang58b7f112020-06-03 15:58:39 -07001103static const char *page_bad_reason(struct page *page, unsigned long flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001104{
Wei Yang82a32412020-06-03 15:58:29 -07001105 const char *bad_reason = NULL;
Dave Hansenf0b791a2014-01-23 15:52:49 -08001106
Kirill A. Shutemov53f92632016-01-15 16:53:42 -08001107 if (unlikely(atomic_read(&page->_mapcount) != -1))
Dave Hansenf0b791a2014-01-23 15:52:49 -08001108 bad_reason = "nonzero mapcount";
1109 if (unlikely(page->mapping != NULL))
1110 bad_reason = "non-NULL mapping";
Joonsoo Kimfe896d12016-03-17 14:19:26 -07001111 if (unlikely(page_ref_count(page) != 0))
Joonsoo Kim0139aa72016-05-19 17:10:49 -07001112 bad_reason = "nonzero _refcount";
Wei Yang58b7f112020-06-03 15:58:39 -07001113 if (unlikely(page->flags & flags)) {
1114 if (flags == PAGE_FLAGS_CHECK_AT_PREP)
1115 bad_reason = "PAGE_FLAGS_CHECK_AT_PREP flag(s) set";
1116 else
1117 bad_reason = "PAGE_FLAGS_CHECK_AT_FREE flag(s) set";
Dave Hansenf0b791a2014-01-23 15:52:49 -08001118 }
Johannes Weiner9edad6e2014-12-10 15:44:58 -08001119#ifdef CONFIG_MEMCG
1120 if (unlikely(page->mem_cgroup))
1121 bad_reason = "page still charged to cgroup";
1122#endif
Wei Yang58b7f112020-06-03 15:58:39 -07001123 return bad_reason;
Mel Gormanbb552ac2016-05-19 17:14:18 -07001124}
1125
Wei Yang58b7f112020-06-03 15:58:39 -07001126static void check_free_page_bad(struct page *page)
1127{
1128 bad_page(page,
1129 page_bad_reason(page, PAGE_FLAGS_CHECK_AT_FREE));
Mel Gormanbb552ac2016-05-19 17:14:18 -07001130}
1131
Wei Yang534fe5e2020-06-03 15:58:36 -07001132static inline int check_free_page(struct page *page)
Mel Gormanbb552ac2016-05-19 17:14:18 -07001133{
Mel Gormanda838d42016-05-19 17:14:21 -07001134 if (likely(page_expected_state(page, PAGE_FLAGS_CHECK_AT_FREE)))
Mel Gormanbb552ac2016-05-19 17:14:18 -07001135 return 0;
Mel Gormanbb552ac2016-05-19 17:14:18 -07001136
1137 /* Something has gone sideways, find it */
Wei Yang0d0c48a2020-06-03 15:58:33 -07001138 check_free_page_bad(page);
Mel Gorman7bfec6f2016-05-19 17:14:15 -07001139 return 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001140}
1141
Mel Gorman4db75482016-05-19 17:14:32 -07001142static int free_tail_pages_check(struct page *head_page, struct page *page)
1143{
1144 int ret = 1;
1145
1146 /*
1147 * We rely page->lru.next never has bit 0 set, unless the page
1148 * is PageTail(). Let's make sure that's true even for poisoned ->lru.
1149 */
1150 BUILD_BUG_ON((unsigned long)LIST_POISON1 & 1);
1151
1152 if (!IS_ENABLED(CONFIG_DEBUG_VM)) {
1153 ret = 0;
1154 goto out;
1155 }
1156 switch (page - head_page) {
1157 case 1:
Matthew Wilcox4da19842018-06-07 17:08:50 -07001158 /* the first tail page: ->mapping may be compound_mapcount() */
Mel Gorman4db75482016-05-19 17:14:32 -07001159 if (unlikely(compound_mapcount(page))) {
Wei Yang82a32412020-06-03 15:58:29 -07001160 bad_page(page, "nonzero compound_mapcount");
Mel Gorman4db75482016-05-19 17:14:32 -07001161 goto out;
1162 }
1163 break;
1164 case 2:
1165 /*
1166 * the second tail page: ->mapping is
Matthew Wilcoxfa3015b2018-06-07 17:08:42 -07001167 * deferred_list.next -- ignore value.
Mel Gorman4db75482016-05-19 17:14:32 -07001168 */
1169 break;
1170 default:
1171 if (page->mapping != TAIL_MAPPING) {
Wei Yang82a32412020-06-03 15:58:29 -07001172 bad_page(page, "corrupted mapping in tail page");
Mel Gorman4db75482016-05-19 17:14:32 -07001173 goto out;
1174 }
1175 break;
1176 }
1177 if (unlikely(!PageTail(page))) {
Wei Yang82a32412020-06-03 15:58:29 -07001178 bad_page(page, "PageTail not set");
Mel Gorman4db75482016-05-19 17:14:32 -07001179 goto out;
1180 }
1181 if (unlikely(compound_head(page) != head_page)) {
Wei Yang82a32412020-06-03 15:58:29 -07001182 bad_page(page, "compound_head not consistent");
Mel Gorman4db75482016-05-19 17:14:32 -07001183 goto out;
1184 }
1185 ret = 0;
1186out:
1187 page->mapping = NULL;
1188 clear_compound_head(page);
1189 return ret;
1190}
1191
Alexander Potapenko64713842019-07-11 20:59:19 -07001192static void kernel_init_free_pages(struct page *page, int numpages)
1193{
1194 int i;
1195
Qian Cai9e15afa2020-08-06 23:25:54 -07001196 /* s390's use of memset() could override KASAN redzones. */
1197 kasan_disable_current();
Alexander Potapenko64713842019-07-11 20:59:19 -07001198 for (i = 0; i < numpages; i++)
1199 clear_highpage(page + i);
Qian Cai9e15afa2020-08-06 23:25:54 -07001200 kasan_enable_current();
Alexander Potapenko64713842019-07-11 20:59:19 -07001201}
1202
Mel Gormane2769db2016-05-19 17:14:38 -07001203static __always_inline bool free_pages_prepare(struct page *page,
1204 unsigned int order, bool check_free)
1205{
1206 int bad = 0;
1207
1208 VM_BUG_ON_PAGE(PageTail(page), page);
1209
1210 trace_mm_page_free(page, order);
Mel Gormane2769db2016-05-19 17:14:38 -07001211
Oscar Salvador79f5f8f2020-10-15 20:07:09 -07001212 if (unlikely(PageHWPoison(page)) && !order) {
1213 /*
1214 * Do not let hwpoison pages hit pcplists/buddy
1215 * Untie memcg state and reset page's owner
1216 */
1217 if (memcg_kmem_enabled() && PageKmemcg(page))
1218 __memcg_kmem_uncharge_page(page, order);
1219 reset_page_owner(page, order);
1220 return false;
1221 }
1222
Mel Gormane2769db2016-05-19 17:14:38 -07001223 /*
1224 * Check tail pages before head page information is cleared to
1225 * avoid checking PageCompound for order-0 pages.
1226 */
1227 if (unlikely(order)) {
1228 bool compound = PageCompound(page);
1229 int i;
1230
1231 VM_BUG_ON_PAGE(compound && compound_order(page) != order, page);
1232
Kirill A. Shutemov9a73f612016-07-26 15:25:53 -07001233 if (compound)
1234 ClearPageDoubleMap(page);
Mel Gormane2769db2016-05-19 17:14:38 -07001235 for (i = 1; i < (1 << order); i++) {
1236 if (compound)
1237 bad += free_tail_pages_check(page, page + i);
Wei Yang534fe5e2020-06-03 15:58:36 -07001238 if (unlikely(check_free_page(page + i))) {
Mel Gormane2769db2016-05-19 17:14:38 -07001239 bad++;
1240 continue;
1241 }
1242 (page + i)->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
1243 }
1244 }
Minchan Kimbda807d2016-07-26 15:23:05 -07001245 if (PageMappingFlags(page))
Mel Gormane2769db2016-05-19 17:14:38 -07001246 page->mapping = NULL;
Vladimir Davydovc4159a72016-08-08 23:03:12 +03001247 if (memcg_kmem_enabled() && PageKmemcg(page))
Roman Gushchinf4b00ea2020-04-01 21:06:46 -07001248 __memcg_kmem_uncharge_page(page, order);
Mel Gormane2769db2016-05-19 17:14:38 -07001249 if (check_free)
Wei Yang534fe5e2020-06-03 15:58:36 -07001250 bad += check_free_page(page);
Mel Gormane2769db2016-05-19 17:14:38 -07001251 if (bad)
1252 return false;
1253
1254 page_cpupid_reset_last(page);
1255 page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
1256 reset_page_owner(page, order);
1257
1258 if (!PageHighMem(page)) {
1259 debug_check_no_locks_freed(page_address(page),
1260 PAGE_SIZE << order);
1261 debug_check_no_obj_freed(page_address(page),
1262 PAGE_SIZE << order);
1263 }
Alexander Potapenko64713842019-07-11 20:59:19 -07001264 if (want_init_on_free())
1265 kernel_init_free_pages(page, 1 << order);
1266
Mel Gormane2769db2016-05-19 17:14:38 -07001267 kernel_poison_pages(page, 1 << order, 0);
Qian Cai234fdce2019-10-06 17:58:25 -07001268 /*
1269 * arch_free_page() can make the page's contents inaccessible. s390
1270 * does this. So nothing which can access the page's contents should
1271 * happen after this.
1272 */
1273 arch_free_page(page, order);
1274
Vlastimil Babka8e57f8a2020-01-13 16:29:20 -08001275 if (debug_pagealloc_enabled_static())
Rick Edgecombed6332692019-04-25 17:11:35 -07001276 kernel_map_pages(page, 1 << order, 0);
1277
Waiman Long3c0c12c2018-12-28 00:38:51 -08001278 kasan_free_nondeferred_pages(page, order);
Mel Gormane2769db2016-05-19 17:14:38 -07001279
1280 return true;
1281}
Mel Gorman4db75482016-05-19 17:14:32 -07001282
1283#ifdef CONFIG_DEBUG_VM
Vlastimil Babka4462b322019-07-11 20:55:09 -07001284/*
1285 * With DEBUG_VM enabled, order-0 pages are checked immediately when being freed
1286 * to pcp lists. With debug_pagealloc also enabled, they are also rechecked when
1287 * moved from pcp lists to free lists.
1288 */
1289static bool free_pcp_prepare(struct page *page)
Mel Gorman4db75482016-05-19 17:14:32 -07001290{
Mel Gormane2769db2016-05-19 17:14:38 -07001291 return free_pages_prepare(page, 0, true);
Mel Gorman4db75482016-05-19 17:14:32 -07001292}
1293
Vlastimil Babka4462b322019-07-11 20:55:09 -07001294static bool bulkfree_pcp_prepare(struct page *page)
Mel Gorman4db75482016-05-19 17:14:32 -07001295{
Vlastimil Babka8e57f8a2020-01-13 16:29:20 -08001296 if (debug_pagealloc_enabled_static())
Wei Yang534fe5e2020-06-03 15:58:36 -07001297 return check_free_page(page);
Vlastimil Babka4462b322019-07-11 20:55:09 -07001298 else
1299 return false;
Mel Gorman4db75482016-05-19 17:14:32 -07001300}
1301#else
Vlastimil Babka4462b322019-07-11 20:55:09 -07001302/*
1303 * With DEBUG_VM disabled, order-0 pages being freed are checked only when
1304 * moving from pcp lists to free list in order to reduce overhead. With
1305 * debug_pagealloc enabled, they are checked also immediately when being freed
1306 * to the pcp lists.
1307 */
Mel Gorman4db75482016-05-19 17:14:32 -07001308static bool free_pcp_prepare(struct page *page)
1309{
Vlastimil Babka8e57f8a2020-01-13 16:29:20 -08001310 if (debug_pagealloc_enabled_static())
Vlastimil Babka4462b322019-07-11 20:55:09 -07001311 return free_pages_prepare(page, 0, true);
1312 else
1313 return free_pages_prepare(page, 0, false);
Mel Gorman4db75482016-05-19 17:14:32 -07001314}
1315
1316static bool bulkfree_pcp_prepare(struct page *page)
1317{
Wei Yang534fe5e2020-06-03 15:58:36 -07001318 return check_free_page(page);
Mel Gorman4db75482016-05-19 17:14:32 -07001319}
1320#endif /* CONFIG_DEBUG_VM */
1321
Aaron Lu97334162018-04-05 16:24:14 -07001322static inline void prefetch_buddy(struct page *page)
1323{
1324 unsigned long pfn = page_to_pfn(page);
1325 unsigned long buddy_pfn = __find_buddy_pfn(pfn, 0);
1326 struct page *buddy = page + (buddy_pfn - pfn);
1327
1328 prefetch(buddy);
1329}
1330
Linus Torvalds1da177e2005-04-16 15:20:36 -07001331/*
Mel Gorman5f8dcc22009-09-21 17:03:19 -07001332 * Frees a number of pages from the PCP lists
Linus Torvalds1da177e2005-04-16 15:20:36 -07001333 * Assumes all pages on list are in same zone, and of same order.
Renaud Lienhart207f36e2005-09-10 00:26:59 -07001334 * count is the number of pages to free.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001335 *
1336 * If the zone was previously in an "all pages pinned" state then look to
1337 * see if this freeing clears that state.
1338 *
1339 * And clear the zone's pages_scanned counter, to hold off the "all pages are
1340 * pinned" detection logic.
1341 */
Mel Gorman5f8dcc22009-09-21 17:03:19 -07001342static void free_pcppages_bulk(struct zone *zone, int count,
1343 struct per_cpu_pages *pcp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001344{
Mel Gorman5f8dcc22009-09-21 17:03:19 -07001345 int migratetype = 0;
Mel Gormana6f9edd62009-09-21 17:03:20 -07001346 int batch_free = 0;
Aaron Lu97334162018-04-05 16:24:14 -07001347 int prefetch_nr = 0;
Mel Gorman37779992016-05-19 17:13:58 -07001348 bool isolated_pageblocks;
Aaron Lu0a5f4e52018-04-05 16:24:10 -07001349 struct page *page, *tmp;
1350 LIST_HEAD(head);
Mel Gormanf2260e62009-06-16 15:32:13 -07001351
Charan Teja Reddy88e8ac12020-08-20 17:42:27 -07001352 /*
1353 * Ensure proper count is passed which otherwise would stuck in the
1354 * below while (list_empty(list)) loop.
1355 */
1356 count = min(pcp->count, count);
Mel Gormane5b31ac2016-05-19 17:14:24 -07001357 while (count) {
Mel Gorman5f8dcc22009-09-21 17:03:19 -07001358 struct list_head *list;
Nick Piggin48db57f2006-01-08 01:00:42 -08001359
Mel Gorman5f8dcc22009-09-21 17:03:19 -07001360 /*
Mel Gormana6f9edd62009-09-21 17:03:20 -07001361 * Remove pages from lists in a round-robin fashion. A
1362 * batch_free count is maintained that is incremented when an
1363 * empty list is encountered. This is so more pages are freed
1364 * off fuller lists instead of spinning excessively around empty
1365 * lists
Mel Gorman5f8dcc22009-09-21 17:03:19 -07001366 */
1367 do {
Mel Gormana6f9edd62009-09-21 17:03:20 -07001368 batch_free++;
Mel Gorman5f8dcc22009-09-21 17:03:19 -07001369 if (++migratetype == MIGRATE_PCPTYPES)
1370 migratetype = 0;
1371 list = &pcp->lists[migratetype];
1372 } while (list_empty(list));
1373
Namhyung Kim1d168712011-03-22 16:32:45 -07001374 /* This is the only non-empty list. Free them all. */
1375 if (batch_free == MIGRATE_PCPTYPES)
Mel Gormane5b31ac2016-05-19 17:14:24 -07001376 batch_free = count;
Namhyung Kim1d168712011-03-22 16:32:45 -07001377
Mel Gormana6f9edd62009-09-21 17:03:20 -07001378 do {
Geliang Tanga16601c2016-01-14 15:20:30 -08001379 page = list_last_entry(list, struct page, lru);
Aaron Lu0a5f4e52018-04-05 16:24:10 -07001380 /* must delete to avoid corrupting pcp list */
Mel Gormana6f9edd62009-09-21 17:03:20 -07001381 list_del(&page->lru);
Aaron Lu77ba9062018-04-05 16:24:06 -07001382 pcp->count--;
Vlastimil Babkaaa016d12015-09-08 15:01:22 -07001383
Mel Gorman4db75482016-05-19 17:14:32 -07001384 if (bulkfree_pcp_prepare(page))
1385 continue;
1386
Aaron Lu0a5f4e52018-04-05 16:24:10 -07001387 list_add_tail(&page->lru, &head);
Aaron Lu97334162018-04-05 16:24:14 -07001388
1389 /*
1390 * We are going to put the page back to the global
1391 * pool, prefetch its buddy to speed up later access
1392 * under zone->lock. It is believed the overhead of
1393 * an additional test and calculating buddy_pfn here
1394 * can be offset by reduced memory latency later. To
1395 * avoid excessive prefetching due to large count, only
1396 * prefetch buddy for the first pcp->batch nr of pages.
1397 */
1398 if (prefetch_nr++ < pcp->batch)
1399 prefetch_buddy(page);
Mel Gormane5b31ac2016-05-19 17:14:24 -07001400 } while (--count && --batch_free && !list_empty(list));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001401 }
Aaron Lu0a5f4e52018-04-05 16:24:10 -07001402
1403 spin_lock(&zone->lock);
1404 isolated_pageblocks = has_isolate_pageblock(zone);
1405
1406 /*
1407 * Use safe version since after __free_one_page(),
1408 * page->lru.next will not point to original list.
1409 */
1410 list_for_each_entry_safe(page, tmp, &head, lru) {
1411 int mt = get_pcppage_migratetype(page);
1412 /* MIGRATE_ISOLATE page should not go to pcplists */
1413 VM_BUG_ON_PAGE(is_migrate_isolate(mt), page);
1414 /* Pageblock could have been isolated meanwhile */
1415 if (unlikely(isolated_pageblocks))
1416 mt = get_pageblock_migratetype(page);
1417
David Hildenbrandf04a5d52020-10-15 20:09:20 -07001418 __free_one_page(page, page_to_pfn(page), zone, 0, mt, FPI_NONE);
Aaron Lu0a5f4e52018-04-05 16:24:10 -07001419 trace_mm_page_pcpu_drain(page, 0, mt);
1420 }
Mel Gormand34b0732017-04-20 14:37:43 -07001421 spin_unlock(&zone->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001422}
1423
Mel Gormandc4b0ca2014-06-04 16:10:17 -07001424static void free_one_page(struct zone *zone,
1425 struct page *page, unsigned long pfn,
Mel Gorman7aeb09f2014-06-04 16:10:21 -07001426 unsigned int order,
David Hildenbrand7fef4312020-10-15 20:09:35 -07001427 int migratetype, fpi_t fpi_flags)
Nick Piggin48db57f2006-01-08 01:00:42 -08001428{
Mel Gormand34b0732017-04-20 14:37:43 -07001429 spin_lock(&zone->lock);
Joonsoo Kimad53f922014-11-13 15:19:11 -08001430 if (unlikely(has_isolate_pageblock(zone) ||
1431 is_migrate_isolate(migratetype))) {
1432 migratetype = get_pfnblock_migratetype(page, pfn);
Joonsoo Kimad53f922014-11-13 15:19:11 -08001433 }
David Hildenbrand7fef4312020-10-15 20:09:35 -07001434 __free_one_page(page, pfn, zone, order, migratetype, fpi_flags);
Mel Gormand34b0732017-04-20 14:37:43 -07001435 spin_unlock(&zone->lock);
Nick Piggin48db57f2006-01-08 01:00:42 -08001436}
1437
Robin Holt1e8ce832015-06-30 14:56:45 -07001438static void __meminit __init_single_page(struct page *page, unsigned long pfn,
Pavel Tatashind0dc12e2018-04-05 16:23:00 -07001439 unsigned long zone, int nid)
Robin Holt1e8ce832015-06-30 14:56:45 -07001440{
Pavel Tatashind0dc12e2018-04-05 16:23:00 -07001441 mm_zero_struct_page(page);
Robin Holt1e8ce832015-06-30 14:56:45 -07001442 set_page_links(page, zone, nid, pfn);
Robin Holt1e8ce832015-06-30 14:56:45 -07001443 init_page_count(page);
1444 page_mapcount_reset(page);
1445 page_cpupid_reset_last(page);
Andrey Konovalov2813b9c2018-12-28 00:30:57 -08001446 page_kasan_tag_reset(page);
Robin Holt1e8ce832015-06-30 14:56:45 -07001447
Robin Holt1e8ce832015-06-30 14:56:45 -07001448 INIT_LIST_HEAD(&page->lru);
1449#ifdef WANT_PAGE_VIRTUAL
1450 /* The shift won't overflow because ZONE_NORMAL is below 4G. */
1451 if (!is_highmem_idx(zone))
1452 set_page_address(page, __va(pfn << PAGE_SHIFT));
1453#endif
1454}
1455
Mel Gorman7e18adb2015-06-30 14:57:05 -07001456#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
Arnd Bergmann57148a62017-10-03 16:15:10 -07001457static void __meminit init_reserved_page(unsigned long pfn)
Mel Gorman7e18adb2015-06-30 14:57:05 -07001458{
1459 pg_data_t *pgdat;
1460 int nid, zid;
1461
1462 if (!early_page_uninitialised(pfn))
1463 return;
1464
1465 nid = early_pfn_to_nid(pfn);
1466 pgdat = NODE_DATA(nid);
1467
1468 for (zid = 0; zid < MAX_NR_ZONES; zid++) {
1469 struct zone *zone = &pgdat->node_zones[zid];
1470
1471 if (pfn >= zone->zone_start_pfn && pfn < zone_end_pfn(zone))
1472 break;
1473 }
Pavel Tatashind0dc12e2018-04-05 16:23:00 -07001474 __init_single_page(pfn_to_page(pfn), pfn, zid, nid);
Mel Gorman7e18adb2015-06-30 14:57:05 -07001475}
1476#else
1477static inline void init_reserved_page(unsigned long pfn)
1478{
1479}
1480#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
1481
Nathan Zimmer92923ca32015-06-30 14:56:48 -07001482/*
1483 * Initialised pages do not have PageReserved set. This function is
1484 * called for each range allocated by the bootmem allocator and
1485 * marks the pages PageReserved. The remaining valid pages are later
1486 * sent to the buddy page allocator.
1487 */
Stefan Bader4b50bcc2016-05-20 16:58:38 -07001488void __meminit reserve_bootmem_region(phys_addr_t start, phys_addr_t end)
Nathan Zimmer92923ca32015-06-30 14:56:48 -07001489{
1490 unsigned long start_pfn = PFN_DOWN(start);
1491 unsigned long end_pfn = PFN_UP(end);
1492
Mel Gorman7e18adb2015-06-30 14:57:05 -07001493 for (; start_pfn < end_pfn; start_pfn++) {
1494 if (pfn_valid(start_pfn)) {
1495 struct page *page = pfn_to_page(start_pfn);
1496
1497 init_reserved_page(start_pfn);
Kirill A. Shutemov1d798ca2015-11-06 16:29:54 -08001498
1499 /* Avoid false-positive PageTail() */
1500 INIT_LIST_HEAD(&page->lru);
1501
Alexander Duyckd483da52018-10-26 15:07:48 -07001502 /*
1503 * no need for atomic set_bit because the struct
1504 * page is not visible yet so nobody should
1505 * access it yet.
1506 */
1507 __SetPageReserved(page);
Mel Gorman7e18adb2015-06-30 14:57:05 -07001508 }
1509 }
Nathan Zimmer92923ca32015-06-30 14:56:48 -07001510}
1511
David Hildenbrand7fef4312020-10-15 20:09:35 -07001512static void __free_pages_ok(struct page *page, unsigned int order,
1513 fpi_t fpi_flags)
KOSAKI Motohiroec95f532010-05-24 14:32:38 -07001514{
Mel Gormand34b0732017-04-20 14:37:43 -07001515 unsigned long flags;
Minchan Kim95e34412012-10-08 16:32:11 -07001516 int migratetype;
Mel Gormandc4b0ca2014-06-04 16:10:17 -07001517 unsigned long pfn = page_to_pfn(page);
KOSAKI Motohiroec95f532010-05-24 14:32:38 -07001518
Mel Gormane2769db2016-05-19 17:14:38 -07001519 if (!free_pages_prepare(page, order, true))
KOSAKI Motohiroec95f532010-05-24 14:32:38 -07001520 return;
1521
Mel Gormancfc47a22014-06-04 16:10:19 -07001522 migratetype = get_pfnblock_migratetype(page, pfn);
Mel Gormand34b0732017-04-20 14:37:43 -07001523 local_irq_save(flags);
1524 __count_vm_events(PGFREE, 1 << order);
David Hildenbrand7fef4312020-10-15 20:09:35 -07001525 free_one_page(page_zone(page), page, pfn, order, migratetype,
1526 fpi_flags);
Mel Gormand34b0732017-04-20 14:37:43 -07001527 local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001528}
1529
Arun KSa9cd4102019-03-05 15:42:14 -08001530void __free_pages_core(struct page *page, unsigned int order)
David Howellsa226f6c2006-01-06 00:11:08 -08001531{
Johannes Weinerc3993072012-01-10 15:08:10 -08001532 unsigned int nr_pages = 1 << order;
Yinghai Lue2d0bd22013-09-11 14:20:37 -07001533 struct page *p = page;
Johannes Weinerc3993072012-01-10 15:08:10 -08001534 unsigned int loop;
David Howellsa226f6c2006-01-06 00:11:08 -08001535
David Hildenbrand7fef4312020-10-15 20:09:35 -07001536 /*
1537 * When initializing the memmap, __init_single_page() sets the refcount
1538 * of all pages to 1 ("allocated"/"not free"). We have to set the
1539 * refcount of all involved pages to 0.
1540 */
Yinghai Lue2d0bd22013-09-11 14:20:37 -07001541 prefetchw(p);
1542 for (loop = 0; loop < (nr_pages - 1); loop++, p++) {
1543 prefetchw(p + 1);
Johannes Weinerc3993072012-01-10 15:08:10 -08001544 __ClearPageReserved(p);
1545 set_page_count(p, 0);
David Howellsa226f6c2006-01-06 00:11:08 -08001546 }
Yinghai Lue2d0bd22013-09-11 14:20:37 -07001547 __ClearPageReserved(p);
1548 set_page_count(p, 0);
Johannes Weinerc3993072012-01-10 15:08:10 -08001549
Arun KS9705bea2018-12-28 00:34:24 -08001550 atomic_long_add(nr_pages, &page_zone(page)->managed_pages);
David Hildenbrand7fef4312020-10-15 20:09:35 -07001551
1552 /*
1553 * Bypass PCP and place fresh pages right to the tail, primarily
1554 * relevant for memory onlining.
1555 */
1556 __free_pages_ok(page, order, FPI_TO_TAIL);
David Howellsa226f6c2006-01-06 00:11:08 -08001557}
1558
Mike Rapoport3f08a302020-06-03 15:57:02 -07001559#ifdef CONFIG_NEED_MULTIPLE_NODES
Mel Gorman7ace9912015-08-06 15:46:13 -07001560
Mel Gorman75a592a2015-06-30 14:56:59 -07001561static struct mminit_pfnnid_cache early_pfnnid_cache __meminitdata;
1562
Mike Rapoport6f24fbd2020-06-03 15:56:57 -07001563#ifndef CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID
1564
1565/*
1566 * Required by SPARSEMEM. Given a PFN, return what node the PFN is on.
1567 */
1568int __meminit __early_pfn_to_nid(unsigned long pfn,
1569 struct mminit_pfnnid_cache *state)
1570{
1571 unsigned long start_pfn, end_pfn;
1572 int nid;
1573
1574 if (state->last_start <= pfn && pfn < state->last_end)
1575 return state->last_nid;
1576
1577 nid = memblock_search_pfn_nid(pfn, &start_pfn, &end_pfn);
1578 if (nid != NUMA_NO_NODE) {
1579 state->last_start = start_pfn;
1580 state->last_end = end_pfn;
1581 state->last_nid = nid;
1582 }
1583
1584 return nid;
1585}
1586#endif /* CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID */
1587
Mel Gorman75a592a2015-06-30 14:56:59 -07001588int __meminit early_pfn_to_nid(unsigned long pfn)
1589{
Mel Gorman7ace9912015-08-06 15:46:13 -07001590 static DEFINE_SPINLOCK(early_pfn_lock);
Mel Gorman75a592a2015-06-30 14:56:59 -07001591 int nid;
1592
Mel Gorman7ace9912015-08-06 15:46:13 -07001593 spin_lock(&early_pfn_lock);
Mel Gorman75a592a2015-06-30 14:56:59 -07001594 nid = __early_pfn_to_nid(pfn, &early_pfnnid_cache);
Mel Gorman7ace9912015-08-06 15:46:13 -07001595 if (nid < 0)
Mel Gormane4568d32016-07-14 12:07:20 -07001596 nid = first_online_node;
Mel Gorman7ace9912015-08-06 15:46:13 -07001597 spin_unlock(&early_pfn_lock);
1598
1599 return nid;
Mel Gorman75a592a2015-06-30 14:56:59 -07001600}
Mike Rapoport3f08a302020-06-03 15:57:02 -07001601#endif /* CONFIG_NEED_MULTIPLE_NODES */
Mel Gorman75a592a2015-06-30 14:56:59 -07001602
Mike Rapoport7c2ee342018-10-30 15:09:36 -07001603void __init memblock_free_pages(struct page *page, unsigned long pfn,
Mel Gorman3a80a7f2015-06-30 14:57:02 -07001604 unsigned int order)
1605{
1606 if (early_page_uninitialised(pfn))
1607 return;
Arun KSa9cd4102019-03-05 15:42:14 -08001608 __free_pages_core(page, order);
Mel Gorman3a80a7f2015-06-30 14:57:02 -07001609}
1610
Joonsoo Kim7cf91a92016-03-15 14:57:51 -07001611/*
1612 * Check that the whole (or subset of) a pageblock given by the interval of
1613 * [start_pfn, end_pfn) is valid and within the same zone, before scanning it
1614 * with the migration of free compaction scanner. The scanners then need to
1615 * use only pfn_valid_within() check for arches that allow holes within
1616 * pageblocks.
1617 *
1618 * Return struct page pointer of start_pfn, or NULL if checks were not passed.
1619 *
1620 * It's possible on some configurations to have a setup like node0 node1 node0
1621 * i.e. it's possible that all pages within a zones range of pages do not
1622 * belong to a single zone. We assume that a border between node0 and node1
1623 * can occur within a single pageblock, but not a node0 node1 node0
1624 * interleaving within a single pageblock. It is therefore sufficient to check
1625 * the first and last page of a pageblock and avoid checking each individual
1626 * page in a pageblock.
1627 */
1628struct page *__pageblock_pfn_to_page(unsigned long start_pfn,
1629 unsigned long end_pfn, struct zone *zone)
1630{
1631 struct page *start_page;
1632 struct page *end_page;
1633
1634 /* end_pfn is one past the range we are checking */
1635 end_pfn--;
1636
1637 if (!pfn_valid(start_pfn) || !pfn_valid(end_pfn))
1638 return NULL;
1639
Michal Hocko2d070ea2017-07-06 15:37:56 -07001640 start_page = pfn_to_online_page(start_pfn);
1641 if (!start_page)
1642 return NULL;
Joonsoo Kim7cf91a92016-03-15 14:57:51 -07001643
1644 if (page_zone(start_page) != zone)
1645 return NULL;
1646
1647 end_page = pfn_to_page(end_pfn);
1648
1649 /* This gives a shorter code than deriving page_zone(end_page) */
1650 if (page_zone_id(start_page) != page_zone_id(end_page))
1651 return NULL;
1652
1653 return start_page;
1654}
1655
1656void set_zone_contiguous(struct zone *zone)
1657{
1658 unsigned long block_start_pfn = zone->zone_start_pfn;
1659 unsigned long block_end_pfn;
1660
1661 block_end_pfn = ALIGN(block_start_pfn + 1, pageblock_nr_pages);
1662 for (; block_start_pfn < zone_end_pfn(zone);
1663 block_start_pfn = block_end_pfn,
1664 block_end_pfn += pageblock_nr_pages) {
1665
1666 block_end_pfn = min(block_end_pfn, zone_end_pfn(zone));
1667
1668 if (!__pageblock_pfn_to_page(block_start_pfn,
1669 block_end_pfn, zone))
1670 return;
David Hildenbrande84fe992020-05-07 18:35:46 -07001671 cond_resched();
Joonsoo Kim7cf91a92016-03-15 14:57:51 -07001672 }
1673
1674 /* We confirm that there is no hole */
1675 zone->contiguous = true;
1676}
1677
1678void clear_zone_contiguous(struct zone *zone)
1679{
1680 zone->contiguous = false;
1681}
1682
Mel Gorman7e18adb2015-06-30 14:57:05 -07001683#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
Pavel Tatashin2f47a912017-11-15 17:36:09 -08001684static void __init deferred_free_range(unsigned long pfn,
1685 unsigned long nr_pages)
Mel Gormana4de83d2015-06-30 14:57:16 -07001686{
Pavel Tatashin2f47a912017-11-15 17:36:09 -08001687 struct page *page;
1688 unsigned long i;
Mel Gormana4de83d2015-06-30 14:57:16 -07001689
Pavel Tatashin2f47a912017-11-15 17:36:09 -08001690 if (!nr_pages)
Mel Gormana4de83d2015-06-30 14:57:16 -07001691 return;
1692
Pavel Tatashin2f47a912017-11-15 17:36:09 -08001693 page = pfn_to_page(pfn);
1694
Mel Gormana4de83d2015-06-30 14:57:16 -07001695 /* Free a large naturally-aligned chunk if possible */
Xishi Qiue7801492016-10-07 16:58:09 -07001696 if (nr_pages == pageblock_nr_pages &&
1697 (pfn & (pageblock_nr_pages - 1)) == 0) {
Mel Gormanac5d2532015-06-30 14:57:20 -07001698 set_pageblock_migratetype(page, MIGRATE_MOVABLE);
Arun KSa9cd4102019-03-05 15:42:14 -08001699 __free_pages_core(page, pageblock_order);
Mel Gormana4de83d2015-06-30 14:57:16 -07001700 return;
1701 }
1702
Xishi Qiue7801492016-10-07 16:58:09 -07001703 for (i = 0; i < nr_pages; i++, page++, pfn++) {
1704 if ((pfn & (pageblock_nr_pages - 1)) == 0)
1705 set_pageblock_migratetype(page, MIGRATE_MOVABLE);
Arun KSa9cd4102019-03-05 15:42:14 -08001706 __free_pages_core(page, 0);
Xishi Qiue7801492016-10-07 16:58:09 -07001707 }
Mel Gormana4de83d2015-06-30 14:57:16 -07001708}
1709
Nicolai Stanged3cd1312015-08-06 15:46:16 -07001710/* Completion tracking for deferred_init_memmap() threads */
1711static atomic_t pgdat_init_n_undone __initdata;
1712static __initdata DECLARE_COMPLETION(pgdat_init_all_done_comp);
1713
1714static inline void __init pgdat_init_report_one_done(void)
1715{
1716 if (atomic_dec_and_test(&pgdat_init_n_undone))
1717 complete(&pgdat_init_all_done_comp);
1718}
Mel Gorman0e1cc952015-06-30 14:57:27 -07001719
Pavel Tatashin2f47a912017-11-15 17:36:09 -08001720/*
Pavel Tatashin80b1f412018-01-31 16:16:30 -08001721 * Returns true if page needs to be initialized or freed to buddy allocator.
1722 *
1723 * First we check if pfn is valid on architectures where it is possible to have
1724 * holes within pageblock_nr_pages. On systems where it is not possible, this
1725 * function is optimized out.
1726 *
1727 * Then, we check if a current large page is valid by only checking the validity
1728 * of the head pfn.
Pavel Tatashin2f47a912017-11-15 17:36:09 -08001729 */
Alexander Duyck56ec43d2019-05-13 17:21:13 -07001730static inline bool __init deferred_pfn_valid(unsigned long pfn)
Pavel Tatashin2f47a912017-11-15 17:36:09 -08001731{
Pavel Tatashin80b1f412018-01-31 16:16:30 -08001732 if (!pfn_valid_within(pfn))
1733 return false;
1734 if (!(pfn & (pageblock_nr_pages - 1)) && !pfn_valid(pfn))
1735 return false;
Pavel Tatashin80b1f412018-01-31 16:16:30 -08001736 return true;
Pavel Tatashin2f47a912017-11-15 17:36:09 -08001737}
1738
Pavel Tatashin80b1f412018-01-31 16:16:30 -08001739/*
1740 * Free pages to buddy allocator. Try to free aligned pages in
1741 * pageblock_nr_pages sizes.
1742 */
Alexander Duyck56ec43d2019-05-13 17:21:13 -07001743static void __init deferred_free_pages(unsigned long pfn,
Pavel Tatashin80b1f412018-01-31 16:16:30 -08001744 unsigned long end_pfn)
Pavel Tatashin2f47a912017-11-15 17:36:09 -08001745{
Pavel Tatashin2f47a912017-11-15 17:36:09 -08001746 unsigned long nr_pgmask = pageblock_nr_pages - 1;
Pavel Tatashin2f47a912017-11-15 17:36:09 -08001747 unsigned long nr_free = 0;
Pavel Tatashin2f47a912017-11-15 17:36:09 -08001748
Pavel Tatashin80b1f412018-01-31 16:16:30 -08001749 for (; pfn < end_pfn; pfn++) {
Alexander Duyck56ec43d2019-05-13 17:21:13 -07001750 if (!deferred_pfn_valid(pfn)) {
Pavel Tatashin80b1f412018-01-31 16:16:30 -08001751 deferred_free_range(pfn - nr_free, nr_free);
1752 nr_free = 0;
1753 } else if (!(pfn & nr_pgmask)) {
1754 deferred_free_range(pfn - nr_free, nr_free);
Pavel Tatashin2f47a912017-11-15 17:36:09 -08001755 nr_free = 1;
Pavel Tatashin80b1f412018-01-31 16:16:30 -08001756 } else {
1757 nr_free++;
Pavel Tatashin2f47a912017-11-15 17:36:09 -08001758 }
1759 }
1760 /* Free the last block of pages to allocator */
Pavel Tatashin80b1f412018-01-31 16:16:30 -08001761 deferred_free_range(pfn - nr_free, nr_free);
1762}
Pavel Tatashin2f47a912017-11-15 17:36:09 -08001763
Pavel Tatashin80b1f412018-01-31 16:16:30 -08001764/*
1765 * Initialize struct pages. We minimize pfn page lookups and scheduler checks
1766 * by performing it only once every pageblock_nr_pages.
1767 * Return number of pages initialized.
1768 */
Alexander Duyck56ec43d2019-05-13 17:21:13 -07001769static unsigned long __init deferred_init_pages(struct zone *zone,
Pavel Tatashin80b1f412018-01-31 16:16:30 -08001770 unsigned long pfn,
1771 unsigned long end_pfn)
1772{
Pavel Tatashin80b1f412018-01-31 16:16:30 -08001773 unsigned long nr_pgmask = pageblock_nr_pages - 1;
Alexander Duyck56ec43d2019-05-13 17:21:13 -07001774 int nid = zone_to_nid(zone);
Pavel Tatashin80b1f412018-01-31 16:16:30 -08001775 unsigned long nr_pages = 0;
Alexander Duyck56ec43d2019-05-13 17:21:13 -07001776 int zid = zone_idx(zone);
Pavel Tatashin80b1f412018-01-31 16:16:30 -08001777 struct page *page = NULL;
1778
1779 for (; pfn < end_pfn; pfn++) {
Alexander Duyck56ec43d2019-05-13 17:21:13 -07001780 if (!deferred_pfn_valid(pfn)) {
Pavel Tatashin80b1f412018-01-31 16:16:30 -08001781 page = NULL;
1782 continue;
1783 } else if (!page || !(pfn & nr_pgmask)) {
1784 page = pfn_to_page(pfn);
Pavel Tatashin80b1f412018-01-31 16:16:30 -08001785 } else {
1786 page++;
1787 }
Pavel Tatashind0dc12e2018-04-05 16:23:00 -07001788 __init_single_page(page, pfn, zid, nid);
Pavel Tatashin80b1f412018-01-31 16:16:30 -08001789 nr_pages++;
1790 }
1791 return (nr_pages);
Pavel Tatashin2f47a912017-11-15 17:36:09 -08001792}
1793
Alexander Duyck0e56aca2019-05-13 17:21:20 -07001794/*
1795 * This function is meant to pre-load the iterator for the zone init.
1796 * Specifically it walks through the ranges until we are caught up to the
1797 * first_init_pfn value and exits there. If we never encounter the value we
1798 * return false indicating there are no valid ranges left.
1799 */
1800static bool __init
1801deferred_init_mem_pfn_range_in_zone(u64 *i, struct zone *zone,
1802 unsigned long *spfn, unsigned long *epfn,
1803 unsigned long first_init_pfn)
1804{
1805 u64 j;
1806
1807 /*
1808 * Start out by walking through the ranges in this zone that have
1809 * already been initialized. We don't need to do anything with them
1810 * so we just need to flush them out of the system.
1811 */
1812 for_each_free_mem_pfn_range_in_zone(j, zone, spfn, epfn) {
1813 if (*epfn <= first_init_pfn)
1814 continue;
1815 if (*spfn < first_init_pfn)
1816 *spfn = first_init_pfn;
1817 *i = j;
1818 return true;
1819 }
1820
1821 return false;
1822}
1823
1824/*
1825 * Initialize and free pages. We do it in two loops: first we initialize
1826 * struct page, then free to buddy allocator, because while we are
1827 * freeing pages we can access pages that are ahead (computing buddy
1828 * page in __free_one_page()).
1829 *
1830 * In order to try and keep some memory in the cache we have the loop
1831 * broken along max page order boundaries. This way we will not cause
1832 * any issues with the buddy page computation.
1833 */
1834static unsigned long __init
1835deferred_init_maxorder(u64 *i, struct zone *zone, unsigned long *start_pfn,
1836 unsigned long *end_pfn)
1837{
1838 unsigned long mo_pfn = ALIGN(*start_pfn + 1, MAX_ORDER_NR_PAGES);
1839 unsigned long spfn = *start_pfn, epfn = *end_pfn;
1840 unsigned long nr_pages = 0;
1841 u64 j = *i;
1842
1843 /* First we loop through and initialize the page values */
1844 for_each_free_mem_pfn_range_in_zone_from(j, zone, start_pfn, end_pfn) {
1845 unsigned long t;
1846
1847 if (mo_pfn <= *start_pfn)
1848 break;
1849
1850 t = min(mo_pfn, *end_pfn);
1851 nr_pages += deferred_init_pages(zone, *start_pfn, t);
1852
1853 if (mo_pfn < *end_pfn) {
1854 *start_pfn = mo_pfn;
1855 break;
1856 }
1857 }
1858
1859 /* Reset values and now loop through freeing pages as needed */
1860 swap(j, *i);
1861
1862 for_each_free_mem_pfn_range_in_zone_from(j, zone, &spfn, &epfn) {
1863 unsigned long t;
1864
1865 if (mo_pfn <= spfn)
1866 break;
1867
1868 t = min(mo_pfn, epfn);
1869 deferred_free_pages(spfn, t);
1870
1871 if (mo_pfn <= epfn)
1872 break;
1873 }
1874
1875 return nr_pages;
1876}
1877
Daniel Jordane4443142020-06-03 15:59:51 -07001878static void __init
1879deferred_init_memmap_chunk(unsigned long start_pfn, unsigned long end_pfn,
1880 void *arg)
1881{
1882 unsigned long spfn, epfn;
1883 struct zone *zone = arg;
1884 u64 i;
1885
1886 deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, start_pfn);
1887
1888 /*
1889 * Initialize and free pages in MAX_ORDER sized increments so that we
1890 * can avoid introducing any issues with the buddy allocator.
1891 */
1892 while (spfn < end_pfn) {
1893 deferred_init_maxorder(&i, zone, &spfn, &epfn);
1894 cond_resched();
1895 }
1896}
1897
Daniel Jordanecd09652020-06-03 15:59:55 -07001898/* An arch may override for more concurrency. */
1899__weak int __init
1900deferred_page_init_max_threads(const struct cpumask *node_cpumask)
1901{
1902 return 1;
1903}
1904
Mel Gorman7e18adb2015-06-30 14:57:05 -07001905/* Initialise remaining memory on a node */
Mel Gorman0e1cc952015-06-30 14:57:27 -07001906static int __init deferred_init_memmap(void *data)
Mel Gorman7e18adb2015-06-30 14:57:05 -07001907{
Mel Gorman0e1cc952015-06-30 14:57:27 -07001908 pg_data_t *pgdat = data;
Mel Gorman0e1cc952015-06-30 14:57:27 -07001909 const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);
Daniel Jordan89c7c402020-06-03 15:59:47 -07001910 unsigned long spfn = 0, epfn = 0;
Alexander Duyck0e56aca2019-05-13 17:21:20 -07001911 unsigned long first_init_pfn, flags;
1912 unsigned long start = jiffies;
1913 struct zone *zone;
Daniel Jordane4443142020-06-03 15:59:51 -07001914 int zid, max_threads;
Pavel Tatashin2f47a912017-11-15 17:36:09 -08001915 u64 i;
Mel Gorman7e18adb2015-06-30 14:57:05 -07001916
Mel Gorman0e1cc952015-06-30 14:57:27 -07001917 /* Bind memory initialisation thread to a local node if possible */
1918 if (!cpumask_empty(cpumask))
1919 set_cpus_allowed_ptr(current, cpumask);
Mel Gorman7e18adb2015-06-30 14:57:05 -07001920
Pavel Tatashin3a2d7fa2018-04-05 16:22:27 -07001921 pgdat_resize_lock(pgdat, &flags);
1922 first_init_pfn = pgdat->first_deferred_pfn;
1923 if (first_init_pfn == ULONG_MAX) {
1924 pgdat_resize_unlock(pgdat, &flags);
1925 pgdat_init_report_one_done();
1926 return 0;
1927 }
1928
Mel Gorman7e18adb2015-06-30 14:57:05 -07001929 /* Sanity check boundaries */
1930 BUG_ON(pgdat->first_deferred_pfn < pgdat->node_start_pfn);
1931 BUG_ON(pgdat->first_deferred_pfn > pgdat_end_pfn(pgdat));
1932 pgdat->first_deferred_pfn = ULONG_MAX;
1933
Pavel Tatashin3d060852020-06-03 15:59:24 -07001934 /*
1935 * Once we unlock here, the zone cannot be grown anymore, thus if an
1936 * interrupt thread must allocate this early in boot, zone must be
1937 * pre-grown prior to start of deferred page initialization.
1938 */
1939 pgdat_resize_unlock(pgdat, &flags);
1940
Mel Gorman7e18adb2015-06-30 14:57:05 -07001941 /* Only the highest zone is deferred so find it */
1942 for (zid = 0; zid < MAX_NR_ZONES; zid++) {
1943 zone = pgdat->node_zones + zid;
1944 if (first_init_pfn < zone_end_pfn(zone))
1945 break;
1946 }
Alexander Duyck0e56aca2019-05-13 17:21:20 -07001947
1948 /* If the zone is empty somebody else may have cleared out the zone */
1949 if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn,
1950 first_init_pfn))
1951 goto zone_empty;
Mel Gorman7e18adb2015-06-30 14:57:05 -07001952
Daniel Jordanecd09652020-06-03 15:59:55 -07001953 max_threads = deferred_page_init_max_threads(cpumask);
Mel Gorman7e18adb2015-06-30 14:57:05 -07001954
Daniel Jordan117003c2020-06-03 15:59:20 -07001955 while (spfn < epfn) {
Daniel Jordane4443142020-06-03 15:59:51 -07001956 unsigned long epfn_align = ALIGN(epfn, PAGES_PER_SECTION);
1957 struct padata_mt_job job = {
1958 .thread_fn = deferred_init_memmap_chunk,
1959 .fn_arg = zone,
1960 .start = spfn,
1961 .size = epfn_align - spfn,
1962 .align = PAGES_PER_SECTION,
1963 .min_chunk = PAGES_PER_SECTION,
1964 .max_threads = max_threads,
1965 };
1966
1967 padata_do_multithreaded(&job);
1968 deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn,
1969 epfn_align);
Daniel Jordan117003c2020-06-03 15:59:20 -07001970 }
Mel Gorman7e18adb2015-06-30 14:57:05 -07001971zone_empty:
Mel Gorman7e18adb2015-06-30 14:57:05 -07001972 /* Sanity check that the next zone really is unpopulated */
1973 WARN_ON(++zid < MAX_NR_ZONES && populated_zone(++zone));
1974
Daniel Jordan89c7c402020-06-03 15:59:47 -07001975 pr_info("node %d deferred pages initialised in %ums\n",
1976 pgdat->node_id, jiffies_to_msecs(jiffies - start));
Nicolai Stanged3cd1312015-08-06 15:46:16 -07001977
1978 pgdat_init_report_one_done();
Mel Gorman0e1cc952015-06-30 14:57:27 -07001979 return 0;
1980}
Pavel Tatashinc9e97a12018-04-05 16:22:31 -07001981
1982/*
Pavel Tatashinc9e97a12018-04-05 16:22:31 -07001983 * If this zone has deferred pages, try to grow it by initializing enough
1984 * deferred pages to satisfy the allocation specified by order, rounded up to
1985 * the nearest PAGES_PER_SECTION boundary. So we're adding memory in increments
1986 * of SECTION_SIZE bytes by initializing struct pages in increments of
1987 * PAGES_PER_SECTION * sizeof(struct page) bytes.
1988 *
1989 * Return true when zone was grown, otherwise return false. We return true even
1990 * when we grow less than requested, to let the caller decide if there are
1991 * enough pages to satisfy the allocation.
1992 *
1993 * Note: We use noinline because this function is needed only during boot, and
1994 * it is called from a __ref function _deferred_grow_zone. This way we are
1995 * making sure that it is not inlined into permanent text section.
1996 */
1997static noinline bool __init
1998deferred_grow_zone(struct zone *zone, unsigned int order)
1999{
Pavel Tatashinc9e97a12018-04-05 16:22:31 -07002000 unsigned long nr_pages_needed = ALIGN(1 << order, PAGES_PER_SECTION);
Alexander Duyck837566e2019-05-13 17:21:17 -07002001 pg_data_t *pgdat = zone->zone_pgdat;
Pavel Tatashinc9e97a12018-04-05 16:22:31 -07002002 unsigned long first_deferred_pfn = pgdat->first_deferred_pfn;
Alexander Duyck0e56aca2019-05-13 17:21:20 -07002003 unsigned long spfn, epfn, flags;
2004 unsigned long nr_pages = 0;
Pavel Tatashinc9e97a12018-04-05 16:22:31 -07002005 u64 i;
2006
2007 /* Only the last zone may have deferred pages */
2008 if (zone_end_pfn(zone) != pgdat_end_pfn(pgdat))
2009 return false;
2010
2011 pgdat_resize_lock(pgdat, &flags);
2012
2013 /*
Pavel Tatashinc9e97a12018-04-05 16:22:31 -07002014 * If someone grew this zone while we were waiting for spinlock, return
2015 * true, as there might be enough pages already.
2016 */
2017 if (first_deferred_pfn != pgdat->first_deferred_pfn) {
2018 pgdat_resize_unlock(pgdat, &flags);
2019 return true;
2020 }
2021
Alexander Duyck0e56aca2019-05-13 17:21:20 -07002022 /* If the zone is empty somebody else may have cleared out the zone */
2023 if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn,
2024 first_deferred_pfn)) {
2025 pgdat->first_deferred_pfn = ULONG_MAX;
Pavel Tatashinc9e97a12018-04-05 16:22:31 -07002026 pgdat_resize_unlock(pgdat, &flags);
Juergen Grossb9705d82019-07-04 15:14:36 -07002027 /* Retry only once. */
2028 return first_deferred_pfn != ULONG_MAX;
Pavel Tatashinc9e97a12018-04-05 16:22:31 -07002029 }
2030
Alexander Duyck0e56aca2019-05-13 17:21:20 -07002031 /*
2032 * Initialize and free pages in MAX_ORDER sized increments so
2033 * that we can avoid introducing any issues with the buddy
2034 * allocator.
2035 */
2036 while (spfn < epfn) {
2037 /* update our first deferred PFN for this section */
2038 first_deferred_pfn = spfn;
Pavel Tatashinc9e97a12018-04-05 16:22:31 -07002039
Alexander Duyck0e56aca2019-05-13 17:21:20 -07002040 nr_pages += deferred_init_maxorder(&i, zone, &spfn, &epfn);
Daniel Jordan117003c2020-06-03 15:59:20 -07002041 touch_nmi_watchdog();
Pavel Tatashinc9e97a12018-04-05 16:22:31 -07002042
Alexander Duyck0e56aca2019-05-13 17:21:20 -07002043 /* We should only stop along section boundaries */
2044 if ((first_deferred_pfn ^ spfn) < PAGES_PER_SECTION)
2045 continue;
2046
2047 /* If our quota has been met we can stop here */
Pavel Tatashinc9e97a12018-04-05 16:22:31 -07002048 if (nr_pages >= nr_pages_needed)
2049 break;
2050 }
2051
Alexander Duyck0e56aca2019-05-13 17:21:20 -07002052 pgdat->first_deferred_pfn = spfn;
Pavel Tatashinc9e97a12018-04-05 16:22:31 -07002053 pgdat_resize_unlock(pgdat, &flags);
2054
2055 return nr_pages > 0;
2056}
2057
2058/*
2059 * deferred_grow_zone() is __init, but it is called from
2060 * get_page_from_freelist() during early boot until deferred_pages permanently
2061 * disables this call. This is why we have refdata wrapper to avoid warning,
2062 * and to ensure that the function body gets unloaded.
2063 */
2064static bool __ref
2065_deferred_grow_zone(struct zone *zone, unsigned int order)
2066{
2067 return deferred_grow_zone(zone, order);
2068}
2069
Joonsoo Kim7cf91a92016-03-15 14:57:51 -07002070#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
Mel Gorman0e1cc952015-06-30 14:57:27 -07002071
2072void __init page_alloc_init_late(void)
2073{
Joonsoo Kim7cf91a92016-03-15 14:57:51 -07002074 struct zone *zone;
Dan Williamse900a912019-05-14 15:41:28 -07002075 int nid;
Joonsoo Kim7cf91a92016-03-15 14:57:51 -07002076
2077#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
Mel Gorman0e1cc952015-06-30 14:57:27 -07002078
Nicolai Stanged3cd1312015-08-06 15:46:16 -07002079 /* There will be num_node_state(N_MEMORY) threads */
2080 atomic_set(&pgdat_init_n_undone, num_node_state(N_MEMORY));
Mel Gorman0e1cc952015-06-30 14:57:27 -07002081 for_each_node_state(nid, N_MEMORY) {
Mel Gorman0e1cc952015-06-30 14:57:27 -07002082 kthread_run(deferred_init_memmap, NODE_DATA(nid), "pgdatinit%d", nid);
2083 }
2084
2085 /* Block until all are initialised */
Nicolai Stanged3cd1312015-08-06 15:46:16 -07002086 wait_for_completion(&pgdat_init_all_done_comp);
Mel Gorman4248b0d2015-08-06 15:46:20 -07002087
Pavel Tatashinc9e97a12018-04-05 16:22:31 -07002088 /*
Mel Gorman3e8fc002019-11-05 21:16:27 -08002089 * The number of managed pages has changed due to the initialisation
2090 * so the pcpu batch and high limits needs to be updated or the limits
2091 * will be artificially small.
2092 */
2093 for_each_populated_zone(zone)
2094 zone_pcp_update(zone);
2095
2096 /*
Pavel Tatashinc9e97a12018-04-05 16:22:31 -07002097 * We initialized the rest of the deferred pages. Permanently disable
2098 * on-demand struct page initialization.
2099 */
2100 static_branch_disable(&deferred_pages);
2101
Mel Gorman4248b0d2015-08-06 15:46:20 -07002102 /* Reinit limits that are based on free pages after the kernel is up */
2103 files_maxfiles_init();
Joonsoo Kim7cf91a92016-03-15 14:57:51 -07002104#endif
Mike Rapoport350e88b2019-05-13 17:22:59 -07002105
Pavel Tatashin3010f872017-08-18 15:16:05 -07002106 /* Discard memblock private memory */
2107 memblock_discard();
Joonsoo Kim7cf91a92016-03-15 14:57:51 -07002108
Dan Williamse900a912019-05-14 15:41:28 -07002109 for_each_node_state(nid, N_MEMORY)
2110 shuffle_free_memory(NODE_DATA(nid));
2111
Joonsoo Kim7cf91a92016-03-15 14:57:51 -07002112 for_each_populated_zone(zone)
2113 set_zone_contiguous(zone);
Mel Gorman7e18adb2015-06-30 14:57:05 -07002114}
Mel Gorman7e18adb2015-06-30 14:57:05 -07002115
Michal Nazarewicz47118af2011-12-29 13:09:50 +01002116#ifdef CONFIG_CMA
Li Zhong9cf510a2013-08-23 13:52:52 +08002117/* Free whole pageblock and set its migration type to MIGRATE_CMA. */
Michal Nazarewicz47118af2011-12-29 13:09:50 +01002118void __init init_cma_reserved_pageblock(struct page *page)
2119{
2120 unsigned i = pageblock_nr_pages;
2121 struct page *p = page;
2122
2123 do {
2124 __ClearPageReserved(p);
2125 set_page_count(p, 0);
Joonsoo Kimd883c6c2018-05-23 10:18:21 +09002126 } while (++p, --i);
Michal Nazarewicz47118af2011-12-29 13:09:50 +01002127
Michal Nazarewicz47118af2011-12-29 13:09:50 +01002128 set_pageblock_migratetype(page, MIGRATE_CMA);
Michal Nazarewiczdc783272014-07-02 15:22:35 -07002129
2130 if (pageblock_order >= MAX_ORDER) {
2131 i = pageblock_nr_pages;
2132 p = page;
2133 do {
2134 set_page_refcounted(p);
2135 __free_pages(p, MAX_ORDER - 1);
2136 p += MAX_ORDER_NR_PAGES;
2137 } while (i -= MAX_ORDER_NR_PAGES);
2138 } else {
2139 set_page_refcounted(page);
2140 __free_pages(page, pageblock_order);
2141 }
2142
Jiang Liu3dcc0572013-07-03 15:03:21 -07002143 adjust_managed_page_count(page, pageblock_nr_pages);
Michal Nazarewicz47118af2011-12-29 13:09:50 +01002144}
2145#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07002146
2147/*
2148 * The order of subdivision here is critical for the IO subsystem.
2149 * Please do not alter this order without good reasons and regression
2150 * testing. Specifically, as large blocks of memory are subdivided,
2151 * the order in which smaller blocks are delivered depends on the order
2152 * they're subdivided in this function. This is the primary factor
2153 * influencing the order in which pages are delivered to the IO
2154 * subsystem according to empirical testing, and this is also justified
2155 * by considering the behavior of a buddy system containing a single
2156 * large block of memory acted on by a series of small allocations.
2157 * This behavior is a critical factor in sglist merging's success.
2158 *
Nadia Yvette Chambers6d49e352012-12-06 10:39:54 +01002159 * -- nyc
Linus Torvalds1da177e2005-04-16 15:20:36 -07002160 */
Nick Piggin085cc7d2006-01-06 00:11:01 -08002161static inline void expand(struct zone *zone, struct page *page,
Alexander Duyck6ab01362020-04-06 20:04:49 -07002162 int low, int high, int migratetype)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002163{
2164 unsigned long size = 1 << high;
2165
2166 while (high > low) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002167 high--;
2168 size >>= 1;
Sasha Levin309381fea2014-01-23 15:52:54 -08002169 VM_BUG_ON_PAGE(bad_range(zone, &page[size]), &page[size]);
Stanislaw Gruszkac0a32fc2012-01-10 15:07:28 -08002170
Joonsoo Kimacbc15a2016-10-07 16:58:15 -07002171 /*
2172 * Mark as guard pages (or page), that will allow to
2173 * merge back to allocator when buddy will be freed.
2174 * Corresponding page table entries will not be touched,
2175 * pages will stay not present in virtual address space
2176 */
2177 if (set_page_guard(zone, &page[size], high, migratetype))
Stanislaw Gruszkac0a32fc2012-01-10 15:07:28 -08002178 continue;
Joonsoo Kimacbc15a2016-10-07 16:58:15 -07002179
Alexander Duyck6ab01362020-04-06 20:04:49 -07002180 add_to_free_list(&page[size], zone, high, migratetype);
Matthew Wilcox (Oracle)ab130f912020-10-15 20:10:15 -07002181 set_buddy_order(&page[size], high);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002182 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002183}
2184
Vlastimil Babka4e611802016-05-19 17:14:41 -07002185static void check_new_page_bad(struct page *page)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002186{
Naoya Horiguchif4c18e62015-08-06 15:47:08 -07002187 if (unlikely(page->flags & __PG_HWPOISON)) {
Naoya Horiguchie570f562016-05-20 16:58:50 -07002188 /* Don't complain about hwpoisoned pages */
2189 page_mapcount_reset(page); /* remove PageBuddy */
2190 return;
Naoya Horiguchif4c18e62015-08-06 15:47:08 -07002191 }
Wei Yang58b7f112020-06-03 15:58:39 -07002192
2193 bad_page(page,
2194 page_bad_reason(page, PAGE_FLAGS_CHECK_AT_PREP));
Vlastimil Babka4e611802016-05-19 17:14:41 -07002195}
2196
2197/*
2198 * This page is about to be returned from the page allocator
2199 */
2200static inline int check_new_page(struct page *page)
2201{
2202 if (likely(page_expected_state(page,
2203 PAGE_FLAGS_CHECK_AT_PREP|__PG_HWPOISON)))
2204 return 0;
2205
2206 check_new_page_bad(page);
2207 return 1;
Wu Fengguang2a7684a2009-09-16 11:50:12 +02002208}
2209
Vinayak Menonbd33ef32017-05-03 14:54:42 -07002210static inline bool free_pages_prezeroed(void)
Laura Abbott1414c7f2016-03-15 14:56:30 -07002211{
Alexander Potapenko64713842019-07-11 20:59:19 -07002212 return (IS_ENABLED(CONFIG_PAGE_POISONING_ZERO) &&
2213 page_poisoning_enabled()) || want_init_on_free();
Laura Abbott1414c7f2016-03-15 14:56:30 -07002214}
2215
Mel Gorman479f8542016-05-19 17:14:35 -07002216#ifdef CONFIG_DEBUG_VM
Vlastimil Babka4462b322019-07-11 20:55:09 -07002217/*
2218 * With DEBUG_VM enabled, order-0 pages are checked for expected state when
2219 * being allocated from pcp lists. With debug_pagealloc also enabled, they are
2220 * also checked when pcp lists are refilled from the free lists.
2221 */
2222static inline bool check_pcp_refill(struct page *page)
Mel Gorman479f8542016-05-19 17:14:35 -07002223{
Vlastimil Babka8e57f8a2020-01-13 16:29:20 -08002224 if (debug_pagealloc_enabled_static())
Vlastimil Babka4462b322019-07-11 20:55:09 -07002225 return check_new_page(page);
2226 else
2227 return false;
Mel Gorman479f8542016-05-19 17:14:35 -07002228}
2229
Vlastimil Babka4462b322019-07-11 20:55:09 -07002230static inline bool check_new_pcp(struct page *page)
Mel Gorman479f8542016-05-19 17:14:35 -07002231{
2232 return check_new_page(page);
2233}
2234#else
Vlastimil Babka4462b322019-07-11 20:55:09 -07002235/*
2236 * With DEBUG_VM disabled, free order-0 pages are checked for expected state
2237 * when pcp lists are being refilled from the free lists. With debug_pagealloc
2238 * enabled, they are also checked when being allocated from the pcp lists.
2239 */
2240static inline bool check_pcp_refill(struct page *page)
Mel Gorman479f8542016-05-19 17:14:35 -07002241{
2242 return check_new_page(page);
2243}
Vlastimil Babka4462b322019-07-11 20:55:09 -07002244static inline bool check_new_pcp(struct page *page)
Mel Gorman479f8542016-05-19 17:14:35 -07002245{
Vlastimil Babka8e57f8a2020-01-13 16:29:20 -08002246 if (debug_pagealloc_enabled_static())
Vlastimil Babka4462b322019-07-11 20:55:09 -07002247 return check_new_page(page);
2248 else
2249 return false;
Mel Gorman479f8542016-05-19 17:14:35 -07002250}
2251#endif /* CONFIG_DEBUG_VM */
2252
2253static bool check_new_pages(struct page *page, unsigned int order)
2254{
2255 int i;
2256 for (i = 0; i < (1 << order); i++) {
2257 struct page *p = page + i;
2258
2259 if (unlikely(check_new_page(p)))
2260 return true;
2261 }
2262
2263 return false;
2264}
2265
Joonsoo Kim46f24fd2016-07-26 15:23:58 -07002266inline void post_alloc_hook(struct page *page, unsigned int order,
2267 gfp_t gfp_flags)
2268{
2269 set_page_private(page, 0);
2270 set_page_refcounted(page);
2271
2272 arch_alloc_page(page, order);
Vlastimil Babka8e57f8a2020-01-13 16:29:20 -08002273 if (debug_pagealloc_enabled_static())
Rick Edgecombed6332692019-04-25 17:11:35 -07002274 kernel_map_pages(page, 1 << order, 1);
Joonsoo Kim46f24fd2016-07-26 15:23:58 -07002275 kasan_alloc_pages(page, order);
Qian Cai41179922019-03-05 15:41:24 -08002276 kernel_poison_pages(page, 1 << order, 1);
Joonsoo Kim46f24fd2016-07-26 15:23:58 -07002277 set_page_owner(page, order, gfp_flags);
2278}
2279
Mel Gorman479f8542016-05-19 17:14:35 -07002280static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags,
Mel Gormanc6038442016-05-19 17:13:38 -07002281 unsigned int alloc_flags)
Wu Fengguang2a7684a2009-09-16 11:50:12 +02002282{
Joonsoo Kim46f24fd2016-07-26 15:23:58 -07002283 post_alloc_hook(page, order, gfp_flags);
Nick Piggin17cf4402006-03-22 00:08:41 -08002284
Alexander Potapenko64713842019-07-11 20:59:19 -07002285 if (!free_pages_prezeroed() && want_init_on_alloc(gfp_flags))
2286 kernel_init_free_pages(page, 1 << order);
Nick Piggin17cf4402006-03-22 00:08:41 -08002287
2288 if (order && (gfp_flags & __GFP_COMP))
2289 prep_compound_page(page, order);
2290
Vlastimil Babka75379192015-02-11 15:25:38 -08002291 /*
Michal Hocko2f064f32015-08-21 14:11:51 -07002292 * page is set pfmemalloc when ALLOC_NO_WATERMARKS was necessary to
Vlastimil Babka75379192015-02-11 15:25:38 -08002293 * allocate the page. The expectation is that the caller is taking
2294 * steps that will free more memory. The caller should avoid the page
2295 * being used for !PFMEMALLOC purposes.
2296 */
Michal Hocko2f064f32015-08-21 14:11:51 -07002297 if (alloc_flags & ALLOC_NO_WATERMARKS)
2298 set_page_pfmemalloc(page);
2299 else
2300 clear_page_pfmemalloc(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002301}
2302
Mel Gorman56fd56b2007-10-16 01:25:58 -07002303/*
2304 * Go through the free lists for the given migratetype and remove
2305 * the smallest available page from the freelists
2306 */
Aaron Lu85ccc8f2017-11-15 17:36:53 -08002307static __always_inline
Mel Gorman728ec982009-06-16 15:32:04 -07002308struct page *__rmqueue_smallest(struct zone *zone, unsigned int order,
Mel Gorman56fd56b2007-10-16 01:25:58 -07002309 int migratetype)
2310{
2311 unsigned int current_order;
Pintu Kumarb8af2942013-09-11 14:20:34 -07002312 struct free_area *area;
Mel Gorman56fd56b2007-10-16 01:25:58 -07002313 struct page *page;
2314
2315 /* Find a page of the appropriate size in the preferred list */
2316 for (current_order = order; current_order < MAX_ORDER; ++current_order) {
2317 area = &(zone->free_area[current_order]);
Dan Williamsb03641a2019-05-14 15:41:32 -07002318 page = get_page_from_free_area(area, migratetype);
Geliang Tanga16601c2016-01-14 15:20:30 -08002319 if (!page)
2320 continue;
Alexander Duyck6ab01362020-04-06 20:04:49 -07002321 del_page_from_free_list(page, zone, current_order);
2322 expand(zone, page, order, current_order, migratetype);
Vlastimil Babkabb14c2c2015-09-08 15:01:25 -07002323 set_pcppage_migratetype(page, migratetype);
Mel Gorman56fd56b2007-10-16 01:25:58 -07002324 return page;
2325 }
2326
2327 return NULL;
2328}
2329
2330
Mel Gormanb2a0ac82007-10-16 01:25:48 -07002331/*
2332 * This array describes the order lists are fallen back to when
2333 * the free lists for the desirable migrate type are depleted
2334 */
Wei Yangda415662020-08-06 23:25:58 -07002335static int fallbacks[MIGRATE_TYPES][3] = {
Mel Gorman974a7862015-11-06 16:28:34 -08002336 [MIGRATE_UNMOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE, MIGRATE_TYPES },
Mel Gorman974a7862015-11-06 16:28:34 -08002337 [MIGRATE_MOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_TYPES },
Huang Shijie7ead3342018-12-28 00:34:46 -08002338 [MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE, MIGRATE_MOVABLE, MIGRATE_TYPES },
Joonsoo Kimdc676472015-04-14 15:45:15 -07002339#ifdef CONFIG_CMA
Mel Gorman974a7862015-11-06 16:28:34 -08002340 [MIGRATE_CMA] = { MIGRATE_TYPES }, /* Never used */
Michal Nazarewicz47118af2011-12-29 13:09:50 +01002341#endif
Minchan Kim194159f2013-02-22 16:33:58 -08002342#ifdef CONFIG_MEMORY_ISOLATION
Mel Gorman974a7862015-11-06 16:28:34 -08002343 [MIGRATE_ISOLATE] = { MIGRATE_TYPES }, /* Never used */
Minchan Kim194159f2013-02-22 16:33:58 -08002344#endif
Mel Gormanb2a0ac82007-10-16 01:25:48 -07002345};
2346
Joonsoo Kimdc676472015-04-14 15:45:15 -07002347#ifdef CONFIG_CMA
Aaron Lu85ccc8f2017-11-15 17:36:53 -08002348static __always_inline struct page *__rmqueue_cma_fallback(struct zone *zone,
Joonsoo Kimdc676472015-04-14 15:45:15 -07002349 unsigned int order)
2350{
2351 return __rmqueue_smallest(zone, order, MIGRATE_CMA);
2352}
2353#else
2354static inline struct page *__rmqueue_cma_fallback(struct zone *zone,
2355 unsigned int order) { return NULL; }
2356#endif
2357
Mel Gormanc361be52007-10-16 01:25:51 -07002358/*
David Hildenbrand293ffa52020-10-15 20:09:30 -07002359 * Move the free pages in a range to the freelist tail of the requested type.
Mel Gormand9c23402007-10-16 01:26:01 -07002360 * Note that start_page and end_pages are not aligned on a pageblock
Mel Gormanc361be52007-10-16 01:25:51 -07002361 * boundary. If alignment is required, use move_freepages_block()
2362 */
Vlastimil Babka02aa0cd2017-05-08 15:54:40 -07002363static int move_freepages(struct zone *zone,
Adrian Bunkb69a7282008-07-23 21:28:12 -07002364 struct page *start_page, struct page *end_page,
Vlastimil Babka02aa0cd2017-05-08 15:54:40 -07002365 int migratetype, int *num_movable)
Mel Gormanc361be52007-10-16 01:25:51 -07002366{
2367 struct page *page;
Kirill A. Shutemovd00181b2015-11-06 16:29:57 -08002368 unsigned int order;
Mel Gormand1003132007-10-16 01:26:00 -07002369 int pages_moved = 0;
Mel Gormanc361be52007-10-16 01:25:51 -07002370
Mel Gormanc361be52007-10-16 01:25:51 -07002371 for (page = start_page; page <= end_page;) {
2372 if (!pfn_valid_within(page_to_pfn(page))) {
2373 page++;
2374 continue;
2375 }
2376
2377 if (!PageBuddy(page)) {
Vlastimil Babka02aa0cd2017-05-08 15:54:40 -07002378 /*
2379 * We assume that pages that could be isolated for
2380 * migration are movable. But we don't actually try
2381 * isolating, as that would be expensive.
2382 */
2383 if (num_movable &&
2384 (PageLRU(page) || __PageMovable(page)))
2385 (*num_movable)++;
2386
Mel Gormanc361be52007-10-16 01:25:51 -07002387 page++;
2388 continue;
2389 }
2390
David Rientjescd961032019-08-24 17:54:40 -07002391 /* Make sure we are not inadvertently changing nodes */
2392 VM_BUG_ON_PAGE(page_to_nid(page) != zone_to_nid(zone), page);
2393 VM_BUG_ON_PAGE(page_zone(page) != zone, page);
2394
Matthew Wilcox (Oracle)ab130f912020-10-15 20:10:15 -07002395 order = buddy_order(page);
Alexander Duyck6ab01362020-04-06 20:04:49 -07002396 move_to_free_list(page, zone, order, migratetype);
Mel Gormanc361be52007-10-16 01:25:51 -07002397 page += 1 << order;
Mel Gormand1003132007-10-16 01:26:00 -07002398 pages_moved += 1 << order;
Mel Gormanc361be52007-10-16 01:25:51 -07002399 }
2400
Mel Gormand1003132007-10-16 01:26:00 -07002401 return pages_moved;
Mel Gormanc361be52007-10-16 01:25:51 -07002402}
2403
Minchan Kimee6f5092012-07-31 16:43:50 -07002404int move_freepages_block(struct zone *zone, struct page *page,
Vlastimil Babka02aa0cd2017-05-08 15:54:40 -07002405 int migratetype, int *num_movable)
Mel Gormanc361be52007-10-16 01:25:51 -07002406{
2407 unsigned long start_pfn, end_pfn;
2408 struct page *start_page, *end_page;
2409
David Rientjes4a222122018-10-26 15:09:24 -07002410 if (num_movable)
2411 *num_movable = 0;
2412
Mel Gormanc361be52007-10-16 01:25:51 -07002413 start_pfn = page_to_pfn(page);
Mel Gormand9c23402007-10-16 01:26:01 -07002414 start_pfn = start_pfn & ~(pageblock_nr_pages-1);
Mel Gormanc361be52007-10-16 01:25:51 -07002415 start_page = pfn_to_page(start_pfn);
Mel Gormand9c23402007-10-16 01:26:01 -07002416 end_page = start_page + pageblock_nr_pages - 1;
2417 end_pfn = start_pfn + pageblock_nr_pages - 1;
Mel Gormanc361be52007-10-16 01:25:51 -07002418
2419 /* Do not cross zone boundaries */
Cody P Schafer108bcc92013-02-22 16:35:23 -08002420 if (!zone_spans_pfn(zone, start_pfn))
Mel Gormanc361be52007-10-16 01:25:51 -07002421 start_page = page;
Cody P Schafer108bcc92013-02-22 16:35:23 -08002422 if (!zone_spans_pfn(zone, end_pfn))
Mel Gormanc361be52007-10-16 01:25:51 -07002423 return 0;
2424
Vlastimil Babka02aa0cd2017-05-08 15:54:40 -07002425 return move_freepages(zone, start_page, end_page, migratetype,
2426 num_movable);
Mel Gormanc361be52007-10-16 01:25:51 -07002427}
2428
Mel Gorman2f66a682009-09-21 17:02:31 -07002429static void change_pageblock_range(struct page *pageblock_page,
2430 int start_order, int migratetype)
2431{
2432 int nr_pageblocks = 1 << (start_order - pageblock_order);
2433
2434 while (nr_pageblocks--) {
2435 set_pageblock_migratetype(pageblock_page, migratetype);
2436 pageblock_page += pageblock_nr_pages;
2437 }
2438}
2439
Srivatsa S. Bhatfef903e2013-09-11 14:20:35 -07002440/*
Vlastimil Babka9c0415e2015-02-11 15:28:21 -08002441 * When we are falling back to another migratetype during allocation, try to
2442 * steal extra free pages from the same pageblocks to satisfy further
2443 * allocations, instead of polluting multiple pageblocks.
2444 *
2445 * If we are stealing a relatively large buddy page, it is likely there will
2446 * be more free pages in the pageblock, so try to steal them all. For
2447 * reclaimable and unmovable allocations, we steal regardless of page size,
2448 * as fragmentation caused by those allocations polluting movable pageblocks
2449 * is worse than movable allocations stealing from unmovable and reclaimable
2450 * pageblocks.
Srivatsa S. Bhatfef903e2013-09-11 14:20:35 -07002451 */
Joonsoo Kim4eb7dce2015-04-14 15:45:18 -07002452static bool can_steal_fallback(unsigned int order, int start_mt)
2453{
2454 /*
2455 * Leaving this order check is intended, although there is
2456 * relaxed order check in next check. The reason is that
2457 * we can actually steal whole pageblock if this condition met,
2458 * but, below check doesn't guarantee it and that is just heuristic
2459 * so could be changed anytime.
2460 */
2461 if (order >= pageblock_order)
2462 return true;
2463
2464 if (order >= pageblock_order / 2 ||
2465 start_mt == MIGRATE_RECLAIMABLE ||
2466 start_mt == MIGRATE_UNMOVABLE ||
2467 page_group_by_mobility_disabled)
2468 return true;
2469
2470 return false;
2471}
2472
Johannes Weinerbd3f4b62020-12-14 19:12:15 -08002473static inline bool boost_watermark(struct zone *zone)
Mel Gorman1c308442018-12-28 00:35:52 -08002474{
2475 unsigned long max_boost;
2476
2477 if (!watermark_boost_factor)
Johannes Weinerbd3f4b62020-12-14 19:12:15 -08002478 return false;
Henry Willard14f69142020-05-07 18:36:27 -07002479 /*
2480 * Don't bother in zones that are unlikely to produce results.
2481 * On small machines, including kdump capture kernels running
2482 * in a small area, boosting the watermark can cause an out of
2483 * memory situation immediately.
2484 */
2485 if ((pageblock_nr_pages * 4) > zone_managed_pages(zone))
Johannes Weinerbd3f4b62020-12-14 19:12:15 -08002486 return false;
Mel Gorman1c308442018-12-28 00:35:52 -08002487
2488 max_boost = mult_frac(zone->_watermark[WMARK_HIGH],
2489 watermark_boost_factor, 10000);
Mel Gorman94b33342019-02-20 22:19:49 -08002490
2491 /*
2492 * high watermark may be uninitialised if fragmentation occurs
2493 * very early in boot so do not boost. We do not fall
2494 * through and boost by pageblock_nr_pages as failing
2495 * allocations that early means that reclaim is not going
2496 * to help and it may even be impossible to reclaim the
2497 * boosted watermark resulting in a hang.
2498 */
2499 if (!max_boost)
Johannes Weinerbd3f4b62020-12-14 19:12:15 -08002500 return false;
Mel Gorman94b33342019-02-20 22:19:49 -08002501
Mel Gorman1c308442018-12-28 00:35:52 -08002502 max_boost = max(pageblock_nr_pages, max_boost);
2503
2504 zone->watermark_boost = min(zone->watermark_boost + pageblock_nr_pages,
2505 max_boost);
Johannes Weinerbd3f4b62020-12-14 19:12:15 -08002506
2507 return true;
Mel Gorman1c308442018-12-28 00:35:52 -08002508}
2509
Joonsoo Kim4eb7dce2015-04-14 15:45:18 -07002510/*
2511 * This function implements actual steal behaviour. If order is large enough,
2512 * we can steal whole pageblock. If not, we first move freepages in this
Vlastimil Babka02aa0cd2017-05-08 15:54:40 -07002513 * pageblock to our migratetype and determine how many already-allocated pages
2514 * are there in the pageblock with a compatible migratetype. If at least half
2515 * of pages are free or compatible, we can change migratetype of the pageblock
2516 * itself, so pages freed in the future will be put on the correct free list.
Joonsoo Kim4eb7dce2015-04-14 15:45:18 -07002517 */
2518static void steal_suitable_fallback(struct zone *zone, struct page *page,
Mel Gorman1c308442018-12-28 00:35:52 -08002519 unsigned int alloc_flags, int start_type, bool whole_block)
Srivatsa S. Bhatfef903e2013-09-11 14:20:35 -07002520{
Matthew Wilcox (Oracle)ab130f912020-10-15 20:10:15 -07002521 unsigned int current_order = buddy_order(page);
Vlastimil Babka02aa0cd2017-05-08 15:54:40 -07002522 int free_pages, movable_pages, alike_pages;
2523 int old_block_type;
2524
2525 old_block_type = get_pageblock_migratetype(page);
Srivatsa S. Bhatfef903e2013-09-11 14:20:35 -07002526
Vlastimil Babka3bc48f92017-05-08 15:54:37 -07002527 /*
2528 * This can happen due to races and we want to prevent broken
2529 * highatomic accounting.
2530 */
Vlastimil Babka02aa0cd2017-05-08 15:54:40 -07002531 if (is_migrate_highatomic(old_block_type))
Vlastimil Babka3bc48f92017-05-08 15:54:37 -07002532 goto single_page;
2533
Srivatsa S. Bhatfef903e2013-09-11 14:20:35 -07002534 /* Take ownership for orders >= pageblock_order */
2535 if (current_order >= pageblock_order) {
2536 change_pageblock_range(page, current_order, start_type);
Vlastimil Babka3bc48f92017-05-08 15:54:37 -07002537 goto single_page;
Srivatsa S. Bhatfef903e2013-09-11 14:20:35 -07002538 }
2539
Mel Gorman1c308442018-12-28 00:35:52 -08002540 /*
2541 * Boost watermarks to increase reclaim pressure to reduce the
2542 * likelihood of future fallbacks. Wake kswapd now as the node
2543 * may be balanced overall and kswapd will not wake naturally.
2544 */
Johannes Weinerbd3f4b62020-12-14 19:12:15 -08002545 if (boost_watermark(zone) && (alloc_flags & ALLOC_KSWAPD))
Mel Gorman73444bc2019-01-08 15:23:39 -08002546 set_bit(ZONE_BOOSTED_WATERMARK, &zone->flags);
Mel Gorman1c308442018-12-28 00:35:52 -08002547
Vlastimil Babka3bc48f92017-05-08 15:54:37 -07002548 /* We are not allowed to try stealing from the whole block */
2549 if (!whole_block)
2550 goto single_page;
2551
Vlastimil Babka02aa0cd2017-05-08 15:54:40 -07002552 free_pages = move_freepages_block(zone, page, start_type,
2553 &movable_pages);
2554 /*
2555 * Determine how many pages are compatible with our allocation.
2556 * For movable allocation, it's the number of movable pages which
2557 * we just obtained. For other types it's a bit more tricky.
2558 */
2559 if (start_type == MIGRATE_MOVABLE) {
2560 alike_pages = movable_pages;
2561 } else {
2562 /*
2563 * If we are falling back a RECLAIMABLE or UNMOVABLE allocation
2564 * to MOVABLE pageblock, consider all non-movable pages as
2565 * compatible. If it's UNMOVABLE falling back to RECLAIMABLE or
2566 * vice versa, be conservative since we can't distinguish the
2567 * exact migratetype of non-movable pages.
2568 */
2569 if (old_block_type == MIGRATE_MOVABLE)
2570 alike_pages = pageblock_nr_pages
2571 - (free_pages + movable_pages);
2572 else
2573 alike_pages = 0;
2574 }
2575
Vlastimil Babka3bc48f92017-05-08 15:54:37 -07002576 /* moving whole block can fail due to zone boundary conditions */
Vlastimil Babka02aa0cd2017-05-08 15:54:40 -07002577 if (!free_pages)
Vlastimil Babka3bc48f92017-05-08 15:54:37 -07002578 goto single_page;
Srivatsa S. Bhatfef903e2013-09-11 14:20:35 -07002579
Vlastimil Babka02aa0cd2017-05-08 15:54:40 -07002580 /*
2581 * If a sufficient number of pages in the block are either free or of
2582 * comparable migratability as our allocation, claim the whole block.
2583 */
2584 if (free_pages + alike_pages >= (1 << (pageblock_order-1)) ||
Joonsoo Kim4eb7dce2015-04-14 15:45:18 -07002585 page_group_by_mobility_disabled)
2586 set_pageblock_migratetype(page, start_type);
Vlastimil Babka3bc48f92017-05-08 15:54:37 -07002587
2588 return;
2589
2590single_page:
Alexander Duyck6ab01362020-04-06 20:04:49 -07002591 move_to_free_list(page, zone, current_order, start_type);
Joonsoo Kim4eb7dce2015-04-14 15:45:18 -07002592}
Srivatsa S. Bhatfef903e2013-09-11 14:20:35 -07002593
Joonsoo Kim2149cda2015-04-14 15:45:21 -07002594/*
2595 * Check whether there is a suitable fallback freepage with requested order.
2596 * If only_stealable is true, this function returns fallback_mt only if
2597 * we can steal other freepages all together. This would help to reduce
2598 * fragmentation due to mixed migratetype pages in one pageblock.
2599 */
2600int find_suitable_fallback(struct free_area *area, unsigned int order,
2601 int migratetype, bool only_stealable, bool *can_steal)
Joonsoo Kim4eb7dce2015-04-14 15:45:18 -07002602{
2603 int i;
2604 int fallback_mt;
2605
2606 if (area->nr_free == 0)
2607 return -1;
2608
2609 *can_steal = false;
2610 for (i = 0;; i++) {
2611 fallback_mt = fallbacks[migratetype][i];
Mel Gorman974a7862015-11-06 16:28:34 -08002612 if (fallback_mt == MIGRATE_TYPES)
Joonsoo Kim4eb7dce2015-04-14 15:45:18 -07002613 break;
2614
Dan Williamsb03641a2019-05-14 15:41:32 -07002615 if (free_area_empty(area, fallback_mt))
Joonsoo Kim4eb7dce2015-04-14 15:45:18 -07002616 continue;
2617
2618 if (can_steal_fallback(order, migratetype))
2619 *can_steal = true;
2620
Joonsoo Kim2149cda2015-04-14 15:45:21 -07002621 if (!only_stealable)
2622 return fallback_mt;
2623
2624 if (*can_steal)
2625 return fallback_mt;
Srivatsa S. Bhatfef903e2013-09-11 14:20:35 -07002626 }
Joonsoo Kim4eb7dce2015-04-14 15:45:18 -07002627
2628 return -1;
Srivatsa S. Bhatfef903e2013-09-11 14:20:35 -07002629}
2630
Mel Gorman0aaa29a2015-11-06 16:28:37 -08002631/*
2632 * Reserve a pageblock for exclusive use of high-order atomic allocations if
2633 * there are no empty page blocks that contain a page with a suitable order
2634 */
2635static void reserve_highatomic_pageblock(struct page *page, struct zone *zone,
2636 unsigned int alloc_order)
2637{
2638 int mt;
2639 unsigned long max_managed, flags;
2640
2641 /*
2642 * Limit the number reserved to 1 pageblock or roughly 1% of a zone.
2643 * Check is race-prone but harmless.
2644 */
Arun KS9705bea2018-12-28 00:34:24 -08002645 max_managed = (zone_managed_pages(zone) / 100) + pageblock_nr_pages;
Mel Gorman0aaa29a2015-11-06 16:28:37 -08002646 if (zone->nr_reserved_highatomic >= max_managed)
2647 return;
2648
2649 spin_lock_irqsave(&zone->lock, flags);
2650
2651 /* Recheck the nr_reserved_highatomic limit under the lock */
2652 if (zone->nr_reserved_highatomic >= max_managed)
2653 goto out_unlock;
2654
2655 /* Yoink! */
2656 mt = get_pageblock_migratetype(page);
Xishi Qiua6ffdc02017-05-03 14:52:52 -07002657 if (!is_migrate_highatomic(mt) && !is_migrate_isolate(mt)
2658 && !is_migrate_cma(mt)) {
Mel Gorman0aaa29a2015-11-06 16:28:37 -08002659 zone->nr_reserved_highatomic += pageblock_nr_pages;
2660 set_pageblock_migratetype(page, MIGRATE_HIGHATOMIC);
Vlastimil Babka02aa0cd2017-05-08 15:54:40 -07002661 move_freepages_block(zone, page, MIGRATE_HIGHATOMIC, NULL);
Mel Gorman0aaa29a2015-11-06 16:28:37 -08002662 }
2663
2664out_unlock:
2665 spin_unlock_irqrestore(&zone->lock, flags);
2666}
2667
2668/*
2669 * Used when an allocation is about to fail under memory pressure. This
2670 * potentially hurts the reliability of high-order allocations when under
2671 * intense memory pressure but failed atomic allocations should be easier
2672 * to recover from than an OOM.
Minchan Kim29fac032016-12-12 16:42:14 -08002673 *
2674 * If @force is true, try to unreserve a pageblock even though highatomic
2675 * pageblock is exhausted.
Mel Gorman0aaa29a2015-11-06 16:28:37 -08002676 */
Minchan Kim29fac032016-12-12 16:42:14 -08002677static bool unreserve_highatomic_pageblock(const struct alloc_context *ac,
2678 bool force)
Mel Gorman0aaa29a2015-11-06 16:28:37 -08002679{
2680 struct zonelist *zonelist = ac->zonelist;
2681 unsigned long flags;
2682 struct zoneref *z;
2683 struct zone *zone;
2684 struct page *page;
2685 int order;
Minchan Kim04c87162016-12-12 16:42:11 -08002686 bool ret;
Mel Gorman0aaa29a2015-11-06 16:28:37 -08002687
Joonsoo Kim97a225e2020-06-03 15:59:01 -07002688 for_each_zone_zonelist_nodemask(zone, z, zonelist, ac->highest_zoneidx,
Mel Gorman0aaa29a2015-11-06 16:28:37 -08002689 ac->nodemask) {
Minchan Kim29fac032016-12-12 16:42:14 -08002690 /*
2691 * Preserve at least one pageblock unless memory pressure
2692 * is really high.
2693 */
2694 if (!force && zone->nr_reserved_highatomic <=
2695 pageblock_nr_pages)
Mel Gorman0aaa29a2015-11-06 16:28:37 -08002696 continue;
2697
2698 spin_lock_irqsave(&zone->lock, flags);
2699 for (order = 0; order < MAX_ORDER; order++) {
2700 struct free_area *area = &(zone->free_area[order]);
2701
Dan Williamsb03641a2019-05-14 15:41:32 -07002702 page = get_page_from_free_area(area, MIGRATE_HIGHATOMIC);
Geliang Tanga16601c2016-01-14 15:20:30 -08002703 if (!page)
Mel Gorman0aaa29a2015-11-06 16:28:37 -08002704 continue;
2705
Mel Gorman0aaa29a2015-11-06 16:28:37 -08002706 /*
Minchan Kim4855e4a2016-12-12 16:42:08 -08002707 * In page freeing path, migratetype change is racy so
2708 * we can counter several free pages in a pageblock
2709 * in this loop althoug we changed the pageblock type
2710 * from highatomic to ac->migratetype. So we should
2711 * adjust the count once.
Mel Gorman0aaa29a2015-11-06 16:28:37 -08002712 */
Xishi Qiua6ffdc02017-05-03 14:52:52 -07002713 if (is_migrate_highatomic_page(page)) {
Minchan Kim4855e4a2016-12-12 16:42:08 -08002714 /*
2715 * It should never happen but changes to
2716 * locking could inadvertently allow a per-cpu
2717 * drain to add pages to MIGRATE_HIGHATOMIC
2718 * while unreserving so be safe and watch for
2719 * underflows.
2720 */
2721 zone->nr_reserved_highatomic -= min(
2722 pageblock_nr_pages,
2723 zone->nr_reserved_highatomic);
2724 }
Mel Gorman0aaa29a2015-11-06 16:28:37 -08002725
2726 /*
2727 * Convert to ac->migratetype and avoid the normal
2728 * pageblock stealing heuristics. Minimally, the caller
2729 * is doing the work and needs the pages. More
2730 * importantly, if the block was always converted to
2731 * MIGRATE_UNMOVABLE or another type then the number
2732 * of pageblocks that cannot be completely freed
2733 * may increase.
2734 */
2735 set_pageblock_migratetype(page, ac->migratetype);
Vlastimil Babka02aa0cd2017-05-08 15:54:40 -07002736 ret = move_freepages_block(zone, page, ac->migratetype,
2737 NULL);
Minchan Kim29fac032016-12-12 16:42:14 -08002738 if (ret) {
2739 spin_unlock_irqrestore(&zone->lock, flags);
2740 return ret;
2741 }
Mel Gorman0aaa29a2015-11-06 16:28:37 -08002742 }
2743 spin_unlock_irqrestore(&zone->lock, flags);
2744 }
Minchan Kim04c87162016-12-12 16:42:11 -08002745
2746 return false;
Mel Gorman0aaa29a2015-11-06 16:28:37 -08002747}
2748
Vlastimil Babka3bc48f92017-05-08 15:54:37 -07002749/*
2750 * Try finding a free buddy page on the fallback list and put it on the free
2751 * list of requested migratetype, possibly along with other pages from the same
2752 * block, depending on fragmentation avoidance heuristics. Returns true if
2753 * fallback was found so that __rmqueue_smallest() can grab it.
Rasmus Villemoesb0025292017-07-10 15:49:26 -07002754 *
2755 * The use of signed ints for order and current_order is a deliberate
2756 * deviation from the rest of this file, to make the for loop
2757 * condition simpler.
Vlastimil Babka3bc48f92017-05-08 15:54:37 -07002758 */
Aaron Lu85ccc8f2017-11-15 17:36:53 -08002759static __always_inline bool
Mel Gorman6bb15452018-12-28 00:35:41 -08002760__rmqueue_fallback(struct zone *zone, int order, int start_migratetype,
2761 unsigned int alloc_flags)
Mel Gormanb2a0ac82007-10-16 01:25:48 -07002762{
Pintu Kumarb8af2942013-09-11 14:20:34 -07002763 struct free_area *area;
Rasmus Villemoesb0025292017-07-10 15:49:26 -07002764 int current_order;
Mel Gorman6bb15452018-12-28 00:35:41 -08002765 int min_order = order;
Mel Gormanb2a0ac82007-10-16 01:25:48 -07002766 struct page *page;
Joonsoo Kim4eb7dce2015-04-14 15:45:18 -07002767 int fallback_mt;
2768 bool can_steal;
Mel Gormanb2a0ac82007-10-16 01:25:48 -07002769
Vlastimil Babka7a8f58f2017-07-10 15:47:14 -07002770 /*
Mel Gorman6bb15452018-12-28 00:35:41 -08002771 * Do not steal pages from freelists belonging to other pageblocks
2772 * i.e. orders < pageblock_order. If there are no local zones free,
2773 * the zonelists will be reiterated without ALLOC_NOFRAGMENT.
2774 */
2775 if (alloc_flags & ALLOC_NOFRAGMENT)
2776 min_order = pageblock_order;
2777
2778 /*
Vlastimil Babka7a8f58f2017-07-10 15:47:14 -07002779 * Find the largest available free page in the other list. This roughly
2780 * approximates finding the pageblock with the most free pages, which
2781 * would be too costly to do exactly.
2782 */
Mel Gorman6bb15452018-12-28 00:35:41 -08002783 for (current_order = MAX_ORDER - 1; current_order >= min_order;
Mel Gorman7aeb09f2014-06-04 16:10:21 -07002784 --current_order) {
Joonsoo Kim4eb7dce2015-04-14 15:45:18 -07002785 area = &(zone->free_area[current_order]);
2786 fallback_mt = find_suitable_fallback(area, current_order,
Joonsoo Kim2149cda2015-04-14 15:45:21 -07002787 start_migratetype, false, &can_steal);
Joonsoo Kim4eb7dce2015-04-14 15:45:18 -07002788 if (fallback_mt == -1)
2789 continue;
Mel Gormanb2a0ac82007-10-16 01:25:48 -07002790
Vlastimil Babka7a8f58f2017-07-10 15:47:14 -07002791 /*
2792 * We cannot steal all free pages from the pageblock and the
2793 * requested migratetype is movable. In that case it's better to
2794 * steal and split the smallest available page instead of the
2795 * largest available page, because even if the next movable
2796 * allocation falls back into a different pageblock than this
2797 * one, it won't cause permanent fragmentation.
2798 */
2799 if (!can_steal && start_migratetype == MIGRATE_MOVABLE
2800 && current_order > order)
2801 goto find_smallest;
Mel Gormane0104872007-10-16 01:25:53 -07002802
Vlastimil Babka7a8f58f2017-07-10 15:47:14 -07002803 goto do_steal;
Mel Gormanb2a0ac82007-10-16 01:25:48 -07002804 }
2805
Vlastimil Babka3bc48f92017-05-08 15:54:37 -07002806 return false;
Vlastimil Babka7a8f58f2017-07-10 15:47:14 -07002807
2808find_smallest:
2809 for (current_order = order; current_order < MAX_ORDER;
2810 current_order++) {
2811 area = &(zone->free_area[current_order]);
2812 fallback_mt = find_suitable_fallback(area, current_order,
2813 start_migratetype, false, &can_steal);
2814 if (fallback_mt != -1)
2815 break;
2816 }
2817
2818 /*
2819 * This should not happen - we already found a suitable fallback
2820 * when looking for the largest page.
2821 */
2822 VM_BUG_ON(current_order == MAX_ORDER);
2823
2824do_steal:
Dan Williamsb03641a2019-05-14 15:41:32 -07002825 page = get_page_from_free_area(area, fallback_mt);
Vlastimil Babka7a8f58f2017-07-10 15:47:14 -07002826
Mel Gorman1c308442018-12-28 00:35:52 -08002827 steal_suitable_fallback(zone, page, alloc_flags, start_migratetype,
2828 can_steal);
Vlastimil Babka7a8f58f2017-07-10 15:47:14 -07002829
2830 trace_mm_page_alloc_extfrag(page, order, current_order,
2831 start_migratetype, fallback_mt);
2832
2833 return true;
2834
Mel Gormanb2a0ac82007-10-16 01:25:48 -07002835}
2836
Mel Gorman56fd56b2007-10-16 01:25:58 -07002837/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07002838 * Do the hard work of removing an element from the buddy allocator.
2839 * Call me with the zone->lock already held.
2840 */
Aaron Lu85ccc8f2017-11-15 17:36:53 -08002841static __always_inline struct page *
Mel Gorman6bb15452018-12-28 00:35:41 -08002842__rmqueue(struct zone *zone, unsigned int order, int migratetype,
2843 unsigned int alloc_flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002844{
Linus Torvalds1da177e2005-04-16 15:20:36 -07002845 struct page *page;
2846
Roman Gushchin16867662020-06-03 15:58:42 -07002847#ifdef CONFIG_CMA
2848 /*
2849 * Balance movable allocations between regular and CMA areas by
2850 * allocating from CMA when over half of the zone's free memory
2851 * is in the CMA area.
2852 */
Joonsoo Kim8510e692020-08-06 23:26:04 -07002853 if (alloc_flags & ALLOC_CMA &&
Roman Gushchin16867662020-06-03 15:58:42 -07002854 zone_page_state(zone, NR_FREE_CMA_PAGES) >
2855 zone_page_state(zone, NR_FREE_PAGES) / 2) {
2856 page = __rmqueue_cma_fallback(zone, order);
2857 if (page)
2858 return page;
2859 }
2860#endif
Vlastimil Babka3bc48f92017-05-08 15:54:37 -07002861retry:
Mel Gorman56fd56b2007-10-16 01:25:58 -07002862 page = __rmqueue_smallest(zone, order, migratetype);
Mel Gorman974a7862015-11-06 16:28:34 -08002863 if (unlikely(!page)) {
Joonsoo Kim8510e692020-08-06 23:26:04 -07002864 if (alloc_flags & ALLOC_CMA)
Joonsoo Kimdc676472015-04-14 15:45:15 -07002865 page = __rmqueue_cma_fallback(zone, order);
2866
Mel Gorman6bb15452018-12-28 00:35:41 -08002867 if (!page && __rmqueue_fallback(zone, order, migratetype,
2868 alloc_flags))
Vlastimil Babka3bc48f92017-05-08 15:54:37 -07002869 goto retry;
Mel Gorman728ec982009-06-16 15:32:04 -07002870 }
2871
Mel Gorman0d3d0622009-09-21 17:02:44 -07002872 trace_mm_page_alloc_zone_locked(page, order, migratetype);
Mel Gormanb2a0ac82007-10-16 01:25:48 -07002873 return page;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002874}
2875
Michal Nazarewicz5f63b722012-01-11 15:16:11 +01002876/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07002877 * Obtain a specified number of elements from the buddy allocator, all under
2878 * a single hold of the lock, for efficiency. Add them to the supplied list.
2879 * Returns the number of new pages which were placed at *list.
2880 */
Michal Nazarewicz5f63b722012-01-11 15:16:11 +01002881static int rmqueue_bulk(struct zone *zone, unsigned int order,
Mel Gormanb2a0ac82007-10-16 01:25:48 -07002882 unsigned long count, struct list_head *list,
Mel Gorman6bb15452018-12-28 00:35:41 -08002883 int migratetype, unsigned int alloc_flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002884{
Mel Gormana6de7342016-12-12 16:44:41 -08002885 int i, alloced = 0;
Michal Nazarewicz5f63b722012-01-11 15:16:11 +01002886
Mel Gormand34b0732017-04-20 14:37:43 -07002887 spin_lock(&zone->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002888 for (i = 0; i < count; ++i) {
Mel Gorman6bb15452018-12-28 00:35:41 -08002889 struct page *page = __rmqueue(zone, order, migratetype,
2890 alloc_flags);
Nick Piggin085cc7d2006-01-06 00:11:01 -08002891 if (unlikely(page == NULL))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002892 break;
Mel Gorman81eabcb2007-12-17 16:20:05 -08002893
Mel Gorman479f8542016-05-19 17:14:35 -07002894 if (unlikely(check_pcp_refill(page)))
2895 continue;
2896
Mel Gorman81eabcb2007-12-17 16:20:05 -08002897 /*
Vlastimil Babka0fac3ba2017-11-15 17:38:07 -08002898 * Split buddy pages returned by expand() are received here in
2899 * physical page order. The page is added to the tail of
2900 * caller's list. From the callers perspective, the linked list
2901 * is ordered by page number under some conditions. This is
2902 * useful for IO devices that can forward direction from the
2903 * head, thus also in the physical page order. This is useful
2904 * for IO devices that can merge IO requests if the physical
2905 * pages are ordered properly.
Mel Gorman81eabcb2007-12-17 16:20:05 -08002906 */
Vlastimil Babka0fac3ba2017-11-15 17:38:07 -08002907 list_add_tail(&page->lru, list);
Mel Gormana6de7342016-12-12 16:44:41 -08002908 alloced++;
Vlastimil Babkabb14c2c2015-09-08 15:01:25 -07002909 if (is_migrate_cma(get_pcppage_migratetype(page)))
Bartlomiej Zolnierkiewiczd1ce7492012-10-08 16:32:02 -07002910 __mod_zone_page_state(zone, NR_FREE_CMA_PAGES,
2911 -(1 << order));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002912 }
Mel Gormana6de7342016-12-12 16:44:41 -08002913
2914 /*
2915 * i pages were removed from the buddy list even if some leak due
2916 * to check_pcp_refill failing so adjust NR_FREE_PAGES based
2917 * on i. Do not confuse with 'alloced' which is the number of
2918 * pages added to the pcp list.
2919 */
Mel Gormanf2260e62009-06-16 15:32:13 -07002920 __mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order));
Mel Gormand34b0732017-04-20 14:37:43 -07002921 spin_unlock(&zone->lock);
Mel Gormana6de7342016-12-12 16:44:41 -08002922 return alloced;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002923}
2924
Christoph Lameter4ae7c032005-06-21 17:14:57 -07002925#ifdef CONFIG_NUMA
Christoph Lameter8fce4d82006-03-09 17:33:54 -08002926/*
Christoph Lameter4037d452007-05-09 02:35:14 -07002927 * Called from the vmstat counter updater to drain pagesets of this
2928 * currently executing processor on remote nodes after they have
2929 * expired.
2930 *
Christoph Lameter879336c2006-03-22 00:09:08 -08002931 * Note that this function must be called with the thread pinned to
2932 * a single processor.
Christoph Lameter8fce4d82006-03-09 17:33:54 -08002933 */
Christoph Lameter4037d452007-05-09 02:35:14 -07002934void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp)
Christoph Lameter4ae7c032005-06-21 17:14:57 -07002935{
Christoph Lameter4ae7c032005-06-21 17:14:57 -07002936 unsigned long flags;
Michal Nazarewicz7be12fc2014-08-06 16:05:15 -07002937 int to_drain, batch;
Christoph Lameter4ae7c032005-06-21 17:14:57 -07002938
Christoph Lameter4037d452007-05-09 02:35:14 -07002939 local_irq_save(flags);
Jason Low4db0c3c2015-04-15 16:14:08 -07002940 batch = READ_ONCE(pcp->batch);
Michal Nazarewicz7be12fc2014-08-06 16:05:15 -07002941 to_drain = min(pcp->count, batch);
Aaron Lu77ba9062018-04-05 16:24:06 -07002942 if (to_drain > 0)
KOSAKI Motohiro2a135152012-07-31 16:42:53 -07002943 free_pcppages_bulk(zone, to_drain, pcp);
Christoph Lameter4037d452007-05-09 02:35:14 -07002944 local_irq_restore(flags);
Christoph Lameter4ae7c032005-06-21 17:14:57 -07002945}
2946#endif
2947
Christoph Lameter9f8f2172008-02-04 22:29:11 -08002948/*
Vlastimil Babka93481ff2014-12-10 15:43:01 -08002949 * Drain pcplists of the indicated processor and zone.
2950 *
2951 * The processor must either be the current processor and the
2952 * thread pinned to the current processor or a processor that
2953 * is not online.
2954 */
2955static void drain_pages_zone(unsigned int cpu, struct zone *zone)
2956{
2957 unsigned long flags;
2958 struct per_cpu_pageset *pset;
2959 struct per_cpu_pages *pcp;
2960
2961 local_irq_save(flags);
2962 pset = per_cpu_ptr(zone->pageset, cpu);
2963
2964 pcp = &pset->pcp;
Aaron Lu77ba9062018-04-05 16:24:06 -07002965 if (pcp->count)
Vlastimil Babka93481ff2014-12-10 15:43:01 -08002966 free_pcppages_bulk(zone, pcp->count, pcp);
Vlastimil Babka93481ff2014-12-10 15:43:01 -08002967 local_irq_restore(flags);
2968}
2969
2970/*
2971 * Drain pcplists of all zones on the indicated processor.
Christoph Lameter9f8f2172008-02-04 22:29:11 -08002972 *
2973 * The processor must either be the current processor and the
2974 * thread pinned to the current processor or a processor that
2975 * is not online.
2976 */
2977static void drain_pages(unsigned int cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002978{
2979 struct zone *zone;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002980
KOSAKI Motohiroee99c712009-03-31 15:19:31 -07002981 for_each_populated_zone(zone) {
Vlastimil Babka93481ff2014-12-10 15:43:01 -08002982 drain_pages_zone(cpu, zone);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002983 }
2984}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002985
Christoph Lameter9f8f2172008-02-04 22:29:11 -08002986/*
2987 * Spill all of this CPU's per-cpu pages back into the buddy allocator.
Vlastimil Babka93481ff2014-12-10 15:43:01 -08002988 *
2989 * The CPU has to be pinned. When zone parameter is non-NULL, spill just
2990 * the single zone's pages.
Christoph Lameter9f8f2172008-02-04 22:29:11 -08002991 */
Vlastimil Babka93481ff2014-12-10 15:43:01 -08002992void drain_local_pages(struct zone *zone)
Christoph Lameter9f8f2172008-02-04 22:29:11 -08002993{
Vlastimil Babka93481ff2014-12-10 15:43:01 -08002994 int cpu = smp_processor_id();
2995
2996 if (zone)
2997 drain_pages_zone(cpu, zone);
2998 else
2999 drain_pages(cpu);
Christoph Lameter9f8f2172008-02-04 22:29:11 -08003000}
3001
Mel Gorman0ccce3b2017-02-24 14:56:32 -08003002static void drain_local_pages_wq(struct work_struct *work)
3003{
Wei Yangd9367bd2018-12-28 00:38:58 -08003004 struct pcpu_drain *drain;
3005
3006 drain = container_of(work, struct pcpu_drain, work);
3007
Michal Hockoa459eeb2017-02-24 14:56:35 -08003008 /*
3009 * drain_all_pages doesn't use proper cpu hotplug protection so
3010 * we can race with cpu offline when the WQ can move this from
3011 * a cpu pinned worker to an unbound one. We can operate on a different
3012 * cpu which is allright but we also have to make sure to not move to
3013 * a different one.
3014 */
3015 preempt_disable();
Wei Yangd9367bd2018-12-28 00:38:58 -08003016 drain_local_pages(drain->zone);
Michal Hockoa459eeb2017-02-24 14:56:35 -08003017 preempt_enable();
Mel Gorman0ccce3b2017-02-24 14:56:32 -08003018}
3019
Christoph Lameter9f8f2172008-02-04 22:29:11 -08003020/*
Gilad Ben-Yossef74046492012-03-28 14:42:45 -07003021 * Spill all the per-cpu pages from all CPUs back into the buddy allocator.
3022 *
Vlastimil Babka93481ff2014-12-10 15:43:01 -08003023 * When zone parameter is non-NULL, spill just the single zone's pages.
3024 *
Mel Gorman0ccce3b2017-02-24 14:56:32 -08003025 * Note that this can be extremely slow as the draining happens in a workqueue.
Christoph Lameter9f8f2172008-02-04 22:29:11 -08003026 */
Vlastimil Babka93481ff2014-12-10 15:43:01 -08003027void drain_all_pages(struct zone *zone)
Christoph Lameter9f8f2172008-02-04 22:29:11 -08003028{
Gilad Ben-Yossef74046492012-03-28 14:42:45 -07003029 int cpu;
Gilad Ben-Yossef74046492012-03-28 14:42:45 -07003030
3031 /*
3032 * Allocate in the BSS so we wont require allocation in
3033 * direct reclaim path for CONFIG_CPUMASK_OFFSTACK=y
3034 */
3035 static cpumask_t cpus_with_pcps;
3036
Michal Hockoce612872017-04-07 16:05:05 -07003037 /*
3038 * Make sure nobody triggers this path before mm_percpu_wq is fully
3039 * initialized.
3040 */
3041 if (WARN_ON_ONCE(!mm_percpu_wq))
3042 return;
3043
Mel Gormanbd233f52017-02-24 14:56:56 -08003044 /*
3045 * Do not drain if one is already in progress unless it's specific to
3046 * a zone. Such callers are primarily CMA and memory hotplug and need
3047 * the drain to be complete when the call returns.
3048 */
3049 if (unlikely(!mutex_trylock(&pcpu_drain_mutex))) {
3050 if (!zone)
3051 return;
3052 mutex_lock(&pcpu_drain_mutex);
3053 }
Mel Gorman0ccce3b2017-02-24 14:56:32 -08003054
Gilad Ben-Yossef74046492012-03-28 14:42:45 -07003055 /*
3056 * We don't care about racing with CPU hotplug event
3057 * as offline notification will cause the notified
3058 * cpu to drain that CPU pcps and on_each_cpu_mask
3059 * disables preemption as part of its processing
3060 */
3061 for_each_online_cpu(cpu) {
Vlastimil Babka93481ff2014-12-10 15:43:01 -08003062 struct per_cpu_pageset *pcp;
3063 struct zone *z;
Gilad Ben-Yossef74046492012-03-28 14:42:45 -07003064 bool has_pcps = false;
Vlastimil Babka93481ff2014-12-10 15:43:01 -08003065
3066 if (zone) {
Gilad Ben-Yossef74046492012-03-28 14:42:45 -07003067 pcp = per_cpu_ptr(zone->pageset, cpu);
Vlastimil Babka93481ff2014-12-10 15:43:01 -08003068 if (pcp->pcp.count)
Gilad Ben-Yossef74046492012-03-28 14:42:45 -07003069 has_pcps = true;
Vlastimil Babka93481ff2014-12-10 15:43:01 -08003070 } else {
3071 for_each_populated_zone(z) {
3072 pcp = per_cpu_ptr(z->pageset, cpu);
3073 if (pcp->pcp.count) {
3074 has_pcps = true;
3075 break;
3076 }
Gilad Ben-Yossef74046492012-03-28 14:42:45 -07003077 }
3078 }
Vlastimil Babka93481ff2014-12-10 15:43:01 -08003079
Gilad Ben-Yossef74046492012-03-28 14:42:45 -07003080 if (has_pcps)
3081 cpumask_set_cpu(cpu, &cpus_with_pcps);
3082 else
3083 cpumask_clear_cpu(cpu, &cpus_with_pcps);
3084 }
Mel Gorman0ccce3b2017-02-24 14:56:32 -08003085
Mel Gormanbd233f52017-02-24 14:56:56 -08003086 for_each_cpu(cpu, &cpus_with_pcps) {
Wei Yangd9367bd2018-12-28 00:38:58 -08003087 struct pcpu_drain *drain = per_cpu_ptr(&pcpu_drain, cpu);
3088
3089 drain->zone = zone;
3090 INIT_WORK(&drain->work, drain_local_pages_wq);
3091 queue_work_on(cpu, mm_percpu_wq, &drain->work);
Mel Gorman0ccce3b2017-02-24 14:56:32 -08003092 }
Mel Gormanbd233f52017-02-24 14:56:56 -08003093 for_each_cpu(cpu, &cpus_with_pcps)
Wei Yangd9367bd2018-12-28 00:38:58 -08003094 flush_work(&per_cpu_ptr(&pcpu_drain, cpu)->work);
Mel Gormanbd233f52017-02-24 14:56:56 -08003095
3096 mutex_unlock(&pcpu_drain_mutex);
Christoph Lameter9f8f2172008-02-04 22:29:11 -08003097}
3098
Rafael J. Wysocki296699d2007-07-29 23:27:18 +02003099#ifdef CONFIG_HIBERNATION
Linus Torvalds1da177e2005-04-16 15:20:36 -07003100
Chen Yu556b9692017-08-25 15:55:30 -07003101/*
3102 * Touch the watchdog for every WD_PAGE_COUNT pages.
3103 */
3104#define WD_PAGE_COUNT (128*1024)
3105
Linus Torvalds1da177e2005-04-16 15:20:36 -07003106void mark_free_pages(struct zone *zone)
3107{
Chen Yu556b9692017-08-25 15:55:30 -07003108 unsigned long pfn, max_zone_pfn, page_count = WD_PAGE_COUNT;
Rafael J. Wysockif623f0d2006-09-25 23:32:49 -07003109 unsigned long flags;
Mel Gorman7aeb09f2014-06-04 16:10:21 -07003110 unsigned int order, t;
Geliang Tang86760a22016-01-14 15:20:33 -08003111 struct page *page;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003112
Xishi Qiu8080fc02013-09-11 14:21:45 -07003113 if (zone_is_empty(zone))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003114 return;
3115
3116 spin_lock_irqsave(&zone->lock, flags);
Rafael J. Wysockif623f0d2006-09-25 23:32:49 -07003117
Cody P Schafer108bcc92013-02-22 16:35:23 -08003118 max_zone_pfn = zone_end_pfn(zone);
Rafael J. Wysockif623f0d2006-09-25 23:32:49 -07003119 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
3120 if (pfn_valid(pfn)) {
Geliang Tang86760a22016-01-14 15:20:33 -08003121 page = pfn_to_page(pfn);
Joonsoo Kimba6b0972016-05-19 17:12:16 -07003122
Chen Yu556b9692017-08-25 15:55:30 -07003123 if (!--page_count) {
3124 touch_nmi_watchdog();
3125 page_count = WD_PAGE_COUNT;
3126 }
3127
Joonsoo Kimba6b0972016-05-19 17:12:16 -07003128 if (page_zone(page) != zone)
3129 continue;
3130
Rafael J. Wysocki7be98232007-05-06 14:50:42 -07003131 if (!swsusp_page_is_forbidden(page))
3132 swsusp_unset_page_free(page);
Rafael J. Wysockif623f0d2006-09-25 23:32:49 -07003133 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003134
Mel Gormanb2a0ac82007-10-16 01:25:48 -07003135 for_each_migratetype_order(order, t) {
Geliang Tang86760a22016-01-14 15:20:33 -08003136 list_for_each_entry(page,
3137 &zone->free_area[order].free_list[t], lru) {
Rafael J. Wysockif623f0d2006-09-25 23:32:49 -07003138 unsigned long i;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003139
Geliang Tang86760a22016-01-14 15:20:33 -08003140 pfn = page_to_pfn(page);
Chen Yu556b9692017-08-25 15:55:30 -07003141 for (i = 0; i < (1UL << order); i++) {
3142 if (!--page_count) {
3143 touch_nmi_watchdog();
3144 page_count = WD_PAGE_COUNT;
3145 }
Rafael J. Wysocki7be98232007-05-06 14:50:42 -07003146 swsusp_set_page_free(pfn_to_page(pfn + i));
Chen Yu556b9692017-08-25 15:55:30 -07003147 }
Rafael J. Wysockif623f0d2006-09-25 23:32:49 -07003148 }
Mel Gormanb2a0ac82007-10-16 01:25:48 -07003149 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003150 spin_unlock_irqrestore(&zone->lock, flags);
3151}
Mel Gormane2c55dc2007-10-16 01:25:50 -07003152#endif /* CONFIG_PM */
Linus Torvalds1da177e2005-04-16 15:20:36 -07003153
Mel Gorman2d4894b2017-11-15 17:37:59 -08003154static bool free_unref_page_prepare(struct page *page, unsigned long pfn)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003155{
Mel Gorman5f8dcc22009-09-21 17:03:19 -07003156 int migratetype;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003157
Mel Gorman4db75482016-05-19 17:14:32 -07003158 if (!free_pcp_prepare(page))
Mel Gorman9cca35d42017-11-15 17:37:37 -08003159 return false;
Hugh Dickins689bceb2005-11-21 21:32:20 -08003160
Mel Gormandc4b0ca2014-06-04 16:10:17 -07003161 migratetype = get_pfnblock_migratetype(page, pfn);
Vlastimil Babkabb14c2c2015-09-08 15:01:25 -07003162 set_pcppage_migratetype(page, migratetype);
Mel Gorman9cca35d42017-11-15 17:37:37 -08003163 return true;
3164}
3165
Mel Gorman2d4894b2017-11-15 17:37:59 -08003166static void free_unref_page_commit(struct page *page, unsigned long pfn)
Mel Gorman9cca35d42017-11-15 17:37:37 -08003167{
3168 struct zone *zone = page_zone(page);
3169 struct per_cpu_pages *pcp;
3170 int migratetype;
3171
3172 migratetype = get_pcppage_migratetype(page);
Mel Gormand34b0732017-04-20 14:37:43 -07003173 __count_vm_event(PGFREE);
Mel Gormanda456f12009-06-16 15:32:08 -07003174
Mel Gorman5f8dcc22009-09-21 17:03:19 -07003175 /*
3176 * We only track unmovable, reclaimable and movable on pcp lists.
3177 * Free ISOLATE pages back to the allocator because they are being
Xishi Qiua6ffdc02017-05-03 14:52:52 -07003178 * offlined but treat HIGHATOMIC as movable pages so we can get those
Mel Gorman5f8dcc22009-09-21 17:03:19 -07003179 * areas back if necessary. Otherwise, we may have to free
3180 * excessively into the page allocator
3181 */
3182 if (migratetype >= MIGRATE_PCPTYPES) {
Minchan Kim194159f2013-02-22 16:33:58 -08003183 if (unlikely(is_migrate_isolate(migratetype))) {
David Hildenbrand7fef4312020-10-15 20:09:35 -07003184 free_one_page(zone, page, pfn, 0, migratetype,
3185 FPI_NONE);
Mel Gorman9cca35d42017-11-15 17:37:37 -08003186 return;
Mel Gorman5f8dcc22009-09-21 17:03:19 -07003187 }
3188 migratetype = MIGRATE_MOVABLE;
3189 }
3190
Christoph Lameter99dcc3e2010-01-05 15:34:51 +09003191 pcp = &this_cpu_ptr(zone->pageset)->pcp;
Mel Gorman2d4894b2017-11-15 17:37:59 -08003192 list_add(&page->lru, &pcp->lists[migratetype]);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003193 pcp->count++;
Nick Piggin48db57f2006-01-08 01:00:42 -08003194 if (pcp->count >= pcp->high) {
Jason Low4db0c3c2015-04-15 16:14:08 -07003195 unsigned long batch = READ_ONCE(pcp->batch);
Cody P Schafer998d39cb2013-07-03 15:01:32 -07003196 free_pcppages_bulk(zone, batch, pcp);
Nick Piggin48db57f2006-01-08 01:00:42 -08003197 }
Mel Gorman9cca35d42017-11-15 17:37:37 -08003198}
Mel Gorman5f8dcc22009-09-21 17:03:19 -07003199
Mel Gorman9cca35d42017-11-15 17:37:37 -08003200/*
3201 * Free a 0-order page
Mel Gorman9cca35d42017-11-15 17:37:37 -08003202 */
Mel Gorman2d4894b2017-11-15 17:37:59 -08003203void free_unref_page(struct page *page)
Mel Gorman9cca35d42017-11-15 17:37:37 -08003204{
3205 unsigned long flags;
3206 unsigned long pfn = page_to_pfn(page);
3207
Mel Gorman2d4894b2017-11-15 17:37:59 -08003208 if (!free_unref_page_prepare(page, pfn))
Mel Gorman9cca35d42017-11-15 17:37:37 -08003209 return;
3210
3211 local_irq_save(flags);
Mel Gorman2d4894b2017-11-15 17:37:59 -08003212 free_unref_page_commit(page, pfn);
Mel Gormand34b0732017-04-20 14:37:43 -07003213 local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003214}
3215
Nick Piggin8dfcc9b2006-03-22 00:08:05 -08003216/*
Konstantin Khlebnikovcc598502012-01-10 15:07:04 -08003217 * Free a list of 0-order pages
3218 */
Mel Gorman2d4894b2017-11-15 17:37:59 -08003219void free_unref_page_list(struct list_head *list)
Konstantin Khlebnikovcc598502012-01-10 15:07:04 -08003220{
3221 struct page *page, *next;
Mel Gorman9cca35d42017-11-15 17:37:37 -08003222 unsigned long flags, pfn;
Lucas Stachc24ad772017-12-14 15:32:55 -08003223 int batch_count = 0;
Konstantin Khlebnikovcc598502012-01-10 15:07:04 -08003224
Mel Gorman9cca35d42017-11-15 17:37:37 -08003225 /* Prepare pages for freeing */
Konstantin Khlebnikovcc598502012-01-10 15:07:04 -08003226 list_for_each_entry_safe(page, next, list, lru) {
Mel Gorman9cca35d42017-11-15 17:37:37 -08003227 pfn = page_to_pfn(page);
Mel Gorman2d4894b2017-11-15 17:37:59 -08003228 if (!free_unref_page_prepare(page, pfn))
Mel Gorman9cca35d42017-11-15 17:37:37 -08003229 list_del(&page->lru);
3230 set_page_private(page, pfn);
Konstantin Khlebnikovcc598502012-01-10 15:07:04 -08003231 }
Mel Gorman9cca35d42017-11-15 17:37:37 -08003232
3233 local_irq_save(flags);
3234 list_for_each_entry_safe(page, next, list, lru) {
3235 unsigned long pfn = page_private(page);
3236
3237 set_page_private(page, 0);
Mel Gorman2d4894b2017-11-15 17:37:59 -08003238 trace_mm_page_free_batched(page);
3239 free_unref_page_commit(page, pfn);
Lucas Stachc24ad772017-12-14 15:32:55 -08003240
3241 /*
3242 * Guard against excessive IRQ disabled times when we get
3243 * a large list of pages to free.
3244 */
3245 if (++batch_count == SWAP_CLUSTER_MAX) {
3246 local_irq_restore(flags);
3247 batch_count = 0;
3248 local_irq_save(flags);
3249 }
Mel Gorman9cca35d42017-11-15 17:37:37 -08003250 }
3251 local_irq_restore(flags);
Konstantin Khlebnikovcc598502012-01-10 15:07:04 -08003252}
3253
3254/*
Nick Piggin8dfcc9b2006-03-22 00:08:05 -08003255 * split_page takes a non-compound higher-order page, and splits it into
3256 * n (1<<order) sub-pages: page[0..n]
3257 * Each sub-page must be freed individually.
3258 *
3259 * Note: this is probably too low level an operation for use in drivers.
3260 * Please consult with lkml before using this in your driver.
3261 */
3262void split_page(struct page *page, unsigned int order)
3263{
3264 int i;
3265
Sasha Levin309381fea2014-01-23 15:52:54 -08003266 VM_BUG_ON_PAGE(PageCompound(page), page);
3267 VM_BUG_ON_PAGE(!page_count(page), page);
Vegard Nossumb1eeab62008-11-25 16:55:53 +01003268
Joonsoo Kima9627bc2016-07-26 15:23:49 -07003269 for (i = 1; i < (1 << order); i++)
Nick Piggin7835e982006-03-22 00:08:40 -08003270 set_page_refcounted(page + i);
Matthew Wilcox (Oracle)8fb156c2020-10-15 20:05:29 -07003271 split_page_owner(page, 1 << order);
Nick Piggin8dfcc9b2006-03-22 00:08:05 -08003272}
K. Y. Srinivasan5853ff22013-03-25 15:47:38 -07003273EXPORT_SYMBOL_GPL(split_page);
Nick Piggin8dfcc9b2006-03-22 00:08:05 -08003274
Joonsoo Kim3c605092014-11-13 15:19:21 -08003275int __isolate_free_page(struct page *page, unsigned int order)
Mel Gorman748446b2010-05-24 14:32:27 -07003276{
Mel Gorman748446b2010-05-24 14:32:27 -07003277 unsigned long watermark;
3278 struct zone *zone;
Bartlomiej Zolnierkiewicz2139cbe2012-10-08 16:32:00 -07003279 int mt;
Mel Gorman748446b2010-05-24 14:32:27 -07003280
3281 BUG_ON(!PageBuddy(page));
3282
3283 zone = page_zone(page);
Marek Szyprowski2e30abd2012-12-11 16:02:57 -08003284 mt = get_pageblock_migratetype(page);
Mel Gorman748446b2010-05-24 14:32:27 -07003285
Minchan Kim194159f2013-02-22 16:33:58 -08003286 if (!is_migrate_isolate(mt)) {
Vlastimil Babka8348faf2016-10-07 16:58:00 -07003287 /*
3288 * Obey watermarks as if the page was being allocated. We can
3289 * emulate a high-order watermark check with a raised order-0
3290 * watermark, because we already know our high-order page
3291 * exists.
3292 */
Mel Gormanfd1444b2019-03-05 15:44:50 -08003293 watermark = zone->_watermark[WMARK_MIN] + (1UL << order);
Joonsoo Kimd883c6c2018-05-23 10:18:21 +09003294 if (!zone_watermark_ok(zone, 0, watermark, 0, ALLOC_CMA))
Marek Szyprowski2e30abd2012-12-11 16:02:57 -08003295 return 0;
3296
Mel Gorman8fb74b92013-01-11 14:32:16 -08003297 __mod_zone_freepage_state(zone, -(1UL << order), mt);
Marek Szyprowski2e30abd2012-12-11 16:02:57 -08003298 }
Mel Gorman748446b2010-05-24 14:32:27 -07003299
3300 /* Remove page from free list */
Dan Williamsb03641a2019-05-14 15:41:32 -07003301
Alexander Duyck6ab01362020-04-06 20:04:49 -07003302 del_page_from_free_list(page, zone, order);
Bartlomiej Zolnierkiewicz2139cbe2012-10-08 16:32:00 -07003303
zhong jiang400bc7f2016-07-28 15:45:07 -07003304 /*
3305 * Set the pageblock if the isolated page is at least half of a
3306 * pageblock
3307 */
Mel Gorman748446b2010-05-24 14:32:27 -07003308 if (order >= pageblock_order - 1) {
3309 struct page *endpage = page + (1 << order) - 1;
Michal Nazarewicz47118af2011-12-29 13:09:50 +01003310 for (; page < endpage; page += pageblock_nr_pages) {
3311 int mt = get_pageblock_migratetype(page);
Minchan Kim88ed3652016-12-12 16:42:05 -08003312 if (!is_migrate_isolate(mt) && !is_migrate_cma(mt)
Xishi Qiua6ffdc02017-05-03 14:52:52 -07003313 && !is_migrate_highatomic(mt))
Michal Nazarewicz47118af2011-12-29 13:09:50 +01003314 set_pageblock_migratetype(page,
3315 MIGRATE_MOVABLE);
3316 }
Mel Gorman748446b2010-05-24 14:32:27 -07003317 }
3318
Joonsoo Kimf3a14ce2015-07-17 16:24:15 -07003319
Mel Gorman8fb74b92013-01-11 14:32:16 -08003320 return 1UL << order;
Mel Gorman1fb3f8c2012-10-08 16:29:12 -07003321}
3322
Alexander Duyck624f58d2020-04-06 20:04:53 -07003323/**
3324 * __putback_isolated_page - Return a now-isolated page back where we got it
3325 * @page: Page that was isolated
3326 * @order: Order of the isolated page
Randy Dunlape6a0a7a2020-04-10 14:32:29 -07003327 * @mt: The page's pageblock's migratetype
Alexander Duyck624f58d2020-04-06 20:04:53 -07003328 *
3329 * This function is meant to return a page pulled from the free lists via
3330 * __isolate_free_page back to the free lists they were pulled from.
3331 */
3332void __putback_isolated_page(struct page *page, unsigned int order, int mt)
3333{
3334 struct zone *zone = page_zone(page);
3335
3336 /* zone lock should be held when this function is called */
3337 lockdep_assert_held(&zone->lock);
3338
3339 /* Return isolated page to tail of freelist. */
David Hildenbrandf04a5d52020-10-15 20:09:20 -07003340 __free_one_page(page, page_to_pfn(page), zone, order, mt,
David Hildenbrand47b6a24a22020-10-15 20:09:26 -07003341 FPI_SKIP_REPORT_NOTIFY | FPI_TO_TAIL);
Alexander Duyck624f58d2020-04-06 20:04:53 -07003342}
3343
Mel Gorman1fb3f8c2012-10-08 16:29:12 -07003344/*
Mel Gorman060e7412016-05-19 17:13:27 -07003345 * Update NUMA hit/miss statistics
3346 *
3347 * Must be called with interrupts disabled.
Mel Gorman060e7412016-05-19 17:13:27 -07003348 */
Michal Hocko41b61672017-01-10 16:57:42 -08003349static inline void zone_statistics(struct zone *preferred_zone, struct zone *z)
Mel Gorman060e7412016-05-19 17:13:27 -07003350{
3351#ifdef CONFIG_NUMA
Kemi Wang3a321d22017-09-08 16:12:48 -07003352 enum numa_stat_item local_stat = NUMA_LOCAL;
Mel Gorman060e7412016-05-19 17:13:27 -07003353
Kemi Wang45180852017-11-15 17:38:22 -08003354 /* skip numa counters update if numa stats is disabled */
3355 if (!static_branch_likely(&vm_numa_stat_key))
3356 return;
3357
Pavel Tatashinc1093b72018-08-21 21:53:32 -07003358 if (zone_to_nid(z) != numa_node_id())
Mel Gorman060e7412016-05-19 17:13:27 -07003359 local_stat = NUMA_OTHER;
Mel Gorman060e7412016-05-19 17:13:27 -07003360
Pavel Tatashinc1093b72018-08-21 21:53:32 -07003361 if (zone_to_nid(z) == zone_to_nid(preferred_zone))
Kemi Wang3a321d22017-09-08 16:12:48 -07003362 __inc_numa_state(z, NUMA_HIT);
Michal Hocko2df26632017-01-10 16:57:39 -08003363 else {
Kemi Wang3a321d22017-09-08 16:12:48 -07003364 __inc_numa_state(z, NUMA_MISS);
3365 __inc_numa_state(preferred_zone, NUMA_FOREIGN);
Mel Gorman060e7412016-05-19 17:13:27 -07003366 }
Kemi Wang3a321d22017-09-08 16:12:48 -07003367 __inc_numa_state(z, local_stat);
Mel Gorman060e7412016-05-19 17:13:27 -07003368#endif
3369}
3370
Mel Gorman066b2392017-02-24 14:56:26 -08003371/* Remove page from the per-cpu list, caller must protect the list */
3372static struct page *__rmqueue_pcplist(struct zone *zone, int migratetype,
Mel Gorman6bb15452018-12-28 00:35:41 -08003373 unsigned int alloc_flags,
Mel Gorman453f85d2017-11-15 17:38:03 -08003374 struct per_cpu_pages *pcp,
Mel Gorman066b2392017-02-24 14:56:26 -08003375 struct list_head *list)
3376{
3377 struct page *page;
3378
3379 do {
3380 if (list_empty(list)) {
3381 pcp->count += rmqueue_bulk(zone, 0,
3382 pcp->batch, list,
Mel Gorman6bb15452018-12-28 00:35:41 -08003383 migratetype, alloc_flags);
Mel Gorman066b2392017-02-24 14:56:26 -08003384 if (unlikely(list_empty(list)))
3385 return NULL;
3386 }
3387
Mel Gorman453f85d2017-11-15 17:38:03 -08003388 page = list_first_entry(list, struct page, lru);
Mel Gorman066b2392017-02-24 14:56:26 -08003389 list_del(&page->lru);
3390 pcp->count--;
3391 } while (check_new_pcp(page));
3392
3393 return page;
3394}
3395
3396/* Lock and remove page from the per-cpu list */
3397static struct page *rmqueue_pcplist(struct zone *preferred_zone,
Yafang Shao1c52e6d2019-05-13 17:22:40 -07003398 struct zone *zone, gfp_t gfp_flags,
3399 int migratetype, unsigned int alloc_flags)
Mel Gorman066b2392017-02-24 14:56:26 -08003400{
3401 struct per_cpu_pages *pcp;
3402 struct list_head *list;
Mel Gorman066b2392017-02-24 14:56:26 -08003403 struct page *page;
Mel Gormand34b0732017-04-20 14:37:43 -07003404 unsigned long flags;
Mel Gorman066b2392017-02-24 14:56:26 -08003405
Mel Gormand34b0732017-04-20 14:37:43 -07003406 local_irq_save(flags);
Mel Gorman066b2392017-02-24 14:56:26 -08003407 pcp = &this_cpu_ptr(zone->pageset)->pcp;
3408 list = &pcp->lists[migratetype];
Mel Gorman6bb15452018-12-28 00:35:41 -08003409 page = __rmqueue_pcplist(zone, migratetype, alloc_flags, pcp, list);
Mel Gorman066b2392017-02-24 14:56:26 -08003410 if (page) {
Yafang Shao1c52e6d2019-05-13 17:22:40 -07003411 __count_zid_vm_events(PGALLOC, page_zonenum(page), 1);
Mel Gorman066b2392017-02-24 14:56:26 -08003412 zone_statistics(preferred_zone, zone);
3413 }
Mel Gormand34b0732017-04-20 14:37:43 -07003414 local_irq_restore(flags);
Mel Gorman066b2392017-02-24 14:56:26 -08003415 return page;
3416}
3417
Mel Gorman060e7412016-05-19 17:13:27 -07003418/*
Vlastimil Babka75379192015-02-11 15:25:38 -08003419 * Allocate a page from the given zone. Use pcplists for order-0 allocations.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003420 */
Mel Gorman0a15c3e2009-06-16 15:32:05 -07003421static inline
Mel Gorman066b2392017-02-24 14:56:26 -08003422struct page *rmqueue(struct zone *preferred_zone,
Mel Gorman7aeb09f2014-06-04 16:10:21 -07003423 struct zone *zone, unsigned int order,
Mel Gormanc6038442016-05-19 17:13:38 -07003424 gfp_t gfp_flags, unsigned int alloc_flags,
3425 int migratetype)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003426{
3427 unsigned long flags;
Hugh Dickins689bceb2005-11-21 21:32:20 -08003428 struct page *page;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003429
Mel Gormand34b0732017-04-20 14:37:43 -07003430 if (likely(order == 0)) {
Joonsoo Kim1d91df82020-10-02 22:21:45 -07003431 /*
3432 * MIGRATE_MOVABLE pcplist could have the pages on CMA area and
3433 * we need to skip it when CMA area isn't allowed.
3434 */
3435 if (!IS_ENABLED(CONFIG_CMA) || alloc_flags & ALLOC_CMA ||
3436 migratetype != MIGRATE_MOVABLE) {
3437 page = rmqueue_pcplist(preferred_zone, zone, gfp_flags,
Yafang Shao1c52e6d2019-05-13 17:22:40 -07003438 migratetype, alloc_flags);
Joonsoo Kim1d91df82020-10-02 22:21:45 -07003439 goto out;
3440 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003441 }
3442
Mel Gorman066b2392017-02-24 14:56:26 -08003443 /*
3444 * We most definitely don't want callers attempting to
3445 * allocate greater than order-1 page units with __GFP_NOFAIL.
3446 */
3447 WARN_ON_ONCE((gfp_flags & __GFP_NOFAIL) && (order > 1));
3448 spin_lock_irqsave(&zone->lock, flags);
3449
3450 do {
3451 page = NULL;
Joonsoo Kim1d91df82020-10-02 22:21:45 -07003452 /*
3453 * order-0 request can reach here when the pcplist is skipped
3454 * due to non-CMA allocation context. HIGHATOMIC area is
3455 * reserved for high-order atomic allocation, so order-0
3456 * request should skip it.
3457 */
3458 if (order > 0 && alloc_flags & ALLOC_HARDER) {
Mel Gorman066b2392017-02-24 14:56:26 -08003459 page = __rmqueue_smallest(zone, order, MIGRATE_HIGHATOMIC);
3460 if (page)
3461 trace_mm_page_alloc_zone_locked(page, order, migratetype);
3462 }
3463 if (!page)
Mel Gorman6bb15452018-12-28 00:35:41 -08003464 page = __rmqueue(zone, order, migratetype, alloc_flags);
Mel Gorman066b2392017-02-24 14:56:26 -08003465 } while (page && check_new_pages(page, order));
3466 spin_unlock(&zone->lock);
3467 if (!page)
3468 goto failed;
3469 __mod_zone_freepage_state(zone, -(1 << order),
3470 get_pcppage_migratetype(page));
3471
Mel Gorman16709d12016-07-28 15:46:56 -07003472 __count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order);
Michal Hocko41b61672017-01-10 16:57:42 -08003473 zone_statistics(preferred_zone, zone);
Nick Piggina74609f2006-01-06 00:11:20 -08003474 local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003475
Mel Gorman066b2392017-02-24 14:56:26 -08003476out:
Mel Gorman73444bc2019-01-08 15:23:39 -08003477 /* Separate test+clear to avoid unnecessary atomics */
3478 if (test_bit(ZONE_BOOSTED_WATERMARK, &zone->flags)) {
3479 clear_bit(ZONE_BOOSTED_WATERMARK, &zone->flags);
3480 wakeup_kswapd(zone, 0, 0, zone_idx(zone));
3481 }
3482
Mel Gorman066b2392017-02-24 14:56:26 -08003483 VM_BUG_ON_PAGE(page && bad_range(zone, page), page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003484 return page;
Nick Piggina74609f2006-01-06 00:11:20 -08003485
3486failed:
3487 local_irq_restore(flags);
Nick Piggina74609f2006-01-06 00:11:20 -08003488 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003489}
3490
Akinobu Mita933e3122006-12-08 02:39:45 -08003491#ifdef CONFIG_FAIL_PAGE_ALLOC
3492
Akinobu Mitab2588c42011-07-26 16:09:03 -07003493static struct {
Akinobu Mita933e3122006-12-08 02:39:45 -08003494 struct fault_attr attr;
3495
Viresh Kumar621a5f72015-09-26 15:04:07 -07003496 bool ignore_gfp_highmem;
Mel Gorman71baba42015-11-06 16:28:28 -08003497 bool ignore_gfp_reclaim;
Akinobu Mita54114992007-07-15 23:40:23 -07003498 u32 min_order;
Akinobu Mita933e3122006-12-08 02:39:45 -08003499} fail_page_alloc = {
3500 .attr = FAULT_ATTR_INITIALIZER,
Mel Gorman71baba42015-11-06 16:28:28 -08003501 .ignore_gfp_reclaim = true,
Viresh Kumar621a5f72015-09-26 15:04:07 -07003502 .ignore_gfp_highmem = true,
Akinobu Mita54114992007-07-15 23:40:23 -07003503 .min_order = 1,
Akinobu Mita933e3122006-12-08 02:39:45 -08003504};
3505
3506static int __init setup_fail_page_alloc(char *str)
3507{
3508 return setup_fault_attr(&fail_page_alloc.attr, str);
3509}
3510__setup("fail_page_alloc=", setup_fail_page_alloc);
3511
Benjamin Poirieraf3b8542018-12-28 00:39:23 -08003512static bool __should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
Akinobu Mita933e3122006-12-08 02:39:45 -08003513{
Akinobu Mita54114992007-07-15 23:40:23 -07003514 if (order < fail_page_alloc.min_order)
Gavin Shandeaf3862012-07-31 16:41:51 -07003515 return false;
Akinobu Mita933e3122006-12-08 02:39:45 -08003516 if (gfp_mask & __GFP_NOFAIL)
Gavin Shandeaf3862012-07-31 16:41:51 -07003517 return false;
Akinobu Mita933e3122006-12-08 02:39:45 -08003518 if (fail_page_alloc.ignore_gfp_highmem && (gfp_mask & __GFP_HIGHMEM))
Gavin Shandeaf3862012-07-31 16:41:51 -07003519 return false;
Mel Gorman71baba42015-11-06 16:28:28 -08003520 if (fail_page_alloc.ignore_gfp_reclaim &&
3521 (gfp_mask & __GFP_DIRECT_RECLAIM))
Gavin Shandeaf3862012-07-31 16:41:51 -07003522 return false;
Akinobu Mita933e3122006-12-08 02:39:45 -08003523
3524 return should_fail(&fail_page_alloc.attr, 1 << order);
3525}
3526
3527#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
3528
3529static int __init fail_page_alloc_debugfs(void)
3530{
Joe Perches0825a6f2018-06-14 15:27:58 -07003531 umode_t mode = S_IFREG | 0600;
Akinobu Mita933e3122006-12-08 02:39:45 -08003532 struct dentry *dir;
Akinobu Mita933e3122006-12-08 02:39:45 -08003533
Akinobu Mitadd48c082011-08-03 16:21:01 -07003534 dir = fault_create_debugfs_attr("fail_page_alloc", NULL,
3535 &fail_page_alloc.attr);
Akinobu Mita933e3122006-12-08 02:39:45 -08003536
Greg Kroah-Hartmand9f79792019-03-05 15:46:09 -08003537 debugfs_create_bool("ignore-gfp-wait", mode, dir,
3538 &fail_page_alloc.ignore_gfp_reclaim);
3539 debugfs_create_bool("ignore-gfp-highmem", mode, dir,
3540 &fail_page_alloc.ignore_gfp_highmem);
3541 debugfs_create_u32("min-order", mode, dir, &fail_page_alloc.min_order);
Akinobu Mita933e3122006-12-08 02:39:45 -08003542
Akinobu Mitab2588c42011-07-26 16:09:03 -07003543 return 0;
Akinobu Mita933e3122006-12-08 02:39:45 -08003544}
3545
3546late_initcall(fail_page_alloc_debugfs);
3547
3548#endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */
3549
3550#else /* CONFIG_FAIL_PAGE_ALLOC */
3551
Benjamin Poirieraf3b8542018-12-28 00:39:23 -08003552static inline bool __should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
Akinobu Mita933e3122006-12-08 02:39:45 -08003553{
Gavin Shandeaf3862012-07-31 16:41:51 -07003554 return false;
Akinobu Mita933e3122006-12-08 02:39:45 -08003555}
3556
3557#endif /* CONFIG_FAIL_PAGE_ALLOC */
3558
Alexei Starovoitov76cd6172020-08-27 15:01:10 -07003559noinline bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
Benjamin Poirieraf3b8542018-12-28 00:39:23 -08003560{
3561 return __should_fail_alloc_page(gfp_mask, order);
3562}
3563ALLOW_ERROR_INJECTION(should_fail_alloc_page, TRUE);
3564
Jaewon Kimf27ce0e2020-08-06 23:25:20 -07003565static inline long __zone_watermark_unusable_free(struct zone *z,
3566 unsigned int order, unsigned int alloc_flags)
3567{
3568 const bool alloc_harder = (alloc_flags & (ALLOC_HARDER|ALLOC_OOM));
3569 long unusable_free = (1 << order) - 1;
3570
3571 /*
3572 * If the caller does not have rights to ALLOC_HARDER then subtract
3573 * the high-atomic reserves. This will over-estimate the size of the
3574 * atomic reserve but it avoids a search.
3575 */
3576 if (likely(!alloc_harder))
3577 unusable_free += z->nr_reserved_highatomic;
3578
3579#ifdef CONFIG_CMA
3580 /* If allocation can't use CMA areas don't use free CMA pages */
3581 if (!(alloc_flags & ALLOC_CMA))
3582 unusable_free += zone_page_state(z, NR_FREE_CMA_PAGES);
3583#endif
3584
3585 return unusable_free;
3586}
3587
Linus Torvalds1da177e2005-04-16 15:20:36 -07003588/*
Mel Gorman97a16fc2015-11-06 16:28:40 -08003589 * Return true if free base pages are above 'mark'. For high-order checks it
3590 * will return true of the order-0 watermark is reached and there is at least
3591 * one free page of a suitable size. Checking now avoids taking the zone lock
3592 * to check in the allocation paths if no pages are free.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003593 */
Michal Hocko86a294a2016-05-20 16:57:12 -07003594bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,
Joonsoo Kim97a225e2020-06-03 15:59:01 -07003595 int highest_zoneidx, unsigned int alloc_flags,
Michal Hocko86a294a2016-05-20 16:57:12 -07003596 long free_pages)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003597{
Christoph Lameterd23ad422007-02-10 01:43:02 -08003598 long min = mark;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003599 int o;
Michal Hockocd04ae12017-09-06 16:24:50 -07003600 const bool alloc_harder = (alloc_flags & (ALLOC_HARDER|ALLOC_OOM));
Linus Torvalds1da177e2005-04-16 15:20:36 -07003601
Mel Gorman0aaa29a2015-11-06 16:28:37 -08003602 /* free_pages may go negative - that's OK */
Jaewon Kimf27ce0e2020-08-06 23:25:20 -07003603 free_pages -= __zone_watermark_unusable_free(z, order, alloc_flags);
Mel Gorman0aaa29a2015-11-06 16:28:37 -08003604
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08003605 if (alloc_flags & ALLOC_HIGH)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003606 min -= min / 2;
Mel Gorman0aaa29a2015-11-06 16:28:37 -08003607
Jaewon Kimf27ce0e2020-08-06 23:25:20 -07003608 if (unlikely(alloc_harder)) {
Michal Hockocd04ae12017-09-06 16:24:50 -07003609 /*
3610 * OOM victims can try even harder than normal ALLOC_HARDER
3611 * users on the grounds that it's definitely going to be in
3612 * the exit path shortly and free memory. Any allocation it
3613 * makes during the free path will be small and short-lived.
3614 */
3615 if (alloc_flags & ALLOC_OOM)
3616 min -= min / 2;
3617 else
3618 min -= min / 4;
3619 }
3620
Mel Gorman97a16fc2015-11-06 16:28:40 -08003621 /*
3622 * Check watermarks for an order-0 allocation request. If these
3623 * are not met, then a high-order request also cannot go ahead
3624 * even if a suitable page happened to be free.
3625 */
Joonsoo Kim97a225e2020-06-03 15:59:01 -07003626 if (free_pages <= min + z->lowmem_reserve[highest_zoneidx])
Mel Gorman88f5acf2011-01-13 15:45:41 -08003627 return false;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003628
Mel Gorman97a16fc2015-11-06 16:28:40 -08003629 /* If this is an order-0 request then the watermark is fine */
3630 if (!order)
3631 return true;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003632
Mel Gorman97a16fc2015-11-06 16:28:40 -08003633 /* For a high-order request, check at least one suitable page is free */
3634 for (o = order; o < MAX_ORDER; o++) {
3635 struct free_area *area = &z->free_area[o];
3636 int mt;
3637
3638 if (!area->nr_free)
3639 continue;
3640
Mel Gorman97a16fc2015-11-06 16:28:40 -08003641 for (mt = 0; mt < MIGRATE_PCPTYPES; mt++) {
Dan Williamsb03641a2019-05-14 15:41:32 -07003642 if (!free_area_empty(area, mt))
Mel Gorman97a16fc2015-11-06 16:28:40 -08003643 return true;
3644 }
3645
3646#ifdef CONFIG_CMA
Joonsoo Kimd883c6c2018-05-23 10:18:21 +09003647 if ((alloc_flags & ALLOC_CMA) &&
Dan Williamsb03641a2019-05-14 15:41:32 -07003648 !free_area_empty(area, MIGRATE_CMA)) {
Mel Gorman97a16fc2015-11-06 16:28:40 -08003649 return true;
Joonsoo Kimd883c6c2018-05-23 10:18:21 +09003650 }
Mel Gorman97a16fc2015-11-06 16:28:40 -08003651#endif
chenqiwu76089d02020-04-01 21:09:50 -07003652 if (alloc_harder && !free_area_empty(area, MIGRATE_HIGHATOMIC))
Vlastimil Babkab050e372017-11-15 17:38:30 -08003653 return true;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003654 }
Mel Gorman97a16fc2015-11-06 16:28:40 -08003655 return false;
Mel Gorman88f5acf2011-01-13 15:45:41 -08003656}
3657
Mel Gorman7aeb09f2014-06-04 16:10:21 -07003658bool zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,
Joonsoo Kim97a225e2020-06-03 15:59:01 -07003659 int highest_zoneidx, unsigned int alloc_flags)
Mel Gorman88f5acf2011-01-13 15:45:41 -08003660{
Joonsoo Kim97a225e2020-06-03 15:59:01 -07003661 return __zone_watermark_ok(z, order, mark, highest_zoneidx, alloc_flags,
Mel Gorman88f5acf2011-01-13 15:45:41 -08003662 zone_page_state(z, NR_FREE_PAGES));
3663}
3664
Mel Gorman48ee5f32016-05-19 17:14:07 -07003665static inline bool zone_watermark_fast(struct zone *z, unsigned int order,
Joonsoo Kim97a225e2020-06-03 15:59:01 -07003666 unsigned long mark, int highest_zoneidx,
Charan Teja Reddyf80b08f2020-08-06 23:25:24 -07003667 unsigned int alloc_flags, gfp_t gfp_mask)
Mel Gorman48ee5f32016-05-19 17:14:07 -07003668{
Jaewon Kimf27ce0e2020-08-06 23:25:20 -07003669 long free_pages;
Joonsoo Kimd883c6c2018-05-23 10:18:21 +09003670
Jaewon Kimf27ce0e2020-08-06 23:25:20 -07003671 free_pages = zone_page_state(z, NR_FREE_PAGES);
Mel Gorman48ee5f32016-05-19 17:14:07 -07003672
3673 /*
3674 * Fast check for order-0 only. If this fails then the reserves
Jaewon Kimf27ce0e2020-08-06 23:25:20 -07003675 * need to be calculated.
Mel Gorman48ee5f32016-05-19 17:14:07 -07003676 */
Jaewon Kimf27ce0e2020-08-06 23:25:20 -07003677 if (!order) {
3678 long fast_free;
3679
3680 fast_free = free_pages;
3681 fast_free -= __zone_watermark_unusable_free(z, 0, alloc_flags);
3682 if (fast_free > mark + z->lowmem_reserve[highest_zoneidx])
3683 return true;
3684 }
Mel Gorman48ee5f32016-05-19 17:14:07 -07003685
Charan Teja Reddyf80b08f2020-08-06 23:25:24 -07003686 if (__zone_watermark_ok(z, order, mark, highest_zoneidx, alloc_flags,
3687 free_pages))
3688 return true;
3689 /*
3690 * Ignore watermark boosting for GFP_ATOMIC order-0 allocations
3691 * when checking the min watermark. The min watermark is the
3692 * point where boosting is ignored so that kswapd is woken up
3693 * when below the low watermark.
3694 */
3695 if (unlikely(!order && (gfp_mask & __GFP_ATOMIC) && z->watermark_boost
3696 && ((alloc_flags & ALLOC_WMARK_MASK) == WMARK_MIN))) {
3697 mark = z->_watermark[WMARK_MIN];
3698 return __zone_watermark_ok(z, order, mark, highest_zoneidx,
3699 alloc_flags, free_pages);
3700 }
3701
3702 return false;
Mel Gorman48ee5f32016-05-19 17:14:07 -07003703}
3704
Mel Gorman7aeb09f2014-06-04 16:10:21 -07003705bool zone_watermark_ok_safe(struct zone *z, unsigned int order,
Joonsoo Kim97a225e2020-06-03 15:59:01 -07003706 unsigned long mark, int highest_zoneidx)
Mel Gorman88f5acf2011-01-13 15:45:41 -08003707{
3708 long free_pages = zone_page_state(z, NR_FREE_PAGES);
3709
3710 if (z->percpu_drift_mark && free_pages < z->percpu_drift_mark)
3711 free_pages = zone_page_state_snapshot(z, NR_FREE_PAGES);
3712
Joonsoo Kim97a225e2020-06-03 15:59:01 -07003713 return __zone_watermark_ok(z, order, mark, highest_zoneidx, 0,
Mel Gorman88f5acf2011-01-13 15:45:41 -08003714 free_pages);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003715}
3716
Paul Jackson9276b1bc2006-12-06 20:31:48 -08003717#ifdef CONFIG_NUMA
David Rientjes957f8222012-10-08 16:33:24 -07003718static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone)
3719{
Gavin Shane02dc012017-02-24 14:59:33 -08003720 return node_distance(zone_to_nid(local_zone), zone_to_nid(zone)) <=
Matt Fleminga55c7452019-08-08 20:53:01 +01003721 node_reclaim_distance;
David Rientjes957f8222012-10-08 16:33:24 -07003722}
Paul Jackson9276b1bc2006-12-06 20:31:48 -08003723#else /* CONFIG_NUMA */
David Rientjes957f8222012-10-08 16:33:24 -07003724static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone)
3725{
3726 return true;
3727}
Paul Jackson9276b1bc2006-12-06 20:31:48 -08003728#endif /* CONFIG_NUMA */
3729
Mel Gorman6bb15452018-12-28 00:35:41 -08003730/*
3731 * The restriction on ZONE_DMA32 as being a suitable zone to use to avoid
3732 * fragmentation is subtle. If the preferred zone was HIGHMEM then
3733 * premature use of a lower zone may cause lowmem pressure problems that
3734 * are worse than fragmentation. If the next zone is ZONE_DMA then it is
3735 * probably too small. It only makes sense to spread allocations to avoid
3736 * fragmentation between the Normal and DMA32 zones.
3737 */
3738static inline unsigned int
Mel Gorman0a79cda2018-12-28 00:35:48 -08003739alloc_flags_nofragment(struct zone *zone, gfp_t gfp_mask)
Mel Gorman6bb15452018-12-28 00:35:41 -08003740{
Mateusz Nosek736838e2020-04-01 21:09:47 -07003741 unsigned int alloc_flags;
Mel Gorman0a79cda2018-12-28 00:35:48 -08003742
Mateusz Nosek736838e2020-04-01 21:09:47 -07003743 /*
3744 * __GFP_KSWAPD_RECLAIM is assumed to be the same as ALLOC_KSWAPD
3745 * to save a branch.
3746 */
3747 alloc_flags = (__force int) (gfp_mask & __GFP_KSWAPD_RECLAIM);
Mel Gorman0a79cda2018-12-28 00:35:48 -08003748
3749#ifdef CONFIG_ZONE_DMA32
Andrey Ryabinin8139ad02019-04-25 22:23:58 -07003750 if (!zone)
3751 return alloc_flags;
3752
Mel Gorman6bb15452018-12-28 00:35:41 -08003753 if (zone_idx(zone) != ZONE_NORMAL)
Andrey Ryabinin8118b822019-04-25 22:24:01 -07003754 return alloc_flags;
Mel Gorman6bb15452018-12-28 00:35:41 -08003755
3756 /*
3757 * If ZONE_DMA32 exists, assume it is the one after ZONE_NORMAL and
3758 * the pointer is within zone->zone_pgdat->node_zones[]. Also assume
3759 * on UMA that if Normal is populated then so is DMA32.
3760 */
3761 BUILD_BUG_ON(ZONE_NORMAL - ZONE_DMA32 != 1);
3762 if (nr_online_nodes > 1 && !populated_zone(--zone))
Andrey Ryabinin8118b822019-04-25 22:24:01 -07003763 return alloc_flags;
Mel Gorman6bb15452018-12-28 00:35:41 -08003764
Andrey Ryabinin8118b822019-04-25 22:24:01 -07003765 alloc_flags |= ALLOC_NOFRAGMENT;
Mel Gorman0a79cda2018-12-28 00:35:48 -08003766#endif /* CONFIG_ZONE_DMA32 */
3767 return alloc_flags;
Mel Gorman6bb15452018-12-28 00:35:41 -08003768}
Mel Gorman6bb15452018-12-28 00:35:41 -08003769
Joonsoo Kim8510e692020-08-06 23:26:04 -07003770static inline unsigned int current_alloc_flags(gfp_t gfp_mask,
3771 unsigned int alloc_flags)
3772{
3773#ifdef CONFIG_CMA
3774 unsigned int pflags = current->flags;
3775
3776 if (!(pflags & PF_MEMALLOC_NOCMA) &&
3777 gfp_migratetype(gfp_mask) == MIGRATE_MOVABLE)
3778 alloc_flags |= ALLOC_CMA;
3779
3780#endif
3781 return alloc_flags;
3782}
3783
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08003784/*
Paul Jackson0798e512006-12-06 20:31:38 -08003785 * get_page_from_freelist goes through the zonelist trying to allocate
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08003786 * a page.
3787 */
3788static struct page *
Vlastimil Babkaa9263752015-02-11 15:25:41 -08003789get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags,
3790 const struct alloc_context *ac)
Martin Hicks753ee722005-06-21 17:14:41 -07003791{
Mel Gorman6bb15452018-12-28 00:35:41 -08003792 struct zoneref *z;
Mel Gorman5117f452009-06-16 15:31:59 -07003793 struct zone *zone;
Mel Gorman3b8c0be2016-07-28 15:46:53 -07003794 struct pglist_data *last_pgdat_dirty_limit = NULL;
Mel Gorman6bb15452018-12-28 00:35:41 -08003795 bool no_fallback;
Mel Gorman3b8c0be2016-07-28 15:46:53 -07003796
Mel Gorman6bb15452018-12-28 00:35:41 -08003797retry:
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08003798 /*
Paul Jackson9276b1bc2006-12-06 20:31:48 -08003799 * Scan zonelist, looking for a zone with enough free.
Vladimir Davydov344736f2014-10-20 15:50:30 +04003800 * See also __cpuset_node_allowed() comment in kernel/cpuset.c.
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08003801 */
Mel Gorman6bb15452018-12-28 00:35:41 -08003802 no_fallback = alloc_flags & ALLOC_NOFRAGMENT;
3803 z = ac->preferred_zoneref;
Mateusz Nosek30d8ec72020-10-13 16:55:57 -07003804 for_next_zone_zonelist_nodemask(zone, z, ac->highest_zoneidx,
3805 ac->nodemask) {
Mel Gormanbe06af02016-05-19 17:13:47 -07003806 struct page *page;
Johannes Weinere085dbc2013-09-11 14:20:46 -07003807 unsigned long mark;
3808
Mel Gorman664eedd2014-06-04 16:10:08 -07003809 if (cpusets_enabled() &&
3810 (alloc_flags & ALLOC_CPUSET) &&
Vlastimil Babka002f2902016-05-19 17:14:30 -07003811 !__cpuset_zone_allowed(zone, gfp_mask))
Mel Gormancd38b112011-07-25 17:12:29 -07003812 continue;
Johannes Weinera756cf52012-01-10 15:07:49 -08003813 /*
3814 * When allocating a page cache page for writing, we
Mel Gorman281e3722016-07-28 15:46:11 -07003815 * want to get it from a node that is within its dirty
3816 * limit, such that no single node holds more than its
Johannes Weinera756cf52012-01-10 15:07:49 -08003817 * proportional share of globally allowed dirty pages.
Mel Gorman281e3722016-07-28 15:46:11 -07003818 * The dirty limits take into account the node's
Johannes Weinera756cf52012-01-10 15:07:49 -08003819 * lowmem reserves and high watermark so that kswapd
3820 * should be able to balance it without having to
3821 * write pages from its LRU list.
3822 *
Johannes Weinera756cf52012-01-10 15:07:49 -08003823 * XXX: For now, allow allocations to potentially
Mel Gorman281e3722016-07-28 15:46:11 -07003824 * exceed the per-node dirty limit in the slowpath
Mel Gormanc9ab0c42015-11-06 16:28:12 -08003825 * (spread_dirty_pages unset) before going into reclaim,
Johannes Weinera756cf52012-01-10 15:07:49 -08003826 * which is important when on a NUMA setup the allowed
Mel Gorman281e3722016-07-28 15:46:11 -07003827 * nodes are together not big enough to reach the
Johannes Weinera756cf52012-01-10 15:07:49 -08003828 * global limit. The proper fix for these situations
Mel Gorman281e3722016-07-28 15:46:11 -07003829 * will require awareness of nodes in the
Johannes Weinera756cf52012-01-10 15:07:49 -08003830 * dirty-throttling and the flusher threads.
3831 */
Mel Gorman3b8c0be2016-07-28 15:46:53 -07003832 if (ac->spread_dirty_pages) {
3833 if (last_pgdat_dirty_limit == zone->zone_pgdat)
3834 continue;
3835
3836 if (!node_dirty_ok(zone->zone_pgdat)) {
3837 last_pgdat_dirty_limit = zone->zone_pgdat;
3838 continue;
3839 }
3840 }
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08003841
Mel Gorman6bb15452018-12-28 00:35:41 -08003842 if (no_fallback && nr_online_nodes > 1 &&
3843 zone != ac->preferred_zoneref->zone) {
3844 int local_nid;
3845
3846 /*
3847 * If moving to a remote node, retry but allow
3848 * fragmenting fallbacks. Locality is more important
3849 * than fragmentation avoidance.
3850 */
3851 local_nid = zone_to_nid(ac->preferred_zoneref->zone);
3852 if (zone_to_nid(zone) != local_nid) {
3853 alloc_flags &= ~ALLOC_NOFRAGMENT;
3854 goto retry;
3855 }
3856 }
3857
Mel Gormana9214442018-12-28 00:35:44 -08003858 mark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK);
Mel Gorman48ee5f32016-05-19 17:14:07 -07003859 if (!zone_watermark_fast(zone, order, mark,
Charan Teja Reddyf80b08f2020-08-06 23:25:24 -07003860 ac->highest_zoneidx, alloc_flags,
3861 gfp_mask)) {
Mel Gormanfa5e0842009-06-16 15:33:22 -07003862 int ret;
3863
Pavel Tatashinc9e97a12018-04-05 16:22:31 -07003864#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
3865 /*
3866 * Watermark failed for this zone, but see if we can
3867 * grow this zone if it contains deferred pages.
3868 */
3869 if (static_branch_unlikely(&deferred_pages)) {
3870 if (_deferred_grow_zone(zone, order))
3871 goto try_this_zone;
3872 }
3873#endif
Mel Gorman5dab2912014-06-04 16:10:14 -07003874 /* Checked here to keep the fast path fast */
3875 BUILD_BUG_ON(ALLOC_NO_WATERMARKS < NR_WMARK);
3876 if (alloc_flags & ALLOC_NO_WATERMARKS)
3877 goto try_this_zone;
3878
Mel Gormana5f5f912016-07-28 15:46:32 -07003879 if (node_reclaim_mode == 0 ||
Mel Gormanc33d6c02016-05-19 17:14:10 -07003880 !zone_allows_reclaim(ac->preferred_zoneref->zone, zone))
Mel Gormancd38b112011-07-25 17:12:29 -07003881 continue;
3882
Mel Gormana5f5f912016-07-28 15:46:32 -07003883 ret = node_reclaim(zone->zone_pgdat, gfp_mask, order);
Mel Gormanfa5e0842009-06-16 15:33:22 -07003884 switch (ret) {
Mel Gormana5f5f912016-07-28 15:46:32 -07003885 case NODE_RECLAIM_NOSCAN:
Mel Gormanfa5e0842009-06-16 15:33:22 -07003886 /* did not scan */
Mel Gormancd38b112011-07-25 17:12:29 -07003887 continue;
Mel Gormana5f5f912016-07-28 15:46:32 -07003888 case NODE_RECLAIM_FULL:
Mel Gormanfa5e0842009-06-16 15:33:22 -07003889 /* scanned but unreclaimable */
Mel Gormancd38b112011-07-25 17:12:29 -07003890 continue;
Mel Gormanfa5e0842009-06-16 15:33:22 -07003891 default:
3892 /* did we reclaim enough */
Mel Gormanfed27192013-04-29 15:07:57 -07003893 if (zone_watermark_ok(zone, order, mark,
Joonsoo Kim97a225e2020-06-03 15:59:01 -07003894 ac->highest_zoneidx, alloc_flags))
Mel Gormanfed27192013-04-29 15:07:57 -07003895 goto try_this_zone;
3896
Mel Gormanfed27192013-04-29 15:07:57 -07003897 continue;
Paul Jackson0798e512006-12-06 20:31:38 -08003898 }
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08003899 }
3900
Mel Gormanfa5e0842009-06-16 15:33:22 -07003901try_this_zone:
Mel Gorman066b2392017-02-24 14:56:26 -08003902 page = rmqueue(ac->preferred_zoneref->zone, zone, order,
Mel Gorman0aaa29a2015-11-06 16:28:37 -08003903 gfp_mask, alloc_flags, ac->migratetype);
Vlastimil Babka75379192015-02-11 15:25:38 -08003904 if (page) {
Mel Gorman479f8542016-05-19 17:14:35 -07003905 prep_new_page(page, order, gfp_mask, alloc_flags);
Mel Gorman0aaa29a2015-11-06 16:28:37 -08003906
3907 /*
3908 * If this is a high-order atomic allocation then check
3909 * if the pageblock should be reserved for the future
3910 */
3911 if (unlikely(order && (alloc_flags & ALLOC_HARDER)))
3912 reserve_highatomic_pageblock(page, zone, order);
3913
Vlastimil Babka75379192015-02-11 15:25:38 -08003914 return page;
Pavel Tatashinc9e97a12018-04-05 16:22:31 -07003915 } else {
3916#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
3917 /* Try again if zone has deferred pages */
3918 if (static_branch_unlikely(&deferred_pages)) {
3919 if (_deferred_grow_zone(zone, order))
3920 goto try_this_zone;
3921 }
3922#endif
Vlastimil Babka75379192015-02-11 15:25:38 -08003923 }
Mel Gorman54a6eb52008-04-28 02:12:16 -07003924 }
Paul Jackson9276b1bc2006-12-06 20:31:48 -08003925
Mel Gorman6bb15452018-12-28 00:35:41 -08003926 /*
3927 * It's possible on a UMA machine to get through all zones that are
3928 * fragmented. If avoiding fragmentation, reset and try again.
3929 */
3930 if (no_fallback) {
3931 alloc_flags &= ~ALLOC_NOFRAGMENT;
3932 goto retry;
3933 }
3934
Mel Gorman4ffeaf32014-08-06 16:07:22 -07003935 return NULL;
Martin Hicks753ee722005-06-21 17:14:41 -07003936}
3937
Michal Hocko9af744d2017-02-22 15:46:16 -08003938static void warn_alloc_show_mem(gfp_t gfp_mask, nodemask_t *nodemask)
Dave Hansena238ab52011-05-24 17:12:16 -07003939{
Dave Hansena238ab52011-05-24 17:12:16 -07003940 unsigned int filter = SHOW_MEM_FILTER_NODES;
Dave Hansena238ab52011-05-24 17:12:16 -07003941
3942 /*
3943 * This documents exceptions given to allocations in certain
3944 * contexts that are allowed to allocate outside current's set
3945 * of allowed nodes.
3946 */
3947 if (!(gfp_mask & __GFP_NOMEMALLOC))
Michal Hockocd04ae12017-09-06 16:24:50 -07003948 if (tsk_is_oom_victim(current) ||
Dave Hansena238ab52011-05-24 17:12:16 -07003949 (current->flags & (PF_MEMALLOC | PF_EXITING)))
3950 filter &= ~SHOW_MEM_FILTER_NODES;
Mel Gormand0164ad2015-11-06 16:28:21 -08003951 if (in_interrupt() || !(gfp_mask & __GFP_DIRECT_RECLAIM))
Dave Hansena238ab52011-05-24 17:12:16 -07003952 filter &= ~SHOW_MEM_FILTER_NODES;
3953
Michal Hocko9af744d2017-02-22 15:46:16 -08003954 show_mem(filter, nodemask);
Michal Hockoaa187502017-02-22 15:41:45 -08003955}
3956
Michal Hockoa8e99252017-02-22 15:46:10 -08003957void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...)
Michal Hockoaa187502017-02-22 15:41:45 -08003958{
3959 struct va_format vaf;
3960 va_list args;
Johannes Weiner1be334e2019-11-05 21:16:51 -08003961 static DEFINE_RATELIMIT_STATE(nopage_rs, 10*HZ, 1);
Michal Hockoaa187502017-02-22 15:41:45 -08003962
Tetsuo Handa0f7896f2017-05-03 14:55:34 -07003963 if ((gfp_mask & __GFP_NOWARN) || !__ratelimit(&nopage_rs))
Michal Hockoaa187502017-02-22 15:41:45 -08003964 return;
3965
Michal Hocko7877cdc2016-10-07 17:01:55 -07003966 va_start(args, fmt);
3967 vaf.fmt = fmt;
3968 vaf.va = &args;
yuzhoujianef8444e2018-12-28 00:36:07 -08003969 pr_warn("%s: %pV, mode:%#x(%pGg), nodemask=%*pbl",
Michal Hocko0205f752017-11-15 17:39:14 -08003970 current->comm, &vaf, gfp_mask, &gfp_mask,
3971 nodemask_pr_args(nodemask));
Michal Hocko7877cdc2016-10-07 17:01:55 -07003972 va_end(args);
Joe Perches3ee9a4f2011-10-31 17:08:35 -07003973
Michal Hockoa8e99252017-02-22 15:46:10 -08003974 cpuset_print_current_mems_allowed();
yuzhoujianef8444e2018-12-28 00:36:07 -08003975 pr_cont("\n");
Dave Hansena238ab52011-05-24 17:12:16 -07003976 dump_stack();
David Rientjes685dbf62017-02-22 15:46:28 -08003977 warn_alloc_show_mem(gfp_mask, nodemask);
Dave Hansena238ab52011-05-24 17:12:16 -07003978}
3979
Mel Gorman11e33f62009-06-16 15:31:57 -07003980static inline struct page *
Michal Hocko6c18ba72017-02-22 15:46:25 -08003981__alloc_pages_cpuset_fallback(gfp_t gfp_mask, unsigned int order,
3982 unsigned int alloc_flags,
3983 const struct alloc_context *ac)
3984{
3985 struct page *page;
3986
3987 page = get_page_from_freelist(gfp_mask, order,
3988 alloc_flags|ALLOC_CPUSET, ac);
3989 /*
3990 * fallback to ignore cpuset restriction if our nodes
3991 * are depleted
3992 */
3993 if (!page)
3994 page = get_page_from_freelist(gfp_mask, order,
3995 alloc_flags, ac);
3996
3997 return page;
3998}
3999
4000static inline struct page *
Mel Gorman11e33f62009-06-16 15:31:57 -07004001__alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order,
Vlastimil Babkaa9263752015-02-11 15:25:41 -08004002 const struct alloc_context *ac, unsigned long *did_some_progress)
Mel Gorman11e33f62009-06-16 15:31:57 -07004003{
David Rientjes6e0fc462015-09-08 15:00:36 -07004004 struct oom_control oc = {
4005 .zonelist = ac->zonelist,
4006 .nodemask = ac->nodemask,
Vladimir Davydov2a966b72016-07-26 15:22:33 -07004007 .memcg = NULL,
David Rientjes6e0fc462015-09-08 15:00:36 -07004008 .gfp_mask = gfp_mask,
4009 .order = order,
David Rientjes6e0fc462015-09-08 15:00:36 -07004010 };
Linus Torvalds1da177e2005-04-16 15:20:36 -07004011 struct page *page;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004012
Johannes Weiner9879de72015-01-26 12:58:32 -08004013 *did_some_progress = 0;
4014
Johannes Weiner9879de72015-01-26 12:58:32 -08004015 /*
Johannes Weinerdc564012015-06-24 16:57:19 -07004016 * Acquire the oom lock. If that fails, somebody else is
4017 * making progress for us.
Johannes Weiner9879de72015-01-26 12:58:32 -08004018 */
Johannes Weinerdc564012015-06-24 16:57:19 -07004019 if (!mutex_trylock(&oom_lock)) {
Johannes Weiner9879de72015-01-26 12:58:32 -08004020 *did_some_progress = 1;
Mel Gorman11e33f62009-06-16 15:31:57 -07004021 schedule_timeout_uninterruptible(1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004022 return NULL;
4023 }
Jens Axboe6b1de912005-11-17 21:35:02 +01004024
Mel Gorman11e33f62009-06-16 15:31:57 -07004025 /*
4026 * Go through the zonelist yet one more time, keep very high watermark
4027 * here, this is only to catch a parallel oom killing, we must fail if
Tetsuo Handae746bf72017-08-31 16:15:20 -07004028 * we're still under heavy pressure. But make sure that this reclaim
4029 * attempt shall not depend on __GFP_DIRECT_RECLAIM && !__GFP_NORETRY
4030 * allocation which will never fail due to oom_lock already held.
Mel Gorman11e33f62009-06-16 15:31:57 -07004031 */
Tetsuo Handae746bf72017-08-31 16:15:20 -07004032 page = get_page_from_freelist((gfp_mask | __GFP_HARDWALL) &
4033 ~__GFP_DIRECT_RECLAIM, order,
4034 ALLOC_WMARK_HIGH|ALLOC_CPUSET, ac);
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08004035 if (page)
Mel Gorman11e33f62009-06-16 15:31:57 -07004036 goto out;
4037
Michal Hocko06ad2762017-02-22 15:46:22 -08004038 /* Coredumps can quickly deplete all memory reserves */
4039 if (current->flags & PF_DUMPCORE)
4040 goto out;
4041 /* The OOM killer will not help higher order allocs */
4042 if (order > PAGE_ALLOC_COSTLY_ORDER)
4043 goto out;
Michal Hockodcda9b02017-07-12 14:36:45 -07004044 /*
4045 * We have already exhausted all our reclaim opportunities without any
4046 * success so it is time to admit defeat. We will skip the OOM killer
4047 * because it is very likely that the caller has a more reasonable
4048 * fallback than shooting a random task.
Mateusz Nosekcfb4a542020-10-13 16:55:45 -07004049 *
4050 * The OOM killer may not free memory on a specific node.
Michal Hockodcda9b02017-07-12 14:36:45 -07004051 */
Mateusz Nosekcfb4a542020-10-13 16:55:45 -07004052 if (gfp_mask & (__GFP_RETRY_MAYFAIL | __GFP_THISNODE))
Michal Hockodcda9b02017-07-12 14:36:45 -07004053 goto out;
Michal Hocko06ad2762017-02-22 15:46:22 -08004054 /* The OOM killer does not needlessly kill tasks for lowmem */
Joonsoo Kim97a225e2020-06-03 15:59:01 -07004055 if (ac->highest_zoneidx < ZONE_NORMAL)
Michal Hocko06ad2762017-02-22 15:46:22 -08004056 goto out;
4057 if (pm_suspended_storage())
4058 goto out;
4059 /*
4060 * XXX: GFP_NOFS allocations should rather fail than rely on
4061 * other request to make a forward progress.
4062 * We are in an unfortunate situation where out_of_memory cannot
4063 * do much for this context but let's try it to at least get
4064 * access to memory reserved if the current task is killed (see
4065 * out_of_memory). Once filesystems are ready to handle allocation
4066 * failures more gracefully we should just bail out here.
4067 */
Michal Hocko3da88fb32016-05-19 17:13:09 -07004068
Shile Zhang3c2c6482018-01-31 16:20:07 -08004069 /* Exhausted what can be done so it's blame time */
Michal Hocko5020e282016-01-14 15:20:36 -08004070 if (out_of_memory(&oc) || WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL)) {
Michal Hockoc32b3cb2015-02-11 15:26:24 -08004071 *did_some_progress = 1;
Michal Hocko5020e282016-01-14 15:20:36 -08004072
Michal Hocko6c18ba72017-02-22 15:46:25 -08004073 /*
4074 * Help non-failing allocations by giving them access to memory
4075 * reserves
4076 */
4077 if (gfp_mask & __GFP_NOFAIL)
4078 page = __alloc_pages_cpuset_fallback(gfp_mask, order,
Michal Hocko5020e282016-01-14 15:20:36 -08004079 ALLOC_NO_WATERMARKS, ac);
Michal Hocko5020e282016-01-14 15:20:36 -08004080 }
Mel Gorman11e33f62009-06-16 15:31:57 -07004081out:
Johannes Weinerdc564012015-06-24 16:57:19 -07004082 mutex_unlock(&oom_lock);
Mel Gorman11e33f62009-06-16 15:31:57 -07004083 return page;
4084}
4085
Michal Hocko33c2d212016-05-20 16:57:06 -07004086/*
4087 * Maximum number of compaction retries wit a progress before OOM
4088 * killer is consider as the only way to move forward.
4089 */
4090#define MAX_COMPACT_RETRIES 16
4091
Mel Gorman56de7262010-05-24 14:32:30 -07004092#ifdef CONFIG_COMPACTION
4093/* Try memory compaction for high-order allocations before reclaim */
4094static struct page *
4095__alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
Mel Gormanc6038442016-05-19 17:13:38 -07004096 unsigned int alloc_flags, const struct alloc_context *ac,
Vlastimil Babkaa5508cd2016-07-28 15:49:28 -07004097 enum compact_priority prio, enum compact_result *compact_result)
Mel Gorman56de7262010-05-24 14:32:30 -07004098{
Mel Gorman5e1f0f02019-03-05 15:45:41 -08004099 struct page *page = NULL;
Johannes Weinereb414682018-10-26 15:06:27 -07004100 unsigned long pflags;
Vlastimil Babka499118e2017-05-08 15:59:50 -07004101 unsigned int noreclaim_flag;
Vlastimil Babka53853e22014-10-09 15:27:02 -07004102
Mel Gorman66199712012-01-12 17:19:41 -08004103 if (!order)
Mel Gorman56de7262010-05-24 14:32:30 -07004104 return NULL;
4105
Johannes Weinereb414682018-10-26 15:06:27 -07004106 psi_memstall_enter(&pflags);
Vlastimil Babka499118e2017-05-08 15:59:50 -07004107 noreclaim_flag = memalloc_noreclaim_save();
Johannes Weinereb414682018-10-26 15:06:27 -07004108
Michal Hockoc5d01d02016-05-20 16:56:53 -07004109 *compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac,
Mel Gorman5e1f0f02019-03-05 15:45:41 -08004110 prio, &page);
Johannes Weinereb414682018-10-26 15:06:27 -07004111
Vlastimil Babka499118e2017-05-08 15:59:50 -07004112 memalloc_noreclaim_restore(noreclaim_flag);
Johannes Weinereb414682018-10-26 15:06:27 -07004113 psi_memstall_leave(&pflags);
Mel Gorman56de7262010-05-24 14:32:30 -07004114
Vlastimil Babka98dd3b42014-10-09 15:27:04 -07004115 /*
4116 * At least in one zone compaction wasn't deferred or skipped, so let's
4117 * count a compaction stall
4118 */
4119 count_vm_event(COMPACTSTALL);
4120
Mel Gorman5e1f0f02019-03-05 15:45:41 -08004121 /* Prep a captured page if available */
4122 if (page)
4123 prep_new_page(page, order, gfp_mask, alloc_flags);
4124
4125 /* Try get a page from the freelist if available */
4126 if (!page)
4127 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
Vlastimil Babka98dd3b42014-10-09 15:27:04 -07004128
4129 if (page) {
4130 struct zone *zone = page_zone(page);
4131
4132 zone->compact_blockskip_flush = false;
4133 compaction_defer_reset(zone, order, true);
4134 count_vm_event(COMPACTSUCCESS);
4135 return page;
4136 }
4137
4138 /*
Vlastimil Babka98dd3b42014-10-09 15:27:04 -07004139 * It's bad if compaction run occurs and fails. The most likely reason
4140 * is that pages exist, but not enough to satisfy watermarks.
4141 */
4142 count_vm_event(COMPACTFAIL);
4143
4144 cond_resched();
4145
Mel Gorman56de7262010-05-24 14:32:30 -07004146 return NULL;
4147}
Michal Hocko33c2d212016-05-20 16:57:06 -07004148
Vlastimil Babka32508452016-10-07 17:00:28 -07004149static inline bool
4150should_compact_retry(struct alloc_context *ac, int order, int alloc_flags,
4151 enum compact_result compact_result,
4152 enum compact_priority *compact_priority,
Vlastimil Babkad9436492016-10-07 17:00:31 -07004153 int *compaction_retries)
Vlastimil Babka32508452016-10-07 17:00:28 -07004154{
4155 int max_retries = MAX_COMPACT_RETRIES;
Vlastimil Babkac2033b02016-10-07 17:00:34 -07004156 int min_priority;
Michal Hocko65190cf2017-02-22 15:42:03 -08004157 bool ret = false;
4158 int retries = *compaction_retries;
4159 enum compact_priority priority = *compact_priority;
Vlastimil Babka32508452016-10-07 17:00:28 -07004160
4161 if (!order)
4162 return false;
4163
Vlastimil Babkad9436492016-10-07 17:00:31 -07004164 if (compaction_made_progress(compact_result))
4165 (*compaction_retries)++;
4166
Vlastimil Babka32508452016-10-07 17:00:28 -07004167 /*
4168 * compaction considers all the zone as desperately out of memory
4169 * so it doesn't really make much sense to retry except when the
4170 * failure could be caused by insufficient priority
4171 */
Vlastimil Babkad9436492016-10-07 17:00:31 -07004172 if (compaction_failed(compact_result))
4173 goto check_priority;
Vlastimil Babka32508452016-10-07 17:00:28 -07004174
4175 /*
Vlastimil Babka494330852019-09-23 15:37:32 -07004176 * compaction was skipped because there are not enough order-0 pages
4177 * to work with, so we retry only if it looks like reclaim can help.
Vlastimil Babka32508452016-10-07 17:00:28 -07004178 */
Vlastimil Babka494330852019-09-23 15:37:32 -07004179 if (compaction_needs_reclaim(compact_result)) {
Michal Hocko65190cf2017-02-22 15:42:03 -08004180 ret = compaction_zonelist_suitable(ac, order, alloc_flags);
4181 goto out;
4182 }
Vlastimil Babka32508452016-10-07 17:00:28 -07004183
4184 /*
Vlastimil Babka494330852019-09-23 15:37:32 -07004185 * make sure the compaction wasn't deferred or didn't bail out early
4186 * due to locks contention before we declare that we should give up.
4187 * But the next retry should use a higher priority if allowed, so
4188 * we don't just keep bailing out endlessly.
4189 */
4190 if (compaction_withdrawn(compact_result)) {
4191 goto check_priority;
4192 }
4193
4194 /*
Michal Hockodcda9b02017-07-12 14:36:45 -07004195 * !costly requests are much more important than __GFP_RETRY_MAYFAIL
Vlastimil Babka32508452016-10-07 17:00:28 -07004196 * costly ones because they are de facto nofail and invoke OOM
4197 * killer to move on while costly can fail and users are ready
4198 * to cope with that. 1/4 retries is rather arbitrary but we
4199 * would need much more detailed feedback from compaction to
4200 * make a better decision.
4201 */
4202 if (order > PAGE_ALLOC_COSTLY_ORDER)
4203 max_retries /= 4;
Michal Hocko65190cf2017-02-22 15:42:03 -08004204 if (*compaction_retries <= max_retries) {
4205 ret = true;
4206 goto out;
4207 }
Vlastimil Babka32508452016-10-07 17:00:28 -07004208
Vlastimil Babkad9436492016-10-07 17:00:31 -07004209 /*
4210 * Make sure there are attempts at the highest priority if we exhausted
4211 * all retries or failed at the lower priorities.
4212 */
4213check_priority:
Vlastimil Babkac2033b02016-10-07 17:00:34 -07004214 min_priority = (order > PAGE_ALLOC_COSTLY_ORDER) ?
4215 MIN_COMPACT_COSTLY_PRIORITY : MIN_COMPACT_PRIORITY;
Michal Hocko65190cf2017-02-22 15:42:03 -08004216
Vlastimil Babkac2033b02016-10-07 17:00:34 -07004217 if (*compact_priority > min_priority) {
Vlastimil Babkad9436492016-10-07 17:00:31 -07004218 (*compact_priority)--;
4219 *compaction_retries = 0;
Michal Hocko65190cf2017-02-22 15:42:03 -08004220 ret = true;
Vlastimil Babkad9436492016-10-07 17:00:31 -07004221 }
Michal Hocko65190cf2017-02-22 15:42:03 -08004222out:
4223 trace_compact_retry(order, priority, compact_result, retries, max_retries, ret);
4224 return ret;
Vlastimil Babka32508452016-10-07 17:00:28 -07004225}
Mel Gorman56de7262010-05-24 14:32:30 -07004226#else
4227static inline struct page *
4228__alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
Mel Gormanc6038442016-05-19 17:13:38 -07004229 unsigned int alloc_flags, const struct alloc_context *ac,
Vlastimil Babkaa5508cd2016-07-28 15:49:28 -07004230 enum compact_priority prio, enum compact_result *compact_result)
Mel Gorman56de7262010-05-24 14:32:30 -07004231{
Michal Hocko33c2d212016-05-20 16:57:06 -07004232 *compact_result = COMPACT_SKIPPED;
Mel Gorman56de7262010-05-24 14:32:30 -07004233 return NULL;
4234}
Michal Hocko33c2d212016-05-20 16:57:06 -07004235
4236static inline bool
Michal Hocko86a294a2016-05-20 16:57:12 -07004237should_compact_retry(struct alloc_context *ac, unsigned int order, int alloc_flags,
4238 enum compact_result compact_result,
Vlastimil Babkaa5508cd2016-07-28 15:49:28 -07004239 enum compact_priority *compact_priority,
Vlastimil Babkad9436492016-10-07 17:00:31 -07004240 int *compaction_retries)
Michal Hocko33c2d212016-05-20 16:57:06 -07004241{
Michal Hocko31e49bf2016-05-20 16:57:15 -07004242 struct zone *zone;
4243 struct zoneref *z;
4244
4245 if (!order || order > PAGE_ALLOC_COSTLY_ORDER)
4246 return false;
4247
4248 /*
4249 * There are setups with compaction disabled which would prefer to loop
4250 * inside the allocator rather than hit the oom killer prematurely.
4251 * Let's give them a good hope and keep retrying while the order-0
4252 * watermarks are OK.
4253 */
Joonsoo Kim97a225e2020-06-03 15:59:01 -07004254 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist,
4255 ac->highest_zoneidx, ac->nodemask) {
Michal Hocko31e49bf2016-05-20 16:57:15 -07004256 if (zone_watermark_ok(zone, 0, min_wmark_pages(zone),
Joonsoo Kim97a225e2020-06-03 15:59:01 -07004257 ac->highest_zoneidx, alloc_flags))
Michal Hocko31e49bf2016-05-20 16:57:15 -07004258 return true;
4259 }
Michal Hocko33c2d212016-05-20 16:57:06 -07004260 return false;
4261}
Vlastimil Babka32508452016-10-07 17:00:28 -07004262#endif /* CONFIG_COMPACTION */
Mel Gorman56de7262010-05-24 14:32:30 -07004263
Peter Zijlstrad92a8cf2017-03-03 10:13:38 +01004264#ifdef CONFIG_LOCKDEP
Omar Sandoval93781322018-06-07 17:07:02 -07004265static struct lockdep_map __fs_reclaim_map =
Peter Zijlstrad92a8cf2017-03-03 10:13:38 +01004266 STATIC_LOCKDEP_MAP_INIT("fs_reclaim", &__fs_reclaim_map);
4267
4268static bool __need_fs_reclaim(gfp_t gfp_mask)
4269{
4270 gfp_mask = current_gfp_context(gfp_mask);
4271
4272 /* no reclaim without waiting on it */
4273 if (!(gfp_mask & __GFP_DIRECT_RECLAIM))
4274 return false;
4275
4276 /* this guy won't enter reclaim */
Tetsuo Handa2e517d682018-03-22 16:17:10 -07004277 if (current->flags & PF_MEMALLOC)
Peter Zijlstrad92a8cf2017-03-03 10:13:38 +01004278 return false;
4279
4280 /* We're only interested __GFP_FS allocations for now */
4281 if (!(gfp_mask & __GFP_FS))
4282 return false;
4283
4284 if (gfp_mask & __GFP_NOLOCKDEP)
4285 return false;
4286
4287 return true;
4288}
4289
Omar Sandoval93781322018-06-07 17:07:02 -07004290void __fs_reclaim_acquire(void)
4291{
4292 lock_map_acquire(&__fs_reclaim_map);
4293}
4294
4295void __fs_reclaim_release(void)
4296{
4297 lock_map_release(&__fs_reclaim_map);
4298}
4299
Peter Zijlstrad92a8cf2017-03-03 10:13:38 +01004300void fs_reclaim_acquire(gfp_t gfp_mask)
4301{
4302 if (__need_fs_reclaim(gfp_mask))
Omar Sandoval93781322018-06-07 17:07:02 -07004303 __fs_reclaim_acquire();
Peter Zijlstrad92a8cf2017-03-03 10:13:38 +01004304}
4305EXPORT_SYMBOL_GPL(fs_reclaim_acquire);
4306
4307void fs_reclaim_release(gfp_t gfp_mask)
4308{
4309 if (__need_fs_reclaim(gfp_mask))
Omar Sandoval93781322018-06-07 17:07:02 -07004310 __fs_reclaim_release();
Peter Zijlstrad92a8cf2017-03-03 10:13:38 +01004311}
4312EXPORT_SYMBOL_GPL(fs_reclaim_release);
4313#endif
4314
Marek Szyprowskibba90712012-01-25 12:09:52 +01004315/* Perform direct synchronous page reclaim */
Yanfei Xu2187e172020-10-13 16:55:54 -07004316static unsigned long
Vlastimil Babkaa9263752015-02-11 15:25:41 -08004317__perform_reclaim(gfp_t gfp_mask, unsigned int order,
4318 const struct alloc_context *ac)
Mel Gorman11e33f62009-06-16 15:31:57 -07004319{
Vlastimil Babka499118e2017-05-08 15:59:50 -07004320 unsigned int noreclaim_flag;
Yanfei Xu2187e172020-10-13 16:55:54 -07004321 unsigned long pflags, progress;
Mel Gorman11e33f62009-06-16 15:31:57 -07004322
4323 cond_resched();
4324
4325 /* We now go into synchronous reclaim */
4326 cpuset_memory_pressure_bump();
Johannes Weinereb414682018-10-26 15:06:27 -07004327 psi_memstall_enter(&pflags);
Peter Zijlstrad92a8cf2017-03-03 10:13:38 +01004328 fs_reclaim_acquire(gfp_mask);
Omar Sandoval93781322018-06-07 17:07:02 -07004329 noreclaim_flag = memalloc_noreclaim_save();
Mel Gorman11e33f62009-06-16 15:31:57 -07004330
Vlastimil Babkaa9263752015-02-11 15:25:41 -08004331 progress = try_to_free_pages(ac->zonelist, order, gfp_mask,
4332 ac->nodemask);
Mel Gorman11e33f62009-06-16 15:31:57 -07004333
Vlastimil Babka499118e2017-05-08 15:59:50 -07004334 memalloc_noreclaim_restore(noreclaim_flag);
Omar Sandoval93781322018-06-07 17:07:02 -07004335 fs_reclaim_release(gfp_mask);
Johannes Weinereb414682018-10-26 15:06:27 -07004336 psi_memstall_leave(&pflags);
Mel Gorman11e33f62009-06-16 15:31:57 -07004337
4338 cond_resched();
4339
Marek Szyprowskibba90712012-01-25 12:09:52 +01004340 return progress;
4341}
4342
4343/* The really slow allocator path where we enter direct reclaim */
4344static inline struct page *
4345__alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order,
Mel Gormanc6038442016-05-19 17:13:38 -07004346 unsigned int alloc_flags, const struct alloc_context *ac,
Vlastimil Babkaa9263752015-02-11 15:25:41 -08004347 unsigned long *did_some_progress)
Marek Szyprowskibba90712012-01-25 12:09:52 +01004348{
4349 struct page *page = NULL;
4350 bool drained = false;
4351
Vlastimil Babkaa9263752015-02-11 15:25:41 -08004352 *did_some_progress = __perform_reclaim(gfp_mask, order, ac);
Mel Gorman9ee493c2010-09-09 16:38:18 -07004353 if (unlikely(!(*did_some_progress)))
4354 return NULL;
Mel Gorman11e33f62009-06-16 15:31:57 -07004355
Mel Gorman9ee493c2010-09-09 16:38:18 -07004356retry:
Vlastimil Babka31a6c192016-07-28 15:49:13 -07004357 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
Mel Gorman9ee493c2010-09-09 16:38:18 -07004358
4359 /*
4360 * If an allocation failed after direct reclaim, it could be because
Mel Gorman0aaa29a2015-11-06 16:28:37 -08004361 * pages are pinned on the per-cpu lists or in high alloc reserves.
Randy Dunlap047b9962020-08-11 18:33:14 -07004362 * Shrink them and try again
Mel Gorman9ee493c2010-09-09 16:38:18 -07004363 */
4364 if (!page && !drained) {
Minchan Kim29fac032016-12-12 16:42:14 -08004365 unreserve_highatomic_pageblock(ac, false);
Vlastimil Babka93481ff2014-12-10 15:43:01 -08004366 drain_all_pages(NULL);
Mel Gorman9ee493c2010-09-09 16:38:18 -07004367 drained = true;
4368 goto retry;
4369 }
4370
Mel Gorman11e33f62009-06-16 15:31:57 -07004371 return page;
4372}
4373
David Rientjes5ecd9d42018-04-05 16:25:16 -07004374static void wake_all_kswapds(unsigned int order, gfp_t gfp_mask,
4375 const struct alloc_context *ac)
Mel Gorman11e33f62009-06-16 15:31:57 -07004376{
4377 struct zoneref *z;
4378 struct zone *zone;
Mel Gormane1a55632016-07-28 15:46:26 -07004379 pg_data_t *last_pgdat = NULL;
Joonsoo Kim97a225e2020-06-03 15:59:01 -07004380 enum zone_type highest_zoneidx = ac->highest_zoneidx;
Mel Gorman11e33f62009-06-16 15:31:57 -07004381
Joonsoo Kim97a225e2020-06-03 15:59:01 -07004382 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, highest_zoneidx,
David Rientjes5ecd9d42018-04-05 16:25:16 -07004383 ac->nodemask) {
Mel Gormane1a55632016-07-28 15:46:26 -07004384 if (last_pgdat != zone->zone_pgdat)
Joonsoo Kim97a225e2020-06-03 15:59:01 -07004385 wakeup_kswapd(zone, gfp_mask, order, highest_zoneidx);
Mel Gormane1a55632016-07-28 15:46:26 -07004386 last_pgdat = zone->zone_pgdat;
4387 }
Mel Gorman11e33f62009-06-16 15:31:57 -07004388}
4389
Mel Gormanc6038442016-05-19 17:13:38 -07004390static inline unsigned int
Peter Zijlstra341ce062009-06-16 15:32:02 -07004391gfp_to_alloc_flags(gfp_t gfp_mask)
4392{
Mel Gormanc6038442016-05-19 17:13:38 -07004393 unsigned int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET;
Peter Zijlstra341ce062009-06-16 15:32:02 -07004394
Mateusz Nosek736838e2020-04-01 21:09:47 -07004395 /*
4396 * __GFP_HIGH is assumed to be the same as ALLOC_HIGH
4397 * and __GFP_KSWAPD_RECLAIM is assumed to be the same as ALLOC_KSWAPD
4398 * to save two branches.
4399 */
Namhyung Kime6223a32010-10-26 14:21:59 -07004400 BUILD_BUG_ON(__GFP_HIGH != (__force gfp_t) ALLOC_HIGH);
Mateusz Nosek736838e2020-04-01 21:09:47 -07004401 BUILD_BUG_ON(__GFP_KSWAPD_RECLAIM != (__force gfp_t) ALLOC_KSWAPD);
Mel Gormana56f57f2009-06-16 15:32:02 -07004402
Peter Zijlstra341ce062009-06-16 15:32:02 -07004403 /*
4404 * The caller may dip into page reserves a bit more if the caller
4405 * cannot run direct reclaim, or if the caller has realtime scheduling
4406 * policy or is asking for __GFP_HIGH memory. GFP_ATOMIC requests will
Mel Gormand0164ad2015-11-06 16:28:21 -08004407 * set both ALLOC_HARDER (__GFP_ATOMIC) and ALLOC_HIGH (__GFP_HIGH).
Peter Zijlstra341ce062009-06-16 15:32:02 -07004408 */
Mateusz Nosek736838e2020-04-01 21:09:47 -07004409 alloc_flags |= (__force int)
4410 (gfp_mask & (__GFP_HIGH | __GFP_KSWAPD_RECLAIM));
Peter Zijlstra341ce062009-06-16 15:32:02 -07004411
Mel Gormand0164ad2015-11-06 16:28:21 -08004412 if (gfp_mask & __GFP_ATOMIC) {
Andrea Arcangeli5c3240d2011-01-13 15:46:49 -08004413 /*
David Rientjesb104a352014-07-30 16:08:24 -07004414 * Not worth trying to allocate harder for __GFP_NOMEMALLOC even
4415 * if it can't schedule.
Andrea Arcangeli5c3240d2011-01-13 15:46:49 -08004416 */
David Rientjesb104a352014-07-30 16:08:24 -07004417 if (!(gfp_mask & __GFP_NOMEMALLOC))
Andrea Arcangeli5c3240d2011-01-13 15:46:49 -08004418 alloc_flags |= ALLOC_HARDER;
Peter Zijlstra341ce062009-06-16 15:32:02 -07004419 /*
David Rientjesb104a352014-07-30 16:08:24 -07004420 * Ignore cpuset mems for GFP_ATOMIC rather than fail, see the
Vladimir Davydov344736f2014-10-20 15:50:30 +04004421 * comment for __cpuset_node_allowed().
Peter Zijlstra341ce062009-06-16 15:32:02 -07004422 */
4423 alloc_flags &= ~ALLOC_CPUSET;
Andrew Mortonc06b1fc2011-01-13 15:47:32 -08004424 } else if (unlikely(rt_task(current)) && !in_interrupt())
Peter Zijlstra341ce062009-06-16 15:32:02 -07004425 alloc_flags |= ALLOC_HARDER;
4426
Joonsoo Kim8510e692020-08-06 23:26:04 -07004427 alloc_flags = current_alloc_flags(gfp_mask, alloc_flags);
4428
Peter Zijlstra341ce062009-06-16 15:32:02 -07004429 return alloc_flags;
4430}
4431
Michal Hockocd04ae12017-09-06 16:24:50 -07004432static bool oom_reserves_allowed(struct task_struct *tsk)
Mel Gorman072bb0a2012-07-31 16:43:58 -07004433{
Michal Hockocd04ae12017-09-06 16:24:50 -07004434 if (!tsk_is_oom_victim(tsk))
Vlastimil Babka31a6c192016-07-28 15:49:13 -07004435 return false;
4436
Michal Hockocd04ae12017-09-06 16:24:50 -07004437 /*
4438 * !MMU doesn't have oom reaper so give access to memory reserves
4439 * only to the thread with TIF_MEMDIE set
4440 */
4441 if (!IS_ENABLED(CONFIG_MMU) && !test_thread_flag(TIF_MEMDIE))
4442 return false;
Vlastimil Babka31a6c192016-07-28 15:49:13 -07004443
Michal Hockocd04ae12017-09-06 16:24:50 -07004444 return true;
4445}
4446
4447/*
4448 * Distinguish requests which really need access to full memory
4449 * reserves from oom victims which can live with a portion of it
4450 */
4451static inline int __gfp_pfmemalloc_flags(gfp_t gfp_mask)
4452{
4453 if (unlikely(gfp_mask & __GFP_NOMEMALLOC))
4454 return 0;
4455 if (gfp_mask & __GFP_MEMALLOC)
4456 return ALLOC_NO_WATERMARKS;
4457 if (in_serving_softirq() && (current->flags & PF_MEMALLOC))
4458 return ALLOC_NO_WATERMARKS;
4459 if (!in_interrupt()) {
4460 if (current->flags & PF_MEMALLOC)
4461 return ALLOC_NO_WATERMARKS;
4462 else if (oom_reserves_allowed(current))
4463 return ALLOC_OOM;
4464 }
4465
4466 return 0;
4467}
4468
4469bool gfp_pfmemalloc_allowed(gfp_t gfp_mask)
4470{
4471 return !!__gfp_pfmemalloc_flags(gfp_mask);
Mel Gorman072bb0a2012-07-31 16:43:58 -07004472}
4473
Michal Hocko0a0337e2016-05-20 16:57:00 -07004474/*
Michal Hocko0a0337e2016-05-20 16:57:00 -07004475 * Checks whether it makes sense to retry the reclaim to make a forward progress
4476 * for the given allocation request.
Johannes Weiner491d79a2017-05-03 14:52:16 -07004477 *
4478 * We give up when we either have tried MAX_RECLAIM_RETRIES in a row
4479 * without success, or when we couldn't even meet the watermark if we
4480 * reclaimed all remaining pages on the LRU lists.
Michal Hocko0a0337e2016-05-20 16:57:00 -07004481 *
4482 * Returns true if a retry is viable or false to enter the oom path.
4483 */
4484static inline bool
4485should_reclaim_retry(gfp_t gfp_mask, unsigned order,
4486 struct alloc_context *ac, int alloc_flags,
Vlastimil Babka423b4522016-10-07 17:00:40 -07004487 bool did_some_progress, int *no_progress_loops)
Michal Hocko0a0337e2016-05-20 16:57:00 -07004488{
4489 struct zone *zone;
4490 struct zoneref *z;
Michal Hocko15f570b2018-10-26 15:03:31 -07004491 bool ret = false;
Michal Hocko0a0337e2016-05-20 16:57:00 -07004492
4493 /*
Vlastimil Babka423b4522016-10-07 17:00:40 -07004494 * Costly allocations might have made a progress but this doesn't mean
4495 * their order will become available due to high fragmentation so
4496 * always increment the no progress counter for them
4497 */
4498 if (did_some_progress && order <= PAGE_ALLOC_COSTLY_ORDER)
4499 *no_progress_loops = 0;
4500 else
4501 (*no_progress_loops)++;
4502
4503 /*
Michal Hocko0a0337e2016-05-20 16:57:00 -07004504 * Make sure we converge to OOM if we cannot make any progress
4505 * several times in the row.
4506 */
Minchan Kim04c87162016-12-12 16:42:11 -08004507 if (*no_progress_loops > MAX_RECLAIM_RETRIES) {
4508 /* Before OOM, exhaust highatomic_reserve */
Minchan Kim29fac032016-12-12 16:42:14 -08004509 return unreserve_highatomic_pageblock(ac, true);
Minchan Kim04c87162016-12-12 16:42:11 -08004510 }
Michal Hocko0a0337e2016-05-20 16:57:00 -07004511
Michal Hocko0a0337e2016-05-20 16:57:00 -07004512 /*
Mel Gormanbca67592016-07-28 15:47:05 -07004513 * Keep reclaiming pages while there is a chance this will lead
4514 * somewhere. If none of the target zones can satisfy our allocation
4515 * request even if all reclaimable pages are considered then we are
4516 * screwed and have to go OOM.
Michal Hocko0a0337e2016-05-20 16:57:00 -07004517 */
Joonsoo Kim97a225e2020-06-03 15:59:01 -07004518 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist,
4519 ac->highest_zoneidx, ac->nodemask) {
Michal Hocko0a0337e2016-05-20 16:57:00 -07004520 unsigned long available;
Michal Hockoede37712016-05-20 16:57:03 -07004521 unsigned long reclaimable;
Michal Hockod379f012017-02-22 15:42:00 -08004522 unsigned long min_wmark = min_wmark_pages(zone);
4523 bool wmark;
Michal Hocko0a0337e2016-05-20 16:57:00 -07004524
Mel Gorman5a1c84b2016-07-28 15:47:31 -07004525 available = reclaimable = zone_reclaimable_pages(zone);
Mel Gorman5a1c84b2016-07-28 15:47:31 -07004526 available += zone_page_state_snapshot(zone, NR_FREE_PAGES);
Michal Hocko0a0337e2016-05-20 16:57:00 -07004527
4528 /*
Johannes Weiner491d79a2017-05-03 14:52:16 -07004529 * Would the allocation succeed if we reclaimed all
4530 * reclaimable pages?
Michal Hocko0a0337e2016-05-20 16:57:00 -07004531 */
Michal Hockod379f012017-02-22 15:42:00 -08004532 wmark = __zone_watermark_ok(zone, order, min_wmark,
Joonsoo Kim97a225e2020-06-03 15:59:01 -07004533 ac->highest_zoneidx, alloc_flags, available);
Michal Hockod379f012017-02-22 15:42:00 -08004534 trace_reclaim_retry_zone(z, order, reclaimable,
4535 available, min_wmark, *no_progress_loops, wmark);
4536 if (wmark) {
Michal Hockoede37712016-05-20 16:57:03 -07004537 /*
4538 * If we didn't make any progress and have a lot of
4539 * dirty + writeback pages then we should wait for
4540 * an IO to complete to slow down the reclaim and
4541 * prevent from pre mature OOM
4542 */
4543 if (!did_some_progress) {
Mel Gorman11fb9982016-07-28 15:46:20 -07004544 unsigned long write_pending;
Michal Hockoede37712016-05-20 16:57:03 -07004545
Mel Gorman5a1c84b2016-07-28 15:47:31 -07004546 write_pending = zone_page_state_snapshot(zone,
4547 NR_ZONE_WRITE_PENDING);
Michal Hockoede37712016-05-20 16:57:03 -07004548
Mel Gorman11fb9982016-07-28 15:46:20 -07004549 if (2 * write_pending > reclaimable) {
Michal Hockoede37712016-05-20 16:57:03 -07004550 congestion_wait(BLK_RW_ASYNC, HZ/10);
4551 return true;
4552 }
4553 }
Mel Gorman5a1c84b2016-07-28 15:47:31 -07004554
Michal Hocko15f570b2018-10-26 15:03:31 -07004555 ret = true;
4556 goto out;
Michal Hocko0a0337e2016-05-20 16:57:00 -07004557 }
4558 }
4559
Michal Hocko15f570b2018-10-26 15:03:31 -07004560out:
4561 /*
4562 * Memory allocation/reclaim might be called from a WQ context and the
4563 * current implementation of the WQ concurrency control doesn't
4564 * recognize that a particular WQ is congested if the worker thread is
4565 * looping without ever sleeping. Therefore we have to do a short sleep
4566 * here rather than calling cond_resched().
4567 */
4568 if (current->flags & PF_WQ_WORKER)
4569 schedule_timeout_uninterruptible(1);
4570 else
4571 cond_resched();
4572 return ret;
Michal Hocko0a0337e2016-05-20 16:57:00 -07004573}
4574
Vlastimil Babka902b6282017-07-06 15:39:56 -07004575static inline bool
4576check_retry_cpuset(int cpuset_mems_cookie, struct alloc_context *ac)
4577{
4578 /*
4579 * It's possible that cpuset's mems_allowed and the nodemask from
4580 * mempolicy don't intersect. This should be normally dealt with by
4581 * policy_nodemask(), but it's possible to race with cpuset update in
4582 * such a way the check therein was true, and then it became false
4583 * before we got our cpuset_mems_cookie here.
4584 * This assumes that for all allocations, ac->nodemask can come only
4585 * from MPOL_BIND mempolicy (whose documented semantics is to be ignored
4586 * when it does not intersect with the cpuset restrictions) or the
4587 * caller can deal with a violated nodemask.
4588 */
4589 if (cpusets_enabled() && ac->nodemask &&
4590 !cpuset_nodemask_valid_mems_allowed(ac->nodemask)) {
4591 ac->nodemask = NULL;
4592 return true;
4593 }
4594
4595 /*
4596 * When updating a task's mems_allowed or mempolicy nodemask, it is
4597 * possible to race with parallel threads in such a way that our
4598 * allocation can fail while the mask is being updated. If we are about
4599 * to fail, check if the cpuset changed during allocation and if so,
4600 * retry.
4601 */
4602 if (read_mems_allowed_retry(cpuset_mems_cookie))
4603 return true;
4604
4605 return false;
4606}
4607
Mel Gorman11e33f62009-06-16 15:31:57 -07004608static inline struct page *
4609__alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
Vlastimil Babkaa9263752015-02-11 15:25:41 -08004610 struct alloc_context *ac)
Mel Gorman11e33f62009-06-16 15:31:57 -07004611{
Mel Gormand0164ad2015-11-06 16:28:21 -08004612 bool can_direct_reclaim = gfp_mask & __GFP_DIRECT_RECLAIM;
Vlastimil Babka282722b2017-05-08 15:54:49 -07004613 const bool costly_order = order > PAGE_ALLOC_COSTLY_ORDER;
Mel Gorman11e33f62009-06-16 15:31:57 -07004614 struct page *page = NULL;
Mel Gormanc6038442016-05-19 17:13:38 -07004615 unsigned int alloc_flags;
Mel Gorman11e33f62009-06-16 15:31:57 -07004616 unsigned long did_some_progress;
Vlastimil Babka5ce9bfe2017-01-24 15:18:38 -08004617 enum compact_priority compact_priority;
Michal Hockoc5d01d02016-05-20 16:56:53 -07004618 enum compact_result compact_result;
Vlastimil Babka5ce9bfe2017-01-24 15:18:38 -08004619 int compaction_retries;
4620 int no_progress_loops;
Vlastimil Babka5ce9bfe2017-01-24 15:18:38 -08004621 unsigned int cpuset_mems_cookie;
Michal Hockocd04ae12017-09-06 16:24:50 -07004622 int reserve_flags;
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08004623
Christoph Lameter952f3b52006-12-06 20:33:26 -08004624 /*
Mel Gormand0164ad2015-11-06 16:28:21 -08004625 * We also sanity check to catch abuse of atomic reserves being used by
4626 * callers that are not in atomic context.
4627 */
4628 if (WARN_ON_ONCE((gfp_mask & (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM)) ==
4629 (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM)))
4630 gfp_mask &= ~__GFP_ATOMIC;
4631
Vlastimil Babka5ce9bfe2017-01-24 15:18:38 -08004632retry_cpuset:
4633 compaction_retries = 0;
4634 no_progress_loops = 0;
4635 compact_priority = DEF_COMPACT_PRIORITY;
4636 cpuset_mems_cookie = read_mems_allowed_begin();
Michal Hocko9a67f642017-02-22 15:46:19 -08004637
4638 /*
4639 * The fast path uses conservative alloc_flags to succeed only until
4640 * kswapd needs to be woken up, and to avoid the cost of setting up
4641 * alloc_flags precisely. So we do that now.
4642 */
4643 alloc_flags = gfp_to_alloc_flags(gfp_mask);
4644
Vlastimil Babkae47483b2017-01-24 15:18:41 -08004645 /*
4646 * We need to recalculate the starting point for the zonelist iterator
4647 * because we might have used different nodemask in the fast path, or
4648 * there was a cpuset modification and we are retrying - otherwise we
4649 * could end up iterating over non-eligible zones endlessly.
4650 */
4651 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist,
Joonsoo Kim97a225e2020-06-03 15:59:01 -07004652 ac->highest_zoneidx, ac->nodemask);
Vlastimil Babkae47483b2017-01-24 15:18:41 -08004653 if (!ac->preferred_zoneref->zone)
4654 goto nopage;
4655
Mel Gorman0a79cda2018-12-28 00:35:48 -08004656 if (alloc_flags & ALLOC_KSWAPD)
David Rientjes5ecd9d42018-04-05 16:25:16 -07004657 wake_all_kswapds(order, gfp_mask, ac);
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08004658
Paul Jackson9bf22292005-09-06 15:18:12 -07004659 /*
Vlastimil Babka23771232016-07-28 15:49:16 -07004660 * The adjusted alloc_flags might result in immediate success, so try
4661 * that first
4662 */
4663 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
4664 if (page)
4665 goto got_pg;
4666
Vlastimil Babkaa8161d12016-07-28 15:49:19 -07004667 /*
4668 * For costly allocations, try direct compaction first, as it's likely
Vlastimil Babka282722b2017-05-08 15:54:49 -07004669 * that we have enough base pages and don't need to reclaim. For non-
4670 * movable high-order allocations, do that as well, as compaction will
4671 * try prevent permanent fragmentation by migrating from blocks of the
4672 * same migratetype.
4673 * Don't try this for allocations that are allowed to ignore
4674 * watermarks, as the ALLOC_NO_WATERMARKS attempt didn't yet happen.
Vlastimil Babkaa8161d12016-07-28 15:49:19 -07004675 */
Vlastimil Babka282722b2017-05-08 15:54:49 -07004676 if (can_direct_reclaim &&
4677 (costly_order ||
4678 (order > 0 && ac->migratetype != MIGRATE_MOVABLE))
4679 && !gfp_pfmemalloc_allowed(gfp_mask)) {
Vlastimil Babkaa8161d12016-07-28 15:49:19 -07004680 page = __alloc_pages_direct_compact(gfp_mask, order,
4681 alloc_flags, ac,
Vlastimil Babkaa5508cd2016-07-28 15:49:28 -07004682 INIT_COMPACT_PRIORITY,
Vlastimil Babkaa8161d12016-07-28 15:49:19 -07004683 &compact_result);
4684 if (page)
4685 goto got_pg;
4686
Vlastimil Babkacc638f32020-01-13 16:29:04 -08004687 /*
4688 * Checks for costly allocations with __GFP_NORETRY, which
4689 * includes some THP page fault allocations
4690 */
4691 if (costly_order && (gfp_mask & __GFP_NORETRY)) {
David Rientjesb39d0ee2019-09-04 12:54:22 -07004692 /*
4693 * If allocating entire pageblock(s) and compaction
4694 * failed because all zones are below low watermarks
4695 * or is prohibited because it recently failed at this
David Rientjes3f36d862019-10-14 14:12:04 -07004696 * order, fail immediately unless the allocator has
4697 * requested compaction and reclaim retry.
David Rientjesb39d0ee2019-09-04 12:54:22 -07004698 *
4699 * Reclaim is
4700 * - potentially very expensive because zones are far
4701 * below their low watermarks or this is part of very
4702 * bursty high order allocations,
4703 * - not guaranteed to help because isolate_freepages()
4704 * may not iterate over freed pages as part of its
4705 * linear scan, and
4706 * - unlikely to make entire pageblocks free on its
4707 * own.
4708 */
4709 if (compact_result == COMPACT_SKIPPED ||
4710 compact_result == COMPACT_DEFERRED)
4711 goto nopage;
Vlastimil Babkaa8161d12016-07-28 15:49:19 -07004712
4713 /*
Vlastimil Babka3eb27712016-07-28 15:49:22 -07004714 * Looks like reclaim/compaction is worth trying, but
4715 * sync compaction could be very expensive, so keep
Vlastimil Babka25160352016-07-28 15:49:25 -07004716 * using async compaction.
Vlastimil Babkaa8161d12016-07-28 15:49:19 -07004717 */
Vlastimil Babkaa5508cd2016-07-28 15:49:28 -07004718 compact_priority = INIT_COMPACT_PRIORITY;
Vlastimil Babkaa8161d12016-07-28 15:49:19 -07004719 }
4720 }
Vlastimil Babka23771232016-07-28 15:49:16 -07004721
4722retry:
4723 /* Ensure kswapd doesn't accidentally go to sleep as long as we loop */
Mel Gorman0a79cda2018-12-28 00:35:48 -08004724 if (alloc_flags & ALLOC_KSWAPD)
David Rientjes5ecd9d42018-04-05 16:25:16 -07004725 wake_all_kswapds(order, gfp_mask, ac);
Vlastimil Babka23771232016-07-28 15:49:16 -07004726
Michal Hockocd04ae12017-09-06 16:24:50 -07004727 reserve_flags = __gfp_pfmemalloc_flags(gfp_mask);
4728 if (reserve_flags)
Joonsoo Kim8510e692020-08-06 23:26:04 -07004729 alloc_flags = current_alloc_flags(gfp_mask, reserve_flags);
Vlastimil Babka23771232016-07-28 15:49:16 -07004730
4731 /*
Vlastimil Babkad6a24df2018-08-17 15:45:05 -07004732 * Reset the nodemask and zonelist iterators if memory policies can be
4733 * ignored. These allocations are high priority and system rather than
4734 * user oriented.
Mel Gormane46e7b72016-06-03 14:56:01 -07004735 */
Michal Hockocd04ae12017-09-06 16:24:50 -07004736 if (!(alloc_flags & ALLOC_CPUSET) || reserve_flags) {
Vlastimil Babkad6a24df2018-08-17 15:45:05 -07004737 ac->nodemask = NULL;
Mel Gormane46e7b72016-06-03 14:56:01 -07004738 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist,
Joonsoo Kim97a225e2020-06-03 15:59:01 -07004739 ac->highest_zoneidx, ac->nodemask);
Mel Gormane46e7b72016-06-03 14:56:01 -07004740 }
4741
Vlastimil Babka23771232016-07-28 15:49:16 -07004742 /* Attempt with potentially adjusted zonelist and alloc_flags */
Vlastimil Babka31a6c192016-07-28 15:49:13 -07004743 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08004744 if (page)
4745 goto got_pg;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004746
Mel Gormand0164ad2015-11-06 16:28:21 -08004747 /* Caller is not willing to reclaim, we can't balance anything */
Michal Hocko9a67f642017-02-22 15:46:19 -08004748 if (!can_direct_reclaim)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004749 goto nopage;
Michal Hocko9a67f642017-02-22 15:46:19 -08004750
Peter Zijlstra341ce062009-06-16 15:32:02 -07004751 /* Avoid recursion of direct reclaim */
Michal Hocko9a67f642017-02-22 15:46:19 -08004752 if (current->flags & PF_MEMALLOC)
Peter Zijlstra341ce062009-06-16 15:32:02 -07004753 goto nopage;
David Rientjes8fe78042014-08-06 16:07:54 -07004754
Mel Gorman11e33f62009-06-16 15:31:57 -07004755 /* Try direct reclaim and then allocating */
Vlastimil Babkaa9263752015-02-11 15:25:41 -08004756 page = __alloc_pages_direct_reclaim(gfp_mask, order, alloc_flags, ac,
4757 &did_some_progress);
Mel Gorman11e33f62009-06-16 15:31:57 -07004758 if (page)
4759 goto got_pg;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004760
Vlastimil Babkaa8161d12016-07-28 15:49:19 -07004761 /* Try direct compaction and then allocating */
4762 page = __alloc_pages_direct_compact(gfp_mask, order, alloc_flags, ac,
Vlastimil Babkaa5508cd2016-07-28 15:49:28 -07004763 compact_priority, &compact_result);
Vlastimil Babkaa8161d12016-07-28 15:49:19 -07004764 if (page)
4765 goto got_pg;
4766
Johannes Weiner90839052015-06-24 16:57:21 -07004767 /* Do not loop if specifically requested */
4768 if (gfp_mask & __GFP_NORETRY)
Vlastimil Babkaa8161d12016-07-28 15:49:19 -07004769 goto nopage;
Johannes Weiner90839052015-06-24 16:57:21 -07004770
Michal Hocko0a0337e2016-05-20 16:57:00 -07004771 /*
4772 * Do not retry costly high order allocations unless they are
Michal Hockodcda9b02017-07-12 14:36:45 -07004773 * __GFP_RETRY_MAYFAIL
Michal Hocko0a0337e2016-05-20 16:57:00 -07004774 */
Michal Hockodcda9b02017-07-12 14:36:45 -07004775 if (costly_order && !(gfp_mask & __GFP_RETRY_MAYFAIL))
Vlastimil Babkaa8161d12016-07-28 15:49:19 -07004776 goto nopage;
Michal Hocko0a0337e2016-05-20 16:57:00 -07004777
Michal Hocko0a0337e2016-05-20 16:57:00 -07004778 if (should_reclaim_retry(gfp_mask, order, ac, alloc_flags,
Vlastimil Babka423b4522016-10-07 17:00:40 -07004779 did_some_progress > 0, &no_progress_loops))
Michal Hocko0a0337e2016-05-20 16:57:00 -07004780 goto retry;
4781
Michal Hocko33c2d212016-05-20 16:57:06 -07004782 /*
4783 * It doesn't make any sense to retry for the compaction if the order-0
4784 * reclaim is not able to make any progress because the current
4785 * implementation of the compaction depends on the sufficient amount
4786 * of free memory (see __compaction_suitable)
4787 */
4788 if (did_some_progress > 0 &&
Michal Hocko86a294a2016-05-20 16:57:12 -07004789 should_compact_retry(ac, order, alloc_flags,
Vlastimil Babkaa5508cd2016-07-28 15:49:28 -07004790 compact_result, &compact_priority,
Vlastimil Babkad9436492016-10-07 17:00:31 -07004791 &compaction_retries))
Michal Hocko33c2d212016-05-20 16:57:06 -07004792 goto retry;
4793
Vlastimil Babka902b6282017-07-06 15:39:56 -07004794
4795 /* Deal with possible cpuset update races before we start OOM killing */
4796 if (check_retry_cpuset(cpuset_mems_cookie, ac))
Vlastimil Babkae47483b2017-01-24 15:18:41 -08004797 goto retry_cpuset;
4798
Johannes Weiner90839052015-06-24 16:57:21 -07004799 /* Reclaim has failed us, start killing things */
4800 page = __alloc_pages_may_oom(gfp_mask, order, ac, &did_some_progress);
4801 if (page)
4802 goto got_pg;
4803
Michal Hocko9a67f642017-02-22 15:46:19 -08004804 /* Avoid allocations with no watermarks from looping endlessly */
Michal Hockocd04ae12017-09-06 16:24:50 -07004805 if (tsk_is_oom_victim(current) &&
Joonsoo Kim8510e692020-08-06 23:26:04 -07004806 (alloc_flags & ALLOC_OOM ||
Tetsuo Handac2889832017-06-02 14:46:31 -07004807 (gfp_mask & __GFP_NOMEMALLOC)))
Michal Hocko9a67f642017-02-22 15:46:19 -08004808 goto nopage;
4809
Johannes Weiner90839052015-06-24 16:57:21 -07004810 /* Retry as long as the OOM killer is making progress */
Michal Hocko0a0337e2016-05-20 16:57:00 -07004811 if (did_some_progress) {
4812 no_progress_loops = 0;
Johannes Weiner90839052015-06-24 16:57:21 -07004813 goto retry;
Michal Hocko0a0337e2016-05-20 16:57:00 -07004814 }
Johannes Weiner90839052015-06-24 16:57:21 -07004815
Linus Torvalds1da177e2005-04-16 15:20:36 -07004816nopage:
Vlastimil Babka902b6282017-07-06 15:39:56 -07004817 /* Deal with possible cpuset update races before we fail */
4818 if (check_retry_cpuset(cpuset_mems_cookie, ac))
Vlastimil Babka5ce9bfe2017-01-24 15:18:38 -08004819 goto retry_cpuset;
4820
Michal Hocko9a67f642017-02-22 15:46:19 -08004821 /*
4822 * Make sure that __GFP_NOFAIL request doesn't leak out and make sure
4823 * we always retry
4824 */
4825 if (gfp_mask & __GFP_NOFAIL) {
4826 /*
4827 * All existing users of the __GFP_NOFAIL are blockable, so warn
4828 * of any new users that actually require GFP_NOWAIT
4829 */
4830 if (WARN_ON_ONCE(!can_direct_reclaim))
4831 goto fail;
4832
4833 /*
4834 * PF_MEMALLOC request from this context is rather bizarre
4835 * because we cannot reclaim anything and only can loop waiting
4836 * for somebody to do a work for us
4837 */
4838 WARN_ON_ONCE(current->flags & PF_MEMALLOC);
4839
4840 /*
4841 * non failing costly orders are a hard requirement which we
4842 * are not prepared for much so let's warn about these users
4843 * so that we can identify them and convert them to something
4844 * else.
4845 */
4846 WARN_ON_ONCE(order > PAGE_ALLOC_COSTLY_ORDER);
4847
Michal Hocko6c18ba72017-02-22 15:46:25 -08004848 /*
4849 * Help non-failing allocations by giving them access to memory
4850 * reserves but do not use ALLOC_NO_WATERMARKS because this
4851 * could deplete whole memory reserves which would just make
4852 * the situation worse
4853 */
4854 page = __alloc_pages_cpuset_fallback(gfp_mask, order, ALLOC_HARDER, ac);
4855 if (page)
4856 goto got_pg;
4857
Michal Hocko9a67f642017-02-22 15:46:19 -08004858 cond_resched();
4859 goto retry;
4860 }
4861fail:
Michal Hockoa8e99252017-02-22 15:46:10 -08004862 warn_alloc(gfp_mask, ac->nodemask,
Michal Hocko7877cdc2016-10-07 17:01:55 -07004863 "page allocation failure: order:%u", order);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004864got_pg:
Mel Gorman072bb0a2012-07-31 16:43:58 -07004865 return page;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004866}
Mel Gorman11e33f62009-06-16 15:31:57 -07004867
Mel Gorman9cd75552017-02-24 14:56:29 -08004868static inline bool prepare_alloc_pages(gfp_t gfp_mask, unsigned int order,
Vlastimil Babka04ec6262017-07-06 15:40:03 -07004869 int preferred_nid, nodemask_t *nodemask,
Mel Gorman9cd75552017-02-24 14:56:29 -08004870 struct alloc_context *ac, gfp_t *alloc_mask,
4871 unsigned int *alloc_flags)
4872{
Joonsoo Kim97a225e2020-06-03 15:59:01 -07004873 ac->highest_zoneidx = gfp_zone(gfp_mask);
Vlastimil Babka04ec6262017-07-06 15:40:03 -07004874 ac->zonelist = node_zonelist(preferred_nid, gfp_mask);
Mel Gorman9cd75552017-02-24 14:56:29 -08004875 ac->nodemask = nodemask;
Wei Yang01c0bfe2020-06-03 15:59:08 -07004876 ac->migratetype = gfp_migratetype(gfp_mask);
Mel Gorman9cd75552017-02-24 14:56:29 -08004877
4878 if (cpusets_enabled()) {
4879 *alloc_mask |= __GFP_HARDWALL;
Muchun Song182f3d72020-08-06 23:26:01 -07004880 /*
4881 * When we are in the interrupt context, it is irrelevant
4882 * to the current task context. It means that any node ok.
4883 */
4884 if (!in_interrupt() && !ac->nodemask)
Mel Gorman9cd75552017-02-24 14:56:29 -08004885 ac->nodemask = &cpuset_current_mems_allowed;
Vlastimil Babka51047822017-02-24 14:56:53 -08004886 else
4887 *alloc_flags |= ALLOC_CPUSET;
Mel Gorman9cd75552017-02-24 14:56:29 -08004888 }
4889
Peter Zijlstrad92a8cf2017-03-03 10:13:38 +01004890 fs_reclaim_acquire(gfp_mask);
4891 fs_reclaim_release(gfp_mask);
Mel Gorman9cd75552017-02-24 14:56:29 -08004892
4893 might_sleep_if(gfp_mask & __GFP_DIRECT_RECLAIM);
4894
4895 if (should_fail_alloc_page(gfp_mask, order))
4896 return false;
4897
Joonsoo Kim8510e692020-08-06 23:26:04 -07004898 *alloc_flags = current_alloc_flags(gfp_mask, *alloc_flags);
Joonsoo Kimd883c6c2018-05-23 10:18:21 +09004899
Mel Gorman9cd75552017-02-24 14:56:29 -08004900 /* Dirty zone balancing only done in the fast path */
4901 ac->spread_dirty_pages = (gfp_mask & __GFP_WRITE);
4902
4903 /*
4904 * The preferred zone is used for statistics but crucially it is
4905 * also used as the starting point for the zonelist iterator. It
4906 * may get reset for allocations that ignore memory policies.
4907 */
4908 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist,
Joonsoo Kim97a225e2020-06-03 15:59:01 -07004909 ac->highest_zoneidx, ac->nodemask);
Mateusz Noseka0622d02020-10-13 16:55:51 -07004910
4911 return true;
Mel Gorman9cd75552017-02-24 14:56:29 -08004912}
4913
Mel Gorman11e33f62009-06-16 15:31:57 -07004914/*
4915 * This is the 'heart' of the zoned buddy allocator.
4916 */
4917struct page *
Vlastimil Babka04ec6262017-07-06 15:40:03 -07004918__alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order, int preferred_nid,
4919 nodemask_t *nodemask)
Mel Gorman11e33f62009-06-16 15:31:57 -07004920{
Mel Gorman5bb1b162016-05-19 17:13:50 -07004921 struct page *page;
Mel Gormane6cbd7f2016-07-28 15:46:50 -07004922 unsigned int alloc_flags = ALLOC_WMARK_LOW;
Tetsuo Handaf19360f2017-09-08 16:13:22 -07004923 gfp_t alloc_mask; /* The gfp_t that was actually used for allocation */
Mel Gorman9cd75552017-02-24 14:56:29 -08004924 struct alloc_context ac = { };
Mel Gorman682a3382016-05-19 17:13:30 -07004925
Michal Hockoc63ae432018-11-16 15:08:53 -08004926 /*
4927 * There are several places where we assume that the order value is sane
4928 * so bail out early if the request is out of bound.
4929 */
4930 if (unlikely(order >= MAX_ORDER)) {
4931 WARN_ON_ONCE(!(gfp_mask & __GFP_NOWARN));
4932 return NULL;
4933 }
4934
Benjamin Herrenschmidtdcce2842009-06-18 13:24:12 +10004935 gfp_mask &= gfp_allowed_mask;
Tetsuo Handaf19360f2017-09-08 16:13:22 -07004936 alloc_mask = gfp_mask;
Vlastimil Babka04ec6262017-07-06 15:40:03 -07004937 if (!prepare_alloc_pages(gfp_mask, order, preferred_nid, nodemask, &ac, &alloc_mask, &alloc_flags))
Mel Gorman11e33f62009-06-16 15:31:57 -07004938 return NULL;
4939
Mel Gorman6bb15452018-12-28 00:35:41 -08004940 /*
4941 * Forbid the first pass from falling back to types that fragment
4942 * memory until all local zones are considered.
4943 */
Mel Gorman0a79cda2018-12-28 00:35:48 -08004944 alloc_flags |= alloc_flags_nofragment(ac.preferred_zoneref->zone, gfp_mask);
Mel Gorman6bb15452018-12-28 00:35:41 -08004945
Mel Gorman5117f452009-06-16 15:31:59 -07004946 /* First allocation attempt */
Vlastimil Babkaa9263752015-02-11 15:25:41 -08004947 page = get_page_from_freelist(alloc_mask, order, alloc_flags, &ac);
Mel Gorman4fcb0972016-05-19 17:14:01 -07004948 if (likely(page))
4949 goto out;
Andrew Morton91fbdc02015-02-11 15:25:04 -08004950
Mel Gorman4fcb0972016-05-19 17:14:01 -07004951 /*
Michal Hocko7dea19f2017-05-03 14:53:15 -07004952 * Apply scoped allocation constraints. This is mainly about GFP_NOFS
4953 * resp. GFP_NOIO which has to be inherited for all allocation requests
4954 * from a particular context which has been marked by
4955 * memalloc_no{fs,io}_{save,restore}.
Mel Gorman4fcb0972016-05-19 17:14:01 -07004956 */
Michal Hocko7dea19f2017-05-03 14:53:15 -07004957 alloc_mask = current_gfp_context(gfp_mask);
Mel Gorman4fcb0972016-05-19 17:14:01 -07004958 ac.spread_dirty_pages = false;
Mel Gorman11e33f62009-06-16 15:31:57 -07004959
Mel Gorman47415262016-05-19 17:14:44 -07004960 /*
4961 * Restore the original nodemask if it was potentially replaced with
4962 * &cpuset_current_mems_allowed to optimize the fast-path attempt.
4963 */
Mateusz Nosek97ce86f2020-04-01 21:09:53 -07004964 ac.nodemask = nodemask;
Vlastimil Babka16096c22017-01-24 15:18:35 -08004965
Mel Gorman4fcb0972016-05-19 17:14:01 -07004966 page = __alloc_pages_slowpath(alloc_mask, order, &ac);
Xishi Qiu23f086f2015-02-11 15:25:07 -08004967
Mel Gorman4fcb0972016-05-19 17:14:01 -07004968out:
Vladimir Davydovc4159a72016-08-08 23:03:12 +03004969 if (memcg_kmem_enabled() && (gfp_mask & __GFP_ACCOUNT) && page &&
Roman Gushchinf4b00ea2020-04-01 21:06:46 -07004970 unlikely(__memcg_kmem_charge_page(page, gfp_mask, order) != 0)) {
Vladimir Davydovc4159a72016-08-08 23:03:12 +03004971 __free_pages(page, order);
4972 page = NULL;
Vladimir Davydov49491482016-07-26 15:24:24 -07004973 }
4974
Mel Gorman4fcb0972016-05-19 17:14:01 -07004975 trace_mm_page_alloc(page, order, alloc_mask, ac.migratetype);
4976
Mel Gorman11e33f62009-06-16 15:31:57 -07004977 return page;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004978}
Mel Gormand2391712009-06-16 15:31:52 -07004979EXPORT_SYMBOL(__alloc_pages_nodemask);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004980
4981/*
Michal Hocko9ea9a682018-08-17 15:46:01 -07004982 * Common helper functions. Never use with __GFP_HIGHMEM because the returned
4983 * address cannot represent highmem pages. Use alloc_pages and then kmap if
4984 * you need to access high mem.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004985 */
Harvey Harrison920c7a52008-02-04 22:29:26 -08004986unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004987{
Akinobu Mita945a1112009-09-21 17:01:47 -07004988 struct page *page;
4989
Michal Hocko9ea9a682018-08-17 15:46:01 -07004990 page = alloc_pages(gfp_mask & ~__GFP_HIGHMEM, order);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004991 if (!page)
4992 return 0;
4993 return (unsigned long) page_address(page);
4994}
Linus Torvalds1da177e2005-04-16 15:20:36 -07004995EXPORT_SYMBOL(__get_free_pages);
4996
Harvey Harrison920c7a52008-02-04 22:29:26 -08004997unsigned long get_zeroed_page(gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004998{
Akinobu Mita945a1112009-09-21 17:01:47 -07004999 return __get_free_pages(gfp_mask | __GFP_ZERO, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005000}
Linus Torvalds1da177e2005-04-16 15:20:36 -07005001EXPORT_SYMBOL(get_zeroed_page);
5002
Aaron Lu742aa7f2018-12-28 00:35:22 -08005003static inline void free_the_page(struct page *page, unsigned int order)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005004{
Aaron Lu742aa7f2018-12-28 00:35:22 -08005005 if (order == 0) /* Via pcp? */
5006 free_unref_page(page);
5007 else
David Hildenbrand7fef4312020-10-15 20:09:35 -07005008 __free_pages_ok(page, order, FPI_NONE);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005009}
5010
Aaron Lu742aa7f2018-12-28 00:35:22 -08005011void __free_pages(struct page *page, unsigned int order)
5012{
5013 if (put_page_testzero(page))
5014 free_the_page(page, order);
Matthew Wilcox (Oracle)e320d302020-10-13 16:56:04 -07005015 else if (!PageHead(page))
5016 while (order-- > 0)
5017 free_the_page(page + (1 << order), order);
Aaron Lu742aa7f2018-12-28 00:35:22 -08005018}
Linus Torvalds1da177e2005-04-16 15:20:36 -07005019EXPORT_SYMBOL(__free_pages);
5020
Harvey Harrison920c7a52008-02-04 22:29:26 -08005021void free_pages(unsigned long addr, unsigned int order)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005022{
5023 if (addr != 0) {
Nick Piggin725d7042006-09-25 23:30:55 -07005024 VM_BUG_ON(!virt_addr_valid((void *)addr));
Linus Torvalds1da177e2005-04-16 15:20:36 -07005025 __free_pages(virt_to_page((void *)addr), order);
5026 }
5027}
5028
5029EXPORT_SYMBOL(free_pages);
5030
Glauber Costa6a1a0d32012-12-18 14:22:00 -08005031/*
Alexander Duyckb63ae8c2015-05-06 21:11:57 -07005032 * Page Fragment:
5033 * An arbitrary-length arbitrary-offset area of memory which resides
5034 * within a 0 or higher order page. Multiple fragments within that page
5035 * are individually refcounted, in the page's reference counter.
5036 *
5037 * The page_frag functions below provide a simple allocation framework for
5038 * page fragments. This is used by the network stack and network device
5039 * drivers to provide a backing region of memory for use as either an
5040 * sk_buff->head, or to be used in the "frags" portion of skb_shared_info.
5041 */
Alexander Duyck2976db82017-01-10 16:58:09 -08005042static struct page *__page_frag_cache_refill(struct page_frag_cache *nc,
5043 gfp_t gfp_mask)
Alexander Duyckb63ae8c2015-05-06 21:11:57 -07005044{
5045 struct page *page = NULL;
5046 gfp_t gfp = gfp_mask;
5047
5048#if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE)
5049 gfp_mask |= __GFP_COMP | __GFP_NOWARN | __GFP_NORETRY |
5050 __GFP_NOMEMALLOC;
5051 page = alloc_pages_node(NUMA_NO_NODE, gfp_mask,
5052 PAGE_FRAG_CACHE_MAX_ORDER);
5053 nc->size = page ? PAGE_FRAG_CACHE_MAX_SIZE : PAGE_SIZE;
5054#endif
5055 if (unlikely(!page))
5056 page = alloc_pages_node(NUMA_NO_NODE, gfp, 0);
5057
5058 nc->va = page ? page_address(page) : NULL;
5059
5060 return page;
5061}
5062
Alexander Duyck2976db82017-01-10 16:58:09 -08005063void __page_frag_cache_drain(struct page *page, unsigned int count)
Alexander Duyck44fdffd2016-12-14 15:05:26 -08005064{
5065 VM_BUG_ON_PAGE(page_ref_count(page) == 0, page);
5066
Aaron Lu742aa7f2018-12-28 00:35:22 -08005067 if (page_ref_sub_and_test(page, count))
5068 free_the_page(page, compound_order(page));
Alexander Duyck44fdffd2016-12-14 15:05:26 -08005069}
Alexander Duyck2976db82017-01-10 16:58:09 -08005070EXPORT_SYMBOL(__page_frag_cache_drain);
Alexander Duyck44fdffd2016-12-14 15:05:26 -08005071
Alexander Duyck8c2dd3e2017-01-10 16:58:06 -08005072void *page_frag_alloc(struct page_frag_cache *nc,
5073 unsigned int fragsz, gfp_t gfp_mask)
Alexander Duyckb63ae8c2015-05-06 21:11:57 -07005074{
5075 unsigned int size = PAGE_SIZE;
5076 struct page *page;
5077 int offset;
5078
5079 if (unlikely(!nc->va)) {
5080refill:
Alexander Duyck2976db82017-01-10 16:58:09 -08005081 page = __page_frag_cache_refill(nc, gfp_mask);
Alexander Duyckb63ae8c2015-05-06 21:11:57 -07005082 if (!page)
5083 return NULL;
5084
5085#if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE)
5086 /* if size can vary use size else just use PAGE_SIZE */
5087 size = nc->size;
5088#endif
5089 /* Even if we own the page, we do not use atomic_set().
5090 * This would break get_page_unless_zero() users.
5091 */
Alexander Duyck86447722019-02-15 14:44:12 -08005092 page_ref_add(page, PAGE_FRAG_CACHE_MAX_SIZE);
Alexander Duyckb63ae8c2015-05-06 21:11:57 -07005093
5094 /* reset page count bias and offset to start of new frag */
Michal Hocko2f064f32015-08-21 14:11:51 -07005095 nc->pfmemalloc = page_is_pfmemalloc(page);
Alexander Duyck86447722019-02-15 14:44:12 -08005096 nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1;
Alexander Duyckb63ae8c2015-05-06 21:11:57 -07005097 nc->offset = size;
5098 }
5099
5100 offset = nc->offset - fragsz;
5101 if (unlikely(offset < 0)) {
5102 page = virt_to_page(nc->va);
5103
Joonsoo Kimfe896d12016-03-17 14:19:26 -07005104 if (!page_ref_sub_and_test(page, nc->pagecnt_bias))
Alexander Duyckb63ae8c2015-05-06 21:11:57 -07005105 goto refill;
5106
Dongli Zhangd8c19012020-11-15 12:10:29 -08005107 if (unlikely(nc->pfmemalloc)) {
5108 free_the_page(page, compound_order(page));
5109 goto refill;
5110 }
5111
Alexander Duyckb63ae8c2015-05-06 21:11:57 -07005112#if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE)
5113 /* if size can vary use size else just use PAGE_SIZE */
5114 size = nc->size;
5115#endif
5116 /* OK, page count is 0, we can safely set it */
Alexander Duyck86447722019-02-15 14:44:12 -08005117 set_page_count(page, PAGE_FRAG_CACHE_MAX_SIZE + 1);
Alexander Duyckb63ae8c2015-05-06 21:11:57 -07005118
5119 /* reset page count bias and offset to start of new frag */
Alexander Duyck86447722019-02-15 14:44:12 -08005120 nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1;
Alexander Duyckb63ae8c2015-05-06 21:11:57 -07005121 offset = size - fragsz;
5122 }
5123
5124 nc->pagecnt_bias--;
5125 nc->offset = offset;
5126
5127 return nc->va + offset;
5128}
Alexander Duyck8c2dd3e2017-01-10 16:58:06 -08005129EXPORT_SYMBOL(page_frag_alloc);
Alexander Duyckb63ae8c2015-05-06 21:11:57 -07005130
5131/*
5132 * Frees a page fragment allocated out of either a compound or order 0 page.
5133 */
Alexander Duyck8c2dd3e2017-01-10 16:58:06 -08005134void page_frag_free(void *addr)
Alexander Duyckb63ae8c2015-05-06 21:11:57 -07005135{
5136 struct page *page = virt_to_head_page(addr);
5137
Aaron Lu742aa7f2018-12-28 00:35:22 -08005138 if (unlikely(put_page_testzero(page)))
5139 free_the_page(page, compound_order(page));
Alexander Duyckb63ae8c2015-05-06 21:11:57 -07005140}
Alexander Duyck8c2dd3e2017-01-10 16:58:06 -08005141EXPORT_SYMBOL(page_frag_free);
Alexander Duyckb63ae8c2015-05-06 21:11:57 -07005142
Kirill A. Shutemovd00181b2015-11-06 16:29:57 -08005143static void *make_alloc_exact(unsigned long addr, unsigned int order,
5144 size_t size)
Andi Kleenee85c2e2011-05-11 15:13:34 -07005145{
5146 if (addr) {
5147 unsigned long alloc_end = addr + (PAGE_SIZE << order);
5148 unsigned long used = addr + PAGE_ALIGN(size);
5149
5150 split_page(virt_to_page((void *)addr), order);
5151 while (used < alloc_end) {
5152 free_page(used);
5153 used += PAGE_SIZE;
5154 }
5155 }
5156 return (void *)addr;
5157}
5158
Timur Tabi2be0ffe2008-07-23 21:28:11 -07005159/**
5160 * alloc_pages_exact - allocate an exact number physically-contiguous pages.
5161 * @size: the number of bytes to allocate
Vlastimil Babka63931eb2019-05-13 17:16:47 -07005162 * @gfp_mask: GFP flags for the allocation, must not contain __GFP_COMP
Timur Tabi2be0ffe2008-07-23 21:28:11 -07005163 *
5164 * This function is similar to alloc_pages(), except that it allocates the
5165 * minimum number of pages to satisfy the request. alloc_pages() can only
5166 * allocate memory in power-of-two pages.
5167 *
5168 * This function is also limited by MAX_ORDER.
5169 *
5170 * Memory allocated by this function must be released by free_pages_exact().
Mike Rapoporta862f682019-03-05 15:48:42 -08005171 *
5172 * Return: pointer to the allocated area or %NULL in case of error.
Timur Tabi2be0ffe2008-07-23 21:28:11 -07005173 */
5174void *alloc_pages_exact(size_t size, gfp_t gfp_mask)
5175{
5176 unsigned int order = get_order(size);
5177 unsigned long addr;
5178
Vlastimil Babka63931eb2019-05-13 17:16:47 -07005179 if (WARN_ON_ONCE(gfp_mask & __GFP_COMP))
5180 gfp_mask &= ~__GFP_COMP;
5181
Timur Tabi2be0ffe2008-07-23 21:28:11 -07005182 addr = __get_free_pages(gfp_mask, order);
Andi Kleenee85c2e2011-05-11 15:13:34 -07005183 return make_alloc_exact(addr, order, size);
Timur Tabi2be0ffe2008-07-23 21:28:11 -07005184}
5185EXPORT_SYMBOL(alloc_pages_exact);
5186
5187/**
Andi Kleenee85c2e2011-05-11 15:13:34 -07005188 * alloc_pages_exact_nid - allocate an exact number of physically-contiguous
5189 * pages on a node.
Randy Dunlapb5e6ab52011-05-16 13:16:54 -07005190 * @nid: the preferred node ID where memory should be allocated
Andi Kleenee85c2e2011-05-11 15:13:34 -07005191 * @size: the number of bytes to allocate
Vlastimil Babka63931eb2019-05-13 17:16:47 -07005192 * @gfp_mask: GFP flags for the allocation, must not contain __GFP_COMP
Andi Kleenee85c2e2011-05-11 15:13:34 -07005193 *
5194 * Like alloc_pages_exact(), but try to allocate on node nid first before falling
5195 * back.
Mike Rapoporta862f682019-03-05 15:48:42 -08005196 *
5197 * Return: pointer to the allocated area or %NULL in case of error.
Andi Kleenee85c2e2011-05-11 15:13:34 -07005198 */
Fabian Fredericke1931812014-08-06 16:04:59 -07005199void * __meminit alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask)
Andi Kleenee85c2e2011-05-11 15:13:34 -07005200{
Kirill A. Shutemovd00181b2015-11-06 16:29:57 -08005201 unsigned int order = get_order(size);
Vlastimil Babka63931eb2019-05-13 17:16:47 -07005202 struct page *p;
5203
5204 if (WARN_ON_ONCE(gfp_mask & __GFP_COMP))
5205 gfp_mask &= ~__GFP_COMP;
5206
5207 p = alloc_pages_node(nid, gfp_mask, order);
Andi Kleenee85c2e2011-05-11 15:13:34 -07005208 if (!p)
5209 return NULL;
5210 return make_alloc_exact((unsigned long)page_address(p), order, size);
5211}
Andi Kleenee85c2e2011-05-11 15:13:34 -07005212
5213/**
Timur Tabi2be0ffe2008-07-23 21:28:11 -07005214 * free_pages_exact - release memory allocated via alloc_pages_exact()
5215 * @virt: the value returned by alloc_pages_exact.
5216 * @size: size of allocation, same value as passed to alloc_pages_exact().
5217 *
5218 * Release the memory allocated by a previous call to alloc_pages_exact.
5219 */
5220void free_pages_exact(void *virt, size_t size)
5221{
5222 unsigned long addr = (unsigned long)virt;
5223 unsigned long end = addr + PAGE_ALIGN(size);
5224
5225 while (addr < end) {
5226 free_page(addr);
5227 addr += PAGE_SIZE;
5228 }
5229}
5230EXPORT_SYMBOL(free_pages_exact);
5231
Zhang Yanfeie0fb5812013-02-22 16:35:54 -08005232/**
5233 * nr_free_zone_pages - count number of pages beyond high watermark
5234 * @offset: The zone index of the highest zone
5235 *
Mike Rapoporta862f682019-03-05 15:48:42 -08005236 * nr_free_zone_pages() counts the number of pages which are beyond the
Zhang Yanfeie0fb5812013-02-22 16:35:54 -08005237 * high watermark within all zones at or below a given zone index. For each
5238 * zone, the number of pages is calculated as:
mchehab@s-opensource.com0e056eb2017-03-30 17:11:36 -03005239 *
5240 * nr_free_zone_pages = managed_pages - high_pages
Mike Rapoporta862f682019-03-05 15:48:42 -08005241 *
5242 * Return: number of pages beyond high watermark.
Zhang Yanfeie0fb5812013-02-22 16:35:54 -08005243 */
Zhang Yanfeiebec3862013-02-22 16:35:43 -08005244static unsigned long nr_free_zone_pages(int offset)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005245{
Mel Gormandd1a2392008-04-28 02:12:17 -07005246 struct zoneref *z;
Mel Gorman54a6eb52008-04-28 02:12:16 -07005247 struct zone *zone;
5248
Martin J. Blighe310fd42005-07-29 22:59:18 -07005249 /* Just pick one node, since fallback list is circular */
Zhang Yanfeiebec3862013-02-22 16:35:43 -08005250 unsigned long sum = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005251
Mel Gorman0e884602008-04-28 02:12:14 -07005252 struct zonelist *zonelist = node_zonelist(numa_node_id(), GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005253
Mel Gorman54a6eb52008-04-28 02:12:16 -07005254 for_each_zone_zonelist(zone, z, zonelist, offset) {
Arun KS9705bea2018-12-28 00:34:24 -08005255 unsigned long size = zone_managed_pages(zone);
Mel Gorman41858962009-06-16 15:32:12 -07005256 unsigned long high = high_wmark_pages(zone);
Martin J. Blighe310fd42005-07-29 22:59:18 -07005257 if (size > high)
5258 sum += size - high;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005259 }
5260
5261 return sum;
5262}
5263
Zhang Yanfeie0fb5812013-02-22 16:35:54 -08005264/**
5265 * nr_free_buffer_pages - count number of pages beyond high watermark
5266 *
5267 * nr_free_buffer_pages() counts the number of pages which are beyond the high
5268 * watermark within ZONE_DMA and ZONE_NORMAL.
Mike Rapoporta862f682019-03-05 15:48:42 -08005269 *
5270 * Return: number of pages beyond high watermark within ZONE_DMA and
5271 * ZONE_NORMAL.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005272 */
Zhang Yanfeiebec3862013-02-22 16:35:43 -08005273unsigned long nr_free_buffer_pages(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005274{
Al Viroaf4ca452005-10-21 02:55:38 -04005275 return nr_free_zone_pages(gfp_zone(GFP_USER));
Linus Torvalds1da177e2005-04-16 15:20:36 -07005276}
Meelap Shahc2f1a552007-07-17 04:04:39 -07005277EXPORT_SYMBOL_GPL(nr_free_buffer_pages);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005278
Christoph Lameter08e0f6a2006-09-27 01:50:06 -07005279static inline void show_node(struct zone *zone)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005280{
Kirill A. Shutemove5adfff2012-12-11 16:00:29 -08005281 if (IS_ENABLED(CONFIG_NUMA))
Andy Whitcroft25ba77c2006-12-06 20:33:03 -08005282 printk("Node %d ", zone_to_nid(zone));
Linus Torvalds1da177e2005-04-16 15:20:36 -07005283}
Linus Torvalds1da177e2005-04-16 15:20:36 -07005284
Igor Redkod02bd272016-03-17 14:19:05 -07005285long si_mem_available(void)
5286{
5287 long available;
5288 unsigned long pagecache;
5289 unsigned long wmark_low = 0;
5290 unsigned long pages[NR_LRU_LISTS];
Vlastimil Babkab29940c2018-10-26 15:05:46 -07005291 unsigned long reclaimable;
Igor Redkod02bd272016-03-17 14:19:05 -07005292 struct zone *zone;
5293 int lru;
5294
5295 for (lru = LRU_BASE; lru < NR_LRU_LISTS; lru++)
Mel Gorman2f95ff92016-08-11 15:32:57 -07005296 pages[lru] = global_node_page_state(NR_LRU_BASE + lru);
Igor Redkod02bd272016-03-17 14:19:05 -07005297
5298 for_each_zone(zone)
Mel Gormana9214442018-12-28 00:35:44 -08005299 wmark_low += low_wmark_pages(zone);
Igor Redkod02bd272016-03-17 14:19:05 -07005300
5301 /*
5302 * Estimate the amount of memory available for userspace allocations,
5303 * without causing swapping.
5304 */
Michal Hockoc41f0122017-09-06 16:23:36 -07005305 available = global_zone_page_state(NR_FREE_PAGES) - totalreserve_pages;
Igor Redkod02bd272016-03-17 14:19:05 -07005306
5307 /*
5308 * Not all the page cache can be freed, otherwise the system will
5309 * start swapping. Assume at least half of the page cache, or the
5310 * low watermark worth of cache, needs to stay.
5311 */
5312 pagecache = pages[LRU_ACTIVE_FILE] + pages[LRU_INACTIVE_FILE];
5313 pagecache -= min(pagecache / 2, wmark_low);
5314 available += pagecache;
5315
5316 /*
Vlastimil Babkab29940c2018-10-26 15:05:46 -07005317 * Part of the reclaimable slab and other kernel memory consists of
5318 * items that are in use, and cannot be freed. Cap this estimate at the
5319 * low watermark.
Igor Redkod02bd272016-03-17 14:19:05 -07005320 */
Roman Gushchind42f3242020-08-06 23:20:39 -07005321 reclaimable = global_node_page_state_pages(NR_SLAB_RECLAIMABLE_B) +
5322 global_node_page_state(NR_KERNEL_MISC_RECLAIMABLE);
Vlastimil Babkab29940c2018-10-26 15:05:46 -07005323 available += reclaimable - min(reclaimable / 2, wmark_low);
Roman Gushchin034ebf62018-04-10 16:27:40 -07005324
Igor Redkod02bd272016-03-17 14:19:05 -07005325 if (available < 0)
5326 available = 0;
5327 return available;
5328}
5329EXPORT_SYMBOL_GPL(si_mem_available);
5330
Linus Torvalds1da177e2005-04-16 15:20:36 -07005331void si_meminfo(struct sysinfo *val)
5332{
Arun KSca79b0c2018-12-28 00:34:29 -08005333 val->totalram = totalram_pages();
Mel Gorman11fb9982016-07-28 15:46:20 -07005334 val->sharedram = global_node_page_state(NR_SHMEM);
Michal Hockoc41f0122017-09-06 16:23:36 -07005335 val->freeram = global_zone_page_state(NR_FREE_PAGES);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005336 val->bufferram = nr_blockdev_pages();
Arun KSca79b0c2018-12-28 00:34:29 -08005337 val->totalhigh = totalhigh_pages();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005338 val->freehigh = nr_free_highpages();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005339 val->mem_unit = PAGE_SIZE;
5340}
5341
5342EXPORT_SYMBOL(si_meminfo);
5343
5344#ifdef CONFIG_NUMA
5345void si_meminfo_node(struct sysinfo *val, int nid)
5346{
Jiang Liucdd91a72013-07-03 15:03:27 -07005347 int zone_type; /* needs to be signed */
5348 unsigned long managed_pages = 0;
Joonsoo Kimfc2bd792016-05-19 17:12:23 -07005349 unsigned long managed_highpages = 0;
5350 unsigned long free_highpages = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005351 pg_data_t *pgdat = NODE_DATA(nid);
5352
Jiang Liucdd91a72013-07-03 15:03:27 -07005353 for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++)
Arun KS9705bea2018-12-28 00:34:24 -08005354 managed_pages += zone_managed_pages(&pgdat->node_zones[zone_type]);
Jiang Liucdd91a72013-07-03 15:03:27 -07005355 val->totalram = managed_pages;
Mel Gorman11fb9982016-07-28 15:46:20 -07005356 val->sharedram = node_page_state(pgdat, NR_SHMEM);
Mel Gorman75ef7182016-07-28 15:45:24 -07005357 val->freeram = sum_zone_node_page_state(nid, NR_FREE_PAGES);
Christoph Lameter98d2b0e2006-09-25 23:31:12 -07005358#ifdef CONFIG_HIGHMEM
Joonsoo Kimfc2bd792016-05-19 17:12:23 -07005359 for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) {
5360 struct zone *zone = &pgdat->node_zones[zone_type];
5361
5362 if (is_highmem(zone)) {
Arun KS9705bea2018-12-28 00:34:24 -08005363 managed_highpages += zone_managed_pages(zone);
Joonsoo Kimfc2bd792016-05-19 17:12:23 -07005364 free_highpages += zone_page_state(zone, NR_FREE_PAGES);
5365 }
5366 }
5367 val->totalhigh = managed_highpages;
5368 val->freehigh = free_highpages;
Christoph Lameter98d2b0e2006-09-25 23:31:12 -07005369#else
Joonsoo Kimfc2bd792016-05-19 17:12:23 -07005370 val->totalhigh = managed_highpages;
5371 val->freehigh = free_highpages;
Christoph Lameter98d2b0e2006-09-25 23:31:12 -07005372#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07005373 val->mem_unit = PAGE_SIZE;
5374}
5375#endif
5376
David Rientjesddd588b2011-03-22 16:30:46 -07005377/*
David Rientjes7bf02ea2011-05-24 17:11:16 -07005378 * Determine whether the node should be displayed or not, depending on whether
5379 * SHOW_MEM_FILTER_NODES was passed to show_free_areas().
David Rientjesddd588b2011-03-22 16:30:46 -07005380 */
Michal Hocko9af744d2017-02-22 15:46:16 -08005381static bool show_mem_node_skip(unsigned int flags, int nid, nodemask_t *nodemask)
David Rientjesddd588b2011-03-22 16:30:46 -07005382{
David Rientjesddd588b2011-03-22 16:30:46 -07005383 if (!(flags & SHOW_MEM_FILTER_NODES))
Michal Hocko9af744d2017-02-22 15:46:16 -08005384 return false;
David Rientjesddd588b2011-03-22 16:30:46 -07005385
Michal Hocko9af744d2017-02-22 15:46:16 -08005386 /*
5387 * no node mask - aka implicit memory numa policy. Do not bother with
5388 * the synchronization - read_mems_allowed_begin - because we do not
5389 * have to be precise here.
5390 */
5391 if (!nodemask)
5392 nodemask = &cpuset_current_mems_allowed;
5393
5394 return !node_isset(nid, *nodemask);
David Rientjesddd588b2011-03-22 16:30:46 -07005395}
5396
Linus Torvalds1da177e2005-04-16 15:20:36 -07005397#define K(x) ((x) << (PAGE_SHIFT-10))
5398
Rabin Vincent377e4f12012-12-11 16:00:24 -08005399static void show_migration_types(unsigned char type)
5400{
5401 static const char types[MIGRATE_TYPES] = {
5402 [MIGRATE_UNMOVABLE] = 'U',
Rabin Vincent377e4f12012-12-11 16:00:24 -08005403 [MIGRATE_MOVABLE] = 'M',
Vlastimil Babka475a2f92015-12-11 13:40:29 -08005404 [MIGRATE_RECLAIMABLE] = 'E',
5405 [MIGRATE_HIGHATOMIC] = 'H',
Rabin Vincent377e4f12012-12-11 16:00:24 -08005406#ifdef CONFIG_CMA
5407 [MIGRATE_CMA] = 'C',
5408#endif
Minchan Kim194159f2013-02-22 16:33:58 -08005409#ifdef CONFIG_MEMORY_ISOLATION
Rabin Vincent377e4f12012-12-11 16:00:24 -08005410 [MIGRATE_ISOLATE] = 'I',
Minchan Kim194159f2013-02-22 16:33:58 -08005411#endif
Rabin Vincent377e4f12012-12-11 16:00:24 -08005412 };
5413 char tmp[MIGRATE_TYPES + 1];
5414 char *p = tmp;
5415 int i;
5416
5417 for (i = 0; i < MIGRATE_TYPES; i++) {
5418 if (type & (1 << i))
5419 *p++ = types[i];
5420 }
5421
5422 *p = '\0';
Joe Perches1f84a182016-10-27 17:46:29 -07005423 printk(KERN_CONT "(%s) ", tmp);
Rabin Vincent377e4f12012-12-11 16:00:24 -08005424}
5425
Linus Torvalds1da177e2005-04-16 15:20:36 -07005426/*
5427 * Show free area list (used inside shift_scroll-lock stuff)
5428 * We also calculate the percentage fragmentation. We do this by counting the
5429 * memory on each free list with the exception of the first item on the list.
Konstantin Khlebnikovd1bfcdb2015-04-14 15:45:30 -07005430 *
5431 * Bits in @filter:
5432 * SHOW_MEM_FILTER_NODES: suppress nodes that are not allowed by current's
5433 * cpuset.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005434 */
Michal Hocko9af744d2017-02-22 15:46:16 -08005435void show_free_areas(unsigned int filter, nodemask_t *nodemask)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005436{
Konstantin Khlebnikovd1bfcdb2015-04-14 15:45:30 -07005437 unsigned long free_pcp = 0;
Jes Sorensenc7241912006-09-27 01:50:05 -07005438 int cpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005439 struct zone *zone;
Mel Gorman599d0c92016-07-28 15:45:31 -07005440 pg_data_t *pgdat;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005441
KOSAKI Motohiroee99c712009-03-31 15:19:31 -07005442 for_each_populated_zone(zone) {
Michal Hocko9af744d2017-02-22 15:46:16 -08005443 if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask))
David Rientjesddd588b2011-03-22 16:30:46 -07005444 continue;
Konstantin Khlebnikovd1bfcdb2015-04-14 15:45:30 -07005445
Konstantin Khlebnikov761b0672015-04-14 15:45:32 -07005446 for_each_online_cpu(cpu)
5447 free_pcp += per_cpu_ptr(zone->pageset, cpu)->pcp.count;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005448 }
5449
KOSAKI Motohiroa7312862009-09-21 17:01:37 -07005450 printk("active_anon:%lu inactive_anon:%lu isolated_anon:%lu\n"
5451 " active_file:%lu inactive_file:%lu isolated_file:%lu\n"
NeilBrown8d928902020-06-01 21:48:21 -07005452 " unevictable:%lu dirty:%lu writeback:%lu\n"
Konstantin Khlebnikovd1bfcdb2015-04-14 15:45:30 -07005453 " slab_reclaimable:%lu slab_unreclaimable:%lu\n"
Bartlomiej Zolnierkiewiczd1ce7492012-10-08 16:32:02 -07005454 " mapped:%lu shmem:%lu pagetables:%lu bounce:%lu\n"
Konstantin Khlebnikovd1bfcdb2015-04-14 15:45:30 -07005455 " free:%lu free_pcp:%lu free_cma:%lu\n",
Mel Gorman599d0c92016-07-28 15:45:31 -07005456 global_node_page_state(NR_ACTIVE_ANON),
5457 global_node_page_state(NR_INACTIVE_ANON),
5458 global_node_page_state(NR_ISOLATED_ANON),
5459 global_node_page_state(NR_ACTIVE_FILE),
5460 global_node_page_state(NR_INACTIVE_FILE),
5461 global_node_page_state(NR_ISOLATED_FILE),
5462 global_node_page_state(NR_UNEVICTABLE),
Mel Gorman11fb9982016-07-28 15:46:20 -07005463 global_node_page_state(NR_FILE_DIRTY),
5464 global_node_page_state(NR_WRITEBACK),
Roman Gushchind42f3242020-08-06 23:20:39 -07005465 global_node_page_state_pages(NR_SLAB_RECLAIMABLE_B),
5466 global_node_page_state_pages(NR_SLAB_UNRECLAIMABLE_B),
Mel Gorman50658e22016-07-28 15:46:14 -07005467 global_node_page_state(NR_FILE_MAPPED),
Mel Gorman11fb9982016-07-28 15:46:20 -07005468 global_node_page_state(NR_SHMEM),
Michal Hockoc41f0122017-09-06 16:23:36 -07005469 global_zone_page_state(NR_PAGETABLE),
5470 global_zone_page_state(NR_BOUNCE),
5471 global_zone_page_state(NR_FREE_PAGES),
Konstantin Khlebnikovd1bfcdb2015-04-14 15:45:30 -07005472 free_pcp,
Michal Hockoc41f0122017-09-06 16:23:36 -07005473 global_zone_page_state(NR_FREE_CMA_PAGES));
Linus Torvalds1da177e2005-04-16 15:20:36 -07005474
Mel Gorman599d0c92016-07-28 15:45:31 -07005475 for_each_online_pgdat(pgdat) {
Michal Hocko9af744d2017-02-22 15:46:16 -08005476 if (show_mem_node_skip(filter, pgdat->node_id, nodemask))
Michal Hockoc02e50b2017-02-22 15:46:07 -08005477 continue;
5478
Mel Gorman599d0c92016-07-28 15:45:31 -07005479 printk("Node %d"
5480 " active_anon:%lukB"
5481 " inactive_anon:%lukB"
5482 " active_file:%lukB"
5483 " inactive_file:%lukB"
5484 " unevictable:%lukB"
5485 " isolated(anon):%lukB"
5486 " isolated(file):%lukB"
Mel Gorman50658e22016-07-28 15:46:14 -07005487 " mapped:%lukB"
Mel Gorman11fb9982016-07-28 15:46:20 -07005488 " dirty:%lukB"
5489 " writeback:%lukB"
5490 " shmem:%lukB"
5491#ifdef CONFIG_TRANSPARENT_HUGEPAGE
5492 " shmem_thp: %lukB"
5493 " shmem_pmdmapped: %lukB"
5494 " anon_thp: %lukB"
5495#endif
5496 " writeback_tmp:%lukB"
Shakeel Butt991e7672020-08-06 23:21:37 -07005497 " kernel_stack:%lukB"
5498#ifdef CONFIG_SHADOW_CALL_STACK
5499 " shadow_call_stack:%lukB"
5500#endif
Mel Gorman599d0c92016-07-28 15:45:31 -07005501 " all_unreclaimable? %s"
5502 "\n",
5503 pgdat->node_id,
5504 K(node_page_state(pgdat, NR_ACTIVE_ANON)),
5505 K(node_page_state(pgdat, NR_INACTIVE_ANON)),
5506 K(node_page_state(pgdat, NR_ACTIVE_FILE)),
5507 K(node_page_state(pgdat, NR_INACTIVE_FILE)),
5508 K(node_page_state(pgdat, NR_UNEVICTABLE)),
5509 K(node_page_state(pgdat, NR_ISOLATED_ANON)),
5510 K(node_page_state(pgdat, NR_ISOLATED_FILE)),
Mel Gorman50658e22016-07-28 15:46:14 -07005511 K(node_page_state(pgdat, NR_FILE_MAPPED)),
Mel Gorman11fb9982016-07-28 15:46:20 -07005512 K(node_page_state(pgdat, NR_FILE_DIRTY)),
5513 K(node_page_state(pgdat, NR_WRITEBACK)),
Alexander Polakov1f06b812017-04-07 16:04:45 -07005514 K(node_page_state(pgdat, NR_SHMEM)),
Mel Gorman11fb9982016-07-28 15:46:20 -07005515#ifdef CONFIG_TRANSPARENT_HUGEPAGE
5516 K(node_page_state(pgdat, NR_SHMEM_THPS) * HPAGE_PMD_NR),
5517 K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED)
5518 * HPAGE_PMD_NR),
5519 K(node_page_state(pgdat, NR_ANON_THPS) * HPAGE_PMD_NR),
5520#endif
Mel Gorman11fb9982016-07-28 15:46:20 -07005521 K(node_page_state(pgdat, NR_WRITEBACK_TEMP)),
Shakeel Butt991e7672020-08-06 23:21:37 -07005522 node_page_state(pgdat, NR_KERNEL_STACK_KB),
5523#ifdef CONFIG_SHADOW_CALL_STACK
5524 node_page_state(pgdat, NR_KERNEL_SCS_KB),
5525#endif
Johannes Weinerc73322d2017-05-03 14:51:51 -07005526 pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES ?
5527 "yes" : "no");
Mel Gorman599d0c92016-07-28 15:45:31 -07005528 }
5529
KOSAKI Motohiroee99c712009-03-31 15:19:31 -07005530 for_each_populated_zone(zone) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07005531 int i;
5532
Michal Hocko9af744d2017-02-22 15:46:16 -08005533 if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask))
David Rientjesddd588b2011-03-22 16:30:46 -07005534 continue;
Konstantin Khlebnikovd1bfcdb2015-04-14 15:45:30 -07005535
5536 free_pcp = 0;
5537 for_each_online_cpu(cpu)
5538 free_pcp += per_cpu_ptr(zone->pageset, cpu)->pcp.count;
5539
Linus Torvalds1da177e2005-04-16 15:20:36 -07005540 show_node(zone);
Joe Perches1f84a182016-10-27 17:46:29 -07005541 printk(KERN_CONT
5542 "%s"
Linus Torvalds1da177e2005-04-16 15:20:36 -07005543 " free:%lukB"
5544 " min:%lukB"
5545 " low:%lukB"
5546 " high:%lukB"
lijiazie47b3462019-11-30 17:55:21 -08005547 " reserved_highatomic:%luKB"
Minchan Kim71c799f2016-07-28 15:47:26 -07005548 " active_anon:%lukB"
5549 " inactive_anon:%lukB"
5550 " active_file:%lukB"
5551 " inactive_file:%lukB"
5552 " unevictable:%lukB"
Mel Gorman5a1c84b2016-07-28 15:47:31 -07005553 " writepending:%lukB"
Linus Torvalds1da177e2005-04-16 15:20:36 -07005554 " present:%lukB"
Jiang Liu9feedc92012-12-12 13:52:12 -08005555 " managed:%lukB"
KOSAKI Motohiro4a0aa732009-09-21 17:01:30 -07005556 " mlocked:%lukB"
KOSAKI Motohiro4a0aa732009-09-21 17:01:30 -07005557 " pagetables:%lukB"
KOSAKI Motohiro4a0aa732009-09-21 17:01:30 -07005558 " bounce:%lukB"
Konstantin Khlebnikovd1bfcdb2015-04-14 15:45:30 -07005559 " free_pcp:%lukB"
5560 " local_pcp:%ukB"
Bartlomiej Zolnierkiewiczd1ce7492012-10-08 16:32:02 -07005561 " free_cma:%lukB"
Linus Torvalds1da177e2005-04-16 15:20:36 -07005562 "\n",
5563 zone->name,
Mel Gorman88f5acf2011-01-13 15:45:41 -08005564 K(zone_page_state(zone, NR_FREE_PAGES)),
Mel Gorman41858962009-06-16 15:32:12 -07005565 K(min_wmark_pages(zone)),
5566 K(low_wmark_pages(zone)),
5567 K(high_wmark_pages(zone)),
lijiazie47b3462019-11-30 17:55:21 -08005568 K(zone->nr_reserved_highatomic),
Minchan Kim71c799f2016-07-28 15:47:26 -07005569 K(zone_page_state(zone, NR_ZONE_ACTIVE_ANON)),
5570 K(zone_page_state(zone, NR_ZONE_INACTIVE_ANON)),
5571 K(zone_page_state(zone, NR_ZONE_ACTIVE_FILE)),
5572 K(zone_page_state(zone, NR_ZONE_INACTIVE_FILE)),
5573 K(zone_page_state(zone, NR_ZONE_UNEVICTABLE)),
Mel Gorman5a1c84b2016-07-28 15:47:31 -07005574 K(zone_page_state(zone, NR_ZONE_WRITE_PENDING)),
Linus Torvalds1da177e2005-04-16 15:20:36 -07005575 K(zone->present_pages),
Arun KS9705bea2018-12-28 00:34:24 -08005576 K(zone_managed_pages(zone)),
KOSAKI Motohiro4a0aa732009-09-21 17:01:30 -07005577 K(zone_page_state(zone, NR_MLOCK)),
KOSAKI Motohiro4a0aa732009-09-21 17:01:30 -07005578 K(zone_page_state(zone, NR_PAGETABLE)),
KOSAKI Motohiro4a0aa732009-09-21 17:01:30 -07005579 K(zone_page_state(zone, NR_BOUNCE)),
Konstantin Khlebnikovd1bfcdb2015-04-14 15:45:30 -07005580 K(free_pcp),
5581 K(this_cpu_read(zone->pageset->pcp.count)),
Minchan Kim33e077b2016-07-28 15:47:14 -07005582 K(zone_page_state(zone, NR_FREE_CMA_PAGES)));
Linus Torvalds1da177e2005-04-16 15:20:36 -07005583 printk("lowmem_reserve[]:");
5584 for (i = 0; i < MAX_NR_ZONES; i++)
Joe Perches1f84a182016-10-27 17:46:29 -07005585 printk(KERN_CONT " %ld", zone->lowmem_reserve[i]);
5586 printk(KERN_CONT "\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -07005587 }
5588
KOSAKI Motohiroee99c712009-03-31 15:19:31 -07005589 for_each_populated_zone(zone) {
Kirill A. Shutemovd00181b2015-11-06 16:29:57 -08005590 unsigned int order;
5591 unsigned long nr[MAX_ORDER], flags, total = 0;
Rabin Vincent377e4f12012-12-11 16:00:24 -08005592 unsigned char types[MAX_ORDER];
Linus Torvalds1da177e2005-04-16 15:20:36 -07005593
Michal Hocko9af744d2017-02-22 15:46:16 -08005594 if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask))
David Rientjesddd588b2011-03-22 16:30:46 -07005595 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005596 show_node(zone);
Joe Perches1f84a182016-10-27 17:46:29 -07005597 printk(KERN_CONT "%s: ", zone->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005598
5599 spin_lock_irqsave(&zone->lock, flags);
5600 for (order = 0; order < MAX_ORDER; order++) {
Rabin Vincent377e4f12012-12-11 16:00:24 -08005601 struct free_area *area = &zone->free_area[order];
5602 int type;
5603
5604 nr[order] = area->nr_free;
Kirill Korotaev8f9de512006-06-23 02:03:50 -07005605 total += nr[order] << order;
Rabin Vincent377e4f12012-12-11 16:00:24 -08005606
5607 types[order] = 0;
5608 for (type = 0; type < MIGRATE_TYPES; type++) {
Dan Williamsb03641a2019-05-14 15:41:32 -07005609 if (!free_area_empty(area, type))
Rabin Vincent377e4f12012-12-11 16:00:24 -08005610 types[order] |= 1 << type;
5611 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07005612 }
5613 spin_unlock_irqrestore(&zone->lock, flags);
Rabin Vincent377e4f12012-12-11 16:00:24 -08005614 for (order = 0; order < MAX_ORDER; order++) {
Joe Perches1f84a182016-10-27 17:46:29 -07005615 printk(KERN_CONT "%lu*%lukB ",
5616 nr[order], K(1UL) << order);
Rabin Vincent377e4f12012-12-11 16:00:24 -08005617 if (nr[order])
5618 show_migration_types(types[order]);
5619 }
Joe Perches1f84a182016-10-27 17:46:29 -07005620 printk(KERN_CONT "= %lukB\n", K(total));
Linus Torvalds1da177e2005-04-16 15:20:36 -07005621 }
5622
David Rientjes949f7ec2013-04-29 15:07:48 -07005623 hugetlb_show_meminfo();
5624
Mel Gorman11fb9982016-07-28 15:46:20 -07005625 printk("%ld total pagecache pages\n", global_node_page_state(NR_FILE_PAGES));
Larry Woodmane6f36022008-02-04 22:29:30 -08005626
Linus Torvalds1da177e2005-04-16 15:20:36 -07005627 show_swap_cache_info();
5628}
5629
Mel Gorman19770b32008-04-28 02:12:18 -07005630static void zoneref_set_zone(struct zone *zone, struct zoneref *zoneref)
5631{
5632 zoneref->zone = zone;
5633 zoneref->zone_idx = zone_idx(zone);
5634}
5635
Linus Torvalds1da177e2005-04-16 15:20:36 -07005636/*
5637 * Builds allocation fallback zone lists.
Christoph Lameter1a932052006-01-06 00:11:16 -08005638 *
5639 * Add all populated zones of a node to the zonelist.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005640 */
Michal Hocko9d3be212017-09-06 16:20:30 -07005641static int build_zonerefs_node(pg_data_t *pgdat, struct zoneref *zonerefs)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005642{
Christoph Lameter1a932052006-01-06 00:11:16 -08005643 struct zone *zone;
Zhang Yanfeibc732f12013-07-08 16:00:06 -07005644 enum zone_type zone_type = MAX_NR_ZONES;
Michal Hocko9d3be212017-09-06 16:20:30 -07005645 int nr_zones = 0;
Christoph Lameter02a68a52006-01-06 00:11:18 -08005646
5647 do {
Christoph Lameter2f6726e2006-09-25 23:31:18 -07005648 zone_type--;
Christoph Lameter070f8032006-01-06 00:11:19 -08005649 zone = pgdat->node_zones + zone_type;
Mel Gorman6aa303d2016-09-01 16:14:55 -07005650 if (managed_zone(zone)) {
Michal Hocko9d3be212017-09-06 16:20:30 -07005651 zoneref_set_zone(zone, &zonerefs[nr_zones++]);
Christoph Lameter070f8032006-01-06 00:11:19 -08005652 check_highest_zone(zone_type);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005653 }
Christoph Lameter2f6726e2006-09-25 23:31:18 -07005654 } while (zone_type);
Zhang Yanfeibc732f12013-07-08 16:00:06 -07005655
Christoph Lameter070f8032006-01-06 00:11:19 -08005656 return nr_zones;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005657}
5658
5659#ifdef CONFIG_NUMA
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07005660
5661static int __parse_numa_zonelist_order(char *s)
5662{
Michal Hockoc9bff3e2017-09-06 16:20:13 -07005663 /*
5664 * We used to support different zonlists modes but they turned
5665 * out to be just not useful. Let's keep the warning in place
5666 * if somebody still use the cmd line parameter so that we do
5667 * not fail it silently
5668 */
5669 if (!(*s == 'd' || *s == 'D' || *s == 'n' || *s == 'N')) {
5670 pr_warn("Ignoring unsupported numa_zonelist_order value: %s\n", s);
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07005671 return -EINVAL;
5672 }
5673 return 0;
5674}
5675
Michal Hockoc9bff3e2017-09-06 16:20:13 -07005676char numa_zonelist_order[] = "Node";
5677
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07005678/*
5679 * sysctl handler for numa_zonelist_order
5680 */
Joe Perchescccad5b2014-06-06 14:38:09 -07005681int numa_zonelist_order_handler(struct ctl_table *table, int write,
Christoph Hellwig32927392020-04-24 08:43:38 +02005682 void *buffer, size_t *length, loff_t *ppos)
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07005683{
Christoph Hellwig32927392020-04-24 08:43:38 +02005684 if (write)
5685 return __parse_numa_zonelist_order(buffer);
5686 return proc_dostring(table, write, buffer, length, ppos);
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07005687}
5688
5689
Christoph Lameter62bc62a2009-06-16 15:32:15 -07005690#define MAX_NODE_LOAD (nr_online_nodes)
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07005691static int node_load[MAX_NUMNODES];
5692
Linus Torvalds1da177e2005-04-16 15:20:36 -07005693/**
Pavel Pisa4dc3b162005-05-01 08:59:25 -07005694 * find_next_best_node - find the next node that should appear in a given node's fallback list
Linus Torvalds1da177e2005-04-16 15:20:36 -07005695 * @node: node whose fallback list we're appending
5696 * @used_node_mask: nodemask_t of already used nodes
5697 *
5698 * We use a number of factors to determine which is the next node that should
5699 * appear on a given node's fallback list. The node should not have appeared
5700 * already in @node's fallback list, and it should be the next closest node
5701 * according to the distance array (which contains arbitrary distance values
5702 * from each node to each node in the system), and should also prefer nodes
5703 * with no CPUs, since presumably they'll have very little allocation pressure
5704 * on them otherwise.
Mike Rapoporta862f682019-03-05 15:48:42 -08005705 *
5706 * Return: node id of the found node or %NUMA_NO_NODE if no node is found.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005707 */
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07005708static int find_next_best_node(int node, nodemask_t *used_node_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005709{
Linus Torvalds4cf808eb2006-02-17 20:38:21 +01005710 int n, val;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005711 int min_val = INT_MAX;
David Rientjes00ef2d22013-02-22 16:35:36 -08005712 int best_node = NUMA_NO_NODE;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005713
Linus Torvalds4cf808eb2006-02-17 20:38:21 +01005714 /* Use the local node if we haven't already */
5715 if (!node_isset(node, *used_node_mask)) {
5716 node_set(node, *used_node_mask);
5717 return node;
5718 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07005719
Lai Jiangshan4b0ef1fe2012-12-12 13:51:46 -08005720 for_each_node_state(n, N_MEMORY) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07005721
5722 /* Don't want a node to appear more than once */
5723 if (node_isset(n, *used_node_mask))
5724 continue;
5725
Linus Torvalds1da177e2005-04-16 15:20:36 -07005726 /* Use the distance array to find the distance */
5727 val = node_distance(node, n);
5728
Linus Torvalds4cf808eb2006-02-17 20:38:21 +01005729 /* Penalize nodes under us ("prefer the next node") */
5730 val += (n < node);
5731
Linus Torvalds1da177e2005-04-16 15:20:36 -07005732 /* Give preference to headless and unused nodes */
Mateusz Nosekb6307492020-10-13 16:55:42 -07005733 if (!cpumask_empty(cpumask_of_node(n)))
Linus Torvalds1da177e2005-04-16 15:20:36 -07005734 val += PENALTY_FOR_NODE_WITH_CPUS;
5735
5736 /* Slight preference for less loaded node */
5737 val *= (MAX_NODE_LOAD*MAX_NUMNODES);
5738 val += node_load[n];
5739
5740 if (val < min_val) {
5741 min_val = val;
5742 best_node = n;
5743 }
5744 }
5745
5746 if (best_node >= 0)
5747 node_set(best_node, *used_node_mask);
5748
5749 return best_node;
5750}
5751
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07005752
5753/*
5754 * Build zonelists ordered by node and zones within node.
5755 * This results in maximum locality--normal zone overflows into local
5756 * DMA zone, if any--but risks exhausting DMA zone.
5757 */
Michal Hocko9d3be212017-09-06 16:20:30 -07005758static void build_zonelists_in_node_order(pg_data_t *pgdat, int *node_order,
5759 unsigned nr_nodes)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005760{
Michal Hocko9d3be212017-09-06 16:20:30 -07005761 struct zoneref *zonerefs;
5762 int i;
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07005763
Michal Hocko9d3be212017-09-06 16:20:30 -07005764 zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK]._zonerefs;
5765
5766 for (i = 0; i < nr_nodes; i++) {
5767 int nr_zones;
5768
5769 pg_data_t *node = NODE_DATA(node_order[i]);
5770
5771 nr_zones = build_zonerefs_node(node, zonerefs);
5772 zonerefs += nr_zones;
5773 }
5774 zonerefs->zone = NULL;
5775 zonerefs->zone_idx = 0;
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07005776}
5777
5778/*
Christoph Lameter523b9452007-10-16 01:25:37 -07005779 * Build gfp_thisnode zonelists
5780 */
5781static void build_thisnode_zonelists(pg_data_t *pgdat)
5782{
Michal Hocko9d3be212017-09-06 16:20:30 -07005783 struct zoneref *zonerefs;
5784 int nr_zones;
Christoph Lameter523b9452007-10-16 01:25:37 -07005785
Michal Hocko9d3be212017-09-06 16:20:30 -07005786 zonerefs = pgdat->node_zonelists[ZONELIST_NOFALLBACK]._zonerefs;
5787 nr_zones = build_zonerefs_node(pgdat, zonerefs);
5788 zonerefs += nr_zones;
5789 zonerefs->zone = NULL;
5790 zonerefs->zone_idx = 0;
Christoph Lameter523b9452007-10-16 01:25:37 -07005791}
5792
5793/*
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07005794 * Build zonelists ordered by zone and nodes within zones.
5795 * This results in conserving DMA zone[s] until all Normal memory is
5796 * exhausted, but results in overflowing to remote node while memory
5797 * may still exist in local DMA zone.
5798 */
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07005799
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07005800static void build_zonelists(pg_data_t *pgdat)
5801{
Michal Hocko9d3be212017-09-06 16:20:30 -07005802 static int node_order[MAX_NUMNODES];
5803 int node, load, nr_nodes = 0;
Wei Yangd0ddf492020-06-03 15:59:05 -07005804 nodemask_t used_mask = NODE_MASK_NONE;
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07005805 int local_node, prev_node;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005806
5807 /* NUMA-aware ordering of nodes */
5808 local_node = pgdat->node_id;
Christoph Lameter62bc62a2009-06-16 15:32:15 -07005809 load = nr_online_nodes;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005810 prev_node = local_node;
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07005811
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07005812 memset(node_order, 0, sizeof(node_order));
Linus Torvalds1da177e2005-04-16 15:20:36 -07005813 while ((node = find_next_best_node(local_node, &used_mask)) >= 0) {
5814 /*
5815 * We don't want to pressure a particular node.
5816 * So adding penalty to the first node in same
5817 * distance group to make it round-robin.
5818 */
David Rientjes957f8222012-10-08 16:33:24 -07005819 if (node_distance(local_node, node) !=
5820 node_distance(local_node, prev_node))
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07005821 node_load[node] = load;
5822
Michal Hocko9d3be212017-09-06 16:20:30 -07005823 node_order[nr_nodes++] = node;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005824 prev_node = node;
5825 load--;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005826 }
Christoph Lameter523b9452007-10-16 01:25:37 -07005827
Michal Hocko9d3be212017-09-06 16:20:30 -07005828 build_zonelists_in_node_order(pgdat, node_order, nr_nodes);
Christoph Lameter523b9452007-10-16 01:25:37 -07005829 build_thisnode_zonelists(pgdat);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005830}
5831
Lee Schermerhorn7aac7892010-05-26 14:45:00 -07005832#ifdef CONFIG_HAVE_MEMORYLESS_NODES
5833/*
5834 * Return node id of node used for "local" allocations.
5835 * I.e., first node id of first zone in arg node's generic zonelist.
5836 * Used for initializing percpu 'numa_mem', which is used primarily
5837 * for kernel allocations, so use GFP_KERNEL flags to locate zonelist.
5838 */
5839int local_memory_node(int node)
5840{
Mel Gormanc33d6c02016-05-19 17:14:10 -07005841 struct zoneref *z;
Lee Schermerhorn7aac7892010-05-26 14:45:00 -07005842
Mel Gormanc33d6c02016-05-19 17:14:10 -07005843 z = first_zones_zonelist(node_zonelist(node, GFP_KERNEL),
Lee Schermerhorn7aac7892010-05-26 14:45:00 -07005844 gfp_zone(GFP_KERNEL),
Mel Gormanc33d6c02016-05-19 17:14:10 -07005845 NULL);
Pavel Tatashinc1093b72018-08-21 21:53:32 -07005846 return zone_to_nid(z->zone);
Lee Schermerhorn7aac7892010-05-26 14:45:00 -07005847}
5848#endif
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07005849
Joonsoo Kim6423aa82016-08-10 16:27:49 -07005850static void setup_min_unmapped_ratio(void);
5851static void setup_min_slab_ratio(void);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005852#else /* CONFIG_NUMA */
5853
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07005854static void build_zonelists(pg_data_t *pgdat)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005855{
Christoph Lameter19655d32006-09-25 23:31:19 -07005856 int node, local_node;
Michal Hocko9d3be212017-09-06 16:20:30 -07005857 struct zoneref *zonerefs;
5858 int nr_zones;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005859
5860 local_node = pgdat->node_id;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005861
Michal Hocko9d3be212017-09-06 16:20:30 -07005862 zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK]._zonerefs;
5863 nr_zones = build_zonerefs_node(pgdat, zonerefs);
5864 zonerefs += nr_zones;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005865
Mel Gorman54a6eb52008-04-28 02:12:16 -07005866 /*
5867 * Now we build the zonelist so that it contains the zones
5868 * of all the other nodes.
5869 * We don't want to pressure a particular node, so when
5870 * building the zones for node N, we make sure that the
5871 * zones coming right after the local ones are those from
5872 * node N+1 (modulo N)
5873 */
5874 for (node = local_node + 1; node < MAX_NUMNODES; node++) {
5875 if (!node_online(node))
5876 continue;
Michal Hocko9d3be212017-09-06 16:20:30 -07005877 nr_zones = build_zonerefs_node(NODE_DATA(node), zonerefs);
5878 zonerefs += nr_zones;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005879 }
Mel Gorman54a6eb52008-04-28 02:12:16 -07005880 for (node = 0; node < local_node; node++) {
5881 if (!node_online(node))
5882 continue;
Michal Hocko9d3be212017-09-06 16:20:30 -07005883 nr_zones = build_zonerefs_node(NODE_DATA(node), zonerefs);
5884 zonerefs += nr_zones;
Mel Gorman54a6eb52008-04-28 02:12:16 -07005885 }
5886
Michal Hocko9d3be212017-09-06 16:20:30 -07005887 zonerefs->zone = NULL;
5888 zonerefs->zone_idx = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005889}
5890
5891#endif /* CONFIG_NUMA */
5892
Christoph Lameter99dcc3e2010-01-05 15:34:51 +09005893/*
5894 * Boot pageset table. One per cpu which is going to be used for all
5895 * zones and all nodes. The parameters will be set in such a way
5896 * that an item put on a list will immediately be handed over to
5897 * the buddy list. This is safe since pageset manipulation is done
5898 * with interrupts disabled.
5899 *
5900 * The boot_pagesets must be kept even after bootup is complete for
5901 * unused processors and/or zones. They do play a role for bootstrapping
5902 * hotplugged processors.
5903 *
5904 * zoneinfo_show() and maybe other functions do
5905 * not check if the processor is online before following the pageset pointer.
5906 * Other parts of the kernel may not check if the zone is available.
5907 */
5908static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch);
5909static DEFINE_PER_CPU(struct per_cpu_pageset, boot_pageset);
Johannes Weiner385386c2017-07-06 15:40:43 -07005910static DEFINE_PER_CPU(struct per_cpu_nodestat, boot_nodestats);
Christoph Lameter99dcc3e2010-01-05 15:34:51 +09005911
Michal Hocko11cd8632017-09-06 16:20:34 -07005912static void __build_all_zonelists(void *data)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005913{
Yasunori Goto68113782006-06-23 02:03:11 -07005914 int nid;
Michal Hockoafb6ebb2017-09-06 16:20:17 -07005915 int __maybe_unused cpu;
Jiang Liu9adb62a2012-07-31 16:43:28 -07005916 pg_data_t *self = data;
Michal Hockob93e0f32017-09-06 16:20:37 -07005917 static DEFINE_SPINLOCK(lock);
5918
5919 spin_lock(&lock);
Paul Jackson9276b1bc2006-12-06 20:31:48 -08005920
Bo Liu7f9cfb32009-08-18 14:11:19 -07005921#ifdef CONFIG_NUMA
5922 memset(node_load, 0, sizeof(node_load));
5923#endif
Jiang Liu9adb62a2012-07-31 16:43:28 -07005924
Wei Yangc1152582017-09-06 16:19:33 -07005925 /*
5926 * This node is hotadded and no memory is yet present. So just
5927 * building zonelists is fine - no need to touch other nodes.
5928 */
Jiang Liu9adb62a2012-07-31 16:43:28 -07005929 if (self && !node_online(self->node_id)) {
5930 build_zonelists(self);
Wei Yangc1152582017-09-06 16:19:33 -07005931 } else {
5932 for_each_online_node(nid) {
5933 pg_data_t *pgdat = NODE_DATA(nid);
Jiang Liu9adb62a2012-07-31 16:43:28 -07005934
Wei Yangc1152582017-09-06 16:19:33 -07005935 build_zonelists(pgdat);
5936 }
Christoph Lameter99dcc3e2010-01-05 15:34:51 +09005937
Michal Hockoafb6ebb2017-09-06 16:20:17 -07005938#ifdef CONFIG_HAVE_MEMORYLESS_NODES
Michal Hockoafb6ebb2017-09-06 16:20:17 -07005939 /*
5940 * We now know the "local memory node" for each node--
5941 * i.e., the node of the first zone in the generic zonelist.
5942 * Set up numa_mem percpu variable for on-line cpus. During
5943 * boot, only the boot cpu should be on-line; we'll init the
5944 * secondary cpus' numa_mem as they come on-line. During
5945 * node/memory hotplug, we'll fixup all on-line cpus.
5946 */
Michal Hockod9c9a0b2017-09-06 16:20:20 -07005947 for_each_online_cpu(cpu)
Michal Hockoafb6ebb2017-09-06 16:20:17 -07005948 set_cpu_numa_mem(cpu, local_memory_node(cpu_to_node(cpu)));
Michal Hockoafb6ebb2017-09-06 16:20:17 -07005949#endif
Michal Hockod9c9a0b2017-09-06 16:20:20 -07005950 }
Michal Hockob93e0f32017-09-06 16:20:37 -07005951
5952 spin_unlock(&lock);
Michal Hockoafb6ebb2017-09-06 16:20:17 -07005953}
5954
5955static noinline void __init
5956build_all_zonelists_init(void)
5957{
5958 int cpu;
5959
5960 __build_all_zonelists(NULL);
5961
Christoph Lameter99dcc3e2010-01-05 15:34:51 +09005962 /*
5963 * Initialize the boot_pagesets that are going to be used
5964 * for bootstrapping processors. The real pagesets for
5965 * each zone will be allocated later when the per cpu
5966 * allocator is available.
5967 *
5968 * boot_pagesets are used also for bootstrapping offline
5969 * cpus if the system is already booted because the pagesets
5970 * are needed to initialize allocators on a specific cpu too.
5971 * F.e. the percpu allocator needs the page allocator which
5972 * needs the percpu allocator in order to allocate its pagesets
5973 * (a chicken-egg dilemma).
5974 */
Michal Hockoafb6ebb2017-09-06 16:20:17 -07005975 for_each_possible_cpu(cpu)
Christoph Lameter99dcc3e2010-01-05 15:34:51 +09005976 setup_pageset(&per_cpu(boot_pageset, cpu), 0);
5977
Rasmus Villemoes061f67b2015-02-12 15:00:06 -08005978 mminit_verify_zonelist();
5979 cpuset_init_current_mems_allowed();
5980}
5981
Haicheng Li4eaf3f62010-05-24 14:32:52 -07005982/*
Haicheng Li4eaf3f62010-05-24 14:32:52 -07005983 * unless system_state == SYSTEM_BOOTING.
Rasmus Villemoes061f67b2015-02-12 15:00:06 -08005984 *
Michal Hocko72675e12017-09-06 16:20:24 -07005985 * __ref due to call of __init annotated helper build_all_zonelists_init
Rasmus Villemoes061f67b2015-02-12 15:00:06 -08005986 * [protected by SYSTEM_BOOTING].
Haicheng Li4eaf3f62010-05-24 14:32:52 -07005987 */
Michal Hocko72675e12017-09-06 16:20:24 -07005988void __ref build_all_zonelists(pg_data_t *pgdat)
Yasunori Goto68113782006-06-23 02:03:11 -07005989{
David Hildenbrand0a18e602020-08-06 23:25:27 -07005990 unsigned long vm_total_pages;
5991
Yasunori Goto68113782006-06-23 02:03:11 -07005992 if (system_state == SYSTEM_BOOTING) {
Rasmus Villemoes061f67b2015-02-12 15:00:06 -08005993 build_all_zonelists_init();
Yasunori Goto68113782006-06-23 02:03:11 -07005994 } else {
Michal Hocko11cd8632017-09-06 16:20:34 -07005995 __build_all_zonelists(pgdat);
Yasunori Goto68113782006-06-23 02:03:11 -07005996 /* cpuset refresh routine should be here */
5997 }
David Hildenbrand56b94132020-08-06 23:25:30 -07005998 /* Get the number of free pages beyond high watermark in all zones. */
5999 vm_total_pages = nr_free_zone_pages(gfp_zone(GFP_HIGHUSER_MOVABLE));
Mel Gorman9ef9acb2007-10-16 01:25:54 -07006000 /*
6001 * Disable grouping by mobility if the number of pages in the
6002 * system is too low to allow the mechanism to work. It would be
6003 * more accurate, but expensive to check per-zone. This check is
6004 * made on memory-hotadd so a system can start with mobility
6005 * disabled and enable it later
6006 */
Mel Gormand9c23402007-10-16 01:26:01 -07006007 if (vm_total_pages < (pageblock_nr_pages * MIGRATE_TYPES))
Mel Gorman9ef9acb2007-10-16 01:25:54 -07006008 page_group_by_mobility_disabled = 1;
6009 else
6010 page_group_by_mobility_disabled = 0;
6011
Alexey Dobriyance0725f2019-03-05 15:48:29 -08006012 pr_info("Built %u zonelists, mobility grouping %s. Total pages: %ld\n",
Joe Perches756a0252016-03-17 14:19:47 -07006013 nr_online_nodes,
Joe Perches756a0252016-03-17 14:19:47 -07006014 page_group_by_mobility_disabled ? "off" : "on",
6015 vm_total_pages);
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07006016#ifdef CONFIG_NUMA
Anton Blanchardf88dfff2014-12-10 15:42:53 -08006017 pr_info("Policy zone: %s\n", zone_names[policy_zone]);
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07006018#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07006019}
6020
Pavel Tatashina9a9e772018-10-26 15:09:40 -07006021/* If zone is ZONE_MOVABLE but memory is mirrored, it is an overlapped init */
6022static bool __meminit
6023overlap_memmap_init(unsigned long zone, unsigned long *pfn)
6024{
Pavel Tatashina9a9e772018-10-26 15:09:40 -07006025 static struct memblock_region *r;
6026
6027 if (mirrored_kernelcore && zone == ZONE_MOVABLE) {
6028 if (!r || *pfn >= memblock_region_memory_end_pfn(r)) {
Mike Rapoportcc6de162020-10-13 16:58:30 -07006029 for_each_mem_region(r) {
Pavel Tatashina9a9e772018-10-26 15:09:40 -07006030 if (*pfn < memblock_region_memory_end_pfn(r))
6031 break;
6032 }
6033 }
6034 if (*pfn >= memblock_region_memory_base_pfn(r) &&
6035 memblock_is_mirror(r)) {
6036 *pfn = memblock_region_memory_end_pfn(r);
6037 return true;
6038 }
6039 }
Pavel Tatashina9a9e772018-10-26 15:09:40 -07006040 return false;
6041}
6042
Linus Torvalds1da177e2005-04-16 15:20:36 -07006043/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07006044 * Initially all pages are reserved - free ones are freed
Mike Rapoportc6ffc5c2018-10-30 15:09:30 -07006045 * up by memblock_free_all() once the early boot process is
Linus Torvalds1da177e2005-04-16 15:20:36 -07006046 * done. Non-atomic initialization, single-pass.
David Hildenbrandd882c002020-10-15 20:08:19 -07006047 *
6048 * All aligned pageblocks are initialized to the specified migratetype
6049 * (usually MIGRATE_MOVABLE). Besides setting the migratetype, no related
6050 * zone stats (e.g., nr_isolate_pageblock) are touched.
Linus Torvalds1da177e2005-04-16 15:20:36 -07006051 */
Matt Tolentinoc09b4242006-01-17 07:03:44 +01006052void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone,
David Hildenbrandd882c002020-10-15 20:08:19 -07006053 unsigned long start_pfn,
6054 enum meminit_context context,
6055 struct vmem_altmap *altmap, int migratetype)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006056{
Pavel Tatashina9a9e772018-10-26 15:09:40 -07006057 unsigned long pfn, end_pfn = start_pfn + size;
Pavel Tatashind0dc12e2018-04-05 16:23:00 -07006058 struct page *page;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006059
Hugh Dickins22b31ee2009-01-06 14:40:09 -08006060 if (highest_memmap_pfn < end_pfn - 1)
6061 highest_memmap_pfn = end_pfn - 1;
6062
Alexander Duyck966cf442018-10-26 15:07:52 -07006063#ifdef CONFIG_ZONE_DEVICE
Dan Williams4b94ffd2016-01-15 16:56:22 -08006064 /*
6065 * Honor reservation requested by the driver for this ZONE_DEVICE
Alexander Duyck966cf442018-10-26 15:07:52 -07006066 * memory. We limit the total number of pages to initialize to just
6067 * those that might contain the memory mapping. We will defer the
6068 * ZONE_DEVICE page initialization until after we have released
6069 * the hotplug lock.
Dan Williams4b94ffd2016-01-15 16:56:22 -08006070 */
Alexander Duyck966cf442018-10-26 15:07:52 -07006071 if (zone == ZONE_DEVICE) {
6072 if (!altmap)
6073 return;
6074
6075 if (start_pfn == altmap->base_pfn)
6076 start_pfn += altmap->reserve;
6077 end_pfn = altmap->base_pfn + vmem_altmap_offset(altmap);
6078 }
6079#endif
Dan Williams4b94ffd2016-01-15 16:56:22 -08006080
David Hildenbrand948c4362020-02-03 17:33:59 -08006081 for (pfn = start_pfn; pfn < end_pfn; ) {
Dave Hansena2f3aa022007-01-10 23:15:30 -08006082 /*
Andrew Mortonb72d0ff2016-03-15 14:55:25 -07006083 * There can be holes in boot-time mem_map[]s handed to this
6084 * function. They do not exist on hotplugged memory.
Dave Hansena2f3aa022007-01-10 23:15:30 -08006085 */
Laurent Dufourc1d0da82020-09-25 21:19:28 -07006086 if (context == MEMINIT_EARLY) {
Pavel Tatashina9a9e772018-10-26 15:09:40 -07006087 if (overlap_memmap_init(zone, &pfn))
6088 continue;
6089 if (defer_init(nid, pfn, end_pfn))
6090 break;
Dave Hansena2f3aa022007-01-10 23:15:30 -08006091 }
Mel Gormanac5d2532015-06-30 14:57:20 -07006092
Pavel Tatashind0dc12e2018-04-05 16:23:00 -07006093 page = pfn_to_page(pfn);
6094 __init_single_page(page, pfn, zone, nid);
Laurent Dufourc1d0da82020-09-25 21:19:28 -07006095 if (context == MEMINIT_HOTPLUG)
Alexander Duyckd483da52018-10-26 15:07:48 -07006096 __SetPageReserved(page);
Pavel Tatashind0dc12e2018-04-05 16:23:00 -07006097
Mel Gormanac5d2532015-06-30 14:57:20 -07006098 /*
David Hildenbrandd882c002020-10-15 20:08:19 -07006099 * Usually, we want to mark the pageblock MIGRATE_MOVABLE,
6100 * such that unmovable allocations won't be scattered all
6101 * over the place during system boot.
Mel Gormanac5d2532015-06-30 14:57:20 -07006102 */
David Hildenbrand4eb29bd2020-10-15 20:08:15 -07006103 if (IS_ALIGNED(pfn, pageblock_nr_pages)) {
David Hildenbrandd882c002020-10-15 20:08:19 -07006104 set_pageblock_migratetype(page, migratetype);
Michal Hocko9b6e63c2017-10-03 16:16:19 -07006105 cond_resched();
Mel Gormanac5d2532015-06-30 14:57:20 -07006106 }
David Hildenbrand948c4362020-02-03 17:33:59 -08006107 pfn++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006108 }
6109}
6110
Alexander Duyck966cf442018-10-26 15:07:52 -07006111#ifdef CONFIG_ZONE_DEVICE
6112void __ref memmap_init_zone_device(struct zone *zone,
6113 unsigned long start_pfn,
Aneesh Kumar K.V1f8d75c2020-02-03 17:34:06 -08006114 unsigned long nr_pages,
Alexander Duyck966cf442018-10-26 15:07:52 -07006115 struct dev_pagemap *pgmap)
6116{
Aneesh Kumar K.V1f8d75c2020-02-03 17:34:06 -08006117 unsigned long pfn, end_pfn = start_pfn + nr_pages;
Alexander Duyck966cf442018-10-26 15:07:52 -07006118 struct pglist_data *pgdat = zone->zone_pgdat;
Christoph Hellwig514caf22019-06-26 14:27:13 +02006119 struct vmem_altmap *altmap = pgmap_altmap(pgmap);
Alexander Duyck966cf442018-10-26 15:07:52 -07006120 unsigned long zone_idx = zone_idx(zone);
6121 unsigned long start = jiffies;
6122 int nid = pgdat->node_id;
6123
Dan Williams46d945a2019-07-18 15:58:18 -07006124 if (WARN_ON_ONCE(!pgmap || zone_idx(zone) != ZONE_DEVICE))
Alexander Duyck966cf442018-10-26 15:07:52 -07006125 return;
6126
6127 /*
6128 * The call to memmap_init_zone should have already taken care
6129 * of the pages reserved for the memmap, so we can just jump to
6130 * the end of that region and start processing the device pages.
6131 */
Christoph Hellwig514caf22019-06-26 14:27:13 +02006132 if (altmap) {
Alexander Duyck966cf442018-10-26 15:07:52 -07006133 start_pfn = altmap->base_pfn + vmem_altmap_offset(altmap);
Aneesh Kumar K.V1f8d75c2020-02-03 17:34:06 -08006134 nr_pages = end_pfn - start_pfn;
Alexander Duyck966cf442018-10-26 15:07:52 -07006135 }
6136
6137 for (pfn = start_pfn; pfn < end_pfn; pfn++) {
6138 struct page *page = pfn_to_page(pfn);
6139
6140 __init_single_page(page, pfn, zone_idx, nid);
6141
6142 /*
6143 * Mark page reserved as it will need to wait for onlining
6144 * phase for it to be fully associated with a zone.
6145 *
6146 * We can use the non-atomic __set_bit operation for setting
6147 * the flag as we are still initializing the pages.
6148 */
6149 __SetPageReserved(page);
6150
6151 /*
Christoph Hellwig8a164fe2019-06-26 14:27:21 +02006152 * ZONE_DEVICE pages union ->lru with a ->pgmap back pointer
6153 * and zone_device_data. It is a bug if a ZONE_DEVICE page is
6154 * ever freed or placed on a driver-private list.
Alexander Duyck966cf442018-10-26 15:07:52 -07006155 */
6156 page->pgmap = pgmap;
Christoph Hellwig8a164fe2019-06-26 14:27:21 +02006157 page->zone_device_data = NULL;
Alexander Duyck966cf442018-10-26 15:07:52 -07006158
6159 /*
6160 * Mark the block movable so that blocks are reserved for
6161 * movable at startup. This will force kernel allocations
6162 * to reserve their blocks rather than leaking throughout
6163 * the address space during boot when many long-lived
6164 * kernel allocations are made.
6165 *
Laurent Dufourc1d0da82020-09-25 21:19:28 -07006166 * Please note that MEMINIT_HOTPLUG path doesn't clear memmap
Dan Williamsba72b4c2019-07-18 15:58:26 -07006167 * because this is done early in section_activate()
Alexander Duyck966cf442018-10-26 15:07:52 -07006168 */
David Hildenbrand4eb29bd2020-10-15 20:08:15 -07006169 if (IS_ALIGNED(pfn, pageblock_nr_pages)) {
Alexander Duyck966cf442018-10-26 15:07:52 -07006170 set_pageblock_migratetype(page, MIGRATE_MOVABLE);
6171 cond_resched();
6172 }
6173 }
6174
Christoph Hellwigfdc029b2019-08-18 11:05:55 +02006175 pr_info("%s initialised %lu pages in %ums\n", __func__,
Aneesh Kumar K.V1f8d75c2020-02-03 17:34:06 -08006176 nr_pages, jiffies_to_msecs(jiffies - start));
Alexander Duyck966cf442018-10-26 15:07:52 -07006177}
6178
6179#endif
Andi Kleen1e548de2008-02-04 22:29:26 -08006180static void __meminit zone_init_free_lists(struct zone *zone)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006181{
Mel Gorman7aeb09f2014-06-04 16:10:21 -07006182 unsigned int order, t;
Mel Gormanb2a0ac82007-10-16 01:25:48 -07006183 for_each_migratetype_order(order, t) {
6184 INIT_LIST_HEAD(&zone->free_area[order].free_list[t]);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006185 zone->free_area[order].nr_free = 0;
6186 }
6187}
6188
Pavel Tatashindfb3ccd2018-10-26 15:09:32 -07006189void __meminit __weak memmap_init(unsigned long size, int nid,
Baoquan He73a6e472020-06-03 15:57:55 -07006190 unsigned long zone,
6191 unsigned long range_start_pfn)
Pavel Tatashindfb3ccd2018-10-26 15:09:32 -07006192{
Baoquan He73a6e472020-06-03 15:57:55 -07006193 unsigned long start_pfn, end_pfn;
6194 unsigned long range_end_pfn = range_start_pfn + size;
6195 int i;
6196
6197 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
6198 start_pfn = clamp(start_pfn, range_start_pfn, range_end_pfn);
6199 end_pfn = clamp(end_pfn, range_start_pfn, range_end_pfn);
6200
6201 if (end_pfn > start_pfn) {
6202 size = end_pfn - start_pfn;
6203 memmap_init_zone(size, nid, zone, start_pfn,
David Hildenbrandd882c002020-10-15 20:08:19 -07006204 MEMINIT_EARLY, NULL, MIGRATE_MOVABLE);
Baoquan He73a6e472020-06-03 15:57:55 -07006205 }
6206 }
Pavel Tatashindfb3ccd2018-10-26 15:09:32 -07006207}
Linus Torvalds1da177e2005-04-16 15:20:36 -07006208
David Rientjes7cd2b0a2014-06-23 13:22:04 -07006209static int zone_batchsize(struct zone *zone)
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07006210{
David Howells3a6be872009-05-06 16:03:03 -07006211#ifdef CONFIG_MMU
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07006212 int batch;
6213
6214 /*
6215 * The per-cpu-pages pools are set to around 1000th of the
Aaron Lud8a759b2018-08-17 15:49:14 -07006216 * size of the zone.
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07006217 */
Arun KS9705bea2018-12-28 00:34:24 -08006218 batch = zone_managed_pages(zone) / 1024;
Aaron Lud8a759b2018-08-17 15:49:14 -07006219 /* But no more than a meg. */
6220 if (batch * PAGE_SIZE > 1024 * 1024)
6221 batch = (1024 * 1024) / PAGE_SIZE;
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07006222 batch /= 4; /* We effectively *= 4 below */
6223 if (batch < 1)
6224 batch = 1;
6225
6226 /*
Nick Piggin0ceaacc2005-12-04 13:55:25 +11006227 * Clamp the batch to a 2^n - 1 value. Having a power
6228 * of 2 value was found to be more likely to have
6229 * suboptimal cache aliasing properties in some cases.
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07006230 *
Nick Piggin0ceaacc2005-12-04 13:55:25 +11006231 * For example if 2 tasks are alternately allocating
6232 * batches of pages, one task can end up with a lot
6233 * of pages of one half of the possible page colors
6234 * and the other with pages of the other colors.
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07006235 */
David Howells91552032009-05-06 16:03:02 -07006236 batch = rounddown_pow_of_two(batch + batch/2) - 1;
Seth, Rohitba56e912005-10-29 18:15:47 -07006237
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07006238 return batch;
David Howells3a6be872009-05-06 16:03:03 -07006239
6240#else
6241 /* The deferral and batching of frees should be suppressed under NOMMU
6242 * conditions.
6243 *
6244 * The problem is that NOMMU needs to be able to allocate large chunks
6245 * of contiguous memory as there's no hardware page translation to
6246 * assemble apparent contiguous memory from discontiguous pages.
6247 *
6248 * Queueing large contiguous runs of pages for batching, however,
6249 * causes the pages to actually be freed in smaller chunks. As there
6250 * can be a significant delay between the individual batches being
6251 * recycled, this leads to the once large chunks of space being
6252 * fragmented and becoming unavailable for high-order allocations.
6253 */
6254 return 0;
6255#endif
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07006256}
6257
Cody P Schafer8d7a8fa2013-07-03 15:01:31 -07006258/*
6259 * pcp->high and pcp->batch values are related and dependent on one another:
6260 * ->batch must never be higher then ->high.
6261 * The following function updates them in a safe manner without read side
6262 * locking.
6263 *
6264 * Any new users of pcp->batch and pcp->high should ensure they can cope with
Randy Dunlap047b9962020-08-11 18:33:14 -07006265 * those fields changing asynchronously (acording to the above rule).
Cody P Schafer8d7a8fa2013-07-03 15:01:31 -07006266 *
6267 * mutex_is_locked(&pcp_batch_high_lock) required when calling this function
6268 * outside of boot time (or some other assurance that no concurrent updaters
6269 * exist).
6270 */
6271static void pageset_update(struct per_cpu_pages *pcp, unsigned long high,
6272 unsigned long batch)
6273{
6274 /* start with a fail safe value for batch */
6275 pcp->batch = 1;
6276 smp_wmb();
6277
6278 /* Update high, then batch, in order */
6279 pcp->high = high;
6280 smp_wmb();
6281
6282 pcp->batch = batch;
6283}
6284
Cody P Schafer36640332013-07-03 15:01:40 -07006285/* a companion to pageset_set_high() */
Cody P Schafer4008bab2013-07-03 15:01:28 -07006286static void pageset_set_batch(struct per_cpu_pageset *p, unsigned long batch)
6287{
Cody P Schafer8d7a8fa2013-07-03 15:01:31 -07006288 pageset_update(&p->pcp, 6 * batch, max(1UL, 1 * batch));
Cody P Schafer4008bab2013-07-03 15:01:28 -07006289}
6290
Cody P Schafer88c90db2013-07-03 15:01:35 -07006291static void pageset_init(struct per_cpu_pageset *p)
Christoph Lameter2caaad42005-06-21 17:15:00 -07006292{
6293 struct per_cpu_pages *pcp;
Mel Gorman5f8dcc22009-09-21 17:03:19 -07006294 int migratetype;
Christoph Lameter2caaad42005-06-21 17:15:00 -07006295
Magnus Damm1c6fe942005-10-26 01:58:59 -07006296 memset(p, 0, sizeof(*p));
6297
Christoph Lameter3dfa5722008-02-04 22:29:19 -08006298 pcp = &p->pcp;
Mel Gorman5f8dcc22009-09-21 17:03:19 -07006299 for (migratetype = 0; migratetype < MIGRATE_PCPTYPES; migratetype++)
6300 INIT_LIST_HEAD(&pcp->lists[migratetype]);
Christoph Lameter2caaad42005-06-21 17:15:00 -07006301}
6302
Cody P Schafer88c90db2013-07-03 15:01:35 -07006303static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch)
6304{
6305 pageset_init(p);
6306 pageset_set_batch(p, batch);
6307}
6308
Rohit Seth8ad4b1f2006-01-08 01:00:40 -08006309/*
Cody P Schafer36640332013-07-03 15:01:40 -07006310 * pageset_set_high() sets the high water mark for hot per_cpu_pagelist
Rohit Seth8ad4b1f2006-01-08 01:00:40 -08006311 * to the value high for the pageset p.
6312 */
Cody P Schafer36640332013-07-03 15:01:40 -07006313static void pageset_set_high(struct per_cpu_pageset *p,
Rohit Seth8ad4b1f2006-01-08 01:00:40 -08006314 unsigned long high)
6315{
Cody P Schafer8d7a8fa2013-07-03 15:01:31 -07006316 unsigned long batch = max(1UL, high / 4);
6317 if ((high / 4) > (PAGE_SHIFT * 8))
6318 batch = PAGE_SHIFT * 8;
Rohit Seth8ad4b1f2006-01-08 01:00:40 -08006319
Cody P Schafer8d7a8fa2013-07-03 15:01:31 -07006320 pageset_update(&p->pcp, high, batch);
Rohit Seth8ad4b1f2006-01-08 01:00:40 -08006321}
6322
David Rientjes7cd2b0a2014-06-23 13:22:04 -07006323static void pageset_set_high_and_batch(struct zone *zone,
6324 struct per_cpu_pageset *pcp)
Cody P Schafer56cef2b2013-07-03 15:01:38 -07006325{
Cody P Schafer56cef2b2013-07-03 15:01:38 -07006326 if (percpu_pagelist_fraction)
Cody P Schafer36640332013-07-03 15:01:40 -07006327 pageset_set_high(pcp,
Arun KS9705bea2018-12-28 00:34:24 -08006328 (zone_managed_pages(zone) /
Cody P Schafer56cef2b2013-07-03 15:01:38 -07006329 percpu_pagelist_fraction));
6330 else
6331 pageset_set_batch(pcp, zone_batchsize(zone));
6332}
6333
Cody P Schafer169f6c12013-07-03 15:01:41 -07006334static void __meminit zone_pageset_init(struct zone *zone, int cpu)
6335{
6336 struct per_cpu_pageset *pcp = per_cpu_ptr(zone->pageset, cpu);
6337
6338 pageset_init(pcp);
6339 pageset_set_high_and_batch(zone, pcp);
6340}
6341
Michal Hocko72675e12017-09-06 16:20:24 -07006342void __meminit setup_zone_pageset(struct zone *zone)
Wu Fengguang319774e2010-05-24 14:32:49 -07006343{
6344 int cpu;
Wu Fengguang319774e2010-05-24 14:32:49 -07006345 zone->pageset = alloc_percpu(struct per_cpu_pageset);
Cody P Schafer56cef2b2013-07-03 15:01:38 -07006346 for_each_possible_cpu(cpu)
6347 zone_pageset_init(zone, cpu);
Wu Fengguang319774e2010-05-24 14:32:49 -07006348}
6349
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07006350/*
Christoph Lameter99dcc3e2010-01-05 15:34:51 +09006351 * Allocate per cpu pagesets and initialize them.
6352 * Before this call only boot pagesets were available.
Christoph Lameter2caaad42005-06-21 17:15:00 -07006353 */
Al Viro78d99552005-12-15 09:18:25 +00006354void __init setup_per_cpu_pageset(void)
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07006355{
Mel Gormanb4911ea2016-08-04 15:31:49 -07006356 struct pglist_data *pgdat;
Christoph Lameter99dcc3e2010-01-05 15:34:51 +09006357 struct zone *zone;
Sandipan Dasb418a0f2020-06-03 15:59:11 -07006358 int __maybe_unused cpu;
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07006359
Wu Fengguang319774e2010-05-24 14:32:49 -07006360 for_each_populated_zone(zone)
6361 setup_zone_pageset(zone);
Mel Gormanb4911ea2016-08-04 15:31:49 -07006362
Sandipan Dasb418a0f2020-06-03 15:59:11 -07006363#ifdef CONFIG_NUMA
6364 /*
6365 * Unpopulated zones continue using the boot pagesets.
6366 * The numa stats for these pagesets need to be reset.
6367 * Otherwise, they will end up skewing the stats of
6368 * the nodes these zones are associated with.
6369 */
6370 for_each_possible_cpu(cpu) {
6371 struct per_cpu_pageset *pcp = &per_cpu(boot_pageset, cpu);
6372 memset(pcp->vm_numa_stat_diff, 0,
6373 sizeof(pcp->vm_numa_stat_diff));
6374 }
6375#endif
6376
Mel Gormanb4911ea2016-08-04 15:31:49 -07006377 for_each_online_pgdat(pgdat)
6378 pgdat->per_cpu_nodestats =
6379 alloc_percpu(struct per_cpu_nodestat);
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07006380}
6381
Matt Tolentinoc09b4242006-01-17 07:03:44 +01006382static __meminit void zone_pcp_init(struct zone *zone)
Dave Hansened8ece22005-10-29 18:16:50 -07006383{
Christoph Lameter99dcc3e2010-01-05 15:34:51 +09006384 /*
6385 * per cpu subsystem is not up at this point. The following code
6386 * relies on the ability of the linker to provide the
6387 * offset of a (static) per cpu variable into the per cpu area.
6388 */
6389 zone->pageset = &boot_pageset;
Dave Hansened8ece22005-10-29 18:16:50 -07006390
Xishi Qiub38a8722013-11-12 15:07:20 -08006391 if (populated_zone(zone))
Christoph Lameter99dcc3e2010-01-05 15:34:51 +09006392 printk(KERN_DEBUG " %s zone: %lu pages, LIFO batch:%u\n",
6393 zone->name, zone->present_pages,
6394 zone_batchsize(zone));
Dave Hansened8ece22005-10-29 18:16:50 -07006395}
6396
Michal Hockodc0bbf32017-07-06 15:37:35 -07006397void __meminit init_currently_empty_zone(struct zone *zone,
Yasunori Goto718127c2006-06-23 02:03:10 -07006398 unsigned long zone_start_pfn,
Yaowei Baib171e402015-11-05 18:47:06 -08006399 unsigned long size)
Dave Hansened8ece22005-10-29 18:16:50 -07006400{
6401 struct pglist_data *pgdat = zone->zone_pgdat;
Wei Yang8f416832018-11-30 14:09:07 -08006402 int zone_idx = zone_idx(zone) + 1;
Linus Torvalds9dcb8b62016-10-26 10:15:30 -07006403
Wei Yang8f416832018-11-30 14:09:07 -08006404 if (zone_idx > pgdat->nr_zones)
6405 pgdat->nr_zones = zone_idx;
Dave Hansened8ece22005-10-29 18:16:50 -07006406
Dave Hansened8ece22005-10-29 18:16:50 -07006407 zone->zone_start_pfn = zone_start_pfn;
6408
Mel Gorman708614e2008-07-23 21:26:51 -07006409 mminit_dprintk(MMINIT_TRACE, "memmap_init",
6410 "Initialising map node %d zone %lu pfns %lu -> %lu\n",
6411 pgdat->node_id,
6412 (unsigned long)zone_idx(zone),
6413 zone_start_pfn, (zone_start_pfn + size));
6414
Andi Kleen1e548de2008-02-04 22:29:26 -08006415 zone_init_free_lists(zone);
Linus Torvalds9dcb8b62016-10-26 10:15:30 -07006416 zone->initialized = 1;
Dave Hansened8ece22005-10-29 18:16:50 -07006417}
6418
Mel Gormanc7132162006-09-27 01:49:43 -07006419/**
Mel Gormanc7132162006-09-27 01:49:43 -07006420 * get_pfn_range_for_nid - Return the start and end page frames for a node
Randy Dunlap88ca3b92006-10-04 02:15:25 -07006421 * @nid: The nid to return the range for. If MAX_NUMNODES, the min and max PFN are returned.
6422 * @start_pfn: Passed by reference. On return, it will have the node start_pfn.
6423 * @end_pfn: Passed by reference. On return, it will have the node end_pfn.
Mel Gormanc7132162006-09-27 01:49:43 -07006424 *
6425 * It returns the start and end page frame of a node based on information
Zhang Zhen7d018172014-06-04 16:10:53 -07006426 * provided by memblock_set_node(). If called for a node
Mel Gormanc7132162006-09-27 01:49:43 -07006427 * with no available memory, a warning is printed and the start and end
Randy Dunlap88ca3b92006-10-04 02:15:25 -07006428 * PFNs will be 0.
Mel Gormanc7132162006-09-27 01:49:43 -07006429 */
Oscar Salvadorbbe5d992018-12-28 00:37:24 -08006430void __init get_pfn_range_for_nid(unsigned int nid,
Mel Gormanc7132162006-09-27 01:49:43 -07006431 unsigned long *start_pfn, unsigned long *end_pfn)
6432{
Tejun Heoc13291a2011-07-12 10:46:30 +02006433 unsigned long this_start_pfn, this_end_pfn;
Mel Gormanc7132162006-09-27 01:49:43 -07006434 int i;
Tejun Heoc13291a2011-07-12 10:46:30 +02006435
Mel Gormanc7132162006-09-27 01:49:43 -07006436 *start_pfn = -1UL;
6437 *end_pfn = 0;
6438
Tejun Heoc13291a2011-07-12 10:46:30 +02006439 for_each_mem_pfn_range(i, nid, &this_start_pfn, &this_end_pfn, NULL) {
6440 *start_pfn = min(*start_pfn, this_start_pfn);
6441 *end_pfn = max(*end_pfn, this_end_pfn);
Mel Gormanc7132162006-09-27 01:49:43 -07006442 }
6443
Christoph Lameter633c0662007-10-16 01:25:37 -07006444 if (*start_pfn == -1UL)
Mel Gormanc7132162006-09-27 01:49:43 -07006445 *start_pfn = 0;
Mel Gormanc7132162006-09-27 01:49:43 -07006446}
6447
6448/*
Mel Gorman2a1e2742007-07-17 04:03:12 -07006449 * This finds a zone that can be used for ZONE_MOVABLE pages. The
6450 * assumption is made that zones within a node are ordered in monotonic
6451 * increasing memory addresses so that the "highest" populated zone is used
6452 */
Adrian Bunkb69a7282008-07-23 21:28:12 -07006453static void __init find_usable_zone_for_movable(void)
Mel Gorman2a1e2742007-07-17 04:03:12 -07006454{
6455 int zone_index;
6456 for (zone_index = MAX_NR_ZONES - 1; zone_index >= 0; zone_index--) {
6457 if (zone_index == ZONE_MOVABLE)
6458 continue;
6459
6460 if (arch_zone_highest_possible_pfn[zone_index] >
6461 arch_zone_lowest_possible_pfn[zone_index])
6462 break;
6463 }
6464
6465 VM_BUG_ON(zone_index == -1);
6466 movable_zone = zone_index;
6467}
6468
6469/*
6470 * The zone ranges provided by the architecture do not include ZONE_MOVABLE
Lucas De Marchi25985ed2011-03-30 22:57:33 -03006471 * because it is sized independent of architecture. Unlike the other zones,
Mel Gorman2a1e2742007-07-17 04:03:12 -07006472 * the starting point for ZONE_MOVABLE is not fixed. It may be different
6473 * in each node depending on the size of each node and how evenly kernelcore
6474 * is distributed. This helper function adjusts the zone ranges
6475 * provided by the architecture for a given node by using the end of the
6476 * highest usable zone for ZONE_MOVABLE. This preserves the assumption that
6477 * zones within a node are in order of monotonic increases memory addresses
6478 */
Oscar Salvadorbbe5d992018-12-28 00:37:24 -08006479static void __init adjust_zone_range_for_zone_movable(int nid,
Mel Gorman2a1e2742007-07-17 04:03:12 -07006480 unsigned long zone_type,
6481 unsigned long node_start_pfn,
6482 unsigned long node_end_pfn,
6483 unsigned long *zone_start_pfn,
6484 unsigned long *zone_end_pfn)
6485{
6486 /* Only adjust if ZONE_MOVABLE is on this node */
6487 if (zone_movable_pfn[nid]) {
6488 /* Size ZONE_MOVABLE */
6489 if (zone_type == ZONE_MOVABLE) {
6490 *zone_start_pfn = zone_movable_pfn[nid];
6491 *zone_end_pfn = min(node_end_pfn,
6492 arch_zone_highest_possible_pfn[movable_zone]);
6493
Xishi Qiue506b992016-10-07 16:58:06 -07006494 /* Adjust for ZONE_MOVABLE starting within this range */
6495 } else if (!mirrored_kernelcore &&
6496 *zone_start_pfn < zone_movable_pfn[nid] &&
6497 *zone_end_pfn > zone_movable_pfn[nid]) {
6498 *zone_end_pfn = zone_movable_pfn[nid];
6499
Mel Gorman2a1e2742007-07-17 04:03:12 -07006500 /* Check if this whole range is within ZONE_MOVABLE */
6501 } else if (*zone_start_pfn >= zone_movable_pfn[nid])
6502 *zone_start_pfn = *zone_end_pfn;
6503 }
6504}
6505
6506/*
Mel Gormanc7132162006-09-27 01:49:43 -07006507 * Return the number of pages a zone spans in a node, including holes
6508 * present_pages = zone_spanned_pages_in_node() - zone_absent_pages_in_node()
6509 */
Oscar Salvadorbbe5d992018-12-28 00:37:24 -08006510static unsigned long __init zone_spanned_pages_in_node(int nid,
Mel Gormanc7132162006-09-27 01:49:43 -07006511 unsigned long zone_type,
Zhang Yanfei7960aed2013-07-08 15:59:52 -07006512 unsigned long node_start_pfn,
6513 unsigned long node_end_pfn,
Taku Izumid91749c2016-03-15 14:55:18 -07006514 unsigned long *zone_start_pfn,
Mike Rapoport854e8842020-06-03 15:58:13 -07006515 unsigned long *zone_end_pfn)
Mel Gormanc7132162006-09-27 01:49:43 -07006516{
Linxu Fang299c83d2019-05-13 17:19:17 -07006517 unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type];
6518 unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type];
Xishi Qiub5685e92015-09-08 15:04:16 -07006519 /* When hotadd a new node from cpu_up(), the node should be empty */
Xishi Qiuf9126ab2015-08-14 15:35:16 -07006520 if (!node_start_pfn && !node_end_pfn)
6521 return 0;
6522
Zhang Yanfei7960aed2013-07-08 15:59:52 -07006523 /* Get the start and end of the zone */
Linxu Fang299c83d2019-05-13 17:19:17 -07006524 *zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high);
6525 *zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high);
Mel Gorman2a1e2742007-07-17 04:03:12 -07006526 adjust_zone_range_for_zone_movable(nid, zone_type,
6527 node_start_pfn, node_end_pfn,
Taku Izumid91749c2016-03-15 14:55:18 -07006528 zone_start_pfn, zone_end_pfn);
Mel Gormanc7132162006-09-27 01:49:43 -07006529
6530 /* Check that this node has pages within the zone's required range */
Taku Izumid91749c2016-03-15 14:55:18 -07006531 if (*zone_end_pfn < node_start_pfn || *zone_start_pfn > node_end_pfn)
Mel Gormanc7132162006-09-27 01:49:43 -07006532 return 0;
6533
6534 /* Move the zone boundaries inside the node if necessary */
Taku Izumid91749c2016-03-15 14:55:18 -07006535 *zone_end_pfn = min(*zone_end_pfn, node_end_pfn);
6536 *zone_start_pfn = max(*zone_start_pfn, node_start_pfn);
Mel Gormanc7132162006-09-27 01:49:43 -07006537
6538 /* Return the spanned pages */
Taku Izumid91749c2016-03-15 14:55:18 -07006539 return *zone_end_pfn - *zone_start_pfn;
Mel Gormanc7132162006-09-27 01:49:43 -07006540}
6541
6542/*
6543 * Return the number of holes in a range on a node. If nid is MAX_NUMNODES,
Randy Dunlap88ca3b92006-10-04 02:15:25 -07006544 * then all holes in the requested range will be accounted for.
Mel Gormanc7132162006-09-27 01:49:43 -07006545 */
Oscar Salvadorbbe5d992018-12-28 00:37:24 -08006546unsigned long __init __absent_pages_in_range(int nid,
Mel Gormanc7132162006-09-27 01:49:43 -07006547 unsigned long range_start_pfn,
6548 unsigned long range_end_pfn)
6549{
Tejun Heo96e907d2011-07-12 10:46:29 +02006550 unsigned long nr_absent = range_end_pfn - range_start_pfn;
6551 unsigned long start_pfn, end_pfn;
6552 int i;
Mel Gormanc7132162006-09-27 01:49:43 -07006553
Tejun Heo96e907d2011-07-12 10:46:29 +02006554 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
6555 start_pfn = clamp(start_pfn, range_start_pfn, range_end_pfn);
6556 end_pfn = clamp(end_pfn, range_start_pfn, range_end_pfn);
6557 nr_absent -= end_pfn - start_pfn;
Mel Gormanc7132162006-09-27 01:49:43 -07006558 }
Tejun Heo96e907d2011-07-12 10:46:29 +02006559 return nr_absent;
Mel Gormanc7132162006-09-27 01:49:43 -07006560}
6561
6562/**
6563 * absent_pages_in_range - Return number of page frames in holes within a range
6564 * @start_pfn: The start PFN to start searching for holes
6565 * @end_pfn: The end PFN to stop searching for holes
6566 *
Mike Rapoporta862f682019-03-05 15:48:42 -08006567 * Return: the number of pages frames in memory holes within a range.
Mel Gormanc7132162006-09-27 01:49:43 -07006568 */
6569unsigned long __init absent_pages_in_range(unsigned long start_pfn,
6570 unsigned long end_pfn)
6571{
6572 return __absent_pages_in_range(MAX_NUMNODES, start_pfn, end_pfn);
6573}
6574
6575/* Return the number of page frames in holes in a zone on a node */
Oscar Salvadorbbe5d992018-12-28 00:37:24 -08006576static unsigned long __init zone_absent_pages_in_node(int nid,
Mel Gormanc7132162006-09-27 01:49:43 -07006577 unsigned long zone_type,
Zhang Yanfei7960aed2013-07-08 15:59:52 -07006578 unsigned long node_start_pfn,
Mike Rapoport854e8842020-06-03 15:58:13 -07006579 unsigned long node_end_pfn)
Mel Gormanc7132162006-09-27 01:49:43 -07006580{
Tejun Heo96e907d2011-07-12 10:46:29 +02006581 unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type];
6582 unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type];
Mel Gorman9c7cd682006-09-27 01:49:58 -07006583 unsigned long zone_start_pfn, zone_end_pfn;
Taku Izumi342332e2016-03-15 14:55:22 -07006584 unsigned long nr_absent;
Mel Gorman9c7cd682006-09-27 01:49:58 -07006585
Xishi Qiub5685e92015-09-08 15:04:16 -07006586 /* When hotadd a new node from cpu_up(), the node should be empty */
Xishi Qiuf9126ab2015-08-14 15:35:16 -07006587 if (!node_start_pfn && !node_end_pfn)
6588 return 0;
6589
Tejun Heo96e907d2011-07-12 10:46:29 +02006590 zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high);
6591 zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high);
Mel Gorman9c7cd682006-09-27 01:49:58 -07006592
Mel Gorman2a1e2742007-07-17 04:03:12 -07006593 adjust_zone_range_for_zone_movable(nid, zone_type,
6594 node_start_pfn, node_end_pfn,
6595 &zone_start_pfn, &zone_end_pfn);
Taku Izumi342332e2016-03-15 14:55:22 -07006596 nr_absent = __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn);
6597
6598 /*
6599 * ZONE_MOVABLE handling.
6600 * Treat pages to be ZONE_MOVABLE in ZONE_NORMAL as absent pages
6601 * and vice versa.
6602 */
Xishi Qiue506b992016-10-07 16:58:06 -07006603 if (mirrored_kernelcore && zone_movable_pfn[nid]) {
6604 unsigned long start_pfn, end_pfn;
6605 struct memblock_region *r;
Taku Izumi342332e2016-03-15 14:55:22 -07006606
Mike Rapoportcc6de162020-10-13 16:58:30 -07006607 for_each_mem_region(r) {
Xishi Qiue506b992016-10-07 16:58:06 -07006608 start_pfn = clamp(memblock_region_memory_base_pfn(r),
6609 zone_start_pfn, zone_end_pfn);
6610 end_pfn = clamp(memblock_region_memory_end_pfn(r),
6611 zone_start_pfn, zone_end_pfn);
Taku Izumi342332e2016-03-15 14:55:22 -07006612
Xishi Qiue506b992016-10-07 16:58:06 -07006613 if (zone_type == ZONE_MOVABLE &&
6614 memblock_is_mirror(r))
6615 nr_absent += end_pfn - start_pfn;
Taku Izumi342332e2016-03-15 14:55:22 -07006616
Xishi Qiue506b992016-10-07 16:58:06 -07006617 if (zone_type == ZONE_NORMAL &&
6618 !memblock_is_mirror(r))
6619 nr_absent += end_pfn - start_pfn;
Taku Izumi342332e2016-03-15 14:55:22 -07006620 }
6621 }
6622
6623 return nr_absent;
Mel Gormanc7132162006-09-27 01:49:43 -07006624}
Mel Gorman0e0b8642006-09-27 01:49:56 -07006625
Oscar Salvadorbbe5d992018-12-28 00:37:24 -08006626static void __init calculate_node_totalpages(struct pglist_data *pgdat,
Zhang Yanfei7960aed2013-07-08 15:59:52 -07006627 unsigned long node_start_pfn,
Mike Rapoport854e8842020-06-03 15:58:13 -07006628 unsigned long node_end_pfn)
Mel Gormanc7132162006-09-27 01:49:43 -07006629{
Gu Zhengfebd5942015-06-24 16:57:02 -07006630 unsigned long realtotalpages = 0, totalpages = 0;
Mel Gormanc7132162006-09-27 01:49:43 -07006631 enum zone_type i;
6632
Gu Zhengfebd5942015-06-24 16:57:02 -07006633 for (i = 0; i < MAX_NR_ZONES; i++) {
6634 struct zone *zone = pgdat->node_zones + i;
Taku Izumid91749c2016-03-15 14:55:18 -07006635 unsigned long zone_start_pfn, zone_end_pfn;
Mike Rapoport3f08a302020-06-03 15:57:02 -07006636 unsigned long spanned, absent;
Gu Zhengfebd5942015-06-24 16:57:02 -07006637 unsigned long size, real_size;
Mel Gormanc7132162006-09-27 01:49:43 -07006638
Mike Rapoport854e8842020-06-03 15:58:13 -07006639 spanned = zone_spanned_pages_in_node(pgdat->node_id, i,
6640 node_start_pfn,
6641 node_end_pfn,
6642 &zone_start_pfn,
6643 &zone_end_pfn);
6644 absent = zone_absent_pages_in_node(pgdat->node_id, i,
6645 node_start_pfn,
6646 node_end_pfn);
Mike Rapoport3f08a302020-06-03 15:57:02 -07006647
6648 size = spanned;
6649 real_size = size - absent;
6650
Taku Izumid91749c2016-03-15 14:55:18 -07006651 if (size)
6652 zone->zone_start_pfn = zone_start_pfn;
6653 else
6654 zone->zone_start_pfn = 0;
Gu Zhengfebd5942015-06-24 16:57:02 -07006655 zone->spanned_pages = size;
6656 zone->present_pages = real_size;
6657
6658 totalpages += size;
6659 realtotalpages += real_size;
6660 }
6661
6662 pgdat->node_spanned_pages = totalpages;
Mel Gormanc7132162006-09-27 01:49:43 -07006663 pgdat->node_present_pages = realtotalpages;
6664 printk(KERN_DEBUG "On node %d totalpages: %lu\n", pgdat->node_id,
6665 realtotalpages);
6666}
6667
Mel Gorman835c1342007-10-16 01:25:47 -07006668#ifndef CONFIG_SPARSEMEM
6669/*
6670 * Calculate the size of the zone->blockflags rounded to an unsigned long
Mel Gormand9c23402007-10-16 01:26:01 -07006671 * Start by making sure zonesize is a multiple of pageblock_order by rounding
6672 * up. Then use 1 NR_PAGEBLOCK_BITS worth of bits per pageblock, finally
Mel Gorman835c1342007-10-16 01:25:47 -07006673 * round what is now in bits to nearest long in bits, then return it in
6674 * bytes.
6675 */
Linus Torvalds7c455122013-02-18 09:58:02 -08006676static unsigned long __init usemap_size(unsigned long zone_start_pfn, unsigned long zonesize)
Mel Gorman835c1342007-10-16 01:25:47 -07006677{
6678 unsigned long usemapsize;
6679
Linus Torvalds7c455122013-02-18 09:58:02 -08006680 zonesize += zone_start_pfn & (pageblock_nr_pages-1);
Mel Gormand9c23402007-10-16 01:26:01 -07006681 usemapsize = roundup(zonesize, pageblock_nr_pages);
6682 usemapsize = usemapsize >> pageblock_order;
Mel Gorman835c1342007-10-16 01:25:47 -07006683 usemapsize *= NR_PAGEBLOCK_BITS;
6684 usemapsize = roundup(usemapsize, 8 * sizeof(unsigned long));
6685
6686 return usemapsize / 8;
6687}
6688
Pavel Tatashin7cc2a952018-08-21 21:53:36 -07006689static void __ref setup_usemap(struct pglist_data *pgdat,
Linus Torvalds7c455122013-02-18 09:58:02 -08006690 struct zone *zone,
6691 unsigned long zone_start_pfn,
6692 unsigned long zonesize)
Mel Gorman835c1342007-10-16 01:25:47 -07006693{
Linus Torvalds7c455122013-02-18 09:58:02 -08006694 unsigned long usemapsize = usemap_size(zone_start_pfn, zonesize);
Mel Gorman835c1342007-10-16 01:25:47 -07006695 zone->pageblock_flags = NULL;
Mike Rapoport23a70522019-03-05 15:46:43 -08006696 if (usemapsize) {
Santosh Shilimkar67828322014-01-21 15:50:25 -08006697 zone->pageblock_flags =
Mike Rapoport26fb3da2019-03-11 23:30:42 -07006698 memblock_alloc_node(usemapsize, SMP_CACHE_BYTES,
6699 pgdat->node_id);
Mike Rapoport23a70522019-03-05 15:46:43 -08006700 if (!zone->pageblock_flags)
6701 panic("Failed to allocate %ld bytes for zone %s pageblock flags on node %d\n",
6702 usemapsize, zone->name, pgdat->node_id);
6703 }
Mel Gorman835c1342007-10-16 01:25:47 -07006704}
6705#else
Linus Torvalds7c455122013-02-18 09:58:02 -08006706static inline void setup_usemap(struct pglist_data *pgdat, struct zone *zone,
6707 unsigned long zone_start_pfn, unsigned long zonesize) {}
Mel Gorman835c1342007-10-16 01:25:47 -07006708#endif /* CONFIG_SPARSEMEM */
6709
Mel Gormand9c23402007-10-16 01:26:01 -07006710#ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
Mel Gormanba72cb82007-11-28 16:21:13 -08006711
Mel Gormand9c23402007-10-16 01:26:01 -07006712/* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */
Oscar Salvador03e85f92018-08-21 21:53:43 -07006713void __init set_pageblock_order(void)
Mel Gormand9c23402007-10-16 01:26:01 -07006714{
Andrew Morton955c1cd2012-05-29 15:06:31 -07006715 unsigned int order;
6716
Mel Gormand9c23402007-10-16 01:26:01 -07006717 /* Check that pageblock_nr_pages has not already been setup */
6718 if (pageblock_order)
6719 return;
6720
Andrew Morton955c1cd2012-05-29 15:06:31 -07006721 if (HPAGE_SHIFT > PAGE_SHIFT)
6722 order = HUGETLB_PAGE_ORDER;
6723 else
6724 order = MAX_ORDER - 1;
6725
Mel Gormand9c23402007-10-16 01:26:01 -07006726 /*
6727 * Assume the largest contiguous order of interest is a huge page.
Andrew Morton955c1cd2012-05-29 15:06:31 -07006728 * This value may be variable depending on boot parameters on IA64 and
6729 * powerpc.
Mel Gormand9c23402007-10-16 01:26:01 -07006730 */
6731 pageblock_order = order;
6732}
6733#else /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */
6734
Mel Gormanba72cb82007-11-28 16:21:13 -08006735/*
6736 * When CONFIG_HUGETLB_PAGE_SIZE_VARIABLE is not set, set_pageblock_order()
Andrew Morton955c1cd2012-05-29 15:06:31 -07006737 * is unused as pageblock_order is set at compile-time. See
6738 * include/linux/pageblock-flags.h for the values of pageblock_order based on
6739 * the kernel config
Mel Gormanba72cb82007-11-28 16:21:13 -08006740 */
Oscar Salvador03e85f92018-08-21 21:53:43 -07006741void __init set_pageblock_order(void)
Mel Gormanba72cb82007-11-28 16:21:13 -08006742{
Mel Gormanba72cb82007-11-28 16:21:13 -08006743}
Mel Gormand9c23402007-10-16 01:26:01 -07006744
6745#endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */
6746
Oscar Salvador03e85f92018-08-21 21:53:43 -07006747static unsigned long __init calc_memmap_size(unsigned long spanned_pages,
Pavel Tatashin7cc2a952018-08-21 21:53:36 -07006748 unsigned long present_pages)
Jiang Liu01cefae2012-12-12 13:52:19 -08006749{
6750 unsigned long pages = spanned_pages;
6751
6752 /*
6753 * Provide a more accurate estimation if there are holes within
6754 * the zone and SPARSEMEM is in use. If there are holes within the
6755 * zone, each populated memory region may cost us one or two extra
6756 * memmap pages due to alignment because memmap pages for each
Masahiro Yamada89d790a2017-02-27 14:29:01 -08006757 * populated regions may not be naturally aligned on page boundary.
Jiang Liu01cefae2012-12-12 13:52:19 -08006758 * So the (present_pages >> 4) heuristic is a tradeoff for that.
6759 */
6760 if (spanned_pages > present_pages + (present_pages >> 4) &&
6761 IS_ENABLED(CONFIG_SPARSEMEM))
6762 pages = present_pages;
6763
6764 return PAGE_ALIGN(pages * sizeof(struct page)) >> PAGE_SHIFT;
6765}
6766
Oscar Salvadorace1db32018-08-21 21:53:29 -07006767#ifdef CONFIG_TRANSPARENT_HUGEPAGE
6768static void pgdat_init_split_queue(struct pglist_data *pgdat)
6769{
Yang Shi364c1ee2019-09-23 15:38:06 -07006770 struct deferred_split *ds_queue = &pgdat->deferred_split_queue;
6771
6772 spin_lock_init(&ds_queue->split_queue_lock);
6773 INIT_LIST_HEAD(&ds_queue->split_queue);
6774 ds_queue->split_queue_len = 0;
Oscar Salvadorace1db32018-08-21 21:53:29 -07006775}
6776#else
6777static void pgdat_init_split_queue(struct pglist_data *pgdat) {}
6778#endif
6779
6780#ifdef CONFIG_COMPACTION
6781static void pgdat_init_kcompactd(struct pglist_data *pgdat)
6782{
6783 init_waitqueue_head(&pgdat->kcompactd_wait);
6784}
6785#else
6786static void pgdat_init_kcompactd(struct pglist_data *pgdat) {}
6787#endif
6788
Oscar Salvador03e85f92018-08-21 21:53:43 -07006789static void __meminit pgdat_init_internals(struct pglist_data *pgdat)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006790{
Dave Hansen208d54e2005-10-29 18:16:52 -07006791 pgdat_resize_init(pgdat);
Oscar Salvadorace1db32018-08-21 21:53:29 -07006792
Oscar Salvadorace1db32018-08-21 21:53:29 -07006793 pgdat_init_split_queue(pgdat);
6794 pgdat_init_kcompactd(pgdat);
6795
Linus Torvalds1da177e2005-04-16 15:20:36 -07006796 init_waitqueue_head(&pgdat->kswapd_wait);
Mel Gorman55150612012-07-31 16:44:35 -07006797 init_waitqueue_head(&pgdat->pfmemalloc_wait);
Oscar Salvadorace1db32018-08-21 21:53:29 -07006798
Joonsoo Kimeefa864b2014-12-12 16:55:46 -08006799 pgdat_page_ext_init(pgdat);
Mel Gormana52633d2016-07-28 15:45:28 -07006800 spin_lock_init(&pgdat->lru_lock);
Johannes Weiner867e5e12019-11-30 17:55:34 -08006801 lruvec_init(&pgdat->__lruvec);
Oscar Salvador03e85f92018-08-21 21:53:43 -07006802}
Michal Nazarewicz5f63b722012-01-11 15:16:11 +01006803
Oscar Salvador03e85f92018-08-21 21:53:43 -07006804static void __meminit zone_init_internals(struct zone *zone, enum zone_type idx, int nid,
6805 unsigned long remaining_pages)
6806{
Arun KS9705bea2018-12-28 00:34:24 -08006807 atomic_long_set(&zone->managed_pages, remaining_pages);
Oscar Salvador03e85f92018-08-21 21:53:43 -07006808 zone_set_nid(zone, nid);
6809 zone->name = zone_names[idx];
6810 zone->zone_pgdat = NODE_DATA(nid);
6811 spin_lock_init(&zone->lock);
6812 zone_seqlock_init(zone);
6813 zone_pcp_init(zone);
6814}
6815
6816/*
6817 * Set up the zone data structures
6818 * - init pgdat internals
6819 * - init all zones belonging to this node
6820 *
6821 * NOTE: this function is only called during memory hotplug
6822 */
6823#ifdef CONFIG_MEMORY_HOTPLUG
6824void __ref free_area_init_core_hotplug(int nid)
6825{
6826 enum zone_type z;
6827 pg_data_t *pgdat = NODE_DATA(nid);
6828
6829 pgdat_init_internals(pgdat);
6830 for (z = 0; z < MAX_NR_ZONES; z++)
6831 zone_init_internals(&pgdat->node_zones[z], z, nid, 0);
6832}
6833#endif
6834
6835/*
6836 * Set up the zone data structures:
6837 * - mark all pages reserved
6838 * - mark all memory queues empty
6839 * - clear the memory bitmaps
6840 *
6841 * NOTE: pgdat should get zeroed by caller.
6842 * NOTE: this function is only called during early init.
6843 */
6844static void __init free_area_init_core(struct pglist_data *pgdat)
6845{
6846 enum zone_type j;
6847 int nid = pgdat->node_id;
6848
6849 pgdat_init_internals(pgdat);
Johannes Weiner385386c2017-07-06 15:40:43 -07006850 pgdat->per_cpu_nodestats = &boot_nodestats;
6851
Linus Torvalds1da177e2005-04-16 15:20:36 -07006852 for (j = 0; j < MAX_NR_ZONES; j++) {
6853 struct zone *zone = pgdat->node_zones + j;
Wei Yange6943852018-06-07 17:06:04 -07006854 unsigned long size, freesize, memmap_pages;
Taku Izumid91749c2016-03-15 14:55:18 -07006855 unsigned long zone_start_pfn = zone->zone_start_pfn;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006856
Gu Zhengfebd5942015-06-24 16:57:02 -07006857 size = zone->spanned_pages;
Wei Yange6943852018-06-07 17:06:04 -07006858 freesize = zone->present_pages;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006859
Mel Gorman0e0b8642006-09-27 01:49:56 -07006860 /*
Jiang Liu9feedc92012-12-12 13:52:12 -08006861 * Adjust freesize so that it accounts for how much memory
Mel Gorman0e0b8642006-09-27 01:49:56 -07006862 * is used by this zone for memmap. This affects the watermark
6863 * and per-cpu initialisations
6864 */
Wei Yange6943852018-06-07 17:06:04 -07006865 memmap_pages = calc_memmap_size(size, freesize);
Zhong Hongboba914f42014-12-12 16:56:21 -08006866 if (!is_highmem_idx(j)) {
6867 if (freesize >= memmap_pages) {
6868 freesize -= memmap_pages;
6869 if (memmap_pages)
6870 printk(KERN_DEBUG
6871 " %s zone: %lu pages used for memmap\n",
6872 zone_names[j], memmap_pages);
6873 } else
Joe Perches11705322016-03-17 14:19:50 -07006874 pr_warn(" %s zone: %lu pages exceeds freesize %lu\n",
Zhong Hongboba914f42014-12-12 16:56:21 -08006875 zone_names[j], memmap_pages, freesize);
6876 }
Mel Gorman0e0b8642006-09-27 01:49:56 -07006877
Christoph Lameter62672762007-02-10 01:43:07 -08006878 /* Account for reserved pages */
Jiang Liu9feedc92012-12-12 13:52:12 -08006879 if (j == 0 && freesize > dma_reserve) {
6880 freesize -= dma_reserve;
Yinghai Lud903ef92008-10-18 20:27:06 -07006881 printk(KERN_DEBUG " %s zone: %lu pages reserved\n",
Christoph Lameter62672762007-02-10 01:43:07 -08006882 zone_names[0], dma_reserve);
Mel Gorman0e0b8642006-09-27 01:49:56 -07006883 }
6884
Christoph Lameter98d2b0e2006-09-25 23:31:12 -07006885 if (!is_highmem_idx(j))
Jiang Liu9feedc92012-12-12 13:52:12 -08006886 nr_kernel_pages += freesize;
Jiang Liu01cefae2012-12-12 13:52:19 -08006887 /* Charge for highmem memmap if there are enough kernel pages */
6888 else if (nr_kernel_pages > memmap_pages * 2)
6889 nr_kernel_pages -= memmap_pages;
Jiang Liu9feedc92012-12-12 13:52:12 -08006890 nr_all_pages += freesize;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006891
Jiang Liu9feedc92012-12-12 13:52:12 -08006892 /*
6893 * Set an approximate value for lowmem here, it will be adjusted
6894 * when the bootmem allocator frees pages into the buddy system.
6895 * And all highmem pages will be managed by the buddy system.
6896 */
Oscar Salvador03e85f92018-08-21 21:53:43 -07006897 zone_init_internals(zone, j, nid, freesize);
Johannes Weiner81c0a2b2013-09-11 14:20:47 -07006898
Joonsoo Kimd883c6c2018-05-23 10:18:21 +09006899 if (!size)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006900 continue;
6901
Andrew Morton955c1cd2012-05-29 15:06:31 -07006902 set_pageblock_order();
Joonsoo Kimd883c6c2018-05-23 10:18:21 +09006903 setup_usemap(pgdat, zone, zone_start_pfn, size);
6904 init_currently_empty_zone(zone, zone_start_pfn, size);
Heiko Carstens76cdd582008-05-14 16:05:52 -07006905 memmap_init(size, nid, j, zone_start_pfn);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006906 }
6907}
6908
Oscar Salvador0cd842f2017-11-15 17:39:18 -08006909#ifdef CONFIG_FLAT_NODE_MEM_MAP
Fabian Frederickbd721ea2016-08-02 14:03:33 -07006910static void __ref alloc_node_mem_map(struct pglist_data *pgdat)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006911{
Tony Luckb0aeba72015-11-10 10:09:47 -08006912 unsigned long __maybe_unused start = 0;
Laura Abbotta1c34a32015-11-05 18:48:46 -08006913 unsigned long __maybe_unused offset = 0;
6914
Linus Torvalds1da177e2005-04-16 15:20:36 -07006915 /* Skip empty nodes */
6916 if (!pgdat->node_spanned_pages)
6917 return;
6918
Tony Luckb0aeba72015-11-10 10:09:47 -08006919 start = pgdat->node_start_pfn & ~(MAX_ORDER_NR_PAGES - 1);
6920 offset = pgdat->node_start_pfn - start;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006921 /* ia64 gets its own node_mem_map, before this, without bootmem */
6922 if (!pgdat->node_mem_map) {
Tony Luckb0aeba72015-11-10 10:09:47 -08006923 unsigned long size, end;
Andy Whitcroftd41dee32005-06-23 00:07:54 -07006924 struct page *map;
6925
Bob Piccoe984bb42006-05-20 15:00:31 -07006926 /*
6927 * The zone's endpoints aren't required to be MAX_ORDER
6928 * aligned but the node_mem_map endpoints must be in order
6929 * for the buddy allocator to function correctly.
6930 */
Cody P Schafer108bcc92013-02-22 16:35:23 -08006931 end = pgdat_end_pfn(pgdat);
Bob Piccoe984bb42006-05-20 15:00:31 -07006932 end = ALIGN(end, MAX_ORDER_NR_PAGES);
6933 size = (end - start) * sizeof(struct page);
Mike Rapoport26fb3da2019-03-11 23:30:42 -07006934 map = memblock_alloc_node(size, SMP_CACHE_BYTES,
6935 pgdat->node_id);
Mike Rapoport23a70522019-03-05 15:46:43 -08006936 if (!map)
6937 panic("Failed to allocate %ld bytes for node %d memory map\n",
6938 size, pgdat->node_id);
Laura Abbotta1c34a32015-11-05 18:48:46 -08006939 pgdat->node_mem_map = map + offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006940 }
Oscar Salvador0cd842f2017-11-15 17:39:18 -08006941 pr_debug("%s: node %d, pgdat %08lx, node_mem_map %08lx\n",
6942 __func__, pgdat->node_id, (unsigned long)pgdat,
6943 (unsigned long)pgdat->node_mem_map);
Roman Zippel12d810c2007-05-31 00:40:54 -07006944#ifndef CONFIG_NEED_MULTIPLE_NODES
Linus Torvalds1da177e2005-04-16 15:20:36 -07006945 /*
6946 * With no DISCONTIG, the global mem_map is just set as node 0's
6947 */
Mel Gormanc7132162006-09-27 01:49:43 -07006948 if (pgdat == NODE_DATA(0)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07006949 mem_map = NODE_DATA(0)->node_mem_map;
Mel Gormanc7132162006-09-27 01:49:43 -07006950 if (page_to_pfn(mem_map) != pgdat->node_start_pfn)
Laura Abbotta1c34a32015-11-05 18:48:46 -08006951 mem_map -= offset;
Mel Gormanc7132162006-09-27 01:49:43 -07006952 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07006953#endif
6954}
Oscar Salvador0cd842f2017-11-15 17:39:18 -08006955#else
6956static void __ref alloc_node_mem_map(struct pglist_data *pgdat) { }
6957#endif /* CONFIG_FLAT_NODE_MEM_MAP */
Linus Torvalds1da177e2005-04-16 15:20:36 -07006958
Oscar Salvador0188dc92018-08-21 21:53:39 -07006959#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
6960static inline void pgdat_set_deferred_range(pg_data_t *pgdat)
6961{
Oscar Salvador0188dc92018-08-21 21:53:39 -07006962 pgdat->first_deferred_pfn = ULONG_MAX;
6963}
6964#else
6965static inline void pgdat_set_deferred_range(pg_data_t *pgdat) {}
6966#endif
6967
Mike Rapoport854e8842020-06-03 15:58:13 -07006968static void __init free_area_init_node(int nid)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006969{
Johannes Weiner9109fb72008-07-23 21:27:20 -07006970 pg_data_t *pgdat = NODE_DATA(nid);
Zhang Yanfei7960aed2013-07-08 15:59:52 -07006971 unsigned long start_pfn = 0;
6972 unsigned long end_pfn = 0;
Johannes Weiner9109fb72008-07-23 21:27:20 -07006973
Minchan Kim88fdf752012-07-31 16:46:14 -07006974 /* pg_data_t should be reset to zero when it's allocated */
Joonsoo Kim97a225e2020-06-03 15:59:01 -07006975 WARN_ON(pgdat->nr_zones || pgdat->kswapd_highest_zoneidx);
Minchan Kim88fdf752012-07-31 16:46:14 -07006976
Mike Rapoport854e8842020-06-03 15:58:13 -07006977 get_pfn_range_for_nid(nid, &start_pfn, &end_pfn);
Mel Gormanc7132162006-09-27 01:49:43 -07006978
Linus Torvalds1da177e2005-04-16 15:20:36 -07006979 pgdat->node_id = nid;
Mike Rapoport854e8842020-06-03 15:58:13 -07006980 pgdat->node_start_pfn = start_pfn;
Mel Gorman75ef7182016-07-28 15:45:24 -07006981 pgdat->per_cpu_nodestats = NULL;
Mike Rapoport854e8842020-06-03 15:58:13 -07006982
Juergen Gross8d29e182015-02-11 15:26:01 -08006983 pr_info("Initmem setup node %d [mem %#018Lx-%#018Lx]\n", nid,
Zhen Lei4ada0c52015-09-08 15:04:19 -07006984 (u64)start_pfn << PAGE_SHIFT,
6985 end_pfn ? ((u64)end_pfn << PAGE_SHIFT) - 1 : 0);
Mike Rapoport854e8842020-06-03 15:58:13 -07006986 calculate_node_totalpages(pgdat, start_pfn, end_pfn);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006987
6988 alloc_node_mem_map(pgdat);
Oscar Salvador0188dc92018-08-21 21:53:39 -07006989 pgdat_set_deferred_range(pgdat);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006990
Wei Yang7f3eb552015-09-08 14:59:50 -07006991 free_area_init_core(pgdat);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006992}
6993
Mike Rapoportbc9331a2020-06-03 15:58:09 -07006994void __init free_area_init_memoryless_node(int nid)
Mike Rapoport3f08a302020-06-03 15:57:02 -07006995{
Mike Rapoport854e8842020-06-03 15:58:13 -07006996 free_area_init_node(nid);
Mike Rapoport3f08a302020-06-03 15:57:02 -07006997}
6998
Mike Rapoportaca52c32018-10-30 15:07:44 -07006999#if !defined(CONFIG_FLAT_NODE_MEM_MAP)
Pavel Tatashinec393a0f2018-10-26 15:10:21 -07007000/*
David Hildenbrand4b094b72020-02-03 17:33:55 -08007001 * Initialize all valid struct pages in the range [spfn, epfn) and mark them
7002 * PageReserved(). Return the number of struct pages that were initialized.
Pavel Tatashinec393a0f2018-10-26 15:10:21 -07007003 */
David Hildenbrand4b094b72020-02-03 17:33:55 -08007004static u64 __init init_unavailable_range(unsigned long spfn, unsigned long epfn)
Pavel Tatashinec393a0f2018-10-26 15:10:21 -07007005{
7006 unsigned long pfn;
7007 u64 pgcnt = 0;
7008
7009 for (pfn = spfn; pfn < epfn; pfn++) {
7010 if (!pfn_valid(ALIGN_DOWN(pfn, pageblock_nr_pages))) {
7011 pfn = ALIGN_DOWN(pfn, pageblock_nr_pages)
7012 + pageblock_nr_pages - 1;
7013 continue;
7014 }
David Hildenbrand4b094b72020-02-03 17:33:55 -08007015 /*
7016 * Use a fake node/zone (0) for now. Some of these pages
7017 * (in memblock.reserved but not in memblock.memory) will
7018 * get re-initialized via reserve_bootmem_region() later.
7019 */
7020 __init_single_page(pfn_to_page(pfn), pfn, 0, 0);
7021 __SetPageReserved(pfn_to_page(pfn));
Pavel Tatashinec393a0f2018-10-26 15:10:21 -07007022 pgcnt++;
7023 }
7024
7025 return pgcnt;
7026}
7027
Pavel Tatashina4a3ede2017-11-15 17:36:31 -08007028/*
7029 * Only struct pages that are backed by physical memory are zeroed and
7030 * initialized by going through __init_single_page(). But, there are some
7031 * struct pages which are reserved in memblock allocator and their fields
7032 * may be accessed (for example page_to_pfn() on some configuration accesses
David Hildenbrand4b094b72020-02-03 17:33:55 -08007033 * flags). We must explicitly initialize those struct pages.
Naoya Horiguchi907ec5f2018-10-26 15:10:15 -07007034 *
7035 * This function also addresses a similar issue where struct pages are left
7036 * uninitialized because the physical address range is not covered by
7037 * memblock.memory or memblock.reserved. That could happen when memblock
David Hildenbrande8229692020-02-03 17:33:48 -08007038 * layout is manually configured via memmap=, or when the highest physical
7039 * address (max_pfn) does not end on a section boundary.
Pavel Tatashina4a3ede2017-11-15 17:36:31 -08007040 */
David Hildenbrand4b094b72020-02-03 17:33:55 -08007041static void __init init_unavailable_mem(void)
Pavel Tatashina4a3ede2017-11-15 17:36:31 -08007042{
7043 phys_addr_t start, end;
Pavel Tatashina4a3ede2017-11-15 17:36:31 -08007044 u64 i, pgcnt;
Naoya Horiguchi907ec5f2018-10-26 15:10:15 -07007045 phys_addr_t next = 0;
Pavel Tatashina4a3ede2017-11-15 17:36:31 -08007046
7047 /*
Naoya Horiguchi907ec5f2018-10-26 15:10:15 -07007048 * Loop through unavailable ranges not covered by memblock.memory.
Pavel Tatashina4a3ede2017-11-15 17:36:31 -08007049 */
7050 pgcnt = 0;
Mike Rapoport6e245ad2020-10-13 16:57:59 -07007051 for_each_mem_range(i, &start, &end) {
Pavel Tatashinec393a0f2018-10-26 15:10:21 -07007052 if (next < start)
David Hildenbrand4b094b72020-02-03 17:33:55 -08007053 pgcnt += init_unavailable_range(PFN_DOWN(next),
7054 PFN_UP(start));
Naoya Horiguchi907ec5f2018-10-26 15:10:15 -07007055 next = end;
Pavel Tatashina4a3ede2017-11-15 17:36:31 -08007056 }
David Hildenbrande8229692020-02-03 17:33:48 -08007057
7058 /*
7059 * Early sections always have a fully populated memmap for the whole
7060 * section - see pfn_valid(). If the last section has holes at the
7061 * end and that section is marked "online", the memmap will be
7062 * considered initialized. Make sure that memmap has a well defined
7063 * state.
7064 */
David Hildenbrand4b094b72020-02-03 17:33:55 -08007065 pgcnt += init_unavailable_range(PFN_DOWN(next),
7066 round_up(max_pfn, PAGES_PER_SECTION));
Pavel Tatashina4a3ede2017-11-15 17:36:31 -08007067
7068 /*
7069 * Struct pages that do not have backing memory. This could be because
7070 * firmware is using some of this memory, or for some other reasons.
Pavel Tatashina4a3ede2017-11-15 17:36:31 -08007071 */
7072 if (pgcnt)
Naoya Horiguchi907ec5f2018-10-26 15:10:15 -07007073 pr_info("Zeroed struct page in unavailable ranges: %lld pages", pgcnt);
Pavel Tatashina4a3ede2017-11-15 17:36:31 -08007074}
David Hildenbrand4b094b72020-02-03 17:33:55 -08007075#else
7076static inline void __init init_unavailable_mem(void)
7077{
7078}
Mike Rapoportaca52c32018-10-30 15:07:44 -07007079#endif /* !CONFIG_FLAT_NODE_MEM_MAP */
Pavel Tatashina4a3ede2017-11-15 17:36:31 -08007080
Miklos Szeredi418508c2007-05-23 13:57:55 -07007081#if MAX_NUMNODES > 1
7082/*
7083 * Figure out the number of possible node ids.
7084 */
Cody P Schaferf9872ca2013-04-29 15:08:01 -07007085void __init setup_nr_node_ids(void)
Miklos Szeredi418508c2007-05-23 13:57:55 -07007086{
Wei Yang904a9552015-09-08 14:59:48 -07007087 unsigned int highest;
Miklos Szeredi418508c2007-05-23 13:57:55 -07007088
Wei Yang904a9552015-09-08 14:59:48 -07007089 highest = find_last_bit(node_possible_map.bits, MAX_NUMNODES);
Miklos Szeredi418508c2007-05-23 13:57:55 -07007090 nr_node_ids = highest + 1;
7091}
Miklos Szeredi418508c2007-05-23 13:57:55 -07007092#endif
7093
Mel Gormanc7132162006-09-27 01:49:43 -07007094/**
Tejun Heo1e019792011-07-12 09:45:34 +02007095 * node_map_pfn_alignment - determine the maximum internode alignment
7096 *
7097 * This function should be called after node map is populated and sorted.
7098 * It calculates the maximum power of two alignment which can distinguish
7099 * all the nodes.
7100 *
7101 * For example, if all nodes are 1GiB and aligned to 1GiB, the return value
7102 * would indicate 1GiB alignment with (1 << (30 - PAGE_SHIFT)). If the
7103 * nodes are shifted by 256MiB, 256MiB. Note that if only the last node is
7104 * shifted, 1GiB is enough and this function will indicate so.
7105 *
7106 * This is used to test whether pfn -> nid mapping of the chosen memory
7107 * model has fine enough granularity to avoid incorrect mapping for the
7108 * populated node map.
7109 *
Mike Rapoporta862f682019-03-05 15:48:42 -08007110 * Return: the determined alignment in pfn's. 0 if there is no alignment
Tejun Heo1e019792011-07-12 09:45:34 +02007111 * requirement (single node).
7112 */
7113unsigned long __init node_map_pfn_alignment(void)
7114{
7115 unsigned long accl_mask = 0, last_end = 0;
Tejun Heoc13291a2011-07-12 10:46:30 +02007116 unsigned long start, end, mask;
Anshuman Khandual98fa15f2019-03-05 15:42:58 -08007117 int last_nid = NUMA_NO_NODE;
Tejun Heoc13291a2011-07-12 10:46:30 +02007118 int i, nid;
Tejun Heo1e019792011-07-12 09:45:34 +02007119
Tejun Heoc13291a2011-07-12 10:46:30 +02007120 for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) {
Tejun Heo1e019792011-07-12 09:45:34 +02007121 if (!start || last_nid < 0 || last_nid == nid) {
7122 last_nid = nid;
7123 last_end = end;
7124 continue;
7125 }
7126
7127 /*
7128 * Start with a mask granular enough to pin-point to the
7129 * start pfn and tick off bits one-by-one until it becomes
7130 * too coarse to separate the current node from the last.
7131 */
7132 mask = ~((1 << __ffs(start)) - 1);
7133 while (mask && last_end <= (start & (mask << 1)))
7134 mask <<= 1;
7135
7136 /* accumulate all internode masks */
7137 accl_mask |= mask;
7138 }
7139
7140 /* convert mask to number of pages */
7141 return ~accl_mask + 1;
7142}
7143
Mel Gormanc7132162006-09-27 01:49:43 -07007144/**
7145 * find_min_pfn_with_active_regions - Find the minimum PFN registered
7146 *
Mike Rapoporta862f682019-03-05 15:48:42 -08007147 * Return: the minimum PFN based on information provided via
Zhang Zhen7d018172014-06-04 16:10:53 -07007148 * memblock_set_node().
Mel Gormanc7132162006-09-27 01:49:43 -07007149 */
7150unsigned long __init find_min_pfn_with_active_regions(void)
7151{
Mike Rapoport8a1b25f2020-06-03 15:58:18 -07007152 return PHYS_PFN(memblock_start_of_DRAM());
Mel Gormanc7132162006-09-27 01:49:43 -07007153}
7154
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07007155/*
7156 * early_calculate_totalpages()
7157 * Sum pages in active regions for movable zone.
Lai Jiangshan4b0ef1fe2012-12-12 13:51:46 -08007158 * Populate N_MEMORY for calculating usable_nodes.
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07007159 */
Adrian Bunk484f51f2007-10-16 01:26:03 -07007160static unsigned long __init early_calculate_totalpages(void)
Mel Gorman7e63efe2007-07-17 04:03:15 -07007161{
Mel Gorman7e63efe2007-07-17 04:03:15 -07007162 unsigned long totalpages = 0;
Tejun Heoc13291a2011-07-12 10:46:30 +02007163 unsigned long start_pfn, end_pfn;
7164 int i, nid;
Mel Gorman7e63efe2007-07-17 04:03:15 -07007165
Tejun Heoc13291a2011-07-12 10:46:30 +02007166 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) {
7167 unsigned long pages = end_pfn - start_pfn;
7168
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07007169 totalpages += pages;
7170 if (pages)
Lai Jiangshan4b0ef1fe2012-12-12 13:51:46 -08007171 node_set_state(nid, N_MEMORY);
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07007172 }
Pintu Kumarb8af2942013-09-11 14:20:34 -07007173 return totalpages;
Mel Gorman7e63efe2007-07-17 04:03:15 -07007174}
7175
Mel Gorman2a1e2742007-07-17 04:03:12 -07007176/*
7177 * Find the PFN the Movable zone begins in each node. Kernel memory
7178 * is spread evenly between nodes as long as the nodes have enough
7179 * memory. When they don't, some nodes will have more kernelcore than
7180 * others
7181 */
Kautuk Consulb224ef82012-03-21 16:34:15 -07007182static void __init find_zone_movable_pfns_for_nodes(void)
Mel Gorman2a1e2742007-07-17 04:03:12 -07007183{
7184 int i, nid;
7185 unsigned long usable_startpfn;
7186 unsigned long kernelcore_node, kernelcore_remaining;
Yinghai Lu66918dc2009-06-30 11:41:37 -07007187 /* save the state before borrow the nodemask */
Lai Jiangshan4b0ef1fe2012-12-12 13:51:46 -08007188 nodemask_t saved_node_state = node_states[N_MEMORY];
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07007189 unsigned long totalpages = early_calculate_totalpages();
Lai Jiangshan4b0ef1fe2012-12-12 13:51:46 -08007190 int usable_nodes = nodes_weight(node_states[N_MEMORY]);
Emil Medve136199f2014-04-07 15:37:52 -07007191 struct memblock_region *r;
Tang Chenb2f3eeb2014-01-21 15:49:38 -08007192
7193 /* Need to find movable_zone earlier when movable_node is specified. */
7194 find_usable_zone_for_movable();
Mel Gorman2a1e2742007-07-17 04:03:12 -07007195
Mel Gorman7e63efe2007-07-17 04:03:15 -07007196 /*
Tang Chenb2f3eeb2014-01-21 15:49:38 -08007197 * If movable_node is specified, ignore kernelcore and movablecore
7198 * options.
7199 */
7200 if (movable_node_is_enabled()) {
Mike Rapoportcc6de162020-10-13 16:58:30 -07007201 for_each_mem_region(r) {
Emil Medve136199f2014-04-07 15:37:52 -07007202 if (!memblock_is_hotpluggable(r))
Tang Chenb2f3eeb2014-01-21 15:49:38 -08007203 continue;
7204
Mike Rapoportd622abf2020-06-03 15:56:53 -07007205 nid = memblock_get_region_node(r);
Tang Chenb2f3eeb2014-01-21 15:49:38 -08007206
Emil Medve136199f2014-04-07 15:37:52 -07007207 usable_startpfn = PFN_DOWN(r->base);
Tang Chenb2f3eeb2014-01-21 15:49:38 -08007208 zone_movable_pfn[nid] = zone_movable_pfn[nid] ?
7209 min(usable_startpfn, zone_movable_pfn[nid]) :
7210 usable_startpfn;
7211 }
7212
7213 goto out2;
7214 }
7215
7216 /*
Taku Izumi342332e2016-03-15 14:55:22 -07007217 * If kernelcore=mirror is specified, ignore movablecore option
7218 */
7219 if (mirrored_kernelcore) {
7220 bool mem_below_4gb_not_mirrored = false;
7221
Mike Rapoportcc6de162020-10-13 16:58:30 -07007222 for_each_mem_region(r) {
Taku Izumi342332e2016-03-15 14:55:22 -07007223 if (memblock_is_mirror(r))
7224 continue;
7225
Mike Rapoportd622abf2020-06-03 15:56:53 -07007226 nid = memblock_get_region_node(r);
Taku Izumi342332e2016-03-15 14:55:22 -07007227
7228 usable_startpfn = memblock_region_memory_base_pfn(r);
7229
7230 if (usable_startpfn < 0x100000) {
7231 mem_below_4gb_not_mirrored = true;
7232 continue;
7233 }
7234
7235 zone_movable_pfn[nid] = zone_movable_pfn[nid] ?
7236 min(usable_startpfn, zone_movable_pfn[nid]) :
7237 usable_startpfn;
7238 }
7239
7240 if (mem_below_4gb_not_mirrored)
Chen Tao633bf2f2020-06-03 16:00:02 -07007241 pr_warn("This configuration results in unmirrored kernel memory.\n");
Taku Izumi342332e2016-03-15 14:55:22 -07007242
7243 goto out2;
7244 }
7245
7246 /*
David Rientjesa5c6d652018-04-05 16:23:09 -07007247 * If kernelcore=nn% or movablecore=nn% was specified, calculate the
7248 * amount of necessary memory.
7249 */
7250 if (required_kernelcore_percent)
7251 required_kernelcore = (totalpages * 100 * required_kernelcore_percent) /
7252 10000UL;
7253 if (required_movablecore_percent)
7254 required_movablecore = (totalpages * 100 * required_movablecore_percent) /
7255 10000UL;
7256
7257 /*
7258 * If movablecore= was specified, calculate what size of
Mel Gorman7e63efe2007-07-17 04:03:15 -07007259 * kernelcore that corresponds so that memory usable for
7260 * any allocation type is evenly spread. If both kernelcore
7261 * and movablecore are specified, then the value of kernelcore
7262 * will be used for required_kernelcore if it's greater than
7263 * what movablecore would have allowed.
7264 */
7265 if (required_movablecore) {
Mel Gorman7e63efe2007-07-17 04:03:15 -07007266 unsigned long corepages;
7267
7268 /*
7269 * Round-up so that ZONE_MOVABLE is at least as large as what
7270 * was requested by the user
7271 */
7272 required_movablecore =
7273 roundup(required_movablecore, MAX_ORDER_NR_PAGES);
Xishi Qiu9fd745d2015-11-05 18:48:11 -08007274 required_movablecore = min(totalpages, required_movablecore);
Mel Gorman7e63efe2007-07-17 04:03:15 -07007275 corepages = totalpages - required_movablecore;
7276
7277 required_kernelcore = max(required_kernelcore, corepages);
7278 }
7279
Xishi Qiubde304b2015-11-05 18:48:56 -08007280 /*
7281 * If kernelcore was not specified or kernelcore size is larger
7282 * than totalpages, there is no ZONE_MOVABLE.
7283 */
7284 if (!required_kernelcore || required_kernelcore >= totalpages)
Yinghai Lu66918dc2009-06-30 11:41:37 -07007285 goto out;
Mel Gorman2a1e2742007-07-17 04:03:12 -07007286
7287 /* usable_startpfn is the lowest possible pfn ZONE_MOVABLE can be at */
Mel Gorman2a1e2742007-07-17 04:03:12 -07007288 usable_startpfn = arch_zone_lowest_possible_pfn[movable_zone];
7289
7290restart:
7291 /* Spread kernelcore memory as evenly as possible throughout nodes */
7292 kernelcore_node = required_kernelcore / usable_nodes;
Lai Jiangshan4b0ef1fe2012-12-12 13:51:46 -08007293 for_each_node_state(nid, N_MEMORY) {
Tejun Heoc13291a2011-07-12 10:46:30 +02007294 unsigned long start_pfn, end_pfn;
7295
Mel Gorman2a1e2742007-07-17 04:03:12 -07007296 /*
7297 * Recalculate kernelcore_node if the division per node
7298 * now exceeds what is necessary to satisfy the requested
7299 * amount of memory for the kernel
7300 */
7301 if (required_kernelcore < kernelcore_node)
7302 kernelcore_node = required_kernelcore / usable_nodes;
7303
7304 /*
7305 * As the map is walked, we track how much memory is usable
7306 * by the kernel using kernelcore_remaining. When it is
7307 * 0, the rest of the node is usable by ZONE_MOVABLE
7308 */
7309 kernelcore_remaining = kernelcore_node;
7310
7311 /* Go through each range of PFNs within this node */
Tejun Heoc13291a2011-07-12 10:46:30 +02007312 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
Mel Gorman2a1e2742007-07-17 04:03:12 -07007313 unsigned long size_pages;
7314
Tejun Heoc13291a2011-07-12 10:46:30 +02007315 start_pfn = max(start_pfn, zone_movable_pfn[nid]);
Mel Gorman2a1e2742007-07-17 04:03:12 -07007316 if (start_pfn >= end_pfn)
7317 continue;
7318
7319 /* Account for what is only usable for kernelcore */
7320 if (start_pfn < usable_startpfn) {
7321 unsigned long kernel_pages;
7322 kernel_pages = min(end_pfn, usable_startpfn)
7323 - start_pfn;
7324
7325 kernelcore_remaining -= min(kernel_pages,
7326 kernelcore_remaining);
7327 required_kernelcore -= min(kernel_pages,
7328 required_kernelcore);
7329
7330 /* Continue if range is now fully accounted */
7331 if (end_pfn <= usable_startpfn) {
7332
7333 /*
7334 * Push zone_movable_pfn to the end so
7335 * that if we have to rebalance
7336 * kernelcore across nodes, we will
7337 * not double account here
7338 */
7339 zone_movable_pfn[nid] = end_pfn;
7340 continue;
7341 }
7342 start_pfn = usable_startpfn;
7343 }
7344
7345 /*
7346 * The usable PFN range for ZONE_MOVABLE is from
7347 * start_pfn->end_pfn. Calculate size_pages as the
7348 * number of pages used as kernelcore
7349 */
7350 size_pages = end_pfn - start_pfn;
7351 if (size_pages > kernelcore_remaining)
7352 size_pages = kernelcore_remaining;
7353 zone_movable_pfn[nid] = start_pfn + size_pages;
7354
7355 /*
7356 * Some kernelcore has been met, update counts and
7357 * break if the kernelcore for this node has been
Pintu Kumarb8af2942013-09-11 14:20:34 -07007358 * satisfied
Mel Gorman2a1e2742007-07-17 04:03:12 -07007359 */
7360 required_kernelcore -= min(required_kernelcore,
7361 size_pages);
7362 kernelcore_remaining -= size_pages;
7363 if (!kernelcore_remaining)
7364 break;
7365 }
7366 }
7367
7368 /*
7369 * If there is still required_kernelcore, we do another pass with one
7370 * less node in the count. This will push zone_movable_pfn[nid] further
7371 * along on the nodes that still have memory until kernelcore is
Pintu Kumarb8af2942013-09-11 14:20:34 -07007372 * satisfied
Mel Gorman2a1e2742007-07-17 04:03:12 -07007373 */
7374 usable_nodes--;
7375 if (usable_nodes && required_kernelcore > usable_nodes)
7376 goto restart;
7377
Tang Chenb2f3eeb2014-01-21 15:49:38 -08007378out2:
Mel Gorman2a1e2742007-07-17 04:03:12 -07007379 /* Align start of ZONE_MOVABLE on all nids to MAX_ORDER_NR_PAGES */
7380 for (nid = 0; nid < MAX_NUMNODES; nid++)
7381 zone_movable_pfn[nid] =
7382 roundup(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES);
Yinghai Lu66918dc2009-06-30 11:41:37 -07007383
Yinghai Lu20e69262013-03-01 14:51:27 -08007384out:
Yinghai Lu66918dc2009-06-30 11:41:37 -07007385 /* restore the node_state */
Lai Jiangshan4b0ef1fe2012-12-12 13:51:46 -08007386 node_states[N_MEMORY] = saved_node_state;
Mel Gorman2a1e2742007-07-17 04:03:12 -07007387}
7388
Lai Jiangshan4b0ef1fe2012-12-12 13:51:46 -08007389/* Any regular or high memory on that node ? */
7390static void check_for_memory(pg_data_t *pgdat, int nid)
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07007391{
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07007392 enum zone_type zone_type;
7393
Lai Jiangshan4b0ef1fe2012-12-12 13:51:46 -08007394 for (zone_type = 0; zone_type <= ZONE_MOVABLE - 1; zone_type++) {
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07007395 struct zone *zone = &pgdat->node_zones[zone_type];
Xishi Qiub38a8722013-11-12 15:07:20 -08007396 if (populated_zone(zone)) {
Oscar Salvador7b0e0c02018-10-26 15:03:58 -07007397 if (IS_ENABLED(CONFIG_HIGHMEM))
7398 node_set_state(nid, N_HIGH_MEMORY);
7399 if (zone_type <= ZONE_NORMAL)
Lai Jiangshan4b0ef1fe2012-12-12 13:51:46 -08007400 node_set_state(nid, N_NORMAL_MEMORY);
Bob Liud0048b02012-01-12 17:19:07 -08007401 break;
7402 }
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07007403 }
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07007404}
7405
Mike Rapoport51930df2020-06-03 15:58:03 -07007406/*
7407 * Some architecturs, e.g. ARC may have ZONE_HIGHMEM below ZONE_NORMAL. For
7408 * such cases we allow max_zone_pfn sorted in the descending order
7409 */
7410bool __weak arch_has_descending_max_zone_pfns(void)
7411{
7412 return false;
7413}
7414
Mel Gormanc7132162006-09-27 01:49:43 -07007415/**
Mike Rapoport9691a072020-06-03 15:57:10 -07007416 * free_area_init - Initialise all pg_data_t and zone data
Randy Dunlap88ca3b92006-10-04 02:15:25 -07007417 * @max_zone_pfn: an array of max PFNs for each zone
Mel Gormanc7132162006-09-27 01:49:43 -07007418 *
7419 * This will call free_area_init_node() for each active node in the system.
Zhang Zhen7d018172014-06-04 16:10:53 -07007420 * Using the page ranges provided by memblock_set_node(), the size of each
Mel Gormanc7132162006-09-27 01:49:43 -07007421 * zone in each node and their holes is calculated. If the maximum PFN
7422 * between two adjacent zones match, it is assumed that the zone is empty.
7423 * For example, if arch_max_dma_pfn == arch_max_dma32_pfn, it is assumed
7424 * that arch_max_dma32_pfn has no pages. It is also assumed that a zone
7425 * starts where the previous one ended. For example, ZONE_DMA32 starts
7426 * at arch_max_dma_pfn.
7427 */
Mike Rapoport9691a072020-06-03 15:57:10 -07007428void __init free_area_init(unsigned long *max_zone_pfn)
Mel Gormanc7132162006-09-27 01:49:43 -07007429{
Tejun Heoc13291a2011-07-12 10:46:30 +02007430 unsigned long start_pfn, end_pfn;
Mike Rapoport51930df2020-06-03 15:58:03 -07007431 int i, nid, zone;
7432 bool descending;
Mel Gormana6af2bc2007-02-10 01:42:57 -08007433
Mel Gormanc7132162006-09-27 01:49:43 -07007434 /* Record where the zone boundaries are */
7435 memset(arch_zone_lowest_possible_pfn, 0,
7436 sizeof(arch_zone_lowest_possible_pfn));
7437 memset(arch_zone_highest_possible_pfn, 0,
7438 sizeof(arch_zone_highest_possible_pfn));
Oliver O'Halloran90cae1f2016-07-26 15:22:17 -07007439
7440 start_pfn = find_min_pfn_with_active_regions();
Mike Rapoport51930df2020-06-03 15:58:03 -07007441 descending = arch_has_descending_max_zone_pfns();
Oliver O'Halloran90cae1f2016-07-26 15:22:17 -07007442
7443 for (i = 0; i < MAX_NR_ZONES; i++) {
Mike Rapoport51930df2020-06-03 15:58:03 -07007444 if (descending)
7445 zone = MAX_NR_ZONES - i - 1;
7446 else
7447 zone = i;
7448
7449 if (zone == ZONE_MOVABLE)
Mel Gorman2a1e2742007-07-17 04:03:12 -07007450 continue;
Oliver O'Halloran90cae1f2016-07-26 15:22:17 -07007451
Mike Rapoport51930df2020-06-03 15:58:03 -07007452 end_pfn = max(max_zone_pfn[zone], start_pfn);
7453 arch_zone_lowest_possible_pfn[zone] = start_pfn;
7454 arch_zone_highest_possible_pfn[zone] = end_pfn;
Oliver O'Halloran90cae1f2016-07-26 15:22:17 -07007455
7456 start_pfn = end_pfn;
Mel Gormanc7132162006-09-27 01:49:43 -07007457 }
Mel Gorman2a1e2742007-07-17 04:03:12 -07007458
7459 /* Find the PFNs that ZONE_MOVABLE begins at in each node */
7460 memset(zone_movable_pfn, 0, sizeof(zone_movable_pfn));
Kautuk Consulb224ef82012-03-21 16:34:15 -07007461 find_zone_movable_pfns_for_nodes();
Mel Gormanc7132162006-09-27 01:49:43 -07007462
Mel Gormanc7132162006-09-27 01:49:43 -07007463 /* Print out the zone ranges */
Anton Blanchardf88dfff2014-12-10 15:42:53 -08007464 pr_info("Zone ranges:\n");
Mel Gorman2a1e2742007-07-17 04:03:12 -07007465 for (i = 0; i < MAX_NR_ZONES; i++) {
7466 if (i == ZONE_MOVABLE)
7467 continue;
Anton Blanchardf88dfff2014-12-10 15:42:53 -08007468 pr_info(" %-8s ", zone_names[i]);
David Rientjes72f0ba02010-03-05 13:42:14 -08007469 if (arch_zone_lowest_possible_pfn[i] ==
7470 arch_zone_highest_possible_pfn[i])
Anton Blanchardf88dfff2014-12-10 15:42:53 -08007471 pr_cont("empty\n");
David Rientjes72f0ba02010-03-05 13:42:14 -08007472 else
Juergen Gross8d29e182015-02-11 15:26:01 -08007473 pr_cont("[mem %#018Lx-%#018Lx]\n",
7474 (u64)arch_zone_lowest_possible_pfn[i]
7475 << PAGE_SHIFT,
7476 ((u64)arch_zone_highest_possible_pfn[i]
Bjorn Helgaasa62e2f42012-05-29 15:06:30 -07007477 << PAGE_SHIFT) - 1);
Mel Gorman2a1e2742007-07-17 04:03:12 -07007478 }
7479
7480 /* Print out the PFNs ZONE_MOVABLE begins at in each node */
Anton Blanchardf88dfff2014-12-10 15:42:53 -08007481 pr_info("Movable zone start for each node\n");
Mel Gorman2a1e2742007-07-17 04:03:12 -07007482 for (i = 0; i < MAX_NUMNODES; i++) {
7483 if (zone_movable_pfn[i])
Juergen Gross8d29e182015-02-11 15:26:01 -08007484 pr_info(" Node %d: %#018Lx\n", i,
7485 (u64)zone_movable_pfn[i] << PAGE_SHIFT);
Mel Gorman2a1e2742007-07-17 04:03:12 -07007486 }
Mel Gormanc7132162006-09-27 01:49:43 -07007487
Dan Williamsf46edbd2019-07-18 15:58:04 -07007488 /*
7489 * Print out the early node map, and initialize the
7490 * subsection-map relative to active online memory ranges to
7491 * enable future "sub-section" extensions of the memory map.
7492 */
Anton Blanchardf88dfff2014-12-10 15:42:53 -08007493 pr_info("Early memory node ranges\n");
Dan Williamsf46edbd2019-07-18 15:58:04 -07007494 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) {
Juergen Gross8d29e182015-02-11 15:26:01 -08007495 pr_info(" node %3d: [mem %#018Lx-%#018Lx]\n", nid,
7496 (u64)start_pfn << PAGE_SHIFT,
7497 ((u64)end_pfn << PAGE_SHIFT) - 1);
Dan Williamsf46edbd2019-07-18 15:58:04 -07007498 subsection_map_init(start_pfn, end_pfn - start_pfn);
7499 }
Mel Gormanc7132162006-09-27 01:49:43 -07007500
7501 /* Initialise every node */
Mel Gorman708614e2008-07-23 21:26:51 -07007502 mminit_verify_pageflags_layout();
Christoph Lameter8ef82862007-02-20 13:57:52 -08007503 setup_nr_node_ids();
David Hildenbrand4b094b72020-02-03 17:33:55 -08007504 init_unavailable_mem();
Mel Gormanc7132162006-09-27 01:49:43 -07007505 for_each_online_node(nid) {
7506 pg_data_t *pgdat = NODE_DATA(nid);
Mike Rapoport854e8842020-06-03 15:58:13 -07007507 free_area_init_node(nid);
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07007508
7509 /* Any memory on that node */
7510 if (pgdat->node_present_pages)
Lai Jiangshan4b0ef1fe2012-12-12 13:51:46 -08007511 node_set_state(nid, N_MEMORY);
7512 check_for_memory(pgdat, nid);
Mel Gormanc7132162006-09-27 01:49:43 -07007513 }
7514}
Mel Gorman2a1e2742007-07-17 04:03:12 -07007515
David Rientjesa5c6d652018-04-05 16:23:09 -07007516static int __init cmdline_parse_core(char *p, unsigned long *core,
7517 unsigned long *percent)
Mel Gorman2a1e2742007-07-17 04:03:12 -07007518{
7519 unsigned long long coremem;
David Rientjesa5c6d652018-04-05 16:23:09 -07007520 char *endptr;
7521
Mel Gorman2a1e2742007-07-17 04:03:12 -07007522 if (!p)
7523 return -EINVAL;
7524
David Rientjesa5c6d652018-04-05 16:23:09 -07007525 /* Value may be a percentage of total memory, otherwise bytes */
7526 coremem = simple_strtoull(p, &endptr, 0);
7527 if (*endptr == '%') {
7528 /* Paranoid check for percent values greater than 100 */
7529 WARN_ON(coremem > 100);
Mel Gorman2a1e2742007-07-17 04:03:12 -07007530
David Rientjesa5c6d652018-04-05 16:23:09 -07007531 *percent = coremem;
7532 } else {
7533 coremem = memparse(p, &p);
7534 /* Paranoid check that UL is enough for the coremem value */
7535 WARN_ON((coremem >> PAGE_SHIFT) > ULONG_MAX);
Mel Gorman2a1e2742007-07-17 04:03:12 -07007536
David Rientjesa5c6d652018-04-05 16:23:09 -07007537 *core = coremem >> PAGE_SHIFT;
7538 *percent = 0UL;
7539 }
Mel Gorman2a1e2742007-07-17 04:03:12 -07007540 return 0;
7541}
Mel Gormaned7ed362007-07-17 04:03:14 -07007542
Mel Gorman7e63efe2007-07-17 04:03:15 -07007543/*
7544 * kernelcore=size sets the amount of memory for use for allocations that
7545 * cannot be reclaimed or migrated.
7546 */
7547static int __init cmdline_parse_kernelcore(char *p)
7548{
Taku Izumi342332e2016-03-15 14:55:22 -07007549 /* parse kernelcore=mirror */
7550 if (parse_option_str(p, "mirror")) {
7551 mirrored_kernelcore = true;
7552 return 0;
7553 }
7554
David Rientjesa5c6d652018-04-05 16:23:09 -07007555 return cmdline_parse_core(p, &required_kernelcore,
7556 &required_kernelcore_percent);
Mel Gorman7e63efe2007-07-17 04:03:15 -07007557}
7558
7559/*
7560 * movablecore=size sets the amount of memory for use for allocations that
7561 * can be reclaimed or migrated.
7562 */
7563static int __init cmdline_parse_movablecore(char *p)
7564{
David Rientjesa5c6d652018-04-05 16:23:09 -07007565 return cmdline_parse_core(p, &required_movablecore,
7566 &required_movablecore_percent);
Mel Gorman7e63efe2007-07-17 04:03:15 -07007567}
7568
Mel Gormaned7ed362007-07-17 04:03:14 -07007569early_param("kernelcore", cmdline_parse_kernelcore);
Mel Gorman7e63efe2007-07-17 04:03:15 -07007570early_param("movablecore", cmdline_parse_movablecore);
Mel Gormaned7ed362007-07-17 04:03:14 -07007571
Jiang Liuc3d5f5f2013-07-03 15:03:14 -07007572void adjust_managed_page_count(struct page *page, long count)
7573{
Arun KS9705bea2018-12-28 00:34:24 -08007574 atomic_long_add(count, &page_zone(page)->managed_pages);
Arun KSca79b0c2018-12-28 00:34:29 -08007575 totalram_pages_add(count);
Jiang Liu3dcc0572013-07-03 15:03:21 -07007576#ifdef CONFIG_HIGHMEM
7577 if (PageHighMem(page))
Arun KSca79b0c2018-12-28 00:34:29 -08007578 totalhigh_pages_add(count);
Jiang Liu3dcc0572013-07-03 15:03:21 -07007579#endif
Jiang Liuc3d5f5f2013-07-03 15:03:14 -07007580}
Jiang Liu3dcc0572013-07-03 15:03:21 -07007581EXPORT_SYMBOL(adjust_managed_page_count);
Jiang Liuc3d5f5f2013-07-03 15:03:14 -07007582
Alexey Dobriyane5cb1132018-12-28 00:36:03 -08007583unsigned long free_reserved_area(void *start, void *end, int poison, const char *s)
Jiang Liu69afade2013-04-29 15:06:21 -07007584{
Jiang Liu11199692013-07-03 15:02:48 -07007585 void *pos;
7586 unsigned long pages = 0;
Jiang Liu69afade2013-04-29 15:06:21 -07007587
Jiang Liu11199692013-07-03 15:02:48 -07007588 start = (void *)PAGE_ALIGN((unsigned long)start);
7589 end = (void *)((unsigned long)end & PAGE_MASK);
7590 for (pos = start; pos < end; pos += PAGE_SIZE, pages++) {
Dave Hansen0d834322018-08-02 15:58:26 -07007591 struct page *page = virt_to_page(pos);
7592 void *direct_map_addr;
7593
7594 /*
7595 * 'direct_map_addr' might be different from 'pos'
7596 * because some architectures' virt_to_page()
7597 * work with aliases. Getting the direct map
7598 * address ensures that we get a _writeable_
7599 * alias for the memset().
7600 */
7601 direct_map_addr = page_address(page);
Jiang Liudbe67df2013-07-03 15:02:51 -07007602 if ((unsigned int)poison <= 0xFF)
Dave Hansen0d834322018-08-02 15:58:26 -07007603 memset(direct_map_addr, poison, PAGE_SIZE);
7604
7605 free_reserved_page(page);
Jiang Liu69afade2013-04-29 15:06:21 -07007606 }
7607
7608 if (pages && s)
Josh Poimboeufadb1fe92016-10-25 09:51:14 -05007609 pr_info("Freeing %s memory: %ldK\n",
7610 s, pages << (PAGE_SHIFT - 10));
Jiang Liu69afade2013-04-29 15:06:21 -07007611
7612 return pages;
7613}
7614
Jiang Liucfa11e02013-04-29 15:07:00 -07007615#ifdef CONFIG_HIGHMEM
7616void free_highmem_page(struct page *page)
7617{
7618 __free_reserved_page(page);
Arun KSca79b0c2018-12-28 00:34:29 -08007619 totalram_pages_inc();
Arun KS9705bea2018-12-28 00:34:24 -08007620 atomic_long_inc(&page_zone(page)->managed_pages);
Arun KSca79b0c2018-12-28 00:34:29 -08007621 totalhigh_pages_inc();
Jiang Liucfa11e02013-04-29 15:07:00 -07007622}
7623#endif
7624
Jiang Liu7ee3d4e2013-07-03 15:03:41 -07007625
7626void __init mem_init_print_info(const char *str)
7627{
7628 unsigned long physpages, codesize, datasize, rosize, bss_size;
7629 unsigned long init_code_size, init_data_size;
7630
7631 physpages = get_num_physpages();
7632 codesize = _etext - _stext;
7633 datasize = _edata - _sdata;
7634 rosize = __end_rodata - __start_rodata;
7635 bss_size = __bss_stop - __bss_start;
7636 init_data_size = __init_end - __init_begin;
7637 init_code_size = _einittext - _sinittext;
7638
7639 /*
7640 * Detect special cases and adjust section sizes accordingly:
7641 * 1) .init.* may be embedded into .data sections
7642 * 2) .init.text.* may be out of [__init_begin, __init_end],
7643 * please refer to arch/tile/kernel/vmlinux.lds.S.
7644 * 3) .rodata.* may be embedded into .text or .data sections.
7645 */
7646#define adj_init_size(start, end, size, pos, adj) \
Pintu Kumarb8af2942013-09-11 14:20:34 -07007647 do { \
7648 if (start <= pos && pos < end && size > adj) \
7649 size -= adj; \
7650 } while (0)
Jiang Liu7ee3d4e2013-07-03 15:03:41 -07007651
7652 adj_init_size(__init_begin, __init_end, init_data_size,
7653 _sinittext, init_code_size);
7654 adj_init_size(_stext, _etext, codesize, _sinittext, init_code_size);
7655 adj_init_size(_sdata, _edata, datasize, __init_begin, init_data_size);
7656 adj_init_size(_stext, _etext, codesize, __start_rodata, rosize);
7657 adj_init_size(_sdata, _edata, datasize, __start_rodata, rosize);
7658
7659#undef adj_init_size
7660
Joe Perches756a0252016-03-17 14:19:47 -07007661 pr_info("Memory: %luK/%luK available (%luK kernel code, %luK rwdata, %luK rodata, %luK init, %luK bss, %luK reserved, %luK cma-reserved"
Jiang Liu7ee3d4e2013-07-03 15:03:41 -07007662#ifdef CONFIG_HIGHMEM
Joe Perches756a0252016-03-17 14:19:47 -07007663 ", %luK highmem"
Jiang Liu7ee3d4e2013-07-03 15:03:41 -07007664#endif
Joe Perches756a0252016-03-17 14:19:47 -07007665 "%s%s)\n",
7666 nr_free_pages() << (PAGE_SHIFT - 10),
7667 physpages << (PAGE_SHIFT - 10),
7668 codesize >> 10, datasize >> 10, rosize >> 10,
7669 (init_data_size + init_code_size) >> 10, bss_size >> 10,
Arun KSca79b0c2018-12-28 00:34:29 -08007670 (physpages - totalram_pages() - totalcma_pages) << (PAGE_SHIFT - 10),
Joe Perches756a0252016-03-17 14:19:47 -07007671 totalcma_pages << (PAGE_SHIFT - 10),
Jiang Liu7ee3d4e2013-07-03 15:03:41 -07007672#ifdef CONFIG_HIGHMEM
Arun KSca79b0c2018-12-28 00:34:29 -08007673 totalhigh_pages() << (PAGE_SHIFT - 10),
Jiang Liu7ee3d4e2013-07-03 15:03:41 -07007674#endif
Joe Perches756a0252016-03-17 14:19:47 -07007675 str ? ", " : "", str ? str : "");
Jiang Liu7ee3d4e2013-07-03 15:03:41 -07007676}
7677
Mel Gorman0e0b8642006-09-27 01:49:56 -07007678/**
Randy Dunlap88ca3b92006-10-04 02:15:25 -07007679 * set_dma_reserve - set the specified number of pages reserved in the first zone
7680 * @new_dma_reserve: The number of pages to mark reserved
Mel Gorman0e0b8642006-09-27 01:49:56 -07007681 *
Yaowei Bai013110a2015-09-08 15:04:10 -07007682 * The per-cpu batchsize and zone watermarks are determined by managed_pages.
Mel Gorman0e0b8642006-09-27 01:49:56 -07007683 * In the DMA zone, a significant percentage may be consumed by kernel image
7684 * and other unfreeable allocations which can skew the watermarks badly. This
Randy Dunlap88ca3b92006-10-04 02:15:25 -07007685 * function may optionally be used to account for unfreeable pages in the
7686 * first zone (e.g., ZONE_DMA). The effect will be lower watermarks and
7687 * smaller per-cpu batchsize.
Mel Gorman0e0b8642006-09-27 01:49:56 -07007688 */
7689void __init set_dma_reserve(unsigned long new_dma_reserve)
7690{
7691 dma_reserve = new_dma_reserve;
7692}
7693
Sebastian Andrzej Siewior005fd4b2016-11-03 15:50:02 +01007694static int page_alloc_cpu_dead(unsigned int cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007695{
Linus Torvalds1da177e2005-04-16 15:20:36 -07007696
Sebastian Andrzej Siewior005fd4b2016-11-03 15:50:02 +01007697 lru_add_drain_cpu(cpu);
7698 drain_pages(cpu);
Christoph Lameter9f8f2172008-02-04 22:29:11 -08007699
Sebastian Andrzej Siewior005fd4b2016-11-03 15:50:02 +01007700 /*
7701 * Spill the event counters of the dead processor
7702 * into the current processors event counters.
7703 * This artificially elevates the count of the current
7704 * processor.
7705 */
7706 vm_events_fold_cpu(cpu);
Christoph Lameter9f8f2172008-02-04 22:29:11 -08007707
Sebastian Andrzej Siewior005fd4b2016-11-03 15:50:02 +01007708 /*
7709 * Zero the differential counters of the dead processor
7710 * so that the vm statistics are consistent.
7711 *
7712 * This is only okay since the processor is dead and cannot
7713 * race with what we are doing.
7714 */
7715 cpu_vm_stats_fold(cpu);
7716 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007717}
Linus Torvalds1da177e2005-04-16 15:20:36 -07007718
Nicholas Piggine03a5122019-07-11 20:59:12 -07007719#ifdef CONFIG_NUMA
7720int hashdist = HASHDIST_DEFAULT;
7721
7722static int __init set_hashdist(char *str)
7723{
7724 if (!str)
7725 return 0;
7726 hashdist = simple_strtoul(str, &str, 0);
7727 return 1;
7728}
7729__setup("hashdist=", set_hashdist);
7730#endif
7731
Linus Torvalds1da177e2005-04-16 15:20:36 -07007732void __init page_alloc_init(void)
7733{
Sebastian Andrzej Siewior005fd4b2016-11-03 15:50:02 +01007734 int ret;
7735
Nicholas Piggine03a5122019-07-11 20:59:12 -07007736#ifdef CONFIG_NUMA
7737 if (num_node_state(N_MEMORY) == 1)
7738 hashdist = 0;
7739#endif
7740
Sebastian Andrzej Siewior005fd4b2016-11-03 15:50:02 +01007741 ret = cpuhp_setup_state_nocalls(CPUHP_PAGE_ALLOC_DEAD,
7742 "mm/page_alloc:dead", NULL,
7743 page_alloc_cpu_dead);
7744 WARN_ON(ret < 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007745}
7746
7747/*
Yaowei Bai34b10062015-09-08 15:04:13 -07007748 * calculate_totalreserve_pages - called when sysctl_lowmem_reserve_ratio
Hideo AOKIcb45b0e2006-04-10 22:52:59 -07007749 * or min_free_kbytes changes.
7750 */
7751static void calculate_totalreserve_pages(void)
7752{
7753 struct pglist_data *pgdat;
7754 unsigned long reserve_pages = 0;
Christoph Lameter2f6726e2006-09-25 23:31:18 -07007755 enum zone_type i, j;
Hideo AOKIcb45b0e2006-04-10 22:52:59 -07007756
7757 for_each_online_pgdat(pgdat) {
Mel Gorman281e3722016-07-28 15:46:11 -07007758
7759 pgdat->totalreserve_pages = 0;
7760
Hideo AOKIcb45b0e2006-04-10 22:52:59 -07007761 for (i = 0; i < MAX_NR_ZONES; i++) {
7762 struct zone *zone = pgdat->node_zones + i;
Mel Gorman3484b2d2014-08-06 16:07:14 -07007763 long max = 0;
Arun KS9705bea2018-12-28 00:34:24 -08007764 unsigned long managed_pages = zone_managed_pages(zone);
Hideo AOKIcb45b0e2006-04-10 22:52:59 -07007765
7766 /* Find valid and maximum lowmem_reserve in the zone */
7767 for (j = i; j < MAX_NR_ZONES; j++) {
7768 if (zone->lowmem_reserve[j] > max)
7769 max = zone->lowmem_reserve[j];
7770 }
7771
Mel Gorman41858962009-06-16 15:32:12 -07007772 /* we treat the high watermark as reserved pages. */
7773 max += high_wmark_pages(zone);
Hideo AOKIcb45b0e2006-04-10 22:52:59 -07007774
Arun KS3d6357d2018-12-28 00:34:20 -08007775 if (max > managed_pages)
7776 max = managed_pages;
Johannes Weinera8d01432016-01-14 15:20:15 -08007777
Mel Gorman281e3722016-07-28 15:46:11 -07007778 pgdat->totalreserve_pages += max;
Johannes Weinera8d01432016-01-14 15:20:15 -08007779
Hideo AOKIcb45b0e2006-04-10 22:52:59 -07007780 reserve_pages += max;
7781 }
7782 }
7783 totalreserve_pages = reserve_pages;
7784}
7785
7786/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07007787 * setup_per_zone_lowmem_reserve - called whenever
Yaowei Bai34b10062015-09-08 15:04:13 -07007788 * sysctl_lowmem_reserve_ratio changes. Ensures that each zone
Linus Torvalds1da177e2005-04-16 15:20:36 -07007789 * has a correct pages reserved value, so an adequate number of
7790 * pages are left in the zone after a successful __alloc_pages().
7791 */
7792static void setup_per_zone_lowmem_reserve(void)
7793{
7794 struct pglist_data *pgdat;
Christoph Lameter2f6726e2006-09-25 23:31:18 -07007795 enum zone_type j, idx;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007796
KAMEZAWA Hiroyukiec936fc2006-03-27 01:15:59 -08007797 for_each_online_pgdat(pgdat) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07007798 for (j = 0; j < MAX_NR_ZONES; j++) {
7799 struct zone *zone = pgdat->node_zones + j;
Arun KS9705bea2018-12-28 00:34:24 -08007800 unsigned long managed_pages = zone_managed_pages(zone);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007801
7802 zone->lowmem_reserve[j] = 0;
7803
Christoph Lameter2f6726e2006-09-25 23:31:18 -07007804 idx = j;
7805 while (idx) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07007806 struct zone *lower_zone;
7807
Christoph Lameter2f6726e2006-09-25 23:31:18 -07007808 idx--;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007809 lower_zone = pgdat->node_zones + idx;
Joonsoo Kimd3cda232018-04-10 16:30:11 -07007810
Baoquan Hef6366152020-06-03 15:58:52 -07007811 if (!sysctl_lowmem_reserve_ratio[idx] ||
7812 !zone_managed_pages(lower_zone)) {
Joonsoo Kimd3cda232018-04-10 16:30:11 -07007813 lower_zone->lowmem_reserve[j] = 0;
Baoquan Hef6366152020-06-03 15:58:52 -07007814 continue;
Joonsoo Kimd3cda232018-04-10 16:30:11 -07007815 } else {
7816 lower_zone->lowmem_reserve[j] =
7817 managed_pages / sysctl_lowmem_reserve_ratio[idx];
7818 }
Arun KS9705bea2018-12-28 00:34:24 -08007819 managed_pages += zone_managed_pages(lower_zone);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007820 }
7821 }
7822 }
Hideo AOKIcb45b0e2006-04-10 22:52:59 -07007823
7824 /* update totalreserve_pages */
7825 calculate_totalreserve_pages();
Linus Torvalds1da177e2005-04-16 15:20:36 -07007826}
7827
Mel Gormancfd3da12011-04-25 21:36:42 +00007828static void __setup_per_zone_wmarks(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007829{
7830 unsigned long pages_min = min_free_kbytes >> (PAGE_SHIFT - 10);
7831 unsigned long lowmem_pages = 0;
7832 struct zone *zone;
7833 unsigned long flags;
7834
7835 /* Calculate total number of !ZONE_HIGHMEM pages */
7836 for_each_zone(zone) {
7837 if (!is_highmem(zone))
Arun KS9705bea2018-12-28 00:34:24 -08007838 lowmem_pages += zone_managed_pages(zone);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007839 }
7840
7841 for_each_zone(zone) {
Andrew Mortonac924c62006-05-15 09:43:59 -07007842 u64 tmp;
7843
Gerald Schaefer1125b4e2008-10-18 20:27:11 -07007844 spin_lock_irqsave(&zone->lock, flags);
Arun KS9705bea2018-12-28 00:34:24 -08007845 tmp = (u64)pages_min * zone_managed_pages(zone);
Andrew Mortonac924c62006-05-15 09:43:59 -07007846 do_div(tmp, lowmem_pages);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007847 if (is_highmem(zone)) {
7848 /*
Nick Piggin669ed172005-11-13 16:06:45 -08007849 * __GFP_HIGH and PF_MEMALLOC allocations usually don't
7850 * need highmem pages, so cap pages_min to a small
7851 * value here.
7852 *
Mel Gorman41858962009-06-16 15:32:12 -07007853 * The WMARK_HIGH-WMARK_LOW and (WMARK_LOW-WMARK_MIN)
Wei Yang8bb4e7a2019-03-05 15:46:22 -08007854 * deltas control async page reclaim, and so should
Nick Piggin669ed172005-11-13 16:06:45 -08007855 * not be capped for highmem.
Linus Torvalds1da177e2005-04-16 15:20:36 -07007856 */
Andrew Morton90ae8d62013-02-22 16:32:22 -08007857 unsigned long min_pages;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007858
Arun KS9705bea2018-12-28 00:34:24 -08007859 min_pages = zone_managed_pages(zone) / 1024;
Andrew Morton90ae8d62013-02-22 16:32:22 -08007860 min_pages = clamp(min_pages, SWAP_CLUSTER_MAX, 128UL);
Mel Gormana9214442018-12-28 00:35:44 -08007861 zone->_watermark[WMARK_MIN] = min_pages;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007862 } else {
Nick Piggin669ed172005-11-13 16:06:45 -08007863 /*
7864 * If it's a lowmem zone, reserve a number of pages
Linus Torvalds1da177e2005-04-16 15:20:36 -07007865 * proportionate to the zone's size.
7866 */
Mel Gormana9214442018-12-28 00:35:44 -08007867 zone->_watermark[WMARK_MIN] = tmp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007868 }
7869
Johannes Weiner795ae7a2016-03-17 14:19:14 -07007870 /*
7871 * Set the kswapd watermarks distance according to the
7872 * scale factor in proportion to available memory, but
7873 * ensure a minimum size on small systems.
7874 */
7875 tmp = max_t(u64, tmp >> 2,
Arun KS9705bea2018-12-28 00:34:24 -08007876 mult_frac(zone_managed_pages(zone),
Johannes Weiner795ae7a2016-03-17 14:19:14 -07007877 watermark_scale_factor, 10000));
7878
Charan Teja Reddyaa092592020-06-03 15:59:14 -07007879 zone->watermark_boost = 0;
Mel Gormana9214442018-12-28 00:35:44 -08007880 zone->_watermark[WMARK_LOW] = min_wmark_pages(zone) + tmp;
7881 zone->_watermark[WMARK_HIGH] = min_wmark_pages(zone) + tmp * 2;
Marek Szyprowski49f223a2012-01-25 12:49:24 +01007882
Gerald Schaefer1125b4e2008-10-18 20:27:11 -07007883 spin_unlock_irqrestore(&zone->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007884 }
Hideo AOKIcb45b0e2006-04-10 22:52:59 -07007885
7886 /* update totalreserve_pages */
7887 calculate_totalreserve_pages();
Linus Torvalds1da177e2005-04-16 15:20:36 -07007888}
7889
Mel Gormancfd3da12011-04-25 21:36:42 +00007890/**
7891 * setup_per_zone_wmarks - called when min_free_kbytes changes
7892 * or when memory is hot-{added|removed}
7893 *
7894 * Ensures that the watermark[min,low,high] values for each zone are set
7895 * correctly with respect to min_free_kbytes.
7896 */
7897void setup_per_zone_wmarks(void)
7898{
Michal Hockob93e0f32017-09-06 16:20:37 -07007899 static DEFINE_SPINLOCK(lock);
7900
7901 spin_lock(&lock);
Mel Gormancfd3da12011-04-25 21:36:42 +00007902 __setup_per_zone_wmarks();
Michal Hockob93e0f32017-09-06 16:20:37 -07007903 spin_unlock(&lock);
Mel Gormancfd3da12011-04-25 21:36:42 +00007904}
7905
Randy Dunlap55a44622009-09-21 17:01:20 -07007906/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07007907 * Initialise min_free_kbytes.
7908 *
7909 * For small machines we want it small (128k min). For large machines
Joel Savitz8beeae82020-07-03 15:15:30 -07007910 * we want it large (256MB max). But it is not linear, because network
Linus Torvalds1da177e2005-04-16 15:20:36 -07007911 * bandwidth does not increase linearly with machine size. We use
7912 *
Pintu Kumarb8af2942013-09-11 14:20:34 -07007913 * min_free_kbytes = 4 * sqrt(lowmem_kbytes), for better accuracy:
Linus Torvalds1da177e2005-04-16 15:20:36 -07007914 * min_free_kbytes = sqrt(lowmem_kbytes * 16)
7915 *
7916 * which yields
7917 *
7918 * 16MB: 512k
7919 * 32MB: 724k
7920 * 64MB: 1024k
7921 * 128MB: 1448k
7922 * 256MB: 2048k
7923 * 512MB: 2896k
7924 * 1024MB: 4096k
7925 * 2048MB: 5792k
7926 * 4096MB: 8192k
7927 * 8192MB: 11584k
7928 * 16384MB: 16384k
7929 */
KOSAKI Motohiro1b79acc2011-05-24 17:11:32 -07007930int __meminit init_per_zone_wmark_min(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007931{
7932 unsigned long lowmem_kbytes;
Michal Hocko5f127332013-07-08 16:00:40 -07007933 int new_min_free_kbytes;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007934
7935 lowmem_kbytes = nr_free_buffer_pages() * (PAGE_SIZE >> 10);
Michal Hocko5f127332013-07-08 16:00:40 -07007936 new_min_free_kbytes = int_sqrt(lowmem_kbytes * 16);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007937
Michal Hocko5f127332013-07-08 16:00:40 -07007938 if (new_min_free_kbytes > user_min_free_kbytes) {
7939 min_free_kbytes = new_min_free_kbytes;
7940 if (min_free_kbytes < 128)
7941 min_free_kbytes = 128;
Joel Savitzee8eb9a2020-04-01 21:09:44 -07007942 if (min_free_kbytes > 262144)
7943 min_free_kbytes = 262144;
Michal Hocko5f127332013-07-08 16:00:40 -07007944 } else {
7945 pr_warn("min_free_kbytes is not updated to %d because user defined value %d is preferred\n",
7946 new_min_free_kbytes, user_min_free_kbytes);
7947 }
Minchan Kimbc75d332009-06-16 15:32:48 -07007948 setup_per_zone_wmarks();
KOSAKI Motohiroa6cccdc2011-05-24 17:11:33 -07007949 refresh_zone_stat_thresholds();
Linus Torvalds1da177e2005-04-16 15:20:36 -07007950 setup_per_zone_lowmem_reserve();
Joonsoo Kim6423aa82016-08-10 16:27:49 -07007951
7952#ifdef CONFIG_NUMA
7953 setup_min_unmapped_ratio();
7954 setup_min_slab_ratio();
7955#endif
7956
Vijay Balakrishna4aab2be2020-10-10 23:16:40 -07007957 khugepaged_min_free_kbytes_update();
7958
Linus Torvalds1da177e2005-04-16 15:20:36 -07007959 return 0;
7960}
Doug Bergere08d3fd2020-08-20 17:42:24 -07007961postcore_initcall(init_per_zone_wmark_min)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007962
7963/*
Pintu Kumarb8af2942013-09-11 14:20:34 -07007964 * min_free_kbytes_sysctl_handler - just a wrapper around proc_dointvec() so
Linus Torvalds1da177e2005-04-16 15:20:36 -07007965 * that we can call two helper functions whenever min_free_kbytes
7966 * changes.
7967 */
Joe Perchescccad5b2014-06-06 14:38:09 -07007968int min_free_kbytes_sysctl_handler(struct ctl_table *table, int write,
Christoph Hellwig32927392020-04-24 08:43:38 +02007969 void *buffer, size_t *length, loff_t *ppos)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007970{
Han Pingtianda8c7572014-01-23 15:53:17 -08007971 int rc;
7972
7973 rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
7974 if (rc)
7975 return rc;
7976
Michal Hocko5f127332013-07-08 16:00:40 -07007977 if (write) {
7978 user_min_free_kbytes = min_free_kbytes;
Minchan Kimbc75d332009-06-16 15:32:48 -07007979 setup_per_zone_wmarks();
Michal Hocko5f127332013-07-08 16:00:40 -07007980 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07007981 return 0;
7982}
7983
Johannes Weiner795ae7a2016-03-17 14:19:14 -07007984int watermark_scale_factor_sysctl_handler(struct ctl_table *table, int write,
Christoph Hellwig32927392020-04-24 08:43:38 +02007985 void *buffer, size_t *length, loff_t *ppos)
Johannes Weiner795ae7a2016-03-17 14:19:14 -07007986{
7987 int rc;
7988
7989 rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
7990 if (rc)
7991 return rc;
7992
7993 if (write)
7994 setup_per_zone_wmarks();
7995
7996 return 0;
7997}
7998
Christoph Lameter96146342006-07-03 00:24:13 -07007999#ifdef CONFIG_NUMA
Joonsoo Kim6423aa82016-08-10 16:27:49 -07008000static void setup_min_unmapped_ratio(void)
Christoph Lameter96146342006-07-03 00:24:13 -07008001{
Joonsoo Kim6423aa82016-08-10 16:27:49 -07008002 pg_data_t *pgdat;
Christoph Lameter96146342006-07-03 00:24:13 -07008003 struct zone *zone;
Christoph Lameter96146342006-07-03 00:24:13 -07008004
Mel Gormana5f5f912016-07-28 15:46:32 -07008005 for_each_online_pgdat(pgdat)
Joonsoo Kim81cbcbc2016-08-10 16:27:46 -07008006 pgdat->min_unmapped_pages = 0;
Mel Gormana5f5f912016-07-28 15:46:32 -07008007
Christoph Lameter96146342006-07-03 00:24:13 -07008008 for_each_zone(zone)
Arun KS9705bea2018-12-28 00:34:24 -08008009 zone->zone_pgdat->min_unmapped_pages += (zone_managed_pages(zone) *
8010 sysctl_min_unmapped_ratio) / 100;
Christoph Lameter96146342006-07-03 00:24:13 -07008011}
Christoph Lameter0ff38492006-09-25 23:31:52 -07008012
Joonsoo Kim6423aa82016-08-10 16:27:49 -07008013
8014int sysctl_min_unmapped_ratio_sysctl_handler(struct ctl_table *table, int write,
Christoph Hellwig32927392020-04-24 08:43:38 +02008015 void *buffer, size_t *length, loff_t *ppos)
Christoph Lameter0ff38492006-09-25 23:31:52 -07008016{
Christoph Lameter0ff38492006-09-25 23:31:52 -07008017 int rc;
8018
Alexey Dobriyan8d65af72009-09-23 15:57:19 -07008019 rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
Christoph Lameter0ff38492006-09-25 23:31:52 -07008020 if (rc)
8021 return rc;
8022
Joonsoo Kim6423aa82016-08-10 16:27:49 -07008023 setup_min_unmapped_ratio();
8024
8025 return 0;
8026}
8027
8028static void setup_min_slab_ratio(void)
8029{
8030 pg_data_t *pgdat;
8031 struct zone *zone;
8032
Mel Gormana5f5f912016-07-28 15:46:32 -07008033 for_each_online_pgdat(pgdat)
8034 pgdat->min_slab_pages = 0;
8035
Christoph Lameter0ff38492006-09-25 23:31:52 -07008036 for_each_zone(zone)
Arun KS9705bea2018-12-28 00:34:24 -08008037 zone->zone_pgdat->min_slab_pages += (zone_managed_pages(zone) *
8038 sysctl_min_slab_ratio) / 100;
Joonsoo Kim6423aa82016-08-10 16:27:49 -07008039}
8040
8041int sysctl_min_slab_ratio_sysctl_handler(struct ctl_table *table, int write,
Christoph Hellwig32927392020-04-24 08:43:38 +02008042 void *buffer, size_t *length, loff_t *ppos)
Joonsoo Kim6423aa82016-08-10 16:27:49 -07008043{
8044 int rc;
8045
8046 rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
8047 if (rc)
8048 return rc;
8049
8050 setup_min_slab_ratio();
8051
Christoph Lameter0ff38492006-09-25 23:31:52 -07008052 return 0;
8053}
Christoph Lameter96146342006-07-03 00:24:13 -07008054#endif
8055
Linus Torvalds1da177e2005-04-16 15:20:36 -07008056/*
8057 * lowmem_reserve_ratio_sysctl_handler - just a wrapper around
8058 * proc_dointvec() so that we can call setup_per_zone_lowmem_reserve()
8059 * whenever sysctl_lowmem_reserve_ratio changes.
8060 *
8061 * The reserve ratio obviously has absolutely no relation with the
Mel Gorman41858962009-06-16 15:32:12 -07008062 * minimum watermarks. The lowmem reserve ratio can only make sense
Linus Torvalds1da177e2005-04-16 15:20:36 -07008063 * if in function of the boot time zone sizes.
8064 */
Joe Perchescccad5b2014-06-06 14:38:09 -07008065int lowmem_reserve_ratio_sysctl_handler(struct ctl_table *table, int write,
Christoph Hellwig32927392020-04-24 08:43:38 +02008066 void *buffer, size_t *length, loff_t *ppos)
Linus Torvalds1da177e2005-04-16 15:20:36 -07008067{
Baoquan He86aaf252020-06-03 15:58:48 -07008068 int i;
8069
Alexey Dobriyan8d65af72009-09-23 15:57:19 -07008070 proc_dointvec_minmax(table, write, buffer, length, ppos);
Baoquan He86aaf252020-06-03 15:58:48 -07008071
8072 for (i = 0; i < MAX_NR_ZONES; i++) {
8073 if (sysctl_lowmem_reserve_ratio[i] < 1)
8074 sysctl_lowmem_reserve_ratio[i] = 0;
8075 }
8076
Linus Torvalds1da177e2005-04-16 15:20:36 -07008077 setup_per_zone_lowmem_reserve();
8078 return 0;
8079}
8080
Mel Gormancb1ef532019-11-30 17:55:11 -08008081static void __zone_pcp_update(struct zone *zone)
8082{
8083 unsigned int cpu;
8084
8085 for_each_possible_cpu(cpu)
8086 pageset_set_high_and_batch(zone,
8087 per_cpu_ptr(zone->pageset, cpu));
8088}
8089
Rohit Seth8ad4b1f2006-01-08 01:00:40 -08008090/*
8091 * percpu_pagelist_fraction - changes the pcp->high for each zone on each
Pintu Kumarb8af2942013-09-11 14:20:34 -07008092 * cpu. It is the fraction of total pages in each zone that a hot per cpu
8093 * pagelist can have before it gets flushed back to buddy allocator.
Rohit Seth8ad4b1f2006-01-08 01:00:40 -08008094 */
Joe Perchescccad5b2014-06-06 14:38:09 -07008095int percpu_pagelist_fraction_sysctl_handler(struct ctl_table *table, int write,
Christoph Hellwig32927392020-04-24 08:43:38 +02008096 void *buffer, size_t *length, loff_t *ppos)
Rohit Seth8ad4b1f2006-01-08 01:00:40 -08008097{
8098 struct zone *zone;
David Rientjes7cd2b0a2014-06-23 13:22:04 -07008099 int old_percpu_pagelist_fraction;
Rohit Seth8ad4b1f2006-01-08 01:00:40 -08008100 int ret;
8101
Cody P Schaferc8e251f2013-07-03 15:01:29 -07008102 mutex_lock(&pcp_batch_high_lock);
David Rientjes7cd2b0a2014-06-23 13:22:04 -07008103 old_percpu_pagelist_fraction = percpu_pagelist_fraction;
8104
8105 ret = proc_dointvec_minmax(table, write, buffer, length, ppos);
8106 if (!write || ret < 0)
8107 goto out;
8108
8109 /* Sanity checking to avoid pcp imbalance */
8110 if (percpu_pagelist_fraction &&
8111 percpu_pagelist_fraction < MIN_PERCPU_PAGELIST_FRACTION) {
8112 percpu_pagelist_fraction = old_percpu_pagelist_fraction;
8113 ret = -EINVAL;
8114 goto out;
Rohit Seth8ad4b1f2006-01-08 01:00:40 -08008115 }
David Rientjes7cd2b0a2014-06-23 13:22:04 -07008116
8117 /* No change? */
8118 if (percpu_pagelist_fraction == old_percpu_pagelist_fraction)
8119 goto out;
8120
Mel Gormancb1ef532019-11-30 17:55:11 -08008121 for_each_populated_zone(zone)
8122 __zone_pcp_update(zone);
David Rientjes7cd2b0a2014-06-23 13:22:04 -07008123out:
Cody P Schaferc8e251f2013-07-03 15:01:29 -07008124 mutex_unlock(&pcp_batch_high_lock);
David Rientjes7cd2b0a2014-06-23 13:22:04 -07008125 return ret;
Rohit Seth8ad4b1f2006-01-08 01:00:40 -08008126}
8127
Srikar Dronamrajuf6f34b42016-10-07 16:59:15 -07008128#ifndef __HAVE_ARCH_RESERVED_KERNEL_PAGES
8129/*
8130 * Returns the number of pages that arch has reserved but
8131 * is not known to alloc_large_system_hash().
8132 */
8133static unsigned long __init arch_reserved_kernel_pages(void)
8134{
8135 return 0;
8136}
8137#endif
8138
Linus Torvalds1da177e2005-04-16 15:20:36 -07008139/*
Pavel Tatashin90172172017-07-06 15:39:14 -07008140 * Adaptive scale is meant to reduce sizes of hash tables on large memory
8141 * machines. As memory size is increased the scale is also increased but at
8142 * slower pace. Starting from ADAPT_SCALE_BASE (64G), every time memory
8143 * quadruples the scale is increased by one, which means the size of hash table
8144 * only doubles, instead of quadrupling as well.
8145 * Because 32-bit systems cannot have large physical memory, where this scaling
8146 * makes sense, it is disabled on such platforms.
8147 */
8148#if __BITS_PER_LONG > 32
8149#define ADAPT_SCALE_BASE (64ul << 30)
8150#define ADAPT_SCALE_SHIFT 2
8151#define ADAPT_SCALE_NPAGES (ADAPT_SCALE_BASE >> PAGE_SHIFT)
8152#endif
8153
8154/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07008155 * allocate a large system hash table from bootmem
8156 * - it is assumed that the hash table must contain an exact power-of-2
8157 * quantity of entries
8158 * - limit is the number of hash buckets, not the total allocation size
8159 */
8160void *__init alloc_large_system_hash(const char *tablename,
8161 unsigned long bucketsize,
8162 unsigned long numentries,
8163 int scale,
8164 int flags,
8165 unsigned int *_hash_shift,
8166 unsigned int *_hash_mask,
Tim Bird31fe62b2012-05-23 13:33:35 +00008167 unsigned long low_limit,
8168 unsigned long high_limit)
Linus Torvalds1da177e2005-04-16 15:20:36 -07008169{
Tim Bird31fe62b2012-05-23 13:33:35 +00008170 unsigned long long max = high_limit;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008171 unsigned long log2qty, size;
8172 void *table = NULL;
Pavel Tatashin3749a8f2017-07-06 15:39:08 -07008173 gfp_t gfp_flags;
Nicholas Pigginec114082019-07-11 20:59:09 -07008174 bool virt;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008175
8176 /* allow the kernel cmdline to have a say */
8177 if (!numentries) {
8178 /* round applicable memory size up to nearest megabyte */
Andrew Morton04903662006-12-06 20:37:33 -08008179 numentries = nr_kernel_pages;
Srikar Dronamrajuf6f34b42016-10-07 16:59:15 -07008180 numentries -= arch_reserved_kernel_pages();
Jerry Zhoua7e83312013-09-11 14:20:26 -07008181
8182 /* It isn't necessary when PAGE_SIZE >= 1MB */
8183 if (PAGE_SHIFT < 20)
8184 numentries = round_up(numentries, (1<<20)/PAGE_SIZE);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008185
Pavel Tatashin90172172017-07-06 15:39:14 -07008186#if __BITS_PER_LONG > 32
8187 if (!high_limit) {
8188 unsigned long adapt;
8189
8190 for (adapt = ADAPT_SCALE_NPAGES; adapt < numentries;
8191 adapt <<= ADAPT_SCALE_SHIFT)
8192 scale++;
8193 }
8194#endif
8195
Linus Torvalds1da177e2005-04-16 15:20:36 -07008196 /* limit to 1 bucket per 2^scale bytes of low memory */
8197 if (scale > PAGE_SHIFT)
8198 numentries >>= (scale - PAGE_SHIFT);
8199 else
8200 numentries <<= (PAGE_SHIFT - scale);
Paul Mundt9ab37b82007-01-05 16:36:30 -08008201
8202 /* Make sure we've got at least a 0-order allocation.. */
Jan Beulich2c85f512009-09-21 17:03:07 -07008203 if (unlikely(flags & HASH_SMALL)) {
8204 /* Makes no sense without HASH_EARLY */
8205 WARN_ON(!(flags & HASH_EARLY));
8206 if (!(numentries >> *_hash_shift)) {
8207 numentries = 1UL << *_hash_shift;
8208 BUG_ON(!numentries);
8209 }
8210 } else if (unlikely((numentries * bucketsize) < PAGE_SIZE))
Paul Mundt9ab37b82007-01-05 16:36:30 -08008211 numentries = PAGE_SIZE / bucketsize;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008212 }
John Hawkes6e692ed2006-03-25 03:08:02 -08008213 numentries = roundup_pow_of_two(numentries);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008214
8215 /* limit allocation size to 1/16 total memory by default */
8216 if (max == 0) {
8217 max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4;
8218 do_div(max, bucketsize);
8219 }
Dimitri Sivanich074b8512012-02-08 12:39:07 -08008220 max = min(max, 0x80000000ULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008221
Tim Bird31fe62b2012-05-23 13:33:35 +00008222 if (numentries < low_limit)
8223 numentries = low_limit;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008224 if (numentries > max)
8225 numentries = max;
8226
David Howellsf0d1b0b2006-12-08 02:37:49 -08008227 log2qty = ilog2(numentries);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008228
Pavel Tatashin3749a8f2017-07-06 15:39:08 -07008229 gfp_flags = (flags & HASH_ZERO) ? GFP_ATOMIC | __GFP_ZERO : GFP_ATOMIC;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008230 do {
Nicholas Pigginec114082019-07-11 20:59:09 -07008231 virt = false;
Linus Torvalds1da177e2005-04-16 15:20:36 -07008232 size = bucketsize << log2qty;
Pavel Tatashinea1f5f32017-11-15 17:36:27 -08008233 if (flags & HASH_EARLY) {
8234 if (flags & HASH_ZERO)
Mike Rapoport26fb3da2019-03-11 23:30:42 -07008235 table = memblock_alloc(size, SMP_CACHE_BYTES);
Pavel Tatashinea1f5f32017-11-15 17:36:27 -08008236 else
Mike Rapoport7e1c4e22018-10-30 15:09:57 -07008237 table = memblock_alloc_raw(size,
8238 SMP_CACHE_BYTES);
Nicholas Pigginec114082019-07-11 20:59:09 -07008239 } else if (get_order(size) >= MAX_ORDER || hashdist) {
Christoph Hellwig88dca4c2020-06-01 21:51:40 -07008240 table = __vmalloc(size, gfp_flags);
Nicholas Pigginec114082019-07-11 20:59:09 -07008241 virt = true;
Pavel Tatashinea1f5f32017-11-15 17:36:27 -08008242 } else {
Eric Dumazet1037b832007-07-15 23:38:05 -07008243 /*
8244 * If bucketsize is not a power-of-two, we may free
Mel Gormana1dd2682009-06-16 15:32:19 -07008245 * some pages at the end of hash table which
8246 * alloc_pages_exact() automatically does
Eric Dumazet1037b832007-07-15 23:38:05 -07008247 */
Nicholas Pigginec114082019-07-11 20:59:09 -07008248 table = alloc_pages_exact(size, gfp_flags);
8249 kmemleak_alloc(table, size, 1, gfp_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07008250 }
8251 } while (!table && size > PAGE_SIZE && --log2qty);
8252
8253 if (!table)
8254 panic("Failed to allocate %s hash table\n", tablename);
8255
Nicholas Pigginec114082019-07-11 20:59:09 -07008256 pr_info("%s hash table entries: %ld (order: %d, %lu bytes, %s)\n",
8257 tablename, 1UL << log2qty, ilog2(size) - PAGE_SHIFT, size,
8258 virt ? "vmalloc" : "linear");
Linus Torvalds1da177e2005-04-16 15:20:36 -07008259
8260 if (_hash_shift)
8261 *_hash_shift = log2qty;
8262 if (_hash_mask)
8263 *_hash_mask = (1 << log2qty) - 1;
8264
8265 return table;
8266}
KAMEZAWA Hiroyukia117e662006-03-27 01:15:25 -08008267
KAMEZAWA Hiroyukia5d76b542007-10-16 01:26:11 -07008268/*
Minchan Kim80934512012-07-31 16:43:01 -07008269 * This function checks whether pageblock includes unmovable pages or not.
Minchan Kim80934512012-07-31 16:43:01 -07008270 *
Pintu Kumarb8af2942013-09-11 14:20:34 -07008271 * PageLRU check without isolation or lru_lock could race so that
Yisheng Xie0efadf42017-02-24 14:57:39 -08008272 * MIGRATE_MOVABLE block might include unmovable pages. And __PageMovable
8273 * check without lock_page also may miss some movable non-lru pages at
8274 * race condition. So you can't expect this function should be exact.
Qian Cai4a55c042020-01-30 22:14:57 -08008275 *
8276 * Returns a page without holding a reference. If the caller wants to
Randy Dunlap047b9962020-08-11 18:33:14 -07008277 * dereference that page (e.g., dumping), it has to make sure that it
Qian Cai4a55c042020-01-30 22:14:57 -08008278 * cannot get removed (e.g., via memory unplug) concurrently.
8279 *
KAMEZAWA Hiroyukia5d76b542007-10-16 01:26:11 -07008280 */
Qian Cai4a55c042020-01-30 22:14:57 -08008281struct page *has_unmovable_pages(struct zone *zone, struct page *page,
8282 int migratetype, int flags)
KAMEZAWA Hiroyuki49ac8252010-10-26 14:21:30 -07008283{
Qian Cai1a9f2192019-04-18 17:50:30 -07008284 unsigned long iter = 0;
8285 unsigned long pfn = page_to_pfn(page);
Li Xinhai6a654e32020-10-13 16:55:39 -07008286 unsigned long offset = pfn % pageblock_nr_pages;
Michal Nazarewicz47118af2011-12-29 13:09:50 +01008287
Qian Cai1a9f2192019-04-18 17:50:30 -07008288 if (is_migrate_cma_page(page)) {
8289 /*
8290 * CMA allocations (alloc_contig_range) really need to mark
8291 * isolate CMA pageblocks even when they are not movable in fact
8292 * so consider them movable here.
8293 */
8294 if (is_migrate_cma(migratetype))
Qian Cai4a55c042020-01-30 22:14:57 -08008295 return NULL;
Michal Hocko4da2ce22017-11-15 17:33:26 -08008296
Qian Cai3d680bd2020-01-30 22:15:01 -08008297 return page;
Qian Cai1a9f2192019-04-18 17:50:30 -07008298 }
8299
Li Xinhai6a654e32020-10-13 16:55:39 -07008300 for (; iter < pageblock_nr_pages - offset; iter++) {
David Hildenbrandfe4c86c2020-01-30 22:14:04 -08008301 if (!pfn_valid_within(pfn + iter))
KAMEZAWA Hiroyuki49ac8252010-10-26 14:21:30 -07008302 continue;
Namhyung Kim29723fc2011-02-25 14:44:25 -08008303
David Hildenbrandfe4c86c2020-01-30 22:14:04 -08008304 page = pfn_to_page(pfn + iter);
Naoya Horiguchic8721bb2013-09-11 14:22:09 -07008305
David Hildenbrandc9c510d2020-10-13 16:55:17 -07008306 /*
8307 * Both, bootmem allocations and memory holes are marked
8308 * PG_reserved and are unmovable. We can even have unmovable
8309 * allocations inside ZONE_MOVABLE, for example when
8310 * specifying "movablecore".
8311 */
Michal Hockod7ab3672017-11-15 17:33:30 -08008312 if (PageReserved(page))
Qian Cai3d680bd2020-01-30 22:15:01 -08008313 return page;
Michal Hockod7ab3672017-11-15 17:33:30 -08008314
Naoya Horiguchic8721bb2013-09-11 14:22:09 -07008315 /*
Michal Hocko9d789992018-11-16 15:08:15 -08008316 * If the zone is movable and we have ruled out all reserved
8317 * pages then it should be reasonably safe to assume the rest
8318 * is movable.
8319 */
8320 if (zone_idx(zone) == ZONE_MOVABLE)
8321 continue;
8322
8323 /*
Naoya Horiguchic8721bb2013-09-11 14:22:09 -07008324 * Hugepages are not in LRU lists, but they're movable.
Rik van Riel1da2f322020-04-01 21:10:31 -07008325 * THPs are on the LRU, but need to be counted as #small pages.
Wei Yang8bb4e7a2019-03-05 15:46:22 -08008326 * We need not scan over tail pages because we don't
Naoya Horiguchic8721bb2013-09-11 14:22:09 -07008327 * handle each tail page individually in migration.
8328 */
Rik van Riel1da2f322020-04-01 21:10:31 -07008329 if (PageHuge(page) || PageTransCompound(page)) {
Oscar Salvador17e2e7d2018-12-21 14:31:00 -08008330 struct page *head = compound_head(page);
8331 unsigned int skip_pages;
Aneesh Kumar K.V464c7ff2018-09-04 15:45:59 -07008332
Rik van Riel1da2f322020-04-01 21:10:31 -07008333 if (PageHuge(page)) {
8334 if (!hugepage_migration_supported(page_hstate(head)))
8335 return page;
8336 } else if (!PageLRU(head) && !__PageMovable(head)) {
Qian Cai3d680bd2020-01-30 22:15:01 -08008337 return page;
Rik van Riel1da2f322020-04-01 21:10:31 -07008338 }
Aneesh Kumar K.V464c7ff2018-09-04 15:45:59 -07008339
Matthew Wilcox (Oracle)d8c65462019-09-23 15:34:30 -07008340 skip_pages = compound_nr(head) - (page - head);
Oscar Salvador17e2e7d2018-12-21 14:31:00 -08008341 iter += skip_pages - 1;
Naoya Horiguchic8721bb2013-09-11 14:22:09 -07008342 continue;
8343 }
8344
Minchan Kim97d255c2012-07-31 16:42:59 -07008345 /*
8346 * We can't use page_count without pin a page
8347 * because another CPU can free compound page.
8348 * This check already skips compound tails of THP
Joonsoo Kim0139aa72016-05-19 17:10:49 -07008349 * because their page->_refcount is zero at all time.
Minchan Kim97d255c2012-07-31 16:42:59 -07008350 */
Joonsoo Kimfe896d12016-03-17 14:19:26 -07008351 if (!page_ref_count(page)) {
KAMEZAWA Hiroyuki49ac8252010-10-26 14:21:30 -07008352 if (PageBuddy(page))
Matthew Wilcox (Oracle)ab130f912020-10-15 20:10:15 -07008353 iter += (1 << buddy_order(page)) - 1;
KAMEZAWA Hiroyuki49ac8252010-10-26 14:21:30 -07008354 continue;
8355 }
Minchan Kim97d255c2012-07-31 16:42:59 -07008356
Wen Congyangb023f462012-12-11 16:00:45 -08008357 /*
8358 * The HWPoisoned page may be not in buddy system, and
8359 * page_count() is not 0.
8360 */
David Hildenbrand756d25be2019-11-30 17:54:07 -08008361 if ((flags & MEMORY_OFFLINE) && PageHWPoison(page))
Wen Congyangb023f462012-12-11 16:00:45 -08008362 continue;
8363
David Hildenbrandaa218792020-05-07 16:01:30 +02008364 /*
8365 * We treat all PageOffline() pages as movable when offlining
8366 * to give drivers a chance to decrement their reference count
8367 * in MEM_GOING_OFFLINE in order to indicate that these pages
8368 * can be offlined as there are no direct references anymore.
8369 * For actually unmovable PageOffline() where the driver does
8370 * not support this, we will fail later when trying to actually
8371 * move these pages that still have a reference count > 0.
8372 * (false negatives in this function only)
8373 */
8374 if ((flags & MEMORY_OFFLINE) && PageOffline(page))
8375 continue;
8376
David Hildenbrandfe4c86c2020-01-30 22:14:04 -08008377 if (__PageMovable(page) || PageLRU(page))
Yisheng Xie0efadf42017-02-24 14:57:39 -08008378 continue;
8379
KAMEZAWA Hiroyuki49ac8252010-10-26 14:21:30 -07008380 /*
Johannes Weiner6b4f7792014-12-12 16:56:13 -08008381 * If there are RECLAIMABLE pages, we need to check
8382 * it. But now, memory offline itself doesn't call
8383 * shrink_node_slabs() and it still to be fixed.
KAMEZAWA Hiroyuki49ac8252010-10-26 14:21:30 -07008384 */
Qian Cai3d680bd2020-01-30 22:15:01 -08008385 return page;
KAMEZAWA Hiroyuki49ac8252010-10-26 14:21:30 -07008386 }
Qian Cai4a55c042020-01-30 22:14:57 -08008387 return NULL;
KAMEZAWA Hiroyuki49ac8252010-10-26 14:21:30 -07008388}
8389
Alexandre Ghiti8df995f2019-05-13 17:19:00 -07008390#ifdef CONFIG_CONTIG_ALLOC
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008391static unsigned long pfn_max_align_down(unsigned long pfn)
8392{
8393 return pfn & ~(max_t(unsigned long, MAX_ORDER_NR_PAGES,
8394 pageblock_nr_pages) - 1);
8395}
8396
8397static unsigned long pfn_max_align_up(unsigned long pfn)
8398{
8399 return ALIGN(pfn, max_t(unsigned long, MAX_ORDER_NR_PAGES,
8400 pageblock_nr_pages));
8401}
8402
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008403/* [start, end) must belong to a single zone. */
Mel Gormanbb13ffe2012-10-08 16:32:41 -07008404static int __alloc_contig_migrate_range(struct compact_control *cc,
8405 unsigned long start, unsigned long end)
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008406{
8407 /* This function is based on compact_zone() from compaction.c. */
Maninder Singh730ec8c2020-06-03 16:01:18 -07008408 unsigned int nr_reclaimed;
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008409 unsigned long pfn = start;
8410 unsigned int tries = 0;
8411 int ret = 0;
Joonsoo Kim8b94e0b2020-08-11 18:37:31 -07008412 struct migration_target_control mtc = {
8413 .nid = zone_to_nid(cc->zone),
8414 .gfp_mask = GFP_USER | __GFP_MOVABLE | __GFP_RETRY_MAYFAIL,
8415 };
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008416
Marek Szyprowskibe49a6e2012-12-12 13:51:19 -08008417 migrate_prep();
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008418
Mel Gormanbb13ffe2012-10-08 16:32:41 -07008419 while (pfn < end || !list_empty(&cc->migratepages)) {
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008420 if (fatal_signal_pending(current)) {
8421 ret = -EINTR;
8422 break;
8423 }
8424
Mel Gormanbb13ffe2012-10-08 16:32:41 -07008425 if (list_empty(&cc->migratepages)) {
8426 cc->nr_migratepages = 0;
Vlastimil Babkaedc2ca62014-10-09 15:27:09 -07008427 pfn = isolate_migratepages_range(cc, pfn, end);
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008428 if (!pfn) {
8429 ret = -EINTR;
8430 break;
8431 }
8432 tries = 0;
8433 } else if (++tries == 5) {
8434 ret = ret < 0 ? ret : -EBUSY;
8435 break;
8436 }
8437
Minchan Kimbeb51ea2012-10-08 16:33:51 -07008438 nr_reclaimed = reclaim_clean_pages_from_list(cc->zone,
8439 &cc->migratepages);
8440 cc->nr_migratepages -= nr_reclaimed;
Minchan Kim02c6de82012-10-08 16:31:55 -07008441
Joonsoo Kim8b94e0b2020-08-11 18:37:31 -07008442 ret = migrate_pages(&cc->migratepages, alloc_migration_target,
8443 NULL, (unsigned long)&mtc, cc->mode, MR_CONTIG_RANGE);
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008444 }
Srinivas Pandruvada2a6f5122013-02-22 16:32:09 -08008445 if (ret < 0) {
8446 putback_movable_pages(&cc->migratepages);
8447 return ret;
8448 }
8449 return 0;
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008450}
8451
8452/**
8453 * alloc_contig_range() -- tries to allocate given range of pages
8454 * @start: start PFN to allocate
8455 * @end: one-past-the-last PFN to allocate
Michal Nazarewicz0815f3d2012-04-03 15:06:15 +02008456 * @migratetype: migratetype of the underlaying pageblocks (either
8457 * #MIGRATE_MOVABLE or #MIGRATE_CMA). All pageblocks
8458 * in range must have the same migratetype and it must
8459 * be either of the two.
Lucas Stachca96b622017-02-24 14:58:37 -08008460 * @gfp_mask: GFP mask to use during compaction
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008461 *
8462 * The PFN range does not have to be pageblock or MAX_ORDER_NR_PAGES
Mike Kravetz2c7452a2018-04-05 16:25:26 -07008463 * aligned. The PFN range must belong to a single zone.
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008464 *
Mike Kravetz2c7452a2018-04-05 16:25:26 -07008465 * The first thing this routine does is attempt to MIGRATE_ISOLATE all
8466 * pageblocks in the range. Once isolated, the pageblocks should not
8467 * be modified by others.
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008468 *
Mike Rapoporta862f682019-03-05 15:48:42 -08008469 * Return: zero on success or negative error code. On success all
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008470 * pages which PFN is in [start, end) are allocated for the caller and
8471 * need to be freed with free_contig_range().
8472 */
Michal Nazarewicz0815f3d2012-04-03 15:06:15 +02008473int alloc_contig_range(unsigned long start, unsigned long end,
Lucas Stachca96b622017-02-24 14:58:37 -08008474 unsigned migratetype, gfp_t gfp_mask)
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008475{
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008476 unsigned long outer_start, outer_end;
Kirill A. Shutemovd00181b2015-11-06 16:29:57 -08008477 unsigned int order;
8478 int ret = 0;
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008479
Mel Gormanbb13ffe2012-10-08 16:32:41 -07008480 struct compact_control cc = {
8481 .nr_migratepages = 0,
8482 .order = -1,
8483 .zone = page_zone(pfn_to_page(start)),
David Rientjese0b9dae2014-06-04 16:08:28 -07008484 .mode = MIGRATE_SYNC,
Mel Gormanbb13ffe2012-10-08 16:32:41 -07008485 .ignore_skip_hint = true,
Vlastimil Babka2583d672017-11-17 15:26:38 -08008486 .no_set_skip_hint = true,
Michal Hocko7dea19f2017-05-03 14:53:15 -07008487 .gfp_mask = current_gfp_context(gfp_mask),
Rik van Rielb06eda02020-04-01 21:10:28 -07008488 .alloc_contig = true,
Mel Gormanbb13ffe2012-10-08 16:32:41 -07008489 };
8490 INIT_LIST_HEAD(&cc.migratepages);
8491
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008492 /*
8493 * What we do here is we mark all pageblocks in range as
8494 * MIGRATE_ISOLATE. Because pageblock and max order pages may
8495 * have different sizes, and due to the way page allocator
8496 * work, we align the range to biggest of the two pages so
8497 * that page allocator won't try to merge buddies from
8498 * different pageblocks and change MIGRATE_ISOLATE to some
8499 * other migration type.
8500 *
8501 * Once the pageblocks are marked as MIGRATE_ISOLATE, we
8502 * migrate the pages from an unaligned range (ie. pages that
8503 * we are interested in). This will put all the pages in
8504 * range back to page allocator as MIGRATE_ISOLATE.
8505 *
8506 * When this is done, we take the pages in range from page
8507 * allocator removing them from the buddy system. This way
8508 * page allocator will never consider using them.
8509 *
8510 * This lets us mark the pageblocks back as
8511 * MIGRATE_CMA/MIGRATE_MOVABLE so that free pages in the
8512 * aligned range but not in the unaligned, original range are
8513 * put back to page allocator so that buddy can use them.
8514 */
8515
8516 ret = start_isolate_page_range(pfn_max_align_down(start),
Michal Hockod381c542018-12-28 00:33:56 -08008517 pfn_max_align_up(end), migratetype, 0);
David Hildenbrand3fa0c7c2020-10-15 20:08:07 -07008518 if (ret)
Bob Liu86a595f2012-10-25 13:37:56 -07008519 return ret;
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008520
Joonsoo Kim8ef58492016-01-14 15:18:45 -08008521 /*
8522 * In case of -EBUSY, we'd like to know which page causes problem.
Mike Kravetz63cd4482017-11-29 16:10:01 -08008523 * So, just fall through. test_pages_isolated() has a tracepoint
8524 * which will report the busy page.
8525 *
8526 * It is possible that busy pages could become available before
8527 * the call to test_pages_isolated, and the range will actually be
8528 * allocated. So, if we fall through be sure to clear ret so that
8529 * -EBUSY is not accidentally used or returned to caller.
Joonsoo Kim8ef58492016-01-14 15:18:45 -08008530 */
Mel Gormanbb13ffe2012-10-08 16:32:41 -07008531 ret = __alloc_contig_migrate_range(&cc, start, end);
Joonsoo Kim8ef58492016-01-14 15:18:45 -08008532 if (ret && ret != -EBUSY)
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008533 goto done;
Mike Kravetz63cd4482017-11-29 16:10:01 -08008534 ret =0;
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008535
8536 /*
8537 * Pages from [start, end) are within a MAX_ORDER_NR_PAGES
8538 * aligned blocks that are marked as MIGRATE_ISOLATE. What's
8539 * more, all pages in [start, end) are free in page allocator.
8540 * What we are going to do is to allocate all pages from
8541 * [start, end) (that is remove them from page allocator).
8542 *
8543 * The only problem is that pages at the beginning and at the
8544 * end of interesting range may be not aligned with pages that
8545 * page allocator holds, ie. they can be part of higher order
8546 * pages. Because of this, we reserve the bigger range and
8547 * once this is done free the pages we are not interested in.
8548 *
8549 * We don't have to hold zone->lock here because the pages are
8550 * isolated thus they won't get removed from buddy.
8551 */
8552
8553 lru_add_drain_all();
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008554
8555 order = 0;
8556 outer_start = start;
8557 while (!PageBuddy(pfn_to_page(outer_start))) {
8558 if (++order >= MAX_ORDER) {
Joonsoo Kim8ef58492016-01-14 15:18:45 -08008559 outer_start = start;
8560 break;
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008561 }
8562 outer_start &= ~0UL << order;
8563 }
8564
Joonsoo Kim8ef58492016-01-14 15:18:45 -08008565 if (outer_start != start) {
Matthew Wilcox (Oracle)ab130f912020-10-15 20:10:15 -07008566 order = buddy_order(pfn_to_page(outer_start));
Joonsoo Kim8ef58492016-01-14 15:18:45 -08008567
8568 /*
8569 * outer_start page could be small order buddy page and
8570 * it doesn't include start page. Adjust outer_start
8571 * in this case to report failed page properly
8572 * on tracepoint in test_pages_isolated()
8573 */
8574 if (outer_start + (1UL << order) <= start)
8575 outer_start = start;
8576 }
8577
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008578 /* Make sure the range is really isolated. */
David Hildenbrand756d25be2019-11-30 17:54:07 -08008579 if (test_pages_isolated(outer_start, end, 0)) {
Jonathan Toppins75dddef2017-08-10 15:23:35 -07008580 pr_info_ratelimited("%s: [%lx, %lx) PFNs busy\n",
Michal Nazarewiczdae803e2014-11-13 15:19:27 -08008581 __func__, outer_start, end);
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008582 ret = -EBUSY;
8583 goto done;
8584 }
8585
Marek Szyprowski49f223a2012-01-25 12:49:24 +01008586 /* Grab isolated pages from freelists. */
Mel Gormanbb13ffe2012-10-08 16:32:41 -07008587 outer_end = isolate_freepages_range(&cc, outer_start, end);
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008588 if (!outer_end) {
8589 ret = -EBUSY;
8590 goto done;
8591 }
8592
8593 /* Free head and tail (if any) */
8594 if (start != outer_start)
8595 free_contig_range(outer_start, start - outer_start);
8596 if (end != outer_end)
8597 free_contig_range(end, outer_end - end);
8598
8599done:
8600 undo_isolate_page_range(pfn_max_align_down(start),
Michal Nazarewicz0815f3d2012-04-03 15:06:15 +02008601 pfn_max_align_up(end), migratetype);
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008602 return ret;
8603}
David Hildenbrand255f5982020-05-07 16:01:29 +02008604EXPORT_SYMBOL(alloc_contig_range);
Anshuman Khandual5e27a2d2019-11-30 17:55:06 -08008605
8606static int __alloc_contig_pages(unsigned long start_pfn,
8607 unsigned long nr_pages, gfp_t gfp_mask)
8608{
8609 unsigned long end_pfn = start_pfn + nr_pages;
8610
8611 return alloc_contig_range(start_pfn, end_pfn, MIGRATE_MOVABLE,
8612 gfp_mask);
8613}
8614
8615static bool pfn_range_valid_contig(struct zone *z, unsigned long start_pfn,
8616 unsigned long nr_pages)
8617{
8618 unsigned long i, end_pfn = start_pfn + nr_pages;
8619 struct page *page;
8620
8621 for (i = start_pfn; i < end_pfn; i++) {
8622 page = pfn_to_online_page(i);
8623 if (!page)
8624 return false;
8625
8626 if (page_zone(page) != z)
8627 return false;
8628
8629 if (PageReserved(page))
8630 return false;
8631
8632 if (page_count(page) > 0)
8633 return false;
8634
8635 if (PageHuge(page))
8636 return false;
8637 }
8638 return true;
8639}
8640
8641static bool zone_spans_last_pfn(const struct zone *zone,
8642 unsigned long start_pfn, unsigned long nr_pages)
8643{
8644 unsigned long last_pfn = start_pfn + nr_pages - 1;
8645
8646 return zone_spans_pfn(zone, last_pfn);
8647}
8648
8649/**
8650 * alloc_contig_pages() -- tries to find and allocate contiguous range of pages
8651 * @nr_pages: Number of contiguous pages to allocate
8652 * @gfp_mask: GFP mask to limit search and used during compaction
8653 * @nid: Target node
8654 * @nodemask: Mask for other possible nodes
8655 *
8656 * This routine is a wrapper around alloc_contig_range(). It scans over zones
8657 * on an applicable zonelist to find a contiguous pfn range which can then be
8658 * tried for allocation with alloc_contig_range(). This routine is intended
8659 * for allocation requests which can not be fulfilled with the buddy allocator.
8660 *
8661 * The allocated memory is always aligned to a page boundary. If nr_pages is a
8662 * power of two then the alignment is guaranteed to be to the given nr_pages
8663 * (e.g. 1GB request would be aligned to 1GB).
8664 *
8665 * Allocated pages can be freed with free_contig_range() or by manually calling
8666 * __free_page() on each allocated page.
8667 *
8668 * Return: pointer to contiguous pages on success, or NULL if not successful.
8669 */
8670struct page *alloc_contig_pages(unsigned long nr_pages, gfp_t gfp_mask,
8671 int nid, nodemask_t *nodemask)
8672{
8673 unsigned long ret, pfn, flags;
8674 struct zonelist *zonelist;
8675 struct zone *zone;
8676 struct zoneref *z;
8677
8678 zonelist = node_zonelist(nid, gfp_mask);
8679 for_each_zone_zonelist_nodemask(zone, z, zonelist,
8680 gfp_zone(gfp_mask), nodemask) {
8681 spin_lock_irqsave(&zone->lock, flags);
8682
8683 pfn = ALIGN(zone->zone_start_pfn, nr_pages);
8684 while (zone_spans_last_pfn(zone, pfn, nr_pages)) {
8685 if (pfn_range_valid_contig(zone, pfn, nr_pages)) {
8686 /*
8687 * We release the zone lock here because
8688 * alloc_contig_range() will also lock the zone
8689 * at some point. If there's an allocation
8690 * spinning on this lock, it may win the race
8691 * and cause alloc_contig_range() to fail...
8692 */
8693 spin_unlock_irqrestore(&zone->lock, flags);
8694 ret = __alloc_contig_pages(pfn, nr_pages,
8695 gfp_mask);
8696 if (!ret)
8697 return pfn_to_page(pfn);
8698 spin_lock_irqsave(&zone->lock, flags);
8699 }
8700 pfn += nr_pages;
8701 }
8702 spin_unlock_irqrestore(&zone->lock, flags);
8703 }
8704 return NULL;
8705}
Alexandre Ghiti4eb07162019-05-13 17:19:04 -07008706#endif /* CONFIG_CONTIG_ALLOC */
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008707
Alexandre Ghiti4eb07162019-05-13 17:19:04 -07008708void free_contig_range(unsigned long pfn, unsigned int nr_pages)
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008709{
Marek Szyprowskibcc2b022012-12-20 15:05:18 -08008710 unsigned int count = 0;
8711
8712 for (; nr_pages--; pfn++) {
8713 struct page *page = pfn_to_page(pfn);
8714
8715 count += page_count(page) != 1;
8716 __free_page(page);
8717 }
8718 WARN(count != 0, "%d pages are still in use!\n", count);
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008719}
David Hildenbrand255f5982020-05-07 16:01:29 +02008720EXPORT_SYMBOL(free_contig_range);
Michal Nazarewicz041d3a82011-12-29 13:09:50 +01008721
Cody P Schafer0a647f32013-07-03 15:01:33 -07008722/*
8723 * The zone indicated has a new number of managed_pages; batch sizes and percpu
8724 * page high values need to be recalulated.
8725 */
Jiang Liu4ed7e022012-07-31 16:43:35 -07008726void __meminit zone_pcp_update(struct zone *zone)
8727{
Cody P Schaferc8e251f2013-07-03 15:01:29 -07008728 mutex_lock(&pcp_batch_high_lock);
Mel Gormancb1ef532019-11-30 17:55:11 -08008729 __zone_pcp_update(zone);
Cody P Schaferc8e251f2013-07-03 15:01:29 -07008730 mutex_unlock(&pcp_batch_high_lock);
Jiang Liu4ed7e022012-07-31 16:43:35 -07008731}
Jiang Liu4ed7e022012-07-31 16:43:35 -07008732
Jiang Liu340175b2012-07-31 16:43:32 -07008733void zone_pcp_reset(struct zone *zone)
8734{
8735 unsigned long flags;
Minchan Kim5a883812012-10-08 16:33:39 -07008736 int cpu;
8737 struct per_cpu_pageset *pset;
Jiang Liu340175b2012-07-31 16:43:32 -07008738
8739 /* avoid races with drain_pages() */
8740 local_irq_save(flags);
8741 if (zone->pageset != &boot_pageset) {
Minchan Kim5a883812012-10-08 16:33:39 -07008742 for_each_online_cpu(cpu) {
8743 pset = per_cpu_ptr(zone->pageset, cpu);
8744 drain_zonestat(zone, pset);
8745 }
Jiang Liu340175b2012-07-31 16:43:32 -07008746 free_percpu(zone->pageset);
8747 zone->pageset = &boot_pageset;
8748 }
8749 local_irq_restore(flags);
8750}
8751
Wen Congyang6dcd73d2012-12-11 16:01:01 -08008752#ifdef CONFIG_MEMORY_HOTREMOVE
KAMEZAWA Hiroyuki0c0e6192007-10-16 01:26:12 -07008753/*
David Hildenbrand257bea72020-10-15 20:07:59 -07008754 * All pages in the range must be in a single zone, must not contain holes,
8755 * must span full sections, and must be isolated before calling this function.
KAMEZAWA Hiroyuki0c0e6192007-10-16 01:26:12 -07008756 */
David Hildenbrand257bea72020-10-15 20:07:59 -07008757void __offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn)
KAMEZAWA Hiroyuki0c0e6192007-10-16 01:26:12 -07008758{
David Hildenbrand257bea72020-10-15 20:07:59 -07008759 unsigned long pfn = start_pfn;
KAMEZAWA Hiroyuki0c0e6192007-10-16 01:26:12 -07008760 struct page *page;
8761 struct zone *zone;
David Hildenbrand0ee5f4f2019-11-30 17:54:03 -08008762 unsigned int order;
KAMEZAWA Hiroyuki0c0e6192007-10-16 01:26:12 -07008763 unsigned long flags;
Michal Hocko5557c762019-05-13 17:21:24 -07008764
Michal Hocko2d070ea2017-07-06 15:37:56 -07008765 offline_mem_sections(pfn, end_pfn);
KAMEZAWA Hiroyuki0c0e6192007-10-16 01:26:12 -07008766 zone = page_zone(pfn_to_page(pfn));
8767 spin_lock_irqsave(&zone->lock, flags);
KAMEZAWA Hiroyuki0c0e6192007-10-16 01:26:12 -07008768 while (pfn < end_pfn) {
KAMEZAWA Hiroyuki0c0e6192007-10-16 01:26:12 -07008769 page = pfn_to_page(pfn);
Wen Congyangb023f462012-12-11 16:00:45 -08008770 /*
8771 * The HWPoisoned page may be not in buddy system, and
8772 * page_count() is not 0.
8773 */
8774 if (unlikely(!PageBuddy(page) && PageHWPoison(page))) {
8775 pfn++;
Wen Congyangb023f462012-12-11 16:00:45 -08008776 continue;
8777 }
David Hildenbrandaa218792020-05-07 16:01:30 +02008778 /*
8779 * At this point all remaining PageOffline() pages have a
8780 * reference count of 0 and can simply be skipped.
8781 */
8782 if (PageOffline(page)) {
8783 BUG_ON(page_count(page));
8784 BUG_ON(PageBuddy(page));
8785 pfn++;
David Hildenbrandaa218792020-05-07 16:01:30 +02008786 continue;
8787 }
Wen Congyangb023f462012-12-11 16:00:45 -08008788
KAMEZAWA Hiroyuki0c0e6192007-10-16 01:26:12 -07008789 BUG_ON(page_count(page));
8790 BUG_ON(!PageBuddy(page));
Matthew Wilcox (Oracle)ab130f912020-10-15 20:10:15 -07008791 order = buddy_order(page);
Alexander Duyck6ab01362020-04-06 20:04:49 -07008792 del_page_from_free_list(page, zone, order);
KAMEZAWA Hiroyuki0c0e6192007-10-16 01:26:12 -07008793 pfn += (1 << order);
8794 }
8795 spin_unlock_irqrestore(&zone->lock, flags);
8796}
8797#endif
Wu Fengguang8d22ba12009-12-16 12:19:58 +01008798
Wu Fengguang8d22ba12009-12-16 12:19:58 +01008799bool is_free_buddy_page(struct page *page)
8800{
8801 struct zone *zone = page_zone(page);
8802 unsigned long pfn = page_to_pfn(page);
8803 unsigned long flags;
Mel Gorman7aeb09f2014-06-04 16:10:21 -07008804 unsigned int order;
Wu Fengguang8d22ba12009-12-16 12:19:58 +01008805
8806 spin_lock_irqsave(&zone->lock, flags);
8807 for (order = 0; order < MAX_ORDER; order++) {
8808 struct page *page_head = page - (pfn & ((1 << order) - 1));
8809
Matthew Wilcox (Oracle)ab130f912020-10-15 20:10:15 -07008810 if (PageBuddy(page_head) && buddy_order(page_head) >= order)
Wu Fengguang8d22ba12009-12-16 12:19:58 +01008811 break;
8812 }
8813 spin_unlock_irqrestore(&zone->lock, flags);
8814
8815 return order < MAX_ORDER;
8816}
Naoya Horiguchid4ae9912018-08-23 17:00:42 -07008817
8818#ifdef CONFIG_MEMORY_FAILURE
8819/*
Oscar Salvador06be6ff2020-10-15 20:07:05 -07008820 * Break down a higher-order page in sub-pages, and keep our target out of
8821 * buddy allocator.
Naoya Horiguchid4ae9912018-08-23 17:00:42 -07008822 */
Oscar Salvador06be6ff2020-10-15 20:07:05 -07008823static void break_down_buddy_pages(struct zone *zone, struct page *page,
8824 struct page *target, int low, int high,
8825 int migratetype)
8826{
8827 unsigned long size = 1 << high;
8828 struct page *current_buddy, *next_page;
8829
8830 while (high > low) {
8831 high--;
8832 size >>= 1;
8833
8834 if (target >= &page[size]) {
8835 next_page = page + size;
8836 current_buddy = page;
8837 } else {
8838 next_page = page;
8839 current_buddy = page + size;
8840 }
8841
8842 if (set_page_guard(zone, current_buddy, high, migratetype))
8843 continue;
8844
8845 if (current_buddy != target) {
8846 add_to_free_list(current_buddy, zone, high, migratetype);
Matthew Wilcox (Oracle)ab130f912020-10-15 20:10:15 -07008847 set_buddy_order(current_buddy, high);
Oscar Salvador06be6ff2020-10-15 20:07:05 -07008848 page = next_page;
8849 }
8850 }
8851}
8852
8853/*
8854 * Take a page that will be marked as poisoned off the buddy allocator.
8855 */
8856bool take_page_off_buddy(struct page *page)
Naoya Horiguchid4ae9912018-08-23 17:00:42 -07008857{
8858 struct zone *zone = page_zone(page);
8859 unsigned long pfn = page_to_pfn(page);
8860 unsigned long flags;
8861 unsigned int order;
Oscar Salvador06be6ff2020-10-15 20:07:05 -07008862 bool ret = false;
Naoya Horiguchid4ae9912018-08-23 17:00:42 -07008863
8864 spin_lock_irqsave(&zone->lock, flags);
8865 for (order = 0; order < MAX_ORDER; order++) {
8866 struct page *page_head = page - (pfn & ((1 << order) - 1));
Matthew Wilcox (Oracle)ab130f912020-10-15 20:10:15 -07008867 int page_order = buddy_order(page_head);
Naoya Horiguchid4ae9912018-08-23 17:00:42 -07008868
Matthew Wilcox (Oracle)ab130f912020-10-15 20:10:15 -07008869 if (PageBuddy(page_head) && page_order >= order) {
Oscar Salvador06be6ff2020-10-15 20:07:05 -07008870 unsigned long pfn_head = page_to_pfn(page_head);
8871 int migratetype = get_pfnblock_migratetype(page_head,
8872 pfn_head);
8873
Matthew Wilcox (Oracle)ab130f912020-10-15 20:10:15 -07008874 del_page_from_free_list(page_head, zone, page_order);
Oscar Salvador06be6ff2020-10-15 20:07:05 -07008875 break_down_buddy_pages(zone, page_head, page, 0,
Matthew Wilcox (Oracle)ab130f912020-10-15 20:10:15 -07008876 page_order, migratetype);
Oscar Salvador06be6ff2020-10-15 20:07:05 -07008877 ret = true;
Naoya Horiguchid4ae9912018-08-23 17:00:42 -07008878 break;
8879 }
Oscar Salvador06be6ff2020-10-15 20:07:05 -07008880 if (page_count(page_head) > 0)
8881 break;
Naoya Horiguchid4ae9912018-08-23 17:00:42 -07008882 }
8883 spin_unlock_irqrestore(&zone->lock, flags);
Oscar Salvador06be6ff2020-10-15 20:07:05 -07008884 return ret;
Naoya Horiguchid4ae9912018-08-23 17:00:42 -07008885}
8886#endif