blob: e3bf6432ed2586cbfcef35407c741fbc60d74790 [file] [log] [blame]
Andi Kleen6a460792009-09-16 11:50:15 +02001/*
2 * Copyright (C) 2008, 2009 Intel Corporation
3 * Authors: Andi Kleen, Fengguang Wu
4 *
5 * This software may be redistributed and/or modified under the terms of
6 * the GNU General Public License ("GPL") version 2 only as published by the
7 * Free Software Foundation.
8 *
9 * High level machine check handler. Handles pages reported by the
Andi Kleen1c80b992010-09-27 23:09:51 +020010 * hardware as being corrupted usually due to a multi-bit ECC memory or cache
Andi Kleen6a460792009-09-16 11:50:15 +020011 * failure.
Andi Kleen1c80b992010-09-27 23:09:51 +020012 *
13 * In addition there is a "soft offline" entry point that allows stop using
14 * not-yet-corrupted-by-suspicious pages without killing anything.
Andi Kleen6a460792009-09-16 11:50:15 +020015 *
16 * Handles page cache pages in various states. The tricky part
Andi Kleen1c80b992010-09-27 23:09:51 +020017 * here is that we can access any page asynchronously in respect to
18 * other VM users, because memory failures could happen anytime and
19 * anywhere. This could violate some of their assumptions. This is why
20 * this code has to be extremely careful. Generally it tries to use
21 * normal locking rules, as in get the standard locks, even if that means
22 * the error handling takes potentially a long time.
Andi Kleene0de78d2015-06-24 16:56:02 -070023 *
24 * It can be very tempting to add handling for obscure cases here.
25 * In general any code for handling new cases should only be added iff:
26 * - You know how to test it.
27 * - You have a test that can be added to mce-test
28 * https://git.kernel.org/cgit/utils/cpu/mce/mce-test.git/
29 * - The case actually shows up as a frequent (top 10) page state in
30 * tools/vm/page-types when running a real workload.
Andi Kleen1c80b992010-09-27 23:09:51 +020031 *
32 * There are several operations here with exponential complexity because
33 * of unsuitable VM data structures. For example the operation to map back
34 * from RMAP chains to processes has to walk the complete process list and
35 * has non linear complexity with the number. But since memory corruptions
36 * are rare we hope to get away with this. This avoids impacting the core
37 * VM.
Andi Kleen6a460792009-09-16 11:50:15 +020038 */
Andi Kleen6a460792009-09-16 11:50:15 +020039#include <linux/kernel.h>
40#include <linux/mm.h>
41#include <linux/page-flags.h>
Wu Fengguang478c5ff2009-12-16 12:19:59 +010042#include <linux/kernel-page-flags.h>
Ingo Molnar3f07c012017-02-08 18:51:30 +010043#include <linux/sched/signal.h>
Ingo Molnar29930022017-02-08 18:51:36 +010044#include <linux/sched/task.h>
Hugh Dickins01e00f82009-10-13 15:02:11 +010045#include <linux/ksm.h>
Andi Kleen6a460792009-09-16 11:50:15 +020046#include <linux/rmap.h>
Paul Gortmakerb9e15ba2011-05-26 16:00:52 -040047#include <linux/export.h>
Andi Kleen6a460792009-09-16 11:50:15 +020048#include <linux/pagemap.h>
49#include <linux/swap.h>
50#include <linux/backing-dev.h>
Andi Kleenfacb6012009-12-16 12:20:00 +010051#include <linux/migrate.h>
Andi Kleenfacb6012009-12-16 12:20:00 +010052#include <linux/suspend.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090053#include <linux/slab.h>
Huang Yingbf998152010-05-31 14:28:19 +080054#include <linux/swapops.h>
Naoya Horiguchi7af446a2010-05-28 09:29:17 +090055#include <linux/hugetlb.h>
KOSAKI Motohiro20d6c962010-12-02 14:31:19 -080056#include <linux/memory_hotplug.h>
Minchan Kim5db8a732011-06-15 15:08:48 -070057#include <linux/mm_inline.h>
Huang Yingea8f5fb2011-07-13 13:14:27 +080058#include <linux/kfifo.h>
Naoya Horiguchia5f65102015-11-05 18:47:26 -080059#include <linux/ratelimit.h>
Andi Kleen6a460792009-09-16 11:50:15 +020060#include "internal.h"
Xie XiuQi97f0b132015-06-24 16:57:36 -070061#include "ras/ras_event.h"
Andi Kleen6a460792009-09-16 11:50:15 +020062
63int sysctl_memory_failure_early_kill __read_mostly = 0;
64
65int sysctl_memory_failure_recovery __read_mostly = 1;
66
Xishi Qiu293c07e2013-02-22 16:34:02 -080067atomic_long_t num_poisoned_pages __read_mostly = ATOMIC_LONG_INIT(0);
Andi Kleen6a460792009-09-16 11:50:15 +020068
Andi Kleen27df5062009-12-21 19:56:42 +010069#if defined(CONFIG_HWPOISON_INJECT) || defined(CONFIG_HWPOISON_INJECT_MODULE)
70
Haicheng Li1bfe5fe2009-12-16 12:19:59 +010071u32 hwpoison_filter_enable = 0;
Wu Fengguang7c116f22009-12-16 12:19:59 +010072u32 hwpoison_filter_dev_major = ~0U;
73u32 hwpoison_filter_dev_minor = ~0U;
Wu Fengguang478c5ff2009-12-16 12:19:59 +010074u64 hwpoison_filter_flags_mask;
75u64 hwpoison_filter_flags_value;
Haicheng Li1bfe5fe2009-12-16 12:19:59 +010076EXPORT_SYMBOL_GPL(hwpoison_filter_enable);
Wu Fengguang7c116f22009-12-16 12:19:59 +010077EXPORT_SYMBOL_GPL(hwpoison_filter_dev_major);
78EXPORT_SYMBOL_GPL(hwpoison_filter_dev_minor);
Wu Fengguang478c5ff2009-12-16 12:19:59 +010079EXPORT_SYMBOL_GPL(hwpoison_filter_flags_mask);
80EXPORT_SYMBOL_GPL(hwpoison_filter_flags_value);
Wu Fengguang7c116f22009-12-16 12:19:59 +010081
82static int hwpoison_filter_dev(struct page *p)
83{
84 struct address_space *mapping;
85 dev_t dev;
86
87 if (hwpoison_filter_dev_major == ~0U &&
88 hwpoison_filter_dev_minor == ~0U)
89 return 0;
90
91 /*
Andi Kleen1c80b992010-09-27 23:09:51 +020092 * page_mapping() does not accept slab pages.
Wu Fengguang7c116f22009-12-16 12:19:59 +010093 */
94 if (PageSlab(p))
95 return -EINVAL;
96
97 mapping = page_mapping(p);
98 if (mapping == NULL || mapping->host == NULL)
99 return -EINVAL;
100
101 dev = mapping->host->i_sb->s_dev;
102 if (hwpoison_filter_dev_major != ~0U &&
103 hwpoison_filter_dev_major != MAJOR(dev))
104 return -EINVAL;
105 if (hwpoison_filter_dev_minor != ~0U &&
106 hwpoison_filter_dev_minor != MINOR(dev))
107 return -EINVAL;
108
109 return 0;
110}
111
Wu Fengguang478c5ff2009-12-16 12:19:59 +0100112static int hwpoison_filter_flags(struct page *p)
113{
114 if (!hwpoison_filter_flags_mask)
115 return 0;
116
117 if ((stable_page_flags(p) & hwpoison_filter_flags_mask) ==
118 hwpoison_filter_flags_value)
119 return 0;
120 else
121 return -EINVAL;
122}
123
Andi Kleen4fd466e2009-12-16 12:19:59 +0100124/*
125 * This allows stress tests to limit test scope to a collection of tasks
126 * by putting them under some memcg. This prevents killing unrelated/important
127 * processes such as /sbin/init. Note that the target task may share clean
128 * pages with init (eg. libc text), which is harmless. If the target task
129 * share _dirty_ pages with another task B, the test scheme must make sure B
130 * is also included in the memcg. At last, due to race conditions this filter
131 * can only guarantee that the page either belongs to the memcg tasks, or is
132 * a freed page.
133 */
Vladimir Davydov94a59fb2015-09-09 15:35:31 -0700134#ifdef CONFIG_MEMCG
Andi Kleen4fd466e2009-12-16 12:19:59 +0100135u64 hwpoison_filter_memcg;
136EXPORT_SYMBOL_GPL(hwpoison_filter_memcg);
137static int hwpoison_filter_task(struct page *p)
138{
Andi Kleen4fd466e2009-12-16 12:19:59 +0100139 if (!hwpoison_filter_memcg)
140 return 0;
141
Vladimir Davydov94a59fb2015-09-09 15:35:31 -0700142 if (page_cgroup_ino(p) != hwpoison_filter_memcg)
Andi Kleen4fd466e2009-12-16 12:19:59 +0100143 return -EINVAL;
144
145 return 0;
146}
147#else
148static int hwpoison_filter_task(struct page *p) { return 0; }
149#endif
150
Wu Fengguang7c116f22009-12-16 12:19:59 +0100151int hwpoison_filter(struct page *p)
152{
Haicheng Li1bfe5fe2009-12-16 12:19:59 +0100153 if (!hwpoison_filter_enable)
154 return 0;
155
Wu Fengguang7c116f22009-12-16 12:19:59 +0100156 if (hwpoison_filter_dev(p))
157 return -EINVAL;
158
Wu Fengguang478c5ff2009-12-16 12:19:59 +0100159 if (hwpoison_filter_flags(p))
160 return -EINVAL;
161
Andi Kleen4fd466e2009-12-16 12:19:59 +0100162 if (hwpoison_filter_task(p))
163 return -EINVAL;
164
Wu Fengguang7c116f22009-12-16 12:19:59 +0100165 return 0;
166}
Andi Kleen27df5062009-12-21 19:56:42 +0100167#else
168int hwpoison_filter(struct page *p)
169{
170 return 0;
171}
172#endif
173
Wu Fengguang7c116f22009-12-16 12:19:59 +0100174EXPORT_SYMBOL_GPL(hwpoison_filter);
175
Andi Kleen6a460792009-09-16 11:50:15 +0200176/*
Tony Luck7329bbe2011-12-13 09:27:58 -0800177 * Send all the processes who have the page mapped a signal.
178 * ``action optional'' if they are not immediately affected by the error
179 * ``action required'' if error happened in current execution context
Andi Kleen6a460792009-09-16 11:50:15 +0200180 */
Tony Luck7329bbe2011-12-13 09:27:58 -0800181static int kill_proc(struct task_struct *t, unsigned long addr, int trapno,
182 unsigned long pfn, struct page *page, int flags)
Andi Kleen6a460792009-09-16 11:50:15 +0200183{
184 struct siginfo si;
185 int ret;
186
Chen Yucong495367c02016-05-20 16:57:32 -0700187 pr_err("Memory failure: %#lx: Killing %s:%d due to hardware memory corruption\n",
188 pfn, t->comm, t->pid);
Andi Kleen6a460792009-09-16 11:50:15 +0200189 si.si_signo = SIGBUS;
190 si.si_errno = 0;
Andi Kleen6a460792009-09-16 11:50:15 +0200191 si.si_addr = (void *)addr;
192#ifdef __ARCH_SI_TRAPNO
193 si.si_trapno = trapno;
194#endif
Wanpeng Lif9121152013-09-11 14:22:52 -0700195 si.si_addr_lsb = compound_order(compound_head(page)) + PAGE_SHIFT;
Tony Luck7329bbe2011-12-13 09:27:58 -0800196
Tony Lucka70ffca2014-06-04 16:10:59 -0700197 if ((flags & MF_ACTION_REQUIRED) && t->mm == current->mm) {
Tony Luck7329bbe2011-12-13 09:27:58 -0800198 si.si_code = BUS_MCEERR_AR;
Tony Lucka70ffca2014-06-04 16:10:59 -0700199 ret = force_sig_info(SIGBUS, &si, current);
Tony Luck7329bbe2011-12-13 09:27:58 -0800200 } else {
201 /*
202 * Don't use force here, it's convenient if the signal
203 * can be temporarily blocked.
204 * This could cause a loop when the user sets SIGBUS
205 * to SIG_IGN, but hopefully no one will do that?
206 */
207 si.si_code = BUS_MCEERR_AO;
208 ret = send_sig_info(SIGBUS, &si, t); /* synchronous? */
209 }
Andi Kleen6a460792009-09-16 11:50:15 +0200210 if (ret < 0)
Chen Yucong495367c02016-05-20 16:57:32 -0700211 pr_info("Memory failure: Error sending signal to %s:%d: %d\n",
Joe Perches11705322016-03-17 14:19:50 -0700212 t->comm, t->pid, ret);
Andi Kleen6a460792009-09-16 11:50:15 +0200213 return ret;
214}
215
216/*
Andi Kleen588f9ce2009-12-16 12:19:57 +0100217 * When a unknown page type is encountered drain as many buffers as possible
218 * in the hope to turn the page into a LRU or free page, which we can handle.
219 */
Andi Kleenfacb6012009-12-16 12:20:00 +0100220void shake_page(struct page *p, int access)
Andi Kleen588f9ce2009-12-16 12:19:57 +0100221{
Naoya Horiguchi8bcb74d2017-05-03 14:56:19 -0700222 if (PageHuge(p))
223 return;
224
Andi Kleen588f9ce2009-12-16 12:19:57 +0100225 if (!PageSlab(p)) {
226 lru_add_drain_all();
227 if (PageLRU(p))
228 return;
Vlastimil Babkac0554322014-12-10 15:43:10 -0800229 drain_all_pages(page_zone(p));
Andi Kleen588f9ce2009-12-16 12:19:57 +0100230 if (PageLRU(p) || is_free_buddy_page(p))
231 return;
232 }
Andi Kleenfacb6012009-12-16 12:20:00 +0100233
Andi Kleen588f9ce2009-12-16 12:19:57 +0100234 /*
Johannes Weiner6b4f7792014-12-12 16:56:13 -0800235 * Only call shrink_node_slabs here (which would also shrink
236 * other caches) if access is not potentially fatal.
Andi Kleen588f9ce2009-12-16 12:19:57 +0100237 */
Vladimir Davydovcb731d62015-02-12 14:58:54 -0800238 if (access)
239 drop_slab_node(page_to_nid(p));
Andi Kleen588f9ce2009-12-16 12:19:57 +0100240}
241EXPORT_SYMBOL_GPL(shake_page);
242
243/*
Andi Kleen6a460792009-09-16 11:50:15 +0200244 * Kill all processes that have a poisoned page mapped and then isolate
245 * the page.
246 *
247 * General strategy:
248 * Find all processes having the page mapped and kill them.
249 * But we keep a page reference around so that the page is not
250 * actually freed yet.
251 * Then stash the page away
252 *
253 * There's no convenient way to get back to mapped processes
254 * from the VMAs. So do a brute-force search over all
255 * running processes.
256 *
257 * Remember that machine checks are not common (or rather
258 * if they are common you have other problems), so this shouldn't
259 * be a performance issue.
260 *
261 * Also there are some races possible while we get from the
262 * error detection to actually handle it.
263 */
264
265struct to_kill {
266 struct list_head nd;
267 struct task_struct *tsk;
268 unsigned long addr;
Andi Kleen9033ae12010-09-27 23:36:05 +0200269 char addr_valid;
Andi Kleen6a460792009-09-16 11:50:15 +0200270};
271
272/*
273 * Failure handling: if we can't find or can't kill a process there's
274 * not much we can do. We just print a message and ignore otherwise.
275 */
276
277/*
278 * Schedule a process for later kill.
279 * Uses GFP_ATOMIC allocations to avoid potential recursions in the VM.
280 * TBD would GFP_NOIO be enough?
281 */
282static void add_to_kill(struct task_struct *tsk, struct page *p,
283 struct vm_area_struct *vma,
284 struct list_head *to_kill,
285 struct to_kill **tkc)
286{
287 struct to_kill *tk;
288
289 if (*tkc) {
290 tk = *tkc;
291 *tkc = NULL;
292 } else {
293 tk = kmalloc(sizeof(struct to_kill), GFP_ATOMIC);
294 if (!tk) {
Chen Yucong495367c02016-05-20 16:57:32 -0700295 pr_err("Memory failure: Out of memory while machine check handling\n");
Andi Kleen6a460792009-09-16 11:50:15 +0200296 return;
297 }
298 }
299 tk->addr = page_address_in_vma(p, vma);
300 tk->addr_valid = 1;
301
302 /*
303 * In theory we don't have to kill when the page was
304 * munmaped. But it could be also a mremap. Since that's
305 * likely very rare kill anyways just out of paranoia, but use
306 * a SIGKILL because the error is not contained anymore.
307 */
308 if (tk->addr == -EFAULT) {
Chen Yucong495367c02016-05-20 16:57:32 -0700309 pr_info("Memory failure: Unable to find user space address %lx in %s\n",
Andi Kleen6a460792009-09-16 11:50:15 +0200310 page_to_pfn(p), tsk->comm);
311 tk->addr_valid = 0;
312 }
313 get_task_struct(tsk);
314 tk->tsk = tsk;
315 list_add_tail(&tk->nd, to_kill);
316}
317
318/*
319 * Kill the processes that have been collected earlier.
320 *
321 * Only do anything when DOIT is set, otherwise just free the list
322 * (this is used for clean pages which do not need killing)
323 * Also when FAIL is set do a force kill because something went
324 * wrong earlier.
325 */
Tony Luck6751ed62012-07-11 10:20:47 -0700326static void kill_procs(struct list_head *to_kill, int forcekill, int trapno,
Minchan Kim666e5a42017-05-03 14:54:20 -0700327 bool fail, struct page *page, unsigned long pfn,
Tony Luck7329bbe2011-12-13 09:27:58 -0800328 int flags)
Andi Kleen6a460792009-09-16 11:50:15 +0200329{
330 struct to_kill *tk, *next;
331
332 list_for_each_entry_safe (tk, next, to_kill, nd) {
Tony Luck6751ed62012-07-11 10:20:47 -0700333 if (forcekill) {
Andi Kleen6a460792009-09-16 11:50:15 +0200334 /*
André Goddard Rosaaf901ca2009-11-14 13:09:05 -0200335 * In case something went wrong with munmapping
Andi Kleen6a460792009-09-16 11:50:15 +0200336 * make sure the process doesn't catch the
337 * signal and then access the memory. Just kill it.
Andi Kleen6a460792009-09-16 11:50:15 +0200338 */
339 if (fail || tk->addr_valid == 0) {
Chen Yucong495367c02016-05-20 16:57:32 -0700340 pr_err("Memory failure: %#lx: forcibly killing %s:%d because of failure to unmap corrupted page\n",
Joe Perches11705322016-03-17 14:19:50 -0700341 pfn, tk->tsk->comm, tk->tsk->pid);
Andi Kleen6a460792009-09-16 11:50:15 +0200342 force_sig(SIGKILL, tk->tsk);
343 }
344
345 /*
346 * In theory the process could have mapped
347 * something else on the address in-between. We could
348 * check for that, but we need to tell the
349 * process anyways.
350 */
Tony Luck7329bbe2011-12-13 09:27:58 -0800351 else if (kill_proc(tk->tsk, tk->addr, trapno,
352 pfn, page, flags) < 0)
Chen Yucong495367c02016-05-20 16:57:32 -0700353 pr_err("Memory failure: %#lx: Cannot send advisory machine check signal to %s:%d\n",
Joe Perches11705322016-03-17 14:19:50 -0700354 pfn, tk->tsk->comm, tk->tsk->pid);
Andi Kleen6a460792009-09-16 11:50:15 +0200355 }
356 put_task_struct(tk->tsk);
357 kfree(tk);
358 }
359}
360
Naoya Horiguchi3ba08122014-06-04 16:11:02 -0700361/*
362 * Find a dedicated thread which is supposed to handle SIGBUS(BUS_MCEERR_AO)
363 * on behalf of the thread group. Return task_struct of the (first found)
364 * dedicated thread if found, and return NULL otherwise.
365 *
366 * We already hold read_lock(&tasklist_lock) in the caller, so we don't
367 * have to call rcu_read_lock/unlock() in this function.
368 */
369static struct task_struct *find_early_kill_thread(struct task_struct *tsk)
Andi Kleen6a460792009-09-16 11:50:15 +0200370{
Naoya Horiguchi3ba08122014-06-04 16:11:02 -0700371 struct task_struct *t;
372
373 for_each_thread(tsk, t)
374 if ((t->flags & PF_MCE_PROCESS) && (t->flags & PF_MCE_EARLY))
375 return t;
376 return NULL;
377}
378
379/*
380 * Determine whether a given process is "early kill" process which expects
381 * to be signaled when some page under the process is hwpoisoned.
382 * Return task_struct of the dedicated thread (main thread unless explicitly
383 * specified) if the process is "early kill," and otherwise returns NULL.
384 */
385static struct task_struct *task_early_kill(struct task_struct *tsk,
386 int force_early)
387{
388 struct task_struct *t;
Andi Kleen6a460792009-09-16 11:50:15 +0200389 if (!tsk->mm)
Naoya Horiguchi3ba08122014-06-04 16:11:02 -0700390 return NULL;
Tony Luck74614de2014-06-04 16:11:01 -0700391 if (force_early)
Naoya Horiguchi3ba08122014-06-04 16:11:02 -0700392 return tsk;
393 t = find_early_kill_thread(tsk);
394 if (t)
395 return t;
396 if (sysctl_memory_failure_early_kill)
397 return tsk;
398 return NULL;
Andi Kleen6a460792009-09-16 11:50:15 +0200399}
400
401/*
402 * Collect processes when the error hit an anonymous page.
403 */
404static void collect_procs_anon(struct page *page, struct list_head *to_kill,
Tony Luck74614de2014-06-04 16:11:01 -0700405 struct to_kill **tkc, int force_early)
Andi Kleen6a460792009-09-16 11:50:15 +0200406{
407 struct vm_area_struct *vma;
408 struct task_struct *tsk;
409 struct anon_vma *av;
Michel Lespinassebf181b92012-10-08 16:31:39 -0700410 pgoff_t pgoff;
Andi Kleen6a460792009-09-16 11:50:15 +0200411
Ingo Molnar4fc3f1d2012-12-02 19:56:50 +0000412 av = page_lock_anon_vma_read(page);
Andi Kleen6a460792009-09-16 11:50:15 +0200413 if (av == NULL) /* Not actually mapped anymore */
Peter Zijlstra9b679322011-06-27 16:18:09 -0700414 return;
415
Naoya Horiguchia0f7a752014-07-23 14:00:01 -0700416 pgoff = page_to_pgoff(page);
Peter Zijlstra9b679322011-06-27 16:18:09 -0700417 read_lock(&tasklist_lock);
Andi Kleen6a460792009-09-16 11:50:15 +0200418 for_each_process (tsk) {
Rik van Riel5beb4932010-03-05 13:42:07 -0800419 struct anon_vma_chain *vmac;
Naoya Horiguchi3ba08122014-06-04 16:11:02 -0700420 struct task_struct *t = task_early_kill(tsk, force_early);
Rik van Riel5beb4932010-03-05 13:42:07 -0800421
Naoya Horiguchi3ba08122014-06-04 16:11:02 -0700422 if (!t)
Andi Kleen6a460792009-09-16 11:50:15 +0200423 continue;
Michel Lespinassebf181b92012-10-08 16:31:39 -0700424 anon_vma_interval_tree_foreach(vmac, &av->rb_root,
425 pgoff, pgoff) {
Rik van Riel5beb4932010-03-05 13:42:07 -0800426 vma = vmac->vma;
Andi Kleen6a460792009-09-16 11:50:15 +0200427 if (!page_mapped_in_vma(page, vma))
428 continue;
Naoya Horiguchi3ba08122014-06-04 16:11:02 -0700429 if (vma->vm_mm == t->mm)
430 add_to_kill(t, page, vma, to_kill, tkc);
Andi Kleen6a460792009-09-16 11:50:15 +0200431 }
432 }
Andi Kleen6a460792009-09-16 11:50:15 +0200433 read_unlock(&tasklist_lock);
Ingo Molnar4fc3f1d2012-12-02 19:56:50 +0000434 page_unlock_anon_vma_read(av);
Andi Kleen6a460792009-09-16 11:50:15 +0200435}
436
437/*
438 * Collect processes when the error hit a file mapped page.
439 */
440static void collect_procs_file(struct page *page, struct list_head *to_kill,
Tony Luck74614de2014-06-04 16:11:01 -0700441 struct to_kill **tkc, int force_early)
Andi Kleen6a460792009-09-16 11:50:15 +0200442{
443 struct vm_area_struct *vma;
444 struct task_struct *tsk;
Andi Kleen6a460792009-09-16 11:50:15 +0200445 struct address_space *mapping = page->mapping;
446
Davidlohr Buesod28eb9c2014-12-12 16:54:36 -0800447 i_mmap_lock_read(mapping);
Peter Zijlstra9b679322011-06-27 16:18:09 -0700448 read_lock(&tasklist_lock);
Andi Kleen6a460792009-09-16 11:50:15 +0200449 for_each_process(tsk) {
Naoya Horiguchia0f7a752014-07-23 14:00:01 -0700450 pgoff_t pgoff = page_to_pgoff(page);
Naoya Horiguchi3ba08122014-06-04 16:11:02 -0700451 struct task_struct *t = task_early_kill(tsk, force_early);
Andi Kleen6a460792009-09-16 11:50:15 +0200452
Naoya Horiguchi3ba08122014-06-04 16:11:02 -0700453 if (!t)
Andi Kleen6a460792009-09-16 11:50:15 +0200454 continue;
Michel Lespinasse6b2dbba2012-10-08 16:31:25 -0700455 vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff,
Andi Kleen6a460792009-09-16 11:50:15 +0200456 pgoff) {
457 /*
458 * Send early kill signal to tasks where a vma covers
459 * the page but the corrupted page is not necessarily
460 * mapped it in its pte.
461 * Assume applications who requested early kill want
462 * to be informed of all such data corruptions.
463 */
Naoya Horiguchi3ba08122014-06-04 16:11:02 -0700464 if (vma->vm_mm == t->mm)
465 add_to_kill(t, page, vma, to_kill, tkc);
Andi Kleen6a460792009-09-16 11:50:15 +0200466 }
467 }
Andi Kleen6a460792009-09-16 11:50:15 +0200468 read_unlock(&tasklist_lock);
Davidlohr Buesod28eb9c2014-12-12 16:54:36 -0800469 i_mmap_unlock_read(mapping);
Andi Kleen6a460792009-09-16 11:50:15 +0200470}
471
472/*
473 * Collect the processes who have the corrupted page mapped to kill.
474 * This is done in two steps for locking reasons.
475 * First preallocate one tokill structure outside the spin locks,
476 * so that we can kill at least one process reasonably reliable.
477 */
Tony Luck74614de2014-06-04 16:11:01 -0700478static void collect_procs(struct page *page, struct list_head *tokill,
479 int force_early)
Andi Kleen6a460792009-09-16 11:50:15 +0200480{
481 struct to_kill *tk;
482
483 if (!page->mapping)
484 return;
485
486 tk = kmalloc(sizeof(struct to_kill), GFP_NOIO);
487 if (!tk)
488 return;
489 if (PageAnon(page))
Tony Luck74614de2014-06-04 16:11:01 -0700490 collect_procs_anon(page, tokill, &tk, force_early);
Andi Kleen6a460792009-09-16 11:50:15 +0200491 else
Tony Luck74614de2014-06-04 16:11:01 -0700492 collect_procs_file(page, tokill, &tk, force_early);
Andi Kleen6a460792009-09-16 11:50:15 +0200493 kfree(tk);
494}
495
Andi Kleen6a460792009-09-16 11:50:15 +0200496static const char *action_name[] = {
Xie XiuQicc637b12015-06-24 16:57:30 -0700497 [MF_IGNORED] = "Ignored",
498 [MF_FAILED] = "Failed",
499 [MF_DELAYED] = "Delayed",
500 [MF_RECOVERED] = "Recovered",
Naoya Horiguchi64d37a22015-04-15 16:13:05 -0700501};
502
503static const char * const action_page_types[] = {
Xie XiuQicc637b12015-06-24 16:57:30 -0700504 [MF_MSG_KERNEL] = "reserved kernel page",
505 [MF_MSG_KERNEL_HIGH_ORDER] = "high-order kernel page",
506 [MF_MSG_SLAB] = "kernel slab page",
507 [MF_MSG_DIFFERENT_COMPOUND] = "different compound page after locking",
508 [MF_MSG_POISONED_HUGE] = "huge page already hardware poisoned",
509 [MF_MSG_HUGE] = "huge page",
510 [MF_MSG_FREE_HUGE] = "free huge page",
511 [MF_MSG_UNMAP_FAILED] = "unmapping failed page",
512 [MF_MSG_DIRTY_SWAPCACHE] = "dirty swapcache page",
513 [MF_MSG_CLEAN_SWAPCACHE] = "clean swapcache page",
514 [MF_MSG_DIRTY_MLOCKED_LRU] = "dirty mlocked LRU page",
515 [MF_MSG_CLEAN_MLOCKED_LRU] = "clean mlocked LRU page",
516 [MF_MSG_DIRTY_UNEVICTABLE_LRU] = "dirty unevictable LRU page",
517 [MF_MSG_CLEAN_UNEVICTABLE_LRU] = "clean unevictable LRU page",
518 [MF_MSG_DIRTY_LRU] = "dirty LRU page",
519 [MF_MSG_CLEAN_LRU] = "clean LRU page",
520 [MF_MSG_TRUNCATED_LRU] = "already truncated LRU page",
521 [MF_MSG_BUDDY] = "free buddy page",
522 [MF_MSG_BUDDY_2ND] = "free buddy page (2nd try)",
523 [MF_MSG_UNKNOWN] = "unknown page",
Naoya Horiguchi64d37a22015-04-15 16:13:05 -0700524};
525
Andi Kleen6a460792009-09-16 11:50:15 +0200526/*
Wu Fengguangdc2a1cb2009-12-16 12:19:58 +0100527 * XXX: It is possible that a page is isolated from LRU cache,
528 * and then kept in swap cache or failed to remove from page cache.
529 * The page count will stop it from being freed by unpoison.
530 * Stress tests should be aware of this memory leak problem.
531 */
532static int delete_from_lru_cache(struct page *p)
533{
534 if (!isolate_lru_page(p)) {
535 /*
536 * Clear sensible page flags, so that the buddy system won't
537 * complain when the page is unpoison-and-freed.
538 */
539 ClearPageActive(p);
540 ClearPageUnevictable(p);
Michal Hocko18365222017-05-12 15:46:26 -0700541
542 /*
543 * Poisoned page might never drop its ref count to 0 so we have
544 * to uncharge it manually from its memcg.
545 */
546 mem_cgroup_uncharge(p);
547
Wu Fengguangdc2a1cb2009-12-16 12:19:58 +0100548 /*
549 * drop the page count elevated by isolate_lru_page()
550 */
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +0300551 put_page(p);
Wu Fengguangdc2a1cb2009-12-16 12:19:58 +0100552 return 0;
553 }
554 return -EIO;
555}
556
557/*
Andi Kleen6a460792009-09-16 11:50:15 +0200558 * Error hit kernel page.
559 * Do nothing, try to be lucky and not touch this instead. For a few cases we
560 * could be more sophisticated.
561 */
562static int me_kernel(struct page *p, unsigned long pfn)
563{
Xie XiuQicc637b12015-06-24 16:57:30 -0700564 return MF_IGNORED;
Andi Kleen6a460792009-09-16 11:50:15 +0200565}
566
567/*
568 * Page in unknown state. Do nothing.
569 */
570static int me_unknown(struct page *p, unsigned long pfn)
571{
Chen Yucong495367c02016-05-20 16:57:32 -0700572 pr_err("Memory failure: %#lx: Unknown page state\n", pfn);
Xie XiuQicc637b12015-06-24 16:57:30 -0700573 return MF_FAILED;
Andi Kleen6a460792009-09-16 11:50:15 +0200574}
575
576/*
Andi Kleen6a460792009-09-16 11:50:15 +0200577 * Clean (or cleaned) page cache page.
578 */
579static int me_pagecache_clean(struct page *p, unsigned long pfn)
580{
581 int err;
Xie XiuQicc637b12015-06-24 16:57:30 -0700582 int ret = MF_FAILED;
Andi Kleen6a460792009-09-16 11:50:15 +0200583 struct address_space *mapping;
584
Wu Fengguangdc2a1cb2009-12-16 12:19:58 +0100585 delete_from_lru_cache(p);
586
Andi Kleen6a460792009-09-16 11:50:15 +0200587 /*
588 * For anonymous pages we're done the only reference left
589 * should be the one m_f() holds.
590 */
591 if (PageAnon(p))
Xie XiuQicc637b12015-06-24 16:57:30 -0700592 return MF_RECOVERED;
Andi Kleen6a460792009-09-16 11:50:15 +0200593
594 /*
595 * Now truncate the page in the page cache. This is really
596 * more like a "temporary hole punch"
597 * Don't do this for block devices when someone else
598 * has a reference, because it could be file system metadata
599 * and that's not safe to truncate.
600 */
601 mapping = page_mapping(p);
602 if (!mapping) {
603 /*
604 * Page has been teared down in the meanwhile
605 */
Xie XiuQicc637b12015-06-24 16:57:30 -0700606 return MF_FAILED;
Andi Kleen6a460792009-09-16 11:50:15 +0200607 }
608
609 /*
610 * Truncation is a bit tricky. Enable it per file system for now.
611 *
612 * Open: to take i_mutex or not for this? Right now we don't.
613 */
614 if (mapping->a_ops->error_remove_page) {
615 err = mapping->a_ops->error_remove_page(mapping, p);
616 if (err != 0) {
Chen Yucong495367c02016-05-20 16:57:32 -0700617 pr_info("Memory failure: %#lx: Failed to punch page: %d\n",
Joe Perches11705322016-03-17 14:19:50 -0700618 pfn, err);
Andi Kleen6a460792009-09-16 11:50:15 +0200619 } else if (page_has_private(p) &&
620 !try_to_release_page(p, GFP_NOIO)) {
Chen Yucong495367c02016-05-20 16:57:32 -0700621 pr_info("Memory failure: %#lx: failed to release buffers\n",
622 pfn);
Andi Kleen6a460792009-09-16 11:50:15 +0200623 } else {
Xie XiuQicc637b12015-06-24 16:57:30 -0700624 ret = MF_RECOVERED;
Andi Kleen6a460792009-09-16 11:50:15 +0200625 }
626 } else {
627 /*
628 * If the file system doesn't support it just invalidate
629 * This fails on dirty or anything with private pages
630 */
631 if (invalidate_inode_page(p))
Xie XiuQicc637b12015-06-24 16:57:30 -0700632 ret = MF_RECOVERED;
Andi Kleen6a460792009-09-16 11:50:15 +0200633 else
Chen Yucong495367c02016-05-20 16:57:32 -0700634 pr_info("Memory failure: %#lx: Failed to invalidate\n",
635 pfn);
Andi Kleen6a460792009-09-16 11:50:15 +0200636 }
637 return ret;
638}
639
640/*
Zhi Yong Wu549543d2014-01-21 15:49:08 -0800641 * Dirty pagecache page
Andi Kleen6a460792009-09-16 11:50:15 +0200642 * Issues: when the error hit a hole page the error is not properly
643 * propagated.
644 */
645static int me_pagecache_dirty(struct page *p, unsigned long pfn)
646{
647 struct address_space *mapping = page_mapping(p);
648
649 SetPageError(p);
650 /* TBD: print more information about the file. */
651 if (mapping) {
652 /*
653 * IO error will be reported by write(), fsync(), etc.
654 * who check the mapping.
655 * This way the application knows that something went
656 * wrong with its dirty file data.
657 *
658 * There's one open issue:
659 *
660 * The EIO will be only reported on the next IO
661 * operation and then cleared through the IO map.
662 * Normally Linux has two mechanisms to pass IO error
663 * first through the AS_EIO flag in the address space
664 * and then through the PageError flag in the page.
665 * Since we drop pages on memory failure handling the
666 * only mechanism open to use is through AS_AIO.
667 *
668 * This has the disadvantage that it gets cleared on
669 * the first operation that returns an error, while
670 * the PageError bit is more sticky and only cleared
671 * when the page is reread or dropped. If an
672 * application assumes it will always get error on
673 * fsync, but does other operations on the fd before
Lucas De Marchi25985ed2011-03-30 22:57:33 -0300674 * and the page is dropped between then the error
Andi Kleen6a460792009-09-16 11:50:15 +0200675 * will not be properly reported.
676 *
677 * This can already happen even without hwpoisoned
678 * pages: first on metadata IO errors (which only
679 * report through AS_EIO) or when the page is dropped
680 * at the wrong time.
681 *
682 * So right now we assume that the application DTRT on
683 * the first EIO, but we're not worse than other parts
684 * of the kernel.
685 */
Jeff Laytonaf21bfa2017-07-06 07:02:19 -0400686 mapping_set_error(mapping, -EIO);
Andi Kleen6a460792009-09-16 11:50:15 +0200687 }
688
689 return me_pagecache_clean(p, pfn);
690}
691
692/*
693 * Clean and dirty swap cache.
694 *
695 * Dirty swap cache page is tricky to handle. The page could live both in page
696 * cache and swap cache(ie. page is freshly swapped in). So it could be
697 * referenced concurrently by 2 types of PTEs:
698 * normal PTEs and swap PTEs. We try to handle them consistently by calling
699 * try_to_unmap(TTU_IGNORE_HWPOISON) to convert the normal PTEs to swap PTEs,
700 * and then
701 * - clear dirty bit to prevent IO
702 * - remove from LRU
703 * - but keep in the swap cache, so that when we return to it on
704 * a later page fault, we know the application is accessing
705 * corrupted data and shall be killed (we installed simple
706 * interception code in do_swap_page to catch it).
707 *
708 * Clean swap cache pages can be directly isolated. A later page fault will
709 * bring in the known good data from disk.
710 */
711static int me_swapcache_dirty(struct page *p, unsigned long pfn)
712{
Andi Kleen6a460792009-09-16 11:50:15 +0200713 ClearPageDirty(p);
714 /* Trigger EIO in shmem: */
715 ClearPageUptodate(p);
716
Wu Fengguangdc2a1cb2009-12-16 12:19:58 +0100717 if (!delete_from_lru_cache(p))
Xie XiuQicc637b12015-06-24 16:57:30 -0700718 return MF_DELAYED;
Wu Fengguangdc2a1cb2009-12-16 12:19:58 +0100719 else
Xie XiuQicc637b12015-06-24 16:57:30 -0700720 return MF_FAILED;
Andi Kleen6a460792009-09-16 11:50:15 +0200721}
722
723static int me_swapcache_clean(struct page *p, unsigned long pfn)
724{
Andi Kleen6a460792009-09-16 11:50:15 +0200725 delete_from_swap_cache(p);
Wu Fengguange43c3af2009-09-29 13:16:20 +0800726
Wu Fengguangdc2a1cb2009-12-16 12:19:58 +0100727 if (!delete_from_lru_cache(p))
Xie XiuQicc637b12015-06-24 16:57:30 -0700728 return MF_RECOVERED;
Wu Fengguangdc2a1cb2009-12-16 12:19:58 +0100729 else
Xie XiuQicc637b12015-06-24 16:57:30 -0700730 return MF_FAILED;
Andi Kleen6a460792009-09-16 11:50:15 +0200731}
732
733/*
734 * Huge pages. Needs work.
735 * Issues:
Naoya Horiguchi93f70f92010-05-28 09:29:20 +0900736 * - Error on hugepage is contained in hugepage unit (not in raw page unit.)
737 * To narrow down kill region to one page, we need to break up pmd.
Andi Kleen6a460792009-09-16 11:50:15 +0200738 */
739static int me_huge_page(struct page *p, unsigned long pfn)
740{
Naoya Horiguchi6de2b1a2010-09-08 10:19:36 +0900741 int res = 0;
Naoya Horiguchi93f70f92010-05-28 09:29:20 +0900742 struct page *hpage = compound_head(p);
Naoya Horiguchi2491ffe2015-06-24 16:56:53 -0700743
744 if (!PageHuge(hpage))
745 return MF_DELAYED;
746
Naoya Horiguchi93f70f92010-05-28 09:29:20 +0900747 /*
748 * We can safely recover from error on free or reserved (i.e.
749 * not in-use) hugepage by dequeuing it from freelist.
750 * To check whether a hugepage is in-use or not, we can't use
751 * page->lru because it can be used in other hugepage operations,
752 * such as __unmap_hugepage_range() and gather_surplus_pages().
753 * So instead we use page_mapping() and PageAnon().
Naoya Horiguchi93f70f92010-05-28 09:29:20 +0900754 */
755 if (!(page_mapping(hpage) || PageAnon(hpage))) {
Naoya Horiguchi6de2b1a2010-09-08 10:19:36 +0900756 res = dequeue_hwpoisoned_huge_page(hpage);
757 if (!res)
Xie XiuQicc637b12015-06-24 16:57:30 -0700758 return MF_RECOVERED;
Naoya Horiguchi93f70f92010-05-28 09:29:20 +0900759 }
Xie XiuQicc637b12015-06-24 16:57:30 -0700760 return MF_DELAYED;
Andi Kleen6a460792009-09-16 11:50:15 +0200761}
762
763/*
764 * Various page states we can handle.
765 *
766 * A page state is defined by its current page->flags bits.
767 * The table matches them in order and calls the right handler.
768 *
769 * This is quite tricky because we can access page at any time
Lucas De Marchi25985ed2011-03-30 22:57:33 -0300770 * in its live cycle, so all accesses have to be extremely careful.
Andi Kleen6a460792009-09-16 11:50:15 +0200771 *
772 * This is not complete. More states could be added.
773 * For any missing state don't attempt recovery.
774 */
775
776#define dirty (1UL << PG_dirty)
Nicholas Piggin6326fec2016-12-25 13:00:29 +1000777#define sc ((1UL << PG_swapcache) | (1UL << PG_swapbacked))
Andi Kleen6a460792009-09-16 11:50:15 +0200778#define unevict (1UL << PG_unevictable)
779#define mlock (1UL << PG_mlocked)
780#define writeback (1UL << PG_writeback)
781#define lru (1UL << PG_lru)
Andi Kleen6a460792009-09-16 11:50:15 +0200782#define head (1UL << PG_head)
Andi Kleen6a460792009-09-16 11:50:15 +0200783#define slab (1UL << PG_slab)
Andi Kleen6a460792009-09-16 11:50:15 +0200784#define reserved (1UL << PG_reserved)
785
786static struct page_state {
787 unsigned long mask;
788 unsigned long res;
Xie XiuQicc637b12015-06-24 16:57:30 -0700789 enum mf_action_page_type type;
Andi Kleen6a460792009-09-16 11:50:15 +0200790 int (*action)(struct page *p, unsigned long pfn);
791} error_states[] = {
Xie XiuQicc637b12015-06-24 16:57:30 -0700792 { reserved, reserved, MF_MSG_KERNEL, me_kernel },
Wu Fengguang95d01fc2009-12-16 12:19:58 +0100793 /*
794 * free pages are specially detected outside this table:
795 * PG_buddy pages only make a small fraction of all free pages.
796 */
Andi Kleen6a460792009-09-16 11:50:15 +0200797
798 /*
799 * Could in theory check if slab page is free or if we can drop
800 * currently unused objects without touching them. But just
801 * treat it as standard kernel for now.
802 */
Xie XiuQicc637b12015-06-24 16:57:30 -0700803 { slab, slab, MF_MSG_SLAB, me_kernel },
Andi Kleen6a460792009-09-16 11:50:15 +0200804
Xie XiuQicc637b12015-06-24 16:57:30 -0700805 { head, head, MF_MSG_HUGE, me_huge_page },
Andi Kleen6a460792009-09-16 11:50:15 +0200806
Xie XiuQicc637b12015-06-24 16:57:30 -0700807 { sc|dirty, sc|dirty, MF_MSG_DIRTY_SWAPCACHE, me_swapcache_dirty },
808 { sc|dirty, sc, MF_MSG_CLEAN_SWAPCACHE, me_swapcache_clean },
Andi Kleen6a460792009-09-16 11:50:15 +0200809
Xie XiuQicc637b12015-06-24 16:57:30 -0700810 { mlock|dirty, mlock|dirty, MF_MSG_DIRTY_MLOCKED_LRU, me_pagecache_dirty },
811 { mlock|dirty, mlock, MF_MSG_CLEAN_MLOCKED_LRU, me_pagecache_clean },
Andi Kleen6a460792009-09-16 11:50:15 +0200812
Xie XiuQicc637b12015-06-24 16:57:30 -0700813 { unevict|dirty, unevict|dirty, MF_MSG_DIRTY_UNEVICTABLE_LRU, me_pagecache_dirty },
814 { unevict|dirty, unevict, MF_MSG_CLEAN_UNEVICTABLE_LRU, me_pagecache_clean },
Naoya Horiguchi5f4b9fc2013-02-22 16:35:53 -0800815
Xie XiuQicc637b12015-06-24 16:57:30 -0700816 { lru|dirty, lru|dirty, MF_MSG_DIRTY_LRU, me_pagecache_dirty },
817 { lru|dirty, lru, MF_MSG_CLEAN_LRU, me_pagecache_clean },
Andi Kleen6a460792009-09-16 11:50:15 +0200818
819 /*
820 * Catchall entry: must be at end.
821 */
Xie XiuQicc637b12015-06-24 16:57:30 -0700822 { 0, 0, MF_MSG_UNKNOWN, me_unknown },
Andi Kleen6a460792009-09-16 11:50:15 +0200823};
824
Andi Kleen2326c462009-12-16 12:20:00 +0100825#undef dirty
826#undef sc
827#undef unevict
828#undef mlock
829#undef writeback
830#undef lru
Andi Kleen2326c462009-12-16 12:20:00 +0100831#undef head
Andi Kleen2326c462009-12-16 12:20:00 +0100832#undef slab
833#undef reserved
834
Naoya Horiguchiff604cf2012-12-11 16:01:32 -0800835/*
836 * "Dirty/Clean" indication is not 100% accurate due to the possibility of
837 * setting PG_dirty outside page lock. See also comment above set_page_dirty().
838 */
Xie XiuQicc3e2af2015-06-24 16:57:33 -0700839static void action_result(unsigned long pfn, enum mf_action_page_type type,
840 enum mf_result result)
Andi Kleen6a460792009-09-16 11:50:15 +0200841{
Xie XiuQi97f0b132015-06-24 16:57:36 -0700842 trace_memory_failure_event(pfn, type, result);
843
Chen Yucong495367c02016-05-20 16:57:32 -0700844 pr_err("Memory failure: %#lx: recovery action for %s: %s\n",
Naoya Horiguchi64d37a22015-04-15 16:13:05 -0700845 pfn, action_page_types[type], action_name[result]);
Andi Kleen6a460792009-09-16 11:50:15 +0200846}
847
848static int page_action(struct page_state *ps, struct page *p,
Wu Fengguangbd1ce5f2009-12-16 12:19:57 +0100849 unsigned long pfn)
Andi Kleen6a460792009-09-16 11:50:15 +0200850{
851 int result;
Wu Fengguang7456b042009-10-19 08:15:01 +0200852 int count;
Andi Kleen6a460792009-09-16 11:50:15 +0200853
854 result = ps->action(p, pfn);
Wu Fengguang7456b042009-10-19 08:15:01 +0200855
Wu Fengguangbd1ce5f2009-12-16 12:19:57 +0100856 count = page_count(p) - 1;
Xie XiuQicc637b12015-06-24 16:57:30 -0700857 if (ps->action == me_swapcache_dirty && result == MF_DELAYED)
Wu Fengguang138ce282009-12-16 12:19:58 +0100858 count--;
859 if (count != 0) {
Chen Yucong495367c02016-05-20 16:57:32 -0700860 pr_err("Memory failure: %#lx: %s still referenced by %d users\n",
Naoya Horiguchi64d37a22015-04-15 16:13:05 -0700861 pfn, action_page_types[ps->type], count);
Xie XiuQicc637b12015-06-24 16:57:30 -0700862 result = MF_FAILED;
Wu Fengguang138ce282009-12-16 12:19:58 +0100863 }
Naoya Horiguchi64d37a22015-04-15 16:13:05 -0700864 action_result(pfn, ps->type, result);
Andi Kleen6a460792009-09-16 11:50:15 +0200865
866 /* Could do more checks here if page looks ok */
867 /*
868 * Could adjust zone counters here to correct for the missing page.
869 */
870
Xie XiuQicc637b12015-06-24 16:57:30 -0700871 return (result == MF_RECOVERED || result == MF_DELAYED) ? 0 : -EBUSY;
Andi Kleen6a460792009-09-16 11:50:15 +0200872}
873
Naoya Horiguchiead07f62015-06-24 16:56:48 -0700874/**
875 * get_hwpoison_page() - Get refcount for memory error handling:
876 * @page: raw error page (hit by memory error)
877 *
878 * Return: return 0 if failed to grab the refcount, otherwise true (some
879 * non-zero value.)
880 */
881int get_hwpoison_page(struct page *page)
882{
883 struct page *head = compound_head(page);
884
Naoya Horiguchi4e41a302016-01-15 16:54:07 -0800885 if (!PageHuge(head) && PageTransHuge(head)) {
Naoya Horiguchi98ed2b02015-08-06 15:47:04 -0700886 /*
887 * Non anonymous thp exists only in allocation/free time. We
888 * can't handle such a case correctly, so let's give it up.
889 * This should be better than triggering BUG_ON when kernel
890 * tries to touch the "partially handled" page.
891 */
892 if (!PageAnon(head)) {
Chen Yucong495367c02016-05-20 16:57:32 -0700893 pr_err("Memory failure: %#lx: non anonymous thp\n",
Naoya Horiguchi98ed2b02015-08-06 15:47:04 -0700894 page_to_pfn(page));
895 return 0;
896 }
Naoya Horiguchiead07f62015-06-24 16:56:48 -0700897 }
898
Konstantin Khlebnikovc2e7e002016-04-28 16:19:03 -0700899 if (get_page_unless_zero(head)) {
900 if (head == compound_head(page))
901 return 1;
902
Chen Yucong495367c02016-05-20 16:57:32 -0700903 pr_info("Memory failure: %#lx cannot catch tail\n",
904 page_to_pfn(page));
Konstantin Khlebnikovc2e7e002016-04-28 16:19:03 -0700905 put_page(head);
906 }
907
908 return 0;
Naoya Horiguchiead07f62015-06-24 16:56:48 -0700909}
910EXPORT_SYMBOL_GPL(get_hwpoison_page);
911
Andi Kleen6a460792009-09-16 11:50:15 +0200912/*
913 * Do all that is necessary to remove user space mappings. Unmap
914 * the pages and send SIGBUS to the processes if the data was dirty.
915 */
Minchan Kim666e5a42017-05-03 14:54:20 -0700916static bool hwpoison_user_mappings(struct page *p, unsigned long pfn,
Naoya Horiguchi54b9dd12014-01-23 15:53:14 -0800917 int trapno, int flags, struct page **hpagep)
Andi Kleen6a460792009-09-16 11:50:15 +0200918{
Shaohua Lia128ca72017-05-03 14:52:22 -0700919 enum ttu_flags ttu = TTU_IGNORE_MLOCK | TTU_IGNORE_ACCESS;
Andi Kleen6a460792009-09-16 11:50:15 +0200920 struct address_space *mapping;
921 LIST_HEAD(tokill);
Minchan Kim666e5a42017-05-03 14:54:20 -0700922 bool unmap_success;
Tony Luck6751ed62012-07-11 10:20:47 -0700923 int kill = 1, forcekill;
Naoya Horiguchi54b9dd12014-01-23 15:53:14 -0800924 struct page *hpage = *hpagep;
Naoya Horiguchi286c4692017-05-03 14:56:22 -0700925 bool mlocked = PageMlocked(hpage);
Andi Kleen6a460792009-09-16 11:50:15 +0200926
Naoya Horiguchi93a9eb32014-07-30 16:08:28 -0700927 /*
928 * Here we are interested only in user-mapped pages, so skip any
929 * other types of pages.
930 */
931 if (PageReserved(p) || PageSlab(p))
Minchan Kim666e5a42017-05-03 14:54:20 -0700932 return true;
Naoya Horiguchi93a9eb32014-07-30 16:08:28 -0700933 if (!(PageLRU(hpage) || PageHuge(p)))
Minchan Kim666e5a42017-05-03 14:54:20 -0700934 return true;
Andi Kleen6a460792009-09-16 11:50:15 +0200935
Andi Kleen6a460792009-09-16 11:50:15 +0200936 /*
937 * This check implies we don't kill processes if their pages
938 * are in the swap cache early. Those are always late kills.
939 */
Naoya Horiguchi7af446a2010-05-28 09:29:17 +0900940 if (!page_mapped(hpage))
Minchan Kim666e5a42017-05-03 14:54:20 -0700941 return true;
Wu Fengguang1668bfd2009-12-16 12:19:58 +0100942
Naoya Horiguchi52089b12014-07-30 16:08:30 -0700943 if (PageKsm(p)) {
Chen Yucong495367c02016-05-20 16:57:32 -0700944 pr_err("Memory failure: %#lx: can't handle KSM pages.\n", pfn);
Minchan Kim666e5a42017-05-03 14:54:20 -0700945 return false;
Naoya Horiguchi52089b12014-07-30 16:08:30 -0700946 }
Andi Kleen6a460792009-09-16 11:50:15 +0200947
948 if (PageSwapCache(p)) {
Chen Yucong495367c02016-05-20 16:57:32 -0700949 pr_err("Memory failure: %#lx: keeping poisoned page in swap cache\n",
950 pfn);
Andi Kleen6a460792009-09-16 11:50:15 +0200951 ttu |= TTU_IGNORE_HWPOISON;
952 }
953
954 /*
955 * Propagate the dirty bit from PTEs to struct page first, because we
956 * need this to decide if we should kill or just drop the page.
Wu Fengguangdb0480b2009-12-16 12:19:58 +0100957 * XXX: the dirty test could be racy: set_page_dirty() may not always
958 * be called inside page lock (it's recommended but not enforced).
Andi Kleen6a460792009-09-16 11:50:15 +0200959 */
Naoya Horiguchi7af446a2010-05-28 09:29:17 +0900960 mapping = page_mapping(hpage);
Tony Luck6751ed62012-07-11 10:20:47 -0700961 if (!(flags & MF_MUST_KILL) && !PageDirty(hpage) && mapping &&
Naoya Horiguchi7af446a2010-05-28 09:29:17 +0900962 mapping_cap_writeback_dirty(mapping)) {
963 if (page_mkclean(hpage)) {
964 SetPageDirty(hpage);
Andi Kleen6a460792009-09-16 11:50:15 +0200965 } else {
966 kill = 0;
967 ttu |= TTU_IGNORE_HWPOISON;
Chen Yucong495367c02016-05-20 16:57:32 -0700968 pr_info("Memory failure: %#lx: corrupted page was clean: dropped without side effects\n",
Andi Kleen6a460792009-09-16 11:50:15 +0200969 pfn);
970 }
971 }
972
Jin Dongminga6d30dd2011-02-01 15:52:40 -0800973 /*
Andi Kleen6a460792009-09-16 11:50:15 +0200974 * First collect all the processes that have the page
975 * mapped in dirty form. This has to be done before try_to_unmap,
976 * because ttu takes the rmap data structures down.
977 *
978 * Error handling: We ignore errors here because
979 * there's nothing that can be done.
980 */
981 if (kill)
Naoya Horiguchi415c64c2015-06-24 16:56:45 -0700982 collect_procs(hpage, &tokill, flags & MF_ACTION_REQUIRED);
Andi Kleen6a460792009-09-16 11:50:15 +0200983
Minchan Kim666e5a42017-05-03 14:54:20 -0700984 unmap_success = try_to_unmap(hpage, ttu);
985 if (!unmap_success)
Chen Yucong495367c02016-05-20 16:57:32 -0700986 pr_err("Memory failure: %#lx: failed to unmap page (mapcount=%d)\n",
Joe Perches11705322016-03-17 14:19:50 -0700987 pfn, page_mapcount(hpage));
Jin Dongminga6d30dd2011-02-01 15:52:40 -0800988
Andi Kleen6a460792009-09-16 11:50:15 +0200989 /*
Naoya Horiguchi286c4692017-05-03 14:56:22 -0700990 * try_to_unmap() might put mlocked page in lru cache, so call
991 * shake_page() again to ensure that it's flushed.
992 */
993 if (mlocked)
994 shake_page(hpage, 0);
995
996 /*
Andi Kleen6a460792009-09-16 11:50:15 +0200997 * Now that the dirty bit has been propagated to the
998 * struct page and all unmaps done we can decide if
999 * killing is needed or not. Only kill when the page
Tony Luck6751ed62012-07-11 10:20:47 -07001000 * was dirty or the process is not restartable,
1001 * otherwise the tokill list is merely
Andi Kleen6a460792009-09-16 11:50:15 +02001002 * freed. When there was a problem unmapping earlier
1003 * use a more force-full uncatchable kill to prevent
1004 * any accesses to the poisoned memory.
1005 */
Naoya Horiguchi415c64c2015-06-24 16:56:45 -07001006 forcekill = PageDirty(hpage) || (flags & MF_MUST_KILL);
Minchan Kim666e5a42017-05-03 14:54:20 -07001007 kill_procs(&tokill, forcekill, trapno, !unmap_success, p, pfn, flags);
Wu Fengguang1668bfd2009-12-16 12:19:58 +01001008
Minchan Kim666e5a42017-05-03 14:54:20 -07001009 return unmap_success;
Andi Kleen6a460792009-09-16 11:50:15 +02001010}
1011
Naoya Horiguchi7013feb2010-05-28 09:29:18 +09001012static void set_page_hwpoison_huge_page(struct page *hpage)
1013{
1014 int i;
Wanpeng Lif9121152013-09-11 14:22:52 -07001015 int nr_pages = 1 << compound_order(hpage);
Naoya Horiguchi7013feb2010-05-28 09:29:18 +09001016 for (i = 0; i < nr_pages; i++)
1017 SetPageHWPoison(hpage + i);
1018}
1019
1020static void clear_page_hwpoison_huge_page(struct page *hpage)
1021{
1022 int i;
Wanpeng Lif9121152013-09-11 14:22:52 -07001023 int nr_pages = 1 << compound_order(hpage);
Naoya Horiguchi7013feb2010-05-28 09:29:18 +09001024 for (i = 0; i < nr_pages; i++)
1025 ClearPageHWPoison(hpage + i);
1026}
1027
Tony Luckcd42f4a2011-12-15 10:48:12 -08001028/**
1029 * memory_failure - Handle memory failure of a page.
1030 * @pfn: Page Number of the corrupted page
1031 * @trapno: Trap number reported in the signal to user space.
1032 * @flags: fine tune action taken
1033 *
1034 * This function is called by the low level machine check code
1035 * of an architecture when it detects hardware memory corruption
1036 * of a page. It tries its best to recover, which includes
1037 * dropping pages, killing processes etc.
1038 *
1039 * The function is primarily of use for corruptions that
1040 * happen outside the current execution context (e.g. when
1041 * detected by a background scrubber)
1042 *
1043 * Must run in process context (e.g. a work queue) with interrupts
1044 * enabled and no spinlocks hold.
1045 */
1046int memory_failure(unsigned long pfn, int trapno, int flags)
Andi Kleen6a460792009-09-16 11:50:15 +02001047{
1048 struct page_state *ps;
1049 struct page *p;
Naoya Horiguchi7af446a2010-05-28 09:29:17 +09001050 struct page *hpage;
Naoya Horiguchi415c64c2015-06-24 16:56:45 -07001051 struct page *orig_head;
Andi Kleen6a460792009-09-16 11:50:15 +02001052 int res;
Naoya Horiguchic9fbdd52010-05-28 09:29:19 +09001053 unsigned int nr_pages;
Naoya Horiguchi524fca12013-02-22 16:35:51 -08001054 unsigned long page_flags;
Andi Kleen6a460792009-09-16 11:50:15 +02001055
1056 if (!sysctl_memory_failure_recovery)
1057 panic("Memory failure from trap %d on page %lx", trapno, pfn);
1058
1059 if (!pfn_valid(pfn)) {
Chen Yucong495367c02016-05-20 16:57:32 -07001060 pr_err("Memory failure: %#lx: memory outside kernel control\n",
1061 pfn);
Wu Fengguanga7560fc2009-12-16 12:19:57 +01001062 return -ENXIO;
Andi Kleen6a460792009-09-16 11:50:15 +02001063 }
1064
1065 p = pfn_to_page(pfn);
Naoya Horiguchi415c64c2015-06-24 16:56:45 -07001066 orig_head = hpage = compound_head(p);
Andi Kleen6a460792009-09-16 11:50:15 +02001067 if (TestSetPageHWPoison(p)) {
Chen Yucong495367c02016-05-20 16:57:32 -07001068 pr_err("Memory failure: %#lx: already hardware poisoned\n",
1069 pfn);
Andi Kleen6a460792009-09-16 11:50:15 +02001070 return 0;
1071 }
1072
Naoya Horiguchi4db0e952013-02-22 16:34:05 -08001073 /*
1074 * Currently errors on hugetlbfs pages are measured in hugepage units,
1075 * so nr_pages should be 1 << compound_order. OTOH when errors are on
1076 * transparent hugepages, they are supposed to be split and error
1077 * measurement is done in normal page units. So nr_pages should be one
1078 * in this case.
1079 */
1080 if (PageHuge(p))
1081 nr_pages = 1 << compound_order(hpage);
1082 else /* normal page or thp */
1083 nr_pages = 1;
Naoya Horiguchi8e304562015-09-08 15:03:24 -07001084 num_poisoned_pages_add(nr_pages);
Andi Kleen6a460792009-09-16 11:50:15 +02001085
1086 /*
1087 * We need/can do nothing about count=0 pages.
1088 * 1) it's a free page, and therefore in safe hand:
1089 * prep_new_page() will be the gate keeper.
Naoya Horiguchi8c6c2ec2010-09-08 10:19:38 +09001090 * 2) it's a free hugepage, which is also safe:
1091 * an affected hugepage will be dequeued from hugepage freelist,
1092 * so there's no concern about reusing it ever after.
1093 * 3) it's part of a non-compound high order page.
Andi Kleen6a460792009-09-16 11:50:15 +02001094 * Implies some kernel user: cannot stop them from
1095 * R/W the page; let's pray that the page has been
1096 * used and will be freed some time later.
1097 * In fact it's dangerous to directly bump up page count from 0,
1098 * that may make page_freeze_refs()/page_unfreeze_refs() mismatch.
1099 */
Naoya Horiguchiead07f62015-06-24 16:56:48 -07001100 if (!(flags & MF_COUNT_INCREASED) && !get_hwpoison_page(p)) {
Wu Fengguang8d22ba12009-12-16 12:19:58 +01001101 if (is_free_buddy_page(p)) {
Xie XiuQicc637b12015-06-24 16:57:30 -07001102 action_result(pfn, MF_MSG_BUDDY, MF_DELAYED);
Wu Fengguang8d22ba12009-12-16 12:19:58 +01001103 return 0;
Naoya Horiguchi8c6c2ec2010-09-08 10:19:38 +09001104 } else if (PageHuge(hpage)) {
1105 /*
Chen Yucongb9851942014-05-22 11:54:15 -07001106 * Check "filter hit" and "race with other subpage."
Naoya Horiguchi8c6c2ec2010-09-08 10:19:38 +09001107 */
Jens Axboe7eaceac2011-03-10 08:52:07 +01001108 lock_page(hpage);
Chen Yucongb9851942014-05-22 11:54:15 -07001109 if (PageHWPoison(hpage)) {
1110 if ((hwpoison_filter(p) && TestClearPageHWPoison(p))
1111 || (p != hpage && TestSetPageHWPoison(hpage))) {
Naoya Horiguchi8e304562015-09-08 15:03:24 -07001112 num_poisoned_pages_sub(nr_pages);
Chen Yucongb9851942014-05-22 11:54:15 -07001113 unlock_page(hpage);
1114 return 0;
1115 }
Naoya Horiguchi8c6c2ec2010-09-08 10:19:38 +09001116 }
1117 set_page_hwpoison_huge_page(hpage);
1118 res = dequeue_hwpoisoned_huge_page(hpage);
Xie XiuQicc637b12015-06-24 16:57:30 -07001119 action_result(pfn, MF_MSG_FREE_HUGE,
1120 res ? MF_IGNORED : MF_DELAYED);
Naoya Horiguchi8c6c2ec2010-09-08 10:19:38 +09001121 unlock_page(hpage);
1122 return res;
Wu Fengguang8d22ba12009-12-16 12:19:58 +01001123 } else {
Xie XiuQicc637b12015-06-24 16:57:30 -07001124 action_result(pfn, MF_MSG_KERNEL_HIGH_ORDER, MF_IGNORED);
Wu Fengguang8d22ba12009-12-16 12:19:58 +01001125 return -EBUSY;
1126 }
Andi Kleen6a460792009-09-16 11:50:15 +02001127 }
1128
Naoya Horiguchi415c64c2015-06-24 16:56:45 -07001129 if (!PageHuge(p) && PageTransHuge(hpage)) {
Naoya Horiguchic3901e72016-11-10 10:46:23 -08001130 lock_page(p);
1131 if (!PageAnon(p) || unlikely(split_huge_page(p))) {
1132 unlock_page(p);
1133 if (!PageAnon(p))
Chen Yucong495367c02016-05-20 16:57:32 -07001134 pr_err("Memory failure: %#lx: non anonymous thp\n",
1135 pfn);
Wanpeng Li7f6bf392015-08-14 15:35:08 -07001136 else
Chen Yucong495367c02016-05-20 16:57:32 -07001137 pr_err("Memory failure: %#lx: thp split failed\n",
1138 pfn);
Naoya Horiguchiead07f62015-06-24 16:56:48 -07001139 if (TestClearPageHWPoison(p))
Naoya Horiguchi8e304562015-09-08 15:03:24 -07001140 num_poisoned_pages_sub(nr_pages);
Wanpeng Li665d9da2015-09-08 15:03:21 -07001141 put_hwpoison_page(p);
Naoya Horiguchi415c64c2015-06-24 16:56:45 -07001142 return -EBUSY;
1143 }
Naoya Horiguchic3901e72016-11-10 10:46:23 -08001144 unlock_page(p);
Naoya Horiguchi415c64c2015-06-24 16:56:45 -07001145 VM_BUG_ON_PAGE(!page_count(p), p);
1146 hpage = compound_head(p);
1147 }
1148
Andi Kleen6a460792009-09-16 11:50:15 +02001149 /*
Wu Fengguange43c3af2009-09-29 13:16:20 +08001150 * We ignore non-LRU pages for good reasons.
1151 * - PG_locked is only well defined for LRU pages and a few others
Kirill A. Shutemov48c935a2016-01-15 16:51:24 -08001152 * - to avoid races with __SetPageLocked()
Wu Fengguange43c3af2009-09-29 13:16:20 +08001153 * - to avoid races with __SetPageSlab*() (and more non-atomic ops)
1154 * The check (unnecessarily) ignores LRU pages being isolated and
1155 * walked by the page reclaim code, however that's not a big loss.
1156 */
Naoya Horiguchi8bcb74d2017-05-03 14:56:19 -07001157 shake_page(p, 0);
1158 /* shake_page could have turned it free. */
1159 if (!PageLRU(p) && is_free_buddy_page(p)) {
1160 if (flags & MF_COUNT_INCREASED)
1161 action_result(pfn, MF_MSG_BUDDY, MF_DELAYED);
1162 else
1163 action_result(pfn, MF_MSG_BUDDY_2ND, MF_DELAYED);
1164 return 0;
Wu Fengguange43c3af2009-09-29 13:16:20 +08001165 }
Wu Fengguange43c3af2009-09-29 13:16:20 +08001166
Jens Axboe7eaceac2011-03-10 08:52:07 +01001167 lock_page(hpage);
Wu Fengguang847ce402009-12-16 12:19:58 +01001168
1169 /*
Andi Kleenf37d4292014-08-06 16:06:49 -07001170 * The page could have changed compound pages during the locking.
1171 * If this happens just bail out.
1172 */
Naoya Horiguchi415c64c2015-06-24 16:56:45 -07001173 if (PageCompound(p) && compound_head(p) != orig_head) {
Xie XiuQicc637b12015-06-24 16:57:30 -07001174 action_result(pfn, MF_MSG_DIFFERENT_COMPOUND, MF_IGNORED);
Andi Kleenf37d4292014-08-06 16:06:49 -07001175 res = -EBUSY;
1176 goto out;
1177 }
1178
1179 /*
Naoya Horiguchi524fca12013-02-22 16:35:51 -08001180 * We use page flags to determine what action should be taken, but
1181 * the flags can be modified by the error containment action. One
1182 * example is an mlocked page, where PG_mlocked is cleared by
1183 * page_remove_rmap() in try_to_unmap_one(). So to determine page status
1184 * correctly, we save a copy of the page flags at this time.
1185 */
James Morse7258ae52017-06-16 14:02:29 -07001186 if (PageHuge(p))
1187 page_flags = hpage->flags;
1188 else
1189 page_flags = p->flags;
Naoya Horiguchi524fca12013-02-22 16:35:51 -08001190
1191 /*
Wu Fengguang847ce402009-12-16 12:19:58 +01001192 * unpoison always clear PG_hwpoison inside page lock
1193 */
1194 if (!PageHWPoison(p)) {
Chen Yucong495367c02016-05-20 16:57:32 -07001195 pr_err("Memory failure: %#lx: just unpoisoned\n", pfn);
Naoya Horiguchi8e304562015-09-08 15:03:24 -07001196 num_poisoned_pages_sub(nr_pages);
Naoya Horiguchia09233f2015-08-06 15:46:58 -07001197 unlock_page(hpage);
Wanpeng Li665d9da2015-09-08 15:03:21 -07001198 put_hwpoison_page(hpage);
Naoya Horiguchia09233f2015-08-06 15:46:58 -07001199 return 0;
Wu Fengguang847ce402009-12-16 12:19:58 +01001200 }
Wu Fengguang7c116f22009-12-16 12:19:59 +01001201 if (hwpoison_filter(p)) {
1202 if (TestClearPageHWPoison(p))
Naoya Horiguchi8e304562015-09-08 15:03:24 -07001203 num_poisoned_pages_sub(nr_pages);
Naoya Horiguchi7af446a2010-05-28 09:29:17 +09001204 unlock_page(hpage);
Wanpeng Li665d9da2015-09-08 15:03:21 -07001205 put_hwpoison_page(hpage);
Wu Fengguang7c116f22009-12-16 12:19:59 +01001206 return 0;
1207 }
Wu Fengguang847ce402009-12-16 12:19:58 +01001208
Chen Yucong0bc1f8b2014-07-02 15:22:37 -07001209 if (!PageHuge(p) && !PageTransTail(p) && !PageLRU(p))
1210 goto identify_page_state;
1211
Naoya Horiguchi7013feb2010-05-28 09:29:18 +09001212 /*
1213 * For error on the tail page, we should set PG_hwpoison
1214 * on the head page to show that the hugepage is hwpoisoned
1215 */
Jin Dongminga6d30dd2011-02-01 15:52:40 -08001216 if (PageHuge(p) && PageTail(p) && TestSetPageHWPoison(hpage)) {
Xie XiuQicc637b12015-06-24 16:57:30 -07001217 action_result(pfn, MF_MSG_POISONED_HUGE, MF_IGNORED);
Naoya Horiguchi7013feb2010-05-28 09:29:18 +09001218 unlock_page(hpage);
Wanpeng Li665d9da2015-09-08 15:03:21 -07001219 put_hwpoison_page(hpage);
Naoya Horiguchi7013feb2010-05-28 09:29:18 +09001220 return 0;
1221 }
1222 /*
1223 * Set PG_hwpoison on all pages in an error hugepage,
1224 * because containment is done in hugepage unit for now.
1225 * Since we have done TestSetPageHWPoison() for the head page with
1226 * page lock held, we can safely set PG_hwpoison bits on tail pages.
1227 */
1228 if (PageHuge(p))
1229 set_page_hwpoison_huge_page(hpage);
1230
Naoya Horiguchi6edd6cc2014-06-04 16:10:35 -07001231 /*
1232 * It's very difficult to mess with pages currently under IO
1233 * and in many cases impossible, so we just avoid it here.
1234 */
Andi Kleen6a460792009-09-16 11:50:15 +02001235 wait_on_page_writeback(p);
1236
1237 /*
1238 * Now take care of user space mappings.
Minchan Kime64a7822011-03-22 16:32:44 -07001239 * Abort on fail: __delete_from_page_cache() assumes unmapped page.
Naoya Horiguchi54b9dd12014-01-23 15:53:14 -08001240 *
1241 * When the raw error page is thp tail page, hpage points to the raw
1242 * page after thp split.
Andi Kleen6a460792009-09-16 11:50:15 +02001243 */
Minchan Kim666e5a42017-05-03 14:54:20 -07001244 if (!hwpoison_user_mappings(p, pfn, trapno, flags, &hpage)) {
Xie XiuQicc637b12015-06-24 16:57:30 -07001245 action_result(pfn, MF_MSG_UNMAP_FAILED, MF_IGNORED);
Wu Fengguang1668bfd2009-12-16 12:19:58 +01001246 res = -EBUSY;
1247 goto out;
1248 }
Andi Kleen6a460792009-09-16 11:50:15 +02001249
1250 /*
1251 * Torn down by someone else?
1252 */
Wu Fengguangdc2a1cb2009-12-16 12:19:58 +01001253 if (PageLRU(p) && !PageSwapCache(p) && p->mapping == NULL) {
Xie XiuQicc637b12015-06-24 16:57:30 -07001254 action_result(pfn, MF_MSG_TRUNCATED_LRU, MF_IGNORED);
Wu Fengguangd95ea512009-12-16 12:19:58 +01001255 res = -EBUSY;
Andi Kleen6a460792009-09-16 11:50:15 +02001256 goto out;
1257 }
1258
Chen Yucong0bc1f8b2014-07-02 15:22:37 -07001259identify_page_state:
Andi Kleen6a460792009-09-16 11:50:15 +02001260 res = -EBUSY;
Naoya Horiguchi524fca12013-02-22 16:35:51 -08001261 /*
1262 * The first check uses the current page flags which may not have any
1263 * relevant information. The second check with the saved page flagss is
1264 * carried out only if the first check can't determine the page status.
1265 */
1266 for (ps = error_states;; ps++)
1267 if ((p->flags & ps->mask) == ps->res)
Andi Kleen6a460792009-09-16 11:50:15 +02001268 break;
Wanpeng Li841fcc52013-09-11 14:22:50 -07001269
1270 page_flags |= (p->flags & (1UL << PG_dirty));
1271
Naoya Horiguchi524fca12013-02-22 16:35:51 -08001272 if (!ps->mask)
1273 for (ps = error_states;; ps++)
1274 if ((page_flags & ps->mask) == ps->res)
1275 break;
1276 res = page_action(ps, p, pfn);
Andi Kleen6a460792009-09-16 11:50:15 +02001277out:
Naoya Horiguchi7af446a2010-05-28 09:29:17 +09001278 unlock_page(hpage);
Andi Kleen6a460792009-09-16 11:50:15 +02001279 return res;
1280}
Tony Luckcd42f4a2011-12-15 10:48:12 -08001281EXPORT_SYMBOL_GPL(memory_failure);
Wu Fengguang847ce402009-12-16 12:19:58 +01001282
Huang Yingea8f5fb2011-07-13 13:14:27 +08001283#define MEMORY_FAILURE_FIFO_ORDER 4
1284#define MEMORY_FAILURE_FIFO_SIZE (1 << MEMORY_FAILURE_FIFO_ORDER)
1285
1286struct memory_failure_entry {
1287 unsigned long pfn;
1288 int trapno;
1289 int flags;
1290};
1291
1292struct memory_failure_cpu {
1293 DECLARE_KFIFO(fifo, struct memory_failure_entry,
1294 MEMORY_FAILURE_FIFO_SIZE);
1295 spinlock_t lock;
1296 struct work_struct work;
1297};
1298
1299static DEFINE_PER_CPU(struct memory_failure_cpu, memory_failure_cpu);
1300
1301/**
1302 * memory_failure_queue - Schedule handling memory failure of a page.
1303 * @pfn: Page Number of the corrupted page
1304 * @trapno: Trap number reported in the signal to user space.
1305 * @flags: Flags for memory failure handling
1306 *
1307 * This function is called by the low level hardware error handler
1308 * when it detects hardware memory corruption of a page. It schedules
1309 * the recovering of error page, including dropping pages, killing
1310 * processes etc.
1311 *
1312 * The function is primarily of use for corruptions that
1313 * happen outside the current execution context (e.g. when
1314 * detected by a background scrubber)
1315 *
1316 * Can run in IRQ context.
1317 */
1318void memory_failure_queue(unsigned long pfn, int trapno, int flags)
1319{
1320 struct memory_failure_cpu *mf_cpu;
1321 unsigned long proc_flags;
1322 struct memory_failure_entry entry = {
1323 .pfn = pfn,
1324 .trapno = trapno,
1325 .flags = flags,
1326 };
1327
1328 mf_cpu = &get_cpu_var(memory_failure_cpu);
1329 spin_lock_irqsave(&mf_cpu->lock, proc_flags);
Stefani Seibold498d3192013-11-14 14:32:17 -08001330 if (kfifo_put(&mf_cpu->fifo, entry))
Huang Yingea8f5fb2011-07-13 13:14:27 +08001331 schedule_work_on(smp_processor_id(), &mf_cpu->work);
1332 else
Joe Perches8e33a522013-07-25 11:53:25 -07001333 pr_err("Memory failure: buffer overflow when queuing memory failure at %#lx\n",
Huang Yingea8f5fb2011-07-13 13:14:27 +08001334 pfn);
1335 spin_unlock_irqrestore(&mf_cpu->lock, proc_flags);
1336 put_cpu_var(memory_failure_cpu);
1337}
1338EXPORT_SYMBOL_GPL(memory_failure_queue);
1339
1340static void memory_failure_work_func(struct work_struct *work)
1341{
1342 struct memory_failure_cpu *mf_cpu;
1343 struct memory_failure_entry entry = { 0, };
1344 unsigned long proc_flags;
1345 int gotten;
1346
Christoph Lameter7c8e0182014-06-04 16:07:56 -07001347 mf_cpu = this_cpu_ptr(&memory_failure_cpu);
Huang Yingea8f5fb2011-07-13 13:14:27 +08001348 for (;;) {
1349 spin_lock_irqsave(&mf_cpu->lock, proc_flags);
1350 gotten = kfifo_get(&mf_cpu->fifo, &entry);
1351 spin_unlock_irqrestore(&mf_cpu->lock, proc_flags);
1352 if (!gotten)
1353 break;
Naveen N. Raocf870c72013-07-10 14:57:01 +05301354 if (entry.flags & MF_SOFT_OFFLINE)
1355 soft_offline_page(pfn_to_page(entry.pfn), entry.flags);
1356 else
1357 memory_failure(entry.pfn, entry.trapno, entry.flags);
Huang Yingea8f5fb2011-07-13 13:14:27 +08001358 }
1359}
1360
1361static int __init memory_failure_init(void)
1362{
1363 struct memory_failure_cpu *mf_cpu;
1364 int cpu;
1365
1366 for_each_possible_cpu(cpu) {
1367 mf_cpu = &per_cpu(memory_failure_cpu, cpu);
1368 spin_lock_init(&mf_cpu->lock);
1369 INIT_KFIFO(mf_cpu->fifo);
1370 INIT_WORK(&mf_cpu->work, memory_failure_work_func);
1371 }
1372
1373 return 0;
1374}
1375core_initcall(memory_failure_init);
1376
Naoya Horiguchia5f65102015-11-05 18:47:26 -08001377#define unpoison_pr_info(fmt, pfn, rs) \
1378({ \
1379 if (__ratelimit(rs)) \
1380 pr_info(fmt, pfn); \
1381})
1382
Wu Fengguang847ce402009-12-16 12:19:58 +01001383/**
1384 * unpoison_memory - Unpoison a previously poisoned page
1385 * @pfn: Page number of the to be unpoisoned page
1386 *
1387 * Software-unpoison a page that has been poisoned by
1388 * memory_failure() earlier.
1389 *
1390 * This is only done on the software-level, so it only works
1391 * for linux injected failures, not real hardware failures
1392 *
1393 * Returns 0 for success, otherwise -errno.
1394 */
1395int unpoison_memory(unsigned long pfn)
1396{
1397 struct page *page;
1398 struct page *p;
1399 int freeit = 0;
Naoya Horiguchic9fbdd52010-05-28 09:29:19 +09001400 unsigned int nr_pages;
Naoya Horiguchia5f65102015-11-05 18:47:26 -08001401 static DEFINE_RATELIMIT_STATE(unpoison_rs, DEFAULT_RATELIMIT_INTERVAL,
1402 DEFAULT_RATELIMIT_BURST);
Wu Fengguang847ce402009-12-16 12:19:58 +01001403
1404 if (!pfn_valid(pfn))
1405 return -ENXIO;
1406
1407 p = pfn_to_page(pfn);
1408 page = compound_head(p);
1409
1410 if (!PageHWPoison(p)) {
Chen Yucong495367c02016-05-20 16:57:32 -07001411 unpoison_pr_info("Unpoison: Page was already unpoisoned %#lx\n",
Naoya Horiguchia5f65102015-11-05 18:47:26 -08001412 pfn, &unpoison_rs);
Wu Fengguang847ce402009-12-16 12:19:58 +01001413 return 0;
1414 }
1415
Naoya Horiguchi230ac712015-09-08 15:03:29 -07001416 if (page_count(page) > 1) {
Chen Yucong495367c02016-05-20 16:57:32 -07001417 unpoison_pr_info("Unpoison: Someone grabs the hwpoison page %#lx\n",
Naoya Horiguchia5f65102015-11-05 18:47:26 -08001418 pfn, &unpoison_rs);
Naoya Horiguchi230ac712015-09-08 15:03:29 -07001419 return 0;
1420 }
1421
1422 if (page_mapped(page)) {
Chen Yucong495367c02016-05-20 16:57:32 -07001423 unpoison_pr_info("Unpoison: Someone maps the hwpoison page %#lx\n",
Naoya Horiguchia5f65102015-11-05 18:47:26 -08001424 pfn, &unpoison_rs);
Naoya Horiguchi230ac712015-09-08 15:03:29 -07001425 return 0;
1426 }
1427
1428 if (page_mapping(page)) {
Chen Yucong495367c02016-05-20 16:57:32 -07001429 unpoison_pr_info("Unpoison: the hwpoison page has non-NULL mapping %#lx\n",
Naoya Horiguchia5f65102015-11-05 18:47:26 -08001430 pfn, &unpoison_rs);
Naoya Horiguchi230ac712015-09-08 15:03:29 -07001431 return 0;
1432 }
1433
Wanpeng Li0cea3fd2013-09-11 14:22:53 -07001434 /*
1435 * unpoison_memory() can encounter thp only when the thp is being
1436 * worked by memory_failure() and the page lock is not held yet.
1437 * In such case, we yield to memory_failure() and make unpoison fail.
1438 */
Wanpeng Lie76d30e2013-09-30 13:45:22 -07001439 if (!PageHuge(page) && PageTransHuge(page)) {
Chen Yucong495367c02016-05-20 16:57:32 -07001440 unpoison_pr_info("Unpoison: Memory failure is now running on %#lx\n",
Naoya Horiguchia5f65102015-11-05 18:47:26 -08001441 pfn, &unpoison_rs);
Naoya Horiguchiead07f62015-06-24 16:56:48 -07001442 return 0;
Wanpeng Li0cea3fd2013-09-11 14:22:53 -07001443 }
1444
Wanpeng Lif9121152013-09-11 14:22:52 -07001445 nr_pages = 1 << compound_order(page);
Naoya Horiguchic9fbdd52010-05-28 09:29:19 +09001446
Naoya Horiguchiead07f62015-06-24 16:56:48 -07001447 if (!get_hwpoison_page(p)) {
Naoya Horiguchi8c6c2ec2010-09-08 10:19:38 +09001448 /*
1449 * Since HWPoisoned hugepage should have non-zero refcount,
1450 * race between memory failure and unpoison seems to happen.
1451 * In such case unpoison fails and memory failure runs
1452 * to the end.
1453 */
1454 if (PageHuge(page)) {
Chen Yucong495367c02016-05-20 16:57:32 -07001455 unpoison_pr_info("Unpoison: Memory failure is now running on free hugepage %#lx\n",
Naoya Horiguchia5f65102015-11-05 18:47:26 -08001456 pfn, &unpoison_rs);
Naoya Horiguchi8c6c2ec2010-09-08 10:19:38 +09001457 return 0;
1458 }
Wu Fengguang847ce402009-12-16 12:19:58 +01001459 if (TestClearPageHWPoison(p))
Naoya Horiguchi8e304562015-09-08 15:03:24 -07001460 num_poisoned_pages_dec();
Chen Yucong495367c02016-05-20 16:57:32 -07001461 unpoison_pr_info("Unpoison: Software-unpoisoned free page %#lx\n",
Naoya Horiguchia5f65102015-11-05 18:47:26 -08001462 pfn, &unpoison_rs);
Wu Fengguang847ce402009-12-16 12:19:58 +01001463 return 0;
1464 }
1465
Jens Axboe7eaceac2011-03-10 08:52:07 +01001466 lock_page(page);
Wu Fengguang847ce402009-12-16 12:19:58 +01001467 /*
1468 * This test is racy because PG_hwpoison is set outside of page lock.
1469 * That's acceptable because that won't trigger kernel panic. Instead,
1470 * the PG_hwpoison page will be caught and isolated on the entrance to
1471 * the free buddy page pool.
1472 */
Naoya Horiguchic9fbdd52010-05-28 09:29:19 +09001473 if (TestClearPageHWPoison(page)) {
Chen Yucong495367c02016-05-20 16:57:32 -07001474 unpoison_pr_info("Unpoison: Software-unpoisoned page %#lx\n",
Naoya Horiguchia5f65102015-11-05 18:47:26 -08001475 pfn, &unpoison_rs);
Naoya Horiguchi8e304562015-09-08 15:03:24 -07001476 num_poisoned_pages_sub(nr_pages);
Wu Fengguang847ce402009-12-16 12:19:58 +01001477 freeit = 1;
Naoya Horiguchi6a901812010-09-08 10:19:40 +09001478 if (PageHuge(page))
1479 clear_page_hwpoison_huge_page(page);
Wu Fengguang847ce402009-12-16 12:19:58 +01001480 }
1481 unlock_page(page);
1482
Wanpeng Li665d9da2015-09-08 15:03:21 -07001483 put_hwpoison_page(page);
Wanpeng Li3ba5eeb2013-09-11 14:23:01 -07001484 if (freeit && !(pfn == my_zero_pfn(0) && page_count(p) == 1))
Wanpeng Li665d9da2015-09-08 15:03:21 -07001485 put_hwpoison_page(page);
Wu Fengguang847ce402009-12-16 12:19:58 +01001486
1487 return 0;
1488}
1489EXPORT_SYMBOL(unpoison_memory);
Andi Kleenfacb6012009-12-16 12:20:00 +01001490
1491static struct page *new_page(struct page *p, unsigned long private, int **x)
1492{
Andi Kleen12686d12009-12-16 12:20:01 +01001493 int nid = page_to_nid(p);
Anshuman Khandual94310cb2017-07-06 15:38:38 -07001494 if (PageHuge(p)) {
1495 struct hstate *hstate = page_hstate(compound_head(p));
1496
1497 if (hstate_is_gigantic(hstate))
1498 return alloc_huge_page_node(hstate, NUMA_NO_NODE);
1499
1500 return alloc_huge_page_node(hstate, nid);
1501 } else {
Vlastimil Babka96db8002015-09-08 15:03:50 -07001502 return __alloc_pages_node(nid, GFP_HIGHUSER_MOVABLE, 0);
Anshuman Khandual94310cb2017-07-06 15:38:38 -07001503 }
Andi Kleenfacb6012009-12-16 12:20:00 +01001504}
1505
1506/*
1507 * Safely get reference count of an arbitrary page.
1508 * Returns 0 for a free page, -EIO for a zero refcount page
1509 * that is not free, and 1 for any other page type.
1510 * For 1 the page is returned with increased page count, otherwise not.
1511 */
Naoya Horiguchiaf8fae72013-02-22 16:34:03 -08001512static int __get_any_page(struct page *p, unsigned long pfn, int flags)
Andi Kleenfacb6012009-12-16 12:20:00 +01001513{
1514 int ret;
1515
1516 if (flags & MF_COUNT_INCREASED)
1517 return 1;
1518
1519 /*
Naoya Horiguchid950b952010-09-08 10:19:39 +09001520 * When the target page is a free hugepage, just remove it
1521 * from free hugepage list.
1522 */
Naoya Horiguchiead07f62015-06-24 16:56:48 -07001523 if (!get_hwpoison_page(p)) {
Naoya Horiguchid950b952010-09-08 10:19:39 +09001524 if (PageHuge(p)) {
Borislav Petkov71dd0b82012-05-29 15:06:16 -07001525 pr_info("%s: %#lx free huge page\n", __func__, pfn);
Naoya Horiguchiaf8fae72013-02-22 16:34:03 -08001526 ret = 0;
Naoya Horiguchid950b952010-09-08 10:19:39 +09001527 } else if (is_free_buddy_page(p)) {
Borislav Petkov71dd0b82012-05-29 15:06:16 -07001528 pr_info("%s: %#lx free buddy page\n", __func__, pfn);
Andi Kleenfacb6012009-12-16 12:20:00 +01001529 ret = 0;
1530 } else {
Borislav Petkov71dd0b82012-05-29 15:06:16 -07001531 pr_info("%s: %#lx: unknown zero refcount page type %lx\n",
1532 __func__, pfn, p->flags);
Andi Kleenfacb6012009-12-16 12:20:00 +01001533 ret = -EIO;
1534 }
1535 } else {
1536 /* Not a free page */
1537 ret = 1;
1538 }
Andi Kleenfacb6012009-12-16 12:20:00 +01001539 return ret;
1540}
1541
Naoya Horiguchiaf8fae72013-02-22 16:34:03 -08001542static int get_any_page(struct page *page, unsigned long pfn, int flags)
1543{
1544 int ret = __get_any_page(page, pfn, flags);
1545
Yisheng Xie85fbe5d2017-02-24 14:57:35 -08001546 if (ret == 1 && !PageHuge(page) &&
1547 !PageLRU(page) && !__PageMovable(page)) {
Naoya Horiguchiaf8fae72013-02-22 16:34:03 -08001548 /*
1549 * Try to free it.
1550 */
Wanpeng Li665d9da2015-09-08 15:03:21 -07001551 put_hwpoison_page(page);
Naoya Horiguchiaf8fae72013-02-22 16:34:03 -08001552 shake_page(page, 1);
1553
1554 /*
1555 * Did it turn free?
1556 */
1557 ret = __get_any_page(page, pfn, 0);
Naoya Horiguchid96b3392016-01-15 16:54:03 -08001558 if (ret == 1 && !PageLRU(page)) {
Wanpeng Li4f32be62015-08-14 15:34:56 -07001559 /* Drop page reference which is from __get_any_page() */
Wanpeng Li665d9da2015-09-08 15:03:21 -07001560 put_hwpoison_page(page);
Anshuman Khandual82a24812017-05-03 14:55:31 -07001561 pr_info("soft_offline: %#lx: unknown non LRU page type %lx (%pGp)\n",
1562 pfn, page->flags, &page->flags);
Naoya Horiguchiaf8fae72013-02-22 16:34:03 -08001563 return -EIO;
1564 }
1565 }
1566 return ret;
1567}
1568
Naoya Horiguchid950b952010-09-08 10:19:39 +09001569static int soft_offline_huge_page(struct page *page, int flags)
1570{
1571 int ret;
1572 unsigned long pfn = page_to_pfn(page);
1573 struct page *hpage = compound_head(page);
Naoya Horiguchib8ec1ce2013-09-11 14:22:01 -07001574 LIST_HEAD(pagelist);
Naoya Horiguchid950b952010-09-08 10:19:39 +09001575
Naoya Horiguchiaf8fae72013-02-22 16:34:03 -08001576 /*
1577 * This double-check of PageHWPoison is to avoid the race with
1578 * memory_failure(). See also comment in __soft_offline_page().
1579 */
1580 lock_page(hpage);
Xishi Qiu0ebff322013-02-22 16:33:59 -08001581 if (PageHWPoison(hpage)) {
Naoya Horiguchiaf8fae72013-02-22 16:34:03 -08001582 unlock_page(hpage);
Wanpeng Li665d9da2015-09-08 15:03:21 -07001583 put_hwpoison_page(hpage);
Xishi Qiu0ebff322013-02-22 16:33:59 -08001584 pr_info("soft offline: %#lx hugepage already poisoned\n", pfn);
Naoya Horiguchiaf8fae72013-02-22 16:34:03 -08001585 return -EBUSY;
Xishi Qiu0ebff322013-02-22 16:33:59 -08001586 }
Naoya Horiguchiaf8fae72013-02-22 16:34:03 -08001587 unlock_page(hpage);
Naoya Horiguchid950b952010-09-08 10:19:39 +09001588
Naoya Horiguchibcc54222015-04-15 16:14:38 -07001589 ret = isolate_huge_page(hpage, &pagelist);
Wanpeng Li03613802015-08-14 15:34:59 -07001590 /*
1591 * get_any_page() and isolate_huge_page() takes a refcount each,
1592 * so need to drop one here.
1593 */
Wanpeng Li665d9da2015-09-08 15:03:21 -07001594 put_hwpoison_page(hpage);
Wanpeng Li03613802015-08-14 15:34:59 -07001595 if (!ret) {
Naoya Horiguchibcc54222015-04-15 16:14:38 -07001596 pr_info("soft offline: %#lx hugepage failed to isolate\n", pfn);
1597 return -EBUSY;
1598 }
1599
David Rientjes68711a72014-06-04 16:08:25 -07001600 ret = migrate_pages(&pagelist, new_page, NULL, MPOL_MF_MOVE_ALL,
Naoya Horiguchib8ec1ce2013-09-11 14:22:01 -07001601 MIGRATE_SYNC, MR_MEMORY_FAILURE);
Naoya Horiguchid950b952010-09-08 10:19:39 +09001602 if (ret) {
Anshuman Khandual82a24812017-05-03 14:55:31 -07001603 pr_info("soft offline: %#lx: migration failed %d, type %lx (%pGp)\n",
1604 pfn, ret, page->flags, &page->flags);
Punit Agrawal30809f52017-06-02 14:46:40 -07001605 if (!list_empty(&pagelist))
1606 putback_movable_pages(&pagelist);
Naoya Horiguchib8ec1ce2013-09-11 14:22:01 -07001607 if (ret > 0)
1608 ret = -EIO;
Naoya Horiguchiaf8fae72013-02-22 16:34:03 -08001609 } else {
Jianguo Wua49ecbc2013-12-18 17:08:54 -08001610 /* overcommit hugetlb page will be freed to buddy */
1611 if (PageHuge(page)) {
1612 set_page_hwpoison_huge_page(hpage);
1613 dequeue_hwpoisoned_huge_page(hpage);
Naoya Horiguchi8e304562015-09-08 15:03:24 -07001614 num_poisoned_pages_add(1 << compound_order(hpage));
Jianguo Wua49ecbc2013-12-18 17:08:54 -08001615 } else {
1616 SetPageHWPoison(page);
Naoya Horiguchi8e304562015-09-08 15:03:24 -07001617 num_poisoned_pages_inc();
Jianguo Wua49ecbc2013-12-18 17:08:54 -08001618 }
Naoya Horiguchid950b952010-09-08 10:19:39 +09001619 }
Naoya Horiguchid950b952010-09-08 10:19:39 +09001620 return ret;
1621}
1622
Naoya Horiguchiaf8fae72013-02-22 16:34:03 -08001623static int __soft_offline_page(struct page *page, int flags)
1624{
1625 int ret;
1626 unsigned long pfn = page_to_pfn(page);
Andi Kleenfacb6012009-12-16 12:20:00 +01001627
1628 /*
Naoya Horiguchiaf8fae72013-02-22 16:34:03 -08001629 * Check PageHWPoison again inside page lock because PageHWPoison
1630 * is set by memory_failure() outside page lock. Note that
1631 * memory_failure() also double-checks PageHWPoison inside page lock,
1632 * so there's no race between soft_offline_page() and memory_failure().
Andi Kleenfacb6012009-12-16 12:20:00 +01001633 */
Xishi Qiu0ebff322013-02-22 16:33:59 -08001634 lock_page(page);
1635 wait_on_page_writeback(page);
Naoya Horiguchiaf8fae72013-02-22 16:34:03 -08001636 if (PageHWPoison(page)) {
1637 unlock_page(page);
Wanpeng Li665d9da2015-09-08 15:03:21 -07001638 put_hwpoison_page(page);
Naoya Horiguchiaf8fae72013-02-22 16:34:03 -08001639 pr_info("soft offline: %#lx page already poisoned\n", pfn);
1640 return -EBUSY;
1641 }
Andi Kleenfacb6012009-12-16 12:20:00 +01001642 /*
1643 * Try to invalidate first. This should work for
1644 * non dirty unmapped page cache pages.
1645 */
1646 ret = invalidate_inode_page(page);
1647 unlock_page(page);
Andi Kleenfacb6012009-12-16 12:20:00 +01001648 /*
Andi Kleenfacb6012009-12-16 12:20:00 +01001649 * RED-PEN would be better to keep it isolated here, but we
1650 * would need to fix isolation locking first.
1651 */
Andi Kleenfacb6012009-12-16 12:20:00 +01001652 if (ret == 1) {
Wanpeng Li665d9da2015-09-08 15:03:21 -07001653 put_hwpoison_page(page);
Andi Kleenfb46e732010-09-27 23:31:30 +02001654 pr_info("soft_offline: %#lx: invalidated\n", pfn);
Naoya Horiguchiaf8fae72013-02-22 16:34:03 -08001655 SetPageHWPoison(page);
Naoya Horiguchi8e304562015-09-08 15:03:24 -07001656 num_poisoned_pages_inc();
Naoya Horiguchiaf8fae72013-02-22 16:34:03 -08001657 return 0;
Andi Kleenfacb6012009-12-16 12:20:00 +01001658 }
1659
1660 /*
1661 * Simple invalidation didn't work.
1662 * Try to migrate to a new page instead. migrate.c
1663 * handles a large number of cases for us.
1664 */
Yisheng Xie85fbe5d2017-02-24 14:57:35 -08001665 if (PageLRU(page))
1666 ret = isolate_lru_page(page);
1667 else
1668 ret = isolate_movable_page(page, ISOLATE_UNEVICTABLE);
Konstantin Khlebnikovbd486282011-05-24 17:12:20 -07001669 /*
1670 * Drop page reference which is came from get_any_page()
1671 * successful isolate_lru_page() already took another one.
1672 */
Wanpeng Li665d9da2015-09-08 15:03:21 -07001673 put_hwpoison_page(page);
Andi Kleenfacb6012009-12-16 12:20:00 +01001674 if (!ret) {
1675 LIST_HEAD(pagelist);
Yisheng Xie85fbe5d2017-02-24 14:57:35 -08001676 /*
1677 * After isolated lru page, the PageLRU will be cleared,
1678 * so use !__PageMovable instead for LRU page's mapping
1679 * cannot have PAGE_MAPPING_MOVABLE.
1680 */
1681 if (!__PageMovable(page))
1682 inc_node_page_state(page, NR_ISOLATED_ANON +
1683 page_is_file_cache(page));
Andi Kleenfacb6012009-12-16 12:20:00 +01001684 list_add(&page->lru, &pagelist);
David Rientjes68711a72014-06-04 16:08:25 -07001685 ret = migrate_pages(&pagelist, new_page, NULL, MPOL_MF_MOVE_ALL,
Hugh Dickins9c620e22013-02-22 16:35:14 -08001686 MIGRATE_SYNC, MR_MEMORY_FAILURE);
Andi Kleenfacb6012009-12-16 12:20:00 +01001687 if (ret) {
Yisheng Xie85fbe5d2017-02-24 14:57:35 -08001688 if (!list_empty(&pagelist))
1689 putback_movable_pages(&pagelist);
Joonsoo Kim59c82b72014-01-21 15:51:17 -08001690
Anshuman Khandual82a24812017-05-03 14:55:31 -07001691 pr_info("soft offline: %#lx: migration failed %d, type %lx (%pGp)\n",
1692 pfn, ret, page->flags, &page->flags);
Andi Kleenfacb6012009-12-16 12:20:00 +01001693 if (ret > 0)
1694 ret = -EIO;
1695 }
1696 } else {
Anshuman Khandual82a24812017-05-03 14:55:31 -07001697 pr_info("soft offline: %#lx: isolation failed: %d, page count %d, type %lx (%pGp)\n",
1698 pfn, ret, page_count(page), page->flags, &page->flags);
Andi Kleenfacb6012009-12-16 12:20:00 +01001699 }
Andi Kleenfacb6012009-12-16 12:20:00 +01001700 return ret;
1701}
Wanpeng Li86e05772013-09-11 14:22:56 -07001702
Naoya Horiguchiacc14dc2016-01-15 16:57:43 -08001703static int soft_offline_in_use_page(struct page *page, int flags)
1704{
1705 int ret;
1706 struct page *hpage = compound_head(page);
1707
1708 if (!PageHuge(page) && PageTransHuge(hpage)) {
1709 lock_page(hpage);
Naoya Horiguchi98fd1ef2016-01-15 16:57:46 -08001710 if (!PageAnon(hpage) || unlikely(split_huge_page(hpage))) {
1711 unlock_page(hpage);
1712 if (!PageAnon(hpage))
1713 pr_info("soft offline: %#lx: non anonymous thp\n", page_to_pfn(page));
1714 else
1715 pr_info("soft offline: %#lx: thp split failed\n", page_to_pfn(page));
1716 put_hwpoison_page(hpage);
Naoya Horiguchiacc14dc2016-01-15 16:57:43 -08001717 return -EBUSY;
1718 }
Naoya Horiguchi98fd1ef2016-01-15 16:57:46 -08001719 unlock_page(hpage);
Naoya Horiguchiacc14dc2016-01-15 16:57:43 -08001720 get_hwpoison_page(page);
1721 put_hwpoison_page(hpage);
1722 }
1723
1724 if (PageHuge(page))
1725 ret = soft_offline_huge_page(page, flags);
1726 else
1727 ret = __soft_offline_page(page, flags);
1728
1729 return ret;
1730}
1731
1732static void soft_offline_free_page(struct page *page)
1733{
1734 if (PageHuge(page)) {
1735 struct page *hpage = compound_head(page);
1736
1737 set_page_hwpoison_huge_page(hpage);
1738 if (!dequeue_hwpoisoned_huge_page(hpage))
1739 num_poisoned_pages_add(1 << compound_order(hpage));
1740 } else {
1741 if (!TestSetPageHWPoison(page))
1742 num_poisoned_pages_inc();
1743 }
1744}
1745
Wanpeng Li86e05772013-09-11 14:22:56 -07001746/**
1747 * soft_offline_page - Soft offline a page.
1748 * @page: page to offline
1749 * @flags: flags. Same as memory_failure().
1750 *
1751 * Returns 0 on success, otherwise negated errno.
1752 *
1753 * Soft offline a page, by migration or invalidation,
1754 * without killing anything. This is for the case when
1755 * a page is not corrupted yet (so it's still valid to access),
1756 * but has had a number of corrected errors and is better taken
1757 * out.
1758 *
1759 * The actual policy on when to do that is maintained by
1760 * user space.
1761 *
1762 * This should never impact any application or cause data loss,
1763 * however it might take some time.
1764 *
1765 * This is not a 100% solution for all memory, but tries to be
1766 * ``good enough'' for the majority of memory.
1767 */
1768int soft_offline_page(struct page *page, int flags)
1769{
1770 int ret;
1771 unsigned long pfn = page_to_pfn(page);
Wanpeng Li86e05772013-09-11 14:22:56 -07001772
1773 if (PageHWPoison(page)) {
1774 pr_info("soft offline: %#lx page already poisoned\n", pfn);
Wanpeng Li1e0e6352015-09-08 15:03:13 -07001775 if (flags & MF_COUNT_INCREASED)
Wanpeng Li665d9da2015-09-08 15:03:21 -07001776 put_hwpoison_page(page);
Wanpeng Li86e05772013-09-11 14:22:56 -07001777 return -EBUSY;
1778 }
Wanpeng Li86e05772013-09-11 14:22:56 -07001779
Vladimir Davydovbfc8c902014-06-04 16:07:18 -07001780 get_online_mems();
Wanpeng Li86e05772013-09-11 14:22:56 -07001781 ret = get_any_page(page, pfn, flags);
Vladimir Davydovbfc8c902014-06-04 16:07:18 -07001782 put_online_mems();
Naoya Horiguchi4e41a302016-01-15 16:54:07 -08001783
Naoya Horiguchiacc14dc2016-01-15 16:57:43 -08001784 if (ret > 0)
1785 ret = soft_offline_in_use_page(page, flags);
1786 else if (ret == 0)
1787 soft_offline_free_page(page);
Naoya Horiguchi4e41a302016-01-15 16:54:07 -08001788
Wanpeng Li86e05772013-09-11 14:22:56 -07001789 return ret;
1790}