blob: 92df149c88a875b61b2d3b2b1c5930ba698e83af [file] [log] [blame]
Russell Kingd111e8f2006-09-27 15:27:33 +01001/*
2 * linux/arch/arm/mm/mmu.c
3 *
4 * Copyright (C) 1995-2005 Russell King
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 */
Russell Kingae8f1542006-09-27 15:38:34 +010010#include <linux/module.h>
Russell Kingd111e8f2006-09-27 15:27:33 +010011#include <linux/kernel.h>
12#include <linux/errno.h>
13#include <linux/init.h>
Russell Kingd111e8f2006-09-27 15:27:33 +010014#include <linux/mman.h>
15#include <linux/nodemask.h>
Russell King2778f622010-07-09 16:27:52 +010016#include <linux/memblock.h>
Catalin Marinasd9073872010-09-13 16:01:24 +010017#include <linux/fs.h>
Nicolas Pitre0536bdf2011-08-25 00:35:59 -040018#include <linux/vmalloc.h>
Alessandro Rubini158e8bf2012-06-24 12:46:26 +010019#include <linux/sizes.h>
Russell Kingd111e8f2006-09-27 15:27:33 +010020
Russell King15d07dc2012-03-28 18:30:01 +010021#include <asm/cp15.h>
Russell King0ba8b9b2008-08-10 18:08:10 +010022#include <asm/cputype.h>
Russell King37efe642008-12-01 11:53:07 +000023#include <asm/sections.h>
Nicolas Pitre3f973e22008-11-04 00:48:42 -050024#include <asm/cachetype.h>
Russell Kingebd49222013-10-24 08:12:39 +010025#include <asm/sections.h>
Russell Kingd111e8f2006-09-27 15:27:33 +010026#include <asm/setup.h>
Russell Kinge616c592009-09-27 20:55:43 +010027#include <asm/smp_plat.h>
Russell Kingd111e8f2006-09-27 15:27:33 +010028#include <asm/tlb.h>
Nicolas Pitred73cd422008-09-15 16:44:55 -040029#include <asm/highmem.h>
David Howells9f97da72012-03-28 18:30:01 +010030#include <asm/system_info.h>
Catalin Marinas247055a2010-09-13 16:03:21 +010031#include <asm/traps.h>
Santosh Shilimkara77e0c72013-07-31 12:44:46 -040032#include <asm/procinfo.h>
33#include <asm/memory.h>
Russell Kingd111e8f2006-09-27 15:27:33 +010034
35#include <asm/mach/arch.h>
36#include <asm/mach/map.h>
Rob Herringc2794432012-02-29 18:10:58 -060037#include <asm/mach/pci.h>
Russell Kingd111e8f2006-09-27 15:27:33 +010038
39#include "mm.h"
Joonsoo Kimde40614e2013-04-05 03:16:51 +010040#include "tcm.h"
Russell Kingd111e8f2006-09-27 15:27:33 +010041
Russell Kingd111e8f2006-09-27 15:27:33 +010042/*
43 * empty_zero_page is a special page that is used for
44 * zero-initialized data and COW.
45 */
46struct page *empty_zero_page;
Aneesh Kumar K.V3653f3a2008-04-29 08:11:12 -040047EXPORT_SYMBOL(empty_zero_page);
Russell Kingd111e8f2006-09-27 15:27:33 +010048
49/*
50 * The pmd table for the upper-most set of pages.
51 */
52pmd_t *top_pmd;
53
Russell Kingae8f1542006-09-27 15:38:34 +010054#define CPOLICY_UNCACHED 0
55#define CPOLICY_BUFFERED 1
56#define CPOLICY_WRITETHROUGH 2
57#define CPOLICY_WRITEBACK 3
58#define CPOLICY_WRITEALLOC 4
59
60static unsigned int cachepolicy __initdata = CPOLICY_WRITEBACK;
61static unsigned int ecc_mask __initdata = 0;
Imre_Deak44b18692007-02-11 13:45:13 +010062pgprot_t pgprot_user;
Russell Kingae8f1542006-09-27 15:38:34 +010063pgprot_t pgprot_kernel;
Christoffer Dallcc577c22013-01-20 18:28:04 -050064pgprot_t pgprot_hyp_device;
65pgprot_t pgprot_s2;
66pgprot_t pgprot_s2_device;
Russell Kingae8f1542006-09-27 15:38:34 +010067
Imre_Deak44b18692007-02-11 13:45:13 +010068EXPORT_SYMBOL(pgprot_user);
Russell Kingae8f1542006-09-27 15:38:34 +010069EXPORT_SYMBOL(pgprot_kernel);
70
71struct cachepolicy {
72 const char policy[16];
73 unsigned int cr_mask;
Catalin Marinas442e70c2011-09-05 17:51:56 +010074 pmdval_t pmd;
Russell Kingf6e33542010-11-16 00:22:09 +000075 pteval_t pte;
Christoffer Dallcc577c22013-01-20 18:28:04 -050076 pteval_t pte_s2;
Russell Kingae8f1542006-09-27 15:38:34 +010077};
78
Christoffer Dallcc577c22013-01-20 18:28:04 -050079#ifdef CONFIG_ARM_LPAE
80#define s2_policy(policy) policy
81#else
82#define s2_policy(policy) 0
83#endif
84
Russell Kingae8f1542006-09-27 15:38:34 +010085static struct cachepolicy cache_policies[] __initdata = {
86 {
87 .policy = "uncached",
88 .cr_mask = CR_W|CR_C,
89 .pmd = PMD_SECT_UNCACHED,
Russell Kingbb30f362008-09-06 20:04:59 +010090 .pte = L_PTE_MT_UNCACHED,
Christoffer Dallcc577c22013-01-20 18:28:04 -050091 .pte_s2 = s2_policy(L_PTE_S2_MT_UNCACHED),
Russell Kingae8f1542006-09-27 15:38:34 +010092 }, {
93 .policy = "buffered",
94 .cr_mask = CR_C,
95 .pmd = PMD_SECT_BUFFERED,
Russell Kingbb30f362008-09-06 20:04:59 +010096 .pte = L_PTE_MT_BUFFERABLE,
Christoffer Dallcc577c22013-01-20 18:28:04 -050097 .pte_s2 = s2_policy(L_PTE_S2_MT_UNCACHED),
Russell Kingae8f1542006-09-27 15:38:34 +010098 }, {
99 .policy = "writethrough",
100 .cr_mask = 0,
101 .pmd = PMD_SECT_WT,
Russell Kingbb30f362008-09-06 20:04:59 +0100102 .pte = L_PTE_MT_WRITETHROUGH,
Christoffer Dallcc577c22013-01-20 18:28:04 -0500103 .pte_s2 = s2_policy(L_PTE_S2_MT_WRITETHROUGH),
Russell Kingae8f1542006-09-27 15:38:34 +0100104 }, {
105 .policy = "writeback",
106 .cr_mask = 0,
107 .pmd = PMD_SECT_WB,
Russell Kingbb30f362008-09-06 20:04:59 +0100108 .pte = L_PTE_MT_WRITEBACK,
Christoffer Dallcc577c22013-01-20 18:28:04 -0500109 .pte_s2 = s2_policy(L_PTE_S2_MT_WRITEBACK),
Russell Kingae8f1542006-09-27 15:38:34 +0100110 }, {
111 .policy = "writealloc",
112 .cr_mask = 0,
113 .pmd = PMD_SECT_WBWA,
Russell Kingbb30f362008-09-06 20:04:59 +0100114 .pte = L_PTE_MT_WRITEALLOC,
Christoffer Dallcc577c22013-01-20 18:28:04 -0500115 .pte_s2 = s2_policy(L_PTE_S2_MT_WRITEBACK),
Russell Kingae8f1542006-09-27 15:38:34 +0100116 }
117};
118
Uwe Kleine-Königb849a602012-01-16 10:34:31 +0100119#ifdef CONFIG_CPU_CP15
Russell Kingae8f1542006-09-27 15:38:34 +0100120/*
Russell Kingca8f0b02014-05-27 20:34:28 +0100121 * Initialise the cache_policy variable with the initial state specified
122 * via the "pmd" value. This is used to ensure that on ARMv6 and later,
123 * the C code sets the page tables up with the same policy as the head
124 * assembly code, which avoids an illegal state where the TLBs can get
125 * confused. See comments in early_cachepolicy() for more information.
126 */
127void __init init_default_cache_policy(unsigned long pmd)
128{
129 int i;
130
131 pmd &= PMD_SECT_TEX(1) | PMD_SECT_BUFFERABLE | PMD_SECT_CACHEABLE;
132
133 for (i = 0; i < ARRAY_SIZE(cache_policies); i++)
134 if (cache_policies[i].pmd == pmd) {
135 cachepolicy = i;
136 break;
137 }
138
139 if (i == ARRAY_SIZE(cache_policies))
140 pr_err("ERROR: could not find cache policy\n");
141}
142
143/*
144 * These are useful for identifying cache coherency problems by allowing
145 * the cache or the cache and writebuffer to be turned off. (Note: the
146 * write buffer should not be on and the cache off).
Russell Kingae8f1542006-09-27 15:38:34 +0100147 */
Jeremy Kerr2b0d8c22010-01-11 23:17:34 +0100148static int __init early_cachepolicy(char *p)
Russell Kingae8f1542006-09-27 15:38:34 +0100149{
Russell Kingca8f0b02014-05-27 20:34:28 +0100150 int i, selected = -1;
Russell Kingae8f1542006-09-27 15:38:34 +0100151
152 for (i = 0; i < ARRAY_SIZE(cache_policies); i++) {
153 int len = strlen(cache_policies[i].policy);
154
Jeremy Kerr2b0d8c22010-01-11 23:17:34 +0100155 if (memcmp(p, cache_policies[i].policy, len) == 0) {
Russell Kingca8f0b02014-05-27 20:34:28 +0100156 selected = i;
Russell Kingae8f1542006-09-27 15:38:34 +0100157 break;
158 }
159 }
Russell Kingca8f0b02014-05-27 20:34:28 +0100160
161 if (selected == -1)
162 pr_err("ERROR: unknown or unsupported cache policy\n");
163
Russell King4b46d642009-11-01 17:44:24 +0000164 /*
165 * This restriction is partly to do with the way we boot; it is
166 * unpredictable to have memory mapped using two different sets of
167 * memory attributes (shared, type, and cache attribs). We can not
168 * change these attributes once the initial assembly has setup the
169 * page tables.
170 */
Russell Kingca8f0b02014-05-27 20:34:28 +0100171 if (cpu_architecture() >= CPU_ARCH_ARMv6 && selected != cachepolicy) {
172 pr_warn("Only cachepolicy=%s supported on ARMv6 and later\n",
173 cache_policies[cachepolicy].policy);
174 return 0;
Catalin Marinas11179d82007-07-20 11:42:24 +0100175 }
Russell Kingca8f0b02014-05-27 20:34:28 +0100176
177 if (selected != cachepolicy) {
178 unsigned long cr = __clear_cr(cache_policies[selected].cr_mask);
179 cachepolicy = selected;
180 flush_cache_all();
181 set_cr(cr);
182 }
Jeremy Kerr2b0d8c22010-01-11 23:17:34 +0100183 return 0;
Russell Kingae8f1542006-09-27 15:38:34 +0100184}
Jeremy Kerr2b0d8c22010-01-11 23:17:34 +0100185early_param("cachepolicy", early_cachepolicy);
Russell Kingae8f1542006-09-27 15:38:34 +0100186
Jeremy Kerr2b0d8c22010-01-11 23:17:34 +0100187static int __init early_nocache(char *__unused)
Russell Kingae8f1542006-09-27 15:38:34 +0100188{
189 char *p = "buffered";
190 printk(KERN_WARNING "nocache is deprecated; use cachepolicy=%s\n", p);
Jeremy Kerr2b0d8c22010-01-11 23:17:34 +0100191 early_cachepolicy(p);
192 return 0;
Russell Kingae8f1542006-09-27 15:38:34 +0100193}
Jeremy Kerr2b0d8c22010-01-11 23:17:34 +0100194early_param("nocache", early_nocache);
Russell Kingae8f1542006-09-27 15:38:34 +0100195
Jeremy Kerr2b0d8c22010-01-11 23:17:34 +0100196static int __init early_nowrite(char *__unused)
Russell Kingae8f1542006-09-27 15:38:34 +0100197{
198 char *p = "uncached";
199 printk(KERN_WARNING "nowb is deprecated; use cachepolicy=%s\n", p);
Jeremy Kerr2b0d8c22010-01-11 23:17:34 +0100200 early_cachepolicy(p);
201 return 0;
Russell Kingae8f1542006-09-27 15:38:34 +0100202}
Jeremy Kerr2b0d8c22010-01-11 23:17:34 +0100203early_param("nowb", early_nowrite);
Russell Kingae8f1542006-09-27 15:38:34 +0100204
Catalin Marinas1b6ba462011-11-22 17:30:29 +0000205#ifndef CONFIG_ARM_LPAE
Jeremy Kerr2b0d8c22010-01-11 23:17:34 +0100206static int __init early_ecc(char *p)
Russell Kingae8f1542006-09-27 15:38:34 +0100207{
Jeremy Kerr2b0d8c22010-01-11 23:17:34 +0100208 if (memcmp(p, "on", 2) == 0)
Russell Kingae8f1542006-09-27 15:38:34 +0100209 ecc_mask = PMD_PROTECTION;
Jeremy Kerr2b0d8c22010-01-11 23:17:34 +0100210 else if (memcmp(p, "off", 3) == 0)
Russell Kingae8f1542006-09-27 15:38:34 +0100211 ecc_mask = 0;
Jeremy Kerr2b0d8c22010-01-11 23:17:34 +0100212 return 0;
Russell Kingae8f1542006-09-27 15:38:34 +0100213}
Jeremy Kerr2b0d8c22010-01-11 23:17:34 +0100214early_param("ecc", early_ecc);
Catalin Marinas1b6ba462011-11-22 17:30:29 +0000215#endif
Russell Kingae8f1542006-09-27 15:38:34 +0100216
Uwe Kleine-Königb849a602012-01-16 10:34:31 +0100217#else /* ifdef CONFIG_CPU_CP15 */
218
219static int __init early_cachepolicy(char *p)
220{
221 pr_warning("cachepolicy kernel parameter not supported without cp15\n");
222}
223early_param("cachepolicy", early_cachepolicy);
224
225static int __init noalign_setup(char *__unused)
226{
227 pr_warning("noalign kernel parameter not supported without cp15\n");
228}
229__setup("noalign", noalign_setup);
230
231#endif /* ifdef CONFIG_CPU_CP15 / else */
232
Russell King36bb94b2010-11-16 08:40:36 +0000233#define PROT_PTE_DEVICE L_PTE_PRESENT|L_PTE_YOUNG|L_PTE_DIRTY|L_PTE_XN
Christoffer Dall4d9c5b82014-02-02 22:21:31 +0100234#define PROT_PTE_S2_DEVICE PROT_PTE_DEVICE
Russell Kingb1cce6b2008-11-04 10:52:28 +0000235#define PROT_SECT_DEVICE PMD_TYPE_SECT|PMD_SECT_AP_WRITE
Russell King0af92be2007-05-05 20:28:16 +0100236
Russell Kingb29e9f52007-04-21 10:47:29 +0100237static struct mem_type mem_types[] = {
Russell King0af92be2007-05-05 20:28:16 +0100238 [MT_DEVICE] = { /* Strongly ordered / ARMv6 shared device */
Russell Kingbb30f362008-09-06 20:04:59 +0100239 .prot_pte = PROT_PTE_DEVICE | L_PTE_MT_DEV_SHARED |
240 L_PTE_SHARED,
Christoffer Dall4d9c5b82014-02-02 22:21:31 +0100241 .prot_pte_s2 = s2_policy(PROT_PTE_S2_DEVICE) |
242 s2_policy(L_PTE_S2_MT_DEV_SHARED) |
243 L_PTE_SHARED,
Russell King0af92be2007-05-05 20:28:16 +0100244 .prot_l1 = PMD_TYPE_TABLE,
Russell Kingb1cce6b2008-11-04 10:52:28 +0000245 .prot_sect = PROT_SECT_DEVICE | PMD_SECT_S,
Russell King0af92be2007-05-05 20:28:16 +0100246 .domain = DOMAIN_IO,
247 },
248 [MT_DEVICE_NONSHARED] = { /* ARMv6 non-shared device */
Russell Kingbb30f362008-09-06 20:04:59 +0100249 .prot_pte = PROT_PTE_DEVICE | L_PTE_MT_DEV_NONSHARED,
Russell King0af92be2007-05-05 20:28:16 +0100250 .prot_l1 = PMD_TYPE_TABLE,
Russell Kingb1cce6b2008-11-04 10:52:28 +0000251 .prot_sect = PROT_SECT_DEVICE,
Russell King0af92be2007-05-05 20:28:16 +0100252 .domain = DOMAIN_IO,
253 },
254 [MT_DEVICE_CACHED] = { /* ioremap_cached */
Russell Kingbb30f362008-09-06 20:04:59 +0100255 .prot_pte = PROT_PTE_DEVICE | L_PTE_MT_DEV_CACHED,
Russell King0af92be2007-05-05 20:28:16 +0100256 .prot_l1 = PMD_TYPE_TABLE,
257 .prot_sect = PROT_SECT_DEVICE | PMD_SECT_WB,
258 .domain = DOMAIN_IO,
Rob Herringc2794432012-02-29 18:10:58 -0600259 },
Lennert Buytenhek1ad77a82008-09-05 13:17:11 +0100260 [MT_DEVICE_WC] = { /* ioremap_wc */
Russell Kingbb30f362008-09-06 20:04:59 +0100261 .prot_pte = PROT_PTE_DEVICE | L_PTE_MT_DEV_WC,
Russell King0af92be2007-05-05 20:28:16 +0100262 .prot_l1 = PMD_TYPE_TABLE,
Russell Kingb1cce6b2008-11-04 10:52:28 +0000263 .prot_sect = PROT_SECT_DEVICE,
Russell King0af92be2007-05-05 20:28:16 +0100264 .domain = DOMAIN_IO,
Russell Kingae8f1542006-09-27 15:38:34 +0100265 },
Russell Kingebb4c652008-11-09 11:18:36 +0000266 [MT_UNCACHED] = {
267 .prot_pte = PROT_PTE_DEVICE,
268 .prot_l1 = PMD_TYPE_TABLE,
269 .prot_sect = PMD_TYPE_SECT | PMD_SECT_XN,
270 .domain = DOMAIN_IO,
271 },
Russell Kingae8f1542006-09-27 15:38:34 +0100272 [MT_CACHECLEAN] = {
Russell King9ef79632007-05-05 20:03:35 +0100273 .prot_sect = PMD_TYPE_SECT | PMD_SECT_XN,
Russell Kingae8f1542006-09-27 15:38:34 +0100274 .domain = DOMAIN_KERNEL,
275 },
Catalin Marinas1b6ba462011-11-22 17:30:29 +0000276#ifndef CONFIG_ARM_LPAE
Russell Kingae8f1542006-09-27 15:38:34 +0100277 [MT_MINICLEAN] = {
Russell King9ef79632007-05-05 20:03:35 +0100278 .prot_sect = PMD_TYPE_SECT | PMD_SECT_XN | PMD_SECT_MINICACHE,
Russell Kingae8f1542006-09-27 15:38:34 +0100279 .domain = DOMAIN_KERNEL,
280 },
Catalin Marinas1b6ba462011-11-22 17:30:29 +0000281#endif
Russell Kingae8f1542006-09-27 15:38:34 +0100282 [MT_LOW_VECTORS] = {
283 .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
Russell King36bb94b2010-11-16 08:40:36 +0000284 L_PTE_RDONLY,
Russell Kingae8f1542006-09-27 15:38:34 +0100285 .prot_l1 = PMD_TYPE_TABLE,
286 .domain = DOMAIN_USER,
287 },
288 [MT_HIGH_VECTORS] = {
289 .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
Russell King36bb94b2010-11-16 08:40:36 +0000290 L_PTE_USER | L_PTE_RDONLY,
Russell Kingae8f1542006-09-27 15:38:34 +0100291 .prot_l1 = PMD_TYPE_TABLE,
292 .domain = DOMAIN_USER,
293 },
Russell King2e2c9de2013-10-24 10:26:40 +0100294 [MT_MEMORY_RWX] = {
Russell King36bb94b2010-11-16 08:40:36 +0000295 .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY,
Santosh Shilimkarf1a24812010-09-24 07:18:22 +0100296 .prot_l1 = PMD_TYPE_TABLE,
Russell King9ef79632007-05-05 20:03:35 +0100297 .prot_sect = PMD_TYPE_SECT | PMD_SECT_AP_WRITE,
Russell Kingae8f1542006-09-27 15:38:34 +0100298 .domain = DOMAIN_KERNEL,
299 },
Russell Kingebd49222013-10-24 08:12:39 +0100300 [MT_MEMORY_RW] = {
301 .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
302 L_PTE_XN,
303 .prot_l1 = PMD_TYPE_TABLE,
304 .prot_sect = PMD_TYPE_SECT | PMD_SECT_AP_WRITE,
305 .domain = DOMAIN_KERNEL,
306 },
Russell Kingae8f1542006-09-27 15:38:34 +0100307 [MT_ROM] = {
Russell King9ef79632007-05-05 20:03:35 +0100308 .prot_sect = PMD_TYPE_SECT,
Russell Kingae8f1542006-09-27 15:38:34 +0100309 .domain = DOMAIN_KERNEL,
310 },
Russell King2e2c9de2013-10-24 10:26:40 +0100311 [MT_MEMORY_RWX_NONCACHED] = {
Santosh Shilimkarf1a24812010-09-24 07:18:22 +0100312 .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
Russell King36bb94b2010-11-16 08:40:36 +0000313 L_PTE_MT_BUFFERABLE,
Santosh Shilimkarf1a24812010-09-24 07:18:22 +0100314 .prot_l1 = PMD_TYPE_TABLE,
Paul Walmsleye4707dd2009-03-12 20:11:43 +0100315 .prot_sect = PMD_TYPE_SECT | PMD_SECT_AP_WRITE,
316 .domain = DOMAIN_KERNEL,
317 },
Russell King2e2c9de2013-10-24 10:26:40 +0100318 [MT_MEMORY_RW_DTCM] = {
Linus Walleijf444fce2010-10-18 09:03:03 +0100319 .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
Russell King36bb94b2010-11-16 08:40:36 +0000320 L_PTE_XN,
Linus Walleijf444fce2010-10-18 09:03:03 +0100321 .prot_l1 = PMD_TYPE_TABLE,
322 .prot_sect = PMD_TYPE_SECT | PMD_SECT_XN,
323 .domain = DOMAIN_KERNEL,
Linus Walleijcb9d7702010-07-12 21:50:59 +0100324 },
Russell King2e2c9de2013-10-24 10:26:40 +0100325 [MT_MEMORY_RWX_ITCM] = {
Russell King36bb94b2010-11-16 08:40:36 +0000326 .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY,
Linus Walleijcb9d7702010-07-12 21:50:59 +0100327 .prot_l1 = PMD_TYPE_TABLE,
Linus Walleijf444fce2010-10-18 09:03:03 +0100328 .domain = DOMAIN_KERNEL,
Linus Walleijcb9d7702010-07-12 21:50:59 +0100329 },
Russell King2e2c9de2013-10-24 10:26:40 +0100330 [MT_MEMORY_RW_SO] = {
Santosh Shilimkar8fb54282011-06-28 12:42:56 -0700331 .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
Santosh Shilimkar93d5bf02013-01-17 07:18:04 +0100332 L_PTE_MT_UNCACHED | L_PTE_XN,
Santosh Shilimkar8fb54282011-06-28 12:42:56 -0700333 .prot_l1 = PMD_TYPE_TABLE,
334 .prot_sect = PMD_TYPE_SECT | PMD_SECT_AP_WRITE | PMD_SECT_S |
335 PMD_SECT_UNCACHED | PMD_SECT_XN,
336 .domain = DOMAIN_KERNEL,
337 },
Marek Szyprowskic7909502011-12-29 13:09:51 +0100338 [MT_MEMORY_DMA_READY] = {
Russell King71b55662013-11-25 12:01:03 +0000339 .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
340 L_PTE_XN,
Marek Szyprowskic7909502011-12-29 13:09:51 +0100341 .prot_l1 = PMD_TYPE_TABLE,
342 .domain = DOMAIN_KERNEL,
343 },
Russell Kingae8f1542006-09-27 15:38:34 +0100344};
345
Russell Kingb29e9f52007-04-21 10:47:29 +0100346const struct mem_type *get_mem_type(unsigned int type)
347{
348 return type < ARRAY_SIZE(mem_types) ? &mem_types[type] : NULL;
349}
Hiroshi DOYU69d3a842009-01-28 21:32:08 +0200350EXPORT_SYMBOL(get_mem_type);
Russell Kingb29e9f52007-04-21 10:47:29 +0100351
Laura Abbott75374ad2013-06-17 10:29:13 -0700352#define PTE_SET_FN(_name, pteop) \
353static int pte_set_##_name(pte_t *ptep, pgtable_t token, unsigned long addr, \
354 void *data) \
355{ \
356 pte_t pte = pteop(*ptep); \
357\
358 set_pte_ext(ptep, pte, 0); \
359 return 0; \
360} \
361
362#define SET_MEMORY_FN(_name, callback) \
363int set_memory_##_name(unsigned long addr, int numpages) \
364{ \
365 unsigned long start = addr; \
366 unsigned long size = PAGE_SIZE*numpages; \
367 unsigned end = start + size; \
368\
369 if (start < MODULES_VADDR || start >= MODULES_END) \
370 return -EINVAL;\
371\
372 if (end < MODULES_VADDR || end >= MODULES_END) \
373 return -EINVAL; \
374\
375 apply_to_page_range(&init_mm, start, size, callback, NULL); \
376 flush_tlb_kernel_range(start, end); \
377 return 0;\
378}
379
380PTE_SET_FN(ro, pte_wrprotect)
381PTE_SET_FN(rw, pte_mkwrite)
382PTE_SET_FN(x, pte_mkexec)
383PTE_SET_FN(nx, pte_mknexec)
384
385SET_MEMORY_FN(ro, pte_set_ro)
386SET_MEMORY_FN(rw, pte_set_rw)
387SET_MEMORY_FN(x, pte_set_x)
388SET_MEMORY_FN(nx, pte_set_nx)
389
Russell Kingae8f1542006-09-27 15:38:34 +0100390/*
391 * Adjust the PMD section entries according to the CPU in use.
392 */
393static void __init build_mem_type_table(void)
394{
395 struct cachepolicy *cp;
396 unsigned int cr = get_cr();
Catalin Marinas442e70c2011-09-05 17:51:56 +0100397 pteval_t user_pgprot, kern_pgprot, vecs_pgprot;
Christoffer Dallcc577c22013-01-20 18:28:04 -0500398 pteval_t hyp_device_pgprot, s2_pgprot, s2_device_pgprot;
Russell Kingae8f1542006-09-27 15:38:34 +0100399 int cpu_arch = cpu_architecture();
400 int i;
401
Catalin Marinas11179d82007-07-20 11:42:24 +0100402 if (cpu_arch < CPU_ARCH_ARMv6) {
Russell Kingae8f1542006-09-27 15:38:34 +0100403#if defined(CONFIG_CPU_DCACHE_DISABLE)
Catalin Marinas11179d82007-07-20 11:42:24 +0100404 if (cachepolicy > CPOLICY_BUFFERED)
405 cachepolicy = CPOLICY_BUFFERED;
Russell Kingae8f1542006-09-27 15:38:34 +0100406#elif defined(CONFIG_CPU_DCACHE_WRITETHROUGH)
Catalin Marinas11179d82007-07-20 11:42:24 +0100407 if (cachepolicy > CPOLICY_WRITETHROUGH)
408 cachepolicy = CPOLICY_WRITETHROUGH;
Russell Kingae8f1542006-09-27 15:38:34 +0100409#endif
Catalin Marinas11179d82007-07-20 11:42:24 +0100410 }
Russell Kingae8f1542006-09-27 15:38:34 +0100411 if (cpu_arch < CPU_ARCH_ARMv5) {
412 if (cachepolicy >= CPOLICY_WRITEALLOC)
413 cachepolicy = CPOLICY_WRITEBACK;
414 ecc_mask = 0;
415 }
Russell Kingca8f0b02014-05-27 20:34:28 +0100416
417 if (is_smp() && cachepolicy != CPOLICY_WRITEALLOC) {
418 pr_warn("Forcing write-allocate cache policy for SMP\n");
Russell Kingf00ec482010-09-04 10:47:48 +0100419 cachepolicy = CPOLICY_WRITEALLOC;
Russell Kingca8f0b02014-05-27 20:34:28 +0100420 }
Russell Kingae8f1542006-09-27 15:38:34 +0100421
422 /*
Russell Kingb1cce6b2008-11-04 10:52:28 +0000423 * Strip out features not present on earlier architectures.
424 * Pre-ARMv5 CPUs don't have TEX bits. Pre-ARMv6 CPUs or those
425 * without extended page tables don't have the 'Shared' bit.
Lennert Buytenhek1ad77a82008-09-05 13:17:11 +0100426 */
Russell Kingb1cce6b2008-11-04 10:52:28 +0000427 if (cpu_arch < CPU_ARCH_ARMv5)
428 for (i = 0; i < ARRAY_SIZE(mem_types); i++)
429 mem_types[i].prot_sect &= ~PMD_SECT_TEX(7);
430 if ((cpu_arch < CPU_ARCH_ARMv6 || !(cr & CR_XP)) && !cpu_is_xsc3())
431 for (i = 0; i < ARRAY_SIZE(mem_types); i++)
432 mem_types[i].prot_sect &= ~PMD_SECT_S;
Russell Kingae8f1542006-09-27 15:38:34 +0100433
434 /*
Russell Kingb1cce6b2008-11-04 10:52:28 +0000435 * ARMv5 and lower, bit 4 must be set for page tables (was: cache
436 * "update-able on write" bit on ARM610). However, Xscale and
437 * Xscale3 require this bit to be cleared.
Russell Kingae8f1542006-09-27 15:38:34 +0100438 */
Russell Kingb1cce6b2008-11-04 10:52:28 +0000439 if (cpu_is_xscale() || cpu_is_xsc3()) {
Russell King9ef79632007-05-05 20:03:35 +0100440 for (i = 0; i < ARRAY_SIZE(mem_types); i++) {
Russell Kingae8f1542006-09-27 15:38:34 +0100441 mem_types[i].prot_sect &= ~PMD_BIT4;
Russell King9ef79632007-05-05 20:03:35 +0100442 mem_types[i].prot_l1 &= ~PMD_BIT4;
443 }
444 } else if (cpu_arch < CPU_ARCH_ARMv6) {
445 for (i = 0; i < ARRAY_SIZE(mem_types); i++) {
Russell Kingae8f1542006-09-27 15:38:34 +0100446 if (mem_types[i].prot_l1)
447 mem_types[i].prot_l1 |= PMD_BIT4;
Russell King9ef79632007-05-05 20:03:35 +0100448 if (mem_types[i].prot_sect)
449 mem_types[i].prot_sect |= PMD_BIT4;
450 }
451 }
Russell Kingae8f1542006-09-27 15:38:34 +0100452
Russell Kingb1cce6b2008-11-04 10:52:28 +0000453 /*
454 * Mark the device areas according to the CPU/architecture.
455 */
456 if (cpu_is_xsc3() || (cpu_arch >= CPU_ARCH_ARMv6 && (cr & CR_XP))) {
457 if (!cpu_is_xsc3()) {
458 /*
459 * Mark device regions on ARMv6+ as execute-never
460 * to prevent speculative instruction fetches.
461 */
462 mem_types[MT_DEVICE].prot_sect |= PMD_SECT_XN;
463 mem_types[MT_DEVICE_NONSHARED].prot_sect |= PMD_SECT_XN;
464 mem_types[MT_DEVICE_CACHED].prot_sect |= PMD_SECT_XN;
465 mem_types[MT_DEVICE_WC].prot_sect |= PMD_SECT_XN;
Russell Kingebd49222013-10-24 08:12:39 +0100466
467 /* Also setup NX memory mapping */
468 mem_types[MT_MEMORY_RW].prot_sect |= PMD_SECT_XN;
Russell Kingb1cce6b2008-11-04 10:52:28 +0000469 }
470 if (cpu_arch >= CPU_ARCH_ARMv7 && (cr & CR_TRE)) {
471 /*
472 * For ARMv7 with TEX remapping,
473 * - shared device is SXCB=1100
474 * - nonshared device is SXCB=0100
475 * - write combine device mem is SXCB=0001
476 * (Uncached Normal memory)
477 */
478 mem_types[MT_DEVICE].prot_sect |= PMD_SECT_TEX(1);
479 mem_types[MT_DEVICE_NONSHARED].prot_sect |= PMD_SECT_TEX(1);
480 mem_types[MT_DEVICE_WC].prot_sect |= PMD_SECT_BUFFERABLE;
481 } else if (cpu_is_xsc3()) {
482 /*
483 * For Xscale3,
484 * - shared device is TEXCB=00101
485 * - nonshared device is TEXCB=01000
486 * - write combine device mem is TEXCB=00100
487 * (Inner/Outer Uncacheable in xsc3 parlance)
488 */
489 mem_types[MT_DEVICE].prot_sect |= PMD_SECT_TEX(1) | PMD_SECT_BUFFERED;
490 mem_types[MT_DEVICE_NONSHARED].prot_sect |= PMD_SECT_TEX(2);
491 mem_types[MT_DEVICE_WC].prot_sect |= PMD_SECT_TEX(1);
492 } else {
493 /*
494 * For ARMv6 and ARMv7 without TEX remapping,
495 * - shared device is TEXCB=00001
496 * - nonshared device is TEXCB=01000
497 * - write combine device mem is TEXCB=00100
498 * (Uncached Normal in ARMv6 parlance).
499 */
500 mem_types[MT_DEVICE].prot_sect |= PMD_SECT_BUFFERED;
501 mem_types[MT_DEVICE_NONSHARED].prot_sect |= PMD_SECT_TEX(2);
502 mem_types[MT_DEVICE_WC].prot_sect |= PMD_SECT_TEX(1);
503 }
504 } else {
505 /*
506 * On others, write combining is "Uncached/Buffered"
507 */
508 mem_types[MT_DEVICE_WC].prot_sect |= PMD_SECT_BUFFERABLE;
509 }
510
511 /*
512 * Now deal with the memory-type mappings
513 */
Russell Kingae8f1542006-09-27 15:38:34 +0100514 cp = &cache_policies[cachepolicy];
Russell Kingbb30f362008-09-06 20:04:59 +0100515 vecs_pgprot = kern_pgprot = user_pgprot = cp->pte;
Christoffer Dallcc577c22013-01-20 18:28:04 -0500516 s2_pgprot = cp->pte_s2;
Christoffer Dall4d9c5b82014-02-02 22:21:31 +0100517 hyp_device_pgprot = mem_types[MT_DEVICE].prot_pte;
518 s2_device_pgprot = mem_types[MT_DEVICE].prot_pte_s2;
Russell Kingbb30f362008-09-06 20:04:59 +0100519
Russell Kingbb30f362008-09-06 20:04:59 +0100520 /*
Will Deaconb6ccb982014-02-07 19:12:27 +0100521 * We don't use domains on ARMv6 (since this causes problems with
522 * v6/v7 kernels), so we must use a separate memory type for user
523 * r/o, kernel r/w to map the vectors page.
524 */
525#ifndef CONFIG_ARM_LPAE
526 if (cpu_arch == CPU_ARCH_ARMv6)
527 vecs_pgprot |= L_PTE_MT_VECTORS;
528#endif
Russell Kingbb30f362008-09-06 20:04:59 +0100529
530 /*
Russell Kingae8f1542006-09-27 15:38:34 +0100531 * ARMv6 and above have extended page tables.
532 */
533 if (cpu_arch >= CPU_ARCH_ARMv6 && (cr & CR_XP)) {
Catalin Marinas1b6ba462011-11-22 17:30:29 +0000534#ifndef CONFIG_ARM_LPAE
Russell Kingae8f1542006-09-27 15:38:34 +0100535 /*
Russell Kingae8f1542006-09-27 15:38:34 +0100536 * Mark cache clean areas and XIP ROM read only
537 * from SVC mode and no access from userspace.
538 */
539 mem_types[MT_ROM].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE;
540 mem_types[MT_MINICLEAN].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE;
541 mem_types[MT_CACHECLEAN].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE;
Catalin Marinas1b6ba462011-11-22 17:30:29 +0000542#endif
Russell Kingae8f1542006-09-27 15:38:34 +0100543
Russell Kingf00ec482010-09-04 10:47:48 +0100544 if (is_smp()) {
545 /*
546 * Mark memory with the "shared" attribute
547 * for SMP systems
548 */
549 user_pgprot |= L_PTE_SHARED;
550 kern_pgprot |= L_PTE_SHARED;
551 vecs_pgprot |= L_PTE_SHARED;
Christoffer Dallcc577c22013-01-20 18:28:04 -0500552 s2_pgprot |= L_PTE_SHARED;
Russell Kingf00ec482010-09-04 10:47:48 +0100553 mem_types[MT_DEVICE_WC].prot_sect |= PMD_SECT_S;
554 mem_types[MT_DEVICE_WC].prot_pte |= L_PTE_SHARED;
555 mem_types[MT_DEVICE_CACHED].prot_sect |= PMD_SECT_S;
556 mem_types[MT_DEVICE_CACHED].prot_pte |= L_PTE_SHARED;
Russell King2e2c9de2013-10-24 10:26:40 +0100557 mem_types[MT_MEMORY_RWX].prot_sect |= PMD_SECT_S;
558 mem_types[MT_MEMORY_RWX].prot_pte |= L_PTE_SHARED;
Russell Kingebd49222013-10-24 08:12:39 +0100559 mem_types[MT_MEMORY_RW].prot_sect |= PMD_SECT_S;
560 mem_types[MT_MEMORY_RW].prot_pte |= L_PTE_SHARED;
Marek Szyprowskic7909502011-12-29 13:09:51 +0100561 mem_types[MT_MEMORY_DMA_READY].prot_pte |= L_PTE_SHARED;
Russell King2e2c9de2013-10-24 10:26:40 +0100562 mem_types[MT_MEMORY_RWX_NONCACHED].prot_sect |= PMD_SECT_S;
563 mem_types[MT_MEMORY_RWX_NONCACHED].prot_pte |= L_PTE_SHARED;
Russell Kingf00ec482010-09-04 10:47:48 +0100564 }
Russell Kingae8f1542006-09-27 15:38:34 +0100565 }
566
Paul Walmsleye4707dd2009-03-12 20:11:43 +0100567 /*
568 * Non-cacheable Normal - intended for memory areas that must
569 * not cause dirty cache line writebacks when used
570 */
571 if (cpu_arch >= CPU_ARCH_ARMv6) {
572 if (cpu_arch >= CPU_ARCH_ARMv7 && (cr & CR_TRE)) {
573 /* Non-cacheable Normal is XCB = 001 */
Russell King2e2c9de2013-10-24 10:26:40 +0100574 mem_types[MT_MEMORY_RWX_NONCACHED].prot_sect |=
Paul Walmsleye4707dd2009-03-12 20:11:43 +0100575 PMD_SECT_BUFFERED;
576 } else {
577 /* For both ARMv6 and non-TEX-remapping ARMv7 */
Russell King2e2c9de2013-10-24 10:26:40 +0100578 mem_types[MT_MEMORY_RWX_NONCACHED].prot_sect |=
Paul Walmsleye4707dd2009-03-12 20:11:43 +0100579 PMD_SECT_TEX(1);
580 }
581 } else {
Russell King2e2c9de2013-10-24 10:26:40 +0100582 mem_types[MT_MEMORY_RWX_NONCACHED].prot_sect |= PMD_SECT_BUFFERABLE;
Paul Walmsleye4707dd2009-03-12 20:11:43 +0100583 }
584
Catalin Marinas1b6ba462011-11-22 17:30:29 +0000585#ifdef CONFIG_ARM_LPAE
586 /*
587 * Do not generate access flag faults for the kernel mappings.
588 */
589 for (i = 0; i < ARRAY_SIZE(mem_types); i++) {
590 mem_types[i].prot_pte |= PTE_EXT_AF;
Vitaly Andrianov1a3abcf2012-05-15 15:01:16 +0100591 if (mem_types[i].prot_sect)
592 mem_types[i].prot_sect |= PMD_SECT_AF;
Catalin Marinas1b6ba462011-11-22 17:30:29 +0000593 }
594 kern_pgprot |= PTE_EXT_AF;
595 vecs_pgprot |= PTE_EXT_AF;
596#endif
597
Russell Kingae8f1542006-09-27 15:38:34 +0100598 for (i = 0; i < 16; i++) {
Will Deacon864aa042012-09-18 19:18:35 +0100599 pteval_t v = pgprot_val(protection_map[i]);
Russell Kingbb30f362008-09-06 20:04:59 +0100600 protection_map[i] = __pgprot(v | user_pgprot);
Russell Kingae8f1542006-09-27 15:38:34 +0100601 }
602
Russell Kingbb30f362008-09-06 20:04:59 +0100603 mem_types[MT_LOW_VECTORS].prot_pte |= vecs_pgprot;
604 mem_types[MT_HIGH_VECTORS].prot_pte |= vecs_pgprot;
Russell Kingae8f1542006-09-27 15:38:34 +0100605
Imre_Deak44b18692007-02-11 13:45:13 +0100606 pgprot_user = __pgprot(L_PTE_PRESENT | L_PTE_YOUNG | user_pgprot);
Russell Kingae8f1542006-09-27 15:38:34 +0100607 pgprot_kernel = __pgprot(L_PTE_PRESENT | L_PTE_YOUNG |
Russell King36bb94b2010-11-16 08:40:36 +0000608 L_PTE_DIRTY | kern_pgprot);
Christoffer Dallcc577c22013-01-20 18:28:04 -0500609 pgprot_s2 = __pgprot(L_PTE_PRESENT | L_PTE_YOUNG | s2_pgprot);
610 pgprot_s2_device = __pgprot(s2_device_pgprot);
611 pgprot_hyp_device = __pgprot(hyp_device_pgprot);
Russell Kingae8f1542006-09-27 15:38:34 +0100612
613 mem_types[MT_LOW_VECTORS].prot_l1 |= ecc_mask;
614 mem_types[MT_HIGH_VECTORS].prot_l1 |= ecc_mask;
Russell King2e2c9de2013-10-24 10:26:40 +0100615 mem_types[MT_MEMORY_RWX].prot_sect |= ecc_mask | cp->pmd;
616 mem_types[MT_MEMORY_RWX].prot_pte |= kern_pgprot;
Russell Kingebd49222013-10-24 08:12:39 +0100617 mem_types[MT_MEMORY_RW].prot_sect |= ecc_mask | cp->pmd;
618 mem_types[MT_MEMORY_RW].prot_pte |= kern_pgprot;
Marek Szyprowskic7909502011-12-29 13:09:51 +0100619 mem_types[MT_MEMORY_DMA_READY].prot_pte |= kern_pgprot;
Russell King2e2c9de2013-10-24 10:26:40 +0100620 mem_types[MT_MEMORY_RWX_NONCACHED].prot_sect |= ecc_mask;
Russell Kingae8f1542006-09-27 15:38:34 +0100621 mem_types[MT_ROM].prot_sect |= cp->pmd;
622
623 switch (cp->pmd) {
624 case PMD_SECT_WT:
625 mem_types[MT_CACHECLEAN].prot_sect |= PMD_SECT_WT;
626 break;
627 case PMD_SECT_WB:
628 case PMD_SECT_WBWA:
629 mem_types[MT_CACHECLEAN].prot_sect |= PMD_SECT_WB;
630 break;
631 }
Michal Simek905b5792013-11-07 12:49:53 +0100632 pr_info("Memory policy: %sData cache %s\n",
633 ecc_mask ? "ECC enabled, " : "", cp->policy);
Russell King2497f0a2007-04-21 09:59:44 +0100634
635 for (i = 0; i < ARRAY_SIZE(mem_types); i++) {
636 struct mem_type *t = &mem_types[i];
637 if (t->prot_l1)
638 t->prot_l1 |= PMD_DOMAIN(t->domain);
639 if (t->prot_sect)
640 t->prot_sect |= PMD_DOMAIN(t->domain);
641 }
Russell Kingae8f1542006-09-27 15:38:34 +0100642}
643
Catalin Marinasd9073872010-09-13 16:01:24 +0100644#ifdef CONFIG_ARM_DMA_MEM_BUFFERABLE
645pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
646 unsigned long size, pgprot_t vma_prot)
647{
648 if (!pfn_valid(pfn))
649 return pgprot_noncached(vma_prot);
650 else if (file->f_flags & O_SYNC)
651 return pgprot_writecombine(vma_prot);
652 return vma_prot;
653}
654EXPORT_SYMBOL(phys_mem_access_prot);
655#endif
656
Russell Kingae8f1542006-09-27 15:38:34 +0100657#define vectors_base() (vectors_high() ? 0xffff0000 : 0)
658
Nicolas Pitre0536bdf2011-08-25 00:35:59 -0400659static void __init *early_alloc_aligned(unsigned long sz, unsigned long align)
Russell King3abe9d32010-03-25 17:02:59 +0000660{
Nicolas Pitre0536bdf2011-08-25 00:35:59 -0400661 void *ptr = __va(memblock_alloc(sz, align));
Russell King2778f622010-07-09 16:27:52 +0100662 memset(ptr, 0, sz);
663 return ptr;
Russell King3abe9d32010-03-25 17:02:59 +0000664}
665
Nicolas Pitre0536bdf2011-08-25 00:35:59 -0400666static void __init *early_alloc(unsigned long sz)
667{
668 return early_alloc_aligned(sz, sz);
669}
670
Russell King4bb2e272010-07-01 18:33:29 +0100671static pte_t * __init early_pte_alloc(pmd_t *pmd, unsigned long addr, unsigned long prot)
672{
673 if (pmd_none(*pmd)) {
Catalin Marinas410f1482011-02-14 12:58:04 +0100674 pte_t *pte = early_alloc(PTE_HWTABLE_OFF + PTE_HWTABLE_SIZE);
Russell King97092e02010-11-16 00:16:01 +0000675 __pmd_populate(pmd, __pa(pte), prot);
Russell King4bb2e272010-07-01 18:33:29 +0100676 }
677 BUG_ON(pmd_bad(*pmd));
678 return pte_offset_kernel(pmd, addr);
679}
680
Russell King24e6c692007-04-21 10:21:28 +0100681static void __init alloc_init_pte(pmd_t *pmd, unsigned long addr,
682 unsigned long end, unsigned long pfn,
683 const struct mem_type *type)
Russell Kingae8f1542006-09-27 15:38:34 +0100684{
Russell King4bb2e272010-07-01 18:33:29 +0100685 pte_t *pte = early_pte_alloc(pmd, addr, type->prot_l1);
Russell King24e6c692007-04-21 10:21:28 +0100686 do {
Russell King40d192b2008-09-06 21:15:56 +0100687 set_pte_ext(pte, pfn_pte(pfn, __pgprot(type->prot_pte)), 0);
Russell King24e6c692007-04-21 10:21:28 +0100688 pfn++;
689 } while (pte++, addr += PAGE_SIZE, addr != end);
Russell Kingae8f1542006-09-27 15:38:34 +0100690}
691
Po-Yu Chuang37468b32013-06-07 12:15:45 +0100692static void __init __map_init_section(pmd_t *pmd, unsigned long addr,
Sricharan Re651eab2013-03-18 12:24:04 +0100693 unsigned long end, phys_addr_t phys,
694 const struct mem_type *type)
695{
Po-Yu Chuang37468b32013-06-07 12:15:45 +0100696 pmd_t *p = pmd;
697
Sricharan Re651eab2013-03-18 12:24:04 +0100698#ifndef CONFIG_ARM_LPAE
699 /*
700 * In classic MMU format, puds and pmds are folded in to
701 * the pgds. pmd_offset gives the PGD entry. PGDs refer to a
702 * group of L1 entries making up one logical pointer to
703 * an L2 table (2MB), where as PMDs refer to the individual
704 * L1 entries (1MB). Hence increment to get the correct
705 * offset for odd 1MB sections.
706 * (See arch/arm/include/asm/pgtable-2level.h)
707 */
708 if (addr & SECTION_SIZE)
709 pmd++;
710#endif
711 do {
712 *pmd = __pmd(phys | type->prot_sect);
713 phys += SECTION_SIZE;
714 } while (pmd++, addr += SECTION_SIZE, addr != end);
715
Po-Yu Chuang37468b32013-06-07 12:15:45 +0100716 flush_pmd_entry(p);
Sricharan Re651eab2013-03-18 12:24:04 +0100717}
718
719static void __init alloc_init_pmd(pud_t *pud, unsigned long addr,
Russell King97092e02010-11-16 00:16:01 +0000720 unsigned long end, phys_addr_t phys,
Russell King24e6c692007-04-21 10:21:28 +0100721 const struct mem_type *type)
Russell Kingae8f1542006-09-27 15:38:34 +0100722{
Russell King516295e2010-11-21 16:27:49 +0000723 pmd_t *pmd = pmd_offset(pud, addr);
Sricharan Re651eab2013-03-18 12:24:04 +0100724 unsigned long next;
Russell Kingae8f1542006-09-27 15:38:34 +0100725
Sricharan Re651eab2013-03-18 12:24:04 +0100726 do {
Russell King24e6c692007-04-21 10:21:28 +0100727 /*
Sricharan Re651eab2013-03-18 12:24:04 +0100728 * With LPAE, we must loop over to map
729 * all the pmds for the given range.
Russell King24e6c692007-04-21 10:21:28 +0100730 */
Sricharan Re651eab2013-03-18 12:24:04 +0100731 next = pmd_addr_end(addr, end);
732
733 /*
734 * Try a section mapping - addr, next and phys must all be
735 * aligned to a section boundary.
736 */
737 if (type->prot_sect &&
738 ((addr | next | phys) & ~SECTION_MASK) == 0) {
Po-Yu Chuang37468b32013-06-07 12:15:45 +0100739 __map_init_section(pmd, addr, next, phys, type);
Sricharan Re651eab2013-03-18 12:24:04 +0100740 } else {
741 alloc_init_pte(pmd, addr, next,
742 __phys_to_pfn(phys), type);
743 }
744
745 phys += next - addr;
746
747 } while (pmd++, addr = next, addr != end);
Russell Kingae8f1542006-09-27 15:38:34 +0100748}
749
Stephen Boyd14904922012-04-27 01:40:10 +0100750static void __init alloc_init_pud(pgd_t *pgd, unsigned long addr,
Vitaly Andrianov20d69562012-07-10 14:41:17 -0400751 unsigned long end, phys_addr_t phys,
752 const struct mem_type *type)
Russell King516295e2010-11-21 16:27:49 +0000753{
754 pud_t *pud = pud_offset(pgd, addr);
755 unsigned long next;
756
757 do {
758 next = pud_addr_end(addr, end);
Sricharan Re651eab2013-03-18 12:24:04 +0100759 alloc_init_pmd(pud, addr, next, phys, type);
Russell King516295e2010-11-21 16:27:49 +0000760 phys += next - addr;
761 } while (pud++, addr = next, addr != end);
762}
763
Catalin Marinas1b6ba462011-11-22 17:30:29 +0000764#ifndef CONFIG_ARM_LPAE
Russell King4a56c1e2007-04-21 10:16:48 +0100765static void __init create_36bit_mapping(struct map_desc *md,
766 const struct mem_type *type)
767{
Russell King97092e02010-11-16 00:16:01 +0000768 unsigned long addr, length, end;
769 phys_addr_t phys;
Russell King4a56c1e2007-04-21 10:16:48 +0100770 pgd_t *pgd;
771
772 addr = md->virtual;
Will Deaconcae62922011-02-15 12:42:57 +0100773 phys = __pfn_to_phys(md->pfn);
Russell King4a56c1e2007-04-21 10:16:48 +0100774 length = PAGE_ALIGN(md->length);
775
776 if (!(cpu_architecture() >= CPU_ARCH_ARMv6 || cpu_is_xsc3())) {
777 printk(KERN_ERR "MM: CPU does not support supersection "
778 "mapping for 0x%08llx at 0x%08lx\n",
Will Deacon29a38192011-02-15 14:31:37 +0100779 (long long)__pfn_to_phys((u64)md->pfn), addr);
Russell King4a56c1e2007-04-21 10:16:48 +0100780 return;
781 }
782
783 /* N.B. ARMv6 supersections are only defined to work with domain 0.
784 * Since domain assignments can in fact be arbitrary, the
785 * 'domain == 0' check below is required to insure that ARMv6
786 * supersections are only allocated for domain 0 regardless
787 * of the actual domain assignments in use.
788 */
789 if (type->domain) {
790 printk(KERN_ERR "MM: invalid domain in supersection "
791 "mapping for 0x%08llx at 0x%08lx\n",
Will Deacon29a38192011-02-15 14:31:37 +0100792 (long long)__pfn_to_phys((u64)md->pfn), addr);
Russell King4a56c1e2007-04-21 10:16:48 +0100793 return;
794 }
795
796 if ((addr | length | __pfn_to_phys(md->pfn)) & ~SUPERSECTION_MASK) {
Will Deacon29a38192011-02-15 14:31:37 +0100797 printk(KERN_ERR "MM: cannot create mapping for 0x%08llx"
798 " at 0x%08lx invalid alignment\n",
799 (long long)__pfn_to_phys((u64)md->pfn), addr);
Russell King4a56c1e2007-04-21 10:16:48 +0100800 return;
801 }
802
803 /*
804 * Shift bits [35:32] of address into bits [23:20] of PMD
805 * (See ARMv6 spec).
806 */
807 phys |= (((md->pfn >> (32 - PAGE_SHIFT)) & 0xF) << 20);
808
809 pgd = pgd_offset_k(addr);
810 end = addr + length;
811 do {
Russell King516295e2010-11-21 16:27:49 +0000812 pud_t *pud = pud_offset(pgd, addr);
813 pmd_t *pmd = pmd_offset(pud, addr);
Russell King4a56c1e2007-04-21 10:16:48 +0100814 int i;
815
816 for (i = 0; i < 16; i++)
817 *pmd++ = __pmd(phys | type->prot_sect | PMD_SECT_SUPER);
818
819 addr += SUPERSECTION_SIZE;
820 phys += SUPERSECTION_SIZE;
821 pgd += SUPERSECTION_SIZE >> PGDIR_SHIFT;
822 } while (addr != end);
823}
Catalin Marinas1b6ba462011-11-22 17:30:29 +0000824#endif /* !CONFIG_ARM_LPAE */
Russell King4a56c1e2007-04-21 10:16:48 +0100825
Russell Kingae8f1542006-09-27 15:38:34 +0100826/*
827 * Create the page directory entries and any necessary
828 * page tables for the mapping specified by `md'. We
829 * are able to cope here with varying sizes and address
830 * offsets, and we take full advantage of sections and
831 * supersections.
832 */
Russell Kinga2227122010-03-25 18:56:05 +0000833static void __init create_mapping(struct map_desc *md)
Russell Kingae8f1542006-09-27 15:38:34 +0100834{
Will Deaconcae62922011-02-15 12:42:57 +0100835 unsigned long addr, length, end;
836 phys_addr_t phys;
Russell Kingd5c98172007-04-21 10:05:32 +0100837 const struct mem_type *type;
Russell King24e6c692007-04-21 10:21:28 +0100838 pgd_t *pgd;
Russell Kingae8f1542006-09-27 15:38:34 +0100839
840 if (md->virtual != vectors_base() && md->virtual < TASK_SIZE) {
Will Deacon29a38192011-02-15 14:31:37 +0100841 printk(KERN_WARNING "BUG: not creating mapping for 0x%08llx"
842 " at 0x%08lx in user region\n",
843 (long long)__pfn_to_phys((u64)md->pfn), md->virtual);
Russell Kingae8f1542006-09-27 15:38:34 +0100844 return;
845 }
846
847 if ((md->type == MT_DEVICE || md->type == MT_ROM) &&
Nicolas Pitre0536bdf2011-08-25 00:35:59 -0400848 md->virtual >= PAGE_OFFSET &&
849 (md->virtual < VMALLOC_START || md->virtual >= VMALLOC_END)) {
Will Deacon29a38192011-02-15 14:31:37 +0100850 printk(KERN_WARNING "BUG: mapping for 0x%08llx"
Nicolas Pitre0536bdf2011-08-25 00:35:59 -0400851 " at 0x%08lx out of vmalloc space\n",
Will Deacon29a38192011-02-15 14:31:37 +0100852 (long long)__pfn_to_phys((u64)md->pfn), md->virtual);
Russell Kingae8f1542006-09-27 15:38:34 +0100853 }
854
Russell Kingd5c98172007-04-21 10:05:32 +0100855 type = &mem_types[md->type];
Russell Kingae8f1542006-09-27 15:38:34 +0100856
Catalin Marinas1b6ba462011-11-22 17:30:29 +0000857#ifndef CONFIG_ARM_LPAE
Russell Kingae8f1542006-09-27 15:38:34 +0100858 /*
859 * Catch 36-bit addresses
860 */
Russell King4a56c1e2007-04-21 10:16:48 +0100861 if (md->pfn >= 0x100000) {
862 create_36bit_mapping(md, type);
863 return;
Russell Kingae8f1542006-09-27 15:38:34 +0100864 }
Catalin Marinas1b6ba462011-11-22 17:30:29 +0000865#endif
Russell Kingae8f1542006-09-27 15:38:34 +0100866
Russell King7b9c7b42007-07-04 21:16:33 +0100867 addr = md->virtual & PAGE_MASK;
Will Deaconcae62922011-02-15 12:42:57 +0100868 phys = __pfn_to_phys(md->pfn);
Russell King7b9c7b42007-07-04 21:16:33 +0100869 length = PAGE_ALIGN(md->length + (md->virtual & ~PAGE_MASK));
Russell Kingae8f1542006-09-27 15:38:34 +0100870
Russell King24e6c692007-04-21 10:21:28 +0100871 if (type->prot_l1 == 0 && ((addr | phys | length) & ~SECTION_MASK)) {
Will Deacon29a38192011-02-15 14:31:37 +0100872 printk(KERN_WARNING "BUG: map for 0x%08llx at 0x%08lx can not "
Russell Kingae8f1542006-09-27 15:38:34 +0100873 "be mapped using pages, ignoring.\n",
Will Deacon29a38192011-02-15 14:31:37 +0100874 (long long)__pfn_to_phys(md->pfn), addr);
Russell Kingae8f1542006-09-27 15:38:34 +0100875 return;
876 }
877
Russell King24e6c692007-04-21 10:21:28 +0100878 pgd = pgd_offset_k(addr);
879 end = addr + length;
880 do {
881 unsigned long next = pgd_addr_end(addr, end);
Russell Kingae8f1542006-09-27 15:38:34 +0100882
Russell King516295e2010-11-21 16:27:49 +0000883 alloc_init_pud(pgd, addr, next, phys, type);
Russell Kingae8f1542006-09-27 15:38:34 +0100884
Russell King24e6c692007-04-21 10:21:28 +0100885 phys += next - addr;
886 addr = next;
887 } while (pgd++, addr != end);
Russell Kingae8f1542006-09-27 15:38:34 +0100888}
889
890/*
891 * Create the architecture specific mappings
892 */
893void __init iotable_init(struct map_desc *io_desc, int nr)
894{
Nicolas Pitre0536bdf2011-08-25 00:35:59 -0400895 struct map_desc *md;
896 struct vm_struct *vm;
Joonsoo Kim101eeda2013-02-09 06:28:06 +0100897 struct static_vm *svm;
Russell Kingae8f1542006-09-27 15:38:34 +0100898
Nicolas Pitre0536bdf2011-08-25 00:35:59 -0400899 if (!nr)
900 return;
901
Joonsoo Kim101eeda2013-02-09 06:28:06 +0100902 svm = early_alloc_aligned(sizeof(*svm) * nr, __alignof__(*svm));
Nicolas Pitre0536bdf2011-08-25 00:35:59 -0400903
904 for (md = io_desc; nr; md++, nr--) {
905 create_mapping(md);
Joonsoo Kim101eeda2013-02-09 06:28:06 +0100906
907 vm = &svm->vm;
Nicolas Pitre0536bdf2011-08-25 00:35:59 -0400908 vm->addr = (void *)(md->virtual & PAGE_MASK);
909 vm->size = PAGE_ALIGN(md->length + (md->virtual & ~PAGE_MASK));
Rob Herringc2794432012-02-29 18:10:58 -0600910 vm->phys_addr = __pfn_to_phys(md->pfn);
911 vm->flags = VM_IOREMAP | VM_ARM_STATIC_MAPPING;
Nicolas Pitre576d2f22011-09-16 01:14:23 -0400912 vm->flags |= VM_ARM_MTYPE(md->type);
Nicolas Pitre0536bdf2011-08-25 00:35:59 -0400913 vm->caller = iotable_init;
Joonsoo Kim101eeda2013-02-09 06:28:06 +0100914 add_static_vm_early(svm++);
Nicolas Pitre0536bdf2011-08-25 00:35:59 -0400915 }
Russell Kingae8f1542006-09-27 15:38:34 +0100916}
917
Rob Herringc2794432012-02-29 18:10:58 -0600918void __init vm_reserve_area_early(unsigned long addr, unsigned long size,
919 void *caller)
920{
921 struct vm_struct *vm;
Joonsoo Kim101eeda2013-02-09 06:28:06 +0100922 struct static_vm *svm;
Rob Herringc2794432012-02-29 18:10:58 -0600923
Joonsoo Kim101eeda2013-02-09 06:28:06 +0100924 svm = early_alloc_aligned(sizeof(*svm), __alignof__(*svm));
925
926 vm = &svm->vm;
Rob Herringc2794432012-02-29 18:10:58 -0600927 vm->addr = (void *)addr;
928 vm->size = size;
Arnd Bergmann863e99a2012-09-04 15:01:37 +0200929 vm->flags = VM_IOREMAP | VM_ARM_EMPTY_MAPPING;
Rob Herringc2794432012-02-29 18:10:58 -0600930 vm->caller = caller;
Joonsoo Kim101eeda2013-02-09 06:28:06 +0100931 add_static_vm_early(svm);
Rob Herringc2794432012-02-29 18:10:58 -0600932}
933
Nicolas Pitre19b52ab2012-06-27 17:28:57 +0100934#ifndef CONFIG_ARM_LPAE
935
936/*
937 * The Linux PMD is made of two consecutive section entries covering 2MB
938 * (see definition in include/asm/pgtable-2level.h). However a call to
939 * create_mapping() may optimize static mappings by using individual
940 * 1MB section mappings. This leaves the actual PMD potentially half
941 * initialized if the top or bottom section entry isn't used, leaving it
942 * open to problems if a subsequent ioremap() or vmalloc() tries to use
943 * the virtual space left free by that unused section entry.
944 *
945 * Let's avoid the issue by inserting dummy vm entries covering the unused
946 * PMD halves once the static mappings are in place.
947 */
948
949static void __init pmd_empty_section_gap(unsigned long addr)
950{
Rob Herringc2794432012-02-29 18:10:58 -0600951 vm_reserve_area_early(addr, SECTION_SIZE, pmd_empty_section_gap);
Nicolas Pitre19b52ab2012-06-27 17:28:57 +0100952}
953
954static void __init fill_pmd_gaps(void)
955{
Joonsoo Kim101eeda2013-02-09 06:28:06 +0100956 struct static_vm *svm;
Nicolas Pitre19b52ab2012-06-27 17:28:57 +0100957 struct vm_struct *vm;
958 unsigned long addr, next = 0;
959 pmd_t *pmd;
960
Joonsoo Kim101eeda2013-02-09 06:28:06 +0100961 list_for_each_entry(svm, &static_vmlist, list) {
962 vm = &svm->vm;
Nicolas Pitre19b52ab2012-06-27 17:28:57 +0100963 addr = (unsigned long)vm->addr;
964 if (addr < next)
965 continue;
966
967 /*
968 * Check if this vm starts on an odd section boundary.
969 * If so and the first section entry for this PMD is free
970 * then we block the corresponding virtual address.
971 */
972 if ((addr & ~PMD_MASK) == SECTION_SIZE) {
973 pmd = pmd_off_k(addr);
974 if (pmd_none(*pmd))
975 pmd_empty_section_gap(addr & PMD_MASK);
976 }
977
978 /*
979 * Then check if this vm ends on an odd section boundary.
980 * If so and the second section entry for this PMD is empty
981 * then we block the corresponding virtual address.
982 */
983 addr += vm->size;
984 if ((addr & ~PMD_MASK) == SECTION_SIZE) {
985 pmd = pmd_off_k(addr) + 1;
986 if (pmd_none(*pmd))
987 pmd_empty_section_gap(addr);
988 }
989
990 /* no need to look at any vm entry until we hit the next PMD */
991 next = (addr + PMD_SIZE - 1) & PMD_MASK;
992 }
993}
994
995#else
996#define fill_pmd_gaps() do { } while (0)
997#endif
998
Rob Herringc2794432012-02-29 18:10:58 -0600999#if defined(CONFIG_PCI) && !defined(CONFIG_NEED_MACH_IO_H)
1000static void __init pci_reserve_io(void)
1001{
Joonsoo Kim101eeda2013-02-09 06:28:06 +01001002 struct static_vm *svm;
Rob Herringc2794432012-02-29 18:10:58 -06001003
Joonsoo Kim101eeda2013-02-09 06:28:06 +01001004 svm = find_static_vm_vaddr((void *)PCI_IO_VIRT_BASE);
1005 if (svm)
1006 return;
Rob Herringc2794432012-02-29 18:10:58 -06001007
Rob Herringc2794432012-02-29 18:10:58 -06001008 vm_reserve_area_early(PCI_IO_VIRT_BASE, SZ_2M, pci_reserve_io);
1009}
1010#else
1011#define pci_reserve_io() do { } while (0)
1012#endif
1013
Rob Herringe5c5f2a2012-10-22 11:42:54 -06001014#ifdef CONFIG_DEBUG_LL
1015void __init debug_ll_io_init(void)
1016{
1017 struct map_desc map;
1018
1019 debug_ll_addr(&map.pfn, &map.virtual);
1020 if (!map.pfn || !map.virtual)
1021 return;
1022 map.pfn = __phys_to_pfn(map.pfn);
1023 map.virtual &= PAGE_MASK;
1024 map.length = PAGE_SIZE;
1025 map.type = MT_DEVICE;
Stephen Boydee4de5d2013-07-06 00:25:51 +01001026 iotable_init(&map, 1);
Rob Herringe5c5f2a2012-10-22 11:42:54 -06001027}
1028#endif
1029
Nicolas Pitre0536bdf2011-08-25 00:35:59 -04001030static void * __initdata vmalloc_min =
1031 (void *)(VMALLOC_END - (240 << 20) - VMALLOC_OFFSET);
Russell King6c5da7a2008-09-30 19:31:44 +01001032
1033/*
1034 * vmalloc=size forces the vmalloc area to be exactly 'size'
1035 * bytes. This can be used to increase (or decrease) the vmalloc
Nicolas Pitre0536bdf2011-08-25 00:35:59 -04001036 * area - the default is 240m.
Russell King6c5da7a2008-09-30 19:31:44 +01001037 */
Jeremy Kerr2b0d8c22010-01-11 23:17:34 +01001038static int __init early_vmalloc(char *arg)
Russell King6c5da7a2008-09-30 19:31:44 +01001039{
Russell King79612392010-05-22 16:20:14 +01001040 unsigned long vmalloc_reserve = memparse(arg, NULL);
Russell King6c5da7a2008-09-30 19:31:44 +01001041
1042 if (vmalloc_reserve < SZ_16M) {
1043 vmalloc_reserve = SZ_16M;
1044 printk(KERN_WARNING
1045 "vmalloc area too small, limiting to %luMB\n",
1046 vmalloc_reserve >> 20);
1047 }
Nicolas Pitre92108072008-09-19 10:43:06 -04001048
1049 if (vmalloc_reserve > VMALLOC_END - (PAGE_OFFSET + SZ_32M)) {
1050 vmalloc_reserve = VMALLOC_END - (PAGE_OFFSET + SZ_32M);
1051 printk(KERN_WARNING
1052 "vmalloc area is too big, limiting to %luMB\n",
1053 vmalloc_reserve >> 20);
1054 }
Russell King79612392010-05-22 16:20:14 +01001055
1056 vmalloc_min = (void *)(VMALLOC_END - vmalloc_reserve);
Jeremy Kerr2b0d8c22010-01-11 23:17:34 +01001057 return 0;
Russell King6c5da7a2008-09-30 19:31:44 +01001058}
Jeremy Kerr2b0d8c22010-01-11 23:17:34 +01001059early_param("vmalloc", early_vmalloc);
Russell King6c5da7a2008-09-30 19:31:44 +01001060
Marek Szyprowskic7909502011-12-29 13:09:51 +01001061phys_addr_t arm_lowmem_limit __initdata = 0;
Russell King8df65162010-10-27 19:57:38 +01001062
Russell King0371d3f2011-07-05 19:58:29 +01001063void __init sanity_check_meminfo(void)
Lennert Buytenhek60296c72008-08-05 01:56:13 +02001064{
Russell Kingc65b7e92013-07-17 17:53:04 +01001065 phys_addr_t memblock_limit = 0;
Russell Kingdde58282009-08-15 12:36:00 +01001066 int i, j, highmem = 0;
Cyril Chemparathy82f66702012-07-20 12:01:23 -04001067 phys_addr_t vmalloc_limit = __pa(vmalloc_min - 1) + 1;
Lennert Buytenhek60296c72008-08-05 01:56:13 +02001068
Nicolas Pitre4b5f32c2008-10-06 13:24:40 -04001069 for (i = 0, j = 0; i < meminfo.nr_banks; i++) {
Nicolas Pitrea1bbaec2008-09-02 11:44:21 -04001070 struct membank *bank = &meminfo.bank[j];
Cyril Chemparathy28d4bf72012-07-20 13:16:41 -04001071 phys_addr_t size_limit;
1072
Nicolas Pitrea1bbaec2008-09-02 11:44:21 -04001073 *bank = meminfo.bank[i];
Cyril Chemparathy28d4bf72012-07-20 13:16:41 -04001074 size_limit = bank->size;
Nicolas Pitrea1bbaec2008-09-02 11:44:21 -04001075
Cyril Chemparathy82f66702012-07-20 12:01:23 -04001076 if (bank->start >= vmalloc_limit)
Will Deacon77f73a22011-11-22 17:30:32 +00001077 highmem = 1;
Cyril Chemparathy28d4bf72012-07-20 13:16:41 -04001078 else
1079 size_limit = vmalloc_limit - bank->start;
Russell Kingdde58282009-08-15 12:36:00 +01001080
1081 bank->highmem = highmem;
1082
Cyril Chemparathyadf2e9f2012-07-20 12:24:45 -04001083#ifdef CONFIG_HIGHMEM
Nicolas Pitrea1bbaec2008-09-02 11:44:21 -04001084 /*
1085 * Split those memory banks which are partially overlapping
1086 * the vmalloc area greatly simplifying things later.
1087 */
Cyril Chemparathy28d4bf72012-07-20 13:16:41 -04001088 if (!highmem && bank->size > size_limit) {
Nicolas Pitrea1bbaec2008-09-02 11:44:21 -04001089 if (meminfo.nr_banks >= NR_BANKS) {
1090 printk(KERN_CRIT "NR_BANKS too low, "
1091 "ignoring high memory\n");
1092 } else {
1093 memmove(bank + 1, bank,
1094 (meminfo.nr_banks - i) * sizeof(*bank));
1095 meminfo.nr_banks++;
1096 i++;
Cyril Chemparathy28d4bf72012-07-20 13:16:41 -04001097 bank[1].size -= size_limit;
Cyril Chemparathy82f66702012-07-20 12:01:23 -04001098 bank[1].start = vmalloc_limit;
Russell Kingdde58282009-08-15 12:36:00 +01001099 bank[1].highmem = highmem = 1;
Nicolas Pitrea1bbaec2008-09-02 11:44:21 -04001100 j++;
1101 }
Cyril Chemparathy28d4bf72012-07-20 13:16:41 -04001102 bank->size = size_limit;
Nicolas Pitrea1bbaec2008-09-02 11:44:21 -04001103 }
1104#else
1105 /*
Will Deacon77f73a22011-11-22 17:30:32 +00001106 * Highmem banks not allowed with !CONFIG_HIGHMEM.
1107 */
1108 if (highmem) {
1109 printk(KERN_NOTICE "Ignoring RAM at %.8llx-%.8llx "
1110 "(!CONFIG_HIGHMEM).\n",
1111 (unsigned long long)bank->start,
1112 (unsigned long long)bank->start + bank->size - 1);
1113 continue;
1114 }
1115
1116 /*
Nicolas Pitrea1bbaec2008-09-02 11:44:21 -04001117 * Check whether this memory bank would partially overlap
1118 * the vmalloc area.
1119 */
Cyril Chemparathy28d4bf72012-07-20 13:16:41 -04001120 if (bank->size > size_limit) {
Russell Kinge33b9d02011-02-20 11:47:41 +00001121 printk(KERN_NOTICE "Truncating RAM at %.8llx-%.8llx "
1122 "to -%.8llx (vmalloc region overlap).\n",
1123 (unsigned long long)bank->start,
1124 (unsigned long long)bank->start + bank->size - 1,
Cyril Chemparathy28d4bf72012-07-20 13:16:41 -04001125 (unsigned long long)bank->start + size_limit - 1);
1126 bank->size = size_limit;
Nicolas Pitrea1bbaec2008-09-02 11:44:21 -04001127 }
1128#endif
Russell Kingc65b7e92013-07-17 17:53:04 +01001129 if (!bank->highmem) {
1130 phys_addr_t bank_end = bank->start + bank->size;
Will Deacon40f7bfe2011-05-19 13:22:48 +01001131
Russell Kingc65b7e92013-07-17 17:53:04 +01001132 if (bank_end > arm_lowmem_limit)
1133 arm_lowmem_limit = bank_end;
1134
1135 /*
1136 * Find the first non-section-aligned page, and point
1137 * memblock_limit at it. This relies on rounding the
1138 * limit down to be section-aligned, which happens at
1139 * the end of this function.
1140 *
1141 * With this algorithm, the start or end of almost any
1142 * bank can be non-section-aligned. The only exception
1143 * is that the start of the bank 0 must be section-
1144 * aligned, since otherwise memory would need to be
1145 * allocated when mapping the start of bank 0, which
1146 * occurs before any free memory is mapped.
1147 */
1148 if (!memblock_limit) {
1149 if (!IS_ALIGNED(bank->start, SECTION_SIZE))
1150 memblock_limit = bank->start;
1151 else if (!IS_ALIGNED(bank_end, SECTION_SIZE))
1152 memblock_limit = bank_end;
1153 }
1154 }
Nicolas Pitrea1bbaec2008-09-02 11:44:21 -04001155 j++;
Lennert Buytenhek60296c72008-08-05 01:56:13 +02001156 }
Russell Kinge616c592009-09-27 20:55:43 +01001157#ifdef CONFIG_HIGHMEM
1158 if (highmem) {
1159 const char *reason = NULL;
1160
1161 if (cache_is_vipt_aliasing()) {
1162 /*
1163 * Interactions between kmap and other mappings
1164 * make highmem support with aliasing VIPT caches
1165 * rather difficult.
1166 */
1167 reason = "with VIPT aliasing cache";
Russell Kinge616c592009-09-27 20:55:43 +01001168 }
1169 if (reason) {
1170 printk(KERN_CRIT "HIGHMEM is not supported %s, ignoring high memory\n",
1171 reason);
1172 while (j > 0 && meminfo.bank[j - 1].highmem)
1173 j--;
1174 }
1175 }
1176#endif
Nicolas Pitre4b5f32c2008-10-06 13:24:40 -04001177 meminfo.nr_banks = j;
Marek Szyprowskic7909502011-12-29 13:09:51 +01001178 high_memory = __va(arm_lowmem_limit - 1) + 1;
Russell Kingc65b7e92013-07-17 17:53:04 +01001179
1180 /*
1181 * Round the memblock limit down to a section size. This
1182 * helps to ensure that we will allocate memory from the
1183 * last full section, which should be mapped.
1184 */
1185 if (memblock_limit)
1186 memblock_limit = round_down(memblock_limit, SECTION_SIZE);
1187 if (!memblock_limit)
1188 memblock_limit = arm_lowmem_limit;
1189
1190 memblock_set_current_limit(memblock_limit);
Lennert Buytenhek60296c72008-08-05 01:56:13 +02001191}
1192
Nicolas Pitre4b5f32c2008-10-06 13:24:40 -04001193static inline void prepare_page_table(void)
Russell Kingd111e8f2006-09-27 15:27:33 +01001194{
1195 unsigned long addr;
Russell King8df65162010-10-27 19:57:38 +01001196 phys_addr_t end;
Russell Kingd111e8f2006-09-27 15:27:33 +01001197
1198 /*
1199 * Clear out all the mappings below the kernel image.
1200 */
Catalin Marinase73fc882011-08-23 14:07:23 +01001201 for (addr = 0; addr < MODULES_VADDR; addr += PMD_SIZE)
Russell Kingd111e8f2006-09-27 15:27:33 +01001202 pmd_clear(pmd_off_k(addr));
1203
1204#ifdef CONFIG_XIP_KERNEL
1205 /* The XIP kernel is mapped in the module area -- skip over it */
Catalin Marinase73fc882011-08-23 14:07:23 +01001206 addr = ((unsigned long)_etext + PMD_SIZE - 1) & PMD_MASK;
Russell Kingd111e8f2006-09-27 15:27:33 +01001207#endif
Catalin Marinase73fc882011-08-23 14:07:23 +01001208 for ( ; addr < PAGE_OFFSET; addr += PMD_SIZE)
Russell Kingd111e8f2006-09-27 15:27:33 +01001209 pmd_clear(pmd_off_k(addr));
1210
1211 /*
Russell King8df65162010-10-27 19:57:38 +01001212 * Find the end of the first block of lowmem.
1213 */
1214 end = memblock.memory.regions[0].base + memblock.memory.regions[0].size;
Marek Szyprowskic7909502011-12-29 13:09:51 +01001215 if (end >= arm_lowmem_limit)
1216 end = arm_lowmem_limit;
Russell King8df65162010-10-27 19:57:38 +01001217
1218 /*
Russell Kingd111e8f2006-09-27 15:27:33 +01001219 * Clear out all the kernel space mappings, except for the first
Nicolas Pitre0536bdf2011-08-25 00:35:59 -04001220 * memory bank, up to the vmalloc region.
Russell Kingd111e8f2006-09-27 15:27:33 +01001221 */
Russell King8df65162010-10-27 19:57:38 +01001222 for (addr = __phys_to_virt(end);
Nicolas Pitre0536bdf2011-08-25 00:35:59 -04001223 addr < VMALLOC_START; addr += PMD_SIZE)
Russell Kingd111e8f2006-09-27 15:27:33 +01001224 pmd_clear(pmd_off_k(addr));
1225}
1226
Catalin Marinas1b6ba462011-11-22 17:30:29 +00001227#ifdef CONFIG_ARM_LPAE
1228/* the first page is reserved for pgd */
1229#define SWAPPER_PG_DIR_SIZE (PAGE_SIZE + \
1230 PTRS_PER_PGD * PTRS_PER_PMD * sizeof(pmd_t))
1231#else
Catalin Marinase73fc882011-08-23 14:07:23 +01001232#define SWAPPER_PG_DIR_SIZE (PTRS_PER_PGD * sizeof(pgd_t))
Catalin Marinas1b6ba462011-11-22 17:30:29 +00001233#endif
Catalin Marinase73fc882011-08-23 14:07:23 +01001234
Russell Kingd111e8f2006-09-27 15:27:33 +01001235/*
Russell King2778f622010-07-09 16:27:52 +01001236 * Reserve the special regions of memory
Russell Kingd111e8f2006-09-27 15:27:33 +01001237 */
Russell King2778f622010-07-09 16:27:52 +01001238void __init arm_mm_memblock_reserve(void)
Russell Kingd111e8f2006-09-27 15:27:33 +01001239{
Russell Kingd111e8f2006-09-27 15:27:33 +01001240 /*
Russell Kingd111e8f2006-09-27 15:27:33 +01001241 * Reserve the page tables. These are already in use,
1242 * and can only be in node 0.
1243 */
Catalin Marinase73fc882011-08-23 14:07:23 +01001244 memblock_reserve(__pa(swapper_pg_dir), SWAPPER_PG_DIR_SIZE);
Russell Kingd111e8f2006-09-27 15:27:33 +01001245
Russell Kingd111e8f2006-09-27 15:27:33 +01001246#ifdef CONFIG_SA1111
1247 /*
1248 * Because of the SA1111 DMA bug, we want to preserve our
1249 * precious DMA-able memory...
1250 */
Russell King2778f622010-07-09 16:27:52 +01001251 memblock_reserve(PHYS_OFFSET, __pa(swapper_pg_dir) - PHYS_OFFSET);
Russell Kingd111e8f2006-09-27 15:27:33 +01001252#endif
Russell Kingd111e8f2006-09-27 15:27:33 +01001253}
1254
1255/*
Nicolas Pitre0536bdf2011-08-25 00:35:59 -04001256 * Set up the device mappings. Since we clear out the page tables for all
1257 * mappings above VMALLOC_START, we will remove any debug device mappings.
Russell Kingd111e8f2006-09-27 15:27:33 +01001258 * This means you have to be careful how you debug this function, or any
1259 * called function. This means you can't use any function or debugging
1260 * method which may touch any device, otherwise the kernel _will_ crash.
1261 */
Russell Kingff69a4c2013-07-26 14:55:59 +01001262static void __init devicemaps_init(const struct machine_desc *mdesc)
Russell Kingd111e8f2006-09-27 15:27:33 +01001263{
1264 struct map_desc map;
1265 unsigned long addr;
Russell King94e5a852012-01-18 15:32:49 +00001266 void *vectors;
Russell Kingd111e8f2006-09-27 15:27:33 +01001267
1268 /*
1269 * Allocate the vector page early.
1270 */
Russell King19accfd2013-07-04 11:40:32 +01001271 vectors = early_alloc(PAGE_SIZE * 2);
Russell King94e5a852012-01-18 15:32:49 +00001272
1273 early_trap_init(vectors);
Russell Kingd111e8f2006-09-27 15:27:33 +01001274
Nicolas Pitre0536bdf2011-08-25 00:35:59 -04001275 for (addr = VMALLOC_START; addr; addr += PMD_SIZE)
Russell Kingd111e8f2006-09-27 15:27:33 +01001276 pmd_clear(pmd_off_k(addr));
1277
1278 /*
1279 * Map the kernel if it is XIP.
1280 * It is always first in the modulearea.
1281 */
1282#ifdef CONFIG_XIP_KERNEL
1283 map.pfn = __phys_to_pfn(CONFIG_XIP_PHYS_ADDR & SECTION_MASK);
Russell Kingab4f2ee2008-11-06 17:11:07 +00001284 map.virtual = MODULES_VADDR;
Russell King37efe642008-12-01 11:53:07 +00001285 map.length = ((unsigned long)_etext - map.virtual + ~SECTION_MASK) & SECTION_MASK;
Russell Kingd111e8f2006-09-27 15:27:33 +01001286 map.type = MT_ROM;
1287 create_mapping(&map);
1288#endif
1289
1290 /*
1291 * Map the cache flushing regions.
1292 */
1293#ifdef FLUSH_BASE
1294 map.pfn = __phys_to_pfn(FLUSH_BASE_PHYS);
1295 map.virtual = FLUSH_BASE;
1296 map.length = SZ_1M;
1297 map.type = MT_CACHECLEAN;
1298 create_mapping(&map);
1299#endif
1300#ifdef FLUSH_BASE_MINICACHE
1301 map.pfn = __phys_to_pfn(FLUSH_BASE_PHYS + SZ_1M);
1302 map.virtual = FLUSH_BASE_MINICACHE;
1303 map.length = SZ_1M;
1304 map.type = MT_MINICLEAN;
1305 create_mapping(&map);
1306#endif
1307
1308 /*
1309 * Create a mapping for the machine vectors at the high-vectors
1310 * location (0xffff0000). If we aren't using high-vectors, also
1311 * create a mapping at the low-vectors virtual address.
1312 */
Russell King94e5a852012-01-18 15:32:49 +00001313 map.pfn = __phys_to_pfn(virt_to_phys(vectors));
Russell Kingd111e8f2006-09-27 15:27:33 +01001314 map.virtual = 0xffff0000;
1315 map.length = PAGE_SIZE;
Russell Kinga5463cd2013-07-31 21:58:56 +01001316#ifdef CONFIG_KUSER_HELPERS
Russell Kingd111e8f2006-09-27 15:27:33 +01001317 map.type = MT_HIGH_VECTORS;
Russell Kinga5463cd2013-07-31 21:58:56 +01001318#else
1319 map.type = MT_LOW_VECTORS;
1320#endif
Russell Kingd111e8f2006-09-27 15:27:33 +01001321 create_mapping(&map);
1322
1323 if (!vectors_high()) {
1324 map.virtual = 0;
Russell King19accfd2013-07-04 11:40:32 +01001325 map.length = PAGE_SIZE * 2;
Russell Kingd111e8f2006-09-27 15:27:33 +01001326 map.type = MT_LOW_VECTORS;
1327 create_mapping(&map);
1328 }
1329
Russell King19accfd2013-07-04 11:40:32 +01001330 /* Now create a kernel read-only mapping */
1331 map.pfn += 1;
1332 map.virtual = 0xffff0000 + PAGE_SIZE;
1333 map.length = PAGE_SIZE;
1334 map.type = MT_LOW_VECTORS;
1335 create_mapping(&map);
1336
Russell Kingd111e8f2006-09-27 15:27:33 +01001337 /*
1338 * Ask the machine support to map in the statically mapped devices.
1339 */
1340 if (mdesc->map_io)
1341 mdesc->map_io();
Maxime Ripardbc373242013-04-18 21:52:23 +02001342 else
1343 debug_ll_io_init();
Nicolas Pitre19b52ab2012-06-27 17:28:57 +01001344 fill_pmd_gaps();
Russell Kingd111e8f2006-09-27 15:27:33 +01001345
Rob Herringc2794432012-02-29 18:10:58 -06001346 /* Reserve fixed i/o space in VMALLOC region */
1347 pci_reserve_io();
1348
Russell Kingd111e8f2006-09-27 15:27:33 +01001349 /*
1350 * Finally flush the caches and tlb to ensure that we're in a
1351 * consistent state wrt the writebuffer. This also ensures that
1352 * any write-allocated cache lines in the vector page are written
1353 * back. After this point, we can start to touch devices again.
1354 */
1355 local_flush_tlb_all();
1356 flush_cache_all();
1357}
1358
Nicolas Pitred73cd422008-09-15 16:44:55 -04001359static void __init kmap_init(void)
1360{
1361#ifdef CONFIG_HIGHMEM
Russell King4bb2e272010-07-01 18:33:29 +01001362 pkmap_page_table = early_pte_alloc(pmd_off_k(PKMAP_BASE),
1363 PKMAP_BASE, _PAGE_KERNEL_TABLE);
Nicolas Pitred73cd422008-09-15 16:44:55 -04001364#endif
1365}
1366
Russell Kinga2227122010-03-25 18:56:05 +00001367static void __init map_lowmem(void)
1368{
Russell King8df65162010-10-27 19:57:38 +01001369 struct memblock_region *reg;
Russell Kingebd49222013-10-24 08:12:39 +01001370 unsigned long kernel_x_start = round_down(__pa(_stext), SECTION_SIZE);
1371 unsigned long kernel_x_end = round_up(__pa(__init_end), SECTION_SIZE);
Russell Kinga2227122010-03-25 18:56:05 +00001372
1373 /* Map all the lowmem memory banks. */
Russell King8df65162010-10-27 19:57:38 +01001374 for_each_memblock(memory, reg) {
1375 phys_addr_t start = reg->base;
1376 phys_addr_t end = start + reg->size;
1377 struct map_desc map;
Russell Kinga2227122010-03-25 18:56:05 +00001378
Marek Szyprowskic7909502011-12-29 13:09:51 +01001379 if (end > arm_lowmem_limit)
1380 end = arm_lowmem_limit;
Russell King8df65162010-10-27 19:57:38 +01001381 if (start >= end)
1382 break;
1383
Russell Kingebd49222013-10-24 08:12:39 +01001384 if (end < kernel_x_start || start >= kernel_x_end) {
1385 map.pfn = __phys_to_pfn(start);
1386 map.virtual = __phys_to_virt(start);
1387 map.length = end - start;
1388 map.type = MT_MEMORY_RWX;
Russell King8df65162010-10-27 19:57:38 +01001389
Russell Kingebd49222013-10-24 08:12:39 +01001390 create_mapping(&map);
1391 } else {
1392 /* This better cover the entire kernel */
1393 if (start < kernel_x_start) {
1394 map.pfn = __phys_to_pfn(start);
1395 map.virtual = __phys_to_virt(start);
1396 map.length = kernel_x_start - start;
1397 map.type = MT_MEMORY_RW;
1398
1399 create_mapping(&map);
1400 }
1401
1402 map.pfn = __phys_to_pfn(kernel_x_start);
1403 map.virtual = __phys_to_virt(kernel_x_start);
1404 map.length = kernel_x_end - kernel_x_start;
1405 map.type = MT_MEMORY_RWX;
1406
1407 create_mapping(&map);
1408
1409 if (kernel_x_end < end) {
1410 map.pfn = __phys_to_pfn(kernel_x_end);
1411 map.virtual = __phys_to_virt(kernel_x_end);
1412 map.length = end - kernel_x_end;
1413 map.type = MT_MEMORY_RW;
1414
1415 create_mapping(&map);
1416 }
1417 }
Russell Kinga2227122010-03-25 18:56:05 +00001418 }
1419}
1420
Santosh Shilimkara77e0c72013-07-31 12:44:46 -04001421#ifdef CONFIG_ARM_LPAE
1422/*
1423 * early_paging_init() recreates boot time page table setup, allowing machines
1424 * to switch over to a high (>4G) address space on LPAE systems
1425 */
1426void __init early_paging_init(const struct machine_desc *mdesc,
1427 struct proc_info_list *procinfo)
1428{
1429 pmdval_t pmdprot = procinfo->__cpu_mm_mmu_flags;
1430 unsigned long map_start, map_end;
1431 pgd_t *pgd0, *pgdk;
1432 pud_t *pud0, *pudk, *pud_start;
1433 pmd_t *pmd0, *pmdk;
1434 phys_addr_t phys;
1435 int i;
1436
1437 if (!(mdesc->init_meminfo))
1438 return;
1439
1440 /* remap kernel code and data */
1441 map_start = init_mm.start_code;
1442 map_end = init_mm.brk;
1443
1444 /* get a handle on things... */
1445 pgd0 = pgd_offset_k(0);
1446 pud_start = pud0 = pud_offset(pgd0, 0);
1447 pmd0 = pmd_offset(pud0, 0);
1448
1449 pgdk = pgd_offset_k(map_start);
1450 pudk = pud_offset(pgdk, map_start);
1451 pmdk = pmd_offset(pudk, map_start);
1452
1453 mdesc->init_meminfo();
1454
1455 /* Run the patch stub to update the constants */
1456 fixup_pv_table(&__pv_table_begin,
1457 (&__pv_table_end - &__pv_table_begin) << 2);
1458
1459 /*
1460 * Cache cleaning operations for self-modifying code
1461 * We should clean the entries by MVA but running a
1462 * for loop over every pv_table entry pointer would
1463 * just complicate the code.
1464 */
1465 flush_cache_louis();
1466 dsb();
1467 isb();
1468
1469 /* remap level 1 table */
1470 for (i = 0; i < PTRS_PER_PGD; pud0++, i++) {
1471 set_pud(pud0,
1472 __pud(__pa(pmd0) | PMD_TYPE_TABLE | L_PGD_SWAPPER));
1473 pmd0 += PTRS_PER_PMD;
1474 }
1475
1476 /* remap pmds for kernel mapping */
1477 phys = __pa(map_start) & PMD_MASK;
1478 do {
1479 *pmdk++ = __pmd(phys | pmdprot);
1480 phys += PMD_SIZE;
1481 } while (phys < map_end);
1482
1483 flush_cache_all();
1484 cpu_switch_mm(pgd0, &init_mm);
1485 cpu_set_ttbr(1, __pa(pgd0) + TTBR1_OFFSET);
1486 local_flush_bp_all();
1487 local_flush_tlb_all();
1488}
1489
1490#else
1491
1492void __init early_paging_init(const struct machine_desc *mdesc,
1493 struct proc_info_list *procinfo)
1494{
1495 if (mdesc->init_meminfo)
1496 mdesc->init_meminfo();
1497}
1498
1499#endif
1500
Russell Kingd111e8f2006-09-27 15:27:33 +01001501/*
1502 * paging_init() sets up the page tables, initialises the zone memory
1503 * maps, and sets up the zero page, bad page and bad page tables.
1504 */
Russell Kingff69a4c2013-07-26 14:55:59 +01001505void __init paging_init(const struct machine_desc *mdesc)
Russell Kingd111e8f2006-09-27 15:27:33 +01001506{
1507 void *zero_page;
1508
1509 build_mem_type_table();
Nicolas Pitre4b5f32c2008-10-06 13:24:40 -04001510 prepare_page_table();
Russell Kinga2227122010-03-25 18:56:05 +00001511 map_lowmem();
Marek Szyprowskic7909502011-12-29 13:09:51 +01001512 dma_contiguous_remap();
Russell Kingd111e8f2006-09-27 15:27:33 +01001513 devicemaps_init(mdesc);
Nicolas Pitred73cd422008-09-15 16:44:55 -04001514 kmap_init();
Joonsoo Kimde40614e2013-04-05 03:16:51 +01001515 tcm_init();
Russell Kingd111e8f2006-09-27 15:27:33 +01001516
1517 top_pmd = pmd_off_k(0xffff0000);
1518
Russell King3abe9d32010-03-25 17:02:59 +00001519 /* allocate the zero page. */
1520 zero_page = early_alloc(PAGE_SIZE);
Russell King2778f622010-07-09 16:27:52 +01001521
Russell King8d717a52010-05-22 19:47:18 +01001522 bootmem_init();
Russell King2778f622010-07-09 16:27:52 +01001523
Russell Kingd111e8f2006-09-27 15:27:33 +01001524 empty_zero_page = virt_to_page(zero_page);
Russell King421fe932009-10-25 10:23:04 +00001525 __flush_dcache_page(NULL, empty_zero_page);
Russell Kingd111e8f2006-09-27 15:27:33 +01001526}