Christoffer Dall | 342cd0a | 2013-01-20 18:28:06 -0500 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 2012 - Virtual Open Systems and Columbia University |
| 3 | * Author: Christoffer Dall <c.dall@virtualopensystems.com> |
| 4 | * |
| 5 | * This program is free software; you can redistribute it and/or modify |
| 6 | * it under the terms of the GNU General Public License, version 2, as |
| 7 | * published by the Free Software Foundation. |
| 8 | * |
| 9 | * This program is distributed in the hope that it will be useful, |
| 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 12 | * GNU General Public License for more details. |
| 13 | * |
| 14 | * You should have received a copy of the GNU General Public License |
| 15 | * along with this program; if not, write to the Free Software |
| 16 | * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. |
| 17 | */ |
| 18 | |
| 19 | #ifndef __ARM_KVM_MMU_H__ |
| 20 | #define __ARM_KVM_MMU_H__ |
| 21 | |
Marc Zyngier | 5a677ce | 2013-04-12 19:12:06 +0100 | [diff] [blame] | 22 | #include <asm/memory.h> |
| 23 | #include <asm/page.h> |
Marc Zyngier | c62ee2b | 2012-10-15 11:27:37 +0100 | [diff] [blame] | 24 | |
Marc Zyngier | 06e8c3b | 2012-10-28 01:09:14 +0100 | [diff] [blame] | 25 | /* |
| 26 | * We directly use the kernel VA for the HYP, as we can directly share |
| 27 | * the mapping (HTTBR "covers" TTBR1). |
| 28 | */ |
Marc Zyngier | 6c41a41 | 2016-06-30 18:40:51 +0100 | [diff] [blame] | 29 | #define kern_hyp_va(kva) (kva) |
Marc Zyngier | 06e8c3b | 2012-10-28 01:09:14 +0100 | [diff] [blame] | 30 | |
Marc Zyngier | 44a497a | 2017-12-03 19:28:56 +0000 | [diff] [blame] | 31 | /* Contrary to arm64, there is no need to generate a PC-relative address */ |
| 32 | #define hyp_symbol_addr(s) \ |
| 33 | ({ \ |
| 34 | typeof(s) *addr = &(s); \ |
| 35 | addr; \ |
| 36 | }) |
| 37 | |
Marc Zyngier | 5a677ce | 2013-04-12 19:12:06 +0100 | [diff] [blame] | 38 | #ifndef __ASSEMBLY__ |
| 39 | |
Marc Zyngier | 363ef89 | 2014-12-19 16:48:06 +0000 | [diff] [blame] | 40 | #include <linux/highmem.h> |
Marc Zyngier | 5a677ce | 2013-04-12 19:12:06 +0100 | [diff] [blame] | 41 | #include <asm/cacheflush.h> |
Marc Zyngier | 91c703e | 2017-10-23 17:11:17 +0100 | [diff] [blame] | 42 | #include <asm/cputype.h> |
Suzuki K Poulose | e55cac5 | 2018-09-26 17:32:44 +0100 | [diff] [blame] | 43 | #include <asm/kvm_arm.h> |
Marc Zyngier | 91c703e | 2017-10-23 17:11:17 +0100 | [diff] [blame] | 44 | #include <asm/kvm_hyp.h> |
Marc Zyngier | 5a677ce | 2013-04-12 19:12:06 +0100 | [diff] [blame] | 45 | #include <asm/pgalloc.h> |
Suzuki K Poulose | b1ae9a3 | 2016-03-22 14:08:17 +0000 | [diff] [blame] | 46 | #include <asm/stage2_pgtable.h> |
Marc Zyngier | 5a677ce | 2013-04-12 19:12:06 +0100 | [diff] [blame] | 47 | |
Marc Zyngier | e3f019b | 2017-12-04 17:04:38 +0000 | [diff] [blame] | 48 | /* Ensure compatibility with arm64 */ |
| 49 | #define VA_BITS 32 |
| 50 | |
Suzuki K Poulose | e55cac5 | 2018-09-26 17:32:44 +0100 | [diff] [blame] | 51 | #define kvm_phys_shift(kvm) KVM_PHYS_SHIFT |
| 52 | #define kvm_phys_size(kvm) (1ULL << kvm_phys_shift(kvm)) |
| 53 | #define kvm_phys_mask(kvm) (kvm_phys_size(kvm) - 1ULL) |
| 54 | #define kvm_vttbr_baddr_mask(kvm) VTTBR_BADDR_MASK |
| 55 | |
| 56 | #define stage2_pgd_size(kvm) (PTRS_PER_S2_PGD * sizeof(pgd_t)) |
| 57 | |
Marc Zyngier | c8dddec | 2016-06-13 15:00:45 +0100 | [diff] [blame] | 58 | int create_hyp_mappings(void *from, void *to, pgprot_t prot); |
Marc Zyngier | 807a378 | 2017-12-04 16:26:09 +0000 | [diff] [blame] | 59 | int create_hyp_io_mappings(phys_addr_t phys_addr, size_t size, |
Marc Zyngier | 1bb32a4 | 2017-12-04 16:43:23 +0000 | [diff] [blame] | 60 | void __iomem **kaddr, |
| 61 | void __iomem **haddr); |
Marc Zyngier | dc2e463 | 2018-02-13 11:00:29 +0000 | [diff] [blame] | 62 | int create_hyp_exec_mappings(phys_addr_t phys_addr, size_t size, |
| 63 | void **haddr); |
Marc Zyngier | 4f72827 | 2013-04-12 19:12:05 +0100 | [diff] [blame] | 64 | void free_hyp_pgds(void); |
Christoffer Dall | 342cd0a | 2013-01-20 18:28:06 -0500 | [diff] [blame] | 65 | |
Christoffer Dall | 957db10 | 2014-11-27 10:35:03 +0100 | [diff] [blame] | 66 | void stage2_unmap_vm(struct kvm *kvm); |
Christoffer Dall | d5d8184 | 2013-01-20 18:28:07 -0500 | [diff] [blame] | 67 | int kvm_alloc_stage2_pgd(struct kvm *kvm); |
| 68 | void kvm_free_stage2_pgd(struct kvm *kvm); |
| 69 | int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa, |
Ard Biesheuvel | c40f2f8 | 2014-09-17 14:56:18 -0700 | [diff] [blame] | 70 | phys_addr_t pa, unsigned long size, bool writable); |
Christoffer Dall | d5d8184 | 2013-01-20 18:28:07 -0500 | [diff] [blame] | 71 | |
| 72 | int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run); |
| 73 | |
| 74 | void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu); |
| 75 | |
Christoffer Dall | 342cd0a | 2013-01-20 18:28:06 -0500 | [diff] [blame] | 76 | phys_addr_t kvm_mmu_get_httbr(void); |
Marc Zyngier | 5a677ce | 2013-04-12 19:12:06 +0100 | [diff] [blame] | 77 | phys_addr_t kvm_get_idmap_vector(void); |
Christoffer Dall | 342cd0a | 2013-01-20 18:28:06 -0500 | [diff] [blame] | 78 | int kvm_mmu_init(void); |
| 79 | void kvm_clear_hyp_idmap(void); |
Christoffer Dall | 94f8e64 | 2013-01-20 18:28:12 -0500 | [diff] [blame] | 80 | |
Marc Zyngier | 0db9dd8 | 2018-06-27 15:51:05 +0100 | [diff] [blame] | 81 | #define kvm_mk_pmd(ptep) __pmd(__pa(ptep) | PMD_TYPE_TABLE) |
| 82 | #define kvm_mk_pud(pmdp) __pud(__pa(pmdp) | PMD_TYPE_TABLE) |
| 83 | #define kvm_mk_pgd(pudp) ({ BUILD_BUG(); 0; }) |
| 84 | |
Punit Agrawal | f8df733 | 2018-12-11 17:10:36 +0000 | [diff] [blame] | 85 | #define kvm_pfn_pte(pfn, prot) pfn_pte(pfn, prot) |
| 86 | #define kvm_pfn_pmd(pfn, prot) pfn_pmd(pfn, prot) |
Punit Agrawal | b8e0ba7 | 2018-12-11 17:10:41 +0000 | [diff] [blame] | 87 | #define kvm_pfn_pud(pfn, prot) (__pud(0)) |
Punit Agrawal | f8df733 | 2018-12-11 17:10:36 +0000 | [diff] [blame] | 88 | |
Punit Agrawal | eb3f0624 | 2018-12-11 17:10:39 +0000 | [diff] [blame] | 89 | #define kvm_pud_pfn(pud) ({ WARN_ON(1); 0; }) |
| 90 | |
| 91 | |
Punit Agrawal | f8df733 | 2018-12-11 17:10:36 +0000 | [diff] [blame] | 92 | #define kvm_pmd_mkhuge(pmd) pmd_mkhuge(pmd) |
Punit Agrawal | b8e0ba7 | 2018-12-11 17:10:41 +0000 | [diff] [blame] | 93 | /* No support for pud hugepages */ |
| 94 | #define kvm_pud_mkhuge(pud) ( {WARN_ON(1); pud; }) |
Punit Agrawal | f8df733 | 2018-12-11 17:10:36 +0000 | [diff] [blame] | 95 | |
Punit Agrawal | 4ea5af5 | 2018-12-11 17:10:37 +0000 | [diff] [blame] | 96 | /* |
| 97 | * The following kvm_*pud*() functions are provided strictly to allow |
| 98 | * sharing code with arm64. They should never be called in practice. |
| 99 | */ |
| 100 | static inline void kvm_set_s2pud_readonly(pud_t *pud) |
| 101 | { |
| 102 | WARN_ON(1); |
| 103 | } |
| 104 | |
| 105 | static inline bool kvm_s2pud_readonly(pud_t *pud) |
| 106 | { |
| 107 | WARN_ON(1); |
| 108 | return false; |
| 109 | } |
| 110 | |
Punit Agrawal | b8e0ba7 | 2018-12-11 17:10:41 +0000 | [diff] [blame] | 111 | static inline void kvm_set_pud(pud_t *pud, pud_t new_pud) |
| 112 | { |
| 113 | WARN_ON(1); |
| 114 | } |
| 115 | |
| 116 | static inline pud_t kvm_s2pud_mkwrite(pud_t pud) |
| 117 | { |
| 118 | WARN_ON(1); |
| 119 | return pud; |
| 120 | } |
| 121 | |
| 122 | static inline pud_t kvm_s2pud_mkexec(pud_t pud) |
| 123 | { |
| 124 | WARN_ON(1); |
| 125 | return pud; |
| 126 | } |
| 127 | |
Punit Agrawal | 86d1c55 | 2018-12-11 17:10:38 +0000 | [diff] [blame] | 128 | static inline bool kvm_s2pud_exec(pud_t *pud) |
| 129 | { |
| 130 | WARN_ON(1); |
| 131 | return false; |
| 132 | } |
| 133 | |
Punit Agrawal | eb3f0624 | 2018-12-11 17:10:39 +0000 | [diff] [blame] | 134 | static inline pud_t kvm_s2pud_mkyoung(pud_t pud) |
| 135 | { |
| 136 | BUG(); |
| 137 | return pud; |
| 138 | } |
| 139 | |
Punit Agrawal | 35a6396 | 2018-12-11 17:10:40 +0000 | [diff] [blame] | 140 | static inline bool kvm_s2pud_young(pud_t pud) |
| 141 | { |
| 142 | WARN_ON(1); |
| 143 | return false; |
| 144 | } |
| 145 | |
Catalin Marinas | 0648505 | 2016-04-13 17:57:37 +0100 | [diff] [blame] | 146 | static inline pte_t kvm_s2pte_mkwrite(pte_t pte) |
Marc Zyngier | c62ee2b | 2012-10-15 11:27:37 +0100 | [diff] [blame] | 147 | { |
Catalin Marinas | 0648505 | 2016-04-13 17:57:37 +0100 | [diff] [blame] | 148 | pte_val(pte) |= L_PTE_S2_RDWR; |
| 149 | return pte; |
Marc Zyngier | c62ee2b | 2012-10-15 11:27:37 +0100 | [diff] [blame] | 150 | } |
| 151 | |
Catalin Marinas | 0648505 | 2016-04-13 17:57:37 +0100 | [diff] [blame] | 152 | static inline pmd_t kvm_s2pmd_mkwrite(pmd_t pmd) |
Christoffer Dall | ad361f0 | 2012-11-01 17:14:45 +0100 | [diff] [blame] | 153 | { |
Catalin Marinas | 0648505 | 2016-04-13 17:57:37 +0100 | [diff] [blame] | 154 | pmd_val(pmd) |= L_PMD_S2_RDWR; |
| 155 | return pmd; |
Christoffer Dall | ad361f0 | 2012-11-01 17:14:45 +0100 | [diff] [blame] | 156 | } |
| 157 | |
Marc Zyngier | d0e22b4 | 2017-10-23 17:11:19 +0100 | [diff] [blame] | 158 | static inline pte_t kvm_s2pte_mkexec(pte_t pte) |
| 159 | { |
| 160 | pte_val(pte) &= ~L_PTE_XN; |
| 161 | return pte; |
| 162 | } |
| 163 | |
| 164 | static inline pmd_t kvm_s2pmd_mkexec(pmd_t pmd) |
| 165 | { |
| 166 | pmd_val(pmd) &= ~PMD_SECT_XN; |
| 167 | return pmd; |
| 168 | } |
| 169 | |
Mario Smarduch | c647355 | 2015-01-15 15:58:56 -0800 | [diff] [blame] | 170 | static inline void kvm_set_s2pte_readonly(pte_t *pte) |
| 171 | { |
| 172 | pte_val(*pte) = (pte_val(*pte) & ~L_PTE_S2_RDWR) | L_PTE_S2_RDONLY; |
| 173 | } |
| 174 | |
| 175 | static inline bool kvm_s2pte_readonly(pte_t *pte) |
| 176 | { |
| 177 | return (pte_val(*pte) & L_PTE_S2_RDWR) == L_PTE_S2_RDONLY; |
| 178 | } |
| 179 | |
Marc Zyngier | 7a3796d | 2017-10-23 17:11:21 +0100 | [diff] [blame] | 180 | static inline bool kvm_s2pte_exec(pte_t *pte) |
| 181 | { |
| 182 | return !(pte_val(*pte) & L_PTE_XN); |
| 183 | } |
| 184 | |
Mario Smarduch | c647355 | 2015-01-15 15:58:56 -0800 | [diff] [blame] | 185 | static inline void kvm_set_s2pmd_readonly(pmd_t *pmd) |
| 186 | { |
| 187 | pmd_val(*pmd) = (pmd_val(*pmd) & ~L_PMD_S2_RDWR) | L_PMD_S2_RDONLY; |
| 188 | } |
| 189 | |
| 190 | static inline bool kvm_s2pmd_readonly(pmd_t *pmd) |
| 191 | { |
| 192 | return (pmd_val(*pmd) & L_PMD_S2_RDWR) == L_PMD_S2_RDONLY; |
| 193 | } |
| 194 | |
Marc Zyngier | 7a3796d | 2017-10-23 17:11:21 +0100 | [diff] [blame] | 195 | static inline bool kvm_s2pmd_exec(pmd_t *pmd) |
| 196 | { |
| 197 | return !(pmd_val(*pmd) & PMD_SECT_XN); |
| 198 | } |
| 199 | |
Christoffer Dall | 4f853a7 | 2014-05-09 23:31:31 +0200 | [diff] [blame] | 200 | static inline bool kvm_page_empty(void *ptr) |
| 201 | { |
| 202 | struct page *ptr_page = virt_to_page(ptr); |
| 203 | return page_count(ptr_page) == 1; |
| 204 | } |
| 205 | |
Christoffer Dall | 38f791a | 2014-10-10 12:14:28 +0200 | [diff] [blame] | 206 | #define kvm_pte_table_empty(kvm, ptep) kvm_page_empty(ptep) |
| 207 | #define kvm_pmd_table_empty(kvm, pmdp) kvm_page_empty(pmdp) |
Suzuki K Poulose | b1d030a | 2016-03-22 17:15:55 +0000 | [diff] [blame] | 208 | #define kvm_pud_table_empty(kvm, pudp) false |
Christoffer Dall | 4f853a7 | 2014-05-09 23:31:31 +0200 | [diff] [blame] | 209 | |
Suzuki K Poulose | b1d030a | 2016-03-22 17:15:55 +0000 | [diff] [blame] | 210 | #define hyp_pte_table_empty(ptep) kvm_page_empty(ptep) |
| 211 | #define hyp_pmd_table_empty(pmdp) kvm_page_empty(pmdp) |
| 212 | #define hyp_pud_table_empty(pudp) false |
Marc Zyngier | a987370 | 2015-03-10 19:06:59 +0000 | [diff] [blame] | 213 | |
Marc Zyngier | c62ee2b | 2012-10-15 11:27:37 +0100 | [diff] [blame] | 214 | struct kvm; |
| 215 | |
Marc Zyngier | 1597930 | 2014-01-14 19:13:10 +0000 | [diff] [blame] | 216 | #define kvm_flush_dcache_to_poc(a,l) __cpuc_flush_dcache_area((a), (l)) |
| 217 | |
| 218 | static inline bool vcpu_has_cache_enabled(struct kvm_vcpu *vcpu) |
| 219 | { |
Marc Zyngier | fb32a52 | 2016-01-03 11:26:01 +0000 | [diff] [blame] | 220 | return (vcpu_cp15(vcpu, c1_SCTLR) & 0b101) == 0b101; |
Marc Zyngier | 1597930 | 2014-01-14 19:13:10 +0000 | [diff] [blame] | 221 | } |
| 222 | |
Marc Zyngier | 17ab9d5 | 2017-10-23 17:11:22 +0100 | [diff] [blame] | 223 | static inline void __clean_dcache_guest_page(kvm_pfn_t pfn, unsigned long size) |
Marc Zyngier | c62ee2b | 2012-10-15 11:27:37 +0100 | [diff] [blame] | 224 | { |
| 225 | /* |
Marc Zyngier | a15f693 | 2017-10-23 17:11:15 +0100 | [diff] [blame] | 226 | * Clean the dcache to the Point of Coherency. |
Marc Zyngier | 0d3e4d4 | 2015-01-05 21:13:24 +0000 | [diff] [blame] | 227 | * |
| 228 | * We need to do this through a kernel mapping (using the |
| 229 | * user-space mapping has proved to be the wrong |
| 230 | * solution). For that, we need to kmap one page at a time, |
| 231 | * and iterate over the range. |
Marc Zyngier | c62ee2b | 2012-10-15 11:27:37 +0100 | [diff] [blame] | 232 | */ |
Marc Zyngier | 0d3e4d4 | 2015-01-05 21:13:24 +0000 | [diff] [blame] | 233 | |
Jan Kiszka | a050dfb | 2015-02-07 22:21:20 +0100 | [diff] [blame] | 234 | VM_BUG_ON(size & ~PAGE_MASK); |
Marc Zyngier | 0d3e4d4 | 2015-01-05 21:13:24 +0000 | [diff] [blame] | 235 | |
Marc Zyngier | 0d3e4d4 | 2015-01-05 21:13:24 +0000 | [diff] [blame] | 236 | while (size) { |
| 237 | void *va = kmap_atomic_pfn(pfn); |
| 238 | |
Marc Zyngier | 8f36eba | 2017-01-25 12:29:59 +0000 | [diff] [blame] | 239 | kvm_flush_dcache_to_poc(va, PAGE_SIZE); |
Marc Zyngier | 0d3e4d4 | 2015-01-05 21:13:24 +0000 | [diff] [blame] | 240 | |
Marc Zyngier | 0d3e4d4 | 2015-01-05 21:13:24 +0000 | [diff] [blame] | 241 | size -= PAGE_SIZE; |
| 242 | pfn++; |
| 243 | |
| 244 | kunmap_atomic(va); |
| 245 | } |
Marc Zyngier | a15f693 | 2017-10-23 17:11:15 +0100 | [diff] [blame] | 246 | } |
Marc Zyngier | 0d3e4d4 | 2015-01-05 21:13:24 +0000 | [diff] [blame] | 247 | |
Marc Zyngier | 17ab9d5 | 2017-10-23 17:11:22 +0100 | [diff] [blame] | 248 | static inline void __invalidate_icache_guest_page(kvm_pfn_t pfn, |
Marc Zyngier | a15f693 | 2017-10-23 17:11:15 +0100 | [diff] [blame] | 249 | unsigned long size) |
| 250 | { |
Marc Zyngier | 91c703e | 2017-10-23 17:11:17 +0100 | [diff] [blame] | 251 | u32 iclsz; |
| 252 | |
Marc Zyngier | a15f693 | 2017-10-23 17:11:15 +0100 | [diff] [blame] | 253 | /* |
| 254 | * If we are going to insert an instruction page and the icache is |
| 255 | * either VIPT or PIPT, there is a potential problem where the host |
| 256 | * (or another VM) may have used the same page as this guest, and we |
| 257 | * read incorrect data from the icache. If we're using a PIPT cache, |
| 258 | * we can invalidate just that page, but if we are using a VIPT cache |
| 259 | * we need to invalidate the entire icache - damn shame - as written |
| 260 | * in the ARM ARM (DDI 0406C.b - Page B3-1393). |
| 261 | * |
| 262 | * VIVT caches are tagged using both the ASID and the VMID and doesn't |
| 263 | * need any kind of flushing (DDI 0406C.b - Page B3-1392). |
| 264 | */ |
| 265 | |
| 266 | VM_BUG_ON(size & ~PAGE_MASK); |
| 267 | |
| 268 | if (icache_is_vivt_asid_tagged()) |
| 269 | return; |
| 270 | |
| 271 | if (!icache_is_pipt()) { |
Marc Zyngier | c62ee2b | 2012-10-15 11:27:37 +0100 | [diff] [blame] | 272 | /* any kind of VIPT cache */ |
| 273 | __flush_icache_all(); |
Marc Zyngier | a15f693 | 2017-10-23 17:11:15 +0100 | [diff] [blame] | 274 | return; |
| 275 | } |
| 276 | |
Marc Zyngier | 91c703e | 2017-10-23 17:11:17 +0100 | [diff] [blame] | 277 | /* |
| 278 | * CTR IminLine contains Log2 of the number of words in the |
| 279 | * cache line, so we can get the number of words as |
| 280 | * 2 << (IminLine - 1). To get the number of bytes, we |
| 281 | * multiply by 4 (the number of bytes in a 32-bit word), and |
| 282 | * get 4 << (IminLine). |
| 283 | */ |
| 284 | iclsz = 4 << (read_cpuid(CPUID_CACHETYPE) & 0xf); |
| 285 | |
Marc Zyngier | a15f693 | 2017-10-23 17:11:15 +0100 | [diff] [blame] | 286 | while (size) { |
| 287 | void *va = kmap_atomic_pfn(pfn); |
Marc Zyngier | 91c703e | 2017-10-23 17:11:17 +0100 | [diff] [blame] | 288 | void *end = va + PAGE_SIZE; |
| 289 | void *addr = va; |
Marc Zyngier | a15f693 | 2017-10-23 17:11:15 +0100 | [diff] [blame] | 290 | |
Marc Zyngier | 91c703e | 2017-10-23 17:11:17 +0100 | [diff] [blame] | 291 | do { |
| 292 | write_sysreg(addr, ICIMVAU); |
| 293 | addr += iclsz; |
| 294 | } while (addr < end); |
| 295 | |
| 296 | dsb(ishst); |
| 297 | isb(); |
Marc Zyngier | 363ef89 | 2014-12-19 16:48:06 +0000 | [diff] [blame] | 298 | |
| 299 | size -= PAGE_SIZE; |
| 300 | pfn++; |
| 301 | |
| 302 | kunmap_atomic(va); |
| 303 | } |
| 304 | |
Marc Zyngier | 91c703e | 2017-10-23 17:11:17 +0100 | [diff] [blame] | 305 | /* Check if we need to invalidate the BTB */ |
| 306 | if ((read_cpuid_ext(CPUID_EXT_MMFR1) >> 28) != 4) { |
| 307 | write_sysreg(0, BPIALLIS); |
| 308 | dsb(ishst); |
| 309 | isb(); |
Marc Zyngier | c62ee2b | 2012-10-15 11:27:37 +0100 | [diff] [blame] | 310 | } |
| 311 | } |
| 312 | |
Marc Zyngier | 363ef89 | 2014-12-19 16:48:06 +0000 | [diff] [blame] | 313 | static inline void __kvm_flush_dcache_pte(pte_t pte) |
| 314 | { |
| 315 | void *va = kmap_atomic(pte_page(pte)); |
| 316 | |
| 317 | kvm_flush_dcache_to_poc(va, PAGE_SIZE); |
| 318 | |
| 319 | kunmap_atomic(va); |
| 320 | } |
| 321 | |
| 322 | static inline void __kvm_flush_dcache_pmd(pmd_t pmd) |
| 323 | { |
| 324 | unsigned long size = PMD_SIZE; |
Dan Williams | ba049e9 | 2016-01-15 16:56:11 -0800 | [diff] [blame] | 325 | kvm_pfn_t pfn = pmd_pfn(pmd); |
Marc Zyngier | 363ef89 | 2014-12-19 16:48:06 +0000 | [diff] [blame] | 326 | |
| 327 | while (size) { |
| 328 | void *va = kmap_atomic_pfn(pfn); |
| 329 | |
| 330 | kvm_flush_dcache_to_poc(va, PAGE_SIZE); |
| 331 | |
| 332 | pfn++; |
| 333 | size -= PAGE_SIZE; |
| 334 | |
| 335 | kunmap_atomic(va); |
| 336 | } |
| 337 | } |
| 338 | |
| 339 | static inline void __kvm_flush_dcache_pud(pud_t pud) |
| 340 | { |
| 341 | } |
| 342 | |
Santosh Shilimkar | 4fda342 | 2013-11-19 14:59:12 -0500 | [diff] [blame] | 343 | #define kvm_virt_to_phys(x) virt_to_idmap((unsigned long)(x)) |
Marc Zyngier | 5a677ce | 2013-04-12 19:12:06 +0100 | [diff] [blame] | 344 | |
Marc Zyngier | 3c1e716 | 2014-12-19 16:05:31 +0000 | [diff] [blame] | 345 | void kvm_set_way_flush(struct kvm_vcpu *vcpu); |
| 346 | void kvm_toggle_cache(struct kvm_vcpu *vcpu, bool was_enabled); |
Marc Zyngier | 9d218a1 | 2014-01-15 12:50:23 +0000 | [diff] [blame] | 347 | |
Ard Biesheuvel | e4c5a68 | 2015-03-19 16:42:28 +0000 | [diff] [blame] | 348 | static inline bool __kvm_cpu_uses_extended_idmap(void) |
| 349 | { |
| 350 | return false; |
| 351 | } |
| 352 | |
Kristina Martsenko | fa2a844 | 2017-12-13 17:07:24 +0000 | [diff] [blame] | 353 | static inline unsigned long __kvm_idmap_ptrs_per_pgd(void) |
| 354 | { |
| 355 | return PTRS_PER_PGD; |
| 356 | } |
| 357 | |
Ard Biesheuvel | e4c5a68 | 2015-03-19 16:42:28 +0000 | [diff] [blame] | 358 | static inline void __kvm_extend_hypmap(pgd_t *boot_hyp_pgd, |
| 359 | pgd_t *hyp_pgd, |
| 360 | pgd_t *merged_hyp_pgd, |
| 361 | unsigned long hyp_idmap_start) { } |
| 362 | |
Vladimir Murzin | 20475f7 | 2015-11-16 11:28:18 +0000 | [diff] [blame] | 363 | static inline unsigned int kvm_get_vmid_bits(void) |
| 364 | { |
| 365 | return 8; |
| 366 | } |
| 367 | |
Andre Przywara | bf30824 | 2018-05-11 15:20:14 +0100 | [diff] [blame] | 368 | /* |
| 369 | * We are not in the kvm->srcu critical section most of the time, so we take |
| 370 | * the SRCU read lock here. Since we copy the data from the user page, we |
| 371 | * can immediately drop the lock again. |
| 372 | */ |
| 373 | static inline int kvm_read_guest_lock(struct kvm *kvm, |
| 374 | gpa_t gpa, void *data, unsigned long len) |
| 375 | { |
| 376 | int srcu_idx = srcu_read_lock(&kvm->srcu); |
| 377 | int ret = kvm_read_guest(kvm, gpa, data, len); |
| 378 | |
| 379 | srcu_read_unlock(&kvm->srcu, srcu_idx); |
| 380 | |
| 381 | return ret; |
| 382 | } |
| 383 | |
Marc Zyngier | a6ecfb1 | 2019-03-19 12:47:11 +0000 | [diff] [blame] | 384 | static inline int kvm_write_guest_lock(struct kvm *kvm, gpa_t gpa, |
| 385 | const void *data, unsigned long len) |
| 386 | { |
| 387 | int srcu_idx = srcu_read_lock(&kvm->srcu); |
| 388 | int ret = kvm_write_guest(kvm, gpa, data, len); |
| 389 | |
| 390 | srcu_read_unlock(&kvm->srcu, srcu_idx); |
| 391 | |
| 392 | return ret; |
| 393 | } |
| 394 | |
Marc Zyngier | 6840bdd | 2018-01-03 16:38:35 +0000 | [diff] [blame] | 395 | static inline void *kvm_get_hyp_vector(void) |
| 396 | { |
Marc Zyngier | 3f7e8e2 | 2018-02-01 11:07:35 +0000 | [diff] [blame] | 397 | switch(read_cpuid_part()) { |
| 398 | #ifdef CONFIG_HARDEN_BRANCH_PREDICTOR |
| 399 | case ARM_CPU_PART_CORTEX_A12: |
| 400 | case ARM_CPU_PART_CORTEX_A17: |
| 401 | { |
| 402 | extern char __kvm_hyp_vector_bp_inv[]; |
| 403 | return kvm_ksym_ref(__kvm_hyp_vector_bp_inv); |
| 404 | } |
| 405 | |
Russell King | 3c908e16 | 2018-05-10 17:52:18 +0100 | [diff] [blame] | 406 | case ARM_CPU_PART_BRAHMA_B15: |
Marc Zyngier | 0c47ac8 | 2018-02-01 11:07:38 +0000 | [diff] [blame] | 407 | case ARM_CPU_PART_CORTEX_A15: |
| 408 | { |
| 409 | extern char __kvm_hyp_vector_ic_inv[]; |
| 410 | return kvm_ksym_ref(__kvm_hyp_vector_ic_inv); |
| 411 | } |
Marc Zyngier | 3f7e8e2 | 2018-02-01 11:07:35 +0000 | [diff] [blame] | 412 | #endif |
| 413 | default: |
| 414 | { |
| 415 | extern char __kvm_hyp_vector[]; |
| 416 | return kvm_ksym_ref(__kvm_hyp_vector); |
| 417 | } |
| 418 | } |
Marc Zyngier | 6840bdd | 2018-01-03 16:38:35 +0000 | [diff] [blame] | 419 | } |
| 420 | |
| 421 | static inline int kvm_map_vectors(void) |
| 422 | { |
| 423 | return 0; |
| 424 | } |
| 425 | |
Marc Zyngier | 55e3748 | 2018-05-29 13:11:16 +0100 | [diff] [blame] | 426 | static inline int hyp_map_aux_data(void) |
| 427 | { |
| 428 | return 0; |
| 429 | } |
| 430 | |
Kristina Martsenko | 529c4b0 | 2017-12-13 17:07:18 +0000 | [diff] [blame] | 431 | #define kvm_phys_to_vttbr(addr) (addr) |
| 432 | |
Suzuki K Poulose | 0f62f0e | 2018-09-26 17:32:52 +0100 | [diff] [blame] | 433 | static inline void kvm_set_ipa_limit(void) {} |
| 434 | |
Christoffer Dall | e329fb7 | 2018-12-11 15:26:31 +0100 | [diff] [blame] | 435 | static __always_inline u64 kvm_get_vttbr(struct kvm *kvm) |
Vladimir Murzin | ab51002 | 2018-07-31 14:08:57 +0100 | [diff] [blame] | 436 | { |
Christoffer Dall | e329fb7 | 2018-12-11 15:26:31 +0100 | [diff] [blame] | 437 | struct kvm_vmid *vmid = &kvm->arch.vmid; |
| 438 | u64 vmid_field, baddr; |
| 439 | |
| 440 | baddr = kvm->arch.pgd_phys; |
| 441 | vmid_field = (u64)vmid->vmid << VTTBR_VMID_SHIFT; |
| 442 | return kvm_phys_to_vttbr(baddr) | vmid_field; |
Vladimir Murzin | ab51002 | 2018-07-31 14:08:57 +0100 | [diff] [blame] | 443 | } |
| 444 | |
Marc Zyngier | 5a677ce | 2013-04-12 19:12:06 +0100 | [diff] [blame] | 445 | #endif /* !__ASSEMBLY__ */ |
| 446 | |
Christoffer Dall | 342cd0a | 2013-01-20 18:28:06 -0500 | [diff] [blame] | 447 | #endif /* __ARM_KVM_MMU_H__ */ |