Russell King | e388b80 | 2018-05-10 13:09:54 +0100 | [diff] [blame] | 1 | // SPDX-License-Identifier: GPL-2.0 |
Russell King | 1011510 | 2018-05-14 15:38:55 +0100 | [diff] [blame] | 2 | #include <linux/arm-smccc.h> |
Russell King | e388b80 | 2018-05-10 13:09:54 +0100 | [diff] [blame] | 3 | #include <linux/kernel.h> |
Russell King | 1011510 | 2018-05-14 15:38:55 +0100 | [diff] [blame] | 4 | #include <linux/psci.h> |
Russell King | e388b80 | 2018-05-10 13:09:54 +0100 | [diff] [blame] | 5 | #include <linux/smp.h> |
| 6 | |
Russell King | f5fe12b | 2018-05-14 14:20:21 +0100 | [diff] [blame] | 7 | #include <asm/cp15.h> |
| 8 | #include <asm/cputype.h> |
Russell King | 1011510 | 2018-05-14 15:38:55 +0100 | [diff] [blame] | 9 | #include <asm/proc-fns.h> |
Russell King | f5fe12b | 2018-05-14 14:20:21 +0100 | [diff] [blame] | 10 | #include <asm/system_misc.h> |
| 11 | |
| 12 | #ifdef CONFIG_HARDEN_BRANCH_PREDICTOR |
| 13 | DEFINE_PER_CPU(harden_branch_predictor_fn_t, harden_branch_predictor_fn); |
| 14 | |
Russell King | c44f366 | 2018-05-25 14:40:14 +0100 | [diff] [blame] | 15 | extern void cpu_v7_iciallu_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm); |
| 16 | extern void cpu_v7_bpiall_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm); |
Russell King | 1011510 | 2018-05-14 15:38:55 +0100 | [diff] [blame] | 17 | extern void cpu_v7_smc_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm); |
| 18 | extern void cpu_v7_hvc_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm); |
| 19 | |
Russell King | f5fe12b | 2018-05-14 14:20:21 +0100 | [diff] [blame] | 20 | static void harden_branch_predictor_bpiall(void) |
| 21 | { |
| 22 | write_sysreg(0, BPIALL); |
| 23 | } |
| 24 | |
| 25 | static void harden_branch_predictor_iciallu(void) |
| 26 | { |
| 27 | write_sysreg(0, ICIALLU); |
| 28 | } |
| 29 | |
Russell King | 1011510 | 2018-05-14 15:38:55 +0100 | [diff] [blame] | 30 | static void __maybe_unused call_smc_arch_workaround_1(void) |
| 31 | { |
| 32 | arm_smccc_1_1_smc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL); |
| 33 | } |
| 34 | |
| 35 | static void __maybe_unused call_hvc_arch_workaround_1(void) |
| 36 | { |
| 37 | arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL); |
| 38 | } |
| 39 | |
Russell King | f5fe12b | 2018-05-14 14:20:21 +0100 | [diff] [blame] | 40 | static void cpu_v7_spectre_init(void) |
| 41 | { |
| 42 | const char *spectre_v2_method = NULL; |
| 43 | int cpu = smp_processor_id(); |
| 44 | |
| 45 | if (per_cpu(harden_branch_predictor_fn, cpu)) |
| 46 | return; |
| 47 | |
| 48 | switch (read_cpuid_part()) { |
| 49 | case ARM_CPU_PART_CORTEX_A8: |
| 50 | case ARM_CPU_PART_CORTEX_A9: |
| 51 | case ARM_CPU_PART_CORTEX_A12: |
| 52 | case ARM_CPU_PART_CORTEX_A17: |
| 53 | case ARM_CPU_PART_CORTEX_A73: |
| 54 | case ARM_CPU_PART_CORTEX_A75: |
| 55 | per_cpu(harden_branch_predictor_fn, cpu) = |
| 56 | harden_branch_predictor_bpiall; |
| 57 | spectre_v2_method = "BPIALL"; |
| 58 | break; |
| 59 | |
| 60 | case ARM_CPU_PART_CORTEX_A15: |
| 61 | case ARM_CPU_PART_BRAHMA_B15: |
| 62 | per_cpu(harden_branch_predictor_fn, cpu) = |
| 63 | harden_branch_predictor_iciallu; |
| 64 | spectre_v2_method = "ICIALLU"; |
| 65 | break; |
Russell King | 1011510 | 2018-05-14 15:38:55 +0100 | [diff] [blame] | 66 | |
| 67 | #ifdef CONFIG_ARM_PSCI |
| 68 | default: |
| 69 | /* Other ARM CPUs require no workaround */ |
| 70 | if (read_cpuid_implementor() == ARM_CPU_IMP_ARM) |
| 71 | break; |
| 72 | /* fallthrough */ |
| 73 | /* Cortex A57/A72 require firmware workaround */ |
| 74 | case ARM_CPU_PART_CORTEX_A57: |
| 75 | case ARM_CPU_PART_CORTEX_A72: { |
| 76 | struct arm_smccc_res res; |
| 77 | |
| 78 | if (psci_ops.smccc_version == SMCCC_VERSION_1_0) |
| 79 | break; |
| 80 | |
| 81 | switch (psci_ops.conduit) { |
| 82 | case PSCI_CONDUIT_HVC: |
| 83 | arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_FEATURES_FUNC_ID, |
| 84 | ARM_SMCCC_ARCH_WORKAROUND_1, &res); |
| 85 | if ((int)res.a0 != 0) |
| 86 | break; |
| 87 | per_cpu(harden_branch_predictor_fn, cpu) = |
| 88 | call_hvc_arch_workaround_1; |
Russell King | 383fb3e | 2018-07-19 12:21:31 +0100 | [diff] [blame] | 89 | cpu_do_switch_mm = cpu_v7_hvc_switch_mm; |
Russell King | 1011510 | 2018-05-14 15:38:55 +0100 | [diff] [blame] | 90 | spectre_v2_method = "hypervisor"; |
| 91 | break; |
| 92 | |
| 93 | case PSCI_CONDUIT_SMC: |
| 94 | arm_smccc_1_1_smc(ARM_SMCCC_ARCH_FEATURES_FUNC_ID, |
| 95 | ARM_SMCCC_ARCH_WORKAROUND_1, &res); |
| 96 | if ((int)res.a0 != 0) |
| 97 | break; |
| 98 | per_cpu(harden_branch_predictor_fn, cpu) = |
| 99 | call_smc_arch_workaround_1; |
Russell King | 383fb3e | 2018-07-19 12:21:31 +0100 | [diff] [blame] | 100 | cpu_do_switch_mm = cpu_v7_smc_switch_mm; |
Russell King | 1011510 | 2018-05-14 15:38:55 +0100 | [diff] [blame] | 101 | spectre_v2_method = "firmware"; |
| 102 | break; |
| 103 | |
| 104 | default: |
| 105 | break; |
| 106 | } |
Russell King | f5fe12b | 2018-05-14 14:20:21 +0100 | [diff] [blame] | 107 | } |
Russell King | 1011510 | 2018-05-14 15:38:55 +0100 | [diff] [blame] | 108 | #endif |
| 109 | } |
| 110 | |
Russell King | f5fe12b | 2018-05-14 14:20:21 +0100 | [diff] [blame] | 111 | if (spectre_v2_method) |
| 112 | pr_info("CPU%u: Spectre v2: using %s workaround\n", |
| 113 | smp_processor_id(), spectre_v2_method); |
| 114 | } |
| 115 | #else |
| 116 | static void cpu_v7_spectre_init(void) |
| 117 | { |
| 118 | } |
| 119 | #endif |
| 120 | |
| 121 | static __maybe_unused bool cpu_v7_check_auxcr_set(bool *warned, |
Russell King | e388b80 | 2018-05-10 13:09:54 +0100 | [diff] [blame] | 122 | u32 mask, const char *msg) |
| 123 | { |
| 124 | u32 aux_cr; |
| 125 | |
| 126 | asm("mrc p15, 0, %0, c1, c0, 1" : "=r" (aux_cr)); |
| 127 | |
| 128 | if ((aux_cr & mask) != mask) { |
| 129 | if (!*warned) |
| 130 | pr_err("CPU%u: %s", smp_processor_id(), msg); |
| 131 | *warned = true; |
Russell King | f5fe12b | 2018-05-14 14:20:21 +0100 | [diff] [blame] | 132 | return false; |
Russell King | e388b80 | 2018-05-10 13:09:54 +0100 | [diff] [blame] | 133 | } |
Russell King | f5fe12b | 2018-05-14 14:20:21 +0100 | [diff] [blame] | 134 | return true; |
Russell King | e388b80 | 2018-05-10 13:09:54 +0100 | [diff] [blame] | 135 | } |
| 136 | |
| 137 | static DEFINE_PER_CPU(bool, spectre_warned); |
| 138 | |
Russell King | f5fe12b | 2018-05-14 14:20:21 +0100 | [diff] [blame] | 139 | static bool check_spectre_auxcr(bool *warned, u32 bit) |
Russell King | e388b80 | 2018-05-10 13:09:54 +0100 | [diff] [blame] | 140 | { |
Russell King | f5fe12b | 2018-05-14 14:20:21 +0100 | [diff] [blame] | 141 | return IS_ENABLED(CONFIG_HARDEN_BRANCH_PREDICTOR) && |
Russell King | e388b80 | 2018-05-10 13:09:54 +0100 | [diff] [blame] | 142 | cpu_v7_check_auxcr_set(warned, bit, |
| 143 | "Spectre v2: firmware did not set auxiliary control register IBE bit, system vulnerable\n"); |
| 144 | } |
| 145 | |
| 146 | void cpu_v7_ca8_ibe(void) |
| 147 | { |
Russell King | f5fe12b | 2018-05-14 14:20:21 +0100 | [diff] [blame] | 148 | if (check_spectre_auxcr(this_cpu_ptr(&spectre_warned), BIT(6))) |
| 149 | cpu_v7_spectre_init(); |
Russell King | e388b80 | 2018-05-10 13:09:54 +0100 | [diff] [blame] | 150 | } |
| 151 | |
| 152 | void cpu_v7_ca15_ibe(void) |
| 153 | { |
Russell King | f5fe12b | 2018-05-14 14:20:21 +0100 | [diff] [blame] | 154 | if (check_spectre_auxcr(this_cpu_ptr(&spectre_warned), BIT(0))) |
| 155 | cpu_v7_spectre_init(); |
| 156 | } |
| 157 | |
| 158 | void cpu_v7_bugs_init(void) |
| 159 | { |
| 160 | cpu_v7_spectre_init(); |
Russell King | e388b80 | 2018-05-10 13:09:54 +0100 | [diff] [blame] | 161 | } |