blob: 38bddb9a2076942e6c961151b266d49459d7bccb [file] [log] [blame]
Mark Rutland8a4da6e2012-11-12 14:33:44 +00001/*
2 * linux/drivers/clocksource/arm_arch_timer.c
3 *
4 * Copyright (C) 2011 ARM Ltd.
5 * All Rights Reserved
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 */
Marc Zyngierf005bd72016-08-01 10:54:15 +010011
12#define pr_fmt(fmt) "arm_arch_timer: " fmt
13
Mark Rutland8a4da6e2012-11-12 14:33:44 +000014#include <linux/init.h>
15#include <linux/kernel.h>
16#include <linux/device.h>
17#include <linux/smp.h>
18#include <linux/cpu.h>
Sudeep KarkadaNagesha346e7482013-08-23 15:53:15 +010019#include <linux/cpu_pm.h>
Mark Rutland8a4da6e2012-11-12 14:33:44 +000020#include <linux/clockchips.h>
Richard Cochran7c8f1e72015-01-06 14:26:13 +010021#include <linux/clocksource.h>
Mark Rutland8a4da6e2012-11-12 14:33:44 +000022#include <linux/interrupt.h>
23#include <linux/of_irq.h>
Stephen Boyd22006992013-07-18 16:59:32 -070024#include <linux/of_address.h>
Mark Rutland8a4da6e2012-11-12 14:33:44 +000025#include <linux/io.h>
Stephen Boyd22006992013-07-18 16:59:32 -070026#include <linux/slab.h>
Ingo Molnare6017572017-02-01 16:36:40 +010027#include <linux/sched/clock.h>
Stephen Boyd65cd4f62013-07-18 16:21:18 -070028#include <linux/sched_clock.h>
Hanjun Guob09ca1e2015-03-24 14:02:50 +000029#include <linux/acpi.h>
Mark Rutland8a4da6e2012-11-12 14:33:44 +000030
31#include <asm/arch_timer.h>
Marc Zyngier82668912013-01-10 11:13:07 +000032#include <asm/virt.h>
Mark Rutland8a4da6e2012-11-12 14:33:44 +000033
34#include <clocksource/arm_arch_timer.h>
35
Fu Weided24012017-01-18 21:25:25 +080036#undef pr_fmt
37#define pr_fmt(fmt) "arch_timer: " fmt
38
Stephen Boyd22006992013-07-18 16:59:32 -070039#define CNTTIDR 0x08
40#define CNTTIDR_VIRT(n) (BIT(1) << ((n) * 4))
41
Robin Murphye392d602016-02-01 12:00:48 +000042#define CNTACR(n) (0x40 + ((n) * 4))
43#define CNTACR_RPCT BIT(0)
44#define CNTACR_RVCT BIT(1)
45#define CNTACR_RFRQ BIT(2)
46#define CNTACR_RVOFF BIT(3)
47#define CNTACR_RWVT BIT(4)
48#define CNTACR_RWPT BIT(5)
49
Stephen Boyd22006992013-07-18 16:59:32 -070050#define CNTVCT_LO 0x08
51#define CNTVCT_HI 0x0c
52#define CNTFRQ 0x10
53#define CNTP_TVAL 0x28
54#define CNTP_CTL 0x2c
55#define CNTV_TVAL 0x38
56#define CNTV_CTL 0x3c
57
Fu Wei8a5c21d2017-01-18 21:25:26 +080058#define ARCH_TIMER_TYPE_CP15 BIT(0)
59#define ARCH_TIMER_TYPE_MEM BIT(1)
Stephen Boyd22006992013-07-18 16:59:32 -070060static unsigned arch_timers_present __initdata;
61
62static void __iomem *arch_counter_base;
63
64struct arch_timer {
65 void __iomem *base;
66 struct clock_event_device evt;
67};
68
69#define to_arch_timer(e) container_of(e, struct arch_timer, evt)
70
Mark Rutland8a4da6e2012-11-12 14:33:44 +000071static u32 arch_timer_rate;
72
73enum ppi_nr {
74 PHYS_SECURE_PPI,
75 PHYS_NONSECURE_PPI,
76 VIRT_PPI,
77 HYP_PPI,
78 MAX_TIMER_PPI
79};
80
81static int arch_timer_ppi[MAX_TIMER_PPI];
82
83static struct clock_event_device __percpu *arch_timer_evt;
84
Marc Zyngierf81f03f2014-02-20 15:21:23 +000085static enum ppi_nr arch_timer_uses_ppi = VIRT_PPI;
Lorenzo Pieralisi82a561942014-04-08 10:04:32 +010086static bool arch_timer_c3stop;
Stephen Boyd22006992013-07-18 16:59:32 -070087static bool arch_timer_mem_use_virtual;
Brian Norrisd8ec7592016-10-04 11:12:09 -070088static bool arch_counter_suspend_stop;
Marc Zyngiera86bd132017-02-01 12:07:15 +000089static bool vdso_default = true;
Mark Rutland8a4da6e2012-11-12 14:33:44 +000090
Will Deacon46fd5c62016-06-27 17:30:13 +010091static bool evtstrm_enable = IS_ENABLED(CONFIG_ARM_ARCH_TIMER_EVTSTREAM);
92
93static int __init early_evtstrm_cfg(char *buf)
94{
95 return strtobool(buf, &evtstrm_enable);
96}
97early_param("clocksource.arm_arch_timer.evtstrm", early_evtstrm_cfg);
98
Mark Rutland8a4da6e2012-11-12 14:33:44 +000099/*
100 * Architected system timer support.
101 */
102
Marc Zyngierf4e00a12017-01-20 18:28:32 +0000103static __always_inline
104void arch_timer_reg_write(int access, enum arch_timer_reg reg, u32 val,
105 struct clock_event_device *clk)
106{
107 if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {
108 struct arch_timer *timer = to_arch_timer(clk);
109 switch (reg) {
110 case ARCH_TIMER_REG_CTRL:
111 writel_relaxed(val, timer->base + CNTP_CTL);
112 break;
113 case ARCH_TIMER_REG_TVAL:
114 writel_relaxed(val, timer->base + CNTP_TVAL);
115 break;
116 }
117 } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {
118 struct arch_timer *timer = to_arch_timer(clk);
119 switch (reg) {
120 case ARCH_TIMER_REG_CTRL:
121 writel_relaxed(val, timer->base + CNTV_CTL);
122 break;
123 case ARCH_TIMER_REG_TVAL:
124 writel_relaxed(val, timer->base + CNTV_TVAL);
125 break;
126 }
127 } else {
128 arch_timer_reg_write_cp15(access, reg, val);
129 }
130}
131
132static __always_inline
133u32 arch_timer_reg_read(int access, enum arch_timer_reg reg,
134 struct clock_event_device *clk)
135{
136 u32 val;
137
138 if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {
139 struct arch_timer *timer = to_arch_timer(clk);
140 switch (reg) {
141 case ARCH_TIMER_REG_CTRL:
142 val = readl_relaxed(timer->base + CNTP_CTL);
143 break;
144 case ARCH_TIMER_REG_TVAL:
145 val = readl_relaxed(timer->base + CNTP_TVAL);
146 break;
147 }
148 } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {
149 struct arch_timer *timer = to_arch_timer(clk);
150 switch (reg) {
151 case ARCH_TIMER_REG_CTRL:
152 val = readl_relaxed(timer->base + CNTV_CTL);
153 break;
154 case ARCH_TIMER_REG_TVAL:
155 val = readl_relaxed(timer->base + CNTV_TVAL);
156 break;
157 }
158 } else {
159 val = arch_timer_reg_read_cp15(access, reg);
160 }
161
162 return val;
163}
164
Marc Zyngier992dd162017-02-01 11:53:46 +0000165/*
166 * Default to cp15 based access because arm64 uses this function for
167 * sched_clock() before DT is probed and the cp15 method is guaranteed
168 * to exist on arm64. arm doesn't use this before DT is probed so even
169 * if we don't have the cp15 accessors we won't have a problem.
170 */
171u64 (*arch_timer_read_counter)(void) = arch_counter_get_cntvct;
172
173static u64 arch_counter_read(struct clocksource *cs)
174{
175 return arch_timer_read_counter();
176}
177
178static u64 arch_counter_read_cc(const struct cyclecounter *cc)
179{
180 return arch_timer_read_counter();
181}
182
183static struct clocksource clocksource_counter = {
184 .name = "arch_sys_counter",
185 .rating = 400,
186 .read = arch_counter_read,
187 .mask = CLOCKSOURCE_MASK(56),
188 .flags = CLOCK_SOURCE_IS_CONTINUOUS,
189};
190
191static struct cyclecounter cyclecounter __ro_after_init = {
192 .read = arch_counter_read_cc,
193 .mask = CLOCKSOURCE_MASK(56),
194};
195
Marc Zyngier5a38bca2017-02-21 14:37:30 +0000196struct ate_acpi_oem_info {
197 char oem_id[ACPI_OEM_ID_SIZE + 1];
198 char oem_table_id[ACPI_OEM_TABLE_ID_SIZE + 1];
199 u32 oem_revision;
200};
201
Scott Woodf6dc1572016-09-22 03:35:17 -0500202#ifdef CONFIG_FSL_ERRATUM_A008585
Ding Tianhong16d10ef2017-02-06 16:47:41 +0000203/*
204 * The number of retries is an arbitrary value well beyond the highest number
205 * of iterations the loop has been observed to take.
206 */
207#define __fsl_a008585_read_reg(reg) ({ \
208 u64 _old, _new; \
209 int _retries = 200; \
210 \
211 do { \
212 _old = read_sysreg(reg); \
213 _new = read_sysreg(reg); \
214 _retries--; \
215 } while (unlikely(_old != _new) && _retries); \
216 \
217 WARN_ON_ONCE(!_retries); \
218 _new; \
219})
Scott Woodf6dc1572016-09-22 03:35:17 -0500220
Ding Tianhong16d10ef2017-02-06 16:47:41 +0000221static u32 notrace fsl_a008585_read_cntp_tval_el0(void)
Scott Woodf6dc1572016-09-22 03:35:17 -0500222{
223 return __fsl_a008585_read_reg(cntp_tval_el0);
224}
225
Ding Tianhong16d10ef2017-02-06 16:47:41 +0000226static u32 notrace fsl_a008585_read_cntv_tval_el0(void)
Scott Woodf6dc1572016-09-22 03:35:17 -0500227{
228 return __fsl_a008585_read_reg(cntv_tval_el0);
229}
230
Ding Tianhong16d10ef2017-02-06 16:47:41 +0000231static u64 notrace fsl_a008585_read_cntvct_el0(void)
Scott Woodf6dc1572016-09-22 03:35:17 -0500232{
233 return __fsl_a008585_read_reg(cntvct_el0);
234}
Ding Tianhong16d10ef2017-02-06 16:47:41 +0000235#endif
236
Ding Tianhongbb42ca42017-02-06 16:47:42 +0000237#ifdef CONFIG_HISILICON_ERRATUM_161010101
238/*
239 * Verify whether the value of the second read is larger than the first by
240 * less than 32 is the only way to confirm the value is correct, so clear the
241 * lower 5 bits to check whether the difference is greater than 32 or not.
242 * Theoretically the erratum should not occur more than twice in succession
243 * when reading the system counter, but it is possible that some interrupts
244 * may lead to more than twice read errors, triggering the warning, so setting
245 * the number of retries far beyond the number of iterations the loop has been
246 * observed to take.
247 */
248#define __hisi_161010101_read_reg(reg) ({ \
249 u64 _old, _new; \
250 int _retries = 50; \
251 \
252 do { \
253 _old = read_sysreg(reg); \
254 _new = read_sysreg(reg); \
255 _retries--; \
256 } while (unlikely((_new - _old) >> 5) && _retries); \
257 \
258 WARN_ON_ONCE(!_retries); \
259 _new; \
260})
261
262static u32 notrace hisi_161010101_read_cntp_tval_el0(void)
263{
264 return __hisi_161010101_read_reg(cntp_tval_el0);
265}
266
267static u32 notrace hisi_161010101_read_cntv_tval_el0(void)
268{
269 return __hisi_161010101_read_reg(cntv_tval_el0);
270}
271
272static u64 notrace hisi_161010101_read_cntvct_el0(void)
273{
274 return __hisi_161010101_read_reg(cntvct_el0);
275}
Marc Zyngierd003d022017-02-21 15:04:27 +0000276
277static struct ate_acpi_oem_info hisi_161010101_oem_info[] = {
278 /*
279 * Note that trailing spaces are required to properly match
280 * the OEM table information.
281 */
282 {
283 .oem_id = "HISI ",
284 .oem_table_id = "HIP05 ",
285 .oem_revision = 0,
286 },
287 {
288 .oem_id = "HISI ",
289 .oem_table_id = "HIP06 ",
290 .oem_revision = 0,
291 },
292 {
293 .oem_id = "HISI ",
294 .oem_table_id = "HIP07 ",
295 .oem_revision = 0,
296 },
297 { /* Sentinel indicating the end of the OEM array */ },
298};
Ding Tianhongbb42ca42017-02-06 16:47:42 +0000299#endif
300
Marc Zyngierfa8d8152017-01-27 12:52:31 +0000301#ifdef CONFIG_ARM64_ERRATUM_858921
302static u64 notrace arm64_858921_read_cntvct_el0(void)
303{
304 u64 old, new;
305
306 old = read_sysreg(cntvct_el0);
307 new = read_sysreg(cntvct_el0);
308 return (((old ^ new) >> 32) & 1) ? old : new;
309}
310#endif
311
Ding Tianhong16d10ef2017-02-06 16:47:41 +0000312#ifdef CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND
Marc Zyngier6acc71c2017-02-20 18:34:48 +0000313DEFINE_PER_CPU(const struct arch_timer_erratum_workaround *,
314 timer_unstable_counter_workaround);
Ding Tianhong16d10ef2017-02-06 16:47:41 +0000315EXPORT_SYMBOL_GPL(timer_unstable_counter_workaround);
316
317DEFINE_STATIC_KEY_FALSE(arch_timer_read_ool_enabled);
318EXPORT_SYMBOL_GPL(arch_timer_read_ool_enabled);
319
Marc Zyngier83280892017-01-27 10:27:09 +0000320static void erratum_set_next_event_tval_generic(const int access, unsigned long evt,
321 struct clock_event_device *clk)
322{
323 unsigned long ctrl;
324 u64 cval = evt + arch_counter_get_cntvct();
325
326 ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
327 ctrl |= ARCH_TIMER_CTRL_ENABLE;
328 ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;
329
330 if (access == ARCH_TIMER_PHYS_ACCESS)
331 write_sysreg(cval, cntp_cval_el0);
332 else
333 write_sysreg(cval, cntv_cval_el0);
334
335 arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
336}
337
338static int erratum_set_next_event_tval_virt(unsigned long evt,
339 struct clock_event_device *clk)
340{
341 erratum_set_next_event_tval_generic(ARCH_TIMER_VIRT_ACCESS, evt, clk);
342 return 0;
343}
344
345static int erratum_set_next_event_tval_phys(unsigned long evt,
346 struct clock_event_device *clk)
347{
348 erratum_set_next_event_tval_generic(ARCH_TIMER_PHYS_ACCESS, evt, clk);
349 return 0;
350}
351
Ding Tianhong16d10ef2017-02-06 16:47:41 +0000352static const struct arch_timer_erratum_workaround ool_workarounds[] = {
353#ifdef CONFIG_FSL_ERRATUM_A008585
354 {
Marc Zyngier651bb2e2017-01-19 17:20:59 +0000355 .match_type = ate_match_dt,
Ding Tianhong16d10ef2017-02-06 16:47:41 +0000356 .id = "fsl,erratum-a008585",
Marc Zyngier651bb2e2017-01-19 17:20:59 +0000357 .desc = "Freescale erratum a005858",
Ding Tianhong16d10ef2017-02-06 16:47:41 +0000358 .read_cntp_tval_el0 = fsl_a008585_read_cntp_tval_el0,
359 .read_cntv_tval_el0 = fsl_a008585_read_cntv_tval_el0,
360 .read_cntvct_el0 = fsl_a008585_read_cntvct_el0,
Marc Zyngier01d3e3f2017-01-27 10:27:09 +0000361 .set_next_event_phys = erratum_set_next_event_tval_phys,
362 .set_next_event_virt = erratum_set_next_event_tval_virt,
Ding Tianhong16d10ef2017-02-06 16:47:41 +0000363 },
364#endif
Ding Tianhongbb42ca42017-02-06 16:47:42 +0000365#ifdef CONFIG_HISILICON_ERRATUM_161010101
366 {
Marc Zyngier651bb2e2017-01-19 17:20:59 +0000367 .match_type = ate_match_dt,
Ding Tianhongbb42ca42017-02-06 16:47:42 +0000368 .id = "hisilicon,erratum-161010101",
Marc Zyngier651bb2e2017-01-19 17:20:59 +0000369 .desc = "HiSilicon erratum 161010101",
Ding Tianhongbb42ca42017-02-06 16:47:42 +0000370 .read_cntp_tval_el0 = hisi_161010101_read_cntp_tval_el0,
371 .read_cntv_tval_el0 = hisi_161010101_read_cntv_tval_el0,
372 .read_cntvct_el0 = hisi_161010101_read_cntvct_el0,
Marc Zyngier01d3e3f2017-01-27 10:27:09 +0000373 .set_next_event_phys = erratum_set_next_event_tval_phys,
374 .set_next_event_virt = erratum_set_next_event_tval_virt,
Ding Tianhongbb42ca42017-02-06 16:47:42 +0000375 },
Marc Zyngierd003d022017-02-21 15:04:27 +0000376 {
377 .match_type = ate_match_acpi_oem_info,
378 .id = hisi_161010101_oem_info,
379 .desc = "HiSilicon erratum 161010101",
380 .read_cntp_tval_el0 = hisi_161010101_read_cntp_tval_el0,
381 .read_cntv_tval_el0 = hisi_161010101_read_cntv_tval_el0,
382 .read_cntvct_el0 = hisi_161010101_read_cntvct_el0,
383 .set_next_event_phys = erratum_set_next_event_tval_phys,
384 .set_next_event_virt = erratum_set_next_event_tval_virt,
385 },
Ding Tianhongbb42ca42017-02-06 16:47:42 +0000386#endif
Marc Zyngierfa8d8152017-01-27 12:52:31 +0000387#ifdef CONFIG_ARM64_ERRATUM_858921
388 {
389 .match_type = ate_match_local_cap_id,
390 .id = (void *)ARM64_WORKAROUND_858921,
391 .desc = "ARM erratum 858921",
392 .read_cntvct_el0 = arm64_858921_read_cntvct_el0,
393 },
394#endif
Ding Tianhong16d10ef2017-02-06 16:47:41 +0000395};
Marc Zyngier651bb2e2017-01-19 17:20:59 +0000396
397typedef bool (*ate_match_fn_t)(const struct arch_timer_erratum_workaround *,
398 const void *);
399
400static
401bool arch_timer_check_dt_erratum(const struct arch_timer_erratum_workaround *wa,
402 const void *arg)
403{
404 const struct device_node *np = arg;
405
406 return of_property_read_bool(np, wa->id);
407}
408
Marc Zyngier00640302017-03-20 16:47:59 +0000409static
410bool arch_timer_check_local_cap_erratum(const struct arch_timer_erratum_workaround *wa,
411 const void *arg)
412{
413 return this_cpu_has_cap((uintptr_t)wa->id);
414}
415
Marc Zyngier5a38bca2017-02-21 14:37:30 +0000416
417static
418bool arch_timer_check_acpi_oem_erratum(const struct arch_timer_erratum_workaround *wa,
419 const void *arg)
420{
421 static const struct ate_acpi_oem_info empty_oem_info = {};
422 const struct ate_acpi_oem_info *info = wa->id;
423 const struct acpi_table_header *table = arg;
424
425 /* Iterate over the ACPI OEM info array, looking for a match */
426 while (memcmp(info, &empty_oem_info, sizeof(*info))) {
427 if (!memcmp(info->oem_id, table->oem_id, ACPI_OEM_ID_SIZE) &&
428 !memcmp(info->oem_table_id, table->oem_table_id, ACPI_OEM_TABLE_ID_SIZE) &&
429 info->oem_revision == table->oem_revision)
430 return true;
431
432 info++;
433 }
434
435 return false;
436}
437
Marc Zyngier651bb2e2017-01-19 17:20:59 +0000438static const struct arch_timer_erratum_workaround *
439arch_timer_iterate_errata(enum arch_timer_erratum_match_type type,
440 ate_match_fn_t match_fn,
441 void *arg)
442{
443 int i;
444
445 for (i = 0; i < ARRAY_SIZE(ool_workarounds); i++) {
446 if (ool_workarounds[i].match_type != type)
447 continue;
448
449 if (match_fn(&ool_workarounds[i], arg))
450 return &ool_workarounds[i];
451 }
452
453 return NULL;
454}
455
456static
Marc Zyngier6acc71c2017-02-20 18:34:48 +0000457void arch_timer_enable_workaround(const struct arch_timer_erratum_workaround *wa,
458 bool local)
Marc Zyngier651bb2e2017-01-19 17:20:59 +0000459{
Marc Zyngier6acc71c2017-02-20 18:34:48 +0000460 int i;
461
462 if (local) {
463 __this_cpu_write(timer_unstable_counter_workaround, wa);
464 } else {
465 for_each_possible_cpu(i)
466 per_cpu(timer_unstable_counter_workaround, i) = wa;
467 }
468
Marc Zyngier651bb2e2017-01-19 17:20:59 +0000469 static_branch_enable(&arch_timer_read_ool_enabled);
Marc Zyngiera86bd132017-02-01 12:07:15 +0000470
471 /*
472 * Don't use the vdso fastpath if errata require using the
473 * out-of-line counter accessor. We may change our mind pretty
474 * late in the game (with a per-CPU erratum, for example), so
475 * change both the default value and the vdso itself.
476 */
477 if (wa->read_cntvct_el0) {
478 clocksource_counter.archdata.vdso_direct = false;
479 vdso_default = false;
480 }
Marc Zyngier651bb2e2017-01-19 17:20:59 +0000481}
482
483static void arch_timer_check_ool_workaround(enum arch_timer_erratum_match_type type,
484 void *arg)
485{
486 const struct arch_timer_erratum_workaround *wa;
487 ate_match_fn_t match_fn = NULL;
Marc Zyngier00640302017-03-20 16:47:59 +0000488 bool local = false;
Marc Zyngier651bb2e2017-01-19 17:20:59 +0000489
490 switch (type) {
491 case ate_match_dt:
492 match_fn = arch_timer_check_dt_erratum;
493 break;
Marc Zyngier00640302017-03-20 16:47:59 +0000494 case ate_match_local_cap_id:
495 match_fn = arch_timer_check_local_cap_erratum;
496 local = true;
497 break;
Marc Zyngier5a38bca2017-02-21 14:37:30 +0000498 case ate_match_acpi_oem_info:
499 match_fn = arch_timer_check_acpi_oem_erratum;
500 break;
Marc Zyngier651bb2e2017-01-19 17:20:59 +0000501 default:
502 WARN_ON(1);
503 return;
504 }
505
506 wa = arch_timer_iterate_errata(type, match_fn, arg);
507 if (!wa)
508 return;
509
Marc Zyngier00640302017-03-20 16:47:59 +0000510 if (needs_unstable_timer_counter_workaround()) {
Marc Zyngier6acc71c2017-02-20 18:34:48 +0000511 const struct arch_timer_erratum_workaround *__wa;
512 __wa = __this_cpu_read(timer_unstable_counter_workaround);
513 if (__wa && wa != __wa)
Marc Zyngier00640302017-03-20 16:47:59 +0000514 pr_warn("Can't enable workaround for %s (clashes with %s\n)",
Marc Zyngier6acc71c2017-02-20 18:34:48 +0000515 wa->desc, __wa->desc);
516
517 if (__wa)
518 return;
Marc Zyngier00640302017-03-20 16:47:59 +0000519 }
520
Marc Zyngier6acc71c2017-02-20 18:34:48 +0000521 arch_timer_enable_workaround(wa, local);
Marc Zyngier00640302017-03-20 16:47:59 +0000522 pr_info("Enabling %s workaround for %s\n",
523 local ? "local" : "global", wa->desc);
Marc Zyngier651bb2e2017-01-19 17:20:59 +0000524}
525
Marc Zyngier01d3e3f2017-01-27 10:27:09 +0000526#define erratum_handler(fn, r, ...) \
527({ \
528 bool __val; \
Marc Zyngier6acc71c2017-02-20 18:34:48 +0000529 if (needs_unstable_timer_counter_workaround()) { \
530 const struct arch_timer_erratum_workaround *__wa; \
531 __wa = __this_cpu_read(timer_unstable_counter_workaround); \
532 if (__wa && __wa->fn) { \
533 r = __wa->fn(__VA_ARGS__); \
534 __val = true; \
535 } else { \
536 __val = false; \
537 } \
Marc Zyngier01d3e3f2017-01-27 10:27:09 +0000538 } else { \
539 __val = false; \
540 } \
541 __val; \
542})
543
Marc Zyngiera86bd132017-02-01 12:07:15 +0000544static bool arch_timer_this_cpu_has_cntvct_wa(void)
545{
546 const struct arch_timer_erratum_workaround *wa;
547
548 wa = __this_cpu_read(timer_unstable_counter_workaround);
549 return wa && wa->read_cntvct_el0;
550}
Marc Zyngier651bb2e2017-01-19 17:20:59 +0000551#else
552#define arch_timer_check_ool_workaround(t,a) do { } while(0)
Marc Zyngier83280892017-01-27 10:27:09 +0000553#define erratum_set_next_event_tval_virt(...) ({BUG(); 0;})
554#define erratum_set_next_event_tval_phys(...) ({BUG(); 0;})
Marc Zyngier01d3e3f2017-01-27 10:27:09 +0000555#define erratum_handler(fn, r, ...) ({false;})
Marc Zyngiera86bd132017-02-01 12:07:15 +0000556#define arch_timer_this_cpu_has_cntvct_wa() ({false;})
Ding Tianhong16d10ef2017-02-06 16:47:41 +0000557#endif /* CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND */
Scott Woodf6dc1572016-09-22 03:35:17 -0500558
Stephen Boyde09f3cc2013-07-18 16:59:28 -0700559static __always_inline irqreturn_t timer_handler(const int access,
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000560 struct clock_event_device *evt)
561{
562 unsigned long ctrl;
Thomas Gleixnercfb6d652013-08-21 14:59:23 +0200563
Stephen Boyd60faddf2013-07-18 16:59:31 -0700564 ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, evt);
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000565 if (ctrl & ARCH_TIMER_CTRL_IT_STAT) {
566 ctrl |= ARCH_TIMER_CTRL_IT_MASK;
Stephen Boyd60faddf2013-07-18 16:59:31 -0700567 arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, evt);
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000568 evt->event_handler(evt);
569 return IRQ_HANDLED;
570 }
571
572 return IRQ_NONE;
573}
574
575static irqreturn_t arch_timer_handler_virt(int irq, void *dev_id)
576{
577 struct clock_event_device *evt = dev_id;
578
579 return timer_handler(ARCH_TIMER_VIRT_ACCESS, evt);
580}
581
582static irqreturn_t arch_timer_handler_phys(int irq, void *dev_id)
583{
584 struct clock_event_device *evt = dev_id;
585
586 return timer_handler(ARCH_TIMER_PHYS_ACCESS, evt);
587}
588
Stephen Boyd22006992013-07-18 16:59:32 -0700589static irqreturn_t arch_timer_handler_phys_mem(int irq, void *dev_id)
590{
591 struct clock_event_device *evt = dev_id;
592
593 return timer_handler(ARCH_TIMER_MEM_PHYS_ACCESS, evt);
594}
595
596static irqreturn_t arch_timer_handler_virt_mem(int irq, void *dev_id)
597{
598 struct clock_event_device *evt = dev_id;
599
600 return timer_handler(ARCH_TIMER_MEM_VIRT_ACCESS, evt);
601}
602
Viresh Kumar46c5bfd2015-06-12 13:30:12 +0530603static __always_inline int timer_shutdown(const int access,
604 struct clock_event_device *clk)
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000605{
606 unsigned long ctrl;
Viresh Kumar46c5bfd2015-06-12 13:30:12 +0530607
608 ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
609 ctrl &= ~ARCH_TIMER_CTRL_ENABLE;
610 arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
611
612 return 0;
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000613}
614
Viresh Kumar46c5bfd2015-06-12 13:30:12 +0530615static int arch_timer_shutdown_virt(struct clock_event_device *clk)
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000616{
Viresh Kumar46c5bfd2015-06-12 13:30:12 +0530617 return timer_shutdown(ARCH_TIMER_VIRT_ACCESS, clk);
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000618}
619
Viresh Kumar46c5bfd2015-06-12 13:30:12 +0530620static int arch_timer_shutdown_phys(struct clock_event_device *clk)
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000621{
Viresh Kumar46c5bfd2015-06-12 13:30:12 +0530622 return timer_shutdown(ARCH_TIMER_PHYS_ACCESS, clk);
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000623}
624
Viresh Kumar46c5bfd2015-06-12 13:30:12 +0530625static int arch_timer_shutdown_virt_mem(struct clock_event_device *clk)
Stephen Boyd22006992013-07-18 16:59:32 -0700626{
Viresh Kumar46c5bfd2015-06-12 13:30:12 +0530627 return timer_shutdown(ARCH_TIMER_MEM_VIRT_ACCESS, clk);
Stephen Boyd22006992013-07-18 16:59:32 -0700628}
629
Viresh Kumar46c5bfd2015-06-12 13:30:12 +0530630static int arch_timer_shutdown_phys_mem(struct clock_event_device *clk)
Stephen Boyd22006992013-07-18 16:59:32 -0700631{
Viresh Kumar46c5bfd2015-06-12 13:30:12 +0530632 return timer_shutdown(ARCH_TIMER_MEM_PHYS_ACCESS, clk);
Stephen Boyd22006992013-07-18 16:59:32 -0700633}
634
Stephen Boyd60faddf2013-07-18 16:59:31 -0700635static __always_inline void set_next_event(const int access, unsigned long evt,
Thomas Gleixnercfb6d652013-08-21 14:59:23 +0200636 struct clock_event_device *clk)
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000637{
638 unsigned long ctrl;
Stephen Boyd60faddf2013-07-18 16:59:31 -0700639 ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000640 ctrl |= ARCH_TIMER_CTRL_ENABLE;
641 ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;
Stephen Boyd60faddf2013-07-18 16:59:31 -0700642 arch_timer_reg_write(access, ARCH_TIMER_REG_TVAL, evt, clk);
643 arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000644}
645
646static int arch_timer_set_next_event_virt(unsigned long evt,
Stephen Boyd60faddf2013-07-18 16:59:31 -0700647 struct clock_event_device *clk)
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000648{
Marc Zyngier01d3e3f2017-01-27 10:27:09 +0000649 int ret;
650
651 if (erratum_handler(set_next_event_virt, ret, evt, clk))
652 return ret;
Marc Zyngier83280892017-01-27 10:27:09 +0000653
Stephen Boyd60faddf2013-07-18 16:59:31 -0700654 set_next_event(ARCH_TIMER_VIRT_ACCESS, evt, clk);
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000655 return 0;
656}
657
658static int arch_timer_set_next_event_phys(unsigned long evt,
Stephen Boyd60faddf2013-07-18 16:59:31 -0700659 struct clock_event_device *clk)
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000660{
Marc Zyngier01d3e3f2017-01-27 10:27:09 +0000661 int ret;
662
663 if (erratum_handler(set_next_event_phys, ret, evt, clk))
664 return ret;
Marc Zyngier83280892017-01-27 10:27:09 +0000665
Stephen Boyd60faddf2013-07-18 16:59:31 -0700666 set_next_event(ARCH_TIMER_PHYS_ACCESS, evt, clk);
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000667 return 0;
668}
669
Stephen Boyd22006992013-07-18 16:59:32 -0700670static int arch_timer_set_next_event_virt_mem(unsigned long evt,
671 struct clock_event_device *clk)
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000672{
Stephen Boyd22006992013-07-18 16:59:32 -0700673 set_next_event(ARCH_TIMER_MEM_VIRT_ACCESS, evt, clk);
674 return 0;
675}
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000676
Stephen Boyd22006992013-07-18 16:59:32 -0700677static int arch_timer_set_next_event_phys_mem(unsigned long evt,
678 struct clock_event_device *clk)
679{
680 set_next_event(ARCH_TIMER_MEM_PHYS_ACCESS, evt, clk);
681 return 0;
682}
683
Thomas Gleixnercfb6d652013-08-21 14:59:23 +0200684static void __arch_timer_setup(unsigned type,
685 struct clock_event_device *clk)
Stephen Boyd22006992013-07-18 16:59:32 -0700686{
687 clk->features = CLOCK_EVT_FEAT_ONESHOT;
688
Fu Wei8a5c21d2017-01-18 21:25:26 +0800689 if (type == ARCH_TIMER_TYPE_CP15) {
Lorenzo Pieralisi82a561942014-04-08 10:04:32 +0100690 if (arch_timer_c3stop)
691 clk->features |= CLOCK_EVT_FEAT_C3STOP;
Stephen Boyd22006992013-07-18 16:59:32 -0700692 clk->name = "arch_sys_timer";
693 clk->rating = 450;
694 clk->cpumask = cpumask_of(smp_processor_id());
Marc Zyngierf81f03f2014-02-20 15:21:23 +0000695 clk->irq = arch_timer_ppi[arch_timer_uses_ppi];
696 switch (arch_timer_uses_ppi) {
697 case VIRT_PPI:
Viresh Kumar46c5bfd2015-06-12 13:30:12 +0530698 clk->set_state_shutdown = arch_timer_shutdown_virt;
Viresh Kumarcf8c5002015-12-23 16:59:12 +0530699 clk->set_state_oneshot_stopped = arch_timer_shutdown_virt;
Stephen Boyd22006992013-07-18 16:59:32 -0700700 clk->set_next_event = arch_timer_set_next_event_virt;
Marc Zyngierf81f03f2014-02-20 15:21:23 +0000701 break;
702 case PHYS_SECURE_PPI:
703 case PHYS_NONSECURE_PPI:
704 case HYP_PPI:
Viresh Kumar46c5bfd2015-06-12 13:30:12 +0530705 clk->set_state_shutdown = arch_timer_shutdown_phys;
Viresh Kumarcf8c5002015-12-23 16:59:12 +0530706 clk->set_state_oneshot_stopped = arch_timer_shutdown_phys;
Stephen Boyd22006992013-07-18 16:59:32 -0700707 clk->set_next_event = arch_timer_set_next_event_phys;
Marc Zyngierf81f03f2014-02-20 15:21:23 +0000708 break;
709 default:
710 BUG();
Stephen Boyd22006992013-07-18 16:59:32 -0700711 }
Scott Woodf6dc1572016-09-22 03:35:17 -0500712
Marc Zyngier00640302017-03-20 16:47:59 +0000713 arch_timer_check_ool_workaround(ate_match_local_cap_id, NULL);
Stephen Boyd22006992013-07-18 16:59:32 -0700714 } else {
Stephen Boyd7b52ad22014-01-06 14:56:17 -0800715 clk->features |= CLOCK_EVT_FEAT_DYNIRQ;
Stephen Boyd22006992013-07-18 16:59:32 -0700716 clk->name = "arch_mem_timer";
717 clk->rating = 400;
718 clk->cpumask = cpu_all_mask;
719 if (arch_timer_mem_use_virtual) {
Viresh Kumar46c5bfd2015-06-12 13:30:12 +0530720 clk->set_state_shutdown = arch_timer_shutdown_virt_mem;
Viresh Kumarcf8c5002015-12-23 16:59:12 +0530721 clk->set_state_oneshot_stopped = arch_timer_shutdown_virt_mem;
Stephen Boyd22006992013-07-18 16:59:32 -0700722 clk->set_next_event =
723 arch_timer_set_next_event_virt_mem;
724 } else {
Viresh Kumar46c5bfd2015-06-12 13:30:12 +0530725 clk->set_state_shutdown = arch_timer_shutdown_phys_mem;
Viresh Kumarcf8c5002015-12-23 16:59:12 +0530726 clk->set_state_oneshot_stopped = arch_timer_shutdown_phys_mem;
Stephen Boyd22006992013-07-18 16:59:32 -0700727 clk->set_next_event =
728 arch_timer_set_next_event_phys_mem;
729 }
730 }
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000731
Viresh Kumar46c5bfd2015-06-12 13:30:12 +0530732 clk->set_state_shutdown(clk);
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000733
Stephen Boyd22006992013-07-18 16:59:32 -0700734 clockevents_config_and_register(clk, arch_timer_rate, 0xf, 0x7fffffff);
735}
736
Nathan Lynche1ce5c72014-09-29 01:50:06 +0200737static void arch_timer_evtstrm_enable(int divider)
738{
739 u32 cntkctl = arch_timer_get_cntkctl();
740
741 cntkctl &= ~ARCH_TIMER_EVT_TRIGGER_MASK;
742 /* Set the divider and enable virtual event stream */
743 cntkctl |= (divider << ARCH_TIMER_EVT_TRIGGER_SHIFT)
744 | ARCH_TIMER_VIRT_EVT_EN;
745 arch_timer_set_cntkctl(cntkctl);
746 elf_hwcap |= HWCAP_EVTSTRM;
747#ifdef CONFIG_COMPAT
748 compat_elf_hwcap |= COMPAT_HWCAP_EVTSTRM;
749#endif
750}
751
Will Deacon037f6372013-08-23 15:32:29 +0100752static void arch_timer_configure_evtstream(void)
753{
754 int evt_stream_div, pos;
755
756 /* Find the closest power of two to the divisor */
757 evt_stream_div = arch_timer_rate / ARCH_TIMER_EVT_STREAM_FREQ;
758 pos = fls(evt_stream_div);
759 if (pos > 1 && !(evt_stream_div & (1 << (pos - 2))))
760 pos--;
761 /* enable event stream */
762 arch_timer_evtstrm_enable(min(pos, 15));
763}
764
Nathan Lynch8b8dde02014-09-29 01:50:06 +0200765static void arch_counter_set_user_access(void)
766{
767 u32 cntkctl = arch_timer_get_cntkctl();
768
Marc Zyngiera86bd132017-02-01 12:07:15 +0000769 /* Disable user access to the timers and both counters */
Nathan Lynch8b8dde02014-09-29 01:50:06 +0200770 /* Also disable virtual event stream */
771 cntkctl &= ~(ARCH_TIMER_USR_PT_ACCESS_EN
772 | ARCH_TIMER_USR_VT_ACCESS_EN
Marc Zyngiera86bd132017-02-01 12:07:15 +0000773 | ARCH_TIMER_USR_VCT_ACCESS_EN
Nathan Lynch8b8dde02014-09-29 01:50:06 +0200774 | ARCH_TIMER_VIRT_EVT_EN
775 | ARCH_TIMER_USR_PCT_ACCESS_EN);
776
Marc Zyngiera86bd132017-02-01 12:07:15 +0000777 /*
778 * Enable user access to the virtual counter if it doesn't
779 * need to be workaround. The vdso may have been already
780 * disabled though.
781 */
782 if (arch_timer_this_cpu_has_cntvct_wa())
783 pr_info("CPU%d: Trapping CNTVCT access\n", smp_processor_id());
784 else
785 cntkctl |= ARCH_TIMER_USR_VCT_ACCESS_EN;
Nathan Lynch8b8dde02014-09-29 01:50:06 +0200786
787 arch_timer_set_cntkctl(cntkctl);
788}
789
Marc Zyngierf81f03f2014-02-20 15:21:23 +0000790static bool arch_timer_has_nonsecure_ppi(void)
791{
792 return (arch_timer_uses_ppi == PHYS_SECURE_PPI &&
793 arch_timer_ppi[PHYS_NONSECURE_PPI]);
794}
795
Marc Zyngierf005bd72016-08-01 10:54:15 +0100796static u32 check_ppi_trigger(int irq)
797{
798 u32 flags = irq_get_trigger_type(irq);
799
800 if (flags != IRQF_TRIGGER_HIGH && flags != IRQF_TRIGGER_LOW) {
801 pr_warn("WARNING: Invalid trigger for IRQ%d, assuming level low\n", irq);
802 pr_warn("WARNING: Please fix your firmware\n");
803 flags = IRQF_TRIGGER_LOW;
804 }
805
806 return flags;
807}
808
Richard Cochran7e86e8b2016-07-13 17:16:39 +0000809static int arch_timer_starting_cpu(unsigned int cpu)
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000810{
Richard Cochran7e86e8b2016-07-13 17:16:39 +0000811 struct clock_event_device *clk = this_cpu_ptr(arch_timer_evt);
Marc Zyngierf005bd72016-08-01 10:54:15 +0100812 u32 flags;
Richard Cochran7e86e8b2016-07-13 17:16:39 +0000813
Fu Wei8a5c21d2017-01-18 21:25:26 +0800814 __arch_timer_setup(ARCH_TIMER_TYPE_CP15, clk);
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000815
Marc Zyngierf005bd72016-08-01 10:54:15 +0100816 flags = check_ppi_trigger(arch_timer_ppi[arch_timer_uses_ppi]);
817 enable_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi], flags);
Marc Zyngierf81f03f2014-02-20 15:21:23 +0000818
Marc Zyngierf005bd72016-08-01 10:54:15 +0100819 if (arch_timer_has_nonsecure_ppi()) {
820 flags = check_ppi_trigger(arch_timer_ppi[PHYS_NONSECURE_PPI]);
821 enable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI], flags);
822 }
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000823
824 arch_counter_set_user_access();
Will Deacon46fd5c62016-06-27 17:30:13 +0100825 if (evtstrm_enable)
Will Deacon037f6372013-08-23 15:32:29 +0100826 arch_timer_configure_evtstream();
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000827
828 return 0;
829}
830
Stephen Boyd22006992013-07-18 16:59:32 -0700831static void
832arch_timer_detect_rate(void __iomem *cntbase, struct device_node *np)
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000833{
Stephen Boyd22006992013-07-18 16:59:32 -0700834 /* Who has more than one independent system counter? */
835 if (arch_timer_rate)
836 return;
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000837
Hanjun Guob09ca1e2015-03-24 14:02:50 +0000838 /*
839 * Try to determine the frequency from the device tree or CNTFRQ,
840 * if ACPI is enabled, get the frequency from CNTFRQ ONLY.
841 */
842 if (!acpi_disabled ||
843 of_property_read_u32(np, "clock-frequency", &arch_timer_rate)) {
Stephen Boyd22006992013-07-18 16:59:32 -0700844 if (cntbase)
845 arch_timer_rate = readl_relaxed(cntbase + CNTFRQ);
846 else
847 arch_timer_rate = arch_timer_get_cntfrq();
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000848 }
849
Stephen Boyd22006992013-07-18 16:59:32 -0700850 /* Check the timer frequency. */
851 if (arch_timer_rate == 0)
Fu Weided24012017-01-18 21:25:25 +0800852 pr_warn("frequency not available\n");
Stephen Boyd22006992013-07-18 16:59:32 -0700853}
854
855static void arch_timer_banner(unsigned type)
856{
Fu Weided24012017-01-18 21:25:25 +0800857 pr_info("%s%s%s timer(s) running at %lu.%02luMHz (%s%s%s).\n",
Fu Wei8a5c21d2017-01-18 21:25:26 +0800858 type & ARCH_TIMER_TYPE_CP15 ? "cp15" : "",
859 type == (ARCH_TIMER_TYPE_CP15 | ARCH_TIMER_TYPE_MEM) ?
860 " and " : "",
861 type & ARCH_TIMER_TYPE_MEM ? "mmio" : "",
Fu Weided24012017-01-18 21:25:25 +0800862 (unsigned long)arch_timer_rate / 1000000,
863 (unsigned long)(arch_timer_rate / 10000) % 100,
Fu Wei8a5c21d2017-01-18 21:25:26 +0800864 type & ARCH_TIMER_TYPE_CP15 ?
Fu Weided24012017-01-18 21:25:25 +0800865 (arch_timer_uses_ppi == VIRT_PPI) ? "virt" : "phys" :
Stephen Boyd22006992013-07-18 16:59:32 -0700866 "",
Fu Wei8a5c21d2017-01-18 21:25:26 +0800867 type == (ARCH_TIMER_TYPE_CP15 | ARCH_TIMER_TYPE_MEM) ? "/" : "",
868 type & ARCH_TIMER_TYPE_MEM ?
Stephen Boyd22006992013-07-18 16:59:32 -0700869 arch_timer_mem_use_virtual ? "virt" : "phys" :
870 "");
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000871}
872
873u32 arch_timer_get_rate(void)
874{
875 return arch_timer_rate;
876}
877
Stephen Boyd22006992013-07-18 16:59:32 -0700878static u64 arch_counter_get_cntvct_mem(void)
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000879{
Stephen Boyd22006992013-07-18 16:59:32 -0700880 u32 vct_lo, vct_hi, tmp_hi;
881
882 do {
883 vct_hi = readl_relaxed(arch_counter_base + CNTVCT_HI);
884 vct_lo = readl_relaxed(arch_counter_base + CNTVCT_LO);
885 tmp_hi = readl_relaxed(arch_counter_base + CNTVCT_HI);
886 } while (vct_hi != tmp_hi);
887
888 return ((u64) vct_hi << 32) | vct_lo;
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000889}
890
Julien Grallb4d6ce92016-04-11 16:32:51 +0100891static struct arch_timer_kvm_info arch_timer_kvm_info;
892
893struct arch_timer_kvm_info *arch_timer_get_kvm_info(void)
894{
895 return &arch_timer_kvm_info;
896}
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000897
Stephen Boyd22006992013-07-18 16:59:32 -0700898static void __init arch_counter_register(unsigned type)
899{
900 u64 start_count;
901
902 /* Register the CP15 based counter if we have one */
Fu Wei8a5c21d2017-01-18 21:25:26 +0800903 if (type & ARCH_TIMER_TYPE_CP15) {
Marc Zyngierf81f03f2014-02-20 15:21:23 +0000904 if (IS_ENABLED(CONFIG_ARM64) || arch_timer_uses_ppi == VIRT_PPI)
Sonny Rao0b46b8a2014-11-23 23:02:44 -0800905 arch_timer_read_counter = arch_counter_get_cntvct;
906 else
907 arch_timer_read_counter = arch_counter_get_cntpct;
Scott Woodf6dc1572016-09-22 03:35:17 -0500908
Marc Zyngiera86bd132017-02-01 12:07:15 +0000909 clocksource_counter.archdata.vdso_direct = vdso_default;
Nathan Lynch423bd692014-09-29 01:50:06 +0200910 } else {
Stephen Boyd22006992013-07-18 16:59:32 -0700911 arch_timer_read_counter = arch_counter_get_cntvct_mem;
Nathan Lynch423bd692014-09-29 01:50:06 +0200912 }
913
Brian Norrisd8ec7592016-10-04 11:12:09 -0700914 if (!arch_counter_suspend_stop)
915 clocksource_counter.flags |= CLOCK_SOURCE_SUSPEND_NONSTOP;
Stephen Boyd22006992013-07-18 16:59:32 -0700916 start_count = arch_timer_read_counter();
917 clocksource_register_hz(&clocksource_counter, arch_timer_rate);
918 cyclecounter.mult = clocksource_counter.mult;
919 cyclecounter.shift = clocksource_counter.shift;
Julien Grallb4d6ce92016-04-11 16:32:51 +0100920 timecounter_init(&arch_timer_kvm_info.timecounter,
921 &cyclecounter, start_count);
Thierry Reding4a7d3e82013-10-15 15:31:51 +0200922
923 /* 56 bits minimum, so we assume worst case rollover */
924 sched_clock_register(arch_timer_read_counter, 56, arch_timer_rate);
Stephen Boyd22006992013-07-18 16:59:32 -0700925}
926
Paul Gortmaker8c37bb32013-06-19 11:32:08 -0400927static void arch_timer_stop(struct clock_event_device *clk)
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000928{
Fu Weided24012017-01-18 21:25:25 +0800929 pr_debug("disable IRQ%d cpu #%d\n", clk->irq, smp_processor_id());
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000930
Marc Zyngierf81f03f2014-02-20 15:21:23 +0000931 disable_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi]);
932 if (arch_timer_has_nonsecure_ppi())
933 disable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI]);
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000934
Viresh Kumar46c5bfd2015-06-12 13:30:12 +0530935 clk->set_state_shutdown(clk);
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000936}
937
Richard Cochran7e86e8b2016-07-13 17:16:39 +0000938static int arch_timer_dying_cpu(unsigned int cpu)
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000939{
Richard Cochran7e86e8b2016-07-13 17:16:39 +0000940 struct clock_event_device *clk = this_cpu_ptr(arch_timer_evt);
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000941
Richard Cochran7e86e8b2016-07-13 17:16:39 +0000942 arch_timer_stop(clk);
943 return 0;
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000944}
945
Sudeep KarkadaNagesha346e7482013-08-23 15:53:15 +0100946#ifdef CONFIG_CPU_PM
Marc Zyngierbee67c52017-04-04 17:05:16 +0100947static DEFINE_PER_CPU(unsigned long, saved_cntkctl);
Sudeep KarkadaNagesha346e7482013-08-23 15:53:15 +0100948static int arch_timer_cpu_pm_notify(struct notifier_block *self,
949 unsigned long action, void *hcpu)
950{
951 if (action == CPU_PM_ENTER)
Marc Zyngierbee67c52017-04-04 17:05:16 +0100952 __this_cpu_write(saved_cntkctl, arch_timer_get_cntkctl());
Sudeep KarkadaNagesha346e7482013-08-23 15:53:15 +0100953 else if (action == CPU_PM_ENTER_FAILED || action == CPU_PM_EXIT)
Marc Zyngierbee67c52017-04-04 17:05:16 +0100954 arch_timer_set_cntkctl(__this_cpu_read(saved_cntkctl));
Sudeep KarkadaNagesha346e7482013-08-23 15:53:15 +0100955 return NOTIFY_OK;
956}
957
958static struct notifier_block arch_timer_cpu_pm_notifier = {
959 .notifier_call = arch_timer_cpu_pm_notify,
960};
961
962static int __init arch_timer_cpu_pm_init(void)
963{
964 return cpu_pm_register_notifier(&arch_timer_cpu_pm_notifier);
965}
Richard Cochran7e86e8b2016-07-13 17:16:39 +0000966
967static void __init arch_timer_cpu_pm_deinit(void)
968{
969 WARN_ON(cpu_pm_unregister_notifier(&arch_timer_cpu_pm_notifier));
970}
971
Sudeep KarkadaNagesha346e7482013-08-23 15:53:15 +0100972#else
973static int __init arch_timer_cpu_pm_init(void)
974{
975 return 0;
976}
Richard Cochran7e86e8b2016-07-13 17:16:39 +0000977
978static void __init arch_timer_cpu_pm_deinit(void)
979{
980}
Sudeep KarkadaNagesha346e7482013-08-23 15:53:15 +0100981#endif
982
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000983static int __init arch_timer_register(void)
984{
985 int err;
986 int ppi;
987
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000988 arch_timer_evt = alloc_percpu(struct clock_event_device);
989 if (!arch_timer_evt) {
990 err = -ENOMEM;
991 goto out;
992 }
993
Marc Zyngierf81f03f2014-02-20 15:21:23 +0000994 ppi = arch_timer_ppi[arch_timer_uses_ppi];
995 switch (arch_timer_uses_ppi) {
996 case VIRT_PPI:
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000997 err = request_percpu_irq(ppi, arch_timer_handler_virt,
998 "arch_timer", arch_timer_evt);
Marc Zyngierf81f03f2014-02-20 15:21:23 +0000999 break;
1000 case PHYS_SECURE_PPI:
1001 case PHYS_NONSECURE_PPI:
Mark Rutland8a4da6e2012-11-12 14:33:44 +00001002 err = request_percpu_irq(ppi, arch_timer_handler_phys,
1003 "arch_timer", arch_timer_evt);
1004 if (!err && arch_timer_ppi[PHYS_NONSECURE_PPI]) {
1005 ppi = arch_timer_ppi[PHYS_NONSECURE_PPI];
1006 err = request_percpu_irq(ppi, arch_timer_handler_phys,
1007 "arch_timer", arch_timer_evt);
1008 if (err)
1009 free_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI],
1010 arch_timer_evt);
1011 }
Marc Zyngierf81f03f2014-02-20 15:21:23 +00001012 break;
1013 case HYP_PPI:
1014 err = request_percpu_irq(ppi, arch_timer_handler_phys,
1015 "arch_timer", arch_timer_evt);
1016 break;
1017 default:
1018 BUG();
Mark Rutland8a4da6e2012-11-12 14:33:44 +00001019 }
1020
1021 if (err) {
Fu Weided24012017-01-18 21:25:25 +08001022 pr_err("can't register interrupt %d (%d)\n", ppi, err);
Mark Rutland8a4da6e2012-11-12 14:33:44 +00001023 goto out_free;
1024 }
1025
Sudeep KarkadaNagesha346e7482013-08-23 15:53:15 +01001026 err = arch_timer_cpu_pm_init();
1027 if (err)
1028 goto out_unreg_notify;
1029
Mark Rutland8a4da6e2012-11-12 14:33:44 +00001030
Richard Cochran7e86e8b2016-07-13 17:16:39 +00001031 /* Register and immediately configure the timer on the boot CPU */
1032 err = cpuhp_setup_state(CPUHP_AP_ARM_ARCH_TIMER_STARTING,
Thomas Gleixner73c1b412016-12-21 20:19:54 +01001033 "clockevents/arm/arch_timer:starting",
Richard Cochran7e86e8b2016-07-13 17:16:39 +00001034 arch_timer_starting_cpu, arch_timer_dying_cpu);
1035 if (err)
1036 goto out_unreg_cpupm;
Mark Rutland8a4da6e2012-11-12 14:33:44 +00001037 return 0;
1038
Richard Cochran7e86e8b2016-07-13 17:16:39 +00001039out_unreg_cpupm:
1040 arch_timer_cpu_pm_deinit();
1041
Sudeep KarkadaNagesha346e7482013-08-23 15:53:15 +01001042out_unreg_notify:
Marc Zyngierf81f03f2014-02-20 15:21:23 +00001043 free_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi], arch_timer_evt);
1044 if (arch_timer_has_nonsecure_ppi())
1045 free_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI],
Mark Rutland8a4da6e2012-11-12 14:33:44 +00001046 arch_timer_evt);
Mark Rutland8a4da6e2012-11-12 14:33:44 +00001047
1048out_free:
1049 free_percpu(arch_timer_evt);
1050out:
1051 return err;
1052}
1053
Stephen Boyd22006992013-07-18 16:59:32 -07001054static int __init arch_timer_mem_register(void __iomem *base, unsigned int irq)
1055{
1056 int ret;
1057 irq_handler_t func;
1058 struct arch_timer *t;
1059
1060 t = kzalloc(sizeof(*t), GFP_KERNEL);
1061 if (!t)
1062 return -ENOMEM;
1063
1064 t->base = base;
1065 t->evt.irq = irq;
Fu Wei8a5c21d2017-01-18 21:25:26 +08001066 __arch_timer_setup(ARCH_TIMER_TYPE_MEM, &t->evt);
Stephen Boyd22006992013-07-18 16:59:32 -07001067
1068 if (arch_timer_mem_use_virtual)
1069 func = arch_timer_handler_virt_mem;
1070 else
1071 func = arch_timer_handler_phys_mem;
1072
1073 ret = request_irq(irq, func, IRQF_TIMER, "arch_mem_timer", &t->evt);
1074 if (ret) {
Fu Weided24012017-01-18 21:25:25 +08001075 pr_err("Failed to request mem timer irq\n");
Stephen Boyd22006992013-07-18 16:59:32 -07001076 kfree(t);
1077 }
1078
1079 return ret;
1080}
1081
1082static const struct of_device_id arch_timer_of_match[] __initconst = {
1083 { .compatible = "arm,armv7-timer", },
1084 { .compatible = "arm,armv8-timer", },
1085 {},
1086};
1087
1088static const struct of_device_id arch_timer_mem_of_match[] __initconst = {
1089 { .compatible = "arm,armv7-timer-mem", },
1090 {},
1091};
1092
Sudeep Hollac387f072014-09-29 01:50:05 +02001093static bool __init
Laurent Pinchart566e6df2015-03-31 12:12:22 +02001094arch_timer_needs_probing(int type, const struct of_device_id *matches)
Sudeep Hollac387f072014-09-29 01:50:05 +02001095{
1096 struct device_node *dn;
Laurent Pinchart566e6df2015-03-31 12:12:22 +02001097 bool needs_probing = false;
Sudeep Hollac387f072014-09-29 01:50:05 +02001098
1099 dn = of_find_matching_node(NULL, matches);
Marc Zyngier59aa8962014-10-15 16:06:20 +01001100 if (dn && of_device_is_available(dn) && !(arch_timers_present & type))
Laurent Pinchart566e6df2015-03-31 12:12:22 +02001101 needs_probing = true;
Sudeep Hollac387f072014-09-29 01:50:05 +02001102 of_node_put(dn);
1103
Laurent Pinchart566e6df2015-03-31 12:12:22 +02001104 return needs_probing;
Sudeep Hollac387f072014-09-29 01:50:05 +02001105}
1106
Daniel Lezcano3c0731d2016-06-06 17:55:40 +02001107static int __init arch_timer_common_init(void)
Stephen Boyd22006992013-07-18 16:59:32 -07001108{
Fu Wei8a5c21d2017-01-18 21:25:26 +08001109 unsigned mask = ARCH_TIMER_TYPE_CP15 | ARCH_TIMER_TYPE_MEM;
Stephen Boyd22006992013-07-18 16:59:32 -07001110
1111 /* Wait until both nodes are probed if we have two timers */
1112 if ((arch_timers_present & mask) != mask) {
Fu Wei8a5c21d2017-01-18 21:25:26 +08001113 if (arch_timer_needs_probing(ARCH_TIMER_TYPE_MEM,
1114 arch_timer_mem_of_match))
Daniel Lezcano3c0731d2016-06-06 17:55:40 +02001115 return 0;
Fu Wei8a5c21d2017-01-18 21:25:26 +08001116 if (arch_timer_needs_probing(ARCH_TIMER_TYPE_CP15,
1117 arch_timer_of_match))
Daniel Lezcano3c0731d2016-06-06 17:55:40 +02001118 return 0;
Stephen Boyd22006992013-07-18 16:59:32 -07001119 }
1120
1121 arch_timer_banner(arch_timers_present);
1122 arch_counter_register(arch_timers_present);
Daniel Lezcano3c0731d2016-06-06 17:55:40 +02001123 return arch_timer_arch_init();
Stephen Boyd22006992013-07-18 16:59:32 -07001124}
1125
Daniel Lezcano3c0731d2016-06-06 17:55:40 +02001126static int __init arch_timer_init(void)
Mark Rutland8a4da6e2012-11-12 14:33:44 +00001127{
Daniel Lezcano3c0731d2016-06-06 17:55:40 +02001128 int ret;
Doug Anderson65b57322014-10-08 00:33:47 -07001129 /*
Marc Zyngier82668912013-01-10 11:13:07 +00001130 * If HYP mode is available, we know that the physical timer
1131 * has been configured to be accessible from PL1. Use it, so
1132 * that a guest can use the virtual timer instead.
1133 *
Mark Rutland8a4da6e2012-11-12 14:33:44 +00001134 * If no interrupt provided for virtual timer, we'll have to
1135 * stick to the physical timer. It'd better be accessible...
Marc Zyngierf81f03f2014-02-20 15:21:23 +00001136 *
1137 * On ARMv8.1 with VH extensions, the kernel runs in HYP. VHE
1138 * accesses to CNTP_*_EL1 registers are silently redirected to
1139 * their CNTHP_*_EL2 counterparts, and use a different PPI
1140 * number.
Mark Rutland8a4da6e2012-11-12 14:33:44 +00001141 */
Marc Zyngier82668912013-01-10 11:13:07 +00001142 if (is_hyp_mode_available() || !arch_timer_ppi[VIRT_PPI]) {
Marc Zyngierf81f03f2014-02-20 15:21:23 +00001143 bool has_ppi;
Mark Rutland8a4da6e2012-11-12 14:33:44 +00001144
Marc Zyngierf81f03f2014-02-20 15:21:23 +00001145 if (is_kernel_in_hyp_mode()) {
1146 arch_timer_uses_ppi = HYP_PPI;
1147 has_ppi = !!arch_timer_ppi[HYP_PPI];
1148 } else {
1149 arch_timer_uses_ppi = PHYS_SECURE_PPI;
1150 has_ppi = (!!arch_timer_ppi[PHYS_SECURE_PPI] ||
1151 !!arch_timer_ppi[PHYS_NONSECURE_PPI]);
1152 }
1153
1154 if (!has_ppi) {
Fu Weided24012017-01-18 21:25:25 +08001155 pr_warn("No interrupt available, giving up\n");
Daniel Lezcano3c0731d2016-06-06 17:55:40 +02001156 return -EINVAL;
Mark Rutland8a4da6e2012-11-12 14:33:44 +00001157 }
1158 }
1159
Daniel Lezcano3c0731d2016-06-06 17:55:40 +02001160 ret = arch_timer_register();
1161 if (ret)
1162 return ret;
1163
1164 ret = arch_timer_common_init();
1165 if (ret)
1166 return ret;
Julien Gralld9b5e412016-04-11 16:32:52 +01001167
1168 arch_timer_kvm_info.virtual_irq = arch_timer_ppi[VIRT_PPI];
Fu Weided24012017-01-18 21:25:25 +08001169
Daniel Lezcano3c0731d2016-06-06 17:55:40 +02001170 return 0;
Mark Rutland8a4da6e2012-11-12 14:33:44 +00001171}
Hanjun Guob09ca1e2015-03-24 14:02:50 +00001172
Daniel Lezcano3c0731d2016-06-06 17:55:40 +02001173static int __init arch_timer_of_init(struct device_node *np)
Hanjun Guob09ca1e2015-03-24 14:02:50 +00001174{
1175 int i;
1176
Fu Wei8a5c21d2017-01-18 21:25:26 +08001177 if (arch_timers_present & ARCH_TIMER_TYPE_CP15) {
Fu Weided24012017-01-18 21:25:25 +08001178 pr_warn("multiple nodes in dt, skipping\n");
Daniel Lezcano3c0731d2016-06-06 17:55:40 +02001179 return 0;
Hanjun Guob09ca1e2015-03-24 14:02:50 +00001180 }
1181
Fu Wei8a5c21d2017-01-18 21:25:26 +08001182 arch_timers_present |= ARCH_TIMER_TYPE_CP15;
Hanjun Guob09ca1e2015-03-24 14:02:50 +00001183 for (i = PHYS_SECURE_PPI; i < MAX_TIMER_PPI; i++)
1184 arch_timer_ppi[i] = irq_of_parse_and_map(np, i);
1185
1186 arch_timer_detect_rate(NULL, np);
1187
1188 arch_timer_c3stop = !of_property_read_bool(np, "always-on");
1189
Marc Zyngier651bb2e2017-01-19 17:20:59 +00001190 /* Check for globally applicable workarounds */
1191 arch_timer_check_ool_workaround(ate_match_dt, np);
Scott Woodf6dc1572016-09-22 03:35:17 -05001192
Hanjun Guob09ca1e2015-03-24 14:02:50 +00001193 /*
1194 * If we cannot rely on firmware initializing the timer registers then
1195 * we should use the physical timers instead.
1196 */
1197 if (IS_ENABLED(CONFIG_ARM) &&
1198 of_property_read_bool(np, "arm,cpu-registers-not-fw-configured"))
Marc Zyngierf81f03f2014-02-20 15:21:23 +00001199 arch_timer_uses_ppi = PHYS_SECURE_PPI;
Hanjun Guob09ca1e2015-03-24 14:02:50 +00001200
Brian Norrisd8ec7592016-10-04 11:12:09 -07001201 /* On some systems, the counter stops ticking when in suspend. */
1202 arch_counter_suspend_stop = of_property_read_bool(np,
1203 "arm,no-tick-in-suspend");
1204
Daniel Lezcano3c0731d2016-06-06 17:55:40 +02001205 return arch_timer_init();
Hanjun Guob09ca1e2015-03-24 14:02:50 +00001206}
Daniel Lezcano177cf6e2016-06-07 00:27:44 +02001207CLOCKSOURCE_OF_DECLARE(armv7_arch_timer, "arm,armv7-timer", arch_timer_of_init);
1208CLOCKSOURCE_OF_DECLARE(armv8_arch_timer, "arm,armv8-timer", arch_timer_of_init);
Stephen Boyd22006992013-07-18 16:59:32 -07001209
Daniel Lezcano3c0731d2016-06-06 17:55:40 +02001210static int __init arch_timer_mem_init(struct device_node *np)
Stephen Boyd22006992013-07-18 16:59:32 -07001211{
1212 struct device_node *frame, *best_frame = NULL;
1213 void __iomem *cntctlbase, *base;
Daniel Lezcano3c0731d2016-06-06 17:55:40 +02001214 unsigned int irq, ret = -EINVAL;
Stephen Boyd22006992013-07-18 16:59:32 -07001215 u32 cnttidr;
1216
Fu Wei8a5c21d2017-01-18 21:25:26 +08001217 arch_timers_present |= ARCH_TIMER_TYPE_MEM;
Stephen Boyd22006992013-07-18 16:59:32 -07001218 cntctlbase = of_iomap(np, 0);
1219 if (!cntctlbase) {
Fu Weided24012017-01-18 21:25:25 +08001220 pr_err("Can't find CNTCTLBase\n");
Daniel Lezcano3c0731d2016-06-06 17:55:40 +02001221 return -ENXIO;
Stephen Boyd22006992013-07-18 16:59:32 -07001222 }
1223
1224 cnttidr = readl_relaxed(cntctlbase + CNTTIDR);
Stephen Boyd22006992013-07-18 16:59:32 -07001225
1226 /*
1227 * Try to find a virtual capable frame. Otherwise fall back to a
1228 * physical capable frame.
1229 */
1230 for_each_available_child_of_node(np, frame) {
1231 int n;
Robin Murphye392d602016-02-01 12:00:48 +00001232 u32 cntacr;
Stephen Boyd22006992013-07-18 16:59:32 -07001233
1234 if (of_property_read_u32(frame, "frame-number", &n)) {
Fu Weided24012017-01-18 21:25:25 +08001235 pr_err("Missing frame-number\n");
Stephen Boyd22006992013-07-18 16:59:32 -07001236 of_node_put(frame);
Robin Murphye392d602016-02-01 12:00:48 +00001237 goto out;
Stephen Boyd22006992013-07-18 16:59:32 -07001238 }
1239
Robin Murphye392d602016-02-01 12:00:48 +00001240 /* Try enabling everything, and see what sticks */
1241 cntacr = CNTACR_RFRQ | CNTACR_RWPT | CNTACR_RPCT |
1242 CNTACR_RWVT | CNTACR_RVOFF | CNTACR_RVCT;
1243 writel_relaxed(cntacr, cntctlbase + CNTACR(n));
1244 cntacr = readl_relaxed(cntctlbase + CNTACR(n));
1245
1246 if ((cnttidr & CNTTIDR_VIRT(n)) &&
1247 !(~cntacr & (CNTACR_RWVT | CNTACR_RVCT))) {
Stephen Boyd22006992013-07-18 16:59:32 -07001248 of_node_put(best_frame);
1249 best_frame = frame;
1250 arch_timer_mem_use_virtual = true;
1251 break;
1252 }
Robin Murphye392d602016-02-01 12:00:48 +00001253
1254 if (~cntacr & (CNTACR_RWPT | CNTACR_RPCT))
1255 continue;
1256
Stephen Boyd22006992013-07-18 16:59:32 -07001257 of_node_put(best_frame);
1258 best_frame = of_node_get(frame);
1259 }
1260
Daniel Lezcano3c0731d2016-06-06 17:55:40 +02001261 ret= -ENXIO;
Stephen Boydf947ee12016-10-26 00:35:50 -07001262 base = arch_counter_base = of_io_request_and_map(best_frame, 0,
1263 "arch_mem_timer");
1264 if (IS_ERR(base)) {
Fu Weided24012017-01-18 21:25:25 +08001265 pr_err("Can't map frame's registers\n");
Robin Murphye392d602016-02-01 12:00:48 +00001266 goto out;
Stephen Boyd22006992013-07-18 16:59:32 -07001267 }
1268
1269 if (arch_timer_mem_use_virtual)
1270 irq = irq_of_parse_and_map(best_frame, 1);
1271 else
1272 irq = irq_of_parse_and_map(best_frame, 0);
Robin Murphye392d602016-02-01 12:00:48 +00001273
Daniel Lezcano3c0731d2016-06-06 17:55:40 +02001274 ret = -EINVAL;
Stephen Boyd22006992013-07-18 16:59:32 -07001275 if (!irq) {
Fu Weided24012017-01-18 21:25:25 +08001276 pr_err("Frame missing %s irq.\n",
Thomas Gleixnercfb6d652013-08-21 14:59:23 +02001277 arch_timer_mem_use_virtual ? "virt" : "phys");
Robin Murphye392d602016-02-01 12:00:48 +00001278 goto out;
Stephen Boyd22006992013-07-18 16:59:32 -07001279 }
1280
1281 arch_timer_detect_rate(base, np);
Daniel Lezcano3c0731d2016-06-06 17:55:40 +02001282 ret = arch_timer_mem_register(base, irq);
1283 if (ret)
1284 goto out;
1285
1286 return arch_timer_common_init();
Robin Murphye392d602016-02-01 12:00:48 +00001287out:
1288 iounmap(cntctlbase);
1289 of_node_put(best_frame);
Daniel Lezcano3c0731d2016-06-06 17:55:40 +02001290 return ret;
Stephen Boyd22006992013-07-18 16:59:32 -07001291}
Daniel Lezcano177cf6e2016-06-07 00:27:44 +02001292CLOCKSOURCE_OF_DECLARE(armv7_arch_timer_mem, "arm,armv7-timer-mem",
Stephen Boyd22006992013-07-18 16:59:32 -07001293 arch_timer_mem_init);
Hanjun Guob09ca1e2015-03-24 14:02:50 +00001294
1295#ifdef CONFIG_ACPI
1296static int __init map_generic_timer_interrupt(u32 interrupt, u32 flags)
1297{
1298 int trigger, polarity;
1299
1300 if (!interrupt)
1301 return 0;
1302
1303 trigger = (flags & ACPI_GTDT_INTERRUPT_MODE) ? ACPI_EDGE_SENSITIVE
1304 : ACPI_LEVEL_SENSITIVE;
1305
1306 polarity = (flags & ACPI_GTDT_INTERRUPT_POLARITY) ? ACPI_ACTIVE_LOW
1307 : ACPI_ACTIVE_HIGH;
1308
1309 return acpi_register_gsi(NULL, interrupt, trigger, polarity);
1310}
1311
1312/* Initialize per-processor generic timer */
1313static int __init arch_timer_acpi_init(struct acpi_table_header *table)
1314{
1315 struct acpi_table_gtdt *gtdt;
1316
Fu Wei8a5c21d2017-01-18 21:25:26 +08001317 if (arch_timers_present & ARCH_TIMER_TYPE_CP15) {
Fu Weided24012017-01-18 21:25:25 +08001318 pr_warn("already initialized, skipping\n");
Hanjun Guob09ca1e2015-03-24 14:02:50 +00001319 return -EINVAL;
1320 }
1321
1322 gtdt = container_of(table, struct acpi_table_gtdt, header);
1323
Fu Wei8a5c21d2017-01-18 21:25:26 +08001324 arch_timers_present |= ARCH_TIMER_TYPE_CP15;
Hanjun Guob09ca1e2015-03-24 14:02:50 +00001325
1326 arch_timer_ppi[PHYS_SECURE_PPI] =
1327 map_generic_timer_interrupt(gtdt->secure_el1_interrupt,
1328 gtdt->secure_el1_flags);
1329
1330 arch_timer_ppi[PHYS_NONSECURE_PPI] =
1331 map_generic_timer_interrupt(gtdt->non_secure_el1_interrupt,
1332 gtdt->non_secure_el1_flags);
1333
1334 arch_timer_ppi[VIRT_PPI] =
1335 map_generic_timer_interrupt(gtdt->virtual_timer_interrupt,
1336 gtdt->virtual_timer_flags);
1337
1338 arch_timer_ppi[HYP_PPI] =
1339 map_generic_timer_interrupt(gtdt->non_secure_el2_interrupt,
1340 gtdt->non_secure_el2_flags);
1341
1342 /* Get the frequency from CNTFRQ */
1343 arch_timer_detect_rate(NULL, NULL);
1344
1345 /* Always-on capability */
1346 arch_timer_c3stop = !(gtdt->non_secure_el1_flags & ACPI_GTDT_ALWAYS_ON);
1347
Marc Zyngier5a38bca2017-02-21 14:37:30 +00001348 /* Check for globally applicable workarounds */
1349 arch_timer_check_ool_workaround(ate_match_acpi_oem_info, table);
1350
Hanjun Guob09ca1e2015-03-24 14:02:50 +00001351 arch_timer_init();
1352 return 0;
1353}
Marc Zyngierae281cb2015-09-28 15:49:17 +01001354CLOCKSOURCE_ACPI_DECLARE(arch_timer, ACPI_SIG_GTDT, arch_timer_acpi_init);
Hanjun Guob09ca1e2015-03-24 14:02:50 +00001355#endif