blob: f655036b524f430b00b643f693eb35a49b02086f [file] [log] [blame]
Mark Rutland8a4da6e2012-11-12 14:33:44 +00001/*
2 * linux/drivers/clocksource/arm_arch_timer.c
3 *
4 * Copyright (C) 2011 ARM Ltd.
5 * All Rights Reserved
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 */
11#include <linux/init.h>
12#include <linux/kernel.h>
13#include <linux/device.h>
14#include <linux/smp.h>
15#include <linux/cpu.h>
Sudeep KarkadaNagesha346e7482013-08-23 15:53:15 +010016#include <linux/cpu_pm.h>
Mark Rutland8a4da6e2012-11-12 14:33:44 +000017#include <linux/clockchips.h>
18#include <linux/interrupt.h>
19#include <linux/of_irq.h>
Stephen Boyd22006992013-07-18 16:59:32 -070020#include <linux/of_address.h>
Mark Rutland8a4da6e2012-11-12 14:33:44 +000021#include <linux/io.h>
Stephen Boyd22006992013-07-18 16:59:32 -070022#include <linux/slab.h>
Stephen Boyd65cd4f62013-07-18 16:21:18 -070023#include <linux/sched_clock.h>
Mark Rutland8a4da6e2012-11-12 14:33:44 +000024
25#include <asm/arch_timer.h>
Marc Zyngier82668912013-01-10 11:13:07 +000026#include <asm/virt.h>
Mark Rutland8a4da6e2012-11-12 14:33:44 +000027
28#include <clocksource/arm_arch_timer.h>
29
Stephen Boyd22006992013-07-18 16:59:32 -070030#define CNTTIDR 0x08
31#define CNTTIDR_VIRT(n) (BIT(1) << ((n) * 4))
32
33#define CNTVCT_LO 0x08
34#define CNTVCT_HI 0x0c
35#define CNTFRQ 0x10
36#define CNTP_TVAL 0x28
37#define CNTP_CTL 0x2c
38#define CNTV_TVAL 0x38
39#define CNTV_CTL 0x3c
40
41#define ARCH_CP15_TIMER BIT(0)
42#define ARCH_MEM_TIMER BIT(1)
43static unsigned arch_timers_present __initdata;
44
45static void __iomem *arch_counter_base;
46
47struct arch_timer {
48 void __iomem *base;
49 struct clock_event_device evt;
50};
51
52#define to_arch_timer(e) container_of(e, struct arch_timer, evt)
53
Mark Rutland8a4da6e2012-11-12 14:33:44 +000054static u32 arch_timer_rate;
55
56enum ppi_nr {
57 PHYS_SECURE_PPI,
58 PHYS_NONSECURE_PPI,
59 VIRT_PPI,
60 HYP_PPI,
61 MAX_TIMER_PPI
62};
63
64static int arch_timer_ppi[MAX_TIMER_PPI];
65
66static struct clock_event_device __percpu *arch_timer_evt;
67
68static bool arch_timer_use_virtual = true;
Stephen Boyd22006992013-07-18 16:59:32 -070069static bool arch_timer_mem_use_virtual;
Mark Rutland8a4da6e2012-11-12 14:33:44 +000070
71/*
72 * Architected system timer support.
73 */
74
Stephen Boyd60faddf2013-07-18 16:59:31 -070075static __always_inline
76void arch_timer_reg_write(int access, enum arch_timer_reg reg, u32 val,
Thomas Gleixnercfb6d652013-08-21 14:59:23 +020077 struct clock_event_device *clk)
Stephen Boyd60faddf2013-07-18 16:59:31 -070078{
Stephen Boyd22006992013-07-18 16:59:32 -070079 if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {
80 struct arch_timer *timer = to_arch_timer(clk);
81 switch (reg) {
82 case ARCH_TIMER_REG_CTRL:
83 writel_relaxed(val, timer->base + CNTP_CTL);
84 break;
85 case ARCH_TIMER_REG_TVAL:
86 writel_relaxed(val, timer->base + CNTP_TVAL);
87 break;
88 }
89 } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {
90 struct arch_timer *timer = to_arch_timer(clk);
91 switch (reg) {
92 case ARCH_TIMER_REG_CTRL:
93 writel_relaxed(val, timer->base + CNTV_CTL);
94 break;
95 case ARCH_TIMER_REG_TVAL:
96 writel_relaxed(val, timer->base + CNTV_TVAL);
97 break;
98 }
99 } else {
100 arch_timer_reg_write_cp15(access, reg, val);
101 }
Stephen Boyd60faddf2013-07-18 16:59:31 -0700102}
103
104static __always_inline
105u32 arch_timer_reg_read(int access, enum arch_timer_reg reg,
Thomas Gleixnercfb6d652013-08-21 14:59:23 +0200106 struct clock_event_device *clk)
Stephen Boyd60faddf2013-07-18 16:59:31 -0700107{
Stephen Boyd22006992013-07-18 16:59:32 -0700108 u32 val;
109
110 if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {
111 struct arch_timer *timer = to_arch_timer(clk);
112 switch (reg) {
113 case ARCH_TIMER_REG_CTRL:
114 val = readl_relaxed(timer->base + CNTP_CTL);
115 break;
116 case ARCH_TIMER_REG_TVAL:
117 val = readl_relaxed(timer->base + CNTP_TVAL);
118 break;
119 }
120 } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {
121 struct arch_timer *timer = to_arch_timer(clk);
122 switch (reg) {
123 case ARCH_TIMER_REG_CTRL:
124 val = readl_relaxed(timer->base + CNTV_CTL);
125 break;
126 case ARCH_TIMER_REG_TVAL:
127 val = readl_relaxed(timer->base + CNTV_TVAL);
128 break;
129 }
130 } else {
131 val = arch_timer_reg_read_cp15(access, reg);
132 }
133
134 return val;
Stephen Boyd60faddf2013-07-18 16:59:31 -0700135}
136
Stephen Boyde09f3cc2013-07-18 16:59:28 -0700137static __always_inline irqreturn_t timer_handler(const int access,
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000138 struct clock_event_device *evt)
139{
140 unsigned long ctrl;
Thomas Gleixnercfb6d652013-08-21 14:59:23 +0200141
Stephen Boyd60faddf2013-07-18 16:59:31 -0700142 ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, evt);
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000143 if (ctrl & ARCH_TIMER_CTRL_IT_STAT) {
144 ctrl |= ARCH_TIMER_CTRL_IT_MASK;
Stephen Boyd60faddf2013-07-18 16:59:31 -0700145 arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, evt);
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000146 evt->event_handler(evt);
147 return IRQ_HANDLED;
148 }
149
150 return IRQ_NONE;
151}
152
153static irqreturn_t arch_timer_handler_virt(int irq, void *dev_id)
154{
155 struct clock_event_device *evt = dev_id;
156
157 return timer_handler(ARCH_TIMER_VIRT_ACCESS, evt);
158}
159
160static irqreturn_t arch_timer_handler_phys(int irq, void *dev_id)
161{
162 struct clock_event_device *evt = dev_id;
163
164 return timer_handler(ARCH_TIMER_PHYS_ACCESS, evt);
165}
166
Stephen Boyd22006992013-07-18 16:59:32 -0700167static irqreturn_t arch_timer_handler_phys_mem(int irq, void *dev_id)
168{
169 struct clock_event_device *evt = dev_id;
170
171 return timer_handler(ARCH_TIMER_MEM_PHYS_ACCESS, evt);
172}
173
174static irqreturn_t arch_timer_handler_virt_mem(int irq, void *dev_id)
175{
176 struct clock_event_device *evt = dev_id;
177
178 return timer_handler(ARCH_TIMER_MEM_VIRT_ACCESS, evt);
179}
180
Stephen Boyd60faddf2013-07-18 16:59:31 -0700181static __always_inline void timer_set_mode(const int access, int mode,
182 struct clock_event_device *clk)
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000183{
184 unsigned long ctrl;
185 switch (mode) {
186 case CLOCK_EVT_MODE_UNUSED:
187 case CLOCK_EVT_MODE_SHUTDOWN:
Stephen Boyd60faddf2013-07-18 16:59:31 -0700188 ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000189 ctrl &= ~ARCH_TIMER_CTRL_ENABLE;
Stephen Boyd60faddf2013-07-18 16:59:31 -0700190 arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000191 break;
192 default:
193 break;
194 }
195}
196
197static void arch_timer_set_mode_virt(enum clock_event_mode mode,
198 struct clock_event_device *clk)
199{
Stephen Boyd60faddf2013-07-18 16:59:31 -0700200 timer_set_mode(ARCH_TIMER_VIRT_ACCESS, mode, clk);
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000201}
202
203static void arch_timer_set_mode_phys(enum clock_event_mode mode,
204 struct clock_event_device *clk)
205{
Stephen Boyd60faddf2013-07-18 16:59:31 -0700206 timer_set_mode(ARCH_TIMER_PHYS_ACCESS, mode, clk);
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000207}
208
Stephen Boyd22006992013-07-18 16:59:32 -0700209static void arch_timer_set_mode_virt_mem(enum clock_event_mode mode,
210 struct clock_event_device *clk)
211{
212 timer_set_mode(ARCH_TIMER_MEM_VIRT_ACCESS, mode, clk);
213}
214
215static void arch_timer_set_mode_phys_mem(enum clock_event_mode mode,
216 struct clock_event_device *clk)
217{
218 timer_set_mode(ARCH_TIMER_MEM_PHYS_ACCESS, mode, clk);
219}
220
Stephen Boyd60faddf2013-07-18 16:59:31 -0700221static __always_inline void set_next_event(const int access, unsigned long evt,
Thomas Gleixnercfb6d652013-08-21 14:59:23 +0200222 struct clock_event_device *clk)
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000223{
224 unsigned long ctrl;
Stephen Boyd60faddf2013-07-18 16:59:31 -0700225 ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000226 ctrl |= ARCH_TIMER_CTRL_ENABLE;
227 ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;
Stephen Boyd60faddf2013-07-18 16:59:31 -0700228 arch_timer_reg_write(access, ARCH_TIMER_REG_TVAL, evt, clk);
229 arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000230}
231
232static int arch_timer_set_next_event_virt(unsigned long evt,
Stephen Boyd60faddf2013-07-18 16:59:31 -0700233 struct clock_event_device *clk)
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000234{
Stephen Boyd60faddf2013-07-18 16:59:31 -0700235 set_next_event(ARCH_TIMER_VIRT_ACCESS, evt, clk);
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000236 return 0;
237}
238
239static int arch_timer_set_next_event_phys(unsigned long evt,
Stephen Boyd60faddf2013-07-18 16:59:31 -0700240 struct clock_event_device *clk)
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000241{
Stephen Boyd60faddf2013-07-18 16:59:31 -0700242 set_next_event(ARCH_TIMER_PHYS_ACCESS, evt, clk);
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000243 return 0;
244}
245
Stephen Boyd22006992013-07-18 16:59:32 -0700246static int arch_timer_set_next_event_virt_mem(unsigned long evt,
247 struct clock_event_device *clk)
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000248{
Stephen Boyd22006992013-07-18 16:59:32 -0700249 set_next_event(ARCH_TIMER_MEM_VIRT_ACCESS, evt, clk);
250 return 0;
251}
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000252
Stephen Boyd22006992013-07-18 16:59:32 -0700253static int arch_timer_set_next_event_phys_mem(unsigned long evt,
254 struct clock_event_device *clk)
255{
256 set_next_event(ARCH_TIMER_MEM_PHYS_ACCESS, evt, clk);
257 return 0;
258}
259
Thomas Gleixnercfb6d652013-08-21 14:59:23 +0200260static void __arch_timer_setup(unsigned type,
261 struct clock_event_device *clk)
Stephen Boyd22006992013-07-18 16:59:32 -0700262{
263 clk->features = CLOCK_EVT_FEAT_ONESHOT;
264
265 if (type == ARCH_CP15_TIMER) {
266 clk->features |= CLOCK_EVT_FEAT_C3STOP;
267 clk->name = "arch_sys_timer";
268 clk->rating = 450;
269 clk->cpumask = cpumask_of(smp_processor_id());
270 if (arch_timer_use_virtual) {
271 clk->irq = arch_timer_ppi[VIRT_PPI];
272 clk->set_mode = arch_timer_set_mode_virt;
273 clk->set_next_event = arch_timer_set_next_event_virt;
274 } else {
275 clk->irq = arch_timer_ppi[PHYS_SECURE_PPI];
276 clk->set_mode = arch_timer_set_mode_phys;
277 clk->set_next_event = arch_timer_set_next_event_phys;
278 }
279 } else {
280 clk->name = "arch_mem_timer";
281 clk->rating = 400;
282 clk->cpumask = cpu_all_mask;
283 if (arch_timer_mem_use_virtual) {
284 clk->set_mode = arch_timer_set_mode_virt_mem;
285 clk->set_next_event =
286 arch_timer_set_next_event_virt_mem;
287 } else {
288 clk->set_mode = arch_timer_set_mode_phys_mem;
289 clk->set_next_event =
290 arch_timer_set_next_event_phys_mem;
291 }
292 }
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000293
Stephen Boyd1ff99ea2013-07-18 16:59:30 -0700294 clk->set_mode(CLOCK_EVT_MODE_SHUTDOWN, clk);
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000295
Stephen Boyd22006992013-07-18 16:59:32 -0700296 clockevents_config_and_register(clk, arch_timer_rate, 0xf, 0x7fffffff);
297}
298
Will Deacon037f6372013-08-23 15:32:29 +0100299static void arch_timer_configure_evtstream(void)
300{
301 int evt_stream_div, pos;
302
303 /* Find the closest power of two to the divisor */
304 evt_stream_div = arch_timer_rate / ARCH_TIMER_EVT_STREAM_FREQ;
305 pos = fls(evt_stream_div);
306 if (pos > 1 && !(evt_stream_div & (1 << (pos - 2))))
307 pos--;
308 /* enable event stream */
309 arch_timer_evtstrm_enable(min(pos, 15));
310}
311
Paul Gortmaker8c37bb32013-06-19 11:32:08 -0400312static int arch_timer_setup(struct clock_event_device *clk)
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000313{
Stephen Boyd22006992013-07-18 16:59:32 -0700314 __arch_timer_setup(ARCH_CP15_TIMER, clk);
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000315
316 if (arch_timer_use_virtual)
317 enable_percpu_irq(arch_timer_ppi[VIRT_PPI], 0);
318 else {
319 enable_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI], 0);
320 if (arch_timer_ppi[PHYS_NONSECURE_PPI])
321 enable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI], 0);
322 }
323
324 arch_counter_set_user_access();
Will Deacon037f6372013-08-23 15:32:29 +0100325 if (IS_ENABLED(CONFIG_ARM_ARCH_TIMER_EVTSTREAM))
326 arch_timer_configure_evtstream();
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000327
328 return 0;
329}
330
Stephen Boyd22006992013-07-18 16:59:32 -0700331static void
332arch_timer_detect_rate(void __iomem *cntbase, struct device_node *np)
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000333{
Stephen Boyd22006992013-07-18 16:59:32 -0700334 /* Who has more than one independent system counter? */
335 if (arch_timer_rate)
336 return;
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000337
Stephen Boyd22006992013-07-18 16:59:32 -0700338 /* Try to determine the frequency from the device tree or CNTFRQ */
339 if (of_property_read_u32(np, "clock-frequency", &arch_timer_rate)) {
340 if (cntbase)
341 arch_timer_rate = readl_relaxed(cntbase + CNTFRQ);
342 else
343 arch_timer_rate = arch_timer_get_cntfrq();
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000344 }
345
Stephen Boyd22006992013-07-18 16:59:32 -0700346 /* Check the timer frequency. */
347 if (arch_timer_rate == 0)
348 pr_warn("Architected timer frequency not available\n");
349}
350
351static void arch_timer_banner(unsigned type)
352{
353 pr_info("Architected %s%s%s timer(s) running at %lu.%02luMHz (%s%s%s).\n",
354 type & ARCH_CP15_TIMER ? "cp15" : "",
355 type == (ARCH_CP15_TIMER | ARCH_MEM_TIMER) ? " and " : "",
356 type & ARCH_MEM_TIMER ? "mmio" : "",
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000357 (unsigned long)arch_timer_rate / 1000000,
358 (unsigned long)(arch_timer_rate / 10000) % 100,
Stephen Boyd22006992013-07-18 16:59:32 -0700359 type & ARCH_CP15_TIMER ?
360 arch_timer_use_virtual ? "virt" : "phys" :
361 "",
362 type == (ARCH_CP15_TIMER | ARCH_MEM_TIMER) ? "/" : "",
363 type & ARCH_MEM_TIMER ?
364 arch_timer_mem_use_virtual ? "virt" : "phys" :
365 "");
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000366}
367
368u32 arch_timer_get_rate(void)
369{
370 return arch_timer_rate;
371}
372
Stephen Boyd22006992013-07-18 16:59:32 -0700373static u64 arch_counter_get_cntvct_mem(void)
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000374{
Stephen Boyd22006992013-07-18 16:59:32 -0700375 u32 vct_lo, vct_hi, tmp_hi;
376
377 do {
378 vct_hi = readl_relaxed(arch_counter_base + CNTVCT_HI);
379 vct_lo = readl_relaxed(arch_counter_base + CNTVCT_LO);
380 tmp_hi = readl_relaxed(arch_counter_base + CNTVCT_HI);
381 } while (vct_hi != tmp_hi);
382
383 return ((u64) vct_hi << 32) | vct_lo;
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000384}
385
Stephen Boyd22006992013-07-18 16:59:32 -0700386/*
387 * Default to cp15 based access because arm64 uses this function for
388 * sched_clock() before DT is probed and the cp15 method is guaranteed
389 * to exist on arm64. arm doesn't use this before DT is probed so even
390 * if we don't have the cp15 accessors we won't have a problem.
391 */
392u64 (*arch_timer_read_counter)(void) = arch_counter_get_cntvct;
393
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000394static cycle_t arch_counter_read(struct clocksource *cs)
395{
Stephen Boyd22006992013-07-18 16:59:32 -0700396 return arch_timer_read_counter();
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000397}
398
399static cycle_t arch_counter_read_cc(const struct cyclecounter *cc)
400{
Stephen Boyd22006992013-07-18 16:59:32 -0700401 return arch_timer_read_counter();
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000402}
403
404static struct clocksource clocksource_counter = {
405 .name = "arch_sys_counter",
406 .rating = 400,
407 .read = arch_counter_read,
408 .mask = CLOCKSOURCE_MASK(56),
Stephen Boyd4fbcdc82013-09-27 13:13:12 -0700409 .flags = CLOCK_SOURCE_IS_CONTINUOUS | CLOCK_SOURCE_SUSPEND_NONSTOP,
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000410};
411
412static struct cyclecounter cyclecounter = {
413 .read = arch_counter_read_cc,
414 .mask = CLOCKSOURCE_MASK(56),
415};
416
417static struct timecounter timecounter;
418
419struct timecounter *arch_timer_get_timecounter(void)
420{
421 return &timecounter;
422}
423
Stephen Boyd22006992013-07-18 16:59:32 -0700424static void __init arch_counter_register(unsigned type)
425{
426 u64 start_count;
427
428 /* Register the CP15 based counter if we have one */
429 if (type & ARCH_CP15_TIMER)
430 arch_timer_read_counter = arch_counter_get_cntvct;
431 else
432 arch_timer_read_counter = arch_counter_get_cntvct_mem;
433
434 start_count = arch_timer_read_counter();
435 clocksource_register_hz(&clocksource_counter, arch_timer_rate);
436 cyclecounter.mult = clocksource_counter.mult;
437 cyclecounter.shift = clocksource_counter.shift;
438 timecounter_init(&timecounter, &cyclecounter, start_count);
439}
440
Paul Gortmaker8c37bb32013-06-19 11:32:08 -0400441static void arch_timer_stop(struct clock_event_device *clk)
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000442{
443 pr_debug("arch_timer_teardown disable IRQ%d cpu #%d\n",
444 clk->irq, smp_processor_id());
445
446 if (arch_timer_use_virtual)
447 disable_percpu_irq(arch_timer_ppi[VIRT_PPI]);
448 else {
449 disable_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI]);
450 if (arch_timer_ppi[PHYS_NONSECURE_PPI])
451 disable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI]);
452 }
453
454 clk->set_mode(CLOCK_EVT_MODE_UNUSED, clk);
455}
456
Paul Gortmaker8c37bb32013-06-19 11:32:08 -0400457static int arch_timer_cpu_notify(struct notifier_block *self,
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000458 unsigned long action, void *hcpu)
459{
Stephen Boydf31c2f12013-04-17 16:26:18 -0700460 /*
461 * Grab cpu pointer in each case to avoid spurious
462 * preemptible warnings
463 */
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000464 switch (action & ~CPU_TASKS_FROZEN) {
465 case CPU_STARTING:
Stephen Boydf31c2f12013-04-17 16:26:18 -0700466 arch_timer_setup(this_cpu_ptr(arch_timer_evt));
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000467 break;
468 case CPU_DYING:
Stephen Boydf31c2f12013-04-17 16:26:18 -0700469 arch_timer_stop(this_cpu_ptr(arch_timer_evt));
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000470 break;
471 }
472
473 return NOTIFY_OK;
474}
475
Paul Gortmaker8c37bb32013-06-19 11:32:08 -0400476static struct notifier_block arch_timer_cpu_nb = {
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000477 .notifier_call = arch_timer_cpu_notify,
478};
479
Sudeep KarkadaNagesha346e7482013-08-23 15:53:15 +0100480#ifdef CONFIG_CPU_PM
481static unsigned int saved_cntkctl;
482static int arch_timer_cpu_pm_notify(struct notifier_block *self,
483 unsigned long action, void *hcpu)
484{
485 if (action == CPU_PM_ENTER)
486 saved_cntkctl = arch_timer_get_cntkctl();
487 else if (action == CPU_PM_ENTER_FAILED || action == CPU_PM_EXIT)
488 arch_timer_set_cntkctl(saved_cntkctl);
489 return NOTIFY_OK;
490}
491
492static struct notifier_block arch_timer_cpu_pm_notifier = {
493 .notifier_call = arch_timer_cpu_pm_notify,
494};
495
496static int __init arch_timer_cpu_pm_init(void)
497{
498 return cpu_pm_register_notifier(&arch_timer_cpu_pm_notifier);
499}
500#else
501static int __init arch_timer_cpu_pm_init(void)
502{
503 return 0;
504}
505#endif
506
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000507static int __init arch_timer_register(void)
508{
509 int err;
510 int ppi;
511
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000512 arch_timer_evt = alloc_percpu(struct clock_event_device);
513 if (!arch_timer_evt) {
514 err = -ENOMEM;
515 goto out;
516 }
517
Stephen Boyd65cd4f62013-07-18 16:21:18 -0700518 clocksource_register_hz(&clocksource_counter, arch_timer_rate);
519 cyclecounter.mult = clocksource_counter.mult;
520 cyclecounter.shift = clocksource_counter.shift;
521 timecounter_init(&timecounter, &cyclecounter,
522 arch_counter_get_cntvct());
523
524 /* 56 bits minimum, so we assume worst case rollover */
525 sched_clock_register(arch_timer_read_counter, 56, arch_timer_rate);
526
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000527 if (arch_timer_use_virtual) {
528 ppi = arch_timer_ppi[VIRT_PPI];
529 err = request_percpu_irq(ppi, arch_timer_handler_virt,
530 "arch_timer", arch_timer_evt);
531 } else {
532 ppi = arch_timer_ppi[PHYS_SECURE_PPI];
533 err = request_percpu_irq(ppi, arch_timer_handler_phys,
534 "arch_timer", arch_timer_evt);
535 if (!err && arch_timer_ppi[PHYS_NONSECURE_PPI]) {
536 ppi = arch_timer_ppi[PHYS_NONSECURE_PPI];
537 err = request_percpu_irq(ppi, arch_timer_handler_phys,
538 "arch_timer", arch_timer_evt);
539 if (err)
540 free_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI],
541 arch_timer_evt);
542 }
543 }
544
545 if (err) {
546 pr_err("arch_timer: can't register interrupt %d (%d)\n",
547 ppi, err);
548 goto out_free;
549 }
550
551 err = register_cpu_notifier(&arch_timer_cpu_nb);
552 if (err)
553 goto out_free_irq;
554
Sudeep KarkadaNagesha346e7482013-08-23 15:53:15 +0100555 err = arch_timer_cpu_pm_init();
556 if (err)
557 goto out_unreg_notify;
558
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000559 /* Immediately configure the timer on the boot CPU */
560 arch_timer_setup(this_cpu_ptr(arch_timer_evt));
561
562 return 0;
563
Sudeep KarkadaNagesha346e7482013-08-23 15:53:15 +0100564out_unreg_notify:
565 unregister_cpu_notifier(&arch_timer_cpu_nb);
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000566out_free_irq:
567 if (arch_timer_use_virtual)
568 free_percpu_irq(arch_timer_ppi[VIRT_PPI], arch_timer_evt);
569 else {
570 free_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI],
571 arch_timer_evt);
572 if (arch_timer_ppi[PHYS_NONSECURE_PPI])
573 free_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI],
574 arch_timer_evt);
575 }
576
577out_free:
578 free_percpu(arch_timer_evt);
579out:
580 return err;
581}
582
Stephen Boyd22006992013-07-18 16:59:32 -0700583static int __init arch_timer_mem_register(void __iomem *base, unsigned int irq)
584{
585 int ret;
586 irq_handler_t func;
587 struct arch_timer *t;
588
589 t = kzalloc(sizeof(*t), GFP_KERNEL);
590 if (!t)
591 return -ENOMEM;
592
593 t->base = base;
594 t->evt.irq = irq;
595 __arch_timer_setup(ARCH_MEM_TIMER, &t->evt);
596
597 if (arch_timer_mem_use_virtual)
598 func = arch_timer_handler_virt_mem;
599 else
600 func = arch_timer_handler_phys_mem;
601
602 ret = request_irq(irq, func, IRQF_TIMER, "arch_mem_timer", &t->evt);
603 if (ret) {
604 pr_err("arch_timer: Failed to request mem timer irq\n");
605 kfree(t);
606 }
607
608 return ret;
609}
610
611static const struct of_device_id arch_timer_of_match[] __initconst = {
612 { .compatible = "arm,armv7-timer", },
613 { .compatible = "arm,armv8-timer", },
614 {},
615};
616
617static const struct of_device_id arch_timer_mem_of_match[] __initconst = {
618 { .compatible = "arm,armv7-timer-mem", },
619 {},
620};
621
622static void __init arch_timer_common_init(void)
623{
624 unsigned mask = ARCH_CP15_TIMER | ARCH_MEM_TIMER;
625
626 /* Wait until both nodes are probed if we have two timers */
627 if ((arch_timers_present & mask) != mask) {
628 if (of_find_matching_node(NULL, arch_timer_mem_of_match) &&
629 !(arch_timers_present & ARCH_MEM_TIMER))
630 return;
631 if (of_find_matching_node(NULL, arch_timer_of_match) &&
632 !(arch_timers_present & ARCH_CP15_TIMER))
633 return;
634 }
635
636 arch_timer_banner(arch_timers_present);
637 arch_counter_register(arch_timers_present);
638 arch_timer_arch_init();
639}
640
Rob Herring0583fe42013-04-10 18:27:51 -0500641static void __init arch_timer_init(struct device_node *np)
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000642{
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000643 int i;
644
Stephen Boyd22006992013-07-18 16:59:32 -0700645 if (arch_timers_present & ARCH_CP15_TIMER) {
Rob Herring0583fe42013-04-10 18:27:51 -0500646 pr_warn("arch_timer: multiple nodes in dt, skipping\n");
647 return;
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000648 }
649
Stephen Boyd22006992013-07-18 16:59:32 -0700650 arch_timers_present |= ARCH_CP15_TIMER;
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000651 for (i = PHYS_SECURE_PPI; i < MAX_TIMER_PPI; i++)
652 arch_timer_ppi[i] = irq_of_parse_and_map(np, i);
Stephen Boyd22006992013-07-18 16:59:32 -0700653 arch_timer_detect_rate(NULL, np);
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000654
655 /*
Marc Zyngier82668912013-01-10 11:13:07 +0000656 * If HYP mode is available, we know that the physical timer
657 * has been configured to be accessible from PL1. Use it, so
658 * that a guest can use the virtual timer instead.
659 *
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000660 * If no interrupt provided for virtual timer, we'll have to
661 * stick to the physical timer. It'd better be accessible...
662 */
Marc Zyngier82668912013-01-10 11:13:07 +0000663 if (is_hyp_mode_available() || !arch_timer_ppi[VIRT_PPI]) {
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000664 arch_timer_use_virtual = false;
665
666 if (!arch_timer_ppi[PHYS_SECURE_PPI] ||
667 !arch_timer_ppi[PHYS_NONSECURE_PPI]) {
668 pr_warn("arch_timer: No interrupt available, giving up\n");
Rob Herring0583fe42013-04-10 18:27:51 -0500669 return;
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000670 }
671 }
672
Rob Herring0583fe42013-04-10 18:27:51 -0500673 arch_timer_register();
Stephen Boyd22006992013-07-18 16:59:32 -0700674 arch_timer_common_init();
Mark Rutland8a4da6e2012-11-12 14:33:44 +0000675}
Rob Herring0583fe42013-04-10 18:27:51 -0500676CLOCKSOURCE_OF_DECLARE(armv7_arch_timer, "arm,armv7-timer", arch_timer_init);
677CLOCKSOURCE_OF_DECLARE(armv8_arch_timer, "arm,armv8-timer", arch_timer_init);
Stephen Boyd22006992013-07-18 16:59:32 -0700678
679static void __init arch_timer_mem_init(struct device_node *np)
680{
681 struct device_node *frame, *best_frame = NULL;
682 void __iomem *cntctlbase, *base;
683 unsigned int irq;
684 u32 cnttidr;
685
686 arch_timers_present |= ARCH_MEM_TIMER;
687 cntctlbase = of_iomap(np, 0);
688 if (!cntctlbase) {
689 pr_err("arch_timer: Can't find CNTCTLBase\n");
690 return;
691 }
692
693 cnttidr = readl_relaxed(cntctlbase + CNTTIDR);
694 iounmap(cntctlbase);
695
696 /*
697 * Try to find a virtual capable frame. Otherwise fall back to a
698 * physical capable frame.
699 */
700 for_each_available_child_of_node(np, frame) {
701 int n;
702
703 if (of_property_read_u32(frame, "frame-number", &n)) {
704 pr_err("arch_timer: Missing frame-number\n");
705 of_node_put(best_frame);
706 of_node_put(frame);
707 return;
708 }
709
710 if (cnttidr & CNTTIDR_VIRT(n)) {
711 of_node_put(best_frame);
712 best_frame = frame;
713 arch_timer_mem_use_virtual = true;
714 break;
715 }
716 of_node_put(best_frame);
717 best_frame = of_node_get(frame);
718 }
719
720 base = arch_counter_base = of_iomap(best_frame, 0);
721 if (!base) {
722 pr_err("arch_timer: Can't map frame's registers\n");
723 of_node_put(best_frame);
724 return;
725 }
726
727 if (arch_timer_mem_use_virtual)
728 irq = irq_of_parse_and_map(best_frame, 1);
729 else
730 irq = irq_of_parse_and_map(best_frame, 0);
731 of_node_put(best_frame);
732 if (!irq) {
733 pr_err("arch_timer: Frame missing %s irq",
Thomas Gleixnercfb6d652013-08-21 14:59:23 +0200734 arch_timer_mem_use_virtual ? "virt" : "phys");
Stephen Boyd22006992013-07-18 16:59:32 -0700735 return;
736 }
737
738 arch_timer_detect_rate(base, np);
739 arch_timer_mem_register(base, irq);
740 arch_timer_common_init();
741}
742CLOCKSOURCE_OF_DECLARE(armv7_arch_timer_mem, "arm,armv7-timer-mem",
743 arch_timer_mem_init);