blob: 6a5b0ec460da8c5c6f4fe83074c6b2ec916bae80 [file] [log] [blame]
Greg Kroah-Hartmanb2441312017-11-01 15:07:57 +01001/* SPDX-License-Identifier: GPL-2.0 */
Brian Gerst1a3b1d82010-01-07 11:53:33 -05002#ifndef _ASM_X86_ATOMIC64_32_H
3#define _ASM_X86_ATOMIC64_32_H
4
5#include <linux/compiler.h>
6#include <linux/types.h>
Brian Gerst1a3b1d82010-01-07 11:53:33 -05007//#include <asm/cmpxchg.h>
8
9/* An 64bit atomic type */
10
11typedef struct {
12 u64 __aligned(8) counter;
13} atomic64_t;
14
15#define ATOMIC64_INIT(val) { (val) }
16
Jan Beulich819165f2012-01-20 16:21:41 +000017#define __ATOMIC64_DECL(sym) void atomic64_##sym(atomic64_t *, ...)
18#ifndef ATOMIC64_EXPORT
19#define ATOMIC64_DECL_ONE __ATOMIC64_DECL
Luca Barbieria7e926a2010-02-24 10:54:25 +010020#else
Jan Beulich819165f2012-01-20 16:21:41 +000021#define ATOMIC64_DECL_ONE(sym) __ATOMIC64_DECL(sym); \
22 ATOMIC64_EXPORT(atomic64_##sym)
Luca Barbieria7e926a2010-02-24 10:54:25 +010023#endif
24
Jan Beulich819165f2012-01-20 16:21:41 +000025#ifdef CONFIG_X86_CMPXCHG64
26#define __alternative_atomic64(f, g, out, in...) \
27 asm volatile("call %P[func]" \
28 : out : [func] "i" (atomic64_##g##_cx8), ## in)
29
30#define ATOMIC64_DECL(sym) ATOMIC64_DECL_ONE(sym##_cx8)
31#else
32#define __alternative_atomic64(f, g, out, in...) \
33 alternative_call(atomic64_##f##_386, atomic64_##g##_cx8, \
34 X86_FEATURE_CX8, ASM_OUTPUT2(out), ## in)
35
36#define ATOMIC64_DECL(sym) ATOMIC64_DECL_ONE(sym##_cx8); \
37 ATOMIC64_DECL_ONE(sym##_386)
38
39ATOMIC64_DECL_ONE(add_386);
40ATOMIC64_DECL_ONE(sub_386);
41ATOMIC64_DECL_ONE(inc_386);
42ATOMIC64_DECL_ONE(dec_386);
43#endif
44
45#define alternative_atomic64(f, out, in...) \
46 __alternative_atomic64(f, f, ASM_OUTPUT2(out), ## in)
47
48ATOMIC64_DECL(read);
49ATOMIC64_DECL(set);
50ATOMIC64_DECL(xchg);
51ATOMIC64_DECL(add_return);
52ATOMIC64_DECL(sub_return);
53ATOMIC64_DECL(inc_return);
54ATOMIC64_DECL(dec_return);
55ATOMIC64_DECL(dec_if_positive);
56ATOMIC64_DECL(inc_not_zero);
57ATOMIC64_DECL(add_unless);
58
59#undef ATOMIC64_DECL
60#undef ATOMIC64_DECL_ONE
61#undef __ATOMIC64_DECL
62#undef ATOMIC64_EXPORT
Luca Barbieria7e926a2010-02-24 10:54:25 +010063
64/**
Dmitry Vyukov8bf705d2018-01-29 18:26:05 +010065 * arch_atomic64_cmpxchg - cmpxchg atomic64 variable
Philipp Hahn1f045972012-05-02 18:09:35 +020066 * @v: pointer to type atomic64_t
Luca Barbieria7e926a2010-02-24 10:54:25 +010067 * @o: expected value
68 * @n: new value
69 *
70 * Atomically sets @v to @n if it was equal to @o and returns
71 * the old value.
72 */
73
Dmitry Vyukov8bf705d2018-01-29 18:26:05 +010074static inline long long arch_atomic64_cmpxchg(atomic64_t *v, long long o,
75 long long n)
Luca Barbieria7e926a2010-02-24 10:54:25 +010076{
Dmitry Vyukov8bf705d2018-01-29 18:26:05 +010077 return arch_cmpxchg64(&v->counter, o, n);
Luca Barbieria7e926a2010-02-24 10:54:25 +010078}
Brian Gerst1a3b1d82010-01-07 11:53:33 -050079
80/**
Dmitry Vyukov8bf705d2018-01-29 18:26:05 +010081 * arch_atomic64_xchg - xchg atomic64 variable
Luca Barbieria7e926a2010-02-24 10:54:25 +010082 * @v: pointer to type atomic64_t
83 * @n: value to assign
Brian Gerst1a3b1d82010-01-07 11:53:33 -050084 *
Luca Barbieria7e926a2010-02-24 10:54:25 +010085 * Atomically xchgs the value of @v to @n and returns
Brian Gerst1a3b1d82010-01-07 11:53:33 -050086 * the old value.
87 */
Dmitry Vyukov8bf705d2018-01-29 18:26:05 +010088static inline long long arch_atomic64_xchg(atomic64_t *v, long long n)
Luca Barbieria7e926a2010-02-24 10:54:25 +010089{
90 long long o;
91 unsigned high = (unsigned)(n >> 32);
92 unsigned low = (unsigned)n;
Jan Beulich819165f2012-01-20 16:21:41 +000093 alternative_atomic64(xchg, "=&A" (o),
94 "S" (v), "b" (low), "c" (high)
95 : "memory");
Luca Barbieria7e926a2010-02-24 10:54:25 +010096 return o;
97}
Brian Gerst1a3b1d82010-01-07 11:53:33 -050098
99/**
Dmitry Vyukov8bf705d2018-01-29 18:26:05 +0100100 * arch_atomic64_set - set atomic64 variable
Luca Barbieria7e926a2010-02-24 10:54:25 +0100101 * @v: pointer to type atomic64_t
Philipp Hahn1f045972012-05-02 18:09:35 +0200102 * @i: value to assign
Brian Gerst1a3b1d82010-01-07 11:53:33 -0500103 *
Luca Barbieria7e926a2010-02-24 10:54:25 +0100104 * Atomically sets the value of @v to @n.
Brian Gerst1a3b1d82010-01-07 11:53:33 -0500105 */
Dmitry Vyukov8bf705d2018-01-29 18:26:05 +0100106static inline void arch_atomic64_set(atomic64_t *v, long long i)
Luca Barbieria7e926a2010-02-24 10:54:25 +0100107{
108 unsigned high = (unsigned)(i >> 32);
109 unsigned low = (unsigned)i;
Jan Beulich819165f2012-01-20 16:21:41 +0000110 alternative_atomic64(set, /* no output */,
111 "S" (v), "b" (low), "c" (high)
112 : "eax", "edx", "memory");
Luca Barbieria7e926a2010-02-24 10:54:25 +0100113}
Brian Gerst1a3b1d82010-01-07 11:53:33 -0500114
115/**
Dmitry Vyukov8bf705d2018-01-29 18:26:05 +0100116 * arch_atomic64_read - read atomic64 variable
Luca Barbieria7e926a2010-02-24 10:54:25 +0100117 * @v: pointer to type atomic64_t
Brian Gerst1a3b1d82010-01-07 11:53:33 -0500118 *
Luca Barbieria7e926a2010-02-24 10:54:25 +0100119 * Atomically reads the value of @v and returns it.
Brian Gerst1a3b1d82010-01-07 11:53:33 -0500120 */
Dmitry Vyukov8bf705d2018-01-29 18:26:05 +0100121static inline long long arch_atomic64_read(const atomic64_t *v)
Brian Gerst1a3b1d82010-01-07 11:53:33 -0500122{
Luca Barbieria7e926a2010-02-24 10:54:25 +0100123 long long r;
Jan Beulich819165f2012-01-20 16:21:41 +0000124 alternative_atomic64(read, "=&A" (r), "c" (v) : "memory");
Luca Barbieria7e926a2010-02-24 10:54:25 +0100125 return r;
Joe Perches447a5642018-03-21 15:09:32 -0700126}
Brian Gerst1a3b1d82010-01-07 11:53:33 -0500127
128/**
Dmitry Vyukov8bf705d2018-01-29 18:26:05 +0100129 * arch_atomic64_add_return - add and return
Luca Barbieria7e926a2010-02-24 10:54:25 +0100130 * @i: integer value to add
131 * @v: pointer to type atomic64_t
Brian Gerst1a3b1d82010-01-07 11:53:33 -0500132 *
Luca Barbieria7e926a2010-02-24 10:54:25 +0100133 * Atomically adds @i to @v and returns @i + *@v
Brian Gerst1a3b1d82010-01-07 11:53:33 -0500134 */
Dmitry Vyukov8bf705d2018-01-29 18:26:05 +0100135static inline long long arch_atomic64_add_return(long long i, atomic64_t *v)
Luca Barbieria7e926a2010-02-24 10:54:25 +0100136{
Jan Beulich819165f2012-01-20 16:21:41 +0000137 alternative_atomic64(add_return,
138 ASM_OUTPUT2("+A" (i), "+c" (v)),
139 ASM_NO_INPUT_CLOBBER("memory"));
Luca Barbieria7e926a2010-02-24 10:54:25 +0100140 return i;
141}
Brian Gerst1a3b1d82010-01-07 11:53:33 -0500142
143/*
144 * Other variants with different arithmetic operators:
145 */
Dmitry Vyukov8bf705d2018-01-29 18:26:05 +0100146static inline long long arch_atomic64_sub_return(long long i, atomic64_t *v)
Luca Barbieria7e926a2010-02-24 10:54:25 +0100147{
Jan Beulich819165f2012-01-20 16:21:41 +0000148 alternative_atomic64(sub_return,
149 ASM_OUTPUT2("+A" (i), "+c" (v)),
150 ASM_NO_INPUT_CLOBBER("memory"));
Luca Barbieria7e926a2010-02-24 10:54:25 +0100151 return i;
152}
153
Dmitry Vyukov8bf705d2018-01-29 18:26:05 +0100154static inline long long arch_atomic64_inc_return(atomic64_t *v)
Luca Barbieria7e926a2010-02-24 10:54:25 +0100155{
156 long long a;
Jan Beulich819165f2012-01-20 16:21:41 +0000157 alternative_atomic64(inc_return, "=&A" (a),
158 "S" (v) : "memory", "ecx");
Luca Barbieria7e926a2010-02-24 10:54:25 +0100159 return a;
160}
Mark Rutland98375592018-06-21 13:13:19 +0100161#define arch_atomic64_inc_return arch_atomic64_inc_return
Luca Barbieria7e926a2010-02-24 10:54:25 +0100162
Dmitry Vyukov8bf705d2018-01-29 18:26:05 +0100163static inline long long arch_atomic64_dec_return(atomic64_t *v)
Luca Barbieria7e926a2010-02-24 10:54:25 +0100164{
165 long long a;
Jan Beulich819165f2012-01-20 16:21:41 +0000166 alternative_atomic64(dec_return, "=&A" (a),
167 "S" (v) : "memory", "ecx");
Luca Barbieria7e926a2010-02-24 10:54:25 +0100168 return a;
169}
Mark Rutland98375592018-06-21 13:13:19 +0100170#define arch_atomic64_dec_return arch_atomic64_dec_return
Brian Gerst1a3b1d82010-01-07 11:53:33 -0500171
172/**
Dmitry Vyukov8bf705d2018-01-29 18:26:05 +0100173 * arch_atomic64_add - add integer to atomic64 variable
Luca Barbieria7e926a2010-02-24 10:54:25 +0100174 * @i: integer value to add
175 * @v: pointer to type atomic64_t
Brian Gerst1a3b1d82010-01-07 11:53:33 -0500176 *
Luca Barbieria7e926a2010-02-24 10:54:25 +0100177 * Atomically adds @i to @v.
Brian Gerst1a3b1d82010-01-07 11:53:33 -0500178 */
Dmitry Vyukov8bf705d2018-01-29 18:26:05 +0100179static inline long long arch_atomic64_add(long long i, atomic64_t *v)
Luca Barbieria7e926a2010-02-24 10:54:25 +0100180{
Jan Beulich819165f2012-01-20 16:21:41 +0000181 __alternative_atomic64(add, add_return,
182 ASM_OUTPUT2("+A" (i), "+c" (v)),
183 ASM_NO_INPUT_CLOBBER("memory"));
Luca Barbieria7e926a2010-02-24 10:54:25 +0100184 return i;
185}
Brian Gerst1a3b1d82010-01-07 11:53:33 -0500186
187/**
Dmitry Vyukov8bf705d2018-01-29 18:26:05 +0100188 * arch_atomic64_sub - subtract the atomic64 variable
Luca Barbieria7e926a2010-02-24 10:54:25 +0100189 * @i: integer value to subtract
190 * @v: pointer to type atomic64_t
Brian Gerst1a3b1d82010-01-07 11:53:33 -0500191 *
Luca Barbieria7e926a2010-02-24 10:54:25 +0100192 * Atomically subtracts @i from @v.
Brian Gerst1a3b1d82010-01-07 11:53:33 -0500193 */
Dmitry Vyukov8bf705d2018-01-29 18:26:05 +0100194static inline long long arch_atomic64_sub(long long i, atomic64_t *v)
Luca Barbieria7e926a2010-02-24 10:54:25 +0100195{
Jan Beulich819165f2012-01-20 16:21:41 +0000196 __alternative_atomic64(sub, sub_return,
197 ASM_OUTPUT2("+A" (i), "+c" (v)),
198 ASM_NO_INPUT_CLOBBER("memory"));
Luca Barbieria7e926a2010-02-24 10:54:25 +0100199 return i;
200}
Brian Gerst1a3b1d82010-01-07 11:53:33 -0500201
202/**
Dmitry Vyukov8bf705d2018-01-29 18:26:05 +0100203 * arch_atomic64_inc - increment atomic64 variable
Luca Barbieria7e926a2010-02-24 10:54:25 +0100204 * @v: pointer to type atomic64_t
Brian Gerst1a3b1d82010-01-07 11:53:33 -0500205 *
Luca Barbieria7e926a2010-02-24 10:54:25 +0100206 * Atomically increments @v by 1.
Brian Gerst1a3b1d82010-01-07 11:53:33 -0500207 */
Dmitry Vyukov8bf705d2018-01-29 18:26:05 +0100208static inline void arch_atomic64_inc(atomic64_t *v)
Luca Barbieria7e926a2010-02-24 10:54:25 +0100209{
Jan Beulich819165f2012-01-20 16:21:41 +0000210 __alternative_atomic64(inc, inc_return, /* no output */,
211 "S" (v) : "memory", "eax", "ecx", "edx");
Luca Barbieria7e926a2010-02-24 10:54:25 +0100212}
Randy Dunlap4331f4d2018-09-02 19:30:53 -0700213#define arch_atomic64_inc arch_atomic64_inc
Brian Gerst1a3b1d82010-01-07 11:53:33 -0500214
215/**
Dmitry Vyukov8bf705d2018-01-29 18:26:05 +0100216 * arch_atomic64_dec - decrement atomic64 variable
Philipp Hahn1f045972012-05-02 18:09:35 +0200217 * @v: pointer to type atomic64_t
Brian Gerst1a3b1d82010-01-07 11:53:33 -0500218 *
Philipp Hahn1f045972012-05-02 18:09:35 +0200219 * Atomically decrements @v by 1.
Brian Gerst1a3b1d82010-01-07 11:53:33 -0500220 */
Dmitry Vyukov8bf705d2018-01-29 18:26:05 +0100221static inline void arch_atomic64_dec(atomic64_t *v)
Luca Barbieria7e926a2010-02-24 10:54:25 +0100222{
Jan Beulich819165f2012-01-20 16:21:41 +0000223 __alternative_atomic64(dec, dec_return, /* no output */,
224 "S" (v) : "memory", "eax", "ecx", "edx");
Luca Barbieria7e926a2010-02-24 10:54:25 +0100225}
Randy Dunlap4331f4d2018-09-02 19:30:53 -0700226#define arch_atomic64_dec arch_atomic64_dec
Brian Gerst1a3b1d82010-01-07 11:53:33 -0500227
228/**
Dmitry Vyukov8bf705d2018-01-29 18:26:05 +0100229 * arch_atomic64_add_unless - add unless the number is a given value
Luca Barbieria7e926a2010-02-24 10:54:25 +0100230 * @v: pointer of type atomic64_t
231 * @a: the amount to add to v...
232 * @u: ...unless v is equal to u.
233 *
234 * Atomically adds @a to @v, so long as it was not @u.
Jan Beulich819165f2012-01-20 16:21:41 +0000235 * Returns non-zero if the add was done, zero otherwise.
Luca Barbieria7e926a2010-02-24 10:54:25 +0100236 */
Dmitry Vyukov8bf705d2018-01-29 18:26:05 +0100237static inline int arch_atomic64_add_unless(atomic64_t *v, long long a,
238 long long u)
Luca Barbieria7e926a2010-02-24 10:54:25 +0100239{
240 unsigned low = (unsigned)u;
241 unsigned high = (unsigned)(u >> 32);
Jan Beulich819165f2012-01-20 16:21:41 +0000242 alternative_atomic64(add_unless,
Jan Beulichcb8095b2012-01-20 16:22:04 +0000243 ASM_OUTPUT2("+A" (a), "+c" (low), "+D" (high)),
244 "S" (v) : "memory");
Luca Barbieria7e926a2010-02-24 10:54:25 +0100245 return (int)a;
246}
247
Dmitry Vyukov8bf705d2018-01-29 18:26:05 +0100248static inline int arch_atomic64_inc_not_zero(atomic64_t *v)
Luca Barbieria7e926a2010-02-24 10:54:25 +0100249{
250 int r;
Jan Beulich819165f2012-01-20 16:21:41 +0000251 alternative_atomic64(inc_not_zero, "=&a" (r),
252 "S" (v) : "ecx", "edx", "memory");
Luca Barbieria7e926a2010-02-24 10:54:25 +0100253 return r;
254}
Randy Dunlap4331f4d2018-09-02 19:30:53 -0700255#define arch_atomic64_inc_not_zero arch_atomic64_inc_not_zero
Luca Barbieria7e926a2010-02-24 10:54:25 +0100256
Dmitry Vyukov8bf705d2018-01-29 18:26:05 +0100257static inline long long arch_atomic64_dec_if_positive(atomic64_t *v)
Luca Barbieria7e926a2010-02-24 10:54:25 +0100258{
259 long long r;
Jan Beulich819165f2012-01-20 16:21:41 +0000260 alternative_atomic64(dec_if_positive, "=&A" (r),
261 "S" (v) : "ecx", "memory");
Luca Barbieria7e926a2010-02-24 10:54:25 +0100262 return r;
263}
Randy Dunlap4331f4d2018-09-02 19:30:53 -0700264#define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive
Luca Barbieria7e926a2010-02-24 10:54:25 +0100265
Jan Beulich819165f2012-01-20 16:21:41 +0000266#undef alternative_atomic64
267#undef __alternative_atomic64
Brian Gerst1a3b1d82010-01-07 11:53:33 -0500268
Dmitry Vyukov8bf705d2018-01-29 18:26:05 +0100269static inline void arch_atomic64_and(long long i, atomic64_t *v)
Dmitry Vyukovba1c9f82017-06-17 11:15:27 +0200270{
271 long long old, c = 0;
272
Dmitry Vyukov8bf705d2018-01-29 18:26:05 +0100273 while ((old = arch_atomic64_cmpxchg(v, c, c & i)) != c)
Dmitry Vyukovba1c9f82017-06-17 11:15:27 +0200274 c = old;
Peter Zijlstra7fc18452014-04-23 20:28:37 +0200275}
276
Dmitry Vyukov8bf705d2018-01-29 18:26:05 +0100277static inline long long arch_atomic64_fetch_and(long long i, atomic64_t *v)
Dmitry Vyukovba1c9f82017-06-17 11:15:27 +0200278{
279 long long old, c = 0;
280
Dmitry Vyukov8bf705d2018-01-29 18:26:05 +0100281 while ((old = arch_atomic64_cmpxchg(v, c, c & i)) != c)
Dmitry Vyukovba1c9f82017-06-17 11:15:27 +0200282 c = old;
283
284 return old;
Peter Zijlstraa8bccca2016-04-18 01:16:03 +0200285}
Peter Zijlstra7fc18452014-04-23 20:28:37 +0200286
Dmitry Vyukov8bf705d2018-01-29 18:26:05 +0100287static inline void arch_atomic64_or(long long i, atomic64_t *v)
Dmitry Vyukovba1c9f82017-06-17 11:15:27 +0200288{
289 long long old, c = 0;
290
Dmitry Vyukov8bf705d2018-01-29 18:26:05 +0100291 while ((old = arch_atomic64_cmpxchg(v, c, c | i)) != c)
Dmitry Vyukovba1c9f82017-06-17 11:15:27 +0200292 c = old;
293}
294
Dmitry Vyukov8bf705d2018-01-29 18:26:05 +0100295static inline long long arch_atomic64_fetch_or(long long i, atomic64_t *v)
Dmitry Vyukovba1c9f82017-06-17 11:15:27 +0200296{
297 long long old, c = 0;
298
Dmitry Vyukov8bf705d2018-01-29 18:26:05 +0100299 while ((old = arch_atomic64_cmpxchg(v, c, c | i)) != c)
Dmitry Vyukovba1c9f82017-06-17 11:15:27 +0200300 c = old;
301
302 return old;
303}
304
Dmitry Vyukov8bf705d2018-01-29 18:26:05 +0100305static inline void arch_atomic64_xor(long long i, atomic64_t *v)
Dmitry Vyukovba1c9f82017-06-17 11:15:27 +0200306{
307 long long old, c = 0;
308
Dmitry Vyukov8bf705d2018-01-29 18:26:05 +0100309 while ((old = arch_atomic64_cmpxchg(v, c, c ^ i)) != c)
Dmitry Vyukovba1c9f82017-06-17 11:15:27 +0200310 c = old;
311}
312
Dmitry Vyukov8bf705d2018-01-29 18:26:05 +0100313static inline long long arch_atomic64_fetch_xor(long long i, atomic64_t *v)
Dmitry Vyukovba1c9f82017-06-17 11:15:27 +0200314{
315 long long old, c = 0;
316
Dmitry Vyukov8bf705d2018-01-29 18:26:05 +0100317 while ((old = arch_atomic64_cmpxchg(v, c, c ^ i)) != c)
Dmitry Vyukovba1c9f82017-06-17 11:15:27 +0200318 c = old;
319
320 return old;
321}
322
Dmitry Vyukov8bf705d2018-01-29 18:26:05 +0100323static inline long long arch_atomic64_fetch_add(long long i, atomic64_t *v)
Dmitry Vyukovba1c9f82017-06-17 11:15:27 +0200324{
325 long long old, c = 0;
326
Dmitry Vyukov8bf705d2018-01-29 18:26:05 +0100327 while ((old = arch_atomic64_cmpxchg(v, c, c + i)) != c)
Dmitry Vyukovba1c9f82017-06-17 11:15:27 +0200328 c = old;
329
330 return old;
331}
Peter Zijlstraa8bccca2016-04-18 01:16:03 +0200332
Dmitry Vyukov8bf705d2018-01-29 18:26:05 +0100333#define arch_atomic64_fetch_sub(i, v) arch_atomic64_fetch_add(-(i), (v))
Peter Zijlstraa8bccca2016-04-18 01:16:03 +0200334
Brian Gerst1a3b1d82010-01-07 11:53:33 -0500335#endif /* _ASM_X86_ATOMIC64_32_H */