blob: 4cab9bb823fbbb4511225a695927a9387bce7eec [file] [log] [blame]
Greg Kroah-Hartmanb2441312017-11-01 15:07:57 +01001/* SPDX-License-Identifier: GPL-2.0 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002/*
3 * Copyright 1995, Russell King.
4 * Various bits and pieces copyrights include:
5 * Linus Torvalds (test_bit).
6 * Big endian support: Copyright 2001, Nicolas Pitre
7 * reworked by rmk.
8 *
9 * bit 0 is the LSB of an "unsigned long" quantity.
10 *
11 * Please note that the code in this file should never be included
12 * from user space. Many of these are not implemented in assembler
13 * since they would be too costly. Also, they require privileged
14 * instructions (which are not available from user mode) to ensure
15 * that they are atomic.
16 */
17
18#ifndef __ASM_ARM_BITOPS_H
19#define __ASM_ARM_BITOPS_H
20
21#ifdef __KERNEL__
22
Jiri Slaby06245172007-10-18 23:40:26 -070023#ifndef _LINUX_BITOPS_H
24#error only <linux/bitops.h> can be included directly
25#endif
26
Russell King8dc39b82005-11-16 17:23:57 +000027#include <linux/compiler.h>
David Howells9f97da72012-03-28 18:30:01 +010028#include <linux/irqflags.h>
Peter Zijlstra030d0172014-03-12 17:11:00 +010029#include <asm/barrier.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070030
31/*
32 * These functions are the basis of our bit ops.
33 *
34 * First, the atomic bitops. These use native endian.
35 */
36static inline void ____atomic_set_bit(unsigned int bit, volatile unsigned long *p)
37{
38 unsigned long flags;
Masahiro Yamada89019252015-08-17 03:59:52 +010039 unsigned long mask = BIT_MASK(bit);
Linus Torvalds1da177e2005-04-16 15:20:36 -070040
Masahiro Yamada89019252015-08-17 03:59:52 +010041 p += BIT_WORD(bit);
Linus Torvalds1da177e2005-04-16 15:20:36 -070042
Lennert Buytenheke7cc2c52006-09-21 03:35:20 +010043 raw_local_irq_save(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -070044 *p |= mask;
Lennert Buytenheke7cc2c52006-09-21 03:35:20 +010045 raw_local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -070046}
47
48static inline void ____atomic_clear_bit(unsigned int bit, volatile unsigned long *p)
49{
50 unsigned long flags;
Masahiro Yamada89019252015-08-17 03:59:52 +010051 unsigned long mask = BIT_MASK(bit);
Linus Torvalds1da177e2005-04-16 15:20:36 -070052
Masahiro Yamada89019252015-08-17 03:59:52 +010053 p += BIT_WORD(bit);
Linus Torvalds1da177e2005-04-16 15:20:36 -070054
Lennert Buytenheke7cc2c52006-09-21 03:35:20 +010055 raw_local_irq_save(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -070056 *p &= ~mask;
Lennert Buytenheke7cc2c52006-09-21 03:35:20 +010057 raw_local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -070058}
59
60static inline void ____atomic_change_bit(unsigned int bit, volatile unsigned long *p)
61{
62 unsigned long flags;
Masahiro Yamada89019252015-08-17 03:59:52 +010063 unsigned long mask = BIT_MASK(bit);
Linus Torvalds1da177e2005-04-16 15:20:36 -070064
Masahiro Yamada89019252015-08-17 03:59:52 +010065 p += BIT_WORD(bit);
Linus Torvalds1da177e2005-04-16 15:20:36 -070066
Lennert Buytenheke7cc2c52006-09-21 03:35:20 +010067 raw_local_irq_save(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -070068 *p ^= mask;
Lennert Buytenheke7cc2c52006-09-21 03:35:20 +010069 raw_local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -070070}
71
72static inline int
73____atomic_test_and_set_bit(unsigned int bit, volatile unsigned long *p)
74{
75 unsigned long flags;
76 unsigned int res;
Masahiro Yamada89019252015-08-17 03:59:52 +010077 unsigned long mask = BIT_MASK(bit);
Linus Torvalds1da177e2005-04-16 15:20:36 -070078
Masahiro Yamada89019252015-08-17 03:59:52 +010079 p += BIT_WORD(bit);
Linus Torvalds1da177e2005-04-16 15:20:36 -070080
Lennert Buytenheke7cc2c52006-09-21 03:35:20 +010081 raw_local_irq_save(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -070082 res = *p;
83 *p = res | mask;
Lennert Buytenheke7cc2c52006-09-21 03:35:20 +010084 raw_local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -070085
Johannes Weinere9ac8292009-07-21 17:08:28 +020086 return (res & mask) != 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -070087}
88
89static inline int
90____atomic_test_and_clear_bit(unsigned int bit, volatile unsigned long *p)
91{
92 unsigned long flags;
93 unsigned int res;
Masahiro Yamada89019252015-08-17 03:59:52 +010094 unsigned long mask = BIT_MASK(bit);
Linus Torvalds1da177e2005-04-16 15:20:36 -070095
Masahiro Yamada89019252015-08-17 03:59:52 +010096 p += BIT_WORD(bit);
Linus Torvalds1da177e2005-04-16 15:20:36 -070097
Lennert Buytenheke7cc2c52006-09-21 03:35:20 +010098 raw_local_irq_save(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -070099 res = *p;
100 *p = res & ~mask;
Lennert Buytenheke7cc2c52006-09-21 03:35:20 +0100101 raw_local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700102
Johannes Weinere9ac8292009-07-21 17:08:28 +0200103 return (res & mask) != 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700104}
105
106static inline int
107____atomic_test_and_change_bit(unsigned int bit, volatile unsigned long *p)
108{
109 unsigned long flags;
110 unsigned int res;
Masahiro Yamada89019252015-08-17 03:59:52 +0100111 unsigned long mask = BIT_MASK(bit);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700112
Masahiro Yamada89019252015-08-17 03:59:52 +0100113 p += BIT_WORD(bit);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700114
Lennert Buytenheke7cc2c52006-09-21 03:35:20 +0100115 raw_local_irq_save(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700116 res = *p;
117 *p = res ^ mask;
Lennert Buytenheke7cc2c52006-09-21 03:35:20 +0100118 raw_local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700119
Johannes Weinere9ac8292009-07-21 17:08:28 +0200120 return (res & mask) != 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700121}
122
Akinobu Mitab89c3b12006-03-26 01:39:19 -0800123#include <asm-generic/bitops/non-atomic.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700124
125/*
126 * A note about Endian-ness.
127 * -------------------------
128 *
129 * When the ARM is put into big endian mode via CR15, the processor
130 * merely swaps the order of bytes within words, thus:
131 *
132 * ------------ physical data bus bits -----------
133 * D31 ... D24 D23 ... D16 D15 ... D8 D7 ... D0
134 * little byte 3 byte 2 byte 1 byte 0
135 * big byte 0 byte 1 byte 2 byte 3
136 *
137 * This means that reading a 32-bit word at address 0 returns the same
138 * value irrespective of the endian mode bit.
139 *
140 * Peripheral devices should be connected with the data bus reversed in
141 * "Big Endian" mode. ARM Application Note 61 is applicable, and is
142 * available from http://www.arm.com/.
143 *
144 * The following assumes that the data bus connectivity for big endian
145 * mode has been followed.
146 *
147 * Note that bit 0 is defined to be 32-bit word bit 0, not byte 0 bit 0.
148 */
149
150/*
Russell King6323f0c2011-01-16 18:02:17 +0000151 * Native endian assembly bitops. nr = 0 -> word 0 bit 0.
152 */
153extern void _set_bit(int nr, volatile unsigned long * p);
154extern void _clear_bit(int nr, volatile unsigned long * p);
155extern void _change_bit(int nr, volatile unsigned long * p);
156extern int _test_and_set_bit(int nr, volatile unsigned long * p);
157extern int _test_and_clear_bit(int nr, volatile unsigned long * p);
158extern int _test_and_change_bit(int nr, volatile unsigned long * p);
159
160/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700161 * Little endian assembly bitops. nr = 0 -> byte 0 bit 0.
162 */
Marc Gonzalez2d618fe2017-05-30 16:41:31 +0100163extern int _find_first_zero_bit_le(const unsigned long *p, unsigned size);
164extern int _find_next_zero_bit_le(const unsigned long *p, int size, int offset);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700165extern int _find_first_bit_le(const unsigned long *p, unsigned size);
166extern int _find_next_bit_le(const unsigned long *p, int size, int offset);
167
168/*
169 * Big endian assembly bitops. nr = 0 -> byte 3 bit 0.
170 */
Marc Gonzalez2d618fe2017-05-30 16:41:31 +0100171extern int _find_first_zero_bit_be(const unsigned long *p, unsigned size);
172extern int _find_next_zero_bit_be(const unsigned long *p, int size, int offset);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700173extern int _find_first_bit_be(const unsigned long *p, unsigned size);
174extern int _find_next_bit_be(const unsigned long *p, int size, int offset);
175
Russell Kinge7ec0292005-07-28 20:36:26 +0100176#ifndef CONFIG_SMP
Linus Torvalds1da177e2005-04-16 15:20:36 -0700177/*
178 * The __* form of bitops are non-atomic and may be reordered.
179 */
Russell King6323f0c2011-01-16 18:02:17 +0000180#define ATOMIC_BITOP(name,nr,p) \
181 (__builtin_constant_p(nr) ? ____atomic_##name(nr, p) : _##name(nr,p))
Russell Kinge7ec0292005-07-28 20:36:26 +0100182#else
Russell King6323f0c2011-01-16 18:02:17 +0000183#define ATOMIC_BITOP(name,nr,p) _##name(nr,p)
Russell Kinge7ec0292005-07-28 20:36:26 +0100184#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700185
Russell King6323f0c2011-01-16 18:02:17 +0000186/*
187 * Native endian atomic definitions.
188 */
189#define set_bit(nr,p) ATOMIC_BITOP(set_bit,nr,p)
190#define clear_bit(nr,p) ATOMIC_BITOP(clear_bit,nr,p)
191#define change_bit(nr,p) ATOMIC_BITOP(change_bit,nr,p)
192#define test_and_set_bit(nr,p) ATOMIC_BITOP(test_and_set_bit,nr,p)
193#define test_and_clear_bit(nr,p) ATOMIC_BITOP(test_and_clear_bit,nr,p)
194#define test_and_change_bit(nr,p) ATOMIC_BITOP(test_and_change_bit,nr,p)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700195
196#ifndef __ARMEB__
197/*
198 * These are the little endian, atomic definitions.
199 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700200#define find_first_zero_bit(p,sz) _find_first_zero_bit_le(p,sz)
201#define find_next_zero_bit(p,sz,off) _find_next_zero_bit_le(p,sz,off)
202#define find_first_bit(p,sz) _find_first_bit_le(p,sz)
203#define find_next_bit(p,sz,off) _find_next_bit_le(p,sz,off)
204
Linus Torvalds1da177e2005-04-16 15:20:36 -0700205#else
Linus Torvalds1da177e2005-04-16 15:20:36 -0700206/*
207 * These are the big endian, atomic definitions.
208 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700209#define find_first_zero_bit(p,sz) _find_first_zero_bit_be(p,sz)
210#define find_next_zero_bit(p,sz,off) _find_next_zero_bit_be(p,sz,off)
211#define find_first_bit(p,sz) _find_first_bit_be(p,sz)
212#define find_next_bit(p,sz,off) _find_next_bit_be(p,sz,off)
213
Linus Torvalds1da177e2005-04-16 15:20:36 -0700214#endif
215
216#if __LINUX_ARM_ARCH__ < 5
217
Akinobu Mitab89c3b12006-03-26 01:39:19 -0800218#include <asm-generic/bitops/ffz.h>
Nicolas Pitre94fc7332008-12-04 03:59:41 +0100219#include <asm-generic/bitops/__fls.h>
Akinobu Mitab89c3b12006-03-26 01:39:19 -0800220#include <asm-generic/bitops/__ffs.h>
221#include <asm-generic/bitops/fls.h>
222#include <asm-generic/bitops/ffs.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700223
224#else
225
Akinobu Mita9363513362006-03-26 01:38:59 -0800226static inline int constant_fls(int x)
227{
228 int r = 32;
229
230 if (!x)
231 return 0;
232 if (!(x & 0xffff0000u)) {
233 x <<= 16;
234 r -= 16;
235 }
236 if (!(x & 0xff000000u)) {
237 x <<= 8;
238 r -= 8;
239 }
240 if (!(x & 0xf0000000u)) {
241 x <<= 4;
242 r -= 4;
243 }
244 if (!(x & 0xc0000000u)) {
245 x <<= 2;
246 r -= 2;
247 }
248 if (!(x & 0x80000000u)) {
249 x <<= 1;
250 r -= 1;
251 }
252 return r;
253}
254
Linus Torvalds1da177e2005-04-16 15:20:36 -0700255/*
Russell King23f66202014-01-13 23:34:56 +0000256 * On ARMv5 and above those functions can be implemented around the
257 * clz instruction for much better code efficiency. __clz returns
258 * the number of leading zeros, zero input will return 32, and
259 * 0x80000000 will return 0.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700260 */
Russell King23f66202014-01-13 23:34:56 +0000261static inline unsigned int __clz(unsigned int x)
Andrew Morton0c65f452008-07-23 15:35:22 -0700262{
Russell King23f66202014-01-13 23:34:56 +0000263 unsigned int ret;
Nicolas Pitre94fc7332008-12-04 03:59:41 +0100264
Rabin Vincente163d522011-01-12 14:38:52 +0100265 asm("clz\t%0, %1" : "=r" (ret) : "r" (x));
Russell King23f66202014-01-13 23:34:56 +0000266
Nicolas Pitre94fc7332008-12-04 03:59:41 +0100267 return ret;
Andrew Morton0c65f452008-07-23 15:35:22 -0700268}
269
Russell King23f66202014-01-13 23:34:56 +0000270/*
271 * fls() returns zero if the input is zero, otherwise returns the bit
272 * position of the last set bit, where the LSB is 1 and MSB is 32.
273 */
274static inline int fls(int x)
275{
276 if (__builtin_constant_p(x))
277 return constant_fls(x);
278
279 return 32 - __clz(x);
280}
281
282/*
283 * __fls() returns the bit position of the last bit set, where the
284 * LSB is 0 and MSB is 31. Zero input is undefined.
285 */
286static inline unsigned long __fls(unsigned long x)
287{
288 return fls(x) - 1;
289}
290
291/*
292 * ffs() returns zero if the input was zero, otherwise returns the bit
293 * position of the first set bit, where the LSB is 1 and MSB is 32.
294 */
295static inline int ffs(int x)
296{
297 return fls(x & -x);
298}
299
300/*
301 * __ffs() returns the bit position of the first bit set, where the
302 * LSB is 0 and MSB is 31. Zero input is undefined.
303 */
304static inline unsigned long __ffs(unsigned long x)
305{
306 return ffs(x) - 1;
307}
308
Linus Torvalds1da177e2005-04-16 15:20:36 -0700309#define ffz(x) __ffs( ~(x) )
310
311#endif
312
Akinobu Mitab89c3b12006-03-26 01:39:19 -0800313#include <asm-generic/bitops/fls64.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700314
Akinobu Mitab89c3b12006-03-26 01:39:19 -0800315#include <asm-generic/bitops/sched.h>
316#include <asm-generic/bitops/hweight.h>
Nick Piggin26333572007-10-18 03:06:39 -0700317#include <asm-generic/bitops/lock.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700318
Akinobu Mita04b18ff2011-05-26 16:26:11 -0700319#ifdef __ARMEB__
Akinobu Mitaf6b57e32011-03-23 16:41:57 -0700320
321static inline int find_first_zero_bit_le(const void *p, unsigned size)
322{
323 return _find_first_zero_bit_le(p, size);
324}
Akinobu Mitaa2812e12011-05-26 16:26:06 -0700325#define find_first_zero_bit_le find_first_zero_bit_le
Akinobu Mitaf6b57e32011-03-23 16:41:57 -0700326
327static inline int find_next_zero_bit_le(const void *p, int size, int offset)
328{
329 return _find_next_zero_bit_le(p, size, offset);
330}
Akinobu Mitaa2812e12011-05-26 16:26:06 -0700331#define find_next_zero_bit_le find_next_zero_bit_le
Akinobu Mitaf6b57e32011-03-23 16:41:57 -0700332
333static inline int find_next_bit_le(const void *p, int size, int offset)
334{
335 return _find_next_bit_le(p, size, offset);
336}
Akinobu Mitaa2812e12011-05-26 16:26:06 -0700337#define find_next_bit_le find_next_bit_le
Akinobu Mitaf6b57e32011-03-23 16:41:57 -0700338
Akinobu Mita04b18ff2011-05-26 16:26:11 -0700339#endif
340
Clement Courbet0ade34c2018-02-06 15:38:34 -0800341#include <asm-generic/bitops/find.h>
Akinobu Mita04b18ff2011-05-26 16:26:11 -0700342#include <asm-generic/bitops/le.h>
343
Linus Torvalds1da177e2005-04-16 15:20:36 -0700344/*
345 * Ext2 is defined to use little-endian byte ordering.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700346 */
Akinobu Mita148817b2011-07-26 16:09:04 -0700347#include <asm-generic/bitops/ext2-atomic-setbit.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700348
Linus Torvalds1da177e2005-04-16 15:20:36 -0700349#endif /* __KERNEL__ */
350
351#endif /* _ARM_BITOPS_H */