atomic64.c 4.17 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
/*
 * Generic implementation of 64-bit atomics using spinlocks,
 * useful on processors that don't have 64-bit atomic instructions.
 *
 * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version
 * 2 of the License, or (at your option) any later version.
 */
#include <linux/types.h>
#include <linux/cache.h>
#include <linux/spinlock.h>
#include <linux/init.h>
16
#include <linux/export.h>
17
#include <linux/atomic.h>
18 19 20 21 22 23 24 25 26 27 28 29 30 31

/*
 * We use a hashed array of spinlocks to provide exclusive access
 * to each atomic64_t variable.  Since this is expected to used on
 * systems with small numbers of CPUs (<= 4 or so), we use a
 * relatively small array of 16 spinlocks to avoid wasting too much
 * memory on the spinlock array.
 */
#define NR_LOCKS	16

/*
 * Ensure each lock is in a separate cacheline.
 */
static union {
32
	raw_spinlock_t lock;
33
	char pad[L1_CACHE_BYTES];
34 35 36 37 38
} atomic64_lock[NR_LOCKS] __cacheline_aligned_in_smp = {
	[0 ... (NR_LOCKS - 1)] = {
		.lock =  __RAW_SPIN_LOCK_UNLOCKED(atomic64_lock.lock),
	},
};
39

40
static inline raw_spinlock_t *lock_addr(const atomic64_t *v)
41 42 43 44 45 46 47 48 49 50 51
{
	unsigned long addr = (unsigned long) v;

	addr >>= L1_CACHE_SHIFT;
	addr ^= (addr >> 8) ^ (addr >> 16);
	return &atomic64_lock[addr & (NR_LOCKS - 1)].lock;
}

long long atomic64_read(const atomic64_t *v)
{
	unsigned long flags;
52
	raw_spinlock_t *lock = lock_addr(v);
53 54
	long long val;

55
	raw_spin_lock_irqsave(lock, flags);
56
	val = v->counter;
57
	raw_spin_unlock_irqrestore(lock, flags);
58 59
	return val;
}
60
EXPORT_SYMBOL(atomic64_read);
61 62 63 64

void atomic64_set(atomic64_t *v, long long i)
{
	unsigned long flags;
65
	raw_spinlock_t *lock = lock_addr(v);
66

67
	raw_spin_lock_irqsave(lock, flags);
68
	v->counter = i;
69
	raw_spin_unlock_irqrestore(lock, flags);
70
}
71
EXPORT_SYMBOL(atomic64_set);
72 73 74 75

void atomic64_add(long long a, atomic64_t *v)
{
	unsigned long flags;
76
	raw_spinlock_t *lock = lock_addr(v);
77

78
	raw_spin_lock_irqsave(lock, flags);
79
	v->counter += a;
80
	raw_spin_unlock_irqrestore(lock, flags);
81
}
82
EXPORT_SYMBOL(atomic64_add);
83 84 85 86

long long atomic64_add_return(long long a, atomic64_t *v)
{
	unsigned long flags;
87
	raw_spinlock_t *lock = lock_addr(v);
88 89
	long long val;

90
	raw_spin_lock_irqsave(lock, flags);
91
	val = v->counter += a;
92
	raw_spin_unlock_irqrestore(lock, flags);
93 94
	return val;
}
95
EXPORT_SYMBOL(atomic64_add_return);
96 97 98 99

void atomic64_sub(long long a, atomic64_t *v)
{
	unsigned long flags;
100
	raw_spinlock_t *lock = lock_addr(v);
101

102
	raw_spin_lock_irqsave(lock, flags);
103
	v->counter -= a;
104
	raw_spin_unlock_irqrestore(lock, flags);
105
}
106
EXPORT_SYMBOL(atomic64_sub);
107 108 109 110

long long atomic64_sub_return(long long a, atomic64_t *v)
{
	unsigned long flags;
111
	raw_spinlock_t *lock = lock_addr(v);
112 113
	long long val;

114
	raw_spin_lock_irqsave(lock, flags);
115
	val = v->counter -= a;
116
	raw_spin_unlock_irqrestore(lock, flags);
117 118
	return val;
}
119
EXPORT_SYMBOL(atomic64_sub_return);
120 121 122 123

long long atomic64_dec_if_positive(atomic64_t *v)
{
	unsigned long flags;
124
	raw_spinlock_t *lock = lock_addr(v);
125 126
	long long val;

127
	raw_spin_lock_irqsave(lock, flags);
128 129 130
	val = v->counter - 1;
	if (val >= 0)
		v->counter = val;
131
	raw_spin_unlock_irqrestore(lock, flags);
132 133
	return val;
}
134
EXPORT_SYMBOL(atomic64_dec_if_positive);
135 136 137 138

long long atomic64_cmpxchg(atomic64_t *v, long long o, long long n)
{
	unsigned long flags;
139
	raw_spinlock_t *lock = lock_addr(v);
140 141
	long long val;

142
	raw_spin_lock_irqsave(lock, flags);
143 144 145
	val = v->counter;
	if (val == o)
		v->counter = n;
146
	raw_spin_unlock_irqrestore(lock, flags);
147 148
	return val;
}
149
EXPORT_SYMBOL(atomic64_cmpxchg);
150 151 152 153

long long atomic64_xchg(atomic64_t *v, long long new)
{
	unsigned long flags;
154
	raw_spinlock_t *lock = lock_addr(v);
155 156
	long long val;

157
	raw_spin_lock_irqsave(lock, flags);
158 159
	val = v->counter;
	v->counter = new;
160
	raw_spin_unlock_irqrestore(lock, flags);
161 162
	return val;
}
163
EXPORT_SYMBOL(atomic64_xchg);
164 165 166 167

int atomic64_add_unless(atomic64_t *v, long long a, long long u)
{
	unsigned long flags;
168
	raw_spinlock_t *lock = lock_addr(v);
169
	int ret = 0;
170

171
	raw_spin_lock_irqsave(lock, flags);
172 173
	if (v->counter != u) {
		v->counter += a;
174
		ret = 1;
175
	}
176
	raw_spin_unlock_irqrestore(lock, flags);
177 178
	return ret;
}
179
EXPORT_SYMBOL(atomic64_add_unless);