lockref.c 3.9 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0
2 3 4
#include <linux/export.h>
#include <linux/lockref.h>

5
#if USE_CMPXCHG_LOCKREF
6 7 8 9 10 11 12 13

/*
 * Note that the "cmpxchg()" reloads the "old" value for the
 * failure case.
 */
#define CMPXCHG_LOOP(CODE, SUCCESS) do {					\
	struct lockref old;							\
	BUILD_BUG_ON(sizeof(old) != 8);						\
14
	old.lock_count = READ_ONCE(lockref->lock_count);			\
15 16 17
	while (likely(arch_spin_value_unlocked(old.lock.rlock.raw_lock))) {  	\
		struct lockref new = old, prev = old;				\
		CODE								\
18 19 20
		old.lock_count = cmpxchg64_relaxed(&lockref->lock_count,	\
						   old.lock_count,		\
						   new.lock_count);		\
21 22 23
		if (likely(old.lock_count == prev.lock_count)) {		\
			SUCCESS;						\
		}								\
24
		cpu_relax();							\
25 26 27 28 29 30 31 32 33
	}									\
} while (0)

#else

#define CMPXCHG_LOOP(CODE, SUCCESS) do { } while (0)

#endif

34 35
/**
 * lockref_get - Increments reference count unconditionally
36
 * @lockref: pointer to lockref structure
37 38 39 40 41 42
 *
 * This operation is only valid if you already hold a reference
 * to the object, so you know the count cannot be zero.
 */
void lockref_get(struct lockref *lockref)
{
43 44 45 46 47 48
	CMPXCHG_LOOP(
		new.count++;
	,
		return;
	);

49 50 51 52 53 54 55
	spin_lock(&lockref->lock);
	lockref->count++;
	spin_unlock(&lockref->lock);
}
EXPORT_SYMBOL(lockref_get);

/**
56
 * lockref_get_not_zero - Increments count unless the count is 0 or dead
57
 * @lockref: pointer to lockref structure
58 59 60 61
 * Return: 1 if count updated successfully or 0 if count was zero
 */
int lockref_get_not_zero(struct lockref *lockref)
{
62 63 64 65
	int retval;

	CMPXCHG_LOOP(
		new.count++;
66
		if (old.count <= 0)
67 68 69 70
			return 0;
	,
		return 1;
	);
71 72

	spin_lock(&lockref->lock);
73
	retval = 0;
74
	if (lockref->count > 0) {
75 76 77 78 79 80 81 82 83
		lockref->count++;
		retval = 1;
	}
	spin_unlock(&lockref->lock);
	return retval;
}
EXPORT_SYMBOL(lockref_get_not_zero);

/**
84
 * lockref_get_or_lock - Increments count unless the count is 0 or dead
85
 * @lockref: pointer to lockref structure
86 87 88 89 90
 * Return: 1 if count updated successfully or 0 if count was zero
 * and we got the lock instead.
 */
int lockref_get_or_lock(struct lockref *lockref)
{
91 92
	CMPXCHG_LOOP(
		new.count++;
93
		if (old.count <= 0)
94 95 96 97 98
			break;
	,
		return 1;
	);

99
	spin_lock(&lockref->lock);
100
	if (lockref->count <= 0)
101 102 103 104 105 106 107
		return 0;
	lockref->count++;
	spin_unlock(&lockref->lock);
	return 1;
}
EXPORT_SYMBOL(lockref_get_or_lock);

108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127
/**
 * lockref_put_return - Decrement reference count if possible
 * @lockref: pointer to lockref structure
 *
 * Decrement the reference count and return the new value.
 * If the lockref was dead or locked, return an error.
 */
int lockref_put_return(struct lockref *lockref)
{
	CMPXCHG_LOOP(
		new.count--;
		if (old.count <= 0)
			return -1;
	,
		return new.count;
	);
	return -1;
}
EXPORT_SYMBOL(lockref_put_return);

128 129
/**
 * lockref_put_or_lock - decrements count unless count <= 1 before decrement
130
 * @lockref: pointer to lockref structure
131 132 133 134
 * Return: 1 if count updated successfully or 0 if count <= 1 and lock taken
 */
int lockref_put_or_lock(struct lockref *lockref)
{
135 136 137 138 139 140 141 142
	CMPXCHG_LOOP(
		new.count--;
		if (old.count <= 1)
			break;
	,
		return 1;
	);

143 144 145 146 147 148 149 150
	spin_lock(&lockref->lock);
	if (lockref->count <= 1)
		return 0;
	lockref->count--;
	spin_unlock(&lockref->lock);
	return 1;
}
EXPORT_SYMBOL(lockref_put_or_lock);
151 152 153 154 155 156 157 158 159 160

/**
 * lockref_mark_dead - mark lockref dead
 * @lockref: pointer to lockref structure
 */
void lockref_mark_dead(struct lockref *lockref)
{
	assert_spin_locked(&lockref->lock);
	lockref->count = -128;
}
161
EXPORT_SYMBOL(lockref_mark_dead);
162 163 164 165 166 167 168 169 170 171 172 173

/**
 * lockref_get_not_dead - Increments count unless the ref is dead
 * @lockref: pointer to lockref structure
 * Return: 1 if count updated successfully or 0 if lockref was dead
 */
int lockref_get_not_dead(struct lockref *lockref)
{
	int retval;

	CMPXCHG_LOOP(
		new.count++;
174
		if (old.count < 0)
175 176 177 178 179 180 181
			return 0;
	,
		return 1;
	);

	spin_lock(&lockref->lock);
	retval = 0;
182
	if (lockref->count >= 0) {
183 184 185 186 187 188 189
		lockref->count++;
		retval = 1;
	}
	spin_unlock(&lockref->lock);
	return retval;
}
EXPORT_SYMBOL(lockref_get_not_dead);