refcount_t: Add ACQUIRE ordering on success for dec(sub)_and_test() variants

This adds an smp_acquire__after_ctrl_dep() barrier on successful
decrease of refcounter value from 1 to 0 for refcount_dec(sub)_and_test
variants and therefore gives stronger memory ordering guarantees than
prior versions of these functions.

Co-developed-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Elena Reshetova <elena.reshetova@intel.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Andrea Parri <andrea.parri@amarulasolutions.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Will Deacon <will.deacon@arm.com>
Cc: dvyukov@google.com
Cc: keescook@chromium.org
Cc: stern@rowland.harvard.edu
Link: https://lkml.kernel.org/r/1548847131-27854-2-git-send-email-elena.reshetova@intel.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
This commit is contained in:
Elena Reshetova 2019-01-30 13:18:51 +02:00 committed by Ingo Molnar
parent 412f34a82c
commit 47b8f3ab9c
3 changed files with 52 additions and 12 deletions

View File

@ -54,6 +54,13 @@ must propagate to all other CPUs before the release operation
(A-cumulative property). This is implemented using (A-cumulative property). This is implemented using
:c:func:`smp_store_release`. :c:func:`smp_store_release`.
An ACQUIRE memory ordering guarantees that all post loads and
stores (all po-later instructions) on the same CPU are
completed after the acquire operation. It also guarantees that all
po-later stores on the same CPU must propagate to all other CPUs
after the acquire operation executes. This is implemented using
:c:func:`smp_acquire__after_ctrl_dep`.
A control dependency (on success) for refcounters guarantees that A control dependency (on success) for refcounters guarantees that
if a reference for an object was successfully obtained (reference if a reference for an object was successfully obtained (reference
counter increment or addition happened, function returned true), counter increment or addition happened, function returned true),
@ -119,13 +126,24 @@ Memory ordering guarantees changes:
result of obtaining pointer to the object! result of obtaining pointer to the object!
case 5) - decrement-based RMW ops that return a value case 5) - generic dec/sub decrement-based RMW ops that return a value
----------------------------------------------------- ---------------------------------------------------------------------
Function changes: Function changes:
* :c:func:`atomic_dec_and_test` --> :c:func:`refcount_dec_and_test` * :c:func:`atomic_dec_and_test` --> :c:func:`refcount_dec_and_test`
* :c:func:`atomic_sub_and_test` --> :c:func:`refcount_sub_and_test` * :c:func:`atomic_sub_and_test` --> :c:func:`refcount_sub_and_test`
Memory ordering guarantees changes:
* fully ordered --> RELEASE ordering + ACQUIRE ordering on success
case 6) other decrement-based RMW ops that return a value
---------------------------------------------------------
Function changes:
* no atomic counterpart --> :c:func:`refcount_dec_if_one` * no atomic counterpart --> :c:func:`refcount_dec_if_one`
* ``atomic_add_unless(&var, -1, 1)`` --> ``refcount_dec_not_one(&var)`` * ``atomic_add_unless(&var, -1, 1)`` --> ``refcount_dec_not_one(&var)``
@ -136,7 +154,7 @@ Memory ordering guarantees changes:
.. note:: :c:func:`atomic_add_unless` only provides full order on success. .. note:: :c:func:`atomic_add_unless` only provides full order on success.
case 6) - lock-based RMW case 7) - lock-based RMW
------------------------ ------------------------
Function changes: Function changes:

View File

@ -67,16 +67,30 @@ static __always_inline void refcount_dec(refcount_t *r)
static __always_inline __must_check static __always_inline __must_check
bool refcount_sub_and_test(unsigned int i, refcount_t *r) bool refcount_sub_and_test(unsigned int i, refcount_t *r)
{ {
return GEN_BINARY_SUFFIXED_RMWcc(LOCK_PREFIX "subl", bool ret = GEN_BINARY_SUFFIXED_RMWcc(LOCK_PREFIX "subl",
REFCOUNT_CHECK_LT_ZERO, REFCOUNT_CHECK_LT_ZERO,
r->refs.counter, e, "er", i, "cx"); r->refs.counter, e, "er", i, "cx");
if (ret) {
smp_acquire__after_ctrl_dep();
return true;
}
return false;
} }
static __always_inline __must_check bool refcount_dec_and_test(refcount_t *r) static __always_inline __must_check bool refcount_dec_and_test(refcount_t *r)
{ {
return GEN_UNARY_SUFFIXED_RMWcc(LOCK_PREFIX "decl", bool ret = GEN_UNARY_SUFFIXED_RMWcc(LOCK_PREFIX "decl",
REFCOUNT_CHECK_LT_ZERO, REFCOUNT_CHECK_LT_ZERO,
r->refs.counter, e, "cx"); r->refs.counter, e, "cx");
if (ret) {
smp_acquire__after_ctrl_dep();
return true;
}
return false;
} }
static __always_inline __must_check static __always_inline __must_check

View File

@ -33,6 +33,9 @@
* Note that the allocator is responsible for ordering things between free() * Note that the allocator is responsible for ordering things between free()
* and alloc(). * and alloc().
* *
* The decrements dec_and_test() and sub_and_test() also provide acquire
* ordering on success.
*
*/ */
#include <linux/mutex.h> #include <linux/mutex.h>
@ -164,8 +167,8 @@ EXPORT_SYMBOL(refcount_inc_checked);
* at UINT_MAX. * at UINT_MAX.
* *
* Provides release memory ordering, such that prior loads and stores are done * Provides release memory ordering, such that prior loads and stores are done
* before, and provides a control dependency such that free() must come after. * before, and provides an acquire ordering on success such that free()
* See the comment on top. * must come after.
* *
* Use of this function is not recommended for the normal reference counting * Use of this function is not recommended for the normal reference counting
* use case in which references are taken and released one at a time. In these * use case in which references are taken and released one at a time. In these
@ -190,7 +193,12 @@ bool refcount_sub_and_test_checked(unsigned int i, refcount_t *r)
} while (!atomic_try_cmpxchg_release(&r->refs, &val, new)); } while (!atomic_try_cmpxchg_release(&r->refs, &val, new));
return !new; if (!new) {
smp_acquire__after_ctrl_dep();
return true;
}
return false;
} }
EXPORT_SYMBOL(refcount_sub_and_test_checked); EXPORT_SYMBOL(refcount_sub_and_test_checked);
@ -202,8 +210,8 @@ EXPORT_SYMBOL(refcount_sub_and_test_checked);
* decrement when saturated at UINT_MAX. * decrement when saturated at UINT_MAX.
* *
* Provides release memory ordering, such that prior loads and stores are done * Provides release memory ordering, such that prior loads and stores are done
* before, and provides a control dependency such that free() must come after. * before, and provides an acquire ordering on success such that free()
* See the comment on top. * must come after.
* *
* Return: true if the resulting refcount is 0, false otherwise * Return: true if the resulting refcount is 0, false otherwise
*/ */