[v10,17/30] linux-gen: atomic: simplify locked 64-bit support

Message ID	20171027125014.27313-18-dmitry.ereminsolenikov@linaro.org
State	New
Headers	show Delivered-To: patch@linaro.org Received-SPF: pass (google.com: domain of lng-odp-bounces@lists.linaro.org designates 54.197.127.237 as permitted sender) client-ip=54.197.127.237; From: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov@linaro.org> To: lng-odp@lists.linaro.org Date: Fri, 27 Oct 2017 15:50:01 +0300 Message-Id: <20171027125014.27313-18-dmitry.ereminsolenikov@linaro.org> In-Reply-To: <20171027125014.27313-1-dmitry.ereminsolenikov@linaro.org> References: <20171027125014.27313-1-dmitry.ereminsolenikov@linaro.org> Subject: [lng-odp] [PATCH v10 17/30] linux-gen: atomic: simplify locked 64-bit support Precedence: list Errors-To: lng-odp-bounces@lists.linaro.org Sender: "lng-odp" <lng-odp-bounces@lists.linaro.org>
Series	restructure headers for ABI-compat/platform-optimized modes \| expand [API-NEXT,v10,00/30] restructure headers for ABI-compat/platform-optimized modes [v10,01/30] travis: also run make distcheck in non-ABI-compat mode [v10,02/30] travis: add cross-compiling tests with ABI compatibility disabled [v10,03/30] linux-gen: ipsec: don't use __odp_force [v10,04/30] helper: don't use __odp_force [v10,05/30] api: abi: provide the the rest of abi files [v10,06/30] configure: provide conditional for ABI-compat mode [v10,07/30] build: move ODP include path to common Makefile.inc [v10,08/30] include: move default ABI headers [v10,09/30] include: install ABI headers without additional ARCH_ABI level [v10,10/30] include: install and use ABI headers only in ABI-compat mode [v10,11/30] abi: queue: drop unused odp_queue_group_t [v10,12/30] abi: classification: drop two unused types [v10,13/30] include: change spec guarding define from ODP_API to ODP_API_SPEC [v10,14/30] api: schedule: remove duplication between schedule and schedule_types [v10,16/30] linux-gen, include: progress in switching headers to api+abi pattern [v10,17/30] linux-gen: atomic: simplify locked 64-bit support [v10,18/30] linux-gen, include: switch atomic.h to api+abi [v10,19/30] linux-gen, include: switch byteorder.h to api+abi [v10,20/30] linux-gen, include: switch std_clib.h to api+abi [v10,21/30] linux-gen, include: switch sync.h to api+abi [v10,22/30] linux-gen, include: switch ticketlock.h to api+abi [v10,23/30] linux-gen, include: move more headers from platform to generic [v10,24/30] linux-gen, include: switch packet headers to api+abi [v10,25/30] linux-gen: move several files under ODP_ABI_COMPAT condition [v10,26/30] linux-gen: split odp_packet/odp_packet_flags [v10,27/30] linux-gen: remove static_inline.h header [v10,28/30] linux-gen: don't install inline headers in non-ABI-compat mode [v10,29/30] configure: stop AC_SUBST'ing ODP_ABI_COMPAT [v10,30/30] include: provide formal description of ODP specification

diff --git a/platform/linux-generic/include/odp/api/plat/atomic_inlines.h b/platform/linux-generic/include/odp/api/plat/atomic_inlines.h index 03b2884fdfca..1c58c77db993 100644 --- a/platform/linux-generic/include/odp/api/plat/atomic_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/atomic_inlines.h @@ -109,177 +109,254 @@ _ODP_INLINE void odp_atomic_min_u32(odp_atomic_u32_t *atom, uint32_t new_min) } } +#ifdef ODP_ATOMIC_U64_LOCK + +/** + * @internal + * CAS operation expression for the ATOMIC_OP macro + */ +#define ATOMIC_CAS_OP(ret_ptr, old_val, new_val) \ +({ \ + if (atom->v == (old_val)) { \ + atom->v = (new_val); \ + *(ret_ptr) = 1; \ + } else { \ + *(ret_ptr) = 0; \ + } \ +}) + +/** + * @internal + * Helper macro for lock-based atomic operations on 64-bit integers + * @param[in,out] atom Pointer to the 64-bit atomic variable + * @param expr Expression used update the variable. + * @return The old value of the variable. + */ +#define ATOMIC_OP(atom, expr) \ +({ \ + uint64_t _old_val; \ + /* Loop while lock is already taken, stop when lock becomes clear */ \ + while (__atomic_test_and_set(&(atom)->lock, __ATOMIC_ACQUIRE)) \ + (void)0; \ + _old_val = (atom)->v; \ + (expr); /* Perform whatever update is desired */ \ + __atomic_clear(&(atom)->lock, __ATOMIC_RELEASE); \ + _old_val; /* Return old value */ \ +}) + _ODP_INLINE void odp_atomic_init_u64(odp_atomic_u64_t *atom, uint64_t val) { atom->v = val; -#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 __atomic_clear(&atom->lock, __ATOMIC_RELAXED); -#endif } _ODP_INLINE uint64_t odp_atomic_load_u64(odp_atomic_u64_t *atom) { -#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 return ATOMIC_OP(atom, (void)0); -#else - return __atomic_load_n(&atom->v, __ATOMIC_RELAXED); -#endif } _ODP_INLINE void odp_atomic_store_u64(odp_atomic_u64_t *atom, uint64_t val) { -#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 (void)ATOMIC_OP(atom, atom->v = val); -#else - __atomic_store_n(&atom->v, val, __ATOMIC_RELAXED); -#endif } _ODP_INLINE uint64_t odp_atomic_fetch_add_u64(odp_atomic_u64_t *atom, uint64_t val) { -#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 return ATOMIC_OP(atom, atom->v += val); -#else - return __atomic_fetch_add(&atom->v, val, __ATOMIC_RELAXED); -#endif } _ODP_INLINE void odp_atomic_add_u64(odp_atomic_u64_t *atom, uint64_t val) { -#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 (void)ATOMIC_OP(atom, atom->v += val); -#else - (void)__atomic_fetch_add(&atom->v, val, __ATOMIC_RELAXED); -#endif } _ODP_INLINE uint64_t odp_atomic_fetch_sub_u64(odp_atomic_u64_t *atom, uint64_t val) { -#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 return ATOMIC_OP(atom, atom->v -= val); -#else - return __atomic_fetch_sub(&atom->v, val, __ATOMIC_RELAXED); -#endif } _ODP_INLINE void odp_atomic_sub_u64(odp_atomic_u64_t *atom, uint64_t val) { -#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 (void)ATOMIC_OP(atom, atom->v -= val); -#else - (void)__atomic_fetch_sub(&atom->v, val, __ATOMIC_RELAXED); -#endif } _ODP_INLINE uint64_t odp_atomic_fetch_inc_u64(odp_atomic_u64_t *atom) { -#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 return ATOMIC_OP(atom, atom->v++); -#else - return __atomic_fetch_add(&atom->v, 1, __ATOMIC_RELAXED); -#endif } _ODP_INLINE void odp_atomic_inc_u64(odp_atomic_u64_t *atom) { -#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 (void)ATOMIC_OP(atom, atom->v++); -#else - (void)__atomic_fetch_add(&atom->v, 1, __ATOMIC_RELAXED); -#endif } _ODP_INLINE uint64_t odp_atomic_fetch_dec_u64(odp_atomic_u64_t *atom) { -#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 return ATOMIC_OP(atom, atom->v--); -#else - return __atomic_fetch_sub(&atom->v, 1, __ATOMIC_RELAXED); -#endif } _ODP_INLINE void odp_atomic_dec_u64(odp_atomic_u64_t *atom) { -#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 (void)ATOMIC_OP(atom, atom->v--); -#else - (void)__atomic_fetch_sub(&atom->v, 1, __ATOMIC_RELAXED); -#endif } _ODP_INLINE int odp_atomic_cas_u64(odp_atomic_u64_t *atom, uint64_t *old_val, uint64_t new_val) { -#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 int ret; *old_val = ATOMIC_OP(atom, ATOMIC_CAS_OP(&ret, *old_val, new_val)); return ret; -#else - return __atomic_compare_exchange_n(&atom->v, old_val, new_val, - 0 /* strong */, - __ATOMIC_RELAXED, - __ATOMIC_RELAXED); -#endif } _ODP_INLINE uint64_t odp_atomic_xchg_u64(odp_atomic_u64_t *atom, uint64_t new_val) { -#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 return ATOMIC_OP(atom, atom->v = new_val); -#else - return __atomic_exchange_n(&atom->v, new_val, __ATOMIC_RELAXED); -#endif } -_ODP_INLINE void odp_atomic_max_u64(odp_atomic_u64_t *atom, uint64_t new_max) +_ODP_INLINE uint64_t odp_atomic_load_acq_u64(odp_atomic_u64_t *atom) { - uint64_t old_val; + return ATOMIC_OP(atom, (void)0); +} - old_val = odp_atomic_load_u64(atom); +_ODP_INLINE void odp_atomic_store_rel_u64(odp_atomic_u64_t *atom, uint64_t val) +{ + (void)ATOMIC_OP(atom, atom->v = val); +} - while (new_max > old_val) { - if (odp_atomic_cas_u64(atom, &old_val, new_max)) - break; - } +_ODP_INLINE void odp_atomic_add_rel_u64(odp_atomic_u64_t *atom, uint64_t val) +{ + (void)ATOMIC_OP(atom, atom->v += val); } -_ODP_INLINE void odp_atomic_min_u64(odp_atomic_u64_t *atom, uint64_t new_min) +_ODP_INLINE void odp_atomic_sub_rel_u64(odp_atomic_u64_t *atom, uint64_t val) { - uint64_t old_val; + (void)ATOMIC_OP(atom, atom->v -= val); +} - old_val = odp_atomic_load_u64(atom); +_ODP_INLINE int odp_atomic_cas_acq_u64(odp_atomic_u64_t *atom, + uint64_t *old_val, uint64_t new_val) +{ + int ret; + *old_val = ATOMIC_OP(atom, ATOMIC_CAS_OP(&ret, *old_val, new_val)); + return ret; +} - while (new_min < old_val) { - if (odp_atomic_cas_u64(atom, &old_val, new_min)) - break; - } +_ODP_INLINE int odp_atomic_cas_rel_u64(odp_atomic_u64_t *atom, + uint64_t *old_val, uint64_t new_val) +{ + int ret; + *old_val = ATOMIC_OP(atom, ATOMIC_CAS_OP(&ret, *old_val, new_val)); + return ret; } -_ODP_INLINE uint32_t odp_atomic_load_acq_u32(odp_atomic_u32_t *atom) +_ODP_INLINE int odp_atomic_cas_acq_rel_u64(odp_atomic_u64_t *atom, + uint64_t *old_val, + uint64_t new_val) +{ + int ret; + *old_val = ATOMIC_OP(atom, ATOMIC_CAS_OP(&ret, *old_val, new_val)); + return ret; +} + +#else /* !ODP_ATOMIC_U64_LOCK */ + +_ODP_INLINE void odp_atomic_init_u64(odp_atomic_u64_t *atom, uint64_t val) +{ + atom->v = val; +} + +_ODP_INLINE uint64_t odp_atomic_load_u64(odp_atomic_u64_t *atom) +{ + return __atomic_load_n(&atom->v, __ATOMIC_RELAXED); +} + +_ODP_INLINE void odp_atomic_store_u64(odp_atomic_u64_t *atom, uint64_t val) +{ + __atomic_store_n(&atom->v, val, __ATOMIC_RELAXED); +} + +_ODP_INLINE uint64_t odp_atomic_fetch_add_u64(odp_atomic_u64_t *atom, + uint64_t val) +{ + return __atomic_fetch_add(&atom->v, val, __ATOMIC_RELAXED); +} + +_ODP_INLINE void odp_atomic_add_u64(odp_atomic_u64_t *atom, uint64_t val) +{ + (void)__atomic_fetch_add(&atom->v, val, __ATOMIC_RELAXED); +} + +_ODP_INLINE uint64_t odp_atomic_fetch_sub_u64(odp_atomic_u64_t *atom, + uint64_t val) +{ + return __atomic_fetch_sub(&atom->v, val, __ATOMIC_RELAXED); +} + +_ODP_INLINE void odp_atomic_sub_u64(odp_atomic_u64_t *atom, uint64_t val) +{ + (void)__atomic_fetch_sub(&atom->v, val, __ATOMIC_RELAXED); +} + +_ODP_INLINE uint64_t odp_atomic_fetch_inc_u64(odp_atomic_u64_t *atom) +{ + return __atomic_fetch_add(&atom->v, 1, __ATOMIC_RELAXED); +} + +_ODP_INLINE void odp_atomic_inc_u64(odp_atomic_u64_t *atom) +{ + (void)__atomic_fetch_add(&atom->v, 1, __ATOMIC_RELAXED); +} + +_ODP_INLINE uint64_t odp_atomic_fetch_dec_u64(odp_atomic_u64_t *atom) +{ + return __atomic_fetch_sub(&atom->v, 1, __ATOMIC_RELAXED); +} + +_ODP_INLINE void odp_atomic_dec_u64(odp_atomic_u64_t *atom) +{ + (void)__atomic_fetch_sub(&atom->v, 1, __ATOMIC_RELAXED); +} + +_ODP_INLINE int odp_atomic_cas_u64(odp_atomic_u64_t *atom, uint64_t *old_val, + uint64_t new_val) +{ + return __atomic_compare_exchange_n(&atom->v, old_val, new_val, + 0 /* strong */, + __ATOMIC_RELAXED, + __ATOMIC_RELAXED); +} + +_ODP_INLINE uint64_t odp_atomic_xchg_u64(odp_atomic_u64_t *atom, + uint64_t new_val) +{ + return __atomic_exchange_n(&atom->v, new_val, __ATOMIC_RELAXED); +} + +_ODP_INLINE uint64_t odp_atomic_load_acq_u64(odp_atomic_u64_t *atom) { return __atomic_load_n(&atom->v, __ATOMIC_ACQUIRE); } -_ODP_INLINE void odp_atomic_store_rel_u32(odp_atomic_u32_t *atom, uint32_t val) +_ODP_INLINE void odp_atomic_store_rel_u64(odp_atomic_u64_t *atom, uint64_t val) { __atomic_store_n(&atom->v, val, __ATOMIC_RELEASE); } -_ODP_INLINE void odp_atomic_add_rel_u32(odp_atomic_u32_t *atom, uint32_t val) +_ODP_INLINE void odp_atomic_add_rel_u64(odp_atomic_u64_t *atom, uint64_t val) { (void)__atomic_fetch_add(&atom->v, val, __ATOMIC_RELEASE); } -_ODP_INLINE void odp_atomic_sub_rel_u32(odp_atomic_u32_t *atom, uint32_t val) +_ODP_INLINE void odp_atomic_sub_rel_u64(odp_atomic_u64_t *atom, uint64_t val) { (void)__atomic_fetch_sub(&atom->v, val, __ATOMIC_RELEASE); } -_ODP_INLINE int odp_atomic_cas_acq_u32(odp_atomic_u32_t *atom, - uint32_t *old_val, uint32_t new_val) +_ODP_INLINE int odp_atomic_cas_acq_u64(odp_atomic_u64_t *atom, + uint64_t *old_val, uint64_t new_val) { return __atomic_compare_exchange_n(&atom->v, old_val, new_val, 0 /* strong */, @@ -287,8 +364,8 @@ _ODP_INLINE int odp_atomic_cas_acq_u32(odp_atomic_u32_t *atom, __ATOMIC_RELAXED); } -_ODP_INLINE int odp_atomic_cas_rel_u32(odp_atomic_u32_t *atom, - uint32_t *old_val, uint32_t new_val) +_ODP_INLINE int odp_atomic_cas_rel_u64(odp_atomic_u64_t *atom, + uint64_t *old_val, uint64_t new_val) { return __atomic_compare_exchange_n(&atom->v, old_val, new_val, 0 /* strong */, @@ -296,9 +373,9 @@ _ODP_INLINE int odp_atomic_cas_rel_u32(odp_atomic_u32_t *atom, __ATOMIC_RELAXED); } -_ODP_INLINE int odp_atomic_cas_acq_rel_u32(odp_atomic_u32_t *atom, - uint32_t *old_val, - uint32_t new_val) +_ODP_INLINE int odp_atomic_cas_acq_rel_u64(odp_atomic_u64_t *atom, + uint64_t *old_val, + uint64_t new_val) { return __atomic_compare_exchange_n(&atom->v, old_val, new_val, 0 /* strong */, @@ -306,86 +383,78 @@ _ODP_INLINE int odp_atomic_cas_acq_rel_u32(odp_atomic_u32_t *atom, __ATOMIC_RELAXED); } -_ODP_INLINE uint64_t odp_atomic_load_acq_u64(odp_atomic_u64_t *atom) +#endif /* !ODP_ATOMIC_U64_LOCK */ + +_ODP_INLINE void odp_atomic_max_u64(odp_atomic_u64_t *atom, uint64_t new_max) +{ + uint64_t old_val; + + old_val = odp_atomic_load_u64(atom); + + while (new_max > old_val) { + if (odp_atomic_cas_u64(atom, &old_val, new_max)) + break; + } +} + +_ODP_INLINE void odp_atomic_min_u64(odp_atomic_u64_t *atom, uint64_t new_min) +{ + uint64_t old_val; + + old_val = odp_atomic_load_u64(atom); + + while (new_min < old_val) { + if (odp_atomic_cas_u64(atom, &old_val, new_min)) + break; + } +} + +_ODP_INLINE uint32_t odp_atomic_load_acq_u32(odp_atomic_u32_t *atom) { -#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 - return ATOMIC_OP(atom, (void)0); -#else return __atomic_load_n(&atom->v, __ATOMIC_ACQUIRE); -#endif } -_ODP_INLINE void odp_atomic_store_rel_u64(odp_atomic_u64_t *atom, uint64_t val) +_ODP_INLINE void odp_atomic_store_rel_u32(odp_atomic_u32_t *atom, uint32_t val) { -#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 - (void)ATOMIC_OP(atom, atom->v = val); -#else __atomic_store_n(&atom->v, val, __ATOMIC_RELEASE); -#endif } -_ODP_INLINE void odp_atomic_add_rel_u64(odp_atomic_u64_t *atom, uint64_t val) +_ODP_INLINE void odp_atomic_add_rel_u32(odp_atomic_u32_t *atom, uint32_t val) { -#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 - (void)ATOMIC_OP(atom, atom->v += val); -#else (void)__atomic_fetch_add(&atom->v, val, __ATOMIC_RELEASE); -#endif } -_ODP_INLINE void odp_atomic_sub_rel_u64(odp_atomic_u64_t *atom, uint64_t val) +_ODP_INLINE void odp_atomic_sub_rel_u32(odp_atomic_u32_t *atom, uint32_t val) { -#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 - (void)ATOMIC_OP(atom, atom->v -= val); -#else (void)__atomic_fetch_sub(&atom->v, val, __ATOMIC_RELEASE); -#endif } -_ODP_INLINE int odp_atomic_cas_acq_u64(odp_atomic_u64_t *atom, - uint64_t *old_val, uint64_t new_val) +_ODP_INLINE int odp_atomic_cas_acq_u32(odp_atomic_u32_t *atom, + uint32_t *old_val, uint32_t new_val) { -#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 - int ret; - *old_val = ATOMIC_OP(atom, ATOMIC_CAS_OP(&ret, *old_val, new_val)); - return ret; -#else return __atomic_compare_exchange_n(&atom->v, old_val, new_val, 0 /* strong */, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); -#endif } -_ODP_INLINE int odp_atomic_cas_rel_u64(odp_atomic_u64_t *atom, - uint64_t *old_val, uint64_t new_val) +_ODP_INLINE int odp_atomic_cas_rel_u32(odp_atomic_u32_t *atom, + uint32_t *old_val, uint32_t new_val) { -#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 - int ret; - *old_val = ATOMIC_OP(atom, ATOMIC_CAS_OP(&ret, *old_val, new_val)); - return ret; -#else return __atomic_compare_exchange_n(&atom->v, old_val, new_val, 0 /* strong */, __ATOMIC_RELEASE, __ATOMIC_RELAXED); -#endif } -_ODP_INLINE int odp_atomic_cas_acq_rel_u64(odp_atomic_u64_t *atom, - uint64_t *old_val, - uint64_t new_val) +_ODP_INLINE int odp_atomic_cas_acq_rel_u32(odp_atomic_u32_t *atom, + uint32_t *old_val, + uint32_t new_val) { -#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 - int ret; - *old_val = ATOMIC_OP(atom, ATOMIC_CAS_OP(&ret, *old_val, new_val)); - return ret; -#else return __atomic_compare_exchange_n(&atom->v, old_val, new_val, 0 /* strong */, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED); -#endif } #endif diff --git a/platform/linux-generic/include/odp/api/plat/atomic_types.h b/platform/linux-generic/include/odp/api/plat/atomic_types.h index a674ac9978c4..c0803bf11f63 100644 --- a/platform/linux-generic/include/odp/api/plat/atomic_types.h +++ b/platform/linux-generic/include/odp/api/plat/atomic_types.h @@ -20,61 +20,39 @@ extern "C" { #include <odp/api/std_types.h> #include <odp/api/align.h> -/** - * @internal - * Atomic 64-bit unsigned integer - */ -struct odp_atomic_u64_s { - uint64_t v; /**< Actual storage for the atomic variable */ -#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 - /* Some architectures do not support lock-free operations on 64-bit - * data types. We use a spin lock to ensure atomicity. */ - char lock; /**< Spin lock (if needed) used to ensure atomic access */ -#endif -} ODP_ALIGNED(sizeof(uint64_t)); /* Enforce alignement! */ - /** * @internal * Atomic 32-bit unsigned integer */ struct odp_atomic_u32_s { uint32_t v; /**< Actual storage for the atomic variable */ -} ODP_ALIGNED(sizeof(uint32_t)); /* Enforce alignement! */ +} ODP_ALIGNED(sizeof(uint32_t)); /* Enforce alignment! */ -#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 +#if __GCC_ATOMIC_LLONG_LOCK_FREE >= 2 /** * @internal - * CAS operation expression for the ATOMIC_OP macro + * Atomic 64-bit unsigned integer */ -#define ATOMIC_CAS_OP(ret_ptr, old_val, new_val) \ -({ \ - if (atom->v == (old_val)) { \ - atom->v = (new_val); \ - *(ret_ptr) = 1; \ - } else { \ - *(ret_ptr) = 0; \ - } \ -}) +struct odp_atomic_u64_s { + uint64_t v; /**< Actual storage for the atomic variable */ +} ODP_ALIGNED(sizeof(uint64_t)); /* Enforce alignment! */ + +#else + +#define ODP_ATOMIC_U64_LOCK 1 /** * @internal - * Helper macro for lock-based atomic operations on 64-bit integers - * @param[in,out] atom Pointer to the 64-bit atomic variable - * @param expr Expression used update the variable. - * @return The old value of the variable. + * Atomic 64-bit unsigned integer */ -#define ATOMIC_OP(atom, expr) \ -({ \ - uint64_t _old_val; \ - /* Loop while lock is already taken, stop when lock becomes clear */ \ - while (__atomic_test_and_set(&(atom)->lock, __ATOMIC_ACQUIRE)) \ - (void)0; \ - _old_val = (atom)->v; \ - (expr); /* Perform whatever update is desired */ \ - __atomic_clear(&(atom)->lock, __ATOMIC_RELEASE); \ - _old_val; /* Return old value */ \ -}) +struct odp_atomic_u64_s { + uint64_t v; /**< Actual storage for the atomic variable */ + /* Some architectures do not support lock-free operations on 64-bit + * data types. We use a spin lock to ensure atomicity. */ + char lock; /**< Spin lock (if needed) used to ensure atomic access */ +} ODP_ALIGNED(sizeof(uint64_t)); /* Enforce alignment! */ + #endif typedef struct odp_atomic_u64_s odp_atomic_u64_t; diff --git a/platform/linux-generic/include/odp_atomic_internal.h b/platform/linux-generic/include/odp_atomic_internal.h index dca2175cefbf..f3696a99426f 100644 --- a/platform/linux-generic/include/odp_atomic_internal.h +++ b/platform/linux-generic/include/odp_atomic_internal.h @@ -223,7 +223,7 @@ static inline void _odp_atomic_u32_sub_mm(odp_atomic_u32_t *atom, *****************************************************************************/ /* Check if the compiler support lock-less atomic operations on 64-bit types */ -#if !defined __GCC_ATOMIC_LLONG_LOCK_FREE || __GCC_ATOMIC_LLONG_LOCK_FREE < 2 +#ifdef ODP_ATOMIC_U64_LOCK /** * @internal * Helper macro for lock-based atomic operations on 64-bit integers @@ -247,7 +247,6 @@ static inline void _odp_atomic_u32_sub_mm(odp_atomic_u32_t *atom, __ATOMIC_SEQ_CST : __ATOMIC_RELEASE); \ old_val; /* Return old value */ \ }) -#endif /** * Atomic load of 64-bit atomic variable @@ -258,13 +257,9 @@ static inline void _odp_atomic_u32_sub_mm(odp_atomic_u32_t *atom, * @return Value of the variable */ static inline uint64_t _odp_atomic_u64_load_mm(odp_atomic_u64_t *atom, - _odp_memmodel_t mmodel) + _odp_memmodel_t mmodel) { -#if !defined __GCC_ATOMIC_LLONG_LOCK_FREE || __GCC_ATOMIC_LLONG_LOCK_FREE < 2 return ATOMIC_OP_MM(atom, (void)0, mmodel); -#else - return __atomic_load_n(&atom->v, mmodel); -#endif } /** @@ -275,14 +270,10 @@ static inline uint64_t _odp_atomic_u64_load_mm(odp_atomic_u64_t *atom, * @param mmodel Memory order associated with the store operation */ static inline void _odp_atomic_u64_store_mm(odp_atomic_u64_t *atom, - uint64_t val, - _odp_memmodel_t mmodel) + uint64_t val, + _odp_memmodel_t mmodel) { -#if !defined __GCC_ATOMIC_LLONG_LOCK_FREE || __GCC_ATOMIC_LLONG_LOCK_FREE < 2 (void)ATOMIC_OP_MM(atom, atom->v = val, mmodel); -#else - __atomic_store_n(&atom->v, val, mmodel); -#endif } /** @@ -295,15 +286,11 @@ static inline void _odp_atomic_u64_store_mm(odp_atomic_u64_t *atom, * @return Old value of variable */ static inline uint64_t _odp_atomic_u64_xchg_mm(odp_atomic_u64_t *atom, - uint64_t val, - _odp_memmodel_t mmodel) + uint64_t val, + _odp_memmodel_t mmodel) { -#if !defined __GCC_ATOMIC_LLONG_LOCK_FREE || __GCC_ATOMIC_LLONG_LOCK_FREE < 2 return ATOMIC_OP_MM(atom, atom->v = val, mmodel); -#else - return __atomic_exchange_n(&atom->v, val, mmodel); -#endif } /** @@ -322,12 +309,11 @@ static inline uint64_t _odp_atomic_u64_xchg_mm(odp_atomic_u64_t *atom, * @retval 0 exchange failed and '*exp' updated with current value */ static inline int _odp_atomic_u64_cmp_xchg_strong_mm(odp_atomic_u64_t *atom, - uint64_t *exp, - uint64_t val, - _odp_memmodel_t success, - _odp_memmodel_t failure) + uint64_t *exp, + uint64_t val, + _odp_memmodel_t success, + _odp_memmodel_t failure) { -#if !defined __GCC_ATOMIC_LLONG_LOCK_FREE || __GCC_ATOMIC_LLONG_LOCK_FREE < 2 /* Possibly we are a bit pessimistic with the memory models */ odp_bool_t ret_succ; /* Loop while lock is already taken, stop when lock becomes clear */ @@ -346,10 +332,6 @@ static inline int _odp_atomic_u64_cmp_xchg_strong_mm(odp_atomic_u64_t *atom, (ret_succ ? success : failure) == _ODP_MEMMODEL_SC ? __ATOMIC_SEQ_CST : __ATOMIC_RELEASE); return ret_succ; -#else - return __atomic_compare_exchange_n(&atom->v, exp, val, - false/*strong*/, success, failure); -#endif } /** @@ -362,14 +344,10 @@ static inline int _odp_atomic_u64_cmp_xchg_strong_mm(odp_atomic_u64_t *atom, * @return Value of the atomic variable before the addition */ static inline uint64_t _odp_atomic_u64_fetch_add_mm(odp_atomic_u64_t *atom, - uint64_t val, - _odp_memmodel_t mmodel) + uint64_t val, + _odp_memmodel_t mmodel) { -#if !defined __GCC_ATOMIC_LLONG_LOCK_FREE || __GCC_ATOMIC_LLONG_LOCK_FREE < 2 return ATOMIC_OP_MM(atom, atom->v += val, mmodel); -#else - return __atomic_fetch_add(&atom->v, val, mmodel); -#endif } /** @@ -380,15 +358,11 @@ static inline uint64_t _odp_atomic_u64_fetch_add_mm(odp_atomic_u64_t *atom, * @param mmodel Memory order associated with the add operation. */ static inline void _odp_atomic_u64_add_mm(odp_atomic_u64_t *atom, - uint64_t val, - _odp_memmodel_t mmodel) + uint64_t val, + _odp_memmodel_t mmodel) { -#if !defined __GCC_ATOMIC_LLONG_LOCK_FREE || __GCC_ATOMIC_LLONG_LOCK_FREE < 2 (void)ATOMIC_OP_MM(atom, atom->v += val, mmodel); -#else - (void)__atomic_fetch_add(&atom->v, val, mmodel); -#endif } /** @@ -401,14 +375,10 @@ static inline void _odp_atomic_u64_add_mm(odp_atomic_u64_t *atom, * @return Value of the atomic variable before the subtraction */ static inline uint64_t _odp_atomic_u64_fetch_sub_mm(odp_atomic_u64_t *atom, - uint64_t val, - _odp_memmodel_t mmodel) + uint64_t val, + _odp_memmodel_t mmodel) { -#if !defined __GCC_ATOMIC_LLONG_LOCK_FREE || __GCC_ATOMIC_LLONG_LOCK_FREE < 2 return ATOMIC_OP_MM(atom, atom->v -= val, mmodel); -#else - return __atomic_fetch_sub(&atom->v, val, mmodel); -#endif } /** @@ -419,20 +389,150 @@ static inline uint64_t _odp_atomic_u64_fetch_sub_mm(odp_atomic_u64_t *atom, * @param mmodel Memory order associated with the subtract operation */ static inline void _odp_atomic_u64_sub_mm(odp_atomic_u64_t *atom, - uint64_t val, - _odp_memmodel_t mmodel) + uint64_t val, + _odp_memmodel_t mmodel) { -#if !defined __GCC_ATOMIC_LLONG_LOCK_FREE || __GCC_ATOMIC_LLONG_LOCK_FREE < 2 (void)ATOMIC_OP_MM(atom, atom->v -= val, mmodel); -#else - (void)__atomic_fetch_sub(&atom->v, val, mmodel); -#endif } -#if !defined __GCC_ATOMIC_LLONG_LOCK_FREE || __GCC_ATOMIC_LLONG_LOCK_FREE < 2 #undef ATOMIC_OP_MM -#endif + +#else /* ! ODP_ATOMIC_U64_LOCK */ + +/** + * Atomic load of 64-bit atomic variable + * + * @param atom Pointer to a 64-bit atomic variable + * @param mmodel Memory order associated with the load operation + * + * @return Value of the variable + */ +static inline uint64_t _odp_atomic_u64_load_mm(odp_atomic_u64_t *atom, + _odp_memmodel_t mmodel) +{ + return __atomic_load_n(&atom->v, mmodel); +} + +/** + * Atomic store to 64-bit atomic variable + * + * @param[out] atom Pointer to a 64-bit atomic variable + * @param val Value to write to the atomic variable + * @param mmodel Memory order associated with the store operation + */ +static inline void _odp_atomic_u64_store_mm(odp_atomic_u64_t *atom, + uint64_t val, + _odp_memmodel_t mmodel) +{ + __atomic_store_n(&atom->v, val, mmodel); +} + +/** + * Atomic exchange (swap) of 64-bit atomic variable + * + * @param[in,out] atom Pointer to a 64-bit atomic variable + * @param val New value to write to the atomic variable + * @param mmodel Memory order associated with the exchange operation + * + * @return Old value of variable + */ +static inline uint64_t _odp_atomic_u64_xchg_mm(odp_atomic_u64_t *atom, + uint64_t val, + _odp_memmodel_t mmodel) + +{ + return __atomic_exchange_n(&atom->v, val, mmodel); +} + +/** + * Atomic compare and exchange (swap) of 64-bit atomic variable + * "Strong" semantics, will not fail spuriously. + * + * @param[in,out] atom Pointer to a 64-bit atomic variable + * @param[in,out] exp Pointer to expected value (updated on failure) + * @param val New value to write + * @param success Memory order associated with a successful compare-and-swap + * operation + * @param failure Memory order associated with a failed compare-and-swap + * operation + * + * @retval 1 exchange successful + * @retval 0 exchange failed and '*exp' updated with current value + */ +static inline int _odp_atomic_u64_cmp_xchg_strong_mm(odp_atomic_u64_t *atom, + uint64_t *exp, + uint64_t val, + _odp_memmodel_t success, + _odp_memmodel_t failure) +{ + return __atomic_compare_exchange_n(&atom->v, exp, val, + false/*strong*/, success, failure); +} + +/** + * Atomic fetch and add of 64-bit atomic variable + * + * @param[in,out] atom Pointer to a 64-bit atomic variable + * @param val Value to add to the atomic variable + * @param mmodel Memory order associated with the add operation + * + * @return Value of the atomic variable before the addition + */ +static inline uint64_t _odp_atomic_u64_fetch_add_mm(odp_atomic_u64_t *atom, + uint64_t val, + _odp_memmodel_t mmodel) +{ + return __atomic_fetch_add(&atom->v, val, mmodel); +} + +/** + * Atomic add of 64-bit atomic variable + * + * @param[in,out] atom Pointer to a 64-bit atomic variable + * @param val Value to add to the atomic variable + * @param mmodel Memory order associated with the add operation. + */ +static inline void _odp_atomic_u64_add_mm(odp_atomic_u64_t *atom, + uint64_t val, + _odp_memmodel_t mmodel) + +{ + (void)__atomic_fetch_add(&atom->v, val, mmodel); +} + +/** + * Atomic fetch and subtract of 64-bit atomic variable + * + * @param[in,out] atom Pointer to a 64-bit atomic variable + * @param val Value to subtract from the atomic variable + * @param mmodel Memory order associated with the subtract operation + * + * @return Value of the atomic variable before the subtraction + */ +static inline uint64_t _odp_atomic_u64_fetch_sub_mm(odp_atomic_u64_t *atom, + uint64_t val, + _odp_memmodel_t mmodel) +{ + return __atomic_fetch_sub(&atom->v, val, mmodel); +} + +/** + * Atomic subtract of 64-bit atomic variable + * + * @param[in,out] atom Pointer to a 64-bit atomic variable + * @param val Value to subtract from the atomic variable + * @param mmodel Memory order associated with the subtract operation + */ +static inline void _odp_atomic_u64_sub_mm(odp_atomic_u64_t *atom, + uint64_t val, + _odp_memmodel_t mmodel) + +{ + (void)__atomic_fetch_sub(&atom->v, val, mmodel); +} + +#endif /* ! ODP_ATOMIC_U64_LOCK */ /***************************************************************************** * Operations on pointer atomics

[v10,17/30] linux-gen: atomic: simplify locked 64-bit support

Commit Message

Patch