Skip to content

Commit f33bbfd

Browse files
hjelmnbosilca
hjelmn
authored and
bosilca
committed
atomic: add support for __atomic builtins (#1735)
* atomic: add support for __atomic builtins This commit adds support for the gcc __atomic builtins. The __sync builtins are deprecated and have been replaced by these atomics. In addition, the new atomics support atomic exchange which was not supported by __sync. Signed-off-by: Nathan Hjelm <[email protected]> * atomic: add support for transactional memory This commit adds support for using transactional memory when using opal atomic locks. This feature is enabled if the __HLE__ feature is available and the gcc builtin atomics are in use. Signed-off-by: Nathan Hjelm <[email protected]>
1 parent b85a5e6 commit f33bbfd

File tree

6 files changed

+322
-11
lines changed

6 files changed

+322
-11
lines changed

‎config/opal_config_asm.m4

Lines changed: 82 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,84 @@ __sync_add_and_fetch(&tmp, 1);],
116116
])
117117

118118

119+
AC_DEFUN([OPAL_CHECK_GCC_BUILTIN_CSWAP_INT128], [
120+
121+
OPAL_VAR_SCOPE_PUSH([atomic_compare_exchange_n_128_result CFLAGS_save])
122+
123+
AC_ARG_ENABLE([cross-cmpset128],[AC_HELP_STRING([--enable-cross-cmpset128],
124+
[enable the use of the __sync builtin atomic compare-and-swap 128 when cross compiling])])
125+
126+
atomic_compare_exchange_n_128_result=0
127+
128+
if test ! "$enable_cross_cmpset128" = "yes" ; then
129+
AC_MSG_CHECKING([for processor support of __atomic builtin atomic compare-and-swap on 128-bit values])
130+
131+
AC_RUN_IFELSE([AC_LANG_PROGRAM([], [__int128 x = 0, y = 0; __atomic_compare_exchange_n (&x, &y, 1, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);])],
132+
[AC_MSG_RESULT([yes])
133+
atomic_compare_exchange_n_128_result=1],
134+
[AC_MSG_RESULT([no])],
135+
[AC_MSG_RESULT([no (cross compiling)])])
136+
137+
if test $atomic_compare_exchange_n_128_result = 0 ; then
138+
CFLAGS_save=$CFLAGS
139+
CFLAGS="$CFLAGS -mcx16"
140+
141+
AC_MSG_CHECKING([for __atomic builtin atomic compare-and-swap on 128-bit values with -mcx16 flag])
142+
AC_RUN_IFELSE([AC_LANG_PROGRAM([], [__int128 x = 0, y = 0; __atomic_compare_exchange_n (&x, &y, 1, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);])],
143+
[AC_MSG_RESULT([yes])
144+
atomic_compare_exchange_n_128_result=1
145+
CFLAGS_save="$CFLAGS"],
146+
[AC_MSG_RESULT([no])],
147+
[AC_MSG_RESULT([no (cross compiling)])])
148+
149+
CFLAGS=$CFLAGS_save
150+
fi
151+
else
152+
AC_MSG_CHECKING([for compiler support of __atomic builtin atomic compare-and-swap on 128-bit values])
153+
154+
# Check if the compiler supports the __atomic builtin
155+
AC_TRY_LINK([], [__int128 x = 0; __atomic_bool_compare_and_swap (&x, 0, 1);],
156+
[AC_MSG_RESULT([yes])
157+
atomic_compare_exchange_n_128_result=1],
158+
[AC_MSG_RESULT([no])])
159+
160+
if test $atomic_compare_exchange_n_128_result = 0 ; then
161+
CFLAGS_save=$CFLAGS
162+
CFLAGS="$CFLAGS -mcx16"
163+
164+
AC_MSG_CHECKING([for __atomic builtin atomic compare-and-swap on 128-bit values with -mcx16 flag])
165+
AC_TRY_LINK([], [__int128 x = 0; __atomic_bool_compare_and_swap (&x, 0, 1);],
166+
[AC_MSG_RESULT([yes])
167+
atomic_compare_exchange_n_128_result=1
168+
CFLAGS_save="$CFLAGS"],
169+
[AC_MSG_RESULT([no])])
170+
171+
CFLAGS=$CFLAGS_save
172+
fi
173+
fi
174+
175+
AC_DEFINE_UNQUOTED([OPAL_HAVE_GCC_BUILTIN_CSWAP_INT128], [$atomic_compare_exchange_n_128_result],
176+
[Whether the __atomic builtin atomic compare and swap supports 128-bit values])
177+
178+
OPAL_VAR_SCOPE_POP
179+
])
180+
181+
AC_DEFUN([OPAL_CHECK_GCC_ATOMIC_BUILTINS], [
182+
AC_MSG_CHECKING([for __atomic builtin atomics])
183+
184+
AC_TRY_LINK([long tmp, old = 0;], [__atomic_thread_fence(__ATOMIC_SEQ_CST);
185+
__atomic_compare_exchange_n(&tmp, &old, 1, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
186+
__atomic_add_fetch(&tmp, 1, __ATOMIC_RELAXED);],
187+
[AC_MSG_RESULT([yes])
188+
$1],
189+
[AC_MSG_RESULT([no])
190+
$2])
191+
192+
# Check for 128-bit support
193+
OPAL_CHECK_GCC_BUILTIN_CSWAP_INT128
194+
])
195+
196+
119197
dnl #################################################################
120198
dnl
121199
dnl OPAL_CHECK_ASM_TEXT
@@ -894,6 +972,9 @@ AC_DEFUN([OPAL_CONFIG_ASM],[
894972
[], [enable_osx_builtin_atomics="yes"])
895973
896974
opal_cv_asm_builtin="BUILTIN_NO"
975+
if test "$opal_cv_asm_builtin" = "BUILTIN_NO" && test "$enable_builtin_atomics" = "yes" ; then
976+
OPAL_CHECK_GCC_ATOMIC_BUILTINS([opal_cv_asm_builtin="BUILTIN_GCC"], [])
977+
fi
897978
if test "$opal_cv_asm_builtin" = "BUILTIN_NO" && test "$enable_builtin_atomics" = "yes" ; then
898979
OPAL_CHECK_SYNC_BUILTINS([opal_cv_asm_builtin="BUILTIN_SYNC"], [])
899980
fi
@@ -1032,7 +1113,7 @@ AC_MSG_ERROR([Can not continue.])
10321113
opal_cv_asm_builtin="BUILTIN_NO"
10331114
fi
10341115
1035-
if test "$opal_cv_asm_builtin" = "BUILTIN_SYNC" ; then
1116+
if test "$opal_cv_asm_builtin" = "BUILTIN_SYNC" || test "$opal_cv_asm_builtin" = "BUILTIN_GCC" ; then
10361117
AC_DEFINE([OPAL_C_GCC_INLINE_ASSEMBLY], [1],
10371118
[Whether C compiler supports GCC style inline assembly])
10381119
else

‎opal/include/opal/sys/Makefile.am

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,3 +36,4 @@ include opal/sys/osx/Makefile.am
3636
include opal/sys/powerpc/Makefile.am
3737
include opal/sys/sparcv9/Makefile.am
3838
include opal/sys/sync_builtin/Makefile.am
39+
include opal/sys/gcc_builtin/Makefile.am

‎opal/include/opal/sys/architecture.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,8 @@
3939
#define OPAL_ARM 0100
4040
#define OPAL_BUILTIN_SYNC 0200
4141
#define OPAL_BUILTIN_OSX 0201
42-
#define OPAL_BUILTIN_NO 0202
42+
#define OPAL_BUILTIN_GCC 0202
43+
#define OPAL_BUILTIN_NO 0203
4344

4445
/* Formats */
4546
#define OPAL_DEFAULT 1000 /* standard for given architecture */

‎opal/include/opal/sys/atomic.h

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,14 @@ typedef struct opal_atomic_lock_t opal_atomic_lock_t;
131131
#define OPAL_HAVE_INLINE_ATOMIC_SWAP_64 1
132132
#endif
133133

134+
/**
135+
* Enumeration of lock states
136+
*/
137+
enum {
138+
OPAL_ATOMIC_UNLOCKED = 0,
139+
OPAL_ATOMIC_LOCKED = 1
140+
};
141+
134142
/**********************************************************************
135143
*
136144
* Load the appropriate architecture files and set some reasonable
@@ -141,6 +149,8 @@ typedef struct opal_atomic_lock_t opal_atomic_lock_t;
141149
/* don't include system-level gorp when generating doxygen files */
142150
#elif OPAL_ASSEMBLY_BUILTIN == OPAL_BUILTIN_SYNC
143151
#include "opal/sys/sync_builtin/atomic.h"
152+
#elif OPAL_ASSEMBLY_BUILTIN == OPAL_BUILTIN_GCC
153+
#include "opal/sys/gcc_builtin/atomic.h"
144154
#elif OPAL_ASSEMBLY_BUILTIN == OPAL_BUILTIN_OSX
145155
#include "opal/sys/osx/atomic.h"
146156
#elif OPAL_ASSEMBLY_ARCH == OPAL_AMD64
@@ -264,15 +274,6 @@ void opal_atomic_wmb(void);
264274

265275
#if defined(DOXYGEN) || OPAL_HAVE_ATOMIC_SPINLOCKS || (OPAL_HAVE_ATOMIC_CMPSET_32 || OPAL_HAVE_ATOMIC_CMPSET_64)
266276

267-
/**
268-
* Enumeration of lock states
269-
*/
270-
enum {
271-
OPAL_ATOMIC_UNLOCKED = 0,
272-
OPAL_ATOMIC_LOCKED = 1
273-
};
274-
275-
276277
/**
277278
* Initialize a lock to value
278279
*

‎opal/include/opal/sys/gcc_builtin/Makefile.am

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
#
2+
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
3+
# University Research and Technology
4+
# Corporation. All rights reserved.
5+
# Copyright (c) 2004-2005 The University of Tennessee and The University
6+
# of Tennessee Research Foundation. All rights
7+
# reserved.
8+
# Copyright (c) 2004-2009 High Performance Computing Center Stuttgart,
9+
# University of Stuttgart. All rights reserved.
10+
# Copyright (c) 2004-2005 The Regents of the University of California.
11+
# All rights reserved.
12+
# Copyright (c) 2011 Sandia National Laboratories. All rights reserved.
13+
# Copyright (c) 2016 Los Alamos National Security, LLC. All rights
14+
# reserved.
15+
# $COPYRIGHT$
16+
#
17+
# Additional copyrights may follow
18+
#
19+
# $HEADER$
20+
#
21+
22+
# This makefile.am does not stand on its own - it is included from opal/include/Makefile.am
23+
24+
headers += \
25+
opal/sys/gcc_builtin/atomic.h

‎opal/include/opal/sys/gcc_builtin/atomic.h

Lines changed: 202 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,202 @@
1+
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
2+
/*
3+
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
4+
* University Research and Technology
5+
* Corporation. All rights reserved.
6+
* Copyright (c) 2004-2013 The University of Tennessee and The University
7+
* of Tennessee Research Foundation. All rights
8+
* reserved.
9+
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
10+
* University of Stuttgart. All rights reserved.
11+
* Copyright (c) 2004-2005 The Regents of the University of California.
12+
* All rights reserved.
13+
* Copyright (c) 2011 Sandia National Laboratories. All rights reserved.
14+
* Copyright (c) 2014-2016 Los Alamos National Security, LLC. All rights
15+
* reserved.
16+
* $COPYRIGHT$
17+
*
18+
* Additional copyrights may follow
19+
*
20+
* $HEADER$
21+
*/
22+
23+
#ifndef OPAL_SYS_ARCH_ATOMIC_H
24+
#define OPAL_SYS_ARCH_ATOMIC_H 1
25+
26+
#include <stdbool.h>
27+
28+
/**********************************************************************
29+
*
30+
* Memory Barriers
31+
*
32+
*********************************************************************/
33+
#define OPAL_HAVE_ATOMIC_MEM_BARRIER 1
34+
35+
#define OPAL_HAVE_ATOMIC_MATH_32 1
36+
#define OPAL_HAVE_ATOMIC_CMPSET_32 1
37+
#define OPAL_HAVE_ATOMIC_ADD_32 1
38+
#define OPAL_HAVE_ATOMIC_SUB_32 1
39+
#define OPAL_HAVE_ATOMIC_SWAP_32 1
40+
#define OPAL_HAVE_ATOMIC_MATH_64 1
41+
#define OPAL_HAVE_ATOMIC_CMPSET_64 1
42+
#define OPAL_HAVE_ATOMIC_ADD_64 1
43+
#define OPAL_HAVE_ATOMIC_SUB_64 1
44+
#define OPAL_HAVE_ATOMIC_SWAP_64 1
45+
46+
47+
static inline void opal_atomic_mb(void)
48+
{
49+
__atomic_thread_fence (__ATOMIC_SEQ_CST);
50+
}
51+
52+
static inline void opal_atomic_rmb(void)
53+
{
54+
__atomic_thread_fence (__ATOMIC_ACQUIRE);
55+
}
56+
57+
static inline void opal_atomic_wmb(void)
58+
{
59+
__atomic_thread_fence (__ATOMIC_RELEASE);
60+
}
61+
62+
#define MB() opal_atomic_mb()
63+
64+
/**********************************************************************
65+
*
66+
* Atomic math operations
67+
*
68+
*********************************************************************/
69+
70+
static inline int opal_atomic_cmpset_acq_32( volatile int32_t *addr,
71+
int32_t oldval, int32_t newval)
72+
{
73+
return __atomic_compare_exchange_n (addr, &oldval, newval, false,
74+
__ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
75+
}
76+
77+
78+
static inline int opal_atomic_cmpset_rel_32( volatile int32_t *addr,
79+
int32_t oldval, int32_t newval)
80+
{
81+
return __atomic_compare_exchange_n (addr, &oldval, newval, false,
82+
__ATOMIC_RELEASE, __ATOMIC_RELAXED);
83+
}
84+
85+
static inline int opal_atomic_cmpset_32( volatile int32_t *addr,
86+
int32_t oldval, int32_t newval)
87+
{
88+
return __atomic_compare_exchange_n (addr, &oldval, newval, false,
89+
__ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
90+
}
91+
92+
static inline int32_t opal_atomic_swap_32 (volatile int32_t *addr, int32_t newval)
93+
{
94+
int32_t oldval;
95+
__atomic_exchange (addr, &newval, &oldval, __ATOMIC_RELAXED);
96+
return oldval;
97+
}
98+
99+
static inline int32_t opal_atomic_add_32(volatile int32_t *addr, int32_t delta)
100+
{
101+
return __atomic_add_fetch (addr, delta, __ATOMIC_RELAXED);
102+
}
103+
104+
static inline int32_t opal_atomic_sub_32(volatile int32_t *addr, int32_t delta)
105+
{
106+
return __atomic_sub_fetch (addr, delta, __ATOMIC_RELAXED);
107+
}
108+
109+
static inline int opal_atomic_cmpset_acq_64( volatile int64_t *addr,
110+
int64_t oldval, int64_t newval)
111+
{
112+
return __atomic_compare_exchange_n (addr, &oldval, newval, false,
113+
__ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
114+
}
115+
116+
static inline int opal_atomic_cmpset_rel_64( volatile int64_t *addr,
117+
int64_t oldval, int64_t newval)
118+
{
119+
return __atomic_compare_exchange_n (addr, &oldval, newval, false,
120+
__ATOMIC_RELEASE, __ATOMIC_RELAXED);
121+
}
122+
123+
124+
static inline int opal_atomic_cmpset_64( volatile int64_t *addr,
125+
int64_t oldval, int64_t newval)
126+
{
127+
return __atomic_compare_exchange_n (addr, &oldval, newval, false,
128+
__ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
129+
}
130+
131+
static inline int64_t opal_atomic_swap_64 (volatile int64_t *addr, int64_t newval)
132+
{
133+
int64_t oldval;
134+
__atomic_exchange (addr, &newval, &oldval, __ATOMIC_RELAXED);
135+
return oldval;
136+
}
137+
138+
static inline int64_t opal_atomic_add_64(volatile int64_t *addr, int64_t delta)
139+
{
140+
return __atomic_add_fetch (addr, delta, __ATOMIC_RELAXED);
141+
}
142+
143+
static inline int64_t opal_atomic_sub_64(volatile int64_t *addr, int64_t delta)
144+
{
145+
return __atomic_sub_fetch (addr, delta, __ATOMIC_RELAXED);
146+
}
147+
148+
#if OPAL_HAVE_GCC_BUILTIN_CSWAP_INT128
149+
150+
#define OPAL_HAVE_ATOMIC_CMPSET_128 1
151+
152+
static inline int opal_atomic_cmpset_128 (volatile opal_int128_t *addr,
153+
opal_int128_t oldval, opal_int128_t newval)
154+
{
155+
return __atomic_compare_exchange_n (addr, &oldval, newval, false,
156+
__ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
157+
}
158+
159+
#endif
160+
161+
#if defined(__HLE__)
162+
163+
#include <immintrin.h>
164+
165+
#define OPAL_HAVE_ATOMIC_SPINLOCKS 1
166+
167+
static inline void opal_atomic_init (opal_atomic_lock_t* lock, int32_t value)
168+
{
169+
lock->u.lock = value;
170+
}
171+
172+
static inline int opal_atomic_trylock(opal_atomic_lock_t *lock)
173+
{
174+
int ret = __atomic_exchange_n (&lock->u.lock, OPAL_ATOMIC_LOCKED,
175+
__ATOMIC_ACQUIRE | __ATOMIC_HLE_ACQUIRE);
176+
if (OPAL_ATOMIC_LOCKED == ret) {
177+
/* abort the transaction */
178+
_mm_pause ();
179+
return 1;
180+
}
181+
182+
return 0;
183+
}
184+
185+
static inline void opal_atomic_lock (opal_atomic_lock_t *lock)
186+
{
187+
while (OPAL_ATOMIC_LOCKED == __atomic_exchange_n (&lock->u.lock, OPAL_ATOMIC_LOCKED,
188+
__ATOMIC_ACQUIRE | __ATOMIC_HLE_ACQUIRE)) {
189+
/* abort the transaction */
190+
_mm_pause ();
191+
}
192+
}
193+
194+
static inline void opal_atomic_unlock (opal_atomic_lock_t *lock)
195+
{
196+
__atomic_store_n (&lock->u.lock, OPAL_ATOMIC_UNLOCKED,
197+
__ATOMIC_RELEASE | __ATOMIC_HLE_RELEASE);
198+
}
199+
200+
#endif
201+
202+
#endif /* ! OPAL_SYS_ARCH_ATOMIC_H */

0 commit comments

Comments
 (0)