Location via proxy:   [ UP ]  
[Report a bug]   [Manage cookies]                
Skip to content

Commit 64b1fb5

Browse files
Optimize pg_atomic_exchange_u32 and pg_atomic_exchange_u64.
Presently, all platforms implement atomic exchanges by performing an atomic compare-and-swap in a loop until it succeeds. This can be especially expensive when there is contention on the atomic variable. This commit optimizes atomic exchanges on many platforms by using compiler intrinsics, which should compile into something much less expensive than a compare-and-swap loop. Since these intrinsics have been available for some time, the inline assembly implementations are omitted. Suggested-by: Andres Freund Reviewed-by: Andres Freund Discussion: https://postgr.es/m/20231129212905.GA1258737%40nathanxps13
1 parent 0d1adae commit 64b1fb5

File tree

3 files changed

+66
-0
lines changed

3 files changed

+66
-0
lines changed

src/include/port/atomics/generic-gcc.h

+34
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,23 @@ pg_atomic_compare_exchange_u32_impl(volatile pg_atomic_uint32 *ptr,
176176
}
177177
#endif
178178

179+
/*
180+
* __sync_lock_test_and_set() only supports setting the value to 1 on some
181+
* platforms, so we only provide an __atomic implementation for
182+
* pg_atomic_exchange.
183+
*
184+
* We assume the availability of 32-bit __atomic_compare_exchange_n() implies
185+
* the availability of 32-bit __atomic_exchange_n().
186+
*/
187+
#if !defined(PG_HAVE_ATOMIC_EXCHANGE_U32) && defined(HAVE_GCC__ATOMIC_INT32_CAS)
188+
#define PG_HAVE_ATOMIC_EXCHANGE_U32
189+
static inline uint32
190+
pg_atomic_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 newval)
191+
{
192+
return __atomic_exchange_n(&ptr->value, newval, __ATOMIC_SEQ_CST);
193+
}
194+
#endif
195+
179196
/* if we have 32-bit __sync_val_compare_and_swap, assume we have these too: */
180197

181198
#if !defined(PG_HAVE_ATOMIC_FETCH_ADD_U32) && defined(HAVE_GCC__SYNC_INT32_CAS)
@@ -243,6 +260,23 @@ pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr,
243260
}
244261
#endif
245262

263+
/*
264+
* __sync_lock_test_and_set() only supports setting the value to 1 on some
265+
* platforms, so we only provide an __atomic implementation for
266+
* pg_atomic_exchange.
267+
*
268+
* We assume the availability of 64-bit __atomic_compare_exchange_n() implies
269+
* the availability of 64-bit __atomic_exchange_n().
270+
*/
271+
#if !defined(PG_HAVE_ATOMIC_EXCHANGE_U64) && defined(HAVE_GCC__ATOMIC_INT64_CAS)
272+
#define PG_HAVE_ATOMIC_EXCHANGE_U64
273+
static inline uint64
274+
pg_atomic_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 newval)
275+
{
276+
return __atomic_exchange_n(&ptr->value, newval, __ATOMIC_SEQ_CST);
277+
}
278+
#endif
279+
246280
/* if we have 64-bit __sync_val_compare_and_swap, assume we have these too: */
247281

248282
#if !defined(PG_HAVE_ATOMIC_FETCH_ADD_U64) && defined(HAVE_GCC__SYNC_INT64_CAS)

src/include/port/atomics/generic-msvc.h

+18
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,13 @@ pg_atomic_compare_exchange_u32_impl(volatile pg_atomic_uint32 *ptr,
5858
return ret;
5959
}
6060

61+
#define PG_HAVE_ATOMIC_EXCHANGE_U32
62+
static inline uint32
63+
pg_atomic_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 newval)
64+
{
65+
return InterlockedExchange(&ptr->value, newval);
66+
}
67+
6168
#define PG_HAVE_ATOMIC_FETCH_ADD_U32
6269
static inline uint32
6370
pg_atomic_fetch_add_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_)
@@ -88,6 +95,16 @@ pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr,
8895

8996
/* Only implemented on 64bit builds */
9097
#ifdef _WIN64
98+
99+
#pragma intrinsic(_InterlockedExchange64)
100+
101+
#define PG_HAVE_ATOMIC_EXCHANGE_U64
102+
static inline uint64
103+
pg_atomic_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 newval)
104+
{
105+
return _InterlockedExchange64(&ptr->value, newval);
106+
}
107+
91108
#pragma intrinsic(_InterlockedExchangeAdd64)
92109

93110
#define PG_HAVE_ATOMIC_FETCH_ADD_U64
@@ -96,6 +113,7 @@ pg_atomic_fetch_add_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_)
96113
{
97114
return _InterlockedExchangeAdd64(&ptr->value, add_);
98115
}
116+
99117
#endif /* _WIN64 */
100118

101119
#endif /* HAVE_ATOMICS */

src/include/port/atomics/generic-sunpro.h

+14
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,13 @@ pg_atomic_compare_exchange_u32_impl(volatile pg_atomic_uint32 *ptr,
8787
return ret;
8888
}
8989

90+
#define PG_HAVE_ATOMIC_EXCHANGE_U32
91+
static inline uint32
92+
pg_atomic_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 newval)
93+
{
94+
return atomic_swap_32(&ptr->value, newval);
95+
}
96+
9097
#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64
9198
static inline bool
9299
pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr,
@@ -101,6 +108,13 @@ pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr,
101108
return ret;
102109
}
103110

111+
#define PG_HAVE_ATOMIC_EXCHANGE_U64
112+
static inline uint64
113+
pg_atomic_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 newval)
114+
{
115+
return atomic_swap_64(&ptr->value, newval);
116+
}
117+
104118
#endif /* HAVE_ATOMIC_H */
105119

106120
#endif /* defined(HAVE_ATOMICS) */

0 commit comments

Comments
 (0)