CX Framework
Cross-platform C utility framework
Loading...
Searching...
No Matches
msvc_atomic.h
1#pragma once
2
3#if defined(_M_X64)
4#define CX_MemoryBarrier __faststorefence
5#define CX_ReadWriteBarrier _ReadWriteBarrier
6#elif defined(_M_IX86)
7__forceinline void CX_MemoryBarrier(void)
8{
9 long Barrier;
10
11 _InterlockedOr(&Barrier, 0);
12 return;
13}
14#define CX_ReadWriteBarrier _ReadWriteBarrier
15#else
16#error "Don't know how to create atomics for this platform for MSVC."
17#endif
18
19#include "cx/utils/macros.h"
20
21#define atomicInit(...) \
22 { \
23 __VA_ARGS__ \
24 }
25
26typedef enum {
27 ATOMIC_MO_Relaxed,
28 ATOMIC_MO_Acquire,
29 ATOMIC_MO_Release,
30 ATOMIC_MO_AcqRel,
31 ATOMIC_MO_SeqCst
32} AtmoicMemoryOrder;
33
34typedef char _cx_atomic_repr_0_t;
35typedef short _cx_atomic_repr_1_t;
36typedef long _cx_atomic_repr_2_t;
37typedef __int64 _cx_atomic_repr_3_t;
38
39// Trivial per-size raw load/store, used uniformly by CX_GENERATE_ATOMICS. Sizes 0-2 (and
40// size 3 on x64) are always atomic as plain accesses on x86/x64; size 3 on x86 needs the
41// special handling below, since a plain 8-byte access there can tear.
42_meta_inline _cx_atomic_repr_0_t _cx_atomic_rawLoad_0(const _cx_atomic_repr_0_t* p)
43{
44 return *p;
45}
46_meta_inline void _cx_atomic_rawStore_0(_cx_atomic_repr_0_t* p, _cx_atomic_repr_0_t val)
47{
48 *p = val;
49}
50_meta_inline _cx_atomic_repr_1_t _cx_atomic_rawLoad_1(const _cx_atomic_repr_1_t* p)
51{
52 return *p;
53}
54_meta_inline void _cx_atomic_rawStore_1(_cx_atomic_repr_1_t* p, _cx_atomic_repr_1_t val)
55{
56 *p = val;
57}
58_meta_inline _cx_atomic_repr_2_t _cx_atomic_rawLoad_2(const _cx_atomic_repr_2_t* p)
59{
60 return *p;
61}
62_meta_inline void _cx_atomic_rawStore_2(_cx_atomic_repr_2_t* p, _cx_atomic_repr_2_t val)
63{
64 *p = val;
65}
66
67#if defined(_M_IX86)
68
69// MSVC/x86 has no 64-bit Interlocked intrinsics except _InterlockedCompareExchange64
70// (cmpxchg8b). Route every "*64" name the RMW macros paste through a "_cx_x86"-suffixed
71// stand-in implemented here as a compare-exchange retry loop.
72#define CX_ATOMIC_IL_SUFFIX_3 64_cx_x86
73
74#include <emmintrin.h>
75
76#if _M_IX86_FP >= 2
77
78// SSE2 movq is an unlocked but atomic 8-byte load/store - the default target since
79// VS2012 (/arch:SSE2), so this is the path every shipped x86 build actually takes.
80_meta_inline _cx_atomic_repr_3_t _cx_atomic_rawLoad_3(const _cx_atomic_repr_3_t* p)
81{
82 _cx_atomic_repr_3_t ret;
83 _mm_storel_epi64((__m128i*)&ret, _mm_loadl_epi64((const __m128i*)p));
84 return ret;
85}
86_meta_inline void _cx_atomic_rawStore_3(_cx_atomic_repr_3_t* p, _cx_atomic_repr_3_t val)
87{
88 _mm_storel_epi64((__m128i*)p, _mm_loadl_epi64((const __m128i*)&val));
89}
90
91#else
92
93// /arch:IA32 has no unlocked 8-byte load/store, so fall back to
94// _InterlockedCompareExchange64. The "load" writes back the value it read (a CAS against
95// itself), which is why it needs writable memory - it cannot honor const.
96_meta_inline _cx_atomic_repr_3_t _cx_atomic_rawLoad_3(const _cx_atomic_repr_3_t* p)
97{
98 _cx_atomic_repr_3_t* pw = (_cx_atomic_repr_3_t*)p;
99 return _InterlockedCompareExchange64(pw, 0, 0);
100}
101_meta_inline void _cx_atomic_rawStore_3(_cx_atomic_repr_3_t* p, _cx_atomic_repr_3_t val)
102{
103 _cx_atomic_repr_3_t old = *p;
104 for (;;) {
105 _cx_atomic_repr_3_t prev = _InterlockedCompareExchange64(p, val, old);
106 if (prev == old) {
107 return;
108 }
109 old = prev;
110 }
111}
112
113#endif // _M_IX86_FP >= 2
114
115_meta_inline _cx_atomic_repr_3_t _InterlockedCompareExchange64_cx_x86(_cx_atomic_repr_3_t* p,
116 _cx_atomic_repr_3_t xchg,
117 _cx_atomic_repr_3_t comp)
118{
119 return _InterlockedCompareExchange64(p, xchg, comp);
120}
121
122// One CAS retry loop shared by every 64-bit RMW op MSVC/x86 lacks a native intrinsic
123// for. "expr" computes the new value from "old"; it may reference "old" and "val". The
124// generated name matches what CX_ATOMIC_IL_NAME pastes via the "64_cx_x86" suffix.
125#define _CX_X86_ATOMIC64_RMW(name, expr) \
126 _meta_inline _cx_atomic_repr_3_t name##64_cx_x86(_cx_atomic_repr_3_t * p, \
127 _cx_atomic_repr_3_t val) \
128 { \
129 _cx_atomic_repr_3_t old = *p; \
130 for (;;) { \
131 _cx_atomic_repr_3_t desired = (expr); \
132 _cx_atomic_repr_3_t prev = _InterlockedCompareExchange64(p, desired, old); \
133 if (prev == old) { \
134 return old; \
135 } \
136 old = prev; \
137 } \
138 }
139
140_CX_X86_ATOMIC64_RMW(_InterlockedExchange, val)
141_CX_X86_ATOMIC64_RMW(_InterlockedExchangeAdd,
142 (_cx_atomic_repr_3_t)((unsigned __int64)old + (unsigned __int64)val))
143_CX_X86_ATOMIC64_RMW(_InterlockedAnd, old & val)
144_CX_X86_ATOMIC64_RMW(_InterlockedOr, old | val)
145_CX_X86_ATOMIC64_RMW(_InterlockedXor, old ^ val)
146
147#else // _M_X64
148
149#define CX_ATOMIC_IL_SUFFIX_3 64
150
151_meta_inline _cx_atomic_repr_3_t _cx_atomic_rawLoad_3(const _cx_atomic_repr_3_t* p)
152{
153 return *p;
154}
155_meta_inline void _cx_atomic_rawStore_3(_cx_atomic_repr_3_t* p, _cx_atomic_repr_3_t val)
156{
157 *p = val;
158}
159
160#endif // defined(_M_IX86)
161
162_meta_inline void _atomicFence(AtmoicMemoryOrder mo)
163{
164 CX_ReadWriteBarrier();
165#if defined(_M_ARM) || defined(_M_ARM64)
166 /* ARM needs a barrier for everything but relaxed. */
167 if (mo != ATOMIC_MO_Relaxed) {
168 CX_MemoryBarrier();
169 }
170#elif defined(_M_IX86) || defined(_M_X64)
171 /* x86 needs a barrier only for seq_cst. */
172 if (mo == ATOMIC_MO_SeqCst) {
173 CX_MemoryBarrier();
174 }
175#else
176#error "Don't know how to create atomics for this platform for MSVC."
177#endif
178 CX_ReadWriteBarrier();
179}
180#define atomicFence(order) _atomicFence(ATOMIC_MO_##order)
181
182#define CX_ATOMIC_IL_REPR(lg_size) _cx_atomic_repr_##lg_size##_t
183
184#define CX_ATOMIC_IL_NAME(base_name, lg_size) tokconcat(base_name, CX_ATOMIC_IL_SUFFIX(lg_size))
185
186#define CX_ATOMIC_IL_SUFFIX(lg_size) tokconcat(CX_ATOMIC_IL_SUFFIX_, lg_size)
187
188#define CX_ATOMIC_IL_SUFFIX_0 8
189#define CX_ATOMIC_IL_SUFFIX_1 16
190#define CX_ATOMIC_IL_SUFFIX_2
191// CX_ATOMIC_IL_SUFFIX_3 is defined above, alongside the rawLoad/rawStore split, since its
192// value depends on the same _M_IX86 vs _M_X64 branch.
193
194#define CX_ATOMIC_IL_RAWLOAD(lg_size) tokconcat(_cx_atomic_rawLoad_, lg_size)
195#define CX_ATOMIC_IL_RAWSTORE(lg_size) tokconcat(_cx_atomic_rawStore_, lg_size)
196
197#define CX_ATOMIC_IL_ALIGN(lg_size) tokconcat(CX_ATOMIC_IL_ALIGN_, lg_size)
198#define CX_ATOMIC_IL_ALIGN_0
199#define CX_ATOMIC_IL_ALIGN_1
200#define CX_ATOMIC_IL_ALIGN_2
201#if defined(_M_IX86)
202// The movq/cmpxchg8b paths both require 8-byte alignment; state it explicitly rather
203// than rely on the default struct layout giving it to us.
204#define CX_ATOMIC_IL_ALIGN_3 alignMem(8)
205#else
206#define CX_ATOMIC_IL_ALIGN_3
207#endif
208
209#define atomic(type) cx_atomic_##type
210#define atomicLoad(type, atomic_ptr, order) _atomicLoad_##type(atomic_ptr, ATOMIC_MO_##order)
211#define atomicStore(type, atomic_ptr, val, order) \
212 _atomicStore_##type(atomic_ptr, val, ATOMIC_MO_##order)
213#define atomicExchange(type, atomic_ptr, val, order) \
214 _atomicExchange_##type(atomic_ptr, val, ATOMIC_MO_##order)
215#define atomicCompareExchange(type, \
216 semantics, \
217 atomic_ptr, \
218 expected_ptr, \
219 desired, \
220 successorder, \
221 failorder) \
222 _atomicCompareExchange_##semantics##_##type(atomic_ptr, \
223 expected_ptr, \
224 desired, \
225 ATOMIC_MO_##successorder, \
226 ATOMIC_MO_##failorder)
227
228#define atomicFetchAdd(type, atomic_ptr, val, order) \
229 _atomicFetchAdd_##type(atomic_ptr, val, ATOMIC_MO_##order)
230#define atomicFetchSub(type, atomic_ptr, val, order) \
231 _atomicFetchSub_##type(atomic_ptr, val, ATOMIC_MO_##order)
232#define atomicFetchAnd(type, atomic_ptr, val, order) \
233 _atomicFetchAnd_##type(atomic_ptr, val, ATOMIC_MO_##order)
234#define atomicFetchOr(type, atomic_ptr, val, order) \
235 _atomicFetchOr_##type(atomic_ptr, val, ATOMIC_MO_##order)
236#define atomicFetchXor(type, atomic_ptr, val, order) \
237 _atomicFetchXor_##type(atomic_ptr, val, ATOMIC_MO_##order)
238
239#define CX_GENERATE_ATOMICS(type, short_type, lg_size) \
240 typedef struct { \
241 CX_ATOMIC_IL_ALIGN(lg_size) CX_ATOMIC_IL_REPR(lg_size) repr; \
242 } cx_atomic_##short_type; \
243 \
244 _meta_inline type _atomicLoad_##short_type(const cx_atomic_##short_type* a, \
245 AtmoicMemoryOrder mo) \
246 { \
247 CX_ATOMIC_IL_REPR(lg_size) ret = CX_ATOMIC_IL_RAWLOAD(lg_size)(&a->repr); \
248 if (mo != ATOMIC_MO_Relaxed) { \
249 _atomicFence(ATOMIC_MO_Acquire); \
250 } \
251 return (type)ret; \
252 } \
253 \
254 _meta_inline void _atomicStore_##short_type(cx_atomic_##short_type* a, \
255 type val, \
256 AtmoicMemoryOrder mo) \
257 { \
258 if (mo != ATOMIC_MO_Relaxed) { \
259 _atomicFence(ATOMIC_MO_Release); \
260 } \
261 CX_ATOMIC_IL_RAWSTORE(lg_size)(&a->repr, (CX_ATOMIC_IL_REPR(lg_size))val); \
262 if (mo == ATOMIC_MO_SeqCst) { \
263 _atomicFence(ATOMIC_MO_SeqCst); \
264 } \
265 } \
266 \
267 _meta_inline type \
268 _atomicExchange_##short_type(cx_atomic_##short_type* a, type val, AtmoicMemoryOrder mo) \
269 { \
270 return (type)CX_ATOMIC_IL_NAME(_InterlockedExchange, \
271 lg_size)(&a->repr, (CX_ATOMIC_IL_REPR(lg_size))val); \
272 } \
273 \
274 _meta_inline bool _atomicCompareExchange_weak_##short_type(cx_atomic_##short_type* a, \
275 type* expected, \
276 type desired, \
277 AtmoicMemoryOrder success_mo, \
278 AtmoicMemoryOrder failure_mo) \
279 { \
280 CX_ATOMIC_IL_REPR(lg_size) e = (CX_ATOMIC_IL_REPR(lg_size)) * expected; \
281 CX_ATOMIC_IL_REPR(lg_size) d = (CX_ATOMIC_IL_REPR(lg_size))desired; \
282 CX_ATOMIC_IL_REPR(lg_size) \
283 old = CX_ATOMIC_IL_NAME(_InterlockedCompareExchange, lg_size)(&a->repr, d, e); \
284 if (old == e) { \
285 return true; \
286 } else { \
287 *expected = (type)old; \
288 return false; \
289 } \
290 } \
291 \
292 _meta_inline bool _atomicCompareExchange_strong_##short_type(cx_atomic_##short_type* a, \
293 type* expected, \
294 type desired, \
295 AtmoicMemoryOrder success_mo, \
296 AtmoicMemoryOrder failure_mo) \
297 { \
298 /* We implement the weak version with strong semantics. */ \
299 return _atomicCompareExchange_weak_##short_type(a, \
300 expected, \
301 desired, \
302 success_mo, \
303 failure_mo); \
304 }
305
306#define CX_EXTERN_ATOMICS(type, short_type) \
307 extern inline type _atomicLoad_##short_type(const cx_atomic_##short_type* a, \
308 AtmoicMemoryOrder mo); \
309 \
310 extern inline void _atomicStore_##short_type(cx_atomic_##short_type* a, \
311 type val, \
312 AtmoicMemoryOrder mo); \
313 \
314 extern inline type _atomicExchange_##short_type(cx_atomic_##short_type* a, \
315 type val, \
316 AtmoicMemoryOrder mo); \
317 \
318 extern inline bool _atomicCompareExchange_weak_##short_type(cx_atomic_##short_type* a, \
319 type* expected, \
320 type desired, \
321 AtmoicMemoryOrder success_mo, \
322 AtmoicMemoryOrder failure_mo); \
323 \
324 extern inline bool _atomicCompareExchange_strong_##short_type(cx_atomic_##short_type* a, \
325 type* expected, \
326 type desired, \
327 AtmoicMemoryOrder success_mo, \
328 AtmoicMemoryOrder failure_mo);
329
330#define CX_GENERATE_INT_ATOMICS(type, short_type, lg_size) \
331 CX_GENERATE_ATOMICS(type, short_type, lg_size) \
332 \
333 _meta_inline type \
334 _atomicFetchAdd_##short_type(cx_atomic_##short_type* a, type val, AtmoicMemoryOrder mo) \
335 { \
336 return (type)CX_ATOMIC_IL_NAME(_InterlockedExchangeAdd, \
337 lg_size)(&a->repr, (CX_ATOMIC_IL_REPR(lg_size))val); \
338 } \
339 \
340 _meta_inline type \
341 _atomicFetchSub_##short_type(cx_atomic_##short_type* a, type val, AtmoicMemoryOrder mo) \
342 { \
343 /* \
344 * MSVC warns on negation of unsigned operands, but for us it \
345 * gives exactly the right semantics (MAX_TYPE + 1 - operand). \
346 */ \
347 __pragma(warning(push)) \
348 __pragma(warning(disable : 4146)) return _atomicFetchAdd_##short_type(a, -val, mo); \
349 __pragma(warning(pop)) \
350 } \
351 _meta_inline type \
352 _atomicFetchAnd_##short_type(cx_atomic_##short_type* a, type val, AtmoicMemoryOrder mo) \
353 { \
354 return (type)CX_ATOMIC_IL_NAME(_InterlockedAnd, \
355 lg_size)(&a->repr, (CX_ATOMIC_IL_REPR(lg_size))val); \
356 } \
357 _meta_inline type \
358 _atomicFetchOr_##short_type(cx_atomic_##short_type* a, type val, AtmoicMemoryOrder mo) \
359 { \
360 return (type)CX_ATOMIC_IL_NAME(_InterlockedOr, \
361 lg_size)(&a->repr, (CX_ATOMIC_IL_REPR(lg_size))val); \
362 } \
363 _meta_inline type \
364 _atomicFetchXor_##short_type(cx_atomic_##short_type* a, type val, AtmoicMemoryOrder mo) \
365 { \
366 return (type)CX_ATOMIC_IL_NAME(_InterlockedXor, \
367 lg_size)(&a->repr, (CX_ATOMIC_IL_REPR(lg_size))val); \
368 }
369
370#define CX_EXTERN_INT_ATOMICS(type, short_type) \
371 CX_EXTERN_ATOMICS(type, short_type) \
372 \
373 extern inline type atomicFetchAdd_##short_type(cx_atomic_##short_type* a, \
374 type val, \
375 AtmoicMemoryOrder mo); \
376 \
377 extern inline type atomicFetchSub_##short_type(cx_atomic_##short_type* a, \
378 type val, \
379 AtmoicMemoryOrder mo); \
380 extern inline type atomicFetchAnd_##short_type(cx_atomic_##short_type* a, \
381 type val, \
382 AtmoicMemoryOrder mo); \
383 extern inline type atomicFetchOr_##short_type(cx_atomic_##short_type* a, \
384 type val, \
385 AtmoicMemoryOrder mo); \
386 extern inline type atomicFetchXor_##short_type(cx_atomic_##short_type* a, \
387 type val, \
388 AtmoicMemoryOrder mo);