/* * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ /* XSIZE_compare_and_swap (based on fetch_compare_and_swap) */ #if defined(AO_HAVE_XSIZE_fetch_compare_and_swap_full) \ && !defined(AO_HAVE_XSIZE_compare_and_swap_full) AO_INLINE int AO_XSIZE_compare_and_swap_full(volatile XCTYPE *addr, XCTYPE old_val, XCTYPE new_val) { return AO_XSIZE_fetch_compare_and_swap_full(addr, old_val, new_val) == old_val; } # define AO_HAVE_XSIZE_compare_and_swap_full #endif #if defined(AO_HAVE_XSIZE_fetch_compare_and_swap_acquire) \ && !defined(AO_HAVE_XSIZE_compare_and_swap_acquire) AO_INLINE int AO_XSIZE_compare_and_swap_acquire(volatile XCTYPE *addr, XCTYPE old_val, XCTYPE new_val) { return AO_XSIZE_fetch_compare_and_swap_acquire(addr, old_val, new_val) == old_val; } # define AO_HAVE_XSIZE_compare_and_swap_acquire #endif #if defined(AO_HAVE_XSIZE_fetch_compare_and_swap_release) \ && !defined(AO_HAVE_XSIZE_compare_and_swap_release) AO_INLINE int AO_XSIZE_compare_and_swap_release(volatile XCTYPE *addr, XCTYPE old_val, XCTYPE new_val) { return AO_XSIZE_fetch_compare_and_swap_release(addr, old_val, new_val) == old_val; } # define AO_HAVE_XSIZE_compare_and_swap_release #endif #if defined(AO_HAVE_XSIZE_fetch_compare_and_swap_write) \ && !defined(AO_HAVE_XSIZE_compare_and_swap_write) AO_INLINE int AO_XSIZE_compare_and_swap_write(volatile XCTYPE *addr, XCTYPE old_val, XCTYPE new_val) { return AO_XSIZE_fetch_compare_and_swap_write(addr, old_val, new_val) == old_val; } # define AO_HAVE_XSIZE_compare_and_swap_write #endif #if defined(AO_HAVE_XSIZE_fetch_compare_and_swap_read) \ && !defined(AO_HAVE_XSIZE_compare_and_swap_read) AO_INLINE int AO_XSIZE_compare_and_swap_read(volatile XCTYPE *addr, XCTYPE old_val, XCTYPE new_val) { return AO_XSIZE_fetch_compare_and_swap_read(addr, old_val, new_val) == old_val; } # define AO_HAVE_XSIZE_compare_and_swap_read #endif #if defined(AO_HAVE_XSIZE_fetch_compare_and_swap) \ && !defined(AO_HAVE_XSIZE_compare_and_swap) AO_INLINE int AO_XSIZE_compare_and_swap(volatile XCTYPE *addr, XCTYPE old_val, XCTYPE new_val) { return AO_XSIZE_fetch_compare_and_swap(addr, old_val, new_val) == old_val; } # define AO_HAVE_XSIZE_compare_and_swap #endif #if defined(AO_HAVE_XSIZE_fetch_compare_and_swap_release_write) \ && !defined(AO_HAVE_XSIZE_compare_and_swap_release_write) AO_INLINE int AO_XSIZE_compare_and_swap_release_write(volatile XCTYPE *addr, XCTYPE old_val, XCTYPE new_val) { return AO_XSIZE_fetch_compare_and_swap_release_write(addr, old_val, new_val) == old_val; } # define AO_HAVE_XSIZE_compare_and_swap_release_write #endif #if defined(AO_HAVE_XSIZE_fetch_compare_and_swap_acquire_read) \ && !defined(AO_HAVE_XSIZE_compare_and_swap_acquire_read) AO_INLINE int AO_XSIZE_compare_and_swap_acquire_read(volatile XCTYPE *addr, XCTYPE old_val, XCTYPE new_val) { return AO_XSIZE_fetch_compare_and_swap_acquire_read(addr, old_val, new_val) == old_val; } # define AO_HAVE_XSIZE_compare_and_swap_acquire_read #endif #if defined(AO_HAVE_XSIZE_fetch_compare_and_swap_dd_acquire_read) \ && !defined(AO_HAVE_XSIZE_compare_and_swap_dd_acquire_read) AO_INLINE int AO_XSIZE_compare_and_swap_dd_acquire_read(volatile XCTYPE *addr, XCTYPE old_val, XCTYPE new_val) { return AO_XSIZE_fetch_compare_and_swap_dd_acquire_read(addr, old_val, new_val) == old_val; } # define AO_HAVE_XSIZE_compare_and_swap_dd_acquire_read #endif /* XSIZE_fetch_and_add */ /* We first try to implement fetch_and_add variants in terms of the */ /* corresponding compare_and_swap variants to minimize adding barriers. */ #if defined(AO_HAVE_XSIZE_compare_and_swap_full) \ && !defined(AO_HAVE_XSIZE_fetch_and_add_full) AO_ATTR_NO_SANITIZE_THREAD AO_INLINE XCTYPE AO_XSIZE_fetch_and_add_full(volatile XCTYPE *addr, XCTYPE incr) { XCTYPE old; do { old = *(XCTYPE *)addr; } while (AO_EXPECT_FALSE(!AO_XSIZE_compare_and_swap_full(addr, old, old + incr))); return old; } # define AO_HAVE_XSIZE_fetch_and_add_full #endif #if defined(AO_HAVE_XSIZE_compare_and_swap_acquire) \ && !defined(AO_HAVE_XSIZE_fetch_and_add_acquire) AO_ATTR_NO_SANITIZE_THREAD AO_INLINE XCTYPE AO_XSIZE_fetch_and_add_acquire(volatile XCTYPE *addr, XCTYPE incr) { XCTYPE old; do { old = *(XCTYPE *)addr; } while (AO_EXPECT_FALSE(!AO_XSIZE_compare_and_swap_acquire(addr, old, old + incr))); return old; } # define AO_HAVE_XSIZE_fetch_and_add_acquire #endif #if defined(AO_HAVE_XSIZE_compare_and_swap_release) \ && !defined(AO_HAVE_XSIZE_fetch_and_add_release) AO_ATTR_NO_SANITIZE_THREAD AO_INLINE XCTYPE AO_XSIZE_fetch_and_add_release(volatile XCTYPE *addr, XCTYPE incr) { XCTYPE old; do { old = *(XCTYPE *)addr; } while (AO_EXPECT_FALSE(!AO_XSIZE_compare_and_swap_release(addr, old, old + incr))); return old; } # define AO_HAVE_XSIZE_fetch_and_add_release #endif #if defined(AO_HAVE_XSIZE_compare_and_swap) \ && !defined(AO_HAVE_XSIZE_fetch_and_add) AO_ATTR_NO_SANITIZE_THREAD AO_INLINE XCTYPE AO_XSIZE_fetch_and_add(volatile XCTYPE *addr, XCTYPE incr) { XCTYPE old; do { old = *(XCTYPE *)addr; } while (AO_EXPECT_FALSE(!AO_XSIZE_compare_and_swap(addr, old, old + incr))); return old; } # define AO_HAVE_XSIZE_fetch_and_add #endif #if defined(AO_HAVE_XSIZE_fetch_and_add_full) # if !defined(AO_HAVE_XSIZE_fetch_and_add_release) # define AO_XSIZE_fetch_and_add_release(addr, val) \ AO_XSIZE_fetch_and_add_full(addr, val) # define AO_HAVE_XSIZE_fetch_and_add_release # endif # if !defined(AO_HAVE_XSIZE_fetch_and_add_acquire) # define AO_XSIZE_fetch_and_add_acquire(addr, val) \ AO_XSIZE_fetch_and_add_full(addr, val) # define AO_HAVE_XSIZE_fetch_and_add_acquire # endif # if !defined(AO_HAVE_XSIZE_fetch_and_add_write) # define AO_XSIZE_fetch_and_add_write(addr, val) \ AO_XSIZE_fetch_and_add_full(addr, val) # define AO_HAVE_XSIZE_fetch_and_add_write # endif # if !defined(AO_HAVE_XSIZE_fetch_and_add_read) # define AO_XSIZE_fetch_and_add_read(addr, val) \ AO_XSIZE_fetch_and_add_full(addr, val) # define AO_HAVE_XSIZE_fetch_and_add_read # endif #endif /* AO_HAVE_XSIZE_fetch_and_add_full */ #if defined(AO_HAVE_XSIZE_fetch_and_add) && defined(AO_HAVE_nop_full) \ && !defined(AO_HAVE_XSIZE_fetch_and_add_acquire) AO_INLINE XCTYPE AO_XSIZE_fetch_and_add_acquire(volatile XCTYPE *addr, XCTYPE incr) { XCTYPE result = AO_XSIZE_fetch_and_add(addr, incr); AO_nop_full(); return result; } # define AO_HAVE_XSIZE_fetch_and_add_acquire #endif #if defined(AO_HAVE_XSIZE_fetch_and_add) && defined(AO_HAVE_nop_full) \ && !defined(AO_HAVE_XSIZE_fetch_and_add_release) # define AO_XSIZE_fetch_and_add_release(addr, incr) \ (AO_nop_full(), AO_XSIZE_fetch_and_add(addr, incr)) # define AO_HAVE_XSIZE_fetch_and_add_release #endif #if !defined(AO_HAVE_XSIZE_fetch_and_add) \ && defined(AO_HAVE_XSIZE_fetch_and_add_release) # define AO_XSIZE_fetch_and_add(addr, val) \ AO_XSIZE_fetch_and_add_release(addr, val) # define AO_HAVE_XSIZE_fetch_and_add #endif #if !defined(AO_HAVE_XSIZE_fetch_and_add) \ && defined(AO_HAVE_XSIZE_fetch_and_add_acquire) # define AO_XSIZE_fetch_and_add(addr, val) \ AO_XSIZE_fetch_and_add_acquire(addr, val) # define AO_HAVE_XSIZE_fetch_and_add #endif #if !defined(AO_HAVE_XSIZE_fetch_and_add) \ && defined(AO_HAVE_XSIZE_fetch_and_add_write) # define AO_XSIZE_fetch_and_add(addr, val) \ AO_XSIZE_fetch_and_add_write(addr, val) # define AO_HAVE_XSIZE_fetch_and_add #endif #if !defined(AO_HAVE_XSIZE_fetch_and_add) \ && defined(AO_HAVE_XSIZE_fetch_and_add_read) # define AO_XSIZE_fetch_and_add(addr, val) \ AO_XSIZE_fetch_and_add_read(addr, val) # define AO_HAVE_XSIZE_fetch_and_add #endif #if defined(AO_HAVE_XSIZE_fetch_and_add_acquire) \ && defined(AO_HAVE_nop_full) && !defined(AO_HAVE_XSIZE_fetch_and_add_full) # define AO_XSIZE_fetch_and_add_full(addr, val) \ (AO_nop_full(), AO_XSIZE_fetch_and_add_acquire(addr, val)) # define AO_HAVE_XSIZE_fetch_and_add_full #endif #if !defined(AO_HAVE_XSIZE_fetch_and_add_release_write) \ && defined(AO_HAVE_XSIZE_fetch_and_add_write) # define AO_XSIZE_fetch_and_add_release_write(addr, val) \ AO_XSIZE_fetch_and_add_write(addr, val) # define AO_HAVE_XSIZE_fetch_and_add_release_write #endif #if !defined(AO_HAVE_XSIZE_fetch_and_add_release_write) \ && defined(AO_HAVE_XSIZE_fetch_and_add_release) # define AO_XSIZE_fetch_and_add_release_write(addr, val) \ AO_XSIZE_fetch_and_add_release(addr, val) # define AO_HAVE_XSIZE_fetch_and_add_release_write #endif #if !defined(AO_HAVE_XSIZE_fetch_and_add_acquire_read) \ && defined(AO_HAVE_XSIZE_fetch_and_add_read) # define AO_XSIZE_fetch_and_add_acquire_read(addr, val) \ AO_XSIZE_fetch_and_add_read(addr, val) # define AO_HAVE_XSIZE_fetch_and_add_acquire_read #endif #if !defined(AO_HAVE_XSIZE_fetch_and_add_acquire_read) \ && defined(AO_HAVE_XSIZE_fetch_and_add_acquire) # define AO_XSIZE_fetch_and_add_acquire_read(addr, val) \ AO_XSIZE_fetch_and_add_acquire(addr, val) # define AO_HAVE_XSIZE_fetch_and_add_acquire_read #endif #ifdef AO_NO_DD_ORDERING # if defined(AO_HAVE_XSIZE_fetch_and_add_acquire_read) # define AO_XSIZE_fetch_and_add_dd_acquire_read(addr, val) \ AO_XSIZE_fetch_and_add_acquire_read(addr, val) # define AO_HAVE_XSIZE_fetch_and_add_dd_acquire_read # endif #else # if defined(AO_HAVE_XSIZE_fetch_and_add) # define AO_XSIZE_fetch_and_add_dd_acquire_read(addr, val) \ AO_XSIZE_fetch_and_add(addr, val) # define AO_HAVE_XSIZE_fetch_and_add_dd_acquire_read # endif #endif /* !AO_NO_DD_ORDERING */ /* XSIZE_fetch_and_add1 */ #if defined(AO_HAVE_XSIZE_fetch_and_add_full) \ && !defined(AO_HAVE_XSIZE_fetch_and_add1_full) # define AO_XSIZE_fetch_and_add1_full(addr) \ AO_XSIZE_fetch_and_add_full(addr, 1) # define AO_HAVE_XSIZE_fetch_and_add1_full #endif #if defined(AO_HAVE_XSIZE_fetch_and_add_release) \ && !defined(AO_HAVE_XSIZE_fetch_and_add1_release) # define AO_XSIZE_fetch_and_add1_release(addr) \ AO_XSIZE_fetch_and_add_release(addr, 1) # define AO_HAVE_XSIZE_fetch_and_add1_release #endif #if defined(AO_HAVE_XSIZE_fetch_and_add_acquire) \ && !defined(AO_HAVE_XSIZE_fetch_and_add1_acquire) # define AO_XSIZE_fetch_and_add1_acquire(addr) \ AO_XSIZE_fetch_and_add_acquire(addr, 1) # define AO_HAVE_XSIZE_fetch_and_add1_acquire #endif #if defined(AO_HAVE_XSIZE_fetch_and_add_write) \ && !defined(AO_HAVE_XSIZE_fetch_and_add1_write) # define AO_XSIZE_fetch_and_add1_write(addr) \ AO_XSIZE_fetch_and_add_write(addr, 1) # define AO_HAVE_XSIZE_fetch_and_add1_write #endif #if defined(AO_HAVE_XSIZE_fetch_and_add_read) \ && !defined(AO_HAVE_XSIZE_fetch_and_add1_read) # define AO_XSIZE_fetch_and_add1_read(addr) \ AO_XSIZE_fetch_and_add_read(addr, 1) # define AO_HAVE_XSIZE_fetch_and_add1_read #endif #if defined(AO_HAVE_XSIZE_fetch_and_add_release_write) \ && !defined(AO_HAVE_XSIZE_fetch_and_add1_release_write) # define AO_XSIZE_fetch_and_add1_release_write(addr) \ AO_XSIZE_fetch_and_add_release_write(addr, 1) # define AO_HAVE_XSIZE_fetch_and_add1_release_write #endif #if defined(AO_HAVE_XSIZE_fetch_and_add_acquire_read) \ && !defined(AO_HAVE_XSIZE_fetch_and_add1_acquire_read) # define AO_XSIZE_fetch_and_add1_acquire_read(addr) \ AO_XSIZE_fetch_and_add_acquire_read(addr, 1) # define AO_HAVE_XSIZE_fetch_and_add1_acquire_read #endif #if defined(AO_HAVE_XSIZE_fetch_and_add) \ && !defined(AO_HAVE_XSIZE_fetch_and_add1) # define AO_XSIZE_fetch_and_add1(addr) AO_XSIZE_fetch_and_add(addr, 1) # define AO_HAVE_XSIZE_fetch_and_add1 #endif #if defined(AO_HAVE_XSIZE_fetch_and_add1_full) # if !defined(AO_HAVE_XSIZE_fetch_and_add1_release) # define AO_XSIZE_fetch_and_add1_release(addr) \ AO_XSIZE_fetch_and_add1_full(addr) # define AO_HAVE_XSIZE_fetch_and_add1_release # endif # if !defined(AO_HAVE_XSIZE_fetch_and_add1_acquire) # define AO_XSIZE_fetch_and_add1_acquire(addr) \ AO_XSIZE_fetch_and_add1_full(addr) # define AO_HAVE_XSIZE_fetch_and_add1_acquire # endif # if !defined(AO_HAVE_XSIZE_fetch_and_add1_write) # define AO_XSIZE_fetch_and_add1_write(addr) \ AO_XSIZE_fetch_and_add1_full(addr) # define AO_HAVE_XSIZE_fetch_and_add1_write # endif # if !defined(AO_HAVE_XSIZE_fetch_and_add1_read) # define AO_XSIZE_fetch_and_add1_read(addr) \ AO_XSIZE_fetch_and_add1_full(addr) # define AO_HAVE_XSIZE_fetch_and_add1_read # endif #endif /* AO_HAVE_XSIZE_fetch_and_add1_full */ #if !defined(AO_HAVE_XSIZE_fetch_and_add1) \ && defined(AO_HAVE_XSIZE_fetch_and_add1_release) # define AO_XSIZE_fetch_and_add1(addr) AO_XSIZE_fetch_and_add1_release(addr) # define AO_HAVE_XSIZE_fetch_and_add1 #endif #if !defined(AO_HAVE_XSIZE_fetch_and_add1) \ && defined(AO_HAVE_XSIZE_fetch_and_add1_acquire) # define AO_XSIZE_fetch_and_add1(addr) AO_XSIZE_fetch_and_add1_acquire(addr) # define AO_HAVE_XSIZE_fetch_and_add1 #endif #if !defined(AO_HAVE_XSIZE_fetch_and_add1) \ && defined(AO_HAVE_XSIZE_fetch_and_add1_write) # define AO_XSIZE_fetch_and_add1(addr) AO_XSIZE_fetch_and_add1_write(addr) # define AO_HAVE_XSIZE_fetch_and_add1 #endif #if !defined(AO_HAVE_XSIZE_fetch_and_add1) \ && defined(AO_HAVE_XSIZE_fetch_and_add1_read) # define AO_XSIZE_fetch_and_add1(addr) AO_XSIZE_fetch_and_add1_read(addr) # define AO_HAVE_XSIZE_fetch_and_add1 #endif #if defined(AO_HAVE_XSIZE_fetch_and_add1_acquire) \ && defined(AO_HAVE_nop_full) \ && !defined(AO_HAVE_XSIZE_fetch_and_add1_full) # define AO_XSIZE_fetch_and_add1_full(addr) \ (AO_nop_full(), AO_XSIZE_fetch_and_add1_acquire(addr)) # define AO_HAVE_XSIZE_fetch_and_add1_full #endif #if !defined(AO_HAVE_XSIZE_fetch_and_add1_release_write) \ && defined(AO_HAVE_XSIZE_fetch_and_add1_write) # define AO_XSIZE_fetch_and_add1_release_write(addr) \ AO_XSIZE_fetch_and_add1_write(addr) # define AO_HAVE_XSIZE_fetch_and_add1_release_write #endif #if !defined(AO_HAVE_XSIZE_fetch_and_add1_release_write) \ && defined(AO_HAVE_XSIZE_fetch_and_add1_release) # define AO_XSIZE_fetch_and_add1_release_write(addr) \ AO_XSIZE_fetch_and_add1_release(addr) # define AO_HAVE_XSIZE_fetch_and_add1_release_write #endif #if !defined(AO_HAVE_XSIZE_fetch_and_add1_acquire_read) \ && defined(AO_HAVE_XSIZE_fetch_and_add1_read) # define AO_XSIZE_fetch_and_add1_acquire_read(addr) \ AO_XSIZE_fetch_and_add1_read(addr) # define AO_HAVE_XSIZE_fetch_and_add1_acquire_read #endif #if !defined(AO_HAVE_XSIZE_fetch_and_add1_acquire_read) \ && defined(AO_HAVE_XSIZE_fetch_and_add1_acquire) # define AO_XSIZE_fetch_and_add1_acquire_read(addr) \ AO_XSIZE_fetch_and_add1_acquire(addr) # define AO_HAVE_XSIZE_fetch_and_add1_acquire_read #endif #ifdef AO_NO_DD_ORDERING # if defined(AO_HAVE_XSIZE_fetch_and_add1_acquire_read) # define AO_XSIZE_fetch_and_add1_dd_acquire_read(addr) \ AO_XSIZE_fetch_and_add1_acquire_read(addr) # define AO_HAVE_XSIZE_fetch_and_add1_dd_acquire_read # endif #else # if defined(AO_HAVE_XSIZE_fetch_and_add1) # define AO_XSIZE_fetch_and_add1_dd_acquire_read(addr) \ AO_XSIZE_fetch_and_add1(addr) # define AO_HAVE_XSIZE_fetch_and_add1_dd_acquire_read # endif #endif /* !AO_NO_DD_ORDERING */ /* XSIZE_fetch_and_sub1 */ #if defined(AO_HAVE_XSIZE_fetch_and_add_full) \ && !defined(AO_HAVE_XSIZE_fetch_and_sub1_full) # define AO_XSIZE_fetch_and_sub1_full(addr) \ AO_XSIZE_fetch_and_add_full(addr, (XCTYPE)(-1)) # define AO_HAVE_XSIZE_fetch_and_sub1_full #endif #if defined(AO_HAVE_XSIZE_fetch_and_add_release) \ && !defined(AO_HAVE_XSIZE_fetch_and_sub1_release) # define AO_XSIZE_fetch_and_sub1_release(addr) \ AO_XSIZE_fetch_and_add_release(addr, (XCTYPE)(-1)) # define AO_HAVE_XSIZE_fetch_and_sub1_release #endif #if defined(AO_HAVE_XSIZE_fetch_and_add_acquire) \ && !defined(AO_HAVE_XSIZE_fetch_and_sub1_acquire) # define AO_XSIZE_fetch_and_sub1_acquire(addr) \ AO_XSIZE_fetch_and_add_acquire(addr, (XCTYPE)(-1)) # define AO_HAVE_XSIZE_fetch_and_sub1_acquire #endif #if defined(AO_HAVE_XSIZE_fetch_and_add_write) \ && !defined(AO_HAVE_XSIZE_fetch_and_sub1_write) # define AO_XSIZE_fetch_and_sub1_write(addr) \ AO_XSIZE_fetch_and_add_write(addr, (XCTYPE)(-1)) # define AO_HAVE_XSIZE_fetch_and_sub1_write #endif #if defined(AO_HAVE_XSIZE_fetch_and_add_read) \ && !defined(AO_HAVE_XSIZE_fetch_and_sub1_read) # define AO_XSIZE_fetch_and_sub1_read(addr) \ AO_XSIZE_fetch_and_add_read(addr, (XCTYPE)(-1)) # define AO_HAVE_XSIZE_fetch_and_sub1_read #endif #if defined(AO_HAVE_XSIZE_fetch_and_add_release_write) \ && !defined(AO_HAVE_XSIZE_fetch_and_sub1_release_write) # define AO_XSIZE_fetch_and_sub1_release_write(addr) \ AO_XSIZE_fetch_and_add_release_write(addr, (XCTYPE)(-1)) # define AO_HAVE_XSIZE_fetch_and_sub1_release_write #endif #if defined(AO_HAVE_XSIZE_fetch_and_add_acquire_read) \ && !defined(AO_HAVE_XSIZE_fetch_and_sub1_acquire_read) # define AO_XSIZE_fetch_and_sub1_acquire_read(addr) \ AO_XSIZE_fetch_and_add_acquire_read(addr, (XCTYPE)(-1)) # define AO_HAVE_XSIZE_fetch_and_sub1_acquire_read #endif #if defined(AO_HAVE_XSIZE_fetch_and_add) \ && !defined(AO_HAVE_XSIZE_fetch_and_sub1) # define AO_XSIZE_fetch_and_sub1(addr) \ AO_XSIZE_fetch_and_add(addr, (XCTYPE)(-1)) # define AO_HAVE_XSIZE_fetch_and_sub1 #endif #if defined(AO_HAVE_XSIZE_fetch_and_sub1_full) # if !defined(AO_HAVE_XSIZE_fetch_and_sub1_release) # define AO_XSIZE_fetch_and_sub1_release(addr) \ AO_XSIZE_fetch_and_sub1_full(addr) # define AO_HAVE_XSIZE_fetch_and_sub1_release # endif # if !defined(AO_HAVE_XSIZE_fetch_and_sub1_acquire) # define AO_XSIZE_fetch_and_sub1_acquire(addr) \ AO_XSIZE_fetch_and_sub1_full(addr) # define AO_HAVE_XSIZE_fetch_and_sub1_acquire # endif # if !defined(AO_HAVE_XSIZE_fetch_and_sub1_write) # define AO_XSIZE_fetch_and_sub1_write(addr) \ AO_XSIZE_fetch_and_sub1_full(addr) # define AO_HAVE_XSIZE_fetch_and_sub1_write # endif # if !defined(AO_HAVE_XSIZE_fetch_and_sub1_read) # define AO_XSIZE_fetch_and_sub1_read(addr) \ AO_XSIZE_fetch_and_sub1_full(addr) # define AO_HAVE_XSIZE_fetch_and_sub1_read # endif #endif /* AO_HAVE_XSIZE_fetch_and_sub1_full */ #if !defined(AO_HAVE_XSIZE_fetch_and_sub1) \ && defined(AO_HAVE_XSIZE_fetch_and_sub1_release) # define AO_XSIZE_fetch_and_sub1(addr) AO_XSIZE_fetch_and_sub1_release(addr) # define AO_HAVE_XSIZE_fetch_and_sub1 #endif #if !defined(AO_HAVE_XSIZE_fetch_and_sub1) \ && defined(AO_HAVE_XSIZE_fetch_and_sub1_acquire) # define AO_XSIZE_fetch_and_sub1(addr) AO_XSIZE_fetch_and_sub1_acquire(addr) # define AO_HAVE_XSIZE_fetch_and_sub1 #endif #if !defined(AO_HAVE_XSIZE_fetch_and_sub1) \ && defined(AO_HAVE_XSIZE_fetch_and_sub1_write) # define AO_XSIZE_fetch_and_sub1(addr) AO_XSIZE_fetch_and_sub1_write(addr) # define AO_HAVE_XSIZE_fetch_and_sub1 #endif #if !defined(AO_HAVE_XSIZE_fetch_and_sub1) \ && defined(AO_HAVE_XSIZE_fetch_and_sub1_read) # define AO_XSIZE_fetch_and_sub1(addr) AO_XSIZE_fetch_and_sub1_read(addr) # define AO_HAVE_XSIZE_fetch_and_sub1 #endif #if defined(AO_HAVE_XSIZE_fetch_and_sub1_acquire) \ && defined(AO_HAVE_nop_full) \ && !defined(AO_HAVE_XSIZE_fetch_and_sub1_full) # define AO_XSIZE_fetch_and_sub1_full(addr) \ (AO_nop_full(), AO_XSIZE_fetch_and_sub1_acquire(addr)) # define AO_HAVE_XSIZE_fetch_and_sub1_full #endif #if !defined(AO_HAVE_XSIZE_fetch_and_sub1_release_write) \ && defined(AO_HAVE_XSIZE_fetch_and_sub1_write) # define AO_XSIZE_fetch_and_sub1_release_write(addr) \ AO_XSIZE_fetch_and_sub1_write(addr) # define AO_HAVE_XSIZE_fetch_and_sub1_release_write #endif #if !defined(AO_HAVE_XSIZE_fetch_and_sub1_release_write) \ && defined(AO_HAVE_XSIZE_fetch_and_sub1_release) # define AO_XSIZE_fetch_and_sub1_release_write(addr) \ AO_XSIZE_fetch_and_sub1_release(addr) # define AO_HAVE_XSIZE_fetch_and_sub1_release_write #endif #if !defined(AO_HAVE_XSIZE_fetch_and_sub1_acquire_read) \ && defined(AO_HAVE_XSIZE_fetch_and_sub1_read) # define AO_XSIZE_fetch_and_sub1_acquire_read(addr) \ AO_XSIZE_fetch_and_sub1_read(addr) # define AO_HAVE_XSIZE_fetch_and_sub1_acquire_read #endif #if !defined(AO_HAVE_XSIZE_fetch_and_sub1_acquire_read) \ && defined(AO_HAVE_XSIZE_fetch_and_sub1_acquire) # define AO_XSIZE_fetch_and_sub1_acquire_read(addr) \ AO_XSIZE_fetch_and_sub1_acquire(addr) # define AO_HAVE_XSIZE_fetch_and_sub1_acquire_read #endif #ifdef AO_NO_DD_ORDERING # if defined(AO_HAVE_XSIZE_fetch_and_sub1_acquire_read) # define AO_XSIZE_fetch_and_sub1_dd_acquire_read(addr) \ AO_XSIZE_fetch_and_sub1_acquire_read(addr) # define AO_HAVE_XSIZE_fetch_and_sub1_dd_acquire_read # endif #else # if defined(AO_HAVE_XSIZE_fetch_and_sub1) # define AO_XSIZE_fetch_and_sub1_dd_acquire_read(addr) \ AO_XSIZE_fetch_and_sub1(addr) # define AO_HAVE_XSIZE_fetch_and_sub1_dd_acquire_read # endif #endif /* !AO_NO_DD_ORDERING */ /* XSIZE_and */ #if defined(AO_HAVE_XSIZE_compare_and_swap_full) \ && !defined(AO_HAVE_XSIZE_and_full) AO_ATTR_NO_SANITIZE_THREAD AO_INLINE void AO_XSIZE_and_full(volatile XCTYPE *addr, XCTYPE value) { XCTYPE old; do { old = *(XCTYPE *)addr; } while (AO_EXPECT_FALSE(!AO_XSIZE_compare_and_swap_full(addr, old, old & value))); } # define AO_HAVE_XSIZE_and_full #endif #if defined(AO_HAVE_XSIZE_and_full) # if !defined(AO_HAVE_XSIZE_and_release) # define AO_XSIZE_and_release(addr, val) AO_XSIZE_and_full(addr, val) # define AO_HAVE_XSIZE_and_release # endif # if !defined(AO_HAVE_XSIZE_and_acquire) # define AO_XSIZE_and_acquire(addr, val) AO_XSIZE_and_full(addr, val) # define AO_HAVE_XSIZE_and_acquire # endif # if !defined(AO_HAVE_XSIZE_and_write) # define AO_XSIZE_and_write(addr, val) AO_XSIZE_and_full(addr, val) # define AO_HAVE_XSIZE_and_write # endif # if !defined(AO_HAVE_XSIZE_and_read) # define AO_XSIZE_and_read(addr, val) AO_XSIZE_and_full(addr, val) # define AO_HAVE_XSIZE_and_read # endif #endif /* AO_HAVE_XSIZE_and_full */ #if !defined(AO_HAVE_XSIZE_and) && defined(AO_HAVE_XSIZE_and_release) # define AO_XSIZE_and(addr, val) AO_XSIZE_and_release(addr, val) # define AO_HAVE_XSIZE_and #endif #if !defined(AO_HAVE_XSIZE_and) && defined(AO_HAVE_XSIZE_and_acquire) # define AO_XSIZE_and(addr, val) AO_XSIZE_and_acquire(addr, val) # define AO_HAVE_XSIZE_and #endif #if !defined(AO_HAVE_XSIZE_and) && defined(AO_HAVE_XSIZE_and_write) # define AO_XSIZE_and(addr, val) AO_XSIZE_and_write(addr, val) # define AO_HAVE_XSIZE_and #endif #if !defined(AO_HAVE_XSIZE_and) && defined(AO_HAVE_XSIZE_and_read) # define AO_XSIZE_and(addr, val) AO_XSIZE_and_read(addr, val) # define AO_HAVE_XSIZE_and #endif #if defined(AO_HAVE_XSIZE_and_acquire) && defined(AO_HAVE_nop_full) \ && !defined(AO_HAVE_XSIZE_and_full) # define AO_XSIZE_and_full(addr, val) \ (AO_nop_full(), AO_XSIZE_and_acquire(addr, val)) # define AO_HAVE_XSIZE_and_full #endif #if !defined(AO_HAVE_XSIZE_and_release_write) \ && defined(AO_HAVE_XSIZE_and_write) # define AO_XSIZE_and_release_write(addr, val) AO_XSIZE_and_write(addr, val) # define AO_HAVE_XSIZE_and_release_write #endif #if !defined(AO_HAVE_XSIZE_and_release_write) \ && defined(AO_HAVE_XSIZE_and_release) # define AO_XSIZE_and_release_write(addr, val) AO_XSIZE_and_release(addr, val) # define AO_HAVE_XSIZE_and_release_write #endif #if !defined(AO_HAVE_XSIZE_and_acquire_read) \ && defined(AO_HAVE_XSIZE_and_read) # define AO_XSIZE_and_acquire_read(addr, val) AO_XSIZE_and_read(addr, val) # define AO_HAVE_XSIZE_and_acquire_read #endif #if !defined(AO_HAVE_XSIZE_and_acquire_read) \ && defined(AO_HAVE_XSIZE_and_acquire) # define AO_XSIZE_and_acquire_read(addr, val) AO_XSIZE_and_acquire(addr, val) # define AO_HAVE_XSIZE_and_acquire_read #endif /* XSIZE_or */ #if defined(AO_HAVE_XSIZE_compare_and_swap_full) \ && !defined(AO_HAVE_XSIZE_or_full) AO_ATTR_NO_SANITIZE_THREAD AO_INLINE void AO_XSIZE_or_full(volatile XCTYPE *addr, XCTYPE value) { XCTYPE old; do { old = *(XCTYPE *)addr; } while (AO_EXPECT_FALSE(!AO_XSIZE_compare_and_swap_full(addr, old, old | value))); } # define AO_HAVE_XSIZE_or_full #endif #if defined(AO_HAVE_XSIZE_or_full) # if !defined(AO_HAVE_XSIZE_or_release) # define AO_XSIZE_or_release(addr, val) AO_XSIZE_or_full(addr, val) # define AO_HAVE_XSIZE_or_release # endif # if !defined(AO_HAVE_XSIZE_or_acquire) # define AO_XSIZE_or_acquire(addr, val) AO_XSIZE_or_full(addr, val) # define AO_HAVE_XSIZE_or_acquire # endif # if !defined(AO_HAVE_XSIZE_or_write) # define AO_XSIZE_or_write(addr, val) AO_XSIZE_or_full(addr, val) # define AO_HAVE_XSIZE_or_write # endif # if !defined(AO_HAVE_XSIZE_or_read) # define AO_XSIZE_or_read(addr, val) AO_XSIZE_or_full(addr, val) # define AO_HAVE_XSIZE_or_read # endif #endif /* AO_HAVE_XSIZE_or_full */ #if !defined(AO_HAVE_XSIZE_or) && defined(AO_HAVE_XSIZE_or_release) # define AO_XSIZE_or(addr, val) AO_XSIZE_or_release(addr, val) # define AO_HAVE_XSIZE_or #endif #if !defined(AO_HAVE_XSIZE_or) && defined(AO_HAVE_XSIZE_or_acquire) # define AO_XSIZE_or(addr, val) AO_XSIZE_or_acquire(addr, val) # define AO_HAVE_XSIZE_or #endif #if !defined(AO_HAVE_XSIZE_or) && defined(AO_HAVE_XSIZE_or_write) # define AO_XSIZE_or(addr, val) AO_XSIZE_or_write(addr, val) # define AO_HAVE_XSIZE_or #endif #if !defined(AO_HAVE_XSIZE_or) && defined(AO_HAVE_XSIZE_or_read) # define AO_XSIZE_or(addr, val) AO_XSIZE_or_read(addr, val) # define AO_HAVE_XSIZE_or #endif #if defined(AO_HAVE_XSIZE_or_acquire) && defined(AO_HAVE_nop_full) \ && !defined(AO_HAVE_XSIZE_or_full) # define AO_XSIZE_or_full(addr, val) \ (AO_nop_full(), AO_XSIZE_or_acquire(addr, val)) # define AO_HAVE_XSIZE_or_full #endif #if !defined(AO_HAVE_XSIZE_or_release_write) \ && defined(AO_HAVE_XSIZE_or_write) # define AO_XSIZE_or_release_write(addr, val) AO_XSIZE_or_write(addr, val) # define AO_HAVE_XSIZE_or_release_write #endif #if !defined(AO_HAVE_XSIZE_or_release_write) \ && defined(AO_HAVE_XSIZE_or_release) # define AO_XSIZE_or_release_write(addr, val) AO_XSIZE_or_release(addr, val) # define AO_HAVE_XSIZE_or_release_write #endif #if !defined(AO_HAVE_XSIZE_or_acquire_read) && defined(AO_HAVE_XSIZE_or_read) # define AO_XSIZE_or_acquire_read(addr, val) AO_XSIZE_or_read(addr, val) # define AO_HAVE_XSIZE_or_acquire_read #endif #if !defined(AO_HAVE_XSIZE_or_acquire_read) \ && defined(AO_HAVE_XSIZE_or_acquire) # define AO_XSIZE_or_acquire_read(addr, val) AO_XSIZE_or_acquire(addr, val) # define AO_HAVE_XSIZE_or_acquire_read #endif /* XSIZE_xor */ #if defined(AO_HAVE_XSIZE_compare_and_swap_full) \ && !defined(AO_HAVE_XSIZE_xor_full) AO_ATTR_NO_SANITIZE_THREAD AO_INLINE void AO_XSIZE_xor_full(volatile XCTYPE *addr, XCTYPE value) { XCTYPE old; do { old = *(XCTYPE *)addr; } while (AO_EXPECT_FALSE(!AO_XSIZE_compare_and_swap_full(addr, old, old ^ value))); } # define AO_HAVE_XSIZE_xor_full #endif #if defined(AO_HAVE_XSIZE_xor_full) # if !defined(AO_HAVE_XSIZE_xor_release) # define AO_XSIZE_xor_release(addr, val) AO_XSIZE_xor_full(addr, val) # define AO_HAVE_XSIZE_xor_release # endif # if !defined(AO_HAVE_XSIZE_xor_acquire) # define AO_XSIZE_xor_acquire(addr, val) AO_XSIZE_xor_full(addr, val) # define AO_HAVE_XSIZE_xor_acquire # endif # if !defined(AO_HAVE_XSIZE_xor_write) # define AO_XSIZE_xor_write(addr, val) AO_XSIZE_xor_full(addr, val) # define AO_HAVE_XSIZE_xor_write # endif # if !defined(AO_HAVE_XSIZE_xor_read) # define AO_XSIZE_xor_read(addr, val) AO_XSIZE_xor_full(addr, val) # define AO_HAVE_XSIZE_xor_read # endif #endif /* AO_HAVE_XSIZE_xor_full */ #if !defined(AO_HAVE_XSIZE_xor) && defined(AO_HAVE_XSIZE_xor_release) # define AO_XSIZE_xor(addr, val) AO_XSIZE_xor_release(addr, val) # define AO_HAVE_XSIZE_xor #endif #if !defined(AO_HAVE_XSIZE_xor) && defined(AO_HAVE_XSIZE_xor_acquire) # define AO_XSIZE_xor(addr, val) AO_XSIZE_xor_acquire(addr, val) # define AO_HAVE_XSIZE_xor #endif #if !defined(AO_HAVE_XSIZE_xor) && defined(AO_HAVE_XSIZE_xor_write) # define AO_XSIZE_xor(addr, val) AO_XSIZE_xor_write(addr, val) # define AO_HAVE_XSIZE_xor #endif #if !defined(AO_HAVE_XSIZE_xor) && defined(AO_HAVE_XSIZE_xor_read) # define AO_XSIZE_xor(addr, val) AO_XSIZE_xor_read(addr, val) # define AO_HAVE_XSIZE_xor #endif #if defined(AO_HAVE_XSIZE_xor_acquire) && defined(AO_HAVE_nop_full) \ && !defined(AO_HAVE_XSIZE_xor_full) # define AO_XSIZE_xor_full(addr, val) \ (AO_nop_full(), AO_XSIZE_xor_acquire(addr, val)) # define AO_HAVE_XSIZE_xor_full #endif #if !defined(AO_HAVE_XSIZE_xor_release_write) \ && defined(AO_HAVE_XSIZE_xor_write) # define AO_XSIZE_xor_release_write(addr, val) AO_XSIZE_xor_write(addr, val) # define AO_HAVE_XSIZE_xor_release_write #endif #if !defined(AO_HAVE_XSIZE_xor_release_write) \ && defined(AO_HAVE_XSIZE_xor_release) # define AO_XSIZE_xor_release_write(addr, val) AO_XSIZE_xor_release(addr, val) # define AO_HAVE_XSIZE_xor_release_write #endif #if !defined(AO_HAVE_XSIZE_xor_acquire_read) \ && defined(AO_HAVE_XSIZE_xor_read) # define AO_XSIZE_xor_acquire_read(addr, val) AO_XSIZE_xor_read(addr, val) # define AO_HAVE_XSIZE_xor_acquire_read #endif #if !defined(AO_HAVE_XSIZE_xor_acquire_read) \ && defined(AO_HAVE_XSIZE_xor_acquire) # define AO_XSIZE_xor_acquire_read(addr, val) AO_XSIZE_xor_acquire(addr, val) # define AO_HAVE_XSIZE_xor_acquire_read #endif /* XSIZE_and/or/xor_dd_acquire_read are meaningless. */