From 0579177988589fafe1a11adf5c85dc49c5b26162 Mon Sep 17 00:00:00 2001 From: Hugh Bellamy Date: Sun, 15 Jan 2017 13:16:58 +0000 Subject: [PATCH] Extract RefCount implementation from header to cpp file --- stdlib/public/SwiftShims/RefCount.h | 297 +++--------------------- stdlib/public/runtime/CMakeLists.txt | 1 + stdlib/public/runtime/RefCount.cpp | 330 +++++++++++++++++++++++++++ 3 files changed, 357 insertions(+), 271 deletions(-) create mode 100644 stdlib/public/runtime/RefCount.cpp diff --git a/stdlib/public/SwiftShims/RefCount.h b/stdlib/public/SwiftShims/RefCount.h index 5f3ddca5cf8d7..0e98c8012ee4a 100644 --- a/stdlib/public/SwiftShims/RefCount.h +++ b/stdlib/public/SwiftShims/RefCount.h @@ -91,26 +91,14 @@ class StrongRefCount { } // Increment the reference count. - void increment() { - __atomic_fetch_add(&refCount, RC_ONE, __ATOMIC_RELAXED); - } + void increment(); - void incrementNonAtomic() { - uint32_t val = __atomic_load_n(&refCount, __ATOMIC_RELAXED); - val += RC_ONE; - __atomic_store_n(&refCount, val, __ATOMIC_RELAXED); - } + void incrementNonAtomic(); // Increment the reference count by n. - void increment(uint32_t n) { - __atomic_fetch_add(&refCount, n << RC_FLAGS_COUNT, __ATOMIC_RELAXED); - } + void increment(uint32_t n); - void incrementNonAtomic(uint32_t n) { - uint32_t val = __atomic_load_n(&refCount, __ATOMIC_RELAXED); - val += n << RC_FLAGS_COUNT; - __atomic_store_n(&refCount, val, __ATOMIC_RELAXED); - } + void incrementNonAtomic(uint32_t n); // Try to simultaneously set the pinned flag and increment the // reference count. If the flag is already set, don't increment the @@ -121,93 +109,39 @@ class StrongRefCount { // Returns true if the flag was set by this operation. // // Postcondition: the flag is set. - bool tryIncrementAndPin() { - uint32_t oldval = __atomic_load_n(&refCount, __ATOMIC_RELAXED); - while (true) { - // If the flag is already set, just fail. - if (oldval & RC_PINNED_FLAG) { - return false; - } - - // Try to simultaneously set the flag and increment the reference count. - uint32_t newval = oldval + (RC_PINNED_FLAG + RC_ONE); - if (__atomic_compare_exchange(&refCount, &oldval, &newval, 0, - __ATOMIC_RELAXED, __ATOMIC_RELAXED)) { - return true; - } - - // Try again; oldval has been updated with the value we saw. - } - } + bool tryIncrementAndPin(); - bool tryIncrementAndPinNonAtomic() { - uint32_t oldval = __atomic_load_n(&refCount, __ATOMIC_RELAXED); - // If the flag is already set, just fail. - if (oldval & RC_PINNED_FLAG) { - return false; - } - - // Try to simultaneously set the flag and increment the reference count. - uint32_t newval = oldval + (RC_PINNED_FLAG + RC_ONE); - __atomic_store_n(&refCount, newval, __ATOMIC_RELAXED); - return true; - } + bool tryIncrementAndPinNonAtomic(); // Increment the reference count, unless the object is deallocating. - bool tryIncrement() { - // FIXME: this could be better on LL/SC architectures like arm64 - uint32_t oldval = __atomic_fetch_add(&refCount, RC_ONE, __ATOMIC_RELAXED); - if (oldval & RC_DEALLOCATING_FLAG) { - __atomic_fetch_sub(&refCount, RC_ONE, __ATOMIC_RELAXED); - return false; - } else { - return true; - } - } + bool tryIncrement(); // Simultaneously clear the pinned flag and decrement the reference // count. // // Precondition: the pinned flag is set. - bool decrementAndUnpinShouldDeallocate() { - return doDecrementShouldDeallocate(); - } + bool decrementAndUnpinShouldDeallocate(); - bool decrementAndUnpinShouldDeallocateNonAtomic() { - return doDecrementShouldDeallocateNonAtomic(); - } + bool decrementAndUnpinShouldDeallocateNonAtomic(); // Decrement the reference count. // Return true if the caller should now deallocate the object. - bool decrementShouldDeallocate() { - return doDecrementShouldDeallocate(); - } + bool decrementShouldDeallocate(); - bool decrementShouldDeallocateNonAtomic() { - return doDecrementShouldDeallocateNonAtomic(); - } + bool decrementShouldDeallocateNonAtomic(); - bool decrementShouldDeallocateN(uint32_t n) { - return doDecrementShouldDeallocateN(n); - } + bool decrementShouldDeallocateN(uint32_t n); // Set the RC_DEALLOCATING_FLAG flag non-atomically. // // Precondition: the reference count must be 1 - void decrementFromOneAndDeallocateNonAtomic() { - assert(refCount == RC_ONE && "Expect a count of 1"); - __atomic_store_n(&refCount, RC_DEALLOCATING_FLAG, __ATOMIC_RELAXED); - } + void decrementFromOneAndDeallocateNonAtomic(); - bool decrementShouldDeallocateNNonAtomic(uint32_t n) { - return doDecrementShouldDeallocateNNonAtomic(n); - } + bool decrementShouldDeallocateNNonAtomic(uint32_t n); // Return the reference count. // During deallocation the reference count is undefined. - uint32_t getCount() const { - return __atomic_load_n(&refCount, __ATOMIC_RELAXED) >> RC_FLAGS_COUNT; - } + uint32_t getCount() const; // Return whether the reference count is exactly 1. // During deallocation the reference count is undefined. @@ -217,178 +151,23 @@ class StrongRefCount { // Return whether the reference count is exactly 1 or the pin flag // is set. During deallocation the reference count is undefined. - bool isUniquelyReferencedOrPinned() const { - auto value = __atomic_load_n(&refCount, __ATOMIC_RELAXED); - // Rotating right by one sets the sign bit to the pinned bit. After - // rotation, the dealloc flag is the least significant bit followed by the - // reference count. A reference count of two or higher means that our value - // is bigger than 3 if the pinned bit is not set. If the pinned bit is set - // the value is negative. - // Note: Because we are using the sign bit for testing pinnedness it - // is important to do a signed comparison below. - static_assert(RC_PINNED_FLAG == 1, - "The pinned flag must be the lowest bit"); - auto rotateRightByOne = ((value >> 1) | (value << 31)); - return (int32_t)rotateRightByOne < (int32_t)RC_ONE; - } + bool isUniquelyReferencedOrPinned() const; // Return true if the object is inside deallocation. - bool isDeallocating() const { - return __atomic_load_n(&refCount, __ATOMIC_RELAXED) & RC_DEALLOCATING_FLAG; - } + bool isDeallocating() const; private: template - bool doDecrementShouldDeallocate() { - // If we're being asked to clear the pinned flag, we can assume - // it's already set. - constexpr uint32_t quantum = - (ClearPinnedFlag ? RC_ONE + RC_PINNED_FLAG : RC_ONE); - uint32_t newval = __atomic_sub_fetch(&refCount, quantum, __ATOMIC_RELEASE); - - assert((!ClearPinnedFlag || !(newval & RC_PINNED_FLAG)) && - "unpinning reference that was not pinned"); - assert(newval + quantum >= RC_ONE && - "releasing reference with a refcount of zero"); - - // If we didn't drop the reference count to zero, or if the - // deallocating flag is already set, we're done; don't start - // deallocation. We can assume that the pinned flag isn't set - // unless the refcount is nonzero, and or'ing it in gives us a - // more efficient mask: the check just becomes "is newval nonzero". - if ((newval & (RC_COUNT_MASK | RC_PINNED_FLAG | RC_DEALLOCATING_FLAG)) - != 0) { - // Refcount is not zero. We definitely do not need to deallocate. - return false; - } - - // Refcount is now 0 and is not already deallocating. Try to set - // the deallocating flag. This must be atomic because it can race - // with weak retains. - // - // This also performs the before-deinit acquire barrier if we set the flag. - static_assert(RC_FLAGS_COUNT == 2, - "fix decrementShouldDeallocate() if you add more flags"); - uint32_t oldval = 0; - newval = RC_DEALLOCATING_FLAG; - return __atomic_compare_exchange(&refCount, &oldval, &newval, 0, - __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); - } + bool doDecrementShouldDeallocate(); template - bool doDecrementShouldDeallocateNonAtomic() { - // If we're being asked to clear the pinned flag, we can assume - // it's already set. - constexpr uint32_t quantum = - (ClearPinnedFlag ? RC_ONE + RC_PINNED_FLAG : RC_ONE); - uint32_t val = __atomic_load_n(&refCount, __ATOMIC_RELAXED); - val -= quantum; - __atomic_store_n(&refCount, val, __ATOMIC_RELEASE); - uint32_t newval = refCount; - - assert((!ClearPinnedFlag || !(newval & RC_PINNED_FLAG)) && - "unpinning reference that was not pinned"); - assert(newval + quantum >= RC_ONE && - "releasing reference with a refcount of zero"); - - // If we didn't drop the reference count to zero, or if the - // deallocating flag is already set, we're done; don't start - // deallocation. We can assume that the pinned flag isn't set - // unless the refcount is nonzero, and or'ing it in gives us a - // more efficient mask: the check just becomes "is newval nonzero". - if ((newval & (RC_COUNT_MASK | RC_PINNED_FLAG | RC_DEALLOCATING_FLAG)) - != 0) { - // Refcount is not zero. We definitely do not need to deallocate. - return false; - } - - // Refcount is now 0 and is not already deallocating. Try to set - // the deallocating flag. This must be atomic because it can race - // with weak retains. - // - // This also performs the before-deinit acquire barrier if we set the flag. - static_assert(RC_FLAGS_COUNT == 2, - "fix decrementShouldDeallocate() if you add more flags"); - uint32_t oldval = 0; - newval = RC_DEALLOCATING_FLAG; - return __atomic_compare_exchange(&refCount, &oldval, &newval, 0, - __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); - } + bool doDecrementShouldDeallocateNonAtomic(); template - bool doDecrementShouldDeallocateN(uint32_t n) { - // If we're being asked to clear the pinned flag, we can assume - // it's already set. - uint32_t delta = (n << RC_FLAGS_COUNT) + (ClearPinnedFlag ? RC_PINNED_FLAG : 0); - uint32_t newval = __atomic_sub_fetch(&refCount, delta, __ATOMIC_RELEASE); - - assert((!ClearPinnedFlag || !(newval & RC_PINNED_FLAG)) && - "unpinning reference that was not pinned"); - assert(newval + delta >= RC_ONE && - "releasing reference with a refcount of zero"); - - // If we didn't drop the reference count to zero, or if the - // deallocating flag is already set, we're done; don't start - // deallocation. We can assume that the pinned flag isn't set - // unless the refcount is nonzero, and or'ing it in gives us a - // more efficient mask: the check just becomes "is newval nonzero". - if ((newval & (RC_COUNT_MASK | RC_PINNED_FLAG | RC_DEALLOCATING_FLAG)) - != 0) { - // Refcount is not zero. We definitely do not need to deallocate. - return false; - } - - // Refcount is now 0 and is not already deallocating. Try to set - // the deallocating flag. This must be atomic because it can race - // with weak retains. - // - // This also performs the before-deinit acquire barrier if we set the flag. - static_assert(RC_FLAGS_COUNT == 2, - "fix decrementShouldDeallocate() if you add more flags"); - uint32_t oldval = 0; - newval = RC_DEALLOCATING_FLAG; - return __atomic_compare_exchange(&refCount, &oldval, &newval, 0, - __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); - } + bool doDecrementShouldDeallocateN(uint32_t n); template - bool doDecrementShouldDeallocateNNonAtomic(uint32_t n) { - // If we're being asked to clear the pinned flag, we can assume - // it's already set. - uint32_t delta = (n << RC_FLAGS_COUNT) + (ClearPinnedFlag ? RC_PINNED_FLAG : 0); - uint32_t val = __atomic_load_n(&refCount, __ATOMIC_RELAXED); - val -= delta; - __atomic_store_n(&refCount, val, __ATOMIC_RELEASE); - uint32_t newval = val; - - assert((!ClearPinnedFlag || !(newval & RC_PINNED_FLAG)) && - "unpinning reference that was not pinned"); - assert(newval + delta >= RC_ONE && - "releasing reference with a refcount of zero"); - - // If we didn't drop the reference count to zero, or if the - // deallocating flag is already set, we're done; don't start - // deallocation. We can assume that the pinned flag isn't set - // unless the refcount is nonzero, and or'ing it in gives us a - // more efficient mask: the check just becomes "is newval nonzero". - if ((newval & (RC_COUNT_MASK | RC_PINNED_FLAG | RC_DEALLOCATING_FLAG)) - != 0) { - // Refcount is not zero. We definitely do not need to deallocate. - return false; - } - - // Refcount is now 0 and is not already deallocating. Try to set - // the deallocating flag. This must be atomic because it can race - // with weak retains. - // - // This also performs the before-deinit acquire barrier if we set the flag. - static_assert(RC_FLAGS_COUNT == 2, - "fix decrementShouldDeallocate() if you add more flags"); - uint32_t oldval = 0; - newval = RC_DEALLOCATING_FLAG; - return __atomic_compare_exchange(&refCount, &oldval, &newval, 0, - __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); - } + bool doDecrementShouldDeallocateNNonAtomic(uint32_t n); }; @@ -436,46 +215,22 @@ class WeakRefCount { } // Increment the weak reference count. - void increment() { - uint32_t newval = __atomic_add_fetch(&refCount, RC_ONE, __ATOMIC_RELAXED); - assert(newval >= RC_ONE && "weak refcount overflow"); - (void)newval; - } + void increment(); /// Increment the weak reference count by n. - void increment(uint32_t n) { - uint32_t addval = (n << RC_FLAGS_COUNT); - uint32_t newval = __atomic_add_fetch(&refCount, addval, __ATOMIC_RELAXED); - assert(newval >= addval && "weak refcount overflow"); - (void)newval; - } + void increment(uint32_t n); // Decrement the weak reference count. // Return true if the caller should deallocate the object. - bool decrementShouldDeallocate() { - uint32_t oldval = __atomic_fetch_sub(&refCount, RC_ONE, __ATOMIC_RELAXED); - assert(oldval >= RC_ONE && "weak refcount underflow"); - - // Should dealloc if count was 1 before decrementing (i.e. it is zero now) - return (oldval & RC_COUNT_MASK) == RC_ONE; - } + bool decrementShouldDeallocate(); /// Decrement the weak reference count. /// Return true if the caller should deallocate the object. - bool decrementShouldDeallocateN(uint32_t n) { - uint32_t subval = (n << RC_FLAGS_COUNT); - uint32_t oldval = __atomic_fetch_sub(&refCount, subval, __ATOMIC_RELAXED); - assert(oldval >= subval && "weak refcount underflow"); - - // Should dealloc if count was subval before decrementing (i.e. it is zero now) - return (oldval & RC_COUNT_MASK) == subval; - } + bool decrementShouldDeallocateN(uint32_t n); // Return weak reference count. // Note that this is not equal to the number of outstanding weak pointers. - uint32_t getCount() const { - return __atomic_load_n(&refCount, __ATOMIC_RELAXED) >> RC_FLAGS_COUNT; - } + uint32_t getCount() const; }; static_assert(swift::IsTriviallyConstructible::value, diff --git a/stdlib/public/runtime/CMakeLists.txt b/stdlib/public/runtime/CMakeLists.txt index 1fa324f16005b..c34ea676e30b5 100644 --- a/stdlib/public/runtime/CMakeLists.txt +++ b/stdlib/public/runtime/CMakeLists.txt @@ -51,6 +51,7 @@ set(swift_runtime_sources Once.cpp Portability.cpp ProtocolConformance.cpp + RefCount.cpp RuntimeEntrySymbols.cpp) # Acknowledge that the following sources are known. diff --git a/stdlib/public/runtime/RefCount.cpp b/stdlib/public/runtime/RefCount.cpp new file mode 100644 index 0000000000000..e410c2c2241ff --- /dev/null +++ b/stdlib/public/runtime/RefCount.cpp @@ -0,0 +1,330 @@ +//===--- RefCount.cpp - Swift Language Reference Counting Support ---------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// +// Reference Counting Shims While the Language is Bootstrapped +// +//===----------------------------------------------------------------------===// + +#include "../../../stdlib/public/SwiftShims/HeapObject.h" + +#include +#include +#include + +#include "swift/Basic/type_traits.h" + +void StrongRefCount::increment() { + __atomic_fetch_add(&refCount, RC_ONE, __ATOMIC_RELAXED); +} + +void StrongRefCount::incrementNonAtomic() { + uint32_t val = __atomic_load_n(&refCount, __ATOMIC_RELAXED); + val += RC_ONE; + __atomic_store_n(&refCount, val, __ATOMIC_RELAXED); +} + +void StrongRefCount::increment(uint32_t n) { + __atomic_fetch_add(&refCount, n << RC_FLAGS_COUNT, __ATOMIC_RELAXED); +} + +void StrongRefCount::incrementNonAtomic(uint32_t n) { + uint32_t val = __atomic_load_n(&refCount, __ATOMIC_RELAXED); + val += n << RC_FLAGS_COUNT; + __atomic_store_n(&refCount, val, __ATOMIC_RELAXED); +} + +bool StrongRefCount::tryIncrementAndPin() { + uint32_t oldval = __atomic_load_n(&refCount, __ATOMIC_RELAXED); + while (true) { + // If the flag is already set, just fail. + if (oldval & RC_PINNED_FLAG) { + return false; + } + + // Try to simultaneously set the flag and increment the reference count. + uint32_t newval = oldval + (RC_PINNED_FLAG + RC_ONE); + if (__atomic_compare_exchange(&refCount, &oldval, &newval, 0, + __ATOMIC_RELAXED, __ATOMIC_RELAXED)) { + return true; + } + + // Try again; oldval has been updated with the value we saw. + } +} + +bool StrongRefCount::tryIncrementAndPinNonAtomic() { + uint32_t oldval = __atomic_load_n(&refCount, __ATOMIC_RELAXED); + // If the flag is already set, just fail. + if (oldval & RC_PINNED_FLAG) { + return false; + } + + // Try to simultaneously set the flag and increment the reference count. + uint32_t newval = oldval + (RC_PINNED_FLAG + RC_ONE); + __atomic_store_n(&refCount, newval, __ATOMIC_RELAXED); + return true; +} + +bool StrongRefCount::tryIncrement() { + // FIXME: this could be better on LL/SC architectures like arm64 + uint32_t oldval = __atomic_fetch_add(&refCount, RC_ONE, __ATOMIC_RELAXED); + if (oldval & RC_DEALLOCATING_FLAG) { + __atomic_fetch_sub(&refCount, RC_ONE, __ATOMIC_RELAXED); + return false; + } else { + return true; + } +} + +bool StrongRefCount::decrementAndUnpinShouldDeallocate() { + return doDecrementShouldDeallocate(); +} + +bool StrongRefCount::decrementAndUnpinShouldDeallocateNonAtomic() { + return doDecrementShouldDeallocateNonAtomic(); +} + +bool StrongRefCount::decrementShouldDeallocate() { + return doDecrementShouldDeallocate(); +} + +bool StrongRefCount::decrementShouldDeallocateNonAtomic() { + return doDecrementShouldDeallocateNonAtomic(); +} + +bool StrongRefCount::decrementShouldDeallocateN(uint32_t n) { + return doDecrementShouldDeallocateN(n); +} + +void StrongRefCount::decrementFromOneAndDeallocateNonAtomic() { + assert(refCount == RC_ONE && "Expect a count of 1"); + __atomic_store_n(&refCount, RC_DEALLOCATING_FLAG, __ATOMIC_RELAXED); +} + +bool StrongRefCount::decrementShouldDeallocateNNonAtomic(uint32_t n) { + return doDecrementShouldDeallocateNNonAtomic(n); +} + +uint32_t StrongRefCount::getCount() const { + return __atomic_load_n(&refCount, __ATOMIC_RELAXED) >> RC_FLAGS_COUNT; +} + +bool StrongRefCount::isUniquelyReferencedOrPinned() const { + auto value = __atomic_load_n(&refCount, __ATOMIC_RELAXED); + // Rotating right by one sets the sign bit to the pinned bit. After + // rotation, the dealloc flag is the least significant bit followed by the + // reference count. A reference count of two or higher means that our value + // is bigger than 3 if the pinned bit is not set. If the pinned bit is set + // the value is negative. + // Note: Because we are using the sign bit for testing pinnedness it + // is important to do a signed comparison below. + static_assert(RC_PINNED_FLAG == 1, + "The pinned flag must be the lowest bit"); + auto rotateRightByOne = ((value >> 1) | (value << 31)); + return (int32_t)rotateRightByOne < (int32_t)RC_ONE; +} + +bool StrongRefCount::isDeallocating() const { + return __atomic_load_n(&refCount, __ATOMIC_RELAXED) & RC_DEALLOCATING_FLAG; +} + +template +bool StrongRefCount::doDecrementShouldDeallocate() { + // If we're being asked to clear the pinned flag, we can assume + // it's already set. + constexpr uint32_t quantum = + (ClearPinnedFlag ? RC_ONE + RC_PINNED_FLAG : RC_ONE); + uint32_t newval = __atomic_sub_fetch(&refCount, quantum, __ATOMIC_RELEASE); + + assert((!ClearPinnedFlag || !(newval & RC_PINNED_FLAG)) && + "unpinning reference that was not pinned"); + assert(newval + quantum >= RC_ONE && + "releasing reference with a refcount of zero"); + + // If we didn't drop the reference count to zero, or if the + // deallocating flag is already set, we're done; don't start + // deallocation. We can assume that the pinned flag isn't set + // unless the refcount is nonzero, and or'ing it in gives us a + // more efficient mask: the check just becomes "is newval nonzero". + if ((newval & (RC_COUNT_MASK | RC_PINNED_FLAG | RC_DEALLOCATING_FLAG)) + != 0) { + // Refcount is not zero. We definitely do not need to deallocate. + return false; + } + + // Refcount is now 0 and is not already deallocating. Try to set + // the deallocating flag. This must be atomic because it can race + // with weak retains. + // + // This also performs the before-deinit acquire barrier if we set the flag. + static_assert(RC_FLAGS_COUNT == 2, + "fix decrementShouldDeallocate() if you add more flags"); + uint32_t oldval = 0; + newval = RC_DEALLOCATING_FLAG; + return __atomic_compare_exchange(&refCount, &oldval, &newval, 0, + __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); +} + +template +bool StrongRefCount::doDecrementShouldDeallocateNonAtomic() { + // If we're being asked to clear the pinned flag, we can assume + // it's already set. + constexpr uint32_t quantum = + (ClearPinnedFlag ? RC_ONE + RC_PINNED_FLAG : RC_ONE); + uint32_t val = __atomic_load_n(&refCount, __ATOMIC_RELAXED); + val -= quantum; + __atomic_store_n(&refCount, val, __ATOMIC_RELEASE); + uint32_t newval = refCount; + + assert((!ClearPinnedFlag || !(newval & RC_PINNED_FLAG)) && + "unpinning reference that was not pinned"); + assert(newval + quantum >= RC_ONE && + "releasing reference with a refcount of zero"); + + // If we didn't drop the reference count to zero, or if the + // deallocating flag is already set, we're done; don't start + // deallocation. We can assume that the pinned flag isn't set + // unless the refcount is nonzero, and or'ing it in gives us a + // more efficient mask: the check just becomes "is newval nonzero". + if ((newval & (RC_COUNT_MASK | RC_PINNED_FLAG | RC_DEALLOCATING_FLAG)) + != 0) { + // Refcount is not zero. We definitely do not need to deallocate. + return false; + } + + // Refcount is now 0 and is not already deallocating. Try to set + // the deallocating flag. This must be atomic because it can race + // with weak retains. + // + // This also performs the before-deinit acquire barrier if we set the flag. + static_assert(RC_FLAGS_COUNT == 2, + "fix decrementShouldDeallocate() if you add more flags"); + uint32_t oldval = 0; + newval = RC_DEALLOCATING_FLAG; + return __atomic_compare_exchange(&refCount, &oldval, &newval, 0, + __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); +} + +template +bool StrongRefCount::doDecrementShouldDeallocateN(uint32_t n) { + // If we're being asked to clear the pinned flag, we can assume + // it's already set. + uint32_t delta = (n << RC_FLAGS_COUNT) + (ClearPinnedFlag ? RC_PINNED_FLAG : 0); + uint32_t newval = __atomic_sub_fetch(&refCount, delta, __ATOMIC_RELEASE); + + assert((!ClearPinnedFlag || !(newval & RC_PINNED_FLAG)) && + "unpinning reference that was not pinned"); + assert(newval + delta >= RC_ONE && + "releasing reference with a refcount of zero"); + + // If we didn't drop the reference count to zero, or if the + // deallocating flag is already set, we're done; don't start + // deallocation. We can assume that the pinned flag isn't set + // unless the refcount is nonzero, and or'ing it in gives us a + // more efficient mask: the check just becomes "is newval nonzero". + if ((newval & (RC_COUNT_MASK | RC_PINNED_FLAG | RC_DEALLOCATING_FLAG)) + != 0) { + // Refcount is not zero. We definitely do not need to deallocate. + return false; + } + + // Refcount is now 0 and is not already deallocating. Try to set + // the deallocating flag. This must be atomic because it can race + // with weak retains. + // + // This also performs the before-deinit acquire barrier if we set the flag. + static_assert(RC_FLAGS_COUNT == 2, + "fix decrementShouldDeallocate() if you add more flags"); + uint32_t oldval = 0; + newval = RC_DEALLOCATING_FLAG; + return __atomic_compare_exchange(&refCount, &oldval, &newval, 0, + __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); +} + +template +bool StrongRefCount::doDecrementShouldDeallocateNNonAtomic(uint32_t n) { + // If we're being asked to clear the pinned flag, we can assume + // it's already set. + uint32_t delta = (n << RC_FLAGS_COUNT) + (ClearPinnedFlag ? RC_PINNED_FLAG : 0); + uint32_t val = __atomic_load_n(&refCount, __ATOMIC_RELAXED); + val -= delta; + __atomic_store_n(&refCount, val, __ATOMIC_RELEASE); + uint32_t newval = val; + + assert((!ClearPinnedFlag || !(newval & RC_PINNED_FLAG)) && + "unpinning reference that was not pinned"); + assert(newval + delta >= RC_ONE && + "releasing reference with a refcount of zero"); + + // If we didn't drop the reference count to zero, or if the + // deallocating flag is already set, we're done; don't start + // deallocation. We can assume that the pinned flag isn't set + // unless the refcount is nonzero, and or'ing it in gives us a + // more efficient mask: the check just becomes "is newval nonzero". + if ((newval & (RC_COUNT_MASK | RC_PINNED_FLAG | RC_DEALLOCATING_FLAG)) + != 0) { + // Refcount is not zero. We definitely do not need to deallocate. + return false; + } + + // Refcount is now 0 and is not already deallocating. Try to set + // the deallocating flag. This must be atomic because it can race + // with weak retains. + // + // This also performs the before-deinit acquire barrier if we set the flag. + static_assert(RC_FLAGS_COUNT == 2, + "fix decrementShouldDeallocate() if you add more flags"); + uint32_t oldval = 0; + newval = RC_DEALLOCATING_FLAG; + return __atomic_compare_exchange(&refCount, &oldval, &newval, 0, + __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); +} + +void WeakRefCount::increment() { + uint32_t newval = __atomic_add_fetch(&refCount, RC_ONE, __ATOMIC_RELAXED); + assert(newval >= RC_ONE && "weak refcount overflow"); + (void)newval; +} + +void WeakRefCount::increment(uint32_t n) { + uint32_t addval = (n << RC_FLAGS_COUNT); + uint32_t newval = __atomic_add_fetch(&refCount, addval, __ATOMIC_RELAXED); + assert(newval >= addval && "weak refcount overflow"); + (void)newval; +} + +// Decrement the weak reference count. +// Return true if the caller should deallocate the object. +bool WeakRefCount::decrementShouldDeallocate() { + uint32_t oldval = __atomic_fetch_sub(&refCount, RC_ONE, __ATOMIC_RELAXED); + assert(oldval >= RC_ONE && "weak refcount underflow"); + + // Should dealloc if count was 1 before decrementing (i.e. it is zero now) + return (oldval & RC_COUNT_MASK) == RC_ONE; +} + +/// Decrement the weak reference count. +/// Return true if the caller should deallocate the object. +bool WeakRefCount::decrementShouldDeallocateN(uint32_t n) { + uint32_t subval = (n << RC_FLAGS_COUNT); + uint32_t oldval = __atomic_fetch_sub(&refCount, subval, __ATOMIC_RELAXED); + assert(oldval >= subval && "weak refcount underflow"); + + // Should dealloc if count was subval before decrementing (i.e. it is zero now) + return (oldval & RC_COUNT_MASK) == subval; +} + +// Return weak reference count. +// Note that this is not equal to the number of outstanding weak pointers. +uint32_t WeakRefCount::getCount() const { + return __atomic_load_n(&refCount, __ATOMIC_RELAXED) >> RC_FLAGS_COUNT; +}