diff --git a/cmake/modules/AddSwift.cmake b/cmake/modules/AddSwift.cmake index 60b5b573cf327..8a13232c2e1ec 100644 --- a/cmake/modules/AddSwift.cmake +++ b/cmake/modules/AddSwift.cmake @@ -329,7 +329,7 @@ function(_add_variant_link_flags) RESULT_VAR_NAME result) if("${LFLAGS_SDK}" STREQUAL "LINUX") - list(APPEND result "-lpthread" "-ldl") + list(APPEND result "-lpthread" "-latomic" "-ldl") elseif("${LFLAGS_SDK}" STREQUAL "FREEBSD") list(APPEND result "-lpthread") elseif("${LFLAGS_SDK}" STREQUAL "CYGWIN") diff --git a/cmake/modules/AddSwiftUnittests.cmake b/cmake/modules/AddSwiftUnittests.cmake index 27b57d44bd8bb..4de5361e22c21 100644 --- a/cmake/modules/AddSwiftUnittests.cmake +++ b/cmake/modules/AddSwiftUnittests.cmake @@ -42,6 +42,9 @@ function(add_swift_unittest test_dirname) if("${CMAKE_SYSTEM_NAME}" STREQUAL "Darwin") set_property(TARGET "${test_dirname}" APPEND_STRING PROPERTY LINK_FLAGS " -Xlinker -rpath -Xlinker ${SWIFT_LIBRARY_OUTPUT_INTDIR}/swift/macosx") + elseif("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux") + set_property(TARGET "${test_dirname}" APPEND_STRING PROPERTY + LINK_FLAGS " -latomic") endif() if(SWIFT_ENABLE_GOLD_LINKER AND diff --git a/include/swift/Runtime/Concurrent.h b/include/swift/Runtime/Concurrent.h index 48e37ab40fd59..1fe1bca7bccd0 100644 --- a/include/swift/Runtime/Concurrent.h +++ b/include/swift/Runtime/Concurrent.h @@ -189,8 +189,9 @@ class ConcurrentMapBase : protected Allocator { // Destroy the node's payload. node->~Node(); - // Deallocate the node. - this->Deallocate(node, allocSize); + // Deallocate the node. The static_cast here is required + // because LLVM's allocator API is insane. + this->Deallocate(static_cast(node), allocSize); } }; diff --git a/stdlib/public/runtime/Metadata.cpp b/stdlib/public/runtime/Metadata.cpp index 59c317ef183d7..57e80fa123758 100644 --- a/stdlib/public/runtime/Metadata.cpp +++ b/stdlib/public/runtime/Metadata.cpp @@ -2809,3 +2809,99 @@ swift::swift_getGenericWitnessTable(GenericWitnessTable *genericTable, } uint64_t swift::RelativeDirectPointerNullPtr = 0; + +/***************************************************************************/ +/*** Allocator implementation **********************************************/ +/***************************************************************************/ + +namespace { + struct PoolRange { + static constexpr uintptr_t PageSize = 16 * 1024; + static constexpr uintptr_t MaxPoolAllocationSize = PageSize / 2; + + /// The start of the allocation. + char *Begin; + + /// The number of bytes remaining. + size_t Remaining; + }; +} + +// A statically-allocated pool. It's zero-initialized, so this +// doesn't cost us anything in binary size. +LLVM_ALIGNAS(alignof(void*)) static char InitialAllocationPool[64*1024]; +static std::atomic +AllocationPool{PoolRange{InitialAllocationPool, + sizeof(InitialAllocationPool)}}; + +void *MetadataAllocator::Allocate(size_t size, size_t alignment) { + assert(alignment <= alignof(void*)); + assert(size % alignof(void*) == 0); + + // If the size is larger than the maximum, just use malloc. + if (size > PoolRange::MaxPoolAllocationSize) + return malloc(size); + + // Allocate out of the pool. + PoolRange curState = AllocationPool.load(std::memory_order_relaxed); + while (true) { + char *allocation; + PoolRange newState; + bool allocatedNewPage; + + // Try to allocate out of the current page. + if (size <= curState.Remaining) { + allocatedNewPage = false; + allocation = curState.Begin; + newState = PoolRange{curState.Begin + size, curState.Remaining - size}; + } else { + allocatedNewPage = true; + allocation = new char[PoolRange::PageSize]; + newState = PoolRange{allocation + size, PoolRange::PageSize - size}; + __asan_poison_memory_region(allocation, PoolRange::PageSize); + } + + // Swap in the new state. + if (std::atomic_compare_exchange_weak_explicit(&AllocationPool, + &curState, newState, + std::memory_order_relaxed, + std::memory_order_relaxed)) { + // If that succeeded, we've successfully allocated. + __msan_allocated_memory(allocation, size); + __asan_poison_memory_region(allocation, size); + return allocation; + } + + // If it failed, go back to a neutral state and try again. + if (allocatedNewPage) { + delete[] allocation; + } + } +} + +void MetadataAllocator::Deallocate(const void *allocation, size_t size) { + __asan_poison_memory_region(allocation, size); + + if (size > PoolRange::MaxPoolAllocationSize) { + free(const_cast(allocation)); + return; + } + + // Check whether the allocation pool is still in the state it was in + // immediately after the given allocation. + PoolRange curState = AllocationPool.load(std::memory_order_relaxed); + if (reinterpret_cast(allocation) + size != curState.Begin) { + return; + } + + // Try to swap back to the pre-allocation state. If this fails, + // don't bother trying again; we'll just leak the allocation. + PoolRange newState = { reinterpret_cast(const_cast(allocation)), + curState.Remaining + size }; + (void) + std::atomic_compare_exchange_strong_explicit(&AllocationPool, + &curState, newState, + std::memory_order_relaxed, + std::memory_order_relaxed); +} + diff --git a/stdlib/public/runtime/MetadataCache.h b/stdlib/public/runtime/MetadataCache.h index aca46968e63e0..2d4e6eb381c78 100644 --- a/stdlib/public/runtime/MetadataCache.h +++ b/stdlib/public/runtime/MetadataCache.h @@ -26,11 +26,18 @@ namespace swift { -// For now, use malloc and free as our standard allocator for -// metadata caches. It might make sense in the future to take -// advantage of the fact that we know that most allocations here -// won't ever be deallocated. -using MetadataAllocator = llvm::MallocAllocator; +class MetadataAllocator : public llvm::AllocatorBase { +public: + void Reset() {} + + LLVM_ATTRIBUTE_RETURNS_NONNULL void *Allocate(size_t size, size_t alignment); + using AllocatorBase::Allocate; + + void Deallocate(const void *Ptr, size_t size); + using AllocatorBase::Deallocate; + + void PrintStats() const {} +}; /// A typedef for simple global caches. template diff --git a/utils/gen-static-stdlib-link-args b/utils/gen-static-stdlib-link-args index 39e518d22b655..ef1423b11c0a3 100755 --- a/utils/gen-static-stdlib-link-args +++ b/utils/gen-static-stdlib-link-args @@ -62,6 +62,7 @@ function write_linkfile { -ldl -lpthread -lswiftCore +-latomic -lswiftImageInspectionShared $ICU_LIBS -Xlinker diff --git a/utils/static-executable-args.lnk b/utils/static-executable-args.lnk index 6987433fc6185..ff408601185c4 100644 --- a/utils/static-executable-args.lnk +++ b/utils/static-executable-args.lnk @@ -8,6 +8,7 @@ -Xlinker --defsym=__import_pthread_key_create=pthread_key_create -lpthread +-latomic -licui18n -licuuc -licudata