diff --git a/include/swift/Runtime/Metadata.h b/include/swift/Runtime/Metadata.h index 07094ef23e7bc..b31a31c2e7f66 100644 --- a/include/swift/Runtime/Metadata.h +++ b/include/swift/Runtime/Metadata.h @@ -666,9 +666,6 @@ void swift_initStructMetadataWithLayoutString(StructMetadata *self, const uint8_t *fieldTags, uint32_t *fieldOffsets); -SWIFT_RUNTIME_STDLIB_INTERNAL -size_t _swift_refCountBytesForMetatype(const Metadata *type); - enum LayoutStringFlags : uint64_t { Empty = 0, // TODO: Track other useful information tha can be used to optimize layout @@ -688,6 +685,9 @@ inline LayoutStringFlags &operator|=(LayoutStringFlags &a, LayoutStringFlags b) return a = (a | b); } +SWIFT_RUNTIME_STDLIB_INTERNAL +size_t _swift_refCountBytesForMetatype(const Metadata *type); + SWIFT_RUNTIME_STDLIB_INTERNAL void _swift_addRefCountStringForMetatype(uint8_t *layoutStr, size_t &layoutStrOffset, diff --git a/stdlib/public/runtime/BytecodeLayouts.cpp b/stdlib/public/runtime/BytecodeLayouts.cpp index a40cab1eb34f5..5e45145f10d0a 100644 --- a/stdlib/public/runtime/BytecodeLayouts.cpp +++ b/stdlib/public/runtime/BytecodeLayouts.cpp @@ -37,26 +37,6 @@ using namespace swift; -static const size_t layoutStringHeaderSize = sizeof(uint64_t) + sizeof(size_t); - -/// Given a pointer and an offset, read the requested data and increment the -/// offset -template -static T readBytes(const uint8_t *typeLayout, size_t &i) { - T returnVal; - memcpy(&returnVal, typeLayout + i, sizeof(T)); - i += sizeof(T); - return returnVal; -} - -/// Given a pointer, a value, and an offset, write the value at the given -/// offset and increment offset by the size of T -template -static void writeBytes(uint8_t *typeLayout, size_t &i, T value) { - memcpy(typeLayout + i, &value, sizeof(T)); - i += sizeof(T); -} - static Metadata *getExistentialTypeMetadata(OpaqueValue *object) { return reinterpret_cast(object)[NumWords_ValueBuffer]; } diff --git a/stdlib/public/runtime/BytecodeLayouts.h b/stdlib/public/runtime/BytecodeLayouts.h index f589352c25a17..f5e487428927b 100644 --- a/stdlib/public/runtime/BytecodeLayouts.h +++ b/stdlib/public/runtime/BytecodeLayouts.h @@ -73,6 +73,23 @@ void swift_resolve_resilientAccessors(uint8_t *layoutStr, size_t layoutStrOffset, const uint8_t *fieldLayoutStr, const Metadata *fieldType); + +template +inline T readBytes(const uint8_t *layoutStr, size_t &i) { + T returnVal; + memcpy(&returnVal, layoutStr + i, sizeof(T)); + i += sizeof(T); + return returnVal; +} + +template +inline void writeBytes(uint8_t *layoutStr, size_t &i, T value) { + memcpy(layoutStr + i, &value, sizeof(T)); + i += sizeof(T); +} + +constexpr size_t layoutStringHeaderSize = sizeof(uint64_t) + sizeof(size_t); + } // namespace swift #endif // SWIFT_BYTECODE_LAYOUTS_H diff --git a/stdlib/public/runtime/Metadata.cpp b/stdlib/public/runtime/Metadata.cpp index cae6faac16b74..9a91f5a23874d 100644 --- a/stdlib/public/runtime/Metadata.cpp +++ b/stdlib/public/runtime/Metadata.cpp @@ -2641,41 +2641,6 @@ void swift::swift_initStructMetadata(StructMetadata *structType, vwtable->publishLayout(layout); } -namespace { - enum LayoutStringFlags : uint64_t { - Empty = 0, - // TODO: Track other useful information tha can be used to optimize layout - // strings, like different reference kinds contained in the string - // number of ref counting operations (maybe up to 4), so we can - // use witness functions optimized for these cases. - HasRelativePointers = (1ULL << 63), - }; - - inline bool operator&(LayoutStringFlags a, LayoutStringFlags b) { - return (uint64_t(a) & uint64_t(b)) != 0; - } - inline LayoutStringFlags operator|(LayoutStringFlags a, LayoutStringFlags b) { - return LayoutStringFlags(uint64_t(a) | uint64_t(b)); - } - inline LayoutStringFlags &operator|=(LayoutStringFlags &a, LayoutStringFlags b) { - return a = (a | b); - } - - template - inline T readBytes(const uint8_t *layoutStr, size_t &i) { - T returnVal; - memcpy(&returnVal, layoutStr + i, sizeof(T)); - i += sizeof(T); - return returnVal; - } - - template - inline void writeBytes(uint8_t *layoutStr, size_t &i, T value) { - memcpy(layoutStr + i, &value, sizeof(T)); - i += sizeof(T); - } -} // end anonymous namespace - void swift::swift_initStructMetadataWithLayoutString( StructMetadata *structType, StructLayoutFlags layoutFlags, size_t numFields, const uint8_t *const *fieldTypes, const uint8_t *fieldTags, @@ -2727,21 +2692,9 @@ void swift::swift_initStructMetadataWithLayoutString( extraInhabitantCount = fieldExtraInhabitantCount; } - if (fieldType->vw_size() == 0) { - continue; - } else if (fieldType->getValueWitnesses()->isPOD()) { - // no extra space required for POD - } else if (fieldType->hasLayoutString()) { - refCountBytes += *(const size_t *)(fieldType->getLayoutString() + - sizeof(uint64_t)); - } else if (fieldType->isClassObject() || fieldType->isAnyExistentialType()) { - refCountBytes += sizeof(uint64_t); - } else { - refCountBytes += sizeof(uint64_t) + sizeof(uintptr_t); - } + refCountBytes += _swift_refCountBytesForMetatype(fieldType); } - const size_t layoutStringHeaderSize = sizeof(uint64_t) + sizeof(size_t); const size_t fixedLayoutStringSize = layoutStringHeaderSize + sizeof(uint64_t) * 2; @@ -2781,93 +2734,9 @@ void swift::swift_initStructMetadataWithLayoutString( const Metadata *fieldType = (const Metadata*)fieldTypes[i]; - fullOffset = roundUpToAlignMask(fullOffset, fieldType->vw_alignment() - 1); - size_t offset = fullOffset - unalignedOffset + previousFieldOffset; - - if (fieldType->vw_size() == 0) { - continue; - } else if (fieldType->getValueWitnesses()->isPOD()) { - // No need to handle PODs - previousFieldOffset = offset + fieldType->vw_size(); - fullOffset += fieldType->vw_size(); - } else if (fieldType->hasLayoutString()) { - const uint8_t *fieldLayoutStr = fieldType->getLayoutString(); - const LayoutStringFlags fieldFlags = - *(const LayoutStringFlags *)fieldLayoutStr; - const size_t fieldRefCountBytes = - *(const size_t *)(fieldLayoutStr + sizeof(uint64_t)); - if (fieldRefCountBytes > 0) { - flags |= fieldFlags; - memcpy(layoutStr + layoutStrOffset, fieldLayoutStr + layoutStringHeaderSize, - fieldRefCountBytes); - - if (fieldFlags & LayoutStringFlags::HasRelativePointers) { - swift_resolve_resilientAccessors(layoutStr, layoutStrOffset, - fieldLayoutStr, fieldType); - } - - if (offset) { - auto layoutStrOffsetCopy = layoutStrOffset; - auto firstTagAndOffset = - readBytes(layoutStr, layoutStrOffsetCopy); - layoutStrOffsetCopy = layoutStrOffset; - firstTagAndOffset += offset; - writeBytes(layoutStr, layoutStrOffsetCopy, firstTagAndOffset); - } - - auto previousFieldOffsetOffset = - layoutStringHeaderSize + fieldRefCountBytes; - previousFieldOffset = readBytes(fieldLayoutStr, - previousFieldOffsetOffset); - layoutStrOffset += fieldRefCountBytes; - } else { - previousFieldOffset += fieldType->vw_size(); - } - fullOffset += fieldType->vw_size(); - } else if (auto *cls = fieldType->getClassObject()) { - RefCountingKind tag; - if (!cls->isTypeMetadata()) { - #if SWIFT_OBJC_INTEROP - tag = RefCountingKind::ObjC; - #else - tag = RefCountingKind::Unknown; - #endif - } else { - auto *vwt = cls->getValueWitnesses(); - if (vwt == &VALUE_WITNESS_SYM(Bo)) { - tag = RefCountingKind::NativeStrong; - } else if (vwt == &VALUE_WITNESS_SYM(BO)) { - #if SWIFT_OBJC_INTEROP - tag = RefCountingKind::ObjC; - #else - tag = RefCountingKind::Unknown; - #endif - } else if (vwt == &VALUE_WITNESS_SYM(Bb)) { - tag = RefCountingKind::Bridge; - } else { - goto metadata; - }; - } - - writeBytes(layoutStr, layoutStrOffset, ((uint64_t)tag << 56) | offset); - previousFieldOffset = fieldType->vw_size(); - fullOffset += previousFieldOffset; - } else if (fieldType->isAnyExistentialType()) { - auto *existential = dyn_cast(fieldType); - assert(existential); - auto tag = existential->isClassBounded() ? RefCountingKind::Unknown - : RefCountingKind::Existential; - writeBytes(layoutStr, layoutStrOffset, ((uint64_t)tag << 56) | offset); - previousFieldOffset = fieldType->vw_size(); - fullOffset += previousFieldOffset; - } else { -metadata: - writeBytes(layoutStr, layoutStrOffset, - ((uint64_t)RefCountingKind::Metatype << 56) | offset); - writeBytes(layoutStr, layoutStrOffset, fieldType); - previousFieldOffset = fieldType->vw_size(); - fullOffset += previousFieldOffset; - } + _swift_addRefCountStringForMetatype(layoutStr, layoutStrOffset, flags, + fieldType, fullOffset, + previousFieldOffset); } writeBytes(layoutStr, layoutStrOffset, previousFieldOffset); @@ -2896,6 +2765,128 @@ void swift::swift_initStructMetadataWithLayoutString( vwtable->publishLayout(layout); } +size_t swift::_swift_refCountBytesForMetatype(const Metadata *type) { + if (type->vw_size() == 0 || type->getValueWitnesses()->isPOD()) { + return 0; + } else if (type->hasLayoutString()) { + size_t offset = sizeof(uint64_t); + return readBytes(type->getLayoutString(), offset); + } else if (type->isClassObject() || type->isAnyExistentialType()) { + return sizeof(uint64_t); + } else if (auto *tuple = dyn_cast(type)) { + size_t res = 0; + for (InProcess::StoredSize i = 0; i < tuple->NumElements; i++) { + res += _swift_refCountBytesForMetatype(tuple->getElement(i).Type); + } + return res; + } else { + return sizeof(uint64_t) + sizeof(uintptr_t); + } +} + +void swift::_swift_addRefCountStringForMetatype(uint8_t *layoutStr, + size_t &layoutStrOffset, + LayoutStringFlags &flags, + const Metadata *fieldType, + size_t &fullOffset, + size_t &previousFieldOffset) { + size_t unalignedOffset = fullOffset; + fullOffset = roundUpToAlignMask(fullOffset, fieldType->vw_alignment() - 1); + size_t offset = fullOffset - unalignedOffset + previousFieldOffset; + if (fieldType->vw_size() == 0) { + return; + } else if (fieldType->getValueWitnesses()->isPOD()) { + // No need to handle PODs + previousFieldOffset = offset + fieldType->vw_size(); + fullOffset += fieldType->vw_size(); + } else if (fieldType->hasLayoutString()) { + const uint8_t *fieldLayoutStr = fieldType->getLayoutString(); + const LayoutStringFlags fieldFlags = + *(const LayoutStringFlags *)fieldLayoutStr; + size_t refCountBytesOffset = sizeof(uint64_t); + const size_t fieldRefCountBytes = readBytes(fieldLayoutStr, + refCountBytesOffset); + if (fieldRefCountBytes > 0) { + flags |= fieldFlags; + memcpy(layoutStr + layoutStrOffset, + fieldLayoutStr + layoutStringHeaderSize, + fieldRefCountBytes); + + if (fieldFlags & LayoutStringFlags::HasRelativePointers) { + swift_resolve_resilientAccessors(layoutStr, layoutStrOffset, + fieldLayoutStr, fieldType); + } + + if (offset) { + auto layoutStrOffsetCopy = layoutStrOffset; + auto firstTagAndOffset = + readBytes(layoutStr, layoutStrOffsetCopy); + layoutStrOffsetCopy = layoutStrOffset; + firstTagAndOffset += offset; + writeBytes(layoutStr, layoutStrOffsetCopy, firstTagAndOffset); + } + + auto previousFieldOffsetOffset = + layoutStringHeaderSize + fieldRefCountBytes; + previousFieldOffset = readBytes(fieldLayoutStr, + previousFieldOffsetOffset); + layoutStrOffset += fieldRefCountBytes; + } else { + previousFieldOffset += fieldType->vw_size(); + } + fullOffset += fieldType->vw_size(); + } else if (auto *tuple = dyn_cast(fieldType)) { + for (InProcess::StoredSize i = 0; i < tuple->NumElements; i++) { + _swift_addRefCountStringForMetatype(layoutStr, layoutStrOffset, flags, + tuple->getElement(i).Type, fullOffset, + previousFieldOffset); + } + } else if (auto *cls = fieldType->getClassObject()) { + RefCountingKind tag; + if (!cls->isTypeMetadata()) { + #if SWIFT_OBJC_INTEROP + tag = RefCountingKind::ObjC; + #else + tag = RefCountingKind::Unknown; + #endif + } else { + auto *vwt = cls->getValueWitnesses(); + if (vwt == &VALUE_WITNESS_SYM(Bo)) { + tag = RefCountingKind::NativeStrong; + } else if (vwt == &VALUE_WITNESS_SYM(BO)) { + #if SWIFT_OBJC_INTEROP + tag = RefCountingKind::ObjC; + #else + tag = RefCountingKind::Unknown; + #endif + } else if (vwt == &VALUE_WITNESS_SYM(Bb)) { + tag = RefCountingKind::Bridge; + } else { + goto metadata; + }; + } + + writeBytes(layoutStr, layoutStrOffset, ((uint64_t)tag << 56) | offset); + previousFieldOffset = fieldType->vw_size(); + fullOffset += previousFieldOffset; + } else if (fieldType->isAnyExistentialType()) { + auto *existential = dyn_cast(fieldType); + assert(existential); + auto tag = existential->isClassBounded() ? RefCountingKind::Unknown + : RefCountingKind::Existential; + writeBytes(layoutStr, layoutStrOffset, ((uint64_t)tag << 56) | offset); + previousFieldOffset = fieldType->vw_size(); + fullOffset += previousFieldOffset; + } else { +metadata: + writeBytes(layoutStr, layoutStrOffset, + ((uint64_t)RefCountingKind::Metatype << 56) | offset); + writeBytes(layoutStr, layoutStrOffset, fieldType); + previousFieldOffset = fieldType->vw_size(); + fullOffset += previousFieldOffset; + } +} + /***************************************************************************/ /*** Classes ***************************************************************/ /***************************************************************************/