diff --git a/stdlib/public/SwiftShims/CoreFoundationShims.h b/stdlib/public/SwiftShims/CoreFoundationShims.h index 03eabe5c24a3e..e71a0c77be4c2 100644 --- a/stdlib/public/SwiftShims/CoreFoundationShims.h +++ b/stdlib/public/SwiftShims/CoreFoundationShims.h @@ -134,9 +134,9 @@ _swift_stdlib_NSStringCStringUsingEncodingTrampoline(id _Nonnull obj, SWIFT_RUNTIME_STDLIB_API __swift_uint8_t _swift_stdlib_NSStringGetCStringTrampoline(id _Nonnull obj, - _swift_shims_UInt8 *buffer, - _swift_shims_CFIndex maxLength, - unsigned long encoding); + _swift_shims_UInt8 *_Nonnull buffer, + _swift_shims_CFIndex maxLength, + unsigned long encoding); SWIFT_RUNTIME_STDLIB_API __swift_uintptr_t diff --git a/stdlib/public/core/SmallString.swift b/stdlib/public/core/SmallString.swift index b3910b936f671..32c10aa126faf 100644 --- a/stdlib/public/core/SmallString.swift +++ b/stdlib/public/core/SmallString.swift @@ -72,18 +72,9 @@ extension _SmallString { } } - @inlinable - internal var discriminator: _StringObject.Discriminator { - @inline(__always) get { - let value = _storage.1 &>> _StringObject.Nibbles.discriminatorShift - return _StringObject.Discriminator(UInt8(truncatingIfNeeded: value)) - } - @inline(__always) set { - _storage.1 &= _StringObject.Nibbles.largeAddressMask - _storage.1 |= ( - UInt64(truncatingIfNeeded: newValue._value) - &<< _StringObject.Nibbles.discriminatorShift) - } + @inlinable @inline(__always) + internal var rawDiscriminatedObject: UInt64 { + return _storage.1 } @inlinable @@ -96,7 +87,7 @@ extension _SmallString { @inlinable internal var count: Int { @inline(__always) get { - return discriminator.smallCount + return _StringObject.getSmallCount(fromRaw: rawDiscriminatedObject) } } @@ -108,27 +99,22 @@ extension _SmallString { @inlinable internal var isASCII: Bool { @inline(__always) get { - return discriminator.smallIsASCII + return _StringObject.getSmallIsASCII(fromRaw: rawDiscriminatedObject) } } // Give raw, nul-terminated code units. This is only for limited internal // usage: it always clears the discriminator and count (in case it's full) - @inlinable + @inlinable @inline(__always) internal var zeroTerminatedRawCodeUnits: RawBitPattern { - @inline(__always) get { - return ( - self._storage.0, - self._storage.1 & _StringObject.Nibbles.largeAddressMask) - } + let smallStringCodeUnitMask: UInt64 = 0x00FF_FFFF_FFFF_FFFF + return (self._storage.0, self._storage.1 & smallStringCodeUnitMask) } - @inlinable internal func computeIsASCII() -> Bool { - // TODO(String micro-performance): Evaluate other expressions, e.g. | first let asciiMask: UInt64 = 0x8080_8080_8080_8080 let raw = zeroTerminatedRawCodeUnits - return (raw.0 & asciiMask == 0) && (raw.1 & asciiMask == 0) + return (raw.0 | raw.1) & asciiMask == 0 } } @@ -220,7 +206,7 @@ extension _SmallString { // Overwrite stored code units, including uninitialized. `f` should return the // new count. - @inlinable @inline(__always) + @inline(__always) internal mutating func withMutableCapacity( _ f: (UnsafeMutableBufferPointer) throws -> Int ) rethrows { @@ -231,14 +217,28 @@ extension _SmallString { return try f(UnsafeMutableBufferPointer( start: ptr, count: _SmallString.capacity)) } - _internalInvariant(len <= _SmallString.capacity) - discriminator = .small(withCount: len, isASCII: self.computeIsASCII()) + + let (leading, trailing) = self.zeroTerminatedRawCodeUnits + self = _SmallString(leading: leading, trailing: trailing, count: len) } } // Creation extension _SmallString { + @inlinable @inline(__always) + internal init(leading: UInt64, trailing: UInt64, count: Int) { + _internalInvariant(count <= _SmallString.capacity) + + let isASCII = (leading | trailing) & 0x8080_8080_8080_8080 == 0 + let countAndDiscriminator = UInt64(truncatingIfNeeded: count) &<< 56 + | _StringObject.Nibbles.small(isASCII: isASCII) + _internalInvariant(trailing & countAndDiscriminator == 0) + + self.init(raw: (leading, trailing | countAndDiscriminator)) + _internalInvariant(self.count == count) + } + // Direct from UTF-8 @inlinable @inline(__always) internal init?(_ input: UnsafeBufferPointer) { @@ -251,11 +251,7 @@ extension _SmallString { let leading = _bytesToUInt64(ptr, Swift.min(input.count, 8)) let trailing = count > 8 ? _bytesToUInt64(ptr + 8, count &- 8) : 0 - let isASCII = (leading | trailing) & 0x8080_8080_8080_8080 == 0 - let discriminator = _StringObject.Discriminator.small( - withCount: count, - isASCII: isASCII) - self.init(raw: (leading, trailing | discriminator.rawBits)) + self.init(leading: leading, trailing: trailing, count: count) } @usableFromInline // @testable @@ -273,13 +269,8 @@ extension _SmallString { } _internalInvariant(writeIdx == totalCount) - let isASCII = base.isASCII && other.isASCII - let discriminator = _StringObject.Discriminator.small( - withCount: totalCount, - isASCII: isASCII) - let (leading, trailing) = result.zeroTerminatedRawCodeUnits - self.init(raw: (leading, trailing | discriminator.rawBits)) + self.init(leading: leading, trailing: trailing, count: totalCount) } } diff --git a/stdlib/public/core/StringBridge.swift b/stdlib/public/core/StringBridge.swift index 4950f5f75920f..99862693802f5 100644 --- a/stdlib/public/core/StringBridge.swift +++ b/stdlib/public/core/StringBridge.swift @@ -2,7 +2,7 @@ // // This source file is part of the Swift.org open source project // -// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors +// Copyright (c) 2014 - 2018 Apple Inc. and the Swift project authors // Licensed under Apache License v2.0 with Runtime Library Exception // // See https://swift.org/LICENSE.txt for license information @@ -47,8 +47,8 @@ internal func _stdlib_binary_CFStringGetCharactersPtr( mutating: _swift_stdlib_CFStringGetCharactersPtr(source)) } -/// Copies a slice of a _CocoaString into contiguous storage of -/// sufficient capacity. +/// Copies a slice of a _CocoaString into contiguous storage of sufficient +/// capacity. @_effects(releasenone) internal func _cocoaStringCopyCharacters( from source: _CocoaString, @@ -81,54 +81,46 @@ internal func _cocoaStringCompare( @_effects(readonly) internal func _cocoaHashString( _ string: _CocoaString - ) -> UInt { +) -> UInt { return _swift_stdlib_CFStringHashNSString(string) } @_effects(readonly) internal func _cocoaHashASCIIBytes( - _ bytes: UnsafePointer, - length: Int - ) -> UInt { + _ bytes: UnsafePointer, length: Int +) -> UInt { return _swift_stdlib_CFStringHashCString(bytes, length) } // These "trampolines" are effectively objc_msgSend_super. -// They bypass our implementations to use NSString's +// They bypass our implementations to use NSString's. @_effects(readonly) internal func _cocoaCStringUsingEncodingTrampoline( - _ string: _CocoaString, - _ encoding: UInt) - -> UnsafePointer? { - return _swift_stdlib_NSStringCStringUsingEncodingTrampoline( - string, - encoding) + _ string: _CocoaString, _ encoding: UInt +) -> UnsafePointer? { + return _swift_stdlib_NSStringCStringUsingEncodingTrampoline(string, encoding) } - - - @_effects(releasenone) internal func _cocoaGetCStringTrampoline( - _ string: _CocoaString, - _ buffer: UnsafeMutablePointer, - _ maxLength: Int, - _ encoding: UInt) - -> Int8 { - return Int8(_swift_stdlib_NSStringGetCStringTrampoline(string, - buffer, - maxLength, - encoding)) + _ string: _CocoaString, + _ buffer: UnsafeMutablePointer, + _ maxLength: Int, + _ encoding: UInt +) -> Int8 { + return Int8(_swift_stdlib_NSStringGetCStringTrampoline( + string, buffer, maxLength, encoding)) } // -// Conversion from NSString to Swift's native representation +// Conversion from NSString to Swift's native representation. // private var kCFStringEncodingASCII : _swift_shims_CFStringEncoding { @inline(__always) get { return 0x0600 } } + private var kCFStringEncodingUTF8 : _swift_shims_CFStringEncoding { @inline(__always) get { return 0x8000100 } } @@ -149,12 +141,12 @@ internal enum _KnownCocoaString { @inline(__always) init(_ str: _CocoaString) { - #if !(arch(i386) || arch(arm)) +#if !(arch(i386) || arch(arm)) if _isObjCTaggedPointer(str) { self = .tagged return } - #endif +#endif switch _unsafeAddressOfCocoaStringClass(str) { case unsafeBitCast(_StringStorage.self, to: UInt.self): @@ -168,7 +160,7 @@ internal enum _KnownCocoaString { } #if !(arch(i386) || arch(arm)) -// Resiliently write a tagged cocoa string's contents into a buffer +// Resiliently write a tagged _CocoaString's contents into a buffer. @_effects(releasenone) // @opaque internal func _bridgeTagged( _ cocoa: _CocoaString, @@ -223,16 +215,18 @@ private func _getCocoaStringPointer( internal func _bridgeCocoaString(_ cocoaString: _CocoaString) -> _StringGuts { switch _KnownCocoaString(cocoaString) { case .storage: - return _unsafeUncheckedDowncast(cocoaString, to: _StringStorage.self).asString._guts + return _unsafeUncheckedDowncast( + cocoaString, to: _StringStorage.self).asString._guts case .shared: - return _unsafeUncheckedDowncast(cocoaString, to: _SharedStringStorage.self).asString._guts + return _unsafeUncheckedDowncast( + cocoaString, to: _SharedStringStorage.self).asString._guts #if !(arch(i386) || arch(arm)) case .tagged: - return _StringGuts(_SmallString(taggedCocoa: cocoaString)) + return _StringGuts(_SmallString(taggedCocoa: cocoaString)) #endif case .cocoa: - // "copy" it into a value to be sure nobody will modify behind - // our backs. In practice, when value is already immutable, this + // "Copy" it into a value to be sure nobody will modify behind + // our backs. In practice, when value is already immutable, this // just does a retain. // // TODO: Only in certain circumstances should we emit this call: @@ -240,15 +234,14 @@ internal func _bridgeCocoaString(_ cocoaString: _CocoaString) -> _StringGuts { // 2) If it's mutable with no associated information, then a copy must // happen; might as well eagerly bridge it in. // 3) If it's mutable with associated information, must make the call - // let immutableCopy = _stdlib_binary_CFStringCreateCopy(cocoaString) as AnyObject - #if !(arch(i386) || arch(arm)) +#if !(arch(i386) || arch(arm)) if _isObjCTaggedPointer(immutableCopy) { return _StringGuts(_SmallString(taggedCocoa: immutableCopy)) } - #endif +#endif let (fastUTF8, isASCII): (Bool, Bool) switch _getCocoaStringPointer(immutableCopy) { @@ -288,9 +281,13 @@ extension String { } } if _guts._object.isImmortal { + // TODO: We'd rather emit a valid ObjC object statically than create a + // shared string class instance. + let gutsCountAndFlags = _guts._object._countAndFlags return _SharedStringStorage( immortal: _guts._object.fastUTF8.baseAddress!, - countAndFlags: _guts._object._countAndFlags) + countAndFlags: _StringObject.CountAndFlags( + sharedCount: _guts.count, isASCII: gutsCountAndFlags.isASCII)) } _internalInvariant(_guts._object.hasObjCBridgeableObject, diff --git a/stdlib/public/core/StringGuts.swift b/stdlib/public/core/StringGuts.swift index 718d1324f400e..03aeb0679466d 100644 --- a/stdlib/public/core/StringGuts.swift +++ b/stdlib/public/core/StringGuts.swift @@ -61,8 +61,7 @@ extension _StringGuts { } internal init(_ storage: _SharedStringStorage) { - // TODO(cleanup): We should probably pass whole perf flags struct around - self.init(_StringObject(storage, isASCII: false)) + self.init(_StringObject(storage)) } internal init( @@ -109,18 +108,15 @@ extension _StringGuts { @inline(__always) get { return isFastUTF8 && _object.isASCII } } - @inlinable - internal var isNFC: Bool { - @inline(__always) get { return _object.isNFC } - } + @inline(__always) + internal var isNFC: Bool { return _object.isNFC } - @inlinable - internal var isNFCFastUTF8: Bool { + @inline(__always) + internal var isNFCFastUTF8: Bool { // TODO(String micro-performance): Consider a dedicated bit for this - @inline(__always) get { return _object.isNFC && isFastUTF8 } + return _object.isNFC && isFastUTF8 } - @inlinable internal var hasNativeStorage: Bool { return _object.hasNativeStorage } internal var hasSharedStorage: Bool { return _object.hasSharedStorage } diff --git a/stdlib/public/core/StringObject.swift b/stdlib/public/core/StringObject.swift index eb2ea32d4dc87..49b94f36af184 100644 --- a/stdlib/public/core/StringObject.swift +++ b/stdlib/public/core/StringObject.swift @@ -17,41 +17,37 @@ // TODO(String docs): Word-level diagram -@_fixed_layout @usableFromInline -internal struct _StringObject { - /* +/* - On 64-bit platforms, the discriminator is the most significant 8 bits of the + On 64-bit platforms, the discriminator is the most significant 4 bits of the bridge object. - ┌─────────────────────╥─────┬─────┬─────┬─────┬─────┬─────┬─────┬─────┐ - │ Form ║ 7 │ 6 │ 5 │ 4 │ 3 │ 2 │ 1 │ 0 │ - ╞═════════════════════╬═════╪═════╪═════╪═════╪═════╧═════╧═════╧═════╡ - │ Immortal, Small ║ 1 │ASCII│ 1 │ 0 │ small count │ - ├─────────────────────╫─────┼─────┼─────┼─────┼─────┬─────┬─────┬─────┤ - │ Immortal, Large ║ 1 │ 0 │ 0 │ 0 │ 0 │ TBD │ TBD │ TBD │ - ╞═════════════════════╬═════╪═════╪═════╪═════╪═════╪═════╪═════╪═════╡ - │ Native ║ 0 │ 0 │ 0 │ 0 │ 0 │ TBD │ TBD │ TBD │ - ├─────────────────────╫─────┼─────┼─────┼─────┼─────┼─────┼─────┼─────┤ - │ Shared ║ x │ 0 │ 0 │ 0 │ 1 │ TBD │ TBD │ TBD │ - ├─────────────────────╫─────┼─────┼─────┼─────┼─────┼─────┼─────┼─────┤ - │ Shared, Bridged ║ 0 │ 1 │ 0 │ 0 │ 1 │ TBD │ TBD │ TBD │ - ╞═════════════════════╬═════╪═════╪═════╪═════╪═════╪═════╪═════╪═════╡ - │ Foreign ║ x │ 0 │ 0 │ 1 │ 1 │ TBD │ TBD │ TBD │ - ├─────────────────────╫─────┼─────┼─────┼─────┼─────┼─────┼─────┼─────┤ - │ Foreign, Bridged ║ 0 │ 1 │ 0 │ 1 │ 1 │ TBD │ TBD │ TBD │ - └─────────────────────╨─────┴─────┴─────┴─────┴─────┴─────┴─────┴─────┘ - - b7: isImmortal: Should the Swift runtime skip ARC + ┌─────────────────────╥─────┬─────┬─────┬─────┐ + │ Form ║ b63 │ b62 │ b61 │ b60 │ + ╞═════════════════════╬═════╪═════╪═════╪═════╡ + │ Immortal, Small ║ 1 │ASCII│ 1 │ 0 │ + ├─────────────────────╫─────┼─────┼─────┼─────┤ + │ Immortal, Large ║ 1 │ 0 │ 0 │ 0 │ + ╞═════════════════════╬═════╪═════╪═════╪═════╡ + │ Native ║ 0 │ 0 │ 0 │ 0 │ + ├─────────────────────╫─────┼─────┼─────┼─────┤ + │ Shared ║ x │ 0 │ 0 │ 0 │ + ├─────────────────────╫─────┼─────┼─────┼─────┤ + │ Shared, Bridged ║ 0 │ 1 │ 0 │ 0 │ + ╞═════════════════════╬═════╪═════╪═════╪═════╡ + │ Foreign ║ x │ 0 │ 0 │ 1 │ + ├─────────────────────╫─────┼─────┼─────┼─────┤ + │ Foreign, Bridged ║ 0 │ 1 │ 0 │ 1 │ + └─────────────────────╨─────┴─────┴─────┴─────┘ + + b63: isImmortal: Should the Swift runtime skip ARC - Small strings are just values, always immortal - Large strings can sometimes be immortal, e.g. literals - b6: (large) isBridged / (small) isASCII + b62: (large) isBridged / (small) isASCII - For large strings, this means lazily-bridged NSString: perform ObjC ARC - Small strings repurpose this as a dedicated bit to remember ASCII-ness - b5: isSmall: Dedicated bit to denote small strings - b4: isForeign: aka isSlow, cannot provide access to contiguous UTF-8 - b3: (large) not isTailAllocated: payload isn't a biased pointer - - Shared strings provide contiguous UTF-8 through extra level of indirection + b61: isSmall: Dedicated bit to denote small strings + b60: isForeign: aka isSlow, cannot provide access to contiguous UTF-8 The canonical empty string is the zero-sized small string. It has a leading nibble of 1110, and all other bits are 0. @@ -60,21 +56,25 @@ internal struct _StringObject { can compile to a fused check-and-branch, even if that burns part of the encoding space. - On 32-bit platforms, we use an explicit discriminator with the same encoding - as above, except bit 7 is omitted from storage -- it is left free, to supply - extra inhabitants in the StringObject structure. The missing bit can be - recovered by looking at `_variant.isImmortal`. + On 32-bit platforms, we store an explicit discriminator (as a UInt8) with the + same encoding as above, placed in the high bits. E.g. `b62` above is in + `_discriminator`'s `b6`. +*/ - */ +@_fixed_layout @usableFromInline +internal struct _StringObject { + // Namespace to hold magic numbers + @usableFromInline @_frozen + enum Nibbles {} + + // Abstract the count and performance-flags containing word @_fixed_layout @usableFromInline - struct Discriminator { + struct CountAndFlags { @usableFromInline - internal var _value: UInt8 + var _storage: UInt64 @inlinable @inline(__always) - internal init(_ value: UInt8) { - self._value = value - } + internal init(zero: ()) { self._storage = 0 } } #if arch(i386) || arch(arm) @@ -99,37 +99,6 @@ internal struct _StringObject { } } - @_fixed_layout @usableFromInline - struct Flags { - @usableFromInline - internal var _value: UInt16 - - @inlinable @inline(__always) - init(_ value: UInt16) { - self._value = value - } - } - - @_fixed_layout @usableFromInline - struct CountAndFlags { - @usableFromInline - internal var count: Int - - @usableFromInline - internal var flags: Flags - - @inlinable @inline(__always) - init(count: Int, flags: Flags) { - self.count = count - self.flags = flags - } - - @inlinable @inline(__always) - internal func _invariantCheck() { - flags._invariantCheck() - } - } - @usableFromInline internal var _count: Int @@ -137,74 +106,63 @@ internal struct _StringObject { internal var _variant: Variant @usableFromInline - internal var _discriminator: Discriminator + internal var _discriminator: UInt8 @usableFromInline - internal var _flags: Flags + internal var _flags: UInt16 @inlinable @inline(__always) - init( - count: Int, - variant: Variant, - discriminator: Discriminator, - flags: Flags - ) { + init(count: Int, variant: Variant, discriminator: UInt64, flags: UInt16) { + _internalInvariant(discriminator & 0xFF00_0000_0000_0000 == discriminator, + "only the top byte can carry the discriminator and small count") + self._count = count self._variant = variant - self._discriminator = discriminator + self._discriminator = UInt8(truncatingIfNeeded: discriminator &>> 56) self._flags = flags + self._invariantCheck() } - @inlinable - internal var _countAndFlags: CountAndFlags { - @inline(__always) get { - return CountAndFlags(count: _count, flags: _flags) - } + @inlinable @inline(__always) + init(variant: Variant, discriminator: UInt64, countAndFlags: CountAndFlags) { + self.init( + count: countAndFlags.count, + variant: variant, + discriminator: discriminator, + flags: countAndFlags.flags) } -#else - // Abstract the count and performance-flags containing word - @_fixed_layout @usableFromInline - struct CountAndFlags { - @usableFromInline - var _storage: UInt - @inlinable @inline(__always) - internal init(zero: ()) { self._storage = 0 } + @inlinable @inline(__always) + internal var _countAndFlagsBits: UInt64 { + let rawBits = UInt64(truncatingIfNeeded: _flags) &<< 48 + | UInt64(truncatingIfNeeded: _count) + return rawBits } +#else // - // Laid out as (_countAndFlags, _object), which allows small string contents to - // naturally start on vector-alignment. + // Laid out as (_countAndFlags, _object), which allows small string contents + // to naturally start on vector-alignment. // + @usableFromInline - internal var _countAndFlags: CountAndFlags + internal var _countAndFlagsBits: UInt64 @usableFromInline internal var _object: Builtin.BridgeObject @inlinable @inline(__always) internal init(zero: ()) { - self._countAndFlags = CountAndFlags(zero:()) + self._countAndFlagsBits = 0 self._object = Builtin.valueToBridgeObject(UInt64(0)._value) } -#endif - // Namespace to hold magic numbers - @usableFromInline @_frozen - enum Nibbles {} -} - -extension _StringObject { - @inlinable - internal var discriminator: Discriminator { - @inline(__always) get { -#if arch(i386) || arch(arm) - return _discriminator -#else - let d = objectRawBits &>> Nibbles.discriminatorShift - return Discriminator(UInt8(truncatingIfNeeded: d)) #endif - } + + @inlinable @inline(__always) + internal var _countAndFlags: CountAndFlags { + _internalInvariant(!isSmall) + return CountAndFlags(rawUnchecked: _countAndFlagsBits) } } @@ -220,9 +178,10 @@ extension _StringObject { internal var rawBits: RawBitPattern { @inline(__always) get { let count = UInt64(truncatingIfNeeded: UInt(bitPattern: _count)) - let payload = UInt64(truncatingIfNeeded: undiscriminatedObjectRawBits) - let flags = UInt64(truncatingIfNeeded: _flags._value) - let discr = UInt64(truncatingIfNeeded: _discriminator._value) + let payload = UInt64(truncatingIfNeeded: discriminatedObjectRawBits) + & _StringObject.Nibbles.largeAddressMask + let flags = UInt64(truncatingIfNeeded: _flags) + let discr = UInt64(truncatingIfNeeded: _discriminator) if isSmall { // Rearrange small strings in a different way, compacting bytes into a // contiguous sequence. See comment on small string layout below. @@ -234,7 +193,9 @@ extension _StringObject { #else @inlinable internal var rawBits: RawBitPattern { - @inline(__always) get { return (_countAndFlags.rawBits, objectRawBits) } + @inline(__always) get { + return (_countAndFlagsBits, discriminatedObjectRawBits) + } } @inlinable @inline(__always) @@ -242,7 +203,7 @@ extension _StringObject { bridgeObject: Builtin.BridgeObject, countAndFlags: CountAndFlags ) { self._object = bridgeObject - self._countAndFlags = countAndFlags + self._countAndFlagsBits = countAndFlags._storage _invariantCheck() } @@ -274,7 +235,7 @@ extension _StringObject { @inlinable @inline(__always) internal init(rawUncheckedValue bits: RawBitPattern) { self.init(zero:()) - self._countAndFlags = CountAndFlags(rawUnchecked: bits.0) + self._countAndFlagsBits = bits.0 self._object = Builtin.valueToBridgeObject(bits.1._value) _internalInvariant(self.rawBits == bits) } @@ -284,47 +245,42 @@ extension _StringObject { self.init(rawUncheckedValue: bits) _invariantCheck() } - - @inlinable @_transparent - internal var objectRawBits: UInt64 { - @inline(__always) get { return Builtin.reinterpretCast(_object) } - } #endif -} -extension _StringObject { @inlinable @_transparent - internal var undiscriminatedObjectRawBits: UInt { - @inline(__always) get { + internal var discriminatedObjectRawBits: UInt64 { #if arch(i386) || arch(arm) - switch _variant { - case .immortal(let bitPattern): - return bitPattern - case .native(let storage): - return Builtin.reinterpretCast(storage) - case .bridged(let object): - return Builtin.reinterpretCast(object) - } + let low32: UInt + switch _variant { + case .immortal(let bitPattern): + low32 = bitPattern + case .native(let storage): + low32 = Builtin.reinterpretCast(storage) + case .bridged(let object): + low32 = Builtin.reinterpretCast(object) + } + + return UInt64(truncatingIfNeeded: _discriminator) &<< 56 + | UInt64(truncatingIfNeeded: low32) #else - return UInt(truncatingIfNeeded: objectRawBits & Nibbles.largeAddressMask) + return Builtin.reinterpretCast(_object) #endif - } } } -#if !(arch(i386) || arch(arm)) +// From/to raw bits for CountAndFlags extension _StringObject.CountAndFlags { @usableFromInline internal typealias RawBitPattern = UInt64 - @inlinable + @inlinable @inline(__always) internal var rawBits: RawBitPattern { - @inline(__always) get { return UInt64(truncatingIfNeeded: _storage) } + return _storage } @inlinable @inline(__always) internal init(rawUnchecked bits: RawBitPattern) { - self._storage = UInt(truncatingIfNeeded: bits) + self._storage = bits } @inlinable @inline(__always) @@ -333,27 +289,13 @@ extension _StringObject.CountAndFlags { _invariantCheck() } } -#endif /* - Encoding is optimized for common fast creation. The canonical empty string, ASCII small strings, as well as most literals, have all consecutive 1s in their high nibble mask, and thus can all be encoded as a logical immediate operand on arm64. - - See docs for _StringOjbect.Discriminator for the layout of the high nibble */ -#if arch(i386) || arch(arm) -extension _StringObject.Discriminator { - @inlinable - internal static var empty: _StringObject.Discriminator { - @inline(__always) get { - return _StringObject.Discriminator.small(withCount: 0, isASCII: true) - } - } -} -#else extension _StringObject.Nibbles { // The canonical empty sting is an empty small string @inlinable @@ -361,7 +303,6 @@ extension _StringObject.Nibbles { @inline(__always) get { return _StringObject.Nibbles.small(isASCII: true) } } } -#endif /* @@ -395,7 +336,7 @@ extension _StringObject.Nibbles { └────────────┘ ┌───────────────┬────────────┐ - │ b63:b56 │ b55:b0 │ + │ b63:b60 │ b60:b0 │ ├───────────────┼────────────┤ │ discriminator │ objectAddr │ └───────────────┴────────────┘ @@ -410,72 +351,14 @@ extension _StringObject.Nibbles { */ extension _StringObject.Nibbles { // Mask for address bits, i.e. non-discriminator and non-extra high bits - @inlinable - static internal var largeAddressMask: UInt64 { - @inline(__always) get { - return 0x00FF_FFFF_FFFF_FFFF - } - } - - // Mask for discriminator bits - @inlinable - static internal var discriminatorMask: UInt64 { - @inline(__always) get { - return ~largeAddressMask - } - } - - // Position of discriminator bits - @inlinable - static internal var discriminatorShift: Int { - @inline(__always) get { - return 56 - } - } -} - -extension _StringObject.Discriminator { - // Discriminator for small strings @inlinable @inline(__always) - internal static func small( - withCount count: Int, - isASCII: Bool - ) -> _StringObject.Discriminator { - _internalInvariant(count >= 0 && count <= _SmallString.capacity) - let c = UInt8(truncatingIfNeeded: count) - return _StringObject.Discriminator((isASCII ? 0xE0 : 0xA0) | c) - } + static internal var largeAddressMask: UInt64 { return 0x0FFF_FFFF_FFFF_FFFF } -#if arch(i386) || arch(arm) - // Discriminator for large, immortal, swift-native strings - @inlinable @inline(__always) - internal static func largeImmortal() -> _StringObject.Discriminator { - return _StringObject.Discriminator(0x80) - } - - // Discriminator for large, mortal (i.e. managed), swift-native strings - @inlinable @inline(__always) - internal static func largeMortal() -> _StringObject.Discriminator { - return _StringObject.Discriminator(0x00) - } - - // Discriminator for large, shared, mortal (i.e. managed), swift-native - // strings + // Mask for address bits, i.e. non-discriminator and non-extra high bits @inlinable @inline(__always) - internal static func largeSharedMortal() -> _StringObject.Discriminator { - return _StringObject.Discriminator(0x08) - } - - internal static func largeCocoa( - providesFastUTF8: Bool - ) -> _StringObject.Discriminator { - return _StringObject.Discriminator(providesFastUTF8 ? 0x48 : 0x58) - } -#endif + static internal var discriminatorMask: UInt64 { return ~largeAddressMask } } -#if !(arch(i386) || arch(arm)) -// FIXME: Can we just switch to using the Discriminator factories above? extension _StringObject.Nibbles { // Discriminator for small strings @inlinable @inline(__always) @@ -502,94 +385,8 @@ extension _StringObject.Nibbles { return 0x0000_0000_0000_0000 } - // Discriminator for large, shared, mortal (i.e. managed), swift-native - // strings - @inlinable @inline(__always) - internal static func largeSharedMortal() -> UInt64 { - return 0x0800_0000_0000_0000 - } - internal static func largeCocoa(providesFastUTF8: Bool) -> UInt64 { - return providesFastUTF8 ? 0x4800_0000_0000_0000 : 0x5800_0000_0000_0000 - } -} -#endif - -extension _StringObject.Discriminator { - @inlinable - internal var isImmortal: Bool { - @inline(__always) get { - return (_value & 0x80) != 0 - } - } - - @inlinable - internal var isSmall: Bool { - @inline(__always) get { - return (_value & 0x20) != 0 - } - } - - @inlinable - internal var smallIsASCII: Bool { - @inline(__always) get { - _internalInvariant(isSmall) - return (_value & 0x40) != 0 - } - } - - @inlinable - internal var smallCount: Int { - @inline(__always) get { - _internalInvariant(isSmall) - return Int(truncatingIfNeeded: _value & 0x0F) - } - } - - @inlinable - internal var providesFastUTF8: Bool { - @inline(__always) get { - return (_value & 0x10) == 0 - } - } - - // Whether we are a mortal, native string - @inlinable - internal var hasNativeStorage: Bool { - @inline(__always) get { - return (_value & 0xF8) == 0 - } - } - - // Whether we are a mortal, shared string (managed by Swift runtime) - internal var hasSharedStorage: Bool { - @inline(__always) get { - return (_value & 0xF8) == 0x08 - } - } - - @inlinable - internal var largeFastIsNative: Bool { - @inline(__always) get { - _internalInvariant(!isSmall && providesFastUTF8) - return (_value & 0x08) == 0 - } - } - - // Whether this string is a lazily-bridged NSString, presupposing it is large - @inlinable - internal var largeIsCocoa: Bool { - @inline(__always) get { - _internalInvariant(!isSmall) - return (_value & 0x40) != 0 - } - } -} - -extension _StringObject.Discriminator { - @inlinable - internal var rawBits: UInt64 { - return UInt64(_value) &<< _StringObject.Nibbles.discriminatorShift + return providesFastUTF8 ? 0x4000_0000_0000_0000 : 0x5000_0000_0000_0000 } } @@ -608,11 +405,7 @@ extension _StringObject { @inlinable internal var isImmortal: Bool { @inline(__always) get { -#if arch(i386) || arch(arm) - return _variant.isImmortal -#else - return (objectRawBits & 0x8000_0000_0000_0000) != 0 -#endif + return (discriminatedObjectRawBits & 0x8000_0000_0000_0000) != 0 } } @@ -624,11 +417,7 @@ extension _StringObject { @inlinable internal var isSmall: Bool { @inline(__always) get { -#if arch(i386) || arch(arm) - return _discriminator.isSmall -#else - return (objectRawBits & 0x2000_0000_0000_0000) != 0 -#endif + return (discriminatedObjectRawBits & 0x2000_0000_0000_0000) != 0 } } @@ -644,11 +433,7 @@ extension _StringObject { @inlinable internal var providesFastUTF8: Bool { @inline(__always) get { -#if arch(i386) || arch(arm) - return _discriminator.providesFastUTF8 -#else - return (objectRawBits & 0x1000_0000_0000_0000) == 0 -#endif + return (discriminatedObjectRawBits & 0x1000_0000_0000_0000) == 0 } } @@ -657,63 +442,48 @@ extension _StringObject { @inline(__always) get { return !providesFastUTF8 } } - // Whether we are a mortal, native string - @inlinable + // Whether we are native or shared, i.e. we have a backing class which + // conforms to `_AbstractStringStorage` + @inline(__always) + internal var hasStorage: Bool { + return (discriminatedObjectRawBits & 0xF000_0000_0000_0000) == 0 + } + + // Whether we are a mortal, native (tail-allocated) string + @inline(__always) internal var hasNativeStorage: Bool { - @inline(__always) get { -#if arch(i386) || arch(arm) - return _discriminator.hasNativeStorage -#else - return (objectRawBits & 0xF800_0000_0000_0000) == 0 -#endif - } + // b61 on the object means isSmall, and on countAndFlags means + // isNativelyStored. We just need to check that b61 is 0 on the object and 1 + // on countAndFlags. + let bits = ~discriminatedObjectRawBits & self._countAndFlagsBits + let result = bits & 0x2000_0000_0000_0000 != 0 + _internalInvariant(!result || hasStorage, "native storage needs storage") + return result } // Whether we are a mortal, shared string (managed by Swift runtime) - internal var hasSharedStorage: Bool { - @inline(__always) get { -#if arch(i386) || arch(arm) - return _discriminator.hasSharedStorage -#else - return (objectRawBits & 0xF800_0000_0000_0000) - == Nibbles.largeSharedMortal() -#endif - } - } + internal var hasSharedStorage: Bool { return hasStorage && !hasNativeStorage } } // Queries conditional on being in a large or fast form. extension _StringObject { - // Whether this string is native, presupposing it is both large and fast - @inlinable - internal var largeFastIsNative: Bool { - @inline(__always) get { - _internalInvariant(isLarge && providesFastUTF8) -#if arch(i386) || arch(arm) - return _discriminator.largeFastIsNative -#else - return (objectRawBits & 0x0800_0000_0000_0000) == 0 -#endif - } + // Whether this string is native, i.e. tail-allocated and nul-terminated, + // presupposing it is both large and fast + @inlinable @inline(__always) + internal var largeFastIsTailAllocated: Bool { + _internalInvariant(isLarge && providesFastUTF8) + return _countAndFlags.isTailAllocated } // Whether this string is shared, presupposing it is both large and fast - @inlinable - internal var largeFastIsShared: Bool { - @inline(__always) get { return !largeFastIsNative } - } + @inline(__always) + internal var largeFastIsShared: Bool { return !largeFastIsTailAllocated } // Whether this string is a lazily-bridged NSString, presupposing it is large - @inlinable + @inline(__always) internal var largeIsCocoa: Bool { - @inline(__always) get { - _internalInvariant(isLarge) -#if arch(i386) || arch(arm) - return _discriminator.largeIsCocoa -#else - return (objectRawBits & 0x4000_0000_0000_0000) != 0 -#endif - } + _internalInvariant(isLarge) + return (discriminatedObjectRawBits & 0x4000_0000_0000_0000) != 0 } } @@ -746,49 +516,59 @@ extension _StringObject { */ extension _StringObject { +#if arch(i386) || arch(arm) + @inlinable @inline(__always) + internal init(_ small: _SmallString) { + // On 32-bit, we need to unpack the small string. + let (word1, word2) = small.rawBits + let smallStringDiscriminatorAndCount: UInt64 = 0xFF00_0000_0000_0000 + + let leadingFour = Int(truncatingIfNeeded: word1) + let nextFour = UInt(truncatingIfNeeded: word1 &>> 32) + let smallDiscriminatorAndCount = word2 & smallStringDiscriminatorAndCount + let trailingTwo = UInt16(truncatingIfNeeded: word2) + self.init( + count: leadingFour, + variant: .immortal(nextFour), + discriminator: smallDiscriminatorAndCount, + flags: trailingTwo) + _internalInvariant(isSmall) + } +#else + @inlinable @inline(__always) + internal init(_ small: _SmallString) { + self.init(rawValue: small.rawBits) + _internalInvariant(isSmall) + } +#endif + + @inlinable + internal static func getSmallCount(fromRaw x: UInt64) -> Int { + return Int(truncatingIfNeeded: (x & 0x0F00_0000_0000_0000) &>> 56) + } + @inlinable internal var smallCount: Int { @inline(__always) get { _internalInvariant(isSmall) - return discriminator.smallCount + return _StringObject.getSmallCount(fromRaw: discriminatedObjectRawBits) } } + @inlinable + internal static func getSmallIsASCII(fromRaw x: UInt64) -> Bool { + return x & 0x4000_0000_0000_0000 != 0 + } @inlinable internal var smallIsASCII: Bool { @inline(__always) get { _internalInvariant(isSmall) -#if arch(i386) || arch(arm) - return _discriminator.smallIsASCII -#else - return objectRawBits & 0x4000_0000_0000_0000 != 0 -#endif + return _StringObject.getSmallIsASCII(fromRaw: discriminatedObjectRawBits) } } - @inlinable @inline(__always) - internal init(_ small: _SmallString) { -#if arch(i386) || arch(arm) - let (word1, word2) = small.rawBits - let countBits = Int(truncatingIfNeeded: word1) - let variantBits = UInt(truncatingIfNeeded: word1 &>> 32) - let flagBits = UInt16(truncatingIfNeeded: word2) - let discriminatorBits = UInt8(truncatingIfNeeded: word2 &>> 56) - _internalInvariant(discriminatorBits & 0xA0 == 0xA0) - self.init( - count: countBits, - variant: .immortal(variantBits), - discriminator: Discriminator(discriminatorBits), - flags: Flags(flagBits) - ) -#else - self.init(rawValue: small.rawBits) -#endif - _internalInvariant(isSmall) - } - @inlinable @inline(__always) internal init(empty:()) { // Canonical empty pattern: small zero-length string @@ -796,10 +576,10 @@ extension _StringObject { self.init( count: 0, variant: .immortal(0), - discriminator: .empty, - flags: Flags(0)) + discriminator: Nibbles.emptyString, + flags: 0) #else - self._countAndFlags = CountAndFlags(zero:()) + self._countAndFlagsBits = 0 self._object = Builtin.valueToBridgeObject(Nibbles.emptyString._value) #endif _internalInvariant(self.smallCount == 0) @@ -820,102 +600,169 @@ extension _StringObject { efficiently on this particular string, and the lower 48 are the code unit count (aka endIndex). -┌─────────┬───────┬────────┬───────┐ -│ b63 │ b62 │ b61:48 │ b47:0 │ -├─────────┼───────┼────────┼───────┤ -│ isASCII │ isNFC │ TBD │ count │ -└─────────┴───────┴────────┴───────┘ +┌─────────┬───────┬──────────────────┬─────────────────┬────────┬───────┐ +│ b63 │ b62 │ b61 │ b60 │ b59:48 │ b47:0 │ +├─────────┼───────┼──────────────────┼─────────────────┼────────┼───────┤ +│ isASCII │ isNFC │ isNativelyStored │ isTailAllocated │ TBD │ count │ +└─────────┴───────┴──────────────────┴─────────────────┴────────┴───────┘ isASCII: set when all code units are known to be ASCII, enabling: - Trivial Unicode scalars, they're just the code units - Trivial UTF-16 transcoding (just bit-extend) - Also, isASCII always implies isNFC - isNFC: set when the contents are in normal form C, enable: - - Trivial lexicographical comparisons: just memcmp - - Allocation of more performance flags is TBD, un-used bits will be reserved for - future use. Count stores the number of code units: corresponds to `endIndex`. + isNFC: set when the contents are in normal form C + - Enables trivial lexicographical comparisons: just memcmp + - `isASCII` always implies `isNFC`, but not vice versa + isNativelyStored: set for native stored strings + - `largeAddressBits` holds an instance of `_StringStorage`. + - I.e. the start of the code units is at the stored address + `nativeBias` + isTailAllocated: start of the code units is at the stored address + `nativeBias` + - `isNativelyStored` always implies `isTailAllocated`, but not vice versa + (e.g. literals) + TBD: Reserved for future usage + - Setting a TBD bit to 1 must be semantically equivalent to 0 + - I.e. it can only be used to "cache" fast-path information in the future + count: stores the number of code units, corresponds to `endIndex`. + + NOTE: isNativelyStored is *specifically* allocated to b61 to align with the + bit-position of isSmall on the BridgeObject. This allows us to check for + native storage without an extra branch guarding against smallness. See + `_StringObject.hasNativeStorage` for this usage. */ -#if arch(i386) || arch(arm) -extension _StringObject.Flags { - @inlinable - internal var isASCII: Bool { - @inline(__always) get { - return _value & 0x8000 != 0 - } - } +extension _StringObject.CountAndFlags { + @inlinable @inline(__always) + internal static var countMask: UInt64 { return 0x0000_FFFF_FFFF_FFFF } - @inlinable - internal var isNFC: Bool { - @inline(__always) get { - return _value & 0x4000 != 0 - } + @inlinable @inline(__always) + internal static var flagsMask: UInt64 { return ~countMask } + + @inlinable @inline(__always) + internal static var isASCIIMask: UInt64 { return 0x8000_0000_0000_0000 } + + @inlinable @inline(__always) + internal static var isNFCMask: UInt64 { return 0x4000_0000_0000_0000 } + + @inlinable @inline(__always) + internal static var isNativelyStoredMask: UInt64 { + return 0x2000_0000_0000_0000 } @inlinable @inline(__always) - init(isASCII: Bool) { - // ASCII also entails NFC - self._value = isASCII ? 0xC000 : 0x0000 + internal static var isTailAllocatedMask: UInt64 { + return 0x1000_0000_0000_0000 } - #if !INTERNAL_CHECKS_ENABLED - @inlinable @inline(__always) internal func _invariantCheck() {} - #else - @usableFromInline @inline(never) @_effects(releasenone) - internal func _invariantCheck() { + // General purpose bottom initializer + @inlinable @inline(__always) + internal init( + count: Int, + isASCII: Bool, + isNFC: Bool, + isNativelyStored: Bool, + isTailAllocated: Bool + ) { + var rawBits = UInt64(truncatingIfNeeded: count) + _internalInvariant(rawBits <= _StringObject.CountAndFlags.countMask) + if isASCII { _internalInvariant(isNFC) + rawBits |= _StringObject.CountAndFlags.isASCIIMask + } + + if isNFC { + rawBits |= _StringObject.CountAndFlags.isNFCMask } + + if isNativelyStored { + _internalInvariant(isTailAllocated) + rawBits |= _StringObject.CountAndFlags.isNativelyStoredMask + } + + if isTailAllocated { + rawBits |= _StringObject.CountAndFlags.isTailAllocatedMask + } + + self.init(raw: rawBits) + _internalInvariant(count == self.count) + _internalInvariant(isASCII == self.isASCII) + _internalInvariant(isNFC == self.isNFC) + _internalInvariant(isNativelyStored == self.isNativelyStored) + _internalInvariant(isTailAllocated == self.isTailAllocated) } - #endif // INTERNAL_CHECKS_ENABLED -} -#else -extension _StringObject.CountAndFlags { + @inlinable @inline(__always) - internal init(count: Int) { - self.init(zero:()) - self.count = count - _invariantCheck() + internal init(count: Int, flags: UInt16) { + // Currently, we only use top 4 flags + _internalInvariant(flags & 0xF000 == flags) + + let rawBits = UInt64(truncatingIfNeeded: flags) &<< 48 + | UInt64(truncatingIfNeeded: count) + self.init(raw: rawBits) + _internalInvariant(self.count == count && self.flags == flags) } + // + // Specialized initializers + // @inlinable @inline(__always) - internal init(count: Int, isASCII: Bool) { - self.init(zero:()) - self.count = count - if isASCII { - // ASCII implies NFC - self._storage |= 0xC000_0000_0000_0000 - } - _invariantCheck() + internal init(immortalCount: Int, isASCII: Bool) { + self.init( + count: immortalCount, + isASCII: isASCII, + isNFC: isASCII, + isNativelyStored: false, + isTailAllocated: true) } - - @inlinable - internal var countMask: UInt { - @inline(__always) get { - return 0x0000_FFFF_FFFF_FFFF - } + @inline(__always) + internal init(mortalCount: Int, isASCII: Bool) { + self.init( + count: mortalCount, + isASCII: isASCII, + isNFC: isASCII, + isNativelyStored: true, + isTailAllocated: true) + } + @inline(__always) + internal init(sharedCount: Int, isASCII: Bool) { + self.init( + count: sharedCount, + isASCII: isASCII, + isNFC: isASCII, + isNativelyStored: false, + isTailAllocated: false) } - @inlinable - internal var flagsMask: UInt { @inline(__always) get { return ~countMask} } + // + // Queries and accessors + // - @inlinable + @inlinable @inline(__always) internal var count: Int { - @inline(__always) get { return Int(bitPattern: _storage & countMask) } - @inline(__always) set { - _internalInvariant(newValue <= countMask, "too large") - _storage = (_storage & flagsMask) | UInt(bitPattern: newValue) - } + return Int( + truncatingIfNeeded: _storage & _StringObject.CountAndFlags.countMask) } - @inlinable + @inlinable @inline(__always) + internal var flags: UInt16 { + return UInt16(truncatingIfNeeded: _storage &>> 48) + } + + @inlinable @inline(__always) internal var isASCII: Bool { - return 0 != _storage & 0x8000_0000_0000_0000 + return 0 != _storage & _StringObject.CountAndFlags.isASCIIMask } - @inlinable + @inlinable @inline(__always) internal var isNFC: Bool { - return 0 != _storage & 0x4000_0000_0000_0000 + return 0 != _storage & _StringObject.CountAndFlags.isNFCMask + } + @inlinable @inline(__always) + internal var isNativelyStored: Bool { + return 0 != _storage & _StringObject.CountAndFlags.isNativelyStoredMask + } + @inlinable @inline(__always) + internal var isTailAllocated: Bool { + return 0 != _storage & _StringObject.CountAndFlags.isTailAllocatedMask } #if !INTERNAL_CHECKS_ENABLED @@ -926,47 +773,34 @@ extension _StringObject.CountAndFlags { if isASCII { _internalInvariant(isNFC) } + if isNativelyStored { + _internalInvariant(isTailAllocated) + } } #endif // INTERNAL_CHECKS_ENABLED } -#endif - // Extract extension _StringObject { - @inlinable + @inlinable @inline(__always) internal var largeCount: Int { - @inline(__always) get { - _internalInvariant(isLarge) -#if arch(i386) || arch(arm) - return _count -#else - return _countAndFlags.count -#endif - } - @inline(__always) set { -#if arch(i386) || arch(arm) - _count = newValue -#else - _countAndFlags.count = newValue -#endif - _internalInvariant(newValue == largeCount) - _invariantCheck() - } + _internalInvariant(isLarge) + return _countAndFlags.count } @inlinable internal var largeAddressBits: UInt { @inline(__always) get { _internalInvariant(isLarge) - return undiscriminatedObjectRawBits + return UInt(truncatingIfNeeded: + discriminatedObjectRawBits & Nibbles.largeAddressMask) } } @inlinable internal var nativeUTF8Start: UnsafePointer { @inline(__always) get { - _internalInvariant(largeFastIsNative) + _internalInvariant(largeFastIsTailAllocated) return UnsafePointer( bitPattern: largeAddressBits &+ _StringObject.nativeBias )._unsafelyUnwrappedUnchecked @@ -976,7 +810,7 @@ extension _StringObject { @inlinable internal var nativeUTF8: UnsafeBufferPointer { @inline(__always) get { - _internalInvariant(largeFastIsNative) + _internalInvariant(largeFastIsTailAllocated) return UnsafeBufferPointer(start: nativeUTF8Start, count: largeCount) } } @@ -1065,42 +899,30 @@ extension _StringObject { internal var isASCII: Bool { @inline(__always) get { if isSmall { return smallIsASCII } -#if arch(i386) || arch(arm) - return _flags.isASCII -#else return _countAndFlags.isASCII -#endif } } - @inlinable + @inline(__always) internal var isNFC: Bool { - @inline(__always) get { - if isSmall { - // TODO(String performance): Worth implementing more sophisiticated - // check, or else performing normalization on- construction. For now, - // approximate it with isASCII - return smallIsASCII - } -#if arch(i386) || arch(arm) - return _flags.isNFC -#else - return _countAndFlags.isNFC -#endif + if isSmall { + // TODO(String performance): Worth implementing more sophisiticated + // check, or else performing normalization on- construction. For now, + // approximate it with isASCII + return smallIsASCII } + return _countAndFlags.isNFC } // Get access to fast UTF-8 contents for large strings which provide it. - @inlinable + @inlinable @inline(__always) internal var fastUTF8: UnsafeBufferPointer { - @inline(__always) get { - _internalInvariant(self.isLarge && self.providesFastUTF8) - if _slowPath(self.largeFastIsShared) { - return sharedUTF8 - } - return UnsafeBufferPointer( - start: self.nativeUTF8Start, count: self.largeCount) + _internalInvariant(self.isLarge && self.providesFastUTF8) + guard _fastPath(self.largeFastIsTailAllocated) else { + return sharedUTF8 } + return UnsafeBufferPointer( + start: self.nativeUTF8Start, count: self.largeCount) } // Whether the object stored can be bridged directly as a NSString @@ -1113,12 +935,10 @@ extension _StringObject { } // Fetch the stored subclass of NSString for bridging - @inlinable + @inline(__always) internal var objCBridgeableObject: AnyObject { - @inline(__always) get { - _internalInvariant(hasObjCBridgeableObject) - return Builtin.reinterpretCast(largeAddressBits) - } + _internalInvariant(hasObjCBridgeableObject) + return Builtin.reinterpretCast(largeAddressBits) } // Whether the object provides fast UTF-8 contents that are nul-terminated @@ -1133,7 +953,7 @@ extension _StringObject { // inclusive. For now, we only know native strings and small strings (when // accessed) are. We could also know about some shared strings. - return largeFastIsNative + return largeFastIsTailAllocated } } @@ -1141,18 +961,18 @@ extension _StringObject { extension _StringObject { @inlinable @inline(__always) internal init(immortal bufPtr: UnsafeBufferPointer, isASCII: Bool) { + let countAndFlags = CountAndFlags( + immortalCount: bufPtr.count, isASCII: isASCII) #if arch(i386) || arch(arm) self.init( - count: bufPtr.count, variant: .immortal(start: bufPtr.baseAddress._unsafelyUnwrappedUnchecked), - discriminator: .largeImmortal(), - flags: Flags(isASCII: isASCII)) + discriminator: Nibbles.largeImmortal(), + countAndFlags: countAndFlags) #else // We bias to align code paths for mortal and immortal strings let biasedAddress = UInt( bitPattern: bufPtr.baseAddress._unsafelyUnwrappedUnchecked ) &- _StringObject.nativeBias - let countAndFlags = CountAndFlags(count: bufPtr.count, isASCII: isASCII) self.init( pointerBits: UInt64(truncatingIfNeeded: biasedAddress), @@ -1165,10 +985,9 @@ extension _StringObject { internal init(_ storage: _StringStorage) { #if arch(i386) || arch(arm) self.init( - count: storage._count, variant: .native(storage), - discriminator: .largeMortal(), - flags: storage._flags) + discriminator: Nibbles.largeMortal(), + countAndFlags: storage._countAndFlags) #else self.init( object: storage, @@ -1177,17 +996,16 @@ extension _StringObject { #endif } - internal init(_ storage: _SharedStringStorage, isASCII: Bool) { + internal init(_ storage: _SharedStringStorage) { #if arch(i386) || arch(arm) self.init( - count: storage._count, variant: .native(storage), - discriminator: .largeSharedMortal(), - flags: storage._flags) + discriminator: Nibbles.largeMortal(), + countAndFlags: storage._countAndFlags) #else self.init( object: storage, - discriminator: Nibbles.largeSharedMortal(), + discriminator: Nibbles.largeMortal(), countAndFlags: storage._countAndFlags) #endif } @@ -1195,15 +1013,14 @@ extension _StringObject { internal init( cocoa: AnyObject, providesFastUTF8: Bool, isASCII: Bool, length: Int ) { + let countAndFlags = CountAndFlags(sharedCount: length, isASCII: isASCII) + let discriminator = Nibbles.largeCocoa(providesFastUTF8: providesFastUTF8) #if arch(i386) || arch(arm) self.init( - count: length, variant: .bridged(cocoa), - discriminator: .largeCocoa(providesFastUTF8: providesFastUTF8), - flags: Flags(isASCII: isASCII)) + discriminator: discriminator, + countAndFlags: countAndFlags) #else - let countAndFlags = CountAndFlags(count: length, isASCII: isASCII) - let discriminator = Nibbles.largeCocoa(providesFastUTF8: providesFastUTF8) self.init( object: cocoa, discriminator: discriminator, countAndFlags: countAndFlags) _internalInvariant(self.largeAddressBits == Builtin.reinterpretCast(cocoa)) @@ -1228,6 +1045,15 @@ extension _StringObject { _internalInvariant(MemoryLayout<_StringObject?>.size == 12) _internalInvariant(MemoryLayout<_StringObject?>.stride == 12) _internalInvariant(MemoryLayout<_StringObject?>.alignment == 4) + + // Non-small-string discriminators are 4 high bits only. Small strings use + // the next 4 for count. + if isSmall { + _internalInvariant(_discriminator & 0xA0 == 0xA0) + } else { + _internalInvariant(_discriminator & 0x0F == 0) + } + #else _internalInvariant(MemoryLayout<_StringObject>.size == 16) @@ -1246,15 +1072,18 @@ extension _StringObject { } else { _internalInvariant(isLarge) _internalInvariant(largeCount == count) - if providesFastUTF8 && largeFastIsNative { + if providesFastUTF8 && largeFastIsTailAllocated { _internalInvariant(!isSmall) _internalInvariant(!largeIsCocoa) + _internalInvariant(_countAndFlags.isTailAllocated) if isImmortal { _internalInvariant(!hasNativeStorage) _internalInvariant(!hasObjCBridgeableObject) + _internalInvariant(!_countAndFlags.isNativelyStored) } else { _internalInvariant(hasNativeStorage) + _internalInvariant(_countAndFlags.isNativelyStored) _internalInvariant(hasObjCBridgeableObject) _internalInvariant(nativeStorage.count == self.count) } @@ -1262,13 +1091,19 @@ extension _StringObject { if largeIsCocoa { _internalInvariant(hasObjCBridgeableObject) _internalInvariant(!isSmall) + _internalInvariant(!_countAndFlags.isNativelyStored) + _internalInvariant(!_countAndFlags.isTailAllocated) if isForeign { - } else { _internalInvariant(largeFastIsShared) } } + if _countAndFlags.isNativelyStored { + let anyObj = Builtin.reinterpretCast(largeAddressBits) as AnyObject + _internalInvariant(anyObj is _StringStorage) + } } + #if arch(i386) || arch(arm) switch _variant { case .immortal: diff --git a/stdlib/public/core/StringStorage.swift b/stdlib/public/core/StringStorage.swift index babb40a5f6aa9..afb33ec52aabb 100644 --- a/stdlib/public/core/StringStorage.swift +++ b/stdlib/public/core/StringStorage.swift @@ -2,7 +2,7 @@ // // This source file is part of the Swift.org open source project // -// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors +// Copyright (c) 2014 - 2018 Apple Inc. and the Swift project authors // Licensed under Apache License v2.0 with Runtime Library Exception // // See https://swift.org/LICENSE.txt for license information @@ -12,7 +12,8 @@ import SwiftShims -//Having @objc stuff in an extension creates an ObjC category, which we don't want +// Having @objc stuff in an extension creates an ObjC category, which we don't +// want. #if _runtime(_ObjC) internal protocol _AbstractStringStorage : _NSCopying { @@ -20,8 +21,7 @@ internal protocol _AbstractStringStorage : _NSCopying { var count: Int { get } var isASCII: Bool { get } var start: UnsafePointer { get } - //in UTF16 code units - var length: Int { get } + var length: Int { get } // In UTF16 code units. } internal let _cocoaASCIIEncoding:UInt = 1 /* NSASCIIStringEncoding */ @@ -45,20 +45,19 @@ internal protocol _AbstractStringStorage { extension _AbstractStringStorage { - // ObjC interfaces - #if _runtime(_ObjC) - +// ObjC interfaces. +#if _runtime(_ObjC) + @inline(__always) @_effects(releasenone) internal func _getCharacters( - _ buffer: UnsafeMutablePointer, - _ aRange: _SwiftNSRange - ) { + _ buffer: UnsafeMutablePointer, _ aRange: _SwiftNSRange + ) { _precondition(aRange.location >= 0 && aRange.length >= 0, "Range out of bounds") _precondition(aRange.location + aRange.length <= Int(count), "Range out of bounds") - + let range = Range( uncheckedBounds: (aRange.location, aRange.location+aRange.length)) let str = asString @@ -70,25 +69,21 @@ extension _AbstractStringStorage { @inline(__always) @_effects(releasenone) internal func _getCString( - _ outputPtr: UnsafeMutablePointer, - _ maxLength: Int, - _ encoding: UInt) - -> Int8 { - switch (encoding, isASCII) { - case (_cocoaASCIIEncoding, true): - fallthrough - case (_cocoaUTF8Encoding, _): - guard maxLength >= count + 1 else { return 0 } - let buffer = UnsafeMutableBufferPointer(start: outputPtr, count: maxLength) - buffer.initialize(from: UnsafeBufferPointer(start: start, count: count)) - buffer[count] = 0 - return 1 - default: - return _cocoaGetCStringTrampoline(self, - outputPtr, - maxLength, - encoding) - } + _ outputPtr: UnsafeMutablePointer, _ maxLength: Int, _ encoding: UInt + ) -> Int8 { + switch (encoding, isASCII) { + case (_cocoaASCIIEncoding, true): + fallthrough + case (_cocoaUTF8Encoding, _): + guard maxLength >= count + 1 else { return 0 } + let buffer = + UnsafeMutableBufferPointer(start: outputPtr, count: maxLength) + buffer.initialize(from: UnsafeBufferPointer(start: start, count: count)) + buffer[count] = 0 + return 1 + default: + return _cocoaGetCStringTrampoline(self, outputPtr, maxLength, encoding) + } } @inline(__always) @@ -103,73 +98,73 @@ extension _AbstractStringStorage { return _cocoaCStringUsingEncodingTrampoline(self, encoding) } } - + @_effects(readonly) - internal func _nativeIsEqual(_ nativeOther: T) -> Int8 { + internal func _nativeIsEqual( + _ nativeOther: T + ) -> Int8 { if count != nativeOther.count { return 0 } - return (start == nativeOther.start || (memcmp(start, nativeOther.start, count) == 0)) ? 1 : 0 + return (start == nativeOther.start || + (memcmp(start, nativeOther.start, count) == 0)) ? 1 : 0 } @inline(__always) @_effects(readonly) - internal func _isEqual(_ other:AnyObject?) - -> Int8 { - guard let other = other else { + internal func _isEqual(_ other: AnyObject?) -> Int8 { + guard let other = other else { + return 0 + } + + if self === other { + return 1 + } + + // Handle the case where both strings were bridged from Swift. + // We can't use String.== because it doesn't match NSString semantics. + let knownOther = _KnownCocoaString(other) + switch knownOther { + case .storage: + return _nativeIsEqual( + _unsafeUncheckedDowncast(other, to: _StringStorage.self)) + case .shared: + return _nativeIsEqual( + _unsafeUncheckedDowncast(other, to: _SharedStringStorage.self)) +#if !(arch(i386) || arch(arm)) + case .tagged: + fallthrough +#endif + case .cocoa: + // We're allowed to crash, but for compatibility reasons NSCFString allows + // non-strings here. + if _isNSString(other) != 1 { return 0 } - - if self === other { - return 1 + // At this point we've proven that it is an NSString of some sort, but not + // one of ours. + if length != _stdlib_binary_CFStringGetLength(other) { + return 0 } - - // Handle the case where both strings were bridged from Swift. - // We can't use String.== because it doesn't match NSString semantics. - let knownOther = _KnownCocoaString(other) - switch knownOther { - case .storage: - return _nativeIsEqual(_unsafeUncheckedDowncast(other, to: _StringStorage.self)) - case .shared: - return _nativeIsEqual(_unsafeUncheckedDowncast(other, to: _SharedStringStorage.self)) -#if !(arch(i386) || arch(arm)) - case .tagged: - fallthrough -#endif - case .cocoa: - //we're allowed to crash, but for compatibility reasons NSCFString allows non-strings here - if _isNSString(other) != 1 { - return 0 - } - - //At this point we've proven that it is an NSString of some sort, but not one of ours - if length != _stdlib_binary_CFStringGetLength(other) { - return 0 - } - - defer { _fixLifetime(other) } - - //CFString will only give us ASCII bytes here, but that's fine - //We already handled non-ASCII UTF8 strings earlier since they're Swift - if let otherStart = _cocoaUTF8Pointer(other) { - return (start == otherStart || (memcmp(start, otherStart, count) == 0)) ? 1 : 0 - } - - /* - The abstract implementation of -isEqualToString: falls back to -compare: - immediately, so when we run out of fast options to try, do the same. - We can likely be more clever here if need be - */ - return _cocoaStringCompare(self, other) == 0 ? 1 : 0 + defer { _fixLifetime(other) } + // CFString will only give us ASCII bytes here, but that's fine. + // We already handled non-ASCII UTF8 strings earlier since they're Swift. + if let otherStart = _cocoaUTF8Pointer(other) { + return (start == otherStart || + (memcmp(start, otherStart, count) == 0)) ? 1 : 0 } + /* + The abstract implementation of -isEqualToString: falls back to -compare: + immediately, so when we run out of fast options to try, do the same. + We can likely be more clever here if need be + */ + return _cocoaStringCompare(self, other) == 0 ? 1 : 0 + } } - - #endif //_runtime(_ObjC) + +#endif //_runtime(_ObjC) } -#if arch(i386) || arch(arm) -private typealias Flags = _StringObject.Flags -#endif private typealias CountAndFlags = _StringObject.CountAndFlags // @@ -178,53 +173,43 @@ private typealias CountAndFlags = _StringObject.CountAndFlags // Optional<_StringBreadcrumbs>. // -final internal class _StringStorage: __SwiftNativeNSString, _AbstractStringStorage { +final internal class _StringStorage + : __SwiftNativeNSString, _AbstractStringStorage { #if arch(i386) || arch(arm) // The total allocated storage capacity. Note that this includes the required - // nul-terminator + // nul-terminator. internal var _realCapacity: Int - internal var _count: Int - - internal var _flags: _StringObject.Flags - + internal var _flags: UInt16 internal var _reserved: UInt16 - internal var count: Int { - @inline(__always) get { return _count } - @inline(__always) set { _count = newValue } + @inline(__always) + internal var count: Int { return _count } + + @inline(__always) + internal var _countAndFlags: _StringObject.CountAndFlags { + return CountAndFlags(count: _count, flags: _flags) } #else // The capacity of our allocation. Note that this includes the nul-terminator, - // which is not available for overridding. + // which is not available for overriding. internal var _realCapacityAndFlags: UInt64 - internal var _countAndFlags: _StringObject.CountAndFlags - internal var count: Int { - @inline(__always) get { return _countAndFlags.count } - @inline(__always) set { _countAndFlags.count = newValue } - } + @inline(__always) + internal var count: Int { return _countAndFlags.count } // The total allocated storage capacity. Note that this includes the required - // nul-terminator + // nul-terminator. + @inline(__always) internal var _realCapacity: Int { - @inline(__always) get { - return Int(truncatingIfNeeded: - _realCapacityAndFlags & _StringObject.Nibbles.largeAddressMask) - } + return Int(truncatingIfNeeded: + _realCapacityAndFlags & CountAndFlags.countMask) } #endif - internal final var isASCII: Bool { - @inline(__always) get { - #if arch(i386) || arch(arm) - return _flags.isASCII - #else - return _countAndFlags.isASCII - #endif - } - } + @inline(__always) + final internal var isASCII: Bool { return _countAndFlags.isASCII } final internal var asString: String { @_effects(readonly) @inline(__always) get { @@ -233,15 +218,16 @@ final internal class _StringStorage: __SwiftNativeNSString, _AbstractStringStora } #if _runtime(_ObjC) - + @objc(length) final internal var length: Int { @_effects(readonly) @inline(__always) get { - return asString.utf16.count //UTF16View special-cases ASCII for us + return asString.utf16.count // UTF16View special-cases ASCII for us. } } - @objc final internal var hash: UInt { + @objc + final internal var hash: UInt { @_effects(readonly) get { if isASCII { return _cocoaHashASCIIBytes(start, length: count) @@ -260,21 +246,22 @@ final internal class _StringStorage: __SwiftNativeNSString, _AbstractStringStora @objc(getCharacters:range:) @_effects(releasenone) final internal func getCharacters( - _ buffer: UnsafeMutablePointer, - range aRange: _SwiftNSRange) { + _ buffer: UnsafeMutablePointer, range aRange: _SwiftNSRange + ) { _getCharacters(buffer, aRange) } @objc(_fastCStringContents:) @_effects(readonly) - final internal func _fastCStringContents(_ requiresNulTermination:Int8) -> UnsafePointer? { + final internal func _fastCStringContents( + _ requiresNulTermination: Int8 + ) -> UnsafePointer? { if isASCII { return start._asCChar } - return nil } - + @objc(UTF8String) @_effects(readonly) final internal func _utf8String() -> UnsafePointer? { @@ -286,13 +273,12 @@ final internal class _StringStorage: __SwiftNativeNSString, _AbstractStringStora final internal func cString(encoding: UInt) -> UnsafePointer? { return _cString(encoding: encoding) } - + @objc(getCString:maxLength:encoding:) @_effects(releasenone) - final internal func getCString(_ outputPtr: UnsafeMutablePointer, - maxLength: Int, - encoding: UInt) - -> Int8 { + final internal func getCString( + _ outputPtr: UnsafeMutablePointer, maxLength: Int, encoding: UInt + ) -> Int8 { return _getCString(outputPtr, maxLength, encoding) } @@ -308,10 +294,10 @@ final internal class _StringStorage: __SwiftNativeNSString, _AbstractStringStora @objc(isEqualToString:) @_effects(readonly) - final internal func isEqual(to other:AnyObject?) -> Int8 { + final internal func isEqual(to other: AnyObject?) -> Int8 { return _isEqual(other) } - + @objc(copyWithZone:) final internal func copy(with zone: _SwiftNSZone?) -> AnyObject { // While _StringStorage instances aren't immutable in general, @@ -352,11 +338,11 @@ private func determineCodeUnitCapacity(_ desiredCapacity: Int) -> Int { _internalInvariant(capacity > desiredCapacity) return capacity #else - // Bigger than _SmallString, and we need 1 extra for nul-terminator + // Bigger than _SmallString, and we need 1 extra for nul-terminator. let minCap = 1 + Swift.max(desiredCapacity, _SmallString.capacity) _internalInvariant(minCap < 0x1_0000_0000_0000, "max 48-bit length") - // Round up to the nearest multiple of 8 that isn't also a multiple of 16 + // Round up to the nearest multiple of 8 that isn't also a multiple of 16. let capacity = ((minCap + 7) & -16) + 8 _internalInvariant( capacity > desiredCapacity && capacity % 8 == 0 && capacity % 16 != 0) @@ -410,12 +396,8 @@ extension _StringStorage { capacity: Int, isASCII: Bool ) -> _StringStorage { -#if arch(i386) || arch(arm) - let flags = Flags(isASCII: isASCII) - let countAndFlags = CountAndFlags(count: bufPtr.count, flags: flags) -#else - let countAndFlags = CountAndFlags(count: bufPtr.count, isASCII: isASCII) -#endif + let countAndFlags = CountAndFlags( + mortalCount: bufPtr.count, isASCII: isASCII) _internalInvariant(capacity >= bufPtr.count) let storage = _StringStorage.create( capacity: capacity, countAndFlags: countAndFlags) @@ -436,26 +418,24 @@ extension _StringStorage { // Usage extension _StringStorage { - @inlinable - internal var mutableStart: UnsafeMutablePointer { - @inline(__always) get { - return UnsafeMutablePointer(Builtin.projectTailElems(self, UInt8.self)) - } + @inline(__always) + private var mutableStart: UnsafeMutablePointer { + return UnsafeMutablePointer(Builtin.projectTailElems(self, UInt8.self)) } private var mutableEnd: UnsafeMutablePointer { @inline(__always) get { return mutableStart + count } } - @inlinable + @inline(__always) internal var start: UnsafePointer { - @inline(__always) get { return UnsafePointer(mutableStart) } + return UnsafePointer(mutableStart) } private final var end: UnsafePointer { @inline(__always) get { return UnsafePointer(mutableEnd) } } - // Point to the nul-terminator + // Point to the nul-terminator. private final var terminator: UnsafeMutablePointer { @inline(__always) get { return mutableEnd } } @@ -477,7 +457,7 @@ extension _StringStorage { } // The total capacity available for code units. Note that this excludes the - // required nul-terminator + // required nul-terminator. internal var capacity: Int { return _realCapacity &- 1 } @@ -494,7 +474,7 @@ extension _StringStorage { } // The capacity available for appending. Note that this excludes the required - // nul-terminator + // nul-terminator. internal var unusedCapacity: Int { get { return _realCapacity &- count &- 1 } } @@ -511,41 +491,40 @@ extension _StringStorage { _internalInvariant(self._realCapacity > self.count, "no room for nul-terminator") _internalInvariant(self.terminator.pointee == 0, "not nul terminated") -#if arch(i386) || arch(arm) - _flags._invariantCheck() -#else _countAndFlags._invariantCheck() -#endif if isASCII { _internalInvariant(_allASCII(self.codeUnits)) } if let crumbs = _breadcrumbsAddress.pointee { crumbs._invariantCheck(for: self.asString) } + _internalInvariant(_countAndFlags.isNativelyStored) + _internalInvariant(_countAndFlags.isTailAllocated) } #endif // INTERNAL_CHECKS_ENABLED } // Appending extension _StringStorage { - // Perform common post-RRC adjustments and invariant enforcement + // Perform common post-RRC adjustments and invariant enforcement. @_effects(releasenone) private func _postRRCAdjust(newCount: Int, newIsASCII: Bool) { + let countAndFlags = CountAndFlags( + mortalCount: newCount, isASCII: newIsASCII) #if arch(i386) || arch(arm) - self._count = newCount - self._flags = Flags(isASCII: newIsASCII) + self._count = countAndFlags.count + self._flags = countAndFlags.flags #else - self._countAndFlags = CountAndFlags( - count: newCount, isASCII: newIsASCII) + self._countAndFlags = countAndFlags #endif self.terminator.pointee = 0 - // TODO(String performance): Consider updating breadcrumbs when feasible + // TODO(String performance): Consider updating breadcrumbs when feasible. self._breadcrumbsAddress.pointee = nil _invariantCheck() } - // Perform common post-append adjustments and invariant enforcement + // Perform common post-append adjustments and invariant enforcement. @_effects(releasenone) private func _postAppendAdjust( appendedCount: Int, appendedIsASCII isASCII: Bool @@ -621,12 +600,12 @@ extension _StringStorage { let replCount = replacement.count _internalInvariant(replCount - (upper - lower) <= unusedCapacity) - // Position the tail + // Position the tail. let lowerPtr = mutableStart + lower let tailCount = _slideTail( src: mutableStart + upper, dst: lowerPtr + replCount) - // Copy in the contents + // Copy in the contents. lowerPtr.moveInitialize( from: UnsafeMutablePointer( mutating: replacement.baseAddress._unsafelyUnwrappedUnchecked), @@ -647,12 +626,12 @@ extension _StringStorage { _internalInvariant(lower <= upper) _internalInvariant(replCount - (upper - lower) <= unusedCapacity) - // Position the tail + // Position the tail. let lowerPtr = mutableStart + lower let tailCount = _slideTail( src: mutableStart + upper, dst: lowerPtr + replCount) - // Copy in the contents + // Copy in the contents. var isASCII = self.isASCII var srcCount = 0 for cu in replacement { @@ -668,21 +647,18 @@ extension _StringStorage { } // For shared storage and bridging literals -final internal class _SharedStringStorage: __SwiftNativeNSString, _AbstractStringStorage { +final internal class _SharedStringStorage + : __SwiftNativeNSString, _AbstractStringStorage { internal var _owner: AnyObject? - internal var start: UnsafePointer #if arch(i386) || arch(arm) internal var _count: Int + internal var _flags: UInt16 - internal var _flags: _StringObject.Flags - - @inlinable + @inline(__always) internal var _countAndFlags: _StringObject.CountAndFlags { - @inline(__always) get { - return CountAndFlags(count: _count, flags: _flags) - } + return CountAndFlags(count: _count, flags: _flags) } #else internal var _countAndFlags: _StringObject.CountAndFlags @@ -690,15 +666,7 @@ final internal class _SharedStringStorage: __SwiftNativeNSString, _AbstractStrin internal var _breadcrumbs: _StringBreadcrumbs? = nil - internal var count: Int { - @_effects(readonly) @inline(__always) get { - #if arch(i386) || arch(arm) - return _count - #else - return _countAndFlags.count - #endif - } - } + internal var count: Int { return _countAndFlags.count } internal init( immortal ptr: UnsafePointer, @@ -715,16 +683,9 @@ final internal class _SharedStringStorage: __SwiftNativeNSString, _AbstractStrin super.init() self._invariantCheck() } - - internal final var isASCII: Bool { - @inline(__always) get { - #if arch(i386) || arch(arm) - return _flags.isASCII - #else - return _countAndFlags.isASCII - #endif - } - } + + @inline(__always) + final internal var isASCII: Bool { return _countAndFlags.isASCII } final internal var asString: String { @_effects(readonly) @inline(__always) get { @@ -733,15 +694,16 @@ final internal class _SharedStringStorage: __SwiftNativeNSString, _AbstractStrin } #if _runtime(_ObjC) - + @objc(length) final internal var length: Int { @_effects(readonly) get { - return asString.utf16.count //UTF16View special-cases ASCII for us + return asString.utf16.count // UTF16View special-cases ASCII for us. } } - - @objc final internal var hash: UInt { + + @objc + final internal var hash: UInt { @_effects(readonly) get { if isASCII { return _cocoaHashASCIIBytes(start, length: count) @@ -749,22 +711,22 @@ final internal class _SharedStringStorage: __SwiftNativeNSString, _AbstractStrin return _cocoaHashString(self) } } - + @objc(characterAtIndex:) @_effects(readonly) final internal func character(at offset: Int) -> UInt16 { let str = asString return str.utf16[str._toUTF16Index(offset)] } - + @objc(getCharacters:range:) @_effects(releasenone) final internal func getCharacters( - _ buffer: UnsafeMutablePointer, - range aRange: _SwiftNSRange) { + _ buffer: UnsafeMutablePointer, range aRange: _SwiftNSRange + ) { _getCharacters(buffer, aRange) } - + @objc final internal var fastestEncoding: UInt { @_effects(readonly) get { @@ -774,17 +736,18 @@ final internal class _SharedStringStorage: __SwiftNativeNSString, _AbstractStrin return _cocoaUTF8Encoding } } - + @objc(_fastCStringContents:) @_effects(readonly) - final internal func _fastCStringContents(_ requiresNulTermination:Int8) -> UnsafePointer? { + final internal func _fastCStringContents( + _ requiresNulTermination: Int8 + ) -> UnsafePointer? { if isASCII { return start._asCChar } - return nil } - + @objc(UTF8String) @_effects(readonly) final internal func _utf8String() -> UnsafePointer? { @@ -796,13 +759,12 @@ final internal class _SharedStringStorage: __SwiftNativeNSString, _AbstractStrin final internal func cString(encoding: UInt) -> UnsafePointer? { return _cString(encoding: encoding) } - + @objc(getCString:maxLength:encoding:) @_effects(releasenone) - final internal func getCString(_ outputPtr: UnsafeMutablePointer, - maxLength: Int, - encoding: UInt) - -> Int8 { + final internal func getCString( + _ outputPtr: UnsafeMutablePointer, maxLength: Int, encoding: UInt + ) -> Int8 { return _getCString(outputPtr, maxLength, encoding) } @@ -811,7 +773,7 @@ final internal class _SharedStringStorage: __SwiftNativeNSString, _AbstractStrin final internal func isEqual(to other:AnyObject?) -> Int8 { return _isEqual(other) } - + @objc(copyWithZone:) final internal func copy(with zone: _SwiftNSZone?) -> AnyObject { // While _StringStorage instances aren't immutable in general, @@ -826,16 +788,17 @@ final internal class _SharedStringStorage: __SwiftNativeNSString, _AbstractStrin } extension _SharedStringStorage { - #if !INTERNAL_CHECKS_ENABLED - @inline(__always) internal func _invariantCheck() {} - #else +#if !INTERNAL_CHECKS_ENABLED + @inline(__always) + internal func _invariantCheck() {} +#else internal func _invariantCheck() { if let crumbs = _breadcrumbs { crumbs._invariantCheck(for: self.asString) } _countAndFlags._invariantCheck() + _internalInvariant(!_countAndFlags.isNativelyStored) + _internalInvariant(!_countAndFlags.isTailAllocated) } - #endif // INTERNAL_CHECKS_ENABLED +#endif // INTERNAL_CHECKS_ENABLED } - - diff --git a/stdlib/public/core/StringTesting.swift b/stdlib/public/core/StringTesting.swift index 0fccea1157621..dee9f7bf8cf18 100644 --- a/stdlib/public/core/StringTesting.swift +++ b/stdlib/public/core/StringTesting.swift @@ -60,13 +60,13 @@ extension _StringGuts { // TODO: shared native _internalInvariant(_object.providesFastUTF8) - _internalInvariant(_object.largeFastIsNative) if _object.isImmortal { result._form = ._immortal( address: UInt(bitPattern: _object.nativeUTF8Start)) return result } if _object.hasNativeStorage { + _internalInvariant(_object.largeFastIsTailAllocated) result._form = ._native(object: _object.nativeStorage) return result } diff --git a/test/SILOptimizer/character_literals.swift b/test/SILOptimizer/character_literals.swift index 311db70be8b22..20c5cae717123 100644 --- a/test/SILOptimizer/character_literals.swift +++ b/test/SILOptimizer/character_literals.swift @@ -39,11 +39,11 @@ public func singleNonAsciiChar() -> Character { } // NOTE: -9223372036854775808 = 0x80 = immortal large discrim -// NOTE: 25 = length in UTF-8 code units +// NOTE: 1152921504606847001 = 25 (code unit length) | `isTailAllocated` perf flag // // CHECK-LABEL: define {{.*}}singleNonSmolChar // CHECK-NEXT: entry: -// CHECK: ret { i64, %swift.bridge* } { i64 25, %swift.bridge* {{.*}}@0{{.*}}i64 -9223372036854775808 +// CHECK: ret { i64, %swift.bridge* } { i64 1152921504606847001, %swift.bridge* {{.*}}@0{{.*}}i64 -9223372036854775808 public func singleNonSmolChar() -> Character { return "👩‍👩‍👦‍👦" } diff --git a/test/SILOptimizer/concat_string_literals.64.swift b/test/SILOptimizer/concat_string_literals.64.swift index bcdb70f6b70d4..ff6a039c6e0ee 100644 --- a/test/SILOptimizer/concat_string_literals.64.swift +++ b/test/SILOptimizer/concat_string_literals.64.swift @@ -1,5 +1,6 @@ // RUN: %target-swift-frontend -O -emit-ir %s | %FileCheck %s // RUN: %target-swift-frontend -Osize -emit-ir %s | %FileCheck %s +// REQUIRES: swift_stdlib_no_asserts,optimized_stdlib // We have a separate test for 32-bit architectures. // REQUIRES: PTRSIZE=64 @@ -33,9 +34,9 @@ public func test_strng_strng2() -> String { return "aé" + "def" } -// NOTE: 43 = code-unit length +// NOTE: 1152921504606847019 = 43 (code-unit length) | `isTailAllocated` perf flag // CHECK-LABEL: test_scalar_strng -// CHECK: ret { i64, %swift.bridge* } { i64 43, %swift.bridge* inttoptr {{.*}}i64 -{{[0-9]+}}{{.*}} to %swift.bridge*) } +// CHECK: ret { i64, %swift.bridge* } { i64 1152921504606847019, %swift.bridge* inttoptr {{.*}}i64 -{{[0-9]+}}{{.*}} to %swift.bridge*) } public func test_scalar_strng() -> String { return "a" + "👨🏿‍💼+🧙🏿‍♂️=🕴🏿" } @@ -47,9 +48,9 @@ public func test_strng_concat_smol() -> String { return "a" + "bc" + "dèf" + "ghī" } -// NOTE: 23 = code-unit length +// NOTE: 1152921504606846999 = 23 (code-unit length) | `isTailAllocated` perf flag // CHECK-LABEL test_strng_concat_large -// CHECK: ret { i64, %swift.bridge* } { i64 23, %swift.bridge* inttoptr {{.*}}i64 -{{[0-9]+}}{{.*}} to %swift.bridge*) } +// CHECK: ret { i64, %swift.bridge* } { i64 1152921504606846999, %swift.bridge* inttoptr {{.*}}i64 -{{[0-9]+}}{{.*}} to %swift.bridge*) } public func test_strng_concat_large() -> String { return "a" + "bc" + "dèf" + "ghī" + "jklmn" + "o" + "𝛒qr" } diff --git a/test/api-digester/Outputs/stability-stdlib-abi.swift.expected b/test/api-digester/Outputs/stability-stdlib-abi.swift.expected index f99ae6cb9ca3a..e85e841390bdf 100644 --- a/test/api-digester/Outputs/stability-stdlib-abi.swift.expected +++ b/test/api-digester/Outputs/stability-stdlib-abi.swift.expected @@ -602,3 +602,34 @@ Func MutableCollection.withContiguousMutableStorageIfAvailable(_:) has been adde Func Sequence.withContiguousStorageIfAvailable(_:) has been added as a protocol requirement Func Strideable....(_:_:) has been removed + +Func _SmallString.computeIsASCII() has been removed +Func _SmallString.withMutableCapacity(_:) has been removed +Struct _StringObject.Discriminator has been removed +Var _SmallString.discriminator has been removed +Var _StringObject.CountAndFlags._storage has declared type change from UInt to UInt64 +Var _StringObject.CountAndFlags.countMask has declared type change from UInt to UInt64 +Var _StringObject.CountAndFlags.countMask is now static +Var _StringObject.CountAndFlags.flagsMask has declared type change from UInt to UInt64 +Var _StringObject.CountAndFlags.flagsMask is now static +Var _StringObject.Nibbles.discriminatorShift has been removed +Var _StringObject.discriminator has been removed +Var _StringObject.objectRawBits has been renamed to Var _StringObject.discriminatedObjectRawBits +Var _StringObject.undiscriminatedObjectRawBits has been removed + +Constructor _StringObject.CountAndFlags.init(count:) has been removed +Constructor _StringObject.CountAndFlags.init(count:isASCII:) has been removed +Func _StringObject.Nibbles.largeSharedMortal() has been removed +Var _StringGuts.hasNativeStorage has been removed +Var _StringGuts.isNFC has been removed +Var _StringGuts.isNFCFastUTF8 has been removed +Var _StringObject.hasNativeStorage has been removed +Var _StringObject.isNFC has been removed +Var _StringObject.largeFastIsNative has been removed + +Var _StringObject.largeFastIsShared has been removed +Var _StringObject.largeIsCocoa has been removed +Var _StringObject.objCBridgeableObject has been removed + +Var _StringObject._countAndFlags is no longer a stored property +Var _StringObject._countAndFlagsBits is added to a non-resilient type diff --git a/test/api-digester/Outputs/stability-stdlib-source.swift.expected b/test/api-digester/Outputs/stability-stdlib-source.swift.expected index 69b8f05172e9e..5648c98daa2d8 100644 --- a/test/api-digester/Outputs/stability-stdlib-source.swift.expected +++ b/test/api-digester/Outputs/stability-stdlib-source.swift.expected @@ -218,3 +218,6 @@ Func MutableCollection.withContiguousMutableStorageIfAvailable(_:) has been adde Func Sequence.withContiguousStorageIfAvailable(_:) has been added as a protocol requirement Func Strideable....(_:_:) has been removed + +Struct Discriminator has been removed + diff --git a/validation-test/Reflection/reflect_Character.swift b/validation-test/Reflection/reflect_Character.swift index c402513c63ad4..4ed4305174f31 100644 --- a/validation-test/Reflection/reflect_Character.swift +++ b/validation-test/Reflection/reflect_Character.swift @@ -34,12 +34,10 @@ reflect(object: obj) // CHECK-64-NEXT: (struct size=16 alignment=8 stride=16 num_extra_inhabitants=2147483647 bitwise_takable=1 // CHECK-64-NEXT: (field name=_object offset=0 // CHECK-64-NEXT: (struct size=16 alignment=8 stride=16 num_extra_inhabitants=2147483647 bitwise_takable=1 -// CHECK-64-NEXT: (field name=_countAndFlags offset=0 +// CHECK-64-NEXT: (field name=_countAndFlagsBits offset=0 // CHECK-64-NEXT: (struct size=8 alignment=8 stride=8 num_extra_inhabitants=0 bitwise_takable=1 -// CHECK-64-NEXT: (field name=_storage offset=0 -// CHECK-64-NEXT: (struct size=8 alignment=8 stride=8 num_extra_inhabitants=0 bitwise_takable=1 -// CHECK-64-NEXT: (field name=_value offset=0 -// CHECK-64-NEXT: (builtin size=8 alignment=8 stride=8 num_extra_inhabitants=0 bitwise_takable=1)))))) +// CHECK-64-NEXT: (field name=_value offset=0 +// CHECK-64-NEXT: (builtin size=8 alignment=8 stride=8 num_extra_inhabitants=0 bitwise_takable=1)))) // CHECK-64-NEXT: (field name=_object offset=8 // CHECK-64-NEXT: (builtin size=8 alignment=8 stride=8 num_extra_inhabitants=2147483647 bitwise_takable=1))))))))))) diff --git a/validation-test/Reflection/reflect_multiple_types.swift b/validation-test/Reflection/reflect_multiple_types.swift index bf4619ad8fb11..687549afa229f 100644 --- a/validation-test/Reflection/reflect_multiple_types.swift +++ b/validation-test/Reflection/reflect_multiple_types.swift @@ -132,12 +132,10 @@ reflect(object: obj) // CHECK-64-NEXT: (struct size=16 alignment=8 stride=16 num_extra_inhabitants=2147483647 bitwise_takable=1 // CHECK-64-NEXT: (field name=_object offset=0 // CHECK-64-NEXT: (struct size=16 alignment=8 stride=16 num_extra_inhabitants=2147483647 bitwise_takable=1 -// CHECK-64-NEXT: (field name=_countAndFlags offset=0 +// CHECK-64-NEXT: (field name=_countAndFlagsBits offset=0 // CHECK-64-NEXT: (struct size=8 alignment=8 stride=8 num_extra_inhabitants=0 bitwise_takable=1 -// CHECK-64-NEXT: (field name=_storage offset=0 -// CHECK-64-NEXT: (struct size=8 alignment=8 stride=8 num_extra_inhabitants=0 bitwise_takable=1 -// CHECK-64-NEXT: (field name=_value offset=0 -// CHECK-64-NEXT: (builtin size=8 alignment=8 stride=8 num_extra_inhabitants=0 bitwise_takable=1)))))) +// CHECK-64-NEXT: (field name=_value offset=0 +// CHECK-64-NEXT: (builtin size=8 alignment=8 stride=8 num_extra_inhabitants=0 bitwise_takable=1)))) // CHECK-64-NEXT: (field name=_object offset=8 // CHECK-64-NEXT: (builtin size=8 alignment=8 stride=8 num_extra_inhabitants=2147483647 bitwise_takable=1))))))))))