[pull] master from apple:master

docs/WindowsBuild.md

@@ -64,9 +64,9 @@ subst S: <path to sources>
 
 ```cmd
 S:
-git clone https://github.com/apple/llvm-project --branch swift/master toolchain
-git clone -c core.autocrlf=input -c core.symlinks=true https://github.com/apple/swift toolchain/swift
-git clone https://github.com/apple/swift-cmark toolchain/cmark
+git clone https://github.com/apple/llvm-project --branch swift/master llvm-project
+git clone -c core.autocrlf=input -c core.symlinks=true https://github.com/apple/swift swift
+git clone https://github.com/apple/swift-cmark cmark
 git clone https://github.com/apple/swift-corelibs-libdispatch swift-corelibs-libdispatch
 git clone https://github.com/apple/swift-corelibs-foundation swift-corelibs-foundation
 git clone https://github.com/apple/swift-corelibs-xctest swift-corelibs-xctest
@@ -78,15 +78,16 @@ git clone https://github.com/compnerd/swift-build swift-build
 
 ## Acquire ICU, SQLite3, curl, libxml2 and zlib
 
-Go to [compnerd's swift-build azure page](https://dev.azure.com/compnerd/swift-build/_build) and open [Pipelines](https://dev.azure.com/compnerd/swift-build/_build) where you'll see bots (hopefully green) for:
-
-- [ICU](https://dev.azure.com/compnerd/swift-build/_build?definitionId=9)
-- [SQLite](https://dev.azure.com/compnerd/swift-build/_build?definitionId=12&_a=summary)
-- [curl](https://dev.azure.com/compnerd/swift-build/_build?definitionId=11&_a=summary)
-- [libxml2](https://dev.azure.com/compnerd/swift-build/_build?definitionId=10&_a=summary)
-- [zlib](https://dev.azure.com/compnerd/swift-build/_build?definitionId=16&_a=summary)
+```
+python -m pip install --user msrest azure-devops tabulate
+python swift-build\utilities\swift-build.py --build-id ICU --latest-artifacts --filter windows-x64 --download
+python swift-build\utilities\swift-build.py --build-id XML2 --latest-artifacts --filter windows-x64 --download
+python swift-build\utilities\swift-build.py --build-id CURL --latest-artifacts --filter windows-x64 --download
+python swift-build\utilities\swift-build.py --build-id zlib --latest-artifacts --filter windows-x64 --download
+python swift-build\utilities\swift-build.py --build-id SQLite --latest-artifacts --filter windows-x64 --download
+```
 
-Download each of the zip files and copy their contents into S:/Library. The directory structure should resemble:
+Extract the zip files, ignoring the top level directory, into `S:/Library`. The directory structure should resemble:
 
 ```
 /Library
@@ -113,29 +114,30 @@ Set up the `ucrt`, `visualc`, and `WinSDK` modules by:
 - and setup the `visualc.apinotes` located at `swift/stdlib/public/Platform/visualc.apinotes` into `${VCToolsInstallDir}/include` as `visualc.apinotes`
 
 ```cmd
-mklink "%UniversalCRTSdkDir%\Include\%UCRTVersion%\ucrt\module.modulemap" S:\toolchain\swift\stdlib\public\Platform\ucrt.modulemap
-mklink "%UniversalCRTSdkDir%\Include\%UCRTVersion%\um\module.modulemap" S:\toolchain\swift\stdlib\public\Platform\winsdk.modulemap
-mklink "%VCToolsInstallDir%\include\module.modulemap" S:\toolchain\swift\stdlib\public\Platform\visualc.modulemap
-mklink "%VCToolsInstallDir%\include\visualc.apinotes" S:\toolchain\swift\stdlib\public\Platform\visualc.apinotes
+mklink "%UniversalCRTSdkDir%\Include\%UCRTVersion%\ucrt\module.modulemap" S:\swift\stdlib\public\Platform\ucrt.modulemap
+mklink "%UniversalCRTSdkDir%\Include\%UCRTVersion%\um\module.modulemap" S:\swift\stdlib\public\Platform\winsdk.modulemap
+mklink "%VCToolsInstallDir%\include\module.modulemap" S:\swift\stdlib\public\Platform\visualc.modulemap
+mklink "%VCToolsInstallDir%\include\visualc.apinotes" S:\swift\stdlib\public\Platform\visualc.apinotes
 ```
 
 Warning: Creating the above links usually requires administrator privileges. The quick and easy way to do this is to open a second developer prompt by right clicking whatever shortcut you used to open the first one, choosing Run As Administrator, and pasting the above commands into the resulting window. You can then close the privileged prompt; this is the only step which requires elevation.
 
 ## Build the toolchain
 
 ```cmd
-md "S:\b\toolchain"
 cmake -B "S:\b\toolchain" ^
   -C S:\swift-build\cmake\caches\windows-x86_64.cmake ^
   -C S:\swift-build\cmake\caches\org.compnerd.dt.cmake ^
   -D CMAKE_BUILD_TYPE=Release ^
   -D LLVM_ENABLE_ASSERTIONS=YES ^
-  -D LLVM_ENABLE_PROJECTS="clang;clang-tools-extra;cmark;swift;lldb;lld" ^
+  -D LLVM_ENABLE_PROJECTS="clang;clang-tools-extra;lldb;lld" ^
   -D LLVM_EXTERNAL_PROJECTS="cmark;swift" ^
   -D SWIFT_PATH_TO_LIBDISPATCH_SOURCE=S:\swift-corelibs-libdispatch ^
   -D LLVM_ENABLE_PDB=YES ^
   -D LLVM_ENABLE_LIBEDIT=NO ^
   -D LLDB_ENABLE_PYTHON=YES ^
+  -D LLVM_EXTERNAL_SWIFT_SOURCE_DIR="S:/swift" ^
+  -D LLVM_EXTERNAL_CMARK_SOURCE_DIR="S:/cmark" ^
   -D SWIFT_WINDOWS_x86_64_ICU_UC_INCLUDE="S:/Library/icu-64/usr/include" ^
   -D SWIFT_WINDOWS_x86_64_ICU_UC="S:/Library/icu-64/usr/lib/icuuc64.lib" ^
   -D SWIFT_WINDOWS_x86_64_ICU_I18N_INCLUDE="S:/Library/icu-64/usr/include" ^
@@ -145,7 +147,7 @@ cmake -B "S:\b\toolchain" ^
   -D SWIFT_BUILD_DYNAMIC_STDLIB=YES ^
   -D SWIFT_BUILD_DYNAMIC_SDK_OVERLAY=YES ^
   -G Ninja ^
-  -S S:\toolchain\llvm
+  -S S:\llvm-project\llvm
 
 ninja -C S:\b\toolchain
 ```

include/swift/Demangling/Demangle.h

@@ -50,6 +50,7 @@ struct DemangleOptions {
   bool DisplayProtocolConformances = true;
   bool DisplayWhereClauses = true;
   bool DisplayEntityTypes = true;
+  bool DisplayLocalNameContexts = true;
   bool ShortenPartialApply = false;
   bool ShortenThunk = false;
   bool ShortenValueWitness = false;

include/swift/Demangling/TypeDecoder.h

@@ -716,19 +716,16 @@ class TypeDecoder {
 
         // If the tuple element is labeled, add its label to 'labels'.
         unsigned typeChildIndex = 0;
-        unsigned nameIdx = 0;
-        if (element->getChild(nameIdx)->getKind() == NodeKind::VariadicMarker) {
-          variadic = true;
-          nameIdx = 1;
-          typeChildIndex = 1;
+        if (element->getChild(typeChildIndex)->getKind() == NodeKind::VariadicMarker) {
+          return BuiltType();
         }
-        if (element->getChild(nameIdx)->getKind() == NodeKind::TupleElementName) {
+        if (element->getChild(typeChildIndex)->getKind() == NodeKind::TupleElementName) {
           // Add spaces to terminate all the previous labels if this
           // is the first we've seen.
           if (labels.empty()) labels.append(elements.size(), ' ');
 
           // Add the label and its terminator.
-          labels += element->getChild(0)->getText();
+          labels += element->getChild(typeChildIndex)->getText();
           labels += ' ';
           typeChildIndex++;
 

include/swift/Reflection/ReflectionContext.h

@@ -34,9 +34,10 @@
 #include "swift/Runtime/Unreachable.h"
 
 #include <set>
-#include <vector>
+#include <sstream>
 #include <unordered_map>
 #include <utility>
+#include <vector>
 
 namespace {
 
@@ -794,7 +795,7 @@ class ReflectionContext
   llvm::Optional<std::string> iterateConformances(
     std::function<void(StoredPointer Type, StoredPointer Proto)> Call) {
     std::string ConformancesPointerName =
-      "__swift_debug_protocolConformanceStatePointer";
+        "_swift_debug_protocolConformanceStatePointer";
     auto ConformancesAddrAddr =
       getReader().getSymbolAddress(ConformancesPointerName);
     if (!ConformancesAddrAddr)
@@ -844,9 +845,9 @@ class ReflectionContext
   llvm::Optional<std::string> iterateMetadataAllocations(
     std::function<void (MetadataAllocation<Runtime>)> Call) {
     std::string IterationEnabledName =
-      "__swift_debug_metadataAllocationIterationEnabled";
+        "_swift_debug_metadataAllocationIterationEnabled";
     std::string AllocationPoolPointerName =
-      "__swift_debug_allocationPoolPointer";
+        "_swift_debug_allocationPoolPointer";
 
     auto IterationEnabledAddr =
       getReader().getSymbolAddress(IterationEnabledName);
@@ -922,6 +923,50 @@ class ReflectionContext
     return llvm::None;
   }
 
+  llvm::Optional<std::string> iterateMetadataAllocationBacktraces(
+      std::function<void(StoredPointer, uint32_t, const StoredPointer *)>
+          Call) {
+    std::string BacktraceListName =
+        "_swift_debug_metadataAllocationBacktraceList";
+
+    auto BacktraceListAddr = getReader().getSymbolAddress(BacktraceListName);
+    if (!BacktraceListAddr)
+      return "unable to look up debug variable " + BacktraceListName;
+    auto BacktraceListNextPtr =
+        getReader().readPointer(BacktraceListAddr, sizeof(StoredPointer));
+    if (!BacktraceListNextPtr)
+      return llvm::None;
+
+    auto BacktraceListNext = BacktraceListNextPtr->getResolvedAddress();
+    while (BacktraceListNext) {
+      auto HeaderBytes = getReader().readBytes(
+          RemoteAddress(BacktraceListNext),
+          sizeof(MetadataAllocationBacktraceHeader<Runtime>));
+      auto HeaderPtr =
+          reinterpret_cast<const MetadataAllocationBacktraceHeader<Runtime> *>(
+              HeaderBytes.get());
+      if (HeaderPtr == nullptr) {
+        std::stringstream stream;
+        stream << "unable to read Next pointer 0x" << std::hex
+               << BacktraceListNext.getAddressData();
+        return stream.str();
+      }
+      auto BacktraceAddrPtr =
+          BacktraceListNext +
+          sizeof(MetadataAllocationBacktraceHeader<Runtime>);
+      auto BacktraceBytes =
+          getReader().readBytes(RemoteAddress(BacktraceAddrPtr),
+                                HeaderPtr->Count * sizeof(StoredPointer));
+      auto BacktracePtr =
+          reinterpret_cast<const StoredPointer *>(BacktraceBytes.get());
+
+      Call(HeaderPtr->Allocation, HeaderPtr->Count, BacktracePtr);
+
+      BacktraceListNext = RemoteAddress(HeaderPtr->Next);
+    }
+    return llvm::None;
+  }
+
 private:
   const TypeInfo *getClosureContextInfo(StoredPointer Context,
                                         const ClosureContextInfo &Info) {

include/swift/Runtime/Debug.h

@@ -17,11 +17,13 @@
 #ifndef SWIFT_RUNTIME_DEBUG_HELPERS_H
 #define SWIFT_RUNTIME_DEBUG_HELPERS_H
 
+#include "swift/Runtime/Config.h"
+#include "swift/Runtime/Unreachable.h"
+#include <atomic>
 #include <cstdarg>
 #include <cstdio>
+#include <functional>
 #include <stdint.h>
-#include "swift/Runtime/Config.h"
-#include "swift/Runtime/Unreachable.h"
 
 #ifdef SWIFT_HAVE_CRASHREPORTERCLIENT
 
@@ -145,6 +147,9 @@ void swift_abortDynamicReplacementDisabling();
 void dumpStackTraceEntry(unsigned index, void *framePC,
                          bool shortOutput = false);
 
+SWIFT_RUNTIME_ATTRIBUTE_NOINLINE
+bool withCurrentBacktrace(std::function<void(void **, int)> call);
+
 SWIFT_RUNTIME_ATTRIBUTE_NOINLINE
 void printCurrentBacktrace(unsigned framesToSkip = 1);
 
@@ -237,6 +242,9 @@ bool _swift_debug_metadataAllocationIterationEnabled;
 SWIFT_RUNTIME_STDLIB_SPI
 const void * const _swift_debug_allocationPoolPointer;
 
+SWIFT_RUNTIME_STDLIB_SPI
+std::atomic<const void *> _swift_debug_metadataAllocationBacktraceList;
+
 SWIFT_RUNTIME_STDLIB_SPI
 const void * const _swift_debug_protocolConformanceStatePointer;
 

include/swift/Runtime/Metadata.h

@@ -54,6 +54,13 @@ enum MetadataAllocatorTags : uint16_t {
   GenericValueMetadataTag,
 };
 
+template <typename Runtime> struct MetadataAllocationBacktraceHeader {
+  TargetPointer<Runtime, const void> Next;
+  TargetPointer<Runtime, void> Allocation;
+  uint32_t Count;
+  // Count backtrace pointers immediately follow.
+};
+
 /// The buffer used by a yield-once coroutine (such as the generalized
 /// accessors `read` and `modify`).
 struct YieldOnceBuffer {

include/swift/SIL/SILVTable.h

@@ -39,43 +39,54 @@ enum IsSerialized_t : unsigned char;
 class SILFunction;
 class SILModule;
 
+// TODO: Entry should include substitutions needed to invoke an overridden
+// generic base class method.
+class SILVTableEntry {
+  /// The declaration reference to the least-derived method visible through
+  /// the class.
+  SILDeclRef Method;
+
+  /// The function which implements the method for the class and the entry kind.
+  llvm::PointerIntPair<SILFunction *, 2, unsigned> ImplAndKind;
+
+public:
+  enum Kind : uint8_t {
+    /// The vtable entry is for a method defined directly in this class.
+    Normal,
+    /// The vtable entry is for a method defined directly in this class, and is
+    /// never overridden by subclasses.
+    NormalNonOverridden,
+    /// The vtable entry is inherited from the superclass.
+    Inherited,
+    /// The vtable entry is inherited from the superclass, and overridden
+    /// in this class.
+    Override,
+
+    // Please update the PointerIntPair above if you add/remove enums.
+  };
+
+  SILVTableEntry() : ImplAndKind(nullptr, Kind::Normal) {}
+
+  SILVTableEntry(SILDeclRef Method, SILFunction *Implementation, Kind TheKind)
+      : Method(Method), ImplAndKind(Implementation, TheKind) {}
+
+  SILDeclRef getMethod() const { return Method; }
+
+  Kind getKind() const { return Kind(ImplAndKind.getInt()); }
+  void setKind(Kind kind) { ImplAndKind.setInt(kind); }
+
+  SILFunction *getImplementation() const { return ImplAndKind.getPointer(); }
+};
+
 /// A mapping from each dynamically-dispatchable method of a class to the
 /// SILFunction that implements the method for that class.
 /// Note that dead methods are completely removed from the vtable.
-class SILVTable : public SILAllocated<SILVTable> {
+class SILVTable final : public SILAllocated<SILVTable>,
+                        llvm::TrailingObjects<SILVTable, SILVTableEntry> {
+  friend TrailingObjects;
+
 public:
-  // TODO: Entry should include substitutions needed to invoke an overridden
-  // generic base class method.
-  struct Entry {
-    enum Kind : uint8_t {
-      /// The vtable entry is for a method defined directly in this class.
-      Normal,
-      /// The vtable entry is for a method defined directly in this class, and is never overridden
-      /// by subclasses.
-      NormalNonOverridden,
-      /// The vtable entry is inherited from the superclass.
-      Inherited,
-      /// The vtable entry is inherited from the superclass, and overridden
-      /// in this class.
-      Override,
-    };
-
-    Entry()
-      : Implementation(nullptr), TheKind(Kind::Normal) { }
-
-    Entry(SILDeclRef Method, SILFunction *Implementation, Kind TheKind)
-      : Method(Method), Implementation(Implementation), TheKind(TheKind) { }
-
-    /// The declaration reference to the least-derived method visible through
-    /// the class.
-    SILDeclRef Method;
-
-    /// The function which implements the method for the class.
-    SILFunction *Implementation;
-
-    /// The entry kind.
-    Kind TheKind;
-  };
+  using Entry = SILVTableEntry;
 
   // Disallow copying into temporary objects.
   SILVTable(const SILVTable &other) = delete;
@@ -92,9 +103,6 @@ class SILVTable : public SILAllocated<SILVTable> {
   /// The number of SILVTables entries.
   unsigned NumEntries : 31;
 
-  /// Tail-allocated SILVTable entries.
-  Entry Entries[1];
-
   /// Private constructor. Create SILVTables by calling SILVTable::create.
   SILVTable(ClassDecl *c, IsSerialized_t serialized, ArrayRef<Entry> entries);
 
@@ -123,29 +131,34 @@ class SILVTable : public SILAllocated<SILVTable> {
   }
 
   /// Return all of the method entries.
-  ArrayRef<Entry> getEntries() const { return {Entries, NumEntries}; }
+  ArrayRef<Entry> getEntries() const {
+    return {getTrailingObjects<SILVTableEntry>(), NumEntries};
+  }
 
   /// Return all of the method entries mutably.
-  MutableArrayRef<Entry> getMutableEntries() { return {Entries, NumEntries}; }
+  MutableArrayRef<Entry> getMutableEntries() {
+    return {getTrailingObjects<SILVTableEntry>(), NumEntries};
+  }
 
   /// Look up the implementation function for the given method.
   Optional<Entry> getEntry(SILModule &M, SILDeclRef method) const;
 
   /// Removes entries from the vtable.
   /// \p predicate Returns true if the passed entry should be removed.
   template <typename Predicate> void removeEntries_if(Predicate predicate) {
-    Entry *end = std::remove_if(Entries, Entries + NumEntries,
-                                [&](Entry &entry) -> bool {
-      if (predicate(entry)) {
-        entry.Implementation->decrementRefCount();
-        removeFromVTableCache(entry);
-        return true;
-      }
-      return false;
-    });
-    NumEntries = end - Entries;
+    auto Entries = getMutableEntries();
+    Entry *end = std::remove_if(
+        Entries.begin(), Entries.end(), [&](Entry &entry) -> bool {
+          if (predicate(entry)) {
+            entry.getImplementation()->decrementRefCount();
+            removeFromVTableCache(entry);
+            return true;
+          }
+          return false;
+        });
+    NumEntries = std::distance(Entries.begin(), end);
   }
-                    
+
   /// Verify that the vtable is well-formed for the given class.
   void verify(const SILModule &M) const;