diff --git a/llvm/lib/Transforms/Coroutines/CoroCloner.h b/llvm/lib/Transforms/Coroutines/CoroCloner.h new file mode 100644 index 0000000000000..d1887980fb3bc --- /dev/null +++ b/llvm/lib/Transforms/Coroutines/CoroCloner.h @@ -0,0 +1,155 @@ +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Helper class for splitting a coroutine into separate functions. For example +// the returned-continuation coroutine is split into separate continuation +// functions. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_TRANSFORMS_COROUTINES_COROCLONER_H +#define LLVM_LIB_TRANSFORMS_COROUTINES_COROCLONER_H + +#include "llvm/IR/Function.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/Support/TimeProfiler.h" +#include "llvm/Transforms/Coroutines/ABI.h" +#include "llvm/Transforms/Coroutines/CoroInstr.h" +#include "llvm/Transforms/Utils/ValueMapper.h" + +namespace llvm { + +namespace coro { + +enum class CloneKind { + /// The shared resume function for a switch lowering. + SwitchResume, + + /// The shared unwind function for a switch lowering. + SwitchUnwind, + + /// The shared cleanup function for a switch lowering. + SwitchCleanup, + + /// An individual continuation function. + Continuation, + + /// An async resume function. + Async, +}; + +class BaseCloner { +protected: + Function &OrigF; + const Twine &Suffix; + coro::Shape &Shape; + CloneKind FKind; + IRBuilder<> Builder; + TargetTransformInfo &TTI; + + ValueToValueMapTy VMap; + Function *NewF = nullptr; + Value *NewFramePtr = nullptr; + + /// The active suspend instruction; meaningful only for continuation and async + /// ABIs. + AnyCoroSuspendInst *ActiveSuspend = nullptr; + + /// Create a cloner for a continuation lowering. + BaseCloner(Function &OrigF, const Twine &Suffix, coro::Shape &Shape, + Function *NewF, AnyCoroSuspendInst *ActiveSuspend, + TargetTransformInfo &TTI) + : OrigF(OrigF), Suffix(Suffix), Shape(Shape), + FKind(Shape.ABI == ABI::Async ? CloneKind::Async + : CloneKind::Continuation), + Builder(OrigF.getContext()), TTI(TTI), NewF(NewF), + ActiveSuspend(ActiveSuspend) { + assert(Shape.ABI == ABI::Retcon || Shape.ABI == ABI::RetconOnce || + Shape.ABI == ABI::Async); + assert(NewF && "need existing function for continuation"); + assert(ActiveSuspend && "need active suspend point for continuation"); + } + +public: + BaseCloner(Function &OrigF, const Twine &Suffix, coro::Shape &Shape, + CloneKind FKind, TargetTransformInfo &TTI) + : OrigF(OrigF), Suffix(Suffix), Shape(Shape), FKind(FKind), + Builder(OrigF.getContext()), TTI(TTI) {} + + virtual ~BaseCloner() {} + + /// Create a clone for a continuation lowering. + static Function *createClone(Function &OrigF, const Twine &Suffix, + coro::Shape &Shape, Function *NewF, + AnyCoroSuspendInst *ActiveSuspend, + TargetTransformInfo &TTI) { + assert(Shape.ABI == ABI::Retcon || Shape.ABI == ABI::RetconOnce || + Shape.ABI == ABI::Async); + TimeTraceScope FunctionScope("BaseCloner"); + + BaseCloner Cloner(OrigF, Suffix, Shape, NewF, ActiveSuspend, TTI); + Cloner.create(); + return Cloner.getFunction(); + } + + Function *getFunction() const { + assert(NewF != nullptr && "declaration not yet set"); + return NewF; + } + + virtual void create(); + +protected: + bool isSwitchDestroyFunction() { + switch (FKind) { + case CloneKind::Async: + case CloneKind::Continuation: + case CloneKind::SwitchResume: + return false; + case CloneKind::SwitchUnwind: + case CloneKind::SwitchCleanup: + return true; + } + llvm_unreachable("Unknown ClonerKind enum"); + } + + void replaceEntryBlock(); + Value *deriveNewFramePointer(); + void replaceRetconOrAsyncSuspendUses(); + void replaceCoroSuspends(); + void replaceCoroEnds(); + void replaceSwiftErrorOps(); + void salvageDebugInfo(); + void handleFinalSuspend(); +}; + +class SwitchCloner : public BaseCloner { +protected: + /// Create a cloner for a switch lowering. + SwitchCloner(Function &OrigF, const Twine &Suffix, coro::Shape &Shape, + CloneKind FKind, TargetTransformInfo &TTI) + : BaseCloner(OrigF, Suffix, Shape, FKind, TTI) {} + + void create() override; + +public: + /// Create a clone for a switch lowering. + static Function *createClone(Function &OrigF, const Twine &Suffix, + coro::Shape &Shape, CloneKind FKind, + TargetTransformInfo &TTI) { + assert(Shape.ABI == ABI::Switch); + TimeTraceScope FunctionScope("SwitchCloner"); + + SwitchCloner Cloner(OrigF, Suffix, Shape, FKind, TTI); + Cloner.create(); + return Cloner.getFunction(); + } +}; + +} // end namespace coro + +} // end namespace llvm + +#endif // LLVM_LIB_TRANSFORMS_COROUTINES_COROCLONER_H diff --git a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp index 441a83310c632..d3732fec603f6 100644 --- a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp +++ b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp @@ -1133,7 +1133,7 @@ static void insertSpills(const FrameDataInfo &FrameData, coro::Shape &Shape) { bool AllowUnresolved = false; // This dbg.declare is preserved for all coro-split function // fragments. It will be unreachable in the main function, and - // processed by coro::salvageDebugInfo() by CoroCloner. + // processed by coro::salvageDebugInfo() by the Cloner. if (UseNewDbgInfoFormat) { DbgVariableRecord *NewDVR = new DbgVariableRecord( ValueAsMetadata::get(CurrentReload), DDI->getVariable(), diff --git a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp index 8a5bae9f6f0d4..3808147fc2600 100644 --- a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp +++ b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp @@ -19,6 +19,7 @@ //===----------------------------------------------------------------------===// #include "llvm/Transforms/Coroutines/CoroSplit.h" +#include "CoroCloner.h" #include "CoroInternal.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/PriorityWorklist.h" @@ -44,10 +45,8 @@ #include "llvm/IR/DataLayout.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/Dominators.h" -#include "llvm/IR/Function.h" #include "llvm/IR/GlobalValue.h" #include "llvm/IR/GlobalVariable.h" -#include "llvm/IR/IRBuilder.h" #include "llvm/IR/InstIterator.h" #include "llvm/IR/InstrTypes.h" #include "llvm/IR/Instruction.h" @@ -61,17 +60,13 @@ #include "llvm/Support/Casting.h" #include "llvm/Support/Debug.h" #include "llvm/Support/PrettyStackTrace.h" -#include "llvm/Support/TimeProfiler.h" #include "llvm/Support/raw_ostream.h" -#include "llvm/Transforms/Coroutines/ABI.h" -#include "llvm/Transforms/Coroutines/CoroInstr.h" #include "llvm/Transforms/Coroutines/MaterializationUtils.h" #include "llvm/Transforms/Scalar.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" #include "llvm/Transforms/Utils/CallGraphUpdater.h" #include "llvm/Transforms/Utils/Cloning.h" #include "llvm/Transforms/Utils/Local.h" -#include "llvm/Transforms/Utils/ValueMapper.h" #include #include #include @@ -82,122 +77,6 @@ using namespace llvm; #define DEBUG_TYPE "coro-split" -namespace { - -/// A little helper class for building -class CoroCloner { -public: - enum class Kind { - /// The shared resume function for a switch lowering. - SwitchResume, - - /// The shared unwind function for a switch lowering. - SwitchUnwind, - - /// The shared cleanup function for a switch lowering. - SwitchCleanup, - - /// An individual continuation function. - Continuation, - - /// An async resume function. - Async, - }; - -private: - Function &OrigF; - Function *NewF; - const Twine &Suffix; - coro::Shape &Shape; - Kind FKind; - ValueToValueMapTy VMap; - IRBuilder<> Builder; - Value *NewFramePtr = nullptr; - - /// The active suspend instruction; meaningful only for continuation and async - /// ABIs. - AnyCoroSuspendInst *ActiveSuspend = nullptr; - - TargetTransformInfo &TTI; - - /// Create a cloner for a switch lowering. - CoroCloner(Function &OrigF, const Twine &Suffix, coro::Shape &Shape, - Kind FKind, TargetTransformInfo &TTI) - : OrigF(OrigF), NewF(nullptr), Suffix(Suffix), Shape(Shape), FKind(FKind), - Builder(OrigF.getContext()), TTI(TTI) { - assert(Shape.ABI == coro::ABI::Switch); - } - - /// Create a cloner for a continuation lowering. - CoroCloner(Function &OrigF, const Twine &Suffix, coro::Shape &Shape, - Function *NewF, AnyCoroSuspendInst *ActiveSuspend, - TargetTransformInfo &TTI) - : OrigF(OrigF), NewF(NewF), Suffix(Suffix), Shape(Shape), - FKind(Shape.ABI == coro::ABI::Async ? Kind::Async : Kind::Continuation), - Builder(OrigF.getContext()), ActiveSuspend(ActiveSuspend), TTI(TTI) { - assert(Shape.ABI == coro::ABI::Retcon || - Shape.ABI == coro::ABI::RetconOnce || Shape.ABI == coro::ABI::Async); - assert(NewF && "need existing function for continuation"); - assert(ActiveSuspend && "need active suspend point for continuation"); - } - -public: - /// Create a clone for a switch lowering. - static Function *createClone(Function &OrigF, const Twine &Suffix, - coro::Shape &Shape, Kind FKind, - TargetTransformInfo &TTI) { - TimeTraceScope FunctionScope("CoroCloner"); - - CoroCloner Cloner(OrigF, Suffix, Shape, FKind, TTI); - Cloner.create(); - return Cloner.getFunction(); - } - - /// Create a clone for a continuation lowering. - static Function *createClone(Function &OrigF, const Twine &Suffix, - coro::Shape &Shape, Function *NewF, - AnyCoroSuspendInst *ActiveSuspend, - TargetTransformInfo &TTI) { - TimeTraceScope FunctionScope("CoroCloner"); - - CoroCloner Cloner(OrigF, Suffix, Shape, NewF, ActiveSuspend, TTI); - Cloner.create(); - return Cloner.getFunction(); - } - - Function *getFunction() const { - assert(NewF != nullptr && "declaration not yet set"); - return NewF; - } - - void create(); - -private: - bool isSwitchDestroyFunction() { - switch (FKind) { - case Kind::Async: - case Kind::Continuation: - case Kind::SwitchResume: - return false; - case Kind::SwitchUnwind: - case Kind::SwitchCleanup: - return true; - } - llvm_unreachable("Unknown CoroCloner::Kind enum"); - } - - void replaceEntryBlock(); - Value *deriveNewFramePointer(); - void replaceRetconOrAsyncSuspendUses(); - void replaceCoroSuspends(); - void replaceCoroEnds(); - void replaceSwiftErrorOps(); - void salvageDebugInfo(); - void handleFinalSuspend(); -}; - -} // end anonymous namespace - // FIXME: // Lower the intrinisc in CoroEarly phase if coroutine frame doesn't escape // and it is known that other transformations, for example, sanitizers @@ -523,7 +402,7 @@ static void replaceCoroEnd(AnyCoroEndInst *End, const coro::Shape &Shape, // is possible since the coroutine is considered suspended at the final suspend // point if promise.unhandled_exception() exits via an exception), we can // remove the last case. -void CoroCloner::handleFinalSuspend() { +void coro::BaseCloner::handleFinalSuspend() { assert(Shape.ABI == coro::ABI::Switch && Shape.SwitchLowering.HasFinalSuspend); @@ -587,7 +466,7 @@ static Function *createCloneDeclaration(Function &OrigF, coro::Shape &Shape, /// arguments to the continuation function. /// /// This assumes that the builder has a meaningful insertion point. -void CoroCloner::replaceRetconOrAsyncSuspendUses() { +void coro::BaseCloner::replaceRetconOrAsyncSuspendUses() { assert(Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce || Shape.ABI == coro::ABI::Async); @@ -635,7 +514,7 @@ void CoroCloner::replaceRetconOrAsyncSuspendUses() { NewS->replaceAllUsesWith(Aggr); } -void CoroCloner::replaceCoroSuspends() { +void coro::BaseCloner::replaceCoroSuspends() { Value *SuspendResult; switch (Shape.ABI) { @@ -672,7 +551,7 @@ void CoroCloner::replaceCoroSuspends() { } } -void CoroCloner::replaceCoroEnds() { +void coro::BaseCloner::replaceCoroEnds() { for (AnyCoroEndInst *CE : Shape.CoroEnds) { // We use a null call graph because there's no call graph node for // the cloned function yet. We'll just be rebuilding that later. @@ -751,11 +630,11 @@ collectDbgVariableIntrinsics(Function &F) { return {Intrinsics, DbgVariableRecords}; } -void CoroCloner::replaceSwiftErrorOps() { +void coro::BaseCloner::replaceSwiftErrorOps() { ::replaceSwiftErrorOps(*NewF, Shape, &VMap); } -void CoroCloner::salvageDebugInfo() { +void coro::BaseCloner::salvageDebugInfo() { auto [Worklist, DbgVariableRecords] = collectDbgVariableIntrinsics(*NewF); SmallDenseMap ArgToAllocaMap; @@ -792,7 +671,7 @@ void CoroCloner::salvageDebugInfo() { for_each(DbgVariableRecords, RemoveOne); } -void CoroCloner::replaceEntryBlock() { +void coro::BaseCloner::replaceEntryBlock() { // In the original function, the AllocaSpillBlock is a block immediately // following the allocation of the frame object which defines GEPs for // all the allocas that have been moved into the frame, and it ends by @@ -860,7 +739,7 @@ void CoroCloner::replaceEntryBlock() { } /// Derive the value of the new frame pointer. -Value *CoroCloner::deriveNewFramePointer() { +Value *coro::BaseCloner::deriveNewFramePointer() { // Builder should be inserting to the front of the new entry block. switch (Shape.ABI) { @@ -984,12 +863,8 @@ static void addSwiftSelfAttrs(AttributeList &Attrs, LLVMContext &Context, /// Clone the body of the original function into a resume function of /// some sort. -void CoroCloner::create() { - // Create the new function if we don't already have one. - if (!NewF) { - NewF = createCloneDeclaration(OrigF, Shape, Suffix, - OrigF.getParent()->end(), ActiveSuspend); - } +void coro::BaseCloner::create() { + assert(NewF); // Replace all args with dummy instructions. If an argument is the old frame // pointer, the dummy will be replaced by the new frame pointer once it is @@ -1213,12 +1088,20 @@ void CoroCloner::create() { // Salvage debug info that points into the coroutine frame. salvageDebugInfo(); +} + +void coro::SwitchCloner::create() { + // Create a new function matching the original type + NewF = createCloneDeclaration(OrigF, Shape, Suffix, OrigF.getParent()->end(), + ActiveSuspend); + + // Clone the function + coro::BaseCloner::create(); // Eliminate coro.free from the clones, replacing it with 'null' in cleanup, // to suppress deallocation code. - if (Shape.ABI == coro::ABI::Switch) - coro::replaceCoroFree(cast(VMap[Shape.CoroBegin->getId()]), - /*Elide=*/FKind == CoroCloner::Kind::SwitchCleanup); + coro::replaceCoroFree(cast(VMap[Shape.CoroBegin->getId()]), + /*Elide=*/FKind == coro::CloneKind::SwitchCleanup); } static void updateAsyncFuncPointerContextSize(coro::Shape &Shape) { @@ -1495,12 +1378,12 @@ struct SwitchCoroutineSplitter { // setting new entry block and replacing coro.suspend an appropriate value // to force resume or cleanup pass for every suspend point. createResumeEntryBlock(F, Shape); - auto *ResumeClone = CoroCloner::createClone( - F, ".resume", Shape, CoroCloner::Kind::SwitchResume, TTI); - auto *DestroyClone = CoroCloner::createClone( - F, ".destroy", Shape, CoroCloner::Kind::SwitchUnwind, TTI); - auto *CleanupClone = CoroCloner::createClone( - F, ".cleanup", Shape, CoroCloner::Kind::SwitchCleanup, TTI); + auto *ResumeClone = coro::SwitchCloner::createClone( + F, ".resume", Shape, coro::CloneKind::SwitchResume, TTI); + auto *DestroyClone = coro::SwitchCloner::createClone( + F, ".destroy", Shape, coro::CloneKind::SwitchUnwind, TTI); + auto *CleanupClone = coro::SwitchCloner::createClone( + F, ".cleanup", Shape, coro::CloneKind::SwitchCleanup, TTI); postSplitCleanup(*ResumeClone); postSplitCleanup(*DestroyClone); @@ -1889,8 +1772,8 @@ void coro::AsyncABI::splitCoroutine(Function &F, coro::Shape &Shape, auto *Suspend = CS; auto *Clone = Clones[Idx]; - CoroCloner::createClone(F, "resume." + Twine(Idx), Shape, Clone, Suspend, - TTI); + coro::BaseCloner::createClone(F, "resume." + Twine(Idx), Shape, Clone, + Suspend, TTI); } } @@ -2020,8 +1903,8 @@ void coro::AnyRetconABI::splitCoroutine(Function &F, coro::Shape &Shape, auto Suspend = CS; auto Clone = Clones[Idx]; - CoroCloner::createClone(F, "resume." + Twine(Idx), Shape, Clone, Suspend, - TTI); + coro::BaseCloner::createClone(F, "resume." + Twine(Idx), Shape, Clone, + Suspend, TTI); } }