diff --git a/llvm/include/llvm/CodeGen/MIRYamlMapping.h b/llvm/include/llvm/CodeGen/MIRYamlMapping.h index dbad3469d047d..772cf448e52f3 100644 --- a/llvm/include/llvm/CodeGen/MIRYamlMapping.h +++ b/llvm/include/llvm/CodeGen/MIRYamlMapping.h @@ -631,6 +631,79 @@ LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::yaml::CalledGlobal) namespace llvm { namespace yaml { +// Struct representing one save/restore point in the +// 'savePoint' / 'restorePoint' list. One point consists of machine basic block +// name and list of saved/restored in this basic block registers. There are +// two forms of Save/Restore point representation: +// 1. Without explicit register enumeration: +// savePoint: '%bb.n' +// restorePoint: '%bb.n' +// in this case we assume that all CalleeSavedRegisters +// are splilled/restored in these points +// 2. With explicit register: +// savePoint: +// - point: '%bb.1' +// registers: +// - '$rbx' +// - '$r12' +// ... +// restorePoint: +// - point: '%bb.1' +// registers: +// - '$rbx' +// - '$r12' +// If this representation form is used and no register is saved/restored in the +// selected BB, the empty list of register should be specified ( i.e. registers: +// []) +struct SaveRestorePointEntry { + StringValue Point; + std::vector Registers; + + bool operator==(const SaveRestorePointEntry &Other) const { + return Point == Other.Point && Registers == Other.Registers; + } +}; + +using SaveRestorePoints = + std::variant, StringValue>; + +template <> struct PolymorphicTraits { + + static NodeKind getKind(const SaveRestorePoints &SRPoints) { + if (std::holds_alternative>(SRPoints)) + return NodeKind::Sequence; + if (std::holds_alternative(SRPoints)) + return NodeKind::Scalar; + llvm_unreachable("Unsupported NodeKind of SaveRestorePoints"); + } + + static SaveRestorePointEntry &getAsMap(SaveRestorePoints &SRPoints) { + llvm_unreachable("SaveRestorePoints can't be represented as Map"); + } + + static std::vector & + getAsSequence(SaveRestorePoints &SRPoints) { + if (!std::holds_alternative>(SRPoints)) + SRPoints = std::vector(); + + return std::get>(SRPoints); + } + + static StringValue &getAsScalar(SaveRestorePoints &SRPoints) { + if (!std::holds_alternative(SRPoints)) + SRPoints = StringValue(); + + return std::get(SRPoints); + } +}; + +template <> struct MappingTraits { + static void mapping(IO &YamlIO, SaveRestorePointEntry &Entry) { + YamlIO.mapRequired("point", Entry.Point); + YamlIO.mapRequired("registers", Entry.Registers); + } +}; + template <> struct MappingTraits { static void mapping(IO &YamlIO, MachineJumpTable &JT) { YamlIO.mapRequired("kind", JT.Kind); @@ -639,6 +712,14 @@ template <> struct MappingTraits { } }; +} // namespace yaml +} // namespace llvm + +LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::yaml::SaveRestorePointEntry) + +namespace llvm { +namespace yaml { + /// Serializable representation of MachineFrameInfo. /// /// Doesn't serialize attributes like 'StackAlignment', 'IsStackRealignable' and @@ -666,8 +747,8 @@ struct MachineFrameInfo { bool HasTailCall = false; bool IsCalleeSavedInfoValid = false; unsigned LocalFrameSize = 0; - StringValue SavePoint; - StringValue RestorePoint; + SaveRestorePoints SavePoints; + SaveRestorePoints RestorePoints; bool operator==(const MachineFrameInfo &Other) const { return IsFrameAddressTaken == Other.IsFrameAddressTaken && @@ -688,7 +769,8 @@ struct MachineFrameInfo { HasMustTailInVarArgFunc == Other.HasMustTailInVarArgFunc && HasTailCall == Other.HasTailCall && LocalFrameSize == Other.LocalFrameSize && - SavePoint == Other.SavePoint && RestorePoint == Other.RestorePoint && + SavePoints == Other.SavePoints && + RestorePoints == Other.RestorePoints && IsCalleeSavedInfoValid == Other.IsCalleeSavedInfoValid; } }; @@ -720,10 +802,14 @@ template <> struct MappingTraits { YamlIO.mapOptional("isCalleeSavedInfoValid", MFI.IsCalleeSavedInfoValid, false); YamlIO.mapOptional("localFrameSize", MFI.LocalFrameSize, (unsigned)0); - YamlIO.mapOptional("savePoint", MFI.SavePoint, - StringValue()); // Don't print it out when it's empty. - YamlIO.mapOptional("restorePoint", MFI.RestorePoint, - StringValue()); // Don't print it out when it's empty. + YamlIO.mapOptional( + "savePoint", MFI.SavePoints, + SaveRestorePoints( + StringValue())); // Don't print it out when it's empty. + YamlIO.mapOptional( + "restorePoint", MFI.RestorePoints, + SaveRestorePoints( + StringValue())); // Don't print it out when it's empty. } }; diff --git a/llvm/include/llvm/CodeGen/MachineFrameInfo.h b/llvm/include/llvm/CodeGen/MachineFrameInfo.h index 213b7ec6b3fbf..546792ee0c30f 100644 --- a/llvm/include/llvm/CodeGen/MachineFrameInfo.h +++ b/llvm/include/llvm/CodeGen/MachineFrameInfo.h @@ -58,9 +58,9 @@ class CalleeSavedInfo { explicit CalleeSavedInfo(unsigned R, int FI = 0) : Reg(R), FrameIdx(FI) {} // Accessors. - Register getReg() const { return Reg; } - int getFrameIdx() const { return FrameIdx; } - unsigned getDstReg() const { return DstReg; } + Register getReg() const { return Reg; } + int getFrameIdx() const { return FrameIdx; } + unsigned getDstReg() const { return DstReg; } void setFrameIdx(int FI) { FrameIdx = FI; SpilledToReg = false; @@ -74,6 +74,36 @@ class CalleeSavedInfo { bool isSpilledToReg() const { return SpilledToReg; } }; +class SaveRestorePoints { +public: + using PointsMap = DenseMap>; + +private: + PointsMap Map; + +public: + const PointsMap &get() const { return Map; } + + const std::vector getCSInfo(MachineBasicBlock *MBB) const { + return Map.lookup(MBB); + } + + void set(PointsMap &&CSI) { Map = std::move(CSI); } + + MachineBasicBlock *findAny(const CalleeSavedInfo &Match) const { + for (auto [BB, CSIV] : Map) { + for (auto &CSI : CSIV) { + if (CSI.getReg() == Match.getReg()) + return BB; + } + } + return nullptr; + } + + void clear() { Map.clear(); } + bool empty() const { return Map.empty(); } +}; + /// The MachineFrameInfo class represents an abstract stack frame until /// prolog/epilog code is inserted. This class is key to allowing stack frame /// representation optimizations, such as frame pointer elimination. It also @@ -331,9 +361,16 @@ class MachineFrameInfo { bool HasTailCall = false; /// Not null, if shrink-wrapping found a better place for the prologue. - MachineBasicBlock *Save = nullptr; + MachineBasicBlock *Prolog = nullptr; /// Not null, if shrink-wrapping found a better place for the epilogue. - MachineBasicBlock *Restore = nullptr; + MachineBasicBlock *Epilog = nullptr; + + /// Not empty, if shrink-wrapping found a better place for saving callee + /// saves. + SaveRestorePoints SavePoints; + /// Not empty, if shrink-wrapping found a better place for restoring callee + /// saves. + SaveRestorePoints RestorePoints; /// Size of the UnsafeStack Frame uint64_t UnsafeStackSize = 0; @@ -809,6 +846,28 @@ class MachineFrameInfo { /// \copydoc getCalleeSavedInfo() std::vector &getCalleeSavedInfo() { return CSInfo; } + /// Returns callee saved info vector for provided save point in + /// the current function. + const std::vector + getSaveCSInfo(MachineBasicBlock *MBB) const { + return SavePoints.getCSInfo(MBB); + } + + /// Returns callee saved info vector for provided restore point + /// in the current function. + const std::vector + getRestoreCSInfo(MachineBasicBlock *MBB) const { + return RestorePoints.getCSInfo(MBB); + } + + MachineBasicBlock *findSpilledIn(const CalleeSavedInfo &CSI) const { + return SavePoints.findAny(CSI); + } + + MachineBasicBlock *findRestoredIn(const CalleeSavedInfo &CSI) const { + return RestorePoints.findAny(CSI); + } + /// Used by prolog/epilog inserter to set the function's callee saved /// information. void setCalleeSavedInfo(std::vector CSI) { @@ -820,10 +879,38 @@ class MachineFrameInfo { void setCalleeSavedInfoValid(bool v) { CSIValid = v; } - MachineBasicBlock *getSavePoint() const { return Save; } - void setSavePoint(MachineBasicBlock *NewSave) { Save = NewSave; } - MachineBasicBlock *getRestorePoint() const { return Restore; } - void setRestorePoint(MachineBasicBlock *NewRestore) { Restore = NewRestore; } + const SaveRestorePoints::PointsMap &getRestorePoints() const { + return RestorePoints.get(); + } + + const SaveRestorePoints::PointsMap &getSavePoints() const { + return SavePoints.get(); + } + + void setSavePoints(SaveRestorePoints::PointsMap NewSavePoints) { + SavePoints.set(std::move(NewSavePoints)); + } + + void setRestorePoints(SaveRestorePoints::PointsMap NewRestorePoints) { + RestorePoints.set(std::move(NewRestorePoints)); + } + + static const SaveRestorePoints::PointsMap constructSaveRestorePoints( + const SaveRestorePoints::PointsMap &SRPoints, + const DenseMap &BBMap) { + SaveRestorePoints::PointsMap Pts{}; + for (auto &Src : SRPoints) + Pts.insert({BBMap.find(Src.first)->second, Src.second}); + return Pts; + } + + MachineBasicBlock *getProlog() const { return Prolog; } + void setProlog(MachineBasicBlock *BB) { Prolog = BB; } + MachineBasicBlock *getEpilog() const { return Epilog; } + void setEpilog(MachineBasicBlock *BB) { Epilog = BB; } + + void clearSavePoints() { SavePoints.clear(); } + void clearRestorePoints() { RestorePoints.clear(); } uint64_t getUnsafeStackSize() const { return UnsafeStackSize; } void setUnsafeStackSize(uint64_t Size) { UnsafeStackSize = Size; } diff --git a/llvm/lib/CodeGen/MIRParser/MIRParser.cpp b/llvm/lib/CodeGen/MIRParser/MIRParser.cpp index de2fe925c2d5c..0c5a8df3f8a22 100644 --- a/llvm/lib/CodeGen/MIRParser/MIRParser.cpp +++ b/llvm/lib/CodeGen/MIRParser/MIRParser.cpp @@ -124,6 +124,10 @@ class MIRParserImpl { bool initializeFrameInfo(PerFunctionMIParsingState &PFS, const yaml::MachineFunction &YamlMF); + bool initializeSaveRestorePoints(PerFunctionMIParsingState &PFS, + const yaml::SaveRestorePoints &YamlSRPoints, + bool IsSavePoints); + bool initializeCallSiteInfo(PerFunctionMIParsingState &PFS, const yaml::MachineFunction &YamlMF); @@ -851,18 +855,12 @@ bool MIRParserImpl::initializeFrameInfo(PerFunctionMIParsingState &PFS, MFI.setHasTailCall(YamlMFI.HasTailCall); MFI.setCalleeSavedInfoValid(YamlMFI.IsCalleeSavedInfoValid); MFI.setLocalFrameSize(YamlMFI.LocalFrameSize); - if (!YamlMFI.SavePoint.Value.empty()) { - MachineBasicBlock *MBB = nullptr; - if (parseMBBReference(PFS, MBB, YamlMFI.SavePoint)) - return true; - MFI.setSavePoint(MBB); - } - if (!YamlMFI.RestorePoint.Value.empty()) { - MachineBasicBlock *MBB = nullptr; - if (parseMBBReference(PFS, MBB, YamlMFI.RestorePoint)) - return true; - MFI.setRestorePoint(MBB); - } + if (initializeSaveRestorePoints(PFS, YamlMFI.SavePoints, + /*IsSavePoints=*/true)) + return true; + if (initializeSaveRestorePoints(PFS, YamlMFI.RestorePoints, + /*IsSavePoints=*/false)) + return true; std::vector CSIInfo; // Initialize the fixed frame objects. @@ -1077,6 +1075,52 @@ bool MIRParserImpl::initializeConstantPool(PerFunctionMIParsingState &PFS, return false; } +// Return true if basic block was incorrectly specified in MIR +bool MIRParserImpl::initializeSaveRestorePoints( + PerFunctionMIParsingState &PFS, const yaml::SaveRestorePoints &YamlSRPoints, + bool IsSavePoints) { + SMDiagnostic Error; + MachineBasicBlock *MBB = nullptr; + llvm::SaveRestorePoints::PointsMap SRPoints; + MachineFunction &MF = PFS.MF; + MachineFrameInfo &MFI = MF.getFrameInfo(); + + if (std::holds_alternative>( + YamlSRPoints)) { + const auto &VectorRepr = + std::get>(YamlSRPoints); + if (VectorRepr.empty()) + return false; + std::vector Registers; + for (const auto &Entry : VectorRepr) { + const auto &MBBSource = Entry.Point; + if (parseMBBReference(PFS, MBB, MBBSource.Value)) + return true; + Registers.clear(); + for (auto &RegStr : Entry.Registers) { + Register Reg; + if (parseNamedRegisterReference(PFS, Reg, RegStr.Value, Error)) + return error(Error, RegStr.SourceRange); + Registers.push_back(CalleeSavedInfo(Reg)); + } + SRPoints.insert(std::make_pair(MBB, Registers)); + } + } else { + yaml::StringValue StringRepr = std::get(YamlSRPoints); + if (StringRepr.Value.empty()) + return false; + if (parseMBBReference(PFS, MBB, StringRepr)) + return true; + SRPoints.insert(std::make_pair(MBB, MFI.getCalleeSavedInfo())); + } + + if (IsSavePoints) + MFI.setSavePoints(SRPoints); + else + MFI.setRestorePoints(SRPoints); + return false; +} + bool MIRParserImpl::initializeJumpTableInfo(PerFunctionMIParsingState &PFS, const yaml::MachineJumpTable &YamlJTI) { MachineJumpTableInfo *JTI = PFS.MF.getOrCreateJumpTableInfo(YamlJTI.Kind); diff --git a/llvm/lib/CodeGen/MIRPrinter.cpp b/llvm/lib/CodeGen/MIRPrinter.cpp index 0b41c90442a5d..e03d780ee7ec2 100644 --- a/llvm/lib/CodeGen/MIRPrinter.cpp +++ b/llvm/lib/CodeGen/MIRPrinter.cpp @@ -117,7 +117,10 @@ class MIRPrinter { const MachineRegisterInfo &RegInfo, const TargetRegisterInfo *TRI); void convert(ModuleSlotTracker &MST, yaml::MachineFrameInfo &YamlMFI, - const MachineFrameInfo &MFI); + const MachineFrameInfo &MFI, const TargetRegisterInfo *TRI); + void convert(ModuleSlotTracker &MST, yaml::SaveRestorePoints &YamlSRPoints, + const llvm::SaveRestorePoints::PointsMap &SRP, + const TargetRegisterInfo *TRI); void convert(yaml::MachineFunction &MF, const MachineConstantPool &ConstantPool); void convert(ModuleSlotTracker &MST, yaml::MachineJumpTable &YamlJTI, @@ -238,7 +241,8 @@ void MIRPrinter::print(const MachineFunction &MF) { convert(YamlMF, MF, MF.getRegInfo(), MF.getSubtarget().getRegisterInfo()); MachineModuleSlotTracker MST(MMI, &MF); MST.incorporateFunction(MF.getFunction()); - convert(MST, YamlMF.FrameInfo, MF.getFrameInfo()); + convert(MST, YamlMF.FrameInfo, MF.getFrameInfo(), + MF.getSubtarget().getRegisterInfo()); convertStackObjects(YamlMF, MF, MST); convertEntryValueObjects(YamlMF, MF, MST); convertCallSiteObjects(YamlMF, MF, MST); @@ -377,7 +381,8 @@ void MIRPrinter::convert(yaml::MachineFunction &YamlMF, void MIRPrinter::convert(ModuleSlotTracker &MST, yaml::MachineFrameInfo &YamlMFI, - const MachineFrameInfo &MFI) { + const MachineFrameInfo &MFI, + const TargetRegisterInfo *TRI) { YamlMFI.IsFrameAddressTaken = MFI.isFrameAddressTaken(); YamlMFI.IsReturnAddressTaken = MFI.isReturnAddressTaken(); YamlMFI.HasStackMap = MFI.hasStackMap(); @@ -397,14 +402,10 @@ void MIRPrinter::convert(ModuleSlotTracker &MST, YamlMFI.HasTailCall = MFI.hasTailCall(); YamlMFI.IsCalleeSavedInfoValid = MFI.isCalleeSavedInfoValid(); YamlMFI.LocalFrameSize = MFI.getLocalFrameSize(); - if (MFI.getSavePoint()) { - raw_string_ostream StrOS(YamlMFI.SavePoint.Value); - StrOS << printMBBReference(*MFI.getSavePoint()); - } - if (MFI.getRestorePoint()) { - raw_string_ostream StrOS(YamlMFI.RestorePoint.Value); - StrOS << printMBBReference(*MFI.getRestorePoint()); - } + if (!MFI.getSavePoints().empty()) + convert(MST, YamlMFI.SavePoints, MFI.getSavePoints(), TRI); + if (!MFI.getRestorePoints().empty()) + convert(MST, YamlMFI.RestorePoints, MFI.getRestorePoints(), TRI); } void MIRPrinter::convertEntryValueObjects(yaml::MachineFunction &YMF, @@ -646,6 +647,41 @@ void MIRPrinter::convert(yaml::MachineFunction &MF, } } +void MIRPrinter::convert(ModuleSlotTracker &MST, + yaml::SaveRestorePoints &YamlSRPoints, + const llvm::SaveRestorePoints::PointsMap &SRPoints, + const TargetRegisterInfo *TRI) { + auto &Points = + std::get>(YamlSRPoints); + for (const auto &MBBEntry : SRPoints) { + SmallString<16> Str; + yaml::SaveRestorePointEntry Entry; + raw_svector_ostream StrOS(Str); + StrOS << printMBBReference(*MBBEntry.first); + Entry.Point = StrOS.str().str(); + Str.clear(); + for (const CalleeSavedInfo &Info : MBBEntry.second) { + if (Info.getReg()) { + StrOS << printReg(Info.getReg(), TRI); + Entry.Registers.push_back(StrOS.str().str()); + Str.clear(); + } + } + // Sort here needed for stable output for lit tests + std::sort(Entry.Registers.begin(), Entry.Registers.end(), + [](const yaml::StringValue &Lhs, const yaml::StringValue &Rhs) { + return Lhs.Value < Rhs.Value; + }); + Points.push_back(Entry); + } + // Sort here needed for stable output for lit tests + std::sort(Points.begin(), Points.end(), + [](const yaml::SaveRestorePointEntry &Lhs, + const yaml::SaveRestorePointEntry &Rhs) { + return Lhs.Point.Value < Rhs.Point.Value; + }); +} + void MIRPrinter::convert(ModuleSlotTracker &MST, yaml::MachineJumpTable &YamlJTI, const MachineJumpTableInfo &JTI) { diff --git a/llvm/lib/CodeGen/MachineFrameInfo.cpp b/llvm/lib/CodeGen/MachineFrameInfo.cpp index e4b993850f73d..e2c44a45e6c77 100644 --- a/llvm/lib/CodeGen/MachineFrameInfo.cpp +++ b/llvm/lib/CodeGen/MachineFrameInfo.cpp @@ -244,6 +244,23 @@ void MachineFrameInfo::print(const MachineFunction &MF, raw_ostream &OS) const{ } OS << "\n"; } + + OS << "save/restore points:\n"; + + if (!SavePoints.empty()) { + OS << "save points:\n"; + + for (auto &item : SavePoints.get()) + OS << printMBBReference(*item.first) << '\n'; + } else + OS << "save points are empty\n"; + + if (!RestorePoints.empty()) { + OS << "restore points:\n"; + for (auto &item : RestorePoints.get()) + OS << printMBBReference(*item.first) << '\n'; + } else + OS << "restore points are empty\n"; } #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) diff --git a/llvm/lib/CodeGen/PrologEpilogInserter.cpp b/llvm/lib/CodeGen/PrologEpilogInserter.cpp index eb8929cae069e..234e2414c871e 100644 --- a/llvm/lib/CodeGen/PrologEpilogInserter.cpp +++ b/llvm/lib/CodeGen/PrologEpilogInserter.cpp @@ -99,8 +99,12 @@ class PEI : public MachineFunctionPass { unsigned MinCSFrameIndex = std::numeric_limits::max(); unsigned MaxCSFrameIndex = 0; - // Save and Restore blocks of the current function. Typically there is a - // single save block, unless Windows EH funclets are involved. + // Prolog and Epilog blocks of the current function. Typically there is a + // single Prolog block, unless Windows EH funclets are involved. + MBBVector PrologBlocks; + MBBVector EpilogBlocks; + + // Save and Restore blocks of the current function. MBBVector SaveBlocks; MBBVector RestoreBlocks; @@ -118,6 +122,7 @@ class PEI : public MachineFunctionPass { void calculateCallFrameInfo(MachineFunction &MF); void calculateSaveRestoreBlocks(MachineFunction &MF); + void calculatePrologEpilogBlocks(MachineFunction &MF); void spillCalleeSavedRegs(MachineFunction &MF); void calculateFrameObjectOffsets(MachineFunction &MF); @@ -234,14 +239,17 @@ bool PEI::runOnMachineFunction(MachineFunction &MF) { // information. Also eliminates call frame pseudo instructions. calculateCallFrameInfo(MF); - // Determine placement of CSR spill/restore code and prolog/epilog code: + // Determine placement of CSR spill/restore code: // place all spills in the entry block, all restores in return blocks. calculateSaveRestoreBlocks(MF); + // Determine placement of prolog/epilog code. + calculatePrologEpilogBlocks(MF); + // Stash away DBG_VALUEs that should not be moved by insertion of prolog code. SavedDbgValuesMap EntryDbgValues; - for (MachineBasicBlock *SaveBlock : SaveBlocks) - stashEntryDbgValues(*SaveBlock, EntryDbgValues); + for (MachineBasicBlock *PrologBlock : PrologBlocks) + stashEntryDbgValues(*PrologBlock, EntryDbgValues); // Handle CSR spilling and restoring, for targets that need it. if (MF.getTarget().usesPhysRegsForValues()) @@ -349,8 +357,10 @@ bool PEI::runOnMachineFunction(MachineFunction &MF) { delete RS; SaveBlocks.clear(); RestoreBlocks.clear(); - MFI.setSavePoint(nullptr); - MFI.setRestorePoint(nullptr); + PrologBlocks.clear(); + EpilogBlocks.clear(); + MFI.clearSavePoints(); + MFI.clearRestorePoints(); return true; } @@ -395,25 +405,48 @@ void PEI::calculateCallFrameInfo(MachineFunction &MF) { } } +/// Compute two sets of blocks for placing prolog and epilog code respectively. +void PEI::calculatePrologEpilogBlocks(MachineFunction &MF) { + const MachineFrameInfo &MFI = MF.getFrameInfo(); + MachineBasicBlock *Prolog = MFI.getProlog(); + MachineBasicBlock *Epilog = MFI.getEpilog(); + + if (Prolog) + PrologBlocks.push_back(Prolog); + + if (Epilog) + EpilogBlocks.push_back(Epilog); + + if (!Prolog && !SaveBlocks.empty()) + PrologBlocks = SaveBlocks; + + if (!Epilog && !RestoreBlocks.empty()) + EpilogBlocks = RestoreBlocks; +} + /// Compute the sets of entry and return blocks for saving and restoring /// callee-saved registers, and placing prolog and epilog code. void PEI::calculateSaveRestoreBlocks(MachineFunction &MF) { const MachineFrameInfo &MFI = MF.getFrameInfo(); - // Even when we do not change any CSR, we still want to insert the // prologue and epilogue of the function. // So set the save points for those. // Use the points found by shrink-wrapping, if any. - if (MFI.getSavePoint()) { - SaveBlocks.push_back(MFI.getSavePoint()); - assert(MFI.getRestorePoint() && "Both restore and save must be set"); - MachineBasicBlock *RestoreBlock = MFI.getRestorePoint(); - // If RestoreBlock does not have any successor and is not a return block - // then the end point is unreachable and we do not need to insert any - // epilogue. - if (!RestoreBlock->succ_empty() || RestoreBlock->isReturnBlock()) - RestoreBlocks.push_back(RestoreBlock); + if (!MFI.getSavePoints().empty()) { + assert(!MFI.getRestorePoints().empty() && + "Both restores and saves must be set"); + for (auto &item : MFI.getSavePoints()) + SaveBlocks.push_back(item.first); + + for (auto &item : MFI.getRestorePoints()) { + MachineBasicBlock *RestoreBlock = item.first; + // If RestoreBlock does not have any successor and is not a return block + // then the end point is unreachable and we do not need to insert any + // epilogue. + if (!RestoreBlock->succ_empty() || RestoreBlock->isReturnBlock()) + RestoreBlocks.push_back(RestoreBlock); + } return; } @@ -522,8 +555,8 @@ static void assignCalleeSavedSpillSlots(MachineFunction &F, /// Helper function to update the liveness information for the callee-saved /// registers. -static void updateLiveness(MachineFunction &MF) { - MachineFrameInfo &MFI = MF.getFrameInfo(); +static void updateLiveness(MachineFunction &MF, MachineBasicBlock *Save, + MachineBasicBlock *Restore, CalleeSavedInfo &Info) { // Visited will contain all the basic blocks that are in the region // where the callee saved registers are alive: // - Anything that is not Save or Restore -> LiveThrough. @@ -534,7 +567,6 @@ static void updateLiveness(MachineFunction &MF) { SmallPtrSet Visited; SmallVector WorkList; MachineBasicBlock *Entry = &MF.front(); - MachineBasicBlock *Save = MFI.getSavePoint(); if (!Save) Save = Entry; @@ -545,7 +577,6 @@ static void updateLiveness(MachineFunction &MF) { } Visited.insert(Save); - MachineBasicBlock *Restore = MFI.getRestorePoint(); if (Restore) // By construction Restore cannot be visited, otherwise it // means there exists a path to Restore that does not go @@ -565,30 +596,27 @@ static void updateLiveness(MachineFunction &MF) { WorkList.push_back(SuccBB); } - const std::vector &CSI = MFI.getCalleeSavedInfo(); - MachineRegisterInfo &MRI = MF.getRegInfo(); - for (const CalleeSavedInfo &I : CSI) { - for (MachineBasicBlock *MBB : Visited) { - MCPhysReg Reg = I.getReg(); - // Add the callee-saved register as live-in. - // It's killed at the spill. - if (!MRI.isReserved(Reg) && !MBB->isLiveIn(Reg)) - MBB->addLiveIn(Reg); - } - // If callee-saved register is spilled to another register rather than - // spilling to stack, the destination register has to be marked as live for - // each MBB between the prologue and epilogue so that it is not clobbered - // before it is reloaded in the epilogue. The Visited set contains all - // blocks outside of the region delimited by prologue/epilogue. - if (I.isSpilledToReg()) { - for (MachineBasicBlock &MBB : MF) { - if (Visited.count(&MBB)) - continue; - MCPhysReg DstReg = I.getDstReg(); - if (!MBB.isLiveIn(DstReg)) - MBB.addLiveIn(DstReg); - } + + for (MachineBasicBlock *MBB : Visited) { + MCPhysReg Reg = Info.getReg(); + // Add the callee-saved register as live-in. + // It's killed at the spill. + if (!MRI.isReserved(Reg) && !MBB->isLiveIn(Reg)) + MBB->addLiveIn(Reg); + } + // If callee-saved register is spilled to another register rather than + // spilling to stack, the destination register has to be marked as live for + // each MBB between the save and restore point so that it is not clobbered + // before it is reloaded in the restore point. The Visited set contains all + // blocks outside of the region delimited by save/restore. + if (Info.isSpilledToReg()) { + for (MachineBasicBlock &MBB : MF) { + if (Visited.count(&MBB)) + continue; + MCPhysReg DstReg = Info.getDstReg(); + if (!MBB.isLiveIn(DstReg)) + MBB.addLiveIn(DstReg); } } } @@ -622,7 +650,7 @@ static void insertCSRSaves(MachineBasicBlock &SaveBlock, /// Insert restore code for the callee-saved registers used in the function. static void insertCSRRestores(MachineBasicBlock &RestoreBlock, - std::vector &CSI) { + std::vector CSI) { MachineFunction &MF = *RestoreBlock.getParent(); const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo(); const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering(); @@ -651,6 +679,67 @@ static void insertCSRRestores(MachineBasicBlock &RestoreBlock, } } +static void fillCSInfoPerBB(MachineFrameInfo &MFI, + DenseMap &RegToInfo, + MBBVector &PrologEpilogBlocks, bool isSave) { + // CaleeSavedInfo list for each point + std::vector CSIV; + // Global CalleeSavedInfo list aggregating CSIVs for all points + std::vector GCSIV; + const SaveRestorePoints::PointsMap &SRPoints = + isSave ? MFI.getSavePoints() : MFI.getRestorePoints(); + SaveRestorePoints::PointsMap Inner; + for (auto [BB, Regs] : SRPoints) { + CSIV.clear(); + for (auto &Reg : Regs) { + auto It = RegToInfo.find(Reg.getReg()); + if (It == RegToInfo.end()) + continue; + CSIV.push_back(*RegToInfo.at(Reg.getReg())); + GCSIV.push_back(*RegToInfo.at(Reg.getReg())); + } + // We need to sort CSIV, because Aarch64 expect CSI list to come sorted by + // frame index + std::sort(CSIV.begin(), CSIV.end(), + [](const CalleeSavedInfo &Lhs, const CalleeSavedInfo &Rhs) { + return Lhs.getFrameIdx() < Rhs.getFrameIdx(); + }); + Inner.insert({BB, CSIV}); + } + + // If in any case not all CSRs listed in MFI.getCalleeSavedInfo are in the + // list of spilled/restored registers (for example AArch64 backend add VG + // registers in the list of CalleeSavedRegs during spill slot assignment), we + // should add them to this list and spill/restore them in Prolog/Epilog. + if (GCSIV.size() < RegToInfo.size()) { + for (auto &RTI : RegToInfo) { + if (find_if(GCSIV, [&RTI](const CalleeSavedInfo &CSI) { + return CSI.getReg() == RTI.first; + }) != std::end(GCSIV)) + continue; + for (MachineBasicBlock *BB : PrologEpilogBlocks) { + if (auto Entry = Inner.find(BB); Entry != Inner.end()) { + auto &CSI = Entry->second; + CSI.push_back(*RTI.second); + std::sort(CSI.begin(), CSI.end(), + [](const CalleeSavedInfo &Lhs, const CalleeSavedInfo &Rhs) { + return Lhs.getFrameIdx() < Rhs.getFrameIdx(); + }); + continue; + } + CSIV.clear(); + CSIV.push_back(*RTI.second); + Inner.insert(std::make_pair(BB, CSIV)); + } + } + } + + if (isSave) + MFI.setSavePoints(Inner); + else + MFI.setRestorePoints(Inner); +} + void PEI::spillCalleeSavedRegs(MachineFunction &MF) { // We can't list this requirement in getRequiredProperties because some // targets (WebAssembly) use virtual registers past this point, and the pass @@ -678,18 +767,57 @@ void PEI::spillCalleeSavedRegs(MachineFunction &MF) { MFI.setCalleeSavedInfoValid(true); std::vector &CSI = MFI.getCalleeSavedInfo(); + DenseMap RegToInfo; + for (auto &CS : CSI) + RegToInfo.insert({CS.getReg(), &CS}); + + if (!MFI.getSavePoints().empty()) { + fillCSInfoPerBB(MFI, RegToInfo, PrologBlocks, /*isSave=*/true); + fillCSInfoPerBB(MFI, RegToInfo, EpilogBlocks, /*isSave=*/false); + } else { + SaveRestorePoints::PointsMap SavePts; + for (MachineBasicBlock *PrologBlock : PrologBlocks) + SavePts.insert({PrologBlock, MFI.getCalleeSavedInfo()}); + MFI.setSavePoints(std::move(SavePts)); + } + if (!CSI.empty()) { if (!MFI.hasCalls()) NumLeafFuncWithSpills++; for (MachineBasicBlock *SaveBlock : SaveBlocks) - insertCSRSaves(*SaveBlock, CSI); + insertCSRSaves(*SaveBlock, MFI.getSavePoints().empty() + ? CSI + : MFI.getSaveCSInfo(SaveBlock)); + + MachineBasicBlock *Save = nullptr; + MachineBasicBlock *Restore = nullptr; + for (auto &CS : CSI) { + if (!MFI.getSavePoints().empty()) { + if (auto BB = MFI.findSpilledIn(CS)) + Save = BB; + + if (auto BB = MFI.findRestoredIn(CS)) + Restore = BB; + } + // Update the live-in information of all the blocks up to the save + // point. + updateLiveness(MF, Save, Restore, CS); + } - // Update the live-in information of all the blocks up to the save point. - updateLiveness(MF); + if (MFI.getRestorePoints().empty()) { + SaveRestorePoints::PointsMap RestorePts; + for (MachineBasicBlock *EpilogBlock : EpilogBlocks) + RestorePts.insert({EpilogBlock, MFI.getCalleeSavedInfo()}); + MFI.setRestorePoints(std::move(RestorePts)); + } - for (MachineBasicBlock *RestoreBlock : RestoreBlocks) - insertCSRRestores(*RestoreBlock, CSI); + for (MachineBasicBlock *RestoreBlock : RestoreBlocks) { + insertCSRRestores(*RestoreBlock, + MFI.getRestorePoints().empty() + ? CSI + : MFI.getRestoreCSInfo(RestoreBlock)); + } } } } @@ -1162,26 +1290,26 @@ void PEI::insertPrologEpilogCode(MachineFunction &MF) { const TargetFrameLowering &TFI = *MF.getSubtarget().getFrameLowering(); // Add prologue to the function... - for (MachineBasicBlock *SaveBlock : SaveBlocks) - TFI.emitPrologue(MF, *SaveBlock); + for (MachineBasicBlock *PrologBlock : PrologBlocks) + TFI.emitPrologue(MF, *PrologBlock); // Add epilogue to restore the callee-save registers in each exiting block. - for (MachineBasicBlock *RestoreBlock : RestoreBlocks) - TFI.emitEpilogue(MF, *RestoreBlock); + for (MachineBasicBlock *EpilogBlock : EpilogBlocks) + TFI.emitEpilogue(MF, *EpilogBlock); // Zero call used registers before restoring callee-saved registers. insertZeroCallUsedRegs(MF); - for (MachineBasicBlock *SaveBlock : SaveBlocks) - TFI.inlineStackProbe(MF, *SaveBlock); + for (MachineBasicBlock *PrologBlock : PrologBlocks) + TFI.inlineStackProbe(MF, *PrologBlock); // Emit additional code that is required to support segmented stacks, if // we've been asked for it. This, when linked with a runtime with support // for segmented stacks (libgcc is one), will result in allocating stack // space in small chunks instead of one large contiguous block. if (MF.shouldSplitStack()) { - for (MachineBasicBlock *SaveBlock : SaveBlocks) - TFI.adjustForSegmentedStacks(MF, *SaveBlock); + for (MachineBasicBlock *PrologBlock : PrologBlocks) + TFI.adjustForSegmentedStacks(MF, *PrologBlock); } // Emit additional code that is required to explicitly handle the stack in @@ -1190,8 +1318,8 @@ void PEI::insertPrologEpilogCode(MachineFunction &MF) { // different conditional check and another BIF for allocating more stack // space. if (MF.getFunction().getCallingConv() == CallingConv::HiPE) - for (MachineBasicBlock *SaveBlock : SaveBlocks) - TFI.adjustForHiPEPrologue(MF, *SaveBlock); + for (MachineBasicBlock *PrologBlock : PrologBlocks) + TFI.adjustForHiPEPrologue(MF, *PrologBlock); } /// insertZeroCallUsedRegs - Zero out call used registers. diff --git a/llvm/lib/CodeGen/ShrinkWrap.cpp b/llvm/lib/CodeGen/ShrinkWrap.cpp index fa57eb30fac43..c59929f387baf 100644 --- a/llvm/lib/CodeGen/ShrinkWrap.cpp +++ b/llvm/lib/CodeGen/ShrinkWrap.cpp @@ -949,6 +949,7 @@ bool ShrinkWrap::runOnMachineFunction(MachineFunction &MF) { bool HasCandidate = performShrinkWrapping(RPOT, RS.get()); StackAddressUsedBlockInfo.clear(); Changed = postShrinkWrapping(HasCandidate, MF, RS.get()); + if (!HasCandidate && !Changed) return false; if (!ArePointsInteresting()) @@ -959,8 +960,17 @@ bool ShrinkWrap::runOnMachineFunction(MachineFunction &MF) { << "\nRestore: " << printMBBReference(*Restore) << '\n'); MachineFrameInfo &MFI = MF.getFrameInfo(); - MFI.setSavePoint(Save); - MFI.setRestorePoint(Restore); + + std::vector CSIVec; + SetOfRegs CSRSet = getCurrentCSRs(RS.get()); + for (unsigned Reg : CSRSet) + CSIVec.push_back(CalleeSavedInfo(Reg)); + + SaveRestorePoints::PointsMap SavePoints({{Save, CSIVec}}); + SaveRestorePoints::PointsMap RestorePoints({{Restore, CSIVec}}); + + MFI.setSavePoints(SavePoints); + MFI.setRestorePoints(RestorePoints); ++NumCandidates; return Changed; } diff --git a/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp b/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp index d27c523425feb..729b4b7093eef 100644 --- a/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp +++ b/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp @@ -193,25 +193,32 @@ void SILowerSGPRSpills::calculateSaveRestoreBlocks(MachineFunction &MF) { // So set the save points for those. // Use the points found by shrink-wrapping, if any. - if (MFI.getSavePoint()) { - SaveBlocks.push_back(MFI.getSavePoint()); - assert(MFI.getRestorePoint() && "Both restore and save must be set"); - MachineBasicBlock *RestoreBlock = MFI.getRestorePoint(); - // If RestoreBlock does not have any successor and is not a return block - // then the end point is unreachable and we do not need to insert any - // epilogue. - if (!RestoreBlock->succ_empty() || RestoreBlock->isReturnBlock()) - RestoreBlocks.push_back(RestoreBlock); + if (!MFI.getSavePoints().empty()) { + assert(!MFI.getRestorePoints().empty() && + "Both restores and saves must be set"); + for (auto &item : MFI.getSavePoints()) + SaveBlocks.push_back(item.first); + + for (auto &item : MFI.getRestorePoints()) { + MachineBasicBlock *RestoreBlock = item.first; + // If RestoreBlock does not have any successor and is not a return block + // then the end point is unreachable and we do not need to insert any + // epilogue. + if (!RestoreBlock->succ_empty() || RestoreBlock->isReturnBlock()) + RestoreBlocks.push_back(RestoreBlock); + } return; } - // Save refs to entry and return blocks. - SaveBlocks.push_back(&MF.front()); - for (MachineBasicBlock &MBB : MF) { - if (MBB.isEHFuncletEntry()) - SaveBlocks.push_back(&MBB); - if (MBB.isReturnBlock()) - RestoreBlocks.push_back(&MBB); + if (MFI.getSavePoints().empty()) { + // Save refs to entry and return blocks. + SaveBlocks.push_back(&MF.front()); + for (MachineBasicBlock &MBB : MF) { + if (MBB.isEHFuncletEntry()) + SaveBlocks.push_back(&MBB); + if (MBB.isReturnBlock()) + RestoreBlocks.push_back(&MBB); + } } } diff --git a/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp b/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp index ba775c4a679d0..729a228a30161 100644 --- a/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp +++ b/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp @@ -2078,10 +2078,9 @@ void PPCFrameLowering::processFunctionBeforeFrameFinalized(MachineFunction &MF, // tail call might not be in the new RestoreBlock, so real branch instruction // won't be generated by emitEpilogue(), because shrink-wrap has chosen new // RestoreBlock. So we handle this case here. - if (MFI.getSavePoint() && MFI.hasTailCall()) { - MachineBasicBlock *RestoreBlock = MFI.getRestorePoint(); + if (!MFI.getSavePoints().empty() && MFI.hasTailCall()) { for (MachineBasicBlock &MBB : MF) { - if (MBB.isReturnBlock() && (&MBB) != RestoreBlock) + if (MBB.isReturnBlock() && (!MFI.getRestorePoints().contains(&MBB))) createTailCallBranchInstr(MBB); } } diff --git a/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp b/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp index bb2e5781c34db..447dee548affe 100644 --- a/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp @@ -30,6 +30,33 @@ using namespace llvm; namespace { +static int64_t calculateCSRSpillOffsets(MachineFrameInfo &MFI, + const TargetFrameLowering *TFI, + int MinCSFI, int FrameIdx) { + int LocalAreaOffset = -TFI->getOffsetOfLocalArea(); + Align MaxAlign = MFI.getMaxAlign(); + Align Alignment = MFI.getObjectAlign(FrameIdx); + MaxAlign = std::max(MaxAlign, Alignment); + int64_t Offset = LocalAreaOffset; + + for (int i = MFI.getObjectIndexBegin(); i != 0; ++i) { + // Only allocate objects on the default stack. + if (MFI.getStackID(i) != TargetStackID::Default) + continue; + + int64_t FixedOff; + FixedOff = -MFI.getObjectOffset(i); + if (FixedOff > Offset) + Offset = FixedOff; + } + + for (int i = MinCSFI; i <= FrameIdx; ++i) + Offset += MFI.getObjectSize(i); + + Offset = alignTo(Offset, Alignment); + return -Offset; +} + class CFISaveRegisterEmitter { MachineFunction &MF; MachineFrameInfo &MFI; @@ -42,7 +69,25 @@ class CFISaveRegisterEmitter { const RISCVRegisterInfo &RI, const RISCVInstrInfo &TII, const DebugLoc &DL, const CalleeSavedInfo &CS) const { int FrameIdx = CS.getFrameIdx(); - int64_t Offset = MFI.getObjectOffset(FrameIdx); + if (FrameIdx >= 0 && + MFI.getStackID(FrameIdx) == TargetStackID::ScalableVector) + return; + + int64_t Offset = 0; + + MachineFrameInfo &MFI = MF.getFrameInfo(); + auto *RVFI = MF.getInfo(); + std::vector GCSI = MFI.getCalleeSavedInfo(); + + if (FrameIdx < 0 && + (RVFI->isPushable(MF) || RVFI->useSaveRestoreLibCalls(MF))) + Offset = MFI.getObjectOffset(FrameIdx); + else { + const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering(); + Offset = calculateCSRSpillOffsets( + MFI, TFI, std::max(GCSI[0].getFrameIdx(), 0), FrameIdx); + } + Register Reg = CS.getReg(); unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::createOffset( nullptr, RI.getDwarfRegNum(Reg, true), Offset)); @@ -450,9 +495,8 @@ uint64_t RISCVFrameLowering::getStackSizeWithRVVPadding( return alignTo(MFI.getStackSize() + RVFI->getRVVPadding(), getStackAlign()); } -static SmallVector -getUnmanagedCSI(const MachineFunction &MF, - const std::vector &CSI) { +SmallVector RISCVFrameLowering::getUnmanagedCSI( + const MachineFunction &MF, const std::vector &CSI) const { const MachineFrameInfo &MFI = MF.getFrameInfo(); SmallVector NonLibcallCSI; @@ -817,13 +861,18 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF, // Determine the correct frame layout determineFrameLayout(MF); - const auto &CSI = MFI.getCalleeSavedInfo(); + const auto &CSI = MFI.getSaveCSInfo(&MBB); // Skip to before the spills of scalar callee-saved registers // FIXME: assumes exactly one instruction is used to restore each // callee-saved register. - MBBI = std::prev(MBBI, getRVVCalleeSavedInfo(MF, CSI).size() + - getUnmanagedCSI(MF, CSI).size()); + // For scalar register spills we skip 2 instrs at once, because right after + // spills there are cfi instructions. At the moment of prolog emission they + // are already inserted for scalar instructions, but not for vector + // instructions. + int ScalarDistance = getUnmanagedCSI(MF, CSI).size() * 2; + int VectorDistance = getRVVCalleeSavedInfo(MF, CSI).size(); + MBBI = std::prev(MBBI, VectorDistance + ScalarDistance); // If libcalls are used to spill and restore callee-saved registers, the frame // has two sections; the opaque section managed by the libcalls, and the @@ -922,11 +971,7 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF, // to the stack, not before. // FIXME: assumes exactly one instruction is used to save each callee-saved // register. - std::advance(MBBI, getUnmanagedCSI(MF, CSI).size()); - - // Iterate over list of callee-saved registers and emit .cfi_offset - // directives. - emitCFIForCSI(MBB, MBBI, getUnmanagedCSI(MF, CSI)); + std::advance(MBBI, ScalarDistance); // Generate new FP. if (hasFP(MF)) { @@ -1087,7 +1132,7 @@ void RISCVFrameLowering::emitEpilogue(MachineFunction &MF, --MBBI; } - const auto &CSI = MFI.getCalleeSavedInfo(); + const auto &CSI = MFI.getRestoreCSInfo(&MBB); // Skip to before the restores of scalar callee-saved registers // FIXME: assumes exactly one instruction is used to restore each @@ -1178,7 +1223,9 @@ void RISCVFrameLowering::emitEpilogue(MachineFunction &MF, // Skip to after the restores of scalar callee-saved registers // FIXME: assumes exactly one instruction is used to restore each // callee-saved register. - MBBI = std::next(FirstScalarCSRRestoreInsn, getUnmanagedCSI(MF, CSI).size()); + // Skip CSR restore instrs + corresponding cfi restore instructions + int ScalarDistance = getUnmanagedCSI(MF, CSI).size() * 2; + MBBI = std::next(FirstScalarCSRRestoreInsn, ScalarDistance); if (getLibCallID(MF, CSI) != -1) { // tail __riscv_restore_[0-12] instruction is considered as a terminator, @@ -1193,9 +1240,6 @@ void RISCVFrameLowering::emitEpilogue(MachineFunction &MF, return; } - // Recover callee-saved registers. - emitCFIForCSI(MBB, MBBI, getUnmanagedCSI(MF, CSI)); - bool ApplyPop = RVFI->isPushable(MF) && MBBI != MBB.end() && MBBI->getOpcode() == RISCV::CM_POP; if (ApplyPop) { @@ -1913,7 +1957,11 @@ bool RISCVFrameLowering::spillCalleeSavedRegisters( MachineInstr::FrameSetup); } }; + storeRegsToStackSlots(UnmanagedCSI); + + emitCFIForCSI(MBB, MI, UnmanagedCSI); + storeRegsToStackSlots(RVVCSI); return true; @@ -2027,8 +2075,12 @@ bool RISCVFrameLowering::restoreCalleeSavedRegisters( } }; loadRegFromStackSlot(RVVCSI); + loadRegFromStackSlot(UnmanagedCSI); + // Recover callee-saved registers. + emitCFIForCSI(MBB, MI, UnmanagedCSI); + RISCVMachineFunctionInfo *RVFI = MF->getInfo(); if (RVFI->isPushable(*MF)) { int RegEnc = RVFI->getRVPushRlist(); diff --git a/llvm/lib/Target/RISCV/RISCVFrameLowering.h b/llvm/lib/Target/RISCV/RISCVFrameLowering.h index d013755ce58a0..e618fe1615e1e 100644 --- a/llvm/lib/Target/RISCV/RISCVFrameLowering.h +++ b/llvm/lib/Target/RISCV/RISCVFrameLowering.h @@ -28,6 +28,10 @@ class RISCVFrameLowering : public TargetFrameLowering { uint64_t getStackSizeWithRVVPadding(const MachineFunction &MF) const; + SmallVector + getUnmanagedCSI(const MachineFunction &MF, + const std::vector &CSI) const; + StackOffset getFrameIndexReference(const MachineFunction &MF, int FI, Register &FrameReg) const override; diff --git a/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp b/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp index 7a99bfd1b2512..a0b6b7e1954fe 100644 --- a/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp @@ -478,12 +478,65 @@ bool RISCVRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II, MachineInstr &MI = *II; MachineFunction &MF = *MI.getParent()->getParent(); MachineRegisterInfo &MRI = MF.getRegInfo(); + const MachineFrameInfo &MFI = MF.getFrameInfo(); DebugLoc DL = MI.getDebugLoc(); int FrameIndex = MI.getOperand(FIOperandNum).getIndex(); Register FrameReg; StackOffset Offset = getFrameLowering(MF)->getFrameIndexReference(MF, FrameIndex, FrameReg); + + const auto &CSI = + getFrameLowering(MF)->getUnmanagedCSI(MF, MFI.getCalleeSavedInfo()); + + if (!CSI.empty()) { + int MinCSFI = CSI.front().getFrameIdx(); + int MaxCSFI = CSI.back().getFrameIdx(); + + // If our FrameIndex is CSI FrameIndex we in some cases need additional + // adjustment + if (FrameIndex >= MinCSFI && FrameIndex <= MaxCSFI) { + MachineBasicBlock *SpilledIn = nullptr; + MachineBasicBlock *RestoredIn = nullptr; + auto It = std::find_if(CSI.begin(), CSI.end(), [FrameIndex](auto &CS) { + return CS.getFrameIdx() == FrameIndex; + }); + + assert(It != CSI.end() && + "Did't find CalleeSavedInfo for CalleeSaved FrameIndex"); + + assert(!(MI.mayLoad() && MI.mayStore()) && + "Instruction with frame index operand may load and store " + "simultaneously!"); + + if (MI.mayStore()) + SpilledIn = MFI.findSpilledIn(*It); + else if (MI.mayLoad()) + RestoredIn = MFI.findRestoredIn(*It); + else + llvm_unreachable( + "Instruction with frame index operand neither loads nor stores!"); + + bool SpilledRestoredInPrologEpilog = true; + // If we didn't managed to find NCD (NCPD) for the list of Save (Restore) + // blocks, spill (restore) will be unconditionally in Prolog (Epilog) + if (MI.mayStore() && MFI.getProlog()) + SpilledRestoredInPrologEpilog = SpilledIn == MFI.getProlog(); + else if (MI.mayLoad() && MFI.getEpilog()) + SpilledRestoredInPrologEpilog = RestoredIn == MFI.getEpilog(); + + // For spills/restores performed not in Prolog/Epilog we need to add full + // SP offset, despite SPAdjusment optimization, because at the end of + // Prolog or at the start of Epilog SP has maximum offset + uint64_t FirstSPAdjustAmount = + getFrameLowering(MF)->getFirstSPAdjustAmount(MF); + if (FirstSPAdjustAmount && !SpilledRestoredInPrologEpilog) + Offset += StackOffset::getFixed( + getFrameLowering(MF)->getStackSizeWithRVVPadding(MF) - + FirstSPAdjustAmount); + } + } + bool IsRVVSpill = RISCV::isRVVSpill(MI); if (!IsRVVSpill) Offset += StackOffset::getFixed(MI.getOperand(FIOperandNum + 1).getImm()); diff --git a/llvm/test/CodeGen/AArch64/dont-shrink-wrap-stack-mayloadorstore.mir b/llvm/test/CodeGen/AArch64/dont-shrink-wrap-stack-mayloadorstore.mir index 1c4447bffd872..b2f6a65016ec3 100644 --- a/llvm/test/CodeGen/AArch64/dont-shrink-wrap-stack-mayloadorstore.mir +++ b/llvm/test/CodeGen/AArch64/dont-shrink-wrap-stack-mayloadorstore.mir @@ -6,17 +6,29 @@ ; RUN: llc -x=mir -simplify-mir -run-pass=shrink-wrap -o - %s | FileCheck %s ; CHECK: name: compiler_pop_stack ; CHECK: frameInfo: - ; CHECK: savePoint: '%bb.1' - ; CHECK: restorePoint: '%bb.7' + ; CHECK: savePoint: + ; CHECK-NEXT: - point: '%bb.1' + ; CHECK-NEXT: registers: [] + ; CHECK: restorePoint: + ; CHECK-NEXT: - point: '%bb.7' + ; CHECK-NEXT: registers: [] ; CHECK: name: compiler_pop_stack_no_memoperands ; CHECK: frameInfo: - ; CHECK: savePoint: '%bb.1' - ; CHECK: restorePoint: '%bb.7' + ; CHECK: savePoint: + ; CHECK-NEXT: - point: '%bb.1' + ; CHECK-NEXT: registers: [] + ; CHECK: restorePoint: + ; CHECK-NEXT: - point: '%bb.7' + ; CHECK-NEXT: registers: [] ; CHECK: name: f ; CHECK: frameInfo: - ; CHECK: savePoint: '%bb.2' - ; CHECK-NEXT: restorePoint: '%bb.4' - ; CHECK-NEXT: stack: + ; CHECK: savePoint: + ; CHECK-NEXT: - point: '%bb.2' + ; CHECK-NEXT: registers: [] + ; CHECK: restorePoint: + ; CHECK-NEXT: - point: '%bb.4' + ; CHECK-NEXT: registers: [] + ; CHECK: stack: target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128" target triple = "aarch64" diff --git a/llvm/test/CodeGen/AArch64/live-debugvalues-sve.mir b/llvm/test/CodeGen/AArch64/live-debugvalues-sve.mir index 612453ab53f43..89f4e8eb161ce 100644 --- a/llvm/test/CodeGen/AArch64/live-debugvalues-sve.mir +++ b/llvm/test/CodeGen/AArch64/live-debugvalues-sve.mir @@ -120,8 +120,18 @@ frameInfo: adjustsStack: true hasCalls: true maxCallFrameSize: 0 - savePoint: '%bb.1' - restorePoint: '%bb.1' + savePoint: + - point: '%bb.1' + registers: + - '$fp' + - '$lr' + - '$x28' + restorePoint: + - point: '%bb.1' + registers: + - '$fp' + - '$lr' + - '$x28' stack: - { id: 0, size: 16, alignment: 16, stack-id: scalable-vector } machineFunctionInfo: {} diff --git a/llvm/test/CodeGen/ARM/invalidated-save-point.ll b/llvm/test/CodeGen/ARM/invalidated-save-point.ll index bb602308a1793..4179316572990 100644 --- a/llvm/test/CodeGen/ARM/invalidated-save-point.ll +++ b/llvm/test/CodeGen/ARM/invalidated-save-point.ll @@ -4,8 +4,8 @@ ; this point. Notably, if it isn't is will be invalid and reference a ; deleted block (%bb.-1.if.end) -; CHECK: savePoint: '' -; CHECK: restorePoint: '' +; CHECK: savePoint: [] +; CHECK: restorePoint: [] target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64" target triple = "thumbv7" diff --git a/llvm/test/CodeGen/MIR/Generic/frame-info.mir b/llvm/test/CodeGen/MIR/Generic/frame-info.mir index d5e014cf62991..e8f3a83fcda89 100644 --- a/llvm/test/CodeGen/MIR/Generic/frame-info.mir +++ b/llvm/test/CodeGen/MIR/Generic/frame-info.mir @@ -46,8 +46,8 @@ tracksRegLiveness: true # CHECK-NEXT: hasTailCall: false # CHECK-NEXT: isCalleeSavedInfoValid: false # CHECK-NEXT: localFrameSize: 0 -# CHECK-NEXT: savePoint: '' -# CHECK-NEXT: restorePoint: '' +# CHECK-NEXT: savePoint: [] +# CHECK-NEXT: restorePoint: [] # CHECK: body frameInfo: maxAlignment: 4 diff --git a/llvm/test/CodeGen/MIR/X86/frame-info-multiple-save-restore-points-parse.mir b/llvm/test/CodeGen/MIR/X86/frame-info-multiple-save-restore-points-parse.mir new file mode 100644 index 0000000000000..55965f8f4cfc1 --- /dev/null +++ b/llvm/test/CodeGen/MIR/X86/frame-info-multiple-save-restore-points-parse.mir @@ -0,0 +1,91 @@ +# RUN: llc -mtriple=x86_64 -run-pass none -o - %s | FileCheck %s +# This test ensures that the MIR parser parses the save and restore points in +# the machine frame info correctly. + +--- | + + define i32 @foo(i32 %a, i32 %b) { + entry: + %tmp = alloca i32, align 4 + %tmp2 = icmp slt i32 %a, %b + br i1 %tmp2, label %true, label %false + + true: + store i32 %a, ptr %tmp, align 4 + %tmp4 = call i32 @doSomething(i32 0, ptr %tmp) + br label %false + + false: + %tmp.0 = phi i32 [ %tmp4, %true ], [ %a, %entry ] + ret i32 %tmp.0 + } + + declare i32 @doSomething(i32, ptr) + +... +--- +name: foo +tracksRegLiveness: true +liveins: + - { reg: '$edi' } + - { reg: '$esi' } +# CHECK: frameInfo: +# CHECK: savePoint: +# CHECK-NEXT: - point: '%bb.1' +# CHECK-NEXT: registers: +# CHECK-NEXT: - '$eax' +# CHECK-NEXT: - point: '%bb.2' +# CHECK-NEXT: registers: +# CHECK-NEXT: - '$edi' +# CHECK-NEXT: restorePoint: +# CHECK-NEXT: - point: '%bb.2' +# CHECK-NEXT: registers: +# CHECK-NEXT: - '$edi' +# CHECK-NEXT: - point: '%bb.3' +# CHECK-NEXT: registers: +# CHECK-NEXT: - '$eax' + +# CHECK: stack +frameInfo: + maxAlignment: 4 + hasCalls: true + savePoint: + - point: '%bb.1' + registers: + - '$eax' + - point: '%bb.2' + registers: + - '$edi' + restorePoint: + - point: '%bb.2' + registers: + - '$edi' + - point: '%bb.3' + registers: + - '$eax' +stack: + - { id: 0, name: tmp, offset: 0, size: 4, alignment: 4 } +body: | + bb.0: + successors: %bb.2, %bb.1 + liveins: $edi, $esi + $eax = COPY $edi + CMP32rr $eax, killed $esi, implicit-def $eflags + JCC_1 %bb.2, 12, implicit killed $eflags + bb.1: + successors: %bb.3 + liveins: $eax + JMP_1 %bb.3 + bb.2.true: + successors: %bb.3 + liveins: $eax + MOV32mr %stack.0.tmp, 1, _, 0, _, killed $eax + ADJCALLSTACKDOWN64 0, 0, 0, implicit-def $rsp, implicit-def $ssp, implicit-def dead $eflags, implicit $rsp, implicit $ssp + $rsi = LEA64r %stack.0.tmp, 1, _, 0, _ + $edi = MOV32r0 implicit-def dead $eflags + CALL64pcrel32 @doSomething, csr_64, implicit $rsp, implicit $ssp, implicit $edi, implicit $rsi, implicit-def $rsp, implicit-def $ssp, implicit-def $eax + ADJCALLSTACKUP64 0, 0, implicit-def $rsp, implicit-def $ssp, implicit-def dead $eflags, implicit $rsp, implicit $ssp + bb.3.false: + liveins: $eax + RET64 $eax +... diff --git a/llvm/test/CodeGen/MIR/X86/frame-info-save-restore-points.mir b/llvm/test/CodeGen/MIR/X86/frame-info-save-restore-points.mir index e26233f946606..400177f7eee32 100644 --- a/llvm/test/CodeGen/MIR/X86/frame-info-save-restore-points.mir +++ b/llvm/test/CodeGen/MIR/X86/frame-info-save-restore-points.mir @@ -30,8 +30,12 @@ liveins: - { reg: '$edi' } - { reg: '$esi' } # CHECK: frameInfo: -# CHECK: savePoint: '%bb.2' -# CHECK-NEXT: restorePoint: '%bb.2' +# CHECK: savePoint: +# CHECK-NEXT: - point: '%bb.2' +# CHECK-NEXT: registers: [] +# CHECK: restorePoint: +# CHECK-NEXT: - point: '%bb.2' +# CHECK-NEXT: registers: [] # CHECK: stack frameInfo: maxAlignment: 4 diff --git a/llvm/test/CodeGen/PowerPC/common-chain-aix32.ll b/llvm/test/CodeGen/PowerPC/common-chain-aix32.ll index 35ddcfd9ba6d6..a61d669b014b5 100644 --- a/llvm/test/CodeGen/PowerPC/common-chain-aix32.ll +++ b/llvm/test/CodeGen/PowerPC/common-chain-aix32.ll @@ -49,9 +49,9 @@ define i64 @two_chain_same_offset_succ_i32(ptr %p, i32 %offset, i32 %base1, i64 ; CHECK-NEXT: slwi r8, r4, 1 ; CHECK-NEXT: li r10, 0 ; CHECK-NEXT: li r11, 0 -; CHECK-NEXT: stw r30, -8(r1) # 4-byte Folded Spill -; CHECK-NEXT: add r8, r4, r8 ; CHECK-NEXT: stw r31, -4(r1) # 4-byte Folded Spill +; CHECK-NEXT: add r8, r4, r8 +; CHECK-NEXT: stw r30, -8(r1) # 4-byte Folded Spill ; CHECK-NEXT: add r9, r5, r8 ; CHECK-NEXT: add r5, r5, r4 ; CHECK-NEXT: add r8, r3, r5 @@ -84,8 +84,8 @@ define i64 @two_chain_same_offset_succ_i32(ptr %p, i32 %offset, i32 %base1, i64 ; CHECK-NEXT: crand 4*cr5+lt, eq, 4*cr1+lt ; CHECK-NEXT: bc 12, 4*cr5+lt, L..BB0_3 ; CHECK-NEXT: # %bb.5: -; CHECK-NEXT: lwz r31, -4(r1) # 4-byte Folded Reload ; CHECK-NEXT: lwz r30, -8(r1) # 4-byte Folded Reload +; CHECK-NEXT: lwz r31, -4(r1) # 4-byte Folded Reload ; CHECK-NEXT: mr r4, r5 ; CHECK-NEXT: blr ; CHECK-NEXT: L..BB0_6: diff --git a/llvm/test/CodeGen/PowerPC/common-chain.ll b/llvm/test/CodeGen/PowerPC/common-chain.ll index b71a360d1be12..bc6401d84cf25 100644 --- a/llvm/test/CodeGen/PowerPC/common-chain.ll +++ b/llvm/test/CodeGen/PowerPC/common-chain.ll @@ -426,11 +426,11 @@ define i64 @not_same_offset_fail(ptr %p, i64 %offset, i64 %base1, i64 %n) { ; CHECK-NEXT: cmpdi r6, 0 ; CHECK-NEXT: ble cr0, .LBB4_4 ; CHECK-NEXT: # %bb.1: # %for.body.preheader -; CHECK-NEXT: std r28, -32(r1) # 8-byte Folded Spill +; CHECK-NEXT: std r30, -16(r1) # 8-byte Folded Spill ; CHECK-NEXT: std r29, -24(r1) # 8-byte Folded Spill ; CHECK-NEXT: add r5, r3, r5 ; CHECK-NEXT: li r3, 0 -; CHECK-NEXT: std r30, -16(r1) # 8-byte Folded Spill +; CHECK-NEXT: std r28, -32(r1) # 8-byte Folded Spill ; CHECK-NEXT: mtctr r6 ; CHECK-NEXT: mulli r11, r4, 10 ; CHECK-NEXT: sldi r8, r4, 2 @@ -455,9 +455,9 @@ define i64 @not_same_offset_fail(ptr %p, i64 %offset, i64 %base1, i64 %n) { ; CHECK-NEXT: maddld r3, r6, r28, r3 ; CHECK-NEXT: bdnz .LBB4_2 ; CHECK-NEXT: # %bb.3: -; CHECK-NEXT: ld r30, -16(r1) # 8-byte Folded Reload -; CHECK-NEXT: ld r29, -24(r1) # 8-byte Folded Reload ; CHECK-NEXT: ld r28, -32(r1) # 8-byte Folded Reload +; CHECK-NEXT: ld r29, -24(r1) # 8-byte Folded Reload +; CHECK-NEXT: ld r30, -16(r1) # 8-byte Folded Reload ; CHECK-NEXT: blr ; CHECK-NEXT: .LBB4_4: ; CHECK-NEXT: li r3, 0 diff --git a/llvm/test/CodeGen/PowerPC/loop-instr-form-prepare.ll b/llvm/test/CodeGen/PowerPC/loop-instr-form-prepare.ll index cc38e250f183f..00627b8434beb 100644 --- a/llvm/test/CodeGen/PowerPC/loop-instr-form-prepare.ll +++ b/llvm/test/CodeGen/PowerPC/loop-instr-form-prepare.ll @@ -189,8 +189,8 @@ define i64 @test_max_number_reminder(ptr %arg, i32 signext %arg1) { ; CHECK-NEXT: cmplwi r4, 0 ; CHECK-NEXT: beq cr0, .LBB2_4 ; CHECK-NEXT: # %bb.1: # %bb3.preheader -; CHECK-NEXT: std r25, -56(r1) # 8-byte Folded Spill -; CHECK-NEXT: std r26, -48(r1) # 8-byte Folded Spill +; CHECK-NEXT: std r30, -16(r1) # 8-byte Folded Spill +; CHECK-NEXT: std r29, -24(r1) # 8-byte Folded Spill ; CHECK-NEXT: addi r10, r3, 4002 ; CHECK-NEXT: li r3, 0 ; CHECK-NEXT: li r5, -1 @@ -198,10 +198,10 @@ define i64 @test_max_number_reminder(ptr %arg, i32 signext %arg1) { ; CHECK-NEXT: li r7, 3 ; CHECK-NEXT: li r8, 5 ; CHECK-NEXT: li r9, 9 -; CHECK-NEXT: std r27, -40(r1) # 8-byte Folded Spill ; CHECK-NEXT: std r28, -32(r1) # 8-byte Folded Spill -; CHECK-NEXT: std r29, -24(r1) # 8-byte Folded Spill -; CHECK-NEXT: std r30, -16(r1) # 8-byte Folded Spill +; CHECK-NEXT: std r27, -40(r1) # 8-byte Folded Spill +; CHECK-NEXT: std r26, -48(r1) # 8-byte Folded Spill +; CHECK-NEXT: std r25, -56(r1) # 8-byte Folded Spill ; CHECK-NEXT: mtctr r4 ; CHECK-NEXT: .p2align 4 ; CHECK-NEXT: .LBB2_2: # %bb3 @@ -226,13 +226,13 @@ define i64 @test_max_number_reminder(ptr %arg, i32 signext %arg1) { ; CHECK-NEXT: maddld r3, r11, r25, r3 ; CHECK-NEXT: bdnz .LBB2_2 ; CHECK-NEXT: # %bb.3: -; CHECK-NEXT: ld r30, -16(r1) # 8-byte Folded Reload -; CHECK-NEXT: ld r29, -24(r1) # 8-byte Folded Reload -; CHECK-NEXT: ld r28, -32(r1) # 8-byte Folded Reload +; CHECK-NEXT: ld r25, -56(r1) # 8-byte Folded Reload +; CHECK-NEXT: ld r26, -48(r1) # 8-byte Folded Reload ; CHECK-NEXT: ld r27, -40(r1) # 8-byte Folded Reload +; CHECK-NEXT: ld r28, -32(r1) # 8-byte Folded Reload ; CHECK-NEXT: add r3, r3, r4 -; CHECK-NEXT: ld r26, -48(r1) # 8-byte Folded Reload -; CHECK-NEXT: ld r25, -56(r1) # 8-byte Folded Reload +; CHECK-NEXT: ld r29, -24(r1) # 8-byte Folded Reload +; CHECK-NEXT: ld r30, -16(r1) # 8-byte Folded Reload ; CHECK-NEXT: blr ; CHECK-NEXT: .LBB2_4: ; CHECK-NEXT: addi r3, r4, 0 @@ -583,10 +583,10 @@ define i64 @test_ds_cross_basic_blocks(ptr %arg, i32 signext %arg1) { ; CHECK-NEXT: beq cr0, .LBB6_9 ; CHECK-NEXT: # %bb.1: # %bb3 ; CHECK-NEXT: addis r5, r2, .LC0@toc@ha -; CHECK-NEXT: std r28, -32(r1) # 8-byte Folded Spill +; CHECK-NEXT: std r30, -16(r1) # 8-byte Folded Spill ; CHECK-NEXT: std r29, -24(r1) # 8-byte Folded Spill ; CHECK-NEXT: ld r5, .LC0@toc@l(r5) -; CHECK-NEXT: std r30, -16(r1) # 8-byte Folded Spill +; CHECK-NEXT: std r28, -32(r1) # 8-byte Folded Spill ; CHECK-NEXT: addi r6, r3, 4009 ; CHECK-NEXT: li r3, 0 ; CHECK-NEXT: li r7, -7 @@ -649,9 +649,9 @@ define i64 @test_ds_cross_basic_blocks(ptr %arg, i32 signext %arg1) { ; CHECK-NEXT: add r4, r30, r4 ; CHECK-NEXT: b .LBB6_3 ; CHECK-NEXT: .LBB6_8: -; CHECK-NEXT: ld r30, -16(r1) # 8-byte Folded Reload -; CHECK-NEXT: ld r29, -24(r1) # 8-byte Folded Reload ; CHECK-NEXT: ld r28, -32(r1) # 8-byte Folded Reload +; CHECK-NEXT: ld r29, -24(r1) # 8-byte Folded Reload +; CHECK-NEXT: ld r30, -16(r1) # 8-byte Folded Reload ; CHECK-NEXT: blr ; CHECK-NEXT: .LBB6_9: ; CHECK-NEXT: li r3, 0 diff --git a/llvm/test/CodeGen/PowerPC/lsr-profitable-chain.ll b/llvm/test/CodeGen/PowerPC/lsr-profitable-chain.ll index 79f2ef3e3746a..3de0fe239021c 100644 --- a/llvm/test/CodeGen/PowerPC/lsr-profitable-chain.ll +++ b/llvm/test/CodeGen/PowerPC/lsr-profitable-chain.ll @@ -8,22 +8,22 @@ define void @foo(ptr readonly %0, ptr %1, i64 %2, i64 %3, i64 %4, i64 %5, i64 %6 ; CHECK-NEXT: cmpd 5, 7 ; CHECK-NEXT: bgelr 0 ; CHECK-NEXT: # %bb.1: # %.preheader -; CHECK-NEXT: std 27, -40(1) # 8-byte Folded Spill -; CHECK-NEXT: addi 27, 5, 2 -; CHECK-NEXT: std 28, -32(1) # 8-byte Folded Spill -; CHECK-NEXT: addi 28, 5, 3 ; CHECK-NEXT: std 30, -16(1) # 8-byte Folded Spill ; CHECK-NEXT: addi 30, 5, 1 +; CHECK-NEXT: std 28, -32(1) # 8-byte Folded Spill +; CHECK-NEXT: addi 28, 5, 3 +; CHECK-NEXT: std 27, -40(1) # 8-byte Folded Spill +; CHECK-NEXT: addi 27, 5, 2 ; CHECK-NEXT: mulld 12, 8, 5 ; CHECK-NEXT: mulld 0, 9, 8 ; CHECK-NEXT: std 29, -24(1) # 8-byte Folded Spill ; CHECK-NEXT: addi 29, 3, 16 ; CHECK-NEXT: sldi 11, 10, 3 -; CHECK-NEXT: std 22, -80(1) # 8-byte Folded Spill -; CHECK-NEXT: std 23, -72(1) # 8-byte Folded Spill -; CHECK-NEXT: std 24, -64(1) # 8-byte Folded Spill -; CHECK-NEXT: std 25, -56(1) # 8-byte Folded Spill ; CHECK-NEXT: std 26, -48(1) # 8-byte Folded Spill +; CHECK-NEXT: std 25, -56(1) # 8-byte Folded Spill +; CHECK-NEXT: std 24, -64(1) # 8-byte Folded Spill +; CHECK-NEXT: std 23, -72(1) # 8-byte Folded Spill +; CHECK-NEXT: std 22, -80(1) # 8-byte Folded Spill ; CHECK-NEXT: mulld 30, 8, 30 ; CHECK-NEXT: mulld 28, 8, 28 ; CHECK-NEXT: mulld 8, 8, 27 @@ -104,15 +104,15 @@ define void @foo(ptr readonly %0, ptr %1, i64 %2, i64 %3, i64 %4, i64 %5, i64 %6 ; CHECK-NEXT: blt 0, .LBB0_5 ; CHECK-NEXT: b .LBB0_2 ; CHECK-NEXT: .LBB0_6: -; CHECK-NEXT: ld 30, -16(1) # 8-byte Folded Reload -; CHECK-NEXT: ld 29, -24(1) # 8-byte Folded Reload -; CHECK-NEXT: ld 28, -32(1) # 8-byte Folded Reload -; CHECK-NEXT: ld 27, -40(1) # 8-byte Folded Reload -; CHECK-NEXT: ld 26, -48(1) # 8-byte Folded Reload -; CHECK-NEXT: ld 25, -56(1) # 8-byte Folded Reload -; CHECK-NEXT: ld 24, -64(1) # 8-byte Folded Reload -; CHECK-NEXT: ld 23, -72(1) # 8-byte Folded Reload ; CHECK-NEXT: ld 22, -80(1) # 8-byte Folded Reload +; CHECK-NEXT: ld 23, -72(1) # 8-byte Folded Reload +; CHECK-NEXT: ld 24, -64(1) # 8-byte Folded Reload +; CHECK-NEXT: ld 25, -56(1) # 8-byte Folded Reload +; CHECK-NEXT: ld 26, -48(1) # 8-byte Folded Reload +; CHECK-NEXT: ld 27, -40(1) # 8-byte Folded Reload +; CHECK-NEXT: ld 28, -32(1) # 8-byte Folded Reload +; CHECK-NEXT: ld 29, -24(1) # 8-byte Folded Reload +; CHECK-NEXT: ld 30, -16(1) # 8-byte Folded Reload ; CHECK-NEXT: blr %9 = icmp slt i64 %2, %4 br i1 %9, label %10, label %97 diff --git a/llvm/test/CodeGen/PowerPC/more-dq-form-prepare.ll b/llvm/test/CodeGen/PowerPC/more-dq-form-prepare.ll index 9f62477ae01df..b6a10aaebdc82 100644 --- a/llvm/test/CodeGen/PowerPC/more-dq-form-prepare.ll +++ b/llvm/test/CodeGen/PowerPC/more-dq-form-prepare.ll @@ -56,39 +56,39 @@ define void @foo(ptr %.m, ptr %.n, ptr %.a, ptr %.x, ptr %.l, ptr %.vy01, ptr %. ; CHECK-NEXT: .cfi_offset v29, -240 ; CHECK-NEXT: .cfi_offset v30, -224 ; CHECK-NEXT: .cfi_offset v31, -208 -; CHECK-NEXT: std 22, 464(1) # 8-byte Folded Spill ; CHECK-NEXT: std 23, 472(1) # 8-byte Folded Spill +; CHECK-NEXT: std 22, 464(1) # 8-byte Folded Spill ; CHECK-NEXT: mr 22, 5 ; CHECK-NEXT: ld 5, 848(1) ; CHECK-NEXT: addi 3, 3, 1 ; CHECK-NEXT: mr 11, 7 ; CHECK-NEXT: ld 23, 688(1) ; CHECK-NEXT: ld 7, 728(1) -; CHECK-NEXT: std 18, 432(1) # 8-byte Folded Spill ; CHECK-NEXT: std 19, 440(1) # 8-byte Folded Spill +; CHECK-NEXT: std 18, 432(1) # 8-byte Folded Spill ; CHECK-NEXT: mr 18, 6 ; CHECK-NEXT: li 6, 9 ; CHECK-NEXT: ld 19, 768(1) ; CHECK-NEXT: ld 2, 760(1) -; CHECK-NEXT: std 26, 496(1) # 8-byte Folded Spill ; CHECK-NEXT: std 27, 504(1) # 8-byte Folded Spill +; CHECK-NEXT: std 26, 496(1) # 8-byte Folded Spill ; CHECK-NEXT: cmpldi 3, 9 ; CHECK-NEXT: ld 27, 816(1) ; CHECK-NEXT: ld 26, 808(1) -; CHECK-NEXT: std 14, 400(1) # 8-byte Folded Spill ; CHECK-NEXT: std 15, 408(1) # 8-byte Folded Spill +; CHECK-NEXT: std 14, 400(1) # 8-byte Folded Spill ; CHECK-NEXT: ld 15, 736(1) ; CHECK-NEXT: lxv 39, 0(8) -; CHECK-NEXT: std 30, 528(1) # 8-byte Folded Spill ; CHECK-NEXT: std 31, 536(1) # 8-byte Folded Spill +; CHECK-NEXT: std 30, 528(1) # 8-byte Folded Spill ; CHECK-NEXT: ld 30, 704(1) ; CHECK-NEXT: lxv 38, 0(9) -; CHECK-NEXT: std 20, 448(1) # 8-byte Folded Spill ; CHECK-NEXT: std 21, 456(1) # 8-byte Folded Spill +; CHECK-NEXT: std 20, 448(1) # 8-byte Folded Spill ; CHECK-NEXT: ld 21, 784(1) ; CHECK-NEXT: ld 20, 776(1) -; CHECK-NEXT: std 24, 480(1) # 8-byte Folded Spill ; CHECK-NEXT: std 25, 488(1) # 8-byte Folded Spill +; CHECK-NEXT: std 24, 480(1) # 8-byte Folded Spill ; CHECK-NEXT: iselgt 3, 3, 6 ; CHECK-NEXT: ld 6, 720(1) ; CHECK-NEXT: ld 24, 792(1) @@ -112,14 +112,14 @@ define void @foo(ptr %.m, ptr %.n, ptr %.a, ptr %.x, ptr %.l, ptr %.vy01, ptr %. ; CHECK-NEXT: lxv 33, 0(10) ; CHECK-NEXT: lxv 32, 0(23) ; CHECK-NEXT: lxv 36, 0(30) -; CHECK-NEXT: std 16, 416(1) # 8-byte Folded Spill -; CHECK-NEXT: std 17, 424(1) # 8-byte Folded Spill -; CHECK-NEXT: ld 17, 752(1) -; CHECK-NEXT: ld 16, 744(1) -; CHECK-NEXT: std 28, 512(1) # 8-byte Folded Spill ; CHECK-NEXT: std 29, 520(1) # 8-byte Folded Spill +; CHECK-NEXT: std 28, 512(1) # 8-byte Folded Spill ; CHECK-NEXT: ld 29, 712(1) ; CHECK-NEXT: ld 28, 696(1) +; CHECK-NEXT: std 17, 424(1) # 8-byte Folded Spill +; CHECK-NEXT: std 16, 416(1) # 8-byte Folded Spill +; CHECK-NEXT: ld 17, 752(1) +; CHECK-NEXT: ld 16, 744(1) ; CHECK-NEXT: std 8, 56(1) # 8-byte Folded Spill ; CHECK-NEXT: std 9, 64(1) # 8-byte Folded Spill ; CHECK-NEXT: lxv 37, 0(28) @@ -132,33 +132,33 @@ define void @foo(ptr %.m, ptr %.n, ptr %.a, ptr %.x, ptr %.l, ptr %.vy01, ptr %. ; CHECK-NEXT: lxv 10, 0(15) ; CHECK-NEXT: lxv 9, 0(16) ; CHECK-NEXT: li 28, 1 -; CHECK-NEXT: stfd 26, 544(1) # 8-byte Folded Spill -; CHECK-NEXT: stfd 27, 552(1) # 8-byte Folded Spill +; CHECK-NEXT: stxv 63, 384(1) # 16-byte Folded Spill +; CHECK-NEXT: stxv 62, 368(1) # 16-byte Folded Spill ; CHECK-NEXT: lxv 8, 0(17) ; CHECK-NEXT: lxv 7, 0(2) -; CHECK-NEXT: stfd 28, 560(1) # 8-byte Folded Spill -; CHECK-NEXT: stfd 29, 568(1) # 8-byte Folded Spill +; CHECK-NEXT: stxv 61, 352(1) # 16-byte Folded Spill +; CHECK-NEXT: stxv 60, 336(1) # 16-byte Folded Spill ; CHECK-NEXT: lxv 5, 0(20) ; CHECK-NEXT: lxv 3, 0(24) -; CHECK-NEXT: stfd 30, 576(1) # 8-byte Folded Spill -; CHECK-NEXT: stfd 31, 584(1) # 8-byte Folded Spill +; CHECK-NEXT: stxv 59, 320(1) # 16-byte Folded Spill +; CHECK-NEXT: stxv 58, 304(1) # 16-byte Folded Spill ; CHECK-NEXT: lxv 2, 0(25) ; CHECK-NEXT: lxv 1, 0(26) -; CHECK-NEXT: stxv 52, 208(1) # 16-byte Folded Spill -; CHECK-NEXT: stxv 53, 224(1) # 16-byte Folded Spill +; CHECK-NEXT: stxv 57, 288(1) # 16-byte Folded Spill +; CHECK-NEXT: stxv 56, 272(1) # 16-byte Folded Spill ; CHECK-NEXT: lxv 0, 0(27) -; CHECK-NEXT: stxv 54, 240(1) # 16-byte Folded Spill ; CHECK-NEXT: stxv 55, 256(1) # 16-byte Folded Spill -; CHECK-NEXT: stxv 56, 272(1) # 16-byte Folded Spill -; CHECK-NEXT: stxv 57, 288(1) # 16-byte Folded Spill -; CHECK-NEXT: stxv 58, 304(1) # 16-byte Folded Spill +; CHECK-NEXT: stxv 54, 240(1) # 16-byte Folded Spill +; CHECK-NEXT: stxv 53, 224(1) # 16-byte Folded Spill +; CHECK-NEXT: stxv 52, 208(1) # 16-byte Folded Spill +; CHECK-NEXT: stfd 31, 584(1) # 8-byte Folded Spill +; CHECK-NEXT: stfd 30, 576(1) # 8-byte Folded Spill ; CHECK-NEXT: std 5, 192(1) # 8-byte Folded Spill ; CHECK-NEXT: ld 5, 832(1) -; CHECK-NEXT: stxv 59, 320(1) # 16-byte Folded Spill -; CHECK-NEXT: stxv 60, 336(1) # 16-byte Folded Spill -; CHECK-NEXT: stxv 61, 352(1) # 16-byte Folded Spill -; CHECK-NEXT: stxv 62, 368(1) # 16-byte Folded Spill -; CHECK-NEXT: stxv 63, 384(1) # 16-byte Folded Spill +; CHECK-NEXT: stfd 29, 568(1) # 8-byte Folded Spill +; CHECK-NEXT: stfd 28, 560(1) # 8-byte Folded Spill +; CHECK-NEXT: stfd 27, 552(1) # 8-byte Folded Spill +; CHECK-NEXT: stfd 26, 544(1) # 8-byte Folded Spill ; CHECK-NEXT: std 15, 88(1) # 8-byte Folded Spill ; CHECK-NEXT: std 16, 96(1) # 8-byte Folded Spill ; CHECK-NEXT: std 17, 104(1) # 8-byte Folded Spill @@ -270,53 +270,53 @@ define void @foo(ptr %.m, ptr %.n, ptr %.a, ptr %.x, ptr %.l, ptr %.vy01, ptr %. ; CHECK-NEXT: ble 0, .LBB0_3 ; CHECK-NEXT: # %bb.6: # %_loop_1_loopHeader_._return_bb_crit_edge.loopexit ; CHECK-NEXT: ld 3, 56(1) # 8-byte Folded Reload -; CHECK-NEXT: lxv 63, 384(1) # 16-byte Folded Reload +; CHECK-NEXT: ld 14, 400(1) # 8-byte Folded Reload ; CHECK-NEXT: stxv 39, 0(3) ; CHECK-NEXT: ld 3, 64(1) # 8-byte Folded Reload -; CHECK-NEXT: lxv 62, 368(1) # 16-byte Folded Reload -; CHECK-NEXT: lxv 61, 352(1) # 16-byte Folded Reload -; CHECK-NEXT: lxv 60, 336(1) # 16-byte Folded Reload -; CHECK-NEXT: lxv 59, 320(1) # 16-byte Folded Reload -; CHECK-NEXT: lxv 58, 304(1) # 16-byte Folded Reload -; CHECK-NEXT: lxv 57, 288(1) # 16-byte Folded Reload -; CHECK-NEXT: lxv 56, 272(1) # 16-byte Folded Reload -; CHECK-NEXT: lxv 55, 256(1) # 16-byte Folded Reload -; CHECK-NEXT: lxv 54, 240(1) # 16-byte Folded Reload -; CHECK-NEXT: lxv 53, 224(1) # 16-byte Folded Reload -; CHECK-NEXT: lxv 52, 208(1) # 16-byte Folded Reload +; CHECK-NEXT: ld 15, 408(1) # 8-byte Folded Reload +; CHECK-NEXT: ld 16, 416(1) # 8-byte Folded Reload +; CHECK-NEXT: ld 17, 424(1) # 8-byte Folded Reload +; CHECK-NEXT: ld 18, 432(1) # 8-byte Folded Reload +; CHECK-NEXT: ld 19, 440(1) # 8-byte Folded Reload +; CHECK-NEXT: ld 20, 448(1) # 8-byte Folded Reload +; CHECK-NEXT: ld 21, 456(1) # 8-byte Folded Reload +; CHECK-NEXT: ld 22, 464(1) # 8-byte Folded Reload +; CHECK-NEXT: ld 23, 472(1) # 8-byte Folded Reload +; CHECK-NEXT: ld 24, 480(1) # 8-byte Folded Reload +; CHECK-NEXT: ld 25, 488(1) # 8-byte Folded Reload ; CHECK-NEXT: stxv 38, 0(3) ; CHECK-NEXT: ld 3, 72(1) # 8-byte Folded Reload -; CHECK-NEXT: lfd 31, 584(1) # 8-byte Folded Reload -; CHECK-NEXT: lfd 30, 576(1) # 8-byte Folded Reload -; CHECK-NEXT: lfd 29, 568(1) # 8-byte Folded Reload -; CHECK-NEXT: lfd 28, 560(1) # 8-byte Folded Reload -; CHECK-NEXT: lfd 27, 552(1) # 8-byte Folded Reload -; CHECK-NEXT: lfd 26, 544(1) # 8-byte Folded Reload -; CHECK-NEXT: ld 31, 536(1) # 8-byte Folded Reload -; CHECK-NEXT: ld 30, 528(1) # 8-byte Folded Reload -; CHECK-NEXT: ld 29, 520(1) # 8-byte Folded Reload -; CHECK-NEXT: ld 28, 512(1) # 8-byte Folded Reload +; CHECK-NEXT: ld 26, 496(1) # 8-byte Folded Reload ; CHECK-NEXT: ld 27, 504(1) # 8-byte Folded Reload +; CHECK-NEXT: ld 28, 512(1) # 8-byte Folded Reload +; CHECK-NEXT: ld 29, 520(1) # 8-byte Folded Reload +; CHECK-NEXT: ld 30, 528(1) # 8-byte Folded Reload +; CHECK-NEXT: ld 31, 536(1) # 8-byte Folded Reload +; CHECK-NEXT: lfd 26, 544(1) # 8-byte Folded Reload +; CHECK-NEXT: lfd 27, 552(1) # 8-byte Folded Reload +; CHECK-NEXT: lfd 28, 560(1) # 8-byte Folded Reload +; CHECK-NEXT: lfd 29, 568(1) # 8-byte Folded Reload +; CHECK-NEXT: lfd 30, 576(1) # 8-byte Folded Reload ; CHECK-NEXT: stxv 33, 0(3) ; CHECK-NEXT: ld 3, 40(1) # 8-byte Folded Reload -; CHECK-NEXT: ld 26, 496(1) # 8-byte Folded Reload -; CHECK-NEXT: ld 25, 488(1) # 8-byte Folded Reload -; CHECK-NEXT: ld 24, 480(1) # 8-byte Folded Reload -; CHECK-NEXT: ld 23, 472(1) # 8-byte Folded Reload -; CHECK-NEXT: ld 22, 464(1) # 8-byte Folded Reload -; CHECK-NEXT: ld 21, 456(1) # 8-byte Folded Reload -; CHECK-NEXT: ld 20, 448(1) # 8-byte Folded Reload -; CHECK-NEXT: ld 19, 440(1) # 8-byte Folded Reload -; CHECK-NEXT: ld 18, 432(1) # 8-byte Folded Reload -; CHECK-NEXT: ld 17, 424(1) # 8-byte Folded Reload -; CHECK-NEXT: ld 16, 416(1) # 8-byte Folded Reload +; CHECK-NEXT: lfd 31, 584(1) # 8-byte Folded Reload +; CHECK-NEXT: lxv 52, 208(1) # 16-byte Folded Reload +; CHECK-NEXT: lxv 53, 224(1) # 16-byte Folded Reload +; CHECK-NEXT: lxv 54, 240(1) # 16-byte Folded Reload +; CHECK-NEXT: lxv 55, 256(1) # 16-byte Folded Reload +; CHECK-NEXT: lxv 56, 272(1) # 16-byte Folded Reload +; CHECK-NEXT: lxv 57, 288(1) # 16-byte Folded Reload +; CHECK-NEXT: lxv 58, 304(1) # 16-byte Folded Reload +; CHECK-NEXT: lxv 59, 320(1) # 16-byte Folded Reload +; CHECK-NEXT: lxv 60, 336(1) # 16-byte Folded Reload +; CHECK-NEXT: lxv 61, 352(1) # 16-byte Folded Reload ; CHECK-NEXT: stxv 32, 0(3) ; CHECK-NEXT: ld 3, 48(1) # 8-byte Folded Reload ; CHECK-NEXT: stxv 37, 0(10) ; CHECK-NEXT: stxv 36, 0(9) ; CHECK-NEXT: stxv 13, 0(8) -; CHECK-NEXT: ld 15, 408(1) # 8-byte Folded Reload -; CHECK-NEXT: ld 14, 400(1) # 8-byte Folded Reload +; CHECK-NEXT: lxv 62, 368(1) # 16-byte Folded Reload +; CHECK-NEXT: lxv 63, 384(1) # 16-byte Folded Reload ; CHECK-NEXT: stxv 12, 0(3) ; CHECK-NEXT: ld 3, 80(1) # 8-byte Folded Reload ; CHECK-NEXT: stxv 11, 0(3) diff --git a/llvm/test/CodeGen/PowerPC/pr43527.ll b/llvm/test/CodeGen/PowerPC/pr43527.ll index 379bd6c070c77..e4b513272b9be 100644 --- a/llvm/test/CodeGen/PowerPC/pr43527.ll +++ b/llvm/test/CodeGen/PowerPC/pr43527.ll @@ -14,8 +14,8 @@ define dso_local void @test(i64 %arg, i64 %arg1) { ; CHECK-NEXT: .cfi_offset lr, 16 ; CHECK-NEXT: .cfi_offset r29, -24 ; CHECK-NEXT: .cfi_offset r30, -16 -; CHECK-NEXT: std r29, -24(r1) # 8-byte Folded Spill ; CHECK-NEXT: std r30, -16(r1) # 8-byte Folded Spill +; CHECK-NEXT: std r29, -24(r1) # 8-byte Folded Spill ; CHECK-NEXT: stdu r1, -64(r1) ; CHECK-NEXT: sub r30, r4, r3 ; CHECK-NEXT: li r29, -4 @@ -33,8 +33,8 @@ define dso_local void @test(i64 %arg, i64 %arg1) { ; CHECK-NEXT: stb r3, 0(r3) ; CHECK-NEXT: addi r1, r1, 64 ; CHECK-NEXT: ld r0, 16(r1) -; CHECK-NEXT: ld r30, -16(r1) # 8-byte Folded Reload ; CHECK-NEXT: ld r29, -24(r1) # 8-byte Folded Reload +; CHECK-NEXT: ld r30, -16(r1) # 8-byte Folded Reload ; CHECK-NEXT: mtlr r0 ; CHECK-NEXT: blr ; CHECK-NEXT: .LBB0_5: # %bb2 diff --git a/llvm/test/CodeGen/PowerPC/shrink-wrap.ll b/llvm/test/CodeGen/PowerPC/shrink-wrap.ll index 12d0b056ca886..44215ce8c4dff 100644 --- a/llvm/test/CodeGen/PowerPC/shrink-wrap.ll +++ b/llvm/test/CodeGen/PowerPC/shrink-wrap.ll @@ -10,26 +10,26 @@ define signext i32 @shrinkwrapme(i32 signext %a, i32 signext %lim) { ; POWERPC64-NEXT: ble 0, .LBB0_4 ; POWERPC64-NEXT: # %bb.1: # %for.body.preheader ; POWERPC64-NEXT: addi 4, 4, -1 -; POWERPC64-NEXT: std 14, -144(1) # 8-byte Folded Spill -; POWERPC64-NEXT: std 15, -136(1) # 8-byte Folded Spill -; POWERPC64-NEXT: std 16, -128(1) # 8-byte Folded Spill -; POWERPC64-NEXT: std 17, -120(1) # 8-byte Folded Spill -; POWERPC64-NEXT: std 18, -112(1) # 8-byte Folded Spill -; POWERPC64-NEXT: std 19, -104(1) # 8-byte Folded Spill -; POWERPC64-NEXT: std 20, -96(1) # 8-byte Folded Spill -; POWERPC64-NEXT: std 21, -88(1) # 8-byte Folded Spill -; POWERPC64-NEXT: std 22, -80(1) # 8-byte Folded Spill -; POWERPC64-NEXT: std 23, -72(1) # 8-byte Folded Spill -; POWERPC64-NEXT: std 24, -64(1) # 8-byte Folded Spill -; POWERPC64-NEXT: std 25, -56(1) # 8-byte Folded Spill -; POWERPC64-NEXT: std 26, -48(1) # 8-byte Folded Spill -; POWERPC64-NEXT: std 27, -40(1) # 8-byte Folded Spill +; POWERPC64-NEXT: std 31, -8(1) # 8-byte Folded Spill +; POWERPC64-NEXT: std 30, -16(1) # 8-byte Folded Spill +; POWERPC64-NEXT: std 29, -24(1) # 8-byte Folded Spill ; POWERPC64-NEXT: std 28, -32(1) # 8-byte Folded Spill +; POWERPC64-NEXT: std 27, -40(1) # 8-byte Folded Spill +; POWERPC64-NEXT: std 26, -48(1) # 8-byte Folded Spill +; POWERPC64-NEXT: std 25, -56(1) # 8-byte Folded Spill +; POWERPC64-NEXT: std 24, -64(1) # 8-byte Folded Spill +; POWERPC64-NEXT: std 23, -72(1) # 8-byte Folded Spill +; POWERPC64-NEXT: std 22, -80(1) # 8-byte Folded Spill +; POWERPC64-NEXT: std 21, -88(1) # 8-byte Folded Spill +; POWERPC64-NEXT: std 20, -96(1) # 8-byte Folded Spill +; POWERPC64-NEXT: std 19, -104(1) # 8-byte Folded Spill +; POWERPC64-NEXT: std 18, -112(1) # 8-byte Folded Spill +; POWERPC64-NEXT: std 17, -120(1) # 8-byte Folded Spill ; POWERPC64-NEXT: clrldi 4, 4, 32 ; POWERPC64-NEXT: addi 4, 4, 1 -; POWERPC64-NEXT: std 29, -24(1) # 8-byte Folded Spill -; POWERPC64-NEXT: std 30, -16(1) # 8-byte Folded Spill -; POWERPC64-NEXT: std 31, -8(1) # 8-byte Folded Spill +; POWERPC64-NEXT: std 16, -128(1) # 8-byte Folded Spill +; POWERPC64-NEXT: std 15, -136(1) # 8-byte Folded Spill +; POWERPC64-NEXT: std 14, -144(1) # 8-byte Folded Spill ; POWERPC64-NEXT: mtctr 4 ; POWERPC64-NEXT: li 4, 0 ; POWERPC64-NEXT: .p2align 4 @@ -40,25 +40,25 @@ define signext i32 @shrinkwrapme(i32 signext %a, i32 signext %lim) { ; POWERPC64-NEXT: #NO_APP ; POWERPC64-NEXT: bdnz .LBB0_2 ; POWERPC64-NEXT: # %bb.3: -; POWERPC64-NEXT: ld 31, -8(1) # 8-byte Folded Reload -; POWERPC64-NEXT: ld 30, -16(1) # 8-byte Folded Reload -; POWERPC64-NEXT: ld 29, -24(1) # 8-byte Folded Reload -; POWERPC64-NEXT: ld 28, -32(1) # 8-byte Folded Reload +; POWERPC64-NEXT: ld 14, -144(1) # 8-byte Folded Reload +; POWERPC64-NEXT: ld 15, -136(1) # 8-byte Folded Reload +; POWERPC64-NEXT: ld 16, -128(1) # 8-byte Folded Reload +; POWERPC64-NEXT: ld 17, -120(1) # 8-byte Folded Reload ; POWERPC64-NEXT: extsw 3, 4 -; POWERPC64-NEXT: ld 27, -40(1) # 8-byte Folded Reload -; POWERPC64-NEXT: ld 26, -48(1) # 8-byte Folded Reload -; POWERPC64-NEXT: ld 25, -56(1) # 8-byte Folded Reload -; POWERPC64-NEXT: ld 24, -64(1) # 8-byte Folded Reload -; POWERPC64-NEXT: ld 23, -72(1) # 8-byte Folded Reload -; POWERPC64-NEXT: ld 22, -80(1) # 8-byte Folded Reload -; POWERPC64-NEXT: ld 21, -88(1) # 8-byte Folded Reload -; POWERPC64-NEXT: ld 20, -96(1) # 8-byte Folded Reload -; POWERPC64-NEXT: ld 19, -104(1) # 8-byte Folded Reload ; POWERPC64-NEXT: ld 18, -112(1) # 8-byte Folded Reload -; POWERPC64-NEXT: ld 17, -120(1) # 8-byte Folded Reload -; POWERPC64-NEXT: ld 16, -128(1) # 8-byte Folded Reload -; POWERPC64-NEXT: ld 15, -136(1) # 8-byte Folded Reload -; POWERPC64-NEXT: ld 14, -144(1) # 8-byte Folded Reload +; POWERPC64-NEXT: ld 19, -104(1) # 8-byte Folded Reload +; POWERPC64-NEXT: ld 20, -96(1) # 8-byte Folded Reload +; POWERPC64-NEXT: ld 21, -88(1) # 8-byte Folded Reload +; POWERPC64-NEXT: ld 22, -80(1) # 8-byte Folded Reload +; POWERPC64-NEXT: ld 23, -72(1) # 8-byte Folded Reload +; POWERPC64-NEXT: ld 24, -64(1) # 8-byte Folded Reload +; POWERPC64-NEXT: ld 25, -56(1) # 8-byte Folded Reload +; POWERPC64-NEXT: ld 26, -48(1) # 8-byte Folded Reload +; POWERPC64-NEXT: ld 27, -40(1) # 8-byte Folded Reload +; POWERPC64-NEXT: ld 28, -32(1) # 8-byte Folded Reload +; POWERPC64-NEXT: ld 29, -24(1) # 8-byte Folded Reload +; POWERPC64-NEXT: ld 30, -16(1) # 8-byte Folded Reload +; POWERPC64-NEXT: ld 31, -8(1) # 8-byte Folded Reload ; POWERPC64-NEXT: blr ; POWERPC64-NEXT: .LBB0_4: ; POWERPC64-NEXT: li 4, 0 @@ -70,24 +70,24 @@ define signext i32 @shrinkwrapme(i32 signext %a, i32 signext %lim) { ; POWERPC32-AIX-NEXT: cmpwi 4, 0 ; POWERPC32-AIX-NEXT: ble 0, L..BB0_4 ; POWERPC32-AIX-NEXT: # %bb.1: # %for.body.preheader -; POWERPC32-AIX-NEXT: stw 14, -72(1) # 4-byte Folded Spill -; POWERPC32-AIX-NEXT: stw 15, -68(1) # 4-byte Folded Spill -; POWERPC32-AIX-NEXT: stw 16, -64(1) # 4-byte Folded Spill -; POWERPC32-AIX-NEXT: stw 17, -60(1) # 4-byte Folded Spill -; POWERPC32-AIX-NEXT: stw 18, -56(1) # 4-byte Folded Spill -; POWERPC32-AIX-NEXT: stw 19, -52(1) # 4-byte Folded Spill -; POWERPC32-AIX-NEXT: stw 20, -48(1) # 4-byte Folded Spill -; POWERPC32-AIX-NEXT: stw 21, -44(1) # 4-byte Folded Spill -; POWERPC32-AIX-NEXT: stw 22, -40(1) # 4-byte Folded Spill -; POWERPC32-AIX-NEXT: stw 23, -36(1) # 4-byte Folded Spill -; POWERPC32-AIX-NEXT: stw 24, -32(1) # 4-byte Folded Spill -; POWERPC32-AIX-NEXT: stw 25, -28(1) # 4-byte Folded Spill -; POWERPC32-AIX-NEXT: stw 26, -24(1) # 4-byte Folded Spill -; POWERPC32-AIX-NEXT: stw 27, -20(1) # 4-byte Folded Spill -; POWERPC32-AIX-NEXT: stw 28, -16(1) # 4-byte Folded Spill -; POWERPC32-AIX-NEXT: stw 29, -12(1) # 4-byte Folded Spill -; POWERPC32-AIX-NEXT: stw 30, -8(1) # 4-byte Folded Spill ; POWERPC32-AIX-NEXT: stw 31, -4(1) # 4-byte Folded Spill +; POWERPC32-AIX-NEXT: stw 30, -8(1) # 4-byte Folded Spill +; POWERPC32-AIX-NEXT: stw 29, -12(1) # 4-byte Folded Spill +; POWERPC32-AIX-NEXT: stw 28, -16(1) # 4-byte Folded Spill +; POWERPC32-AIX-NEXT: stw 27, -20(1) # 4-byte Folded Spill +; POWERPC32-AIX-NEXT: stw 26, -24(1) # 4-byte Folded Spill +; POWERPC32-AIX-NEXT: stw 25, -28(1) # 4-byte Folded Spill +; POWERPC32-AIX-NEXT: stw 24, -32(1) # 4-byte Folded Spill +; POWERPC32-AIX-NEXT: stw 23, -36(1) # 4-byte Folded Spill +; POWERPC32-AIX-NEXT: stw 22, -40(1) # 4-byte Folded Spill +; POWERPC32-AIX-NEXT: stw 21, -44(1) # 4-byte Folded Spill +; POWERPC32-AIX-NEXT: stw 20, -48(1) # 4-byte Folded Spill +; POWERPC32-AIX-NEXT: stw 19, -52(1) # 4-byte Folded Spill +; POWERPC32-AIX-NEXT: stw 18, -56(1) # 4-byte Folded Spill +; POWERPC32-AIX-NEXT: stw 17, -60(1) # 4-byte Folded Spill +; POWERPC32-AIX-NEXT: stw 16, -64(1) # 4-byte Folded Spill +; POWERPC32-AIX-NEXT: stw 15, -68(1) # 4-byte Folded Spill +; POWERPC32-AIX-NEXT: stw 14, -72(1) # 4-byte Folded Spill ; POWERPC32-AIX-NEXT: mtctr 4 ; POWERPC32-AIX-NEXT: li 4, 0 ; POWERPC32-AIX-NEXT: .align 4 @@ -98,25 +98,25 @@ define signext i32 @shrinkwrapme(i32 signext %a, i32 signext %lim) { ; POWERPC32-AIX-NEXT: #NO_APP ; POWERPC32-AIX-NEXT: bdnz L..BB0_2 ; POWERPC32-AIX-NEXT: # %bb.3: -; POWERPC32-AIX-NEXT: lwz 31, -4(1) # 4-byte Folded Reload -; POWERPC32-AIX-NEXT: lwz 30, -8(1) # 4-byte Folded Reload -; POWERPC32-AIX-NEXT: lwz 29, -12(1) # 4-byte Folded Reload -; POWERPC32-AIX-NEXT: lwz 28, -16(1) # 4-byte Folded Reload +; POWERPC32-AIX-NEXT: lwz 14, -72(1) # 4-byte Folded Reload +; POWERPC32-AIX-NEXT: lwz 15, -68(1) # 4-byte Folded Reload +; POWERPC32-AIX-NEXT: lwz 16, -64(1) # 4-byte Folded Reload +; POWERPC32-AIX-NEXT: lwz 17, -60(1) # 4-byte Folded Reload ; POWERPC32-AIX-NEXT: mr 3, 4 -; POWERPC32-AIX-NEXT: lwz 27, -20(1) # 4-byte Folded Reload -; POWERPC32-AIX-NEXT: lwz 26, -24(1) # 4-byte Folded Reload -; POWERPC32-AIX-NEXT: lwz 25, -28(1) # 4-byte Folded Reload -; POWERPC32-AIX-NEXT: lwz 24, -32(1) # 4-byte Folded Reload -; POWERPC32-AIX-NEXT: lwz 23, -36(1) # 4-byte Folded Reload -; POWERPC32-AIX-NEXT: lwz 22, -40(1) # 4-byte Folded Reload -; POWERPC32-AIX-NEXT: lwz 21, -44(1) # 4-byte Folded Reload -; POWERPC32-AIX-NEXT: lwz 20, -48(1) # 4-byte Folded Reload -; POWERPC32-AIX-NEXT: lwz 19, -52(1) # 4-byte Folded Reload ; POWERPC32-AIX-NEXT: lwz 18, -56(1) # 4-byte Folded Reload -; POWERPC32-AIX-NEXT: lwz 17, -60(1) # 4-byte Folded Reload -; POWERPC32-AIX-NEXT: lwz 16, -64(1) # 4-byte Folded Reload -; POWERPC32-AIX-NEXT: lwz 15, -68(1) # 4-byte Folded Reload -; POWERPC32-AIX-NEXT: lwz 14, -72(1) # 4-byte Folded Reload +; POWERPC32-AIX-NEXT: lwz 19, -52(1) # 4-byte Folded Reload +; POWERPC32-AIX-NEXT: lwz 20, -48(1) # 4-byte Folded Reload +; POWERPC32-AIX-NEXT: lwz 21, -44(1) # 4-byte Folded Reload +; POWERPC32-AIX-NEXT: lwz 22, -40(1) # 4-byte Folded Reload +; POWERPC32-AIX-NEXT: lwz 23, -36(1) # 4-byte Folded Reload +; POWERPC32-AIX-NEXT: lwz 24, -32(1) # 4-byte Folded Reload +; POWERPC32-AIX-NEXT: lwz 25, -28(1) # 4-byte Folded Reload +; POWERPC32-AIX-NEXT: lwz 26, -24(1) # 4-byte Folded Reload +; POWERPC32-AIX-NEXT: lwz 27, -20(1) # 4-byte Folded Reload +; POWERPC32-AIX-NEXT: lwz 28, -16(1) # 4-byte Folded Reload +; POWERPC32-AIX-NEXT: lwz 29, -12(1) # 4-byte Folded Reload +; POWERPC32-AIX-NEXT: lwz 30, -8(1) # 4-byte Folded Reload +; POWERPC32-AIX-NEXT: lwz 31, -4(1) # 4-byte Folded Reload ; POWERPC32-AIX-NEXT: blr ; POWERPC32-AIX-NEXT: L..BB0_4: ; POWERPC32-AIX-NEXT: li 3, 0 @@ -128,26 +128,26 @@ define signext i32 @shrinkwrapme(i32 signext %a, i32 signext %lim) { ; POWERPC64-AIX-NEXT: blt 0, L..BB0_4 ; POWERPC64-AIX-NEXT: # %bb.1: # %for.body.preheader ; POWERPC64-AIX-NEXT: addi 4, 4, -1 -; POWERPC64-AIX-NEXT: std 14, -144(1) # 8-byte Folded Spill -; POWERPC64-AIX-NEXT: std 15, -136(1) # 8-byte Folded Spill -; POWERPC64-AIX-NEXT: std 16, -128(1) # 8-byte Folded Spill -; POWERPC64-AIX-NEXT: std 17, -120(1) # 8-byte Folded Spill -; POWERPC64-AIX-NEXT: std 18, -112(1) # 8-byte Folded Spill -; POWERPC64-AIX-NEXT: std 19, -104(1) # 8-byte Folded Spill -; POWERPC64-AIX-NEXT: std 20, -96(1) # 8-byte Folded Spill -; POWERPC64-AIX-NEXT: std 21, -88(1) # 8-byte Folded Spill -; POWERPC64-AIX-NEXT: std 22, -80(1) # 8-byte Folded Spill -; POWERPC64-AIX-NEXT: std 23, -72(1) # 8-byte Folded Spill -; POWERPC64-AIX-NEXT: std 24, -64(1) # 8-byte Folded Spill -; POWERPC64-AIX-NEXT: std 25, -56(1) # 8-byte Folded Spill -; POWERPC64-AIX-NEXT: std 26, -48(1) # 8-byte Folded Spill -; POWERPC64-AIX-NEXT: std 27, -40(1) # 8-byte Folded Spill +; POWERPC64-AIX-NEXT: std 31, -8(1) # 8-byte Folded Spill +; POWERPC64-AIX-NEXT: std 30, -16(1) # 8-byte Folded Spill +; POWERPC64-AIX-NEXT: std 29, -24(1) # 8-byte Folded Spill ; POWERPC64-AIX-NEXT: std 28, -32(1) # 8-byte Folded Spill +; POWERPC64-AIX-NEXT: std 27, -40(1) # 8-byte Folded Spill +; POWERPC64-AIX-NEXT: std 26, -48(1) # 8-byte Folded Spill +; POWERPC64-AIX-NEXT: std 25, -56(1) # 8-byte Folded Spill +; POWERPC64-AIX-NEXT: std 24, -64(1) # 8-byte Folded Spill +; POWERPC64-AIX-NEXT: std 23, -72(1) # 8-byte Folded Spill +; POWERPC64-AIX-NEXT: std 22, -80(1) # 8-byte Folded Spill +; POWERPC64-AIX-NEXT: std 21, -88(1) # 8-byte Folded Spill +; POWERPC64-AIX-NEXT: std 20, -96(1) # 8-byte Folded Spill +; POWERPC64-AIX-NEXT: std 19, -104(1) # 8-byte Folded Spill +; POWERPC64-AIX-NEXT: std 18, -112(1) # 8-byte Folded Spill +; POWERPC64-AIX-NEXT: std 17, -120(1) # 8-byte Folded Spill ; POWERPC64-AIX-NEXT: clrldi 4, 4, 32 ; POWERPC64-AIX-NEXT: addi 4, 4, 1 -; POWERPC64-AIX-NEXT: std 29, -24(1) # 8-byte Folded Spill -; POWERPC64-AIX-NEXT: std 30, -16(1) # 8-byte Folded Spill -; POWERPC64-AIX-NEXT: std 31, -8(1) # 8-byte Folded Spill +; POWERPC64-AIX-NEXT: std 16, -128(1) # 8-byte Folded Spill +; POWERPC64-AIX-NEXT: std 15, -136(1) # 8-byte Folded Spill +; POWERPC64-AIX-NEXT: std 14, -144(1) # 8-byte Folded Spill ; POWERPC64-AIX-NEXT: mtctr 4 ; POWERPC64-AIX-NEXT: li 4, 0 ; POWERPC64-AIX-NEXT: .align 4 @@ -158,25 +158,25 @@ define signext i32 @shrinkwrapme(i32 signext %a, i32 signext %lim) { ; POWERPC64-AIX-NEXT: #NO_APP ; POWERPC64-AIX-NEXT: bdnz L..BB0_2 ; POWERPC64-AIX-NEXT: # %bb.3: -; POWERPC64-AIX-NEXT: ld 31, -8(1) # 8-byte Folded Reload -; POWERPC64-AIX-NEXT: ld 30, -16(1) # 8-byte Folded Reload -; POWERPC64-AIX-NEXT: ld 29, -24(1) # 8-byte Folded Reload -; POWERPC64-AIX-NEXT: ld 28, -32(1) # 8-byte Folded Reload +; POWERPC64-AIX-NEXT: ld 14, -144(1) # 8-byte Folded Reload +; POWERPC64-AIX-NEXT: ld 15, -136(1) # 8-byte Folded Reload +; POWERPC64-AIX-NEXT: ld 16, -128(1) # 8-byte Folded Reload +; POWERPC64-AIX-NEXT: ld 17, -120(1) # 8-byte Folded Reload ; POWERPC64-AIX-NEXT: extsw 3, 4 -; POWERPC64-AIX-NEXT: ld 27, -40(1) # 8-byte Folded Reload -; POWERPC64-AIX-NEXT: ld 26, -48(1) # 8-byte Folded Reload -; POWERPC64-AIX-NEXT: ld 25, -56(1) # 8-byte Folded Reload -; POWERPC64-AIX-NEXT: ld 24, -64(1) # 8-byte Folded Reload -; POWERPC64-AIX-NEXT: ld 23, -72(1) # 8-byte Folded Reload -; POWERPC64-AIX-NEXT: ld 22, -80(1) # 8-byte Folded Reload -; POWERPC64-AIX-NEXT: ld 21, -88(1) # 8-byte Folded Reload -; POWERPC64-AIX-NEXT: ld 20, -96(1) # 8-byte Folded Reload -; POWERPC64-AIX-NEXT: ld 19, -104(1) # 8-byte Folded Reload ; POWERPC64-AIX-NEXT: ld 18, -112(1) # 8-byte Folded Reload -; POWERPC64-AIX-NEXT: ld 17, -120(1) # 8-byte Folded Reload -; POWERPC64-AIX-NEXT: ld 16, -128(1) # 8-byte Folded Reload -; POWERPC64-AIX-NEXT: ld 15, -136(1) # 8-byte Folded Reload -; POWERPC64-AIX-NEXT: ld 14, -144(1) # 8-byte Folded Reload +; POWERPC64-AIX-NEXT: ld 19, -104(1) # 8-byte Folded Reload +; POWERPC64-AIX-NEXT: ld 20, -96(1) # 8-byte Folded Reload +; POWERPC64-AIX-NEXT: ld 21, -88(1) # 8-byte Folded Reload +; POWERPC64-AIX-NEXT: ld 22, -80(1) # 8-byte Folded Reload +; POWERPC64-AIX-NEXT: ld 23, -72(1) # 8-byte Folded Reload +; POWERPC64-AIX-NEXT: ld 24, -64(1) # 8-byte Folded Reload +; POWERPC64-AIX-NEXT: ld 25, -56(1) # 8-byte Folded Reload +; POWERPC64-AIX-NEXT: ld 26, -48(1) # 8-byte Folded Reload +; POWERPC64-AIX-NEXT: ld 27, -40(1) # 8-byte Folded Reload +; POWERPC64-AIX-NEXT: ld 28, -32(1) # 8-byte Folded Reload +; POWERPC64-AIX-NEXT: ld 29, -24(1) # 8-byte Folded Reload +; POWERPC64-AIX-NEXT: ld 30, -16(1) # 8-byte Folded Reload +; POWERPC64-AIX-NEXT: ld 31, -8(1) # 8-byte Folded Reload ; POWERPC64-AIX-NEXT: blr ; POWERPC64-AIX-NEXT: L..BB0_4: ; POWERPC64-AIX-NEXT: li 4, 0 diff --git a/llvm/test/CodeGen/X86/cfi-epilogue-with-return.mir b/llvm/test/CodeGen/X86/cfi-epilogue-with-return.mir index 583e54b097faf..395a2c5ebf915 100644 --- a/llvm/test/CodeGen/X86/cfi-epilogue-with-return.mir +++ b/llvm/test/CodeGen/X86/cfi-epilogue-with-return.mir @@ -21,8 +21,22 @@ liveins: frameInfo: maxAlignment: 1 hasCalls: true - savePoint: '%bb.1' - restorePoint: '%bb.1' + savePoint: + - point: '%bb.1' + registers: + - '$rbx' + - '$r12' + - '$r13' + - '$r14' + - '$r15' + restorePoint: + - point: '%bb.1' + registers: + - '$rbx' + - '$r12' + - '$r13' + - '$r14' + - '$r15' machineFunctionInfo: {} body: | bb.0: diff --git a/llvm/test/CodeGen/X86/cfi-epilogue-without-return.mir b/llvm/test/CodeGen/X86/cfi-epilogue-without-return.mir index 8f04721489608..73e07d0c28335 100644 --- a/llvm/test/CodeGen/X86/cfi-epilogue-without-return.mir +++ b/llvm/test/CodeGen/X86/cfi-epilogue-without-return.mir @@ -28,8 +28,22 @@ liveins: frameInfo: maxAlignment: 1 hasCalls: true - savePoint: '%bb.1' - restorePoint: '%bb.1' + savePoint: + - point: '%bb.1' + registers: + - '$rbx' + - '$r12' + - '$r13' + - '$r14' + - '$r15' + restorePoint: + - point: '%bb.1' + registers: + - '$rbx' + - '$r12' + - '$r13' + - '$r14' + - '$r15' machineFunctionInfo: {} body: | bb.0: diff --git a/llvm/test/CodeGen/X86/shrink_wrap_dbg_value.mir b/llvm/test/CodeGen/X86/shrink_wrap_dbg_value.mir index aa7befc18d4fe..c4f0a5a0a3a1c 100644 --- a/llvm/test/CodeGen/X86/shrink_wrap_dbg_value.mir +++ b/llvm/test/CodeGen/X86/shrink_wrap_dbg_value.mir @@ -119,8 +119,18 @@ frameInfo: hasOpaqueSPAdjustment: false hasVAStart: false hasMustTailInVarArgFunc: false - # CHECK: savePoint: '%bb.1' - # CHECK: restorePoint: '%bb.3' + # CHECK: savePoint: + # CHECK-NEXT: - point: '%bb.1' + # CHECK-NEXT: registers: + # CHECK-NEXT: - '$ebx' + # CHECK-NEXT: - '$edi' + # CHECK-NEXT: - '$esi' + # CHECK: restorePoint: + # CHECK-NEXT: - point: '%bb.3' + # CHECK-NEXT: registers: + # CHECK-NEXT: - '$ebx' + # CHECK-NEXT: - '$edi' + # CHECK-NEXT: - '$esi' savePoint: '' restorePoint: '' fixedStack: diff --git a/llvm/test/tools/llvm-reduce/mir/preserve-frame-info.mir b/llvm/test/tools/llvm-reduce/mir/preserve-frame-info.mir index d7ad5f88874d7..13c4b0ce86ee8 100644 --- a/llvm/test/tools/llvm-reduce/mir/preserve-frame-info.mir +++ b/llvm/test/tools/llvm-reduce/mir/preserve-frame-info.mir @@ -20,8 +20,12 @@ # RESULT-NEXT: hasVAStart: true # RESULT-NEXT: hasMustTailInVarArgFunc: true # RESULT-NEXT: hasTailCall: true -# RESULT-NEXT: savePoint: '%bb.1' -# RESULT-NEXT: restorePoint: '%bb.2' +# RESULT-NEXT: savePoint: +# RESULT-NEXT: - point: '%bb.1' +# RESULT-NEXT: registers: [] +# RESULT-NEXT: restorePoint: +# RESULT-NEXT: - point: '%bb.2' +# RESULT-NEXT: registers: [] # RESULT-NEXT: fixedStack: # RESULT-NEXT: - { id: 0, offset: 56, size: 4, alignment: 8, callee-saved-register: '$sgpr44', diff --git a/llvm/tools/llvm-reduce/ReducerWorkItem.cpp b/llvm/tools/llvm-reduce/ReducerWorkItem.cpp index 7cd974f0cf438..434f88ebd8940 100644 --- a/llvm/tools/llvm-reduce/ReducerWorkItem.cpp +++ b/llvm/tools/llvm-reduce/ReducerWorkItem.cpp @@ -93,11 +93,11 @@ static void cloneFrameInfo( DstMFI.setCVBytesOfCalleeSavedRegisters( SrcMFI.getCVBytesOfCalleeSavedRegisters()); - if (MachineBasicBlock *SavePt = SrcMFI.getSavePoint()) - DstMFI.setSavePoint(Src2DstMBB.find(SavePt)->second); - if (MachineBasicBlock *RestorePt = SrcMFI.getRestorePoint()) - DstMFI.setRestorePoint(Src2DstMBB.find(RestorePt)->second); + DstMFI.setSavePoints(MachineFrameInfo::constructSaveRestorePoints( + SrcMFI.getSavePoints(), Src2DstMBB)); + DstMFI.setRestorePoints(MachineFrameInfo::constructSaveRestorePoints( + SrcMFI.getRestorePoints(), Src2DstMBB)); auto CopyObjectProperties = [](MachineFrameInfo &DstMFI, const MachineFrameInfo &SrcMFI, int FI) {