diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index f2f8a85b7cc23..ef8abdaa8aa76 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -2978,8 +2978,7 @@ LoopVectorizationCostModel::getVectorIntrinsicCost(CallInst *CI, void InnerLoopVectorizer::fixVectorizedLoop(VPTransformState &State) { // Fix widened non-induction PHIs by setting up the PHI operands. - if (EnableVPlanNativePath) - fixNonInductionPHIs(State); + fixNonInductionPHIs(State); // Forget the original basic block. PSE.getSE()->forgetLoop(OrigLoop); diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 88f3f672d3aa3..d15418ab7415e 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -2287,10 +2287,16 @@ class VPWidenPHIRecipe : public VPSingleDefRecipe { /// List of incoming blocks. Only used in the VPlan native path. SmallVector IncomingBlocks; + /// Name to use for the generated IR instruction for the widened IV. + std::string Name; + public: /// Create a new VPWidenPHIRecipe for \p Phi with start value \p Start. - VPWidenPHIRecipe(PHINode *Phi, VPValue *Start = nullptr) - : VPSingleDefRecipe(VPDef::VPWidenPHISC, ArrayRef(), Phi) { + VPWidenPHIRecipe(PHINode *Phi, VPValue *Start = nullptr, + const Twine &Name = "vec.phi") + : VPSingleDefRecipe(VPDef::VPWidenPHISC, ArrayRef(), Phi, + Phi->getDebugLoc()), + Name(Name.str()) { if (Start) addOperand(Start); } @@ -2312,13 +2318,15 @@ class VPWidenPHIRecipe : public VPSingleDefRecipe { VPSlotTracker &SlotTracker) const override; #endif - /// Adds a pair (\p IncomingV, \p IncomingBlock) to the phi. + /// Adds a pair (\p IncomingV, \p IncomingBlock) to the phi. Only used in the + /// VPlan native path. void addIncoming(VPValue *IncomingV, VPBasicBlock *IncomingBlock) { addOperand(IncomingV); IncomingBlocks.push_back(IncomingBlock); } - /// Returns the \p I th incoming VPBasicBlock. + /// Returns the \p I th incoming VPBasicBlock. Only used in the VPlan native + /// path. VPBasicBlock *getIncomingBlock(unsigned I) { return IncomingBlocks[I]; } /// Returns the \p I th incoming VPValue. diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 77c08839dbfa9..b470e228e023a 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -3440,12 +3440,13 @@ void VPReductionPHIRecipe::print(raw_ostream &O, const Twine &Indent, #endif void VPWidenPHIRecipe::execute(VPTransformState &State) { - assert(EnableVPlanNativePath && - "Non-native vplans are not expected to have VPWidenPHIRecipes."); + assert((EnableVPlanNativePath || IncomingBlocks.empty()) && + "Non-native vplans are not expected to have VPWidenPHIRecipes with " + "incoming blocks."); Value *Op0 = State.get(getOperand(0)); Type *VecTy = Op0->getType(); - Value *VecPhi = State.Builder.CreatePHI(VecTy, 2, "vec.phi"); + Value *VecPhi = State.Builder.CreatePHI(VecTy, 2, Name); State.set(this, VecPhi); }