diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp index 165b57c87beb1..280ea47c5d7cc 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -883,6 +883,16 @@ void VPRegionBlock::print(raw_ostream &O, const Twine &Indent, void VPRegionBlock::dissolveToCFGLoop() { auto *Header = cast(getEntry()); + if (auto *CanIV = dyn_cast(&Header->front())) { + assert(this == getPlan()->getVectorLoopRegion() && + "Canonical IV must be in the entry of the top-level loop region"); + auto *ScalarR = VPBuilder(CanIV).createScalarPhi( + {CanIV->getStartValue(), CanIV->getBackedgeValue()}, + CanIV->getDebugLoc(), "index"); + CanIV->replaceAllUsesWith(ScalarR); + CanIV->eraseFromParent(); + } + VPBlockBase *Preheader = getSinglePredecessor(); auto *ExitingLatch = cast(getExiting()); VPBlockBase *Middle = getSingleSuccessor(); diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 348100124ba3c..ea617f042566b 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -2599,14 +2599,10 @@ void VPlanTransforms::convertToConcreteRecipes(VPlan &Plan, for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly( vp_depth_first_deep(Plan.getEntry()))) { for (VPRecipeBase &R : make_early_inc_range(*VPBB)) { - if (isa(&R)) { - auto *PhiR = cast(&R); - StringRef Name = - isa(PhiR) ? "index" : "evl.based.iv"; - VPBuilder Builder(PhiR); - auto *ScalarR = Builder.createScalarPhi( + if (auto *PhiR = dyn_cast(&R)) { + auto *ScalarR = VPBuilder(PhiR).createScalarPhi( {PhiR->getStartValue(), PhiR->getBackedgeValue()}, - PhiR->getDebugLoc(), Name); + PhiR->getDebugLoc(), "evl.based.iv"); PhiR->replaceAllUsesWith(ScalarR); ToRemove.push_back(PhiR); continue;