diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h index ed3e45dd2c6c8..b6f51bc0a3f04 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h +++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h @@ -25,7 +25,6 @@ #define LLVM_TRANSFORMS_VECTORIZE_LOOPVECTORIZATIONPLANNER_H #include "VPlan.h" -#include "llvm/ADT/SmallSet.h" #include "llvm/Support/InstructionCost.h" namespace llvm { @@ -46,6 +45,7 @@ struct VFRange; class VPBuilder { VPBasicBlock *BB = nullptr; VPBasicBlock::iterator InsertPt = VPBasicBlock::iterator(); + [[maybe_unused]] VPlan &Plan; /// Insert \p VPI in BB at InsertPt if BB is set. template T *tryInsertInstruction(T *R) { @@ -67,10 +67,15 @@ class VPBuilder { } public: - VPBuilder() = default; - VPBuilder(VPBasicBlock *InsertBB) { setInsertPoint(InsertBB); } - VPBuilder(VPRecipeBase *InsertPt) { setInsertPoint(InsertPt); } - VPBuilder(VPBasicBlock *TheBB, VPBasicBlock::iterator IP) { + VPBuilder(VPlan &Plan) : Plan(Plan) {} + VPBuilder(VPlan &Plan, VPBasicBlock *InsertBB) : Plan(Plan) { + setInsertPoint(InsertBB); + } + VPBuilder(VPlan &Plan, VPRecipeBase *InsertPt) : Plan(Plan) { + setInsertPoint(InsertPt); + } + VPBuilder(VPlan &Plan, VPBasicBlock *TheBB, VPBasicBlock::iterator IP) + : Plan(Plan) { setInsertPoint(TheBB, IP); } @@ -84,13 +89,6 @@ class VPBuilder { VPBasicBlock *getInsertBlock() const { return BB; } VPBasicBlock::iterator getInsertPoint() const { return InsertPt; } - /// Create a VPBuilder to insert after \p R. - static VPBuilder getToInsertAfter(VPRecipeBase *R) { - VPBuilder B; - B.setInsertPoint(R->getParent(), std::next(R->getIterator())); - return B; - } - /// InsertPoint - A saved insertion point. class VPInsertPoint { VPBasicBlock *Block = nullptr; @@ -394,9 +392,6 @@ class LoopVectorizationPlanner { /// Profitable vector factors. SmallVector ProfitableVFs; - /// A builder used to construct the current plan. - VPBuilder Builder; - /// Computes the cost of \p Plan for vectorization factor \p VF. /// /// The current implementation requires access to the diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 610e4904a80ad..15461aa8e845a 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -8986,7 +8986,7 @@ static void addCanonicalIVRecipes(VPlan &Plan, Type *IdxTy, bool HasNUW, VPBasicBlock *Header = TopRegion->getEntryBasicBlock(); Header->insert(CanonicalIVPHI, Header->begin()); - VPBuilder Builder(TopRegion->getExitingBasicBlock()); + VPBuilder Builder(Plan, TopRegion->getExitingBasicBlock()); // Add a VPInstruction to increment the scalar canonical IV by VF * UF. auto *CanonicalIVIncrement = Builder.createOverflowingOp( Instruction::Add, {CanonicalIVPHI, &Plan.getVFxUF()}, {HasNUW, false}, DL, @@ -9046,9 +9046,9 @@ static void addScalarResumePhis(VPRecipeBuilder &Builder, VPlan &Plan, auto *MiddleVPBB = cast(ScalarPH->getSinglePredecessor()); VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion(); VPBuilder VectorPHBuilder( - cast(VectorRegion->getSinglePredecessor())); - VPBuilder MiddleBuilder(MiddleVPBB, MiddleVPBB->getFirstNonPhi()); - VPBuilder ScalarPHBuilder(ScalarPH); + Plan, cast(VectorRegion->getSinglePredecessor())); + VPBuilder MiddleBuilder(Plan, MiddleVPBB, MiddleVPBB->getFirstNonPhi()); + VPBuilder ScalarPHBuilder(Plan, ScalarPH); VPValue *OneVPV = Plan.getOrAddLiveIn( ConstantInt::get(Plan.getCanonicalIV()->getScalarType(), 1)); for (VPRecipeBase &ScalarPhiR : *Plan.getScalarHeader()) { @@ -9140,7 +9140,7 @@ addUsersInExitBlocks(VPlan &Plan, return; auto *MiddleVPBB = Plan.getMiddleBlock(); - VPBuilder B(MiddleVPBB, MiddleVPBB->getFirstNonPhi()); + VPBuilder B(Plan, MiddleVPBB, MiddleVPBB->getFirstNonPhi()); // Introduce extract for exiting values and update the VPIRInstructions // modeling the corresponding LCSSA phis. @@ -9162,8 +9162,8 @@ static void addExitUsersForFirstOrderRecurrences( VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion(); auto *ScalarPHVPBB = Plan.getScalarPreheader(); auto *MiddleVPBB = Plan.getMiddleBlock(); - VPBuilder ScalarPHBuilder(ScalarPHVPBB); - VPBuilder MiddleBuilder(MiddleVPBB, MiddleVPBB->getFirstNonPhi()); + VPBuilder ScalarPHBuilder(Plan, ScalarPHVPBB); + VPBuilder MiddleBuilder(Plan, MiddleVPBB, MiddleVPBB->getFirstNonPhi()); VPValue *TwoVPV = Plan.getOrAddLiveIn( ConstantInt::get(Plan.getCanonicalIV()->getScalarType(), 2)); @@ -9300,8 +9300,7 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) { bool HasNUW = !IVUpdateMayOverflow || Style == TailFoldingStyle::None; addCanonicalIVRecipes(*Plan, Legal->getWidestInductionType(), HasNUW, DL); - VPRecipeBuilder RecipeBuilder(*Plan, OrigLoop, TLI, &TTI, Legal, CM, PSE, - Builder); + VPRecipeBuilder RecipeBuilder(*Plan, OrigLoop, TLI, &TTI, Legal, CM, PSE); // --------------------------------------------------------------------------- // Pre-construction: record ingredients whose recipes we'll need to further @@ -9357,7 +9356,7 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) { // ingredients and fill a new VPBasicBlock. if (VPBB != HeaderVPBB) VPBB->setName(BB->getName()); - Builder.setInsertPoint(VPBB); + RecipeBuilder.setInsertPoint(VPBB); if (VPBB == HeaderVPBB) RecipeBuilder.createHeaderMask(); @@ -9521,7 +9520,7 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) { // Sink users of fixed-order recurrence past the recipe defining the previous // value and introduce FirstOrderRecurrenceSplice VPInstructions. if (!VPlanTransforms::runPass(VPlanTransforms::adjustFixedOrderRecurrences, - *Plan, Builder)) + *Plan, RecipeBuilder.getIRBuilder())) return nullptr; if (useActiveLaneMask(Style)) { @@ -9571,8 +9570,7 @@ VPlanPtr LoopVectorizationPlanner::buildVPlan(VFRange &Range) { // Collect mapping of IR header phis to header phi recipes, to be used in // addScalarResumePhis. - VPRecipeBuilder RecipeBuilder(*Plan, OrigLoop, TLI, &TTI, Legal, CM, PSE, - Builder); + VPRecipeBuilder RecipeBuilder(*Plan, OrigLoop, TLI, &TTI, Legal, CM, PSE); for (auto &R : Plan->getVectorLoopRegion()->getEntryBasicBlock()->phis()) { if (isa(&R)) continue; @@ -9737,6 +9735,7 @@ void LoopVectorizationPlanner::adjustRecipesForReductions( } } VPBasicBlock *LatchVPBB = VectorLoopRegion->getExitingBasicBlock(); + VPBuilder Builder(*Plan); Builder.setInsertPoint(&*LatchVPBB->begin()); VPBasicBlock::iterator IP = MiddleVPBB->getFirstNonPhi(); for (VPRecipeBase &R : @@ -10248,7 +10247,7 @@ static void preparePlanForMainVectorLoop(VPlan &MainPlan, VPlan &EpiPlan) { m_Specific(VectorTC), m_SpecificInt(0))); })) return; - VPBuilder ScalarPHBuilder(MainScalarPH, MainScalarPH->begin()); + VPBuilder ScalarPHBuilder(MainPlan, MainScalarPH, MainScalarPH->begin()); ScalarPHBuilder.createNaryOp( VPInstruction::ResumePhi, {VectorTC, MainPlan.getCanonicalIV()->getStartValue()}, {}, diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h index e8d3ad89e14cf..6045e7d91ca96 100644 --- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h +++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h @@ -12,7 +12,6 @@ #include "LoopVectorizationPlanner.h" #include "VPlan.h" #include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/PointerUnion.h" #include "llvm/Analysis/ScalarEvolutionExpressions.h" #include "llvm/IR/IRBuilder.h" @@ -66,7 +65,7 @@ class VPRecipeBuilder { PredicatedScalarEvolution &PSE; - VPBuilder &Builder; + VPBuilder Builder; /// When we if-convert we need to create edge masks. We have to cache values /// so that we don't end up with exponential recursion/IR. Note that @@ -155,9 +154,13 @@ class VPRecipeBuilder { const TargetTransformInfo *TTI, LoopVectorizationLegality *Legal, LoopVectorizationCostModel &CM, - PredicatedScalarEvolution &PSE, VPBuilder &Builder) + PredicatedScalarEvolution &PSE) : Plan(Plan), OrigLoop(OrigLoop), TLI(TLI), TTI(TTI), Legal(Legal), - CM(CM), PSE(PSE), Builder(Builder) {} + CM(CM), PSE(PSE), Builder(Plan) {} + + void setInsertPoint(VPBasicBlock *VPBB) { Builder.setInsertPoint(VPBB); } + + VPBuilder &getIRBuilder() { return Builder; } std::optional getScalingForReduction(const Instruction *ExitInst) { auto It = ScaledReductionMap.find(ExitInst); diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp index 5a88ebeffb18b..1dcfff3eed8a1 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -922,7 +922,7 @@ VPlanPtr VPlan::createInitialVPlan(Type *InductionTy, // of the corresponding compare because they may have ended up with // different line numbers and we want to avoid awkward line stepping while // debugging. Eg. if the compare has got a line number inside the loop. - VPBuilder Builder(MiddleVPBB); + VPBuilder Builder(*Plan, MiddleVPBB); VPValue *Cmp = TailFolded ? Plan->getOrAddLiveIn(ConstantInt::getTrue( diff --git a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp b/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp index 5a2e5d7cfee48..a0505e0d1bb8d 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp @@ -72,7 +72,7 @@ class PlainCFGBuilder { public: PlainCFGBuilder(Loop *Lp, LoopInfo *LI, VPlan &P) - : TheLoop(Lp), LI(LI), Plan(P) {} + : TheLoop(Lp), LI(LI), Plan(P), VPIRBuilder(Plan) {} /// Build plain CFG for TheLoop and connects it to Plan's entry. void buildPlainCFG(); diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 6c917e4eef655..b6c4277cd18e9 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -592,7 +592,7 @@ static void legalizeAndOptimizeInductions(VPlan &Plan) { using namespace llvm::VPlanPatternMatch; VPBasicBlock *HeaderVPBB = Plan.getVectorLoopRegion()->getEntryBasicBlock(); bool HasOnlyVectorVFs = !Plan.hasScalarVFOnly(); - VPBuilder Builder(HeaderVPBB, HeaderVPBB->getFirstNonPhi()); + VPBuilder Builder(Plan, HeaderVPBB, HeaderVPBB->getFirstNonPhi()); for (VPRecipeBase &Phi : HeaderVPBB->phis()) { auto *PhiR = dyn_cast(&Phi); if (!PhiR) @@ -745,7 +745,7 @@ void VPlanTransforms::optimizeInductionExitUsers( "predecessor must be the middle block"); VPTypeAnalysis TypeInfo(Plan.getCanonicalIV()->getScalarType()); - VPBuilder B(Plan.getMiddleBlock()->getTerminator()); + VPBuilder B(Plan, Plan.getMiddleBlock()->getTerminator()); for (VPRecipeBase &R : *ExitVPBB) { auto *ExitIRI = cast(&R); if (!isa(ExitIRI->getInstruction())) @@ -1503,7 +1503,7 @@ static VPActiveLaneMaskPHIRecipe *addVPLaneMaskPhiAndUpdateExitBranch( // we have to take unrolling into account. Each part needs to start at // Part * VF auto *VecPreheader = Plan.getVectorPreheader(); - VPBuilder Builder(VecPreheader); + VPBuilder Builder(Plan, VecPreheader); // Create the ActiveLaneMask instruction using the correct start values. VPValue *TC = Plan.getTripCount(); @@ -1622,7 +1622,8 @@ void VPlanTransforms::addActiveLaneMask( LaneMask = addVPLaneMaskPhiAndUpdateExitBranch( Plan, DataAndControlFlowWithoutRuntimeCheck); } else { - VPBuilder B = VPBuilder::getToInsertAfter(WideCanonicalIV); + VPBuilder B(Plan, WideCanonicalIV->getParent(), + std::next(WideCanonicalIV->getIterator())); LaneMask = B.createNaryOp(VPInstruction::ActiveLaneMask, {WideCanonicalIV, Plan.getTripCount()}, nullptr, "active.lane.mask"); @@ -1826,7 +1827,7 @@ bool VPlanTransforms::tryAddExplicitVectorLength( // Create the ExplicitVectorLengthPhi recipe in the main loop. auto *EVLPhi = new VPEVLBasedIVPHIRecipe(StartV, DebugLoc()); EVLPhi->insertAfter(CanonicalIVPHI); - VPBuilder Builder(Header, Header->getFirstNonPhi()); + VPBuilder Builder(Plan, Header, Header->getFirstNonPhi()); // Compute original TC - IV as the AVL (application vector length). VPValue *AVL = Builder.createNaryOp( Instruction::Sub, {Plan.getTripCount(), EVLPhi}, DebugLoc(), "avl"); @@ -1907,7 +1908,7 @@ void VPlanTransforms::dropPoisonGeneratingRecipes( // where the operands are disjoint or poison otherwise. if (match(RecWithFlags, m_BinaryOr(m_VPValue(A), m_VPValue(B))) && RecWithFlags->isDisjoint()) { - VPBuilder Builder(RecWithFlags); + VPBuilder Builder(Plan, RecWithFlags); VPInstruction *New = Builder.createOverflowingOp( Instruction::Add, {A, B}, {false, false}, RecWithFlags->getDebugLoc()); @@ -2021,7 +2022,7 @@ void VPlanTransforms::createInterleaveGroups( /*IsSigned=*/true); VPValue *OffsetVPV = Plan.getOrAddLiveIn( ConstantInt::get(IRInsertPos->getParent()->getContext(), -Offset)); - VPBuilder B(InsertPos); + VPBuilder B(Plan, InsertPos); Addr = InBounds ? B.createInBoundsPtrAdd(InsertPos->getAddr(), OffsetVPV) : B.createPtrAdd(InsertPos->getAddr(), OffsetVPV); } @@ -2067,7 +2068,7 @@ void VPlanTransforms::handleUncountableEarlyExit( BasicBlock *UncountableExitingBlock, VPRecipeBuilder &RecipeBuilder) { VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion(); auto *LatchVPBB = cast(LoopRegion->getExiting()); - VPBuilder Builder(LatchVPBB->getTerminator()); + VPBuilder Builder(Plan, LatchVPBB->getTerminator()); auto *MiddleVPBB = Plan.getMiddleBlock(); VPValue *IsEarlyExitTaken = nullptr; @@ -2108,8 +2109,8 @@ void VPlanTransforms::handleUncountableEarlyExit( VPBlockUtils::connectBlocks(VectorEarlyExitVPBB, VPEarlyExitBlock); // Update the exit phis in the early exit block. - VPBuilder MiddleBuilder(NewMiddle); - VPBuilder EarlyExitB(VectorEarlyExitVPBB); + VPBuilder MiddleBuilder(Plan, NewMiddle); + VPBuilder EarlyExitB(Plan, VectorEarlyExitVPBB); for (VPRecipeBase &R : *VPEarlyExitBlock) { auto *ExitIRI = cast(&R); auto *ExitPhi = dyn_cast(&ExitIRI->getInstruction()); diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp index 89e372d6b46cf..53e086dfe1cbd 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp @@ -156,7 +156,7 @@ void UnrollState::unrollWidenInductionByUF( FMFs = ID.getInductionBinOp()->getFastMathFlags(); VPValue *VectorStep = &Plan.getVF(); - VPBuilder Builder(PH); + VPBuilder Builder(Plan, PH); if (TypeInfo.inferScalarType(VectorStep) != IVTy) { Instruction::CastOps CastOp = IVTy->isFloatingPointTy() ? Instruction::UIToFP : Instruction::Trunc;