diff --git a/llvm/include/llvm/Analysis/IVDescriptors.h b/llvm/include/llvm/Analysis/IVDescriptors.h index 3b627a5140854..85a4f19430873 100644 --- a/llvm/include/llvm/Analysis/IVDescriptors.h +++ b/llvm/include/llvm/Analysis/IVDescriptors.h @@ -57,6 +57,8 @@ enum class RecurKind { FindLastIV, ///< FindLast reduction with select(cmp(),x,y) where one of ///< (x,y) is increasing loop induction, and both x and y are ///< integer type. + MinMaxFirstIdx, ///< Integer Min/Max with first index + MinMaxLastIdx, ///< Integer Min/Max with last index // clang-format on // TODO: Any_of and FindLast reduction need not be restricted to integer type // only. @@ -209,6 +211,26 @@ class RecurrenceDescriptor { LLVM_ABI static bool isFixedOrderRecurrence(PHINode *Phi, Loop *TheLoop, DominatorTree *DT); + /// Returns the recurrence chain if \p Phi is an integer min/max recurrence in + /// \p TheLoop. The RecurrenceDescriptor is returned in \p RecurDes. + static SmallVector + tryToGetMinMaxRecurrenceChain(PHINode *Phi, Loop *TheLoop, + RecurrenceDescriptor &RecurDes); + + /// Returns true if the recurrence is a min/max with index pattern, and + /// updates the recurrence kind to RecurKind::MinMaxFirstIdx or + /// RecurKind::MinMaxLastIdx. + /// + /// \param IdxPhi The phi representing the index recurrence. + /// \param MinMaxPhi The phi representing the min/max recurrence involved + /// in the min/max with index pattern. + /// \param MinMaxDesc The descriptor of the min/max recurrence. + /// \param MinMaxChain The chain of instructions involved in the min/max + /// recurrence. + bool isMinMaxIdxReduction(PHINode *IdxPhi, PHINode *MinMaxPhi, + const RecurrenceDescriptor &MinMaxDesc, + ArrayRef MinMaxChain); + RecurKind getRecurrenceKind() const { return Kind; } unsigned getOpcode() const { return getOpcode(getRecurrenceKind()); } @@ -262,6 +284,20 @@ class RecurrenceDescriptor { return Kind == RecurKind::FindLastIV; } + /// Returns true if the recurrence kind is of the form: + /// select(icmp(a,b),x,y) + /// where one of (x,y) is an increasing loop induction variable, and icmp(a,b) + /// depends on a min/max recurrence. + static bool isMinMaxIdxRecurrenceKind(RecurKind Kind) { + return Kind == RecurKind::MinMaxFirstIdx || + Kind == RecurKind::MinMaxLastIdx; + } + + /// Returns true if the recurrence kind is an integer max kind. + static bool isIntMaxRecurrenceKind(RecurKind Kind) { + return Kind == RecurKind::UMax || Kind == RecurKind::SMax; + } + /// Returns the type of the recurrence. This type can be narrower than the /// actual type of the Phi if the recurrence has been type-promoted. Type *getRecurrenceType() const { return RecurrenceType; } diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h index d654ac3ec9273..3f82a81dd99c0 100644 --- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h +++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h @@ -345,6 +345,9 @@ class LoopVectorizationLegality { /// Returns True if Phi is a fixed-order recurrence in this loop. bool isFixedOrderRecurrence(const PHINode *Phi) const; + /// Returns True if \p Phi is a min/max recurrence in this loop. + bool isMinMaxRecurrence(const PHINode *Phi) const; + /// Return true if the block BB needs to be predicated in order for the loop /// to be vectorized. bool blockNeedsPredication(BasicBlock *BB) const; @@ -519,6 +522,14 @@ class LoopVectorizationLegality { /// specific checks for outer loop vectorization. bool canVectorizeOuterLoop(); + // Min/max recurrences can only be vectorized when involved in a min/max with + // index reduction pattern. This function checks whether the \p Phi, which + // represents the min/max recurrence, can be vectorized based on the given \p + // Chain, which is the recurrence chain for the min/max recurrence. Returns + // true if the min/max recurrence can be vectorized. + bool canVectorizeMinMaxRecurrence(PHINode *Phi, + ArrayRef Chain); + /// Returns true if this is an early exit loop that can be vectorized. /// Currently, a loop with an uncountable early exit is considered /// vectorizable if: @@ -606,6 +617,9 @@ class LoopVectorizationLegality { /// Holds the phi nodes that are fixed-order recurrences. RecurrenceSet FixedOrderRecurrences; + /// Holds the min/max recurrences variables. + RecurrenceSet MinMaxRecurrences; + /// Holds the widest induction type encountered. IntegerType *WidestIndTy = nullptr; diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp index 7232283b9101b..1c195404cc6a3 100644 --- a/llvm/lib/Analysis/IVDescriptors.cpp +++ b/llvm/lib/Analysis/IVDescriptors.cpp @@ -51,6 +51,8 @@ bool RecurrenceDescriptor::isIntegerRecurrenceKind(RecurKind Kind) { case RecurKind::UMin: case RecurKind::AnyOf: case RecurKind::FindLastIV: + case RecurKind::MinMaxFirstIdx: + case RecurKind::MinMaxLastIdx: return true; } return false; @@ -1130,6 +1132,226 @@ bool RecurrenceDescriptor::isFixedOrderRecurrence(PHINode *Phi, Loop *TheLoop, return true; } +/// Return the recurrence kind if \p I is matched by the min/max operation +/// pattern. Otherwise, return RecurKind::None. +static RecurKind isMinMaxRecurOp(const Instruction *I) { + if (match(I, m_UMin(m_Value(), m_Value()))) + return RecurKind::UMin; + if (match(I, m_UMax(m_Value(), m_Value()))) + return RecurKind::UMax; + if (match(I, m_SMax(m_Value(), m_Value()))) + return RecurKind::SMax; + if (match(I, m_SMin(m_Value(), m_Value()))) + return RecurKind::SMin; + // TODO: support fp-min/max + return RecurKind::None; +} + +SmallVector +RecurrenceDescriptor::tryToGetMinMaxRecurrenceChain( + PHINode *Phi, Loop *TheLoop, RecurrenceDescriptor &RecurDes) { + SmallVector Chain; + // Check the phi is in the loop header and has two incoming values. + if (Phi->getParent() != TheLoop->getHeader() || + Phi->getNumIncomingValues() != 2) + return {}; + + // Ensure the loop has a preheader and a latch block. + auto *Preheader = TheLoop->getLoopPreheader(); + auto *Latch = TheLoop->getLoopLatch(); + if (!Preheader || !Latch) + return {}; + + // Ensure that one of the incoming values of the PHI node is from the + // preheader, and the other one is from the loop latch. + if (Phi->getBasicBlockIndex(Preheader) < 0 || + Phi->getBasicBlockIndex(Latch) < 0) + return {}; + + Value *StartValue = Phi->getIncomingValueForBlock(Preheader); + auto *BEValue = dyn_cast(Phi->getIncomingValueForBlock(Latch)); + if (!BEValue || BEValue == Phi) + return {}; + + auto HasLoopExternalUse = [TheLoop](const Instruction *I) { + return any_of(I->users(), [TheLoop](auto *U) { + return !TheLoop->contains(cast(U)); + }); + }; + + // Ensure the recurrence phi has no users outside the loop, as such cases + // cannot be vectorized. + if (HasLoopExternalUse(Phi)) + return {}; + + // Ensure the backedge value of the phi is only used internally by the phi; + // all other users must be outside the loop. + // TODO: support intermediate store. + if (any_of(BEValue->users(), [&](auto *U) { + auto *UI = cast(U); + return TheLoop->contains(UI) && UI != Phi; + })) + return {}; + + // Ensure the backedge value of the phi matches the min/max operation pattern. + RecurKind TargetKind = isMinMaxRecurOp(BEValue); + if (TargetKind == RecurKind::None) + return {}; + + // TODO: type-promoted recurrence + SmallPtrSet CastInsts; + + // Trace the use-def chain from the backedge value to the phi, ensuring a + // unique in-loop path where all operations match the expected recurrence + // kind. + bool FoundRecurPhi = false; + SmallVector Worklist(1, BEValue); + SmallDenseMap VisitedFrom; + + VisitedFrom.try_emplace(BEValue); + + while (!Worklist.empty()) { + Instruction *Cur = Worklist.pop_back_val(); + if (Cur == Phi) { + if (FoundRecurPhi) + return {}; + FoundRecurPhi = true; + continue; + } + + if (!TheLoop->contains(Cur)) + continue; + + // TODO: support the min/max recurrence in cmp-select pattern. + if (!isa(Cur) || isMinMaxRecurOp(Cur) != TargetKind) + continue; + + for (Use &Op : Cur->operands()) { + if (auto *OpInst = dyn_cast(Op)) { + if (!VisitedFrom.try_emplace(OpInst, Cur).second) + return {}; + Worklist.push_back(OpInst); + } + } + } + + if (!FoundRecurPhi) + return {}; + + Instruction *ExitInstruction = nullptr; + // Get the recurrence chain by visited trace. + Instruction *VisitedInst = VisitedFrom.at(Phi); + while (VisitedInst) { + // Ensure that no instruction in the recurrence chain is used outside the + // loop, except for the backedge value, which is permitted. + if (HasLoopExternalUse(VisitedInst)) { + if (VisitedInst != BEValue) + return {}; + ExitInstruction = BEValue; + } + Chain.push_back(VisitedInst); + VisitedInst = VisitedFrom.at(VisitedInst); + } + + RecurDes = RecurrenceDescriptor( + StartValue, ExitInstruction, /*IntermediateStore=*/nullptr, TargetKind, + FastMathFlags(), /*ExactFPMathInst=*/nullptr, Phi->getType(), + /*IsSigned=*/false, /*IsOrdered=*/false, CastInsts, + /*MinWidthCastToRecurTy=*/-1U); + + LLVM_DEBUG(dbgs() << "Found a min/max recurrence PHI: " << *Phi << "\n"); + + return Chain; +} + +bool RecurrenceDescriptor::isMinMaxIdxReduction( + PHINode *IdxPhi, PHINode *MinMaxPhi, const RecurrenceDescriptor &MinMaxDesc, + ArrayRef MinMaxChain) { + // Return early if the recurrence kind is already known to be min/max with + // index. + if (isMinMaxIdxRecurrenceKind(Kind)) + return true; + + if (!isFindLastIVRecurrenceKind(Kind)) + return false; + + // Ensure index reduction phi and min/max recurrence phi are in the same basic + // block. + if (IdxPhi->getParent() != MinMaxPhi->getParent()) + return false; + + RecurKind MinMaxRK = MinMaxDesc.getRecurrenceKind(); + // TODO: support floating-point min/max with index. + if (!isIntMinMaxRecurrenceKind(MinMaxRK)) + return false; + + // FindLastIV only supports a single select operation in the recurrence chain + // so far. Therefore, do not consider min/max recurrences with more than one + // operation in the recurrence chain. + // TODO: support FindLastIV with multiple operations in the recurrence chain. + if (MinMaxChain.size() != 1) + return false; + + Instruction *MinMaxChainCur = MinMaxPhi; + Instruction *MinMaxChainNext = MinMaxChain.front(); + Value *OutOfChain; + bool IsMinMaxOperation = match( + MinMaxChainNext, + m_CombineOr(m_MaxOrMin(m_Specific(MinMaxChainCur), m_Value(OutOfChain)), + m_MaxOrMin(m_Value(OutOfChain), m_Specific(MinMaxChainCur)))); + assert(IsMinMaxOperation && "Unexpected operation in the recurrence chain"); + + auto *IdxExit = cast(LoopExitInstr); + Value *IdxCond = IdxExit->getCondition(); + // Check if the operands used by cmp instruction of index select is the same + // as the operands used by min/max recurrence. + bool IsMatchLHSInMinMaxChain = + match(IdxCond, m_Cmp(m_Specific(MinMaxChainCur), m_Specific(OutOfChain))); + bool IsMatchRHSInMinMaxChain = + match(IdxCond, m_Cmp(m_Specific(OutOfChain), m_Specific(MinMaxChainCur))); + if (!IsMatchLHSInMinMaxChain && !IsMatchRHSInMinMaxChain) + return false; + + CmpInst::Predicate IdxPred = cast(IdxCond)->getPredicate(); + // The predicate of cmp instruction must be relational in min/max with index. + if (CmpInst::isEquality(IdxPred)) + return false; + + // Normalize predicate from + // m_Cmp(pred, out_of_chain, in_chain) + // to + // m_Cmp(swapped_pred, in_chain, out_of_chain). + if (IsMatchRHSInMinMaxChain) + IdxPred = CmpInst::getSwappedPredicate(IdxPred); + + // Verify that the select operation is updated on the correct side based on + // the min/max kind. + bool IsTrueUpdateIdx = IdxExit->getFalseValue() == IdxPhi; + bool IsMaxRK = isIntMaxRecurrenceKind(MinMaxRK); + bool IsLess = ICmpInst::isLT(IdxPred) || ICmpInst::isLE(IdxPred); + bool IsExpectedTrueUpdateIdx = IsMaxRK == IsLess; + if (IsTrueUpdateIdx != IsExpectedTrueUpdateIdx) + return false; + + RecurKind NewIdxRK; + // The index recurrence kind is the same for both the predicate and its + // inverse. + if (!IsLess) + IdxPred = CmpInst::getInversePredicate(IdxPred); + // For max recurrence, a strict less-than predicate indicates that the first + // matching index will be selected. For min recurrence, the opposite holds. + NewIdxRK = IsMaxRK != ICmpInst::isLE(IdxPred) ? RecurKind::MinMaxFirstIdx + : RecurKind::MinMaxLastIdx; + + // Update the kind of index recurrence. + Kind = NewIdxRK; + LLVM_DEBUG( + dbgs() << "Found a min/max with " + << (NewIdxRK == RecurKind::MinMaxFirstIdx ? "first" : "last") + << " index reduction PHI." << *IdxPhi << "\n"); + return true; +} + unsigned RecurrenceDescriptor::getOpcode(RecurKind Kind) { switch (Kind) { case RecurKind::Add: diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp index 8e09e6f8d4935..25894ac14df33 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp @@ -795,6 +795,10 @@ static bool canWidenCallReturnType(Type *Ty) { bool LoopVectorizationLegality::canVectorizeInstrs() { BasicBlock *Header = TheLoop->getHeader(); + // Tracks the operation chain for each min/max recurrence phi that is + // considered vectorizable. + SmallDenseMap> MinMaxRecurrenceChains; + // For each block in the loop. for (BasicBlock *BB : TheLoop->blocks()) { // Scan the instructions in the block and look for hazards. @@ -840,6 +844,18 @@ bool LoopVectorizationLegality::canVectorizeInstrs() { continue; } + RecurrenceDescriptor MinMaxRecurDes; + if (auto Chain = RecurrenceDescriptor::tryToGetMinMaxRecurrenceChain( + Phi, TheLoop, MinMaxRecurDes); + !Chain.empty()) { + if (MinMaxRecurDes.getLoopExitInstr()) + AllowedExit.insert(MinMaxRecurDes.getLoopExitInstr()); + Reductions[Phi] = MinMaxRecurDes; + MinMaxRecurrences.insert(Phi); + MinMaxRecurrenceChains[Phi] = std::move(Chain); + continue; + } + // We prevent matching non-constant strided pointer IVS to preserve // historical vectorizer behavior after a generalization of the // IVDescriptor code. The intent is to remove this check, but we @@ -1069,9 +1085,74 @@ bool LoopVectorizationLegality::canVectorizeInstrs() { if (PrimaryInduction && WidestIndTy != PrimaryInduction->getType()) PrimaryInduction = nullptr; + // The second stage check for reduction. Confirm if the min/max with index + // reduction, involving two PHIs, is legal to vectorize. + for (auto &Entry : MinMaxRecurrenceChains) { + PHINode *Phi = Entry.first; + ArrayRef Chain = Entry.second; + if (!canVectorizeMinMaxRecurrence(Phi, Chain)) + return false; + } + // FIXME: Remove this after the IR generation of min/max with index is + // supported. + if (!MinMaxRecurrences.empty()) + return false; + return true; } +bool LoopVectorizationLegality::canVectorizeMinMaxRecurrence( + PHINode *Phi, ArrayRef Chain) { + assert(!Chain.empty() && "Unexpected empty recurrence chain"); + assert(isMinMaxRecurrence(Phi) && "The PHI is not a min/max recurrence phi"); + + auto IsMinMaxIdxReductionPhi = [this, Phi, &Chain](Value *Candidate) -> bool { + auto *IdxPhi = dyn_cast(Candidate); + if (!IdxPhi || !isReductionVariable(IdxPhi)) + return false; + + RecurrenceDescriptor &IdxRdxDesc = Reductions.find(IdxPhi)->second; + const RecurrenceDescriptor &MinMaxDesc = Reductions.find(Phi)->second; + return IdxRdxDesc.isMinMaxIdxReduction(IdxPhi, Phi, MinMaxDesc, Chain); + }; + + // Find the potential index recurrence chain head. + // Note: Only one chain head can be found since 2-D indexes are not yet + // supported. + SelectInst *IdxChainHead = nullptr; + // TODO: support min/max with 2-D indexes. + if (!Phi->hasNUses(2)) + return false; + + for (User *U : Phi->users()) { + if (auto *Cmp = dyn_cast(U)) { + if (!Cmp->hasOneUse()) + return false; + if (!match(Cmp->user_back(), + m_Select(m_Specific(Cmp), m_Value(), m_Value()))) + return false; + assert(!IdxChainHead && + "Unexpected multiple index recurrence chain head"); + IdxChainHead = cast(Cmp->user_back()); + continue; + } + + // Skip the user in the min/max recurrence chain + if (llvm::is_contained(Chain, cast(U))) + continue; + + // Unexpected user + return false; + } + + if (!IdxChainHead) + return false; + + auto *TrueVal = IdxChainHead->getTrueValue(); + auto *FalseVal = IdxChainHead->getFalseValue(); + return IsMinMaxIdxReductionPhi(TrueVal) || IsMinMaxIdxReductionPhi(FalseVal); +} + /// Find histogram operations that match high-level code in loops: /// \code /// buckets[indices[i]]+=step; @@ -1394,6 +1475,10 @@ bool LoopVectorizationLegality::isFixedOrderRecurrence( return FixedOrderRecurrences.count(Phi); } +bool LoopVectorizationLegality::isMinMaxRecurrence(const PHINode *Phi) const { + return MinMaxRecurrences.contains(Phi); +} + bool LoopVectorizationLegality::blockNeedsPredication(BasicBlock *BB) const { // When vectorizing early exits, create predicates for the latch block only. // The early exiting block must be a direct predecessor of the latch at the diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index ec40124c57a6a..044658f0838d6 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -23103,6 +23103,8 @@ class HorizontalReduction { case RecurKind::FMulAdd: case RecurKind::AnyOf: case RecurKind::FindLastIV: + case RecurKind::MinMaxFirstIdx: + case RecurKind::MinMaxLastIdx: case RecurKind::FMaximumNum: case RecurKind::FMinimumNum: case RecurKind::None: @@ -23237,6 +23239,8 @@ class HorizontalReduction { case RecurKind::FMulAdd: case RecurKind::AnyOf: case RecurKind::FindLastIV: + case RecurKind::MinMaxFirstIdx: + case RecurKind::MinMaxLastIdx: case RecurKind::FMaximumNum: case RecurKind::FMinimumNum: case RecurKind::None: @@ -23336,6 +23340,8 @@ class HorizontalReduction { case RecurKind::FMulAdd: case RecurKind::AnyOf: case RecurKind::FindLastIV: + case RecurKind::MinMaxFirstIdx: + case RecurKind::MinMaxLastIdx: case RecurKind::FMaximumNum: case RecurKind::FMinimumNum: case RecurKind::None: diff --git a/llvm/test/Transforms/LoopVectorize/smax-idx.ll b/llvm/test/Transforms/LoopVectorize/smax-idx.ll index 37dcd7fc7e39f..ce29818d05913 100644 --- a/llvm/test/Transforms/LoopVectorize/smax-idx.ll +++ b/llvm/test/Transforms/LoopVectorize/smax-idx.ll @@ -1,6 +1,38 @@ -; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S < %s | FileCheck %s --check-prefix=CHECK -; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=4 -S < %s | FileCheck %s --check-prefix=CHECK -; RUN: opt -passes=loop-vectorize -force-vector-width=1 -force-vector-interleave=4 -S < %s | FileCheck %s --check-prefix=CHECK +; REQUIRES: asserts +; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S < %s -debug-only=loop-vectorize,iv-descriptors 2>&1 | FileCheck %s --check-prefix=CHECK +; RUN: opt -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=4 -S < %s -debug-only=loop-vectorize,iv-descriptors 2>&1 | FileCheck %s --check-prefix=CHECK +; RUN: opt -passes=loop-vectorize -force-vector-width=1 -force-vector-interleave=4 -S < %s -debug-only=loop-vectorize,iv-descriptors 2>&1 | FileCheck %s --check-prefix=CHECK + +; CHECK-LABEL: LV: Checking a loop in 'smax_idx' +; CHECK: Found a min/max recurrence PHI: %max.09 = phi i64 [ %mm, %entry ], [ %1, %for.body ] +; CHECK: Found a min/max with first index reduction PHI. %idx.011 = phi i64 [ %ii, %entry ], [ %spec.select7, %for.body ] + +; CHECK-LABEL: LV: Checking a loop in 'smax_idx_inverted_phi' +; CHECK: Found a min/max recurrence PHI: %max.09 = phi i64 [ %mm, %entry ], [ %1, %for.body ] +; CHECK: Found a min/max with first index reduction PHI. %idx.011 = phi i64 [ %ii, %entry ], [ %spec.select7, %for.body ] + +; CHECK-LABEL: LV: Checking a loop in 'smax_idx_max_no_exit_user' +; CHECK: Found a min/max recurrence PHI: %max.09 = phi i64 [ %mm, %entry ], [ %1, %for.body ] +; CHECK: Found a min/max with first index reduction PHI. %idx.011 = phi i64 [ %ii, %entry ], [ %spec.select7, %for.body ] + +; CHECK-LABEL: LV: Checking a loop in 'smax_idx_select_cmp' +; CHECK: LV: Not vectorizing: Found an unidentified PHI %max.09 = phi i64 [ %mm, %entry ], [ %spec.select, %for.body ] + +; CHECK-LABEL: LV: Checking a loop in 'smax_idx_inverted_pred' +; CHECK: Found a min/max recurrence PHI: %max.09 = phi i64 [ %mm, %entry ], [ %1, %for.body ] +; CHECK: Found a min/max with last index reduction PHI. %idx.011 = phi i64 [ %ii, %entry ], [ %spec.select7, %for.body ] + +; CHECK-LABEL: LV: Checking a loop in 'smax_idx_extract_last' +; CHECK: Found a min/max recurrence PHI: %max.09 = phi i64 [ %mm, %entry ], [ %1, %for.body ] +; CHECK: Found a min/max with last index reduction PHI. %idx.011 = phi i64 [ %ii, %entry ], [ %spec.select7, %for.body ] + +; CHECK-LABEL: LV: Checking a loop in 'smax_idx_not_vec_1' +; CHECK: Found a min/max recurrence PHI: %max.09 = phi i64 [ %mm, %entry ], [ %2, %for.body ] +; CHECK-NOT: Found a min/max with last index reduction PHI. + +; CHECK-LABEL: LV: Checking a loop in 'smax_idx_not_vec_2' +; CHECK: Found a min/max recurrence PHI: %max.09 = phi i64 [ %mm, %entry ], [ %1, %for.body ] +; CHECK-NOT: Found a min/max with last index reduction PHI. define i64 @smax_idx(ptr nocapture readonly %a, i64 %mm, i64 %ii, ptr nocapture writeonly %res_max, i64 %n) { ; CHECK-LABEL: @smax_idx( @@ -58,11 +90,6 @@ exit: ; Check if it is a min/max with index (MMI) pattern when the ; min/max value is not used outside the loop. ; -; Currently, the vectorizer checks if smax value is used outside -; the loop. However, even if only the index part has external users, -; and smax itself does not have external users, it can still form a -; MMI pattern. -; define i64 @smax_idx_max_no_exit_user(ptr nocapture readonly %a, i64 %mm, i64 %ii, i64 %n) { ; CHECK-LABEL: @smax_idx_max_no_exit_user( ; CHECK-NOT: vector.body: