From ab1faf58e0aeb26f8f3ed5542ad6b0168bf1fa0d Mon Sep 17 00:00:00 2001 From: Han-Kuan Chen Date: Fri, 27 Sep 2024 09:28:51 -0700 Subject: [PATCH 1/4] [RISCV][TTI] Pre-commit test. --- .../RISCV/fixed-vector-insert-subvector.ll | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 llvm/test/Analysis/CostModel/RISCV/fixed-vector-insert-subvector.ll diff --git a/llvm/test/Analysis/CostModel/RISCV/fixed-vector-insert-subvector.ll b/llvm/test/Analysis/CostModel/RISCV/fixed-vector-insert-subvector.ll new file mode 100644 index 0000000000000..e9ba3517456ea --- /dev/null +++ b/llvm/test/Analysis/CostModel/RISCV/fixed-vector-insert-subvector.ll @@ -0,0 +1,18 @@ +; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py +; RUN: opt < %s -passes="print" 2>&1 -disable-output -S -mtriple=riscv64 -mattr=+v | FileCheck %s + +define void @test() { +; CHECK-LABEL: 'test' +; CHECK-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %0 = shufflevector <8 x float> poison, <8 x float> poison, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %1 = shufflevector <4 x i16> poison, <4 x i16> poison, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %2 = shufflevector <4 x float> poison, <4 x float> poison, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %3 = shufflevector <2 x i1> poison, <2 x i1> poison, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +entry: + %0 = shufflevector <8 x float> poison, <8 x float> poison, <16 x i32> + %1 = shufflevector <4 x i16> poison, <4 x i16> poison, <16 x i32> + %2 = shufflevector <4 x float> poison, <4 x float> poison, <8 x i32> + %3 = shufflevector <2 x i1> poison, <2 x i1> poison, <4 x i32> + ret void +} From 22cb3db61011c3f389a2260795e6954336869ae5 Mon Sep 17 00:00:00 2001 From: Han-Kuan Chen Date: Fri, 27 Sep 2024 10:24:54 -0700 Subject: [PATCH 2/4] [RISCV][TTI] Recognize CONCAT_VECTORS if a shufflevector mask is a multiple insert subvector. --- .../Target/RISCV/RISCVTargetTransformInfo.cpp | 38 +++++++++++++++++++ .../Target/RISCV/RISCVTargetTransformInfo.h | 6 +++ .../RISCV/fixed-vector-insert-subvector.ll | 4 +- .../RISCV/remarks-insert-into-small-vector.ll | 2 +- .../RISCV/revec-getGatherCost.ll | 2 +- 5 files changed, 48 insertions(+), 4 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp index c042782389f18..7c990ecb7495b 100644 --- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp @@ -343,6 +343,40 @@ RISCVTTIImpl::getConstantPoolLoadCost(Type *Ty, TTI::TargetCostKind CostKind) { /*AddressSpace=*/0, CostKind); } +InstructionCost +RISCVTTIImpl::isMultipleInsertSubvector(VectorType *Tp, ArrayRef Mask, + TTI::TargetCostKind CostKind) { + if (!isa(Tp)) + return InstructionCost::getInvalid(); + std::pair LT = getTypeLegalizationCost(Tp); + if (LT.second.getScalarSizeInBits() == 1) + return InstructionCost::getInvalid(); + // Try to guess SubTp. + for (unsigned SubVecSize = 1, E = Mask.size(); SubVecSize < E; + SubVecSize <<= 1) { + if (E % SubVecSize != 0) + continue; + SmallVector RepeatedPattern(createSequentialMask(0, SubVecSize, 0)); + bool Skip = false; + for (unsigned I = 0; I != E; I += SubVecSize) + if (!Mask.slice(I, SubVecSize).equals(RepeatedPattern)) { + Skip = true; + break; + } + if (Skip) + continue; + InstructionCost Cost = 0; + FixedVectorType *SubTp = FixedVectorType::get( + cast(Tp)->getElementType(), SubVecSize); + // The cost of extraction from a subvector is 0 if the index is 0. + for (unsigned I = 0; I != E; I += SubVecSize) + Cost += + getShuffleCost(TTI::SK_InsertSubvector, Tp, {}, CostKind, I, SubTp); + return Cost; + } + return InstructionCost::getInvalid(); +} + static VectorType *getVRGatherIndexType(MVT DataVT, const RISCVSubtarget &ST, LLVMContext &C) { assert((DataVT.getScalarSizeInBits() != 8 || @@ -394,6 +428,10 @@ InstructionCost RISCVTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, LT.second, CostKind); } } + if (InstructionCost Cost = + isMultipleInsertSubvector(Tp, Mask, CostKind); + Cost.isValid()) + return Cost; } // vrgather + cost of generating the mask constant. // We model this for an unknown mask with a single vrgather. diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h index 65bbd90550855..9d6317baabc65 100644 --- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h +++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h @@ -55,6 +55,12 @@ class RISCVTTIImpl : public BasicTTIImplBase { /// type. InstructionCost getConstantPoolLoadCost(Type *Ty, TTI::TargetCostKind CostKind); + + /// Return the cost if a shufflevector can be consist of multiple vslideup. + /// Otherwise, return InstructionCost::getInvalid(). + InstructionCost isMultipleInsertSubvector(VectorType *Tp, ArrayRef Mask, + TTI::TargetCostKind CostKind); + public: explicit RISCVTTIImpl(const RISCVTargetMachine *TM, const Function &F) : BaseT(TM, F.getDataLayout()), ST(TM->getSubtargetImpl(F)), diff --git a/llvm/test/Analysis/CostModel/RISCV/fixed-vector-insert-subvector.ll b/llvm/test/Analysis/CostModel/RISCV/fixed-vector-insert-subvector.ll index e9ba3517456ea..a0c2413c8655a 100644 --- a/llvm/test/Analysis/CostModel/RISCV/fixed-vector-insert-subvector.ll +++ b/llvm/test/Analysis/CostModel/RISCV/fixed-vector-insert-subvector.ll @@ -3,9 +3,9 @@ define void @test() { ; CHECK-LABEL: 'test' -; CHECK-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %0 = shufflevector <8 x float> poison, <8 x float> poison, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %0 = shufflevector <8 x float> poison, <8 x float> poison, <16 x i32> ; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %1 = shufflevector <4 x i16> poison, <4 x i16> poison, <16 x i32> -; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %2 = shufflevector <4 x float> poison, <4 x float> poison, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %2 = shufflevector <4 x float> poison, <4 x float> poison, <8 x i32> ; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %3 = shufflevector <2 x i1> poison, <2 x i1> poison, <4 x i32> ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/remarks-insert-into-small-vector.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/remarks-insert-into-small-vector.ll index bb806be15c71c..23a9a654c96f9 100644 --- a/llvm/test/Transforms/SLPVectorizer/RISCV/remarks-insert-into-small-vector.ll +++ b/llvm/test/Transforms/SLPVectorizer/RISCV/remarks-insert-into-small-vector.ll @@ -8,7 +8,7 @@ ; YAML-NEXT: Function: test ; YAML-NEXT: Args: ; YAML-NEXT: - String: 'Stores SLP vectorized with cost ' -; YAML-NEXT: - Cost: '2' +; YAML-NEXT: - Cost: '0' ; YAML-NEXT: - String: ' and with tree size ' ; YAML-NEXT: - TreeSize: '7' diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/revec-getGatherCost.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/revec-getGatherCost.ll index 995cd7cfbc880..fd01494b7b343 100644 --- a/llvm/test/Transforms/SLPVectorizer/RISCV/revec-getGatherCost.ll +++ b/llvm/test/Transforms/SLPVectorizer/RISCV/revec-getGatherCost.ll @@ -8,7 +8,7 @@ ; YAML: Function: test1 ; YAML: Args: ; YAML: - String: 'Stores SLP vectorized with cost ' -; YAML: - Cost: '6' +; YAML: - Cost: '4' ; YAML: - String: ' and with tree size ' ; YAML: - TreeSize: '5' From 5c43d5281bcc47811f44cd97afffb0bc6dfe7f03 Mon Sep 17 00:00:00 2001 From: Han-Kuan Chen Date: Mon, 30 Sep 2024 09:14:04 -0700 Subject: [PATCH 3/4] [RISCV][TTI] Count whole vector register move and match codegen result. --- .../Target/RISCV/RISCVTargetTransformInfo.cpp | 19 ++++++++++++++----- .../RISCV/fixed-vector-insert-subvector.ll | 2 +- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp index 7c990ecb7495b..d72693d76a3b8 100644 --- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp @@ -366,12 +366,21 @@ RISCVTTIImpl::isMultipleInsertSubvector(VectorType *Tp, ArrayRef Mask, if (Skip) continue; InstructionCost Cost = 0; - FixedVectorType *SubTp = FixedVectorType::get( - cast(Tp)->getElementType(), SubVecSize); + unsigned NumSlides = Log2_32(E / SubVecSize); // The cost of extraction from a subvector is 0 if the index is 0. - for (unsigned I = 0; I != E; I += SubVecSize) - Cost += - getShuffleCost(TTI::SK_InsertSubvector, Tp, {}, CostKind, I, SubTp); + for (unsigned I = 0; I != NumSlides; ++I) { + unsigned InsertIndex = SubVecSize * (1 << I); + FixedVectorType *SubTp = FixedVectorType::get( + cast(Tp)->getElementType(), InsertIndex); + FixedVectorType *DesTp = + FixedVectorType::getDoubleElementsVectorType(SubTp); + std::pair DesLT = getTypeLegalizationCost(DesTp); + // Add the cost of whole vector register move because the destination + // vector register group for vslideup cannot overlap the source. + Cost += DesLT.first * TLI->getLMULCost(DesLT.second); + Cost += getShuffleCost(TTI::SK_InsertSubvector, DesTp, {}, CostKind, + InsertIndex, SubTp); + } return Cost; } return InstructionCost::getInvalid(); diff --git a/llvm/test/Analysis/CostModel/RISCV/fixed-vector-insert-subvector.ll b/llvm/test/Analysis/CostModel/RISCV/fixed-vector-insert-subvector.ll index a0c2413c8655a..47a2af92aee95 100644 --- a/llvm/test/Analysis/CostModel/RISCV/fixed-vector-insert-subvector.ll +++ b/llvm/test/Analysis/CostModel/RISCV/fixed-vector-insert-subvector.ll @@ -4,7 +4,7 @@ define void @test() { ; CHECK-LABEL: 'test' ; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %0 = shufflevector <8 x float> poison, <8 x float> poison, <16 x i32> -; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %1 = shufflevector <4 x i16> poison, <4 x i16> poison, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %1 = shufflevector <4 x i16> poison, <4 x i16> poison, <16 x i32> ; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %2 = shufflevector <4 x float> poison, <4 x float> poison, <8 x i32> ; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %3 = shufflevector <2 x i1> poison, <2 x i1> poison, <4 x i32> ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void From 66e4c34b81bcbfaa1ae549e81554c78cb8108f39 Mon Sep 17 00:00:00 2001 From: Han-Kuan Chen Date: Mon, 30 Sep 2024 17:05:52 -0700 Subject: [PATCH 4/4] [RISCV][TTI] Apply to a new test. --- llvm/test/Transforms/SLPVectorizer/RISCV/revec-getGatherCost.ll | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/revec-getGatherCost.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/revec-getGatherCost.ll index fd01494b7b343..a0cb52a853b7e 100644 --- a/llvm/test/Transforms/SLPVectorizer/RISCV/revec-getGatherCost.ll +++ b/llvm/test/Transforms/SLPVectorizer/RISCV/revec-getGatherCost.ll @@ -47,7 +47,7 @@ declare <4 x float> @llvm.fmuladd.v4f32(<4 x float>, <4 x float>, <4 x float>) ; YAML: Function: test2 ; YAML: Args: ; YAML: - String: 'Stores SLP vectorized with cost ' -; YAML: - Cost: '16' +; YAML: - Cost: '12' ; YAML: - String: ' and with tree size ' ; YAML: - TreeSize: '5'