LLVM 24.0.0git
Classes | Macros | Functions | Variables
AArch64TargetTransformInfo.cpp File Reference
#include "AArch64TargetTransformInfo.h"
#include "AArch64ExpandImm.h"
#include "AArch64PerfectShuffle.h"
#include "AArch64SMEAttributes.h"
#include "MCTargetDesc/AArch64AddressingModes.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/bit.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/CodeGen/BasicTTIImpl.h"
#include "llvm/CodeGen/CostTable.h"
#include "llvm/CodeGen/TargetLowering.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/IntrinsicsAArch64.h"
#include "llvm/IR/PatternMatch.h"
#include "llvm/Support/Debug.h"
#include "llvm/TargetParser/AArch64TargetParser.h"
#include "llvm/Transforms/InstCombine/InstCombiner.h"
#include "llvm/Transforms/Utils/UnrollLoop.h"
#include "llvm/Transforms/Vectorize/LoopVectorizationLegality.h"
#include <algorithm>
#include <optional>

Go to the source code of this file.

Classes

struct  SVEIntrinsicInfo

Macros

#define DEBUG_TYPE   "aarch64tti"

Functions

static void extractAttrFeatures (const Function &F, const AArch64TTIImpl *TTI, SmallVectorImpl< StringRef > &Features)
static bool isUnpackedVectorVT (EVT VecVT)
static InstructionCost getHistogramCost (const AArch64Subtarget *ST, const IntrinsicCostAttributes &ICA)
static std::optional< Instruction * > processPhiNode (InstCombiner &IC, IntrinsicInst &II)
 The function will remove redundant reinterprets casting in the presence of the control flow.
static SVEIntrinsicInfo constructSVEIntrinsicInfo (IntrinsicInst &II)
static bool isAllActivePredicate (Value *Pred)
static Value * stripInactiveLanes (Value *V, const Value *Pg)
static std::optional< Instruction * > simplifySVEIntrinsicBinOp (InstCombiner &IC, IntrinsicInst &II, const SVEIntrinsicInfo &IInfo)
static std::optional< Instruction * > simplifySVEIntrinsicCompare (InstCombiner &IC, IntrinsicInst &II, const SVEIntrinsicInfo &IInfo)
static std::optional< Instruction * > simplifySVEIntrinsic (InstCombiner &IC, IntrinsicInst &II, const SVEIntrinsicInfo &IInfo)
static std::optional< Instruction * > tryCombineFromSVBoolBinOp (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > instCombineConvertFromSVBool (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > instCombineSVESel (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > instCombineSVEDup (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > instCombineSVEDupX (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > instCombineXorSVECmpCC (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > instCombineZExtSVECmpNE (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > instCombineSVECmpNE (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > instCombineSVELast (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > instCombineSVECondLast (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > instCombineRDFFR (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > instCombineSVECntElts (InstCombiner &IC, IntrinsicInst &II, unsigned NumElts)
static std::optional< Instruction * > instCombineSMECntsd (InstCombiner &IC, IntrinsicInst &II, const AArch64Subtarget *ST)
static std::optional< Instruction * > instCombineSVEPTest (InstCombiner &IC, IntrinsicInst &II)
template<Intrinsic::ID MulOpc, Intrinsic::ID FuseOpc>
static std::optional< Instruction * > instCombineSVEVectorFuseMulAddSub (InstCombiner &IC, IntrinsicInst &II, bool MergeIntoAddendOp)
static std::optional< Instruction * > instCombineSVELD1 (InstCombiner &IC, IntrinsicInst &II, const DataLayout &DL)
static std::optional< Instruction * > instCombineSVEST1 (InstCombiner &IC, IntrinsicInst &II, const DataLayout &DL)
static Instruction::BinaryOps intrinsicIDToBinOpCode (unsigned Intrinsic)
static std::optional< Instruction * > instCombineSVEVectorBinOp (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > instCombineSVEVectorMlaU (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > instCombineSVEPairwiseAddLong (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > instCombineSVEVectorAdd (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > instCombineSVEVectorFAdd (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > instCombineSVEVectorFAddU (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > instCombineSVEVectorFSub (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > instCombineSVEVectorFSubU (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > instCombineSVEVectorSub (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > instCombineSVEUnpack (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > instCombineSVETBL (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > instCombineSVEUzp1 (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > instCombineSVEZip (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > instCombineLD1GatherIndex (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > instCombineST1ScatterIndex (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > instCombineSVESDIV (InstCombiner &IC, IntrinsicInst &II)
static bool matchBSL (const Instruction *I, Instruction *&MainAnd, Instruction *&OtherAnd, Value *&A, Value *&B)
 Match Or(And(MASK, A), And(Not(MASK), B)), which ISel selects as BSL/BIT/BIF.
bool SimplifyValuePattern (SmallVector< Value * > &Vec, bool AllowPoison)
static std::optional< Instruction * > instCombineSVEDupqLane (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > instCombineMaxMinNM (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > instCombineSVESrshl (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > instCombineSVEInsr (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > instCombineDMB (InstCombiner &IC, IntrinsicInst &II, unsigned LookaheadThreshold)
static std::optional< Instruction * > instCombineWhilelo (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > instCombinePTrue (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > instCombineSVEUxt (InstCombiner &IC, IntrinsicInst &II, unsigned NumBits)
static std::optional< Instruction * > instCombineInStreamingMode (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > instCombineSVEUMin (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > instCombineSVEOrr (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * > instCombineSVEAnd (InstCombiner &IC, IntrinsicInst &II)
static unsigned getSVEGatherScatterOverhead (unsigned Opcode, const AArch64Subtarget *ST)
static void getFalkorUnrollingPreferences (Loop *L, ScalarEvolution &SE, TargetTransformInfo::UnrollingPreferences &UP)
static bool isLoopSizeWithinBudget (Loop *L, const AArch64TTIImpl &TTI, InstructionCost Budget, unsigned *FinalSize)
static bool shouldUnrollMultiExitLoop (Loop *L, ScalarEvolution &SE, const AArch64TTIImpl &TTI)
static void getAppleRuntimeUnrollPreferences (Loop *L, ScalarEvolution &SE, TargetTransformInfo::UnrollingPreferences &UP, const AArch64TTIImpl &TTI)
 For Apple CPUs, we want to runtime-unroll loops to make better use if the OOO engine's wide instruction window and various predictors.
static bool containsDecreasingPointers (Loop *TheLoop, PredicatedScalarEvolution *PSE, const DominatorTree &DT)
static bool isSplatShuffle (Value *V)
static bool areExtractShuffleVectors (Value *Op1, Value *Op2, bool AllowSplat=false)
 Check if both Op1 and Op2 are shufflevector extracts of either the lower or upper half of the vector elements.
static bool areExtractExts (Value *Ext1, Value *Ext2)
 Check if Ext1 and Ext2 are extends of the same type, doubling the bitwidth of the vector elements.
static bool isOperandOfVmullHighP64 (Value *Op)
 Check if Op could be used with vmull_high_p64 intrinsic.
static bool areOperandsOfVmullHighP64 (Value *Op1, Value *Op2)
 Check if Op1 and Op2 could be used with vmull_high_p64 intrinsic.
static bool shouldSinkVectorOfPtrs (Value *Ptrs, SmallVectorImpl< Use * > &Ops)
static bool shouldSinkVScale (Value *Op, SmallVectorImpl< Use * > &Ops)
 We want to sink following cases: (add|sub|gep) A, ((mul|shl) vscale, imm); (add|sub|gep) A, vscale; (add|sub|gep) A, ((mul|shl) zext(vscale), imm);.
static bool isFNeg (Value *Op)

Variables

TailFoldingOption TailFoldingOptionLoc
static cl::opt< TailFoldingOption, true, cl::parser< std::string > > SVETailFolding ("sve-tail-folding", cl::desc("Control the use of vectorisation using tail-folding for SVE where the" " option is specified in the form (Initial)[+(Flag1|Flag2|...)]:" "\ndisabled (Initial) No loop types will vectorize using " "tail-folding" "\ndefault (Initial) Uses the default tail-folding settings for " "the target CPU" "\nall (Initial) All legal loop types will vectorize using " "tail-folding" "\nsimple (Initial) Use tail-folding for simple loops (not " "reductions or recurrences)" "\nreductions Use tail-folding for loops containing reductions" "\nnoreductions Inverse of above" "\nrecurrences Use tail-folding for loops containing fixed order " "recurrences" "\nnorecurrences Inverse of above" "\nreverse Use tail-folding for loops requiring reversed " "predicates" "\nnoreverse Inverse of above"), cl::location(TailFoldingOptionLoc))

Macro Definition Documentation

◆ DEBUG_TYPE

#define DEBUG_TYPE   "aarch64tti"

Definition at line 37 of file AArch64TargetTransformInfo.cpp.

Function Documentation

◆ areExtractExts()

bool areExtractExts ( Value * Ext1,
Value * Ext2 )
static

Check if Ext1 and Ext2 are extends of the same type, doubling the bitwidth of the vector elements.

Definition at line 7917 of file AArch64TargetTransformInfo.cpp.

References llvm::cast(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::m_ZExtOrSExt(), and llvm::PatternMatch::match().

Referenced by llvm::AArch64TTIImpl::isProfitableToSinkOperands(), and llvm::ARMTTIImpl::isProfitableToSinkOperands().

◆ areExtractShuffleVectors()

bool areExtractShuffleVectors ( Value * Op1,
Value * Op2,
bool AllowSplat = false )
static

Check if both Op1 and Op2 are shufflevector extracts of either the lower or upper half of the vector elements.

Definition at line 7857 of file AArch64TargetTransformInfo.cpp.

References llvm::cast(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getFixedValue(), llvm::Type::getPrimitiveSizeInBits(), llvm::Value::getType(), llvm::ShuffleVectorInst::isExtractSubvectorMask(), llvm::Type::isScalableTy(), isSplatShuffle(), llvm::M1(), llvm::PatternMatch::m_Shuffle(), llvm::PatternMatch::m_Undef(), llvm::PatternMatch::m_Value(), and llvm::PatternMatch::match().

Referenced by llvm::AArch64TTIImpl::isProfitableToSinkOperands().

◆ areOperandsOfVmullHighP64()

bool areOperandsOfVmullHighP64 ( Value * Op1,
Value * Op2 )
static

Check if Op1 and Op2 could be used with vmull_high_p64 intrinsic.

Definition at line 7944 of file AArch64TargetTransformInfo.cpp.

References isOperandOfVmullHighP64().

Referenced by llvm::AArch64TTIImpl::isProfitableToSinkOperands().

◆ constructSVEIntrinsicInfo()

SVEIntrinsicInfo constructSVEIntrinsicInfo ( IntrinsicInst & II)
static

Definition at line 1725 of file AArch64TargetTransformInfo.cpp.

References llvm::all_of(), SVEIntrinsicInfo::defaultMergingOp(), SVEIntrinsicInfo::defaultMergingUnaryNarrowingTopOp(), SVEIntrinsicInfo::defaultMergingUnaryOp(), SVEIntrinsicInfo::defaultUndefOp(), SVEIntrinsicInfo::defaultVoidOp(), SVEIntrinsicInfo::defaultZeroingOp(), llvm::CmpInst::FCMP_OEQ, llvm::CmpInst::FCMP_OGE, llvm::CmpInst::FCMP_OGT, llvm::CmpInst::FCMP_UNE, llvm::CmpInst::FCMP_UNO, llvm::CmpInst::ICMP_EQ, llvm::CmpInst::ICMP_NE, llvm::CmpInst::ICMP_SGE, llvm::CmpInst::ICMP_SGT, llvm::CmpInst::ICMP_SLE, llvm::CmpInst::ICMP_SLT, llvm::CmpInst::ICMP_UGE, llvm::CmpInst::ICMP_UGT, llvm::CmpInst::ICMP_ULE, llvm::CmpInst::ICMP_ULT, II, llvm::isa(), SVEIntrinsicInfo::setCmpPredicate(), SVEIntrinsicInfo::setMatchingIROpcode(), and V.

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

◆ containsDecreasingPointers()

bool containsDecreasingPointers ( Loop * TheLoop,
PredicatedScalarEvolution * PSE,
const DominatorTree & DT )
static

Definition at line 7711 of file AArch64TargetTransformInfo.cpp.

References llvm::LoopBase< BlockT, LoopT >::blocks(), llvm::getLoadStorePointerOperand(), llvm::getLoadStoreType(), llvm::getPtrStride(), I, and llvm::isa().

Referenced by llvm::AArch64TTIImpl::preferTailFoldingOverEpilogue().

◆ extractAttrFeatures()

void extractAttrFeatures ( const Function & F,
const AArch64TTIImpl * TTI,
SmallVectorImpl< StringRef > & Features )
static

Definition at line 275 of file AArch64TargetTransformInfo.cpp.

References F, and llvm::StringRef::split().

Referenced by llvm::AArch64TTIImpl::getFeatureMask(), and llvm::AArch64TTIImpl::getPriorityMask().

◆ getAppleRuntimeUnrollPreferences()

void getAppleRuntimeUnrollPreferences ( Loop * L,
ScalarEvolution & SE,
TargetTransformInfo::UnrollingPreferences & UP,
const AArch64TTIImpl & TTI )
static

For Apple CPUs, we want to runtime-unroll loops to make better use if the OOO engine's wide instruction window and various predictors.

Definition at line 6426 of file AArch64TargetTransformInfo.cpp.

References llvm::TargetTransformInfo::UnrollingPreferences::AddAdditionalAccumulators, llvm::any_of(), llvm::cast(), llvm::SmallPtrSetImpl< PtrType >::contains(), llvm::TargetTransformInfo::UnrollingPreferences::DefaultUnrollRuntimeCount, llvm::Depth, llvm::dyn_cast(), llvm::ScalarEvolution::getBackedgeTakenCount(), llvm::getBooleanLoopAttribute(), llvm::getLoadStorePointerOperand(), llvm::ScalarEvolution::getSCEV(), llvm::ScalarEvolution::getSmallConstantMaxTripCount(), llvm::ScalarEvolution::getSmallConstantTripMultiple(), llvm::ScalarEvolution::getSymbolicMaxBackedgeTakenCount(), I, llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::is_contained(), llvm::isa(), llvm::ScalarEvolution::isLoopInvariant(), isLoopSizeWithinBudget(), llvm::PatternMatch::m_Br(), llvm::PatternMatch::m_ICmp(), llvm::PatternMatch::m_Instruction(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::match(), llvm::TargetTransformInfo::UnrollingPreferences::MaxCount, llvm::none_of(), llvm::TargetTransformInfo::UnrollingPreferences::Partial, Pred, llvm::predecessors(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::TargetTransformInfo::UnrollingPreferences::Runtime, llvm::TargetTransformInfo::UnrollingPreferences::SCEVExpansionBudget, llvm::Size, llvm::SmallVectorTemplateCommon< T, typename >::size(), and V.

Referenced by llvm::AArch64TTIImpl::getUnrollingPreferences().

◆ getFalkorUnrollingPreferences()

void getFalkorUnrollingPreferences ( Loop * L,
ScalarEvolution & SE,
TargetTransformInfo::UnrollingPreferences & UP )
static

Definition at line 6311 of file AArch64TargetTransformInfo.cpp.

References llvm::dbgs(), llvm::dyn_cast(), llvm::LoadInst::getPointerOperand(), llvm::ScalarEvolution::getSCEV(), I, llvm::SCEVAddRecExpr::isAffine(), LLVM_DEBUG, llvm::Log2_32(), and llvm::TargetTransformInfo::UnrollingPreferences::MaxCount.

Referenced by llvm::AArch64TTIImpl::getUnrollingPreferences().

◆ getHistogramCost()

InstructionCost getHistogramCost ( const AArch64Subtarget * ST,
const IntrinsicCostAttributes & ICA )
static

Definition at line 635 of file AArch64TargetTransformInfo.cpp.

References llvm::dyn_cast(), llvm::IntrinsicCostAttributes::getArgTypes(), llvm::InstructionCost::getInvalid(), llvm::Type::getScalarSizeInBits(), llvm::InstructionCost, llvm::Type::isIntegerTy(), llvm::Type::isPointerTy(), llvm::isPowerOf2_64(), and llvm::AArch64::SVEBitsPerBlock.

Referenced by llvm::AArch64TTIImpl::getIntrinsicInstrCost().

◆ getSVEGatherScatterOverhead()

unsigned getSVEGatherScatterOverhead ( unsigned Opcode,
const AArch64Subtarget * ST )
static

Definition at line 5893 of file AArch64TargetTransformInfo.cpp.

References assert(), and llvm_unreachable.

Referenced by llvm::AArch64TTIImpl::getGatherScatterOpCost().

◆ instCombineConvertFromSVBool()

std::optional< Instruction * > instCombineConvertFromSVBool ( InstCombiner & IC,
IntrinsicInst & II )
static

Definition at line 2425 of file AArch64TargetTransformInfo.cpp.

References llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::cast(), llvm::dyn_cast(), II, llvm::SmallVectorImpl< T >::insert(), llvm::isa(), processPhiNode(), llvm::InstCombiner::replaceInstUsesWith(), and tryCombineFromSVBoolBinOp().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

◆ instCombineDMB()

std::optional< Instruction * > instCombineDMB ( InstCombiner & IC,
IntrinsicInst & II,
unsigned LookaheadThreshold )
static

Definition at line 3533 of file AArch64TargetTransformInfo.cpp.

References llvm::dyn_cast_or_null(), llvm::InstCombiner::eraseInstFromFunction(), I, and II.

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

◆ instCombineInStreamingMode()

std::optional< Instruction * > instCombineInStreamingMode ( InstCombiner & IC,
IntrinsicInst & II )
static

Definition at line 3597 of file AArch64TargetTransformInfo.cpp.

References llvm::ConstantInt::getBool(), llvm::SMEAttrs::hasStreamingCompatibleInterface(), llvm::SMEAttrs::hasStreamingInterfaceOrBody(), II, and llvm::InstCombiner::replaceInstUsesWith().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

◆ instCombineLD1GatherIndex()

std::optional< Instruction * > instCombineLD1GatherIndex ( InstCombiner & IC,
IntrinsicInst & II )
static

Definition at line 3263 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::cast(), llvm::IRBuilderBase::CreateGEP(), llvm::IRBuilderBase::CreateMaskedLoad(), llvm::ConstantAggregateZero::get(), II, llvm::PatternMatch::m_Intrinsic(), llvm::PatternMatch::m_One(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::match(), llvm::InstCombiner::replaceInstUsesWith(), and llvm::Value::takeName().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

◆ instCombineMaxMinNM()

std::optional< Instruction * > instCombineMaxMinNM ( InstCombiner & IC,
IntrinsicInst & II )
static

Definition at line 3478 of file AArch64TargetTransformInfo.cpp.

References A(), B(), II, and llvm::InstCombiner::replaceInstUsesWith().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

◆ instCombinePTrue()

std::optional< Instruction * > instCombinePTrue ( InstCombiner & IC,
IntrinsicInst & II )
static

Definition at line 3566 of file AArch64TargetTransformInfo.cpp.

References llvm::cast(), llvm::ConstantInt::getTrue(), II, and llvm::InstCombiner::replaceInstUsesWith().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

◆ instCombineRDFFR()

std::optional< Instruction * > instCombineRDFFR ( InstCombiner & IC,
IntrinsicInst & II )
static

Definition at line 2796 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::IRBuilderBase::CreateIntrinsic(), llvm::ConstantInt::getTrue(), II, llvm::InstCombiner::replaceInstUsesWith(), and llvm::Value::takeName().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

◆ instCombineSMECntsd()

std::optional< Instruction * > instCombineSMECntsd ( InstCombiner & IC,
IntrinsicInst & II,
const AArch64Subtarget * ST )
static

Definition at line 2826 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::IRBuilderBase::CreateElementCount(), llvm::ElementCount::getScalable(), II, llvm::InstCombiner::replaceInstUsesWith(), and llvm::Value::takeName().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

◆ instCombineST1ScatterIndex()

std::optional< Instruction * > instCombineST1ScatterIndex ( InstCombiner & IC,
IntrinsicInst & II )
static

Definition at line 3291 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::cast(), llvm::IRBuilderBase::CreateGEP(), llvm::IRBuilderBase::CreateMaskedStore(), llvm::InstCombiner::eraseInstFromFunction(), llvm::Value::getType(), II, llvm::PatternMatch::m_Intrinsic(), llvm::PatternMatch::m_One(), llvm::PatternMatch::m_Value(), and llvm::PatternMatch::match().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

◆ instCombineSVEAnd()

std::optional< Instruction * > instCombineSVEAnd ( InstCombiner & IC,
IntrinsicInst & II )
static

Definition at line 3652 of file AArch64TargetTransformInfo.cpp.

References A(), llvm::sampleprof::Base, llvm::InstCombiner::Builder, llvm::IRBuilderBase::CreateIntrinsic(), llvm::Constant::getNullValue(), II, LHS, llvm::PatternMatch::m_ConstantInt(), llvm::PatternMatch::m_Intrinsic(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::match(), llvm::InstCombiner::replaceInstUsesWith(), RHS, llvm::Sub, and uint64_t.

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

◆ instCombineSVECmpNE()

std::optional< Instruction * > instCombineSVECmpNE ( InstCombiner & IC,
IntrinsicInst & II )
static

Definition at line 2584 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::cast(), llvm::IRBuilderBase::CreateIntrinsic(), llvm::dyn_cast(), llvm::dyn_cast_or_null(), llvm::ScalableVectorType::get(), llvm::Type::getInt1Ty(), llvm::Constant::getNullValue(), llvm::getSplatValue(), llvm::ConstantInt::getTrue(), I, II, instCombineXorSVECmpCC(), instCombineZExtSVECmpNE(), llvm::isa(), isAllActivePredicate(), llvm::InstCombiner::replaceInstUsesWith(), llvm::AArch64::SVEBitsPerBlock, and llvm::Value::takeName().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

◆ instCombineSVECntElts()

std::optional< Instruction * > instCombineSVECntElts ( InstCombiner & IC,
IntrinsicInst & II,
unsigned NumElts )
static

Definition at line 2807 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::cast(), llvm::IRBuilderBase::CreateElementCount(), llvm::getNumElementsFromSVEPredPattern(), llvm::ElementCount::getScalable(), II, llvm::InstCombiner::replaceInstUsesWith(), and llvm::Value::takeName().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

◆ instCombineSVECondLast()

std::optional< Instruction * > instCombineSVECondLast ( InstCombiner & IC,
IntrinsicInst & II )
static

Definition at line 2754 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::cast(), llvm::IRBuilderBase::CreateBitCast(), llvm::IRBuilderBase::CreateIntrinsic(), llvm::VectorType::get(), getBitWidth(), llvm::IRBuilderBase::getDoubleTy(), llvm::IRBuilderBase::getFloatTy(), llvm::IRBuilderBase::getHalfTy(), llvm::Value::getType(), II, and llvm::InstCombiner::replaceInstUsesWith().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

◆ instCombineSVEDup()

std::optional< Instruction * > instCombineSVEDup ( InstCombiner & IC,
IntrinsicInst & II )
static

Definition at line 2489 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::cast(), llvm::IRBuilderBase::CreateInsertElement(), llvm::IRBuilderBase::CreateVectorSplat(), II, isAllActivePredicate(), llvm::PatternMatch::m_Intrinsic(), llvm::PatternMatch::m_SpecificInt(), llvm::PatternMatch::match(), llvm::InstCombiner::replaceInstUsesWith(), llvm::Splat, and uint64_t.

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

◆ instCombineSVEDupqLane()

std::optional< Instruction * > instCombineSVEDupqLane ( InstCombiner & IC,
IntrinsicInst & II )
static

Definition at line 3411 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::cast(), llvm::IRBuilderBase::CreateBitOrPointerCast(), llvm::IRBuilderBase::CreateInsertElement(), llvm::IRBuilderBase::CreateInsertVector(), llvm::IRBuilderBase::CreateShuffleVector(), Default, llvm::dyn_cast(), llvm::ConstantAggregateZero::get(), llvm::PoisonValue::get(), llvm::ScalableVectorType::get(), llvm::IRBuilderBase::getInt32Ty(), llvm::IRBuilderBase::getInt64(), llvm::IRBuilderBase::getIntNTy(), llvm::Value::getType(), I, II, llvm::isa(), llvm::PatternMatch::m_Intrinsic(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::m_Zero(), llvm::PatternMatch::match(), llvm::InstCombiner::replaceInstUsesWith(), SimplifyValuePattern(), llvm::SmallVectorTemplateCommon< T, typename >::size(), and uint64_t.

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

◆ instCombineSVEDupX()

std::optional< Instruction * > instCombineSVEDupX ( InstCombiner & IC,
IntrinsicInst & II )
static

Definition at line 2511 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::cast(), llvm::IRBuilderBase::CreateVectorSplat(), II, llvm::InstCombiner::replaceInstUsesWith(), and llvm::Splat.

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

◆ instCombineSVEInsr()

std::optional< Instruction * > instCombineSVEInsr ( InstCombiner & IC,
IntrinsicInst & II )
static

Definition at line 3522 of file AArch64TargetTransformInfo.cpp.

References llvm::getSplatValue(), II, and llvm::InstCombiner::replaceInstUsesWith().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

◆ instCombineSVELast()

std::optional< Instruction * > instCombineSVELast ( InstCombiner & IC,
IntrinsicInst & II )
static

Definition at line 2680 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, C(), llvm::cast(), llvm::ExtractElementInst::Create(), llvm::IRBuilderBase::CreateIntrinsic(), llvm::BinaryOperator::CreateWithCopiedFlags(), llvm::dyn_cast(), llvm::Type::getInt64Ty(), llvm::getNumElementsFromSVEPredPattern(), llvm::getSplatValue(), llvm::Value::getType(), II, llvm::isSplatValue(), LHS, llvm::PatternMatch::m_BinOp(), llvm::MIPatternMatch::m_OneUse(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::match(), llvm::InstCombiner::replaceInstUsesWith(), and RHS.

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

◆ instCombineSVELD1()

std::optional< Instruction * > instCombineSVELD1 ( InstCombiner & IC,
IntrinsicInst & II,
const DataLayout & DL )
static

Definition at line 2952 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::Instruction::copyMetadata(), llvm::IRBuilderBase::CreateLoad(), llvm::IRBuilderBase::CreateMaskedLoad(), DL, llvm::ConstantAggregateZero::get(), llvm::Value::getPointerAlignment(), II, isAllActivePredicate(), llvm::Load, Pred, and llvm::InstCombiner::replaceInstUsesWith().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

◆ instCombineSVEOrr()

std::optional< Instruction * > instCombineSVEOrr ( InstCombiner & IC,
IntrinsicInst & II )
static

Definition at line 3632 of file AArch64TargetTransformInfo.cpp.

References A(), B(), llvm::InstCombiner::Builder, llvm::IRBuilderBase::CreateIntrinsic(), II, llvm::PatternMatch::m_Intrinsic(), llvm::PatternMatch::m_One(), llvm::MIPatternMatch::m_OneUse(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::match(), and llvm::InstCombiner::replaceInstUsesWith().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

◆ instCombineSVEPairwiseAddLong()

std::optional< Instruction * > instCombineSVEPairwiseAddLong ( InstCombiner & IC,
IntrinsicInst & II )
static

Definition at line 3047 of file AArch64TargetTransformInfo.cpp.

References assert(), llvm::InstCombiner::Builder, llvm::IRBuilderBase::CreateIntrinsic(), llvm::dyn_cast_or_null(), llvm::InstCombiner::eraseInstFromFunction(), II, llvm::PatternMatch::m_c_Add(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::m_Zero(), llvm::PatternMatch::match(), llvm::InstCombiner::replaceInstUsesWith(), and llvm::IRBuilderBase::SetInsertPoint().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

◆ instCombineSVEPTest()

std::optional< Instruction * > instCombineSVEPTest ( InstCombiner & IC,
IntrinsicInst & II )
static

Definition at line 2839 of file AArch64TargetTransformInfo.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::InstCombiner::Builder, llvm::IRBuilderBase::CreateIntrinsic(), llvm::dyn_cast(), llvm::CallBase::getArgOperand(), llvm::IntrinsicInst::getIntrinsicID(), llvm::Value::getType(), II, llvm::InstCombiner::replaceInstUsesWith(), and llvm::Value::takeName().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

◆ instCombineSVESDIV()

std::optional< Instruction * > instCombineSVESDIV ( InstCombiner & IC,
IntrinsicInst & II )
static

Definition at line 3317 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::IRBuilderBase::CreateIntrinsic(), llvm::dyn_cast_or_null(), llvm::IRBuilderBase::getInt32Ty(), llvm::APInt::getSExtValue(), llvm::getSplatValue(), llvm::ConstantInt::getValue(), II, llvm::APInt::isNegatedPowerOf2(), llvm::APInt::isPowerOf2(), llvm::APInt::logBase2(), llvm::APInt::negate(), Pred, and llvm::InstCombiner::replaceInstUsesWith().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

◆ instCombineSVESel()

std::optional< Instruction * > instCombineSVESel ( InstCombiner & IC,
IntrinsicInst & II )
static

Definition at line 2477 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::IRBuilderBase::CreateSelect(), II, isAllActivePredicate(), llvm::InstCombiner::replaceInstUsesWith(), and Select.

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

◆ instCombineSVESrshl()

std::optional< Instruction * > instCombineSVESrshl ( InstCombiner & IC,
IntrinsicInst & II )
static

Definition at line 3488 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::IRBuilderBase::CreateIntrinsic(), II, llvm::isa(), isAllActivePredicate(), llvm::PatternMatch::m_Intrinsic(), llvm::PatternMatch::m_NonNegative(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::match(), Pred, and llvm::InstCombiner::replaceInstUsesWith().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

◆ instCombineSVEST1()

std::optional< Instruction * > instCombineSVEST1 ( InstCombiner & IC,
IntrinsicInst & II,
const DataLayout & DL )
static

Definition at line 2971 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::Instruction::copyMetadata(), llvm::IRBuilderBase::CreateMaskedStore(), llvm::IRBuilderBase::CreateStore(), DL, llvm::InstCombiner::eraseInstFromFunction(), llvm::Value::getPointerAlignment(), II, isAllActivePredicate(), Pred, and llvm::Store.

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

◆ instCombineSVETBL()

std::optional< Instruction * > instCombineSVETBL ( InstCombiner & IC,
IntrinsicInst & II )
static

Definition at line 3194 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::cast(), llvm::IRBuilderBase::CreateExtractElement(), llvm::IRBuilderBase::CreateVectorSplat(), llvm::dyn_cast_or_null(), llvm::getSplatValue(), II, llvm::InstCombiner::replaceInstUsesWith(), and llvm::Value::takeName().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

◆ instCombineSVEUMin()

std::optional< Instruction * > instCombineSVEUMin ( InstCombiner & IC,
IntrinsicInst & II )
static

Definition at line 3606 of file AArch64TargetTransformInfo.cpp.

References A(), B(), llvm::InstCombiner::Builder, llvm::IRBuilderBase::CreateIntrinsic(), II, llvm::PatternMatch::m_Intrinsic(), llvm::PatternMatch::m_One(), llvm::MIPatternMatch::m_OneUse(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::match(), and llvm::InstCombiner::replaceInstUsesWith().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

◆ instCombineSVEUnpack()

std::optional< Instruction * > instCombineSVEUnpack ( InstCombiner & IC,
IntrinsicInst & II )
static

Definition at line 3174 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::cast(), llvm::IRBuilderBase::CreateIntCast(), llvm::IRBuilderBase::CreateVectorSplat(), llvm::getSplatValue(), II, llvm::InstCombiner::replaceInstUsesWith(), and llvm::Value::takeName().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

◆ instCombineSVEUxt()

std::optional< Instruction * > instCombineSVEUxt ( InstCombiner & IC,
IntrinsicInst & II,
unsigned NumBits )
static

Definition at line 3576 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::cast(), llvm::IRBuilderBase::CreateIntrinsic(), llvm::APInt::getLowBitsSet(), II, llvm::isa(), isAllActivePredicate(), and llvm::InstCombiner::replaceInstUsesWith().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

◆ instCombineSVEUzp1()

std::optional< Instruction * > instCombineSVEUzp1 ( InstCombiner & IC,
IntrinsicInst & II )
static

Definition at line 3217 of file AArch64TargetTransformInfo.cpp.

References A(), B(), llvm::InstCombiner::Builder, llvm::cast(), llvm::IRBuilderBase::CreateInsertVector(), llvm::PoisonValue::get(), llvm::ScalableVectorType::getDoubleElementsVectorType(), II, llvm::PatternMatch::m_Intrinsic(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::match(), llvm::InstCombiner::replaceInstUsesWith(), llvm::Value::takeName(), and uint64_t.

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

◆ instCombineSVEVectorAdd()

std::optional< Instruction * > instCombineSVEVectorAdd ( InstCombiner & IC,
IntrinsicInst & II )
static

Definition at line 3072 of file AArch64TargetTransformInfo.cpp.

References II, and instCombineSVEVectorFuseMulAddSub().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

◆ instCombineSVEVectorBinOp()

std::optional< Instruction * > instCombineSVEVectorBinOp ( InstCombiner & IC,
IntrinsicInst & II )
static

Definition at line 3002 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::IRBuilderBase::CreateBinOpFMF(), II, intrinsicIDToBinOpCode(), isAllActivePredicate(), and llvm::InstCombiner::replaceInstUsesWith().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic(), instCombineSVEVectorFAddU(), and instCombineSVEVectorFSubU().

◆ instCombineSVEVectorFAdd()

std::optional< Instruction * > instCombineSVEVectorFAdd ( InstCombiner & IC,
IntrinsicInst & II )
static

Definition at line 3086 of file AArch64TargetTransformInfo.cpp.

References II, and instCombineSVEVectorFuseMulAddSub().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

◆ instCombineSVEVectorFAddU()

std::optional< Instruction * > instCombineSVEVectorFAddU ( InstCombiner & IC,
IntrinsicInst & II )
static

Definition at line 3106 of file AArch64TargetTransformInfo.cpp.

References II, instCombineSVEVectorBinOp(), and instCombineSVEVectorFuseMulAddSub().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

◆ instCombineSVEVectorFSub()

std::optional< Instruction * > instCombineSVEVectorFSub ( InstCombiner & IC,
IntrinsicInst & II )
static

Definition at line 3126 of file AArch64TargetTransformInfo.cpp.

References II, and instCombineSVEVectorFuseMulAddSub().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

◆ instCombineSVEVectorFSubU()

std::optional< Instruction * > instCombineSVEVectorFSubU ( InstCombiner & IC,
IntrinsicInst & II )
static

Definition at line 3146 of file AArch64TargetTransformInfo.cpp.

References II, instCombineSVEVectorBinOp(), and instCombineSVEVectorFuseMulAddSub().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

◆ instCombineSVEVectorFuseMulAddSub()

template<Intrinsic::ID MulOpc, Intrinsic::ID FuseOpc>
std::optional< Instruction * > instCombineSVEVectorFuseMulAddSub ( InstCombiner & IC,
IntrinsicInst & II,
bool MergeIntoAddendOp )
static

Definition at line 2909 of file AArch64TargetTransformInfo.cpp.

References llvm::FastMathFlags::allowContract(), llvm::InstCombiner::Builder, llvm::cast(), llvm::IRBuilderBase::CreateIntrinsic(), getFastMathFlags(), II, llvm::PatternMatch::m_Intrinsic(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::match(), Mul, P, and llvm::InstCombiner::replaceInstUsesWith().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic(), instCombineSVEVectorAdd(), instCombineSVEVectorFAdd(), instCombineSVEVectorFAddU(), instCombineSVEVectorFSub(), instCombineSVEVectorFSubU(), and instCombineSVEVectorSub().

◆ instCombineSVEVectorMlaU()

std::optional< Instruction * > instCombineSVEVectorMlaU ( InstCombiner & IC,
IntrinsicInst & II )
static

Definition at line 3018 of file AArch64TargetTransformInfo.cpp.

References assert(), llvm::InstCombiner::Builder, llvm::IRBuilderBase::CreateAdd(), llvm::IRBuilderBase::CreateSub(), II, llvm::isa(), llvm::MIPatternMatch::m_AllOnes(), llvm::PatternMatch::m_One(), llvm::PatternMatch::match(), and llvm::InstCombiner::replaceInstUsesWith().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

◆ instCombineSVEVectorSub()

std::optional< Instruction * > instCombineSVEVectorSub ( InstCombiner & IC,
IntrinsicInst & II )
static

Definition at line 3165 of file AArch64TargetTransformInfo.cpp.

References II, and instCombineSVEVectorFuseMulAddSub().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

◆ instCombineSVEZip()

std::optional< Instruction * > instCombineSVEZip ( InstCombiner & IC,
IntrinsicInst & II )
static

Definition at line 3247 of file AArch64TargetTransformInfo.cpp.

References A(), B(), II, llvm::PatternMatch::m_Intrinsic(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::match(), and llvm::InstCombiner::replaceInstUsesWith().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

◆ instCombineWhilelo()

std::optional< Instruction * > instCombineWhilelo ( InstCombiner & IC,
IntrinsicInst & II )
static

Definition at line 3557 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::IRBuilderBase::CreateIntrinsic(), II, and llvm::InstCombiner::replaceInstUsesWith().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

◆ instCombineXorSVECmpCC()

std::optional< Instruction * > instCombineXorSVECmpCC ( InstCombiner & IC,
IntrinsicInst & II )
static

Definition at line 2523 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::cast(), llvm::IRBuilderBase::CreateIntrinsic(), llvm::InstCombiner::eraseInstFromFunction(), II, llvm::PatternMatch::m_c_Xor(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::match(), llvm::InstCombiner::replaceInstUsesWith(), and llvm::IRBuilderBase::SetInsertPoint().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic(), and instCombineSVECmpNE().

◆ instCombineZExtSVECmpNE()

std::optional< Instruction * > instCombineZExtSVECmpNE ( InstCombiner & IC,
IntrinsicInst & II )
static

Definition at line 2560 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::cast(), llvm::IRBuilderBase::CreateIntrinsic(), llvm::InstCombiner::eraseInstFromFunction(), llvm::Value::getType(), II, isAllActivePredicate(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::m_Zero(), llvm::PatternMatch::m_ZExt(), llvm::PatternMatch::match(), llvm::InstCombiner::replaceInstUsesWith(), llvm::IRBuilderBase::SetInsertPoint(), and llvm::UMin.

Referenced by instCombineSVECmpNE().

◆ intrinsicIDToBinOpCode()

Instruction::BinaryOps intrinsicIDToBinOpCode ( unsigned Intrinsic)
static

Definition at line 2988 of file AArch64TargetTransformInfo.cpp.

Referenced by instCombineSVEVectorBinOp().

◆ isAllActivePredicate()

bool isAllActivePredicate ( Value * Pred)
static

Definition at line 2111 of file AArch64TargetTransformInfo.cpp.

References C(), llvm::cast(), llvm::dyn_cast(), llvm::Value::getType(), llvm::PatternMatch::m_Intrinsic(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::match(), and Pred.

◆ isFNeg()

bool isFNeg ( Value * Op)
static

Definition at line 7994 of file AArch64TargetTransformInfo.cpp.

References llvm::PatternMatch::m_FNegOrFSubNegZero(), llvm::PatternMatch::m_Value(), and llvm::PatternMatch::match().

Referenced by llvm::AArch64TTIImpl::isProfitableToSinkOperands().

◆ isLoopSizeWithinBudget()

bool isLoopSizeWithinBudget ( Loop * L,
const AArch64TTIImpl & TTI,
InstructionCost Budget,
unsigned * FinalSize )
static

Definition at line 6362 of file AArch64TargetTransformInfo.cpp.

References llvm::InstructionCost::getValue(), I, Operands, and llvm::TargetTransformInfo::TCK_CodeSize.

Referenced by getAppleRuntimeUnrollPreferences(), and shouldUnrollMultiExitLoop().

◆ isOperandOfVmullHighP64()

bool isOperandOfVmullHighP64 ( Value * Op)
static

Check if Op could be used with vmull_high_p64 intrinsic.

Definition at line 7933 of file AArch64TargetTransformInfo.cpp.

References llvm::cast(), llvm::Value::getType(), llvm::ConstantInt::getValue(), llvm::isa(), llvm::PatternMatch::m_ConstantInt(), llvm::PatternMatch::m_ExtractElt(), llvm::PatternMatch::m_Value(), and llvm::PatternMatch::match().

Referenced by areOperandsOfVmullHighP64().

◆ isSplatShuffle()

bool isSplatShuffle ( Value * V)
static

Definition at line 7849 of file AArch64TargetTransformInfo.cpp.

References llvm::all_equal(), llvm::dyn_cast(), and V.

Referenced by areExtractShuffleVectors(), and llvm::AArch64TTIImpl::isProfitableToSinkOperands().

◆ isUnpackedVectorVT()

bool isUnpackedVectorVT ( EVT VecVT)
static

Definition at line 630 of file AArch64TargetTransformInfo.cpp.

References llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), llvm::EVT::getSizeInBits(), llvm::EVT::isScalableVector(), and llvm::AArch64::SVEBitsPerBlock.

Referenced by llvm::AArch64TTIImpl::getIntrinsicInstrCost().

◆ matchBSL()

bool matchBSL ( const Instruction * I,
Instruction *& MainAnd,
Instruction *& OtherAnd,
Value *& A,
Value *& B )
static

Match Or(And(MASK, A), And(Not(MASK), B)), which ISel selects as BSL/BIT/BIF.

Both Ands must have one use and be in the Or's block. MainAnd,

OtherAnd, A and B are out parameters with the corresponding instructions. MainAnd refers to And that has Not as one of its operands.

Definition at line 3360 of file AArch64TargetTransformInfo.cpp.

References A(), B(), llvm::cast(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), I, llvm::PatternMatch::m_c_And(), llvm::PatternMatch::m_c_Or(), llvm::PatternMatch::m_Instruction(), llvm::MIPatternMatch::m_Not(), llvm::MIPatternMatch::m_OneUse(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::m_Value(), and llvm::PatternMatch::match().

Referenced by llvm::AArch64TTIImpl::getArithmeticInstrCost(), and llvm::AArch64TTIImpl::isProfitableToSinkOperands().

◆ processPhiNode()

std::optional< Instruction * > processPhiNode ( InstCombiner & IC,
IntrinsicInst & II )
static

The function will remove redundant reinterprets casting in the presence of the control flow.

Definition at line 1454 of file AArch64TargetTransformInfo.cpp.

References llvm::PHINode::addIncoming(), assert(), llvm::InstCombiner::Builder, llvm::cast(), llvm::IRBuilderBase::CreatePHI(), llvm::dyn_cast(), I, II, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::InstCombiner::replaceInstUsesWith(), and llvm::IRBuilderBase::SetInsertPoint().

Referenced by instCombineConvertFromSVBool().

◆ shouldSinkVectorOfPtrs()

bool shouldSinkVectorOfPtrs ( Value * Ptrs,
SmallVectorImpl< Use * > & Ops )
static

Definition at line 7948 of file AArch64TargetTransformInfo.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::sampleprof::Base, llvm::cast(), llvm::dyn_cast(), GEP, and llvm::isa().

Referenced by llvm::AArch64TTIImpl::isProfitableToSinkOperands().

◆ shouldSinkVScale()

bool shouldSinkVScale ( Value * Op,
SmallVectorImpl< Use * > & Ops )
static

We want to sink following cases: (add|sub|gep) A, ((mul|shl) vscale, imm); (add|sub|gep) A, vscale; (add|sub|gep) A, ((mul|shl) zext(vscale), imm);.

Definition at line 7976 of file AArch64TargetTransformInfo.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::cast(), llvm::PatternMatch::m_ConstantInt(), llvm::PatternMatch::m_Mul(), llvm::PatternMatch::m_Shl(), llvm::PatternMatch::m_VScale(), llvm::PatternMatch::m_ZExt(), and llvm::PatternMatch::match().

Referenced by llvm::AArch64TTIImpl::isProfitableToSinkOperands().

◆ shouldUnrollMultiExitLoop()

bool shouldUnrollMultiExitLoop ( Loop * L,
ScalarEvolution & SE,
const AArch64TTIImpl & TTI )
static

Definition at line 6389 of file AArch64TargetTransformInfo.cpp.

References llvm::any_of(), llvm::ScalarEvolution::getSmallConstantMaxTripCount(), llvm::ScalarEvolution::getSymbolicMaxBackedgeTakenCount(), llvm::BasicBlock::getTerminator(), llvm::isa(), isLoopSizeWithinBudget(), and llvm::ArrayRef< T >::size().

Referenced by llvm::AArch64TTIImpl::getUnrollingPreferences().

◆ simplifySVEIntrinsic()

std::optional< Instruction * > simplifySVEIntrinsic ( InstCombiner & IC,
IntrinsicInst & II,
const SVEIntrinsicInfo & IInfo )
static

Definition at line 2296 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::eraseInstFromFunction(), llvm::UndefValue::get(), SVEIntrinsicInfo::getGoverningPredicateOperandIdx(), SVEIntrinsicInfo::getMatchingIROpode(), SVEIntrinsicInfo::getMatchingUndefIntrinsic(), llvm::Constant::getNullValue(), SVEIntrinsicInfo::getOperandIdxInactiveLanesTakenFrom(), SVEIntrinsicInfo::getOperandIdxWithNoActiveLanes(), llvm::Intrinsic::getOrInsertDeclaration(), SVEIntrinsicInfo::hasGoverningPredicate(), SVEIntrinsicInfo::hasMatchingIROpode(), SVEIntrinsicInfo::hasMatchingUndefIntrinsic(), SVEIntrinsicInfo::hasOperandWithNoActiveLanes(), II, SVEIntrinsicInfo::inactiveLanesAreUnused(), SVEIntrinsicInfo::inactiveLanesTakenFromOperand(), llvm::isa(), isAllActivePredicate(), llvm::Instruction::isBinaryOp(), llvm::MIPatternMatch::m_ZeroInt(), llvm::PatternMatch::match(), Opc, llvm::InstCombiner::replaceInstUsesWith(), llvm::InstCombiner::replaceOperand(), SVEIntrinsicInfo::resultIsZeroInitialized(), simplifySVEIntrinsicBinOp(), and simplifySVEIntrinsicCompare().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

◆ simplifySVEIntrinsicBinOp()

std::optional< Instruction * > simplifySVEIntrinsicBinOp ( InstCombiner & IC,
IntrinsicInst & II,
const SVEIntrinsicInfo & IInfo )
static

Definition at line 2148 of file AArch64TargetTransformInfo.cpp.

References assert(), llvm::InstCombiner::Builder, llvm::IRBuilderBase::CreateIntrinsic(), llvm::IRBuilderBase::CreateSelect(), DL, llvm::dyn_cast(), SVEIntrinsicInfo::getMatchingIROpode(), llvm::Constant::getNullValue(), SVEIntrinsicInfo::getOperandIdxInactiveLanesTakenFrom(), llvm::Value::getType(), II, SVEIntrinsicInfo::inactiveLanesAreNotDefined(), llvm::isa(), llvm::Instruction::isBinaryOp(), llvm::Instruction::isCommutative(), llvm::PatternMatch::m_Intrinsic(), llvm::PatternMatch::m_Value(), llvm::MIPatternMatch::m_ZeroInt(), llvm::PatternMatch::match(), Opc, llvm::InstCombiner::replaceInstUsesWith(), llvm::InstCombiner::replaceOperand(), SVEIntrinsicInfo::resultIsZeroInitialized(), llvm::simplifyBinOp(), and stripInactiveLanes().

Referenced by simplifySVEIntrinsic().

◆ simplifySVEIntrinsicCompare()

std::optional< Instruction * > simplifySVEIntrinsicCompare ( InstCombiner & IC,
IntrinsicInst & II,
const SVEIntrinsicInfo & IInfo )
static

Definition at line 2227 of file AArch64TargetTransformInfo.cpp.

References assert(), llvm::InstCombiner::Builder, llvm::cast(), llvm::IRBuilderBase::CreateLogicalAnd(), DL, llvm::VectorType::get(), SVEIntrinsicInfo::getCmpPredicate(), SVEIntrinsicInfo::getMatchingIROpode(), llvm::APInt::getSExtValue(), llvm::APInt::getZExtValue(), II, llvm::isa(), llvm::FCmpInst::isCommutative(), llvm::ICmpInst::isCommutative(), llvm::CmpInst::isUnsigned(), LHS, llvm::PatternMatch::m_APInt(), llvm::MIPatternMatch::m_ZeroInt(), llvm::PatternMatch::match(), Opc, llvm::InstCombiner::replaceInstUsesWith(), llvm::InstCombiner::replaceOperand(), SVEIntrinsicInfo::resultIsZeroInitialized(), RHS, llvm::simplifyCmpInst(), and stripInactiveLanes().

Referenced by simplifySVEIntrinsic().

◆ SimplifyValuePattern()

bool SimplifyValuePattern ( SmallVector< Value * > & Vec,
bool AllowPoison )

Definition at line 3382 of file AArch64TargetTransformInfo.cpp.

References llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::isPowerOf2_64(), LHS, llvm::SmallVectorImpl< T >::resize(), RHS, SimplifyValuePattern(), and llvm::SmallVectorTemplateCommon< T, typename >::size().

Referenced by instCombineSVEDupqLane(), and SimplifyValuePattern().

◆ stripInactiveLanes()

Value * stripInactiveLanes ( Value * V,
const Value * Pg )
static

Definition at line 2136 of file AArch64TargetTransformInfo.cpp.

References llvm::cast(), llvm::dyn_cast(), llvm::ConstantVector::getSplat(), llvm::isa(), and V.

Referenced by simplifySVEIntrinsicBinOp(), and simplifySVEIntrinsicCompare().

◆ tryCombineFromSVBoolBinOp()

std::optional< Instruction * > tryCombineFromSVBoolBinOp ( InstCombiner & IC,
IntrinsicInst & II )
static

Definition at line 2361 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::IRBuilderBase::CreateIntrinsic(), llvm::IRBuilderBase::CreateLogicalAnd(), llvm::dyn_cast(), II, LHS, llvm::PatternMatch::m_Intrinsic(), llvm::PatternMatch::m_LogicalAnd(), llvm::MIPatternMatch::m_SpecificType(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::match(), P, llvm::InstCombiner::replaceInstUsesWith(), and RHS.

Referenced by instCombineConvertFromSVBool().

Variable Documentation

◆ SVETailFolding

cl::opt< TailFoldingOption, true, cl::parser< std::string > > SVETailFolding("sve-tail-folding", cl::desc( "Control the use of vectorisation using tail-folding for SVE where the" " option is specified in the form (Initial)[+(Flag1|Flag2|...)]:" "\ndisabled (Initial) No loop types will vectorize using " "tail-folding" "\ndefault (Initial) Uses the default tail-folding settings for " "the target CPU" "\nall (Initial) All legal loop types will vectorize using " "tail-folding" "\nsimple (Initial) Use tail-folding for simple loops (not " "reductions or recurrences)" "\nreductions Use tail-folding for loops containing reductions" "\nnoreductions Inverse of above" "\nrecurrences Use tail-folding for loops containing fixed order " "recurrences" "\nnorecurrences Inverse of above" "\nreverse Use tail-folding for loops requiring reversed " "predicates" "\nnoreverse Inverse of above"), cl::location(TailFoldingOptionLoc)) ( "sve-tail-folding" ,
cl::desc( "Control the use of vectorisation using tail-folding for SVE where the" " option is specified in the form (Initial)[+(Flag1|Flag2|...)]:" "\ndisabled (Initial) No loop types will vectorize using " "tail-folding" "\ndefault (Initial) Uses the default tail-folding settings for " "the target CPU" "\nall (Initial) All legal loop types will vectorize using " "tail-folding" "\nsimple (Initial) Use tail-folding for simple loops (not " "reductions or recurrences)" "\nreductions Use tail-folding for loops containing reductions" "\nnoreductions Inverse of above" "\nrecurrences Use tail-folding for loops containing fixed order " "recurrences" "\nnorecurrences Inverse of above" "\nreverse Use tail-folding for loops requiring reversed " "predicates" "\nnoreverse Inverse of above") ,
cl::location(TailFoldingOptionLoc)  )
static

◆ TailFoldingOptionLoc

TailFoldingOption TailFoldingOptionLoc

Definition at line 150 of file AArch64TargetTransformInfo.cpp.

Referenced by llvm::AArch64TTIImpl::preferTailFoldingOverEpilogue().

Generated on for LLVM by doxygen 1.14.0