|
LLVM 24.0.0git
|
#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"#include "llvm/CodeGen/GlobalISel/CallLowering.h"#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"#include "llvm/CodeGen/GlobalISel/LostDebugLocObserver.h"#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"#include "llvm/CodeGen/GlobalISel/Utils.h"#include "llvm/CodeGen/LowLevelTypeUtils.h"#include "llvm/CodeGen/MachineConstantPool.h"#include "llvm/CodeGen/MachineFrameInfo.h"#include "llvm/CodeGen/MachineRegisterInfo.h"#include "llvm/CodeGen/RuntimeLibcallUtil.h"#include "llvm/CodeGen/TargetFrameLowering.h"#include "llvm/CodeGen/TargetInstrInfo.h"#include "llvm/CodeGen/TargetLowering.h"#include "llvm/CodeGen/TargetOpcodes.h"#include "llvm/CodeGen/TargetSubtargetInfo.h"#include "llvm/IR/Instructions.h"#include "llvm/Support/Debug.h"#include "llvm/Support/MathExtras.h"#include "llvm/Support/raw_ostream.h"#include "llvm/Target/TargetMachine.h"#include <cassert>#include <numeric>#include <optional>Go to the source code of this file.
Macros | |
| #define | DEBUG_TYPE "legalizer" |
| #define | RTLIBCASE_INT(LibcallPrefix) |
| #define | RTLIBCASE(LibcallPrefix) |
| #define | LCALLS(A, B) |
| #define | LCALL5(A) |
| #define | RTLIBCASE_CMP(LibcallPrefix, ICmpPred) |
Functions | |
| static MachineBasicBlock::iterator | getPHIIncomingInsertPoint (MachineBasicBlock &MBB, Register Reg, const MachineRegisterInfo &MRI) |
| Choose an insertion point for a conversion of an incoming PHI value. | |
| static std::pair< int, int > | getNarrowTypeBreakDown (LLT OrigTy, LLT NarrowTy, LLT &LeftoverTy) |
Try to break down OrigTy into NarrowTy sized pieces. | |
| static Type * | getFloatTypeForLLT (LLVMContext &Ctx, LLT Ty) |
| static void | getUnmergeResults (SmallVectorImpl< Register > &Regs, const MachineInstr &MI) |
Append the result registers of G_UNMERGE_VALUES MI to Regs. | |
| static RTLIB::Libcall | getRTLibDesc (unsigned Opcode, unsigned Size) |
| static bool | hasSwiftErrorArg (MachineFunction &MF) |
| static bool | isLibCallInTailPosition (const CallLowering::ArgInfo &Result, MachineInstr &MI, const TargetInstrInfo &TII, MachineRegisterInfo &MRI) |
| True if an instruction is in tail position in its caller. | |
| static RTLIB::Libcall | getConvRTLibDesc (unsigned Opcode, Type *ToType, Type *FromType) |
| static RTLIB::Libcall | getOutlineAtomicLibcall (MachineInstr &MI) |
| static RTLIB::Libcall | getStateLibraryFunctionFor (MachineInstr &MI, const TargetLowering &TLI) |
| static std::pair< RTLIB::Libcall, CmpInst::Predicate > | getFCMPLibcallDesc (const CmpInst::Predicate Pred, unsigned Size) |
| Returns the corresponding libcall for the given Pred and the ICMP predicate that should be generated to compare with #0 after the libcall. | |
| static void | getUnmergePieces (SmallVectorImpl< Register > &Pieces, MachineIRBuilder &B, Register Src, LLT Ty) |
| static void | emitLoadFromConstantPool (Register DstReg, const Constant *ConstVal, MachineIRBuilder &MIRBuilder) |
| static Register | getBitcastWiderVectorElementOffset (MachineIRBuilder &B, Register Idx, unsigned NewEltSize, unsigned OldEltSize) |
| Figure out the bit offset into a register when coercing a vector index for the wide element type. | |
| static Register | buildBitFieldInsert (MachineIRBuilder &B, Register TargetReg, Register InsertReg, Register OffsetBits) |
Emit code to insert InsertReg into TargetRet at OffsetBits in TargetReg, while preserving other bits in TargetReg. | |
| static Register | clampVectorIndex (MachineIRBuilder &B, Register IdxReg, LLT VecTy) |
| static bool | hasSameNumEltsOnAllVectorOperands (GenericMachineInstr &MI, MachineRegisterInfo &MRI, std::initializer_list< unsigned > NonVecOpIndices) |
| Check that all vector operands have same number of elements. | |
| static void | makeDstOps (SmallVectorImpl< DstOp > &DstOps, LLT Ty, unsigned NumElts) |
Fill DstOps with DstOps that have same number of elements combined as the Ty. | |
| static void | broadcastSrcOp (SmallVectorImpl< SrcOp > &Ops, unsigned N, MachineOperand &Op) |
Operand Op is used on N sub-instructions. | |
| static bool | isNonZeroModBitWidthOrUndef (const MachineRegisterInfo &MRI, Register Reg, unsigned BW) |
| static bool | isNonZeroModBitWidth (const MachineRegisterInfo &MRI, Register Reg, unsigned BW) |
| static LegalizerHelper::LegalizeResult | loweri64tof16ITOFP (MachineInstr &MI, Register Dst, LLT DstTy, Register Src, LLT SrcTy, MachineIRBuilder &MIRBuilder) |
| i64->fp16 itofp can be lowered to i64->f64,f64->f32,f32->f16. | |
| static CmpInst::Predicate | minMaxToCompare (unsigned Opc) |
| static MachineInstrBuilder | SwapN (unsigned N, DstOp Dst, MachineIRBuilder &B, MachineInstrBuilder Src, const APInt &Mask) |
| static Register | getMemsetValue (Register Val, LLT Ty, MachineIRBuilder &MIB) |
| #define DEBUG_TYPE "legalizer" |
Definition at line 44 of file LegalizerHelper.cpp.
| #define LCALL5 | ( | A | ) |
| #define RTLIBCASE | ( | LibcallPrefix | ) |
Referenced by getRTLibDesc().
| #define RTLIBCASE_CMP | ( | LibcallPrefix, | |
| ICmpPred ) |
Referenced by getFCMPLibcallDesc().
| #define RTLIBCASE_INT | ( | LibcallPrefix | ) |
Referenced by getRTLibDesc().
|
static |
Operand Op is used on N sub-instructions.
Fill Ops with N SrcOps made from Op depending on operand type.
Definition at line 5361 of file LegalizerHelper.cpp.
References AbstractManglingParser< Derived, Alloc >::Ops, llvm_unreachable, and N.
Referenced by llvm::LegalizerHelper::fewerElementsVectorMultiEltType().
|
static |
Emit code to insert InsertReg into TargetRet at OffsetBits in TargetReg, while preserving other bits in TargetReg.
(InsertReg << Offset) | (TargetReg & ~(-1 >> InsertReg.size()) << Offset)
Definition at line 4030 of file LegalizerHelper.cpp.
References B(), llvm::APInt::getLowBitsSet(), and llvm::LLT::getSizeInBits().
Referenced by llvm::LegalizerHelper::bitcastInsertVectorElt().
|
static |
Definition at line 5251 of file LegalizerHelper.cpp.
References B(), llvm::APInt::getLowBitsSet(), llvm::LLT::getNumElements(), llvm::LLT::getSizeInBits(), Imm, llvm::isPowerOf2_32(), llvm::Log2_32(), llvm::MIPatternMatch::m_ICst(), and llvm::MIPatternMatch::mi_match().
Referenced by llvm::LegalizerHelper::getVectorElementPointer().
|
static |
Definition at line 3737 of file LegalizerHelper.cpp.
References llvm::MachineIRBuilder::buildConstantPool(), llvm::MachineIRBuilder::buildLoadInstr(), DL, llvm::MachineFunction::getConstantPool(), llvm::MachinePointerInfo::getConstantPool(), llvm::MachineConstantPool::getConstantPoolIndex(), llvm::MachineIRBuilder::getDataLayout(), llvm::MachineFunction::getMachineMemOperand(), llvm::MachineIRBuilder::getMF(), llvm::MachineIRBuilder::getMRI(), llvm::Value::getType(), llvm::MachineMemOperand::MOLoad, MRI, and llvm::LLT::pointer().
Referenced by llvm::LegalizerHelper::lowerConstant(), and llvm::LegalizerHelper::lowerFConstant().
|
static |
Figure out the bit offset into a register when coercing a vector index for the wide element type.
This is only for the case when promoting vector to one with larger elements. offset_idx = G_AND idx, ~(-1 << Log2(DstEltSize / SrcEltSize)) offset_bits = G_SHL offset_idx, Log2(SrcEltSize)
Definition at line 3902 of file LegalizerHelper.cpp.
References B(), llvm::APInt::getAllOnes(), llvm::LLT::getSizeInBits(), and llvm::Log2_32().
Referenced by llvm::LegalizerHelper::bitcastExtractVectorElt(), and llvm::LegalizerHelper::bitcastInsertVectorElt().
Definition at line 785 of file LegalizerHelper.cpp.
References llvm::RTLIB::getFPEXT(), llvm::RTLIB::getFPROUND(), llvm::RTLIB::getFPTOSINT(), llvm::RTLIB::getFPTOUINT(), llvm::RTLIB::getSINTTOFP(), llvm::RTLIB::getUINTTOFP(), llvm::MVT::getVT(), and llvm_unreachable.
Referenced by llvm::LegalizerHelper::conversionLibcall().
|
static |
Returns the corresponding libcall for the given Pred and the ICMP predicate that should be generated to compare with #0 after the libcall.
Definition at line 1162 of file LegalizerHelper.cpp.
References llvm::CmpInst::BAD_ICMP_PREDICATE, llvm::CmpInst::FCMP_OEQ, llvm::CmpInst::FCMP_OGE, llvm::CmpInst::FCMP_OGT, llvm::CmpInst::FCMP_OLE, llvm::CmpInst::FCMP_OLT, llvm::CmpInst::FCMP_UNE, llvm::CmpInst::FCMP_UNO, llvm::CmpInst::ICMP_EQ, llvm::CmpInst::ICMP_NE, llvm::CmpInst::ICMP_SGE, llvm::CmpInst::ICMP_SGT, llvm::CmpInst::ICMP_SLE, llvm::CmpInst::ICMP_SLT, Pred, RTLIBCASE_CMP, and llvm::Size.
|
static |
Definition at line 105 of file LegalizerHelper.cpp.
References llvm::Type::getDoubleTy(), llvm::Type::getFloatTy(), llvm::Type::getFP128Ty(), llvm::Type::getHalfTy(), and llvm::Type::getX86_FP80Ty().
Referenced by llvm::LegalizerHelper::libcall().
|
static |
Definition at line 11125 of file LegalizerHelper.cpp.
References llvm::MachineIRBuilder::buildConstant(), llvm::MachineIRBuilder::buildMul(), llvm::MachineIRBuilder::buildSplatBuildVector(), llvm::MachineIRBuilder::buildZExtOrTrunc(), llvm::getIConstantVRegValWithLookThrough(), llvm::MachineIRBuilder::getMRI(), llvm::MachineInstrBuilder::getReg(), llvm::LLT::getScalarType(), llvm::APInt::getSplat(), MRI, and llvm::APInt::trunc().
Referenced by getMemsetStores().
Try to break down OrigTy into NarrowTy sized pieces.
Returns the number of NarrowTy elements needed to reconstruct OrigTy, with any leftover piece as type LeftoverTy
Returns -1 in the first element of the pair if the breakdown is not satisfiable.
Definition at line 79 of file LegalizerHelper.cpp.
References assert(), llvm::LLT::changeElementCount(), llvm::ElementCount::getFixed(), llvm::LLT::getScalarSizeInBits(), llvm::LLT::getSizeInBits(), llvm::LLT::integer(), llvm::LLT::isValid(), llvm::LLT::isVector(), NumParts, and llvm::Size.
Referenced by makeDstOps(), and llvm::LegalizerHelper::reduceLoadStoreWidth().
|
static |
Definition at line 921 of file LegalizerHelper.cpp.
References llvm::cast(), llvm::LLT::getSizeInBytes(), llvm::LLT::isVector(), LCALL5, MI, Opc, and uint64_t.
Referenced by llvm::LegalizerHelper::createAtomicLibcall().
|
static |
Choose an insertion point for a conversion of an incoming PHI value.
Values available before an invoke must be converted before it, so the conversion also executes on the unwind edge. An invoke result is defined after G_INVOKE_REGION_START and is only available on the normal edge.
Definition at line 55 of file LegalizerHelper.cpp.
References E(), I, MBB, MRI, and Reg.
Referenced by llvm::LegalizerHelper::fewerElementsVectorPhi(), llvm::LegalizerHelper::moreElementsVectorPhi(), llvm::LegalizerHelper::narrowScalar(), and llvm::LegalizerHelper::widenScalar().
Definition at line 406 of file LegalizerHelper.cpp.
References llvm_unreachable, RTLIBCASE, RTLIBCASE_INT, and llvm::Size.
Referenced by llvm::LegalizerHelper::libcall(), and llvm::LegalizerHelper::simpleLibcall().
|
static |
Definition at line 1052 of file LegalizerHelper.cpp.
References llvm_unreachable, MI, and TLI.
|
static |
Definition at line 3730 of file LegalizerHelper.cpp.
References B(), E(), I, and llvm::SmallVectorTemplateBase< T, bool >::push_back().
Referenced by llvm::LegalizerHelper::lowerBitcast().
|
static |
Append the result registers of G_UNMERGE_VALUES MI to Regs.
Definition at line 249 of file LegalizerHelper.cpp.
References assert(), I, MI, llvm::SmallVectorImpl< T >::resize(), and llvm::SmallVectorTemplateCommon< T, typename >::size().
Referenced by llvm::LegalizerHelper::fewerElementsBitcast().
|
static |
Check that all vector operands have same number of elements.
Other operands should be listed in NonVecOp.
Definition at line 5301 of file LegalizerHelper.cpp.
References llvm::LLT::getNumElements(), llvm::is_contained(), llvm::LLT::isVector(), MI, and MRI.
Referenced by llvm::LegalizerHelper::fewerElementsVectorMultiEltType().
|
static |
Definition at line 542 of file LegalizerHelper.cpp.
References llvm::Function::getAttributes(), llvm::MachineFunction::getFunction(), llvm::MachineFunction::getSubtarget(), llvm::TargetSubtargetInfo::getTargetLowering(), and TLI.
Referenced by llvm::LegalizerHelper::createLibcall(), and llvm::LegalizerHelper::createMemLibcall().
|
static |
True if an instruction is in tail position in its caller.
Intended for legalizing libcalls as tail calls when possible.
Definition at line 551 of file LegalizerHelper.cpp.
References F, llvm::Register::isPhysical(), llvm::Register::isVirtual(), MBB, MI, MRI, llvm::Next, llvm::next_nodbg(), and TII.
Referenced by llvm::LegalizerHelper::createLibcall(), and llvm::LegalizerHelper::createMemLibcall().
|
static |
Definition at line 8263 of file LegalizerHelper.cpp.
References C(), llvm::cast(), llvm::matchUnaryPredicate(), MRI, and Reg.
Referenced by llvm::LegalizerHelper::lowerFunnelShiftAsShifts().
|
static |
Definition at line 8249 of file LegalizerHelper.cpp.
References C(), llvm::dyn_cast_or_null(), llvm::ConstantInt::getValue(), llvm::matchUnaryPredicate(), MRI, Reg, and llvm::APInt::urem().
Referenced by llvm::TargetLowering::expandFunnelShift(), llvm::LegalizerHelper::lowerFunnelShiftAsShifts(), and llvm::LegalizerHelper::lowerFunnelShiftWithInverse().
|
static |
i64->fp16 itofp can be lowered to i64->f64,f64->f32,f32->f16.
We cannot convert fpround f64->f16 without double-rounding, so we manually perform the lowering here where we know it is valid.
Definition at line 8653 of file LegalizerHelper.cpp.
References llvm::MachineIRBuilder::buildFPTrunc(), llvm::MachineIRBuilder::buildSITOFP(), llvm::MachineIRBuilder::buildUITOFP(), llvm::LLT::changeElementSize(), llvm::LLT::floatIEEE(), llvm::LegalizerHelper::Legalized, llvm::M1(), and MI.
Referenced by llvm::LegalizerHelper::lowerSITOFP(), and llvm::LegalizerHelper::lowerUITOFP().
|
static |
Fill DstOps with DstOps that have same number of elements combined as the Ty.
These DstOps have either scalar type when NumElts = 1 or are vectors with NumElts elements. When Ty.getNumElements() is not multiple of NumElts last DstOp (leftover) has fewer then NumElts elements.
Definition at line 5339 of file LegalizerHelper.cpp.
References assert(), llvm::LLT::changeElementCount(), llvm::ElementCount::getFixed(), getNarrowTypeBreakDown(), llvm::LLT::isValid(), NumParts, and llvm::SmallVectorTemplateBase< T, bool >::push_back().
Referenced by llvm::LegalizerHelper::fewerElementsVectorMultiEltType(), and llvm::LegalizerHelper::fewerElementsVectorPhi().
|
static |
Definition at line 9290 of file LegalizerHelper.cpp.
References llvm::CmpInst::ICMP_SGT, llvm::CmpInst::ICMP_SLT, llvm::CmpInst::ICMP_UGT, llvm::CmpInst::ICMP_ULT, llvm_unreachable, and Opc.
Referenced by llvm::LegalizerHelper::lowerMinMax().
|
static |
Definition at line 10472 of file LegalizerHelper.cpp.
References B(), LHS, N, and RHS.
Referenced by llvm::LegalizerHelper::lowerBitreverse().