AutoUpgrade.cpp File Reference

#include "llvm/IR/AutoUpgrade.h"
#include "IROptions.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/StringSwitch.h"
#include "llvm/BinaryFormat/Dwarf.h"
#include "llvm/IR/AttributeMask.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/CallingConv.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DebugInfo.h"
#include "llvm/IR/DebugInfoMetadata.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/InstVisitor.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/IntrinsicsAArch64.h"
#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/IR/IntrinsicsARM.h"
#include "llvm/IR/IntrinsicsNVPTX.h"
#include "llvm/IR/IntrinsicsRISCV.h"
#include "llvm/IR/IntrinsicsWebAssembly.h"
#include "llvm/IR/IntrinsicsX86.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/MDBuilder.h"
#include "llvm/IR/Metadata.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/NVVMIntrinsicUtils.h"
#include "llvm/IR/Value.h"
#include "llvm/IR/Verifier.h"
#include "llvm/Support/AMDGPUAddrSpace.h"
#include "llvm/Support/CodeGen.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/NVPTXAddrSpace.h"
#include "llvm/Support/NVVMAttributes.h"
#include "llvm/Support/Regex.h"
#include "llvm/Support/TimeProfiler.h"
#include "llvm/TargetParser/Triple.h"
#include <cstdint>
#include <cstring>
#include <numeric>

Go to the source code of this file.

Macros
#define	DEBUG_TYPE   "auto-upgrade"
#define	NVVM_TMA_G2S_MODES(M)
#define	G2S_ID(ID_SUFFIX, NAME)
#define	G2S_CTA_ID(ID_SUFFIX, NAME)
#define	G2S_CLUSTER_CASE(ID_SUFFIX, NAME)
#define	G2S_CTA_CASE(ID_SUFFIX, NAME)

Functions
static void	rename (GlobalValue *GV)
static void	reportFatalUsageErrorWithCI (StringRef reason, CallBase *CI)
static bool	upgradePTESTIntrinsic (Function *F, Intrinsic::ID IID, Function *&NewFn)
static bool	upgradeX86IntrinsicsWith8BitMask (Function *F, Intrinsic::ID IID, Function *&NewFn)
static bool	upgradeX86MaskedFPCompare (Function *F, Intrinsic::ID IID, Function *&NewFn)
static bool	upgradeX86MultiplyAddBytes (Function *F, Intrinsic::ID IID, Function *&NewFn)
static bool	upgradeX86MultiplyAddWords (Function *F, Intrinsic::ID IID, Function *&NewFn)
static bool	upgradeX86BF16Intrinsic (Function *F, Intrinsic::ID IID, Function *&NewFn)
static bool	upgradeX86BF16DPIntrinsic (Function *F, Intrinsic::ID IID, Function *&NewFn)
static bool	shouldUpgradeX86Intrinsic (Function *F, StringRef Name)
static bool	upgradeX86IntrinsicFunction (Function *F, StringRef Name, Function *&NewFn)
static bool	upgradeArmOrAarch64IntrinsicFunction (bool IsArm, Function *F, StringRef Name, Function *&NewFn)
static Intrinsic::ID	shouldUpgradeNVPTXTMAG2SIntrinsics (Function *F, StringRef Name, SmallVectorImpl< Type * > &OvlTys)
static Intrinsic::ID	shouldUpgradeNVPTXTMAG2SCTAIntrinsics (Function *F, StringRef Name)
static Intrinsic::ID	shouldUpgradeNVPTXBulkG2SClusterIntrinsic (Function *F, StringRef Name, SmallVectorImpl< Type * > &OvlTys)
static Intrinsic::ID	shouldUpgradeNVPTXBulkG2SCTAIntrinsic (Function *F, StringRef Name)
static std::optional< unsigned >	getNVPTXTMAReductionOp (StringRef Name)
static Intrinsic::ID	shouldUpgradeNVPTXTMAReductionIntrinsics (StringRef Name)
static Intrinsic::ID	shouldUpgradeNVPTXSharedClusterIntrinsic (Function *F, StringRef Name)
static Intrinsic::ID	shouldUpgradeNVPTXTcgen05CommitSharedIntrinsic (Function *F, StringRef Name)
static Intrinsic::ID	shouldUpgradeNVPTXTcgen05AllocSharedIntrinsic (Function *F, StringRef Name)
static Intrinsic::ID	shouldUpgradeNVPTXBF16Intrinsic (StringRef Name)
static bool	isLegacyNVPTXBF16IntSignature (Function *F, Intrinsic::ID IID)
static bool	isValidLegacyNVVMFPArithIntrinsic (const Intrinsic::ID(&IIDs)[2][3], StringRef Type, bool IsRN, bool IsFTZ, bool IsSat, bool IsRelu)
static std::optional< std::pair< Intrinsic::ID, RoundingMode > >	getNVVMFPArithUpgrade (StringRef Name, const Intrinsic::ID(&IIDs)[2][3])
static Intrinsic::ID	shouldUpgradeNVPTXMBarrierInitIntrinsic (StringRef Name)
static bool	consumeNVVMPtrAddrSpace (StringRef &Name)
static unsigned	getFunctionalOpcodeForVP (StringRef Name)
static Intrinsic::ID	getFunctionalIntrinsicIDForVP (StringRef Name)
static bool	shouldUpgradeVPIntrinsic (StringRef Name)
static bool	convertIntrinsicValidType (StringRef Name, const FunctionType *FuncTy)
static unsigned	getFullArgCountForDefaultArgUpgrade (Function *F, Intrinsic::ID IID, SmallVectorImpl< Type * > &OverloadTys)
static bool	upgradeIntrinsicWithDefaultArgs (Function *F, Function *&NewFn)
static bool	upgradeIntrinsicFunction1 (Function *F, Function *&NewFn, bool CanUpgradeDebugIntrinsicsToRecords)
static Value *	upgradeX86PSLLDQIntrinsics (IRBuilder<> &Builder, Value *Op, unsigned Shift)
static Value *	upgradeX86PSRLDQIntrinsics (IRBuilder<> &Builder, Value *Op, unsigned Shift)
static Value *	getX86MaskVec (IRBuilder<> &Builder, Value *Mask, unsigned NumElts)
static Value *	emitX86Select (IRBuilder<> &Builder, Value *Mask, Value *Op0, Value *Op1)
static Value *	emitX86ScalarSelect (IRBuilder<> &Builder, Value *Mask, Value *Op0, Value *Op1)
static Value *	upgradeX86ALIGNIntrinsics (IRBuilder<> &Builder, Value *Op0, Value *Op1, Value *Shift, Value *Passthru, Value *Mask, bool IsVALIGN)
static Value *	upgradeX86VPERMT2Intrinsics (IRBuilder<> &Builder, CallBase &CI, bool ZeroMask, bool IndexForm)
static Value *	upgradeX86BinaryIntrinsics (IRBuilder<> &Builder, CallBase &CI, Intrinsic::ID IID)
static Value *	upgradeX86Rotate (IRBuilder<> &Builder, CallBase &CI, bool IsRotateRight)
static Value *	upgradeX86vpcom (IRBuilder<> &Builder, CallBase &CI, unsigned Imm, bool IsSigned)
static Value *	upgradeX86ConcatShift (IRBuilder<> &Builder, CallBase &CI, bool IsShiftRight, bool ZeroMask)
static Value *	upgradeMaskedStore (IRBuilder<> &Builder, Value *Ptr, Value *Data, Value *Mask, bool Aligned)
static Value *	upgradeMaskedLoad (IRBuilder<> &Builder, Value *Ptr, Value *Passthru, Value *Mask, bool Aligned)
static Value *	upgradeAbs (IRBuilder<> &Builder, CallBase &CI)
static Value *	upgradePMULDQ (IRBuilder<> &Builder, CallBase &CI, bool IsSigned)
static Value *	applyX86MaskOn1BitsVec (IRBuilder<> &Builder, Value *Vec, Value *Mask)
static Value *	upgradeMaskedCompare (IRBuilder<> &Builder, CallBase &CI, unsigned CC, bool Signed)
static Value *	upgradeX86MaskedShift (IRBuilder<> &Builder, CallBase &CI, Intrinsic::ID IID)
static Value *	upgradeMaskedMove (IRBuilder<> &Builder, CallBase &CI)
static Value *	upgradeMaskToInt (IRBuilder<> &Builder, CallBase &CI)
static bool	upgradeAVX512MaskToSelect (StringRef Name, IRBuilder<> &Builder, CallBase &CI, Value *&Rep)
static Value *	upgradeNVVMFPArithCall (IRBuilder<> &Builder, CallBase *CI, StringRef Name, const Intrinsic::ID(&IIDs)[2][3])
static Value *	upgradeNVVMIntrinsicCall (StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
static Value *	upgradeX86IntrinsicCall (StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
static Value *	upgradeAArch64IntrinsicCall (StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
static Value *	upgradeARMIntrinsicCall (StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
static Value *	upgradeAMDGCNIntrinsicCall (StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
static MDNode *	unwrapMAVOp (CallBase *CI, unsigned Op)
	Helper to unwrap intrinsic call MetadataAsValue operands.
static Metadata *	unwrapMAVMetadataOp (CallBase *CI, unsigned Op)
	Helper to unwrap Metadata MetadataAsValue operands, such as the Value field.
static void	upgradeDbgIntrinsicToDbgRecord (StringRef Name, CallBase *CI)
	Convert debug intrinsic calls to non-instruction debug records.
static Value *	upgradeVectorSplice (CallBase *CI, IRBuilder<> &Builder)
static Value *	upgradeConvertIntrinsicCall (StringRef Name, CallBase *CI, Function *F, IRBuilder<> &Builder)
static ICmpInst::Predicate	getVPIntPredicateFromMD (const Value *Op)
static FCmpInst::Predicate	getVPFPPredicateFromMD (const Value *Op)
static Value *	upgradeVPIntrinsicCall (StringRef Name, CallBase *CI, IRBuilder<> &Builder)
static bool	upgradeIntrinsicCallWithDefaultArgs (CallBase *CI, Function *NewFn, IRBuilder<> &Builder)
static std::optional< StringRef >	getModuleFlagNameSafely (const MDNode &Flag)
static void	upgradeNVVMFnVectorAttr (const StringRef Attr, const char DimC, GlobalValue *GV, const Metadata *V)
static bool	isXYZ (StringRef S)
static bool	upgradeSingleNVVMAnnotation (GlobalValue *GV, StringRef K, const Metadata *V)
static bool	upgradeRetainReleaseMarker (Module &M)
	This checks for objc retain release marker which should be upgraded.
static bool	upgradePtrauthInitFiniArrays (Module &M)
static void	setFunctionAttrIfNotSet (Function &F, StringRef FnAttrName, StringRef Value)
static void	ConvertFunctionAttr (Function &F, bool Set, StringRef FnAttrName)
static void	ConvertModuleFlag (Module &M, Module::ModFlagBehavior Behavior, StringRef Key, uint32_t Val)
static const BooleanLoopTags *	getOldBooleanLoopTags (const MDTuple *T)
	Return the replacement tags if T still uses a removed two-operand form.
static Metadata *	makeBooleanLoopNode (LLVMContext &C, const BooleanLoopTags &Tags, const MDOperand &Op)
	Build the single-operand node that replaces a boolean operand: nonzero selects the enable tag, zero the disable tag.
static bool	isOldLoopArgument (Metadata *MD)
static MDString *	upgradeLoopTag (LLVMContext &C, StringRef OldTag)
static Metadata *	upgradeLoopArgument (Metadata *MD)

Variables
static constexpr Intrinsic::ID	NVVMFAddIIDs [2][3]
static constexpr Intrinsic::ID	NVVMFMulIIDs [2][3]
static constexpr Intrinsic::ID	NVVMFMAIIDs [2][3]

Macro Definition Documentation

DEBUG_TYPE

#define DEBUG_TYPE   "auto-upgrade"

Definition at line 65 of file AutoUpgrade.cpp.

G2S_CLUSTER_CASE

#define G2S_CLUSTER_CASE	(		ID_SUFFIX,
			NAME )

Value:

  case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_##ID_SUFFIX:

Referenced by llvm::UpgradeIntrinsicCall().

G2S_CTA_CASE

#define G2S_CTA_CASE	(		ID_SUFFIX,
			NAME )

Value:

  case Intrinsic::nvvm_cp_async_bulk_tensor_g2s_cta_##ID_SUFFIX:

Referenced by llvm::UpgradeIntrinsicCall().

G2S_CTA_ID

#define G2S_CTA_ID	(		ID_SUFFIX,
			NAME )

Value:

  .Case(NAME, Intrinsic::nvvm_cp_async_bulk_tensor_g2s_cta_##ID_SUFFIX)

Referenced by shouldUpgradeNVPTXTMAG2SCTAIntrinsics().

G2S_ID

#define G2S_ID	(		ID_SUFFIX,
			NAME )

Value:

  .Case(NAME, Intrinsic::nvvm_cp_async_bulk_tensor_g2s_##ID_SUFFIX)

Referenced by shouldUpgradeNVPTXTMAG2SIntrinsics().

NVVM_TMA_G2S_MODES

#define NVVM_TMA_G2S_MODES

(

M

)

Value:

  M(tile_1d, "tile.1d")                                                        \
  M(tile_2d, "tile.2d")                                                        \
  M(tile_3d, "tile.3d")                                                        \
  M(tile_4d, "tile.4d")                                                        \
  M(tile_5d, "tile.5d")                                                        \
  M(tile_gather4_2d, "tile.gather4.2d")                                        \
  M(im2col_3d, "im2col.3d")                                                    \
  M(im2col_4d, "im2col.4d")                                                    \
  M(im2col_5d, "im2col.5d")                                                    \
  M(im2col_w_3d, "im2col.w.3d")                                                \
  M(im2col_w_4d, "im2col.w.4d")                                                \
  M(im2col_w_5d, "im2col.w.5d")                                                \
  M(im2col_w_128_3d, "im2col.w.128.3d")                                        \
  M(im2col_w_128_4d, "im2col.w.128.4d")                                        \
  M(im2col_w_128_5d, "im2col.w.128.5d")

Definition at line 1194 of file AutoUpgrade.cpp.

Referenced by shouldUpgradeNVPTXTMAG2SCTAIntrinsics(), shouldUpgradeNVPTXTMAG2SIntrinsics(), and llvm::UpgradeIntrinsicCall().

Function Documentation

applyX86MaskOn1BitsVec()

Value * applyX86MaskOn1BitsVec ( IRBuilder<> & Builder, Value * Vec, Value * Mask )

static

Definition at line 2967 of file AutoUpgrade.cpp.

References C(), llvm::cast(), llvm::dyn_cast(), llvm::Constant::getNullValue(), llvm::Value::getType(), and getX86MaskVec().

Referenced by llvm::UpgradeIntrinsicCall(), upgradeMaskedCompare(), and upgradeX86IntrinsicCall().

consumeNVVMPtrAddrSpace()

bool consumeNVVMPtrAddrSpace ( StringRef & Name )

static

Definition at line 1584 of file AutoUpgrade.cpp.

Referenced by upgradeIntrinsicFunction1(), and upgradeNVVMIntrinsicCall().

ConvertFunctionAttr()

void ConvertFunctionAttr ( Function & F, bool Set, StringRef FnAttrName )

static

Definition at line 7669 of file AutoUpgrade.cpp.

References A(), llvm::dbgs(), F, and LLVM_DEBUG.

Referenced by llvm::copyModuleAttrToFunctions().

convertIntrinsicValidType()

bool convertIntrinsicValidType ( StringRef Name, const FunctionType * FuncTy )

static

Definition at line 1674 of file AutoUpgrade.cpp.

References llvm::CastInst::castIsValid(), and llvm::Type::getHalfTy().

Referenced by upgradeIntrinsicFunction1().

ConvertModuleFlag()

void ConvertModuleFlag ( Module & M, Module::ModFlagBehavior Behavior, StringRef Key, uint32_t Val )

static

Definition at line 7691 of file AutoUpgrade.cpp.

References llvm::dbgs(), llvm::InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key, and LLVM_DEBUG.

Referenced by llvm::copyModuleAttrToFunctions().

emitX86ScalarSelect()

Value * emitX86ScalarSelect ( IRBuilder<> & Builder, Value * Mask, Value * Op0, Value * Op1 )

static

Definition at line 2653 of file AutoUpgrade.cpp.

References C(), llvm::dyn_cast(), llvm::FixedVectorType::get(), and uint64_t.

Referenced by upgradeX86IntrinsicCall().

emitX86Select()

Value * emitX86Select ( IRBuilder<> & Builder, Value * Mask, Value * Op0, Value * Op1 )

static

Definition at line 2641 of file AutoUpgrade.cpp.

References C(), llvm::cast(), llvm::dyn_cast(), llvm::Value::getType(), and getX86MaskVec().

Referenced by upgradeAbs(), upgradeAVX512MaskToSelect(), upgradePMULDQ(), upgradeX86ALIGNIntrinsics(), upgradeX86BinaryIntrinsics(), upgradeX86ConcatShift(), upgradeX86IntrinsicCall(), upgradeX86MaskedShift(), upgradeX86Rotate(), and upgradeX86VPERMT2Intrinsics().

getFullArgCountForDefaultArgUpgrade()

unsigned getFullArgCountForDefaultArgUpgrade ( Function * F, Intrinsic::ID IID, SmallVectorImpl< Type * > & OverloadTys )

static

Definition at line 1695 of file AutoUpgrade.cpp.

References F, llvm::Intrinsic::getAllDefaultArgValues(), and llvm::Intrinsic::isSignatureValid().

Referenced by upgradeIntrinsicWithDefaultArgs().

getFunctionalIntrinsicIDForVP()

Intrinsic::ID getFunctionalIntrinsicIDForVP ( StringRef Name )

static

Definition at line 1627 of file AutoUpgrade.cpp.

References llvm::StringSwitch< T, R >::Default(), and llvm::StringSwitch< T, R >::StartsWith().

Referenced by shouldUpgradeVPIntrinsic(), and upgradeVPIntrinsicCall().

getFunctionalOpcodeForVP()

unsigned getFunctionalOpcodeForVP ( StringRef Name )

static

Definition at line 1590 of file AutoUpgrade.cpp.

References llvm::StringSwitch< T, R >::Default(), and llvm::StringSwitch< T, R >::StartsWith().

Referenced by shouldUpgradeVPIntrinsic(), and upgradeVPIntrinsicCall().

getModuleFlagNameSafely()

std::optional< StringRef > getModuleFlagNameSafely ( const MDNode & Flag )

static

Definition at line 6817 of file AutoUpgrade.cpp.

References llvm::dyn_cast_or_null().

Referenced by upgradePtrauthInitFiniArrays().

getNVPTXTMAReductionOp()

std::optional< unsigned > getNVPTXTMAReductionOp ( StringRef Name )

static

Definition at line 1339 of file AutoUpgrade.cpp.

References llvm::nvvm::ADD, llvm::nvvm::AND, llvm::StringSwitch< T, R >::Case(), llvm::nvvm::DEC, llvm::StringSwitch< T, R >::Default(), llvm::nvvm::INC, llvm::nvvm::MAX, llvm::nvvm::MIN, llvm::nvvm::OR, and llvm::nvvm::XOR.

Referenced by shouldUpgradeNVPTXTMAReductionIntrinsics(), and llvm::UpgradeIntrinsicCall().

getNVVMFPArithUpgrade()

std::optional< std::pair< Intrinsic::ID, RoundingMode > > getNVVMFPArithUpgrade ( StringRef Name, const Intrinsic::ID(&) IIDs[2][3] )

static

Definition at line 1544 of file AutoUpgrade.cpp.

References llvm::StringSwitch< T, R >::Case(), llvm::StringRef::consume_front(), llvm::StringSwitch< T, R >::Default(), llvm::StringRef::empty(), isValidLegacyNVVMFPArithIntrinsic(), llvm::NearestTiesToEven, llvm::Intrinsic::not_intrinsic, llvm::TowardNegative, llvm::TowardPositive, and llvm::TowardZero.

Referenced by upgradeIntrinsicFunction1(), and upgradeNVVMFPArithCall().

getOldBooleanLoopTags()

const BooleanLoopTags * getOldBooleanLoopTags ( const MDTuple * T )

static

Return the replacement tags if T still uses a removed two-operand form.

Definition at line 7793 of file AutoUpgrade.cpp.

References llvm::dyn_cast_or_null(), llvm::findBooleanLoopTags(), llvm::mdconst::hasa(), and T.

Referenced by isOldLoopArgument(), llvm::upgradeInstructionLoopAttachment(), and upgradeLoopArgument().

getVPFPPredicateFromMD()

FCmpInst::Predicate getVPFPPredicateFromMD ( const Value * Op )

static

Definition at line 5711 of file AutoUpgrade.cpp.

References llvm::CmpInst::BAD_FCMP_PREDICATE, llvm::StringSwitch< T, R >::Case(), llvm::cast(), llvm::StringSwitch< T, R >::Default(), llvm::CmpInst::FCMP_OEQ, llvm::CmpInst::FCMP_OGE, llvm::CmpInst::FCMP_OGT, llvm::CmpInst::FCMP_OLE, llvm::CmpInst::FCMP_OLT, llvm::CmpInst::FCMP_ONE, llvm::CmpInst::FCMP_ORD, llvm::CmpInst::FCMP_UEQ, llvm::CmpInst::FCMP_UGE, llvm::CmpInst::FCMP_UGT, llvm::CmpInst::FCMP_ULE, llvm::CmpInst::FCMP_ULT, llvm::CmpInst::FCMP_UNE, llvm::CmpInst::FCMP_UNO, and llvm::isa().

Referenced by upgradeVPIntrinsicCall().

getVPIntPredicateFromMD()

ICmpInst::Predicate getVPIntPredicateFromMD ( const Value * Op )

static

Definition at line 5693 of file AutoUpgrade.cpp.

References llvm::CmpInst::BAD_ICMP_PREDICATE, llvm::StringSwitch< T, R >::Case(), llvm::cast(), llvm::StringSwitch< T, R >::Default(), llvm::CmpInst::ICMP_EQ, llvm::CmpInst::ICMP_NE, llvm::CmpInst::ICMP_SGE, llvm::CmpInst::ICMP_SGT, llvm::CmpInst::ICMP_SLE, llvm::CmpInst::ICMP_SLT, llvm::CmpInst::ICMP_UGE, llvm::CmpInst::ICMP_UGT, llvm::CmpInst::ICMP_ULE, llvm::CmpInst::ICMP_ULT, and llvm::isa().

Referenced by upgradeVPIntrinsicCall().

getX86MaskVec()

Value * getX86MaskVec ( IRBuilder<> & Builder, Value * Mask, unsigned NumElts )

static

Definition at line 2621 of file AutoUpgrade.cpp.

References llvm::ArrayRef(), assert(), llvm::cast(), llvm::FixedVectorType::get(), and llvm::isPowerOf2_32().

Referenced by applyX86MaskOn1BitsVec(), emitX86Select(), llvm::UpgradeIntrinsicCall(), upgradeMaskedLoad(), upgradeMaskedStore(), upgradeMaskToInt(), and upgradeX86IntrinsicCall().

isLegacyNVPTXBF16IntSignature()

bool isLegacyNVPTXBF16IntSignature ( Function * F, Intrinsic::ID IID )

static

Definition at line 1489 of file AutoUpgrade.cpp.

References E(), F, llvm::Type::getPrimitiveSizeInBits(), llvm::Type::getScalarType(), llvm::Intrinsic::getType(), I, and llvm::Type::isIntegerTy().

Referenced by upgradeIntrinsicFunction1(), and upgradeNVVMIntrinsicCall().

isOldLoopArgument()

bool isOldLoopArgument ( Metadata * MD )

static

Definition at line 7810 of file AutoUpgrade.cpp.

References llvm::dyn_cast_or_null(), getOldBooleanLoopTags(), and T.

Referenced by llvm::upgradeInstructionLoopAttachment().

isValidLegacyNVVMFPArithIntrinsic()

bool isValidLegacyNVVMFPArithIntrinsic ( const Intrinsic::ID(&) IIDs[2][3], StringRef Type, bool IsRN, bool IsFTZ, bool IsSat, bool IsRelu )

static

Definition at line 1527 of file AutoUpgrade.cpp.

References llvm::StringSwitch< T, R >::Case(), llvm::StringSwitch< T, R >::Cases(), llvm::Default, NVVMFMAIIDs, and NVVMFMulIIDs.

Referenced by getNVVMFPArithUpgrade().

isXYZ()

bool isXYZ ( StringRef S )

inlinestatic

Definition at line 6904 of file AutoUpgrade.cpp.

Referenced by upgradeSingleNVVMAnnotation().

makeBooleanLoopNode()

Metadata * makeBooleanLoopNode ( LLVMContext & C, const BooleanLoopTags & Tags, const MDOperand & Op )

static

Build the single-operand node that replaces a boolean operand: nonzero selects the enable tag, zero the disable tag.

Definition at line 7802 of file AutoUpgrade.cpp.

References C(), llvm::BooleanLoopTags::Disable, Enable, llvm::BooleanLoopTags::Enable, llvm::mdconst::extract(), llvm::MDString::get(), and llvm::MDTuple::get().

Referenced by upgradeLoopArgument().

rename()

void rename ( GlobalValue * GV )

static

Definition at line 67 of file AutoUpgrade.cpp.

References llvm::Value::getName(), and llvm::Value::setName().

Referenced by upgradeArmOrAarch64IntrinsicFunction(), upgradeIntrinsicFunction1(), upgradeIntrinsicWithDefaultArgs(), upgradeNVVMIntrinsicCall(), upgradePTESTIntrinsic(), upgradeX86BF16DPIntrinsic(), upgradeX86BF16Intrinsic(), upgradeX86IntrinsicFunction(), upgradeX86IntrinsicsWith8BitMask(), upgradeX86MaskedFPCompare(), upgradeX86MultiplyAddBytes(), and upgradeX86MultiplyAddWords().

reportFatalUsageErrorWithCI()

void reportFatalUsageErrorWithCI ( StringRef reason, CallBase * CI )

static

Definition at line 71 of file AutoUpgrade.cpp.

References llvm::errs(), llvm::Value::print(), and llvm::reportFatalUsageError().

Referenced by upgradeX86IntrinsicCall().

setFunctionAttrIfNotSet()

void setFunctionAttrIfNotSet ( Function & F, StringRef FnAttrName, StringRef Value )

static

Definition at line 7657 of file AutoUpgrade.cpp.

References llvm::dbgs(), F, and LLVM_DEBUG.

Referenced by llvm::copyModuleAttrToFunctions().

shouldUpgradeNVPTXBF16Intrinsic()

Intrinsic::ID shouldUpgradeNVPTXBF16Intrinsic ( StringRef Name )

static

Definition at line 1431 of file AutoUpgrade.cpp.

References llvm::StringSwitch< T, R >::Case(), llvm::StringSwitch< T, R >::Default(), and llvm::Intrinsic::not_intrinsic.

Referenced by upgradeIntrinsicFunction1(), and upgradeNVVMIntrinsicCall().

shouldUpgradeNVPTXBulkG2SClusterIntrinsic()

Intrinsic::ID shouldUpgradeNVPTXBulkG2SClusterIntrinsic ( Function * F, StringRef Name, SmallVectorImpl< Type * > & OvlTys )

static

Definition at line 1295 of file AutoUpgrade.cpp.

References F, llvm::Type::isIntegerTy(), llvm::Intrinsic::not_intrinsic, and llvm::SmallVectorTemplateBase< T, bool >::push_back().

Referenced by upgradeIntrinsicFunction1().

shouldUpgradeNVPTXBulkG2SCTAIntrinsic()

Intrinsic::ID shouldUpgradeNVPTXBulkG2SCTAIntrinsic ( Function * F, StringRef Name )

static

Definition at line 1323 of file AutoUpgrade.cpp.

References F, and llvm::Intrinsic::not_intrinsic.

Referenced by upgradeIntrinsicFunction1().

shouldUpgradeNVPTXMBarrierInitIntrinsic()

Intrinsic::ID shouldUpgradeNVPTXMBarrierInitIntrinsic ( StringRef Name )

static

Definition at line 1577 of file AutoUpgrade.cpp.

References llvm::Intrinsic::not_intrinsic.

Referenced by upgradeIntrinsicFunction1().

shouldUpgradeNVPTXSharedClusterIntrinsic()

Intrinsic::ID shouldUpgradeNVPTXSharedClusterIntrinsic ( Function * F, StringRef Name )

static

Definition at line 1372 of file AutoUpgrade.cpp.

References llvm::NVPTXAS::ADDRESS_SPACE_SHARED, llvm::StringSwitch< T, R >::Case(), llvm::StringSwitch< T, R >::Default(), F, and llvm::Intrinsic::not_intrinsic.

Referenced by upgradeIntrinsicFunction1().

shouldUpgradeNVPTXTcgen05AllocSharedIntrinsic()

Intrinsic::ID shouldUpgradeNVPTXTcgen05AllocSharedIntrinsic ( Function * F, StringRef Name )

static

Definition at line 1421 of file AutoUpgrade.cpp.

References llvm::StringSwitch< T, R >::Case(), llvm::StringSwitch< T, R >::Default(), F, and llvm::Intrinsic::not_intrinsic.

Referenced by upgradeIntrinsicFunction1().

shouldUpgradeNVPTXTcgen05CommitSharedIntrinsic()

Intrinsic::ID shouldUpgradeNVPTXTcgen05CommitSharedIntrinsic ( Function * F, StringRef Name )

static

Definition at line 1396 of file AutoUpgrade.cpp.

References llvm::StringSwitch< T, R >::Case(), llvm::StringSwitch< T, R >::Default(), F, and llvm::Intrinsic::not_intrinsic.

Referenced by upgradeIntrinsicFunction1().

shouldUpgradeNVPTXTMAG2SCTAIntrinsics()

Intrinsic::ID shouldUpgradeNVPTXTMAG2SCTAIntrinsics ( Function * F, StringRef Name )

static

Definition at line 1262 of file AutoUpgrade.cpp.

References llvm::StringSwitch< T, R >::Default(), F, G2S_CTA_ID, llvm::Intrinsic::not_intrinsic, and NVVM_TMA_G2S_MODES.

Referenced by upgradeIntrinsicFunction1().

shouldUpgradeNVPTXTMAG2SIntrinsics()

Intrinsic::ID shouldUpgradeNVPTXTMAG2SIntrinsics ( Function * F, StringRef Name, SmallVectorImpl< Type * > & OvlTys )

static

Definition at line 1219 of file AutoUpgrade.cpp.

References assert(), llvm::StringSwitch< T, R >::Default(), F, G2S_ID, llvm::Type::isIntegerTy(), llvm::Intrinsic::not_intrinsic, NVVM_TMA_G2S_MODES, and llvm::SmallVectorTemplateBase< T, bool >::push_back().

Referenced by upgradeIntrinsicFunction1().

shouldUpgradeNVPTXTMAReductionIntrinsics()

Intrinsic::ID shouldUpgradeNVPTXTMAReductionIntrinsics ( StringRef Name )

static

Definition at line 1352 of file AutoUpgrade.cpp.

References llvm::StringSwitch< T, R >::Case(), llvm::StringSwitch< T, R >::Default(), getNVPTXTMAReductionOp(), and llvm::Intrinsic::not_intrinsic.

Referenced by upgradeIntrinsicFunction1().

shouldUpgradeVPIntrinsic()

bool shouldUpgradeVPIntrinsic ( StringRef Name )

static

Definition at line 1670 of file AutoUpgrade.cpp.

References getFunctionalIntrinsicIDForVP(), and getFunctionalOpcodeForVP().

Referenced by llvm::UpgradeIntrinsicCall(), and upgradeIntrinsicFunction1().

shouldUpgradeX86Intrinsic()

bool shouldUpgradeX86Intrinsic ( Function * F, StringRef Name )

static

Definition at line 179 of file AutoUpgrade.cpp.

References F.

Referenced by upgradeX86IntrinsicFunction().

unwrapMAVMetadataOp()

Metadata * unwrapMAVMetadataOp ( CallBase * CI, unsigned Op )

static

Helper to unwrap Metadata MetadataAsValue operands, such as the Value field.

Definition at line 5605 of file AutoUpgrade.cpp.

References llvm::dyn_cast(), and llvm::CallBase::getArgOperand().

Referenced by upgradeDbgIntrinsicToDbgRecord().

unwrapMAVOp()

MDNode * unwrapMAVOp ( CallBase * CI, unsigned Op )

static

Helper to unwrap intrinsic call MetadataAsValue operands.

Return as a plain MDNode, as it's the verifier's job to check these are the correct types later.

Definition at line 5593 of file AutoUpgrade.cpp.

References llvm::dyn_cast(), llvm::dyn_cast_if_present(), and llvm::CallBase::getArgOperand().

Referenced by upgradeDbgIntrinsicToDbgRecord().

upgradeAArch64IntrinsicCall()

Value * upgradeAArch64IntrinsicCall ( StringRef Name, CallBase * CI, Function * F, IRBuilder<> & Builder )

static

Definition at line 5230 of file AutoUpgrade.cpp.

References llvm::CallBase::args(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::StringSwitch< T, R >::Case(), llvm::dbgs(), llvm::StringSwitch< T, R >::Default(), llvm::SmallVectorTemplateCommon< T, typename >::end(), F, llvm::ConstantAggregateZero::get(), llvm::FixedVectorType::get(), llvm::ScalableVectorType::get(), llvm::Type::getBFloatTy(), llvm::Type::getFloatTy(), llvm::Type::getHalfTy(), llvm::Type::getInt16Ty(), llvm::Value::getName(), llvm::User::getOperand(), getType(), llvm::Value::getType(), llvm_unreachable, and llvm::Intrinsic::not_intrinsic.

Referenced by llvm::UpgradeIntrinsicCall().

upgradeAbs()

Value * upgradeAbs ( IRBuilder<> & Builder, CallBase & CI )

static

Definition at line 2927 of file AutoUpgrade.cpp.

References llvm::CallBase::arg_size(), emitX86Select(), llvm::CallBase::getArgOperand(), and llvm::Value::getType().

Referenced by upgradeX86IntrinsicCall().

upgradeAMDGCNIntrinsicCall()

Value * upgradeAMDGCNIntrinsicCall ( StringRef Name, CallBase * CI, Function * F, IRBuilder<> & Builder )

static

Definition at line 5390 of file AutoUpgrade.cpp.

References llvm::CallBase::arg_size(), llvm::CallBase::args(), assert(), llvm::cast(), llvm::Instruction::copyMetadata(), llvm::MDBuilder::createRange(), llvm::dyn_cast(), E(), F, llvm::AtomicRMWInst::FAdd, llvm::AMDGPUAS::FLAT_ADDRESS, llvm::AtomicRMWInst::FMax, llvm::AtomicRMWInst::FMin, llvm::MDNode::get(), llvm::VectorType::get(), llvm::CallBase::getArgOperand(), llvm::CallBase::getAttributes(), llvm::Type::getBFloatTy(), llvm::CallBase::getCallingConv(), llvm::User::getNumOperands(), llvm::CallBase::getOperandBundlesAsDefs(), llvm::Intrinsic::getOrInsertDeclaration(), getType(), llvm::Value::getType(), llvm::ConstantInt::getZExtValue(), I, llvm::Type::isFloatTy(), llvm::Type::isIntegerTy(), llvm::isValidAtomicOrdering(), llvm::ConstantInt::isZero(), LHS, llvm::AMDGPUAS::LOCAL_ADDRESS, llvm::NotAtomic, Pred, llvm::AMDGPUAS::PRIVATE_ADDRESS, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::reportFatalUsageError(), RHS, llvm::SequentiallyConsistent, llvm::CallBase::setCallingConv(), llvm::Instruction::setMetadata(), llvm::CallInst::setTailCallKind(), llvm::AtomicRMWInst::setVolatile(), llvm::StringSwitch< T, R >::StartsWith(), T1, llvm::Value::takeName(), llvm::AtomicRMWInst::UDecWrap, llvm::AtomicRMWInst::UIncWrap, llvm::Unordered, llvm::AtomicRMWInst::USubCond, and llvm::AtomicRMWInst::USubSat.

Referenced by llvm::UpgradeIntrinsicCall().

upgradeARMIntrinsicCall()

Value * upgradeARMIntrinsicCall ( StringRef Name, CallBase * CI, Function * F, IRBuilder<> & Builder )

static

Definition at line 5300 of file AutoUpgrade.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::CallBase::args(), C1, F, llvm::FixedVectorType::get(), llvm::VectorType::get(), llvm::CallBase::getArgOperand(), llvm::CallBase::getIntrinsicID(), llvm::Value::getName(), llvm::User::getOperand(), llvm::Value::getType(), and llvm_unreachable.

Referenced by llvm::UpgradeIntrinsicCall().

upgradeArmOrAarch64IntrinsicFunction()

bool upgradeArmOrAarch64IntrinsicFunction ( bool IsArm, Function * F, StringRef Name, Function *& NewFn )

static

Definition at line 794 of file AutoUpgrade.cpp.

References assert(), llvm::StringSwitch< T, R >::Case(), llvm::StringSwitch< T, R >::Cases(), llvm::cast(), llvm::Default, llvm::StringSwitch< T, R >::Default(), llvm::dyn_cast(), F, llvm::FixedVectorType::get(), llvm::VectorType::get(), llvm::Type::getBFloatTy(), llvm::Intrinsic::getOrInsertDeclaration(), Groups, llvm::Regex::match(), llvm::Intrinsic::not_intrinsic, rename(), llvm::size(), and llvm::StringSwitch< T, R >::StartsWith().

Referenced by upgradeIntrinsicFunction1().

upgradeAVX512MaskToSelect()

bool upgradeAVX512MaskToSelect ( StringRef Name, IRBuilder<> & Builder, CallBase & CI, Value *& Rep )

static

Definition at line 3051 of file AutoUpgrade.cpp.

References llvm::CallBase::arg_size(), llvm::CallBase::args(), assert(), emitX86Select(), llvm::CallBase::getArgOperand(), llvm::Type::getPrimitiveSizeInBits(), llvm::Type::getScalarSizeInBits(), llvm::Value::getType(), llvm::Type::isFPOrFPVectorTy(), llvm_unreachable, and upgradeX86BinaryIntrinsics().

Referenced by upgradeX86IntrinsicCall().

upgradeConvertIntrinsicCall()

Value * upgradeConvertIntrinsicCall ( StringRef Name, CallBase * CI, Function * F, IRBuilder<> & Builder )

static

Definition at line 5676 of file AutoUpgrade.cpp.

References F, llvm::CallBase::getArgOperand(), and llvm::Value::getType().

Referenced by llvm::UpgradeIntrinsicCall().

upgradeDbgIntrinsicToDbgRecord()

void upgradeDbgIntrinsicToDbgRecord ( StringRef Name, CallBase * CI )

static

Convert debug intrinsic calls to non-instruction debug records.

Name - Final part of the intrinsic name, e.g. 'value' in llvm.dbg.value. CI - The debug intrinsic call.

Definition at line 5615 of file AutoUpgrade.cpp.

References llvm::DIExpression::append(), llvm::CallBase::arg_size(), assert(), llvm::DbgVariableRecord::Assign, llvm::DbgLabelRecord::createUnresolvedDbgLabelRecord(), llvm::DbgVariableRecord::createUnresolvedDbgVariableRecord(), llvm::DbgVariableRecord::Declare, llvm::dyn_cast(), llvm::dyn_cast_or_null(), llvm::CallBase::getArgOperand(), llvm::Instruction::getDebugLoc(), llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), llvm::Offset, llvm::DbgRecord::setDebugLoc(), unwrapMAVMetadataOp(), unwrapMAVOp(), and llvm::DbgVariableRecord::Value.

Referenced by llvm::UpgradeIntrinsicCall().

upgradeIntrinsicCallWithDefaultArgs()

bool upgradeIntrinsicCallWithDefaultArgs ( CallBase * CI, Function * NewFn, IRBuilder<> & Builder )

static

Definition at line 5764 of file AutoUpgrade.cpp.

References llvm::CallBase::arg_size(), llvm::Function::arg_size(), llvm::CallBase::args(), assert(), llvm::Instruction::copyMetadata(), llvm::dyn_cast(), llvm::Instruction::eraseFromParent(), llvm::Intrinsic::getAllDefaultArgValues(), llvm::CallBase::getCallingConv(), llvm::CallBase::getFunctionType(), llvm::Function::getFunctionType(), llvm::Function::getIntrinsicID(), llvm::CallBase::getOperandBundlesAsDefs(), llvm::Type::isIntegerTy(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::Value::replaceAllUsesWith(), llvm::CallBase::setCalledFunction(), llvm::CallBase::setCallingConv(), llvm::CallInst::setTailCallKind(), and llvm::Value::takeName().

Referenced by llvm::UpgradeIntrinsicCall().

upgradeIntrinsicFunction1()

bool upgradeIntrinsicFunction1 ( Function * F, Function *& NewFn, bool CanUpgradeDebugIntrinsicsToRecords )

static

Definition at line 1731 of file AutoUpgrade.cpp.

References assert(), llvm::StringSwitch< T, R >::Case(), llvm::StringSwitch< T, R >::Cases(), consumeNVVMPtrAddrSpace(), convertIntrinsicValidType(), llvm::Default, llvm::StringSwitch< T, R >::Default(), F, getNVVMFPArithUpgrade(), llvm::Intrinsic::getOrInsertDeclaration(), Groups, isLegacyNVPTXBF16IntSignature(), llvm::Intrinsic::not_intrinsic, NVVMFAddIIDs, NVVMFMAIIDs, NVVMFMulIIDs, llvm::SmallVectorTemplateBase< T, bool >::push_back(), R2, rename(), shouldUpgradeNVPTXBF16Intrinsic(), shouldUpgradeNVPTXBulkG2SClusterIntrinsic(), shouldUpgradeNVPTXBulkG2SCTAIntrinsic(), shouldUpgradeNVPTXMBarrierInitIntrinsic(), shouldUpgradeNVPTXSharedClusterIntrinsic(), shouldUpgradeNVPTXTcgen05AllocSharedIntrinsic(), shouldUpgradeNVPTXTcgen05CommitSharedIntrinsic(), shouldUpgradeNVPTXTMAG2SCTAIntrinsics(), shouldUpgradeNVPTXTMAG2SIntrinsics(), shouldUpgradeNVPTXTMAReductionIntrinsics(), shouldUpgradeVPIntrinsic(), llvm::StringSwitch< T, R >::StartsWith(), upgradeArmOrAarch64IntrinsicFunction(), and upgradeX86IntrinsicFunction().

Referenced by llvm::UpgradeIntrinsicFunction().

upgradeIntrinsicWithDefaultArgs()

bool upgradeIntrinsicWithDefaultArgs ( Function * F, Function *& NewFn )

static

Definition at line 1715 of file AutoUpgrade.cpp.

References llvm::Function::arg_size(), assert(), F, getFullArgCountForDefaultArgUpgrade(), llvm::Intrinsic::getOrInsertDeclaration(), and rename().

upgradeLoopArgument()

Metadata * upgradeLoopArgument ( Metadata * MD )

static

Rewrite a removed two-operand boolean form to the single-operand pair.

Definition at line 7836 of file AutoUpgrade.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, C(), llvm::dyn_cast_or_null(), E(), llvm::findBooleanLoopTags(), llvm::MDTuple::get(), getOldBooleanLoopTags(), llvm::MDString::getString(), llvm::mdconst::hasa(), I, makeBooleanLoopNode(), T, and upgradeLoopTag().

Referenced by llvm::upgradeInstructionLoopAttachment().

upgradeLoopTag()

MDString * upgradeLoopTag ( LLVMContext & C, StringRef OldTag )

static

Definition at line 7824 of file AutoUpgrade.cpp.

References assert(), C(), llvm::StringRef::drop_front(), llvm::MDString::get(), llvm::StringRef::size(), and llvm::StringRef::starts_with().

Referenced by upgradeLoopArgument().

upgradeMaskedCompare()

Value * upgradeMaskedCompare ( IRBuilder<> & Builder, CallBase & CI, unsigned CC, bool Signed )

static

Definition at line 2989 of file AutoUpgrade.cpp.

References applyX86MaskOn1BitsVec(), llvm::CallBase::arg_size(), llvm::cast(), CC, llvm::FixedVectorType::get(), llvm::Constant::getAllOnesValue(), llvm::CallBase::getArgOperand(), llvm::Constant::getNullValue(), llvm::Value::getType(), llvm::CmpInst::ICMP_EQ, llvm::CmpInst::ICMP_NE, llvm::CmpInst::ICMP_SGE, llvm::CmpInst::ICMP_SGT, llvm::CmpInst::ICMP_SLE, llvm::CmpInst::ICMP_SLT, llvm::CmpInst::ICMP_UGE, llvm::CmpInst::ICMP_UGT, llvm::CmpInst::ICMP_ULE, llvm::CmpInst::ICMP_ULT, llvm_unreachable, Pred, and llvm::Signed.

Referenced by upgradeX86IntrinsicCall().

upgradeMaskedLoad()

Value * upgradeMaskedLoad ( IRBuilder<> & Builder, Value * Ptr, Value * Passthru, Value * Mask, bool Aligned )

static

Definition at line 2906 of file AutoUpgrade.cpp.

References llvm::Aligned, C(), llvm::cast(), llvm::dyn_cast(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getFixedValue(), llvm::Type::getPrimitiveSizeInBits(), llvm::Value::getType(), and getX86MaskVec().

Referenced by upgradeX86IntrinsicCall().

upgradeMaskedMove()

Value * upgradeMaskedMove ( IRBuilder<> & Builder, CallBase & CI )

static

Definition at line 3028 of file AutoUpgrade.cpp.

References A(), B(), llvm::CallBase::getArgOperand(), Select, and uint64_t.

Referenced by upgradeX86IntrinsicCall().

upgradeMaskedStore()

Value * upgradeMaskedStore ( IRBuilder<> & Builder, Value * Ptr, Value * Data, Value * Mask, bool Aligned )

static

Definition at line 2888 of file AutoUpgrade.cpp.

References llvm::Aligned, C(), llvm::cast(), llvm::Data, llvm::dyn_cast(), and getX86MaskVec().

Referenced by upgradeX86IntrinsicCall().

upgradeMaskToInt()

Value * upgradeMaskToInt ( IRBuilder<> & Builder, CallBase & CI )

static

Definition at line 3042 of file AutoUpgrade.cpp.

References llvm::cast(), llvm::CallBase::getArgOperand(), llvm::Value::getType(), and getX86MaskVec().

Referenced by upgradeX86IntrinsicCall().

upgradeNVVMFnVectorAttr()

void upgradeNVVMFnVectorAttr ( const StringRef Attr, const char DimC, GlobalValue * GV, const Metadata * V )

static

Definition at line 6871 of file AutoUpgrade.cpp.

References llvm::ArrayRef(), assert(), llvm::cast(), llvm::StringRef::empty(), llvm::mdconst::extract(), F, llvm::join(), llvm::Length, llvm::StringRef::split(), llvm::StringRef::trim(), uint64_t, llvm::utostr(), and V.

Referenced by upgradeSingleNVVMAnnotation().

upgradeNVVMFPArithCall()

Value * upgradeNVVMFPArithCall ( IRBuilder<> & Builder, CallBase * CI, StringRef Name, const Intrinsic::ID(&) IIDs[2][3] )

static

Definition at line 3291 of file AutoUpgrade.cpp.

References llvm::CallBase::args(), assert(), llvm::FixedVectorType::get(), getNVVMFPArithUpgrade(), llvm::Type::getScalarType(), llvm::Value::getType(), and llvm::Type::isIntegerTy().

Referenced by upgradeNVVMIntrinsicCall().

upgradeNVVMIntrinsicCall()

Value * upgradeNVVMIntrinsicCall ( StringRef Name, CallBase * CI, Function * F, IRBuilder<> & Builder )

static

Definition at line 3317 of file AutoUpgrade.cpp.

References llvm::AtomicRMWInst::Add, llvm::AtomicRMWInst::And, llvm::Function::arg_size(), assert(), llvm::AtomicRMWInst::BAD_BINOP, C(), llvm::StringSwitch< T, R >::Case(), llvm::cast(), consumeNVVMPtrAddrSpace(), llvm::StringSwitch< T, R >::Default(), F, llvm::AtomicRMWInst::FAdd, llvm::FixedVectorType::get(), llvm::MDNode::get(), llvm::Function::getArg(), llvm::CallBase::getArgOperand(), llvm::Type::getBFloatTy(), llvm::Value::getContext(), llvm::Type::getInt64Ty(), llvm::User::getOperand(), llvm::Intrinsic::getOrInsertDeclaration(), llvm::LLVMContext::getOrInsertSyncScopeID(), llvm::Type::getScalarType(), llvm::Value::getType(), I, llvm::Type::isBFloatTy(), llvm::Type::isIntegerTy(), isLegacyNVPTXBF16IntSignature(), llvm::AtomicRMWInst::Max, llvm::AtomicRMWInst::Min, llvm::Monotonic, llvm::Intrinsic::not_intrinsic, NVVMFAddIIDs, NVVMFMAIIDs, NVVMFMulIIDs, llvm::AtomicRMWInst::Or, rename(), llvm::FastMathFlags::setApproxFunc(), llvm::Value::setMetadata(), shouldUpgradeNVPTXBF16Intrinsic(), llvm::size(), llvm::StringSwitch< T, R >::StartsWith(), llvm::AtomicRMWInst::UDecWrap, llvm::AtomicRMWInst::UIncWrap, upgradeNVVMFPArithCall(), llvm::AtomicRMWInst::Xchg, and llvm::AtomicRMWInst::Xor.

Referenced by llvm::UpgradeIntrinsicCall().

upgradePMULDQ()

Value * upgradePMULDQ ( IRBuilder<> & Builder, CallBase & CI, bool IsSigned )

static

Definition at line 2937 of file AutoUpgrade.cpp.

References llvm::CallBase::arg_size(), emitX86Select(), llvm::CallBase::getArgOperand(), llvm::Value::getType(), LHS, and RHS.

Referenced by upgradeX86IntrinsicCall().

upgradePTESTIntrinsic()

bool upgradePTESTIntrinsic ( Function * F, Intrinsic::ID IID, Function *& NewFn )

static

Definition at line 80 of file AutoUpgrade.cpp.

References F, llvm::FixedVectorType::get(), llvm::Type::getFloatTy(), llvm::Intrinsic::getOrInsertDeclaration(), and rename().

Referenced by upgradeX86IntrinsicFunction().

upgradePtrauthInitFiniArrays()

bool upgradePtrauthInitFiniArrays ( Module & M )

static

Definition at line 7165 of file AutoUpgrade.cpp.

References assert(), llvm::dyn_cast(), llvm::dyn_cast_if_present(), llvm::SmallVectorImpl< T >::emplace_back(), llvm::SmallVectorTemplateCommon< T, typename >::empty(), llvm::Module::Error, llvm::ConstantArray::get(), llvm::ConstantStruct::get(), getModuleFlagNameSafely(), llvm::User::getNumOperands(), llvm::User::getOperand(), and llvm::ConstantStruct::getType().

Referenced by llvm::UpgradeModuleFlags().

upgradeRetainReleaseMarker()

bool upgradeRetainReleaseMarker ( Module & M )

static

This checks for objc retain release marker which should be upgraded.

It returns true if module is modified.

Definition at line 7010 of file AutoUpgrade.cpp.

References Changed, llvm::dyn_cast_or_null(), llvm::Module::Error, llvm::MDString::get(), llvm::NamedMDNode::getOperand(), and llvm::SmallVectorTemplateCommon< T, typename >::size().

Referenced by llvm::UpgradeARCRuntime().

upgradeSingleNVVMAnnotation()

bool upgradeSingleNVVMAnnotation ( GlobalValue * GV, StringRef K, const Metadata * V )

static

Definition at line 6908 of file AutoUpgrade.cpp.

References llvm::cast(), llvm::NVVMAttr::ClusterDim(), llvm::mdconst::extract(), llvm::Attribute::get(), llvm::Value::getContext(), llvm::Attribute::getWithStackAlignment(), llvm::NVVMAttr::GridConstant(), isXYZ(), isZero(), llvm::NVVMAttr::MaxClusterRank(), llvm::NVVMAttr::MaxNReg(), llvm::NVVMAttr::MaxNTID(), llvm::NVVMAttr::MinCTASm(), llvm::CallingConv::PTX_Kernel, llvm::NVVMAttr::ReqNTID(), uint64_t, upgradeNVVMFnVectorAttr(), llvm::utostr(), and V.

Referenced by llvm::UpgradeNVVMAnnotations().

upgradeVectorSplice()

Value * upgradeVectorSplice ( CallBase * CI, IRBuilder<> & Builder )

static

Definition at line 5663 of file AutoUpgrade.cpp.

References llvm::dyn_cast(), llvm::CallBase::getArgOperand(), llvm::Value::getType(), llvm::Offset, and llvm::reportFatalUsageError().

Referenced by llvm::UpgradeIntrinsicCall().

upgradeVPIntrinsicCall()

Value * upgradeVPIntrinsicCall ( StringRef Name, CallBase * CI, IRBuilder<> & Builder )

static

Definition at line 5733 of file AutoUpgrade.cpp.

References llvm::CallBase::args(), llvm::drop_end(), llvm::CallBase::getArgOperand(), getFunctionalIntrinsicIDForVP(), getFunctionalOpcodeForVP(), llvm::Value::getType(), getVPFPPredicateFromMD(), getVPIntPredicateFromMD(), llvm::Instruction::isBinaryOp(), llvm::Instruction::isCast(), llvm::Instruction::isUnaryOp(), llvm_unreachable, and llvm::Value::takeName().

Referenced by llvm::UpgradeIntrinsicCall().

upgradeX86ALIGNIntrinsics()

Value * upgradeX86ALIGNIntrinsics ( IRBuilder<> & Builder, Value * Op0, Value * Op1, Value * Shift, Value * Passthru, Value * Mask, bool IsVALIGN )

static

Definition at line 2670 of file AutoUpgrade.cpp.

References llvm::ArrayRef(), assert(), llvm::cast(), emitX86Select(), llvm::Constant::getNullValue(), llvm::Value::getType(), and llvm::isPowerOf2_32().

Referenced by upgradeX86IntrinsicCall().

upgradeX86BF16DPIntrinsic()

bool upgradeX86BF16DPIntrinsic ( Function * F, Intrinsic::ID IID, Function *& NewFn )

static

Definition at line 169 of file AutoUpgrade.cpp.

References F, llvm::Intrinsic::getOrInsertDeclaration(), and rename().

Referenced by upgradeX86IntrinsicFunction().

upgradeX86BF16Intrinsic()

bool upgradeX86BF16Intrinsic ( Function * F, Intrinsic::ID IID, Function *& NewFn )

static

Definition at line 159 of file AutoUpgrade.cpp.

References F, llvm::Intrinsic::getOrInsertDeclaration(), and rename().

Referenced by upgradeX86IntrinsicFunction().

upgradeX86BinaryIntrinsics()

Value * upgradeX86BinaryIntrinsics ( IRBuilder<> & Builder, CallBase & CI, Intrinsic::ID IID )

static

Definition at line 2775 of file AutoUpgrade.cpp.

References llvm::CallBase::arg_size(), emitX86Select(), llvm::User::getOperand(), and llvm::Value::getType().

Referenced by upgradeAVX512MaskToSelect(), and upgradeX86IntrinsicCall().

upgradeX86ConcatShift()

Value * upgradeX86ConcatShift ( IRBuilder<> & Builder, CallBase & CI, bool IsShiftRight, bool ZeroMask )

static

Definition at line 2855 of file AutoUpgrade.cpp.

References llvm::CallBase::arg_size(), llvm::cast(), emitX86Select(), llvm::ConstantAggregateZero::get(), llvm::CallBase::getArgOperand(), llvm::User::getOperand(), llvm::Value::getType(), and std::swap().

Referenced by upgradeX86IntrinsicCall().

upgradeX86IntrinsicCall()

Value * upgradeX86IntrinsicCall ( StringRef Name, CallBase * CI, Function * F, IRBuilder<> & Builder )

static

Definition at line 3532 of file AutoUpgrade.cpp.

References A(), AbstractManglingParser< Derived, Alloc >::Ops, llvm::Aligned, applyX86MaskOn1BitsVec(), llvm::CallBase::arg_size(), llvm::CallBase::args(), llvm::ArrayRef(), assert(), B(), C(), llvm::cast(), llvm::Data, emitX86ScalarSelect(), emitX86Select(), F, llvm::ConstantAggregateZero::get(), llvm::ConstantAsMetadata::get(), llvm::FixedVectorType::get(), llvm::MDNode::get(), llvm::PoisonValue::get(), llvm::VectorType::get(), llvm::Constant::getAllOnesValue(), llvm::CallBase::getArgOperand(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getFixedValue(), llvm::Type::getHalfTy(), llvm::Type::getInt32Ty(), llvm::Type::getInt64Ty(), llvm::VectorType::getInteger(), llvm::Instruction::getModule(), llvm::Constant::getNullValue(), llvm::User::getOperand(), llvm::Intrinsic::getOrInsertDeclaration(), llvm::Type::getPrimitiveSizeInBits(), llvm::Type::getScalarSizeInBits(), llvm::ShuffleVectorInst::getShuffleMask(), getType(), llvm::Value::getType(), getX86MaskVec(), High, I, llvm::CmpInst::ICMP_EQ, llvm::CmpInst::ICMP_NE, llvm::CmpInst::ICMP_SGT, llvm::CmpInst::ICMP_SLT, Imm, llvm::isa(), llvm::Type::isIntegerTy(), LHS, llvm_unreachable, llvm::Load, llvm::Offset, Pred, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::Value::replaceAllUsesWith(), reportFatalUsageErrorWithCI(), RHS, llvm::Instruction::setMetadata(), llvm::Size, std::swap(), uint64_t, upgradeAbs(), upgradeAVX512MaskToSelect(), upgradeMaskedCompare(), upgradeMaskedLoad(), upgradeMaskedMove(), upgradeMaskedStore(), upgradeMaskToInt(), upgradePMULDQ(), upgradeX86ALIGNIntrinsics(), upgradeX86BinaryIntrinsics(), upgradeX86ConcatShift(), upgradeX86MaskedShift(), upgradeX86PSLLDQIntrinsics(), upgradeX86PSRLDQIntrinsics(), upgradeX86Rotate(), upgradeX86vpcom(), upgradeX86VPERMT2Intrinsics(), llvm::V1, and Zero.

Referenced by llvm::UpgradeIntrinsicCall().

upgradeX86IntrinsicFunction()

bool upgradeX86IntrinsicFunction ( Function * F, StringRef Name, Function *& NewFn )

static

Definition at line 556 of file AutoUpgrade.cpp.

References llvm::StringSwitch< T, R >::Case(), llvm::StringSwitch< T, R >::Default(), F, llvm::Intrinsic::getOrInsertDeclaration(), llvm::Intrinsic::not_intrinsic, rename(), shouldUpgradeX86Intrinsic(), upgradePTESTIntrinsic(), upgradeX86BF16DPIntrinsic(), upgradeX86BF16Intrinsic(), upgradeX86IntrinsicsWith8BitMask(), upgradeX86MaskedFPCompare(), upgradeX86MultiplyAddBytes(), and upgradeX86MultiplyAddWords().

Referenced by upgradeIntrinsicFunction1().

upgradeX86IntrinsicsWith8BitMask()

bool upgradeX86IntrinsicsWith8BitMask ( Function * F, Intrinsic::ID IID, Function *& NewFn )

static

Definition at line 96 of file AutoUpgrade.cpp.

References F, llvm::Intrinsic::getOrInsertDeclaration(), llvm::Type::isIntegerTy(), and rename().

Referenced by upgradeX86IntrinsicFunction().

upgradeX86MaskedFPCompare()

bool upgradeX86MaskedFPCompare ( Function * F, Intrinsic::ID IID, Function *& NewFn )

static

Definition at line 112 of file AutoUpgrade.cpp.

References F, llvm::Intrinsic::getOrInsertDeclaration(), and rename().

Referenced by upgradeX86IntrinsicFunction().

upgradeX86MaskedShift()

Value * upgradeX86MaskedShift ( IRBuilder<> & Builder, CallBase & CI, Intrinsic::ID IID )

static

Definition at line 3021 of file AutoUpgrade.cpp.

References emitX86Select(), and llvm::CallBase::getArgOperand().

Referenced by upgradeX86IntrinsicCall().

upgradeX86MultiplyAddBytes()

bool upgradeX86MultiplyAddBytes ( Function * F, Intrinsic::ID IID, Function *& NewFn )

static

Definition at line 125 of file AutoUpgrade.cpp.

References llvm::cast(), F, llvm::Intrinsic::getOrInsertDeclaration(), llvm::Type::isVectorTy(), and rename().

Referenced by upgradeX86IntrinsicFunction().

upgradeX86MultiplyAddWords()

bool upgradeX86MultiplyAddWords ( Function * F, Intrinsic::ID IID, Function *& NewFn )

static

Definition at line 143 of file AutoUpgrade.cpp.

References llvm::cast(), F, llvm::Intrinsic::getOrInsertDeclaration(), llvm::Type::isVectorTy(), and rename().

Referenced by upgradeX86IntrinsicFunction().

upgradeX86PSLLDQIntrinsics()

Value * upgradeX86PSLLDQIntrinsics ( IRBuilder<> & Builder, Value * Op, unsigned Shift )

static

Definition at line 2555 of file AutoUpgrade.cpp.

References llvm::ArrayRef(), llvm::cast(), llvm::FixedVectorType::get(), and llvm::Constant::getNullValue().

Referenced by upgradeX86IntrinsicCall().

upgradeX86PSRLDQIntrinsics()

Value * upgradeX86PSRLDQIntrinsics ( IRBuilder<> & Builder, Value * Op, unsigned Shift )

static

Definition at line 2589 of file AutoUpgrade.cpp.

References llvm::ArrayRef(), llvm::cast(), llvm::FixedVectorType::get(), and llvm::Constant::getNullValue().

Referenced by upgradeX86IntrinsicCall().

upgradeX86Rotate()

Value * upgradeX86Rotate ( IRBuilder<> & Builder, CallBase & CI, bool IsRotateRight )

static

Definition at line 2790 of file AutoUpgrade.cpp.

References llvm::CallBase::arg_size(), llvm::cast(), emitX86Select(), llvm::CallBase::getArgOperand(), llvm::User::getOperand(), and llvm::Value::getType().

Referenced by upgradeX86IntrinsicCall().

upgradeX86vpcom()

Value * upgradeX86vpcom ( IRBuilder<> & Builder, CallBase & CI, unsigned Imm, bool IsSigned )

static

Definition at line 2816 of file AutoUpgrade.cpp.

References llvm::Constant::getAllOnesValue(), llvm::CallBase::getArgOperand(), llvm::Constant::getNullValue(), llvm::Value::getType(), llvm::CmpInst::ICMP_EQ, llvm::CmpInst::ICMP_NE, llvm::CmpInst::ICMP_SGE, llvm::CmpInst::ICMP_SGT, llvm::CmpInst::ICMP_SLE, llvm::CmpInst::ICMP_SLT, llvm::CmpInst::ICMP_UGE, llvm::CmpInst::ICMP_UGT, llvm::CmpInst::ICMP_ULE, llvm::CmpInst::ICMP_ULT, Imm, LHS, llvm_unreachable, Pred, and RHS.

Referenced by upgradeX86IntrinsicCall().

upgradeX86VPERMT2Intrinsics()

Value * upgradeX86VPERMT2Intrinsics ( IRBuilder<> & Builder, CallBase & CI, bool ZeroMask, bool IndexForm )

static

Definition at line 2715 of file AutoUpgrade.cpp.

References emitX86Select(), llvm::ConstantAggregateZero::get(), llvm::CallBase::getArgOperand(), llvm::Type::getPrimitiveSizeInBits(), llvm::Value::getType(), llvm_unreachable, std::swap(), and V.

Referenced by upgradeX86IntrinsicCall().

Variable Documentation

NVVMFAddIIDs

Intrinsic::ID NVVMFAddIIDs[2][3]

staticconstexpr

Initial value:

= {
    {Intrinsic::nvvm_fadd, Intrinsic::nvvm_fadd_sat, Intrinsic::not_intrinsic},
    {Intrinsic::nvvm_fadd_ftz, Intrinsic::nvvm_fadd_ftz_sat,
     Intrinsic::not_intrinsic}}

Definition at line 1512 of file AutoUpgrade.cpp.

Referenced by upgradeIntrinsicFunction1(), and upgradeNVVMIntrinsicCall().

NVVMFMAIIDs

Intrinsic::ID NVVMFMAIIDs[2][3]

staticconstexpr

Initial value:

= {
    {Intrinsic::nvvm_fma, Intrinsic::nvvm_fma_sat, Intrinsic::nvvm_fma_relu},
    {Intrinsic::nvvm_fma_ftz, Intrinsic::nvvm_fma_ftz_sat,
     Intrinsic::nvvm_fma_ftz_relu}}

Definition at line 1520 of file AutoUpgrade.cpp.

Referenced by isValidLegacyNVVMFPArithIntrinsic(), upgradeIntrinsicFunction1(), and upgradeNVVMIntrinsicCall().

NVVMFMulIIDs

Intrinsic::ID NVVMFMulIIDs[2][3]

staticconstexpr

Initial value:

= {
    {Intrinsic::nvvm_fmul, Intrinsic::nvvm_fmul_sat, Intrinsic::not_intrinsic},
    {Intrinsic::nvvm_fmul_ftz, Intrinsic::nvvm_fmul_ftz_sat,
     Intrinsic::not_intrinsic}}

Definition at line 1516 of file AutoUpgrade.cpp.

Referenced by isValidLegacyNVVMFPArithIntrinsic(), upgradeIntrinsicFunction1(), and upgradeNVVMIntrinsicCall().