SIInstrInfo.h

Go to the documentation of this file.

//===- SIInstrInfo.h - SI Instruction Info Interface ------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
/// \file
/// Interface definition for SIInstrInfo.
//
//===----------------------------------------------------------------------===//
 
#ifndef LLVM_LIB_TARGET_AMDGPU_SIINSTRINFO_H
#define LLVM_LIB_TARGET_AMDGPU_SIINSTRINFO_H
 
#include "AMDGPUMIRFormatter.h"
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
#include "SIRegisterInfo.h"
#include "Utils/AMDGPUBaseInfo.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
#include "llvm/CodeGen/TargetSchedule.h"
 
#define GET_INSTRINFO_HEADER
#include "AMDGPUGenInstrInfo.inc"
 
namespace llvm {
 
class APInt;
class GCNSubtarget;
class MachineDominatorTree;
class MachineRegisterInfo;
class RegScavenger;
class SIMachineFunctionInfo;
class MCRegisterClass;
using TargetRegisterClass = MCRegisterClass;
class ScheduleHazardRecognizer;
 
namespace AMDGPU {
enum class AsyncStages : uint32_t;
} // namespace AMDGPU
 
constexpr unsigned DefaultMemoryClusterDWordsLimit = 8;
 
/// Mark the MMO of a uniform load if there are no potentially clobbering stores
/// on any path from the start of an entry function to this load.
static const MachineMemOperand::Flags MONoClobber =
    MachineMemOperand::MOTargetFlag1;
 
/// Mark the MMO of a load as the last use.
static const MachineMemOperand::Flags MOLastUse =
    MachineMemOperand::MOTargetFlag2;
 
/// Mark the MMO of cooperative load/store atomics.
static const MachineMemOperand::Flags MOCooperative =
    MachineMemOperand::MOTargetFlag3;
 
struct V2PhysSCopyInfo {
  // Operands that need to replaced by waterfall
  SmallVector<MachineOperand *> MOs;
  // Target physical registers replacing the MOs
  SmallVector<Register> SGPRs;
};
/// Mark the MMO of accesses to memory locations that are
/// never written to by other threads.
static const MachineMemOperand::Flags MOThreadPrivate =
    MachineMemOperand::MOTargetFlag4;
 
/// Use two MMO flags to encode the 2-bit cache fill size field for GFX13+
static const MachineMemOperand::Flags MOCFSB0 =
    MachineMemOperand::MOTargetFlag5;
 
static const MachineMemOperand::Flags MOCFSB1 =
    MachineMemOperand::MOTargetFlag6;
 
/// Utility to store machine instructions worklist.
struct SIInstrWorklist {
  SIInstrWorklist() = default;
 
  void insert(MachineInstr *MI);
 
  MachineInstr *top() const { return InstrList[Front]; }
 
  void erase_top() {
    InSet.erase(InstrList[Front]);
    ++Front;
  }
 
  bool empty() const { return Front == InstrList.size(); }
 
  void clear() {
    InstrList.clear();
    Front = 0;
    InSet.clear();
    DeferredList.clear();
  }
 
  bool isDeferred(MachineInstr *MI);
 
  SetVector<MachineInstr *> &getDeferredList() { return DeferredList; }
 
private:
  /// InstrList contains the MachineInstrs.
  SmallVector<MachineInstr *> InstrList;
  SmallPtrSet<MachineInstr *, 8> InSet;
  unsigned Front = 0;
  /// Deferred instructions are specific MachineInstr
  /// that will be added by insert method.
  SetVector<MachineInstr *> DeferredList;
};
 
// In namespace llvm so ADL finds it when SIInstrFlags predicates are
// instantiated with MachineInstr (MachineInstr is in namespace llvm).
inline uint64_t getTSFlags(const MachineInstr &MI) {
  return MI.getDesc().TSFlags;
}
 
class SIInstrInfo final : public AMDGPUGenInstrInfo {
  struct ThreeAddressUpdates;
 
private:
  const SIRegisterInfo RI;
  const GCNSubtarget &ST;
  TargetSchedModel SchedModel;
  mutable std::unique_ptr<AMDGPUMIRFormatter> Formatter;
 
  // The inverse predicate should have the negative value.
  enum BranchPredicate {
    INVALID_BR = 0,
    SCC_TRUE = 1,
    SCC_FALSE = -1,
    VCCNZ = 2,
    VCCZ = -2,
    EXECNZ = -3,
    EXECZ = 3
  };
 
  using SetVectorType = SmallSetVector<MachineInstr *, 32>;
 
  static unsigned getBranchOpcode(BranchPredicate Cond);
  static BranchPredicate getBranchPredicate(unsigned Opcode);
 
public:
  unsigned buildExtractSubReg(MachineBasicBlock::iterator MI,
                              MachineRegisterInfo &MRI,
                              const MachineOperand &SuperReg,
                              const TargetRegisterClass *SuperRC,
                              unsigned SubIdx,
                              const TargetRegisterClass *SubRC) const;
  MachineOperand buildExtractSubRegOrImm(
      MachineBasicBlock::iterator MI, MachineRegisterInfo &MRI,
      const MachineOperand &SuperReg, const TargetRegisterClass *SuperRC,
      unsigned SubIdx, const TargetRegisterClass *SubRC) const;
 
private:
  bool optimizeSCC(MachineInstr *SCCValid, MachineInstr *SCCRedefine,
                   bool NeedInversion) const;
 
  bool invertSCCUse(MachineInstr *SCCDef) const;
 
  void swapOperands(MachineInstr &Inst) const;
 
  std::pair<bool, MachineBasicBlock *>
  moveScalarAddSub(SIInstrWorklist &Worklist, MachineInstr &Inst,
                   MachineDominatorTree *MDT = nullptr) const;
 
  void lowerSelect(SIInstrWorklist &Worklist, MachineInstr &Inst,
                   MachineDominatorTree *MDT = nullptr) const;
 
  void lowerScalarAbs(SIInstrWorklist &Worklist, MachineInstr &Inst) const;
 
  void lowerScalarAbsDiff(SIInstrWorklist &Worklist, MachineInstr &Inst) const;
 
  void lowerScalarXnor(SIInstrWorklist &Worklist, MachineInstr &Inst) const;
 
  void splitScalarNotBinop(SIInstrWorklist &Worklist, MachineInstr &Inst,
                           unsigned Opcode) const;
 
  void splitScalarBinOpN2(SIInstrWorklist &Worklist, MachineInstr &Inst,
                          unsigned Opcode) const;
 
  void splitScalar64BitUnaryOp(SIInstrWorklist &Worklist, MachineInstr &Inst,
                               unsigned Opcode, bool Swap = false) const;
 
  void splitScalar64BitBinaryOp(SIInstrWorklist &Worklist, MachineInstr &Inst,
                                unsigned Opcode,
                                MachineDominatorTree *MDT = nullptr) const;
 
  void splitScalarSMulU64(SIInstrWorklist &Worklist, MachineInstr &Inst,
                          MachineDominatorTree *MDT) const;
 
  void splitScalarSMulPseudo(SIInstrWorklist &Worklist, MachineInstr &Inst,
                             MachineDominatorTree *MDT) const;
 
  void splitScalar64BitXnor(SIInstrWorklist &Worklist, MachineInstr &Inst,
                            MachineDominatorTree *MDT = nullptr) const;
 
  void splitScalar64BitBCNT(SIInstrWorklist &Worklist,
                            MachineInstr &Inst) const;
  void splitScalar64BitBFE(SIInstrWorklist &Worklist, MachineInstr &Inst) const;
  void splitScalar64BitCountOp(SIInstrWorklist &Worklist, MachineInstr &Inst,
                               unsigned Opcode,
                               MachineDominatorTree *MDT = nullptr) const;
  void movePackToVALU(SIInstrWorklist &Worklist, MachineRegisterInfo &MRI,
                      MachineInstr &Inst) const;
 
  void addUsersToMoveToVALUWorklist(Register Reg, MachineRegisterInfo &MRI,
                                    SIInstrWorklist &Worklist) const;
 
  void addSCCDefUsersToVALUWorklist(const MachineOperand &Op,
                                    MachineInstr &SCCDefInst,
                                    SIInstrWorklist &Worklist,
                                    Register NewCond = Register()) const;
  void addSCCDefsToVALUWorklist(MachineInstr *SCCUseInst,
                                SIInstrWorklist &Worklist) const;
 
  const TargetRegisterClass *
  getDestEquivalentVGPRClass(const MachineInstr &Inst) const;
 
  bool checkInstOffsetsDoNotOverlap(const MachineInstr &MIa,
                                    const MachineInstr &MIb) const;
 
  Register findUsedSGPR(const MachineInstr &MI, int OpIndices[3]) const;
 
  bool verifyCopy(const MachineInstr &MI, const MachineRegisterInfo &MRI,
                  StringRef &ErrInfo) const;
 
  bool resultDependsOnExec(const MachineInstr &MI) const;
 
  MachineInstr *convertToThreeAddressImpl(MachineInstr &MI,
                                          ThreeAddressUpdates &Updates) const;
 
protected:
  /// If the specific machine instruction is a instruction that moves/copies
  /// value from one register to another register return destination and source
  /// registers as machine operands.
  std::optional<DestSourcePair>
  isCopyInstrImpl(const MachineInstr &MI) const override;
 
  bool swapSourceModifiers(MachineInstr &MI, MachineOperand &Src0,
                           AMDGPU::OpName Src0OpName, MachineOperand &Src1,
                           AMDGPU::OpName Src1OpName) const;
  bool isNonCommutableDPP(const MachineInstr &MI) const;
  MachineInstr *commuteInstructionImpl(MachineInstr &MI, bool NewMI,
                                       unsigned OpIdx0,
                                       unsigned OpIdx1) const override;
 
public:
  bool isLegalToSwap(const MachineInstr &MI, unsigned fromIdx,
                     unsigned toIdx) const;
  enum TargetOperandFlags {
    MO_MASK = 0xf,
 
    MO_NONE = 0,
    // MO_GOTPCREL -> symbol@GOTPCREL -> R_AMDGPU_GOTPCREL.
    MO_GOTPCREL = 1,
    // MO_GOTPCREL32_LO -> symbol@gotpcrel32@lo -> R_AMDGPU_GOTPCREL32_LO.
    MO_GOTPCREL32 = 2,
    MO_GOTPCREL32_LO = 2,
    // MO_GOTPCREL32_HI -> symbol@gotpcrel32@hi -> R_AMDGPU_GOTPCREL32_HI.
    MO_GOTPCREL32_HI = 3,
    // MO_GOTPCREL64 -> symbol@GOTPCREL -> R_AMDGPU_GOTPCREL.
    MO_GOTPCREL64 = 4,
    // MO_REL32_LO -> symbol@rel32@lo -> R_AMDGPU_REL32_LO.
    MO_REL32 = 5,
    MO_REL32_LO = 5,
    // MO_REL32_HI -> symbol@rel32@hi -> R_AMDGPU_REL32_HI.
    MO_REL32_HI = 6,
    MO_REL64 = 7,
 
    MO_FAR_BRANCH_OFFSET = 8,
 
    MO_ABS32_LO = 9,
    MO_ABS32_HI = 10,
    MO_ABS64 = 11,
  };
 
  explicit SIInstrInfo(const GCNSubtarget &ST);
 
  const SIRegisterInfo &getRegisterInfo() const {
    return RI;
  }
 
  // FIXME: This is inaccurate and needs to account for use context. Normal asm
  // constraints should use 64-bit pointers.
  const TargetRegisterClass *getInlineAsmMemoryOperandRegClass(
      InlineAsm::ConstraintCode C) const override {
    return &AMDGPU::VGPR_32RegClass;
  }
 
  const GCNSubtarget &getSubtarget() const {
    return ST;
  }
 
  bool isReMaterializableImpl(const MachineInstr &MI) const override;
 
  bool isIgnorableUse(const MachineInstr &MI, unsigned OpIdx) const override;
 
  bool isSafeToSink(MachineInstr &MI, MachineBasicBlock *SuccToSinkTo,
                    MachineCycleInfo *CI) const override;
 
  bool areLoadsFromSameBasePtr(SDNode *Load0, SDNode *Load1, int64_t &Offset0,
                               int64_t &Offset1) const override;
 
  bool isGlobalMemoryObject(const MachineInstr *MI) const override;
 
  bool getMemOperandsWithOffsetWidth(
      const MachineInstr &LdSt,
      SmallVectorImpl<const MachineOperand *> &BaseOps, int64_t &Offset,
      bool &OffsetIsScalable, LocationSize &Width) const final;
 
  bool shouldClusterMemOps(ArrayRef<const MachineOperand *> BaseOps1,
                           int64_t Offset1, bool OffsetIsScalable1,
                           ArrayRef<const MachineOperand *> BaseOps2,
                           int64_t Offset2, bool OffsetIsScalable2,
                           unsigned ClusterSize,
                           unsigned NumBytes) const override;
 
  /// Return true if the memory operations described by \p BaseOps1 and
  /// \p BaseOps2 (as returned by getMemOperandsWithOffsetWidth) address the
  /// same base. Unlike shouldClusterMemOps, this does not apply the
  /// MaxMemoryClusterDWords budget, which only bounds pre-RA register pressure.
  /// Only the first base operand is compared, unlike
  /// memOpsHaveSameBaseOperands which requires all of them to match.
  bool memOpsHaveSameBase(ArrayRef<const MachineOperand *> BaseOps1,
                          ArrayRef<const MachineOperand *> BaseOps2) const;
 
  bool shouldScheduleLoadsNear(SDNode *Load0, SDNode *Load1, int64_t Offset0,
                               int64_t Offset1, unsigned NumLoads) const override;
 
  void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
                   const DebugLoc &DL, Register DestReg, Register SrcReg,
                   bool KillSrc, bool RenamableDest = false,
                   bool RenamableSrc = false) const override;
 
private:
  void storeRegToStackSlotImpl(MachineBasicBlock &MBB,
                               MachineBasicBlock::iterator MI, Register SrcReg,
                               bool isKill, int FrameIndex,
                               const TargetRegisterClass *RC, Register VReg,
                               MachineInstr::MIFlag Flags, bool NeedsCFI) const;
 
public:
  void storeRegToStackSlotCFI(MachineBasicBlock &MBB,
                              MachineBasicBlock::iterator MI, Register SrcReg,
                              bool isKill, int FrameIndex,
                              const TargetRegisterClass *RC) const;
 
  bool getConstValDefinedInReg(const MachineInstr &MI, const Register Reg,
                               int64_t &ImmVal) const override;
 
  std::optional<int64_t>
  getImmOrMaterializedImm(const MachineRegisterInfo &MRI,
                          const MachineOperand &Op,
                          MachineInstr **DefMI = nullptr) const;
  std::optional<int64_t>
  getImmOrMaterializedImm(const MachineRegisterInfo &MRI, Register Reg,
                          MachineInstr **DefMI = nullptr) const;
 
  unsigned getVectorRegSpillSaveOpcode(Register Reg,
                                       const TargetRegisterClass *RC,
                                       unsigned Size,
                                       const SIMachineFunctionInfo &MFI,
                                       bool NeedsCFI) const;
  unsigned
  getVectorRegSpillRestoreOpcode(Register Reg, const TargetRegisterClass *RC,
                                 unsigned Size,
                                 const SIMachineFunctionInfo &MFI) const;
 
  void storeRegToStackSlot(
      MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register SrcReg,
      bool isKill, int FrameIndex, const TargetRegisterClass *RC, Register VReg,
      MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const override;
 
  void loadRegFromStackSlot(
      MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register DestReg,
      int FrameIndex, const TargetRegisterClass *RC, Register VReg,
      unsigned SubReg = 0,
      MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const override;
 
  bool expandPostRAPseudo(MachineInstr &MI) const override;
 
  void
  reMaterialize(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
                Register DestReg, unsigned SubIdx, const MachineInstr &Orig,
                LaneBitmask UsedLanes = LaneBitmask::getAll()) const override;
 
  // Splits a V_MOV_B64_DPP_PSEUDO opcode into a pair of v_mov_b32_dpp
  // instructions. Returns a pair of generated instructions.
  // Can split either post-RA with physical registers or pre-RA with
  // virtual registers. In latter case IR needs to be in SSA form and
  // and a REG_SEQUENCE is produced to define original register.
  std::pair<MachineInstr*, MachineInstr*>
  expandMovDPP64(MachineInstr &MI) const;
 
  // Returns an opcode that can be used to move a value to a \p DstRC
  // register.  If there is no hardware instruction that can store to \p
  // DstRC, then AMDGPU::COPY is returned.
  unsigned getMovOpcode(const TargetRegisterClass *DstRC) const;
 
  const MCInstrDesc &getIndirectRegWriteMovRelPseudo(unsigned VecSize,
                                                     unsigned EltSize,
                                                     bool IsSGPR) const;
 
  const MCInstrDesc &getIndirectGPRIDXPseudo(unsigned VecSize,
                                             bool IsIndirectSrc) const;
  LLVM_READONLY
  int commuteOpcode(unsigned Opc) const;
 
  LLVM_READONLY
  inline int commuteOpcode(const MachineInstr &MI) const {
    return commuteOpcode(MI.getOpcode());
  }
 
  bool findCommutedOpIndices(const MachineInstr &MI, unsigned &SrcOpIdx0,
                             unsigned &SrcOpIdx1) const override;
 
  bool findCommutedOpIndices(const MCInstrDesc &Desc, unsigned &SrcOpIdx0,
                             unsigned &SrcOpIdx1) const;
 
  bool isBranchOffsetInRange(unsigned BranchOpc,
                             int64_t BrOffset) const override;
 
  MachineBasicBlock *getBranchDestBlock(const MachineInstr &MI) const override;
 
  /// Return whether the block terminate with divergent branch.
  /// Note this only work before lowering the pseudo control flow instructions.
  bool hasDivergentBranch(const MachineBasicBlock *MBB) const;
 
  void insertIndirectBranch(MachineBasicBlock &MBB,
                            MachineBasicBlock &NewDestBB,
                            MachineBasicBlock &RestoreBB, const DebugLoc &DL,
                            int64_t BrOffset, RegScavenger *RS) const override;
 
  bool analyzeBranchImpl(MachineBasicBlock &MBB,
                         MachineBasicBlock::iterator I,
                         MachineBasicBlock *&TBB,
                         MachineBasicBlock *&FBB,
                         SmallVectorImpl<MachineOperand> &Cond,
                         bool AllowModify) const;
 
  bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB,
                     MachineBasicBlock *&FBB,
                     SmallVectorImpl<MachineOperand> &Cond,
                     bool AllowModify = false) const override;
 
  unsigned removeBranch(MachineBasicBlock &MBB,
                        int *BytesRemoved = nullptr) const override;
 
  unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
                        MachineBasicBlock *FBB, ArrayRef<MachineOperand> Cond,
                        const DebugLoc &DL,
                        int *BytesAdded = nullptr) const override;
 
  bool reverseBranchCondition(
    SmallVectorImpl<MachineOperand> &Cond) const override;
 
  std::unique_ptr<PipelinerLoopInfo>
  analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const override;
 
  bool canInsertSelect(const MachineBasicBlock &MBB,
                       ArrayRef<MachineOperand> Cond, Register DstReg,
                       Register TrueReg, Register FalseReg,
                       SelectExpansion &Exp) const override;
 
  void insertSelect(MachineBasicBlock &MBB,
                    MachineBasicBlock::iterator I, const DebugLoc &DL,
                    Register DstReg, ArrayRef<MachineOperand> Cond,
                    Register TrueReg, Register FalseReg) const override;
 
  bool analyzeCompare(const MachineInstr &MI, Register &SrcReg,
                      Register &SrcReg2, int64_t &CmpMask,
                      int64_t &CmpValue) const override;
 
  bool optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
                            Register SrcReg2, int64_t CmpMask, int64_t CmpValue,
                            const MachineRegisterInfo *MRI) const override;
 
  bool
  areMemAccessesTriviallyDisjoint(const MachineInstr &MIa,
                                  const MachineInstr &MIb) const override;
 
  static bool isFoldableCopy(const MachineInstr &MI);
  static unsigned getFoldableCopySrcIdx(const MachineInstr &MI);
 
  void removeModOperands(MachineInstr &MI) const;
 
  void mutateAndCleanupImplicit(MachineInstr &MI,
                                const MCInstrDesc &NewDesc) const;
 
  /// Return the extracted immediate value in a subregister use from a constant
  /// materialized in a super register.
  ///
  /// e.g. %imm = S_MOV_B64 K[0:63]
  ///      USE %imm.sub1
  /// This will return K[32:63]
  static std::optional<int64_t> extractSubregFromImm(int64_t ImmVal,
                                                     unsigned SubRegIndex);
 
  bool foldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, Register Reg,
                     MachineRegisterInfo *MRI) const final;
 
  unsigned getMachineCSELookAheadLimit() const override { return 500; }
 
  MachineInstr *convertToThreeAddress(MachineInstr &MI,
                                      LiveIntervals *LIS) const override;
 
  bool isSchedulingBoundary(const MachineInstr &MI,
                            const MachineBasicBlock *MBB,
                            const MachineFunction &MF) const override;
 
  static bool isSALU(const MachineInstr &MI) {
    return SIInstrFlags::isSALU(MI);
  }
 
  bool isSALU(uint32_t Opcode) const {
    return SIInstrFlags::isSALU(get(Opcode));
  }
 
  static bool isVALU(const MachineInstr &MI, bool AllowLDSDMA) {
    if (!AllowLDSDMA && isLDSDMA(MI))
      return false;
 
    return SIInstrFlags::isVALU(MI);
  }
 
  /// LDSDMA instructions act as both VALU and memory instructions, thus
  /// we also tag them as VALU. However, in many places, we do not actually want
  /// to include LDSDMA instructions in this query. By setting \p AllowLDSDMA to
  /// false, this will return false for LDSDMA instructions.
  bool isVALU(uint32_t Opcode, bool AllowLDSDMA) const {
    if (!AllowLDSDMA && isLDSDMA(Opcode))
      return false;
 
    return SIInstrFlags::isVALU(get(Opcode));
  }
 
  static bool isImage(const MachineInstr &MI) {
    return SIInstrFlags::isImage(MI);
  }
 
  bool isImage(uint32_t Opcode) const {
    return SIInstrFlags::isImage(get(Opcode));
  }
 
  static bool isVMEM(const MachineInstr &MI) {
    return SIInstrFlags::isVMEM(MI);
  }
 
  bool isVMEM(uint32_t Opcode) const {
    return SIInstrFlags::isVMEM(get(Opcode));
  }
 
  /// True if MI implicitly drains XCNT.
  static bool isXcntDrain(const MachineInstr &MI);
 
  static bool isSOP1(const MachineInstr &MI) {
    return SIInstrFlags::isSOP1(MI);
  }
 
  bool isSOP1(uint32_t Opcode) const {
    return SIInstrFlags::isSOP1(get(Opcode));
  }
 
  static bool isSOP2(const MachineInstr &MI) {
    return SIInstrFlags::isSOP2(MI);
  }
 
  bool isSOP2(uint32_t Opcode) const {
    return SIInstrFlags::isSOP2(get(Opcode));
  }
 
  static bool isSOPC(const MachineInstr &MI) {
    return SIInstrFlags::isSOPC(MI);
  }
 
  bool isSOPC(uint32_t Opcode) const {
    return SIInstrFlags::isSOPC(get(Opcode));
  }
 
  static bool isSOPK(const MachineInstr &MI) {
    return SIInstrFlags::isSOPK(MI);
  }
 
  bool isSOPK(uint32_t Opcode) const {
    return SIInstrFlags::isSOPK(get(Opcode));
  }
 
  static bool isSOPP(const MachineInstr &MI) {
    return SIInstrFlags::isSOPP(MI);
  }
 
  bool isSOPP(uint32_t Opcode) const {
    return SIInstrFlags::isSOPP(get(Opcode));
  }
 
  static bool isPacked(const MachineInstr &MI) {
    return SIInstrFlags::isPacked(MI);
  }
 
  bool isPacked(uint32_t Opcode) const {
    return SIInstrFlags::isPacked(get(Opcode));
  }
 
  static bool isVOP1(const MachineInstr &MI) {
    return SIInstrFlags::isVOP1(MI);
  }
 
  bool isVOP1(uint32_t Opcode) const {
    return SIInstrFlags::isVOP1(get(Opcode));
  }
 
  static bool isVOP2(const MachineInstr &MI) {
    return SIInstrFlags::isVOP2(MI);
  }
 
  bool isVOP2(uint32_t Opcode) const {
    return SIInstrFlags::isVOP2(get(Opcode));
  }
 
  static bool isVOP3(const MCInstrDesc &Desc) {
    return SIInstrFlags::isVOP3(Desc);
  }
 
  static bool isVOP3(const MachineInstr &MI) { return isVOP3(MI.getDesc()); }
 
  bool isVOP3(uint32_t Opcode) const { return isVOP3(get(Opcode)); }
 
  static bool isSDWA(const MachineInstr &MI) {
    return SIInstrFlags::isSDWA(MI);
  }
 
  bool isSDWA(uint32_t Opcode) const {
    return SIInstrFlags::isSDWA(get(Opcode));
  }
 
  static bool isVOPC(const MachineInstr &MI) {
    return SIInstrFlags::isVOPC(MI);
  }
 
  bool isVOPC(uint32_t Opcode) const {
    return SIInstrFlags::isVOPC(get(Opcode));
  }
 
  static bool isMUBUF(const MachineInstr &MI) {
    return SIInstrFlags::isMUBUF(MI);
  }
 
  bool isMUBUF(uint32_t Opcode) const {
    return SIInstrFlags::isMUBUF(get(Opcode));
  }
 
  static bool isMTBUF(const MachineInstr &MI) {
    return SIInstrFlags::isMTBUF(MI);
  }
 
  bool isMTBUF(uint32_t Opcode) const {
    return SIInstrFlags::isMTBUF(get(Opcode));
  }
 
  static bool isBUF(const MachineInstr &MI) {
    return isMUBUF(MI) || isMTBUF(MI);
  }
 
  static bool isSMRD(const MachineInstr &MI) {
    return SIInstrFlags::isSMRD(MI);
  }
 
  bool isSMRD(uint32_t Opcode) const {
    return SIInstrFlags::isSMRD(get(Opcode));
  }
 
  bool isBufferSMRD(const MachineInstr &MI) const;
 
  static bool isDS(const MachineInstr &MI) { return SIInstrFlags::isDS(MI); }
 
  bool isDS(uint32_t Opcode) const { return SIInstrFlags::isDS(get(Opcode)); }
 
  static bool isLDSDMA(const MachineInstr &MI) {
    return (SIInstrFlags::isVALU(MI) && (isMUBUF(MI) || isFLAT(MI))) ||
           SIInstrFlags::usesTENSOR_CNT(MI);
  }
 
  bool isLDSDMA(uint32_t Opcode) const {
    return (SIInstrFlags::isVALU(get(Opcode)) &&
            (isMUBUF(Opcode) || isFLAT(Opcode))) ||
           SIInstrFlags::usesTENSOR_CNT(get(Opcode));
  }
 
  static bool isGWS(const MachineInstr &MI) { return SIInstrFlags::isGWS(MI); }
 
  bool isGWS(uint32_t Opcode) const { return SIInstrFlags::isGWS(get(Opcode)); }
 
  bool isAlwaysGDS(uint32_t Opcode) const;
 
  static bool isMIMG(const MachineInstr &MI) {
    return SIInstrFlags::isMIMG(MI);
  }
 
  bool isMIMG(uint32_t Opcode) const {
    return SIInstrFlags::isMIMG(get(Opcode));
  }
 
  static bool isVIMAGE(const MachineInstr &MI) {
    return SIInstrFlags::isVIMAGE(MI);
  }
 
  bool isVIMAGE(uint32_t Opcode) const {
    return SIInstrFlags::isVIMAGE(get(Opcode));
  }
 
  static bool isVSAMPLE(const MachineInstr &MI) {
    return SIInstrFlags::isVSAMPLE(MI);
  }
 
  bool isVSAMPLE(uint32_t Opcode) const {
    return SIInstrFlags::isVSAMPLE(get(Opcode));
  }
 
  static bool isGather4(const MachineInstr &MI) {
    return SIInstrFlags::isGather4(MI);
  }
 
  bool isGather4(uint32_t Opcode) const {
    return SIInstrFlags::isGather4(get(Opcode));
  }
 
  static bool isFLAT(const MachineInstr &MI) {
    return SIInstrFlags::isFLAT(MI);
  }
 
  // Is a FLAT encoded instruction which accesses a specific segment,
  // i.e. global_* or scratch_*.
  static bool isSegmentSpecificFLAT(const MachineInstr &MI) {
    return SIInstrFlags::isSegmentSpecificFLAT(MI);
  }
 
  bool isSegmentSpecificFLAT(uint32_t Opcode) const {
    return SIInstrFlags::isSegmentSpecificFLAT(get(Opcode));
  }
 
  static bool isFLATGlobal(const MachineInstr &MI) {
    return SIInstrFlags::isFlatGlobal(MI);
  }
 
  bool isFLATGlobal(uint32_t Opcode) const {
    return SIInstrFlags::isFlatGlobal(get(Opcode));
  }
 
  static bool isFLATScratch(const MachineInstr &MI) {
    return SIInstrFlags::isFlatScratch(MI);
  }
 
  bool isFLATScratch(uint32_t Opcode) const {
    return SIInstrFlags::isFlatScratch(get(Opcode));
  }
 
  // Any FLAT encoded instruction, including global_* and scratch_*.
  bool isFLAT(uint32_t Opcode) const {
    return SIInstrFlags::isFLAT(get(Opcode));
  }
 
  /// \returns true for SCRATCH_ instructions, or FLAT/BUF instructions unless
  /// the MMOs do not include scratch.
  /// Conservatively correct; will return true if \p MI cannot be proven
  /// to not hit scratch.
  bool mayAccessScratch(const MachineInstr &MI) const;
 
  /// \returns true for FLAT instructions that can access VMEM.
  bool mayAccessVMEMThroughFlat(const MachineInstr &MI) const;
 
  /// \returns true for FLAT instructions that can access LDS.
  bool mayAccessLDSThroughFlat(const MachineInstr &MI, bool TgSplit) const;
 
  static bool isBlockLoadStore(uint32_t Opcode) {
    switch (Opcode) {
    case AMDGPU::SI_BLOCK_SPILL_V1024_SAVE:
    case AMDGPU::SI_BLOCK_SPILL_V1024_CFI_SAVE:
    case AMDGPU::SI_BLOCK_SPILL_V1024_RESTORE:
    case AMDGPU::SCRATCH_STORE_BLOCK_SADDR:
    case AMDGPU::SCRATCH_LOAD_BLOCK_SADDR:
    case AMDGPU::SCRATCH_STORE_BLOCK_SVS:
    case AMDGPU::SCRATCH_LOAD_BLOCK_SVS:
      return true;
    default:
      return false;
    }
  }
 
  static bool setsSCCIfResultIsNonZero(const MachineInstr &MI) {
    switch (MI.getOpcode()) {
    case AMDGPU::S_ABSDIFF_I32:
    case AMDGPU::S_ABS_I32:
    case AMDGPU::S_AND_B32:
    case AMDGPU::S_AND_B64:
    case AMDGPU::S_ANDN2_B32:
    case AMDGPU::S_ANDN2_B64:
    case AMDGPU::S_ASHR_I32:
    case AMDGPU::S_ASHR_I64:
    case AMDGPU::S_BCNT0_I32_B32:
    case AMDGPU::S_BCNT0_I32_B64:
    case AMDGPU::S_BCNT1_I32_B32:
    case AMDGPU::S_BCNT1_I32_B64:
    case AMDGPU::S_BFE_I32:
    case AMDGPU::S_BFE_I64:
    case AMDGPU::S_BFE_U32:
    case AMDGPU::S_BFE_U64:
    case AMDGPU::S_LSHL_B32:
    case AMDGPU::S_LSHL_B64:
    case AMDGPU::S_LSHR_B32:
    case AMDGPU::S_LSHR_B64:
    case AMDGPU::S_NAND_B32:
    case AMDGPU::S_NAND_B64:
    case AMDGPU::S_NOR_B32:
    case AMDGPU::S_NOR_B64:
    case AMDGPU::S_NOT_B32:
    case AMDGPU::S_NOT_B64:
    case AMDGPU::S_OR_B32:
    case AMDGPU::S_OR_B64:
    case AMDGPU::S_ORN2_B32:
    case AMDGPU::S_ORN2_B64:
    case AMDGPU::S_QUADMASK_B32:
    case AMDGPU::S_QUADMASK_B64:
    case AMDGPU::S_WQM_B32:
    case AMDGPU::S_WQM_B64:
    case AMDGPU::S_XNOR_B32:
    case AMDGPU::S_XNOR_B64:
    case AMDGPU::S_XOR_B32:
    case AMDGPU::S_XOR_B64:
      return true;
    default:
      return false;
    }
  }
 
  static bool isEXP(const MachineInstr &MI) { return SIInstrFlags::isEXP(MI); }
 
  static bool isDualSourceBlendEXP(const MachineInstr &MI) {
    if (!isEXP(MI))
      return false;
    unsigned Target = MI.getOperand(0).getImm();
    return Target == AMDGPU::Exp::ET_DUAL_SRC_BLEND0 ||
           Target == AMDGPU::Exp::ET_DUAL_SRC_BLEND1;
  }
 
  bool isEXP(uint32_t Opcode) const { return SIInstrFlags::isEXP(get(Opcode)); }
 
  static bool isAtomicNoRet(const MachineInstr &MI) {
    return SIInstrFlags::isAtomicNoRet(MI);
  }
 
  bool isAtomicNoRet(uint32_t Opcode) const {
    return SIInstrFlags::isAtomicNoRet(get(Opcode));
  }
 
  static bool isAtomicRet(const MachineInstr &MI) {
    return SIInstrFlags::isAtomicRet(MI);
  }
 
  bool isAtomicRet(uint32_t Opcode) const {
    return SIInstrFlags::isAtomicRet(get(Opcode));
  }
 
  static bool isAtomic(const MachineInstr &MI) {
    return SIInstrFlags::isAtomic(MI);
  }
 
  bool isAtomic(uint32_t Opcode) const {
    return SIInstrFlags::isAtomic(get(Opcode));
  }
 
  static bool mayWriteLDSThroughDMA(const MachineInstr &MI) {
    unsigned Opc = MI.getOpcode();
    // Exclude instructions that read FROM LDS (not write to it)
    return isLDSDMA(MI) && Opc != AMDGPU::BUFFER_STORE_LDS_DWORD &&
           Opc != AMDGPU::TENSOR_STORE_FROM_LDS_d2 &&
           Opc != AMDGPU::TENSOR_STORE_FROM_LDS_d4;
  }
 
  bool isNonAsyncLdsDmaWrite(const MachineInstr &MI) const {
    return mayWriteLDSThroughDMA(MI) && !isAsync(MI);
  }
 
  /// \returns true if \p MI creates an async operation.
  bool isAsync(const MachineInstr &MI) const {
    if (!isLDSDMA(MI))
      return false;
    if (usesASYNC_CNT(MI) || usesTENSOR_CNT(MI))
      return true;
    const MachineOperand *Async = getNamedOperand(MI, AMDGPU::OpName::IsAsync);
    return Async && (Async->getImm());
  }
 
  /// \returns the asyncmark stage that the async operation initiated by \p MI
  /// belongs to, or no stage if \p MI does not initiate an async operation.
  AMDGPU::AsyncStages getAsyncStage(const MachineInstr &MI) const;
 
  static bool isSBarrierSCCWrite(unsigned Opcode) {
    return Opcode == AMDGPU::S_BARRIER_LEAVE ||
           Opcode == AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM ||
           Opcode == AMDGPU::S_BARRIER_SIGNAL_ISFIRST_M0;
  }
 
  static bool isCBranchVCCZRead(const MachineInstr &MI) {
    unsigned Opc = MI.getOpcode();
    return (Opc == AMDGPU::S_CBRANCH_VCCNZ || Opc == AMDGPU::S_CBRANCH_VCCZ) &&
           !MI.getOperand(1).isUndef();
  }
 
  static bool isWQM(const MachineInstr &MI) { return SIInstrFlags::isWQM(MI); }
 
  bool isWQM(uint32_t Opcode) const { return SIInstrFlags::isWQM(get(Opcode)); }
 
  static bool isDisableWQM(const MachineInstr &MI) {
    return SIInstrFlags::isDisableWQM(MI);
  }
 
  bool isDisableWQM(uint32_t Opcode) const {
    return SIInstrFlags::isDisableWQM(get(Opcode));
  }
 
  // SI_SPILL_S32_TO_VGPR and SI_RESTORE_S32_FROM_VGPR form a special case of
  // SGPRs spilling to VGPRs which are SGPR spills but from VALU instructions
  // therefore we need an explicit check for them since just checking if the
  // Spill bit is set and what instruction type it came from misclassifies
  // them.
  static bool isVGPRSpill(const MachineInstr &MI) {
    return MI.getOpcode() != AMDGPU::SI_SPILL_S32_TO_VGPR &&
           MI.getOpcode() != AMDGPU::SI_RESTORE_S32_FROM_VGPR &&
           (isSpill(MI) && isVALU(MI, /*AllowLDSDMA=*/false));
  }
 
  bool isVGPRSpill(uint32_t Opcode) const {
    return Opcode != AMDGPU::SI_SPILL_S32_TO_VGPR &&
           Opcode != AMDGPU::SI_RESTORE_S32_FROM_VGPR &&
           (isSpill(Opcode) && isVALU(Opcode, /*AllowLDSDMA=*/false));
  }
 
  static bool isSGPRSpill(const MachineInstr &MI) {
    return MI.getOpcode() == AMDGPU::SI_SPILL_S32_TO_VGPR ||
           MI.getOpcode() == AMDGPU::SI_RESTORE_S32_FROM_VGPR ||
           (isSpill(MI) && isSALU(MI));
  }
 
  bool isSGPRSpill(uint32_t Opcode) const {
    return Opcode == AMDGPU::SI_SPILL_S32_TO_VGPR ||
           Opcode == AMDGPU::SI_RESTORE_S32_FROM_VGPR ||
           (isSpill(Opcode) && isSALU(Opcode));
  }
 
  bool isSpill(uint32_t Opcode) const {
    return SIInstrFlags::isSpill(get(Opcode));
  }
 
  static bool isSpill(const MCInstrDesc &Desc) {
    return SIInstrFlags::isSpill(Desc);
  }
 
  static bool isSpill(const MachineInstr &MI) { return isSpill(MI.getDesc()); }
 
  static bool isWWMRegSpillOpcode(uint32_t Opcode) {
    return Opcode == AMDGPU::SI_SPILL_WWM_V32_SAVE ||
           Opcode == AMDGPU::SI_SPILL_WWM_AV32_SAVE ||
           Opcode == AMDGPU::SI_SPILL_WWM_V32_RESTORE ||
           Opcode == AMDGPU::SI_SPILL_WWM_AV32_RESTORE;
  }
 
  static bool isChainCallOpcode(uint64_t Opcode) {
    return Opcode == AMDGPU::SI_CS_CHAIN_TC_W32 ||
           Opcode == AMDGPU::SI_CS_CHAIN_TC_W64;
  }
 
  static bool isDPP(const MachineInstr &MI) { return SIInstrFlags::isDPP(MI); }
 
  bool isDPP(uint32_t Opcode) const { return SIInstrFlags::isDPP(get(Opcode)); }
 
  // Some opcodes use Src1 for DPP instead of Src0, because the sequencer
  // transforms them and reverse the order of their operands at runtime.
  //
  // Documentation is incomplete on which instructions are effected, so
  // the implementation is derived from experimentation.
  //
  // Listed as target-independent pseudos; the per-subtarget MC opcodes
  // (V_SUBREV_NC_U32_e32_gfx11 and friends) are all reached through these.
  // Defined out of line because GCNSubtarget is incomplete here.
  static bool isSrc1DPPRevOpcode(const GCNSubtarget &ST, uint32_t Opcode);
 
  static bool isTRANS(const MachineInstr &MI) {
    return SIInstrFlags::isTRANS(MI);
  }
 
  bool isTRANS(uint32_t Opcode) const {
    return SIInstrFlags::isTRANS(get(Opcode));
  }
 
  static bool isVOP3P(const MachineInstr &MI) {
    return SIInstrFlags::isVOP3P(MI);
  }
 
  bool isVOP3P(uint32_t Opcode) const {
    return SIInstrFlags::isVOP3P(get(Opcode));
  }
 
  bool isVOP3PMix(const MachineInstr &MI) const {
    return isVOP3PMix(MI.getOpcode());
  }
 
  bool isVOP3PMix(uint16_t Opcode) const {
    switch (Opcode) {
    case AMDGPU::V_FMA_MIXHI_F16:
    case AMDGPU::V_FMA_MIXLO_F16:
    case AMDGPU::V_FMA_MIX_F32:
    case AMDGPU::V_MAD_MIXHI_F16:
    case AMDGPU::V_MAD_MIXLO_F16:
    case AMDGPU::V_MAD_MIX_F32:
      return true;
    default:
      return false;
    }
  }
 
  static bool isVINTRP(const MachineInstr &MI) {
    return SIInstrFlags::isVINTRP(MI);
  }
 
  bool isVINTRP(uint32_t Opcode) const {
    return SIInstrFlags::isVINTRP(get(Opcode));
  }
 
  static bool isMAI(const MCInstrDesc &Desc) {
    return SIInstrFlags::isMAI(Desc);
  }
 
  static bool isMAI(const MachineInstr &MI) { return isMAI(MI.getDesc()); }
 
  bool isMAI(uint32_t Opcode) const { return isMAI(get(Opcode)); }
 
  static bool isMFMA(const MachineInstr &MI) {
    return isMAI(MI) && MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 &&
           MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64;
  }
 
  bool isMFMA(uint32_t Opcode) const {
    return isMAI(Opcode) && Opcode != AMDGPU::V_ACCVGPR_WRITE_B32_e64 &&
           Opcode != AMDGPU::V_ACCVGPR_READ_B32_e64;
  }
 
  static bool isDOT(const MachineInstr &MI) { return SIInstrFlags::isDOT(MI); }
 
  static bool isWMMA(const MachineInstr &MI) {
    return SIInstrFlags::isWMMA(MI);
  }
 
  bool isWMMA(uint32_t Opcode) const {
    return SIInstrFlags::isWMMA(get(Opcode));
  }
 
  static bool isMFMAorWMMA(const MachineInstr &MI) {
    return isMFMA(MI) || isWMMA(MI) || isSWMMAC(MI);
  }
 
  bool isMFMAorWMMA(uint32_t Opcode) const {
    return isMFMA(Opcode) || isWMMA(Opcode) || isSWMMAC(Opcode);
  }
 
  static bool isSWMMAC(const MachineInstr &MI) {
    return SIInstrFlags::isSWMMAC(MI);
  }
 
  bool isSWMMAC(uint32_t Opcode) const {
    return SIInstrFlags::isSWMMAC(get(Opcode));
  }
 
  bool isDOT(uint32_t Opcode) const { return SIInstrFlags::isDOT(get(Opcode)); }
 
  bool isXDLWMMA(const MachineInstr &MI) const;
 
  bool isXDL(const MachineInstr &MI) const;
 
  static bool isDGEMM(unsigned Opcode) { return AMDGPU::getMAIIsDGEMM(Opcode); }
 
  static bool isLDSDIR(const MachineInstr &MI) {
    return SIInstrFlags::isLDSDIR(MI);
  }
 
  bool isLDSDIR(uint32_t Opcode) const {
    return SIInstrFlags::isLDSDIR(get(Opcode));
  }
 
  static bool isVINTERP(const MachineInstr &MI) {
    return SIInstrFlags::isVINTERP(MI);
  }
 
  bool isVINTERP(uint32_t Opcode) const {
    return SIInstrFlags::isVINTERP(get(Opcode));
  }
 
  static bool usesVM_CNT(const MachineInstr &MI) {
    return SIInstrFlags::usesVM_CNT(MI);
  }
 
  static bool usesLGKM_CNT(const MachineInstr &MI) {
    return SIInstrFlags::usesLGKM_CNT(MI);
  }
 
  static bool usesASYNC_CNT(const MachineInstr &MI) {
    return SIInstrFlags::usesASYNC_CNT(MI);
  }
 
  bool usesASYNC_CNT(uint32_t Opcode) const {
    return SIInstrFlags::usesASYNC_CNT(get(Opcode));
  }
 
  static bool usesTENSOR_CNT(const MachineInstr &MI) {
    return SIInstrFlags::usesTENSOR_CNT(MI);
  }
 
  bool usesTENSOR_CNT(uint32_t Opcode) const {
    return SIInstrFlags::usesTENSOR_CNT(get(Opcode));
  }
 
  // Most sopk treat the immediate as a signed 16-bit, however some
  // use it as unsigned.
  static bool sopkIsZext(unsigned Opcode) {
    return Opcode == AMDGPU::S_CMPK_EQ_U32 || Opcode == AMDGPU::S_CMPK_LG_U32 ||
           Opcode == AMDGPU::S_CMPK_GT_U32 || Opcode == AMDGPU::S_CMPK_GE_U32 ||
           Opcode == AMDGPU::S_CMPK_LT_U32 || Opcode == AMDGPU::S_CMPK_LE_U32 ||
           Opcode == AMDGPU::S_GETREG_B32 ||
           Opcode == AMDGPU::S_GETREG_B32_const;
  }
 
  /// \returns true if this is an s_store_dword* instruction. This is more
  /// specific than isSMEM && mayStore.
  static bool isScalarStore(const MachineInstr &MI) {
    return SIInstrFlags::isScalarStore(MI);
  }
 
  bool isScalarStore(uint32_t Opcode) const {
    return SIInstrFlags::isScalarStore(get(Opcode));
  }
 
  static bool isFixedSize(const MachineInstr &MI) {
    return SIInstrFlags::isFixedSize(MI);
  }
 
  bool isFixedSize(uint32_t Opcode) const {
    return SIInstrFlags::isFixedSize(get(Opcode));
  }
 
  static bool hasFPClamp(const MachineInstr &MI) {
    return SIInstrFlags::hasFPClamp(MI);
  }
 
  bool hasFPClamp(uint32_t Opcode) const {
    return SIInstrFlags::hasFPClamp(get(Opcode));
  }
 
  static bool hasIntClamp(const MachineInstr &MI) {
    return SIInstrFlags::hasIntClamp(MI);
  }
 
  static bool hasSameClamp(const MachineInstr &A, const MachineInstr &B) {
    const MCInstrDesc &DA = A.getDesc(), &DB = B.getDesc();
    return SIInstrFlags::hasFPClamp(DA) == SIInstrFlags::hasFPClamp(DB) &&
           SIInstrFlags::hasIntClamp(DA) == SIInstrFlags::hasIntClamp(DB) &&
           SIInstrFlags::hasClampLo(DA) == SIInstrFlags::hasClampLo(DB) &&
           SIInstrFlags::hasClampHi(DA) == SIInstrFlags::hasClampHi(DB);
  }
 
  static bool usesFPDPRounding(const MachineInstr &MI) {
    return SIInstrFlags::usesFPDPRounding(MI);
  }
 
  bool usesFPDPRounding(uint32_t Opcode) const {
    return SIInstrFlags::usesFPDPRounding(get(Opcode));
  }
 
  static bool isFPAtomic(const MachineInstr &MI) {
    return SIInstrFlags::isFPAtomic(MI);
  }
 
  bool isFPAtomic(uint32_t Opcode) const {
    return SIInstrFlags::isFPAtomic(get(Opcode));
  }
 
  static bool isNeverUniform(const MachineInstr &MI) {
    return SIInstrFlags::isNeverUniform(MI);
  }
 
  // Check to see if opcode is for a barrier start. Pre gfx12 this is just the
  // S_BARRIER, but after support for S_BARRIER_SIGNAL* / S_BARRIER_WAIT we want
  // to check for the barrier start (S_BARRIER_SIGNAL*)
  bool isBarrierStart(unsigned Opcode) const {
    return Opcode == AMDGPU::S_BARRIER ||
           Opcode == AMDGPU::S_BARRIER_SIGNAL_M0 ||
           Opcode == AMDGPU::S_BARRIER_SIGNAL_ISFIRST_M0 ||
           Opcode == AMDGPU::S_BARRIER_SIGNAL_IMM ||
           Opcode == AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM;
  }
 
  bool isBarrier(unsigned Opcode) const {
    return isBarrierStart(Opcode) || Opcode == AMDGPU::S_BARRIER_WAIT ||
           Opcode == AMDGPU::S_BARRIER_INIT_M0 ||
           Opcode == AMDGPU::S_BARRIER_INIT_IMM ||
           Opcode == AMDGPU::S_BARRIER_JOIN_IMM ||
           Opcode == AMDGPU::S_BARRIER_LEAVE || Opcode == AMDGPU::DS_GWS_INIT ||
           Opcode == AMDGPU::DS_GWS_BARRIER;
  }
 
  static bool isLoadMonitor(unsigned Opc) {
    switch (Opc) {
    case AMDGPU::GLOBAL_LOAD_MONITOR_B32:
    case AMDGPU::GLOBAL_LOAD_MONITOR_B32_SADDR:
    case AMDGPU::GLOBAL_LOAD_MONITOR_B64:
    case AMDGPU::GLOBAL_LOAD_MONITOR_B64_SADDR:
    case AMDGPU::GLOBAL_LOAD_MONITOR_B128:
    case AMDGPU::GLOBAL_LOAD_MONITOR_B128_SADDR:
    case AMDGPU::FLAT_LOAD_MONITOR_B32:
    case AMDGPU::FLAT_LOAD_MONITOR_B64:
    case AMDGPU::FLAT_LOAD_MONITOR_B128:
      return true;
    default:
      return false;
    }
  }
 
  static bool isGFX12CacheInvOrWBInst(unsigned Opc) {
    return Opc == AMDGPU::GLOBAL_INV || Opc == AMDGPU::GLOBAL_WB ||
           Opc == AMDGPU::GLOBAL_WBINV;
  }
 
  static bool isF16PseudoScalarTrans(unsigned Opcode) {
    return Opcode == AMDGPU::V_S_EXP_F16_e64 ||
           Opcode == AMDGPU::V_S_LOG_F16_e64 ||
           Opcode == AMDGPU::V_S_RCP_F16_e64 ||
           Opcode == AMDGPU::V_S_RSQ_F16_e64 ||
           Opcode == AMDGPU::V_S_SQRT_F16_e64;
  }
 
  static bool isPseudoScalarTrans(unsigned Opcode) {
    return isF16PseudoScalarTrans(Opcode) ||
           Opcode == AMDGPU::V_S_EXP_F32_e64 ||
           Opcode == AMDGPU::V_S_LOG_F32_e64 ||
           Opcode == AMDGPU::V_S_RCP_F32_e64 ||
           Opcode == AMDGPU::V_S_RSQ_F32_e64 ||
           Opcode == AMDGPU::V_S_SQRT_F32_e64;
  }
 
  static bool isVPermPk16(unsigned Opcode) {
    return Opcode == AMDGPU::V_PERM_PK16_B4_U4_e64 ||
           Opcode == AMDGPU::V_PERM_PK16_B6_U4_e64 ||
           Opcode == AMDGPU::V_PERM_PK16_B8_U4_e64;
  }
 
  // \returns true if \p MI clears the V_PERM_PK16 hazard when it immediately
  // follows a V_PERM_PK16 (i.e. \p MI is a "safe" instruction).
  bool isVPermPk16SafeInstr(const MachineInstr &MI) const {
    unsigned Opc = MI.getOpcode();
 
    // Only VALU ops issue on the pipe that clears the V_PERM_PK16 hazard.
    if (!isVALU(MI, /*AllowLDSDMA=*/false))
      return false;
    // OP_XDL: matrix (WMMA/SWMMAC/DOT) ops clear the hazard.
    if (isXDL(MI))
      return true;
    // Pseudo-scalar transcendentals (OP32_SCL_T) do NOT clear the hazard.
    if (isPseudoScalarTrans(Opc))
      return false;
 
    // Use the table lookup, not getBlockingCycles(): occupancy is gated off on
    // gfx1251 but the hazard applies to both gfx1250 and gfx1251. Gfx1250 table
    // is valid enough for gfx1251 w.r.t. v_perm_pk16 safety check here.
    // OP_32_T is in the table at 2 and is safe; other table entries (>= 2) are
    // not.
    unsigned Cycles = getGFX1250BlockingCyclesTable(MI);
    return Cycles < 2 || (Cycles == 2 && isTRANS(MI));
  }
 
  static bool doesNotReadTiedSource(const MachineInstr &MI) {
    return SIInstrFlags::isTiedSourceNotRead(MI);
  }
 
  bool doesNotReadTiedSource(uint32_t Opcode) const {
    return SIInstrFlags::isTiedSourceNotRead(get(Opcode));
  }
 
  bool isIGLP(unsigned Opcode) const {
    return Opcode == AMDGPU::SCHED_BARRIER ||
           Opcode == AMDGPU::SCHED_GROUP_BARRIER || Opcode == AMDGPU::IGLP_OPT;
  }
 
  bool isIGLP(const MachineInstr &MI) const { return isIGLP(MI.getOpcode()); }
 
  // Return true if the instruction is mutually exclusive with all non-IGLP DAG
  // mutations, requiring all other mutations to be disabled.
  bool isIGLPMutationOnly(unsigned Opcode) const {
    return Opcode == AMDGPU::SCHED_GROUP_BARRIER || Opcode == AMDGPU::IGLP_OPT;
  }
 
  static unsigned getNonSoftWaitcntOpcode(unsigned Opcode) {
    switch (Opcode) {
    case AMDGPU::S_WAITCNT_soft:
      return AMDGPU::S_WAITCNT;
    case AMDGPU::S_WAITCNT_VSCNT_soft:
      return AMDGPU::S_WAITCNT_VSCNT;
    case AMDGPU::S_WAIT_LOADCNT_soft:
      return AMDGPU::S_WAIT_LOADCNT;
    case AMDGPU::S_WAIT_STORECNT_soft:
      return AMDGPU::S_WAIT_STORECNT;
    case AMDGPU::S_WAIT_SAMPLECNT_soft:
      return AMDGPU::S_WAIT_SAMPLECNT;
    case AMDGPU::S_WAIT_BVHCNT_soft:
      return AMDGPU::S_WAIT_BVHCNT;
    case AMDGPU::S_WAIT_DSCNT_soft:
      return AMDGPU::S_WAIT_DSCNT;
    case AMDGPU::S_WAIT_KMCNT_soft:
      return AMDGPU::S_WAIT_KMCNT;
    case AMDGPU::S_WAIT_XCNT_soft:
      return AMDGPU::S_WAIT_XCNT;
    default:
      return Opcode;
    }
  }
 
  static bool isWaitcnt(unsigned Opcode) {
    switch (getNonSoftWaitcntOpcode(Opcode)) {
    case AMDGPU::S_WAITCNT:
    case AMDGPU::S_WAITCNT_VSCNT:
    case AMDGPU::S_WAITCNT_VMCNT:
    case AMDGPU::S_WAITCNT_EXPCNT:
    case AMDGPU::S_WAITCNT_LGKMCNT:
    case AMDGPU::S_WAIT_LOADCNT:
    case AMDGPU::S_WAIT_LOADCNT_DSCNT:
    case AMDGPU::S_WAIT_STORECNT:
    case AMDGPU::S_WAIT_STORECNT_DSCNT:
    case AMDGPU::S_WAIT_SAMPLECNT:
    case AMDGPU::S_WAIT_BVHCNT:
    case AMDGPU::S_WAIT_EXPCNT:
    case AMDGPU::S_WAIT_DSCNT:
    case AMDGPU::S_WAIT_KMCNT:
    case AMDGPU::S_WAIT_XCNT:
    case AMDGPU::S_WAIT_ASYNCCNT:
    case AMDGPU::S_WAIT_TENSORCNT:
    case AMDGPU::S_WAIT_IDLE:
      return true;
    default:
      return false;
    }
  }
 
  bool isVGPRCopy(const MachineInstr &MI) const {
    assert(isCopyInstr(MI));
    Register Dest = MI.getOperand(0).getReg();
    const MachineFunction &MF = *MI.getMF();
    const MachineRegisterInfo &MRI = MF.getRegInfo();
    return !RI.isSGPRReg(MRI, Dest);
  }
 
  bool hasVGPRUses(const MachineInstr &MI) const {
    const MachineFunction &MF = *MI.getMF();
    const MachineRegisterInfo &MRI = MF.getRegInfo();
    return llvm::any_of(MI.explicit_uses(),
                        [&MRI, this](const MachineOperand &MO) {
      return MO.isReg() && RI.isVGPR(MRI, MO.getReg());});
  }
 
  /// Return true if the instruction modifies the mode register.q
  static bool modifiesModeRegister(const MachineInstr &MI);
 
  /// This function is used to determine if an instruction can be safely
  /// executed under EXEC = 0 without hardware error, indeterminate results,
  /// and/or visible effects on future vector execution or outside the shader.
  /// Note: as of 2024 the only use of this is SIPreEmitPeephole where it is
  /// used in removing branches over short EXEC = 0 sequences.
  /// As such it embeds certain assumptions which may not apply to every case
  /// of EXEC = 0 execution.
  bool hasUnwantedEffectsWhenEXECEmpty(const MachineInstr &MI) const;
 
  /// Returns true if the instruction could potentially depend on the value of
  /// exec. If false, exec dependencies may safely be ignored.
  bool mayReadEXEC(const MachineRegisterInfo &MRI, const MachineInstr &MI) const;
 
  bool isInlineConstant(const APInt &Imm) const;
 
  bool isInlineConstant(const APFloat &Imm) const;
 
  // Returns true if this non-register operand definitely does not need to be
  // encoded as a 32-bit literal. Note that this function handles all kinds of
  // operands, not just immediates.
  //
  // Some operands like FrameIndexes could resolve to an inline immediate value
  // that will not require an additional 4-bytes; this function assumes that it
  // will.
  bool isInlineConstant(const MachineOperand &MO, uint8_t OperandType) const {
    if (!MO.isImm())
      return false;
    return isInlineConstant(MO.getImm(), OperandType);
  }
  bool isInlineConstant(int64_t ImmVal, uint8_t OperandType) const;
 
  bool isInlineConstant(const MachineOperand &MO,
                        const MCOperandInfo &OpInfo) const {
    return isInlineConstant(MO, OpInfo.OperandType);
  }
 
  /// \p returns true if \p UseMO is substituted with \p DefMO in \p MI it would
  /// be an inline immediate.
  bool isInlineConstant(const MachineInstr &MI,
                        const MachineOperand &UseMO,
                        const MachineOperand &DefMO) const {
    assert(UseMO.getParent() == &MI);
    int OpIdx = UseMO.getOperandNo();
    if (OpIdx >= MI.getDesc().NumOperands)
      return false;
 
    return isInlineConstant(DefMO, MI.getDesc().operands()[OpIdx]);
  }
 
  /// \p returns true if the operand \p OpIdx in \p MI is a valid inline
  /// immediate.
  bool isInlineConstant(const MachineInstr &MI, unsigned OpIdx) const {
    const MachineOperand &MO = MI.getOperand(OpIdx);
    return isInlineConstant(MO, MI.getDesc().operands()[OpIdx].OperandType);
  }
 
  bool isInlineConstant(const MachineInstr &MI, unsigned OpIdx,
                        int64_t ImmVal) const {
    if (OpIdx >= MI.getDesc().NumOperands)
      return false;
 
    if (isCopyInstr(MI)) {
      unsigned Size = getOpSize(MI, OpIdx);
      assert(Size == 8 || Size == 4);
 
      uint8_t OpType = (Size == 8) ?
        AMDGPU::OPERAND_REG_IMM_INT64 : AMDGPU::OPERAND_REG_IMM_INT32;
      return isInlineConstant(ImmVal, OpType);
    }
 
    return isInlineConstant(ImmVal, MI.getDesc().operands()[OpIdx].OperandType);
  }
 
  bool isInlineConstant(const MachineInstr &MI, unsigned OpIdx,
                        const MachineOperand &MO) const {
    return isInlineConstant(MI, OpIdx, MO.getImm());
  }
 
  bool isInlineConstant(const MachineOperand &MO) const {
    return isInlineConstant(*MO.getParent(), MO.getOperandNo());
  }
 
  bool isImmOperandLegal(const MCInstrDesc &InstDesc, unsigned OpNo,
                         const MachineOperand &MO) const;
 
  bool isLiteralOperandLegal(const MCInstrDesc &InstDesc,
                             const MCOperandInfo &OpInfo) const;
 
  bool isImmOperandLegal(const MCInstrDesc &InstDesc, unsigned OpNo,
                         int64_t ImmVal) const;
 
  bool isImmOperandLegal(const MachineInstr &MI, unsigned OpNo,
                         const MachineOperand &MO) const {
    return isImmOperandLegal(MI.getDesc(), OpNo, MO);
  }
 
  bool isNeverCoissue(MachineInstr &MI) const;
 
  /// Check if this immediate value can be used for AV_MOV_B64_IMM_PSEUDO.
  bool isLegalAV64PseudoImm(uint64_t Imm) const;
 
  /// Return true if this 64-bit VALU instruction has a 32-bit encoding.
  /// This function will return false if you pass it a 32-bit instruction.
  bool hasVALU32BitEncoding(unsigned Opcode) const;
 
  /// Return true if \p Reg is a lane mask that already has 0 in every bit
  /// corresponding to a lane that is inactive in EXEC where \p Use executes,
  /// so that ANDing it with EXEC there would be a no-op. Requires SSA form.
  bool isMaskedByExec(Register Reg, const MachineInstr &Use,
                      const MachineRegisterInfo &MRI, unsigned Depth = 0) const;
 
  bool physRegUsesConstantBus(const MachineOperand &Reg) const;
  bool regUsesConstantBus(const MachineOperand &Reg,
                          const MachineRegisterInfo &MRI) const;
 
  /// Returns true if this operand uses the constant bus.
  bool usesConstantBus(const MachineRegisterInfo &MRI,
                       const MachineOperand &MO,
                       const MCOperandInfo &OpInfo) const;
 
  bool usesConstantBus(const MachineRegisterInfo &MRI, const MachineInstr &MI,
                       int OpIdx) const {
    return usesConstantBus(MRI, MI.getOperand(OpIdx),
                           MI.getDesc().operands()[OpIdx]);
  }
 
  /// Return true if this instruction has any modifiers.
  ///  e.g. src[012]_mod, omod, clamp.
  bool hasModifiers(unsigned Opcode) const;
 
  bool hasModifiersSet(const MachineInstr &MI, AMDGPU::OpName OpName) const;
  bool hasAnyModifiersSet(const MachineInstr &MI) const;
 
  bool canShrink(const MachineInstr &MI,
                 const MachineRegisterInfo &MRI) const;
 
  MachineInstr *buildShrunkInst(MachineInstr &MI,
                                unsigned NewOpcode) const;
 
  bool verifyInstruction(const MachineInstr &MI,
                         StringRef &ErrInfo) const override;
 
  unsigned getVALUOp(const MachineInstr &MI) const;
  unsigned getVALUOp(unsigned Opc) const;
 
  void insertScratchExecCopy(MachineFunction &MF, MachineBasicBlock &MBB,
                             MachineBasicBlock::iterator MBBI,
                             const DebugLoc &DL, Register Reg, bool IsSCCLive,
                             SlotIndexes *Indexes = nullptr) const;
 
  void restoreExec(MachineFunction &MF, MachineBasicBlock &MBB,
                   MachineBasicBlock::iterator MBBI, const DebugLoc &DL,
                   Register Reg, SlotIndexes *Indexes = nullptr) const;
 
  MachineInstr *getWholeWaveFunctionSetup(MachineFunction &MF) const;
 
  /// Return the correct register class for \p OpNo.  For target-specific
  /// instructions, this will return the register class that has been defined
  /// in tablegen.  For generic instructions, like REG_SEQUENCE it will return
  /// the register class of its machine operand.
  /// to infer the correct register class base on the other operands.
  const TargetRegisterClass *getOpRegClass(const MachineInstr &MI,
                                           unsigned OpNo) const;
 
  /// Return the size in bytes of the operand OpNo on the given
  // instruction opcode.
  unsigned getOpSize(uint32_t Opcode, unsigned OpNo) const {
    const MCOperandInfo &OpInfo = get(Opcode).operands()[OpNo];
 
    if (OpInfo.RegClass == -1) {
      // If this is an immediate operand, this must be a 32-bit literal.
      assert(OpInfo.OperandType == MCOI::OPERAND_IMMEDIATE);
      return 4;
    }
 
    return RI.getRegSizeInBits(*RI.getRegClass(getOpRegClassID(OpInfo))) / 8;
  }
 
  /// This form should usually be preferred since it handles operands
  /// with unknown register classes.
  unsigned getOpSize(const MachineInstr &MI, unsigned OpNo) const {
    const MachineOperand &MO = MI.getOperand(OpNo);
    if (MO.isReg()) {
      if (unsigned SubReg = MO.getSubReg()) {
        return RI.getSubRegIdxSize(SubReg) / 8;
      }
    }
    return RI.getRegSizeInBits(*getOpRegClass(MI, OpNo)) / 8;
  }
 
  /// Legalize the \p OpIndex operand of this instruction by inserting
  /// a MOV.  For example:
  /// ADD_I32_e32 VGPR0, 15
  /// to
  /// MOV VGPR1, 15
  /// ADD_I32_e32 VGPR0, VGPR1
  ///
  /// If the operand being legalized is a register, then a COPY will be used
  /// instead of MOV.
  void legalizeOpWithMove(MachineInstr &MI, unsigned OpIdx) const;
 
  /// Check if \p MO is a legal operand if it was the \p OpIdx Operand
  /// for \p MI.
  bool isOperandLegal(const MachineInstr &MI, unsigned OpIdx,
                      const MachineOperand *MO = nullptr) const;
 
  /// Check if \p MO (a register operand) is a legal register for the
  /// given operand description or operand index.
  /// The operand index version provide more legality checks
  bool isLegalRegOperand(const MachineRegisterInfo &MRI,
                         const MCOperandInfo &OpInfo,
                         const MachineOperand &MO) const;
  bool isLegalRegOperand(const MachineInstr &MI, unsigned OpIdx,
                         const MachineOperand &MO) const;
 
  /// Check if \p MO would be a legal operand for a single-SGPR-read
  /// instruction.
  ///
  /// Single-SGPR-read instructions typically accept VGPRs, SGPRs, or immediates
  /// as source operands. On gfx12+, if a source operand uses SGPRs, the HW can
  /// only read the first SGPR and replicate the value across all lanes. \p SrcN
  /// can be 0, 1, or 2, representing src0, src1, and src2, respectively. If \p
  /// MO is nullptr, the operand corresponding to \p SrcN will be used. Non-SGPR
  /// operands are always considered legal.
  bool
  isLegalSingleSGPRReadInstOperand(const MachineRegisterInfo &MRI,
                                   const MachineInstr &MI, unsigned SrcN,
                                   const MachineOperand *MO = nullptr) const;
 
  /// Legalize operands in \p MI by either commuting it or inserting a
  /// copy of src1.
  void legalizeOperandsVOP2(MachineRegisterInfo &MRI, MachineInstr &MI) const;
 
  /// Fix operands in \p MI to satisfy constant bus requirements.
  void legalizeOperandsVOP3(MachineRegisterInfo &MRI, MachineInstr &MI) const;
 
  /// Copy a value from a VGPR (\p SrcReg) to SGPR. The desired register class
  /// for the dst register (\p DstRC) can be optionally supplied. This function
  /// can only be used when it is know that the value in SrcReg is same across
  /// all threads in the wave.
  /// \returns The SGPR register that \p SrcReg was copied to.
  Register readlaneVGPRToSGPR(Register SrcReg, MachineInstr &UseMI,
                              MachineRegisterInfo &MRI,
                              const TargetRegisterClass *DstRC = nullptr) const;
 
  void legalizeOperandsSMRD(MachineRegisterInfo &MRI, MachineInstr &MI) const;
  void legalizeOperandsFLAT(MachineRegisterInfo &MRI, MachineInstr &MI) const;
 
  void legalizeGenericOperand(MachineBasicBlock &InsertMBB,
                              MachineBasicBlock::iterator I,
                              const TargetRegisterClass *DstRC,
                              MachineOperand &Op, MachineRegisterInfo &MRI,
                              const DebugLoc &DL) const;
 
  /// Legalize all operands in this instruction.  This function may create new
  /// instructions and control-flow around \p MI.  If present, \p MDT is
  /// updated.
  /// \returns A new basic block that contains \p MI if new blocks were created.
  MachineBasicBlock *
  legalizeOperands(MachineInstr &MI, MachineDominatorTree *MDT = nullptr) const;
 
  /// Change SADDR form of a FLAT \p Inst to its VADDR form if saddr operand
  /// was moved to VGPR. \returns true if succeeded.
  bool moveFlatAddrToVGPR(MachineInstr &Inst) const;
 
  /// Fix operands in Inst to fix 16bit SALU to VALU lowering.
  void legalizeOperandsVALUt16(MachineInstr &Inst,
                               MachineRegisterInfo &MRI) const;
  void legalizeOperandsVALUt16(MachineInstr &Inst, unsigned OpIdx,
                               MachineRegisterInfo &MRI) const;
 
  /// Replace the instructions opcode with the equivalent VALU
  /// opcode.  This function will also move the users of MachineInstruntions
  /// in the \p WorkList to the VALU if necessary. If present, \p MDT is
  /// updated.
  void moveToVALU(SIInstrWorklist &Worklist, MachineDominatorTree *MDT) const;
 
  void
  moveToVALUImpl(SIInstrWorklist &Worklist, MachineDominatorTree *MDT,
                 MachineInstr &Inst,
                 DenseMap<MachineInstr *, V2PhysSCopyInfo> &WaterFalls,
                 DenseMap<MachineInstr *, bool> &V2SPhyCopiesToErase) const;
  /// Wrapper function for generating waterfall for instruction \p MI
  /// This function take into consideration of related pre & succ instructions
  /// (e.g. calling process) into consideratioin
  void createWaterFallForSiCall(MachineInstr *MI, MachineDominatorTree *MDT,
                                ArrayRef<MachineOperand *> ScalarOps,
                                ArrayRef<Register> PhySGPRs = {}) const;
 
  void insertNoop(MachineBasicBlock &MBB,
                  MachineBasicBlock::iterator MI) const override;
 
  void insertNoops(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
                   unsigned Quantity) const override;
 
  /// Build instructions that simulate the behavior of a `s_trap 2` instructions
  /// for hardware (namely, gfx11) that runs in PRIV=1 mode. There, s_trap is
  /// interpreted as a nop.
  MachineBasicBlock *insertSimulatedTrap(MachineRegisterInfo &MRI,
                                         MachineBasicBlock &MBB,
                                         MachineInstr &MI,
                                         const DebugLoc &DL) const;
 
  /// Return the number of wait states that result from executing this
  /// instruction.
  static unsigned getNumWaitStates(const MachineInstr &MI);
 
  /// Returns the operand named \p Op.  If \p MI does not have an
  /// operand named \c Op, this function returns nullptr.
  LLVM_READONLY
  MachineOperand *getNamedOperand(MachineInstr &MI,
                                  AMDGPU::OpName OperandName) const;
 
  LLVM_READONLY
  const MachineOperand *getNamedOperand(const MachineInstr &MI,
                                        AMDGPU::OpName OperandName) const {
    return getNamedOperand(const_cast<MachineInstr &>(MI), OperandName);
  }
 
  /// Get required immediate operand
  int64_t getNamedImmOperand(const MachineInstr &MI,
                             AMDGPU::OpName OperandName) const {
    int Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), OperandName);
    return MI.getOperand(Idx).getImm();
  }
 
  uint64_t getDefaultRsrcDataFormat() const;
  uint64_t getScratchRsrcWords23() const;
 
  bool isLowLatencyInstruction(const MachineInstr &MI) const;
  bool isHighLatencyDef(int Opc) const override;
 
  /// Return the descriptor of the target-specific machine instruction
  /// that corresponds to the specified pseudo or native opcode.
  const MCInstrDesc &getMCOpcodeFromPseudo(unsigned Opcode) const {
    return get(pseudoToMCOpcode(Opcode));
  }
 
  Register isStackAccess(const MachineInstr &MI, int &FrameIndex,
                         TypeSize &MemBytes) const;
  Register isSGPRStackAccess(const MachineInstr &MI, int &FrameIndex,
                             TypeSize &MemBytes) const;
 
  Register isLoadFromStackSlot(const MachineInstr &MI,
                               int &FrameIndex) const override {
    TypeSize MemBytes = TypeSize::getZero();
    return isLoadFromStackSlot(MI, FrameIndex, MemBytes);
  }
 
  Register isLoadFromStackSlot(const MachineInstr &MI, int &FrameIndex,
                               TypeSize &MemBytes) const override;
 
  Register isStoreToStackSlot(const MachineInstr &MI,
                              int &FrameIndex) const override {
    TypeSize MemBytes = TypeSize::getZero();
    return isStoreToStackSlot(MI, FrameIndex, MemBytes);
  }
 
  Register isStoreToStackSlot(const MachineInstr &MI, int &FrameIndex,
                              TypeSize &MemBytes) const override;
 
  unsigned getInstSizeInBytes(const MachineInstr &MI) const override;
 
  InstSizeVerifyMode
  getInstSizeVerifyMode(const MachineInstr &MI) const override;
 
  std::pair<unsigned, unsigned>
  decomposeMachineOperandsTargetFlags(unsigned TF) const override;
 
  ArrayRef<std::pair<int, const char *>>
  getSerializableTargetIndices() const override;
 
  ArrayRef<std::pair<unsigned, const char *>>
  getSerializableDirectMachineOperandTargetFlags() const override;
 
  ArrayRef<std::pair<MachineMemOperand::Flags, const char *>>
  getSerializableMachineMemOperandTargetFlags() const override;
 
  ScheduleHazardRecognizer *
  CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II,
                                 const ScheduleDAG *DAG) const override;
 
  ScheduleHazardRecognizer *
  CreateTargetPostRAHazardRecognizer(const MachineFunction &MF,
                                     MachineLoopInfo *MLI) const override;
 
  ScheduleHazardRecognizer *
  CreateTargetMIHazardRecognizer(const InstrItineraryData *II,
                                 const ScheduleDAGMI *DAG) const override;
 
  unsigned getLiveRangeSplitOpcode(Register Reg,
                                   const MachineFunction &MF) const override;
 
  bool isBasicBlockPrologue(const MachineInstr &MI,
                            Register Reg = Register()) const override;
 
  bool canAddToBBProlog(const MachineInstr &MI) const;
 
  MachineInstr *createPHIDestinationCopy(MachineBasicBlock &MBB,
                                         MachineBasicBlock::iterator InsPt,
                                         const DebugLoc &DL, Register Src,
                                         Register Dst) const override;
 
  MachineInstr *createPHISourceCopy(MachineBasicBlock &MBB,
                                    MachineBasicBlock::iterator InsPt,
                                    const DebugLoc &DL, Register Src,
                                    unsigned SrcSubReg,
                                    Register Dst) const override;
 
  bool isWave32() const;
 
  bool isVOPDAntidependencyAllowed(const MachineInstr &MI) const;
 
  bool hasRAWDependency(const MachineInstr &FirstMI,
                        const MachineInstr &SecondMI) const;
 
  /// Return a partially built integer add instruction without carry.
  /// Caller must add source operands.
  /// For pre-GFX9 it will generate unused carry destination operand.
  /// TODO: After GFX9 it should return a no-carry operation.
  MachineInstrBuilder getAddNoCarry(MachineBasicBlock &MBB,
                                    MachineBasicBlock::iterator I,
                                    const DebugLoc &DL,
                                    Register DestReg) const;
 
  MachineInstrBuilder getAddNoCarry(MachineBasicBlock &MBB,
                                    MachineBasicBlock::iterator I,
                                    const DebugLoc &DL,
                                    Register DestReg,
                                    RegScavenger &RS) const;
 
  static bool isKillTerminator(unsigned Opcode);
  const MCInstrDesc &getKillTerminatorFromPseudo(unsigned Opcode) const;
 
  bool isLegalMUBUFImmOffset(unsigned Imm) const;
 
  static unsigned getMaxMUBUFImmOffset(const GCNSubtarget &ST);
 
  bool splitMUBUFOffset(uint32_t Imm, uint32_t &SOffset, uint32_t &ImmOffset,
                        Align Alignment = Align(4)) const;
 
  /// Returns if \p Offset is legal for the subtarget as the offset to a FLAT
  /// encoded instruction with the given \p FlatVariant.
  bool isLegalFLATOffset(int64_t Offset, unsigned AddrSpace,
                         AMDGPU::FlatAddrSpace FlatVariant) const;
 
  /// Split \p COffsetVal into {immediate offset field, remainder offset}
  /// values.
  std::pair<int64_t, int64_t>
  splitFlatOffset(int64_t COffsetVal, unsigned AddrSpace,
                  AMDGPU::FlatAddrSpace FlatVariant) const;
 
  /// Returns true if negative offsets are allowed for the given \p FlatVariant.
  bool allowNegativeFlatOffset(AMDGPU::FlatAddrSpace FlatVariant) const;
 
  /// \brief Return a target-specific opcode if Opcode is a pseudo instruction.
  /// Return -1 if the target-specific opcode for the pseudo instruction does
  /// not exist. If Opcode is not a pseudo instruction, this is identity.
  int pseudoToMCOpcode(int Opcode) const;
 
  /// \brief Check if this instruction should only be used by assembler.
  /// Return true if this opcode should not be used by codegen.
  bool isAsmOnlyOpcode(int MCOp) const;
 
  void fixImplicitOperands(MachineInstr &MI) const;
 
  MachineInstr *foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI,
                                      ArrayRef<unsigned> Ops, int FrameIndex,
                                      MachineInstr *&CopyMI,
                                      LiveIntervals *LIS = nullptr,
                                      VirtRegMap *VRM = nullptr) const override;
 
  unsigned getInstrLatency(const InstrItineraryData *ItinData,
                           const MachineInstr &MI,
                           unsigned *PredCost = nullptr) const override;
 
  unsigned getBlockingCycles(const MachineInstr &MI) const;
 
  /// GFX1250 blocking-cycles table lookup with no occupancy subtarget gate.
  /// Returns 0 if \p MI is not in the table. Used as a multi-pass VALU denylist
  /// (e.g. V_PERM_PK16 hazard) on both gfx1250 and gfx1251.
  unsigned getGFX1250BlockingCyclesTable(const MachineInstr &MI) const;
 
  const MachineOperand &getCalleeOperand(const MachineInstr &MI) const override;
 
  ValueUniformity getValueUniformity(const MachineInstr &MI) const final;
 
  ValueUniformity getGenericValueUniformity(const MachineInstr &MI) const;
 
  const MIRFormatter *getMIRFormatter() const override;
 
  static unsigned getDSShaderTypeValue(const MachineFunction &MF);
 
  const TargetSchedModel &getSchedModel() const { return SchedModel; }
 
  void createReadFirstLaneFromCopyToPhysReg(MachineRegisterInfo &MRI,
                                            Register DstReg,
                                            MachineInstr &Inst) const;
 
  void handleCopyToPhysHelper(
      SIInstrWorklist &Worklist, Register DstReg, MachineInstr &Inst,
      MachineRegisterInfo &MRI,
      DenseMap<MachineInstr *, V2PhysSCopyInfo> &WaterFalls,
      DenseMap<MachineInstr *, bool> &V2SPhyCopiesToErase) const;
 
  // FIXME: This should be removed
  // Enforce operand's \p OpName even alignment if required by target.
  // This is used if an operand is a 32 bit register but needs to be aligned
  // regardless.
  void enforceOperandRCAlignment(MachineInstr &MI, AMDGPU::OpName OpName) const;
 
  /// Get the repeat rate for a VALU instruction from the scheduling model.
  /// Returns 1 for regular VALU, >1 for long-latency VALU (packed, F64, etc.)
  unsigned getRepeatRate(const MachineInstr &MI) const;
};
 
/// \brief Returns true if a reg:subreg pair P has a TRC class
inline bool isOfRegClass(const TargetInstrInfo::RegSubRegPair &P,
                         const TargetRegisterClass &TRC,
                         MachineRegisterInfo &MRI) {
  auto *RC = MRI.getRegClass(P.Reg);
  if (!P.SubReg)
    return RC == &TRC;
  auto *TRI = MRI.getTargetRegisterInfo();
  return RC == TRI->getMatchingSuperRegClass(RC, &TRC, P.SubReg);
}
 
/// \brief Create RegSubRegPair from a register MachineOperand
inline
TargetInstrInfo::RegSubRegPair getRegSubRegPair(const MachineOperand &O) {
  assert(O.isReg());
  return TargetInstrInfo::RegSubRegPair(O.getReg(), O.getSubReg());
}
 
/// \brief Return the SubReg component from REG_SEQUENCE
TargetInstrInfo::RegSubRegPair getRegSequenceSubReg(MachineInstr &MI,
                                                    unsigned SubReg);
 
/// \brief Return the defining instruction for a given reg:subreg pair
/// skipping copy like instructions and subreg-manipulation pseudos.
/// Following another subreg of a reg:subreg isn't supported.
MachineInstr *getVRegSubRegDef(const TargetInstrInfo::RegSubRegPair &P,
                               const MachineRegisterInfo &MRI);
 
/// \brief Return false if EXEC is not changed between the def of \p VReg at \p
/// DefMI and the use at \p UseMI. Should be run on SSA. Currently does not
/// attempt to track between blocks.
bool execMayBeModifiedBeforeUse(const MachineRegisterInfo &MRI,
                                Register VReg,
                                const MachineInstr &DefMI,
                                const MachineInstr &UseMI);
 
/// \brief Return false if EXEC is not changed between the def of \p VReg at \p
/// DefMI and all its uses. Should be run on SSA. Currently does not attempt to
/// track between blocks.
bool execMayBeModifiedBeforeAnyUse(const MachineRegisterInfo &MRI,
                                   Register VReg,
                                   const MachineInstr &DefMI);
 
namespace AMDGPU {
 
  LLVM_READONLY
  int32_t getVOPe64(uint32_t Opcode);
 
  LLVM_READONLY
  int32_t getVOPe32(uint32_t Opcode);
 
  LLVM_READONLY
  int32_t getSDWAOp(uint32_t Opcode);
 
  LLVM_READONLY
  int32_t getDPPOp32(uint32_t Opcode);
 
  LLVM_READONLY
  int32_t getDPPOp64(uint32_t Opcode);
 
  LLVM_READONLY
  int32_t getBasicFromSDWAOp(uint32_t Opcode);
 
  LLVM_READONLY
  int32_t getCommuteRev(uint32_t Opcode);
 
  LLVM_READONLY
  int32_t getCommuteOrig(uint32_t Opcode);
 
  LLVM_READONLY
  int32_t getAddr64Inst(uint32_t Opcode);
 
  /// Check if \p Opcode is an Addr64 opcode.
  ///
  /// \returns \p Opcode if it is an Addr64 opcode, otherwise -1.
  LLVM_READONLY
  int32_t getIfAddr64Inst(uint32_t Opcode);
 
  LLVM_READONLY
  int32_t getSOPKOp(uint32_t Opcode);
 
  /// \returns SADDR form of a FLAT Global instruction given an \p Opcode
  /// of a VADDR form.
  LLVM_READONLY
  int32_t getGlobalSaddrOp(uint32_t Opcode);
 
  /// \returns VADDR form of a FLAT Global instruction given an \p Opcode
  /// of a SADDR form.
  LLVM_READONLY
  int32_t getGlobalVaddrOp(uint32_t Opcode);
 
  /// \returns ST form with only immediate offset of a FLAT Scratch instruction
  /// given an \p Opcode of an SS (SADDR) form.
  LLVM_READONLY
  int32_t getFlatScratchInstSTfromSS(uint32_t Opcode);
 
  /// \returns SV (VADDR) form of a FLAT Scratch instruction given an \p Opcode
  /// of an SVS (SADDR + VADDR) form.
  LLVM_READONLY
  int32_t getFlatScratchInstSVfromSVS(uint32_t Opcode);
 
  /// \returns SS (SADDR) form of a FLAT Scratch instruction given an \p Opcode
  /// of an SV (VADDR) form.
  LLVM_READONLY
  int32_t getFlatScratchInstSSfromSV(uint32_t Opcode);
 
  /// \returns SV (VADDR) form of a FLAT Scratch instruction given an \p Opcode
  /// of an SS (SADDR) form.
  LLVM_READONLY
  int32_t getFlatScratchInstSVfromSS(uint32_t Opcode);
 
  /// \returns earlyclobber version of a MAC MFMA is exists.
  LLVM_READONLY
  int32_t getMFMAEarlyClobberOp(uint32_t Opcode);
 
  /// \returns Version of an instruction which uses AGPRs for coupled operands
  /// given an \p Opcode which uses VGPRs for coupled operands.
  LLVM_READONLY
  int32_t getAGPRFormOp(uint32_t Opcode);
 
  /// \returns v_cmpx version of a v_cmp instruction.
  LLVM_READONLY
  int32_t getVCMPXOpFromVCMP(uint32_t Opcode);
 
  const uint64_t RSRC_DATA_FORMAT = 0xf00000000000LL;
  const uint64_t RSRC_ELEMENT_SIZE_SHIFT = (32 + 19);
  const uint64_t RSRC_INDEX_STRIDE_SHIFT = (32 + 21);
  const uint64_t RSRC_TID_ENABLE = UINT64_C(1) << (32 + 23);
 
} // end namespace AMDGPU
 
namespace AMDGPU {
enum AsmComments : MachineInstr::AsmPrinterFlagTy {
  // For sgpr to vgpr spill instructions
  SGPR_SPILL = MachineInstr::TAsmComments
};
} // namespace AMDGPU
 
namespace SI {
namespace KernelInputOffsets {
 
/// Offsets in bytes from the start of the input buffer
enum Offsets {
  NGROUPS_X = 0,
  NGROUPS_Y = 4,
  NGROUPS_Z = 8,
  GLOBAL_SIZE_X = 12,
  GLOBAL_SIZE_Y = 16,
  GLOBAL_SIZE_Z = 20,
  LOCAL_SIZE_X = 24,
  LOCAL_SIZE_Y = 28,
  LOCAL_SIZE_Z = 32
};
 
} // end namespace KernelInputOffsets
} // end namespace SI
 
} // end namespace llvm
 
#endif // LLVM_LIB_TARGET_AMDGPU_SIINSTRINFO_H