/*========================== begin_copyright_notice ============================

Copyright (C) 2017-2024 Intel Corporation

SPDX-License-Identifier: MIT

============================= end_copyright_notice ===========================*/

#include "Compiler/CISACodeGen/PatternMatchPass.hpp"
#include "Compiler/CISACodeGen/DeSSA.hpp"
#include "Compiler/CISACodeGen/EmitVISAPass.hpp"
#include "common/igc_regkeys.hpp"
#include "common/LLVMWarningsPush.hpp"
#include <llvm/Analysis/ValueTracking.h>
#include <llvm/IR/InlineAsm.h>
#include <llvm/IR/Dominators.h>
#include <llvm/IR/Constants.h>
#include <llvm/IR/Instruction.h>
#include <llvm/IR/IntrinsicInst.h>
#include <llvm/IR/PatternMatch.h>
#include "llvm/Support/KnownBits.h"
#include "common/LLVMWarningsPop.hpp"
#include <llvmWrapper/IR/Instructions.h>
#include "llvmWrapper/Support/Alignment.h"
#include <llvmWrapper/IR/CmpPredicate.h>
#include "llvmWrapper/IR/Constants.h"
#include "Compiler/IGCPassSupport.h"
#include "Compiler/InitializePasses.h"
#include "Compiler/DebugInfo/ScalarVISAModule.h"
#include "GenISAIntrinsics/GenIntrinsicInst.h"
#include "Probe/Assertion.h"

using namespace llvm;
using namespace IGC;
using namespace IGC::IGCMD;

char CodeGenPatternMatch::ID = 0;
#define PASS_FLAG "CodeGenPatternMatch"
#define PASS_DESCRIPTION "Does pattern matching"
#define PASS_CFG_ONLY true
#define PASS_ANALYSIS true
IGC_INITIALIZE_PASS_BEGIN(CodeGenPatternMatch, PASS_FLAG, PASS_DESCRIPTION, PASS_CFG_ONLY, PASS_ANALYSIS)
IGC_INITIALIZE_PASS_DEPENDENCY(WIAnalysis)
IGC_INITIALIZE_PASS_DEPENDENCY(LiveVarsAnalysis)
IGC_INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
IGC_INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
IGC_INITIALIZE_PASS_DEPENDENCY(MetaDataUtilsWrapper)
IGC_INITIALIZE_PASS_DEPENDENCY(PositionDepAnalysis)
IGC_INITIALIZE_PASS_END(CodeGenPatternMatch, PASS_FLAG, PASS_DESCRIPTION, PASS_CFG_ONLY, PASS_ANALYSIS)

namespace IGC {
CodeGenPatternMatch::CodeGenPatternMatch()
    : FunctionPass(ID), m_rootIsSubspanUse(false), m_blocks(nullptr), m_numBlocks(0), m_root(nullptr),
      m_currentPattern(nullptr), m_Platform(), m_AllowContractions(true), m_NeedVMask(false),
      m_samplertoRenderTargetEnable(false), m_ctx(nullptr), DT(nullptr), PDT(nullptr), LI(nullptr), m_DL(0),
      m_WI(nullptr), m_LivenessInfo(nullptr), m_PosDep(nullptr) {
  initializeCodeGenPatternMatchPass(*PassRegistry::getPassRegistry());
}

CodeGenPatternMatch::~CodeGenPatternMatch() { delete[] m_blocks; }

void CodeGenPatternMatch::CodeGenNode(llvm::DomTreeNode *node) {
  struct NodeRange {
    NodeRange(llvm::DomTreeNode *node, llvm::DomTreeNode::iterator current, llvm::DomTreeNode::iterator end)
        : m_Node(node), m_Current(current), m_End(end) {}
    llvm::DomTreeNode *m_Node;
    llvm::DomTreeNode::iterator m_Current;
    llvm::DomTreeNode::iterator m_End;
  };
  std::list<NodeRange> m_NodeContainer;
  m_NodeContainer.push_front(NodeRange(node, node->begin(), node->end()));

  // Process blocks by processing the dominance tree depth first
  while (!m_NodeContainer.empty()) {
    NodeRange &currNodeRange = m_NodeContainer.front();
    if (currNodeRange.m_Current == currNodeRange.m_End) {
      llvm::BasicBlock *bb = currNodeRange.m_Node->getBlock();
      CodeGenBlock(bb);
      m_NodeContainer.pop_front();
      continue;
    }
    llvm::DomTreeNode *child = *currNodeRange.m_Current;
    m_NodeContainer.push_front(NodeRange(child, child->begin(), child->end()));
    currNodeRange.m_Current++;
  }
}

void CodeGenPatternMatch::releaseMemory() {
  m_blockMap.clear();
  ConstantPlacement.clear();
  PairOutputMap.clear();
  UniformBools.clear();
  LoopExitingBlocksCache.clear();
  delete[] m_blocks;
  m_blocks = nullptr;
}

bool CodeGenPatternMatch::runOnFunction(llvm::Function &F) {
  m_ctx = getAnalysis<CodeGenContextWrapper>().getCodeGenContext();

  MetaDataUtils *pMdUtils = getAnalysis<MetaDataUtilsWrapper>().getMetaDataUtils();
  ModuleMetaData *modMD = getAnalysis<MetaDataUtilsWrapper>().getModuleMetaData();
  if (pMdUtils->findFunctionsInfoItem(&F) == pMdUtils->end_FunctionsInfo()) {
    return false;
  }

  m_AllowContractions = true;
  if (m_ctx->m_DriverInfo.NeedCheckContractionAllowed()) {
    m_AllowContractions = modMD->compOpt.FastRelaxedMath || modMD->compOpt.MadEnable;
  }
  m_Platform = m_ctx->platform;

  DT = &getAnalysis<llvm::DominatorTreeWrapperPass>().getDomTree();
  PDT = &getAnalysis<llvm::PostDominatorTreeWrapperPass>().getPostDomTree();
  LI = &getAnalysis<llvm::LoopInfoWrapperPass>().getLoopInfo();
  m_DL = &F.getParent()->getDataLayout();
  m_WI = &getAnalysis<WIAnalysis>();
  m_PosDep = &getAnalysis<PositionDepAnalysis>();
  // pattern match will update liveness held by LiveVar, which needs
  // WIAnalysis result for uniform variable
  m_LivenessInfo = &getAnalysis<LiveVarsAnalysis>().getLiveVars();
  CreateBasicBlocks(&F);
  CodeGenNode(DT->getRootNode());
  return false;
}

inline bool HasSideEffect(llvm::Instruction &inst) {
  if (inst.mayWriteToMemory() || inst.isTerminator()) {
    return true;
  }
  return false;
}

inline bool HasPhiUse(llvm::Value &inst) {
  for (auto UI = inst.user_begin(), E = inst.user_end(); UI != E; ++UI) {
    llvm::User *U = *UI;
    if (llvm::isa<llvm::PHINode>(U)) {
      return true;
    }
  }
  return false;
}

// If "v" is a canonicalize instruction, return its source argument. Return
// "v" otherwise.
inline Value *SkipCanonicalize(Value *v) {
  IntrinsicInst *intr = dyn_cast<IntrinsicInst>(v);
  if (intr && intr->getIntrinsicID() == Intrinsic::canonicalize) {
    return intr->getOperand(0);
  }
  return v;
}

bool CodeGenPatternMatch::IsDbgInst(llvm::Instruction &inst) const {
  if (llvm::isa<llvm::DbgInfoIntrinsic>(&inst)) {
    // FIXME: We probably don't need that.
    return true;
  }
  // perThreadOffset is special variable we should be alive for O0 runs.
  // This is intended for debug, but we do it for non-debug too to
  // avoid altering generated code by adding -g.
  if (m_ctx->getModuleMetaData()->compOpt.OptDisable && inst.getMetadata("perThreadOffset")) {
    return true;
  }
  return false;
}

bool CodeGenPatternMatch::IsConstOrSimdConstExpr(Value *C) {
  if (isa<ConstantInt>(C)) {
    return true;
  }
  if (Instruction *inst = dyn_cast<Instruction>(C)) {
    return SIMDConstExpr(inst);
  }
  return false;
}

// Checks if denorms are flushed on instruction's output.
// When denorm mode in CR0 is set to flush to zero, denorms are flushed on
// input and output of any floating-point mathematical operation with the
// following exceptions:
// - raw mov retains denorms
// - conversion instructions retain denorms (includes mix-mode instructions)
// - extended math instructions retain half float denorms
bool CodeGenPatternMatch::FlushesDenormsOnOutput(Instruction &I) {
  bool flushesDenorms = false;
  if ((m_ctx->getModuleMetaData()->compOpt.FloatDenormMode16 == IGC::FLOAT_DENORM_FLUSH_TO_ZERO &&
       I.getType()->isHalfTy()) ||
      (m_ctx->getModuleMetaData()->compOpt.FloatDenormMode32 == IGC::FLOAT_DENORM_FLUSH_TO_ZERO &&
       I.getType()->isFloatTy()) ||
      (m_ctx->getModuleMetaData()->compOpt.FloatDenormMode64 == IGC::FLOAT_DENORM_FLUSH_TO_ZERO &&
       I.getType()->isDoubleTy())) {
    switch (GetOpCode(&I)) {
    case llvm_select:
      flushesDenorms = true;
      for (uint i = 1; i < I.getNumOperands(); ++i) {
        Instruction *inst = dyn_cast<Instruction>(I.getOperand(i));
        if (inst && !FlushesDenormsOnOutput(*inst)) {
          flushesDenorms = false;
          break;
        }
      }
      break;
    case llvm_max:
    case llvm_min:
      flushesDenorms = true;
      for (uint i = 0; i < I.getNumOperands(); ++i) {
        Instruction *inst = dyn_cast<Instruction>(I.getOperand(i));
        if (inst && !FlushesDenormsOnOutput(*inst)) {
          flushesDenorms = false;
          break;
        }
      }
      break;
    case llvm_fsat:
    case llvm_fabs:
      if (isa<Instruction>(I.getOperand(0))) {
        flushesDenorms = FlushesDenormsOnOutput(*(cast<Instruction>(I.getOperand(0))));
      }
      break;
    case llvm_canonicalize:
    case llvm_fadd:
    case llvm_fadd_rtz:
    case llvm_fsub:
    case llvm_fmul:
    case llvm_fmul_rtz:
    case llvm_fma:
    case llvm_roundne:
    case llvm_round_z:
    case llvm_floor:
    case llvm_ceil:
    case llvm_frc:
      flushesDenorms = true;
      break;
    case llvm_fdiv:
    case llvm_frem:
    case llvm_cos:
    case llvm_sin:
    case llvm_log:
    case llvm_exp:
    case llvm_pow:
    case llvm_sqrt:
    case llvm_rsq:
      // extended math retain half denorms
      flushesDenorms = !I.getType()->isHalfTy();
      break;
    default:
      break;
    }
  }
  return flushesDenorms;
}
bool CodeGenPatternMatch::FlushesDenormsOnInput(Instruction &I) {
  bool flushesDenorms = false;
  if ((m_ctx->getModuleMetaData()->compOpt.FloatDenormMode16 == IGC::FLOAT_DENORM_FLUSH_TO_ZERO &&
       I.getType()->isHalfTy()) ||
      (m_ctx->getModuleMetaData()->compOpt.FloatDenormMode32 == IGC::FLOAT_DENORM_FLUSH_TO_ZERO &&
       I.getType()->isFloatTy()) ||
      (m_ctx->getModuleMetaData()->compOpt.FloatDenormMode64 == IGC::FLOAT_DENORM_FLUSH_TO_ZERO &&
       I.getType()->isDoubleTy())) {
    switch (GetOpCode(&I)) {
    case llvm_canonicalize:
    case llvm_fadd:
    case llvm_fadd_rtz:
    case llvm_fsub:
    case llvm_fmul:
    case llvm_fmul_rtz:
    case llvm_fma:
    case llvm_roundne:
    case llvm_round_z:
    case llvm_floor:
    case llvm_ceil:
    case llvm_frc:
      flushesDenorms = true;
      break;
    case llvm_fdiv:
    case llvm_frem:
    case llvm_cos:
    case llvm_sin:
    case llvm_log:
    case llvm_exp:
    case llvm_pow:
    case llvm_sqrt:
    case llvm_rsq:
      // extended math retain half denorms
      flushesDenorms = !I.getType()->isHalfTy();
      break;
    default:
      break;
    }
  }
  return flushesDenorms;
}

bool CodeGenPatternMatch::ContractionAllowed(llvm::Instruction &I) const {
  if (m_AllowContractions || (m_ctx->m_DriverInfo.RespectPerInstructionContractFlag() && I.hasAllowContract())) {
    return true;
  }
  return false;
}

// this function need to be in sync with CShader::EvaluateSIMDConstExpr on what can be supported
bool CodeGenPatternMatch::SIMDConstExpr(Instruction *C) {
  auto it = m_IsSIMDConstExpr.find(C);
  if (it != m_IsSIMDConstExpr.end()) {
    return it->second;
  }
  bool isConstExpr = false;
  if (BinaryOperator *op = dyn_cast<BinaryOperator>(C)) {
    switch (op->getOpcode()) {
    case Instruction::Add:
    case Instruction::Sub:
    case Instruction::Mul:
    case Instruction::Shl:
    case Instruction::Or:
    case Instruction::And:
      isConstExpr = IsConstOrSimdConstExpr(op->getOperand(0)) && IsConstOrSimdConstExpr(op->getOperand(1));
      break;
    default:
      break;
    }
  } else if (llvm::GenIntrinsicInst *genInst = dyn_cast<GenIntrinsicInst>(C)) {
    if (genInst->getIntrinsicID() == GenISAIntrinsic::GenISA_simdSize) {
      isConstExpr = true;
    }
  }
  m_IsSIMDConstExpr.insert(std::make_pair(C, isConstExpr));
  return isConstExpr;
}

bool CodeGenPatternMatch::NeedInstruction(llvm::Instruction &I) {
  if (SIMDConstExpr(&I)) {
    return false;
  }
  if (HasPhiUse(I) || HasSideEffect(I) || IsDbgInst(I) || (m_usedInstructions.find(&I) != m_usedInstructions.end())) {
    return true;
  }
  return false;
}

void CodeGenPatternMatch::AddToConstantPool(llvm::BasicBlock *UseBlock, llvm::Value *Val) {
  Constant *C = dyn_cast_or_null<Constant>(Val);
  if (!C)
    return;

  BasicBlock *LCA = UseBlock;
  // Determine where we put the constant initialization.
  // Choose loop pre-header as LICM.
  // XXX: Further investigation/tuning is needed to see whether
  // we need to hoist constant initialization out of the
  // top-level loop within a nested loop. So far, we only hoist
  // one level up.
  if (Loop *L = LI->getLoopFor(LCA)) {
    if (BasicBlock *Preheader = L->getLoopPreheader())
      LCA = Preheader;
  }
  // Find the common dominator as CSE.
  if (BasicBlock *BB = ConstantPlacement.lookup(C))
    LCA = DT->findNearestCommonDominator(LCA, BB);
  IGC_ASSERT_MESSAGE(LCA, "LCA always exists for reachable BBs within a function!");
  ConstantPlacement[C] = LCA;
}

bool CodeGenPatternMatch::supportsLSCImmediateGlobalBaseOffset() {
  bool res = IGC_GET_FLAG_VALUE(LscImmOffsMatch) > 1 || m_Platform.matchImmOffsetsLSC();
  return res;
}

// Check bool values that can be emitted as a single element predicate.
void CodeGenPatternMatch::gatherUniformBools(Value *Val) {
  if (!isUniform(Val) || Val->getType()->getScalarType()->isIntegerTy(1))
    return;

  // Only starts from select instruction for now.
  // It is more complicate for uses in terminators.
  if (SelectInst *SI = dyn_cast<SelectInst>(Val)) {
    Value *Cond = SI->getCondition();
    if (Cond->getType()->isVectorTy() || !Cond->hasOneUse())
      return;

    // All users of bool values.
    DenseSet<Value *> Vals;
    Vals.insert(SI);

    // Grow the list of bool values to be checked.
    std::vector<Value *> ValList;
    ValList.push_back(Cond);

    bool IsLegal = true;
    while (!ValList.empty()) {
      Value *V = ValList.back();
      ValList.pop_back();
      IGC_ASSERT(nullptr != V);
      IGC_ASSERT(nullptr != V->getType());
      IGC_ASSERT(isUniform(V));
      IGC_ASSERT(V->getType()->isIntegerTy(1));

      // Check uses.
      for (auto UI = V->user_begin(), UE = V->user_end(); UI != UE; ++UI) {
        Value *U = *UI;
        if (!Vals.count(U))
          goto FAIL;
      }

      // Check defs.
      Vals.insert(V);
      if (auto CI = dyn_cast<CmpInst>(V)) {
        IGC_ASSERT(isUniform(CI->getOperand(0)));
        IGC_ASSERT(isUniform(CI->getOperand(1)));
        if (CI->getOperand(0)->getType()->getScalarSizeInBits() == 1)
          goto FAIL;
        continue;
      } else if (auto BI = dyn_cast<BinaryOperator>(V)) {
        IGC_ASSERT(isUniform(BI->getOperand(0)));
        IGC_ASSERT(isUniform(BI->getOperand(1)));
        if (isa<Instruction>(BI->getOperand(0)))
          ValList.push_back(BI->getOperand(0));
        if (isa<Instruction>(BI->getOperand(1)))
          ValList.push_back(BI->getOperand(1));
        continue;
      }

    FAIL:
      IsLegal = false;
      break;
    }

    // Populate all boolean values if legal.
    if (IsLegal) {
      for (auto V : Vals) {
        if (V->getType()->isIntegerTy(1))
          UniformBools.insert(V);
      }
    }
  }
}

void CodeGenPatternMatch::CodeGenBlock(llvm::BasicBlock *bb) {
  llvm::BasicBlock::InstListType::reverse_iterator I, E;
  auto it = m_blockMap.find(bb);
  IGC_ASSERT(it != m_blockMap.end());
  SBasicBlock *block = it->second;

  // loop through instructions bottom up
  for (I = bb->rbegin(), E = bb->rend(); I != E; ++I) {
    llvm::Instruction &inst = (*I);

    if (NeedInstruction(inst)) {
      SetPatternRoot(inst);
      Pattern *pattern = Match(inst);
      if (pattern) {
        block->m_dags.push_back(SDAG(pattern, m_root));
        gatherUniformBools(m_root);
      }
    }
  }
}

void CodeGenPatternMatch::CreateBasicBlocks(llvm::Function *pLLVMFunc) {
  m_numBlocks = pLLVMFunc->size();
  m_blocks = new SBasicBlock[m_numBlocks];
  uint i = 0;
  for (BasicBlock &bb : *pLLVMFunc) {
    m_blocks[i].id = i;
    m_blocks[i].bb = &bb;
    m_blockMap.insert(std::pair<llvm::BasicBlock *, SBasicBlock *>(&bb, &m_blocks[i]));
    i++;
  }
}
Pattern *CodeGenPatternMatch::Match(llvm::Instruction &inst) {
  m_currentPattern = nullptr;
  visit(inst);
  return m_currentPattern;
}

void CodeGenPatternMatch::SetPatternRoot(llvm::Instruction &inst) {
  m_root = &inst;
  m_rootIsSubspanUse = IsSubspanUse(m_root);
}

template <typename Op_t, typename ConstTy> struct ClampWithConstants_match {
  typedef ConstTy *ConstPtrTy;

  mutable Op_t Op;
  ConstPtrTy &CMin, &CMax;

  ClampWithConstants_match(const Op_t &OpMatch, ConstPtrTy &Min, ConstPtrTy &Max) : Op(OpMatch), CMin(Min), CMax(Max) {}

  template <typename OpTy> bool match(OpTy *V) const {
    CallInst *GII = dyn_cast<CallInst>(V);
    if (!GII)
      return false;

    EOPCODE op = GetOpCode(GII);

    if (op != llvm_max && op != llvm_min)
      return false;

    Value *X = GII->getOperand(0);
    Value *C = GII->getOperand(1);
    if (isa<ConstTy>(X))
      std::swap(X, C);

    ConstPtrTy C0 = dyn_cast<ConstTy>(C);
    if (!C0)
      return false;

    CallInst *GII2 = dyn_cast<CallInst>(X);
    if (!GII2)
      return false;

    EOPCODE op2 = GetOpCode(GII2);
    if (!(op == llvm_min && op2 == llvm_max) && !(op == llvm_max && op2 == llvm_min))
      return false;

    X = GII2->getOperand(0);
    C = GII2->getOperand(1);
    if (isa<ConstTy>(X))
      std::swap(X, C);

    ConstPtrTy C1 = dyn_cast<ConstTy>(C);
    if (!C1)
      return false;

    if (!Op.match(X))
      return false;

    CMin = (op2 == llvm_min) ? C0 : C1;
    CMax = (op2 == llvm_min) ? C1 : C0;
    return true;
  }
};

template <typename OpTy, typename ConstTy>
inline ClampWithConstants_match<OpTy, ConstTy> m_ClampWithConstants(const OpTy &Op, ConstTy *&Min, ConstTy *&Max) {
  return ClampWithConstants_match<OpTy, ConstTy>(Op, Min, Max);
}

template <typename Op_t> struct IsNaN_match {
  Op_t Op;

  IsNaN_match(const Op_t &OpMatch) : Op(OpMatch) {}

  template <typename OpTy> bool match(OpTy *V) const {
    using namespace llvm::PatternMatch;

    FCmpInst *FCI = dyn_cast<FCmpInst>(V);
    if (!FCI)
      return false;

    switch (FCI->getPredicate()) {
    case FCmpInst::FCMP_UNE:
      return FCI->getOperand(0) == FCI->getOperand(1) && Op.match(FCI->getOperand(0));
    case FCmpInst::FCMP_UNO:
      return m_Zero().match(FCI->getOperand(1)) && Op.match(FCI->getOperand(0));
    default:
      break;
    }

    return false;
  }
};

template <typename OpTy> inline IsNaN_match<OpTy> m_IsNaN(const OpTy &Op) { return IsNaN_match<OpTy>(Op); }

std::tuple<Value *, unsigned, VISA_Type> CodeGenPatternMatch::isFPToIntegerSatWithExactConstant(llvm::CastInst *I) {
  using namespace llvm::PatternMatch; // Scoped using declaration.

  unsigned Opcode = I->getOpcode();
  IGC_ASSERT(Opcode == Instruction::FPToSI || Opcode == Instruction::FPToUI);

  unsigned BitWidth = I->getDestTy()->getIntegerBitWidth();
  APFloat FMin(I->getSrcTy()->getFltSemantics());
  APFloat FMax(I->getSrcTy()->getFltSemantics());
  if (Opcode == Instruction::FPToSI) {
    if (FMax.convertFromAPInt(APInt::getSignedMaxValue(BitWidth), true, APFloat::rmNearestTiesToEven) != APFloat::opOK)
      return std::make_tuple(nullptr, 0, ISA_TYPE_F);
    if (FMin.convertFromAPInt(APInt::getSignedMinValue(BitWidth), true, APFloat::rmNearestTiesToEven) != APFloat::opOK)
      return std::make_tuple(nullptr, 0, ISA_TYPE_F);
  } else {
    if (FMax.convertFromAPInt(APInt::getMaxValue(BitWidth), false, APFloat::rmNearestTiesToEven) != APFloat::opOK)
      return std::make_tuple(nullptr, 0, ISA_TYPE_F);
    if (FMin.convertFromAPInt(APInt::getMinValue(BitWidth), false, APFloat::rmNearestTiesToEven) != APFloat::opOK)
      return std::make_tuple(nullptr, 0, ISA_TYPE_F);
  }

  llvm::ConstantFP *CMin = nullptr;
  llvm::ConstantFP *CMax = nullptr;
  llvm::Value *X = nullptr;

  if (!match(I->getOperand(0), m_ClampWithConstants(m_Value(X), CMin, CMax)))
    return std::make_tuple(nullptr, 0, ISA_TYPE_F);

  if (!CMin->isExactlyValue(FMin) || !CMax->isExactlyValue(FMax))
    return std::make_tuple(nullptr, 0, ISA_TYPE_F);

  return std::make_tuple(X, Opcode, GetType(I->getType(), m_ctx));
}

// The following pattern matching is targeted to the conversion from FP values
// to INTEGER values with saturation where the MAX and/or MIN INTEGER values
// cannot be represented in FP values exactly. E.g., UINT_MAX (2**32-1) in
// 'unsigned' cannot be represented in 'float', where only 23 significant bits
// are available but UINT_MAX needs 32 significant bits. We cannot simply
// express that conversion with saturation as
//
//  o := fptoui(clamp(x, float(UINT_MIN), float(UINT_MAX));
//
// as, in LLVM, fptoui is undefined when the 'unsigned' source cannot fit in
// 'float', where clamp(x, MIN, MAX) is defined as max(min(x, MAX), MIN),
//
// Hence, OCL use the following sequence (over-simplified by excluding the NaN
// case.)
//
//  o := select(fptoui(x), UINT_MIN, x < float(UINT_MIN));
//  o := select(o,         UINT_MAX, x > float(UINT_MAX));
//
// (We SHOULD use 'o := select(o, UINTMAX, x >= float(UINT_MAX))' as
// 'float(UINT_MAX)' will be rounded to UINT_MAX+1, i.e. 2 ** 32, and the next
// smaller value than float(UINT_MAX) in 'float' is (2 ** 24 - 1) << 8. For
// 'int', that's also true for INT_MIN.)

std::tuple<Value *, unsigned, VISA_Type>
CodeGenPatternMatch::isFPToSignedIntSatWithInexactConstant(llvm::SelectInst *SI) {
  using namespace llvm::PatternMatch; // Scoped using declaration.

  // TODO
  return std::make_tuple(nullptr, 0, ISA_TYPE_F);
}

std::tuple<Value *, unsigned, VISA_Type>
CodeGenPatternMatch::isFPToUnsignedIntSatWithInexactConstant(llvm::SelectInst *SI) {
  using namespace llvm::PatternMatch; // Scoped using declaration.

  Constant *C0 = dyn_cast<Constant>(SI->getTrueValue());
  if (!C0)
    return std::make_tuple(nullptr, 0, ISA_TYPE_F);
  if (!isa<ConstantFP>(C0) && !isa<ConstantInt>(C0))
    return std::make_tuple(nullptr, 0, ISA_TYPE_F);
  Value *Cond = SI->getCondition();

  SelectInst *SI2 = dyn_cast<SelectInst>(SI->getFalseValue());
  if (!SI2)
    return std::make_tuple(nullptr, 0, ISA_TYPE_F);
  Constant *C1 = dyn_cast<Constant>(SI2->getTrueValue());
  if (!C1)
    return std::make_tuple(nullptr, 0, ISA_TYPE_F);
  if (!isa<ConstantFP>(C1) && !isa<ConstantInt>(C1))
    return std::make_tuple(nullptr, 0, ISA_TYPE_F);
  Value *Cond2 = SI2->getCondition();

  Value *X = SI2->getFalseValue();
  Type *Ty = X->getType();
  if (Ty->isFloatTy()) {
    BitCastInst *BC = dyn_cast<BitCastInst>(X);
    if (!BC)
      return std::make_tuple(nullptr, 0, ISA_TYPE_F);
    X = BC->getOperand(0);
    Ty = X->getType();
    C1 = ConstantExpr::getBitCast(C1, Ty);
    C0 = ConstantExpr::getBitCast(C0, Ty);
  }
  IntegerType *ITy = dyn_cast<IntegerType>(Ty);
  if (!ITy)
    return std::make_tuple(nullptr, 0, ISA_TYPE_F);
  unsigned BitWidth = ITy->getBitWidth();
  FPToUIInst *CI = dyn_cast<FPToUIInst>(X);
  if (!CI)
    return std::make_tuple(nullptr, 0, ISA_TYPE_F);
  Ty = CI->getSrcTy();
  if (!(Ty->isFloatTy() && BitWidth == 32) && !(Ty->isDoubleTy() && BitWidth == 64))
    return std::make_tuple(nullptr, 0, ISA_TYPE_F);
  X = CI->getOperand(0);

  ConstantInt *CMin = dyn_cast<ConstantInt>(C0);
  ConstantInt *CMax = dyn_cast<ConstantInt>(C1);
  if (!CMax || !CMin || !CMax->isMaxValue(false) || !CMin->isMinValue(false))
    return std::make_tuple(nullptr, 0, ISA_TYPE_F);

  Constant *FMin = IGCLLVM::ConstantExpr::getUIToFP(CMin, Ty);
  Constant *FMax = IGCLLVM::ConstantExpr::getUIToFP(CMax, Ty);

  IGCLLVM::FCmpInstPredicate Pred(FCmpInst::FCMP_FALSE);
  if (!match(Cond2, m_FCmp(Pred, m_Specific(X), m_Specific(FMax))))
    return std::make_tuple(nullptr, 0, ISA_TYPE_F);
  if (Pred != FCmpInst::FCMP_OGT) // FIXME: We should use OGE instead of OGT.
    return std::make_tuple(nullptr, 0, ISA_TYPE_F);

  IGCLLVM::FCmpInstPredicate Pred2 = FCmpInst::FCMP_FALSE;
  if (!match(Cond, m_Or(m_FCmp(Pred, m_Specific(X), m_Specific(FMin)), m_FCmp(Pred2, m_Specific(X), m_Specific(X))))) {
    if (!match(Cond, m_Or(m_FCmp(Pred, m_Specific(X), m_Specific(FMin)), m_Zero()))) {
      return std::make_tuple(nullptr, 0, ISA_TYPE_F);
    }
    // Special case where the staturatured result is bitcasted into float
    // again (due to typedwrite only accepts `float`. So the isNaN(X) is
    // reduced to `false`.
    Pred2 = FCmpInst::FCMP_UNE;
  }
  if (Pred != FCmpInst::FCMP_OLT || Pred2 != FCmpInst::FCMP_UNE)
    return std::make_tuple(nullptr, 0, ISA_TYPE_F);

  VISA_Type type = GetType(CI->getType(), m_ctx);

  // Fold extra clamp.
  Value *X2 = nullptr;
  ConstantFP *CMin2 = nullptr;
  ConstantFP *CMax2 = nullptr;
  if (match(X, m_ClampWithConstants(m_Value(X2), CMin2, CMax2))) {
    if (CMin2 == FMin) {
      if (CMax2->isExactlyValue(255.0)) {
        X = X2;
        type = ISA_TYPE_B;
      } else if (CMax2->isExactlyValue(65535.0)) {
        X = X2;
        type = ISA_TYPE_W;
      }
    }
  }

  return std::make_tuple(X, Instruction::FPToUI, type);
}

bool CodeGenPatternMatch::MatchFPToIntegerWithSaturation(llvm::Instruction &I) {
  Value *X = nullptr;
  unsigned Opcode = 0;
  VISA_Type type = ISA_TYPE_NUM;

  if (CastInst *CI = dyn_cast<CastInst>(&I)) {
    std::tie(X, Opcode, type) = isFPToIntegerSatWithExactConstant(CI);
    if (!X)
      return false;
  } else if (SelectInst *SI = dyn_cast<SelectInst>(&I)) {
    std::tie(X, Opcode, type) = isFPToSignedIntSatWithInexactConstant(SI);
    if (!X) {
      std::tie(X, Opcode, type) = isFPToUnsignedIntSatWithInexactConstant(SI);
      if (!X)
        return false;
    }
  } else {
    return false;
  }

  // Match!
  IGC_ASSERT(Opcode == Instruction::FPToSI || Opcode == Instruction::FPToUI);

  struct FPToIntegerWithSaturationPattern : public Pattern {
    bool isUnsigned, needBitCast;
    VISA_Type type;
    SSource src;
    virtual void Emit(EmitPass *pass, const DstModifier &dstMod) {
      pass->EmitFPToIntWithSat(isUnsigned, needBitCast, type, src, dstMod);
    }
  };

  bool isUnsigned = (Opcode == Instruction::FPToUI);
  FPToIntegerWithSaturationPattern *pat = new (m_allocator) FPToIntegerWithSaturationPattern();
  pat->isUnsigned = isUnsigned;
  pat->needBitCast = !I.getType()->isIntegerTy();
  pat->type = type;
  pat->src = GetSource(X, !isUnsigned, false, IsSourceOfSample(&I));
  AddPattern(pat);

  return true;
}

std::tuple<Value *, bool, bool> CodeGenPatternMatch::isIntegerSatTrunc(llvm::SelectInst *SI) {
  using namespace llvm::PatternMatch; // Scoped using declaration.

  ICmpInst *Cmp = dyn_cast<ICmpInst>(SI->getOperand(0));
  if (!Cmp)
    return std::make_tuple(nullptr, false, false);

  IGCLLVM::ICmpInstPredicate Pred = Cmp->getPredicate();
  if (Pred != ICmpInst::ICMP_SGT && Pred != ICmpInst::ICMP_UGT)
    return std::make_tuple(nullptr, false, false);

  ConstantInt *CI = dyn_cast<ConstantInt>(Cmp->getOperand(1));
  if (!CI)
    return std::make_tuple(nullptr, false, false);

  // Truncate into unsigned integer by default.
  bool isSignedDst = false;
  unsigned DstBitWidth = SI->getType()->getIntegerBitWidth();
  unsigned SrcBitWidth = Cmp->getOperand(0)->getType()->getIntegerBitWidth();
  APInt UMax = APInt::getMaxValue(DstBitWidth);
  APInt UMin = APInt::getMinValue(DstBitWidth);
  APInt SMax = APInt::getSignedMaxValue(DstBitWidth);
  APInt SMin = APInt::getSignedMinValue(DstBitWidth);
  if (SrcBitWidth > DstBitWidth) {
    UMax = UMax.zext(SrcBitWidth);
    UMin = UMin.zext(SrcBitWidth);
    SMax = SMax.sext(SrcBitWidth);
    SMin = SMin.sext(SrcBitWidth);
  } else {
    // SrcBitwidth should be always wider than DstBitwidth,
    // since src is a source of a trunc instruction, and dst
    // have the same width as its destination.
    return std::make_tuple(nullptr, false, false);
  }

  if (CI->getValue() != UMax && CI->getValue() != SMax)
    return std::make_tuple(nullptr, false, false);
  if (CI->getValue() == SMax) // Truncate into signed integer.
    isSignedDst = true;

  APInt MinValue = isSignedDst ? SMin : UMin;
  CI = dyn_cast<ConstantInt>(SI->getOperand(1));
  if (!CI || !CI->isMaxValue(isSignedDst))
    return std::make_tuple(nullptr, false, false);

  TruncInst *TI = dyn_cast<TruncInst>(SI->getOperand(2));
  if (!TI)
    return std::make_tuple(nullptr, false, false);

  Value *Val = TI->getOperand(0);
  if (Val != Cmp->getOperand(0))
    return std::make_tuple(nullptr, false, false);

  // Truncate from unsigned integer.
  if (Pred == ICmpInst::ICMP_UGT)
    return std::make_tuple(Val, isSignedDst, false);

  // Truncate from signed integer. Need to check further for lower bound.
  Value *LHS = nullptr, *RHS = nullptr;
  if (!match(Val, m_SMax(m_Value(LHS), m_Value(RHS))))
    return std::make_tuple(nullptr, false, false);

  if (isa<ConstantInt>(LHS))
    std::swap(LHS, RHS);

  CI = dyn_cast<ConstantInt>(RHS);
  if (!CI || CI->getValue() != MinValue)
    return std::make_tuple(nullptr, false, false);

  return std::make_tuple(LHS, isSignedDst, true);
}

bool CodeGenPatternMatch::MatchIntegerTruncSatModifier(llvm::SelectInst &I) {
  // Only match BYTE or WORD.
  if (!I.getType()->isIntegerTy(8) && !I.getType()->isIntegerTy(16))
    return false;
  auto [Src, isSignedDst, isSignedSrc] = isIntegerSatTrunc(&I);
  if (!Src)
    return false;

  struct IntegerSatTruncPattern : public Pattern {
    SSource src;
    bool isSignedDst;
    bool isSignedSrc;
    virtual void Emit(EmitPass *pass, const DstModifier &dstMod) {
      pass->EmitIntegerTruncWithSat(isSignedDst, isSignedSrc, src, dstMod);
    }
  };

  IntegerSatTruncPattern *pat = new (m_allocator) IntegerSatTruncPattern();
  pat->src = GetSource(Src, isSignedSrc, false, IsSourceOfSample(&I));
  pat->isSignedDst = isSignedDst;
  pat->isSignedSrc = isSignedSrc;
  AddPattern(pat);

  return true;
}

bool CodeGenPatternMatch::MatchShrSatModifier(llvm::SelectInst &I) {
  struct ShrSatPattern : public Pattern {
    SSource sources[2];
    Instruction *inst;
    virtual void Emit(EmitPass *pass, const DstModifier &modifier) {
      DstModifier mod = modifier;
      mod.sat = true;
      pass->EmitSimpleAlu(GetOpCode(inst), sources, mod, true /*isUnsigned*/);
    }
  };

  Value *LHS = nullptr, *RHS = nullptr;
  if (isMax(&I, LHS, RHS)) {
    if (isa<ConstantInt>(LHS) || isa<ConstantInt>(RHS)) {
      ConstantInt *c = isa<ConstantInt>(LHS) ? cast<ConstantInt>(LHS) : cast<ConstantInt>(RHS);
      if (c->getZExtValue() == 0) {
        Value *v = isa<ConstantInt>(LHS) ? RHS : LHS;
        if (isa<LShrOperator, AShrOperator>(v)) {
          Instruction *inst = cast<Instruction>(v);
          ShrSatPattern *pattern = new (m_allocator) ShrSatPattern();
          pattern->inst = inst;
          pattern->sources[0] = GetSource(inst->getOperand(0), false, false, IsSourceOfSample(&I));
          pattern->sources[1] = GetSource(inst->getOperand(1), false, false, IsSourceOfSample(&I));
          AddPattern(pattern);

          return true;
        }
      }
    }
  }
  return false;
}

void CodeGenPatternMatch::visitFPToSIInst(llvm::FPToSIInst &I) {
  [[maybe_unused]] bool match = MatchFPToIntegerWithSaturation(I) || MatchModifier(I);
  IGC_ASSERT_MESSAGE(match, "Pattern match Failed");
}

void CodeGenPatternMatch::visitFPToUIInst(llvm::FPToUIInst &I) {
  [[maybe_unused]] bool match = MatchFPToIntegerWithSaturation(I) || MatchModifier(I);
  IGC_ASSERT_MESSAGE(match, "Pattern match Failed");
}

bool CodeGenPatternMatch::MatchSIToFPZExt(llvm::SIToFPInst *S2FI) {
  ZExtInst *ZEI = dyn_cast<ZExtInst>(S2FI->getOperand(0));
  if (!ZEI)
    return false;
  if (!ZEI->getSrcTy()->isIntegerTy(1))
    return false;

  struct SIToFPExtPattern : public Pattern {
    SSource src;
    virtual void Emit(EmitPass *pass, const DstModifier &dstMod) { pass->EmitSIToFPZExt(src, dstMod); }
  };

  SIToFPExtPattern *pat = new (m_allocator) SIToFPExtPattern();
  pat->src = GetSource(ZEI->getOperand(0), false, false, IsSourceOfSample(S2FI));
  AddPattern(pat);

  return true;
}

void CodeGenPatternMatch::visitCastInst(llvm::CastInst &I) {
  [[maybe_unused]] bool match = 0;
  if (I.getOpcode() == Instruction::SExt) {
    match = MatchUnpack4i8(I) || MatchCmpSext(I) || MatchModifier(I);
  } else if (I.getOpcode() == Instruction::ZExt) {
    match = MatchUnpack4i8(I) || MatchZExtByteLoad(I) || MatchModifier(I);
  } else if (I.getOpcode() == Instruction::SIToFP) {
    match = MatchSIToFPZExt(cast<SIToFPInst>(&I)) || MatchModifier(I);
  } else if (I.getOpcode() == Instruction::Trunc) {
    match = MatchModifier(I);
  } else {
    match = MatchModifier(I);
  }
}

bool CodeGenPatternMatch::NeedVMask() { return m_NeedVMask; }

bool CodeGenPatternMatch::HasUseOutsideLoop(llvm::Value *v) {
  if (Instruction *inst = dyn_cast<Instruction>(v)) {
    if (Loop *L = LI->getLoopFor(inst->getParent())) {
      for (auto UI = inst->user_begin(), E = inst->user_end(); UI != E; ++UI) {
        if (!L->contains(cast<Instruction>(*UI))) {
          return true;
        }
      }
    }
  }
  return false;
}

void CodeGenPatternMatch::HandleSubspanUse(llvm::Value *v, bool isSampleSource) {
  IGC_ASSERT(m_root != nullptr);
  if (m_ctx->type != ShaderType::PIXEL_SHADER) {
    return;
  }
  if (!isa<Constant>(v) && !m_WI->isUniform(v)) {
    if (isa<PHINode>(v) || HasUseOutsideLoop(v)) {
      // If a phi is used in a subspan we cannot propagate the subspan use and need to use VMask
      m_NeedVMask = true;

      if (isSampleSource && isa<PHINode>(v)) {
        m_sampleSource.insert(v);
      }

      if (Instruction *I = dyn_cast<Instruction>(v)) {
        // this is WA for situation where application has early return (not discard) from shader
        // in this case phi node will not set sample coords properly on all simd lanes
        // WA is to initialize register used for phi node with zero, which eliminates corruptions
        // here we collect suspected phi nodes to be resolved later in EmitVISAPass
        if (m_ctx->getModuleMetaData()->compOpt.initializePhiSampleSourceWA && isSampleSource &&
            !PDT->dominates(I->getParent(), &I->getFunction()->getEntryBlock())) {
          m_sampleUnderCFPHIsource.insert(v);
        }
      }
    } else {
      if (isa<SampleIntrinsic>(v)) {
        // Also, we want to apply vector mask instead of no mask to sample instructions,
        // because it should have better performance on workloads with many disabled subspans
        m_NeedVMask = true;
      }

      if (Instruction *I = dyn_cast<Instruction>(v)) {
        if (isSampleSource) {
          m_sampleSource.insert(v);
          if (!PDT->dominates(I->getParent(), &I->getFunction()->getEntryBlock())) {
            m_sampleUnderCFsource.insert(v);
          }

          if (Loop *L = LI->getLoopFor(I->getParent())) {
            if (L->contains(I)) {
              m_sampleUnderCFsource.insert(v);
            }
          }
        }
      }

      m_subSpanUse.insert(v);
      if (LoadInst *load = dyn_cast<LoadInst>(v)) {
        if (load->getPointerAddressSpace() == ADDRESS_SPACE_PRIVATE) {
          m_NeedVMask = true;
        }
      }
      if (HasPhiUse(*v) && m_WI->insideDivergentCF(m_root)) {
        // \todo, more accurate condition for force-isolation
        ForceIsolate(v);
      }
    }
  }
}

bool CodeGenPatternMatch::MatchMinMax(llvm::SelectInst &SI) {
  // Pattern to emit.
  struct MinMaxPattern : public Pattern {
    SSource srcs[2];
    bool isMin, isUnsigned;
    virtual void Emit(EmitPass *pass, const DstModifier &dstMod) {
      // FIXME: We should tell umax/umin from max/min as integers in LLVM
      // have no sign!
      pass->EmitMinMax(isMin, isUnsigned, srcs, dstMod);
    }
  };

  // Skip min/max pattern matching on FP, which needs to either explicitly
  // use intrinsics or convert them into intrinsic in GenIRLower pass.
  if (SI.getType()->isFloatingPointTy())
    return false;

  bool isMin = false, isUnsigned = false;
  llvm::Value *LHS = nullptr, *RHS = nullptr;

  if (!isMinOrMax(&SI, LHS, RHS, isMin, isUnsigned))
    return false;

  MinMaxPattern *pat = new (m_allocator) MinMaxPattern();
  // FIXME: We leave unsigned operand without source modifier so far. When
  // its behavior is detailed and correcty modeled, consider to add source
  // modifier support.
  pat->srcs[0] = GetSource(LHS, !isUnsigned, false, IsSourceOfSample(&SI));
  pat->srcs[1] = GetSource(RHS, !isUnsigned, false, IsSourceOfSample(&SI));
  pat->isMin = isMin;
  pat->isUnsigned = isUnsigned;
  AddPattern(pat);

  return true;
}

void CodeGenPatternMatch::visitSelectInst(SelectInst &I) {
  [[maybe_unused]] bool match = MatchFloatingPointSatModifier(I) || MatchIntegerTruncSatModifier(I) ||
                                MatchShrSatModifier(I) || MatchAbsNeg(I) || MatchFPToIntegerWithSaturation(I) ||
                                MatchMinMax(I) || MatchCmpSelect(I) || MatchSelectModifier(I);
  IGC_ASSERT_MESSAGE(match, "Pattern Match failed");
}

void CodeGenPatternMatch::visitBinaryOperator(llvm::BinaryOperator &I) {

  [[maybe_unused]] bool match = false;
  switch (I.getOpcode()) {
  case Instruction::FSub:
    match = MatchFloor(I) || MatchFrc(I) || MatchLrp(I) || MatchPredAdd(I) || MatchFMad(I) || MatchAbsNeg(I) ||
            MatchModifier(I);
    break;
  case Instruction::Sub:
    match = MatchIMad(I) || MatchAdd3(I) || MatchAbsNeg(I) || MatchMulAdd16(I) || MatchModifier(I);
    break;
  case Instruction::Mul:
    match = matchWideMul64Pair(I) || MatchFullMul32(I) || MatchMulAdd16(I) || MatchModifier(I);
    break;
  case Instruction::Add:
    match = MatchIMad(I) || MatchAdd3(I) || MatchMulAdd16(I) || MatchModifier(I);
    break;
  case Instruction::UDiv:
  case Instruction::SDiv:
    match = MatchAvg(I) || MatchModifier(I);
    break;
  case Instruction::AShr:
    match = MatchAvg(I) || MatchBinaryUnpack4i8(I) || MatchModifier(I);
    break;
  case Instruction::FMul:
    match = MatchArcpFdiv(I) || MatchModifier(I);
    break;
  case Instruction::URem:
  case Instruction::SRem:
  case Instruction::FRem:
    match = MatchModifier(I);
    break;
  case Instruction::Shl:
    match = MatchBinaryUnpack4i8(I) || MatchModifier(I);
    break;
  case Instruction::LShr:
    match = MatchBinaryUnpack4i8(I) || MatchModifier(I, false);
    break;
  case Instruction::FDiv:
    match = MatchRsqrt(I) || MatchModifier(I);
    break;
  case Instruction::FAdd:
    match = MatchLrp(I) || MatchPredAdd(I) || MatchFMad(I) || MatchSimpleAdd(I) || MatchModifier(I);
    break;
  case Instruction::And:
    match = MatchBitwiseI64Halves(I) || MatchBfn(I) || MatchBoolOp(I) || MatchLogicAlu(I);
    break;
  case Instruction::Or:
    match = MatchBitwiseI64Halves(I) || MatchPack4i8(I) || MatchBfn(I) || MatchBoolOp(I) || MatchLogicAlu(I);
    break;
  case Instruction::Xor:
    match = MatchBitwiseI64Halves(I) || MatchBfn(I) || MatchLogicAlu(I);
    break;
  default:
    IGC_ASSERT_MESSAGE(0, "unknown binary instruction");
    break;
  }
  IGC_ASSERT(match == true);
}

void CodeGenPatternMatch::visitCmpInst(llvm::CmpInst &I) {
  [[maybe_unused]] bool match = MatchCondModifier(I) || MatchModifier(I);
  IGC_ASSERT(match);
}

void CodeGenPatternMatch::visitBranchInst(llvm::BranchInst &I) { MatchBranch(I); }

void CodeGenPatternMatch::visitCallInst(CallInst &I) {
  [[maybe_unused]] bool match = false;
  using namespace GenISAIntrinsic;
  if (GenIntrinsicInst *GII = llvm::dyn_cast<GenIntrinsicInst>(&I)) {
    if (isSampleLoadGather4InfoInstruction(GII)) {
      match = MatchSample(*GII);
    } else {
      switch (GII->getIntrinsicID()) {
      case GenISAIntrinsic::GenISA_ROUNDNE:
      case GenISAIntrinsic::GenISA_imulH:
      case GenISAIntrinsic::GenISA_umulH:
      case GenISAIntrinsic::GenISA_uaddc:
      case GenISAIntrinsic::GenISA_usubb:
      case GenISAIntrinsic::GenISA_bfrev:
      case GenISAIntrinsic::GenISA_IEEE_Sqrt:
      case GenISAIntrinsic::GenISA_IEEE_Divide:
      case GenISAIntrinsic::GenISA_rsq:
      case GenISAIntrinsic::GenISA_inv:
      case GenISAIntrinsic::GenISA_tanh:
      case GenISAIntrinsic::GenISA_sigm:
        match = MatchModifier(I);
        break;
      case GenISAIntrinsic::GenISA_WideSMulHi:
      case GenISAIntrinsic::GenISA_WideUMulHi:
        match = matchWideMul64Pair(*GII) || MatchSingleInstruction(*GII);
        break;
      case GenISAIntrinsic::GenISA_intatomicraw:
      case GenISAIntrinsic::GenISA_floatatomicraw:
      case GenISAIntrinsic::GenISA_icmpxchgatomicraw:
      case GenISAIntrinsic::GenISA_fcmpxchgatomicraw:
      case GenISAIntrinsic::GenISA_atomiccounterinc:
      case GenISAIntrinsic::GenISA_atomiccounterpredec:
        if (m_Platform.hasEfficient64bEnabled()) {
          Value *pointerOperand = GII->getArgOperand(0);
          bool direct = false;
          unsigned bufId = 0;
          BufferType bufType = DecodeAS4GFXResource(pointerOperand->getType()->getPointerAddressSpace(), direct, bufId);
          if (IsBindless(bufType)) {
            match = MatchLoadStoreAtomicsStatefulEff64(GII);
            if (match)
              break;
          }
        }
        [[fallthrough]];
      case GenISAIntrinsic::GenISA_intatomicrawA64:
      case GenISAIntrinsic::GenISA_floatatomicrawA64:
      case GenISAIntrinsic::GenISA_icmpxchgatomicrawA64:
      case GenISAIntrinsic::GenISA_fcmpxchgatomicrawA64:
        if (m_Platform.hasEfficient64bEnabled()) {
          match = MatchLoadStoreAtomicsStatelessUniformBase(I);
          if (match)
            break;
        }
        if (supportsLSCImmediateGlobalBaseOffset()) {
          match = MatchImmOffsetLSC(I);
          if (match)
            return;
        }
        match = MatchSingleInstruction(I);
        break;
      case GenISAIntrinsic::GenISA_ldrawvector_indexed:
      case GenISAIntrinsic::GenISA_ldraw_indexed:
      case GenISAIntrinsic::GenISA_storerawvector_indexed:
      case GenISAIntrinsic::GenISA_storeraw_indexed:
        if (m_Platform.hasEfficient64bEnabled()) {
          match = MatchLoadStoreAtomicsStatefulEff64(GII);
          if (match)
            break;
        }
        if (supportsLSCImmediateGlobalBaseOffset()) {
          match = MatchImmOffsetLSC(I);
          if (match)
            break;
        }
        match = MatchSingleInstruction(I);
        break;
      case GenISAIntrinsic::GenISA_typedread:
      case GenISAIntrinsic::GenISA_typedwrite:
      case GenISAIntrinsic::GenISA_typedreadMS:
      case GenISAIntrinsic::GenISA_typedwriteMS:
      case GenISAIntrinsic::GenISA_intatomictyped:
      case GenISAIntrinsic::GenISA_icmpxchgatomictyped:
      case GenISAIntrinsic::GenISA_floatatomictyped:
      case GenISAIntrinsic::GenISA_fcmpxchgatomictyped:
        if (m_Platform.hasEfficient64bEnabled()) {
          match = MatchSampleLoadStoreAtomicTypedEff64(*GII);
          if (match)
            break;
        } else {
          match = MatchSingleInstruction(I);
          if (match)
            break;
        }
      case GenISAIntrinsic::GenISA_GradientX:
      case GenISAIntrinsic::GenISA_GradientY:
      case GenISAIntrinsic::GenISA_GradientXfine:
      case GenISAIntrinsic::GenISA_GradientYfine:
        match = MatchGradient(*GII);
        break;
      case GenISAIntrinsic::GenISA_fsat:
        match = MatchFloatingPointSatModifier(I);
        break;
      case GenISAIntrinsic::GenISA_usat:
      case GenISAIntrinsic::GenISA_isat:
        match = MatchIntegerSatModifier(I);
        break;
      case GenISAIntrinsic::GenISA_WaveShuffleIndex:
      case GenISAIntrinsic::GenISA_WaveBroadcast:
        match = MatchRegisterRegion(*GII) || MatchShuffleBroadCast(*GII) || MatchWaveShuffleIndex(*GII);
        break;
      case GenISAIntrinsic::GenISA_WaveBallot:
      case GenISAIntrinsic::GenISA_WaveInverseBallot:
      case GenISAIntrinsic::GenISA_WaveClusteredBallot:
      case GenISAIntrinsic::GenISA_WaveAll:
      case GenISAIntrinsic::GenISA_WaveClustered:
      case GenISAIntrinsic::GenISA_WaveInterleave:
      case GenISAIntrinsic::GenISA_WaveClusteredInterleave:
      case GenISAIntrinsic::GenISA_WavePrefix:
      case GenISAIntrinsic::GenISA_WaveClusteredPrefix:
        match = MatchWaveInstruction(*GII);
        break;
      case GenISAIntrinsic::GenISA_simdBlockRead:
      case GenISAIntrinsic::GenISA_simdBlockWrite:
        match = MatchBlockReadWritePointer(*GII) || MatchSingleInstruction(*GII);
        break;
      case GenISAIntrinsic::GenISA_UnmaskedRegionBegin:
        match = MatchUnmaskedRegionBoundary(I, true);
        break;
      case GenISAIntrinsic::GenISA_UnmaskedRegionEnd:
        match = MatchUnmaskedRegionBoundary(I, false);
        break;
      case GenISAIntrinsic::GenISA_sub_group_dpas:
      case GenISAIntrinsic::GenISA_dpas:
        match = MatchDpas(*GII);
        break;
      case GenISAIntrinsic::GenISA_dp4a_ss:
      case GenISAIntrinsic::GenISA_dp4a_su:
      case GenISAIntrinsic::GenISA_dp4a_us:
      case GenISAIntrinsic::GenISA_dp4a_uu:
        match = MatchDp4a(*GII);
        break;
      default:
        match = MatchSingleInstruction(I);
        // no pattern for the rest of the intrinsics
        break;
      }
    }
    IGC_ASSERT_MESSAGE(match, "no pattern found for GenISA intrinsic");
  } else {
    Function *Callee = I.getCalledFunction();

    // Match inline asm
    if (I.isInlineAsm()) {
      if (getAnalysis<CodeGenContextWrapper>().getCodeGenContext()->m_DriverInfo.SupportInlineAssembly()) {
        match = MatchSingleInstruction(I);
      }
    }
    // Match indirect call, support declarations for indirect funcs
    else if (!Callee || Callee->hasFnAttribute("referenced-indirectly") ||
             Callee->hasFnAttribute("invoke_simd_target")) {
      match = MatchSingleInstruction(I);
    }
    // Match direct call, skip declarations
    else if (!Callee->isDeclaration()) {
      match = MatchSingleInstruction(I);
    }
  }
  IGC_ASSERT_MESSAGE(match, "no match for this call");
}

void CodeGenPatternMatch::visitUnaryInstruction(llvm::UnaryInstruction &I) {
  [[maybe_unused]] bool match = false;
  switch (I.getOpcode()) {
  case Instruction::Alloca:
  case Instruction::Load:
    match = MatchSingleInstruction(I);
    break;
  case Instruction::FNeg:
    match = MatchAbsNeg(I);
    break;
  }
  IGC_ASSERT(match);
}

void CodeGenPatternMatch::visitIntrinsicInst(llvm::IntrinsicInst &I) {
  [[maybe_unused]] bool match = false;
  switch (I.getIntrinsicID()) {
  case Intrinsic::sqrt:
  case Intrinsic::log2:
  case Intrinsic::cos:
  case Intrinsic::sin:
  case Intrinsic::pow:
  case Intrinsic::powi:
  case Intrinsic::floor:
  case Intrinsic::ceil:
  case Intrinsic::trunc:
  case Intrinsic::ctpop:
  case Intrinsic::ctlz:
  case Intrinsic::cttz:
    match = MatchModifier(I);
    break;
  case Intrinsic::exp2:
    match = MatchPow(I) || MatchModifier(I);
    break;
  case Intrinsic::abs:
  case Intrinsic::fabs:
    match = MatchAbsNeg(I);
    break;
  case Intrinsic::fma:
    match = MatchFMA(I);
    break;
  case Intrinsic::maxnum:
  case Intrinsic::minnum:
    match = MatchFloatingPointSatModifier(I) || MatchModifier(I);
    break;
  case Intrinsic::fshl:
  case Intrinsic::fshr:
    match = MatchFunnelShiftRotate(I);
    break;
  case Intrinsic::canonicalize:
    match = MatchCanonicalizeInstruction(I);
    break;
  default:
    match = MatchSingleInstruction(I);
    // no pattern for the rest of the intrinsics
    break;
  }
  IGC_ASSERT_MESSAGE(match, "no pattern found");
}

void CodeGenPatternMatch::visitStoreInst(StoreInst &I) {
  [[maybe_unused]] bool match = false;
  if (m_Platform.hasEfficient64bEnabled()) {
    match = MatchLoadStoreAtomicsStatelessUniformBase(I);
    if (match)
      return;
  }
  if (supportsLSCImmediateGlobalBaseOffset()) {
    match = MatchImmOffsetLSC(I);
    if (match)
      return;
  }
  match = MatchSingleInstruction(I);
  IGC_ASSERT(match);
}

void CodeGenPatternMatch::visitLoadInst(LoadInst &I) {
  [[maybe_unused]] bool match = false;
  if (m_Platform.hasEfficient64bEnabled()) {
    match = MatchLoadStoreAtomicsStatelessUniformBase(I);
    if (match)
      return;
  }
  if (supportsLSCImmediateGlobalBaseOffset()) {
    match = MatchImmOffsetLSC(I);
    if (match)
      return;
  }
  match = MatchSingleInstruction(I);
  IGC_ASSERT(match);
}

void CodeGenPatternMatch::visitInstruction(llvm::Instruction &I) {
  // use default pattern
  MatchSingleInstruction(I);
}

void CodeGenPatternMatch::visitExtractElementInst(llvm::ExtractElementInst &I) {
  Value *VecOpnd = I.getVectorOperand();
  if (isa<Constant>(VecOpnd)) {
    const Function *F = I.getParent()->getParent();
    unsigned NUse = 0;
    for (auto User : VecOpnd->users()) {
      if (auto Inst = dyn_cast<Instruction>(User)) {
        NUse += (Inst->getParent()->getParent() == F);
      }
    }

    // Only add it to pool when there are multiple uses within this
    // function; otherwise no benefit but to hurt RP.
    if (NUse > 1)
      AddToConstantPool(I.getParent(), VecOpnd);
  }
  MatchSingleInstruction(I);
}

void CodeGenPatternMatch::visitPHINode(PHINode &I) {
  struct PHIPattern : Pattern {
    // bool isInDivergentLoop = false;
    llvm::PHINode *phi = nullptr;
    virtual void Emit(EmitPass *pass, const DstModifier &modifier) {
      IGC_ASSERT(modifier.sat == false);
      IGC_ASSERT(modifier.flag == nullptr);
      pass->EmitInitializePHI(phi);
    }
  };

  if (Loop *L = LI->getLoopFor(I.getParent())) {
    auto &exitingBlocks = LoopExitingBlocksCache[L];
    if (exitingBlocks.empty()) {
      L->getExitingBlocks(exitingBlocks);
    }
    SmallPtrSet<Instruction *, 8> visitedInstructions;
    for (auto *exitingBlock : exitingBlocks) {
      if (!isUniform(exitingBlock->getTerminator())) {
        std::vector<Instruction *> worklist;
        worklist.push_back(&I);
        while (!worklist.empty()) {
          Instruction *currentInst = worklist.back();
          worklist.pop_back();
          for (auto operand : currentInst->operand_values()) {
            if (auto nextInst = dyn_cast<Instruction>(operand)) {
              if (nextInst == &I) {
                return;
              }
              if (L->contains(nextInst->getParent()) && !visitedInstructions.contains(nextInst)) {
                worklist.push_back(nextInst);
                visitedInstructions.insert(nextInst);
              }
            }
          }
        }
        break;
      }
    }
  }
  if (IsSourceOfSample(&I)) {
    PHIPattern *pattern = new (m_allocator) PHIPattern();
    pattern->phi = &I;
    // pattern->isInDivergentLoop = isInDivergentLoop;
    AddPattern(pattern);
  }
}

void CodeGenPatternMatch::visitBitCastInst(BitCastInst &I) {
  // detect
  // %66 = insertelement <2 x i32> <i32 0, i32 undef>, i32 %xor19.i, i32 1
  // %67 = bitcast <2 x i32> % 66 to i64
  // and replace it with a shl 32
  struct Shl32Pattern : public Pattern {
    SSource sources[2];
    virtual void Emit(EmitPass *pass, const DstModifier &modifier) { pass->Binary(EOPCODE_SHL, sources, modifier); }
  };

  if (m_Platform.hasPartialInt64Support() && I.getType()->isIntegerTy(64) && I.getOperand(0)->getType()->isVectorTy() &&
      cast<VectorType>(I.getOperand(0)->getType())->getElementType()->isIntegerTy(32)) {
    if (auto IEI = dyn_cast<InsertElementInst>(I.getOperand(0))) {
      auto vec = dyn_cast<ConstantVector>(IEI->getOperand(0));
      bool isCandidate =
          vec && vec->getNumOperands() == 2 && IsZero(vec->getOperand(0)) && isa<UndefValue>(vec->getOperand(1));
      auto index = dyn_cast<ConstantInt>(IEI->getOperand(2));
      isCandidate &= index && index->getZExtValue() == 1;
      if (isCandidate) {
        Shl32Pattern *Pat = new (m_allocator) Shl32Pattern();
        Pat->sources[0] = GetSource(IEI->getOperand(1), false, false, IsSourceOfSample(&I));
        Pat->sources[1] =
            GetSource(ConstantInt::get(Type::getInt32Ty(I.getContext()), 32), false, false, IsSourceOfSample(&I));
        AddPattern(Pat);
        return;
      }
    }
  }
  if (matchMulPairToI64(I))
    return;

  [[maybe_unused]] bool match = false;
  match = MatchPackBfnI64(I) || MatchPackI64(I) || MatchRepack4i8(I) || MatchPack4i8(I) || MatchSingleInstruction(I);
  IGC_ASSERT_MESSAGE(match, "Unsupported BitCast instruction");
}

void CodeGenPatternMatch::visitIntToPtrInst(IntToPtrInst &I) { MatchSingleInstruction(I); }

void CodeGenPatternMatch::visitPtrToIntInst(PtrToIntInst &I) { MatchSingleInstruction(I); }

void CodeGenPatternMatch::visitAddrSpaceCast(AddrSpaceCastInst &I) { MatchSingleInstruction(I); }

void CodeGenPatternMatch::visitDbgInfoIntrinsic(DbgInfoIntrinsic &I) { MatchDbgInstruction(I); }

void CodeGenPatternMatch::visitExtractValueInst(ExtractValueInst &I) {
  [[maybe_unused]] bool Match = false;

  // Ignore the extract value instruction. Handled in the call inst.
  if (CallInst *call = dyn_cast<CallInst>(I.getOperand(0))) {
    if (call->isInlineAsm() && call->getType()->isStructTy()) {
      MarkAsSource(call, IsSourceOfSample(&I));
      return;
    }
  }

  Match = matchAddPair(&I) || matchSubPair(&I) || matchMulPair(&I) || matchPtrToPair(&I) || MatchSingleInstruction(I);

  IGC_ASSERT_MESSAGE(Match, "Unknown `extractvalue` instruction!");
}

bool CodeGenPatternMatch::matchAddPair(ExtractValueInst *Ex) {
  Value *V = Ex->getOperand(0);
  GenIntrinsicInst *GII = dyn_cast<GenIntrinsicInst>(V);
  if (!GII || GII->getIntrinsicID() != GenISAIntrinsic::GenISA_add_pair)
    return false;

  if (Ex->getNumIndices() != 1)
    return false;
  unsigned Idx = *Ex->idx_begin();
  if (Idx != 0 && Idx != 1)
    return false;

  struct AddPairPattern : public Pattern {
    GenIntrinsicInst *GII;
    SSource Sources[4]; // L0, H0, L1, H1
    virtual void Emit(EmitPass *Pass, const DstModifier &DstMod) { Pass->EmitAddPair(GII, Sources, DstMod); }
  };

  struct AddPairSubPattern : public Pattern {
    virtual void Emit(EmitPass *Pass, const DstModifier &Mod) {
      // DO NOTHING. Dummy pattern.
    }
  };

  auto [MI, New] = PairOutputMap.insert(std::make_pair(GII, PairOutputTy()));
  if (New) {
    AddPairPattern *Pat = new (m_allocator) AddPairPattern();
    Pat->GII = GII;
    Pat->Sources[0] = GetSource(GII->getOperand(0), false, false, IsSourceOfSample(Ex));
    Pat->Sources[1] = GetSource(GII->getOperand(1), false, false, IsSourceOfSample(Ex));
    Pat->Sources[2] = GetSource(GII->getOperand(2), false, false, IsSourceOfSample(Ex));
    Pat->Sources[3] = GetSource(GII->getOperand(3), false, false, IsSourceOfSample(Ex));
    AddPattern(Pat);
  } else {
    AddPairSubPattern *Pat = new (m_allocator) AddPairSubPattern();
    AddPattern(Pat);
  }
  if (Idx == 0)
    MI->second.first = Ex;
  else
    MI->second.second = Ex;

  return true;
}

bool CodeGenPatternMatch::matchSubPair(ExtractValueInst *Ex) {
  Value *V = Ex->getOperand(0);
  GenIntrinsicInst *GII = dyn_cast<GenIntrinsicInst>(V);
  if (!GII || GII->getIntrinsicID() != GenISAIntrinsic::GenISA_sub_pair)
    return false;

  if (Ex->getNumIndices() != 1)
    return false;
  unsigned Idx = *Ex->idx_begin();
  if (Idx != 0 && Idx != 1)
    return false;

  struct SubPairPattern : public Pattern {
    GenIntrinsicInst *GII;
    SSource Sources[4]; // L0, H0, L1, H1
    virtual void Emit(EmitPass *Pass, const DstModifier &DstMod) { Pass->EmitSubPair(GII, Sources, DstMod); }
  };

  struct SubPairSubPattern : public Pattern {
    virtual void Emit(EmitPass *Pass, const DstModifier &Mod) {
      // DO NOTHING. Dummy pattern.
    }
  };

  auto [MI, New] = PairOutputMap.insert(std::make_pair(GII, PairOutputTy()));
  if (New) {
    SubPairPattern *Pat = new (m_allocator) SubPairPattern();
    Pat->GII = GII;
    Pat->Sources[0] = GetSource(GII->getOperand(0), false, false, IsSourceOfSample(Ex));
    Pat->Sources[1] = GetSource(GII->getOperand(1), false, false, IsSourceOfSample(Ex));
    Pat->Sources[2] = GetSource(GII->getOperand(2), false, false, IsSourceOfSample(Ex));
    Pat->Sources[3] = GetSource(GII->getOperand(3), false, false, IsSourceOfSample(Ex));
    AddPattern(Pat);
  } else {
    SubPairSubPattern *Pat = new (m_allocator) SubPairSubPattern();
    AddPattern(Pat);
  }
  if (Idx == 0)
    MI->second.first = Ex;
  else
    MI->second.second = Ex;

  return true;
}

bool CodeGenPatternMatch::matchMulPairToI64(BitCastInst &I) {
  using namespace llvm::PatternMatch;

  // Match: bitcast <2 x i32> (insertelement(insertelement(undef,
  //   extractvalue(mul_pair, 0), 0), extractvalue(mul_pair, 1), 1)) to i64
  // This pattern packs the mul_pair lo/hi results directly into an i64
  // variable using stride-2 regioning, avoiding intermediate mov instructions.
  if (!I.getType()->isIntegerTy(64))
    return false;

  // We do not handle uniform case here
  if (isUniform(&I))
    return false;

  Value *MulPairLo = nullptr;
  Value *MulPairHi = nullptr;
  // Match the nested insertelement/extractvalue structure:
  //   bitcast(insertelement(insertelement(undef, ev0, 0), ev1, 1))
  // where ev0 = extractvalue(X, 0), ev1 = extractvalue(Y, 1)
  if (!match(I.getOperand(0),
             m_OneUse(m_InsertElt(
                 m_OneUse(m_InsertElt(m_Undef(), m_OneUse(m_ExtractValue<0>(m_Value(MulPairLo))), m_SpecificInt(0))),
                 m_OneUse(m_ExtractValue<1>(m_Value(MulPairHi))), m_SpecificInt(1)))))
    return false;

  // Both extractvalue instructions must reference the same mul_pair intrinsic.
  if (MulPairLo != MulPairHi)
    return false;
  auto *GII = dyn_cast<GenIntrinsicInst>(MulPairLo);
  if (!GII || GII->getIntrinsicID() != GenISAIntrinsic::GenISA_mul_pair)
    return false;

  // All matched instructions must be in the same basic block.
  if (GII->getParent() != I.getParent())
    return false;

  struct MulPairToI64Pattern : public Pattern {
    GenIntrinsicInst *GII;
    SSource Sources[4]; // L0, H0, L1, H1
    virtual void Emit(EmitPass *Pass, const DstModifier &DstMod) { Pass->EmitMulPairToI64(GII, Sources, DstMod); }
  };

  MulPairToI64Pattern *Pat = new (m_allocator) MulPairToI64Pattern();
  Pat->GII = GII;
  Pat->Sources[0] = GetSource(GII->getOperand(0), false, false, IsSourceOfSample(&I));
  Pat->Sources[1] = GetSource(GII->getOperand(1), false, false, IsSourceOfSample(&I));
  Pat->Sources[2] = GetSource(GII->getOperand(2), false, false, IsSourceOfSample(&I));
  Pat->Sources[3] = GetSource(GII->getOperand(3), false, false, IsSourceOfSample(&I));
  AddPattern(Pat);
  return true;
}

bool CodeGenPatternMatch::matchMulPair(ExtractValueInst *Ex) {
  Value *V = Ex->getOperand(0);
  GenIntrinsicInst *GII = dyn_cast<GenIntrinsicInst>(V);
  if (!GII || GII->getIntrinsicID() != GenISAIntrinsic::GenISA_mul_pair)
    return false;

  if (Ex->getNumIndices() != 1)
    return false;
  unsigned Idx = *Ex->idx_begin();
  if (Idx != 0 && Idx != 1)
    return false;

  struct MulPairPattern : public Pattern {
    GenIntrinsicInst *GII;
    SSource Sources[4]; // L0, H0, L1, H1
    virtual void Emit(EmitPass *Pass, const DstModifier &DstMod) { Pass->EmitMulPair(GII, Sources, DstMod); }
  };

  struct MulPairSubPattern : public Pattern {
    virtual void Emit(EmitPass *Pass, const DstModifier &Mod) {
      // DO NOTHING. Dummy pattern.
    }
  };

  auto [MI, New] = PairOutputMap.insert(std::make_pair(GII, PairOutputTy()));
  if (New) {
    MulPairPattern *Pat = new (m_allocator) MulPairPattern();
    Pat->GII = GII;
    Pat->Sources[0] = GetSource(GII->getOperand(0), false, false, IsSourceOfSample(Ex));
    Pat->Sources[1] = GetSource(GII->getOperand(1), false, false, IsSourceOfSample(Ex));
    Pat->Sources[2] = GetSource(GII->getOperand(2), false, false, IsSourceOfSample(Ex));
    Pat->Sources[3] = GetSource(GII->getOperand(3), false, false, IsSourceOfSample(Ex));
    AddPattern(Pat);
  } else {
    MulPairSubPattern *Pat = new (m_allocator) MulPairSubPattern();
    AddPattern(Pat);
  }
  if (Idx == 0)
    MI->second.first = Ex;
  else
    MI->second.second = Ex;

  return true;
}

bool CodeGenPatternMatch::matchPtrToPair(ExtractValueInst *Ex) {
  Value *V = Ex->getOperand(0);
  GenIntrinsicInst *GII = dyn_cast<GenIntrinsicInst>(V);
  if (!GII || GII->getIntrinsicID() != GenISAIntrinsic::GenISA_ptr_to_pair)
    return false;

  if (Ex->getNumIndices() != 1)
    return false;
  unsigned Idx = *Ex->idx_begin();
  if (Idx != 0 && Idx != 1)
    return false;

  struct PtrToPairPattern : public Pattern {
    GenIntrinsicInst *GII;
    SSource Sources[1]; // Ptr
    virtual void Emit(EmitPass *Pass, const DstModifier &DstMod) { Pass->EmitPtrToPair(GII, Sources, DstMod); }
  };

  struct PtrToPairSubPattern : public Pattern {
    virtual void Emit(EmitPass *Pass, const DstModifier &Mod) {
      // DO NOTHING. Dummy pattern.
    }
  };

  auto [MI, New] = PairOutputMap.insert(std::make_pair(GII, PairOutputTy()));
  if (New) {
    PtrToPairPattern *Pat = new (m_allocator) PtrToPairPattern();
    Pat->GII = GII;
    Pat->Sources[0] = GetSource(GII->getOperand(0), false, false, IsSourceOfSample(Ex));
    AddPattern(Pat);
  } else {
    PtrToPairSubPattern *Pat = new (m_allocator) PtrToPairSubPattern();
    AddPattern(Pat);
  }
  if (Idx == 0)
    MI->second.first = Ex;
  else
    MI->second.second = Ex;

  return true;
}
bool CodeGenPatternMatch::matchWideMul64Pair(Instruction &I) {
  if (!m_Platform.hasWideMulMad()) {
    return false;
  }

  EOPCODE opCode = GetOpCode(&I);
  if (opCode != llvm_mul && opCode != llvm_wideUMulHi && opCode != llvm_wideSMulHi) {
    return false;
  }

  struct WideMulPattern : public Pattern {
    SSource sources[2];
    Instruction *mul64Lo;
    GenIntrinsicInst *mul64Hi;
    bool isUnsigned;
    virtual void Emit(EmitPass *Pass, const DstModifier &DstMod) {
      Pass->emitWideMul(mul64Lo, sources, EmitPass::ResultScope::All, isUnsigned, mul64Hi);
    }
  };

  struct WideMulSubPattern : public Pattern {
    virtual void Emit(EmitPass *Pass, const DstModifier &Mod) {
      // DO NOTHING. Dummy pattern.
    }
  };

  // Matching 64bit dst instructions only.
  Type *Ty = I.getType();
  if (!(Ty->isIntegerTy(64)))
    return false;

  Value *src0 = I.getOperand(0);
  Value *src1 = I.getOperand(1);

  for (User *user : src0->users()) {
    Instruction *anotherInst = dyn_cast<Instruction>(user);
    if (!anotherInst)
      continue;
    EOPCODE anotherInstOpCode = GetOpCode(anotherInst);
    if ((anotherInstOpCode != llvm_mul && anotherInstOpCode != llvm_wideUMulHi &&
         anotherInstOpCode != llvm_wideSMulHi) ||
        anotherInst->getNumOperands() < 2 || src0 != anotherInst->getOperand(0) || src1 != anotherInst->getOperand(1)) {
      continue;
    }

    GenIntrinsicInst *wideInstHi = nullptr;
    Instruction *wideInstLo = nullptr;
    EOPCODE opCodeHi, opCodeLo;
    switch (opCode) {
    case llvm_mul:
      wideInstHi = dyn_cast<GenIntrinsicInst>(anotherInst);
      opCodeHi = anotherInstOpCode;
      wideInstLo = &I;
      opCodeLo = opCode;
      break;
    case llvm_wideUMulHi:
    case llvm_wideSMulHi:
      wideInstHi = dyn_cast<GenIntrinsicInst>(&I);
      opCodeHi = opCode;
      wideInstLo = anotherInst;
      opCodeLo = anotherInstOpCode;
      break;
    default:
      IGC_ASSERT(0);
      return false;
    }

    if (opCodeLo != llvm_mul || (opCodeHi != llvm_wideSMulHi && opCodeHi != llvm_wideUMulHi)) {
      continue;
    }
    if (!wideInstHi) {
      IGC_ASSERT(0);
      continue;
    }

    auto [MI, New] = WidePairMap.insert(std::make_pair(wideInstHi, wideInstLo));
    if (New) {
      WideMulPattern *pattern = new (m_allocator) WideMulPattern();
      pattern->isUnsigned = (opCodeHi == llvm_wideUMulHi);
      pattern->mul64Lo = wideInstLo;
      pattern->mul64Hi = wideInstHi;
      pattern->sources[0] = GetSource(src0, false, false, IsSourceOfSample(&I));
      pattern->sources[1] = GetSource(src1, false, false, IsSourceOfSample(&I));
      AddPattern(pattern);
      return true;
    } else {
      WideMulSubPattern *pattern = new (m_allocator) WideMulSubPattern();
      AddPattern(pattern);
      return true;
    }
  }

  return false;
}

bool CodeGenPatternMatch::MatchAbsNeg(llvm::Instruction &I) {
  struct MovModifierPattern : public Pattern {
    SSource source;
    virtual void Emit(EmitPass *pass, const DstModifier &modifier) { pass->Mov(source, modifier); }
  };
  [[maybe_unused]] bool match = false;
  e_modifier mod{};
  Value *source = nullptr;
  if (GetModifier(I, mod, source)) {
    MovModifierPattern *pattern = new (m_allocator) MovModifierPattern();
    pattern->source = GetSource(source, mod, false, IsSourceOfSample(&I));
    match = true;
    AddPattern(pattern);
  }
  return match;
}

bool CodeGenPatternMatch::MatchFrc(llvm::BinaryOperator &I) {
  if (!ContractionAllowed(I))
    return false;
  struct FrcPattern : public Pattern {
    SSource source;
    void Emit(EmitPass *pass, const DstModifier &modifier) override { pass->Frc(source, modifier); }

    bool supportsSaturate() override { return false; }
  };
  IGC_ASSERT(I.getOpcode() == Instruction::FSub);
  llvm::Value *source0 = I.getOperand(0);
  llvm::IntrinsicInst *source1 = llvm::dyn_cast<llvm::IntrinsicInst>(I.getOperand(1));
  bool found = false;
  if (source1 && source1->getIntrinsicID() == Intrinsic::floor) {
    if (source1->getOperand(0) == source0) {
      found = true;
    }
  }
  if (found) {
    FrcPattern *pattern = new (m_allocator) FrcPattern();
    pattern->source = GetSource(source0, true, false, IsSourceOfSample(&I));
    AddPattern(pattern);
  }
  return found;
}

/*
below pass handles x - frac(x) = floor(x) pattern. Refer below :

frc (8|M0) r20.0<1>:f r19.0<8;8,1>:f {Compacted, @1}
add (8|M0) (ge)f0.1 r19.0<1>:f r19.0<8;8,1>:f -r20.0<8;8,1>:f {@1}
rndd (8|M0) r21.0<1>:f (abs)r19.0<8;8,1>:f {Compacted, @1}
*/

bool CodeGenPatternMatch::MatchFloor(llvm::BinaryOperator &I) {
  if (IGC_IS_FLAG_ENABLED(DisableMatchFloor)) {
    return false;
  }
  struct FloorPattern : public Pattern {
    SSource source;
    virtual void Emit(EmitPass *pass, const DstModifier &modifier) { pass->Floor(source, modifier); }
  };
  IGC_ASSERT(I.getOpcode() == Instruction::FSub);
  llvm::Value *source0 = I.getOperand(0);
  GenIntrinsicInst *source1 = dyn_cast<GenIntrinsicInst>(I.getOperand(1));
  bool found = false;
  if (source1 && source1->getIntrinsicID() == GenISAIntrinsic::GenISA_frc) {
    if (source1->getOperand(0) == source0) {
      found = true;
    }
  }
  if (found) {
    FloorPattern *pattern = new (m_allocator) FloorPattern();
    pattern->source = GetSource(source0, true, false, IsSourceOfSample(&I));
    AddPattern(pattern);
  }
  return found;
}

SSource CodeGenPatternMatch::GetSource(llvm::Value *value, bool modifier, bool regioning, bool isSampleDerivative) {
  llvm::Value *sourceValue = value;
  e_modifier mod = EMOD_NONE;
  if (modifier) {
    GetModifier(*sourceValue, mod, sourceValue);
  }
  return GetSource(sourceValue, mod, regioning, isSampleDerivative);
}

SSource CodeGenPatternMatch::GetSource(llvm::Value *value, e_modifier mod, bool regioning, bool isSampleDerivative) {
  SSource source;
  GetRegionModifier(source, value, regioning);
  source.value = value;
  source.mod = mod;
  MarkAsSource(value, isSampleDerivative);
  return source;
}

void CodeGenPatternMatch::MarkAsSource(llvm::Value *v, bool isSampleDerivative) {
  // update liveness of the sources
  if (IsConstOrSimdConstExpr(v)) {
    // skip constant
    return;
  }
  if (isa<Instruction, Argument>(v)) {
    m_LivenessInfo->HandleVirtRegUse(v, m_root->getParent(), m_root);
  }
  // mark the source as used so that we know we need to generate this value
  if (llvm::Instruction *inst = llvm::dyn_cast<Instruction>(v)) {
    m_usedInstructions.insert(inst);
  }
  if (m_rootIsSubspanUse) {
    HandleSubspanUse(v, isSampleDerivative);
  }
}

bool CodeGenPatternMatch::IsSubspanUse(llvm::Value *v) { return m_subSpanUse.find(v) != m_subSpanUse.end(); }

bool CodeGenPatternMatch::IsSourceOfSample(llvm::Value *v) { return m_sampleSource.find(v) != m_sampleSource.end(); }

bool CodeGenPatternMatch::IsSourceOfSampleUnderCF(llvm::Value *v) {
  return m_sampleUnderCFsource.find(v) != m_sampleUnderCFsource.end();
}

bool CodeGenPatternMatch::IsPHISourceOfSampleUnderCF(llvm::Value *v) {
  return m_sampleUnderCFPHIsource.find(v) != m_sampleUnderCFPHIsource.end();
}

bool CodeGenPatternMatch::MatchFMA(llvm::IntrinsicInst &I) {
  struct FMAPattern : Pattern {
    SSource sources[3];
    virtual void Emit(EmitPass *pass, const DstModifier &modifier) { pass->Mad(sources, modifier); }
  };

  FMAPattern *pattern = new (m_allocator) FMAPattern();
  for (int i = 0; i < 3; i++) {
    llvm::Value *V = I.getOperand(i);
    pattern->sources[i] = GetSource(V, true, false, IsSourceOfSample(&I));
    if (isa<Constant>(V) &&
        (!m_Platform.support16BitImmSrcForMad() || V->getType()->getTypeID() != llvm::Type::HalfTyID || i == 1)) {
      // CNL+ mad instruction allows 16 bit immediate for src0 and src2
      AddToConstantPool(I.getParent(), V);
      pattern->sources[i].fromConstantPool = true;
    }
  }
  AddPattern(pattern);

  return true;
}

bool CodeGenPatternMatch::MatchPredAdd(llvm::BinaryOperator &I) {
  struct PredAddPattern : Pattern {
    SSource sources[2];
    SSource pred;
    e_predMode predMode;
    bool invertPred;
    void Emit(EmitPass *pass, const DstModifier &modifier) override {
      DstModifier modf = modifier;
      modf.predMode = predMode;
      pass->PredAdd(pred, invertPred, sources, modf);
    }

    // Cannot use the ".sat" modifier as it is applied only to data lanes enabled by the predication
    // while saturation operation matched MatchFloatingPointSatModifier should be executed
    // irrespective of the predicate value matched by MatchPredAdd.
    bool supportsSaturate() override { return false; }
  };

  if (m_ctx->getModuleMetaData()->isPrecise) {
    return false;
  }

  if (m_ctx->type == ShaderType::VERTEX_SHADER || !m_ctx->m_DriverInfo.SupportMatchPredAdd()) {
    return false;
  }

  bool found = false;

  llvm::Value *sources[2] = {nullptr, nullptr};
  llvm::Value *pred = nullptr;
  e_modifier src_mod[2] = {e_modifier::EMOD_NONE, e_modifier::EMOD_NONE};
  e_modifier pred_mod = e_modifier::EMOD_NONE;
  bool invertPred = false;
  if (!ContractionAllowed(I) || IGC_IS_FLAG_ENABLED(DisableMatchPredAdd)) {
    return false;
  }

  // Skip the pattern match if FPTrunc/FPEXt is used right after fadd
  if (I.hasOneUse()) {
    FPTruncInst *FPTrunc = llvm::dyn_cast<llvm::FPTruncInst>(*I.user_begin());
    FPExtInst *FPExt = llvm::dyn_cast<llvm::FPExtInst>(*I.user_begin());

    if (FPTrunc || FPExt) {
      return false;
    }
  }

  IGC_ASSERT(I.getOpcode() == Instruction::FAdd || I.getOpcode() == Instruction::FSub);
  for (uint iAdd = 0; iAdd < 2 && !found; iAdd++) {
    Value *src = I.getOperand(iAdd);
    llvm::BinaryOperator *mul = llvm::dyn_cast<llvm::BinaryOperator>(src);
    if (mul && mul->getOpcode() == Instruction::FMul) {
      if (!mul->hasOneUse() || !ContractionAllowed(*mul)) {
        continue;
      }

      for (uint iMul = 0; iMul < 2; iMul++) {
        if (llvm::SelectInst *selInst = dyn_cast<SelectInst>(mul->getOperand(iMul))) {
          ConstantFP *C1 = dyn_cast<ConstantFP>(selInst->getOperand(1));
          ConstantFP *C2 = dyn_cast<ConstantFP>(selInst->getOperand(2));
          if (C1 && C2 && selInst->hasOneUse()) {
            // select i1 %res_s48, float 1.000000e+00, float 0.000000e+00
            if ((C2->isZero() && C1->isExactlyValue(1.f))) {
              invertPred = false;
            }
            // select i1 %res_s48, float 0.000000e+00, float 1.000000e+00
            else if (C1->isZero() && C2->isExactlyValue(1.f)) {
              invertPred = true;
            } else {
              continue;
            }
          } // if (C1 && C2 && selInst->hasOneUse())
          else {
            continue;
          }

          //   % 97 = select i1 %res_s48, float 1.000000e+00, float 0.000000e+00
          //   %102 = fmul fast float %97 %98

          // case 1 (add)
          // Before match
          //   %105 = fadd %102 %103
          // After match
          //              %105 = %103
          //   (%res_s48) %105 = fadd %105 %98

          // case 2 (fsub match @ iAdd = 0)
          // Before match
          //   %105 = fsub %102 %103
          // After match
          //              %105 = -%103
          //   (%res_s48) %105 = fadd %105 %98

          // case 3 (fsub match @ iAdd = 1)
          // Before match
          //   %105 = fsub %103 %102
          // After match
          //              %105 = %103
          //   (%res_s48) %98 = fadd %105 -%98

          // sources[0]: store add operand (i.e. %103 above)
          // sources[1]: store mul operand (i.e. %98 above)

          sources[0] = I.getOperand(1 ^ iAdd);
          sources[1] = mul->getOperand(1 ^ iMul);
          pred = selInst->getOperand(0);

          GetModifier(*sources[0], src_mod[0], sources[0]);
          GetModifier(*sources[1], src_mod[1], sources[1]);
          GetModifier(*pred, pred_mod, pred);

          if (I.getOpcode() == Instruction::FSub) {
            src_mod[iAdd] = CombineModifier(EMOD_NEG, src_mod[iAdd]);
          }

          found = true;
          break;
        } //  if (llvm::SelectInst* selInst = dyn_cast<SelectInst>(mul->getOperand(iMul)))
      } // for (uint iMul = 0; iMul < 2; iMul++)
    } // if (mul && mul->getOpcode() == Instruction::FMul)
  } // for (uint iAdd = 0; iAdd < 2; iAdd++)

  if (found) {
    PredAddPattern *pattern = new (m_allocator) PredAddPattern();
    pattern->predMode = EPRED_NORMAL;
    pattern->sources[0] = GetSource(sources[0], src_mod[0], false, IsSourceOfSample(&I));
    pattern->sources[1] = GetSource(sources[1], src_mod[1], false, IsSourceOfSample(&I));
    pattern->pred = GetSource(pred, pred_mod, false, IsSourceOfSample(&I));
    pattern->invertPred = invertPred;

    AddPattern(pattern);
  }
  return found;
}

// we match the following pattern
// %c = fcmp %1 %2
// %g = sext %c to i32
// %h = and i32 %g 1065353216
// %m = bitcast i32 %h to float
// %p = fadd %m %n
bool CodeGenPatternMatch::MatchSimpleAdd(llvm::BinaryOperator &I) {
  struct SimpleAddPattern : public Pattern {
    SSource sources[2];
    SSource pred;

    e_predMode predMode;
    bool invertPred;
    virtual void Emit(EmitPass *pass, const DstModifier &modifier) {
      DstModifier modf = modifier;
      modf.predMode = predMode;
      pass->PredAdd(pred, invertPred, sources, modf);
    }
  };

  IGC_ASSERT(I.getOpcode() == Instruction::FAdd);

  if (IGC_IS_FLAG_ENABLED(DisableMatchSimpleAdd)) {
    return false;
  }

  unsigned int repAddOperand = 0;
  BitCastInst *bitCastInst0 = llvm::dyn_cast<llvm::BitCastInst>(I.getOperand(0));
  BitCastInst *bitCastInst1 = llvm::dyn_cast<llvm::BitCastInst>(I.getOperand(1));
  BitCastInst *bitCastInst = NULL;

  for (auto UI = I.user_begin(); UI != I.user_end(); ++UI) {
    if (isa<GenIntrinsicInst>(*UI, GenISAIntrinsic::GenISA_fsat)) {
      return false;
    }
  }

  if (!bitCastInst0 && !bitCastInst1) {
    return false;
  }

  if (bitCastInst1) {
    bitCastInst = bitCastInst1;
    repAddOperand = 1;
  } else {
    bitCastInst = bitCastInst0;
    repAddOperand = 0;
  }

  if (!bitCastInst->hasOneUse()) {
    return false;
  }

  Instruction *andInst = dyn_cast<Instruction>(bitCastInst->getOperand(0));
  if (!andInst || (andInst->getOpcode() != Instruction::And)) {
    return false;
  }

  // check %h = and i32 %g 1065353216
  if (!andInst->getType()->isIntegerTy(32)) {
    return false;
  }

  ConstantInt *CInt = dyn_cast<ConstantInt>(andInst->getOperand(1));
  if (!CInt || (CInt->getZExtValue() != 0x3f800000)) {
    return false;
  }

  SExtInst *SExt = llvm::dyn_cast<llvm::SExtInst>(andInst->getOperand(0));
  if (!SExt) {
    return false;
  }

  CmpInst *cmp = llvm::dyn_cast<CmpInst>(SExt->getOperand(0));
  if (!cmp) {
    return false;
  }

  // match found
  SimpleAddPattern *pattern = new (m_allocator) SimpleAddPattern();
  llvm::Value *sources[2], *pred;
  e_modifier src_mod[2], pred_mod;

  sources[0] = I.getOperand(1 - repAddOperand);
  sources[1] = ConstantFP::get(I.getType(), 1.0);
  pred = cmp;

  GetModifier(*sources[0], src_mod[0], sources[0]);
  GetModifier(*sources[1], src_mod[1], sources[1]);
  GetModifier(*pred, pred_mod, pred);

  pattern->predMode = EPRED_NORMAL;
  pattern->sources[0] = GetSource(sources[0], src_mod[0], false, IsSourceOfSample(&I));
  pattern->sources[1] = GetSource(sources[1], src_mod[1], false, IsSourceOfSample(&I));
  pattern->pred = GetSource(pred, pred_mod, false, IsSourceOfSample(&I));
  pattern->invertPred = false;
  AddPattern(pattern);

  return true;
}

bool CodeGenPatternMatch::CanMatchMad(llvm::BinaryOperator &I) const {
  IGC_ASSERT(I.getOpcode() == Instruction::Add || I.getOpcode() == Instruction::Sub ||
             I.getOpcode() == Instruction::FAdd || I.getOpcode() == Instruction::FSub);

  // General restrictions
  if (IGC_IS_FLAG_ENABLED(DisableMatchMad)) {
    return false;
  }

  if (m_ctx->type == ShaderType::VERTEX_SHADER && m_ctx->m_DriverInfo.DisableMatchMad()) {
    return false;
  }

  if (m_ctx->type == ShaderType::COMPUTE_SHADER && m_ctx->getModuleMetaData()->disableMatchMadOptimizationForCS) {
    return false;
  }

  // Instruction specific restrictions
  bool isFpMad = I.getType()->isFloatingPointTy() ||
                 (I.getType()->isVectorTy() && I.getType()->getScalarType()->isFloatingPointTy() &&
                  IGC_IS_FLAG_ENABLED(VectorizerAllowFMADMatching));

  if (I.getOperand(0) == I.getOperand(1)) {
    // requirement, cannot match mad instruction if both ops of add/sub are the same, but can just do shl or mul by 2
    return false;
  }

  if (isFpMad &&
      (m_ctx->getModuleMetaData()->isPrecise || m_ctx->getModuleMetaData()->compOpt.disableMathRefactoring)) {
    return false;
  }

  bool bothOperandsAreUniform = isUniform(I.getOperand(0)) && isUniform(I.getOperand(1));
  bool canUseMAD = m_Platform.doIntegerMad(bothOperandsAreUniform) && m_ctx->m_DriverInfo.EnableIntegerMad();

  if (I.getType()->isIntOrIntVectorTy() && !canUseMAD) {
    return false;
  }

  if (m_ctx->type == ShaderType::VERTEX_SHADER && m_ctx->m_DriverInfo.PreventZFighting() &&
      !(m_ctx->getModuleMetaData()->allowMatchMadOptimizationforVS ||
        IGC_IS_FLAG_ENABLED(WaAllowMatchMadOptimizationforVS))) {
    if (m_PosDep->PositionDependsOnInst(&I)) {
      return false;
    }
  }

  if (isFpMad && !ContractionAllowed(I)) {
    return false;
  }

  return true;
}

bool CodeGenPatternMatch::MatchFMad(llvm::BinaryOperator &I) {
  struct FMadPattern : Pattern {
    SSource sources[3];
    virtual void Emit(EmitPass *pass, const DstModifier &modifier) {
      if (IGC_IS_FLAG_ENABLED(EnableVectorEmitter) && sources[0].value->getType()->isVectorTy() &&
          sources[1].value->getType()->isVectorTy() && sources[2].value->getType()->isVectorTy())
        pass->VectorMad(sources, modifier);
      else
        pass->Mad(sources, modifier);
    }
  };

  if (!CanMatchMad(I))
    return false;

  bool found = false;
  llvm::Value *sources[3];
  e_modifier src_mod[3];

  IGC_ASSERT(I.getOpcode() == Instruction::FAdd || I.getOpcode() == Instruction::FSub);
  for (uint i = 0; i < 2; i++) {
    Value *src = SkipCanonicalize(I.getOperand(i));
    if (FPExtInst *fpextInst = llvm::dyn_cast<llvm::FPExtInst>(src)) {
      if (!m_Platform.supportMixMode() && fpextInst->getSrcTy()->getTypeID() == llvm::Type::HalfTyID) {
        // no mix mode instructions
      } else if (fpextInst->getSrcTy()->getTypeID() != llvm::Type::DoubleTyID &&
                 fpextInst->getDestTy()->getTypeID() != llvm::Type::DoubleTyID) {
        src = fpextInst->getOperand(0);
      }
    }
    llvm::BinaryOperator *mul = llvm::dyn_cast<llvm::BinaryOperator>(src);

    if (mul && mul->getOpcode() == Instruction::FMul) {
      // in case we know we won't be able to remove the mul we don't merge it
      if (!m_PosDep->PositionDependsOnInst(mul) && NeedInstruction(*mul))
        continue;

      // Check either contraction allowed globally or fmul has allow contract flag (at this point fadd/fsub already has
      // contract flag if disabled at global level)
      if (!ContractionAllowed(*mul))
        continue;

      sources[2] = SkipCanonicalize(I.getOperand(1 - i));
      sources[1] = SkipCanonicalize(mul->getOperand(0));
      sources[0] = SkipCanonicalize(mul->getOperand(1));

      GetModifier(*sources[0], src_mod[0], sources[0]);
      GetModifier(*sources[1], src_mod[1], sources[1]);
      GetModifier(*sources[2], src_mod[2], sources[2]);

      sources[0] = SkipCanonicalize(sources[0]);
      sources[1] = SkipCanonicalize(sources[1]);
      sources[2] = SkipCanonicalize(sources[2]);
      if (I.getOpcode() == Instruction::FSub) {
        if (i == 0) {
          src_mod[2] = CombineModifier(EMOD_NEG, src_mod[2]);
        } else {
          if (llvm::isa<llvm::Constant>(sources[0])) {
            src_mod[1] = CombineModifier(EMOD_NEG, src_mod[1]);
          } else {
            src_mod[0] = CombineModifier(EMOD_NEG, src_mod[0]);
          }
        }
      }
      found = true;
      break;
    }
  }

  if (found) {
    FMadPattern *pattern = new (m_allocator) FMadPattern();
    for (int i = 0; i < 3; i++) {
      pattern->sources[i] = GetSource(sources[i], src_mod[i], false, IsSourceOfSample(&I));
      if (isa<ConstantFP>(sources[i]) &&
          (!m_Platform.support16BitImmSrcForMad() || !sources[i]->getType()->isHalfTy() || i == 1)) {
        // CNL+ mad instruction allows 16 bit immediate for src0 and src2
        AddToConstantPool(I.getParent(), sources[i]);
        pattern->sources[i].fromConstantPool = true;
      }
    }
    AddPattern(pattern);
  }
  return found;
}

bool CodeGenPatternMatch::IsMulCandidateForIMad(llvm::BinaryOperator *Mul, llvm::BinaryOperator *I) {
  IGC_ASSERT(Mul->getOpcode() == Instruction::Mul);
  IGC_ASSERT(I->getOpcode() == Instruction::Add || I->getOpcode() == Instruction::Sub);

  // in case we know we won't be able to remove the mul we don't merge it
  if (!m_PosDep->PositionDependsOnInst(Mul) && NeedInstruction(*Mul))
    return false;

  // This condition prevents creating mad instruction when mul instruction is 64-bit wide operation.
  // We should use WideMulSequence for this case to emit sequence of mul+mad instructions
  if ((Mul->getType()->isIntegerTy(64)) && m_Platform.hasWideMulMad()) {
    return false;
  }

  auto isValueNumBitsWide = [](Value *V, unsigned bitWidth) {
    // Look through casts because LLVM cannot represent mixed type mul and add that may match a mad instruction
    // i.e.
    // a64 = zext i32 a to i64
    // b64 = zext i32 b to i64
    // temp = mul i64 a64, b64
    // res = add i64 temp, c64
    while (isa<ZExtInst, SExtInst, BitCastInst>(V))
      V = cast<Instruction>(V)->getOperand(0);
    Type *T = V->getType();
    return (T->isIntegerTy() && T->getPrimitiveSizeInBits() == bitWidth);
  };

  // Mad instruction doesn't support QW type for mul operands
  if (isValueNumBitsWide(Mul->getOperand(1), 64) || isValueNumBitsWide(Mul->getOperand(0), 64))
    return false;

  if (IsConstOrSimdConstExpr(Mul->getOperand(1)) && IsConstOrSimdConstExpr(Mul->getOperand(0))) {
    // If both Mul operands are constants or simdSize intrinsic, do not match MAD, just match ADD/ADD3 with folded
    // constant.
    return false;
  }

  // In cases where mul has multiple users, do additional check.
  // Always allow single user case, even if cross block.
  if (Mul->getUniqueUndroppableUser() != I) {
    if (IGC_IS_FLAG_DISABLED(AllowMultipleMulUsesMatchMad)) {
      // If Mul has other users, do not match mad, just compute mul and use for the add/sub since it will likely need to
      // be computed to be used in the other instructions.
      return false;
    }

    // Mul has other users and is allowed. However, if mul is not in the same block as I, then matching mad may increase
    // reg pressure. Lifetime of Mul may get extended, matching something else like add3 may reduce reg pressure.
    if (IGC_IS_FLAG_DISABLED(AllowCrossBlockMatchMad) && Mul->getParent() != I->getParent()) {
      return false;
    }
  }

  auto *otherOp = (I->getOperand(0) == Mul) ? I->getOperand(1) : I->getOperand(0);
  // Mad instruction does not support qword type for add/sub operand
  if (isValueNumBitsWide(otherOp, 64) && !m_Platform.hasWideMulMad()) {
    return false;
  }
  return true;
}

bool CodeGenPatternMatch::MatchIMad(llvm::BinaryOperator &I) {
  struct IMadPattern : Pattern {
    SSource sources[3];
    virtual void Emit(EmitPass *pass, const DstModifier &modifier) { pass->Mad(sources, modifier); }
  };

  if (!CanMatchMad(I)) {
    return false;
  }

  if (!I.getType()->isIntegerTy()) {
    // TODO: Support vector IMad
    return false;
  }

  for (int i = 0; i < 2; i++) {
    auto *mul = llvm::dyn_cast<llvm::BinaryOperator>(I.getOperand(i));

    if (mul && mul->getOpcode() == Instruction::Mul) {
      if (!IsMulCandidateForIMad(mul, &I)) {
        continue;
      }

      // Helper to hold relevant info about how to emit each source operand
      struct {
        Value *src = nullptr;         // LLVM operand
        Value *originalSrc = nullptr; // original LLVM operand, before any modification for matching
        e_modifier mod = EMOD_NONE;
        bool isSigned = false;
        bool isConstant16BitEq = false;
        bool isUpcastSrc = false;
        bool isCandidate() { return needsRegioning || isConstant16BitEq || isUpcastSrc; }
        void reset() {
          src = originalSrc;
          mod = EMOD_NONE;
          isSigned = false;
          isConstant16BitEq = false;
          isUpcastSrc = false;
          needsRegioning = false;
          useLower = true;
        }

        // below properties relevant when src is 32-bit but can be represented in 16 bits
        bool needsRegioning = false; // needs regioning to reduce bit width
        bool useLower = true;        // for setting offset
      } oprdInfo[3];

      oprdInfo[2].src = I.getOperand(1 - i);
      oprdInfo[2].originalSrc = oprdInfo[2].src;
      oprdInfo[1].src = mul->getOperand(0);
      oprdInfo[1].originalSrc = oprdInfo[1].src;
      oprdInfo[0].src = mul->getOperand(1); // usually a constant LLVM operand will be 2nd mul op at this point
      oprdInfo[0].originalSrc = oprdInfo[0].src;

      // Check whether we can replace some/all of 32-bit MUL+ADD operands with partial 16-bit MAD operands
      // TODO: Can this also extend to replacing 16-bit operands with 8 bits? Need to be careful for byte regioning,
      // seems like there are more restrictions in vISA for matching 8-bit MAD
      if (IGC_IS_FLAG_ENABLED(EnableMixIntOperands) && I.getType()->isIntegerTy(32)) {
        using namespace llvm::PatternMatch;
        for (int opI = 0; opI < 3; ++opI) {
          Value *L = nullptr;
          if (IsConstOrSimdConstExpr(oprdInfo[opI].src)) {
            if (auto *CI = dyn_cast<ConstantInt>(oprdInfo[opI].src)) {
              int64_t val = CI->isNegative() ? CI->getSExtValue() : CI->getZExtValue();
              if (INT16_MIN <= val && val <= INT16_MAX) {
                oprdInfo[opI].isSigned = true;
                oprdInfo[opI].isConstant16BitEq = true;
              } else if (0 <= val && val <= UINT16_MAX) {
                oprdInfo[opI].isConstant16BitEq = true;
              }
            } else {
              // If the source is a derivative of simdSize intrinsic, compute the known bits
              // Not guaranteed to fit in 16 or 8 bits, because of intermediate operations modifying var range
              KnownBits knownBits = computeKnownBits(oprdInfo[opI].src, I.getModule()->getDataLayout());
              // rudimentary check since this doesn't account for negative values as a result of operations performed on
              // SIMDSize, may not catch all cases
              if (knownBits.countMaxActiveBits() <= 16) {
                oprdInfo[opI].isConstant16BitEq = true;
              }
            }
            // When doing matches for m_Value(L), if L is a captured 32-bit variable, then we set needsRegioning to true
            // to know to set regioning later
          } else if (match(oprdInfo[opI].src, m_And(m_Value(L), m_SpecificInt(0xFFFF)))) {
            oprdInfo[opI].src = L;
            oprdInfo[opI].needsRegioning = true;
          } else if (match(oprdInfo[opI].src, m_LShr(m_Value(L), m_SpecificInt(16)))) {
            oprdInfo[opI].src = L;
            oprdInfo[opI].needsRegioning = true;
            oprdInfo[opI].useLower = false;
          } else if (match(oprdInfo[opI].src, m_AShr(m_Shl(m_Value(L), m_SpecificInt(16)), m_SpecificInt(16)))) {
            oprdInfo[opI].src = L;
            oprdInfo[opI].isSigned = true;
            oprdInfo[opI].needsRegioning = true;
          } else if (PatternMatch::match(oprdInfo[opI].src, m_AShr(m_Value(L), m_SpecificInt(16)))) {
            oprdInfo[opI].src = L;
            oprdInfo[opI].isSigned = true;
            oprdInfo[opI].needsRegioning = true;
            oprdInfo[opI].useLower = false;
          } else if (match(oprdInfo[opI].src, m_ZExt(m_Value(L))) && L->getType()->isIntegerTy(16)) {
            oprdInfo[opI].src = L;
            oprdInfo[opI].isUpcastSrc = true;
          } else if (match(oprdInfo[opI].src, m_SExt(m_Value(L))) && L->getType()->isIntegerTy(16)) {
            oprdInfo[opI].src = L;
            oprdInfo[opI].isSigned = true;
            oprdInfo[opI].isUpcastSrc = true;
          }
        }
      }

      // Preemptive swapping of mul operands to try to get better candidates for src0 and src1 of mad
      // Order of if statements in increasing order of priority
      if (isUniform(oprdInfo[1].src) && !isUniform(oprdInfo[0].src) &&
          !(oprdInfo[1].needsRegioning && !oprdInfo[1].useLower)) {
        // put non-uniform value in src1, since src0 (src2 in gen) has regioning restrictions, but only if new value
        // in src0 will still be GRF aligned
        std::swap(oprdInfo[0], oprdInfo[1]);
      }
      if (IsConstOrSimdConstExpr(oprdInfo[1].src)) {
        // put the constant in src0
        std::swap(oprdInfo[0], oprdInfo[1]);
      }
      if (mul->getType()->isIntegerTy(32) && oprdInfo[1].isCandidate() && !oprdInfo[0].isCandidate()) {
        // put the wider operand in src1
        std::swap(oprdInfo[0], oprdInfo[1]);
      }

      // Need GRF-aligned src0 (in vISA, src2 in gen) for mad. Since preemptive swapping happened earlier, if conditions
      // are not satisfied just don't match mad. Alignment and regioning are limited for src0
      // - dword aligned but not using lower bits
      if (oprdInfo[0].needsRegioning && !oprdInfo[0].useLower &&
          !m_Platform.isCoreChildOf(IGFX_XE3P_CORE) // Simplified regioning
      ) {
        continue;
      }

      // - word aligned
      if (oprdInfo[0].isUpcastSrc) {
        oprdInfo[0].reset();
      }

      // Get source modifiers for all operands if not using regioning
      // Source modifier not supported on src1 for DW and lower precision integer multiply
      bool skipModifierSrc1 = false;
      if (m_Platform.supportsSourceModifierForMixedIntMad() ||
          !(mul->getType()->isIntegerTy(32) && (oprdInfo[0].isCandidate() ^ oprdInfo[1].isCandidate()))) {
        // Source modifier supported, but will only get applied properly if not using special regioning to specify
        // reduced values
        if (!oprdInfo[1].needsRegioning) {
          bool mod = GetModifier(*oprdInfo[1].src, oprdInfo[1].mod, oprdInfo[1].src);
          // Special case: NEG, ABS, or NEGABS modifier on unsigned upcasted source
          // Happens when:
          // B = llvm.abs(A) OR B = sub 0, A OR B = sub 0, llvm.abs(A)
          // C = zext i16 B to i32
          // origSrc=C, src=B, isSigned=false (from zext), mod=NONE -> src=A, isSigned=false, mod=ABS/NEG/NEGABS
          // Mod on A, which is guaranteed signed 16-bit since abs or neg or negabs was applied, undef for unsigned
          // values
          if (mod && oprdInfo[1].isUpcastSrc &&
              (oprdInfo[1].mod == EMOD_NEG || oprdInfo[1].mod == EMOD_ABS || oprdInfo[1].mod == EMOD_NEGABS)) {
            oprdInfo[1].isSigned = true;
          }
        }
      } else {
        skipModifierSrc1 = true;
      }
      if (!oprdInfo[0].needsRegioning) {
        bool mod = GetModifier(*oprdInfo[0].src, oprdInfo[0].mod, oprdInfo[0].src);
        if (mod && oprdInfo[0].isUpcastSrc &&
            (oprdInfo[0].mod == EMOD_NEG || oprdInfo[0].mod == EMOD_ABS || oprdInfo[0].mod == EMOD_NEGABS)) {
          oprdInfo[0].isSigned = true;
        }
      }
      if (!oprdInfo[2].needsRegioning) {
        bool mod = GetModifier(*oprdInfo[2].src, oprdInfo[2].mod, oprdInfo[2].src);
        if (mod && oprdInfo[2].isUpcastSrc &&
            (oprdInfo[2].mod == EMOD_NEG || oprdInfo[2].mod == EMOD_ABS || oprdInfo[2].mod == EMOD_NEGABS)) {
          oprdInfo[2].isSigned = true;
        }
      }

      if (I.getOpcode() == Instruction::Sub) {
        if (i == 0) {
          // Matching sub to mad instruction by trying to apply negative modifier to subtrahend
          if (auto *CI = dyn_cast<ConstantInt>(oprdInfo[2].src)) {
            int64_t val = CI->isNegative() ? CI->getSExtValue() : CI->getZExtValue();
            if (CI->isNegative()) {
              // flip without overflow concerns
              oprdInfo[2].src = ConstantInt::get(CI->getType(), -val);
              oprdInfo[2].isSigned = false;
            } else {
              if (val > (int64_t)(INT32_MAX) + 1 && !m_Platform.hasWideMulMad() ||
                  (uint64_t)val > (uint64_t)(INT64_MAX) + 1) {
                // negating will overflow
                return false;
              } else if (val > (int64_t)(INT16_MAX) + 1) {
                // negating will invalidate status
                oprdInfo[2].isConstant16BitEq = false;
              }
              oprdInfo[2].src = ConstantInt::get(CI->getType(), -val, true);
              oprdInfo[2].isSigned = true;
            }
          } else {
            // If src2 is candidate and unsigned, neg modifier will do 2s complement
            // This will result in incorrect values if src2 value falls between INT16_MAX and UINT16_MAX.
            // Unless NEG already present, fallback to full dword operand
            if (oprdInfo[2].isCandidate() && !oprdInfo[2].isSigned && oprdInfo[2].mod != EMOD_NEG) {
              oprdInfo[2].reset();
              GetModifier(*oprdInfo[2].src, oprdInfo[2].mod, oprdInfo[2].src);
              // fallthrough to combine neg modifier from sub instruction
            }
            oprdInfo[2].mod = CombineModifier(EMOD_NEG, oprdInfo[2].mod);
          }
        } else {
          // Matching sub to mad instruction by trying to apply negative modifier to minuend
          // try to apply neg to src0 if possible
          if (auto *CI = dyn_cast<ConstantInt>(oprdInfo[0].src)) {
            int64_t val = CI->isNegative() ? CI->getSExtValue() : CI->getZExtValue();
            // if src0 is a constant, need to make sure to negate it properly
            if (CI->isNegative()) {
              // flip without overflow concerns
              oprdInfo[0].src = ConstantInt::get(CI->getType(), -val);
              oprdInfo[0].isSigned = false;
            } else {
              if (val > (int64_t)(INT32_MAX) + 1 && !m_Platform.hasWideMulMad() ||
                  (uint64_t)val > (uint64_t)(INT64_MAX) + 1) {
                // negating will overflow
                return false;
              } else if (val > (int64_t)(INT16_MAX) + 1) {
                // can flip to negative, need to recheck 16-bit candidacy
                if (oprdInfo[0].isConstant16BitEq && !skipModifierSrc1 &&
                    (!oprdInfo[1].isCandidate() || oprdInfo[1].isSigned)) {
                  // if src0 was never a candidate, no consequence for flipping. Only consider flipping src1 if src0 is
                  // a candidate not modifier restricted for src1 and won't mess with src1 type/regioning by adding neg
                  // modifier, put modifier on src1 without making src0 non-16-bit candidate
                  oprdInfo[1].mod = CombineModifier(EMOD_NEG, oprdInfo[1].mod);
                } else {
                  // cannot put modifier on src1 or will need to reset src1, just negate src0
                  // this will likely result in mad no longer getting matched, but there is no other way
                  oprdInfo[0].src = ConstantInt::get(CI->getType(), -val, true);
                  oprdInfo[0].isSigned = true;
                  oprdInfo[0].isConstant16BitEq = false;
                }
              } else {
                // negative constant fits in 16 bits
                oprdInfo[0].src = ConstantInt::get(CI->getType(), -val, true);
                oprdInfo[0].isSigned = true;
              }
            }
          } else {
            // if src0 is candidate and unsigned, neg modifier will do 2s complement
            // this will result in incorrect values if src0 value falls between INT16_MAX and UINT16_MAX
            // unless NEG already present, fallback to full dword operand
            if (oprdInfo[0].isCandidate() && !oprdInfo[0].isSigned && oprdInfo[0].mod != EMOD_NEG) {
              if (!skipModifierSrc1) {
                oprdInfo[1].mod = CombineModifier(EMOD_NEG, oprdInfo[1].mod);
              } else {
                // this will likely cause mad not to be matched, but no other way
                oprdInfo[0].reset();
                GetModifier(*oprdInfo[0].src, oprdInfo[0].mod, oprdInfo[0].src);
                // combine neg modifier from sub instruction
                oprdInfo[0].mod = CombineModifier(EMOD_NEG, oprdInfo[0].mod);
              }
            } else {
              oprdInfo[0].mod = CombineModifier(EMOD_NEG, oprdInfo[0].mod);
            }
          }
        }
      }

      // Mad instruction does not support dword * dword, no operands were able to get reduced.
      if (mul->getType()->isIntegerTy(32) && !oprdInfo[0].isCandidate() &&
          !oprdInfo[1].isCandidate()
          /* Unless supported platform is performing dword * dword + {dword or qword} = qword */
          && !m_Platform.hasWideMulMad()
          /* Simplified regioning, dword * dword allowed */
          && !m_Platform.isCoreChildOf(IGFX_XE3P_CORE)) {
        return false;
      }

      auto isValueConstantInt16Bit = [&](Value *V) {
        if (auto *CI = dyn_cast<ConstantInt>(V)) {
          return CI->getValue().isNegative() ? CI->getValue().sge(INT16_MIN) && CI->getValue().sle(INT16_MAX)
                                             : CI->getValue().uge(0) && CI->getValue().ule(UINT16_MAX);
        } else if (isa<Instruction>(V) && SIMDConstExpr(cast<Instruction>(V))) {
          KnownBits knownBits = computeKnownBits(V, I.getModule()->getDataLayout());
          // rudimentary check since this doesn't account for negative values as a result of operations performed on
          // SIMDSize
          return knownBits.countMaxActiveBits() <= 16;
        }
        return false;
      };

      unsigned numConst = 0;
      unsigned num16BitImm = 0;
      // At this point check whether constants can be immediate or they need to be in constant pool
      for (int i = 0; i < 3; ++i) {
        if (IsConstOrSimdConstExpr(oprdInfo[i].src)) {
          numConst++;
          if (m_Platform.support16BitImmSrcForMad() && isValueConstantInt16Bit(oprdInfo[i].src)) {
            num16BitImm++;
          }
        }
      }

      // control whether to allow any constant operand >16-bit to be generated to be used from the constant pool
      // if AllowConstMadOpMovToReg is set, we allow one such constant, otherwise all constants must be 16-bit
      const unsigned constLimit =
          IGC_IS_FLAG_ENABLED(AllowConstMadOpMovToReg) || m_Platform.isCoreChildOf(IGFX_XE3P_CORE) ? 1 : 0;
      if ((numConst - num16BitImm) > constLimit) {
        continue;
      }

      IMadPattern *pattern = new (m_allocator) IMadPattern();

      bool addToConstantPool = false;
      for (int i = 0; i < 3; ++i) {
        // ZExt from a sub-dword LoadInst: use originalSrc (ZExtInst) so MarkAsSource
        // keeps ZExtByteLoadPattern active; otherwise the dead-copy skip in
        // emitLSCVectorLoad_subDW leaves Dest:UW uninitialized.
        // ZExt from non-load (e.g. extractelement): use src (the i16 extractelement)
        // directly. Using the ZExtInst here causes GetSrcVariable to alias its dword
        // CVariable as UW, producing a 64-element UW alias of a 32-element dword var.
        // A mad (M1,32) reading 32 elements from that alias covers only lanes 0-15.
        // SExt: use src (i16/i8 load) directly; originalSrc (sext) maps to a dword
        // CVariable whose stride-1 word alias reads wrong lanes on SIMD32.
        bool useOriginalSrc = oprdInfo[i].isUpcastSrc && !oprdInfo[i].isSigned && !oprdInfo[i].needsRegioning &&
                              isa<LoadInst>(oprdInfo[i].src);
        llvm::Value *srcVal = useOriginalSrc ? oprdInfo[i].originalSrc : oprdInfo[i].src;
        pattern->sources[i] = GetSource(srcVal, oprdInfo[i].mod, false, IsSourceOfSample(&I));
        if (oprdInfo[i].isCandidate()) {
          SSource &thisSrc = pattern->sources[i];

          // useOriginalSrc: GetSource(ZExtInst) returns gatherDst (UD, N elts)
          // which already holds zero-extended i16 values. No ISA_TYPE_UW override
          // or special regioning needed; leave the UD type intact.
          if (!useOriginalSrc) {
            // for now, Use W/UW only if region_set is false or the src is scalar
            if (!thisSrc.region_set || (thisSrc.region[0] == 0 && thisSrc.region[1] == 1 && thisSrc.region[2] == 0)) {
              // Note that SSource's type, if set by GetSource(), should be 32bit type. It's safe to override it with
              // either UW or W. But for SSource's offset, need to re-calculate in term of 16bit, not 32bit.
              thisSrc.type = oprdInfo[i].isSigned ? ISA_TYPE_W : ISA_TYPE_UW;
              if (oprdInfo[i].needsRegioning) {
                thisSrc.elementOffset = (2 * thisSrc.elementOffset) + (oprdInfo[i].useLower ? 0 : 1);
              }
            }
            if (oprdInfo[i].needsRegioning && !isUniform(oprdInfo[i].src)) {
              thisSrc.region_set = true;
              thisSrc.region[0] = 16;
              thisSrc.region[1] = 8;
              thisSrc.region[2] = 2;
            }
          }
        }

        if (isa<ConstantInt>(oprdInfo[i].src)) {
          if (!m_Platform.support16BitImmSrcForMad() || !isValueConstantInt16Bit(oprdInfo[i].src) ||
              addToConstantPool) {
            // Add all constants that are unable to fit in 16-bits to constant pool.
            AddToConstantPool(I.getParent(), oprdInfo[i].src);
            pattern->sources[i].fromConstantPool = true;
          } else if (!addToConstantPool
                     // Pre XE3P only one immediate constant allowed in mad
                     && !m_Platform.isCoreChildOf(IGFX_XE3P_CORE)) {
            addToConstantPool = true;
          }
        }
      }
      AddPattern(pattern);
      return true;
    }
  }

  return false;
}

// match simdblockRead/Write with preceding inttoptr if possible
// to save a copy move
bool CodeGenPatternMatch::MatchBlockReadWritePointer(llvm::GenIntrinsicInst &I) {
  struct BlockReadWritePointerPattern : public Pattern {
    GenIntrinsicInst *inst;
    Value *offset;
    explicit BlockReadWritePointerPattern(GenIntrinsicInst *I, Value *ofst) : inst(I), offset(ofst) {}
    virtual void Emit(EmitPass *pass, const DstModifier &modifier) {
      switch (inst->getIntrinsicID()) {
      case GenISAIntrinsic::GenISA_simdBlockRead:
        pass->emitSimdBlockRead(inst, offset);
        break;
      case GenISAIntrinsic::GenISA_simdBlockWrite:
        pass->emitSimdBlockWrite(inst, offset);
        break;
      default:
        IGC_ASSERT_MESSAGE(0, "unexpected intrinsic");
        break;
      }
    }
  };

  if (I.getIntrinsicID() != GenISAIntrinsic::GenISA_simdBlockRead &&
      I.getIntrinsicID() != GenISAIntrinsic::GenISA_simdBlockWrite) {
    return false;
  }

  // check the address is inttoptr with same dst and src type width
  auto ptrVal = I.getOperand(0);
  auto I2P = dyn_cast<IntToPtrInst>(ptrVal);
  if (I2P && I2P->getOperand(0)->getType()->getIntegerBitWidth() ==
                 m_ctx->getRegisterPointerSizeInBits(I2P->getAddressSpace())) {
    auto addrBase = I2P->getOperand(0);
    BlockReadWritePointerPattern *pattern = new (m_allocator) BlockReadWritePointerPattern(&I, addrBase);
    MarkAsSource(addrBase, IsSourceOfSample(&I));
    // for write mark data ptr as well
    if (I.getIntrinsicID() == GenISAIntrinsic::GenISA_simdBlockWrite) {
      MarkAsSource(I.getOperand(1), IsSourceOfSample(&I));
    }

    AddPattern(pattern);
    return true;
  }
  return false;
}


// Pattern matching to detect and handle immediate offsets in load/store
// instructions.  It detects offsets of the form "add dst, var, imm"
// The add instruction is removed, and imm included in the LSC message descriptor.
//
// OpenCL Load/Store launder the address through an inttoptr....
//   %2 = add i32 %0, 64
//   %3 = inttoptr i32 %2 to i32 addrspace(131072)*
//   store i32 16, i32 addrspace(131072)* %3, align 8
// Fold to vISA lsc_store... [%0 + 64].
//
// Stuff like 3D GenISA_ldrawvector_indexed directly reference the add.
//   %723 = add i32 %713, 16
//   %724 = call <4 x i32> @llvm.genx.GenISA.ldrawvector.indexed.v4i32.p2621448v4f32(
//            <4 x float> addrspace(2621448)* %runtime_value_1, i32 %723, i32 4, i1 false)
// Fold to lsc_load [%713 + 16].
static uint GetNbSources(llvm::Instruction &v) {
  if (llvm::CallInst *intrin = llvm::dyn_cast<llvm::CallInst>(&v)) {
    return IGCLLVM::getNumArgOperands(intrin);
  }
  return v.getNumOperands();
}

// Returns true if the value is an integer constant that can be encoded in
// 32 bits.
inline bool Is32BitConstInt(llvm::Value *val) {
  llvm::ConstantInt *imm = llvm::dyn_cast<ConstantInt>(val);
  if (imm && imm->getSExtValue() <= std::numeric_limits<int32_t>::max() &&
      imm->getSExtValue() >= std::numeric_limits<int32_t>::min()) {
    return true;
  }
  return false;
}

bool CodeGenPatternMatch::MatchImmOffsetLSC(llvm::Instruction &I) {
  struct LSCImmOffsetPattern : public Pattern {
    explicit LSCImmOffsetPattern(Instruction *I, llvm::Value *varOffset, llvm::ConstantInt *immOffset)
        : m_inst(I), m_varOff(varOffset), m_immOff(immOffset) {}

    virtual void Emit(EmitPass *pass, const DstModifier &modifier) {
      if (isa<LoadInst>(m_inst)) {
        pass->emitLoad(cast<LoadInst>(m_inst), m_varOff, m_immOff);
      } else if (isa<StoreInst>(m_inst)) {
        pass->emitStore(cast<StoreInst>(m_inst), m_varOff, m_immOff);
      } else if (isa<LdRawIntrinsic>(m_inst)) {
        pass->emitLoadRawIndexed(cast<LdRawIntrinsic>(m_inst), m_varOff, nullptr, m_immOff);
      } else if (isa<StoreRawIntrinsic>(m_inst)) {
        pass->emitStoreRawIndexed(cast<StoreRawIntrinsic>(m_inst), m_varOff, nullptr, m_immOff);
      } else {
        IGC_ASSERT_MESSAGE(false, "unmatched imm off pattern");
      }
    }

  private:
    llvm::Instruction *m_inst;
    llvm::Value *m_varOff;
    llvm::ConstantInt *m_immOff;
  }; // LSCImmOffsetPattern

  // match:
  //   %addr = (add/sub %var, %imm) | (add %imm, %var)
  //   [ %addr = inttoptr %addr, ... ]
  //   load/store/whatever ... %addr
  int addrOpnd = -1;
  if (llvm::isa<LoadInst>(&I) || llvm::isa<StoreInst>(&I)) {
    // buffer load/store instructions in compute languages are usually
    // buried behind a inttoptr op
    addrOpnd = llvm::isa<LoadInst>(&I) ? 0 : 1;
  } else if (llvm::isa<llvm::GenIntrinsicInst>(I)) {
    llvm::GenIntrinsicInst &GI = llvm::cast<llvm::GenIntrinsicInst>(I);
    switch (GI.getIntrinsicID()) {
    // TODO: incrementally enable others
    //
    // case GenISAIntrinsic::GenISA_intatomicraw:
    // case GenISAIntrinsic::GenISA_floatatomicraw:
    // case GenISAIntrinsic::GenISA_intatomicrawA64:
    // case GenISAIntrinsic::GenISA_floatatomicrawA64:
    // case GenISAIntrinsic::GenISA_icmpxchgatomicraw:
    // case GenISAIntrinsic::GenISA_icmpxchgatomicrawA64:
    // case GenISAIntrinsic::GenISA_fcmpxchgatomicrawA64:
    //  all these will target => emitAtomicRaw
    //
    case GenISAIntrinsic::GenISA_ldrawvector_indexed:
    case GenISAIntrinsic::GenISA_ldraw_indexed:
    case GenISAIntrinsic::GenISA_storerawvector_indexed:
    case GenISAIntrinsic::GenISA_storeraw_indexed:
      addrOpnd = 1;
      if (IGC_IS_FLAG_ENABLED(DisableStatefulFolding))
        return false;
      break;
      // return false;
    // TODO: are these even reachable here???
    // case GenISAIntrinsic::GenISA_ldstructured:
    // case GenISAIntrinsic::GenISA_storestructured1:
    // case GenISAIntrinsic::GenISA_storestructured2:
    // case GenISAIntrinsic::GenISA_storestructured3:
    // case GenISAIntrinsic::GenISA_storestructured4:
    //    addrOpnd = 2;
    //    break;
    default:
      return false;
    }
  } else {
    return false;
  }

  if (addrOpnd < 0) {
    return false;
  }

  llvm::Instruction *addSubInst = llvm::dyn_cast<llvm::Instruction>(I.getOperand((unsigned)addrOpnd));
  if (!addSubInst) {
    // e.g. the address is a constant
    return false;
  }

  Type *addInstType = addSubInst->getType();
  // Note that the A64 addressing mode can be only connected with load and store instructions
  bool isA64AddressingModel = addInstType->isPointerTy() && IGC::isA64Ptr(cast<PointerType>(addInstType), m_ctx);

  llvm::Instruction *intToPtrInst = nullptr;
  if (addSubInst->getOpcode() == llvm::Instruction::IntToPtr) {
    intToPtrInst = addSubInst;
    addSubInst = llvm::dyn_cast<llvm::Instruction>(addSubInst->getOperand(0));
  }
  if (!addSubInst) {
    return false; // e.g. intToPtr of constant
  } else if (addSubInst->getOpcode() != llvm::Instruction::Add && addSubInst->getOpcode() != llvm::Instruction::Sub) {
    return false;
  }

  // VISA interface accepts 32-bit immediate offset values
  const bool isConstant0 = Is32BitConstInt(addSubInst->getOperand(0));
  const bool isConstant1 = Is32BitConstInt(addSubInst->getOperand(1));
  if (isConstant1 || (isConstant0 && addSubInst->getOpcode() == llvm::Instruction::Add)) {
    // YES: var + imm
    // YES: var - imm
    // YES: imm + var
    // NO:  imm - var  (IGC drops the negation otherwise)
    IGC_ASSERT_MESSAGE(!isConstant0 || !isConstant1,
                       "Both operands are immediate - constants should be folded elsewhere.");

    llvm::Value *varOffset = isConstant0 ? addSubInst->getOperand(1) : addSubInst->getOperand(0);

    // HW does an early bounds check on varOffset for A32 messages. Thus, if varOffset
    // is negative, then the bounds check fails early even though the immediate offset
    // would bring the final calculation to a positive number.
    bool disableA32ImmediateGlobalBaseOffset =
        !isA64AddressingModel && !UsedWithoutImmInMemInst(varOffset) && !valueIsPositive(varOffset, m_DL) &&
        IGC_GET_FLAG_VALUE(LscImmOffsMatch) < 3;

    if (disableA32ImmediateGlobalBaseOffset)
      return false;

    MarkAsSource(varOffset, IsSourceOfSample(&I));

    unsigned numSources = GetNbSources(I);
    for (unsigned i = 0; i < numSources; i++) {
      if (I.getOperand(i) != intToPtrInst && I.getOperand(i) != addSubInst) {
        MarkAsSource(I.getOperand(i), IsSourceOfSample(&I));
      }
    }

    llvm::Value *immOffset = isConstant0 ? addSubInst->getOperand(0) : addSubInst->getOperand(1);

    LSCImmOffsetPattern *pattern =
        new (m_allocator) LSCImmOffsetPattern(&I, varOffset, llvm::dyn_cast<llvm::ConstantInt>(immOffset));
    AddPattern(pattern);
    return true;
  }

  return false;
}
// Pattern match to handle efficient 64 bit messages.
// It searches for uniform base variable that can be used in scalar register (s0)
// and scale factors.
// Example, where %testOutputPointer is uniform:
//   %0 = ptrtoint float addrspace(1)* %testOutputPointer to i64
//   %1 = shl nsw i64 %idxprom, 2
//   %2 = add i64 %1, %0
//   %3 = inttoptr i64 %2 to float addrspace(1)*
//   %4 = load float, float addrspace(1)* %3, align 4
//   %mul = fmul float %4, %4
//   store float %mul, float addrspace(1)* %3, align 4
//
// This should be eventually matched to vISA where the uniform pointer is put
// in the "global offset" operand (aka surface index), and we will get the scale from shl inst.
//   e.g. load will look like:
//   lsc_load.ugm (M1, 16)  V0033:d32 flat(testOutputPointer_0)[0x4*V0032]:a64
//
// With this pattern applied we can get rid of additional add instruction that sums the
// uniform base (%testOutputPointer) with offset - derived from %idxprom in above example
// as well as shl instruction that scaled the pointer to type width in bytes.
bool CodeGenPatternMatch::MatchLoadStoreAtomicsStatelessUniformBase(llvm::Instruction &I) {
  using namespace llvm;
  using namespace llvm::PatternMatch;

  struct LSCUniformBasePattern : public Pattern {
    explicit LSCUniformBasePattern(Instruction *I, llvm::Value *uniformBase, llvm::Value *varOffset,
                                   llvm::ConstantInt *immOffset, llvm::ConstantInt *scale, bool signExtendOffset,
                                   bool zeroExtendOffset)
        : m_inst(I), m_uniformBase(uniformBase), m_varOffset(varOffset), m_immOffset(immOffset), m_scale(scale),
          m_signExtendOffset(signExtendOffset), m_zeroExtendOffset(zeroExtendOffset) {}

    virtual void Emit(EmitPass *pass, const DstModifier &modifier) {
      if (isa<LoadInst>(m_inst)) {
        pass->emitLoad(cast<LoadInst>(m_inst), m_varOffset, m_immOffset, m_scale, m_uniformBase, m_signExtendOffset,
                       m_zeroExtendOffset);
      } else if (isa<StoreInst>(m_inst)) {
        pass->emitStore(cast<StoreInst>(m_inst), m_varOffset, m_immOffset, m_scale, m_uniformBase, m_signExtendOffset,
                        m_zeroExtendOffset);
      } else if (isa<GenIntrinsicInst>(m_inst)) {
        pass->emitAtomicRaw(cast<GenIntrinsicInst>(m_inst), m_varOffset, m_immOffset, m_scale, m_uniformBase,
                            m_signExtendOffset, m_zeroExtendOffset);
      } else {
        IGC_ASSERT_MESSAGE(0, "Expected load, store or atomic instruction!");
      }
    }

  private:
    llvm::Instruction *m_inst;
    llvm::Value *m_uniformBase;
    llvm::Value *m_varOffset;
    llvm::ConstantInt *m_immOffset;
    llvm::ConstantInt *m_scale;
    bool m_signExtendOffset;
    bool m_zeroExtendOffset;
  }; // LSCLoadStoreUniformBasePattern

  Value *Base = nullptr;
  Value *Offset = nullptr;
  ConstantInt *ImmOffset = nullptr;
  Value *PointerOperand = nullptr;
  Value *AddressOperand = nullptr;
  ConstantInt *Scale = nullptr;
  Value *NotScaledOffset = nullptr;
  llvm::Align InstAlign = llvm::Align(1);
  llvm::Align DataAlign = llvm::Align(1);
  unsigned DataSizeInBytes = 0;
  bool IsScalar = true;

  if (GenIntrinsicInst *GII = llvm::dyn_cast<GenIntrinsicInst>(&I)) {
    switch (GII->getIntrinsicID()) {
    case GenISAIntrinsic::GenISA_intatomicraw:
    case GenISAIntrinsic::GenISA_floatatomicraw:
    case GenISAIntrinsic::GenISA_icmpxchgatomicraw:
    case GenISAIntrinsic::GenISA_fcmpxchgatomicraw:
    case GenISAIntrinsic::GenISA_atomiccounterinc:
    case GenISAIntrinsic::GenISA_atomiccounterpredec:
      PointerOperand = GII->getOperand(0);
      AddressOperand = GII->getOperand(1);
      // For atomic operations, address needs to be aligned to data size.
      DataAlign = IGCLLVM::getABITypeAlign(I.getModule()->getDataLayout(), I.getType());
      InstAlign = DataAlign;
      DataSizeInBytes = I.getType()->getPrimitiveSizeInBits() / 8;
      break;

    case GenISAIntrinsic::GenISA_intatomicrawA64:
    case GenISAIntrinsic::GenISA_floatatomicrawA64:
    case GenISAIntrinsic::GenISA_icmpxchgatomicrawA64:
    case GenISAIntrinsic::GenISA_fcmpxchgatomicrawA64:
      PointerOperand = GII->getOperand(1);
      // For atomic operations, address needs to be aligned to data size.
      DataAlign = IGCLLVM::getABITypeAlign(I.getModule()->getDataLayout(), I.getType());
      InstAlign = DataAlign;
      DataSizeInBytes = I.getType()->getPrimitiveSizeInBits() / 8;
      break;
    default:
      return false;
    }
  } else if (auto LI = dyn_cast<LoadInst>(&I)) {
    PointerOperand = LI->getPointerOperand();
    InstAlign = IGCLLVM::getAlign(*LI);
    DataAlign = IGCLLVM::getABITypeAlign(LI->getModule()->getDataLayout(), LI->getType());
    IsScalar = !LI->getType()->isVectorTy();
    DataSizeInBytes = LI->getType()->getPrimitiveSizeInBits() / 8;

  } else if (auto SI = dyn_cast<StoreInst>(&I)) {
    PointerOperand = SI->getPointerOperand();
    InstAlign = IGCLLVM::getAlign(*SI);
    DataAlign = IGCLLVM::getABITypeAlign(SI->getModule()->getDataLayout(), SI->getValueOperand()->getType());
    IsScalar = !SI->getValueOperand()->getType()->isVectorTy();
    DataSizeInBytes = SI->getValueOperand()->getType()->getPrimitiveSizeInBits() / 8;
  } else {
    IGC_ASSERT_MESSAGE(0, "Expected load, store or atomic instruction!");
    return false;
  }

  // Base address must be data element aligned for non-scalar types, except for D16 type.
  bool NeedsToBeDataAligned = !IsScalar;
  if (DataSizeInBytes == 2 && IsScalar &&
      (m_Platform.getWATable().Wa_14024686037 || m_Platform.GetProductFamily() == IGFX_CRI)) {
    NeedsToBeDataAligned = true;
  }
  if (NeedsToBeDataAligned) {
    if ((IGCLLVM::getAlignmentValue(InstAlign) % IGCLLVM::getAlignmentValue(DataAlign)) != 0)
      return false;
  }

  // Skip private address space instructions that use scratch memory,
  // as they will use stateful message, where the uniform base can't be
  // applied.
  if (PointerOperand->getType()->getPointerAddressSpace() == ADDRESS_SPACE_PRIVATE &&
      m_ctx->getModuleMetaData()->compOpt.UseScratchSpacePrivateMemory) {
    return false;
  }

  bool isA64AddressingModel = IGC::isA64Ptr(cast<PointerType>(PointerOperand->getType()), m_ctx);

  if (!isa<IntToPtrInst>(PointerOperand))
    return false;
  auto IntToPtr = cast<IntToPtrInst>(PointerOperand);

  if (!AddressOperand)
    AddressOperand = IntToPtr->getOperand(0);

  auto PatternAdd = m_c_Add(m_Value(Base), m_Value(Offset));
  auto ScalePattern = m_Shl(m_Value(NotScaledOffset), m_ConstantInt(Scale));

  if (!match(AddressOperand, PatternAdd))
    return false;

  if (isa<ConstantInt>(Base) || isa<ConstantInt>(Offset)) {
    if (!supportsLSCImmediateGlobalBaseOffset())
      return false;
    IGC_ASSERT_MESSAGE(!(isa<ConstantInt>(Base) && isa<ConstantInt>(Offset)),
                       "Constants should have been folded earlier!");
    if (isa<ConstantInt>(Base)) {
      std::swap(Base, Offset);
    }
    ImmOffset = cast<ConstantInt>(Offset);
    if (!match(Base, PatternAdd))
      return false;
  }

  if (!m_WI->isUniform(Base) && !m_WI->isUniform(Offset))
    return false;

  // We match add in either order, uniform needs to be in the Base
  // variable.
  if (!m_WI->isUniform(Base) && m_WI->isUniform(Offset)) {
    std::swap(Base, Offset);
  }

  if (m_WI->isUniform(Base) && m_WI->isUniform(Offset)) {
    if (match(Base, ScalePattern)) {
      std::swap(Base, Offset);
    }
  }

  // Check if the offset is scaled.
  if (match(Offset, ScalePattern)) {
    auto ScaleImm = 1ll << Scale->getSExtValue();
    if (DataSizeInBytes == ScaleImm) {
      Offset = NotScaledOffset;
      Scale = ConstantInt::get(cast<IntegerType>(Scale->getType()), ScaleImm);
    } else {
      Scale = nullptr;
    }
  }

  bool signExtendOffset = false;
  bool zeroExtendOffset = false;
  if (auto OffsetInst = dyn_cast<Instruction>(Offset)) {
    if ((OffsetInst->getOpcode() == Instruction::SExt || OffsetInst->getOpcode() == Instruction::ZExt) &&
        OffsetInst->getType()->isIntegerTy(64) && OffsetInst->getOperand(0)->getType()->isIntegerTy(32)) {

      Offset = OffsetInst->getOperand(0);
      if (OffsetInst->getOpcode() == Instruction::SExt) {
        signExtendOffset = true;
      } else if (OffsetInst->getOpcode() == Instruction::ZExt) {
        zeroExtendOffset = true;
      }
    }
  }

  if (ImmOffset) {
    bool disableA32ImmediateGlobalBaseOffset =
        !isA64AddressingModel && !valueIsPositive(Offset, m_DL) && IGC_GET_FLAG_VALUE(LscImmOffsMatch) < 3;

    if (disableA32ImmediateGlobalBaseOffset)
      return false;

    if (DataSizeInBytes == 0 || ImmOffset->getSExtValue() % DataSizeInBytes != 0) {
      // check if immOffset is a multiple of data size
      return false;
    }

    // checks if input fits in allocated bits
    // assumes signed
    auto fitsIn = [](int what, int bits) { return what >= -(1 << (bits - 1)) && what <= (1 << (bits - 1)) - 1; };

    // 22 bits in the message descriptor for immediate offset
    if (!fitsIn(static_cast<int>(ImmOffset->getSExtValue() / DataSizeInBytes), 22)) {
      return false;
    }
  }

  LSCUniformBasePattern *pattern =
      new (m_allocator) LSCUniformBasePattern(&I, Base, Offset, ImmOffset, Scale, signExtendOffset, zeroExtendOffset);

  MarkAsSource(Base, IsSourceOfSample(&I));
  MarkAsSource(Offset, IsSourceOfSample(&I));
  if (ImmOffset) {
    MarkAsSource(ImmOffset, IsSourceOfSample(&I));
  }
  if (isa<StoreInst>(I)) {
    MarkAsSource(cast<StoreInst>(I).getValueOperand(), IsSourceOfSample(&I));
  } else if (auto GII = llvm::dyn_cast<GenIntrinsicInst>(&I)) {
    for (unsigned i = 0; i < GII->getNumOperands(); ++i) {
      // Operand 1 ommitted as it was the pointer that we
      // matched against above.
      // If any operand is equal to the pointer operand, it is
      // not marked as a source as well, as it is usually the case in
      // atomic instruction calls.
      if (GII->getOperand(i) == GII->getOperand(1))
        continue;

      MarkAsSource(GII->getOperand(i), IsSourceOfSample(&I));
    }
  }
  AddPattern(pattern);
  return true;
}

std::optional<std::pair<Value *, unsigned>> CodeGenPatternMatch::matchStateIndex(Value *resourcePtr, bool sampler) {
  if (IGC_IS_FLAG_ENABLED(DisableStatefulFolding))
    return {};
  if (resourcePtr == nullptr)
    return {};
  using namespace llvm::PatternMatch;
  Value *TmpBase = nullptr;
  ConstantInt *SurfaceOffset = nullptr;
  if (!match(resourcePtr, m_IntToPtr(m_c_Add(m_Value(TmpBase), m_ConstantInt(SurfaceOffset)))))
    return {};

  unsigned SurfaceStateSize = sampler ? m_Platform.GetBindlessSamplerSize() : m_ctx->m_DriverInfo.getSurfaceStateSize();
  if (SurfaceStateSize == 0)
    return {};

  uint64_t Offset = SurfaceOffset->getZExtValue();
  if (Offset % SurfaceStateSize != 0)
    return {};

  uint64_t Idx = Offset / SurfaceStateSize;
  if ((!sampler && TmpBase->getType()->getPrimitiveSizeInBits() == 64 && Idx < 32) || (sampler && Idx < 7))
    return std::make_pair(TmpBase, unsigned(Idx));
  return {};
}

bool CodeGenPatternMatch::MatchLoadStoreAtomicsStatefulEff64(GenIntrinsicInst *I) {
  if (IGC_IS_FLAG_ENABLED(DisableStatefulFolding))
    return false;

  using namespace llvm::PatternMatch;

  struct LSCStatefulPattern : public Pattern {
    explicit LSCStatefulPattern(GenIntrinsicInst *I, Value *VarOffset, ConstantInt *ImmScale, ConstantInt *ImmOffset)
        : I(I), VarOffset(VarOffset), ImmScale(ImmScale), ImmOffset(ImmOffset) {}

    void Emit(EmitPass *pass, const DstModifier & /* modifier */) override {
      if (auto *LRI = dyn_cast<LdRawIntrinsic>(I)) {
        pass->emitLoadRawIndexed(LRI, VarOffset, ImmScale, ImmOffset);
      } else if (auto *SRI = dyn_cast<StoreRawIntrinsic>(I)) {
        pass->emitStoreRawIndexed(SRI, VarOffset, ImmScale, ImmOffset);
      } else if (auto *ARI = dyn_cast<AtomicRawIntrinsic>(I)) {
        pass->emitAtomicRaw(ARI, VarOffset, ImmOffset, ImmScale);
      } else if (auto *ACI = dyn_cast<AtomicCounterIntrinsic>(I)) {
        pass->emitAtomicCounter(ACI);
      } else {
        IGC_ASSERT_MESSAGE(false, "unmatched pattern");
      }
    }

  private:
    GenIntrinsicInst *I;
    Value *VarOffset;
    ConstantInt *ImmScale;
    ConstantInt *ImmOffset;
  };

  // The subspan-destination workaround only applies to vector LOADS:
  // EmitPass::emitLSCVectorLoad() ANDs off the low bits of the subspan
  // destination, which the folded match does not account for. Atomics and
  // stores have no such destination masking, so they must NOT bail here --
  // otherwise they fall through to MatchLoadStoreAtomicsStatelessUniformBase(),
  // which splits a uniform byte offset into a separate uniform base that the
  // SURF (Efficient64b stateful) encoder cannot carry and silently drops
  // (see CEncoder::LSC_AtomicRaw in CISABuilder.cpp), corrupting the address.
  // TODO: also fold loads once emitLSCVectorLoad() handles the subspan case.
  if (m_ctx->type == ShaderType::PIXEL_SHADER && (I->getIntrinsicID() == GenISAIntrinsic::GenISA_ldraw_indexed ||
                                                  I->getIntrinsicID() == GenISAIntrinsic::GenISA_ldrawvector_indexed))
    return false;

  Type *DataTy = nullptr;

  Value *Base = nullptr;
  Value *VarOffset = nullptr;

  switch (I->getIntrinsicID()) {
  case GenISAIntrinsic::GenISA_intatomicraw:
  case GenISAIntrinsic::GenISA_floatatomicraw:
  case GenISAIntrinsic::GenISA_icmpxchgatomicraw:
  case GenISAIntrinsic::GenISA_fcmpxchgatomicraw: {
    auto *ARI = cast<AtomicRawIntrinsic>(I);
    Base = ARI->getResourceValue();
    VarOffset = ARI->getOperand(1);
    DataTy = ARI->getType();
    break;
  }
  case GenISAIntrinsic::GenISA_atomiccounterinc:
  case GenISAIntrinsic::GenISA_atomiccounterpredec: {
    auto *ACI = cast<AtomicCounterIntrinsic>(I);
    Base = ACI->getResourceValue();
    VarOffset = ACI->getOperand(1);
    DataTy = ACI->getType();
    break;
  }
  case GenISAIntrinsic::GenISA_ldraw_indexed:
  case GenISAIntrinsic::GenISA_ldrawvector_indexed: {
    auto *LRI = cast<LdRawIntrinsic>(I);
    Base = LRI->getResourceValue();
    VarOffset = LRI->getOffsetValue();
    DataTy = LRI->getType();
    break;
  }
  case GenISAIntrinsic::GenISA_storeraw_indexed:
  case GenISAIntrinsic::GenISA_storerawvector_indexed: {
    auto *SRI = cast<StoreRawIntrinsic>(I);
    Base = SRI->getResourceValue();
    VarOffset = SRI->getOffsetValue();
    DataTy = SRI->getStoreValue()->getType();
    break;
  }
  default:
    return false;
  }

  ConstantInt *Scale = nullptr;
  ConstantInt *ImmOffset = nullptr;

  const DataLayout &DL = I->getModule()->getDataLayout();
  unsigned DataSizeInBytes = DL.getTypeSizeInBits(DataTy) / 8;

  if (DataTy->isSingleValueType() && I->getIntrinsicID() != GenISAIntrinsic::GenISA_atomiccounterinc &&
      I->getIntrinsicID() != GenISAIntrinsic::GenISA_atomiccounterpredec) {
    Value *TmpVarOffset = nullptr;
    if (match(VarOffset, m_c_Add(m_Value(TmpVarOffset), m_ConstantInt(ImmOffset)))) {
      // HW does an early bounds check on TmpVarOffset. Thus, if
      // TmpVarOffset is negative, then the bounds check fails early
      // even though the immediate offset would bring the final
      // calculation to a positive number.
      bool CanFoldImmediate =
          valueIsPositive(TmpVarOffset, m_DL);

      // checks if input fits in allocated bits
      // assumes signed
      auto fitsIn = [](int what, int bits) { return what >= -(1 << (bits - 1)) && what <= (1 << (bits - 1)) - 1; };
      if (ImmOffset->isNegative() || !CanFoldImmediate || (ImmOffset->getSExtValue() % DataSizeInBytes) != 0 ||
          !fitsIn(static_cast<int>(ImmOffset->getSExtValue() / DataSizeInBytes), 17))
        ImmOffset = nullptr;
      else
        VarOffset = TmpVarOffset;
    }

    if (m_Platform.supportStatefulScaleFolding()) {
      Value *NotScaledOffset = nullptr;
      // Check if the offset is scaled.
      if (match(VarOffset, m_Shl(m_Value(NotScaledOffset), m_ConstantInt(Scale)))) {
        int64_t ScaleImm = 1ll << Scale->getSExtValue();
        if (DataSizeInBytes == ScaleImm) {
          VarOffset = NotScaledOffset;
          Scale = ConstantInt::get(cast<IntegerType>(Scale->getType()), ScaleImm);
        } else {
          Scale = nullptr;
        }
      }
    }
  }

  auto *Pattern = new (m_allocator) LSCStatefulPattern(I, VarOffset, Scale, ImmOffset);

  // Try to match surface state index.
  // If matched, use the base resource for marking source.
  // This is only for marking source purpose, to avoid emitting redundant add instructions.
  // Surface state index is not used as param for emit instruction here, it will be matched again in emit pass.
  // This is required for GetResourceVariable() in emit pass, as it work on original resource pointer.
  if (auto SurfaceStateIndex = matchStateIndex(Base))
    MarkAsSource(SurfaceStateIndex->first, IsSourceOfSample(I));
  else
    MarkAsSource(Base, IsSourceOfSample(I));
  MarkAsSource(VarOffset, IsSourceOfSample(I));
  if (auto *SRI = dyn_cast<StoreRawIntrinsic>(I))
    MarkAsSource(SRI->getStoreValue(), IsSourceOfSample(I));
  else if (auto *ARI = dyn_cast<AtomicRawIntrinsic>(I)) {
    for (unsigned i = 0; i < ARI->getNumOperands(); ++i) {
      // Operand 1 ommitted as it was the pointer that we
      // matched against above.
      if (ARI->getOperand(i) == ARI->getOperand(1))
        continue;
      MarkAsSource(ARI->getOperand(i), IsSourceOfSample(I));
    }
  }
  AddPattern(Pattern);
  return true;
}

bool CodeGenPatternMatch::MatchLrp(llvm::BinaryOperator &I) {
  struct LRPPattern : public Pattern {
    SSource sources[3];
    virtual void Emit(EmitPass *pass, const DstModifier &modifier) { pass->Lrp(sources, modifier); }
  };

  if (!I.getType()->isFloatTy())
    return false;
  if (!m_Platform.supportLRPInstruction())
    return false;

  if (m_ctx->getModuleMetaData()->isPrecise) {
    return false;
  }

  bool found = false;
  llvm::Value *sources[3];
  e_modifier src_mod[3];

  if (!ContractionAllowed(I)) {
    return false;
  }

  IGC_ASSERT((I.getOpcode() == Instruction::FAdd) || (I.getOpcode() == Instruction::FSub));

  bool startPatternIsAdd = false;
  if (I.getOpcode() == Instruction::FAdd) {
    startPatternIsAdd = true;
  }

  // match the case: dst = src0 (src1 - src2)  + src2;
  for (uint i = 0; i < 2; i++) {
    llvm::BinaryOperator *mul = llvm::dyn_cast<llvm::BinaryOperator>(I.getOperand(i));
    if (mul && mul->getOpcode() == Instruction::FMul && ContractionAllowed(*mul)) {
      for (uint j = 0; j < 2; j++) {
        llvm::BinaryOperator *sub = llvm::dyn_cast<llvm::BinaryOperator>(mul->getOperand(j));
        if (sub && ContractionAllowed(*sub)) {
          llvm::ConstantFP *zero = llvm::dyn_cast<llvm::ConstantFP>(sub->getOperand(0));
          if (zero && zero->isExactlyValue(0.f)) {
            // in this case we can optimize the pattern into fmad and give better result
            continue;
          }

          if ((startPatternIsAdd && sub->getOpcode() == Instruction::FSub) ||
              (!startPatternIsAdd && i == 0 && sub->getOpcode() == Instruction::FAdd)) {
            if (sub->getOperand(1) == I.getOperand(1 - i) && mul->getOperand(0) != mul->getOperand(1)) {
              sources[0] = mul->getOperand(1 - j);
              sources[1] = sub->getOperand(0);
              sources[2] = sub->getOperand(1);
              GetModifier(*sources[0], src_mod[0], sources[0]);
              GetModifier(*sources[1], src_mod[1], sources[1]);
              GetModifier(*sources[2], src_mod[2], sources[2]);

              if (!startPatternIsAdd && i == 0) {
                // handle patterns like this:
                // dst = src0 (src1 + src2) - src2;
                src_mod[2] = CombineModifier(EMOD_NEG, src_mod[2]);
              }

              found = true;
              break;
            }
          }
        }
      }
    }
    if (found) {
      break;
    }
  }

  // match the case: dst = src0 * src1 + src2 * (1.0 - src0);
  if (!found) {
    llvm::BinaryOperator *mul[2];
    mul[0] = llvm::dyn_cast<llvm::BinaryOperator>(I.getOperand(0));
    mul[1] = llvm::dyn_cast<llvm::BinaryOperator>(I.getOperand(1));
    if (mul[0] && mul[0]->getOpcode() == Instruction::FMul && ContractionAllowed(*mul[0]) && mul[1] &&
        mul[1]->getOpcode() == Instruction::FMul && ContractionAllowed(*mul[1]) &&
        !llvm::isa<llvm::ConstantFP>(mul[0]->getOperand(0)) && !llvm::isa<llvm::ConstantFP>(mul[0]->getOperand(1)) &&
        !llvm::isa<llvm::ConstantFP>(mul[1]->getOperand(0)) && !llvm::isa<llvm::ConstantFP>(mul[1]->getOperand(1))) {
      for (uint i = 0; i < 2; i++) {
        for (uint j = 0; j < 2; j++) {
          llvm::BinaryOperator *sub = llvm::dyn_cast<llvm::BinaryOperator>(mul[i]->getOperand(j));
          if (sub && sub->getOpcode() == Instruction::FSub && ContractionAllowed(*sub)) {
            llvm::ConstantFP *one = llvm::dyn_cast<llvm::ConstantFP>(sub->getOperand(0));
            if (one && one->isExactlyValue(1.f)) {
              for (uint k = 0; k < 2; k++) {
                if (sub->getOperand(1) == mul[1 - i]->getOperand(k)) {
                  sources[0] = sub->getOperand(1);
                  sources[1] = mul[1 - i]->getOperand(1 - k);
                  sources[2] = mul[i]->getOperand(1 - j);
                  GetModifier(*sources[0], src_mod[0], sources[0]);
                  GetModifier(*sources[1], src_mod[1], sources[1]);
                  GetModifier(*sources[2], src_mod[2], sources[2]);
                  if (!startPatternIsAdd) {
                    if (i == 1) {
                      // handle patterns like this:
                      // dst = (src1 * src0) - (src2 * (1.0 - src0))
                      src_mod[2] = CombineModifier(EMOD_NEG, src_mod[2]);
                    } else {
                      // handle patterns like this:
                      // dst = (src2 * (1.0 - src0)) - (src1 * src0)
                      src_mod[1] = CombineModifier(EMOD_NEG, src_mod[1]);
                    }
                  }
                  found = true;
                  break;
                }
              }
            }
          }
          if (found) {
            break;
          }
        }
        if (found) {
          break;
        }
      }
    }
  }

  if (!found) {
    // match the case: dst = src2 - (src0 * src2) + (src0 * src1);
    // match the case: dst = (src0 * src1) + src2 - (src0 * src2);
    // match the case: dst = src2 + (src0 * src1) - (src0 * src2);
    if (I.getOpcode() == Instruction::FAdd || I.getOpcode() == Instruction::FSub) {
      // dst = op[0] +/- op[1] +/- op[2]
      llvm::Instruction *op[3];
      llvm::Instruction *addSub1 = llvm::dyn_cast<llvm::Instruction>(I.getOperand(0));
      if (addSub1 && (addSub1->getOpcode() == Instruction::FSub || addSub1->getOpcode() == Instruction::FAdd)) {
        op[0] = llvm::dyn_cast<llvm::Instruction>(addSub1->getOperand(0));
        op[1] = llvm::dyn_cast<llvm::Instruction>(addSub1->getOperand(1));
        op[2] = llvm::dyn_cast<llvm::Instruction>(I.getOperand(1));

        if (op[0] && op[1] && op[2]) {
          for (uint casei = 0; casei < 3; casei++) {
            // i, j, k are the index for op[]
            uint i = (casei == 2 ? 1 : 0);
            uint j = (casei == 0 ? 1 : 2);
            uint k = 2 - casei;

            // op[i] and op[j] should be fMul, and op[k] is src2
            if (op[i]->getOpcode() == Instruction::FMul && op[j]->getOpcode() == Instruction::FMul &&
                ContractionAllowed(*op[i]) && ContractionAllowed(*op[j])) {
              for (uint srci = 0; srci < 2; srci++) {
                for (uint srcj = 0; srcj < 2; srcj++) {
                  // op[i] and op[j] needs to have one common source. this common source will be src0
                  if (op[i]->getOperand(srci) == op[j]->getOperand(srcj)) {
                    // one of the non-common source from op[i] and op[j] needs to be the same as op[k], which is src2
                    if (op[k] == op[i]->getOperand(1 - srci) || op[k] == op[j]->getOperand(1 - srcj)) {
                      // disable if any of the sources is immediate
                      if (llvm::isa<llvm::ConstantFP>(op[i]->getOperand(srci)) ||
                          llvm::isa<llvm::ConstantFP>(op[i]->getOperand(1 - srci)) ||
                          llvm::isa<llvm::ConstantFP>(op[j]->getOperand(srcj)) ||
                          llvm::isa<llvm::ConstantFP>(op[j]->getOperand(1 - srcj)) ||
                          llvm::isa<llvm::ConstantFP>(op[k])) {
                        break;
                      }

                      // check the add/sub cases and add negate to the sources when needed.
                      /*
                      ( src0src1, -src0src2, src2 )   okay
                      ( src0src1, -src0src2, -src2 )  skip
                      ( src0src1, src0src2, src2 )    skip
                      ( src0src1, src0src2, -src2 )   negate src2
                      ( -src0src1, -src0src2, src2 )  negate src1
                      ( -src0src1, -src0src2, -src2 ) skip
                      ( -src0src1, src0src2, src2 )   skip
                      ( -src0src1, src0src2, -src2 )  negate src1 src2
                      */

                      bool SignPositiveOp[3];
                      SignPositiveOp[0] = true;
                      SignPositiveOp[1] = (addSub1->getOpcode() == Instruction::FAdd);
                      SignPositiveOp[2] = (I.getOpcode() == Instruction::FAdd);

                      uint mulSrc0Src1Index = op[k] == op[i]->getOperand(1 - srci) ? j : i;
                      uint mulSrc0Src2Index = op[k] == op[i]->getOperand(1 - srci) ? i : j;

                      if (SignPositiveOp[mulSrc0Src2Index] == SignPositiveOp[k]) {
                        // abort the cases marked as "skip" in the comment above
                        break;
                      }

                      sources[0] = op[i]->getOperand(srci);
                      sources[1] = op[k] == op[i]->getOperand(1 - srci) ? op[j]->getOperand(1 - srcj)
                                                                        : op[i]->getOperand(1 - srci);
                      sources[2] = op[k];
                      GetModifier(*sources[0], src_mod[0], sources[0]);
                      GetModifier(*sources[1], src_mod[1], sources[1]);
                      GetModifier(*sources[2], src_mod[2], sources[2]);

                      if (SignPositiveOp[mulSrc0Src1Index] == false) {
                        src_mod[1] = CombineModifier(EMOD_NEG, src_mod[1]);
                      }
                      if (SignPositiveOp[k] == false) {
                        src_mod[2] = CombineModifier(EMOD_NEG, src_mod[2]);
                      }

                      found = true;
                    }
                  }
                }
                if (found) {
                  break;
                }
              }
            }
            if (found) {
              break;
            }
          }
        }
      }
    }
  }

  if (found) {
    LRPPattern *pattern = new (m_allocator) LRPPattern();
    for (int i = 0; i < 3; i++) {
      pattern->sources[i] = GetSource(sources[i], src_mod[i], false, IsSourceOfSample(&I));
    }
    AddPattern(pattern);
  }
  return found;
}

bool CodeGenPatternMatch::MatchCmpSext(llvm::Instruction &I) {
  /*
      %res_s42 = icmp eq i32 %src1_s41, 0
      %17 = sext i1 %res_s42 to i32
          to
      %res_s42 (i32) = icmp eq i32 %src1_s41, 0


      %res_s73 = fcmp oge float %res_s61, %42
      %46 = sext i1 %res_s73 to i32
          to
      %res_s73 (i32) = fcmp oge float %res_s61, %42
  */

  struct CmpSextPattern : Pattern {
    llvm::CmpInst *inst;
    SSource sources[2];
    virtual void Emit(EmitPass *pass, const DstModifier &modifier) {
      pass->Cmp(inst->getPredicate(), sources, modifier);
    }
  };
  [[maybe_unused]] bool match = false;

  if (CmpInst *cmpInst = dyn_cast<CmpInst>(I.getOperand(0))) {
    if (cmpInst->getOperand(0)->getType()->getPrimitiveSizeInBits() == I.getType()->getPrimitiveSizeInBits()) {
      CmpSextPattern *pattern = new (m_allocator) CmpSextPattern();
      bool supportModifier = SupportsModifier(cmpInst, m_Platform);

      pattern->inst = cmpInst;
      pattern->sources[0] = GetSource(cmpInst->getOperand(0), supportModifier, false, IsSourceOfSample(&I));
      pattern->sources[1] = GetSource(cmpInst->getOperand(1), supportModifier, false, IsSourceOfSample(&I));
      AddPattern(pattern);
      match = true;
    }
  }

  return match;
}

// Matches the following sequence:
//  %src0v4i8 = bitcast i32 %x to <4 x i8>
//  %elem1 = extractelement <4 x i8> %src0v4i8, i32 1
//  %dst = {s|z}ext i8 %elem1 to i32
bool CodeGenPatternMatch::MatchUnpack4i8(Instruction &I) {
  struct UnpackPattern : public Pattern {
    SSource source;
    uint32_t index;
    bool isUnsigned;
    virtual void Emit(EmitPass *pass, const DstModifier &modifier) {
      pass->EmitUnpack4i8(source, index, isUnsigned, modifier);
    }
  };
  if (I.getType()->isIntegerTy(32) && isa<ZExtInst, SExtInst>(&I)) {
    auto extract = dyn_cast<ExtractElementInst>(I.getOperand(0));
    if (extract && isa<ConstantInt>(extract->getIndexOperand()) && isa<BitCastInst>(extract->getVectorOperand()) &&
        cast<BitCastInst>(extract->getVectorOperand())->getType()->isIntOrIntVectorTy(8) &&
        !cast<BitCastInst>(extract->getVectorOperand())->getOperand(0)->getType()->isVectorTy() &&
        cast<BitCastInst>(extract->getVectorOperand())->getOperand(0)->getType()->getPrimitiveSizeInBits() == 32) {
      UnpackPattern *pattern = new (m_allocator) UnpackPattern();
      auto bitcast = cast<BitCastInst>(extract->getVectorOperand());
      uint32_t index = int_cast<uint32_t>(cast<ConstantInt>(extract->getIndexOperand())->getZExtValue());
      pattern->source = GetSource(bitcast->getOperand(0), false, false, IsSourceOfSample(&I));
      pattern->index = index;
      pattern->isUnsigned = isa<ZExtInst>(&I);
      AddPattern(pattern);
      return true;
    }
  }
  return false;
}

// Matches the following sequences:
//  %val = load i8, ptr addrspace(N) %ptr
//  %dst = zext i8 %val to i32
// or:
//  %val = load i16, ptr addrspace(N) %ptr
//  %dst = zext i16 %val to i32
// When an LSC sub-dword load (d8u32 / d16u32) is used, the load result is
// already zero-extended to 32 bits in vISA. This pattern allows the zext to
// directly use the dword-typed load result, eliminating intermediate movs.
bool CodeGenPatternMatch::MatchZExtByteLoad(Instruction &I) {
  struct ZExtByteLoadPattern : public Pattern {
    llvm::LoadInst *loadInst;
    llvm::Instruction *zextInst;
    virtual void Emit(EmitPass *pass, const DstModifier &modifier) {
      pass->EmitZExtByteLoad(loadInst, zextInst, modifier);
    }
  };
  // LSC d8u32/d16u32 sub-dword loads with zero-extension to dword are
  // only available on Xe2+ platforms.
  if (!m_Platform.isCoreChildOf(IGFX_XE2_HPG_CORE))
    return false;
  if (I.getType()->isIntegerTy(32) && isa<ZExtInst>(&I)) {
    auto *load = dyn_cast<LoadInst>(I.getOperand(0));
    if (load && (load->getType()->isIntegerTy(8) || load->getType()->isIntegerTy(16))) {
      // Only match when load, zext, and all zext users are in the same BB.
      BasicBlock *BB = I.getParent();
      if (load->getParent() != BB)
        return false;
      for (auto *U : I.users()) {
        if (auto *UI = dyn_cast<Instruction>(U))
          if (UI->getParent() != BB)
            return false;
      }
      ZExtByteLoadPattern *pattern = new (m_allocator) ZExtByteLoadPattern();
      pattern->loadInst = load;
      pattern->zextInst = &I;
      MarkAsSource(load, IsSourceOfSample(&I));
      AddPattern(pattern);
      return true;
    }
  }
  return false;
}

// Matches the following patterns
//
// Pattern 1, pack 4 i8 into i32
// Match the following sequence:
//  %x8 = trunc i32 %x to i8
//  %y8 = trunc i32 %y to i8
//  %z8 = trunc i32 %z to i8
//  %w8 = trunc i32 %w to i8
//  %vec0 = insertelement <4 x i8> undef, i8 %x8, i64 0
//  %vec1 = insertelement <4 x i8> %vec0, i8 %y8, i64 1
//  %vec2 = insertelement <4 x i8> %vec1, i8 %z8, i64 2
//  %vec3 = insertelement <4 x i8> %vec2, i8 %w8, i64 3
//  %dst = bitcast <4 x i8> %vec3 to i32
// And generate:
//  mov (M1_NM, 1) dst_0(0,0)<4> x(0,0)<0;1,0>
//  mov (M1_NM, 1) dst_0(0,1)<4> y(0,0)<0;1,0>
//  mov (M1_NM, 1) dst_0(0,2)<4> z(0,0)<0;1,0>
//  mov (M1_NM, 1) dst_0(0,3)<4> w(0,0)<0;1,0>
//
// Pattern 2, pack 4 i8 into i32 with saturation
// Match the following sequence:
//  %x1 = max i32 %x0, 0
//  %x2 = min i32 %x1, 127
//  %y1 = max i32 %y0, 0
//  %y2 = min i32 %y1, 127
//  %z1 = max i32 %z0, 0
//  %z2 = min i32 %z1, 127
//  %w1 = max i32 %w0, 0
//  %w2 = min i32 %w1, 127
//  %x8 = trunc i32 %x2 to i8
//  %y8 = trunc i32 %y2 to i8
//  %z8 = trunc i32 %z2 to i8
//  %w8 = trunc i32 %w2 to i8
//  %vec0 = insertelement <4 x i8> undef, i8 %x8, i64 0
//  %vec1 = insertelement <4 x i8> %vec0, i8 %y8, i64 1
//  %vec2 = insertelement <4 x i8> %vec1, i8 %z8, i64 2
//  %vec3 = insertelement <4 x i8> %vec2, i8 %w8, i64 3
//  %dst = bitcast <4 x i8> %vec3 to i32
// And generate:
//  max.sat (M1_NM, 1) xyzw_0(0,0)<4> x(0,0)<0;1,0> 0x0:d
//  max.sat (M1_NM, 1) xyzw_0(0,1)<4> y(0,0)<0;1,0> 0x0:d
//  max.sat (M1_NM, 1) xyzw_0(0,2)<4> z(0,0)<0;1,0> 0x0:d
//  max.sat (M1_NM, 1) xyzw_0(0,3)<4> w(0,0)<0;1,0> 0x0:d
//
// Pattern 3, pack 4 i8 into i32 with saturation
// Match the following sequence:
//  %x1 = max i32 %x0, 0
//  %x2 = min i32 %x1, 127
//  %y1 = max i32 %y0, 0
//  %y2 = min i32 %y1, 127
//  %z1 = max i32 %z0, 0
//  %z2 = min i32 %z1, 127
//  %w1 = max i32 %w0, 0
//  %w2 = min i32 %w1, 127
//  %y8 = shl nuw nsw i32 %y2, 8
//  %xy = or i32 %x2, %y8
//  %z8 = shl nuw nsw i32 %z2, 16
//  %xyz = or i32 %z8, %xy
//  %w8 = shl nuw nsw i32 %w2, 24
//  %dst = or i32 %xyz, %w8
// And generate:
//  max.sat (M1_NM, 1) xyzw_0(0,0)<4> x(0,0)<0;1,0> 0x0:d
//  max.sat (M1_NM, 1) xyzw_0(0,1)<4> y(0,0)<0;1,0> 0x0:d
//  max.sat (M1_NM, 1) xyzw_0(0,2)<4> z(0,0)<0;1,0> 0x0:d
//  max.sat (M1_NM, 1) xyzw_0(0,3)<4> w(0,0)<0;1,0> 0x0:d
//
bool CodeGenPatternMatch::MatchPack4i8(Instruction &I) {
  using namespace llvm::PatternMatch; // Scoped using declaration.

  struct PackPattern : public Pattern {
    std::array<EOPCODE, 4> opcodes;
    std::array<SSource, 4> sources0;
    std::array<SSource, 4> sources1;
    std::array<bool, 4> isSat;
    virtual void Emit(EmitPass *pass, const DstModifier &modifier) {
      pass->EmitPack4i8(opcodes, sources0, sources1, isSat, modifier);
    }
  };

  // Lambda matches a `min` or `max` operation with a specific immediate
  // constant value (equal to `imm`) and returns the other source value
  // in the `otherSrc`.
  auto MatchMinMaxWithImm = [this](Value *v, uint64_t imm, bool matchMin, Value *&otherSrc) -> bool {
    bool isUnsigned;
    bool isMin;
    Value *src[2];
    if (isMinOrMax(v, src[0], src[1], isMin, isUnsigned) && isMin == matchMin &&
        (isa<ConstantInt>(src[0]) || isa<ConstantInt>(src[1]))) {
      ConstantInt *c = isa<ConstantInt>(src[0]) ? cast<ConstantInt>(src[0]) : cast<ConstantInt>(src[1]);
      if (c->getZExtValue() == imm) {
        otherSrc = isa<ConstantInt>(src[0]) ? src[1] : src[0];
        return true;
      }
    }
    return false;
  };
  // Lambda matches clamp(x, MIN, MAX) pattern.
  // If the pattern is found `x` is returned in the `clampedVal` reference.
  auto MatchClampWithImm = [&MatchMinMaxWithImm](Value *v, Value *&clampedVal, uint32_t minVal,
                                                 uint32_t maxVal) -> bool {
    bool matchMin = true;
    bool matchMax = false;
    Value *src[2];
    // Match either of:
    // v = min(max(x, MIN), MAX)
    // v = max(min(x, MIN), MAX)
    if ((MatchMinMaxWithImm(v, maxVal, matchMin, src[0]) && MatchMinMaxWithImm(src[0], minVal, matchMax, src[1])) ||
        (MatchMinMaxWithImm(v, minVal, matchMax, src[0]) && MatchMinMaxWithImm(src[0], maxVal, matchMin, src[1]))) {
      clampedVal = src[1];
      return true;
    }
    return false;
  };
  // Lambda matches clamp(x, 0, 127) pattern.
  // If the pattern is found `x` is returned in the `clampedVal` reference.
  auto MatchClamp0_127 = [&MatchClampWithImm](Value *v, Value *&clampedVal) -> bool {
    return MatchClampWithImm(v, clampedVal, 0, 127);
  };

  EOPCODE opcodes[4] = {};
  Value *sources0[4] = {};
  Value *sources1[4] = {};
  bool isSat[4] = {};

  uint32_t elemsFound = 0;

  // Match patterns 1 and 2
  if (isa<BitCastInst>(I) && !I.getType()->isVectorTy() && I.getType()->getPrimitiveSizeInBits() == 32 &&
      I.getOperand(0)->getType()->isIntOrIntVectorTy(8) && isa<InsertElementInst>(I.getOperand(0))) {
    auto ie = cast<InsertElementInst>(I.getOperand(0));
    // Match sequences of InsertElementInsts, e.g.:
    //  %vec0 = insertelement <4 x i8> undef, i8 %x8, i64 0
    //  %vec1 = insertelement <4 x i8> %vec0, i8 %y8, i64 1
    //  %vec2 = insertelement <4 x i8> %vec1, i8 %z8, i64 2
    //  %vec3 = insertelement <4 x i8> %vec2, i8 %w8, i64 3
    while (ie) {
      auto idxVal = dyn_cast<ConstantInt>(ie->getOperand(2));
      auto idx = idxVal ? idxVal->getZExtValue() : 0;
      if (idxVal && sources0[idx] == nullptr) {
        sources0[idx] = ie->getOperand(1);
        ++elemsFound;
      } else {
        // Non-constant index is not supported.
        break;
      }
      ie = dyn_cast<InsertElementInst>(ie->getOperand(0));
    }
    if (elemsFound == 4) {
      // Match sequences integer trunc, e.g.:
      //  %x8 = trunc i32 %x2 to i8
      elemsFound = 0;
      for (uint32_t i = 0; i < 4; ++i) {
        auto trunc = dyn_cast<TruncInst>(sources0[i]);
        if (trunc) {
          opcodes[i] = llvm_bitcast;
          sources0[i] = trunc->getOperand(0);
          ++elemsFound;
        }
      }
    }
    if (elemsFound == 4) {
      // Match pattern 2
      for (uint32_t i = 0; i < 4; ++i) {
        Value *srcToSat;
        // Match clamping of values to 0..127 range, e.g.:
        //  %x1 = max i32 %x0, 0
        //  %x2 = min i32 %x1, 127
        if (MatchClamp0_127(sources0[i], srcToSat)) {
          opcodes[i] = llvm_max;
          sources0[i] = srcToSat;
          sources1[i] = ConstantInt::get(srcToSat->getType(), 0);
          isSat[i] = true;
          // Match clamping of values to -128..127 range, e.g.:
          //  %x1 = max i32 %x0, -128
          //  %x2 = min i32 %x1, 127
        } else if (MatchClampWithImm(sources0[i], srcToSat, -128, 127)) {
          opcodes[i] = llvm_fptosi;
          sources0[i] = srcToSat;
          isSat[i] = true;
        }
      }
    }
  }
  // Match pattern 3
  else if (I.getType()->getPrimitiveSizeInBits() == 32 && isa<BinaryOperator>(&I) &&
           cast<BinaryOperator>(&I)->getOpcode() == Instruction::Or) {
    auto orInst = cast<BinaryOperator>(&I);
    while (orInst) {
      // Match sequences of or + shl instructions, e.g.
      //  %y8 = shl nuw nsw i32 %y2, 8
      //  %xy = or i32 %x2, %y8
      ConstantInt *shlImm = nullptr;
      Value *shlSrc = nullptr;
      Value *orSrc = nullptr;
      if (match(orInst, m_Or(m_Value(orSrc), m_Shl(m_Value(shlSrc), m_ConstantInt(shlImm)))) ||
          match(orInst, m_Or(m_Shl(m_Value(shlSrc), m_ConstantInt(shlImm)), m_Value(orSrc)))) {
        // Lambda sets source values of a `sel` instruction.
        auto SetSource = [&](uint32_t i, Value *src0) {
          IGC_ASSERT(sources0[i] == nullptr);
          IGC_ASSERT(sources1[i] == nullptr);
          opcodes[i] = llvm_max;
          sources0[i] = src0;
          sources1[i] = ConstantInt::get(src0->getType(), 0);
          isSat[i] = true;
        };
        // Match clamping of the `shl` source to 0..127 range, e.g.:
        //  %x1 = max i32 %x0, 0
        //  %x2 = min i32 %x1, 127
        //  %x8 = shl nuw nsw i32 %x2, 8
        Value *srcToSat;
        uint32_t i = int_cast<uint32_t>(shlImm->getZExtValue());
        if (MatchClamp0_127(shlSrc, srcToSat) && ((i % 8) == 0 && (i / 8) < 4)) {
          SetSource(i / 8, srcToSat);
          ++elemsFound;
        }
        // Match clamping of the `or` source to 0..127 range, e.g.:
        //  %x1 = max i32 %x0, 0
        //  %x2 = min i32 %x1, 127
        //  %xy = or i32 %x2, %y8
        if (MatchClamp0_127(orSrc, srcToSat)) {
          SetSource(0, srcToSat);
          ++elemsFound;
        }
      }
      if (elemsFound != 4 && orSrc && isa<BinaryOperator>(orSrc) &&
          cast<BinaryOperator>(orSrc)->getOpcode() == Instruction::Or) {
        // Continue checking other remaining channels
        orInst = cast<BinaryOperator>(orSrc);
      } else {
        // All elements found, or an unsupported pattern.
        orInst = nullptr;
      }
    }
  }
  // TODO: consider adding support for patterns, where not all 4 i8 values
  // are packed, for example:
  //   %x8 = trunc i32 %x to i8
  //   %y8 = trunc i32 %y to i8
  //   %z8 = trunc i32 %z to i8
  //   %vec0 = insertelement <4 x i8> zeroinitializer, i8 %x8, i64 0
  //   %vec1 = insertelement <4 x i8> %vec0, i8 %y8, i64 1
  //   %vec2 = insertelement <4 x i8> %vec1, i8 %z8, i64 2
  //   %dst = bitcast <4 x i8> %vec3 to i32
  if (elemsFound == 4) {
    PackPattern *pattern = new (m_allocator) PackPattern();
    for (uint32_t i = 0; i < 4; ++i) {
      pattern->opcodes[i] = opcodes[i];
      pattern->sources0[i] = GetSource(sources0[i], false, false, IsSourceOfSample(&I));
      if (sources1[i] != nullptr) {
        pattern->sources1[i] = GetSource(sources1[i], false, false, IsSourceOfSample(&I));
      }
      pattern->isSat[i] = isSat[i];
    }
    AddPattern(pattern);
    return true;
  }
  return false;
}

// Matches repacking of packed <4 x i8> values, for example:
//   %src0v4i8 = bitcast i32 %x to <4 x i8>
//   %src1v4i8 = bitcast i32 %y to <4 x i8>
//   %elem1 = extractelement <4 x i8> %src0v4i8, i32 1
//   %dst0 = insertelement <4 x i8> %src0v4i8, i8 %elem1, i32 1
//   %elem2 = extractelement <4 x i8> %src1v4i8, i32 0
//   %dst1 = insertelement <4 x i8> %dst0, i8 %elem2, i32 2
//   %elem3 = extractelement <4 x i8> %src1v4i8, i32 1
//   %dst2 = insertelement <4 x i8> %dst1, i8 %elem3, i32 3
//   %dst3 = bitcast <4 x i8> %dst2 to i32
bool CodeGenPatternMatch::MatchRepack4i8(BitCastInst &I) {
  struct RepackPattern : public Pattern {
    std::array<SSource, 4> sources;
    std::array<uint32_t, 4> mappings;
    virtual void Emit(EmitPass *pass, const DstModifier &modifier) { pass->EmitRepack4i8(sources, mappings, modifier); }
  };

  if (!I.getType()->isVectorTy() && I.getType()->getPrimitiveSizeInBits() == 32 &&
      I.getOperand(0)->getType()->isIntOrIntVectorTy(8) && isa<InsertElementInst>(I.getOperand(0))) {
    Value *sources[4] = {};
    uint32_t mappings[4] = {};
    auto ie = cast<InsertElementInst>(I.getOperand(0));
    Value *baseVec = nullptr;
    uint32_t elemsFound = 0;
    while (ie) {
      auto idxVal = dyn_cast<ConstantInt>(ie->getOperand(2));
      if (!idxVal) {
        return false;
      }
      auto idx = idxVal->getZExtValue();
      if (idxVal && sources[idx] == nullptr) {
        sources[idx] = ie->getOperand(1);
        ++elemsFound;
      }
      baseVec = ie->getOperand(0);
      ie = dyn_cast<InsertElementInst>(baseVec);
    }

    for (uint32_t i = 0; i < 4; ++i) {
      if (sources[i]) {
        auto ee = dyn_cast<ExtractElementInst>(sources[i]);
        if (!ee || !isa<ConstantInt>(ee->getOperand(1)) || !isa<BitCastInst>(ee->getOperand(0)) ||
            (cast<BitCastInst>(ee->getOperand(0)))->getOperand(0)->getType()->isVectorTy() ||
            ((cast<BitCastInst>(ee->getOperand(0)))->getOperand(0)->getType()->getPrimitiveSizeInBits() != 32)) {
          return false;
        }
        auto bitcast = cast<BitCastInst>(ee->getOperand(0));
        mappings[i] = int_cast<uint32_t>(cast<ConstantInt>(ee->getOperand(1))->getZExtValue());
        sources[i] = bitcast->getOperand(0);
      } else {
        if (!isa<UndefValue>(baseVec) &&
            (!isa<BitCastInst>(baseVec) || (cast<BitCastInst>(baseVec))->getOperand(0)->getType()->isVectorTy() ||
             (cast<BitCastInst>(baseVec))->getOperand(0)->getType()->getPrimitiveSizeInBits() != 32)) {
          return false;
        }
        mappings[i] = i;
        auto bitcast = dyn_cast<BitCastInst>(baseVec);
        sources[i] = bitcast ? bitcast->getOperand(0) : baseVec;
      }
    }
    RepackPattern *pattern = new (m_allocator) RepackPattern();
    for (uint32_t i = 0; i < 4; ++i) {
      pattern->sources[i] = GetSource(sources[i], false, false, IsSourceOfSample(&I));
      pattern->sources[i].type = ISA_TYPE_UB;
      pattern->mappings[i] = mappings[i];
    }
    AddPattern(pattern);
    return true;
  }
  return false;
}

// Matches binary operations on packed <4 x i8> values.
// For example, the following patterns are matched:
//
// Shl+Asr pattern:
//   %x0 = shl i32 %x, 24
//   %dst = ashr i32 %x0, 28
//
// Unpack + shl pattern:
//   %src0v4i8 = bitcast i32 %x to <4 x i8>
//   %elem1 = extractelement <4 x i8> %src0v4i8, i32 1
//   %elem132 = sext i8 %elem1 to i32
//   %dst = shl i32 %elem132, 4
bool CodeGenPatternMatch::MatchBinaryUnpack4i8(Instruction &I) {
  struct BinaryUnpackPattern : public Pattern {
    SSource sources[2];
    Instruction *inst;
    virtual void Emit(EmitPass *pass, const DstModifier &modifier) { pass->BinaryUnary(inst, sources, modifier); }
  };

  Instruction *instToEmit = &I;
  if (I.getType()->isIntegerTy(32)) {
    std::tuple<bool, bool, Value *, uint32_t> sources[2];
    bool foundUnpack = false;
    // Shl+Asr pattern:
    //   %x0 = shl i32 %x, 24
    //   %dst = ashr i32 %x0, 28
    if (isa<LShrOperator, AShrOperator>(&I) && isa<ConstantInt>(cast<Instruction>(&I)->getOperand(1))) {
      auto shr = cast<Instruction>(&I);
      auto shl = dyn_cast<ShlOperator>(shr->getOperand(0));
      if (shl && isa<ConstantInt>(shl->getOperand(1)) && cast<ConstantInt>(shl->getOperand(1))->getZExtValue() == 24) {
        sources[0] = std::make_tuple(true, isa<LShrOperator>(&I), shl->getOperand(0), 0);
        auto shift = cast<ConstantInt>(shr->getOperand(1))->getZExtValue();
        bool isShl = shift < 24;
        if (isShl) {
          instToEmit = cast<Instruction>(shl);
          shift = 24 - shift;
        } else {
          shift = shift - 24;
        }

        sources[1] = std::make_tuple(false, false, ConstantInt::get(shr->getOperand(1)->getType(), shift), 0);
        foundUnpack = true;
      }
    } else {
      for (uint32_t i = 0; i < 2; ++i) {
        Value *op = I.getOperand(i);
        sources[i] = std::make_tuple(false, false, op, 0);
        if (isa<ZExtInst, SExtInst>(op)) {
          auto ext = cast<Instruction>(op);
          if (auto extract = dyn_cast<ExtractElementInst>(ext->getOperand(0))) {
            if (isa<ConstantInt>(extract->getIndexOperand()) && isa<BitCastInst>(extract->getVectorOperand()) &&
                cast<BitCastInst>(extract->getVectorOperand())->getType()->isIntOrIntVectorTy(8) &&
                !cast<BitCastInst>(extract->getVectorOperand())->getOperand(0)->getType()->isVectorTy() &&
                cast<BitCastInst>(extract->getVectorOperand())->getOperand(0)->getType()->getPrimitiveSizeInBits() ==
                    32) {
              sources[i] = std::make_tuple(
                  true, isa<ZExtInst>(op), cast<BitCastInst>(extract->getVectorOperand())->getOperand(0),
                  int_cast<uint32_t>(cast<ConstantInt>(extract->getIndexOperand())->getZExtValue()));
              foundUnpack = true;
            }
          }
        }
      }
    }
    if (foundUnpack) {
      BinaryUnpackPattern *pattern = new (m_allocator) BinaryUnpackPattern();
      pattern->inst = instToEmit;
      for (uint32_t i = 0; i < 2; ++i) {
        auto [isUnpack, isUnsigned, source, subreg] = sources[i];
        pattern->sources[i] = GetSource(source, false, false, IsSourceOfSample(&I));
        if (isUnpack) {
          pattern->sources[i].region_set = true;
          pattern->sources[i].elementOffset = subreg;
          pattern->sources[i].region[0] = 4;
          pattern->sources[i].region[1] = 1;
          pattern->sources[i].region[2] = 0;
          pattern->sources[i].type = isUnsigned ? ISA_TYPE_UB : ISA_TYPE_B;
        }
      }
      AddPattern(pattern);
      return true;
    }
  }
  return false;
}

// Match the pattern of 32 x 32 = 64, a full 32-bit multiplication.
bool CodeGenPatternMatch::MatchFullMul32(llvm::Instruction &I) {
  using namespace llvm::PatternMatch; // Scoped namespace using.

  struct FullMul32Pattern : public Pattern {
    SSource srcs[2];
    bool isUnsigned;
    virtual void Emit(EmitPass *pass, const DstModifier &dstMod) { pass->EmitFullMul32(isUnsigned, srcs, dstMod); }
  };

  IGC_ASSERT_MESSAGE(I.getOpcode() == llvm::Instruction::Mul, "Mul instruction is expected!");

  if (!I.getType()->isIntegerTy(64))
    return false;

  llvm::Value *LHS = I.getOperand(0);
  llvm::Value *RHS = I.getOperand(1);

  // Swap operand to ensure the constant is always RHS.
  if (isa<ConstantInt>(LHS))
    std::swap(LHS, RHS);

  bool IsUnsigned = false;
  llvm::Value *L;
  llvm::Value *R;

  // Check LHS
  if (match(LHS, m_SExt(m_Value(L)))) {
    // Bail out if there's non 32-bit integer.
    if (!L->getType()->isIntegerTy(32))
      return false;
  } else if (match(LHS, m_ZExt(m_Value(L)))) {
    // Bail out if there's non 32-bit integer.
    if (!L->getType()->isIntegerTy(32))
      return false;
    IsUnsigned = true;
  } else {
    // Bailout if it's unknown that LHS have less significant bits than the
    // product.
    // NOTE we don't assertion fail the case where LHS is an constant to prevent
    // the assertion in O0 mode. Otherwise, we expect there's at most 1
    // constant operand.
    return false;
  }

  // Check RHS
  if (match(RHS, m_SExt(m_Value(R)))) {
    // Bail out if there's signedness mismatch or non 32-bit integer.
    if (IsUnsigned || !R->getType()->isIntegerTy(32))
      return false;
  } else if (match(RHS, m_ZExt(m_Value(R)))) {
    // Bail out if there's signedness mismatch or non 32-bit integer.
    if (!IsUnsigned || !R->getType()->isIntegerTy(32))
      return false;
    IsUnsigned = true;
  } else if (ConstantInt *CI = dyn_cast<ConstantInt>(RHS)) {
    APInt Val = CI->getValue();
    // 31-bit unsigned integer could be used as either signed or
    // unsigned one. Otherwise, we need special check how MSB is used.
    if (!Val.isIntN(31)) {
      if (!(IsUnsigned && Val.isIntN(32)) && !(!IsUnsigned && Val.isSignedIntN(32))) {
        return false;
      }
    }
    R = ConstantExpr::getTrunc(CI, L->getType());
  } else {
    // Bailout if it's unknown that RHS have less significant bits than the
    // product.
    return false;
  }

  FullMul32Pattern *Pat = new (m_allocator) FullMul32Pattern();
  Pat->srcs[0] = GetSource(L, !IsUnsigned, false, IsSourceOfSample(&I));
  Pat->srcs[1] = GetSource(R, !IsUnsigned, false, IsSourceOfSample(&I));
  Pat->isUnsigned = IsUnsigned;
  AddPattern(Pat);

  return true;
}

// For 32 bit integer mul/add/sub, use 16bit operands if possible. Thus,
// This will match 16x16->32, 16x32->32, the same for add/sub.
//
// For example:
//   1.  before:
//        %9 = ashr i32 %8, 16
//        %10 = mul nsw i32 %9, -1024
//        ( asr (16|M0)  r19.0<1>:d  r19.0<8;8,1>:d  16:w
//          mul (16|M0)  r19.0<1>:d  r19.0<8;8,1>:d  -1024:w )
//
//      after:
//      --> %10:d = mul %9.1<16;8:2>:w -1024:w
//          (  mul (16|M0)  r23.0<1>:d   r19.1<2;1,0>:w   -1024:w )
//
//  2. before:
//        %9  = lshr i32 %8, 16
//        %10 = and i32 %8, 65535
//        %11 = mul nuw i32 %9, %10
//        ( shr  (16|M0)   r14.0<1>:d  r12.0<8;8,1>:ud   16:w
//          and(16 | M0)   r12.0<1>:d  r12.0<8;8,1>:d  65535:d
//          mul(16 | M0)   r14.0<1>:d  r14.0<8;8,1>:d  r12.0<8;8,1>:d )
//
//     after:
//     --> %11:d = mul %8.1<16;8,2>:uw %8.0<16;8,2>:uw
//         ( mul (16|M0)  r14.0<1>:d   r12.1<2;1,0>:w   r12.0<2;1,0>:w )
//
bool CodeGenPatternMatch::MatchMulAdd16(Instruction &I) {
  using namespace llvm::PatternMatch;

  struct Oprd16Pattern : public Pattern {
    SSource srcs[2];
    Instruction *rootInst;
    virtual void Emit(EmitPass *pass, const DstModifier &dstMod) { pass->emitMulAdd16(rootInst, srcs, dstMod); }
  };

  // The code is under the control of registry key EnableMixIntOperands.
  if (IGC_IS_FLAG_DISABLED(EnableMixIntOperands)) {
    return false;
  }

  unsigned opc = I.getOpcode();
  IGC_ASSERT_MESSAGE((opc == Instruction::Mul) || (opc == Instruction::Add) || (opc == Instruction::Sub),
                     "Mul instruction is expected!");

  // Handle 32 bit integer mul/add/sub only.
  if (!I.getType()->isIntegerTy(32)) {
    return false;
  }

  // Try to replace any source operands with ones of type short if any. As vISA
  // allows the mix of any integer type, each operand is considered separately.
  struct {
    Value *src;
    bool useLower;
    bool isSigned;
  } oprdInfo[2];
  bool isCandidate = false;

  for (int i = 0; i < 2; ++i) {
    Value *oprd = I.getOperand(i);
    Value *L = nullptr;

    // oprdInfo[i].src == null --> no W operand replacement.
    oprdInfo[i].src = nullptr;
    if (ConstantInt *CI = dyn_cast<ConstantInt>(oprd)) {
      int64_t val = CI->isNegative() ? CI->getSExtValue() : CI->getZExtValue();
      // If src needs to be negated (y = x - a = x + (-a), as gen only
      // has add), need to check if the negated src fits into W/UW.
      bool isNegSrc = (opc == Instruction::Sub && i == 1);
      if (isNegSrc) {
        val = -val;
      }
      if (INT16_MIN <= val && val <= INT16_MAX) {
        oprdInfo[i].src = oprd;
        oprdInfo[i].useLower = true; // does not matter for const
        oprdInfo[i].isSigned = true;
        isCandidate = true;
      } else if (0 <= val && val <= UINT16_MAX) {
        oprdInfo[i].src = oprd;
        oprdInfo[i].useLower = true; // does not matter for const
        oprdInfo[i].isSigned = false;
        isCandidate = true;
      }
    } else if (match(oprd, m_And(m_Value(L), m_SpecificInt(0xFFFF)))) {
      oprdInfo[i].src = L;
      oprdInfo[i].useLower = true;
      oprdInfo[i].isSigned = false;
      isCandidate = true;
    } else if (match(oprd, m_LShr(m_Value(L), m_SpecificInt(16)))) {
      oprdInfo[i].src = L;
      oprdInfo[i].useLower = false;
      oprdInfo[i].isSigned = false;
      isCandidate = true;
    } else if (match(oprd, m_AShr(m_Shl(m_Value(L), m_SpecificInt(16)), m_SpecificInt(16)))) {
      oprdInfo[i].src = L;
      oprdInfo[i].useLower = true;
      oprdInfo[i].isSigned = true;
      isCandidate = true;
    } else if (match(oprd, m_AShr(m_Value(L), m_SpecificInt(16)))) {
      oprdInfo[i].src = L;
      oprdInfo[i].useLower = false;
      oprdInfo[i].isSigned = true;
      isCandidate = true;
    }
  }

  if (!isCandidate) {
    return false;
  }

  Oprd16Pattern *Pat = new (m_allocator) Oprd16Pattern();
  for (int i = 0; i < 2; ++i) {
    if (oprdInfo[i].src) {
      Pat->srcs[i] = GetSource(oprdInfo[i].src, false, false, IsSourceOfSample(&I));
      SSource &thisSrc = Pat->srcs[i];

      // for now, Use W/UW only if region_set is false or the src is scalar
      if (thisSrc.region_set && !(thisSrc.region[0] == 0 && thisSrc.region[1] == 1 && thisSrc.region[2] == 0)) {
        Pat->srcs[i] = GetSource(I.getOperand(i), true, false, IsSourceOfSample(&I));
      } else {
        // Note that SSource's type, if set by GetSource(), should be 32bit type. It's
        // safe to override it with either UW or W. But for SSource's offset, need to
        // re-calculate in term of 16bit, not 32bit.
        thisSrc.type = oprdInfo[i].isSigned ? ISA_TYPE_W : ISA_TYPE_UW;
        thisSrc.elementOffset = (2 * thisSrc.elementOffset) + (oprdInfo[i].useLower ? 0 : 1);
      }
    } else {
      Pat->srcs[i] = GetSource(I.getOperand(i), true, false, IsSourceOfSample(&I));
    }
  }
  Pat->rootInst = &I;
  AddPattern(Pat);

  return true;
}

bool CodeGenPatternMatch::BitcastSearch(SSource &source, llvm::Value *&value, bool broadcast) {
  if (auto elemInst = dyn_cast<ExtractElementInst>(value)) {
    if (auto bTInst = dyn_cast<BitCastInst>(elemInst->getOperand(0))) {
      // Pattern Matching (Instruction) + ExtractElem + (Vector)Bitcast
      //
      // In order to set the regioning for the ALU operand
      // I require three things:
      //      -The first is the source number of elements
      //      -The second is the destination number of elements
      //      -The third is the index from the extract element
      //
      // For example if I have <4 x i32> to <16 x i8> all I need is
      // the 4 (vstride) and the i8 (b) in this case the operand would look
      // like this -> r22.x <4;1,0>:b
      // x is calculated below and later on using the simdsize

      uint32_t index, srcNElts, dstNElts, nEltsRatio;
      llvm::Type *srcTy = bTInst->getOperand(0)->getType();
      llvm::Type *dstTy = bTInst->getType();

      srcNElts = (srcTy->isVectorTy()) ? (uint32_t)cast<IGCLLVM::FixedVectorType>(srcTy)->getNumElements() : 1;
      dstNElts = (dstTy->isVectorTy()) ? (uint32_t)cast<IGCLLVM::FixedVectorType>(dstTy)->getNumElements() : 1;

      if (srcNElts < dstNElts && srcTy->getScalarSizeInBits() < 64) {
        if (isa<ConstantInt>(elemInst->getIndexOperand())) {
          index = int_cast<uint>(cast<ConstantInt>(elemInst->getIndexOperand())->getZExtValue());
          nEltsRatio = dstNElts / srcNElts;
          source.value = bTInst->getOperand(0);
          source.SIMDOffset = iSTD::RoundDownNonPow2(index, nEltsRatio);
          source.elementOffset = source.elementOffset * nEltsRatio + index % nEltsRatio;
          value = source.value;
          if (!broadcast) {
            source.region_set = true;
            if (m_WI->isUniform(value))
              source.region[0] = 0;
            else
              source.region[0] = (unsigned char)nEltsRatio;
            source.region[1] = 1;
            source.region[2] = 0;
          }
          return true;
        }
      }
    }
  }
  return false;
}

bool CodeGenPatternMatch::MatchModifier(llvm::Instruction &I, bool SupportSrc0Mod) {
  struct ModifierPattern : public Pattern {
    SSource sources[2];
    llvm::Instruction *instruction;
    virtual void Emit(EmitPass *pass, const DstModifier &modifier) {
      pass->BinaryUnary(instruction, sources, modifier);
    }
  };

  ModifierPattern *pattern = new (m_allocator) ModifierPattern();
  pattern->instruction = &I;
  uint nbSources = GetNbSources(I);

  llvm::SmallVector<llvm::Value *, 4> sources(I.op_begin(), I.op_end());
  if (FlushesDenormsOnInput(I)) {
    // Denorms are flushed at input - skip canonicalize
    std::transform(sources.begin(), sources.end(), sources.begin(),
                   [](llvm::Value *src) -> llvm::Value * { return SkipCanonicalize(src); });
  }
  bool supportModifierSrc0 = SupportsModifier(&I, m_Platform);
  bool supportRegioning = SupportsRegioning(&I);
  llvm::Instruction *src0Inst = llvm::dyn_cast<llvm::Instruction>(I.getOperand(0));
  if (I.getOpcode() == llvm::Instruction::UDiv && src0Inst && src0Inst->getOpcode() == llvm::Instruction::Sub) {
    supportModifierSrc0 = false;
  }
  pattern->sources[0] =
      GetSource(sources[0], supportModifierSrc0 && SupportSrc0Mod, supportRegioning, IsSourceOfSample(&I));
  if (nbSources > 1) {
    bool supportModifierSrc1 = SupportsModifier(&I, m_Platform);
    llvm::Instruction *src1Inst = llvm::dyn_cast<llvm::Instruction>(I.getOperand(1));
    if (I.getOpcode() == llvm::Instruction::UDiv && src1Inst && src1Inst->getOpcode() == llvm::Instruction::Sub) {
      supportModifierSrc1 = false;
    }
    pattern->sources[1] = GetSource(sources[1], supportModifierSrc1, supportRegioning, IsSourceOfSample(&I));

    // add df imm to constant pool for binary/ternary inst
    // we do 64-bit int imm bigger than 32 bits, since smaller may fit in D/W
    for (int i = 0, numSrc = (int)nbSources; i < numSrc; ++i) {
      Value *op = sources[i];
      if (isCandidateForConstantPool(op)) {
        AddToConstantPool(I.getParent(), op);
        pattern->sources[i].fromConstantPool = true;
      }
    }
  }

  AddPattern(pattern);

  return true;
}

bool CodeGenPatternMatch::MatchSingleInstruction(llvm::Instruction &I) {
  struct SingleInstPattern : Pattern {
    llvm::Instruction *inst;
    virtual void Emit(EmitPass *pass, const DstModifier &modifier) {
      IGC_ASSERT(modifier.sat == false);
      IGC_ASSERT(modifier.flag == nullptr);
      pass->EmitNoModifier(inst);
    }
  };
  SingleInstPattern *pattern = new (m_allocator) SingleInstPattern();
  pattern->inst = &I;
  bool flushesDenorms = FlushesDenormsOnInput(I);
  uint numSources = GetNbSources(I);
  for (uint i = 0; i < numSources; i++) {
    Value *src = I.getOperand(i);
    if (flushesDenorms) {
      // Denorms are flushed at input - skip canonicalize
      src = SkipCanonicalize(src);
    }
    MarkAsSource(src, IsSourceOfSample(&I));
  }

  if (CallInst *callinst = dyn_cast<CallInst>(&I)) {
    // Mark the function pointer in indirect calls as a source
    if (!callinst->getCalledFunction()) {
      MarkAsSource(IGCLLVM::getCalledValue(callinst), IsSourceOfSample(&I));
    }
  }
  AddPattern(pattern);
  return true;
}

bool CodeGenPatternMatch::MatchCanonicalizeInstruction(llvm::Instruction &I) {
  struct CanonicalizeInstPattern : Pattern {
    CanonicalizeInstPattern(llvm::Instruction *pInst) : m_pInst(pInst) {}

    llvm::Instruction *m_pInst;
    Pattern *m_pPattern = nullptr;

    virtual void Emit(EmitPass *pass, const DstModifier &modifier) {
      if (m_pPattern) {
        m_pPattern->Emit(pass, modifier);
      } else {
        pass->emitCanonicalize(m_pInst, modifier);
      }
    }
  };

  CanonicalizeInstPattern *pattern = new (m_allocator) CanonicalizeInstPattern(&I);
  IGC_ASSERT(isa<Instruction>(I.getOperand(0)) || isa<Argument>(I.getOperand(0)));
  // Current implementation assumes that mix mode is disabled if
  // half float or 32-bit float denorms must be flushed.
  if (m_ctx->getModuleMetaData()->compOpt.FloatDenormMode16 == IGC::FLOAT_DENORM_FLUSH_TO_ZERO ||
      m_ctx->getModuleMetaData()->compOpt.FloatDenormMode32 == IGC::FLOAT_DENORM_FLUSH_TO_ZERO) {
    IGC_ASSERT(!m_Platform.supportMixMode() || m_ctx->getModuleMetaData()->disableMixMode);
  }
  if (isa<Argument>(I.getOperand(0)) || !FlushesDenormsOnOutput(*(cast<Instruction>(I.getOperand(0))))) {
    MarkAsSource(I.getOperand(0), IsSourceOfSample(&I));
  } else {
    pattern->m_pPattern = Match(*llvm::cast<llvm::Instruction>(I.getOperand(0)));
  }

  AddPattern(pattern);
  return true;
}

bool CodeGenPatternMatch::MatchBranch(llvm::BranchInst &I) {
  using namespace llvm::PatternMatch;
  struct CondBrInstPattern : Pattern {
    SSource cond;
    llvm::BranchInst *inst;
    e_predMode predMode = EPRED_NORMAL;
    bool isDiscardBranch = false;
    virtual void Emit(EmitPass *pass, const DstModifier &modifier) {
      if (isDiscardBranch) {
        pass->emitDiscardBranch(inst, cond);
      } else {
        pass->emitBranch(inst, cond, predMode);
      }
    }
  };
  CondBrInstPattern *pattern = new (m_allocator) CondBrInstPattern();
  pattern->inst = &I;

  if (!I.isUnconditional()) {
    Value *orSrc0 = nullptr;
    Value *orSrc1 = nullptr;
    Value *cond = I.getCondition();
    if (dyn_cast<GenIntrinsicInst>(cond, GenISAIntrinsic::GenISA_UpdateDiscardMask)) {
      pattern->isDiscardBranch = true;
    } else if (match(cond, m_Or(m_Value(orSrc0), m_Value(orSrc1)))) {
      // %6 = call i1 @llvm.genx.GenISA.UpdateDiscardMask(i1 %0, i1 %2)
      // %7 = or i1 %6, %2  (or: %7 = or i1 %2, %6)
      // br i1 %7, label %DiscardRet, label %PostDiscard
      if (auto intr = dyn_cast<GenIntrinsicInst>(orSrc0, GenISAIntrinsic::GenISA_UpdateDiscardMask)) {
        if (intr->getOperand(1) == orSrc1) {
          pattern->isDiscardBranch = true;
        }
      } else if (auto intr = dyn_cast<GenIntrinsicInst>(orSrc1, GenISAIntrinsic::GenISA_UpdateDiscardMask)) {
        if (intr->getOperand(1) == orSrc0) {
          pattern->isDiscardBranch = true;
        }
      }
    }
    pattern->cond = GetSource(I.getCondition(), false, false, IsSourceOfSample(&I));
  }
  AddPattern(pattern);
  return true;
}

bool CodeGenPatternMatch::MatchFloatingPointSatModifier(llvm::Instruction &I) {
  struct SatPattern : Pattern {
    Pattern *pattern;
    SSource source;
    bool isUnsigned;
    virtual void Emit(EmitPass *pass, const DstModifier &modifier) {
      DstModifier mod = modifier;
      mod.sat = true;
      if (pattern) {
        pattern->Emit(pass, mod);
      } else {
        pass->Mov(source, mod);
      }
    }
  };
  [[maybe_unused]] bool match = false;
  llvm::Value *source = nullptr;
  bool isUnsigned = false;
  if (isSat(&I, source, isUnsigned)) {
    SatPattern *satPattern = new (m_allocator) SatPattern();
    if (llvm::Instruction *inst = llvm::dyn_cast<Instruction>(source)) {
      // As an heuristic we only match saturate if the instruction has one use
      // to avoid duplicating expensive instructions and increasing reg pressure
      // without improve code quality this may be refined in the future
      if (inst->hasOneUse() && SupportsSaturate(inst)) {
        bool isSourceOfSample = IsSourceOfSample(&I);
        if (isSourceOfSample) {
          m_sampleSource.insert(inst);
        }
        auto *pattern = Match(*inst);
        IGC_ASSERT_MESSAGE(pattern, "Failed to match pattern");
        // Even though the original `inst` may support saturate,
        // we need to know if the instruction(s) generated from
        // the pattern support it.
        if (pattern->supportsSaturate()) {
          satPattern->pattern = pattern;
          match = true;
        } else if (isSourceOfSample) {
          m_sampleSource.erase(inst);
        }
      }
    }
    if (!match) {
      satPattern->pattern = nullptr;
      satPattern->source = GetSource(source, true, false, IsSourceOfSample(&I));
      match = true;
    }
    if (isUniform(&I) && source->hasOneUse()) {
      gatherUniformBools(source);
    }
    AddPattern(satPattern);
  }
  return match;
}

bool CodeGenPatternMatch::MatchIntegerSatModifier(llvm::Instruction &I) {
  // a default pattern
  struct SatPattern : Pattern {
    Pattern *pattern;
    virtual void Emit(EmitPass *pass, const DstModifier &modifier) {
      DstModifier mod = modifier;
      mod.sat = true;
      pattern->Emit(pass, mod);
    }
  };

  // a special pattern is required because of the fact that the instruction works on unsigned values
  // whereas the default type is signed for arithmetic instructions
  struct UAddPattern : Pattern {
    BinaryOperator *inst;

    virtual void Emit(EmitPass *pass, const DstModifier &modifier) {
      DstModifier mod = modifier;
      mod.sat = true;
      pass->EmitUAdd(inst, mod);
    }
  };

  struct IntegerSatTruncPattern : public Pattern {
    SSource src;
    bool isSigned;
    virtual void Emit(EmitPass *pass, const DstModifier &dstMod) {
      pass->EmitIntegerTruncWithSat(isSigned, isSigned, src, dstMod);
    }
  };

  [[maybe_unused]] bool match = false;
  llvm::Value *source = nullptr;
  bool isUnsigned = false;
  if (isSat(&I, source, isUnsigned)) {
    IGC_ASSERT(llvm::isa<Instruction>(source));

    // As an heuristic we only match saturate if the instruction has one use
    // to avoid duplicating expensive instructions and increasing reg pressure
    // without improve code quality this may be refined in the future.
    if (llvm::Instruction *sourceInst = llvm::cast<llvm::Instruction>(source);
        sourceInst && sourceInst->hasOneUse() && SupportsSaturate(sourceInst)) {
      if (llvm::BinaryOperator *binaryOpInst = llvm::dyn_cast<llvm::BinaryOperator>(source);
          binaryOpInst && (binaryOpInst->getOpcode() == llvm::BinaryOperator::BinaryOps::Add) && isUnsigned) {
        match = true;

        uint numSources = GetNbSources(*sourceInst);
        for (uint i = 0; i < numSources; i++) {
          MarkAsSource(sourceInst->getOperand(i), IsSourceOfSample(&I));
        }

        UAddPattern *uAddPattern = new (m_allocator) UAddPattern();
        uAddPattern->inst = binaryOpInst;
        AddPattern(uAddPattern);
      } else if (binaryOpInst && (binaryOpInst->getOpcode() == llvm::BinaryOperator::BinaryOps::Add) && !isUnsigned) {
        match = true;
        SatPattern *satPattern = new (m_allocator) SatPattern();
        satPattern->pattern = Match(*sourceInst);
        AddPattern(satPattern);
      } else if (llvm::TruncInst *truncInst = llvm::dyn_cast<llvm::TruncInst>(source); truncInst) {
        match = true;
        IntegerSatTruncPattern *satPattern = new (m_allocator) IntegerSatTruncPattern();
        satPattern->isSigned = !isUnsigned;
        satPattern->src = GetSource(truncInst->getOperand(0), !isUnsigned, false, IsSourceOfSample(&I));
        AddPattern(satPattern);
      } else {
        IGC_ASSERT_MESSAGE(0, "An undefined pattern match");
      }
    }
  }
  return match;
}

bool CodeGenPatternMatch::MatchSelectModifier(llvm::SelectInst &I) {
  if (I.getType()->isAggregateType())
    return MatchSingleInstruction(I);

  struct SelectPattern : Pattern {
    SSource sources[3];
    e_predMode predMode;
    virtual void Emit(EmitPass *pass, const DstModifier &modifier) {
      DstModifier modf = modifier;
      modf.predMode = predMode;
      pass->Select(sources, modf);
    }
  };
  SelectPattern *pattern = new (m_allocator) SelectPattern();
  pattern->predMode = EPRED_NORMAL;

  pattern->sources[0] = GetSource(I.getCondition(), false, false, IsSourceOfSample(&I));
  pattern->sources[1] = GetSource(I.getTrueValue(), true, false, IsSourceOfSample(&I));
  pattern->sources[2] = GetSource(I.getFalseValue(), true, false, IsSourceOfSample(&I));

  // try to add to constant pool whatever possible.
  if (isCandidateForConstantPool(I.getTrueValue())) {
    AddToConstantPool(I.getParent(), I.getTrueValue());
    pattern->sources[1].fromConstantPool = true;
  }

  if (isCandidateForConstantPool(I.getFalseValue())) {
    AddToConstantPool(I.getParent(), I.getFalseValue());
    pattern->sources[2].fromConstantPool = true;
  }

  AddPattern(pattern);
  return true;
}

static bool IsPositiveFloat(Value *v, unsigned int depth = 0) {
  if (depth > 3) {
    // limit the depth of recursion to avoid compile time problem
    return false;
  }
  if (ConstantFP *cst = dyn_cast<ConstantFP>(v)) {
    if (!cst->getValueAPF().isNegative()) {
      return true;
    }
  } else if (Instruction *I = dyn_cast<Instruction>(v)) {
    switch (I->getOpcode()) {
    case Instruction::FMul:
    case Instruction::FAdd:
      return IsPositiveFloat(I->getOperand(0), depth + 1) && IsPositiveFloat(I->getOperand(1), depth + 1);
    case Instruction::Call:
      if (IntrinsicInst *intrinsicInst = dyn_cast<IntrinsicInst>(I)) {
        if (intrinsicInst->getIntrinsicID() == Intrinsic::fabs) {
          return true;
        }
      } else if (isa<GenIntrinsicInst>(I, GenISAIntrinsic::GenISA_fsat)) {
        return true;
      }
      break;
    default:
      break;
    }
  }
  return false;
}

bool CodeGenPatternMatch::MatchPow(llvm::IntrinsicInst &I) {
  if (IGC_IS_FLAG_ENABLED(DisableMatchPow)) {
    return false;
  }

  // For this pattern match exp(log(x) * y) = pow
  // if x < 0 and y is an integer (ex: 1.0)
  // with pattern match : pow(x, 1.0) = x
  // without pattern match : exp(log(x) * 1.0) = NaN because log(x) is NaN.
  //
  // Since pow is 2x slower than exp/log, disabling this optimization might not hurt much.
  // Keep the code and disable MatchPow to track any performance change for now.
  struct PowPattern : public Pattern {
    SSource sources[2];
    virtual void Emit(EmitPass *pass, const DstModifier &modifier) { pass->Pow(sources, modifier); }
  };
  bool found = false;
  llvm::Value *source0 = NULL;
  llvm::Value *source1 = NULL;
  if (I.getIntrinsicID() == Intrinsic::exp2) {
    llvm::BinaryOperator *mul = dyn_cast<BinaryOperator>(I.getOperand((0)));
    if (mul && mul->getOpcode() == Instruction::FMul) {
      for (uint j = 0; j < 2; j++) {
        llvm::IntrinsicInst *log = dyn_cast<IntrinsicInst>(mul->getOperand(j));
        if (log && log->getIntrinsicID() == Intrinsic::log2) {
          if (IsPositiveFloat(log->getOperand(0))) {
            source0 = log->getOperand(0);
            source1 = mul->getOperand(1 - j);
            found = true;
            break;
          }
        }
      }
    }
  }
  if (found) {
    PowPattern *pattern = new (m_allocator) PowPattern();
    pattern->sources[0] = GetSource(source0, true, false, IsSourceOfSample(&I));
    pattern->sources[1] = GetSource(source1, true, false, IsSourceOfSample(&I));
    AddPattern(pattern);
  }
  return found;
}

bool CodeGenPatternMatch::MatchCondModifier(llvm::CmpInst &I) {
  struct CondModifierPattern : Pattern {
    Pattern *pattern;
    Instruction *alu;
    CmpInst *cmp;
    int aluOprdNum = 0; // 0: alu is src0; otherwise, alu is src1
    virtual void Emit(EmitPass *pass, const DstModifier &modifier) {
      IGC_ASSERT(modifier.flag == nullptr);
      IGC_ASSERT(modifier.sat == false);
      pass->emitAluConditionMod(pattern, alu, cmp, aluOprdNum);
    }
  };
  bool found = false;
  for (uint i = 0; i < 2; i++) {
    if (IsZero(I.getOperand(i))) {
      llvm::Instruction *alu = dyn_cast<Instruction>(I.getOperand(1 - i));
      if (alu && alu->hasOneUse() && SupportsCondModifier(alu)) {
        CondModifierPattern *pattern = new (m_allocator) CondModifierPattern();
        pattern->pattern = Match(*alu);
        IGC_ASSERT_MESSAGE(pattern->pattern, "Failed to match pattern");
        pattern->alu = alu;
        pattern->cmp = &I;
        pattern->aluOprdNum = 1 - i;
        AddPattern(pattern);
        found = true;
        break;
      }
    }
  }
  return found;
}

// we match the following pattern
// %f = cmp %1 %2
// %o = or/and %f %g
bool CodeGenPatternMatch::MatchBoolOp(llvm::BinaryOperator &I) {
  struct BoolOpPattern : public Pattern {
    Pattern *cmpPattern;
    llvm::BinaryOperator *boolOp;
    SSource binarySource;
    virtual void Emit(EmitPass *pass, const DstModifier &modifier) {
      pass->CmpBoolOp(cmpPattern, boolOp, binarySource, modifier);
    }
  };

  IGC_ASSERT(I.getOpcode() == Instruction::Or || I.getOpcode() == Instruction::And);
  bool found = false;
  if (I.getType()->isIntegerTy(1)) {
    for (uint i = 0; i < 2; i++) {
      if (CmpInst *cmp = llvm::dyn_cast<CmpInst>(I.getOperand(i))) {
        // only beneficial if the other operand only have one use
        if (I.getOperand(1 - i)->hasOneUse()) {
          BoolOpPattern *pattern = new (m_allocator) BoolOpPattern();
          pattern->cmpPattern = Match(*cmp);
          pattern->boolOp = &I;
          pattern->binarySource = GetSource(I.getOperand(1 - i), false, false, IsSourceOfSample(&I));
          AddPattern(pattern);
          found = true;
          break;
        }
      }
    }
  }
  return found;
}

bool CodeGenPatternMatch::MatchFunnelShiftRotate(llvm::IntrinsicInst &I) {
  // Hanlde only funnel shift that can be turned into rotate.
  struct funnelShiftRotatePattern : public Pattern {
    SSource sources[2];
    llvm::IntrinsicInst *instruction;
    virtual void Emit(EmitPass *pass, const DstModifier &modifier) {
      bool isShl = instruction->getIntrinsicID() == Intrinsic::fshl;
      pass->Binary(isShl ? EOPCODE_ROL : EOPCODE_ROR, sources, modifier);
    }
  };

  if (!m_Platform.supportRotateInstruction() || I.getType()->isVectorTy()) {
    return false;
  }

  Value *A = I.getOperand(0);
  Value *B = I.getOperand(1);
  Value *Amt = I.getOperand(2);
  uint32_t typebits = I.getType()->getScalarSizeInBits();
  if (A != B || (typebits != 16 && typebits != 32 && typebits != 64)) {
    return false;
  }
  if (typebits == 64 && !m_Platform.supportQWRotateInstructions()) {
    return false;
  }

  // Found the pattern.
  funnelShiftRotatePattern *pattern = new (m_allocator) funnelShiftRotatePattern();
  pattern->instruction = &I;
  pattern->sources[0] = GetSource(A, true, false, IsSourceOfSample(&I));
  pattern->sources[1] = GetSource(Amt, true, false, IsSourceOfSample(&I));

  AddPattern(pattern);
  return true;
}

// Map LLVM bitwise opcode to IGC e_opcode. Returns false for non-bitwise ops.
static bool matchBitwiseOpcode(unsigned llvmOp, e_opcode &opCode) {
  switch (llvmOp) {
  case Instruction::And:
    opCode = EOPCODE_AND;
    return true;
  case Instruction::Or:
    opCode = EOPCODE_OR;
    return true;
  case Instruction::Xor:
    opCode = EOPCODE_XOR;
    return true;
  default:
    return false;
  }
}

// Match bitwise op (and/or/xor) on lo/hi halves extracted from i64 values:
//   %bc0 = bitcast i64 %a to <2 x i32>
//   %ext0 = extractelement <2 x i32> %bc0, i32 idx
//   %bc1 = bitcast i64 %b to <2 x i32>
//   %ext1 = extractelement <2 x i32> %bc1, i32 idx
//   %res = {and|or|xor} i32 %ext0, %ext1
// Also handles the case where only one operand is an extractelement.
// Emit the op with stride-2 regioning directly on i64 source aliases,
// eliminating intermediate extractelement mov instructions.
bool CodeGenPatternMatch::MatchBitwiseI64Halves(llvm::Instruction &I) {
  using namespace llvm::PatternMatch;

  // Skip stride-2 regioning optimization at O0 for better debuggability.
  if (m_ctx->getModuleMetaData()->compOpt.OptDisable)
    return false;

  // Stride-2 regioning and SIMD32 split logic assumes 64-byte GRFs.
  if (m_Platform.getGRFSize() != 64)
    return false;

  struct BitwiseI64HalvesPattern : public Pattern {
    SSource sources[2];
    uint32_t extractIdx;
    bool isI64Half[2]; // per-operand: true if from extract of bitcast i64
    e_opcode opCode;
    virtual void Emit(EmitPass *pass, const DstModifier &modifier) {
      pass->EmitBitwiseI64Halves(opCode, sources, extractIdx, isI64Half, modifier);
    }
  };

  // Try to match each operand as: extractelement(bitcast(i64 → <2 x i32>), const_idx)
  // Reject constant i64 sources — the emitter aliases via GetNewAlias which
  // does not accept immediates.
  bool isI64Half[2] = {false, false};
  uint32_t extractIdx[2] = {0, 0};
  Value *i64Srcs[2] = {nullptr, nullptr};

  for (unsigned i = 0; i < 2; ++i) {
    Value *src = nullptr;
    ConstantInt *idxCI = nullptr;
    if (!match(I.getOperand(i), m_ExtractElt(m_BitCast(m_Value(src)), m_ConstantInt(idxCI))))
      continue;
    if (!src->getType()->isIntegerTy(64) || isa<Constant>(src))
      continue;
    auto *extElt = cast<ExtractElementInst>(I.getOperand(i));
    auto *bc = cast<BitCastInst>(extElt->getVectorOperand());
    // All matched instructions must be in the same basic block.
    if (extElt->getParent() != I.getParent() || bc->getParent() != I.getParent())
      continue;
    auto *vecTy = dyn_cast<IGCLLVM::FixedVectorType>(bc->getType());
    if (!vecTy || vecTy->getNumElements() != 2 || !vecTy->getElementType()->isIntegerTy(32))
      continue;
    isI64Half[i] = true;
    extractIdx[i] = int_cast<uint32_t>(idxCI->getZExtValue());
    i64Srcs[i] = src;
  }

  // Need at least one operand from extractelement
  if (!isI64Half[0] && !isI64Half[1])
    return false;

  // If both are extractelement, indices must match
  if (isI64Half[0] && isI64Half[1] && extractIdx[0] != extractIdx[1])
    return false;

  uint32_t idx = isI64Half[0] ? extractIdx[0] : extractIdx[1];

  e_opcode opCode;
  if (!matchBitwiseOpcode(I.getOpcode(), opCode))
    return false;

  BitwiseI64HalvesPattern *pattern = new (m_allocator) BitwiseI64HalvesPattern();
  for (unsigned i = 0; i < 2; ++i) {
    pattern->isI64Half[i] = isI64Half[i];
    if (isI64Half[i]) {
      pattern->sources[i] = GetSource(i64Srcs[i], false, false, IsSourceOfSample(&I));
    } else {
      pattern->sources[i] = GetSource(I.getOperand(i), false, false, IsSourceOfSample(&I));
    }
  }
  pattern->extractIdx = idx;
  pattern->opCode = opCode;
  AddPattern(pattern);
  return true;
}

// Match insertelement→bitcast pack of two i32 values into i64, where both
// halves are produced by single-use bitwise ALU ops (and/or/xor).
// The ALU ops are folded into the pack, writing directly into the i64
// destination with stride-2 regioning and eliminating intermediate movs.
//
// Match the following sequence:
//   %lo = {and|or|xor} i32 %a, %b   ; single-use
//   %hi = {and|or|xor} i32 %c, %d   ; single-use
//   %vec0 = insertelement <2 x i32> undef, i32 %lo, i32 0
//   %vec1 = insertelement <2 x i32> %vec0, i32 %hi, i32 1
//   %dst  = bitcast <2 x i32> %vec1 to i64
// And generate:
//   {and|or|xor} (M1_NM, 1) dst_0(0,0)<2> a(0,0)<0;1,0> b(0,0)<0;1,0>
//   {and|or|xor} (M1_NM, 1) dst_0(0,1)<2> c(0,0)<0;1,0> d(0,0)<0;1,0>
//
// When an operand of a folded op is itself an extractelement from a
// bitcast i64→<2 x i32>, stride-2 source regioning is used to read
// the lo/hi half directly from the i64 source, avoiding an intermediate mov.
//

// Check if val is extractelement from bitcast i64 to <2 x i32>.
// On match, populate pattern fields at the given index and return true.
static bool tryMatchI64Half(CodeGenPatternMatch *matcher, Value *val, unsigned idx, bool *isI64Half,
                            uint32_t *extractIdx, SSource *sources, Instruction &I) {
  using namespace llvm::PatternMatch;
  Value *src = nullptr;
  ConstantInt *idxCI = nullptr;
  if (!match(val, m_ExtractElt(m_BitCast(m_Value(src)), m_ConstantInt(idxCI))))
    return false;
  if (!src->getType()->isIntegerTy(64) || isa<Constant>(src))
    return false;
  auto *extElt = cast<ExtractElementInst>(val);
  auto *bc = cast<BitCastInst>(extElt->getVectorOperand());
  // All matched instructions must be in the same basic block.
  if (extElt->getParent() != I.getParent() || bc->getParent() != I.getParent())
    return false;
  auto *vecTy = dyn_cast<IGCLLVM::FixedVectorType>(bc->getType());
  if (!vecTy || vecTy->getNumElements() != 2 || !vecTy->getElementType()->isIntegerTy(32))
    return false;
  isI64Half[idx] = true;
  extractIdx[idx] = int_cast<uint32_t>(idxCI->getZExtValue());
  sources[idx] = matcher->GetSource(src, false, false, matcher->IsSourceOfSample(&I));
  return true;
}

bool CodeGenPatternMatch::MatchPackI64(llvm::Instruction &I) {
  using namespace llvm::PatternMatch;

  // Skip stride-2 regioning optimization at O0 for better debuggability.
  if (m_ctx->getModuleMetaData()->compOpt.OptDisable)
    return false;

  // Stride-2 regioning and SIMD32 split logic assumes 64-byte GRFs.
  if (m_Platform.getGRFSize() != 64)
    return false;

  struct PackI64Pattern : public Pattern {
    SSource sources[4]; // [half*2+op]
    bool isI64Half[4] = {};
    uint32_t extractIdx[4] = {};
    e_opcode foldedOp[2];
    virtual void Emit(EmitPass *pass, const DstModifier &modifier) {
      pass->EmitPackI64(sources, isI64Half, extractIdx, foldedOp, modifier);
    }
  };

  // Check: bitcast <2 x i32> to i64
  if (!I.getType()->isIntegerTy(64))
    return false;
  auto *srcVecTy = dyn_cast<IGCLLVM::FixedVectorType>(I.getOperand(0)->getType());
  if (!srcVecTy || srcVecTy->getNumElements() != 2 || !srcVecTy->getElementType()->isIntegerTy(32))
    return false;

  // Match insertelement chain:
  //   insertelement(insertelement(undef, %innerVal, innerI), %outerVal, outerI)
  Value *outerVal = nullptr, *innerVal = nullptr;
  ConstantInt *outerIdx = nullptr, *innerIdx = nullptr;
  if (!match(I.getOperand(0),
             m_OneUse(m_InsertElt(m_OneUse(m_InsertElt(m_Undef(), m_Value(innerVal), m_ConstantInt(innerIdx))),
                                  m_Value(outerVal), m_ConstantInt(outerIdx)))))
    return false;

  // All matched instructions must be in the same basic block.
  auto *outerInsert = cast<InsertElementInst>(I.getOperand(0));
  auto *innerInsert = cast<InsertElementInst>(outerInsert->getOperand(0));
  BasicBlock *BB = I.getParent();
  if (outerInsert->getParent() != BB || innerInsert->getParent() != BB)
    return false;

  uint64_t outerI = outerIdx->getZExtValue();
  uint64_t innerI = innerIdx->getZExtValue();
  if (outerI > 1 || innerI > 1 || outerI == innerI)
    return false;

  // Identify lo (index 0) and hi (index 1) values.
  // Both must be single-use bitwise BinaryOperators.
  Value *vals[2];
  vals[innerI] = innerVal;
  vals[outerI] = outerVal;

  e_opcode foldedOp[2];
  BinaryOperator *binOps[2];
  for (unsigned half = 0; half < 2; ++half) {
    auto *binOp = dyn_cast<BinaryOperator>(vals[half]);
    if (!binOp || !binOp->hasOneUse())
      return false;
    if (binOp->getParent() != BB)
      return false;
    if (!matchBitwiseOpcode(binOp->getOpcode(), foldedOp[half]))
      return false;
    binOps[half] = binOp;
  }

  // Special case: both halves are XOR with all-ones (NOT).
  // Each half's NOT operand can independently be an extractelement from a
  // bitcast i64 → <2 x i32> (using stride-2 source regioning) or a plain
  // i32 value. NOT uses stride-2 destination regioning.
  {
    struct PackI64NotPattern : public Pattern {
      SSource sources[2];
      bool isI64Half[2] = {};
      uint32_t extractIdx[2] = {};
      virtual void Emit(EmitPass *pass, const DstModifier &modifier) {
        pass->EmitPackI64Not(sources, isI64Half, extractIdx, modifier);
      }
    };

    // Check if both halves are NOT: xor %v, -1
    Value *notSrcs[2] = {};
    if (match(binOps[0], m_Not(m_Value(notSrcs[0]))) && match(binOps[1], m_Not(m_Value(notSrcs[1])))) {
      auto *pattern = new (m_allocator) PackI64NotPattern();
      for (unsigned half = 0; half < 2; ++half) {
        if (!tryMatchI64Half(this, notSrcs[half], half, pattern->isI64Half, pattern->extractIdx, pattern->sources, I))
          pattern->sources[half] = GetSource(notSrcs[half], false, false, IsSourceOfSample(&I));
      }
      AddPattern(pattern);
      return true;
    }
  }

  PackI64Pattern *pattern = new (m_allocator) PackI64Pattern();
  pattern->foldedOp[0] = foldedOp[0];
  pattern->foldedOp[1] = foldedOp[1];

  for (unsigned half = 0; half < 2; ++half) {
    for (unsigned op = 0; op < 2; ++op) {
      unsigned idx = half * 2 + op;
      if (!tryMatchI64Half(this, binOps[half]->getOperand(op), idx, pattern->isI64Half, pattern->extractIdx,
                           pattern->sources, I))
        pattern->sources[idx] = GetSource(binOps[half]->getOperand(op), false, false, IsSourceOfSample(&I));
    }
  }

  AddPattern(pattern);
  return true;
}

// Match insertelement→bitcast pack of two i32 values into i64, where both
// halves are produced by a chain of two nested binary logic ops that can be
// folded into a single BFN instruction per half. Each BFN writes directly
// into the i64 destination with stride-2 regioning.
//
// Match the following sequence:
//   %t0 = {and|or|xor} i32 %a, %b   ; inner op, single-use
//   %lo = {and|or|xor} i32 %t0, %c  ; root op, single-use
//   %t1 = {and|or|xor} i32 %d, %e   ; inner op, single-use
//   %hi = {and|or|xor} i32 %t1, %f  ; root op, single-use
//   %vec0 = insertelement <2 x i32> undef, i32 %lo, i32 0
//   %vec1 = insertelement <2 x i32> %vec0, i32 %hi, i32 1
//   %dst  = bitcast <2 x i32> %vec1 to i64
// And generate:
//   bfn.xNN (M1, 32) dst_0v(0,0)<2> a<2;1,0> b<2;1,0> c<2;1,0>
//   bfn.xNN (M1, 32) dst_0v(0,1)<2> d<2;1,0> e<2;1,0> f<2;1,0>
//
bool CodeGenPatternMatch::MatchPackBfnI64(llvm::Instruction &I) {
  using namespace llvm::PatternMatch;

  // Skip stride-2 regioning optimization at O0 for better debuggability.
  if (m_ctx->getModuleMetaData()->compOpt.OptDisable)
    return false;

  // Stride-2 regioning and SIMD32 split logic assumes 64-byte GRFs.
  if (m_Platform.getGRFSize() != 64)
    return false;

  struct PackBfnI64Pattern : public Pattern {
    SSource sources[6]; // [half*3+srcIdx]
    bool isI64Half[6] = {};
    uint32_t extractIdx[6] = {};
    uint8_t booleanFuncCtrl[2];
    virtual void Emit(EmitPass *pass, const DstModifier &modifier) {
      pass->EmitPackBfnI64(sources, isI64Half, extractIdx, booleanFuncCtrl, modifier);
    }
  };

  // Check platform support for BFN
  if (IGC_IS_FLAG_DISABLED(EnableBfn) || !m_Platform.supportBfnInstruction())
    return false;

  // Check: bitcast <2 x i32> to i64
  if (!I.getType()->isIntegerTy(64))
    return false;
  auto *srcVecTy = dyn_cast<IGCLLVM::FixedVectorType>(I.getOperand(0)->getType());
  if (!srcVecTy || srcVecTy->getNumElements() != 2 || !srcVecTy->getElementType()->isIntegerTy(32))
    return false;

  // Match insertelement chain:
  //   insertelement(insertelement(undef, %innerVal, innerI), %outerVal, outerI)
  Value *outerVal = nullptr, *innerVal = nullptr;
  ConstantInt *outerIdx = nullptr, *innerIdx = nullptr;
  if (!match(I.getOperand(0),
             m_OneUse(m_InsertElt(m_OneUse(m_InsertElt(m_Undef(), m_Value(innerVal), m_ConstantInt(innerIdx))),
                                  m_Value(outerVal), m_ConstantInt(outerIdx)))))
    return false;

  // All matched instructions must be in the same basic block.
  auto *outerInsert = cast<InsertElementInst>(I.getOperand(0));
  auto *innerInsert = cast<InsertElementInst>(outerInsert->getOperand(0));
  BasicBlock *BB = I.getParent();
  if (outerInsert->getParent() != BB || innerInsert->getParent() != BB)
    return false;

  uint64_t outerI = outerIdx->getZExtValue();
  uint64_t innerI = innerIdx->getZExtValue();
  if (outerI > 1 || innerI > 1 || outerI == innerI)
    return false;

  // Identify lo (index 0) and hi (index 1) values
  Value *vals[2];
  vals[innerI] = innerVal;
  vals[outerI] = outerVal;

  auto isBinaryLogic = [](unsigned op) {
    return op == Instruction::Or || op == Instruction::And || op == Instruction::Xor;
  };

  auto applyBfnOp = [](uint8_t a, uint8_t b, unsigned opcode) -> uint8_t {
    switch (opcode) {
    case Instruction::And:
      return a & b;
    case Instruction::Or:
      return a | b;
    case Instruction::Xor:
      return a ^ b;
    default:
      return 0;
    }
  };

  const uint8_t bfnSrc[3] = {0xAA, 0xCC, 0xF0};
  const int useThreshold = 4;

  // For each half, try to match a BFN pattern (3 sources from 2 nested binary logic ops)
  Value *bfnSources[2][3];
  uint8_t bfnCtrl[2];

  for (unsigned half = 0; half < 2; ++half) {
    auto *rootOp = dyn_cast<BinaryOperator>(vals[half]);
    if (!rootOp || !rootOp->hasOneUse() || !isBinaryLogic(rootOp->getOpcode()))
      return false;
    if (rootOp->getParent() != BB)
      return false;

    bool foundBfn = false;
    for (unsigned side = 0; side < 2 && !foundBfn; ++side) {
      auto *innerOp = dyn_cast<BinaryOperator>(rootOp->getOperand(side));
      if (!innerOp || !isBinaryLogic(innerOp->getOpcode()) || innerOp->hasNUsesOrMore(useThreshold))
        continue;
      if (innerOp->getParent() != BB)
        continue;

      if (side == 0) {
        // Tree: (innerOp.lhs innerOpcode innerOp.rhs) rootOpcode rootOp.rhs
        bfnSources[half][0] = innerOp->getOperand(0);
        bfnSources[half][1] = innerOp->getOperand(1);
        bfnSources[half][2] = rootOp->getOperand(1);
        bfnCtrl[half] =
            applyBfnOp(applyBfnOp(bfnSrc[0], bfnSrc[1], innerOp->getOpcode()), bfnSrc[2], rootOp->getOpcode());
      } else {
        // Tree: rootOp.lhs rootOpcode (innerOp.lhs innerOpcode innerOp.rhs)
        bfnSources[half][0] = rootOp->getOperand(0);
        bfnSources[half][1] = innerOp->getOperand(0);
        bfnSources[half][2] = innerOp->getOperand(1);
        bfnCtrl[half] =
            applyBfnOp(bfnSrc[0], applyBfnOp(bfnSrc[1], bfnSrc[2], innerOp->getOpcode()), rootOp->getOpcode());
      }
      foundBfn = true;
    }
    if (!foundBfn)
      return false;
  }

  PackBfnI64Pattern *pattern = new (m_allocator) PackBfnI64Pattern();
  pattern->booleanFuncCtrl[0] = bfnCtrl[0];
  pattern->booleanFuncCtrl[1] = bfnCtrl[1];

  for (unsigned half = 0; half < 2; ++half) {
    for (unsigned s = 0; s < 3; ++s) {
      unsigned idx = half * 3 + s;
      if (!tryMatchI64Half(this, bfnSources[half][s], idx, pattern->isI64Half, pattern->extractIdx, pattern->sources,
                           I)) {
        pattern->sources[idx] = GetSource(bfnSources[half][s], false, false, IsSourceOfSample(&I));
        // BFN can use imm16 in src0 and src2; src1 must always go through
        // the constant pool. Mirror the legalization from MatchBfn.
        if (auto *CI = dyn_cast<ConstantInt>(bfnSources[half][s])) {
          if (s == 1 || !CI->getType()->isIntegerTy(16)) {
            AddToConstantPool(I.getParent(), bfnSources[half][s]);
            pattern->sources[idx].fromConstantPool = true;
          }
        }
      }
    }
  }

  AddPattern(pattern);
  return true;
}

bool CodeGenPatternMatch::MatchUnmaskedRegionBoundary(Instruction &I, bool start) {
  struct UnmaskedBoundaryPattern : public Pattern {
    bool start;
    virtual void Emit(EmitPass *pass, const DstModifier &modifier) {
      (void)modifier;
      pass->emitUnmaskedRegionBoundary(start);
    }
  };

  UnmaskedBoundaryPattern *pattern = new (m_allocator) UnmaskedBoundaryPattern();
  pattern->start = start;
  AddPattern(pattern);
  return true;
}

bool CodeGenPatternMatch::MatchAdd3(Instruction &I) {
  using namespace llvm::PatternMatch;

  struct Add3Pattern : public Pattern {
    SSource sources[3];
    Instruction *instruction;
    virtual void Emit(EmitPass *pass, const DstModifier &modifier) { pass->Tenary(EOPCODE_ADD3, sources, modifier); }
  };

  if (IGC_IS_FLAG_DISABLED(EnableAdd3) || !m_Platform.supportAdd3Instruction()) {
    return false;
  }

  // Only handle D & W integer types.
  Type *Ty = I.getType();
  if (!(Ty->isIntegerTy(16) || Ty->isIntegerTy(32))) {
    return false;
  }

  Value *s0 = I.getOperand(0);
  Value *s1 = nullptr, *s2 = nullptr;
  e_modifier Mod1 = EMOD_NONE, Mod2 = EMOD_NONE;
  Instruction *I0 = dyn_cast<Instruction>(s0);
  Instruction *Add2 = nullptr;
  if (I0) {
    if (I0->getOpcode() == Instruction::Sub) {
      s0 = I0->getOperand(0);
      s1 = I0->getOperand(1);
      Mod1 = EMOD_NEG;
      Add2 = I0;
    } else if (I0->getOpcode() == Instruction::Add) {
      s0 = I0->getOperand(0);
      s1 = I0->getOperand(1);
      Add2 = I0;
    }
  }

  bool isNeg = (I.getOpcode() == Instruction::Sub);
  if (s1 == nullptr) {
    s1 = I.getOperand(1);
    Instruction *I1 = dyn_cast<Instruction>(s1);
    if (I1) {
      if (I1->getOpcode() == Instruction::Sub) {
        s1 = I1->getOperand(0);
        s2 = I1->getOperand(1);
        Add2 = I1;
        if (isNeg) {
          Mod1 = EMOD_NEG;
        } else {
          Mod2 = EMOD_NEG;
        }
      } else if (I1->getOpcode() == Instruction::Add) {
        s1 = I1->getOperand(0);
        s2 = I1->getOperand(1);
        Add2 = I1;
        if (isNeg) {
          Mod1 = EMOD_NEG;
          Mod2 = EMOD_NEG;
        }
      }
    }
  } else {
    s2 = I.getOperand(1);
    if (isNeg) {
      Mod2 = EMOD_NEG;
    }
  }

  if (s2 == nullptr) {
    return false;
  }

  // If source operand corresponding to first add instruction
  // has many uses the add3 pattern match is unlikely to be profitable,
  // as it increases register pressure and makes register bank
  // conflicts more likely. Unless some of the operands are constants
  // then there is no increase of register pressure from add3 construction
  if (m_ctx->type == ShaderType::OPENCL_SHADER) {
    const int Threshold = 3;

    if (!isa<ConstantInt>(s0) && !isa<ConstantInt>(s1) && !isa<ConstantInt>(s2) &&
        (Add2->getOperand(0)->hasOneUse() || Add2->getOperand(1)->hasOneUse()) && Add2->hasNUsesOrMore(Threshold)) {
      // add3 is unlikely to be profitable.
      return false;
    }
  }

  // Found the pattern.
  // Make sure that the middle one is not constant
  if (isa<ConstantInt>(s1)) {
    std::swap(s1, s2);
    std::swap(Mod1, Mod2);
  }

  Add3Pattern *pattern = new (m_allocator) Add3Pattern();
  pattern->instruction = &I;
  pattern->sources[0] = GetSource(s0, true, false, IsSourceOfSample(&I));
  pattern->sources[1] = GetSource(s1, true, false, IsSourceOfSample(&I));
  pattern->sources[2] = GetSource(s2, true, false, IsSourceOfSample(&I));
  if (Mod1 != EMOD_NONE) {
    pattern->sources[1].mod = CombineModifier(Mod1, pattern->sources[1].mod);
  }
  if (Mod2 != EMOD_NONE) {
    pattern->sources[2].mod = CombineModifier(Mod2, pattern->sources[2].mod);
  }
  AddPattern(pattern);
  return true;
}

bool CodeGenPatternMatch::MatchBfn(Instruction &I) {
  using namespace llvm::PatternMatch;

  struct BfnPattern : public Pattern {
    uint8_t booleanFuncCtrl = 0;
    SSource sources[3];
    Instruction *instruction;
    virtual void Emit(EmitPass *pass, const DstModifier &modifier) { pass->Bfn(booleanFuncCtrl, sources, modifier); }
  };

  if (IGC_IS_FLAG_DISABLED(EnableBfn) || !m_Platform.supportBfnInstruction()) {
    return false;
  }

  // Only handle D & W integer types.
  Type *Ty = I.getType();
  if (!(Ty->isIntegerTy(16) || Ty->isIntegerTy(32))) {
    return false;
  }

  struct CtrlCalculator {
    enum OP { NONE, AND, OR, XOR };
    enum SOURCE {
      S0 = 0, // s0, s1, s2
      S1 = 1,
      S2 = 2,
      NS0 = 3, // ~s0, ~s1, ~s2
      NS1 = 4,
      NS2 = 5
    };
    // Value of the sources for calculating BooleanFuncCtrl. The
    // index maps to enum SOURCE
    const uint8_t s[6] = {0xAA, 0xCC, 0xF0, 0x55, 0x33, 0x0F};

    typedef llvm::SmallVector<OP, 2> OpVecType;
    typedef llvm::SmallVector<SOURCE, 3> SourceVecType;

    // The sequence of matched operators, follow the calculation order
    OpVecType ops;
    // The sequence of matched sources, follow the calculation order
    SourceVecType source_idx;

    void addSource(SOURCE src) { source_idx.push_back(src); }
    void addOPFromLLVMOp(unsigned llvmOp) {
      if (llvmOp == Instruction::And)
        ops.push_back(AND);
      else if (llvmOp == Instruction::Or)
        ops.push_back(OR);
      else if (llvmOp == Instruction::Xor)
        ops.push_back(XOR);
      else {
        IGC_ASSERT_MESSAGE(0, "MatchBfn: inccorect opcode");
        return;
      }
    }

    uint8_t getBooleanFuncCtrl() {
      IGC_ASSERT_MESSAGE((source_idx.size() == (ops.size() + 1)), "MatchBfn: OPs and Sources length missmatched");
      SourceVecType::iterator s_it = source_idx.begin(), s_end = source_idx.end();
      // start from the first source
      uint8_t result = s[*s_it];
      ++s_it;

      // iterate through the matched sequence and compute the BooleanFuncCtrl
      OpVecType::iterator op_it = ops.begin();
      for (; s_it != s_end; ++s_it, ++op_it) {
        switch (*op_it) {
        case AND:
          result = result & s[*s_it];
          break;
        case OR:
          result = result | s[*s_it];
          break;
        case XOR:
          result = result ^ s[*s_it];
          break;
        default:
          IGC_ASSERT_MESSAGE(0, "MatchBfn:CtrlCalculator: inccorect OP");
          break;
        }
      }
      return result;
    }
  };

  auto isBinaryLogic = [](Instruction::BinaryOps op) {
    return op == Instruction::Or || op == Instruction::And || op == Instruction::Xor;
  };
  // Find BFN patterns. Matched patterns: (op0 and op1 are boolean operations)
  // ~s0  ~s1                  ~s1  ~s2   <-- extra check to matchNot on each found src
  //  |    |                    |    |
  // s0   s1  ~s2         ~s0   s1  s2
  //   \ /    |            |     \  /
  //    op    s2     OR    s0   op1
  //     \   /              \   /
  //      op0                op0   <-- match to BFN

  // if source operand has many uses the bfn pattern match is unlikely to be profitable,
  // as it increases register pressure and makes register bank conflicts more likely
  // ToDo: tune the value of N
  const int useThreshold = 4;
  // if both operands of the root is binary logic op, use simple heuristics
  // to fold one of them
  if (isa<BinaryOperator>(I.getOperand(0)) && isa<BinaryOperator>(I.getOperand(1))) {
    auto I0 = cast<BinaryOperator>(I.getOperand(0));
    auto I1 = cast<BinaryOperator>(I.getOperand(1));
    if (I0->hasNUsesOrMore(useThreshold) && I1->hasNUsesOrMore(useThreshold)) {
      // bfn is unlikely to be profitable.
      return false;
    } else if (I0->getNumUses() > I1->getNumUses()) {
      I.setOperand(0, I1);
      I.setOperand(1, I0);
    }
  }

  // Further match s to not operation. Update s if matched.
  auto matchNot = [](Value *&s) {
    BinaryOperator *bo = dyn_cast<BinaryOperator>(s);
    if (bo && match(s, m_Not(m_Specific(bo->getOperand(0))))) {
      s = bo->getOperand(0);
      return true;
    }
    return false;
  };

  CtrlCalculator ctrlcal;
  Value *s0 = I.getOperand(0);
  Value *s1 = nullptr, *s2 = nullptr;
  BinaryOperator *I0 = dyn_cast<BinaryOperator>(s0);
  if (I0) {
    if (isBinaryLogic(I0->getOpcode()) && !I0->hasNUsesOrMore(useThreshold)) {
      s0 = I0->getOperand(0);
      s1 = I0->getOperand(1);
    }
  }

  if (s1 == nullptr) {
    s1 = I.getOperand(1);
    BinaryOperator *I1 = dyn_cast<BinaryOperator>(s1);
    if (I1) {
      if (isBinaryLogic(I1->getOpcode()) && !I1->hasNUsesOrMore(useThreshold)) {
        s1 = I1->getOperand(0);
        s2 = I1->getOperand(1);

        // add ops and sources by execution order
        ctrlcal.addSource(matchNot(s1) ? CtrlCalculator::NS1 : CtrlCalculator::S1);
        ctrlcal.addOPFromLLVMOp(I1->getOpcode());
        ctrlcal.addSource(matchNot(s2) ? CtrlCalculator::NS2 : CtrlCalculator::S2);
        ctrlcal.addOPFromLLVMOp(I.getOpcode());
        ctrlcal.addSource(matchNot(s0) ? CtrlCalculator::NS0 : CtrlCalculator::S0);
      }
    }
  } else {
    s2 = I.getOperand(1);

    // add ops and sources by execution order
    ctrlcal.addSource(matchNot(s0) ? CtrlCalculator::NS0 : CtrlCalculator::S0);
    ctrlcal.addOPFromLLVMOp(I0->getOpcode());
    ctrlcal.addSource(matchNot(s1) ? CtrlCalculator::NS1 : CtrlCalculator::S1);
    ctrlcal.addOPFromLLVMOp(I.getOpcode());
    ctrlcal.addSource(matchNot(s2) ? CtrlCalculator::NS2 : CtrlCalculator::S2);
  }

  if (s2 == nullptr)
    return false;

  BfnPattern *pattern = new (m_allocator) BfnPattern();
  pattern->booleanFuncCtrl = ctrlcal.getBooleanFuncCtrl();
  pattern->instruction = &I;
  pattern->sources[0] = GetSource(s0, false, false, IsSourceOfSample(&I));
  pattern->sources[1] = GetSource(s1, false, false, IsSourceOfSample(&I));
  pattern->sources[2] = GetSource(s2, false, false, IsSourceOfSample(&I));

  // BFN can use imm16 in src0 and src2, check for those;
  // otherwise try to add to constant pool even int32.
  if (dyn_cast<ConstantInt>(s0) && !s0->getType()->isIntegerTy(16)) {
    AddToConstantPool(I.getParent(), s0);
    pattern->sources[0].fromConstantPool = true;
  }
  if (dyn_cast<ConstantInt>(s1)) {
    AddToConstantPool(I.getParent(), s1);
    pattern->sources[1].fromConstantPool = true;
  }
  if (dyn_cast<ConstantInt>(s2) && !s2->getType()->isIntegerTy(16)) {
    AddToConstantPool(I.getParent(), s2);
    pattern->sources[2].fromConstantPool = true;
  }

  AddPattern(pattern);
  return true;
}

// Match this pattern
// %1 = cmp %2 %3
// %6 = select %1 $4 %5
// to
// %1 = cmp %2 %3
// %6  = bfn %1 %4 %5
//
// For the original cmp-sel sequence, a flag-based sequence is generated.
// We instead want to generate a non-flag cmp-bfn sequence which has shorter latency.
bool CodeGenPatternMatch::MatchCmpSelect(llvm::SelectInst &I) {
  struct CmpSelectPattern : public Pattern {
    uint8_t execSize;
    llvm::CmpInst::Predicate predicate;
    SSource cmpSources[2];
    uint8_t booleanFuncCtrl = 0xD8; // represents s0&s1|~s0&s2
    SSource bfnSources[3];
    virtual void Emit(EmitPass *pass, const DstModifier &modifier) {
      pass->CmpBfn(predicate, cmpSources, booleanFuncCtrl, bfnSources, modifier);
    }
  };

  if (IGC_IS_FLAG_DISABLED(EnableBfn) || !m_Platform.supportBfnInstruction()) {
    return false;
  }

  if (llvm::CmpInst *cmp = llvm::dyn_cast<llvm::CmpInst>(I.getOperand(0))) {
    // handle one use for now
    if (!cmp->hasOneUse()) {
      return false;
    }

    // BFN only supports D & W types.
    Type *selTy = I.getType();
    if (!(selTy->isIntegerTy(16) || selTy->isIntegerTy(32))) {
      return false;
    }

    Type *cmpS0Ty = cmp->getOperand(0)->getType();
    if (selTy->getPrimitiveSizeInBits() != cmpS0Ty->getPrimitiveSizeInBits()) {
      return false;
    }

    llvm::Value *selSources[2] = {};
    e_modifier selMod[2] = {};
    selSources[0] = I.getOperand(1);
    selSources[1] = I.getOperand(2);

    // BFN only supports 16bit immediate
    if ((isa<Constant>(selSources[0]) && selSources[0]->getType()->isIntegerTy(32)) ||
        (isa<Constant>(selSources[1]) && selSources[1]->getType()->isIntegerTy(32))) {
      return false;
    }

    // As BFN doesn't support src modifier, it is not worth to generate the cmp-bfn
    // sequence if one of its sources will need an extra move.
    if (GetModifier(*selSources[0], selMod[0], selSources[0]) ||
        GetModifier(*selSources[1], selMod[1], selSources[1])) {
      return false;
    }

    CmpSelectPattern *pattern = new (m_allocator) CmpSelectPattern();
    pattern->predicate = cmp->getPredicate();
    bool supportsModifier = SupportsModifier(cmp, m_Platform);
    pattern->cmpSources[0] = GetSource(cmp->getOperand(0), supportsModifier, false, IsSourceOfSample(&I));
    pattern->cmpSources[1] = GetSource(cmp->getOperand(1), supportsModifier, false, IsSourceOfSample(&I));

    pattern->bfnSources[1] = GetSource(selSources[0], false, false, IsSourceOfSample(&I));
    pattern->bfnSources[2] = GetSource(selSources[1], false, false, IsSourceOfSample(&I));
    AddPattern(pattern);

    return true;
  }
  return false;
}

bool CodeGenPatternMatch::MatchDpas(GenIntrinsicInst &I) {
  struct DpasPattern : public Pattern {
    SSource source[3];
    GenIntrinsicInst *instruction;
    virtual void Emit(EmitPass *pass, const DstModifier &modifier) {
      pass->emitDpas(instruction, source, modifier);
      // pass->BinaryUnary(instruction, source, modifier);
    }
  };

  GenISAIntrinsic::ID dpasID = I.getIntrinsicID();
  IGC_ASSERT_MESSAGE((dpasID == GenISAIntrinsic::GenISA_dpas || dpasID == GenISAIntrinsic::GenISA_sub_group_dpas),
                     "Unexpected DPAS intrinsic!");

  Value *src0 = I.getOperand(0); // input
  Value *src1 = I.getOperand(1); // activation. operand(3) is its precision
  Value *src2 = I.getOperand(2); // weight. operand(4) is its precision
  // ConstantInt* pa = dyn_cast<ConstantInt>(I.getOperand(3));
  // ConstantInt* pb = dyn_cast<ConstantInt>(I.getOperand(4));
  ConstantInt *sdepth = cast<ConstantInt>(I.getOperand(5));
  ConstantInt *rcount = cast<ConstantInt>(I.getOperand(6));
  int SD = (int)sdepth->getZExtValue();
  int RC = (int)rcount->getZExtValue();

  if (dpasID == GenISAIntrinsic::GenISA_dpas && RC == 1 && SD == 8) {
    // Special-handling of activation (src1 - uniform). The case handled is:
    //
    // %s0 = call i32 @llvm.genx.GenISA.WaveShuffleIndex.i32(i32 %5, i32 0)
    // %s1 = call i32 @llvm.genx.GenISA.WaveShuffleIndex.i32(i32 %5, i32 1)
    // %s2 = call i32 @llvm.genx.GenISA.WaveShuffleIndex.i32(i32 %5, i32 2)
    // %s3 = call i32 @llvm.genx.GenISA.WaveShuffleIndex.i32(i32 %5, i32 3)
    // %s4 = call i32 @llvm.genx.GenISA.WaveShuffleIndex.i32(i32 %5, i32 4)
    // %s5 = call i32 @llvm.genx.GenISA.WaveShuffleIndex.i32(i32 %5, i32 5)
    // %s6 = call i32 @llvm.genx.GenISA.WaveShuffleIndex.i32(i32 %5, i32 6)
    // %s7 = call i32 @llvm.genx.GenISA.WaveShuffleIndex.i32(i32 %5, i32 7)
    // %a0 = insertelement <8 x i32> undef, i32 %s0, i32 0
    // %a1 = insertelement <8 x i32> %a0,   i32 %s1, i32 1
    // %a2 = insertelement <8 x i32> %a1,   i32 %s2, i32 2
    // %a3 = insertelement <8 x i32> %a2,   i32 %s3, i32 3
    // %a4 = insertelement <8 x i32> %a3,   i32 %s4, i32 4
    // %a5 = insertelement <8 x i32> %a4,   i32 %s5, i32 5
    // %a6 = insertelement <8 x i32> %a5,   i32 %s6, i32 6
    // %a7 = insertelement <8 x i32> %a6,   i32 %s7, i32 7
    //
    // %c0 = call i32 @llvm.genx.GenISA.dpas.8(i32 %9, <8 x i32> %a7, i32 7, <8 x i32> %14, i32 7)
    //
    // %a7 will be replaced with %5 in vISA.
    //

    InsertElementInst *rootIEI = dyn_cast<InsertElementInst>(src1);
    if (rootIEI) {
      // Currently, the src1 can be at most int8.
      Value *srcVec[8];
      Value *WSVal = nullptr;
      for (int i = 0; i < 8; srcVec[i] = nullptr, ++i)
        ;
      InsertElementInst *IEI = rootIEI;
      while (IEI) {
        Value *elem = IEI->getOperand(1);
        ConstantInt *CI = dyn_cast<ConstantInt>(IEI->getOperand(2));
        if (!CI) {
          // Only handle constant index
          WSVal = nullptr;
          break;
        }
        uint32_t ix = (uint32_t)CI->getZExtValue();
        if (ix > 7 || srcVec[ix]) {
          // Assume each element is inserted once.
          WSVal = nullptr;
          break;
        }

        // Check if the inserted element is created
        // by WaveShuffleIndex intrinsic
        GenIntrinsicInst *WSI = dyn_cast<GenIntrinsicInst>(elem);
        if (!WSI || (WSI->getIntrinsicID() != GenISAIntrinsic::GenISA_WaveShuffleIndex &&
                     WSI->getIntrinsicID() != GenISAIntrinsic::GenISA_WaveBroadcast)) {
          WSVal = nullptr;
          break;
        }
        Value *shuffleVal = WSI->getOperand(0);
        Value *ixVal = WSI->getOperand(1);
        if (WSVal == nullptr) {
          WSVal = shuffleVal;
        } else if (WSVal != shuffleVal) {
          WSVal = nullptr;
          break;
        }
        if (ConstantInt *CIX = dyn_cast<ConstantInt>(ixVal)) {
          if ((uint32_t)CIX->getZExtValue() != ix) {
            WSVal = nullptr;
            break;
          }
        }
        srcVec[ix] = elem;
        IEI = dyn_cast<InsertElementInst>(IEI->getOperand(0));
      }

      if (WSVal) {
        for (int i = 0; i < SD; ++i) {
          if (srcVec[i] == nullptr) {
            WSVal = nullptr;
            break;
          }
        }
      }

      // If WSVal is set at this point, it is one that will
      // replace src1.
      if (WSVal) {
        src1 = WSVal;
      }
    }
  }

  DpasPattern *pattern = new (m_allocator) DpasPattern();
  pattern->instruction = &I;
  pattern->source[0] = GetSource(src0, false, false, IsSourceOfSample(&I));
  pattern->source[1] = GetSource(src1, false, false, IsSourceOfSample(&I));
  pattern->source[2] = GetSource(src2, false, false, IsSourceOfSample(&I));
  AddPattern(pattern);

  return true;
}

bool CodeGenPatternMatch::MatchDp4a(GenIntrinsicInst &I) {
  struct MatchDp4a : public Pattern {
    SSource source[3];
    GenIntrinsicInst *instruction;
    virtual void Emit(EmitPass *pass, const DstModifier &modifier) { pass->emitDP4A(instruction, source, modifier); }
  };

  // Attempt to find a pattern like this:
  // %scalar52.6.2474 = extractelement <4 x i8> %145, i32 0
  // %scalar53.6.2475 = extractelement <4 x i8> % 145, i32 1
  // %scalar54.6.2476 = extractelement <4 x i8> % 145, i32 2
  // %scalar55.6.2477 = extractelement <4 x i8> % 145, i32 3
  // %simdShuffle.6.2480 = call i8 @llvm.genx.GenISA.WaveShuffleIndex.i8(i8 % scalar52.6.2474, i32 0, i32 0)
  // %simdShuffle33.6.2481 = call i8 @llvm.genx.GenISA.WaveShuffleIndex.i8(i8 % scalar53.6.2475, i32 0, i32 0)
  // %simdShuffle34.6.2482 = call i8 @llvm.genx.GenISA.WaveShuffleIndex.i8(i8 % scalar54.6.2476, i32 0, i32 0)
  // %simdShuffle35.6.2483 = call i8 @llvm.genx.GenISA.WaveShuffleIndex.i8(i8 % scalar55.6.2477, i32 0, i32 0)
  // %assembled.vect.6.2484 = insertelement <4 x i8> undef, i8 % simdShuffle.6.2480, i32 0
  // %assembled.vect56.6.2485 = insertelement <4 x i8> % assembled.vect.6.2484, i8 % simdShuffle33.6.2481, i32 1
  // %assembled.vect57.6.2486 = insertelement <4 x i8> % assembled.vect56.6.2485, i8 % simdShuffle34.6.2482, i32 2
  // %assembled.vect58.6.2487 = insertelement <4 x i8> % assembled.vect57.6.2486, i8 % simdShuffle35.6.2483, i32 3
  // %astype.i45.6.2489 = bitcast <4 x i8> % assembled.vect58.6.2487 to i32
  // %call.i4738.6.2490 = call i32 @llvm.genx.GenISA.dp4a.uu(i32 % call.i4738.6.3.1, i32 % astype.i45.6.2489, i32 % 135)
  // %call.i1239.6.2492 = call i32 @llvm.genx.GenISA.dp4a.uu(i32 % call.i1239.6.3.1, i32 % astype.i45.6.2489, i32
  // 16843009)
  //
  // If the pattern is found, we can avoid creating extra movs by changing the source of the dp4a
  // to use the input from the bitcast instead of using the result of the bitcast.
  // This pattern only happens if the WaveShuffleIndex uses 0,0 for the arguments.
  if (llvm::BitCastInst *bcInst = llvm::dyn_cast<BitCastInst>(I.getOperand(1))) {
    llvm::CallInst *waveInst[4] = {nullptr};
    llvm::InsertElementInst *insertInst[4] = {nullptr};

    // Find the 4 x insertelement
    insertInst[3] = llvm::dyn_cast<InsertElementInst>(bcInst->getOperand(0));
    if (insertInst[3]) {
      for (int i = 2; i >= 0; i--) {
        insertInst[i] = llvm::dyn_cast<InsertElementInst>(insertInst[i + 1]->getOperand(0));
        if (!insertInst[i])
          break;
      }
    }

    // Find the 4 x WaveShuffleIndex
    for (int i = 3; i >= 0; i--) {
      if (!insertInst[i])
        break;
      CallInst *temp = llvm::dyn_cast<CallInst>(insertInst[i]->getOperand(1));
      if (!temp)
        break;

      llvm::GenIntrinsicInst *intrin = llvm::dyn_cast<llvm::GenIntrinsicInst>(temp);
      if (!intrin || (intrin->getIntrinsicID() != GenISAIntrinsic::GenISA_WaveShuffleIndex &&
                      intrin->getIntrinsicID() != GenISAIntrinsic::GenISA_WaveBroadcast))
        break;
      waveInst[i] = temp;
    }

    // Check to see if the WaveShuffleIndex uses 0,0
    llvm::Constant *wavesrc1 = nullptr, *wavesrc2 = nullptr;
    if (waveInst[0]) {
      wavesrc1 = llvm::dyn_cast<llvm::Constant>(waveInst[0]->getOperand(1));
      wavesrc2 = llvm::dyn_cast<llvm::Constant>(waveInst[0]->getOperand(2));
    }

    if (wavesrc1 && wavesrc2 && wavesrc1->isZeroValue() && wavesrc2->isZeroValue()) {
      if (llvm::isa<llvm::ExtractElementInst>(waveInst[0]->getOperand(0))) {
        llvm::ExtractElementInst *extractInst[4];

        // Find the 4 x extractelement
        extractInst[0] = llvm::dyn_cast<llvm::ExtractElementInst>(waveInst[0]->getOperand(0));
        extractInst[1] = llvm::dyn_cast<llvm::ExtractElementInst>(waveInst[1]->getOperand(0));
        extractInst[2] = llvm::dyn_cast<llvm::ExtractElementInst>(waveInst[2]->getOperand(0));
        extractInst[3] = llvm::dyn_cast<llvm::ExtractElementInst>(waveInst[3]->getOperand(0));

        if (extractInst[0] && extractInst[1] && extractInst[2] && extractInst[3] &&
            extractInst[0]->getOperand(0) == extractInst[1]->getOperand(0) &&
            extractInst[0]->getOperand(0) == extractInst[2]->getOperand(0) &&
            extractInst[0]->getOperand(0) == extractInst[3]->getOperand(0)) {
          MatchDp4a *pattern = new (m_allocator) MatchDp4a();
          pattern->instruction = &I;
          pattern->source[0] = GetSource(I.getOperand(0), false, false, IsSourceOfSample(&I));

          // set regioning to: <0;1,0>, as if we were still going to use the result of the WaveShuffleIndex
          llvm::BitCastInst *bitCast = llvm::dyn_cast<BitCastInst>(extractInst[0]->getOperand(0));
          if (bitCast) {
            pattern->source[1] = GetSource(bitCast->getOperand(0), false, false, IsSourceOfSample(&I));
            pattern->source[1].region[0] = 0;
            pattern->source[1].region[1] = 1;
            pattern->source[1].region[2] = 0;
            pattern->source[1].region_set = true;

            pattern->source[2] = GetSource(I.getOperand(2), false, false, IsSourceOfSample(&I));
            AddPattern(pattern);
            return true;
          }
        }
      }
    }
  }
  return MatchSingleInstruction(I);
}

bool CodeGenPatternMatch::MatchLogicAlu(llvm::BinaryOperator &I) {
  struct LogicInstPattern : public Pattern {
    SSource sources[2];
    llvm::Instruction *instruction;
    virtual void Emit(EmitPass *pass, const DstModifier &modifier) {
      pass->BinaryUnary(instruction, sources, modifier);
    }
  };
  LogicInstPattern *pattern = new (m_allocator) LogicInstPattern();
  pattern->instruction = &I;
  for (unsigned int i = 0; i < 2; ++i) {
    e_modifier mod = EMOD_NONE;
    Value *src = I.getOperand(i);
    if (!I.getType()->isIntegerTy(1)) {
      if (BinaryOperator *notInst = dyn_cast<BinaryOperator>(src)) {
        if (notInst->getOpcode() == Instruction::Xor) {
          if (ConstantInt *minusOne = dyn_cast<ConstantInt>(notInst->getOperand(1))) {
            if (minusOne->isMinusOne()) {
              mod = EMOD_NOT;
              src = notInst->getOperand(0);
            }
          }
        }
      }
    }
    pattern->sources[i] = GetSource(src, mod, false, IsSourceOfSample(&I));

    if (isCandidateForConstantPool(src)) {
      AddToConstantPool(I.getParent(), src);
      pattern->sources[i].fromConstantPool = true;
    }
  }
  AddPattern(pattern);
  return true;
}

bool CodeGenPatternMatch::MatchRsqrt(llvm::BinaryOperator &I) {
  struct RsqrtPattern : public Pattern {
    SSource source;
    virtual void Emit(EmitPass *pass, const DstModifier &modifier) { pass->Rsqrt(source, modifier); }
  };

  bool found = false;
  llvm::Value *source = NULL;
  if (I.getOpcode() == Instruction::FDiv) {
    // by vISA document, rsqrt doesn't support double type
    if (isOne(I.getOperand(0)) && I.getType()->getTypeID() != Type::DoubleTyID) {
      if (llvm::IntrinsicInst *sqrt = dyn_cast<IntrinsicInst>(I.getOperand(1))) {
        if (sqrt->getIntrinsicID() == Intrinsic::sqrt) {
          source = sqrt->getOperand(0);
          found = true;
        }
      }
    }
  }
  if (found) {
    RsqrtPattern *pattern = new (m_allocator) RsqrtPattern();
    pattern->source = GetSource(source, true, false, IsSourceOfSample(&I));
    AddPattern(pattern);
  }
  return found;
}

bool CodeGenPatternMatch::MatchArcpFdiv(llvm::BinaryOperator &I) {

  using namespace llvm::PatternMatch;

  struct ArcpFdivPattern : public Pattern {
    SSource sources[2];
    virtual void Emit(EmitPass *pass, const DstModifier &modifier) { pass->FDiv(sources, modifier); }
  };

  if (!I.getType()->isDoubleTy() || !I.hasAllowReciprocal())
    return false;

  // Look for fdiv.
  Instruction *fdiv = nullptr;
  Value *dividend = nullptr, *divisor = nullptr;

  auto fdivPattern = m_OneUse(m_FDiv(m_FPOne(), m_Value(divisor)));

  if (match(I.getOperand(0), fdivPattern)) {
    fdiv = dyn_cast<Instruction>(I.getOperand(0));
    dividend = I.getOperand(1);
  } else if (match(I.getOperand(1), fdivPattern)) {
    fdiv = dyn_cast<Instruction>(I.getOperand(1));
    dividend = I.getOperand(0);
  }

  if (!fdiv || !fdiv->hasAllowReciprocal())
    return false;

  // Pattern found.
  ArcpFdivPattern *pattern = new (m_allocator) ArcpFdivPattern();
  Value *sources[2] = {dividend, divisor};
  e_modifier src_mod[2] = {};

  if (FlushesDenormsOnInput(*fdiv)) {
    sources[0] = SkipCanonicalize(sources[0]);
    sources[1] = SkipCanonicalize(sources[1]);
  }

  GetModifier(*sources[0], src_mod[0], sources[0]);
  GetModifier(*sources[1], src_mod[1], sources[1]);

  pattern->sources[0] = GetSource(sources[0], src_mod[0], false, IsSourceOfSample(&I));
  pattern->sources[1] = GetSource(sources[1], src_mod[1], false, IsSourceOfSample(&I));

  // Try to add to constant pool whatever possible.
  if (isCandidateForConstantPool(sources[0])) {
    AddToConstantPool(I.getParent(), sources[0]);
    pattern->sources[0].fromConstantPool = true;
  }
  if (isCandidateForConstantPool(sources[1])) {
    AddToConstantPool(I.getParent(), sources[1]);
    pattern->sources[1].fromConstantPool = true;
  }

  AddPattern(pattern);

  return true;
}

bool CodeGenPatternMatch::MatchGradient(llvm::GenIntrinsicInst &I) {
  struct GradientPattern : public Pattern {
    SSource sources[2];
    llvm::Instruction *instruction;
    virtual void Emit(EmitPass *pass, const DstModifier &modifier) {
      pass->BinaryUnary(instruction, sources, modifier);
    }
  };
  GradientPattern *pattern = new (m_allocator) GradientPattern();
  pattern->instruction = &I;
  pattern->sources[0] = GetSource(I.getOperand(0), true, false, IsSourceOfSample(&I));
  pattern->sources[1] = {};
  AddPattern(pattern);
  // mark the source as subspan use
  HandleSubspanUse(pattern->sources[0].value, IsSourceOfSample(&I));
  return true;
}

bool CodeGenPatternMatch::MatchSampleLoadStoreAtomicTypedEff64(llvm::GenIntrinsicInst &I) {
  if (IGC_IS_FLAG_ENABLED(DisableStatefulFolding))
    return false;

  using namespace llvm::PatternMatch;

  struct SampleEff64bPattern : public Pattern {
    explicit SampleEff64bPattern(GenIntrinsicInst *I) : I(I) {}

    void Emit(EmitPass *pass, const DstModifier & /* modifier */) override {
      if (auto *SI = dyn_cast<SampleIntrinsic>(I)) {
        pass->emitSampleInstruction(SI);
      } else if (auto *GI = dyn_cast<SamplerGatherIntrinsic>(I)) {
        pass->emitGather4Instruction(GI);
      } else if (auto *II = dyn_cast<InfoIntrinsic>(I)) {
        pass->emitInfoInstruction(II);
      } else if (isa<AtomicTypedIntrinsic>(I)) {
        if (pass->shouldGenerateLSC(I)) {
          pass->emitLSCAtomicTyped(I);
        } else
          pass->emitAtomicTyped(I);
      } else {
        switch (I->getIntrinsicID()) {
        case GenISAIntrinsic::GenISA_ldptr:
        case GenISAIntrinsic::GenISA_ldlptr:
          pass->emitLdInstruction(I);
          break;
        case GenISAIntrinsic::GenISA_ldmsptr:
        case GenISAIntrinsic::GenISA_ldmcsptr:
        case GenISAIntrinsic::GenISA_ldmsptr16bit:
          pass->emitLdmsInstruction(I);
          break;
        case GenISAIntrinsic::GenISA_typedread:
          if (pass->shouldGenerateLSC(I, true) && IGC_IS_FLAG_DISABLED(DisableLSCForTypedUAV)) {
            pass->emitLSCTypedRead(I);
            break;
          }
          pass->emitTypedRead(I);
          break;
        case GenISAIntrinsic::GenISA_typedwrite:
          if (pass->shouldGenerateLSC(I, true) && IGC_IS_FLAG_DISABLED(DisableLSCForTypedUAV)) {
            pass->emitLSCTypedWrite(I);
            break;
          }
          pass->emitTypedWrite(I);
          break;
        case GenISAIntrinsic::GenISA_typedreadMS:
          IGC_ASSERT(pass->shouldGenerateLSC(I));
          pass->emitLSCTypedRead(I);
          break;
        case GenISAIntrinsic::GenISA_typedwriteMS:
          IGC_ASSERT(pass->shouldGenerateLSC(I));
          pass->emitLSCTypedWrite(I);
          break;
        default:
          IGC_ASSERT_MESSAGE(false, "unmatched pattern");
          break;
        }
      }
    }

  private:
    GenIntrinsicInst *I;
  };

  Value *texture = nullptr;
  Value *sampler = nullptr;

  if (auto *SI = dyn_cast<SampleIntrinsic>(&I)) {
    texture = SI->getTextureValue();
    sampler = SI->getSamplerValue();
  } else if (auto *GI = dyn_cast<SamplerGatherIntrinsic>(&I)) {
    texture = GI->getTextureValue();
    sampler = GI->getSamplerValue();
  } else if (auto *II = dyn_cast<InfoIntrinsic>(&I)) {
    texture = II->getOperand(0);
  } else if (auto *LI = dyn_cast<SamplerLoadIntrinsic>(&I)) {
    uint32_t numOperands = LI->getNumOperands();
    uint32_t textureArgIdx = numOperands - 5;
    texture = LI->getOperand(textureArgIdx);
  } else if (auto *ATI = dyn_cast<AtomicTypedIntrinsic>(&I)) {
    texture = ATI->getResourceValue();
  } else {
    texture = I.getOperand(0);
  }

  // Try to match texture and sampler to state index.
  // If matched, use the base resource/sampler for marking source.
  // This is only for marking source purpose, to avoid emitting redundant add instructions.
  // Sampler/texture index is not used as param for emit instruction here, it will be matched again in emit pass.
  // This is required for GetResourceVariable() and GetSamplerVariable() in emit pass, as it work on original
  // texture/sampler pointer.
  Value *baseTexture = nullptr;
  Value *baseSampler = nullptr;
  if (auto surfaceAndIndex = matchStateIndex(texture)) {
    baseTexture = surfaceAndIndex->first;
  }

  if (auto samplerAndIndex = matchStateIndex(sampler, true)) {
    baseSampler = samplerAndIndex->first;
  }

  auto *Pattern = new (m_allocator) SampleEff64bPattern(&I);

  for (auto &Op : I.operands()) {
    if (Op == texture) {
      if (baseTexture) {
        MarkAsSource(baseTexture, IsSourceOfSample(&I));
      } else {
        MarkAsSource(Op, IsSourceOfSample(&I));
      }
    } else if (Op == sampler) {
      if (baseSampler) {
        MarkAsSource(baseSampler, IsSourceOfSample(&I));
      } else {
        MarkAsSource(Op, IsSourceOfSample(&I));
      }
    } else {
      MarkAsSource(Op, IsSourceOfSample(&I));
    }
  }

  AddPattern(Pattern);
  return true;
}

bool CodeGenPatternMatch::MatchSample(llvm::GenIntrinsicInst &I) {
  switch (I.getIntrinsicID()) {
  case GenISAIntrinsic::GenISA_sampleptr:
  case GenISAIntrinsic::GenISA_sampleBptr:
  case GenISAIntrinsic::GenISA_sampleBCptr:
  case GenISAIntrinsic::GenISA_sampleCptr:
  case GenISAIntrinsic::GenISA_gather4Bptr:
  case GenISAIntrinsic::GenISA_gather4BPOptr:
  case GenISAIntrinsic::GenISA_gather4Iptr:
  case GenISAIntrinsic::GenISA_gather4IPOptr:
  case GenISAIntrinsic::GenISA_gather4ICptr:
  case GenISAIntrinsic::GenISA_gather4ICPOptr:
  case GenISAIntrinsic::GenISA_sampleMlodptr:
  case GenISAIntrinsic::GenISA_sampleCMlodptr:
  case GenISAIntrinsic::GenISA_sampleBCMlodptr:
  case GenISAIntrinsic::GenISA_samplePOptr:
  case GenISAIntrinsic::GenISA_samplePOBptr:
  case GenISAIntrinsic::GenISA_samplePOCptr:
  case GenISAIntrinsic::GenISA_gather4POPackedBptr:
  case GenISAIntrinsic::GenISA_gather4POPackedIptr:
  case GenISAIntrinsic::GenISA_gather4POPackedICptr:
  case GenISAIntrinsic::GenISA_lodptr:
  case GenISAIntrinsic::GenISA_sampleKillPix:
    HandleSampleDerivative(I);
    [[fallthrough]];
  default:
    if (m_Platform.hasEfficient64bEnabled()) {
      return MatchSampleLoadStoreAtomicTypedEff64(I);
    } else {
      return MatchSingleInstruction(I);
    }
  }
}

bool CodeGenPatternMatch::MatchDbgInstruction(llvm::DbgInfoIntrinsic &I) {
  struct DbgInstPattern : Pattern {
    virtual void Emit(EmitPass *pass, const DstModifier &modifier) {
      // Nothing to emit.
    }
  };
  DbgInstPattern *pattern = new (m_allocator) DbgInstPattern();
  AddPattern(pattern);
  return true;
}

bool CodeGenPatternMatch::MatchAvg(llvm::Instruction &I) {
  // "Average value" pattern:
  // (x + y + 1) / 2  -->  avg(x, y)
  //
  // We're looking for patterns like this:
  //    % 14 = add nsw i32 % 10, % 13
  //    % 15 = add nsw i32 % 14, 1
  //    % 16 = ashr i32 % 15, 1

  struct AvgPattern : Pattern {
    SSource sources[2];
    virtual void Emit(EmitPass *pass, const DstModifier &modifier) { pass->Avg(sources, modifier); }
  };

  bool found = false;
  llvm::Value *sources[2] = {};
  e_modifier src_mod[2] = {};

  IGC_ASSERT(I.getOpcode() == Instruction::SDiv || I.getOpcode() == Instruction::UDiv ||
             I.getOpcode() == Instruction::AShr);

  // HW avg instruction only supports types up to 32 bits (B/UB/W/UW/D/UD).
  if (I.getType()->getScalarSizeInBits() > 32)
    return false;

  // We expect 2 for "div" and 1 for "right shift".
  int expectedVal = (I.getOpcode() == Instruction::SDiv ? 2 : 1);
  Value *opnd1 = I.getOperand(1); // Divisor or shift factor.
  if (!isa<ConstantInt>(opnd1) || (cast<ConstantInt>(opnd1))->getZExtValue() != expectedVal) {
    return false;
  }

  if (Instruction *divSrc = dyn_cast<Instruction>(I.getOperand(0))) {
    if (divSrc->getOpcode() == Instruction::Add && !NeedInstruction(*divSrc)) {
      Instruction *instAdd = cast<Instruction>(divSrc);
      for (int i = 0; i < 2; i++) {
        if (ConstantInt *cnst = dyn_cast<ConstantInt>(instAdd->getOperand(i))) {
          // "otherArg" is the second argument of "instAdd" (which is not constant).
          Value *otherArg = instAdd->getOperand(i == 0 ? 1 : 0);
          if (cnst->getZExtValue() == 1 && isa<AddOperator>(otherArg) &&
              !NeedInstruction(*cast<Instruction>(otherArg))) {
            Instruction *firstAdd = cast<Instruction>(otherArg);
            sources[0] = firstAdd->getOperand(0);
            sources[1] = firstAdd->getOperand(1);
            GetModifier(*sources[0], src_mod[0], sources[0]);
            GetModifier(*sources[1], src_mod[1], sources[1]);
            found = true;
            break;
          }
        }
      }
    }
  }

  if (found) {
    AvgPattern *pattern = new (m_allocator) AvgPattern();
    pattern->sources[0] = GetSource(sources[0], src_mod[0], false, IsSourceOfSample(&I));
    pattern->sources[1] = GetSource(sources[1], src_mod[1], false, IsSourceOfSample(&I));
    AddPattern(pattern);
  }
  return found;
}

bool CodeGenPatternMatch::MatchShuffleBroadCast(llvm::GenIntrinsicInst &I) {
  // Match cases like:
  //    %84 = bitcast <2 x i32> %vCastload to <4 x half>
  //    %scalar269 = extractelement <4 x half> % 84, i32 0
  //    %simdShuffle = call half @llvm.genx.GenISA.simdShuffle.f.f16(half %scalar269, i32 0)
  //
  // to mov with region and offset
  struct BroadCastPattern : public Pattern {
    SSource source;
    virtual void Emit(EmitPass *pass, const DstModifier &modifier) { pass->Mov(source, modifier); }
  };
  [[maybe_unused]] bool match = false;
  SSource source;
  Value *sourceV = &I;
  if (GetRegionModifier(source, sourceV, true)) {
    BroadCastPattern *pattern = new (m_allocator) BroadCastPattern();
    GetModifier(*sourceV, source.mod, sourceV);
    source.value = sourceV;
    pattern->source = source;
    MarkAsSource(sourceV, IsSourceOfSample(&I));
    match = true;
    AddPattern(pattern);
  }
  return match;
}

bool CodeGenPatternMatch::MatchWaveShuffleIndex(llvm::GenIntrinsicInst &I) {
  auto helperLaneMode = cast<ConstantInt>(I.getOperand(2));
  IGC_ASSERT(helperLaneMode);
  if (int_cast<int>(helperLaneMode->getSExtValue()) == 1) {
    // only if helperLaneMode==1, we enable helper lane under some shuffleindex cases (not for all cases).
    HandleSubspanUse(I.getArgOperand(0), IsSourceOfSample(&I));
    HandleSubspanUse(I.getArgOperand(1), IsSourceOfSample(&I));
    HandleSubspanUse(&I, IsSourceOfSample(&I));
  }
  return MatchSingleInstruction(I);
}

bool CodeGenPatternMatch::MatchWaveInstruction(llvm::GenIntrinsicInst &I) {
  if (subgroupIntrinsicHasHelperLanes(I)) {
    m_NeedVMask = true;
  }
  return MatchSingleInstruction(I);
}

bool CodeGenPatternMatch::MatchRegisterRegion(llvm::GenIntrinsicInst &I) {
  struct MatchRegionPattern : public Pattern {
    SSource source;
    virtual void Emit(EmitPass *pass, const DstModifier &modifier) {
      const bool isSimd32Dispatch = (pass->m_currShader->m_State.m_dispatchSize == SIMDMode::SIMD32);
      if (isSimd32Dispatch && pass->m_currShader->m_numberInstance == 2) {
        pass->emitCrossInstanceMov(source, modifier);
      } else {
        pass->Mov(source, modifier);
      }
    }
  };

  /*
  * Match case 1 - With SubReg Offset: Shuffle( data, (laneID << x) + y )
  *   %25 = call i16 @llvm.genx.GenISA.simdLaneId()
  *   %30 = zext i16 %25 to i32
  *   %31 = shl nuw nsw i32 %30, 1  - Current LaneID shifted by x
  *   %36 = add i32 %31, 1          - Current LaneID shifted by x + y  Shuffle( data, (laneID << x) + 1 )
  *   %37 = call float @llvm.genx.GenISA.WaveShuffleIndex.f32(float %21, i32 %36)

  * Match case 2(Special case of Match Case 1) - No SubReg Offset: Shuffle( data, (laneID << x) + 0 )
  *    %25 = call i16 @llvm.genx.GenISA.simdLaneId()
  *    %30 = zext i16 %25 to i32
  *    %31 = shl nuw nsw i32 %30, 1 - Current LaneID shifted by x
  *    %32 = call float @llvm.genx.GenISA.WaveShuffleIndex.f32(float %21, i32 %31)
  */

  Value *data = I.getOperand(0);
  Value *source = I.getOperand(1);
  uint typeByteSize = data->getType()->getScalarSizeInBits() / 8;
  bool isMatch = false;
  int subReg = 0;
  uint verticalStride = 1; // Default value for special case  Shuffle( data, (laneID << x) + y )  when x = 0

  if (auto binaryInst = dyn_cast<BinaryOperator>(source)) {
    // Will be skipped for match case 2
    if (binaryInst->getOpcode() == Instruction::Add) {
      if (llvm::ConstantInt *simDOffSetInst = llvm::dyn_cast<llvm::ConstantInt>(binaryInst->getOperand(1))) {
        subReg = int_cast<int>(cast<ConstantInt>(simDOffSetInst)->getSExtValue());

        // Subregister must be a number between 0 and 15 for a valid region
        //  We could support up to 31 but we need to handle reading from different SIMD16 var chunks
        if (subReg >= 0 && subReg < 16) {
          source = binaryInst->getOperand(0);
        }
      }
    }
  }

  if (auto binaryInst = dyn_cast<BinaryOperator>(source)) {
    if (binaryInst->getOpcode() == Instruction::Shl) {
      source = binaryInst->getOperand(0);

      if (llvm::ConstantInt *simDOffSetInst = llvm::dyn_cast<llvm::ConstantInt>(binaryInst->getOperand(1))) {
        uint shiftFactor = int_cast<uint>(simDOffSetInst->getZExtValue());
        // Check to make sure we dont end up with an invalid Vertical Stride.
        // Only 1, 2, 4, 8, 16 are supported.
        if (shiftFactor <= 4)
          verticalStride = (1U << shiftFactor);
        else
          return false;
      }
    }
  }

  if (auto zExtInst = llvm::dyn_cast<llvm::ZExtInst>(source)) {
    source = zExtInst->getOperand(0);
  }

  llvm::GenIntrinsicInst *intrin = llvm::dyn_cast<llvm::GenIntrinsicInst>(source);

  // Finally check for simLaneID intrisic
  if (intrin && (intrin->getIntrinsicID() == GenISAIntrinsic::GenISA_simdLaneId)) {
    // To avoid compiler crash, pattern match with direct mov will be disable
    // Conservetively, we assum simd16 for 32 bytes GRF platforms and simd32 for 64 bytes GRF platforms
    bool cross2GRFs = typeByteSize * (subReg + verticalStride * (m_Platform.getGRFSize() > 32 ? 32 : 16)) >
                      (2 * m_Platform.getGRFSize());
    if (!cross2GRFs) {
      MatchRegionPattern *pattern = new (m_allocator) MatchRegionPattern();
      pattern->source.elementOffset = subReg;

      // Set Region Parameters <VerString;Width,HorzString>
      pattern->source.region_set = true;
      pattern->source.region[0] = verticalStride;
      pattern->source.region[1] = 1;
      pattern->source.region[2] = 0;

      pattern->source.value = data;
      MarkAsSource(data, IsSourceOfSample(&I));
      HandleSubspanUse(data, IsSourceOfSample(&I));
      AddPattern(pattern);

      isMatch = true;
    }
  }

  return isMatch;
}

bool CodeGenPatternMatch::GetRegionModifier(SSource &sourceMod, llvm::Value *&source, bool regioning) {
  bool found = false;
  Value *OrignalSource = source;
  if (llvm::BitCastInst *bitCast = llvm::dyn_cast<BitCastInst>(source)) {
    if (!bitCast->getType()->isVectorTy() && !bitCast->getOperand(0)->getType()->isVectorTy()) {
      source = bitCast->getOperand(0);
      found = true;
    }
  }

  if (llvm::GenIntrinsicInst *intrin = llvm::dyn_cast<llvm::GenIntrinsicInst>(source)) {
    GenISAIntrinsic::ID id = intrin->getIntrinsicID();
    if (id == GenISAIntrinsic::GenISA_WaveShuffleIndex || id == GenISAIntrinsic::GenISA_WaveBroadcast) {
      if (llvm::ConstantInt *channelVal = llvm::dyn_cast<llvm::ConstantInt>(intrin->getOperand(1))) {
        unsigned int offset = int_cast<unsigned int>(channelVal->getZExtValue());
        if (offset < 16 && !isUniform(intrin->getOperand(0))) {
          sourceMod.elementOffset = offset;
          // SIMD shuffle force region <0,1;0>
          sourceMod.region_set = true;
          sourceMod.region[0] = 0;
          sourceMod.region[1] = 1;
          sourceMod.region[2] = 0;
          sourceMod.instance = EINSTANCE_FIRST_HALF;
          source = intrin->getOperand(0);
          found = true;
          BitcastSearch(sourceMod, source, true);
        }
      }
    }
  }
  if (regioning && !sourceMod.region_set) {
    found |= BitcastSearch(sourceMod, source, false);
  }
  if (found && sourceMod.type == VISA_Type::ISA_TYPE_NUM) {
    // keep the original type
    sourceMod.type = GetType(OrignalSource->getType(), m_ctx);
  }
  return found;
}

void CodeGenPatternMatch::HandleSampleDerivative(llvm::GenIntrinsicInst &I) {
  switch (I.getIntrinsicID()) {
  case GenISAIntrinsic::GenISA_sampleptr:
  case GenISAIntrinsic::GenISA_lodptr:
  case GenISAIntrinsic::GenISA_sampleKillPix:
    HandleSubspanUse(I.getOperand(0), true);
    HandleSubspanUse(I.getOperand(1), true);
    HandleSubspanUse(I.getOperand(2), true);
    break;
  case GenISAIntrinsic::GenISA_sampleBptr:
  case GenISAIntrinsic::GenISA_sampleCptr:
    HandleSubspanUse(I.getOperand(1), true);
    HandleSubspanUse(I.getOperand(2), true);
    HandleSubspanUse(I.getOperand(3), true);
    break;
  case GenISAIntrinsic::GenISA_sampleBCptr:
    HandleSubspanUse(I.getOperand(2), true);
    HandleSubspanUse(I.getOperand(3), true);
    HandleSubspanUse(I.getOperand(4), true);
    break;
  case GenISAIntrinsic::GenISA_gather4Iptr:
  case GenISAIntrinsic::GenISA_gather4IPOptr:
  case GenISAIntrinsic::GenISA_gather4POPackedIptr:
    HandleSubspanUse(I.getOperand(0), true);
    HandleSubspanUse(I.getOperand(1), true);
    HandleSubspanUse(I.getOperand(2), true);
    break;
  case GenISAIntrinsic::GenISA_samplePOptr:
    HandleSubspanUse(I.getOperand(0), true);
    HandleSubspanUse(I.getOperand(1), true);
    HandleSubspanUse(I.getOperand(2), true);
    break;
  case GenISAIntrinsic::GenISA_gather4Bptr:
  case GenISAIntrinsic::GenISA_gather4BPOptr:
  case GenISAIntrinsic::GenISA_gather4ICptr:
  case GenISAIntrinsic::GenISA_gather4ICPOptr:
  case GenISAIntrinsic::GenISA_gather4POPackedBptr:
  case GenISAIntrinsic::GenISA_gather4POPackedICptr:
    HandleSubspanUse(I.getOperand(1), true);
    HandleSubspanUse(I.getOperand(2), true);
    HandleSubspanUse(I.getOperand(3), true);
    break;
  case GenISAIntrinsic::GenISA_sampleMlodptr:
    HandleSubspanUse(I.getOperand(1), true);
    HandleSubspanUse(I.getOperand(2), true);
    HandleSubspanUse(I.getOperand(3), true);
    break;
  case GenISAIntrinsic::GenISA_sampleCMlodptr:
  case GenISAIntrinsic::GenISA_sampleBCMlodptr:
    HandleSubspanUse(I.getOperand(2), true);
    HandleSubspanUse(I.getOperand(3), true);
    HandleSubspanUse(I.getOperand(4), true);
    break;
  case GenISAIntrinsic::GenISA_samplePOBptr:
  case GenISAIntrinsic::GenISA_samplePOCptr:
    HandleSubspanUse(I.getOperand(1), true);
    HandleSubspanUse(I.getOperand(2), true);
    HandleSubspanUse(I.getOperand(3), true);
    break;
  default:
    break;
  }
}

// helper function for pattern match
static inline bool isLowerPredicate(llvm::CmpInst::Predicate pred) {
  switch (pred) {
  case llvm::CmpInst::FCMP_ULT:
  case llvm::CmpInst::FCMP_ULE:
  case llvm::CmpInst::FCMP_OLT:
  case llvm::CmpInst::FCMP_OLE:
  case llvm::CmpInst::ICMP_ULT:
  case llvm::CmpInst::ICMP_ULE:
  case llvm::CmpInst::ICMP_SLT:
  case llvm::CmpInst::ICMP_SLE:
    return true;
  default:
    break;
  }
  return false;
}

// helper function for pattern match
static inline bool isGreaterOrLowerPredicate(llvm::CmpInst::Predicate pred) {
  switch (pred) {
  case llvm::CmpInst::FCMP_UGT:
  case llvm::CmpInst::FCMP_UGE:
  case llvm::CmpInst::FCMP_ULT:
  case llvm::CmpInst::FCMP_ULE:
  case llvm::CmpInst::FCMP_OGT:
  case llvm::CmpInst::FCMP_OGE:
  case llvm::CmpInst::FCMP_OLT:
  case llvm::CmpInst::FCMP_OLE:
  case llvm::CmpInst::ICMP_UGT:
  case llvm::CmpInst::ICMP_UGE:
  case llvm::CmpInst::ICMP_ULT:
  case llvm::CmpInst::ICMP_ULE:
  case llvm::CmpInst::ICMP_SGT:
  case llvm::CmpInst::ICMP_SGE:
  case llvm::CmpInst::ICMP_SLT:
  case llvm::CmpInst::ICMP_SLE:
    return true;
  default:
    break;
  }
  return false;
}

static bool isIntegerAbs(SelectInst *SI, e_modifier &mod, Value *&source) {
  using namespace llvm::PatternMatch; // Scoped using declaration.

  Value *Cond = SI->getOperand(0);
  Value *TVal = SI->getOperand(1);
  Value *FVal = SI->getOperand(2);

  IGCLLVM::ICmpInstPredicate IPred(FCmpInst::FCMP_FALSE);
  Value *LHS = nullptr;
  Value *RHS = nullptr;

  if (!match(Cond, m_ICmp(IPred, m_Value(LHS), m_Value(RHS))))
    return false;

  if (!ICmpInst::isSigned(IPred))
    return false;

  if (match(LHS, m_Zero())) {
    IPred = ICmpInst::getSwappedPredicate(IPred);
    std::swap(LHS, RHS);
  }

  if (!match(RHS, m_Zero()))
    return false;

  if (match(TVal, m_Neg(m_Specific(FVal)))) {
    IPred = ICmpInst::getInversePredicate(IPred);
    std::swap(TVal, FVal);
  }

  if (!match(FVal, m_Neg(m_Specific(TVal))))
    return false;

  if (LHS != TVal)
    return false;

  source = TVal;
  mod = (IPred == ICmpInst::ICMP_SGT || IPred == ICmpInst::ICMP_SGE) ? EMOD_ABS : EMOD_NEGABS;

  return true;
}

bool isAbs(llvm::Value *abs, e_modifier &mod, llvm::Value *&source) {
  bool found = false;

  if (IntrinsicInst *intrinsicInst = dyn_cast<IntrinsicInst>(abs)) {
    if (intrinsicInst->getIntrinsicID() == Intrinsic::fabs) {
      source = intrinsicInst->getOperand(0);
      mod = EMOD_ABS;
      return true;
    }
    if (intrinsicInst->getIntrinsicID() == Intrinsic::abs) {
      source = intrinsicInst->getOperand(0);
      mod = EMOD_ABS;
      return true;
    }
  }

  llvm::SelectInst *select = llvm::dyn_cast<llvm::SelectInst>(abs);
  if (!select)
    return false;

  // Try to find floating point abs first
  if (llvm::FCmpInst *cmp = llvm::dyn_cast<llvm::FCmpInst>(select->getOperand(0))) {
    llvm::CmpInst::Predicate pred = cmp->getPredicate();
    if (isGreaterOrLowerPredicate(pred)) {
      for (int zeroIndex = 0; zeroIndex < 2; zeroIndex++) {
        llvm::ConstantFP *zero = llvm::dyn_cast<llvm::ConstantFP>(cmp->getOperand(zeroIndex));
        if (zero && zero->isZero()) {
          llvm::Value *cmpSource = cmp->getOperand(1 - zeroIndex);
          for (int sourceIndex = 0; sourceIndex < 2; sourceIndex++) {
            if (cmpSource == select->getOperand(1 + sourceIndex)) {
              llvm::Instruction *opnd = llvm::dyn_cast<llvm::Instruction>(select->getOperand(1 + (1 - sourceIndex)));
              llvm::Value *negateSource = NULL;
              if (opnd && IsNegate(opnd, negateSource) && negateSource == cmpSource) {
                found = true;
                source = cmpSource;
                // depending on the order source in cmp/select it can abs() or -abs()
                bool isNegateAbs = (zeroIndex == 0) ^ isLowerPredicate(pred) ^ (sourceIndex == 1);
                mod = isNegateAbs ? EMOD_NEGABS : EMOD_ABS;
              }
              break;
            }
          }
          break;
        }
      }
    }
  }

  // If not found, try integer abs
  return found || isIntegerAbs(select, mod, source);
}

// combine two modifiers, this function is *not* communtative
e_modifier CombineModifier(e_modifier mod1, e_modifier mod2) {
  e_modifier mod = EMOD_NONE;
  switch (mod1) {
  case EMOD_ABS:
  case EMOD_NEGABS:
    mod = mod1;
    break;
  case EMOD_NEG:
    if (mod2 == EMOD_NEGABS) {
      mod = EMOD_ABS;
    } else if (mod2 == EMOD_ABS) {
      mod = EMOD_NEGABS;
    } else if (mod2 == EMOD_NEG) {
      mod = EMOD_NONE;
    } else {
      mod = EMOD_NEG;
    }
    break;
  default:
    mod = mod2;
  }
  return mod;
}

bool GetModifier(llvm::Value &modifier, e_modifier &mod, llvm::Value *&source) {
  mod = EMOD_NONE;
  if (llvm::Instruction *bin = llvm::dyn_cast<llvm::Instruction>(&modifier)) {
    return GetModifier(*bin, mod, source);
  }
  return false;
}

bool GetModifier(llvm::Instruction &modifier, e_modifier &mod, llvm::Value *&source) {
  llvm::Value *modifierSource = NULL;
  mod = EMOD_NONE;
  if (IsNegate(&modifier, modifierSource)) {
    e_modifier absModifier = EMOD_NONE;
    llvm::Value *absSource = NULL;
    if (isAbs(modifierSource, absModifier, absSource)) {
      source = absSource;
      mod = IGC::CombineModifier(EMOD_NEG, absModifier);
    } else {
      source = modifierSource;
      mod = EMOD_NEG;
    }
    return true;
  } else if (isAbs(&modifier, mod, modifierSource)) {
    source = modifierSource;
    return true;
  }
  return false;
}

bool IsNegate(llvm::Instruction *inst, llvm::Value *&negateSource) {
  BinaryOperator *binop = dyn_cast<BinaryOperator>(inst);
  if (binop && (inst->getOpcode() == Instruction::FSub || inst->getOpcode() == Instruction::Sub)) {
    if (IsZero(inst->getOperand(0))) {
      negateSource = inst->getOperand(1);
      return true;
    }
  }
  UnaryOperator *unop = dyn_cast<UnaryOperator>(inst);
  if (unop && inst->getOpcode() == Instruction::FNeg) {
    negateSource = inst->getOperand(0);
    return true;
  }
  return false;
}

bool IsZero(llvm::Value *zero) {
  if (llvm::ConstantFP *FCst = llvm::dyn_cast<llvm::ConstantFP>(zero)) {
    if (FCst->isZero()) {
      return true;
    }
  }
  if (llvm::ConstantInt *ICst = llvm::dyn_cast<llvm::ConstantInt>(zero)) {
    if (ICst->isZero()) {
      return true;
    }
  }
  return false;
}

inline bool isMinOrMax(llvm::Value *inst, llvm::Value *&source0, llvm::Value *&source1, bool &isMin, bool &isUnsigned) {
  bool found = false;
  llvm::Instruction *max = llvm::dyn_cast<llvm::Instruction>(inst);
  if (!max)
    return false;

  EOPCODE op = GetOpCode(max);
  if (op == llvm_min || op == llvm_max) {
    source0 = max->getOperand(0);
    source1 = max->getOperand(1);
    isUnsigned = false;
    isMin = (op == llvm_min);
    return true;
  } else if (op == llvm_select) {
    if (llvm::CmpInst *cmp = llvm::dyn_cast<llvm::CmpInst>(max->getOperand(0))) {
      auto SkipAsr = [max](uint32_t idx) {
        ConstantInt *c1 = dyn_cast<ConstantInt>(max->getOperand(1));
        ConstantInt *c2 = dyn_cast<ConstantInt>(max->getOperand(2));
        Instruction *op = dyn_cast<Instruction>(max->getOperand(idx));
        if (((c1 && c1->isZeroValue()) || (c2 && c2->isZeroValue())) && (op && op->getOpcode() == Instruction::AShr)) {
          return op->getOperand(0);
        }
        return max->getOperand(idx);
      };
      if (isGreaterOrLowerPredicate(cmp->getPredicate())) {
        if ((cmp->getOperand(0) == max->getOperand(1) && cmp->getOperand(1) == max->getOperand(2)) ||
            (cmp->getOperand(0) == max->getOperand(2) && cmp->getOperand(1) == max->getOperand(1))) {
          isMin = isLowerPredicate(cmp->getPredicate()) ^ (cmp->getOperand(0) == max->getOperand(2));
          found = true;
        } else if ((cmp->getOperand(0) == SkipAsr(1) && cmp->getOperand(1) == SkipAsr(2)) ||
                   (cmp->getOperand(0) == SkipAsr(2) && cmp->getOperand(1) == SkipAsr(1))) {
          isMin = isLowerPredicate(cmp->getPredicate()) ^ (cmp->getOperand(0) == SkipAsr(2));
          found = true;
        }
        if (found) {
          source0 = max->getOperand(1);
          source1 = max->getOperand(2);
          isUnsigned = IsUnsignedCmp(cmp->getPredicate());
        }
      }
    }
  }
  return found;
}

bool isMax(llvm::Value *max, llvm::Value *&source0, llvm::Value *&source1) {
  bool isMin, isUnsigned;
  llvm::Value *maxSource0;
  llvm::Value *maxSource1;
  if (isMinOrMax(max, maxSource0, maxSource1, isMin, isUnsigned)) {
    if (!isMin) {
      source0 = maxSource0;
      source1 = maxSource1;
      return true;
    }
  }
  return false;
}

bool isMin(llvm::Value *min, llvm::Value *&source0, llvm::Value *&source1) {
  bool isMin, isUnsigned;
  llvm::Value *maxSource0;
  llvm::Value *maxSource1;
  if (isMinOrMax(min, maxSource0, maxSource1, isMin, isUnsigned)) {
    if (isMin) {
      source0 = maxSource0;
      source1 = maxSource1;
      return true;
    }
  }
  return false;
}

bool isOne(llvm::Value *zero) {
  if (llvm::ConstantFP *FCst = llvm::dyn_cast<llvm::ConstantFP>(zero)) {
    if (FCst->isExactlyValue(1.f)) {
      return true;
    }
  }
  if (llvm::ConstantInt *ICst = llvm::dyn_cast<llvm::ConstantInt>(zero)) {
    if (ICst->isOne()) {
      return true;
    }
  }
  return false;
}

bool isSat(llvm::Instruction *sat, llvm::Value *&source, bool &isUnsigned) {
  bool found = false;
  llvm::Value *sources[2] = {0};
  bool floatMatch = sat->getType()->isFloatingPointTy();
  GenIntrinsicInst *intrin = dyn_cast<GenIntrinsicInst>(sat);
  if (intrin && (intrin->getIntrinsicID() == GenISAIntrinsic::GenISA_fsat ||
                 intrin->getIntrinsicID() == GenISAIntrinsic::GenISA_usat ||
                 intrin->getIntrinsicID() == GenISAIntrinsic::GenISA_isat)) {
    source = intrin->getOperand(0);
    found = true;
    isUnsigned = intrin->getIntrinsicID() == GenISAIntrinsic::GenISA_usat;
  } else if (floatMatch && isMax(sat, sources[0], sources[1])) {
    for (int i = 0; i < 2; i++) {
      if (IsZero(sources[i])) {
        llvm::Value *maxSources[2] = {0};
        if (isMin(sources[1 - i], maxSources[0], maxSources[1])) {
          for (int j = 0; j < 2; j++) {
            if (isOne(maxSources[j])) {
              found = true;
              source = maxSources[1 - j];
              isUnsigned = false;
              break;
            }
          }
        }
        break;
      }
    }
  } else if (floatMatch && isMin(sat, sources[0], sources[1])) {
    for (int i = 0; i < 2; i++) {
      if (isOne(sources[i])) {
        llvm::Value *maxSources[2] = {0};
        if (isMax(sources[1 - i], maxSources[0], maxSources[1])) {
          for (int j = 0; j < 2; j++) {
            if (IsZero(maxSources[j])) {
              found = true;
              source = maxSources[1 - j];
              isUnsigned = false;
              break;
            }
          }
        }
        break;
      }
    }
  }
  return found;
}

bool isCandidateForConstantPool(llvm::Value *val) {
  auto ci = dyn_cast<ConstantInt>(val);
  bool isBigQW = ci && !ci->getValue().isZero() && !ci->getValue().isSignedIntN(32);
  bool isDF = val->getType()->isDoubleTy();
  return (isBigQW || isDF);
}

uint CodeGenPatternMatch::GetBlockId(llvm::BasicBlock *block) {
  auto it = m_blockMap.find(block);
  IGC_ASSERT(it != m_blockMap.end());

  uint blockID = it->second->id;
  return blockID;
}

} // namespace IGC
