/*========================== begin_copyright_notice ============================

Copyright (C) 2021-2025 Intel Corporation

SPDX-License-Identifier: MIT

============================= end_copyright_notice ===========================*/

#include "common/LLVMWarningsPush.hpp"
#include "llvm/PassInfo.h"
#include "llvm/PassRegistry.h"
#include "common/LLVMWarningsPop.hpp"
#include "GenISAIntrinsics/GenIntrinsicInst.h"
#include "IGC/Compiler/CodeGenPublic.h"
#include "Probe/Assertion.h"
#include "Compiler/CISACodeGen/helper.h"
#include "Compiler/IGCPassSupport.h"
#include "SynchronizationObjectCoalescing.hpp"
#include "visa_igc_common_header.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/Analysis/CFG.h"
#include "llvm/Analysis/LoopInfo.h"
#include <utility>
#include <map>

using namespace llvm;
namespace IGC {

#if _DEBUG
////////////////////////////////////////////////////////////////////////
struct ExplanationEntry {
  ////////////////////////////////////////////////////////////////////////
  enum Cause { GlobalMemoryRedundancy, L1CacheInvalidationRedundancy, StrictRedundancy, FastStrictRedundancy };

  Cause m_Cause;

  typedef std::tuple<const llvm::BasicBlock *, uint32_t /*index*/, std::string> SyncInstDescription;

  SyncInstDescription m_SynchronizationDescription;
  std::vector<SyncInstDescription> m_ForwardBoundaries;
  std::vector<SyncInstDescription> m_BackwardBoundaries;
  std::vector<SyncInstDescription> m_ThreadGroupBarriersBoundaries;
  std::vector<const llvm::Instruction *> m_ForwardMemoryInstructions;
  std::vector<const llvm::Instruction *> m_BackwardMemoryInstructions;
};
#endif // _DEBUG

////////////////////////////////////////////////////////////////////////////////
enum InstructionMask : uint32_t {
  None = 0x0,
  AtomicOperation = (1 << 0),
  TypedReadOperation = (1 << 1),
  TypedWriteOperation = (1 << 2),
  OutputUrbReadOperation = (1 << 3),
  UrbWriteOperation = (1 << 4),
  BufferReadOperation = (1 << 5),
  BufferWriteOperation = (1 << 6),
  SharedMemoryReadOperation = (1 << 7),
  SharedMemoryWriteOperation = (1 << 8),
  EndOfThreadOperation = (1 << 9),
  ExtendedCacheControlOperation = (1 << 10),
  LastMaskPlusOne,
};

////////////////////////////////////////////////////////////////////////////////
enum class SyncInstMask : uint32_t {
  None = 0x0,
  UntypedMemoryFence = (1 << 0),
  TypedMemoryFence = (1 << 1),
  SharedMemoryFence = (1 << 2),
  UrbMemoryFence = (1 << 3),
  ThreadGroupBarrier = (1 << 4)
};

constexpr InstructionMask AllInstructionsMask = InstructionMask{((InstructionMask::LastMaskPlusOne - 1) << 1) - 1};

constexpr InstructionMask AllNoAtomicMask = InstructionMask{AllInstructionsMask & ~InstructionMask::AtomicOperation};

inline constexpr InstructionMask operator|(InstructionMask a, InstructionMask b) {
  return InstructionMask(uint32_t(a) | uint32_t(b));
}
inline constexpr InstructionMask &operator|=(InstructionMask &a, InstructionMask b) {
  a = a | b;
  return a;
}
inline constexpr SyncInstMask operator|(SyncInstMask a, SyncInstMask b) {
  return SyncInstMask(uint32_t(a) | uint32_t(b));
}
inline constexpr SyncInstMask &operator|=(SyncInstMask &a, SyncInstMask b) {
  a = a | b;
  return a;
}

inline constexpr InstructionMask operator&(InstructionMask a, InstructionMask b) {
  return InstructionMask(uint32_t(a) & uint32_t(b));
}
inline constexpr InstructionMask &operator&=(InstructionMask &a, InstructionMask b) {
  a = a & b;
  return a;
}

using InstIdxLookupTableT = llvm::DenseMap<llvm::Instruction *, uint32_t>;
using InstMaskLookupTableT = llvm::DenseMap<llvm::Instruction *, InstructionMask>;
using BasicBlockMaskLookupTableT = llvm::DenseMap<llvm::BasicBlock *, InstructionMask>;
using InstLookupTableInBasicBlockT = std::map<uint32_t, llvm::Instruction *>;
using OrderedInstructionLookupTableT = llvm::DenseMap<llvm::BasicBlock *, InstLookupTableInBasicBlockT>;

////////////////////////////////////////////////////////////////////////
/// @brief Synchronization objects prevents from hazardous situations in kernels.
/// They cannot be removed if one of hazards is possible:
/// - Read-after-write (RAW) - "read too soon",
/// - Write-after-read(WAR) - "written too soon",
/// - Write-after-write(WAW) - "written out-of-order".
/// Synchronization objects are following:
/// - memory fences - they synchronize caches with destination memory to the defined
///   extent and ensures that memory accesses finish before the execution
///   of the synchronization,
/// - control barriers - they ensures not until all the threads of the defined scope
/// reached can any of them pass this instruction.
/// Atomic instructions can also be used for synchronizing or predicting the order of
/// execution. Such instructions introduces a complex way of synchronization which
/// can hide some potential hazards. In this context they must be handled as well.
/// This pass analyzes all paths of the shaders to find dependencies
/// between memory read and write instructions and synchronization instructions.
/// The result of this analysis indicates redundancies in the following synchronization
/// instructions:
/// - thread group barriers:  llvm.genx.GenISA.threadgroupbarrier;
///   (interacts with global, typed, shared memory buffers, outputs and atomics)
/// - memory fence: llvm.genx.GenISA.memoryfence;
///   depending on the HW platforms and intrinsic arguments the fence can synchronize
///   global(untyped and typed) memory and/or local memory(SLM)
///   (interacts with global, typed, shared memory buffers, outputs and atomics)
/// - typed memory fences:    llvm.genx.GenISA.typedmemoryfence;
///   (interacts with typed buffers and atomics)
/// - URB memory fences:      llvm.genx.GenISA.urbfence;
///   (interacts with outputs and atomics)
/// - LSC fences:      llvm.genx.GenISA.LSCfence;
///   Current implementation of the pass treats LSC fences separately from
///   legacy memoryfence intrinsics. The assumption is that frontends will
///   either use legacy memoryfence intrinsics or LSC fences and not a mix.
///   Note: a simple pass could be added to translate the legacy memoryfence
///   intrinsics to LSC intrinsics if needed.
/// Effects of this are following:
/// 1) [a strict redundancy] removing unnecessary usage of the synchronization instructions can be
///    performed safely if none of such paths which meet at least one of the following order of visitation
///    (not crossed by a substitutive synchronization**):
///    - any write instruction, this synchronization instruction, any read instruction (RAW),
///    - any write instruction, this synchronization instruction, any write instruction (WAW),
///    - any write instruction, this synchronization instruction, any atomic instruction (RAW or WAW),
///    - any atomic instruction, this synchronization instruction, any read instruction (RAW),
///    - any atomic instruction, this synchronization instruction, any write instruction (WAR or WAW),
///    - any write instruction, this synchronization instruction, any return instruction (WAE),
///    - (only for barriers or dependent fences****) any read instruction, this synchronization instruction,
///      any write instruction (WAR),
///    - any read instruction, this synchronization instruction, any atomic instruction (RAW),
///    - (only for barriers) any atomic instruction, this synchronization instruction, any atomic instruction (RAW, WAW
///    or WAR).
///    * in this context read and write instructions contribute to the same type of memory.
///    ** [Only for the strict redundancy case] If a fence neighbors with barriers forwardly (there is no path between
///    this fence and a barrier which
///      crosses a substitute of this fence), the area of searching is increased to next synchronization area which can
///      be synchronized (there is a substitute of this fence). This area can be depicted in this parenthesis: this
///      fence -> ((barrier -> substitute*** -> barrier***))
///    *** this elements can be missing in the shader
///    **** dependent fences means that such fences neighbor with barriers forwardly (there is no path between these
///    fences and a barrier which
///      crosses a substitute of these fences).
/// 2) [a partial redundancy] removing the global memory association with untyped memory fences (for platforms with
/// dependent SLM fences)
///    can be performed safely if none of such paths which meet at least one of the following order of visitation
///    (not crossed by a substitutive synchronization):
///    - any buffer write instruction, this synchronization instruction, any buffer read instruction (RAW),
///    - any buffer write instruction, this synchronization instruction, any buffer write instruction (WAW),
///    - any buffer write instruction, this synchronization instruction, any atomic instruction (RAW or WAW),
///    - any atomic instruction, this synchronization instruction, any buffer read instruction (RAW).
/// 3) [a partial redundancy] disabling L1 cache invalidation (invalidation functionality) can be performed
///    safely if none of such paths which meet at least one of the following order of visitation (not crossed
///    by a substitutive synchronization):
///    - any buffer write instruction, this synchronization instruction, any buffer read instruction (RAW),
///    - any buffer write instruction, this synchronization instruction, any atomic instruction (RAW or WAW),
///    - any atomic instruction, this synchronization instruction, any buffer read instruction (WAR).
///    * L1 cache invalidation synchronizes read instructions.
/// Examples:
/// a) a required thread group barrier (write-barrier-read):
///      store float 5.000000e-01, float addrspace(3)* %store_address, align 4
///      call void @llvm.genx.GenISA.memoryfence(i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false)
///      call void @llvm.genx.GenISA.threadgroupbarrier()
///      load float 5.000000e-01, float addrspace(3)* %load_address, align 4
/// b) a required thread group barrier (read-barrier-write):
///      load float 5.000000e-01, float addrspace(3)* %load_address, align 4
///      call void @llvm.genx.GenISA.memoryfence(i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false)
///      call void @llvm.genx.GenISA.threadgroupbarrier()
///      store float 5.000000e-01, float addrspace(3)* %store_address, align 4
/// c) a required thread group barrier (write-barrier-write):
///      store float 5.000000e-01, float addrspace(3)* %store_address, align 4
///      call void @llvm.genx.GenISA.memoryfence(i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false)
///      call void @llvm.genx.GenISA.threadgroupbarrier()
///      store float 5.000000e-01, float addrspace(3)* %store_address1, align 4
/// d) an unneeded thread group barrier (read-barrier-read):
///      load float 5.000000e-01, float addrspace(3)* %load_address, align 4
///      call void @llvm.genx.GenISA.memoryfence(i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false) ;
///      an unneeded instruction call void @llvm.genx.GenISA.threadgroupbarrier() ; an unneeded instruction load
///      float 5.000000e-01, float addrspace(3)* %load_address1, align 4
/// e) an unneeded thread group barrier (write-barrier-no read/write):
///      store float 5.000000e-01, float addrspace(3)* %store_address, align 4
///      call void @llvm.genx.GenISA.memoryfence(i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false) ;
///      an unneeded instruction call void @llvm.genx.GenISA.threadgroupbarrier() ; an unneeded instruction
/// f) an unneeded thread group barrier (no read/write-barrier-write):
///      call void @llvm.genx.GenISA.memoryfence(i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false) ;
///      an unneeded instruction call void @llvm.genx.GenISA.threadgroupbarrier() ; an unneeded instruction store
///      float 5.000000e-01, float addrspace(3)* %store_address1, align 4
/// f) an unneeded thread group barrier (no read/write-barrier-no read/write):
///      call void @llvm.genx.GenISA.memoryfence(i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false) ;
///      an unneeded instruction call void @llvm.genx.GenISA.threadgroupbarrier() ; an unneeded instruction
///
/// Note: The current implementation analyzes with low granularity. It means there is no detection if
/// there exist synchronization dependencies between the same area of memory.
/// Moreover, unsynchronized read instructions can be such instructions which are used only after thread
/// group barriers.
/// Additionally, the pass assumes all possible call instructions are inlined.
///
/// Dictionary for some terms:
/// 1) synchronization area - this is such an area where all threads in the same workgroup can be run independently.
/// However the start point and
///    end point is the same for the bunch of threads. Such synchronization area can be drawn by an entry point, thread
///    group barriers and end of thread instructions.
/// 2) unsynchronized instruction - this is such a memory instruction which can finish after an imaginatively removed
/// synchronization point. Read
///    instructions must be finished before the synchronization point or first usage. Writes instructions are only
///    synchronized with the first encountered synchronization point. The missing synchronization point for such an
///    instruction can be destructive for the expected order of memory operations.
/// 3) boundary instruction - this is such a synchronization instruction which draws a border for searching.
/// 4) substitute instruction - this is such a synchronization instruction which work at least so extensively as the
/// reference instruction. It
///    means that the substitute instruction can replace the reference instruction without any regressive functional
///    changes in the shader.
class SynchronizationObjectCoalescing : public llvm::FunctionPass {
public:
  static char ID; ///< ID used by the llvm PassManager (the value is not important)

  SynchronizationObjectCoalescing();

  ////////////////////////////////////////////////////////////////////////
  virtual bool runOnFunction(llvm::Function &F);

  ////////////////////////////////////////////////////////////////////////
  virtual void getAnalysisUsage(llvm::AnalysisUsage &AU) const;

  ////////////////////////////////////////////////////////////////////////
  const llvm::DenseSet<llvm::Instruction *> &GetRedundantInstructions() const;

  ////////////////////////////////////////////////////////////////////////
  const llvm::DenseSet<llvm::Instruction *> &GetInvalidationFunctionalityRedundancies() const;

  ////////////////////////////////////////////////////////////////////////
  const llvm::DenseSet<llvm::Instruction *> &GetGlobalMemoryRedundancies() const;

#if _DEBUG
  ////////////////////////////////////////////////////////////////////////
  void print(llvm::raw_ostream &stream, bool onlyMemoryInstructionMask) const;

  ////////////////////////////////////////////////////////////////////////
  void dump(bool onlyMemoryInstructionMask = true) const;
#endif // _DEBUG

  ////////////////////////////////////////////////////////////////////////
  enum SynchronizationCaseMask : uint32_t {
    Empty = 0x0,
    ReadSyncWrite = 0x1,
    WriteSyncWrite = 0x2,
    AtomicSyncRead = 0x4,
    AtomicSyncWrite = 0x8,
    WriteSyncAtomic = 0x10,
    ReadSyncAtomic = 0x20,
    WriteSyncRead = 0x40,
    AtomicSyncAtomic = 0x80,
    WriteSyncRet = 0x100,
    ExtendedCacheControlSyncRet = 0x200
  };

private:
  static constexpr InstructionMask sc_MemoryWriteInstructionMask = static_cast<InstructionMask>(
      AtomicOperation | TypedWriteOperation | UrbWriteOperation | BufferWriteOperation | SharedMemoryWriteOperation);

  static constexpr InstructionMask sc_MemoryReadInstructionMask = static_cast<InstructionMask>(
      AtomicOperation | TypedReadOperation | OutputUrbReadOperation | BufferReadOperation | SharedMemoryReadOperation);

  static constexpr SynchronizationCaseMask sc_FullSynchronizationCaseMask =
      static_cast<SynchronizationCaseMask>(WriteSyncRead | WriteSyncWrite | AtomicSyncRead | AtomicSyncWrite |
                                           WriteSyncAtomic | ReadSyncAtomic | ReadSyncWrite | AtomicSyncAtomic);

  ////////////////////////////////////////////////////////////////////////
  void CreateSourceValueInst(std::vector<llvm::Instruction *> &pAtomicInstToBeSourced, llvm::Instruction *pFenceInst);

  ////////////////////////////////////////////////////////////////////////
  void EraseRedundantInst(llvm::Instruction *pInst);

  ////////////////////////////////////////////////////////////////////////
  void EraseRedundantL1CacheInvalidation(llvm::Instruction *pInst);

  ////////////////////////////////////////////////////////////////////////
  void EraseRedundantGlobalScope(llvm::Instruction *pInst);

  ////////////////////////////////////////////////////////////////////////
  bool ProcessFunction();

  ////////////////////////////////////////////////////////////////////////
  void InvalidateMembers();

  ////////////////////////////////////////////////////////////////////////
  void GatherInstructions();

  ////////////////////////////////////////////////////////////////////////
  bool FindRedundancies();

  ////////////////////////////////////////////////////////////////////////
  void GetVisibleMemoryInstructions(const llvm::Instruction *pSourceInst, bool forwardDirection,
                                    std::vector<const llvm::Instruction *> &boundaryInstructions,
                                    std::vector<const llvm::Instruction *> &memoryInstructions) const;

  ////////////////////////////////////////////////////////////////////////
  void GetAllUnsynchronizedMemoryInstructions(const llvm::Instruction *pSourceInst,
                                              std::vector<const llvm::Instruction *> &threadGroupBarriers,
                                              std::vector<const llvm::Instruction *> &memoryInstructions) const;

  ////////////////////////////////////////////////////////////////////////
  InstructionMask GetInstructionMask(const llvm::Instruction *pSourceInst, bool forwardDirection) const;

  ////////////////////////////////////////////////////////////////////////
  InstructionMask GetAtomicInstructionMaskFromPointer(const llvm::Instruction *pSourceInst) const;

  ////////////////////////////////////////////////////////////////////////
  InstructionMask GetUnsynchronizedForwardInstructionMask(const llvm::Instruction *pSourceInst) const;

  ////////////////////////////////////////////////////////////////////////
  bool IsRequiredForAtomicOperationsOrdering(const llvm::Instruction *pSourceInst,
                                             std::vector<llvm::Instruction *> &pAtomicInstToBeSourced,
                                             bool onlyGlobalAtomics = false) const;

  ////////////////////////////////////////////////////////////////////////
  InstructionMask GetInstructionMask(const std::vector<const llvm::Instruction *> &input) const;

  ////////////////////////////////////////////////////////////////////////
  InstructionMask GetDefaultWriteMemoryInstructionMask(const llvm::Instruction *pSourceInst) const;

  ////////////////////////////////////////////////////////////////////////
  InstructionMask GetDefaultMemoryInstructionMask(const llvm::Instruction *pSourceInst) const;

  ////////////////////////////////////////////////////////////////////////
  bool IsSubstituteInstruction(const llvm::Instruction *pEvaluatedInst, const llvm::Instruction *pReferenceInst) const;

  ////////////////////////////////////////////////////////////////////////
  InstructionMask GetInstructionMask(const llvm::Instruction *pInst) const;

  ////////////////////////////////////////////////////////////////////////
  SynchronizationCaseMask GetSynchronizationMask(InstructionMask localForwardMemoryInstructionMask,
                                                 InstructionMask localBackwardMemoryInstructionMask,
                                                 InstructionMask readBit, InstructionMask writeBit) const;

  ////////////////////////////////////////////////////////////////////////
  SynchronizationCaseMask GetStrictSynchronizationMask(llvm::Instruction *pInst) const;

  ////////////////////////////////////////////////////////////////////////
  SynchronizationCaseMask GetL1CacheInvalidatioSynchronizationMask() const;

  ////////////////////////////////////////////////////////////////////////
  SynchronizationCaseMask
  GetSynchronizationMaskForAllResources(InstructionMask localForwardMemoryInstructionMask,
                                        InstructionMask localBackwardMemoryInstructionMask) const;

  ////////////////////////////////////////////////////////////////////////
  static bool IsAtomicOperation(const llvm::Instruction *pInst);

  ////////////////////////////////////////////////////////////////////////
  static bool IsTypedReadOperation(const llvm::Instruction *pInst);

  ////////////////////////////////////////////////////////////////////////
  static bool IsTypedWriteOperation(const llvm::Instruction *pInst);

  ////////////////////////////////////////////////////////////////////////
  static bool IsOutputUrbReadOperation(const llvm::Instruction *pInst);

  ////////////////////////////////////////////////////////////////////////
  static bool IsUrbWriteOperation(const llvm::Instruction *pInst);

  ////////////////////////////////////////////////////////////////////////
  static bool IsBufferReadOperation(const llvm::Instruction *pInst);

  ////////////////////////////////////////////////////////////////////////
  static bool IsBufferWriteOperation(const llvm::Instruction *pInst);

  ////////////////////////////////////////////////////////////////////////
  static bool IsSharedMemoryReadOperation(const llvm::Instruction *pInst);

  ////////////////////////////////////////////////////////////////////////
  static bool IsSharedMemoryWriteOperation(const llvm::Instruction *pInst);

  ////////////////////////////////////////////////////////////////////////
  static bool IsExtendedCacheControlOperation(const llvm::Instruction *pInst);

  ////////////////////////////////////////////////////////////////////////
  static bool IsReturnOperation(const llvm::Instruction *pInst);

  ////////////////////////////////////////////////////////////////////////
  static bool IsThreadBarrierOperation(const llvm::Instruction *pInst);

  ////////////////////////////////////////////////////////////////////////
  static bool IsUntypedMemoryFenceOperation(const llvm::Instruction *pInst);

  ////////////////////////////////////////////////////////////////////////
  bool IsUntypedMemoryFenceOperationWithInvalidationFunctionality(const llvm::Instruction *pInst) const;

  ////////////////////////////////////////////////////////////////////////
  bool IsUntypedMemoryFenceOperationForSharedMemoryAccess(const llvm::Instruction *pInst) const;

  ////////////////////////////////////////////////////////////////////////
  bool IsUntypedMemoryFenceOperationForGlobalAccess(const llvm::Instruction *pInst) const;

  ////////////////////////////////////////////////////////////////////////
  bool IsUntypedMemoryLscFenceOperationForGlobalAccess(const llvm::Instruction *pInst) const;

  ////////////////////////////////////////////////////////////////////////
  bool IsSharedLscFenceOperation(const llvm::Instruction *pInst) const;

  ////////////////////////////////////////////////////////////////////////
  bool IsUrbLscFenceOperation(const llvm::Instruction *pInst) const;

  ////////////////////////////////////////////////////////////////////////
  static bool IsTypedMemoryFenceOperation(const llvm::Instruction *pInst);

  ////////////////////////////////////////////////////////////////////////
  bool IsTypedMemoryLscFenceOperation(const llvm::Instruction *pInst) const;

  ////////////////////////////////////////////////////////////////////////
  bool IsTypedMemoryFenceOperationWithInvalidationFunctionality(const llvm::Instruction *pInst) const;

  ////////////////////////////////////////////////////////////////////////
  static bool IsUrbFenceOperation(const llvm::Instruction *pInst);

  ////////////////////////////////////////////////////////////////////////
  static bool IsFenceOperation(const llvm::Instruction *pInst);

  ////////////////////////////////////////////////////////////////////////
  static bool IsGlobalWritableResource(llvm::Type *pResourePointerType);

  ////////////////////////////////////////////////////////////////////////
  static bool IsSharedMemoryResource(llvm::Type *pResourePointerType);

  std::vector<llvm::Instruction *> m_TypedMemoryFences;
  std::vector<llvm::Instruction *> m_UrbMemoryFences;
  std::vector<llvm::Instruction *> m_LscMemoryFences;
  std::vector<llvm::Instruction *> m_UntypedMemoryFences;
  std::vector<llvm::Instruction *> m_ThreadGroupBarriers;
  std::unordered_set<llvm::Instruction *> m_SourcedAtomicInstructions;

  // this variable holds a mapping from a basic block to its memory instructions ordered by their occurrences in it
  // (the initial index of line of this basic block - the number of instructions preceding an instruction it its basic
  // block)
  OrderedInstructionLookupTableT m_OrderedMemoryInstructionsInBasicBlockCache;

  // this variable holds a mapping from a basic block to the aggregated memory mask of its memory instructions
  BasicBlockMaskLookupTableT m_BasicBlockMemoryInstructionMaskCache;

  // this variable holds a mapping from a basic block to its fence instructions ordered by their occurrences in it
  // (the initial index of line of this basic block - the number of instructions preceding an instruction it its basic
  // block)
  OrderedInstructionLookupTableT m_OrderedFenceInstructionsInBasicBlockCache;

  // this variable holds a mapping from a basic block to its barrier instructions ordered by their occurrences in it
  // (the initial index of line of this basic block - the number of instructions preceding an instruction it its basic
  // block)
  OrderedInstructionLookupTableT m_OrderedBarrierInstructionsInBasicBlockCache;

  // this variable holds a mapping from a instruction to its initial number of line of its basic block
  InstIdxLookupTableT m_InstIdxLookupTable;

  // this variable holds a mapping from a instruction to its memory instruction mask
  InstMaskLookupTableT m_InstMaskLookupTable;

  llvm::Function *m_CurrentFunction = nullptr;
  const CodeGenContext *ctx = nullptr;
  bool m_HasIndependentSharedMemoryFenceFunctionality = false;
  bool m_HasTypedMemoryFenceFunctionality = false;
  bool m_HasUrbFenceFunctionality = false;
  InstructionMask m_GlobalMemoryInstructionMask = InstructionMask::None;
  ShaderType m_ShaderType = ShaderType::UNKNOWN;

#if _DEBUG
  std::vector<ExplanationEntry> m_ExplanationEntries;

  ////////////////////////////////////////////////////////////////////////
  void RegisterRedundancyExplanation(const llvm::Instruction *pInst, ExplanationEntry::Cause cause);
#endif // _DEBUG
};

inline constexpr SynchronizationObjectCoalescing::SynchronizationCaseMask
operator|(SynchronizationObjectCoalescing::SynchronizationCaseMask a,
          SynchronizationObjectCoalescing::SynchronizationCaseMask b) {
  return SynchronizationObjectCoalescing::SynchronizationCaseMask(uint32_t(a) | uint32_t(b));
}
inline constexpr SynchronizationObjectCoalescing::SynchronizationCaseMask &
operator|=(SynchronizationObjectCoalescing::SynchronizationCaseMask &a,
           SynchronizationObjectCoalescing::SynchronizationCaseMask b) {
  a = a | b;
  return a;
}

static inline bool IsLscFenceOperation(const Instruction *pInst);
static inline LSC_SFID GetLscMem(const Instruction *pInst);
static inline LSC_SCOPE GetLscScope(const Instruction *pInst);
static inline LSC_FENCE_OP GetLscFenceOp(const Instruction *pInst);

char SynchronizationObjectCoalescing::ID = 0;

////////////////////////////////////////////////////////////////////////////
SynchronizationObjectCoalescing::SynchronizationObjectCoalescing() : llvm::FunctionPass(ID) {
  initializeSynchronizationObjectCoalescingPass(*PassRegistry::getPassRegistry());
}

////////////////////////////////////////////////////////////////////////
bool SynchronizationObjectCoalescing::runOnFunction(llvm::Function &F) {
  m_CurrentFunction = &F;
  ctx = getAnalysis<CodeGenContextWrapper>().getCodeGenContext();
  const ModuleMetaData *const md = ctx->getModuleMetaData();
  m_HasIndependentSharedMemoryFenceFunctionality =
      !ctx->platform.hasSLMFence() ||
      (ctx->platform.hasSLMFence() && ctx->platform.hasIndependentSharedMemoryFenceFunctionality()) ||
      md->compOpt.EnableIndependentSharedMemoryFenceFunctionality ||
      IGC_IS_FLAG_ENABLED(EnableIndependentSharedMemoryFenceFunctionality);
  m_ShaderType = ctx->type;
  m_HasTypedMemoryFenceFunctionality = ctx->platform.hasLSC() && ctx->platform.LSCEnabled();
  m_HasUrbFenceFunctionality = ctx->platform.hasURBFence();
  bool isModified = ProcessFunction();
  return isModified;
}

////////////////////////////////////////////////////////////////////////
/// @brief Processes an analysis which results in pointing out redundancies
/// among synchronization instructions appearing in the analyzed function.
bool SynchronizationObjectCoalescing::ProcessFunction() {
  const CodeGenContext *const pContext = getAnalysis<CodeGenContextWrapper>().getCodeGenContext();
  bool isDisabled = IsStage1FastestCompile(pContext->m_CgFlag, pContext->m_StagingCtx) ||
                    IGC_GET_FLAG_VALUE(ForceFastestSIMD) ||
                    IGC_IS_FLAG_ENABLED(DisableSynchronizationObjectCoalescingPass);
  if (isDisabled) {
    return false;
  }

  InvalidateMembers();
  GatherInstructions();
  return FindRedundancies();
}

// Referenced from MemoryModelPass
static inline PHINode *FindDominatingPhi(DominatorTree &DT, Instruction *def, BasicBlock *postDominator) {
  IGC_ASSERT(def->getParent() != postDominator);
  IGC_ASSERT(!DT.dominates(def, postDominator));
  SmallPtrSet<PHINode *, 8> seen;
  SmallVector<User *, 8> worklist(def->users());
  while (!worklist.empty()) {
    PHINode *phi = dyn_cast<PHINode>(worklist.pop_back_val());
    if (phi == nullptr || seen.count(phi) > 0) {
      continue;
    }
    if (phi->getParent() == postDominator || DT.dominates(phi, postDominator)) {
      return phi;
    }
    seen.insert(phi);
  }
  return nullptr;
}

////////////////////////////////////////////////////////////////////////
/// @brief Fence Instruction responsible for only ordering of atomic Instructions
/// can be replaced with Source Value Intrinsic which will still maintain
/// the order of Instructions
void SynchronizationObjectCoalescing::CreateSourceValueInst(std::vector<llvm::Instruction *> &pAtomicInstToBeSourced,
                                                            llvm::Instruction *pFenceInst) {
  if (pAtomicInstToBeSourced.size() == 0) {
    return;
  }
  // reversing the list to source the atomic instructions in the order
  reverse(pAtomicInstToBeSourced.begin(), pAtomicInstToBeSourced.end());
  Function *funcPtr = GenISAIntrinsic::getDeclaration(pFenceInst->getModule(), GenISAIntrinsic::GenISA_source_value);
  BasicBlock *fenceBB = pFenceInst->getParent();

  Function *F = pAtomicInstToBeSourced[0]->getFunction();
  DominatorTree DT(*F);
  PostDominatorTree PDT(*F);
  LoopInfo LI(DT);

  std::unordered_set<llvm::Value *> m_SourcedValues;

  for (llvm::Instruction *atomicInst : pAtomicInstToBeSourced) {
    // Making sure that the Fence Inst is potentially reachable from the atomic Instruction.
    if (!isPotentiallyReachable(atomicInst, pFenceInst, nullptr, &DT, &LI)) {
      continue;
    }

    // In few shaders, the atomic instructions were already sourced before unification. Skip creating one in this case
    bool atomicSourced = false;
    for (User *U : atomicInst->users()) {
      if (GenIntrinsicInst *Inst = dyn_cast<GenIntrinsicInst>(U)) {
        // TODO: If dominates fail, then either move the source_value to BB that dominates fence and post dominates
        // atomic or delete them and let the below code create new ones. Requires further testing.
        if (Inst->getIntrinsicID() == GenISAIntrinsic::GenISA_source_value &&
            DT.dominates(Inst->getParent(), fenceBB)) {
          atomicSourced = true;
          break;
        }
      }
    }
    if (!atomicSourced) {
      BasicBlock *atomicBB = atomicInst->getParent();
      BasicBlock *fenceDominator = fenceBB;
      Instruction *insertPoint = atomicBB->getTerminator();
      Value *sourceVal = cast<GenIntrinsicInst>(atomicInst);

      // TODO: Determining Insert point can be improved which can postpone the source value intrinsic as long as
      // possible. Similar analysis is done in FindOptimalInsertPoints() in ApplyCacheControls.cpp

      // Check if fence Instruction BB post dominates atomic Instruction BB
      // Else find the BB that is a predecessor of fence BB and post dominates atomic BB.
      // If we don't find one, then the insert point is near the terminator of atomic BB
      while (fenceDominator && fenceDominator != atomicBB) {
        if (PDT.dominates(fenceDominator, atomicBB)) {
          // If fence instruction is in same BB, then use fence as insert point
          // Else use the terminator of fenceDominator as insert point
          insertPoint = fenceBB == fenceDominator ? pFenceInst : fenceDominator->getTerminator();
          // It's possible that the atomic instruction does not dominate
          // the post-dominator, find a PHI user of the atomic instruction
          // that dominates the post-dominator.
          if (!DT.dominates(atomicBB, fenceDominator)) {
            PHINode *phi = FindDominatingPhi(DT, atomicInst, fenceDominator);
            if (phi) {
              sourceVal = phi;
            } else {
              // Fallback to inserting the source value in the basic
              // block with the atomic instruction.
              insertPoint = atomicBB->getTerminator();
            }
          }
          break;
        }
        fenceDominator = fenceDominator->getSinglePredecessor();
      }
      // A Fence can synchronize a phi node which is coming from atomic instructions in different branches.
      // Avoid creating a duplicate source value intrinsic in this case.
      if (m_SourcedValues.find(sourceVal) == m_SourcedValues.end()) {
        m_SourcedValues.insert(sourceVal);
        // If Fence is present in same BB as atomic, then insert at Fence Instruction
        if (fenceBB == atomicBB) {
          insertPoint = pFenceInst;
        }

        IRBuilder<> builder(insertPoint);
        Type *sourceValType = sourceVal->getType();

        // Source value intrinsic accepts only i32.
        if (sourceValType->isIntegerTy()) {
          sourceVal = builder.CreateZExtOrTrunc(sourceVal, builder.getInt32Ty());
        } else if (sourceValType->isFloatingPointTy()) {
          if (sourceValType->isFloatTy()) {
            sourceVal = builder.CreateBitCast(sourceVal, builder.getInt32Ty());
          } else {
            sourceVal = builder.CreateFPToUI(sourceVal, builder.getInt32Ty());
          }
        } else {
          IGC_ASSERT_MESSAGE(0, "Unexpected type");
        }

        builder.CreateCall(funcPtr, {sourceVal});
      }
    }
    m_SourcedAtomicInstructions.insert(atomicInst);
  }
}

////////////////////////////////////////////////////////////////////////
void SynchronizationObjectCoalescing::EraseRedundantInst(llvm::Instruction *pInst) {
  bool isFence = IsFenceOperation(pInst);
  llvm::DenseMap<llvm::BasicBlock *, std::map<uint32_t, llvm::Instruction *>> &container =
      isFence ? m_OrderedFenceInstructionsInBasicBlockCache : m_OrderedBarrierInstructionsInBasicBlockCache;
  IGC_ASSERT(container.count(pInst->getParent()) > 0);
  std::map<uint32_t, llvm::Instruction *> &innerContainer = container.find(pInst->getParent())->second;
  auto index = m_InstIdxLookupTable.lookup(pInst);
  innerContainer.erase(index);
  if (innerContainer.empty()) {
    container.erase(pInst->getParent());
  }
  pInst->eraseFromParent();
}

////////////////////////////////////////////////////////////////////////
void SynchronizationObjectCoalescing::EraseRedundantL1CacheInvalidation(llvm::Instruction *pInst) {
  llvm::GenIntrinsicInst *pGenIntrinsicInst = llvm::cast<llvm::GenIntrinsicInst>(pInst);

  switch (pGenIntrinsicInst->getIntrinsicID()) {
  case llvm::GenISAIntrinsic::GenISA_memoryfence: {
    constexpr uint32_t L1CacheInvalidateArg = 6;
    pGenIntrinsicInst->setOperand(
        L1CacheInvalidateArg,
        llvm::ConstantInt::getFalse(pGenIntrinsicInst->getOperand(L1CacheInvalidateArg)->getType()));
    break;
  }
  case llvm::GenISAIntrinsic::GenISA_typedmemoryfence: {
    constexpr uint32_t L1CacheInvalidateArg = 0;
    pGenIntrinsicInst->setOperand(
        L1CacheInvalidateArg,
        llvm::ConstantInt::getFalse(pGenIntrinsicInst->getOperand(L1CacheInvalidateArg)->getType()));
    break;
  }
  case llvm::GenISAIntrinsic::GenISA_LSCFence: {
    [[maybe_unused]] LSC_SFID mem = GetLscMem(pGenIntrinsicInst);
    [[maybe_unused]] LSC_FENCE_OP op = GetLscFenceOp(pGenIntrinsicInst);
    IGC_ASSERT(mem == LSC_TGM || mem == LSC_UGM);
    IGC_ASSERT(op == LSC_FENCE_OP_INVALIDATE);
    constexpr uint32_t fenceOpArg = 2;
    llvm::Type *type = pGenIntrinsicInst->getOperand(fenceOpArg)->getType();
    pGenIntrinsicInst->setOperand(fenceOpArg, llvm::ConstantInt::get(type, LSC_FENCE_OP_NONE));
    break;
  }
  default:
    IGC_ASSERT(0);
    break;
  }
}

////////////////////////////////////////////////////////////////////////
void SynchronizationObjectCoalescing::EraseRedundantGlobalScope(llvm::Instruction *pInst) {
  llvm::GenIntrinsicInst *pGenIntrinsicInst = llvm::cast<llvm::GenIntrinsicInst>(pInst);

  switch (pGenIntrinsicInst->getIntrinsicID()) {
  case llvm::GenISAIntrinsic::GenISA_memoryfence: {
    constexpr uint32_t globalMemFenceArg = 5;
    pGenIntrinsicInst->setOperand(
        globalMemFenceArg, llvm::ConstantInt::getFalse(pGenIntrinsicInst->getOperand(globalMemFenceArg)->getType()));
    constexpr uint32_t scopeMemFenceArg = 8;
    pGenIntrinsicInst->setOperand(scopeMemFenceArg,
                                  llvm::ConstantInt::get(pGenIntrinsicInst->getOperand(scopeMemFenceArg)->getType(),
                                                         static_cast<uint32_t>(LSC_SCOPE::LSC_SCOPE_GROUP)));
    break;
  }
  case llvm::GenISAIntrinsic::GenISA_LSCFence: {
    constexpr uint32_t globalMemFenceArg = 0;
    pGenIntrinsicInst->setOperand(globalMemFenceArg,
                                  llvm::ConstantInt::get(pGenIntrinsicInst->getOperand(globalMemFenceArg)->getType(),
                                                         static_cast<uint32_t>(LSC_SFID::LSC_SLM)));
    constexpr uint32_t scopeMemFenceArg = 1;
    pGenIntrinsicInst->setOperand(scopeMemFenceArg,
                                  llvm::ConstantInt::get(pGenIntrinsicInst->getOperand(scopeMemFenceArg)->getType(),
                                                         static_cast<uint32_t>(LSC_SCOPE::LSC_SCOPE_GROUP)));
    constexpr uint32_t fenceOpArg = 2;
    pGenIntrinsicInst->setOperand(fenceOpArg,
                                  llvm::ConstantInt::get(pGenIntrinsicInst->getOperand(fenceOpArg)->getType(),
                                                         static_cast<uint32_t>(LSC_FENCE_OP_NONE)));
    break;
  }
  default:
    IGC_ASSERT(0);
    break;
  }
}

////////////////////////////////////////////////////////////////////////
/// @brief Identifies explicit redundancies among synchronization instructions.
bool SynchronizationObjectCoalescing::FindRedundancies() {
  bool isModified = false;
  // This lambda function identifies partial and strict redundancies among synchronization operations
  auto GatherRedundancies = [this, &isModified](std::vector<llvm::Instruction *> &synchronizationOperations) {
    constexpr bool forwardDirection = true;
    constexpr bool backwardDirection = false;

    for (llvm::Instruction *pInst : synchronizationOperations) {
      SyncInstMask instType = SyncInstMask::None;
      if (IsUntypedMemoryFenceOperationForGlobalAccess(pInst) ||
          IsUntypedMemoryLscFenceOperationForGlobalAccess(pInst)) {
        instType |= SyncInstMask::UntypedMemoryFence;
      }
      if (IsTypedMemoryFenceOperation(pInst) || IsTypedMemoryLscFenceOperation(pInst)) {
        instType |= SyncInstMask::TypedMemoryFence;
      }
      if (IsUntypedMemoryFenceOperationForSharedMemoryAccess(pInst) || IsSharedLscFenceOperation(pInst)) {
        instType |= SyncInstMask::SharedMemoryFence;
      }
      if (IsUrbFenceOperation(pInst) || IsUrbLscFenceOperation(pInst)) {
        instType |= SyncInstMask::UrbMemoryFence;
      }
      if (isBarrierIntrinsic(pInst)) {
        instType |= SyncInstMask::ThreadGroupBarrier;
      }
      IGC_ASSERT(instType != SyncInstMask::None);
      bool isDisabled =
          instType != SyncInstMask::None && (IGC_GET_FLAG_VALUE(DisableCoalescingSynchronizationObjectMask) &
                                             static_cast<DWORD>(instType)) == static_cast<DWORD>(instType);
      if (isDisabled) {
        continue;
      }

      if ((m_GlobalMemoryInstructionMask & GetDefaultWriteMemoryInstructionMask(pInst)) == 0) {
#if _DEBUG
        RegisterRedundancyExplanation(pInst, ExplanationEntry::FastStrictRedundancy);
#endif // _DEBUG
        EraseRedundantInst(pInst);
        isModified = true;
        continue;
      }
      InstructionMask localForwardMemoryInstructionMask = InstructionMask::None;
      InstructionMask localBackwardMemoryInstructionMask = InstructionMask::None;

      auto SetLocalMemoryInstructionMask = [this, pInst, forwardDirection, backwardDirection,
                                            &localForwardMemoryInstructionMask, &localBackwardMemoryInstructionMask]() {
        localForwardMemoryInstructionMask =
            GetDefaultMemoryInstructionMask(pInst) & GetInstructionMask(pInst, forwardDirection);
        localBackwardMemoryInstructionMask =
            GetDefaultMemoryInstructionMask(pInst) & GetInstructionMask(pInst, backwardDirection);
      };
      SetLocalMemoryInstructionMask();

      // Partial redundancies:
      // 1) Applies to untyped and typed memory fences with invalidation
      //    functionality. Removing this functionality is possible if there
      //    is no read instruction which can depend on this synchronization
      //    instruction. Such a reduction can result in removing the whole
      //    instruction because of changing classification of the instruction.
      // 2) Applies to untyped memory fences interacting with global memory.
      //    Removing this association is possible if there is no global
      //    write instruction which can depend on this synchronization
      //    instruction. Such a reduction can result in removing the whole
      //    instruction because of changing classification of the instruction.

      // identify partial redundancies for untyped memory fences
      if (IsUntypedMemoryFenceOperationForGlobalAccess(pInst) ||
          IsUntypedMemoryLscFenceOperationForGlobalAccess(pInst)) {
        if (IsUntypedMemoryFenceOperationWithInvalidationFunctionality(pInst)) {
          // buffer access
          SynchronizationCaseMask syncCaseMask =
              GetSynchronizationMask(localForwardMemoryInstructionMask, localBackwardMemoryInstructionMask,
                                     BufferReadOperation, BufferWriteOperation);
          if (!m_HasTypedMemoryFenceFunctionality) {
            syncCaseMask |=
                GetSynchronizationMask(localForwardMemoryInstructionMask, localBackwardMemoryInstructionMask,
                                       TypedReadOperation, TypedWriteOperation);
          }
          if (!m_HasUrbFenceFunctionality) {
            syncCaseMask |=
                GetSynchronizationMask(localForwardMemoryInstructionMask, localBackwardMemoryInstructionMask,
                                       OutputUrbReadOperation, UrbWriteOperation);
          }
          syncCaseMask |= GetSynchronizationMask(localForwardMemoryInstructionMask, localBackwardMemoryInstructionMask,
                                                 OutputUrbReadOperation, UrbWriteOperation);

          bool isObligatory = (syncCaseMask & GetL1CacheInvalidatioSynchronizationMask()) != 0;
          if (!isObligatory) {
#if _DEBUG
            RegisterRedundancyExplanation(pInst, ExplanationEntry::L1CacheInvalidationRedundancy);
#endif // _DEBUG
            EraseRedundantL1CacheInvalidation(pInst);
            isModified = true;
            SetLocalMemoryInstructionMask();
          }
        }

        if (!m_HasIndependentSharedMemoryFenceFunctionality) {
          // buffer access
          SynchronizationCaseMask syncCaseMask =
              GetSynchronizationMask(localForwardMemoryInstructionMask, localBackwardMemoryInstructionMask,
                                     BufferReadOperation, BufferWriteOperation);
          syncCaseMask = static_cast<SynchronizationCaseMask>(
              syncCaseMask |
              GetSynchronizationMask(localForwardMemoryInstructionMask, localBackwardMemoryInstructionMask, None,
                                     ExtendedCacheControlOperation));
          if (!m_HasTypedMemoryFenceFunctionality) {
            syncCaseMask |=
                GetSynchronizationMask(localForwardMemoryInstructionMask, localBackwardMemoryInstructionMask,
                                       TypedReadOperation, TypedWriteOperation);
          }
          if (!m_HasUrbFenceFunctionality) {
            syncCaseMask |=
                GetSynchronizationMask(localForwardMemoryInstructionMask, localBackwardMemoryInstructionMask,
                                       OutputUrbReadOperation, UrbWriteOperation);
          }
          SynchronizationCaseMask referenceSyncCaseMask = GetStrictSynchronizationMask(pInst);
          bool isObligatory = (syncCaseMask & referenceSyncCaseMask) != 0;

          std::vector<llvm::Instruction *> atomicInstToBeSourced;
          if (!isObligatory) {
            isObligatory =
                IsRequiredForAtomicOperationsOrdering(pInst, atomicInstToBeSourced, true /*onlyGlobalAtomics*/);
          }

          bool verifyUnsynchronizedInstructions = IsFenceOperation(pInst);
          verifyUnsynchronizedInstructions &= (!isObligatory || syncCaseMask == ReadSyncWrite);

          if (verifyUnsynchronizedInstructions) {
            InstructionMask unsynchronizedForwardMemoryInstructionMask = GetUnsynchronizedForwardInstructionMask(pInst);
            SynchronizationCaseMask syncCaseMaskForUnsynchronizedInstructions =
                GetSynchronizationMask(unsynchronizedForwardMemoryInstructionMask, localBackwardMemoryInstructionMask,
                                       BufferReadOperation, BufferWriteOperation);
            if (!m_HasTypedMemoryFenceFunctionality) {
              syncCaseMaskForUnsynchronizedInstructions |=
                  GetSynchronizationMask(unsynchronizedForwardMemoryInstructionMask, localBackwardMemoryInstructionMask,
                                         TypedReadOperation, TypedWriteOperation);
            }
            if (!m_HasUrbFenceFunctionality) {
              syncCaseMaskForUnsynchronizedInstructions |=
                  GetSynchronizationMask(unsynchronizedForwardMemoryInstructionMask, localBackwardMemoryInstructionMask,
                                         OutputUrbReadOperation, UrbWriteOperation);
            }
            syncCaseMask = syncCaseMaskForUnsynchronizedInstructions;
            isObligatory = (syncCaseMask & referenceSyncCaseMask) != 0;
          }

          if (!isObligatory) {
#if _DEBUG
            RegisterRedundancyExplanation(pInst, ExplanationEntry::GlobalMemoryRedundancy);
#endif // _DEBUG
            if (ctx->platform.enableReplaceAtomicFenceWithSourceValue()) {
              CreateSourceValueInst(atomicInstToBeSourced, const_cast<Instruction *>(pInst));
            }
            EraseRedundantGlobalScope(pInst);
            isModified = true;
            SetLocalMemoryInstructionMask();
          }
        }
      }

      // identify partial redundancies for typed memory fences
      if (IsTypedMemoryFenceOperationWithInvalidationFunctionality(pInst)) {
        // typed access
        SynchronizationCaseMask syncCaseMask =
            GetSynchronizationMask(localForwardMemoryInstructionMask, localBackwardMemoryInstructionMask,
                                   TypedReadOperation, TypedWriteOperation);
        bool isObligatory = (syncCaseMask & GetL1CacheInvalidatioSynchronizationMask()) != 0;

        if (!isObligatory) {
#if _DEBUG
          RegisterRedundancyExplanation(pInst, ExplanationEntry::L1CacheInvalidationRedundancy);
#endif // _DEBUG
          EraseRedundantL1CacheInvalidation(pInst);
          isModified = true;
          SetLocalMemoryInstructionMask();
        }
      }

      if (IsLscFenceOperation(pInst)) {
        LSC_SFID mem = GetLscMem(pInst);
        LSC_FENCE_OP op = GetLscFenceOp(pInst);
        if (op == LSC_FENCE_OP_INVALIDATE && (mem == LSC_UGM || mem == LSC_TGM)) {
          SynchronizationCaseMask syncCaseMask =
              GetSynchronizationMask(localForwardMemoryInstructionMask, localBackwardMemoryInstructionMask,
                                     (mem == LSC_TGM ? TypedReadOperation : BufferReadOperation),
                                     (mem == LSC_TGM ? TypedWriteOperation : BufferWriteOperation));
          bool isObligatory = (syncCaseMask & GetL1CacheInvalidatioSynchronizationMask()) != 0;
          if (!isObligatory) {
#if _DEBUG
            RegisterRedundancyExplanation(pInst, ExplanationEntry::L1CacheInvalidationRedundancy);
#endif // _DEBUG
            EraseRedundantL1CacheInvalidation(pInst);
            isModified = true;
            SetLocalMemoryInstructionMask();
          }
        }
      }

      // Strict Redundancies
      /// It means removing such a set of instructions safely if none of such paths which meet at least one of the
      /// following order of visitation (not crossed by a substitutive synchronization):
      /// - any write instruction, this synchronization instruction, any read instruction,
      /// - any write instruction, this synchronization instruction, any write instruction,
      /// - any write instruction, this synchronization instruction, any atomic instruction,
      /// - any atomic instruction, this synchronization instruction, any read instruction,
      /// - any atomic instruction, this synchronization instruction, any write instruction,
      /// - (only for barriers or dependent fences) any read instruction, this synchronization instruction, any write
      /// instruction,
      /// - (only for barriers) any atomic instruction, this synchronization instruction, any atomic instruction.
      /// * in this context read and write instructions contributes to the same type of memory.

      // identify strict redundancies

      SynchronizationCaseMask syncCaseMask =
          GetSynchronizationMaskForAllResources(localForwardMemoryInstructionMask, localBackwardMemoryInstructionMask);
      SynchronizationCaseMask referenceSyncCaseMask = GetStrictSynchronizationMask(pInst);
      bool isObligatory = (syncCaseMask & referenceSyncCaseMask) != 0;

      std::vector<llvm::Instruction *> atomicInstToBeSourced;
      if (!isObligatory) {
        isObligatory = IsRequiredForAtomicOperationsOrdering(pInst, atomicInstToBeSourced);
      }

      bool verifyUnsynchronizedInstructions = IsFenceOperation(pInst);
      verifyUnsynchronizedInstructions &= (!isObligatory || syncCaseMask == ReadSyncWrite);

      if (verifyUnsynchronizedInstructions) {
        InstructionMask unsynchronizedForwardMemoryInstructionMask = GetUnsynchronizedForwardInstructionMask(pInst);
        SynchronizationCaseMask syncCaseMaskForUnsynchronizedInstructions = GetSynchronizationMaskForAllResources(
            unsynchronizedForwardMemoryInstructionMask, localBackwardMemoryInstructionMask);
        syncCaseMask = syncCaseMaskForUnsynchronizedInstructions;
        isObligatory = (syncCaseMask & referenceSyncCaseMask) != 0;
      }

      if (!isObligatory) {
#if _DEBUG
        RegisterRedundancyExplanation(pInst, ExplanationEntry::StrictRedundancy);
#endif // _DEBUG
        if (ctx->platform.enableReplaceAtomicFenceWithSourceValue()) {
          CreateSourceValueInst(atomicInstToBeSourced, const_cast<Instruction *>(pInst));
        }
        EraseRedundantInst(pInst);
        isModified = true;
      }
    }
  };

  GatherRedundancies(m_ThreadGroupBarriers);
  GatherRedundancies(m_UntypedMemoryFences);
  GatherRedundancies(m_TypedMemoryFences);
  GatherRedundancies(m_UrbMemoryFences);
  GatherRedundancies(m_LscMemoryFences);
  return isModified;
}

////////////////////////////////////////////////////////////////////////
/// @brief Provides write memory instructions mask which are synchronized
/// by the instruction.
InstructionMask
SynchronizationObjectCoalescing::GetDefaultWriteMemoryInstructionMask(const llvm::Instruction *pSourceInst) const {
  InstructionMask result = InstructionMask::None;
  if (IsUntypedMemoryFenceOperation(pSourceInst)) {
    [[maybe_unused]] const bool hasInvalidationFunctionality =
        IsUntypedMemoryFenceOperationWithInvalidationFunctionality(pSourceInst);
    const bool hasSharedMemoryInfluence = IsUntypedMemoryFenceOperationForSharedMemoryAccess(pSourceInst);
    const bool hasGlobalInfluence = IsUntypedMemoryFenceOperationForGlobalAccess(pSourceInst);
    if (hasSharedMemoryInfluence) {
      result = static_cast<InstructionMask>(result | SharedMemoryWriteOperation);
      if (m_ShaderType == ShaderType::HULL_SHADER && !m_HasUrbFenceFunctionality &&
          m_HasIndependentSharedMemoryFenceFunctionality) {
        // This is for ICL+ but should not harm on pre-ICL
        result = static_cast<InstructionMask>(result | UrbWriteOperation);
      }
    }
    if (hasGlobalInfluence) {
      result = static_cast<InstructionMask>(result | BufferWriteOperation);
      if (!m_HasTypedMemoryFenceFunctionality) {
        result = static_cast<InstructionMask>(result | TypedWriteOperation);
      }
      if (!m_HasUrbFenceFunctionality) {
        // This is for ICL+ but should not harm on pre-ICL
        result = static_cast<InstructionMask>(result | UrbWriteOperation);
      }
    }
  } else if (IsLscFenceOperation(pSourceInst)) {
    IGC_ASSERT(m_HasUrbFenceFunctionality);
    switch (GetLscMem(pSourceInst)) {
    case LSC_UGM:  // .ugm
    case LSC_UGML: // .ugml
      result |= ExtendedCacheControlOperation | BufferWriteOperation;
      if (!m_HasIndependentSharedMemoryFenceFunctionality) {
        result |= SharedMemoryWriteOperation;
      }
      break;
    case LSC_TGM: // .tgm
      result |= TypedWriteOperation;
      break;
    case LSC_SLM: // .slm
      result |= SharedMemoryWriteOperation;
      break;
    case LSC_URB: // .urb
      result |= UrbWriteOperation;
      break;
    }
  } else if (IsTypedMemoryFenceOperation(pSourceInst)) {
    [[maybe_unused]] const bool hasInvalidationFunctionality =
        IsTypedMemoryFenceOperationWithInvalidationFunctionality(pSourceInst);
    result = static_cast<InstructionMask>(result | TypedWriteOperation);
  } else if (IsUrbFenceOperation(pSourceInst)) {
    result = static_cast<InstructionMask>(result | UrbWriteOperation);
  } else if (IsThreadBarrierOperation(pSourceInst)) {
    result = static_cast<InstructionMask>(sc_MemoryWriteInstructionMask);
  } else {
    IGC_ASSERT(0);
  }
  return result;
}

////////////////////////////////////////////////////////////////////////
/// @brief Provides default memory instruction mask which is used for
/// graph searching.
InstructionMask
SynchronizationObjectCoalescing::GetDefaultMemoryInstructionMask(const llvm::Instruction *pSourceInst) const {
  // All the rules stems from assumptions in the main comment of this analysis (the paragraph about a strict
  // redundancy).
  InstructionMask result = InstructionMask::None;
  if (IsUntypedMemoryFenceOperation(pSourceInst)) {
    const bool hasSharedMemoryInfluence = IsUntypedMemoryFenceOperationForSharedMemoryAccess(pSourceInst);
    const bool hasGlobalInfluence = IsUntypedMemoryFenceOperationForGlobalAccess(pSourceInst);
    result = static_cast<InstructionMask>(result | AtomicOperation);

    if (hasSharedMemoryInfluence) {
      result = static_cast<InstructionMask>(result | SharedMemoryWriteOperation | SharedMemoryReadOperation);
      if (m_ShaderType == ShaderType::HULL_SHADER && !m_HasUrbFenceFunctionality &&
          m_HasIndependentSharedMemoryFenceFunctionality) {
        // This is for ICL+ but should not harm on pre-ICL
        result = static_cast<InstructionMask>(result | UrbWriteOperation | OutputUrbReadOperation);
      }
    }

    if (hasGlobalInfluence) {
      result = static_cast<InstructionMask>(result | BufferWriteOperation | BufferReadOperation);
      if (!m_HasTypedMemoryFenceFunctionality) {
        result = static_cast<InstructionMask>(result | TypedWriteOperation | TypedReadOperation);
      }
      if (!m_HasUrbFenceFunctionality) {
        // This is for ICL+ but should not harm on pre-ICL
        result = static_cast<InstructionMask>(result | UrbWriteOperation | OutputUrbReadOperation);
      }
    }
  } else if (IsLscFenceOperation(pSourceInst)) {
    IGC_ASSERT(m_HasUrbFenceFunctionality);
    switch (GetLscMem(pSourceInst)) {
    case LSC_UGM:  // .ugm
    case LSC_UGML: // .ugml
      result |= ExtendedCacheControlOperation | AtomicOperation | BufferWriteOperation | BufferReadOperation;
      if (!m_HasIndependentSharedMemoryFenceFunctionality) {
        result |= SharedMemoryWriteOperation | SharedMemoryReadOperation;
      }
      break;
    case LSC_TGM: // .tgm
      result |= AtomicOperation | TypedWriteOperation | TypedReadOperation;
      break;
    case LSC_SLM: // .slm
      result |= AtomicOperation | SharedMemoryWriteOperation | SharedMemoryReadOperation;
      break;
    case LSC_URB: // .urb
      result |= UrbWriteOperation | OutputUrbReadOperation;
      break;
    }
  } else if (IsTypedMemoryFenceOperation(pSourceInst)) {
    [[maybe_unused]] const bool hasInvalidationFunctionality =
        IsTypedMemoryFenceOperationWithInvalidationFunctionality(pSourceInst);
    result = static_cast<InstructionMask>(result | AtomicOperation);

    result = static_cast<InstructionMask>(result | TypedWriteOperation | TypedReadOperation);
  } else if (IsUrbFenceOperation(pSourceInst)) {
    result = static_cast<InstructionMask>(result | OutputUrbReadOperation | UrbWriteOperation | AtomicOperation);
  } else if (IsThreadBarrierOperation(pSourceInst)) {
    result = static_cast<InstructionMask>(sc_MemoryReadInstructionMask | sc_MemoryWriteInstructionMask);
  } else {
    IGC_ASSERT(0);
  }

  constexpr InstructionMask maskToIncludeEOT =
      ExtendedCacheControlOperation | SharedMemoryWriteOperation | BufferWriteOperation | TypedWriteOperation;
  if (static_cast<uint32_t>(result & maskToIncludeEOT) != 0) {

    result = static_cast<InstructionMask>(result | EndOfThreadOperation);
  }

  return result;
}

////////////////////////////////////////////////////////////////////////
bool SynchronizationObjectCoalescing::IsExtendedCacheControlOperation(const llvm::Instruction *pInst) {
  if (llvm::isa<llvm::GenIntrinsicInst>(pInst)) {
    const llvm::GenIntrinsicInst *pGenIntrinsicInst = llvm::cast<llvm::GenIntrinsicInst>(pInst);

    switch (pGenIntrinsicInst->getIntrinsicID()) {
    case llvm::GenISAIntrinsic::GenISA_ExtendedCacheControl:
      return true;
    default:
      break;
    }
  }

  return false;
}

////////////////////////////////////////////////////////////////////////
bool SynchronizationObjectCoalescing::IsReturnOperation(const llvm::Instruction *pInst) {
  return llvm::isa<llvm::ReturnInst>(pInst);
}

//////////////////////////////////////////////////////////////////////////
template <class... Ts> struct overloaded : Ts... {
  template <typename T> overloaded<Ts...> &operator=(const T &lambda) {
    ((static_cast<Ts &>(*this) = lambda), ...);
    return *this;
  }

  using Ts::operator()...;
};
template <class... Ts> overloaded(Ts...) -> overloaded<Ts...>;

//////////////////////////////////////////////////////////////////////////
using ForwardIterationCallbackT =
    std::function<void(llvm::BasicBlock::const_iterator, llvm::BasicBlock::const_iterator)>;

//////////////////////////////////////////////////////////////////////////
using BackwardIterationCallbackT =
    std::function<void(llvm::BasicBlock::const_reverse_iterator, llvm::BasicBlock::const_reverse_iterator)>;

//////////////////////////////////////////////////////////////////////////
using IterationCallbackT = overloaded<ForwardIterationCallbackT, BackwardIterationCallbackT>;

//////////////////////////////////////////////////////////////////////////
using ForwardBoundaryCallbackT =
    std::function<llvm::BasicBlock::const_iterator(llvm::BasicBlock::const_iterator, llvm::BasicBlock::const_iterator)>;

//////////////////////////////////////////////////////////////////////////
using BackwardBoundaryCallbackT = std::function<llvm::BasicBlock::const_reverse_iterator(
    llvm::BasicBlock::const_reverse_iterator, llvm::BasicBlock::const_reverse_iterator)>;

//////////////////////////////////////////////////////////////////////////
using BoundaryCallbackT = overloaded<ForwardBoundaryCallbackT, BackwardBoundaryCallbackT>;

//////////////////////////////////////////////////////////////////////////
using ForwardProcessCallbackT = std::function<bool(llvm::BasicBlock::const_iterator, llvm::BasicBlock::const_iterator)>;

//////////////////////////////////////////////////////////////////////////
using BackwardProcessCallbackT =
    std::function<bool(llvm::BasicBlock::const_reverse_iterator, llvm::BasicBlock::const_reverse_iterator)>;

//////////////////////////////////////////////////////////////////////////
using ProcessCallbackT = overloaded<ForwardProcessCallbackT, BackwardProcessCallbackT>;

////////////////////////////////////////////////////////////////////////
/// @brief Scan over basic blocks using a custom function which
/// analyzes instructions between the beginning and ending iterator
/// and decides if the scan should be continued. The direction of the
/// scanning process is defined by the kind of used iterators.
/// @param workList holds a collection with the beginning points of searching
/// @param visitedBasicBlocks holds a collection of fully visited basic blocks
/// @param ProcessInstructions holds a function returning information
/// based on the beginning and ending iterators of the currently scanned
/// basic blocks if adjacent basic blocks also should be scanned
/// @tparam BasicBlockterator a basic block iterator type indicating the scan direction
template <typename BasicBlockterator>
static void SearchInstructions(std::list<BasicBlockterator> &workList,
                               llvm::DenseSet<const llvm::BasicBlock *> &visitedBasicBlocks,
                               std::function<bool(BasicBlockterator, BasicBlockterator)> &ProcessInstructions) {
  auto GetBeginIt = [](const llvm::BasicBlock *pBasicBlock) -> BasicBlockterator {
    constexpr bool isForwardDirection = std::is_same_v<BasicBlockterator, llvm::BasicBlock::const_iterator>;
    if constexpr (isForwardDirection) {
      return pBasicBlock->begin();
    } else {
      return pBasicBlock->rbegin();
    }
  };

  auto GetEndIt = [](const llvm::BasicBlock *pBasicBlock) -> BasicBlockterator {
    constexpr bool isForwardDirection = std::is_same_v<BasicBlockterator, llvm::BasicBlock::const_iterator>;
    if constexpr (isForwardDirection) {
      return pBasicBlock->end();
    } else {
      return pBasicBlock->rend();
    }
  };

  auto GetSuccessors = [](const llvm::BasicBlock *pBasicBlock) {
    constexpr bool isForwardDirection = std::is_same_v<BasicBlockterator, llvm::BasicBlock::const_iterator>;
    if constexpr (isForwardDirection) {
      return llvm::successors(pBasicBlock);
    } else {
      return llvm::predecessors(pBasicBlock);
    }
  };

  for (const auto &it : workList) {
    const llvm::BasicBlock *pCurrentBasicBlock = it->getParent();
    // use the iterator only if it wasn't visited or restricted
    auto bbIt = visitedBasicBlocks.find(pCurrentBasicBlock);
    if (bbIt != visitedBasicBlocks.end()) {
      continue;
    } else if (GetBeginIt(pCurrentBasicBlock) == it) {
      visitedBasicBlocks.insert(pCurrentBasicBlock);
    }
    BasicBlockterator end = GetEndIt(pCurrentBasicBlock);
    if (ProcessInstructions(it, end)) {
      for (const llvm::BasicBlock *pSuccessor : GetSuccessors(pCurrentBasicBlock)) {
        if (visitedBasicBlocks.find(pSuccessor) == visitedBasicBlocks.end()) {
          workList.push_back(GetBeginIt(pSuccessor));
        }
      }
    }
  }
}

////////////////////////////////////////////////////////////////////////
/// @brief Scan over basic blocks using custom functions which
/// one of them analyzes instructions between the beginning and ending iterator
/// and second of them decides if the scan should be continued. The direction of the
/// scanning process is defined by the kind of used iterators.
/// @param workList holds a collection with the beginning points of searching
/// @param visitedBasicBlocks holds a collection of fully visited basic blocks
/// @param IterateOverMemoryInsts holds a function analyzing the content
/// between the beginning iterator and the boundary iterator.
/// @param GetBoundaryInst holds a function returning information
/// about the boundary of scanning if it is present in the analyzed basic
/// block, otherwise, it returns the ending iterator and it means the
/// scan process is proceeded.
/// based on the beginning and ending iterators of the currently scanned
/// basic blocks if adjacent basic blocks also should be scanned
/// @tparam BasicBlockterator a basic block iterator type indicating the scan direction
template <typename BasicBlockterator>
static void
SearchInstructions(std::list<BasicBlockterator> &workList, llvm::DenseSet<const llvm::BasicBlock *> &visitedBasicBlocks,
                   std::function<void(BasicBlockterator, BasicBlockterator)> &IterateOverMemoryInsts,
                   std::function<BasicBlockterator(BasicBlockterator, BasicBlockterator)> &GetBoundaryInst) {
  std::function<bool(BasicBlockterator, BasicBlockterator)> ProcessInstructions{};
  ProcessInstructions = [&GetBoundaryInst, &IterateOverMemoryInsts](auto it, auto end) {
    auto beg = it;
    it = GetBoundaryInst(beg, end);
    IterateOverMemoryInsts(beg, it);
    return it == end;
  };
  SearchInstructions(workList, visitedBasicBlocks, ProcessInstructions);
}

////////////////////////////////////////////////////////////////////////
/// @brief Returns the following information as a pair:
/// - the map range of the container determined by the basic block iterators,
/// - the answer to the question whether the map range covers the entire container.
/// @param container holds ordered filtered instructions inside the analyzed basic block
/// @param lookupTable holds a mapping from a instruction to its number of line of its basic block
/// @param it indicates the beginning instruction of the map range
/// @param end indicates the ending instruction of the map range
/// @tparam BasicBlockterator a basic block iterator type indicating the scan direction
template <typename BasicBlockterator>
auto GetMapRange(const InstLookupTableInBasicBlockT &container, const InstIdxLookupTableT &lookupTable,
                 BasicBlockterator it, BasicBlockterator end) {
  IGC_ASSERT(it != end);
  auto GetEndIt = [](const llvm::BasicBlock *pBasicBlock) -> BasicBlockterator {
    constexpr bool isForwardDirection = std::is_same_v<BasicBlockterator, llvm::BasicBlock::const_iterator>;
    if constexpr (isForwardDirection) {
      return pBasicBlock->end();
    } else {
      return pBasicBlock->rend();
    }
  };
  constexpr bool isForwardDirection = std::is_same_v<decltype(it), llvm::BasicBlock::const_iterator>;
  auto currBeg = isForwardDirection ? it : end;
  const llvm::BasicBlock *pBB = currBeg->getParent();
  auto lowIt = currBeg != GetEndIt(pBB) ? lookupTable.find(&(*(currBeg))) : lookupTable.end();
  auto lowBound = lowIt == lookupTable.end() ? container.begin() : container.lower_bound(lowIt->second);
  auto currEnd = !isForwardDirection ? it : end;
  auto highIt = currEnd != GetEndIt(pBB) ? lookupTable.find(&(*(currEnd))) : lookupTable.end();
  auto highBound = highIt == lookupTable.end() ? container.end() : container.upper_bound(highIt->second);
  auto insts = llvm::make_range(lowBound, highBound);
  bool allEntries = lowBound == container.begin() && highBound == container.end();
  if constexpr (isForwardDirection) {
    return std::make_pair(insts, allEntries);
  } else {
    return std::make_pair(llvm::reverse(insts), allEntries);
  }
}

////////////////////////////////////////////////////////////////////////
/// @brief Returns the boundary callback to determine the boundary instruction
/// in the basic block.
/// @param container holds ordered filtered instructions inside the analyzed basic block
/// @param lookupTable holds a mapping from a instruction to its number of line of its basic block
/// @param IsBoundaryInst a callback to check if the instruction is a boundary instruction
/// @param boundaryInstructions the container of the visited boundary instructions filled by the output callback
auto GetBoundaryFunc(const OrderedInstructionLookupTableT &container, const InstIdxLookupTableT &lookupTable,
                     std::function<bool(const llvm::Instruction *)> &IsBoundaryInst,
                     std::vector<const llvm::Instruction *> &boundaryInstructions) {
  BoundaryCallbackT GetBoundary{};
  GetBoundary = [&container, &IsBoundaryInst, &boundaryInstructions, &lookupTable](auto beg,
                                                                                   auto end) -> decltype(beg) {
    if (beg == end) {
      return end;
    }
    constexpr bool isForwardDirection = std::is_same_v<decltype(beg), llvm::BasicBlock::const_iterator>;
    const llvm::BasicBlock *pCurrentBasicBlock = beg->getParent();
    const auto &currBBInstsIt = container.find(pCurrentBasicBlock);
    if (currBBInstsIt == container.end() || currBBInstsIt->second.empty()) {
      return end;
    }
    auto currBBInstsRange = GetMapRange(currBBInstsIt->second, lookupTable, beg, end);
    for (auto [index, pInst] : currBBInstsRange.first) {
      if (IsBoundaryInst(pInst)) {
        boundaryInstructions.push_back(pInst);
        if constexpr (isForwardDirection) {
          return pInst->getIterator();
        } else {
          return pInst->getReverseIterator();
        }
      }
    }
    return end;
  };
  return GetBoundary;
}

////////////////////////////////////////////////////////////////////////
/// @brief Returns the boundary callback to determine a boundary instruction
/// in the basic block.
/// @param container holds ordered filtered instructions inside the analyzed basic block
/// @param lookupTable holds a mapping from a instruction to its number of line of its basic block
auto GetBoundaryFunc(const OrderedInstructionLookupTableT &container, const InstIdxLookupTableT &lookupTable) {
  BoundaryCallbackT GetBoundary{};
  GetBoundary = [&](auto beg, auto end) -> decltype(beg) {
    if (beg == end) {
      return end;
    }
    constexpr bool isForwardDirection = std::is_same_v<decltype(beg), llvm::BasicBlock::const_iterator>;
    const llvm::BasicBlock *pCurrentBasicBlock = beg->getParent();
    const auto &currBBInstsIt = container.find(pCurrentBasicBlock);
    if (currBBInstsIt == container.end() || currBBInstsIt->second.empty()) {
      return end;
    }
    auto currBBInstsRange = GetMapRange(currBBInstsIt->second, lookupTable, beg, end);
    for (auto [index, pInst] : currBBInstsRange.first) {
      if constexpr (isForwardDirection) {
        return pInst->getIterator();
      } else {
        return pInst->getReverseIterator();
      }
    }
    return end;
  };
  return GetBoundary;
}

////////////////////////////////////////////////////////////////////////
/// @brief Returns the boundary callback to determine a boundary instruction
/// in the basic block.
/// @param container holds ordered filtered instructions inside the analyzed basic block
/// @param lookupTable holds a mapping from a instruction to its number of line of its basic block
/// @param IsBoundaryInst a callback to check if the instruction is a boundary instruction
auto GetBoundaryFunc(const OrderedInstructionLookupTableT &container, const InstIdxLookupTableT &lookupTable,
                     std::function<bool(const llvm::Instruction *)> &IsBoundaryInst) {
  BoundaryCallbackT GetBoundary{};
  GetBoundary = [&](auto beg, auto end) -> decltype(beg) {
    if (beg == end) {
      return end;
    }
    constexpr bool isForwardDirection = std::is_same_v<decltype(beg), llvm::BasicBlock::const_iterator>;
    const llvm::BasicBlock *pCurrentBasicBlock = beg->getParent();
    const auto &currBBInstsIt = container.find(pCurrentBasicBlock);
    if (currBBInstsIt == container.end() || currBBInstsIt->second.empty()) {
      return end;
    }
    auto currBBInstsRange = GetMapRange(currBBInstsIt->second, lookupTable, beg, end);
    for (auto [index, pInst] : currBBInstsRange.first) {
      if (IsBoundaryInst(pInst)) {
        if constexpr (isForwardDirection) {
          return pInst->getIterator();
        } else {
          return pInst->getReverseIterator();
        }
      }
    }
    return end;
  };
  return GetBoundary;
}

////////////////////////////////////////////////////////////////////////
/// @brief Returns the boundary callback to iterate over a container between
/// indicated basic block iterators.
/// @param container holds ordered filtered instructions inside the analyzed basic block
/// @param lookupTable holds a mapping from a instruction to its number of line of its basic block
/// @param IsExpectedInst a callback to check if the instruction is an expected instruction
/// @param instructions the container of the visited expected instructions from the container filled by the output
/// callback
auto GetIterationFunc(const OrderedInstructionLookupTableT &container, const InstIdxLookupTableT &lookupTable,
                      std::function<bool(const llvm::Instruction *)> &IsExpectedInst,
                      std::vector<const llvm::Instruction *> &instructions) {
  IterationCallbackT IterateFunc{};
  IterateFunc = [&](auto it, auto end) {
    if (it == end) {
      return;
    }
    const llvm::BasicBlock *pCurrentBasicBlock = it->getParent();
    const auto &currBBInstsIt = container.find(pCurrentBasicBlock);
    if (currBBInstsIt == container.end() || currBBInstsIt->second.empty()) {
      return;
    }
    auto currBBInstsRange = GetMapRange(currBBInstsIt->second, lookupTable, it, end);
    for (const auto &[index, val] : currBBInstsRange.first) {
      if (IsExpectedInst(val)) {
        instructions.push_back(val);
      }
    }
  };
  return IterateFunc;
}

////////////////////////////////////////////////////////////////////////
/// @brief Returns the boundary callback to iterate over a container between
/// indicated basic block iterators.
/// @param container holds ordered filtered instructions inside the analyzed basic block
/// @param lookupTable holds a mapping from a instruction to its number of line of its basic block
/// @param instructions the container of the visited instructions from the container filled by the output callback
auto GetIterationFunc(const OrderedInstructionLookupTableT &container, const InstIdxLookupTableT &lookupTable,
                      std::vector<const llvm::Instruction *> &instructions) {
  IterationCallbackT IterateFunc{};
  IterateFunc = [&](auto it, auto end) {
    if (it == end) {
      return;
    }
    const llvm::BasicBlock *pCurrentBasicBlock = it->getParent();
    const auto &currBBInstsIt = container.find(pCurrentBasicBlock);
    if (currBBInstsIt == container.end() || currBBInstsIt->second.empty()) {
      return;
    }
    auto currBBInstsRange = GetMapRange(currBBInstsIt->second, lookupTable, it, end);
    for (const auto &[index, val] : currBBInstsRange.first) {
      instructions.push_back(val);
    }
  };
  return IterateFunc;
}

////////////////////////////////////////////////////////////////////////
/// @brief Returns the boundary callback to iterate over a container between
/// indicated basic block iterators.
/// @param container holds ordered filtered instructions inside the analyzed basic block
/// @param lookupTable holds a mapping from a instruction to its number of line of its basic block
/// @param instMaskLookupTable holds a mapping from a instruction to its memory mask
/// @param bbLookupTable holds a mapping from a basic block to the aggregated memory mask of its memory instructions
/// @param instructionMask is the aggregated instruction mask by the output callback
IterationCallbackT GetIterationFunc(const OrderedInstructionLookupTableT &container,
                                    const InstIdxLookupTableT &lookupTable,
                                    const InstMaskLookupTableT &instMaskLookupTable,
                                    const BasicBlockMaskLookupTableT &bbLookupTable, InstructionMask &instructionMask) {
  IterationCallbackT IterateFunc{};
  IterateFunc = [&container, &lookupTable, &instMaskLookupTable, &bbLookupTable, &instructionMask](auto it, auto end) {
    if (it == end) {
      return;
    }
    const llvm::BasicBlock *pCurrentBasicBlock = it->getParent();
    const auto &currBBInstsIt = container.find(pCurrentBasicBlock);
    if (currBBInstsIt == container.end() || currBBInstsIt->second.empty()) {
      return;
    }
    auto currBBInstsRange = GetMapRange(currBBInstsIt->second, lookupTable, it, end);
    if (currBBInstsRange.second) {
      instructionMask |= bbLookupTable.lookup(pCurrentBasicBlock);
      return;
    }
    for (const auto &[index, val] : currBBInstsRange.first) {
      instructionMask |= instMaskLookupTable.lookup(val);
    }
  };
  return IterateFunc;
}

////////////////////////////////////////////////////////////////////////
/// @brief Provides reachable memory instructions from this source instruction.
/// The boundary for this searching are drawn by visible substitutes.
/// The visibility term means such a reachability which is determined by
/// the fact if any path between this source instruction and a substitute
/// is not crossed by another substitute.
/// The searching relies on DFS algorithm.
/// @param pSourceInst the source synchronization instruction
/// @param forwardDirection the direction of searching
/// @param boundaryInstructions the collection of substitute instructions
/// @param memoryInstructions the collection of unsynchronized memory instructions
void SynchronizationObjectCoalescing::GetVisibleMemoryInstructions(
    const llvm::Instruction *pSourceInst, bool forwardDirection,
    std::vector<const llvm::Instruction *> &boundaryInstructions,
    std::vector<const llvm::Instruction *> &memoryInstructions) const {
  InstructionMask memoryInstructionMask = GetDefaultMemoryInstructionMask(pSourceInst);
  llvm::DenseSet<const llvm::BasicBlock *> visitedBasicBlocks;

  std::function<bool(const llvm::Instruction *)> IsBoundaryInst =
      [this, pSourceInst](const llvm::Instruction *pEvaluatedInst) {
        return IsSubstituteInstruction(pEvaluatedInst, pSourceInst);
      };

  std::function<bool(const llvm::Instruction *)> IsMemoryInst =
      [this, memoryInstructionMask](const llvm::Instruction *pEvaluatedInst) {
        return (m_InstMaskLookupTable.lookup(pEvaluatedInst) & memoryInstructionMask) != 0;
      };
  auto GetBoundaries = GetBoundaryFunc(IsFenceOperation(pSourceInst) ? m_OrderedFenceInstructionsInBasicBlockCache
                                                                     : m_OrderedBarrierInstructionsInBasicBlockCache,
                                       m_InstIdxLookupTable, IsBoundaryInst, boundaryInstructions);
  auto GetMemInsts = GetIterationFunc(m_OrderedMemoryInstructionsInBasicBlockCache, m_InstIdxLookupTable, IsMemoryInst,
                                      memoryInstructions);

  if (forwardDirection) {
    llvm::BasicBlock::const_iterator firstIt = ++pSourceInst->getIterator();
    std::list<llvm::BasicBlock::const_iterator> workList{firstIt};
    SearchInstructions(workList, visitedBasicBlocks, static_cast<ForwardIterationCallbackT &>(GetMemInsts),
                       static_cast<ForwardBoundaryCallbackT &>(GetBoundaries));
  } else {
    llvm::BasicBlock::const_reverse_iterator firstIt = ++pSourceInst->getReverseIterator();
    std::list<llvm::BasicBlock::const_reverse_iterator> workList{};
    if (firstIt == pSourceInst->getParent()->rend()) {
      for (const llvm::BasicBlock *pPredecessor : llvm::predecessors(pSourceInst->getParent())) {
        workList.push_back(pPredecessor->rbegin());
      }
    } else {
      workList.push_back(firstIt);
    }
    SearchInstructions(workList, visitedBasicBlocks, static_cast<BackwardIterationCallbackT &>(GetMemInsts),
                       static_cast<BackwardBoundaryCallbackT &>(GetBoundaries));
  }
}

////////////////////////////////////////////////////////////////////////
/// @brief Provides reachable memory instructions in the next synchronization area
/// from this source instruction. Such searching is executed because fences
/// ensures that all started memory operations are finished together.
/// These potentially unsynchronized instructions should be known to make decision
/// if the fence is redundant. This function search such patterns: this fence -> (barrier ->
/// substitute* -> barrier*). All unsynchronized instructions are inside the paranthesis.
/// * these instructions can be missed in the shader.
/// @param pSourceInst a collection with begin points of searching
/// @param threadGroupBarriers holds boundary thread group barrier instructions
/// @param memoryInstructions the collection of unsynchronized memory instructions
void SynchronizationObjectCoalescing::GetAllUnsynchronizedMemoryInstructions(
    const llvm::Instruction *pSourceInst, std::vector<const llvm::Instruction *> &threadGroupBarriers,
    std::vector<const llvm::Instruction *> &memoryInstructions) const {
  std::function<bool(const llvm::Instruction *)> IsBoundaryInst =
      [this, pSourceInst](const llvm::Instruction *pEvaluatedInst) {
        return IsSubstituteInstruction(pEvaluatedInst, pSourceInst);
      };
  auto GetBoundaries =
      GetBoundaryFunc(m_OrderedFenceInstructionsInBasicBlockCache, m_InstIdxLookupTable, IsBoundaryInst);
  auto GetBarrierInsts =
      GetIterationFunc(m_OrderedBarrierInstructionsInBasicBlockCache, m_InstIdxLookupTable, threadGroupBarriers);

  // gather all visible thread group barriers
  {
    llvm::DenseSet<const llvm::BasicBlock *> visitedBasicBlocks;
    llvm::BasicBlock::const_iterator firstIt = ++pSourceInst->getIterator();
    std::list<llvm::BasicBlock::const_iterator> workList{firstIt};
    SearchInstructions(workList, visitedBasicBlocks, static_cast<ForwardIterationCallbackT &>(GetBarrierInsts),
                       static_cast<ForwardBoundaryCallbackT &>(GetBoundaries));
  }

  // gather all memory instructions between thread group barriers which goes through a substitute (thread group barrier
  // -> fence -> thread group barrier)
  if (!threadGroupBarriers.empty()) {
    InstructionMask memoryInstructionMask = GetDefaultMemoryInstructionMask(pSourceInst);
    std::function<bool(const llvm::Instruction *)> IsMemoryInst =
        [this, memoryInstructionMask](const llvm::Instruction *pEvaluatedInst) {
          return (m_InstMaskLookupTable.lookup(pEvaluatedInst) & memoryInstructionMask) != 0;
        };
    auto GetMemInsts = GetIterationFunc(m_OrderedMemoryInstructionsInBasicBlockCache, m_InstIdxLookupTable,
                                        IsMemoryInst, memoryInstructions);
    llvm::DenseSet<const llvm::BasicBlock *> visitedBasicBlocks;

    // find reachable substitutes from these thread group barriers
    std::list<llvm::BasicBlock::const_iterator> workList;
    for (const llvm::Instruction *pBarrier : threadGroupBarriers) {
      workList.push_back(pBarrier->getIterator());
    }
    std::vector<const llvm::Instruction *> visibleSubstitues;
    auto GetBoundariesAfterBarriers = GetBoundaryFunc(m_OrderedFenceInstructionsInBasicBlockCache, m_InstIdxLookupTable,
                                                      IsBoundaryInst, visibleSubstitues);
    SearchInstructions(workList, visitedBasicBlocks, static_cast<ForwardIterationCallbackT &>(GetMemInsts),
                       static_cast<ForwardBoundaryCallbackT &>(GetBoundariesAfterBarriers));

    // find reachable thread group barriers from these substitutes
    workList.clear();
    for (const llvm::Instruction *pSubstitute : visibleSubstitues) {
      workList.push_back(pSubstitute->getIterator());
      if (visitedBasicBlocks.find(pSubstitute->getParent()) != visitedBasicBlocks.end()) {
        visitedBasicBlocks.erase(pSubstitute->getParent());
      }
    }
    auto GetBarrierBoundaries = GetBoundaryFunc(m_OrderedBarrierInstructionsInBasicBlockCache, m_InstIdxLookupTable);
    SearchInstructions(workList, visitedBasicBlocks, static_cast<ForwardIterationCallbackT &>(GetMemInsts),
                       static_cast<ForwardBoundaryCallbackT &>(GetBarrierBoundaries));
  }
}

////////////////////////////////////////////////////////////////////////
/// @brief Provides the memory instruction mask from reachable memory instructions
/// of this source instruction. The boundary for this searching are drawn
/// by visible substitutes. The visibility term means such a reachability which
/// is determined by the fact if any path between this source instruction and
/// a substitute is not crossed by another substitute.
/// @param pSourceInst the source synchronization instruction
/// @param forwardDirection the direction of searching
InstructionMask SynchronizationObjectCoalescing::GetInstructionMask(const llvm::Instruction *pSourceInst,
                                                                    bool forwardDirection) const {
  InstructionMask result{};
  llvm::DenseSet<const llvm::BasicBlock *> visitedBasicBlocks;
  std::function<bool(const llvm::Instruction *)> IsBoundaryInst =
      [this, pSourceInst](const llvm::Instruction *pEvaluatedInst) {
        return IsSubstituteInstruction(pEvaluatedInst, pSourceInst);
      };
  auto GetBoundaries = GetBoundaryFunc(IsFenceOperation(pSourceInst) ? m_OrderedFenceInstructionsInBasicBlockCache
                                                                     : m_OrderedBarrierInstructionsInBasicBlockCache,
                                       m_InstIdxLookupTable, IsBoundaryInst);
  auto GetMemInsts = GetIterationFunc(m_OrderedMemoryInstructionsInBasicBlockCache, m_InstIdxLookupTable,
                                      m_InstMaskLookupTable, m_BasicBlockMemoryInstructionMaskCache, result);
  if (forwardDirection) {
    llvm::BasicBlock::const_iterator firstIt = ++pSourceInst->getIterator();
    std::list<llvm::BasicBlock::const_iterator> workList{firstIt};
    SearchInstructions(workList, visitedBasicBlocks, static_cast<ForwardIterationCallbackT &>(GetMemInsts),
                       static_cast<ForwardBoundaryCallbackT &>(GetBoundaries));
  } else {
    llvm::BasicBlock::const_reverse_iterator firstIt = ++pSourceInst->getReverseIterator();
    std::list<llvm::BasicBlock::const_reverse_iterator> workList{};
    if (firstIt == pSourceInst->getParent()->rend()) {
      for (const llvm::BasicBlock *pPredecessor : llvm::predecessors(pSourceInst->getParent())) {
        workList.push_back(pPredecessor->rbegin());
      }
    } else {
      workList.push_back(firstIt);
    }
    SearchInstructions(workList, visitedBasicBlocks, static_cast<BackwardIterationCallbackT &>(GetMemInsts),
                       static_cast<BackwardBoundaryCallbackT &>(GetBoundaries));
  }
  return result;
}

////////////////////////////////////////////////////////////////////////
/// @brief Provides the memory instruction mask from the atomic operation
/// instruction based on the destination memory address.
/// @param pSourceInst the atomic operation operation instruction
IGC::InstructionMask
SynchronizationObjectCoalescing::GetAtomicInstructionMaskFromPointer(const llvm::Instruction *pSourceInst) const {
  InstructionMask memoryInstructionMask = GetInstructionMask(pSourceInst);
  InstructionMask result{};
  if (memoryInstructionMask == InstructionMask::AtomicOperation) {
    const llvm::GenIntrinsicInst *pGenIntrinsicInst = llvm::cast<llvm::GenIntrinsicInst>(pSourceInst);
    if (pGenIntrinsicInst->getIntrinsicID() == llvm::GenISAIntrinsic::GenISA_intatomictyped ||
        pGenIntrinsicInst->getIntrinsicID() == llvm::GenISAIntrinsic::GenISA_floatatomictyped ||
        pGenIntrinsicInst->getIntrinsicID() == llvm::GenISAIntrinsic::GenISA_fcmpxchgatomictyped ||
        pGenIntrinsicInst->getIntrinsicID() == llvm::GenISAIntrinsic::GenISA_icmpxchgatomictyped) {
      result = static_cast<InstructionMask>(result | InstructionMask::TypedReadOperation);
      result = static_cast<InstructionMask>(result | InstructionMask::TypedWriteOperation);
    } else {
      llvm::Type *pPointerType = pGenIntrinsicInst->getOperand(0)->getType();
      if (IsGlobalWritableResource(pPointerType)) {
        result = static_cast<InstructionMask>(result | InstructionMask::BufferReadOperation);
        result = static_cast<InstructionMask>(result | InstructionMask::BufferWriteOperation);
      } else if (IsSharedMemoryResource(pPointerType)) {
        result = static_cast<InstructionMask>(result | InstructionMask::SharedMemoryReadOperation);
        result = static_cast<InstructionMask>(result | InstructionMask::SharedMemoryWriteOperation);
      }
    }
  }
  return result;
}

////////////////////////////////////////////////////////////////////////
/// @brief This function verifies which cases of synchronization can be
/// attributed to this synchronization instruction in the case of the particular resource.
/// This stems from the forwardly and backwardly visible memory instructions and
/// their interaction with the synchronization instruction.
/// @param localForwardMemoryInstructionMask the mask with forwardly visible memory instructions
/// @param localBackwardMemoryInstructionMask the mask with backwardly visible memory instructions
/// @param readBit the read instruction bit for the particular resource
/// @param writeBit the write instruction bit for the particular resource
SynchronizationObjectCoalescing::SynchronizationCaseMask
SynchronizationObjectCoalescing::GetSynchronizationMask(InstructionMask localForwardMemoryInstructionMask,
                                                        InstructionMask localBackwardMemoryInstructionMask,
                                                        InstructionMask readBit, InstructionMask writeBit) const {
  SynchronizationCaseMask result = SynchronizationCaseMask::Empty;
  // write -> barrier/fence -> read
  bool isWriteSyncReadCase =
      ((localBackwardMemoryInstructionMask & writeBit) != 0 && (localForwardMemoryInstructionMask & readBit) != 0);
  if (isWriteSyncReadCase) {
    result = static_cast<SynchronizationCaseMask>(result | SynchronizationCaseMask::WriteSyncRead);
  }

  // write -> barrier/fence -> write
  bool isWriteSyncWriteCase =
      ((localBackwardMemoryInstructionMask & writeBit) != 0 && (localForwardMemoryInstructionMask & writeBit) != 0);
  if (isWriteSyncWriteCase) {
    result = static_cast<SynchronizationCaseMask>(result | SynchronizationCaseMask::WriteSyncWrite);
  }

  // write -> fence -> ret
  bool requiresFlushWrites =
      static_cast<uint32_t>(writeBit & (SharedMemoryWriteOperation | BufferWriteOperation | TypedWriteOperation)) != 0;
  bool isWriteSyncRetCase = requiresFlushWrites && ((localBackwardMemoryInstructionMask & writeBit) != 0 &&
                                                    (localForwardMemoryInstructionMask & EndOfThreadOperation) != 0);
  if (isWriteSyncRetCase) {
    result = static_cast<SynchronizationCaseMask>(result | SynchronizationCaseMask::WriteSyncRet);
  }

  // extended cache control -> fence -> ret
  bool requiresFlushExtendedCacheControls = static_cast<uint32_t>(writeBit & (ExtendedCacheControlOperation)) != 0;
  bool isExtendedCacheControlSyncRetCase =
      requiresFlushExtendedCacheControls && ((localBackwardMemoryInstructionMask & writeBit) != 0 &&
                                             (localForwardMemoryInstructionMask & EndOfThreadOperation) != 0);
  if (isExtendedCacheControlSyncRetCase) {
    result = static_cast<SynchronizationCaseMask>(result | SynchronizationCaseMask::ExtendedCacheControlSyncRet);
  }

  // atomic -> barrier/fence -> read
  bool isAtomicSyncReadCase = ((localBackwardMemoryInstructionMask & AtomicOperation) != 0 &&
                               (localForwardMemoryInstructionMask & readBit) != 0);
  if (isAtomicSyncReadCase) {
    result = static_cast<SynchronizationCaseMask>(result | SynchronizationCaseMask::AtomicSyncRead);
  }

  // atomic -> barrier/fence -> write
  bool isAtomicSyncWriteCase = ((localBackwardMemoryInstructionMask & AtomicOperation) != 0 &&
                                (localForwardMemoryInstructionMask & writeBit) != 0);
  if (isAtomicSyncWriteCase) {
    result = static_cast<SynchronizationCaseMask>(result | SynchronizationCaseMask::AtomicSyncWrite);
  }

  // write -> barrier/fence -> atomic
  bool isWriteSyncAtomicCase = ((localBackwardMemoryInstructionMask & writeBit) != 0 &&
                                (localForwardMemoryInstructionMask & AtomicOperation) != 0);
  if (isWriteSyncAtomicCase) {
    result = static_cast<SynchronizationCaseMask>(result | SynchronizationCaseMask::WriteSyncAtomic);
  }

  // read -> barrier -> atomic
  bool isReadSyncAtomicCase = ((localBackwardMemoryInstructionMask & readBit) != 0 &&
                               (localForwardMemoryInstructionMask & AtomicOperation) != 0);
  if (isReadSyncAtomicCase) {
    result = static_cast<SynchronizationCaseMask>(result | SynchronizationCaseMask::ReadSyncAtomic);
  }

  // read -> barrier -> write
  bool isReadSyncWriteCase =
      ((localBackwardMemoryInstructionMask & readBit) != 0 && (localForwardMemoryInstructionMask & writeBit) != 0);
  if (isReadSyncWriteCase) {
    result = static_cast<SynchronizationCaseMask>(result | SynchronizationCaseMask::ReadSyncWrite);
  }

  // atomic -> barrier -> atomic
  bool isAtomicSyncAtomicCase = ((localBackwardMemoryInstructionMask & AtomicOperation) != 0 &&
                                 (localForwardMemoryInstructionMask & AtomicOperation) != 0);
  if (isAtomicSyncAtomicCase) {
    result = static_cast<SynchronizationCaseMask>(result | SynchronizationCaseMask::AtomicSyncAtomic);
  }

  return result;
}

////////////////////////////////////////////////////////////////////////
/// @brief Provides the synchronization case mask for determining strict
/// redundancy.
SynchronizationObjectCoalescing::SynchronizationCaseMask
SynchronizationObjectCoalescing::GetStrictSynchronizationMask(llvm::Instruction *pInst) const {
  SynchronizationCaseMask strictSynchronizationCaseMask = sc_FullSynchronizationCaseMask;

  if (IsFenceOperation(pInst)) {
    // fences doesn't provide any guarantees for the order of instruction execution between threads
    strictSynchronizationCaseMask = static_cast<SynchronizationCaseMask>((~SynchronizationCaseMask::AtomicSyncAtomic) &
                                                                         strictSynchronizationCaseMask);

    strictSynchronizationCaseMask =
        static_cast<SynchronizationCaseMask>((SynchronizationCaseMask::WriteSyncRet) | strictSynchronizationCaseMask);

    strictSynchronizationCaseMask = static_cast<SynchronizationCaseMask>(
        (SynchronizationCaseMask::ExtendedCacheControlSyncRet) | strictSynchronizationCaseMask);

    // Note: Please change the description in igc flags if the value is changed.
    static_assert(SynchronizationCaseMask::ReadSyncWrite == 0x01);
    bool disableReadFenceWriteCase =
        (IGC_GET_FLAG_VALUE(SynchronizationObjectCoalescingConfig) & SynchronizationCaseMask::ReadSyncWrite) != 0;
    if (disableReadFenceWriteCase) {
      strictSynchronizationCaseMask = static_cast<SynchronizationCaseMask>((~SynchronizationCaseMask::ReadSyncWrite) &
                                                                           strictSynchronizationCaseMask);
    }
  }

  return strictSynchronizationCaseMask;
}

////////////////////////////////////////////////////////////////////////
/// @brief Provides the synchronization case mask for determining L1 Cache
/// invalidation redundancy.
IGC::SynchronizationObjectCoalescing::SynchronizationCaseMask
SynchronizationObjectCoalescing::GetL1CacheInvalidatioSynchronizationMask() const {
  constexpr SynchronizationCaseMask L1CacheInvalidationCaseMask = static_cast<SynchronizationCaseMask>(
      SynchronizationCaseMask::WriteSyncRead | SynchronizationCaseMask::WriteSyncAtomic |
      SynchronizationCaseMask::AtomicSyncRead);
  return L1CacheInvalidationCaseMask;
}

////////////////////////////////////////////////////////////////////////
/// @brief Provides the synchronization case mask for all resources.
SynchronizationObjectCoalescing::SynchronizationCaseMask
SynchronizationObjectCoalescing::GetSynchronizationMaskForAllResources(
    InstructionMask localForwardMemoryInstructionMask, InstructionMask localBackwardMemoryInstructionMask) const {
  // buffer access
  SynchronizationCaseMask syncCaseMask = GetSynchronizationMask(
      localForwardMemoryInstructionMask, localBackwardMemoryInstructionMask, BufferReadOperation, BufferWriteOperation);
  // shared memory access
  syncCaseMask = static_cast<SynchronizationCaseMask>(
      syncCaseMask | GetSynchronizationMask(localForwardMemoryInstructionMask, localBackwardMemoryInstructionMask,
                                            SharedMemoryReadOperation, SharedMemoryWriteOperation));
  // typed access
  syncCaseMask = static_cast<SynchronizationCaseMask>(
      syncCaseMask | GetSynchronizationMask(localForwardMemoryInstructionMask, localBackwardMemoryInstructionMask,
                                            TypedReadOperation, TypedWriteOperation));
  // URB access
  syncCaseMask = static_cast<SynchronizationCaseMask>(
      syncCaseMask | GetSynchronizationMask(localForwardMemoryInstructionMask, localBackwardMemoryInstructionMask,
                                            OutputUrbReadOperation, UrbWriteOperation));

  // extended cache control access
  syncCaseMask = static_cast<SynchronizationCaseMask>(
      syncCaseMask | GetSynchronizationMask(localForwardMemoryInstructionMask, localBackwardMemoryInstructionMask, None,
                                            ExtendedCacheControlOperation));

  return syncCaseMask;
}

////////////////////////////////////////////////////////////////////////
/// @brief Provides the memory instruction mask from unsynchronized forward
/// memory instructions (in the next synchronization block delineated by
/// thread group barriers)
/// @param pSourceInst the source synchronization instruction
InstructionMask
SynchronizationObjectCoalescing::GetUnsynchronizedForwardInstructionMask(const llvm::Instruction *pSourceInst) const {
  std::vector<const llvm::Instruction *> boundaryInstructions;
  std::vector<const llvm::Instruction *> memoryInstructions;
  GetAllUnsynchronizedMemoryInstructions(pSourceInst, boundaryInstructions, memoryInstructions);
  return GetInstructionMask(memoryInstructions);
}

////////////////////////////////////////////////////////////////////////////////
/// @brief Checks if the fence is required to assure correct ordering of atomic
/// operations present before the fence (in program order)
/// @param pSourceInst the source synchronization instruction
/// @param onlyGlobalAtomics check only TGM and UGM atomic operations
bool SynchronizationObjectCoalescing::IsRequiredForAtomicOperationsOrdering(
    const llvm::Instruction *pSourceInst, std::vector<llvm::Instruction *> &pAtomicInstToBeSourced,
    bool onlyGlobalAtomics /*= false*/) const {
  if (!IsFenceOperation(pSourceInst)) {
    // Not a fence, nothing to check
    return false;
  }
  InstructionMask defaultSourceInstructionMask = GetDefaultMemoryInstructionMask(pSourceInst);
  if (onlyGlobalAtomics) {
    // Change the mask to include only global memory operations
    InstructionMask globalOnlyMask = InstructionMask::None;
    InstructionMask typedReadOrWrite = static_cast<InstructionMask>(TypedReadOperation | TypedWriteOperation);
    InstructionMask globalReadOrWrite = static_cast<InstructionMask>(BufferReadOperation | BufferWriteOperation);
    if ((defaultSourceInstructionMask & typedReadOrWrite) != InstructionMask::None) {
      globalOnlyMask = typedReadOrWrite;
    }
    if ((defaultSourceInstructionMask & globalReadOrWrite) != InstructionMask::None) {
      globalOnlyMask |= globalReadOrWrite;
    }
    defaultSourceInstructionMask = globalOnlyMask;
  }
  if (defaultSourceInstructionMask == InstructionMask::None) {
    return false;
  }

  // Gather all instructions that the source fence operation affects.
  constexpr bool backwardDirection = false;
  std::vector<const llvm::Instruction *> boundaryInstructions;
  std::vector<const llvm::Instruction *> memoryInstructions;
  GetVisibleMemoryInstructions(pSourceInst, backwardDirection, boundaryInstructions, memoryInstructions);
  for (const llvm::Instruction *pInst : memoryInstructions) {
    InstructionMask memoryInstructionMask = GetInstructionMask(pInst);
    if (memoryInstructionMask != InstructionMask::AtomicOperation) {
      // Only atomic operations are checked.
      continue;
    }
    InstructionMask atomicPointerMemoryInstructionMask = GetAtomicInstructionMaskFromPointer(pInst);
    bool isPotentiallyUnsynchronizedAtomic = (atomicPointerMemoryInstructionMask & defaultSourceInstructionMask) != 0;
    if (!isPotentiallyUnsynchronizedAtomic) {
      // The source fence does not affect the atomic instruction.
      continue;
    }
    // Check if the atomic operation is not synchronized by a different
    // fence operation with different scope or memory operation.
    // e.g.:
    // %0 = call i32 @llvm.genx.GenISA.atomiccounterinc.p2490368i8(i8 addrspace(2490368)* %atomic_address)
    // call void @llvm.genx.GenISA.memoryfence(i1 true, i1 false, i1 false, i1 false, i1 false, i1 true, i1 false)
    // call void @llvm.genx.GenISA.memoryfence(i1 true, i1 false, i1 false, i1 false, i1 false, i1 true, i1 true)
    // store float %value, float addrspace(1)* %global_address
    {
      isPotentiallyUnsynchronizedAtomic = false;
      // Lambda that checks if a fence operation synchronizes the atomic operation.
      // This can be improved to detect the users of atomic instruction and end the search for fences once we find the
      // user. This user is essentially same as Source Value Intrinsic, however it can be reordered in visa affecting
      // the execution order of atomic instructions. If we can find a way to treat this user as a special instruction
      // and avoid reordering, we can skip creating new source value instruction.
      std::function<bool(const llvm::Instruction *)> IsBoundaryInst = [this, &atomicPointerMemoryInstructionMask,
                                                                       &isPotentiallyUnsynchronizedAtomic,
                                                                       pSourceInst](const llvm::Instruction *pInst) {
        if (pInst == pSourceInst) {
          // Instruction search reached the source instruction, there is
          // no other fence instruction that assures ordering of the atomic.
          isPotentiallyUnsynchronizedAtomic = true;
        }
        bool isFence = IsFenceOperation(pInst);
        if (isFence) {
          InstructionMask memoryInstructionMask = GetDefaultMemoryInstructionMask(pInst);
          bool isBoundary = (atomicPointerMemoryInstructionMask & memoryInstructionMask) != 0;
          return isBoundary;
        }
        return false;
      };
      std::function<bool(const llvm::Instruction *)> IsMemoryInst =
          [this, memoryInstructionMask](const llvm::Instruction *pEvaluatedInst) {
            return (m_InstMaskLookupTable.lookup(pEvaluatedInst) & memoryInstructionMask) != 0;
          };
      BoundaryCallbackT GetBoundaries = GetBoundaryFunc(m_OrderedFenceInstructionsInBasicBlockCache,
                                                        m_InstIdxLookupTable, IsBoundaryInst, boundaryInstructions);
      ProcessCallbackT ProcessInstructions{};
      ProcessInstructions = [&GetBoundaries](auto beg, auto end) { return GetBoundaries(beg, end) == end; };

      llvm::DenseSet<const llvm::BasicBlock *> visitedBasicBlocks;
      llvm::BasicBlock::const_iterator firstIt = ++pInst->getIterator();
      std::list<llvm::BasicBlock::const_iterator> workList{firstIt};

      // Start from the atomic operation and check all instruction in the
      // forward direction.
      SearchInstructions(workList, visitedBasicBlocks, static_cast<ForwardProcessCallbackT &>(ProcessInstructions));
      IGC_ASSERT(boundaryInstructions.size() > 0);
    }

    // Check if any of the instructions that immediately follow the fence
    // instruction is a fence that assures ordering of current atomic
    // operation.
    // Note: this approach addresses the simplest cases like the example
    // below and can be improved in the future:
    // %0 = call i32 @llvm.genx.GenISA.atomiccounterinc.p2490368i8(i8 addrspace(2490368)* %atomic_address)
    // call void @llvm.genx.GenISA.memoryfence(i1 true, i1 false, i1 false, i1 false, i1 false, i1 true, i1 false)
    // call void @llvm.genx.GenISA.memoryfence(i1 true, i1 false, i1 false, i1 false, i1 false, i1 true, i1 true)
    // store float %value, float addrspace(1)* %global_address
    // %1 = load float, float addrspace(1)* %global_address
    if (isPotentiallyUnsynchronizedAtomic) {
      bool substituteFenceFound = false;
      for (llvm::BasicBlock::const_iterator it = ++pSourceInst->getIterator(); it != pSourceInst->getParent()->end();
           ++it) {
        const llvm::Instruction *pCurrInst = &(*it);
        // If we encounter an atomic instruction after pSourceInst (Source Fence), then the fence is required to execute
        // pInst (initial atomic) before pCurrInst (current atomic)
        if (IsAtomicOperation(pCurrInst)) {
          break;
        }
        if (IsFenceOperation(pCurrInst) && IsSubstituteInstruction(pCurrInst, pSourceInst)) {
          substituteFenceFound = true;
          break;
        }
      }
      if (!substituteFenceFound) {
        // Found an atomic operation that requires the source fence
        // instruction for correct memory ordering.

        // If ReplaceAtomicFenceWithSourceValue is true, we can replace this fence with GenISA_source_value.
        // This will source the atomic instruction and still maintains the order of atomic instructions.
        // Else return true marking the fence instruction as Obligatory.

        if (ctx->platform.enableReplaceAtomicFenceWithSourceValue()) {
          // If a previous fence was replaced with source value intrinsic, GetVisibleMemoryInstructions will add the
          // same atomic instruction again for the next fence resulting in multiple source value intrinsics but we need
          // it to be sourced only once. Hence we check if it was already sourced previously. Continues to check all
          // valid atomic Instructions to be sourced.
          if (m_SourcedAtomicInstructions.find(const_cast<Instruction *>(pInst)) == m_SourcedAtomicInstructions.end()) {
            pAtomicInstToBeSourced.push_back(const_cast<Instruction *>(pInst));
          }
        } else {
          return true;
        }
      }
    }
  }
  return false;
}

////////////////////////////////////////////////////////////////////////
/// @brief Provides the memory instruction mask from the instruction container
/// @param input the instruction container
InstructionMask
SynchronizationObjectCoalescing::GetInstructionMask(const std::vector<const llvm::Instruction *> &input) const {
  InstructionMask result = InstructionMask::None;
  for (const llvm::Instruction *pInst : input) {
    result |= GetInstructionMask(pInst);
  }
  return result;
}

////////////////////////////////////////////////////////////////////////
/// @brief Checks if the reference instruction can be replaced by the evaluated instruction.
/// @param pEvaluatedInst the instruction which is evaluated if it can replace the reference one
/// @param pReferenceInst represents the reference instruction which is an object of the replacement
/// and it means that this instruction must be equal or weaker than the evaluated one.
bool SynchronizationObjectCoalescing::IsSubstituteInstruction(const llvm::Instruction *pEvaluatedInst,
                                                              const llvm::Instruction *pReferenceInst) const {
  if (pEvaluatedInst == pReferenceInst) {
    return false;
  }
  if (IsUntypedMemoryFenceOperation(pEvaluatedInst) && IsUntypedMemoryFenceOperation(pReferenceInst)) {
    const uint32_t commitEnableArg = 0;
    const uint32_t L3FlushRWDataArg = 1;
    const uint32_t L3FlushConstantDataArg = 2;
    const uint32_t L3FlushTextureDataArg = 3;
    const uint32_t L3FlushInstructionsArg = 4;
    bool isDuplicate = pEvaluatedInst->getOperand(commitEnableArg) == pReferenceInst->getOperand(commitEnableArg);
    isDuplicate &= pEvaluatedInst->getOperand(L3FlushRWDataArg) == pReferenceInst->getOperand(L3FlushRWDataArg);
    isDuplicate &=
        pEvaluatedInst->getOperand(L3FlushConstantDataArg) == pReferenceInst->getOperand(L3FlushConstantDataArg);
    isDuplicate &=
        pEvaluatedInst->getOperand(L3FlushTextureDataArg) == pReferenceInst->getOperand(L3FlushTextureDataArg);
    isDuplicate &=
        pEvaluatedInst->getOperand(L3FlushInstructionsArg) == pReferenceInst->getOperand(L3FlushInstructionsArg);

    isDuplicate &= ((IsUntypedMemoryFenceOperationForGlobalAccess(pEvaluatedInst) &&
                     IsUntypedMemoryFenceOperationForGlobalAccess(pReferenceInst)) ||
                    (m_HasIndependentSharedMemoryFenceFunctionality &&
                     (IsUntypedMemoryFenceOperationForSharedMemoryAccess(pEvaluatedInst) &&
                      IsUntypedMemoryFenceOperationForSharedMemoryAccess(pReferenceInst)))) ||
                   (!m_HasIndependentSharedMemoryFenceFunctionality &&
                    (IsUntypedMemoryFenceOperationForGlobalAccess(pEvaluatedInst) ||
                     !IsUntypedMemoryFenceOperationForGlobalAccess(pReferenceInst)));

    if (IsUntypedMemoryFenceOperationForGlobalAccess(pReferenceInst)) {
      isDuplicate &= (IsUntypedMemoryFenceOperationWithInvalidationFunctionality(pEvaluatedInst) ||
                      !IsUntypedMemoryFenceOperationWithInvalidationFunctionality(pReferenceInst));
    }

    return isDuplicate;
  } else if (IsLscFenceOperation(pEvaluatedInst) && IsLscFenceOperation(pReferenceInst)) {
    IGC_ASSERT(m_HasUrbFenceFunctionality);
    LSC_SFID evalLscMemSync = GetLscMem(pEvaluatedInst);
    LSC_SFID refLscMemSync = GetLscMem(pReferenceInst);
    bool isEvaluatedStrongerOrEqual = evalLscMemSync == refLscMemSync;
    if (!m_HasIndependentSharedMemoryFenceFunctionality) {
      isEvaluatedStrongerOrEqual |= evalLscMemSync == LSC_SFID::LSC_UGM && refLscMemSync == LSC_SFID::LSC_SLM;
    }
    bool isDuplicate = isEvaluatedStrongerOrEqual;
    isDuplicate &= GetLscScope(pEvaluatedInst) >= GetLscScope(pReferenceInst);
    LSC_FENCE_OP opEvaluated = GetLscFenceOp(pEvaluatedInst);
    LSC_FENCE_OP opReference = GetLscFenceOp(pReferenceInst);
    bool referenceIsOpNone = opReference == LSC_FENCE_OP_NONE;
    bool evaluatedIsOpNone = opEvaluated == LSC_FENCE_OP_NONE;
    // Current implementation allows replacing the reference LSC fence with
    // the evaluated LSC fence if any of the following conditions is true:
    // 1.) both fences have the same operation type
    // 2.) reference fence is a .none operation
    // 3.) reference fence is an .invalidate and evaluated fence is
    //     an .invalidate or an .evict
    bool cond1 = opReference == opEvaluated;
    bool cond2 = referenceIsOpNone;
    bool cond3 = !evaluatedIsOpNone && opReference == LSC_FENCE_OP_INVALIDATE &&
                 (opEvaluated == LSC_FENCE_OP_INVALIDATE || opEvaluated == LSC_FENCE_OP_EVICT);
    isDuplicate &= (cond1 || cond2 || cond3);

    return isDuplicate;
  } else if (IsTypedMemoryFenceOperation(pEvaluatedInst) && IsTypedMemoryFenceOperation(pReferenceInst)) {
    bool isDuplicate = IsTypedMemoryFenceOperationWithInvalidationFunctionality(pEvaluatedInst) ||
                       !IsTypedMemoryFenceOperationWithInvalidationFunctionality(pReferenceInst);
    return isDuplicate;
  } else if (IsUrbFenceOperation(pEvaluatedInst) && IsUrbFenceOperation(pReferenceInst)) {
    return true;
  } else if (IsThreadBarrierOperation(pEvaluatedInst) && IsThreadBarrierOperation(pReferenceInst)) {
    return true;
  }

  return false;
}

////////////////////////////////////////////////////////////////////////
void SynchronizationObjectCoalescing::getAnalysisUsage(llvm::AnalysisUsage &AU) const {
  AU.setPreservesCFG();
  AU.addRequired<CodeGenContextWrapper>();
}

////////////////////////////////////////////////////////////////////////
/// @brief Gathers synchronization instructions from the current
/// function.
void SynchronizationObjectCoalescing::GatherInstructions() {
  for (llvm::BasicBlock &basicBlock : *m_CurrentFunction) {
    uint32_t i = 0;
    for (llvm::Instruction &inst : basicBlock) {
      bool isSyncInst = false;
      m_InstIdxLookupTable[&inst] = i;
      if (IsThreadBarrierOperation(&inst)) {
        m_ThreadGroupBarriers.push_back(&inst);
        m_OrderedBarrierInstructionsInBasicBlockCache[&basicBlock][i] = &inst;
      } else if (IsUntypedMemoryFenceOperation(&inst)) {
        m_UntypedMemoryFences.push_back(&inst);
        isSyncInst = true;
      } else if (IsTypedMemoryFenceOperation(&inst)) {
        m_TypedMemoryFences.push_back(&inst);
        isSyncInst = true;
      } else if (IsUrbFenceOperation(&inst)) {
        m_UrbMemoryFences.push_back(&inst);
        isSyncInst = true;
      } else if (IsLscFenceOperation(&inst)) {
        m_LscMemoryFences.push_back(&inst);
        isSyncInst = true;
      } else if (InstructionMask memoryInstructionMask = GetInstructionMask(&inst);
                 memoryInstructionMask != InstructionMask::None) {
        InstructionMask pointerMemoryInstructionMask = GetAtomicInstructionMaskFromPointer(&inst);
        m_BasicBlockMemoryInstructionMaskCache[&basicBlock] |= memoryInstructionMask;
        m_OrderedMemoryInstructionsInBasicBlockCache[&basicBlock][i] = &inst;
        m_InstMaskLookupTable[&inst] = memoryInstructionMask;
        m_GlobalMemoryInstructionMask |= memoryInstructionMask | pointerMemoryInstructionMask;
      }

      if (isSyncInst) {
        m_OrderedFenceInstructionsInBasicBlockCache[&basicBlock][i] = &inst;
      }

      i++;
    }
  }
  if (!m_LscMemoryFences.empty()) {
    // Current implementation does not expect a mix of legacy memory fences
    // and LSC fences.
    IGC_ASSERT(m_UntypedMemoryFences.empty());
    IGC_ASSERT(m_TypedMemoryFences.empty());
  }
}

////////////////////////////////////////////////////////////////////////
void SynchronizationObjectCoalescing::InvalidateMembers() {
  m_UntypedMemoryFences.clear();
  m_ThreadGroupBarriers.clear();
  m_TypedMemoryFences.clear();
  m_UrbMemoryFences.clear();
  m_LscMemoryFences.clear();

  m_OrderedMemoryInstructionsInBasicBlockCache.clear();
  m_OrderedFenceInstructionsInBasicBlockCache.clear();
  m_OrderedBarrierInstructionsInBasicBlockCache.clear();
  m_BasicBlockMemoryInstructionMaskCache.clear();
  m_SourcedAtomicInstructions.clear();
#if _DEBUG
  m_ExplanationEntries.clear();
#endif // _DEBUG
  m_GlobalMemoryInstructionMask = InstructionMask::None;
}

////////////////////////////////////////////////////////////////////////
bool SynchronizationObjectCoalescing::IsAtomicOperation(const llvm::Instruction *pInst) {
  if (llvm::isa<llvm::GenIntrinsicInst>(pInst)) {
    const llvm::GenIntrinsicInst *pGenIntrinsicInst = llvm::cast<llvm::GenIntrinsicInst>(pInst);

    switch (pGenIntrinsicInst->getIntrinsicID()) {
    case llvm::GenISAIntrinsic::GenISA_atomiccounterinc:
    case llvm::GenISAIntrinsic::GenISA_atomiccounterpredec:
    case llvm::GenISAIntrinsic::GenISA_icmpxchgatomicraw:
    case llvm::GenISAIntrinsic::GenISA_icmpxchgatomicrawA64:
    case llvm::GenISAIntrinsic::GenISA_cmpxchgatomicstructured:
    case llvm::GenISAIntrinsic::GenISA_icmpxchgatomictyped:
    case llvm::GenISAIntrinsic::GenISA_intatomicraw:
    case llvm::GenISAIntrinsic::GenISA_intatomicrawA64:
    case llvm::GenISAIntrinsic::GenISA_dwordatomicstructured:
    case llvm::GenISAIntrinsic::GenISA_intatomictyped:
    case llvm::GenISAIntrinsic::GenISA_fcmpxchgatomicraw:
    case llvm::GenISAIntrinsic::GenISA_fcmpxchgatomicrawA64:
    case llvm::GenISAIntrinsic::GenISA_fcmpxchgatomicstructured:
    case llvm::GenISAIntrinsic::GenISA_floatatomicraw:
    case llvm::GenISAIntrinsic::GenISA_floatatomicrawA64:
    case llvm::GenISAIntrinsic::GenISA_floatatomicstructured:
    case llvm::GenISAIntrinsic::GenISA_floatatomictyped:
    case llvm::GenISAIntrinsic::GenISA_fcmpxchgatomictyped:
    case llvm::GenISAIntrinsic::GenISA_LSCAtomicFP64:
    case llvm::GenISAIntrinsic::GenISA_LSCAtomicFP32:
    case llvm::GenISAIntrinsic::GenISA_LSCAtomicBF16:
    case llvm::GenISAIntrinsic::GenISA_LSCAtomicInts:
      return true;
    default:
      break;
    }
  }

  return false;
}

////////////////////////////////////////////////////////////////////////
bool SynchronizationObjectCoalescing::IsTypedReadOperation(const llvm::Instruction *pInst) {
  if (llvm::isa<llvm::GenIntrinsicInst>(pInst)) {
    const llvm::GenIntrinsicInst *pGenIntrinsicInst = llvm::cast<llvm::GenIntrinsicInst>(pInst);

    switch (pGenIntrinsicInst->getIntrinsicID()) {
    case llvm::GenISAIntrinsic::GenISA_typedread:
    case llvm::GenISAIntrinsic::GenISA_typedreadMS:
      return IsGlobalWritableResource(pGenIntrinsicInst->getOperand(0)->getType());
    default:
      break;
    }
  }

  return false;
}

bool SynchronizationObjectCoalescing::IsTypedWriteOperation(const llvm::Instruction *pInst) {
  if (llvm::isa<llvm::GenIntrinsicInst>(pInst)) {
    const llvm::GenIntrinsicInst *pGenIntrinsicInst = llvm::cast<llvm::GenIntrinsicInst>(pInst);

    switch (pGenIntrinsicInst->getIntrinsicID()) {
    case llvm::GenISAIntrinsic::GenISA_typedwrite:
    case llvm::GenISAIntrinsic::GenISA_typedwriteMS:
      return true;
    default:
      break;
    }
  }

  return false;
}

////////////////////////////////////////////////////////////////////////
bool SynchronizationObjectCoalescing::IsOutputUrbReadOperation(const llvm::Instruction *pInst) {
  if (llvm::isa<llvm::GenIntrinsicInst>(pInst)) {
    const llvm::GenIntrinsicInst *pGenIntrinsicInst = llvm::cast<llvm::GenIntrinsicInst>(pInst);

    switch (pGenIntrinsicInst->getIntrinsicID()) {
    case llvm::GenISAIntrinsic::GenISA_OutputTaskDataInput:
    case llvm::GenISAIntrinsic::GenISA_OutputMeshPrimitiveDataInput:
    case llvm::GenISAIntrinsic::GenISA_OutputMeshVertexDataInput:
    case llvm::GenISAIntrinsic::GenISA_OutputMeshSivDataInput:
    case llvm::GenISAIntrinsic::GenISA_DCL_HSOutputCntrlPtInputVec:
    case llvm::GenISAIntrinsic::GenISA_DCL_HSPatchConstInputVec:
      return true;
    default:
      break;
    }
  }

  return false;
}

////////////////////////////////////////////////////////////////////////
bool SynchronizationObjectCoalescing::IsUrbWriteOperation(const llvm::Instruction *pInst) {
  if (llvm::isa<llvm::GenIntrinsicInst>(pInst)) {
    const llvm::GenIntrinsicInst *pGenIntrinsicInst = llvm::cast<llvm::GenIntrinsicInst>(pInst);

    switch (pGenIntrinsicInst->getIntrinsicID()) {
    case llvm::GenISAIntrinsic::GenISA_OUTPUT:
    case llvm::GenISAIntrinsic::GenISA_OutputTessControlPoint:
    case llvm::GenISAIntrinsic::GenISA_PatchConstantOutput:
      return true;
    default:
      break;
    }
  }

  return false;
}

////////////////////////////////////////////////////////////////////////////////
inline llvm::Type *GetReadOperationPointerType(const llvm::Instruction *pInst) {
  if (llvm::isa<llvm::GenIntrinsicInst>(pInst)) {
    const llvm::GenIntrinsicInst *pGenIntrinsicInst = llvm::cast<llvm::GenIntrinsicInst>(pInst);

    switch (pGenIntrinsicInst->getIntrinsicID()) {
    case llvm::GenISAIntrinsic::GenISA_ldraw_indexed:
    case llvm::GenISAIntrinsic::GenISA_ldrawvector_indexed:
    case llvm::GenISAIntrinsic::GenISA_simdBlockRead:
    case llvm::GenISAIntrinsic::GenISA_simdBlockReadBindless:
    case llvm::GenISAIntrinsic::GenISA_LSCLoad:
    case llvm::GenISAIntrinsic::GenISA_LSCLoadWithSideEffects:
    case llvm::GenISAIntrinsic::GenISA_LSCLoadBlock:
    case llvm::GenISAIntrinsic::GenISA_LSCLoadCmask:
      return pGenIntrinsicInst->getOperand(0)->getType();
    default:
      break;
    }
  } else if (llvm::isa<llvm::LoadInst>(pInst)) {
    return llvm::cast<llvm::LoadInst>(pInst)->getPointerOperandType();
  }
  return nullptr;
}

////////////////////////////////////////////////////////////////////////////////
inline llvm::Type *GetWriteOperationPointerType(const llvm::Instruction *pInst) {
  if (llvm::isa<llvm::GenIntrinsicInst>(pInst)) {
    const llvm::GenIntrinsicInst *pGenIntrinsicInst = llvm::cast<llvm::GenIntrinsicInst>(pInst);

    switch (pGenIntrinsicInst->getIntrinsicID()) {
    case llvm::GenISAIntrinsic::GenISA_storeraw_indexed:
    case llvm::GenISAIntrinsic::GenISA_storerawvector_indexed:
    case llvm::GenISAIntrinsic::GenISA_simdBlockWrite:
    case llvm::GenISAIntrinsic::GenISA_simdBlockWriteBindless:
    case llvm::GenISAIntrinsic::GenISA_LSCStore:
    case llvm::GenISAIntrinsic::GenISA_LSCStoreBlock:
    case llvm::GenISAIntrinsic::GenISA_LSCStoreCmask:
      return pGenIntrinsicInst->getOperand(0)->getType();
    default:
      break;
    }
  } else if (llvm::isa<llvm::StoreInst>(pInst)) {
    return llvm::cast<llvm::StoreInst>(pInst)->getPointerOperandType();
  }

  return nullptr;
}

////////////////////////////////////////////////////////////////////////
bool SynchronizationObjectCoalescing::IsBufferReadOperation(const llvm::Instruction *pInst) {
  if (llvm::Type *pPtrType = GetReadOperationPointerType(pInst)) {
    return IsGlobalWritableResource(pPtrType);
  }
  return false;
}

////////////////////////////////////////////////////////////////////////
bool SynchronizationObjectCoalescing::IsBufferWriteOperation(const llvm::Instruction *pInst) {
  if (llvm::Type *pPtrType = GetWriteOperationPointerType(pInst)) {
    return IsGlobalWritableResource(pPtrType);
  }
  return false;
}

////////////////////////////////////////////////////////////////////////
bool SynchronizationObjectCoalescing::IsSharedMemoryReadOperation(const llvm::Instruction *pInst) {
  if (llvm::Type *pPtrType = GetReadOperationPointerType(pInst)) {
    return IsSharedMemoryResource(pPtrType);
  }
  return false;
}

////////////////////////////////////////////////////////////////////////
bool SynchronizationObjectCoalescing::IsSharedMemoryWriteOperation(const llvm::Instruction *pInst) {
  if (llvm::Type *pPtrType = GetWriteOperationPointerType(pInst)) {
    return IsSharedMemoryResource(pPtrType);
  }
  return false;
}

////////////////////////////////////////////////////////////////////////
bool SynchronizationObjectCoalescing::IsThreadBarrierOperation(const llvm::Instruction *pInst) {
  if (llvm::isa<llvm::GenIntrinsicInst>(pInst)) {
    const llvm::GenIntrinsicInst *pGenIntrinsicInst = llvm::cast<llvm::GenIntrinsicInst>(pInst);

    switch (pGenIntrinsicInst->getIntrinsicID()) {
    case llvm::GenISAIntrinsic::GenISA_threadgroupbarrier:
      return true;
    default:
      break;
    }
  }

  return false;
}

////////////////////////////////////////////////////////////////////////
bool SynchronizationObjectCoalescing::IsUntypedMemoryFenceOperation(const llvm::Instruction *pInst) {
  if (llvm::isa<llvm::GenIntrinsicInst>(pInst)) {
    const llvm::GenIntrinsicInst *pGenIntrinsicInst = llvm::cast<llvm::GenIntrinsicInst>(pInst);

    switch (pGenIntrinsicInst->getIntrinsicID()) {
    case llvm::GenISAIntrinsic::GenISA_memoryfence:
      return true;
    default:
      break;
    }
  }
  return false;
}

////////////////////////////////////////////////////////////////////////
bool SynchronizationObjectCoalescing::IsUntypedMemoryFenceOperationWithInvalidationFunctionality(
    const llvm::Instruction *pInst) const {
  constexpr uint32_t L1CacheInvalidateArg = 6;
  return IsUntypedMemoryFenceOperation(pInst) &&
         llvm::cast<llvm::ConstantInt>(pInst->getOperand(L1CacheInvalidateArg))->getValue().getBoolValue();
}

////////////////////////////////////////////////////////////////////////
bool SynchronizationObjectCoalescing::IsUntypedMemoryFenceOperationForSharedMemoryAccess(
    const llvm::Instruction *pInst) const {
  constexpr uint32_t globalMemFenceArg = 5;
  return IsUntypedMemoryFenceOperation(pInst) &&
         (!m_HasIndependentSharedMemoryFenceFunctionality ||
          llvm::cast<llvm::ConstantInt>(pInst->getOperand(globalMemFenceArg))->getValue().getBoolValue() == false);
}

////////////////////////////////////////////////////////////////////////
bool SynchronizationObjectCoalescing::IsUntypedMemoryFenceOperationForGlobalAccess(
    const llvm::Instruction *pInst) const {
  constexpr uint32_t globalMemFenceArg = 5;
  return IsUntypedMemoryFenceOperation(pInst) &&
         llvm::cast<llvm::ConstantInt>(pInst->getOperand(globalMemFenceArg))->getValue().getBoolValue();
}

////////////////////////////////////////////////////////////////////////
bool SynchronizationObjectCoalescing::IsUntypedMemoryLscFenceOperationForGlobalAccess(
    const llvm::Instruction *pInst) const {
  if (IsLscFenceOperation(pInst)) {
    LSC_SFID mem = GetLscMem(pInst);
    return mem == LSC_SFID::LSC_UGM || mem == LSC_SFID::LSC_UGML;
  }
  return false;
}

////////////////////////////////////////////////////////////////////////
bool SynchronizationObjectCoalescing::IsSharedLscFenceOperation(const llvm::Instruction *pInst) const {
  if (IsLscFenceOperation(pInst)) {
    LSC_SFID mem = GetLscMem(pInst);
    return mem == LSC_SFID::LSC_SLM;
  }
  return false;
}

////////////////////////////////////////////////////////////////////////
bool SynchronizationObjectCoalescing::IsTypedMemoryLscFenceOperation(const llvm::Instruction *pInst) const {
  if (IsLscFenceOperation(pInst)) {
    LSC_SFID mem = GetLscMem(pInst);
    return mem == LSC_SFID::LSC_TGM;
  }
  return false;
}

////////////////////////////////////////////////////////////////////////
bool SynchronizationObjectCoalescing::IsTypedMemoryFenceOperation(const llvm::Instruction *pInst) {
  if (llvm::isa<llvm::GenIntrinsicInst>(pInst)) {
    const llvm::GenIntrinsicInst *pGenIntrinsicInst = llvm::cast<llvm::GenIntrinsicInst>(pInst);

    switch (pGenIntrinsicInst->getIntrinsicID()) {
    case llvm::GenISAIntrinsic::GenISA_typedmemoryfence:
      return true;
    default:
      break;
    }
  }

  return false;
}

////////////////////////////////////////////////////////////////////////
bool SynchronizationObjectCoalescing::IsTypedMemoryFenceOperationWithInvalidationFunctionality(
    const llvm::Instruction *pInst) const {
  constexpr uint32_t L1CacheInvalidateArg = 0;
  return IsTypedMemoryFenceOperation(pInst) &&
         llvm::cast<llvm::ConstantInt>(pInst->getOperand(L1CacheInvalidateArg))->getValue().getBoolValue();
}

////////////////////////////////////////////////////////////////////////
bool SynchronizationObjectCoalescing::IsUrbLscFenceOperation(const llvm::Instruction *pInst) const {
  if (IsLscFenceOperation(pInst)) {
    LSC_SFID mem = GetLscMem(pInst);
    return mem == LSC_SFID::LSC_URB;
  }
  return false;
}

////////////////////////////////////////////////////////////////////////
bool SynchronizationObjectCoalescing::IsUrbFenceOperation(const llvm::Instruction *pInst) {
  if (llvm::isa<llvm::GenIntrinsicInst>(pInst)) {
    const llvm::GenIntrinsicInst *pGenIntrinsicInst = llvm::cast<llvm::GenIntrinsicInst>(pInst);

    switch (pGenIntrinsicInst->getIntrinsicID()) {
    case llvm::GenISAIntrinsic::GenISA_urbfence:
      return true;
    default:
      break;
    }
  }

  return false;
}

////////////////////////////////////////////////////////////////////////////////
static inline bool IsLscFenceOperation(const Instruction *pInst) {
  const GenIntrinsicInst *pIntr = dyn_cast<GenIntrinsicInst>(pInst);
  if (pIntr && pIntr->isGenIntrinsic(GenISAIntrinsic::GenISA_LSCFence)) {
    return true;
  }
  return false;
}

static bool IsAsyncRaytracingOperation(const Instruction *pInst) {
  if (auto *GII = dyn_cast<GenIntrinsicInst>(pInst)) {
    switch (GII->getIntrinsicID()) {
    case GenISAIntrinsic::GenISA_TraceRayAsync:
    case GenISAIntrinsic::GenISA_BindlessThreadDispatch_1_0:
      return true;
    default:
      break;
    }
  }

  return false;
}

static bool IsSyncRaytracingOperation(const Instruction *pInst) {
  if (auto *GII = dyn_cast<GenIntrinsicInst>(pInst)) {
    switch (GII->getIntrinsicID()) {
    case GenISAIntrinsic::GenISA_TraceRaySync:
      return true;
    default:
      break;
    }
  }

  return false;
}

////////////////////////////////////////////////////////////////////////////////
static inline LSC_SFID GetLscMem(const Instruction *pInst) {
  IGC_ASSERT(IsLscFenceOperation(pInst));
  return getImmValueEnum<LSC_SFID>(pInst->getOperand(0));
}

////////////////////////////////////////////////////////////////////////////////
static inline LSC_SCOPE GetLscScope(const Instruction *pInst) {
  IGC_ASSERT(IsLscFenceOperation(pInst));
  return getImmValueEnum<LSC_SCOPE>(pInst->getOperand(1));
}

////////////////////////////////////////////////////////////////////////////////
static inline LSC_FENCE_OP GetLscFenceOp(const Instruction *pInst) {
  IGC_ASSERT(IsLscFenceOperation(pInst));
  return getImmValueEnum<LSC_FENCE_OP>(pInst->getOperand(2));
}
////////////////////////////////////////////////////////////////////////
InstructionMask SynchronizationObjectCoalescing::GetInstructionMask(const llvm::Instruction *pInst) const {
  if (IsAtomicOperation(pInst)) {
    return InstructionMask::AtomicOperation;
  } else if (IsTypedReadOperation(pInst)) {
    return InstructionMask::TypedReadOperation;
  } else if (IsTypedWriteOperation(pInst)) {
    return InstructionMask::TypedWriteOperation;
  } else if (IsOutputUrbReadOperation(pInst)) {
    return InstructionMask::OutputUrbReadOperation;
  } else if (IsUrbWriteOperation(pInst)) {
    return InstructionMask::UrbWriteOperation;
  } else if (IsBufferReadOperation(pInst)) {
    return InstructionMask::BufferReadOperation;
  } else if (IsBufferWriteOperation(pInst)) {
    return InstructionMask::BufferWriteOperation;
  } else if (IsSharedMemoryReadOperation(pInst)) {
    return InstructionMask::SharedMemoryReadOperation;
  } else if (IsSharedMemoryWriteOperation(pInst)) {
    return InstructionMask::SharedMemoryWriteOperation;
  } else if (IsAsyncRaytracingOperation(pInst)) {
    return AllNoAtomicMask;
  } else if (IsExtendedCacheControlOperation(pInst)) {
    return InstructionMask::ExtendedCacheControlOperation;
  } else if (IsSyncRaytracingOperation(pInst)) {
    return InstructionMask::BufferReadOperation | InstructionMask::BufferWriteOperation;
  } else if (IsReturnOperation(pInst)) {
    return InstructionMask::EndOfThreadOperation;
  }

  return InstructionMask::None;
}

////////////////////////////////////////////////////////////////////////
bool SynchronizationObjectCoalescing::IsFenceOperation(const llvm::Instruction *pInst) {
  return IsTypedMemoryFenceOperation(pInst) || IsUrbFenceOperation(pInst) || IsLscFenceOperation(pInst) ||
         IsUntypedMemoryFenceOperation(pInst);
}

////////////////////////////////////////////////////////////////////////
/// Returns true for resources in the global memory (storage buffers or
/// storage images) that are not marked as ReadOnly in the shader.
bool SynchronizationObjectCoalescing::IsGlobalWritableResource(llvm::Type *pResourePointerType) {
  uint as = pResourePointerType->getPointerAddressSpace();
  switch (as) {
  case ADDRESS_SPACE_LOCAL:
  case ADDRESS_SPACE_GENERIC:
    return false;
  case ADDRESS_SPACE_GLOBAL:
    return true;
  default: {
    BufferType bufferType = DecodeBufferType(as);
    switch (bufferType) {
    case IGC::UAV:
    case IGC::BINDLESS:
    case IGC::BINDLESS_WRITEONLY:
    case IGC::STATELESS:
    case IGC::SSH_BINDLESS:
      return true;
    case IGC::CONSTANT_BUFFER:
    case IGC::RESOURCE:
    case IGC::SLM:
    case IGC::POINTER:
    case IGC::BINDLESS_CONSTANT_BUFFER:
    case IGC::BINDLESS_TEXTURE:
    case IGC::BINDLESS_READONLY:
    case IGC::SAMPLER:
    case IGC::BINDLESS_SAMPLER:
    case IGC::RENDER_TARGET:
    case IGC::STATELESS_READONLY:
    case IGC::STATELESS_A32:
    case IGC::SSH_BINDLESS_CONSTANT_BUFFER:
    case IGC::SSH_BINDLESS_TEXTURE:
    case IGC::BUFFER_TYPE_UNKNOWN:
      return false;
    }
  }
  }
  return false;
}

////////////////////////////////////////////////////////////////////////
bool SynchronizationObjectCoalescing::IsSharedMemoryResource(llvm::Type *pResourePointerType) {
  uint as = pResourePointerType->getPointerAddressSpace();
  switch (as) {
  case ADDRESS_SPACE_LOCAL:
    return true;
  default:
    return false;
  }
}

#if _DEBUG
////////////////////////////////////////////////////////////////////////
/// @brief Registers information which allows understanding the decision
/// about identifying redundancy for this synchronization instruction.
void SynchronizationObjectCoalescing::RegisterRedundancyExplanation(const llvm::Instruction *pInst,
                                                                    ExplanationEntry::Cause cause) {
  constexpr bool forwardDirection = true;
  constexpr bool backwardDirection = false;

  auto GetIndex = [](const llvm::Instruction *pInst) {
    uint32_t i = 0;
    for (auto it = pInst->getParent()->begin(); it != pInst->getIterator(); it++, i++) {
    }
    return i;
  };

  auto GetBoundaryInst = [this, GetIndex](const llvm::Instruction *pInst) -> ExplanationEntry::SyncInstDescription {
    std::string stringRepresentation;
    llvm::raw_string_ostream stream(stringRepresentation);
    pInst->print(stream, true /*isForDeubg*/);
    return {pInst->getParent(), GetIndex(pInst), stringRepresentation};
  };

  m_ExplanationEntries.emplace_back();
  ExplanationEntry &explanationEntry = m_ExplanationEntries.back();
  explanationEntry.m_SynchronizationDescription = GetBoundaryInst(pInst);
  explanationEntry.m_Cause = cause;

  if (explanationEntry.m_Cause == ExplanationEntry::Cause::FastStrictRedundancy) {
    return;
  }

  std::vector<const llvm::Instruction *> forwardBoundaryInstructions;
  std::vector<const llvm::Instruction *> backwardBoundaryInstructions;
  std::vector<const llvm::Instruction *> threadGroupBarriersBoundaries;
  GetVisibleMemoryInstructions(pInst, forwardDirection, forwardBoundaryInstructions,
                               explanationEntry.m_ForwardMemoryInstructions);
  GetVisibleMemoryInstructions(pInst, backwardDirection, backwardBoundaryInstructions,
                               explanationEntry.m_BackwardMemoryInstructions);
  if (!IsThreadBarrierOperation(pInst) && cause == ExplanationEntry::Cause::StrictRedundancy) {
    GetAllUnsynchronizedMemoryInstructions(pInst, threadGroupBarriersBoundaries,
                                           explanationEntry.m_ForwardMemoryInstructions);
  }

  for (const llvm::Instruction *pSubstituteInst : forwardBoundaryInstructions) {
    explanationEntry.m_ForwardBoundaries.push_back(GetBoundaryInst(pSubstituteInst));
  }

  for (const llvm::Instruction *pSubstituteInst : backwardBoundaryInstructions) {
    explanationEntry.m_BackwardBoundaries.push_back(GetBoundaryInst(pSubstituteInst));
  }

  for (const llvm::Instruction *pThreadGroupBarrier : threadGroupBarriersBoundaries) {
    explanationEntry.m_ThreadGroupBarriersBoundaries.push_back(GetBoundaryInst(pThreadGroupBarrier));
  }
}

////////////////////////////////////////////////////////////////////////
/// @brief Print explanation for all redundancies.
void SynchronizationObjectCoalescing::print(llvm::raw_ostream &stream,
                                            bool onlyMemoryInstructionMask /* = true*/) const {
  auto GetRedundancyCauseName = [](ExplanationEntry::Cause cause) {
    switch (cause) {
    case ExplanationEntry::GlobalMemoryRedundancy:
      return "global memory";
    case ExplanationEntry::L1CacheInvalidationRedundancy:
      return "L1 cache invalidation";
    case ExplanationEntry::StrictRedundancy:
      return "strict";
    case ExplanationEntry::FastStrictRedundancy:
      return "fast strict";
    default:
      return "unknown";
    }
  };

  auto GetSyncInstName = [this, GetRedundancyCauseName](const ExplanationEntry::SyncInstDescription &syncInstDesc) {
    std::stringstream syncDescName;
    syncDescName << std::get<0>(syncInstDesc)->getName().str() << ", " << std::get<1>(syncInstDesc) << ", "
                 << std::get<2>(syncInstDesc);
    return syncDescName.str();
  };

  auto GetInstructionMaskBitName = [this](InstructionMask bit) {
    switch (bit) {
    case InstructionMask::None:
      return "None";
    case InstructionMask::AtomicOperation:
      return "AtomicOperation";
    case InstructionMask::TypedReadOperation:
      return "TypedReadOperation";
    case InstructionMask::TypedWriteOperation:
      return "TypedWriteOperation";
    case InstructionMask::OutputUrbReadOperation:
      return "OutputUrbReadOperation";
    case InstructionMask::UrbWriteOperation:
      return "UrbWriteOperation";
    case InstructionMask::BufferReadOperation:
      return "BufferReadOperation";
    case InstructionMask::BufferWriteOperation:
      return "BufferWriteOperation";
    case InstructionMask::SharedMemoryReadOperation:
      return "SharedMemoryReadOperation";
    case InstructionMask::SharedMemoryWriteOperation:
      return "SharedMemoryWriteOperation";
    default:
      return "Unknown";
    }
  };

  auto GetInstructionMaskName = [this, GetInstructionMaskBitName](InstructionMask mask) {
    std::stringstream name;
    if (mask != 0) {
      bool appendSeparator = false;
      for (uint32_t bit = 1; bit <= InstructionMask::SharedMemoryWriteOperation; bit <<= 1) {
        if ((bit & mask) != 0) {
          if (appendSeparator) {
            name << " | ";
          }
          name << GetInstructionMaskBitName(static_cast<InstructionMask>(bit));
          appendSeparator = true;
        }
      }
    } else {
      name << GetInstructionMaskBitName(InstructionMask::None);
    }
    return name.str();
  };

  auto GetInstructionDescription = [this, GetInstructionMaskBitName](const llvm::Instruction *pInst) {
    constexpr bool isForDebug = true;
    std::string result;
    llvm::raw_string_ostream stream(result);
    pInst->print(stream, isForDebug);
    stream << " (" << GetInstructionMaskBitName(GetInstructionMask(pInst)) << ")";
    stream.flush();
    return result;
  };

  uint32_t index = 0;
  std::string indentation = "  ";

  for (const ExplanationEntry &explanationEntry : m_ExplanationEntries) {
    stream << index << ". A " << GetRedundancyCauseName(explanationEntry.m_Cause) << " redundancy." << "\n";
    stream << indentation << GetSyncInstName(explanationEntry.m_SynchronizationDescription) << "\n";
    if (explanationEntry.m_Cause == ExplanationEntry::Cause::FastStrictRedundancy) {
      stream << indentation << "The global instruction mask: " << GetInstructionMaskName(m_GlobalMemoryInstructionMask)
             << "\n";
      stream << "\n";
      continue;
    }
    stream << indentation << "The forward instruction mask: "
           << GetInstructionMaskName(GetInstructionMask(explanationEntry.m_ForwardMemoryInstructions)) << "\n";
    stream << indentation << "The backward instruction mask: "
           << GetInstructionMaskName(GetInstructionMask(explanationEntry.m_BackwardMemoryInstructions)) << "\n";
    stream << "\n";
    if (!onlyMemoryInstructionMask) {
      if (!explanationEntry.m_ForwardBoundaries.empty()) {
        stream << indentation << "The forward boundary instructions: " << "\n";
        for (const ExplanationEntry::SyncInstDescription &syncDesc : explanationEntry.m_ForwardBoundaries) {
          stream << indentation << indentation << GetSyncInstName(syncDesc) << "\n";
        }
        stream << "\n";
      } else {
        stream << indentation << "No forward boundary instructions." << "\n\n";
      }
      if (!explanationEntry.m_ThreadGroupBarriersBoundaries.empty()) {
        stream << indentation << "The boundary thread group barrier instructions:" << "\n";
        for (const ExplanationEntry::SyncInstDescription &syncDesc : explanationEntry.m_ThreadGroupBarriersBoundaries) {
          stream << indentation << indentation << GetSyncInstName(syncDesc) << "\n";
        }
        stream << "\n";
      } else {
        stream << indentation << "No boundary thread group barrier instructions." << "\n\n";
      }
      if (!explanationEntry.m_BackwardBoundaries.empty()) {
        stream << indentation << "The backward boundary instructions:" << "\n";
        for (const ExplanationEntry::SyncInstDescription &syncDesc : explanationEntry.m_BackwardBoundaries) {
          stream << indentation << indentation << GetSyncInstName(syncDesc) << "\n";
        }
        stream << "\n";
      } else {
        stream << indentation << "No backward boundary instructions." << "\n\n";
      }
      if (!explanationEntry.m_ForwardMemoryInstructions.empty()) {
        stream << indentation << "The forward reachable memory instructions:" << "\n";
        for (const llvm::Instruction *pInst : explanationEntry.m_ForwardMemoryInstructions) {
          stream << indentation << indentation << GetInstructionDescription(pInst) << "\n";
        }
        stream << "\n";
      } else {
        stream << indentation << "No forward reachable memory instructions." << "\n\n";
      }
      if (!explanationEntry.m_BackwardMemoryInstructions.empty()) {
        stream << indentation << "The backward reachable memory instructions:" << "\n";
        for (const llvm::Instruction *pInst : explanationEntry.m_BackwardMemoryInstructions) {
          stream << indentation << indentation << GetInstructionDescription(pInst) << "\n";
        }
        stream << "\n";
      } else {
        stream << indentation << "No backward reachable memory instructions." << "\n\n";
      }
    }

    index++;
  }
}

////////////////////////////////////////////////////////////////////////
/// @brief Dumps explanation for all redundancies.
void SynchronizationObjectCoalescing::dump(bool onlyMemoryInstructionMask /*= true*/) const {
  print(llvm::dbgs(), onlyMemoryInstructionMask);
}
#endif // _DEBUG

////////////////////////////////////////////////////////////////////////
llvm::Pass *createSynchronizationObjectCoalescing() { return new SynchronizationObjectCoalescing(); }

} // namespace IGC

using namespace llvm;
using namespace IGC;

#define PASS_FLAG "igc-synchronization-object-coalescing"
#define PASS_DESCRIPTION "SynchronizationObjectCoalescing"
#define PASS_CFG_ONLY false
#define PASS_ANALYSIS false
IGC_INITIALIZE_PASS_BEGIN(SynchronizationObjectCoalescing, PASS_FLAG, PASS_DESCRIPTION, PASS_CFG_ONLY, PASS_ANALYSIS)
IGC_INITIALIZE_PASS_DEPENDENCY(CodeGenContextWrapper)
IGC_INITIALIZE_PASS_END(SynchronizationObjectCoalescing, PASS_FLAG, PASS_DESCRIPTION, PASS_CFG_ONLY, PASS_ANALYSIS)
