/*========================== begin_copyright_notice ============================
Copyright (C) 2024-2026 Intel Corporation

SPDX-License-Identifier: MIT
============================= end_copyright_notice ===========================*/

/*!
 * @file A tool that profiles accessed cachelines
 * The following modes are supported:
 * 1. Cacheline histogram    - amount of cache lines accessed by the kernel by each SEND instruction. This mode supports all addressing modes.
 * 2. Memory footprint       - detects cache lines accessed by the kernel per read and per write separately. This mode supports A64 addressing mode only.
 * 3. Memory cache model     - models a direct-mapped cache and detects amount of cache hits and misses. This mode supports A64 addressing mode only.
 * 4. Memory access analysis - detects read-after-read, read-after-write, write-after-read, and write-after-write dependencies between memory accesses.
 *                             This mode supports A64 addressing mode only.
 */

#include <cstring>
#include <map>
#include <vector>
#include <list>
#include <set>
#include <algorithm>

#include "cachelineprof.h"
#include "gen_send_decoder.h"

#include "gtpin_api.h"
#include "gtpin_tool_utils.h"
#include "gt_basic_utils.h"
#include "ged.h"

enum MODE {
    MODE_CACHE_HISTOGRAM                           = 1,
    MODE_MEMORY_FOOTPRINT                          = 2,
    MODE_MEMORY_CACHE_MODEL                        = 3,
    MODE_MEMORY_ACCESS_ANALYSIS                    = 4,
    MODE_MEMORY_RACE_DETECTION                     = 5,
    MODE_MEMORY_ACCESS_ANALYSIS_AND_RACE_DETECTION = 6,
};

constexpr uint32_t ONE_KB  = (1024ull);
constexpr uint64_t FOUR_GB = (4 * 1024ull * 1024ull * 1024ull);
constexpr bool     hasComboParam = true;

/* ============================================================================================= */
// Configuration
/* ============================================================================================= */
Knob<bool> KNOB_NO_COUT("no_cout", false, "Do not send profiling results to standard output device");
Knob<bool> KNOB_IGNORE_SLM("ignore_slm", false, "Do not profile SLM accesses");
Knob<bool> KNOB_NO_CACHE_INVALIDATE("no_cache_invalidate", false, "No cache invalidation between enqueues");
Knob<bool> KNOB_ANALYSIS_INIT_TO_WRITE("init_to_write", false, "Initialize bitvector for analysis as write");
Knob<bool> KNOB_NO_INVALIDATE("no_invalidate", false, "No data invalidation between enqueues");
Knob<int>  KNOB_MODE("mode", MODE_CACHE_HISTOGRAM, "Profiling mode");
Knob<int>  KNOB_CACHE_SIZE("cache_size", 2048, "Size in KB of the cache (should be power of 2)");
Knob<int>  KNOB_CACHELINE_SIZE("cacheline_size", 64, "Size in bytes of the cacheline (should be power of 2)");
Knob<int>  KNOB_SUPPORTED_ADDRESS_BITS("supported_address_bits", 38, "Number of supported least significant bits within 64-bit address");
Knob<int>  KNOB_CACHE_MODEL_HASH_SHIFT("cache_model_hash_shift", 0, "Amount of bits to shift left the cache hash");

uint32_t gSupportedAddressBits = 0;
uint32_t gLog2CacheLineSize = 0;
size_t   gBitVectorSizeInBytes = 0;
size_t   gRaceDataVectorSize = 0;
size_t   gNumOfCacheBuckets = 0;
uint32_t gCacheSize = 0;

using BitVector = std::vector<uint8_t>;
using RaceDetectionDataVector = std::vector<uint32_t>;
using CacheBucketTagsVector = std::vector<uint64_t>;

/* ============================================================================================= */
// Class MemAccess
/* ============================================================================================= */
/// Information about memory access instruction
struct MemAccess
{
    /// Constructor. If memory access is unsupported, the reason can be queried by Error()
    explicit MemAccess(const IGtIns &ins);

    bool         IsValid()         const { return _isValid; }            ///< @return true for a supported memory access
    InsId        Id()              const { return _insId; }              ///< @return Instruction ID
    GtRegNum     FirstAddrReg()    const { return _firstAddrReg; }       ///< @return First register in the address payload
    uint32_t     NumAccesses()     const { return _numAccesses; }        ///< @return Number of elements in the address payload
    uint32_t     DataSize()        const { return _dataSize; }           ///< @return Data size, in bytes, per each address
    bool         IsSlm()           const { return _isSlm; }              ///< @return true for SLM access
    bool         IsBlock2DAccess() const { return _isBlock2D;  }         ///< @return true for block2d access
    const GtMemoryAddrModel& AddrModel() const { return _addrModel; } ///< @return Address model of the memory access
    const std::string&       Error()     const { return _errMsg; }    ///< @return Error message on unsupported memory access

    /*!
     * @return Arguments of the HLI function that compute cache line access histograms
     * @note This object owns the CachelineProfArgs structure, but its content is controlled externally
     */
    const CachelineProfArgs& GetCachelineProfArgs() const { return _clArgs; }
    CachelineProfArgs&       GetCachelineProfArgs()       { return _clArgs; }

private:
    bool                _isValid = false;       ///< True, if this structure represents supported memory access
    InsId               _insId;                 ///< ID of the memory access instruction
    GtMemoryAddrModel   _addrModel;             ///< Address model of the memory access
    GtRegNum            _firstAddrReg;          ///< First register in the address payload of the instruction
    uint32_t            _numAccesses = 0;       ///< Number of elements in the address payload of the instruction
    uint32_t            _dataSize = 0;          ///< Size, in bytes, of the memory range referenced by a single address
    CachelineProfArgs   _clArgs;                ///< Common argument of cacheline profiling functions

    std::string         _errMsg;                ///< Error message on unsupported memory access
    bool                _isSlm = false;         ///< True if the memory access is to SLM
    bool                _isBlock2D = false;     ///< True if the memory access is load_block2d or store_block2d
};

/* ============================================================================================= */
// Struct ProfileResults
/* ============================================================================================= */
/*!
 * Profile results per kernel dispatch / per insruction
 */
struct ProfileResults
{
    ProfileResults(const IGtKernelDispatch& dispatcher, const MemAccess& memAccess);

    uint64_t          dispatchId;                                      ///< Unique ID of the kernel dispatch assigned by GTPin
    GtKernelExecDesc  kernelExecDesc;                                  ///< Kernel execution descriptor
    InsId             insId;                                           ///< ID of the memory access instruction
    uint32_t          clHist[MAX_SUPPORTED_CACHE_LINE_HISTOGRAM_SIZE]; ///< Cacheline histogram
    uint32_t          outOfHistogramValues;                            ///< Counter how many times the value was out of histogram size
};

struct ProfileResultsMemFootprint
{
    ProfileResultsMemFootprint(const IGtKernelDispatch& dispatcher, const MemoryFootprintArgs& memfootprintArg, BitVector& bitvector);

    uint64_t                   dispatchId;       ///< Unique ID of the kernel dispatch assigned by GTPin
    GtKernelExecDesc           kernelExecDesc;   ///< Kernel execution descriptor
    BitVector                  clBitvector;      ///< Cacheline bitvector
    uint64_t                   upperAddressBits; ///< Upper bits of the addresses (assumed to be invariant for all kernel accesses per dispatch)
};

struct ProfileResultsMemCacheModel
{
    ProfileResultsMemCacheModel(const IGtKernelDispatch& dispatcher, const MemoryCacheModelArgs& memCacheModelArg);

    uint64_t                   dispatchId;       ///< Unique ID of the kernel dispatch assigned by GTPin
    GtKernelExecDesc           kernelExecDesc;   ///< Kernel execution descriptor
    uint64_t                   hits;             ///< Amount of counted cache hits
    uint64_t                   misses;           ///< Amount of counted cache misses
};

struct ProfileResultsMemAccessAnalysis
{
    ProfileResultsMemAccessAnalysis(const IGtKernelDispatch& dispatcher, const MemoryAccessAnalysisArgs& memAccessAnalysisArg);

    uint64_t                   dispatchId;       ///< Unique ID of the kernel dispatch assigned by GTPin
    GtKernelExecDesc           kernelExecDesc;   ///< Kernel execution descriptor
    uint64_t                   rarCount;         ///< Number of Read-After-Read accesses
    uint64_t                   rawCount;         ///< Number of Read-After-Write accesses
    uint64_t                   warCount;         ///< Number of Write-After-Read accesses
    uint64_t                   wawCount;         ///< Number of Write-After-Write accesses
};

struct ProfileResultsMemRaceDetection
{
    ProfileResultsMemRaceDetection(const IGtKernelDispatch& dispatcher, const MemoryRaceDetectionArgs& memRaceDetectionArg, BitVector& bitvector);

    uint64_t                   dispatchId;       ///< Unique ID of the kernel dispatch assigned by GTPin
    GtKernelExecDesc           kernelExecDesc;   ///< Kernel execution descriptor
    BitVector                  clBitvector;      ///< Cacheline bitvector
    uint64_t                   raceCount;        ///< Number of detected race conditions
    uint64_t                   rarCount;         ///< Number of Read-After-Read accesses
    uint64_t                   rawCount;         ///< Number of Read-After-Write accesses
    uint64_t                   warCount;         ///< Number of Write-After-Read accesses
    uint64_t                   wawCount;         ///< Number of Write-After-Write accesses
    uint64_t                   upperAddressBits; ///< Upper bits of the addresses (assumed to be invariant for all kernel accesses per dispatch)
};

/* ============================================================================================= */
// Class KernelProfile
/* ============================================================================================= */
/// Static properties of the kernel, and its profile data updated on each kernel run
class KernelProfile
{
public:
    using MemAccessMap = std::map<InsId, MemAccess>;              ///< Information about memory accesses by kernel instructions

public:
    KernelProfile(const IGtKernel& kernel, const IGtCfg& cfg);    ///< Constructor
    ~KernelProfile() = default;

    inline GtKernelId                 Id()                    const; ///< @return Unique identifier of the kernel
    inline GtGpuPlatform              Platform()              const; ///< @return Kernel's platform
    inline const std::string&         Name()                  const; ///< @return Name of the kernel
    inline const std::string&         UniqueName()            const; ///< @return Unique name of the kernel
    inline std::string                CachelineProfResults()  const; ///< @return Profiling results of kernel runs, in text format
    inline void                       DumpAsm()               const; ///< Store kernel's assembly text in the file
    inline MemAccessMap&              GetMemAccessMap();             ///< @return Information about memory accesses in the kernel
    inline BitVector&                 GetMemoryFootprintBitvector(); ///< @return memory footprint bitvector
    inline RaceDetectionDataVector&   GetRaceDetectionDataVector();  ///< @return race detection data vector
    inline CacheBucketTagsVector&     GetCacheBucketTagsVector();    ///< @return memory cache bucket tags vector
    inline MemoryFootprintArgs&       GetMemoryFootprintArg();       ///< @return argument for memory footprint
    inline MemoryCacheModelArgs&      GetMemoryCacheModelArg();      ///< @return argument for memory cache model
    inline MemoryAccessAnalysisArgs&  GetMemoryAccessAnalysisArg();  ///< @return argument for memory access analysis
    inline MemoryRaceDetectionArgs&   GetMemoryRaceDetectionArg();   ///< @return argument for memory race detection


    void RecordCachelineProfResults(IGtKernelDispatch& dispatcher); ///< Update profile data with the latest memory check results
    void RecordUnsupportedInstruction(const IGtIns& ins, const std::string& errMsg); ///< Record unsupported instruction

private:
    GtKernelId                                 _id;                              ///< Unique identifier of the kernel
    GtGpuPlatform                              _platform;                        ///< Kernel's platform
    std::string                                _name;                            ///< Name of the kernel
    std::string                                _uniqueName;                      ///< Unique name of the kernel
    std::string                                _asmText;                         ///< Assembly text of the kernel
    std::string                                _unhandledAccesses;               ///< List of unhadled memory accesses, in text format

    MemAccessMap                               _memAccessMap;                    ///< Map: Instruction ID to memory access information
    MemoryFootprintArgs                        _memFootprintArgs;                ///< Argument for memory footprint mode
    MemoryCacheModelArgs                       _memCacheModelArgs;               ///< Argument for memory cache model mode
    MemoryAccessAnalysisArgs                   _memAccessAnalysisArgs;           ///< Argument for memory access analysis mode
    MemoryRaceDetectionArgs                    _memRaceDetectionArgs;            ///< Argument for memory race condition detection mode
    BitVector                                  _memFootprintBitvector;           ///< Bitvector for memory footprint
    RaceDetectionDataVector                    _memRaceDetectionDataVector;      ///< Vector of race detection data
    CacheBucketTagsVector                      _memCacheBucketTagsVector;        ///< Vector of cache buckets tags
    std::list<ProfileResults>                  _profileResultsHist;              ///< Profile results per kernel dispatch / per insruction 
    std::list<ProfileResultsMemFootprint>      _profileResultsMemFootprint;      ///< Profile results per kernel dispatch for memory footprint mode
    std::list<ProfileResultsMemCacheModel>     _profileResultsMemCacheModel;     ///< Profile results per kernel dispatch for memory cache model mode
    std::list<ProfileResultsMemAccessAnalysis> _profileResultsMemAccessAnalysis; ///< Profile results per kernel dispatch for memory access analysis mode
    std::list<ProfileResultsMemRaceDetection>
                                               _profileResultsMemRaceDetection; ///< Profile results per kernel dispatch for memory race condition detection mode
};

/* ============================================================================================= */
// Class CachelineProf
/* ============================================================================================= */
/*!
 * A tool that detects amount of cache lines accessed by kernels
 */
class CachelineProf : public GtTool
{
public:
    // Implementation of the IGtTool interface
    const char* Name()          const                   override { return "Cachelineprof"; }
    void        OnKernelBuild(IGtKernelInstrument&)     override;
    void        OnKernelRun(IGtKernelDispatch&)         override;
    void        OnKernelComplete(IGtKernelDispatch&)    override;

    void                  LoadHliLibrary();                   ///< Compile and load library of HLI functions
    static CachelineProf* Instance();                         ///< Return single instance of this class
    static void           OnFini()    { Instance()->Fini(); } ///< Termination handler registered with atexit()

private:
    
    CachelineProf();                                            ///< Default constructor
    CachelineProf(const CachelineProf&) = delete;               ///< Disabled copy constructor
    CachelineProf& operator = (const CachelineProf&) = delete;  ///< Disabled assignment operator
    ~CachelineProf();                                           ///< Destructor
    void Fini();                                                ///< Post process and dump profiling data

    /*!
     * Insert a call to HLI function that detects amount of cache lines accessed by the specified instruction
     * @param[in]   ins             The memory access instruction
     * @param[in]   memAccess       Information about memory access
     * @param[in]   instrumentor    Instrumentation interface
     * @return true - success, false - the instruction or memory operation is not supported
     */
    bool InsertCachelineProf(const IGtIns &ins, const MemAccess& memAccess, IGtKernelInstrument& instrumentor);
    bool InsertMemoryFootprintProf(const IGtIns&              ins,
                                   const MemAccess&           memAccess,
                                   const MemoryFootprintArgs& memfootprintArg,
                                   const BitVector&           bitvector,
                                   IGtKernelInstrument&       instrumentor);
    bool InsertMemoryCacheModelProf(const IGtIns&                ins,
                                    const MemAccess&             memAccess,
                                    const MemoryCacheModelArgs&  memCacheModelArgs,
                                    const CacheBucketTagsVector& cacheBucketTagsVector,
                                    IGtKernelInstrument&         instrumentor);
    bool InsertMemoryAccessAnalysisProf(const IGtIns&                   ins,
                                        const MemAccess&                memAccess,
                                        const MemoryAccessAnalysisArgs& memAccessAnalysisArg,
                                        const BitVector&                bitvector,
                                        IGtKernelInstrument&            instrumentor);
    bool InsertRaceDetectionProf(const IGtIns& ins,
                                 const MemAccess&     memAccess,
                                 const MemoryRaceDetectionArgs& memRaceDetectionArg,
                                 const RaceDetectionDataVector& raceDetectionDataVector,
                                 const BitVector&               bitvector,
                                 IGtKernelInstrument&           instrumentor);

private:
    // Cacheline profiling functions
    CheckScatterA32AccessFunc                       _checkScatterA32AccessFunc;
    CheckScatterA64AccessFunc                       _checkScatterA64AccessFunc;

    GlobalMemoryFootprintA64AccessFunc              _globalMemoryFootprintA64AccessFunc;
    GlobalMemoryFootprintBlock2DAccessFunc          _globalMemoryFootprintBlock2DAccessFunc;

    GlobalMemoryCacheModelA64AccessFunc             _globalMemoryCacheModelA64AccessFunc;
    GlobalMemoryCacheModelBlock2DAccessFunc         _globalMemoryCacheModelBlock2DAccessFunc;

    GlobalMemoryAccessAnalysisA64AccessFunc         _globalMemoryAccessAnalysisA64AccessFunc;
    GlobalMemoryAccessAnalysisBlock2DAccessFunc     _globalMemoryAccessAnalysisBlock2DAccessFunc;

    GlobalMemoryRaceDetectionA64AccessFunc          _globalMemoryRaceDetectionA64AccessFunc;
    GlobalMemoryAccessAnalysisAndRaceDetectionA64AccessFunc  _globalMemoryAccessAnalysisAndRaceDetectionA64AccessFunc;

    IGtHliModuleHandle                              _hliModule = nullptr;        ///< Module of HLI functions
    std::map<GtKernelId, KernelProfile>             _kernels;                    ///< Collection of kernel profiles
};

/* ============================================================================================= */
// MemAccess implementation
/* ============================================================================================= */
MemAccess::MemAccess(const IGtIns &ins) : _insId(ins.Id())
{
    // Get and check data port (SFID)
    GtSfid sfid = ins.Sfid();
    if ((sfid != GED_SFID_UGM) && (sfid != GED_SFID_SLM) && (sfid != GED_SFID_DP_DC0) && (sfid != GED_SFID_DP_DC1))
    {
        _errMsg = "Unsupported data port " + std::string(sfid.ToString());
        return;
    }

    _isSlm = (sfid == GED_SFID_SLM);
    if (_isSlm && (KNOB_IGNORE_SLM || (KNOB_MODE == MODE_MEMORY_FOOTPRINT) || (KNOB_MODE == MODE_MEMORY_CACHE_MODEL) || (KNOB_MODE == MODE_MEMORY_ACCESS_ANALYSIS)))
    {
        _errMsg = "SLM access - ignored";
        return;
    }

    // Retrieve message descriptor
    if (!ins.MsgDescRegFile().IsImm())
    {
        _errMsg = "SEND message descriptor is not immediate";
        return;
    }

    // Initialize address model
    _addrModel = ins.MemAddrModel();
    if (!_addrModel.IsValid())
    {
        _errMsg = "Unsupported/unknown address model";
        return;
    }

    if (((KNOB_MODE == MODE_MEMORY_FOOTPRINT) || (KNOB_MODE == MODE_MEMORY_CACHE_MODEL) || (KNOB_MODE == MODE_MEMORY_ACCESS_ANALYSIS)) && !_addrModel.IsA64())
    {
        _errMsg = "Memory footprint, cache model and memory access analysis modes are supported for A64 access mode only";
        return;
    }

    // Finally, initialize the rest of data members...
    GTPIN_ASSERT(ins.SrcRegFile(0).IsGrf());
    _firstAddrReg = ins.SrcRegOperand(0).Reg().RegNum(); 
    _numAccesses  = ins.NumAccesses(); GTPIN_ASSERT(_numAccesses != 0);
    _isBlock2D    = ins.IsBlock2DAccess();

    DcSendMsg msg = DcSendMsg::Decode(ins.GetGedIns());
    _dataSize = msg.ElementSize() * msg.NumElements();

    if (_dataSize == 0)
    {
        _errMsg = "Unsupported SEND operation";
        return;
    }

    _isValid = true;
}

/* ============================================================================================= */
// ProfileResults implementation
/* ============================================================================================= */
ProfileResults::ProfileResults(const IGtKernelDispatch& dispatcher, const MemAccess& memAccess) :
    dispatchId(dispatcher.DispatchId()), insId(memAccess.Id())
{
    auto& clArg = memAccess.GetCachelineProfArgs();
    memcpy(&clHist, clArg.out.cachelineHistogram, sizeof(clArg.out.cachelineHistogram));
    outOfHistogramValues = clArg.out.outOfHistogramValues;
    dispatcher.GetExecDescriptor(kernelExecDesc);
}

ProfileResultsMemFootprint::ProfileResultsMemFootprint(const IGtKernelDispatch& dispatcher, const MemoryFootprintArgs& arg, BitVector& bitvector) :
    dispatchId(dispatcher.DispatchId())
{
    clBitvector.assign(bitvector.begin(), bitvector.end());
    upperAddressBits = arg.out.upperAddressBits;
    dispatcher.GetExecDescriptor(kernelExecDesc);
}

ProfileResultsMemCacheModel::ProfileResultsMemCacheModel(const IGtKernelDispatch& dispatcher, const MemoryCacheModelArgs& memCacheModelArg) :
    dispatchId(dispatcher.DispatchId())
{
    hits = memCacheModelArg.out.hits;
    misses = memCacheModelArg.out.misses;
    dispatcher.GetExecDescriptor(kernelExecDesc);
}

ProfileResultsMemAccessAnalysis::ProfileResultsMemAccessAnalysis(const IGtKernelDispatch& dispatcher, const MemoryAccessAnalysisArgs& memAccessAnalysisArg) :
    dispatchId(dispatcher.DispatchId())
{
    rarCount = memAccessAnalysisArg.out.rarCount;
    rawCount = memAccessAnalysisArg.out.rawCount;
    warCount = memAccessAnalysisArg.out.warCount;
    wawCount = memAccessAnalysisArg.out.wawCount;
    dispatcher.GetExecDescriptor(kernelExecDesc);
}

ProfileResultsMemRaceDetection::ProfileResultsMemRaceDetection(const IGtKernelDispatch& dispatcher, const MemoryRaceDetectionArgs& memRaceDetectionArg, BitVector& bitvector) :
    dispatchId(dispatcher.DispatchId())
{
    clBitvector.assign(bitvector.begin(), bitvector.end());
    raceCount        = memRaceDetectionArg.out.raceCount;
    rarCount         = memRaceDetectionArg.out.rarCount;
    rawCount         = memRaceDetectionArg.out.rawCount;
    warCount         = memRaceDetectionArg.out.warCount;
    wawCount         = memRaceDetectionArg.out.wawCount;
    upperAddressBits = memRaceDetectionArg.out.upperAddressBits;
    dispatcher.GetExecDescriptor(kernelExecDesc);
}

/* ============================================================================================= */
// KernelProfile implementation
/* ============================================================================================= */
KernelProfile::KernelProfile(const IGtKernel& kernel, const IGtCfg& cfg) :
    _id(kernel.Id()), _platform(kernel.GpuPlatform()), _name(GlueString(kernel.Name())), _uniqueName(kernel.UniqueName()),
    _asmText(CfgAsmText(cfg))
{
    // Populate this object with the information about memory accesses
    for (auto bblPtr : cfg.Bbls())
    {
        for (auto insPtr : bblPtr->Instructions())
        {
            const IGtIns& ins = *insPtr;
            if (ins.IsMemAccess() && !ins.IsEot())
            {
                MemAccess memAccess(ins);

                if (memAccess.IsValid())
                {
                    _memAccessMap.emplace(ins.Id(), memAccess);
                }
                else
                {
                    RecordUnsupportedInstruction(ins, memAccess.Error());
                }
            }
        }
    }

    if (KNOB_MODE == MODE_MEMORY_FOOTPRINT)
    {
        _memFootprintBitvector.resize(gBitVectorSizeInBytes, 0);
    }

    if (KNOB_MODE == MODE_MEMORY_CACHE_MODEL)
    {
        _memCacheBucketTagsVector.resize(gNumOfCacheBuckets, 0);
    }

    if (KNOB_MODE == MODE_MEMORY_ACCESS_ANALYSIS)
    {
        _memFootprintBitvector.resize(gBitVectorSizeInBytes, KNOB_ANALYSIS_INIT_TO_WRITE ? 0xFF : 0);
    }

    if (KNOB_MODE == MODE_MEMORY_RACE_DETECTION || KNOB_MODE == MODE_MEMORY_ACCESS_ANALYSIS_AND_RACE_DETECTION)
    {
        _memFootprintBitvector.resize(gBitVectorSizeInBytes, 0);
        _memRaceDetectionDataVector.resize(gRaceDataVectorSize, 0xFFFFFFFF);
    }
}

GtKernelId         KernelProfile::Id()          const { return _id; }
GtGpuPlatform      KernelProfile::Platform()    const { return _platform; }
const std::string& KernelProfile::Name()        const { return _name; }
const std::string& KernelProfile::UniqueName()  const { return _uniqueName; }
void               KernelProfile::DumpAsm()     const { DumpKernelAsmText(_name, _uniqueName, _asmText); }
KernelProfile::MemAccessMap&      KernelProfile::GetMemAccessMap()                    { return _memAccessMap; }
BitVector&                        KernelProfile::GetMemoryFootprintBitvector()        { return _memFootprintBitvector; }
RaceDetectionDataVector&          KernelProfile::GetRaceDetectionDataVector()         { return _memRaceDetectionDataVector; }
CacheBucketTagsVector&            KernelProfile::GetCacheBucketTagsVector()           { return _memCacheBucketTagsVector; }
MemoryFootprintArgs&              KernelProfile::GetMemoryFootprintArg()              { return _memFootprintArgs; }
MemoryCacheModelArgs&             KernelProfile::GetMemoryCacheModelArg()             { return _memCacheModelArgs; }
MemoryAccessAnalysisArgs&         KernelProfile::GetMemoryAccessAnalysisArg()         { return _memAccessAnalysisArgs; }
MemoryRaceDetectionArgs&          KernelProfile::GetMemoryRaceDetectionArg()          { return _memRaceDetectionArgs; }


std::string KernelProfile::CachelineProfResults() const
{
    std::ostringstream os;

    bool emptyResults = true;

    switch (KNOB_MODE) {
    case MODE_CACHE_HISTOGRAM:        emptyResults = _profileResultsHist.empty(); break;
    case MODE_MEMORY_FOOTPRINT:       emptyResults = _profileResultsMemFootprint.empty(); break;
    case MODE_MEMORY_CACHE_MODEL:     emptyResults = _profileResultsMemCacheModel.empty(); break;
    case MODE_MEMORY_ACCESS_ANALYSIS: emptyResults = _profileResultsMemAccessAnalysis.empty(); break;
    case MODE_MEMORY_RACE_DETECTION:
    case MODE_MEMORY_ACCESS_ANALYSIS_AND_RACE_DETECTION:
                                      emptyResults = _profileResultsMemRaceDetection.empty(); break;
    default: break;
    }

    if (emptyResults)
    {
        return os.str();
    }

    os << std::string(120, '-') << std::endl;
    os << std::setw(4) << _id << ":    " << _name << "___" << _uniqueName << std::endl;
    os << std::string(120, '-') << std::endl;

    if (KNOB_MODE == MODE_CACHE_HISTOGRAM)
    {
        for (const auto& res : _profileResultsHist)
        {
            bool clFound = false;
            for (uint32_t i = 0; i < MAX_SUPPORTED_CACHE_LINE_HISTOGRAM_SIZE; i++)
            {
                if (res.clHist[i])
                {
                    clFound = true;
                    break;
                }
            }
            if (clFound || res.outOfHistogramValues) {
                os << std::setw(10) << "Dispatch Id = " << std::dec << res.dispatchId << std::setw(20) << " Instruction Id = " << res.insId << std::endl;

                if (clFound)
                {
                    for (uint32_t i = 0; i < MAX_SUPPORTED_CACHE_LINE_HISTOGRAM_SIZE; i++)
                    {
                        if (res.clHist[i])
                        {
                            os << std::setw(20) << std::setw(4) << i << " cachelines were accessed " << std::setw(10) << res.clHist[i] << " times" << std::endl;
                        }
                    }
                }
                if (res.outOfHistogramValues)
                {
                    os << std::setw(20) << "A value bigger than the histogram size was detected " << std::setw(10) << res.outOfHistogramValues << " times" << std::endl;
                }
            }
        }
    }
    
    struct CacheLine {
        uint64_t start;
        uint64_t end;
        CacheLine(uint64_t s, uint64_t e) : start(s), end(e) {}
    };

    if (KNOB_MODE == MODE_MEMORY_FOOTPRINT)
    {
        for (const auto& res : _profileResultsMemFootprint)
        {
            uint32_t cacheLineSize = KNOB_CACHELINE_SIZE;

            auto GetCacheLines = [&](size_t size, uint8_t* reads, uint8_t* writes, std::list<CacheLine>& readOnly, std::list<CacheLine>& writesOnly, std::list<CacheLine>& readWrites)
            {
                for (uint32_t i = 0; i < size; i++)
                {
                    uint8_t readByte = reads[i];
                    uint8_t writeByte = writes ? writes[i] : 0;

                    if (readByte == 0 && writeByte == 0)
                    {
                        continue;
                    }
                    for (uint32_t j = 0; j < 8; j++)
                    {
                        // we assume the upper bits of the addresses are the same for all kernel accesses in the specific dispatch
                        uint64_t start = ((uint64_t)i * cacheLineSize * 8 + j * cacheLineSize) | res.upperAddressBits;
                        uint64_t end = start + cacheLineSize;

                        uint8_t readBit = readByte & (1 << j);
                        uint8_t writeBit = writeByte & (1 << j);
                        if (readBit && writeBit)
                        {
                            readWrites.push_back({ start, end });
                        }
                        else if (readBit)
                        {
                            readOnly.push_back({ start, end });
                        }
                        else if (writeBit)
                        {
                            writesOnly.push_back({ start, end });
                        }
                    }
                }
            };

            auto PrintAccessedLines = [&](std::string title, std::list<CacheLine>& accessedLines)
            {
                if (accessedLines.empty())
                {
                    return;
                }
                CacheLine first = accessedLines.front();
                accessedLines.pop_front();

                uint64_t totalAccessedSize = 0;

                os << title << std::endl;

                while (!accessedLines.empty())
                {
                    CacheLine next = accessedLines.front();
                    accessedLines.pop_front();

                    if (first.end == next.start)
                    {
                        first.end = next.end;
                    }
                    else
                    {
                        totalAccessedSize += first.end - first.start;
                        os << "  " << std::hex << first.start << " - " << first.end << std::endl;
                        first = next;
                    }
                }
                totalAccessedSize += first.end - first.start;
                os << "  " << std::hex << first.start << " - " << first.end << std::endl;
                os << "  " << "Total accessed size = 0x" << totalAccessedSize << std::endl;
            };

            os << std::setw(10) << "\nDispatch Id = " << std::dec << res.dispatchId << std::endl;

            size_t size = gBitVectorSizeInBytes >> 1;

            std::list<CacheLine> accessedLinesReadsOnly;
            std::list<CacheLine> accessedLinesWritesOnly;
            std::list<CacheLine> accessedLinesReadWrites;

            uint8_t* reads = (uint8_t*)res.clBitvector.data();
            uint8_t* writes = reads + size;

            GetCacheLines(size, reads, writes, accessedLinesReadsOnly, accessedLinesWritesOnly, accessedLinesReadWrites);

            if (accessedLinesReadsOnly.empty() && accessedLinesWritesOnly.empty() && accessedLinesReadWrites.empty())
            {
                os << "Total accessed size = 0" << std::endl;
                continue;
            }

            PrintAccessedLines("Reads cachelines:", accessedLinesReadsOnly);
            PrintAccessedLines("Writes cachelines:", accessedLinesWritesOnly);
            PrintAccessedLines("Read-Write cachelines:", accessedLinesReadWrites);
        }
    }

    if (KNOB_MODE == MODE_MEMORY_CACHE_MODEL)
    {
        uint64_t totalHits   = 0;
        uint64_t totalMisses = 0;

        for (const auto& res : _profileResultsMemCacheModel)
        {
            os << std::setw(10) << "Dispatch Id = " << std::dec << res.dispatchId << std::endl;
            os << "             Cache hits   = " << std::dec << res.hits << std::endl;
            os << "             Cache misses = " << std::dec << res.misses << std::endl;

            totalHits   += res.hits;
            totalMisses += res.misses;
        }

        os << "=====================" << std::endl;
        os << "Total cache hits   = " << std::dec << totalHits << std::endl;
        os << "Total cache misses = " << std::dec << totalMisses << std::endl;
    }

    if (KNOB_MODE == MODE_MEMORY_ACCESS_ANALYSIS)
    {
        uint64_t totalRAR = 0;
        uint64_t totalRAW = 0;
        uint64_t totalWAR = 0;
        uint64_t totalWAW = 0;
        for (const auto& res : _profileResultsMemAccessAnalysis)
        {
            os << std::setw(10) << "Dispatch Id = " << std::dec << res.dispatchId << std::endl;
            os << "             Read-After-Read   accesses = " << std::dec << res.rarCount << std::endl;
            os << "             Read-After-Write  accesses = " << std::dec << res.rawCount << std::endl;
            os << "             Write-After-Read  accesses = " << std::dec << res.warCount << std::endl;
            os << "             Write-After-Write accesses = " << std::dec << res.wawCount << std::endl;

            totalRAR += res.rarCount;
            totalRAW += res.rawCount;
            totalWAR += res.warCount;
            totalWAW += res.wawCount;
        }

        os << "=====================" << std::endl;
        os << "Total Read-After-Read   accesses = " << std::dec << totalRAR << std::endl;
        os << "Total Read-After-Write  accesses = " << std::dec << totalRAW << std::endl;
        os << "Total Write-After-Read  accesses = " << std::dec << totalWAR << std::endl;
        os << "Total Write-After-Write accesses = " << std::dec << totalWAW << std::endl;
    }

    if (KNOB_MODE == MODE_MEMORY_RACE_DETECTION || KNOB_MODE == MODE_MEMORY_ACCESS_ANALYSIS_AND_RACE_DETECTION)
    {
        uint64_t totalDetectedRaces = 0;
        for (const auto& res : _profileResultsMemRaceDetection)
        {
            auto PrintAccessedLines = [&](std::string title, std::list<CacheLine>& accessedLines)
            {
                if (accessedLines.empty())
                {
                    return;
                }

                os << title << std::endl;

                while (!accessedLines.empty())
                {
                    CacheLine cl = accessedLines.front();
                    accessedLines.pop_front();
                    os << "  " << std::hex << cl.start << std::endl;
                }
            };

            os << std::setw(10) << "Dispatch Id = " << std::dec << res.dispatchId << std::endl;

            if (KNOB_MODE == MODE_MEMORY_ACCESS_ANALYSIS_AND_RACE_DETECTION)
            {
                os << "             Read-After-Read   accesses = " << std::dec << res.rarCount << std::endl;
                os << "             Read-After-Write  accesses = " << std::dec << res.rawCount << std::endl;
                os << "             Write-After-Read  accesses = " << std::dec << res.warCount << std::endl;
                os << "             Write-After-Write accesses = " << std::dec << res.wawCount << std::endl;
            }

            if (res.raceCount == 0)
            {
                os << "             No race conditions detected" << std::endl;
                continue;
            }

            os << "             Race conditions detected   = " << std::dec << res.raceCount << std::endl << std::endl;

            std::list<CacheLine> cacheLines;
            const uint8_t* bitvector = res.clBitvector.data();
            uint32_t cacheLineSize = KNOB_CACHELINE_SIZE;

            for (uint32_t i = 0; i < res.clBitvector.size(); i++)
            {
                uint8_t byte = bitvector[i];

                if (byte == 0)
                {
                    continue;
                }
                for (uint32_t j = 0; j < 8; j++)
                {
                    if (!(byte & (1 << j)))
                    {
                        continue;
                    }
                    // we assume the upper bits of the addresses are the same for all kernel accesses in the specific dispatch
                    uint64_t start = ((uint64_t)i * cacheLineSize * 8 + j * cacheLineSize) | res.upperAddressBits;
                    uint64_t end = start + cacheLineSize;

                    cacheLines.push_back({ start, end });
                }
            }

            PrintAccessedLines("Races detected at these addresses:", cacheLines);

            totalDetectedRaces += res.raceCount;
        }

        os << "=====================" << std::endl;
        os << "Total Race conditions detected   = " << std::dec << totalDetectedRaces << std::endl;
    }

    return os.str();
}

void KernelProfile::RecordCachelineProfResults(IGtKernelDispatch& dispatcher)
{
    if (KNOB_MODE == MODE_CACHE_HISTOGRAM)
    {
        for (auto& entry : _memAccessMap)
        {
            MemAccess& memAccess = entry.second;
            _profileResultsHist.emplace_back(dispatcher, memAccess);
        }
    }
    if (KNOB_MODE == MODE_MEMORY_FOOTPRINT)
    {
        _profileResultsMemFootprint.emplace_back(dispatcher, _memFootprintArgs, _memFootprintBitvector);
    }
    if (KNOB_MODE == MODE_MEMORY_CACHE_MODEL)
    {
        _profileResultsMemCacheModel.emplace_back(dispatcher, _memCacheModelArgs);
    }
    if (KNOB_MODE == MODE_MEMORY_ACCESS_ANALYSIS)
    {
        _profileResultsMemAccessAnalysis.emplace_back(dispatcher, _memAccessAnalysisArgs);
    }
    if (KNOB_MODE == MODE_MEMORY_RACE_DETECTION || KNOB_MODE == MODE_MEMORY_ACCESS_ANALYSIS_AND_RACE_DETECTION)
    {
        _profileResultsMemRaceDetection.emplace_back(dispatcher, _memRaceDetectionArgs, _memFootprintBitvector);
    }
}

void KernelProfile::RecordUnsupportedInstruction(const IGtIns& ins, const std::string& errMsg)
{
    if (!errMsg.empty())
    {
        std::ostringstream os;
        os << errMsg << ": [" << std::setw(3) << ins.Id() << "] " << ins.ToString() << std::endl;
        _unhandledAccesses.append(os.str());
    }
}

/* ============================================================================================= */
// CachelineProf implementation
/* ============================================================================================= */
CachelineProf::CachelineProf() :
    _checkScatterA32AccessFunc("CheckScatterA32Access"),
    _checkScatterA64AccessFunc("CheckScatterA64Access"),
    _globalMemoryFootprintA64AccessFunc("GlobalMemoryFootprintA64Access", GtHliCallStd::IGC_STACK, hasComboParam),
    _globalMemoryFootprintBlock2DAccessFunc("GlobalMemoryFootprintBlock2DAccess", GtHliCallStd::IGC_STACK, hasComboParam),
    _globalMemoryCacheModelA64AccessFunc("GlobalMemoryCacheModelA64Access", GtHliCallStd::IGC_STACK, hasComboParam),
    _globalMemoryCacheModelBlock2DAccessFunc("GlobalMemoryCacheModelBlock2DAccess", GtHliCallStd::IGC_STACK, hasComboParam),
    _globalMemoryAccessAnalysisA64AccessFunc("GlobalMemoryAccessAnalysisA64Access", GtHliCallStd::IGC_STACK, hasComboParam),
    _globalMemoryAccessAnalysisBlock2DAccessFunc("GlobalMemoryAccessAnalysisBlock2DAccess", GtHliCallStd::IGC_STACK, hasComboParam),
    _globalMemoryRaceDetectionA64AccessFunc("GlobalMemoryRaceDetectionA64Access", GtHliCallStd::IGC_STACK, hasComboParam),
    _globalMemoryAccessAnalysisAndRaceDetectionA64AccessFunc("GlobalMemoryAccessAnalysisAndRaceDetectionA64Access", GtHliCallStd::IGC_STACK, hasComboParam)
{}

CachelineProf::~CachelineProf() {}

CachelineProf* CachelineProf::Instance()
{
    static CachelineProf instance;
    return &instance;
}

void CachelineProf::OnKernelBuild(IGtKernelInstrument& instrumentor)
{
    const IGtKernel& kernel     = instrumentor.Kernel();
    const IGtCfg&    cfg        = instrumentor.Cfg();
    IGtMemoryMapper& memMapper  = instrumentor.MemoryMapper();

    // Create profile for this kernel
    auto result = _kernels.emplace(std::piecewise_construct,
                                   std::forward_as_tuple(instrumentor.Kernel().Id()),
                                   std::forward_as_tuple(kernel, cfg));
    GTPIN_ASSERT(result.second);

    KernelProfile& kernelProfile = result.first->second;

    // Instrument memory accesses and share per-access arguments with HLI functions
    for (const auto& entry : kernelProfile.GetMemAccessMap())
    {
        const auto& memAccess = entry.second;
        auto        insId     = entry.first;

        if (int32_t(insId) < knobMinInstrumentIns || knobMaxInstrumentIns < int32_t(insId))
        {
            continue;
        }
        const IGtIns& ins = cfg.GetInstruction(insId);

        if (memAccess.IsValid())
        {
            switch (KNOB_MODE) {
            case MODE_CACHE_HISTOGRAM:
            {
                InsertCachelineProf(ins, memAccess, instrumentor);
                // Share per-access HLI arguments.
                // They will be initialized at the start of the kernel, and copied back to the host memory at completion of the kernel
                memMapper.Map(memAccess.GetCachelineProfArgs(), GT_MMAP_SHARE);
                break;
            }
            case MODE_MEMORY_FOOTPRINT:
            {
                InsertMemoryFootprintProf(ins,
                                          memAccess,
                                          kernelProfile.GetMemoryFootprintArg(),
                                          kernelProfile.GetMemoryFootprintBitvector(),
                                          instrumentor);
                break;
            }
            case MODE_MEMORY_CACHE_MODEL:
            {
                InsertMemoryCacheModelProf(ins,
                                            memAccess,
                                            kernelProfile.GetMemoryCacheModelArg(),
                                            kernelProfile.GetCacheBucketTagsVector(),
                                            instrumentor);
                break;
            }
            case MODE_MEMORY_ACCESS_ANALYSIS:
            {
                InsertMemoryAccessAnalysisProf(ins,
                                               memAccess,
                                               kernelProfile.GetMemoryAccessAnalysisArg(),
                                               kernelProfile.GetMemoryFootprintBitvector(),
                                               instrumentor);
                break;
            }
            case MODE_MEMORY_RACE_DETECTION:
            case MODE_MEMORY_ACCESS_ANALYSIS_AND_RACE_DETECTION:
            {
                InsertRaceDetectionProf(ins,
                                        memAccess,
                                        kernelProfile.GetMemoryRaceDetectionArg(),
                                        kernelProfile.GetRaceDetectionDataVector(),
                                        kernelProfile.GetMemoryFootprintBitvector(),
                                        instrumentor);
                break;
            }
            default: GTPIN_ERROR();
            }
        }
        else
        {
            GTPIN_ERROR();
        }
    }

    if (KNOB_MODE == MODE_MEMORY_FOOTPRINT)
    {
        // Share bitvector argument.
        // It will be initialized at the start of the kernel, and copied back to the host memory at completion of the kernel
        memMapper.Map(kernelProfile.GetMemoryFootprintArg(), GT_MMAP_SHARE);
        memMapper.Map(kernelProfile.GetMemoryFootprintBitvector(), GT_MMAP_SHARE);
    }

    if (KNOB_MODE == MODE_MEMORY_CACHE_MODEL)
    {
        // Share cache model tags vector argument.
        // It will be initialized at the start of the kernel, and copied back to the host memory at completion of the kernel
        memMapper.Map(kernelProfile.GetMemoryCacheModelArg(), GT_MMAP_SHARE);
        memMapper.Map(kernelProfile.GetCacheBucketTagsVector(), GT_MMAP_SHARE);
    }

    if (KNOB_MODE == MODE_MEMORY_ACCESS_ANALYSIS)
    {
        // Share memory access analysis argument.
        // It will be initialized at the start of the kernel, and copied back to the host memory at completion of the kernel
        memMapper.Map(kernelProfile.GetMemoryAccessAnalysisArg(), GT_MMAP_SHARE);
        memMapper.Map(kernelProfile.GetMemoryFootprintBitvector(), GT_MMAP_SHARE);
    }

    if (KNOB_MODE == MODE_MEMORY_RACE_DETECTION || KNOB_MODE == MODE_MEMORY_ACCESS_ANALYSIS_AND_RACE_DETECTION)
    {
        // Share memory access analysis argument.
        // It will be initialized at the start of the kernel, and copied back to the host memory at completion of the kernel
        memMapper.Map(kernelProfile.GetMemoryFootprintBitvector(), GT_MMAP_SHARE);
        memMapper.Map(kernelProfile.GetRaceDetectionDataVector(), GT_MMAP_SHARE);
        memMapper.Map(kernelProfile.GetMemoryRaceDetectionArg(), GT_MMAP_SHARE);
    }

    // Link the kernel with the library of HLI functions
    instrumentor.LinkHliModule(_hliModule);
}

void CachelineProf::OnKernelRun(IGtKernelDispatch& dispatcher)
{
    const IGtKernel& kernel        = dispatcher.Kernel();

    auto it = _kernels.find(kernel.Id());
    if (it == _kernels.end())
    {
        return;
    }

    KernelProfile&   kernelProfile = it->second;

    if (dispatcher.ExecStage().IsDispatch())
    {
        GtKernelExecDesc execDesc; dispatcher.GetExecDescriptor(execDesc);
        if (kernel.IsInstrumented() && IsKernelExecProfileEnabled(execDesc, kernel.GpuPlatform()))
        {
            dispatcher.SetProfilingMode(true);  // Enable instrumentation
        }
        else
        {
            dispatcher.SetProfilingMode(false); // Disable instrumentation
            return;
        }
    }

    IGtMemoryMapper& memMapper = dispatcher.MemoryMapper();

    if (KNOB_MODE == MODE_CACHE_HISTOGRAM)
    {
        // Initialize per-access arguments of HLI functions
        for (auto& entry : kernelProfile.GetMemAccessMap())
        {
            auto        insId = entry.first;

            if (int32_t(insId) < knobMinInstrumentIns || knobMaxInstrumentIns < int32_t(insId))
            {
                continue;
            }

            MemAccess& memAccess = entry.second;

            CachelineProfArgs& clArgs = memAccess.GetCachelineProfArgs();

            memset(&clArgs.out, 0, sizeof(clArgs.out));
            clArgs.in.dataSize = memAccess.DataSize();
            clArgs.in.numAccesses = memAccess.NumAccesses();
            clArgs.in.log2CachelineSize = gLog2CacheLineSize;

            memMapper.Write(&clArgs, sizeof(clArgs));
        }
    }
    
    if (KNOB_MODE == MODE_MEMORY_FOOTPRINT)
    {
        MemoryFootprintArgs& mfprntArgs = kernelProfile.GetMemoryFootprintArg();

        mfprntArgs.in.log2CachelineSize    = gLog2CacheLineSize;
        mfprntArgs.in.supportedAddressBits = gSupportedAddressBits;
        mfprntArgs.in.bitVectorSizeInBytes = (uint32_t)gBitVectorSizeInBytes;
        memMapper.Write(&mfprntArgs, sizeof(mfprntArgs));

        auto& bitvector = kernelProfile.GetMemoryFootprintBitvector();
        memset(bitvector.data(), 0, bitvector.size());
        memMapper.Write(bitvector.data(), (uint32_t)bitvector.size());
    }

    if (KNOB_MODE == MODE_MEMORY_CACHE_MODEL)
    {
        MemoryCacheModelArgs& mCacheModelArgs = kernelProfile.GetMemoryCacheModelArg();

        mCacheModelArgs.in.log2CachelineSize = gLog2CacheLineSize;
        mCacheModelArgs.in.cacheSize         = gCacheSize;
        mCacheModelArgs.in.hashMaskShift     = KNOB_CACHE_MODEL_HASH_SHIFT;
        mCacheModelArgs.out.hits             = 0;
        mCacheModelArgs.out.misses           = 0;
        memMapper.Write(&mCacheModelArgs, sizeof(mCacheModelArgs));

        if (!KNOB_NO_CACHE_INVALIDATE)
        {
            // Initialize cache model per each enqueue
            auto& tagsVector = kernelProfile.GetCacheBucketTagsVector();
            memset(tagsVector.data(), 0, tagsVector.size() * sizeof(uint64_t));
            memMapper.Write(tagsVector.data(), (uint32_t)tagsVector.size() * sizeof(uint64_t));
        }
    }

    if (KNOB_MODE == MODE_MEMORY_ACCESS_ANALYSIS)
    {
        MemoryAccessAnalysisArgs& mAccessAnalysisArgs = kernelProfile.GetMemoryAccessAnalysisArg();

        mAccessAnalysisArgs.in.log2CachelineSize    = gLog2CacheLineSize;
        mAccessAnalysisArgs.in.supportedAddressBits = gSupportedAddressBits;
        mAccessAnalysisArgs.in.bitVectorSizeInBytes = (uint32_t)gBitVectorSizeInBytes;

        mAccessAnalysisArgs.out.rarCount = 0;
        mAccessAnalysisArgs.out.rawCount = 0;
        mAccessAnalysisArgs.out.warCount = 0;
        mAccessAnalysisArgs.out.wawCount = 0;
        memMapper.Write(&mAccessAnalysisArgs, sizeof(mAccessAnalysisArgs));

        if (!KNOB_NO_INVALIDATE)
        {
            auto& bitvector = kernelProfile.GetMemoryFootprintBitvector();
            memset(bitvector.data(), KNOB_ANALYSIS_INIT_TO_WRITE ? 0xFF : 0, bitvector.size());
            memMapper.Write(bitvector.data(), (uint32_t)bitvector.size());
        }
    }

    if (KNOB_MODE == MODE_MEMORY_RACE_DETECTION || KNOB_MODE == MODE_MEMORY_ACCESS_ANALYSIS_AND_RACE_DETECTION)
    {
        MemoryRaceDetectionArgs& mRaceDetectionArgs = kernelProfile.GetMemoryRaceDetectionArg();

        mRaceDetectionArgs.in.log2CachelineSize = gLog2CacheLineSize;
        mRaceDetectionArgs.in.supportedAddressBits = gSupportedAddressBits;
        mRaceDetectionArgs.in.bitVectorSizeInBytes = (uint32_t)gBitVectorSizeInBytes;

        mRaceDetectionArgs.out.raceCount = 0;
        mRaceDetectionArgs.out.upperAddressBits = 0;

        if (KNOB_MODE == MODE_MEMORY_ACCESS_ANALYSIS_AND_RACE_DETECTION)
        {
            mRaceDetectionArgs.out.rarCount = 0;
            mRaceDetectionArgs.out.rawCount = 0;
            mRaceDetectionArgs.out.warCount = 0;
            mRaceDetectionArgs.out.wawCount = 0;
        }
        memMapper.Write(&mRaceDetectionArgs, sizeof(mRaceDetectionArgs));

        if (!KNOB_NO_INVALIDATE)
        {
            auto& bitvector = kernelProfile.GetMemoryFootprintBitvector();
            memset(bitvector.data(), 0, bitvector.size());
            memMapper.Write(bitvector.data(), (uint32_t)bitvector.size());

            auto& raceDataVector = kernelProfile.GetRaceDetectionDataVector();
            memset(raceDataVector.data(), 0xFF, raceDataVector.size() * sizeof(uint32_t));
            memMapper.Write(raceDataVector.data(), (uint32_t)raceDataVector.size());
        }
    }
}

void CachelineProf::OnKernelComplete(IGtKernelDispatch& dispatcher)
{
    if (dispatcher.IsProfilingEnabled())
    {
        KernelProfile& kernelProfile = _kernels.at(dispatcher.Kernel().Id());
        kernelProfile.RecordCachelineProfResults(dispatcher);
    }
}

bool CachelineProf::InsertCachelineProf(const IGtIns &ins, const MemAccess& memAccess, IGtKernelInstrument& instrumentor)
{
    GTPIN_ASSERT(memAccess.IsValid() && (memAccess.Id() == ins.Id()));

    uint32_t numAccesses = memAccess.NumAccesses();
    if (numAccesses == 0)
    {
        return false; // Nothing to check
    }

    uint32_t                 regSize              = instrumentor.Kernel().GenModel().GrfRegSize();
    const GtMemoryAddrModel& addrModel            = memAccess.AddrModel();
    uint32_t                 addrSize             = addrModel.PtrSize();
    GtReg                    firstAddrReg         = GrfReg(memAccess.FirstAddrReg(), 0, regSize);
    uint32_t                 numAddrRegs          = RoundUp(numAccesses * addrSize, regSize) / regSize;
    CachelineProfArgs*       checkArgs            = const_cast<CachelineProfArgs*>(&memAccess.GetCachelineProfArgs());
    IargConstGrfRange        addrPayload(firstAddrReg.RegNum(), numAddrRegs);
    IargInsOpMask            accessMask(ins);

    if (addrModel.IsA64())
    {
        _checkScatterA64AccessFunc.InsertCallAtInstruction(instrumentor, ins, GtIpoint::Before(),
            NullReg(),          // Unused return value
            addrPayload,        // arg[1]: Base address of the accessed memory range
            accessMask,         // arg[2]: Per-channel mask of memory accesses
            checkArgs           // arg[3]: Cacheline check arguments
        );
    }
    else
    {
        _checkScatterA32AccessFunc.InsertCallAtInstruction(instrumentor, ins, GtIpoint::Before(),
            NullReg(),          // Unused return value
            addrPayload,        // arg[1]: Base address of the accessed memory range
            accessMask,         // arg[2]: Per-channel mask of memory accesses
            checkArgs           // arg[3]: Cacheline check arguments
        );
    }
    return true;
}

bool CachelineProf::InsertMemoryFootprintProf(const IGtIns& ins, const MemAccess& memAccess, const MemoryFootprintArgs& memFootprintArgs,  const BitVector& bitvector, IGtKernelInstrument& instrumentor)
{
    GTPIN_ASSERT(memAccess.IsValid() && (memAccess.Id() == ins.Id()) && memAccess.AddrModel().IsA64());

    uint32_t numAccesses = memAccess.NumAccesses();
    if (numAccesses == 0)
    {
        return false; // Nothing to check
    }

    uint32_t                 regSize      = instrumentor.Kernel().GenModel().GrfRegSize();
    GtReg                    firstAddrReg = GrfReg(memAccess.FirstAddrReg(), 0, regSize);
    uint32_t                 numAddrRegs  = RoundUp(numAccesses * 8, regSize) / regSize;
    uint32_t                 dataSize     = memAccess.DataSize();
    MemoryAccessType         accessType   = ins.IsAtomic() ? MemoryAccessType::ReadModifyWrite : (ins.IsMemRead() ? MemoryAccessType::Load : MemoryAccessType::Store);
    MemoryFootprintArgs*     args         = const_cast<MemoryFootprintArgs*>(&memFootprintArgs);
    IargHostPtr              footprint    = bitvector.data();
    IargConstGrfRange        addrPayload(firstAddrReg.RegNum(), numAddrRegs);
    IargInsOpMask            accessMask(ins);

    if (memAccess.IsBlock2DAccess())
    {
        _globalMemoryFootprintBlock2DAccessFunc.InsertCallAtInstruction(instrumentor, ins, GtIpoint::Before(),
            NullReg(),          // Unused return value
            addrPayload,        // arg[1]: Base address of the accessed memory range
            accessMask,         // arg[2]: Per-channel mask of memory accesses
            dataSize,           // arg[3]: Access data size
            accessType,         // arg[4]: Access type
            args,               // arg[5]: Memory footprint arguments
            footprint           // arg[6]: bitvector
        );
    }
    else
    {
        _globalMemoryFootprintA64AccessFunc.InsertCallAtInstruction(instrumentor, ins, GtIpoint::Before(),
            NullReg(),          // Unused return value
            addrPayload,        // arg[1]: Base address of the accessed memory range
            accessMask,         // arg[2]: Per-channel mask of memory accesses
            dataSize,           // arg[3]: Access data size
            accessType,         // arg[4]: Access type
            args,               // arg[5]: Memory footprint arguments
            footprint           // arg[6]: bitvector
        );
    }

    return true;
}

bool CachelineProf::InsertMemoryAccessAnalysisProf(const IGtIns& ins, const MemAccess& memAccess, const MemoryAccessAnalysisArgs& memAccessAnalysisArgs, const BitVector& bitvector, IGtKernelInstrument& instrumentor)
{
    GTPIN_ASSERT(memAccess.IsValid() && (memAccess.Id() == ins.Id()) && memAccess.AddrModel().IsA64());

    uint32_t numAccesses = memAccess.NumAccesses();
    if (numAccesses == 0)
    {
        return false; // Nothing to check
    }

    uint32_t                  regSize      = instrumentor.Kernel().GenModel().GrfRegSize();
    GtReg                     firstAddrReg = GrfReg(memAccess.FirstAddrReg(), 0, regSize);
    uint32_t                  numAddrRegs  = RoundUp(numAccesses * 8, regSize) / regSize;
    uint32_t                  dataSize     = memAccess.DataSize();
    MemoryAccessAnalysisArgs* args         = const_cast<MemoryAccessAnalysisArgs*>(&memAccessAnalysisArgs);
    IargHostPtr               footprint    = bitvector.data();
    MemoryAccessType          accessType   = ins.IsAtomic() ? MemoryAccessType::ReadModifyWrite : (ins.IsMemRead() ? MemoryAccessType::Load : MemoryAccessType::Store);
    IargConstGrfRange         addrPayload(firstAddrReg.RegNum(), numAddrRegs);
    IargInsOpMask             accessMask(ins);

    if (memAccess.IsBlock2DAccess())
    {
        _globalMemoryAccessAnalysisBlock2DAccessFunc.InsertCallAtInstruction(instrumentor, ins, GtIpoint::Before(),
            NullReg(),          // Unused return value
            addrPayload,        // arg[1]: Base address of the accessed memory range
            accessMask,         // arg[2]: Per-channel mask of memory accesses
            dataSize,           // arg[3]: Access data size
            accessType,         // arg[4]: Access type
            args,               // arg[5]: Memory access analysis arguments
            footprint           // arg[6]: bitvector
            );
    }
    else
    {
        _globalMemoryAccessAnalysisA64AccessFunc.InsertCallAtInstruction(instrumentor, ins, GtIpoint::Before(),
            NullReg(),          // Unused return value
            addrPayload,        // arg[1]: Base address of the accessed memory range
            accessMask,         // arg[2]: Per-channel mask of memory accesses
            dataSize,           // arg[3]: Access data size
            accessType,         // arg[4]: Access type
            args,               // arg[5]: Memory access analysis arguments
            footprint           // arg[6]: bitvector
        );
    }

    return true;
}

bool CachelineProf::InsertRaceDetectionProf(const IGtIns&                  ins,
                                            const MemAccess&               memAccess,
                                            const MemoryRaceDetectionArgs& memRaceDetectionArg,
                                            const RaceDetectionDataVector& raceDetectionDataVector,
                                            const BitVector&               bitvector,
                                            IGtKernelInstrument&           instrumentor)
{
    GTPIN_ASSERT(memAccess.IsValid() && (memAccess.Id() == ins.Id()) && memAccess.AddrModel().IsA64());

    uint32_t numAccesses = memAccess.NumAccesses();
    if (numAccesses == 0)
    {
        return false; // Nothing to check
    }

    uint32_t                  regSize = instrumentor.Kernel().GenModel().GrfRegSize();
    GtReg                     firstAddrReg = GrfReg(memAccess.FirstAddrReg(), 0, regSize);
    uint32_t                  numAddrRegs = RoundUp(memAccess.NumAccesses() * 8, regSize) / regSize;
    uint32_t                  dataSize = memAccess.DataSize();
    MemoryAccessType          accessType = ins.IsAtomic() ? MemoryAccessType::ReadModifyWrite : (ins.IsMemRead() ? MemoryAccessType::Load : MemoryAccessType::Store);
    MemoryRaceDetectionArgs*  args = const_cast<MemoryRaceDetectionArgs*>(&memRaceDetectionArg);
    IargHostPtr               detectedCacheLines = bitvector.data();
    IargHostPtr               raceData = raceDetectionDataVector.data();
    IargConstGrfRange         addrPayload(firstAddrReg.RegNum(), numAddrRegs);
    IargInsOpMask             accessMask(ins);
    IargTid                   threadId;

    if (memAccess.IsBlock2DAccess())
    {
    }
    else
    {
        if (KNOB_MODE == MODE_MEMORY_RACE_DETECTION)
        {
            _globalMemoryRaceDetectionA64AccessFunc.InsertCallAtInstruction(instrumentor, ins, GtIpoint::Before(),
                NullReg(),                // Unused return value
                addrPayload,              // arg[1]: Base address of the accessed memory range
                accessMask,               // arg[2]: Per-channel mask of memory accesses
                dataSize,                 // arg[3]: Access data size
                accessType,               // arg[4]: Access type
                threadId,                 // arg[5]: HW Thread ID
                args,                     // arg[6]: Memory race condition detection arguments
                raceData,                 // arg[7]: race data vector
                detectedCacheLines        // arg[8]: bitvector - detected cache lines
            );
        }
        if (KNOB_MODE == MODE_MEMORY_ACCESS_ANALYSIS_AND_RACE_DETECTION)
        {
            _globalMemoryAccessAnalysisAndRaceDetectionA64AccessFunc.InsertCallAtInstruction(instrumentor, ins, GtIpoint::Before(),
                NullReg(),                // Unused return value
                addrPayload,              // arg[1]: Base address of the accessed memory range
                accessMask,               // arg[2]: Per-channel mask of memory accesses
                dataSize,                 // arg[3]: Access data size
                accessType,               // arg[4]: Access type
                threadId,                 // arg[5]: HW Thread ID
                args,                     // arg[6]: Memory race condition detection arguments
                raceData,                 // arg[7]: race data vector
                detectedCacheLines        // arg[8]: bitvector - detected cache lines
            );
        }
    }

    return true;
}

bool CachelineProf::InsertMemoryCacheModelProf(const IGtIns& ins, const MemAccess& memAccess, const MemoryCacheModelArgs& memCacheModelArgs, const CacheBucketTagsVector& cacheBucketTagsVector, IGtKernelInstrument& instrumentor)
{
    GTPIN_ASSERT(memAccess.IsValid() && (memAccess.Id() == ins.Id()) && memAccess.AddrModel().IsA64());

    uint32_t numAccesses = memAccess.NumAccesses();
    if (numAccesses == 0)
    {
        return false; // Nothing to check
    }
         
    uint32_t                 regSize      = instrumentor.Kernel().GenModel().GrfRegSize();
    GtReg                    firstAddrReg = GrfReg(memAccess.FirstAddrReg(), 0, regSize);
    uint32_t                 numAddrRegs  = RoundUp(numAccesses * 8, regSize) / regSize;
    uint32_t                 dataSize     = memAccess.DataSize();
    MemoryCacheModelArgs*    args         = const_cast<MemoryCacheModelArgs*>(&memCacheModelArgs);
    IargHostPtr              tags         = cacheBucketTagsVector.data();
    IargConstGrfRange        addrPayload(firstAddrReg.RegNum(), numAddrRegs);
    IargInsOpMask            accessMask(ins);

    if (memAccess.IsBlock2DAccess())
    {
        _globalMemoryCacheModelBlock2DAccessFunc.InsertCallAtInstruction(instrumentor, ins, GtIpoint::Before(),
            NullReg(),          // Unused return value
            addrPayload,        // arg[1]: Base address of the accessed memory range
            accessMask,         // arg[2]: Per-channel mask of memory accesses
            dataSize,           // arg[3]: Access data size
            args,               // arg[4]: Memory footprint arguments
            tags                // arg[5]: tags vector
        );
    }
    else
    {
        _globalMemoryCacheModelA64AccessFunc.InsertCallAtInstruction(instrumentor, ins, GtIpoint::Before(),
            NullReg(),          // Unused return value
            addrPayload,        // arg[1]: Base address of the accessed memory range
            accessMask,         // arg[2]: Per-channel mask of memory accesses
            dataSize,           // arg[3]: Access data size
            args,               // arg[4]: Memory footprint arguments
            tags                // arg[5]: tags vector
        );
    }

    return true;
}

void CachelineProf::LoadHliLibrary()
{
    std::string modulePath = JoinPath(GetKnobValue<std::string>("installDir"), "Examples", "cachelineprof.cl");
    _hliModule = GTPin_GetCore()->HliLibrary().CompileModuleFromFile(modulePath.c_str());
    GTPIN_ASSERT_MSG(_hliModule != nullptr, "Could not load HLI module " + modulePath);
}

void CachelineProf::Fini()
{
    std::string str;

    // Dump profiling results and assembly code of all kernels
    for (const auto& entry : _kernels)
    {
        const auto& kernelProfile = entry.second;
        str += kernelProfile.CachelineProfResults();
        kernelProfile.DumpAsm();
    }

    std::ofstream fs(JoinPath(GTPin_GetCore()->ProfileDir(), "cachelineprof.txt"));
    GTPIN_ASSERT(fs.is_open());
    fs << str;

    if (!KNOB_NO_COUT)
    {
        std::cout << str;
    }
}

/* ============================================================================================= */
// GTPin_Entry
/* ============================================================================================= */
EXPORT_C_FUNC void GTPin_Entry(int argc, const char *argv[])
{
    SetKnobValue<int>(0, "scalarize_hlif");
    ConfigureGTPin(argc, argv);

    // Register the tool (callbacks) with the GTPin core
    CachelineProf::Instance()->Register();

    // Compile and load library of HLI functions
    CachelineProf::Instance()->LoadHliLibrary();

    // Register the termination function
    atexit(CachelineProf::OnFini);

    if (KNOB_MODE != MODE_CACHE_HISTOGRAM)
    {
        // All the modes except cache histogram support A64 address mode only.
        // To make sure the compiler generates A64 accesses, we force IGC to do that.
        SetKnobValue<bool>(true, "igc_force_a64");
    }

    gLog2CacheLineSize = Bsr32(KNOB_CACHELINE_SIZE);

    gSupportedAddressBits = KNOB_SUPPORTED_ADDRESS_BITS;

    if (KNOB_MODE == MODE_MEMORY_FOOTPRINT)
    {
        // Memory footprint mode distinguishes between read and write accesses. For that we allocate a bitvector of the double size.
        // The first half of the bitvector is used for read accesses and the second half for write accesses. Since the bitvector is big,
        // to save buffer size we reduce one bit from the supported address bits, and thus doubling the buffer size we get the same number of bytes.
        gSupportedAddressBits -= 1;

        uint64_t bitVectorSizeInBytes = 2 * (((uint64_t)1 << gSupportedAddressBits) >> gLog2CacheLineSize) / 8;
        GTPIN_ASSERT_MSG(bitVectorSizeInBytes < FOUR_GB, "Too big buffer - please reduce supported address bits"); // we don't want to support buffers bigger than 4GB

        gBitVectorSizeInBytes = (size_t)bitVectorSizeInBytes;
    }

    if (KNOB_MODE == MODE_MEMORY_CACHE_MODEL)
    {
        gCacheSize = KNOB_CACHE_SIZE * ONE_KB;
        gNumOfCacheBuckets = gCacheSize >> gLog2CacheLineSize;
    }

    if (KNOB_MODE == MODE_MEMORY_ACCESS_ANALYSIS)
    {
        uint64_t bitVectorSizeInBytes = (((uint64_t)1 << gSupportedAddressBits) >> gLog2CacheLineSize) / 8;
        GTPIN_ASSERT_MSG(bitVectorSizeInBytes < FOUR_GB, "Too big buffer - please reduce supported address bits"); // we don't want to support buffers bigger than 4GB

        gBitVectorSizeInBytes = (size_t)bitVectorSizeInBytes;
    }

    if (KNOB_MODE == MODE_MEMORY_RACE_DETECTION || KNOB_MODE == MODE_MEMORY_ACCESS_ANALYSIS_AND_RACE_DETECTION)
    {
        // This mode assumes dword accesses and sets maximum of supported address bits to 31 to not exceed 4GB of profiling buffer.
        SetKnobValue<int>(4, "cacheline_size");
        SetKnobValue<int>(31, "supported_address_bits");

        gSupportedAddressBits = KNOB_SUPPORTED_ADDRESS_BITS;
        gLog2CacheLineSize = Bsr32(KNOB_CACHELINE_SIZE);

        uint64_t raceDataVectorSize   = (((uint64_t)1 << gSupportedAddressBits) >> gLog2CacheLineSize);
        uint64_t bitVectorSizeInBytes = raceDataVectorSize / 8;
        GTPIN_ASSERT_MSG(bitVectorSizeInBytes + raceDataVectorSize * sizeof(uint32_t) < FOUR_GB, "Too big buffer - please reduce supported address bits");
        gBitVectorSizeInBytes = (size_t)bitVectorSizeInBytes;
        gRaceDataVectorSize   = (size_t)raceDataVectorSize;
    }
}
