/*========================== begin_copyright_notice ============================
Copyright (C) 2021-2026 Intel Corporation

SPDX-License-Identifier: MIT
============================= end_copyright_notice ===========================*/

/*!
 * @file GEN kernel information, instrumentation, and dispatching interface
 */

#ifndef IGT_KERNEL_H_
#define IGT_KERNEL_H_

#include "gt_basic_defs.h"
#include "gt_gpu_defs.h"
#include "igt_core.h"
#include "igt_hli_library.h"
#include "igt_cfg.h"

namespace gtpin
{
#pragma pack(push, 8)

using GtKernelId     = Uint32Validatable<struct _KernelId>; ///< Unique kernel identifier
using GtKernelBinary = ConstByteSpan;                       ///< Kernel code bytes
using GtKernelElf    = ConstByteSpan;                       ///< Kernel ELF image bytes

/* ===================================================================================================== */
// External declarations
/* ===================================================================================================== */
class IGtGenModel;
class IGtCfg;
class IGtBbl;
class IGtIns;
class IGtGenCoder;
class IGtGenProcedure;
class IGtProfileBuffer;
class IGtProfileBufferAllocator;
class IGtMemoryMapper;
class IGtIargFactory;

/* ===================================================================================================== */
// Class GtKernelType
/* ===================================================================================================== */
/*! @ingroup GTPIN_KERNEL
 * @brief Type of a kernel (shader)
 */
enum KERNEL_TYPE :uint32_t
{
    KERNEL_TYPE_INVALID,
    KERNEL_TYPE_HS,       ///< Hull Shader
    KERNEL_TYPE_DS,       ///< Domain Shader
    KERNEL_TYPE_VS,       ///< Vertex Shader
    KERNEL_TYPE_PS,       ///< Pixel Shader
    KERNEL_TYPE_CS,       ///< Compute Shader (GPGPU)
    KERNEL_TYPE_GS,       ///< Geometry Shader
    KERNEL_TYPE_MS,       ///< Mesh Shader
    KERNEL_TYPE_TS,       ///< Task Shader
    KERNEL_TYPE_RT,       ///< Ray Tracing
};

/*! @ingroup GTPIN_KERNEL
 * @brief Type of a kernel (shader)
 */
class GtKernelType
{
public:
    inline constexpr GtKernelType(KERNEL_TYPE v = KERNEL_TYPE_INVALID) : _value(v) {}   ///< Conversion constructor
    inline constexpr operator KERNEL_TYPE() const { return _value; }                    ///< Conversion operator

    inline constexpr bool IsValid()   const { return _value != KERNEL_TYPE_INVALID; }
    inline const char*    ToString()  const;                                            ///< Return string representation

    static inline GtKernelType MakeFromString(const char* typeCstr);                    ///< Construct from string

private:
    KERNEL_TYPE _value;
};
GTPIN_ASSERT_STANDARD_LAYOUT(GtKernelType);

/* ===================================================================================================== */
// Class GtRtKernelType
/* ===================================================================================================== */
/*! @ingroup GTPIN_KERNEL
 * @brief Type of a kernel (shader)
 */
enum KERNEL_RT_TYPE :uint32_t
{
    KERNEL_RT_TYPE_INVALID,
    KERNEL_RT_TYPE_PROLOGUE,
    KERNEL_RT_TYPE_RAYGEN,
    KERNEL_RT_TYPE_MISS,
    KERNEL_RT_TYPE_HIT,
    KERNEL_RT_TYPE_ANYHIT,
    KERNEL_RT_TYPE_INTERSECTION,
    KERNEL_RT_TYPE_CALLABLE,
    KERNEL_RT_TYPE_CALLSTACKHANDLER
};

/*! @ingroup GTPIN_KERNEL
 * @brief Type of a kernel (shader)
 */
class GtRtKernelType
{
public:
    inline constexpr GtRtKernelType(KERNEL_RT_TYPE v = KERNEL_RT_TYPE_INVALID) : _value(v) {}   ///< Conversion constructor
    inline constexpr operator KERNEL_RT_TYPE() const { return _value; }                    ///< Conversion operator

    inline constexpr bool IsValid()   const { return _value != KERNEL_RT_TYPE_INVALID; }
    inline const char* ToString()  const;                                            ///< Return string representation

    static inline GtRtKernelType MakeFromString(const char* typeCstr);                 ///< Construct from string

private:
    KERNEL_RT_TYPE _value;
};
GTPIN_ASSERT_STANDARD_LAYOUT(GtRtKernelType);


/* ===================================================================================================== */
// Struct GtKernelArgument
/* ===================================================================================================== */
/*! @ingroup GTPIN_KERNEL
 * @brief Descriptor of the kernel's payload argument
 */
struct GtKernelArgument
{
    const char*         type   = nullptr;   ///< Name of the argument type, e.g. 'arg_byvalue', 'arg_bypointer', 'local_id', etc.
    uint32_t            size   = 0;         ///< Size of the argument value, in bytes
    uint32_t            offset = 0;         ///< Offset of the argument value in the kernel payload
    Uint32Validatable<> index;              ///< Index of the argument in the kernel payload. Valid for explicit arguments
    GtAddrSpace         addrSpace;          ///< Address space referenced by the argument. Valid for IsPointer() arguments
    GtMemoryAddrModel   addrModel;          ///< Address model of the memory space access. Valid for IsMemoryPointer() arguments
    GtAccessType        accessType;         ///< Type of the access to the space referenced by the IsPointer() argument

public:
    /// Default constructor
    GtKernelArgument() = default;

    /// Constructor
    GtKernelArgument(const char* argType, uint32_t argSize, uint32_t argOffset, uint32_t argIndex,
                     GtAddrSpace argAddrSpace = {}, GtMemoryAddrModel argAddrModel = {}, GtAccessType argAccessType = {}):
                     type(argType), size(argSize), offset(argOffset), index(argIndex),
                     addrSpace(argAddrSpace), addrModel(argAddrModel), accessType(argAccessType) {}

    /// @return true for a valid argument descriptor
    bool IsValid()                const { return (type != nullptr); }

    /// @return true if this is an explicit argument (has a valid index)
    inline bool IsExplicit()      const { return index.IsValid(); }

    /// @return true if this argument references an object within an address space
    bool IsPointer()              const { return addrSpace.IsValid(); }

    /// @return true if this argument references a memory region
    inline bool IsMemoryPointer() const { return addrSpace.IsMemory() && addrModel.IsValid(); }
};
GTPIN_ASSERT_STANDARD_LAYOUT(GtKernelArgument);

/// Span of the kernel payload argument descriptors
using  GtKernelArgSpan = Span<const GtKernelArgument>;

/* ===================================================================================================== */
// Class IGtKernel
/* ===================================================================================================== */
/*! @ingroup GTPIN_KERNEL
 * @brief Basic properties of the GEN kernel as a compilation/instrumentation unit
 * 
 * @par Availability:
 * - IGtTool::OnKernelBuild
 * - IGtTool::OnKernelRun
 * - IGtTool::OnKernelComplete
 */
class IGtKernel
{
public:
    /// @return Unique ID of the kernel
    virtual GtKernelId Id() const = 0;
        
    /// @return Name of the kernel function 
    virtual GtCoreString Name() const = 0;

    /// @return Unique symbolic identifier of this kernel among all kernels in the current program
    virtual GtCoreString UniqueName() const = 0;

    /// @return Type of the kernel (shader)
    virtual GtKernelType Type() const = 0;

    /// @return SIMD width of the kernel
    virtual GtSimdWidth SimdWidth() const = 0;

    /// @return GPU platform the kernel is intended to run on
    virtual GtGpuPlatform GpuPlatform() const = 0;

    /// @return GEN model, the kernel is compiled for
    virtual const IGtGenModel& GenModel() const = 0;

    /// @return Hash identifier of the kernel, as reported by IGC/driver 
    virtual uint64_t HashId() const = 0;

    /// @return Hash value of the original kernel's binary code, as calculated by GTPin 
    virtual uint64_t BinarySignature() const = 0;

    /// @return Format of the module that contains the kernel
    virtual GtBinaryFormat BinaryFormat() const = 0;

    /// @return Binary code of the kernel
    virtual GtKernelBinary Binary() const = 0;

    /// @return ELF image of the kernel
    virtual GtKernelElf Elf() const = 0;

    /// @return true, if instrumented version of the kernel function exists; false, otherwise
    virtual bool IsInstrumented() const = 0;

    /// @return Address model of profile buffer accesses
    virtual GtMemoryAddrModel ProfileBufferAddrModel() const = 0;

    /// @return Number of registers in the kernel's input payload
    virtual uint32_t GetInputPayloadSize() const = 0;

    ///< @return Number of GRF registers used by the kernel
    virtual uint32_t GetNumGrfRegs() const = 0;

    /// @return true, if the kernel is a Ray Tracing kernel; false, otherwise
    virtual bool IsRayTracing() const = 0;

    /// @return true, if the kernel is the primary (not continuation) part of a Ray Tracing kernel; false, otherwise
    virtual bool IsRayTracingPrimary() const = 0;

    virtual GtRtKernelType              GetRtType()                          const = 0;
    virtual Uint32Validatable<>         GetGlobalRootOffset()                const = 0;
    virtual Uint32Validatable<>         GetGlobalArgsPtrOffset()             const = 0;
    virtual Uint32Validatable<>         GetLocalArgsPtrOffset()              const = 0;
    virtual Uint32Validatable<>         GetStackIdOffset()                   const = 0;
    virtual Uint32Validatable<>         GetBaseSurfaceStatePointerOffset()   const = 0;
    virtual Uint32Validatable<>         GetSwHotZoneSize()                   const = 0;
    virtual Uint32Validatable<>         GetDispatchRayIndexOffset()          const = 0;
    virtual Uint32Validatable<>         GetSwHotZoneSurfaceStateByteOffset() const = 0;
    virtual GtMemoryAddrModel           GetSwHotZoneMemoryAddrModel()        const = 0;
    virtual bool                        IsDispatchRayIndexSupported()        const = 0;
    virtual uint32_t                    GetNumRTStacksBits()                 const = 0;

    /*!
     * @return  Sequence of the kernel payload argument descriptors.
     *          On some GPU platforms, the kernel payload is unknown to GTPin. In this case, the function returns an empty
     *          sequence, and the corresponding failure reason can be examined by the IGtCore::LastError() function.
     */
    virtual GtKernelArgSpan PayloadArguments() const = 0;

    /*!
     * @brief Get number of explicit arguments in the kernel payload.
     * 
     * Explicit argument is identified by its index, which uniquely references the argument value in GFX runtime APIs.
     * The value of the explicit argument can be examined by the IGtKernelDispatch::GetPayloadArgumentValue function.
     * Note, explicit argument can be referred by multiple GtKernelArgument descriptors that provide various information
     * (metadata) pertaining to a single explicit argument.
     * 
     * @return Number of explicit arguments in the kernel payload
     */
    virtual uint32_t NumExplicitPayloadArguments() const = 0;

    /*!
     * @return GPU address of the instrumented kernel code or uint64_t(-1) in the case the address is not known
     *
     */
    virtual uint64_t GetInstrumentedCodeBaseInGpu() const = 0;

protected:
    /// @internal Protected destructor
    virtual ~IGtKernel() = default;
};
GTPIN_ASSERT_PURE_ABSTRACT(IGtKernel);

/* ===================================================================================================== */
// Class GtIpoint
/* ===================================================================================================== */
/*! @ingroup GTPIN_KERNEL_INSTRUMENT
 * @brief Relative location within the code at which instrumentation is inserted
 */
enum GT_IPOINT : uint32_t
{
    GT_IPOINT_INVALID,///< Invalid location
    GT_IPOINT_BEFORE, ///< Location before instruction/BBL, etc.
    GT_IPOINT_AFTER   ///< Location after instruction/BBL, etc.
};

/*! @ingroup GTPIN_KERNEL_INSTRUMENT
 * @brief Relative location within the code at which instrumentation is inserted
 */
class GtIpoint
{
public:
    constexpr GtIpoint(GT_IPOINT v = GT_IPOINT_INVALID) : _value(v) {}          ///< Conversion constructor
    constexpr operator GT_IPOINT() const { return _value; }                     ///< Conversion operator

    constexpr bool IsValid()   const { return _value != GT_IPOINT_INVALID; }
    constexpr bool IsBefore()  const { return _value == GT_IPOINT_BEFORE; }
    constexpr bool IsAfter()   const { return _value == GT_IPOINT_AFTER; }

    static constexpr GtIpoint Invalid() { return {GT_IPOINT_INVALID}; }
    static constexpr GtIpoint Before()  { return {GT_IPOINT_BEFORE}; }          ///< Make "before" location
    static constexpr GtIpoint After()   { return {GT_IPOINT_AFTER}; }           ///< Make "after" location

private:
    GT_IPOINT _value;
};
GTPIN_ASSERT_STANDARD_LAYOUT(GtIpoint);

/* ===================================================================================================== */
// Class IGtKernelInstrument
/* ===================================================================================================== */
/*! @ingroup GTPIN_KERNEL_INSTRUMENT
 * @brief Instrumentor of the GEN kernel
 * 
 * For each new binary kernel created by the compiler, GTPin invokes the IGtTool::OnKernelBuild function.
 * This function, implemented by the tool, receives the IGtKernelInstrument object that refers to the kernel
 * being compiled and provides methods for instrumenting the kernel. Part of these methods are implemented
 * by the IGtKernelInstrument object itself, while many other methods are implemented by referred objects
 * that provide interfaces like IGtCfg, IGtGenCoder, IGtInsFactory, etc.
 * 
 * @par Availability:
 * - IGtTool::OnKernelBuild
 */
class IGtKernelInstrument
{
public:
    /*!
     * @brief Get IGtKernel interface to the kernel being instrumented
     * @return Object that provides access to basic properties of the kernel
     */
    virtual const IGtKernel& Kernel() const = 0;

    /*!
     * @brief Get IGtCfg interface to the kernel being instrumented
     * @return Object that provides interface to control flow graph (CFG) of the kernel
     */
    virtual const IGtCfg& Cfg() const = 0;

    /*!
     * @brief Get IGtGenCoder interface to the kernel being instrumented
     * @return Object that provides code generation interface and assembles instrumentation procedures
     */
    virtual const IGtGenCoder& Coder() const = 0;

    /*!
     * @brief Get IGtProfileBufferAllocator interface to the kernel being instrumented
     * @return Allocator of memory blocks within kernel's profile buffer
     */
    virtual IGtProfileBufferAllocator& ProfileBufferAllocator() const = 0;

    /*!
     * @brief Get IGtMemoryMapper interface to the kernel being instrumented
     * @return Object that provides host-to-device memory mapping service
     */
    virtual IGtMemoryMapper& MemoryMapper() = 0;

     /*!
      * Get Get IGtIargFactory interface to the factory of arguments of the High-Level Instrumentation (HLI) functions
      * @return Factory of HLI arguments associated with this object
      */
    virtual IGtIargFactory& IargFactory() const = 0;

   /*!
     * @brief Insert the specified instrumentation procedure before/after the specified instruction
     * 
     * @param[in] ins       Instruction in the original code of the kernel
     * @param[in] ipoint    Location in the original code, relative to 'ins'
     * @param[in] proc      Instrumentation procedure to be inserted at the specified location
     * @return true - success, false - failure
     *         In case of error, the IGtCore::LastError() function provides the specific failure reason
     */
    virtual bool InstrumentInstruction(const IGtIns& ins, GtIpoint ipoint, const IGtGenProcedure& proc) = 0;

    /*!
     * @brief Insert the specified instrumentation procedure before/after the specified BBL
     * 
     * @param[in] bbl       BBL in the original code of the kernel
     * @param[in] ipoint    Location in the original code, relative to 'bbl'
     * @param[in] proc      Instrumentation procedure to be inserted at the specified location
     *
     * @note If 'ipoint' specifies location after BBL, the insertion position is as following:
     *       - before the last BBL's instruction, if BBL ends with a control flow or EOT instruction
     *       - after the last BBL's instruction, if BBL falls through to the next block
     * @return true - success, false - failure
     *         In case of error, the IGtCore::LastError() function provides the specific failure reason
     */
    virtual bool InstrumentBbl(const IGtBbl& bbl, GtIpoint ipoint, const IGtGenProcedure& proc) = 0;

    /*!
     * @brief Insert the specified instrumentation procedure to the destination bbl and execute
     *        it only when entering from the source bbl of the given edge
     *
     * @param[in] edge      edge between two basic blocks in the original code of the kernel
     * @param[in] proc      Instrumentation procedure to be inserted at the specified location
     *
     * @return true - success, false - failure
     *         In case of error, the IGtCore::LastError() function provides the specific failure reason
     */
    virtual bool InstrumentEdge(const IGtEdge& igtEdge, const IGtGenProcedure& proc) = 0;

    /*!
     * @brief Insert the specified instrumentation procedure at all entries to the kernel.
     * 
     * @param[in] proc      Instrumentation procedure to be inserted
     * 
     * @note This function guaranties that instrumentation procedure will be executed once per each
     *       execution of the kernel. This feature makes it different from the insertion of instrumentation
     *       procedure before each Cfg().EntryBbls(). The difference could be important in kernels with
     *       multiple (conditionally) connected entries, or kernels looping through entry point.
     * @return true - success, false - failure
     *         In case of error, the IGtCore::LastError() function provides the specific failure reason
     */
    virtual bool InstrumentEntries(const IGtGenProcedure& proc) = 0;

    /*!
     * @brief Insert the specified instrumentation procedure at all exits of the kernel - before each EOT
     *        instruction in the original code of the kernel
     * 
     * @param[in] proc      Instrumentation procedure to be inserted
     * @return true - success, false - failure
     *         In case of error, the IGtCore::LastError() function provides the specific failure reason
     */
    virtual bool InstrumentExits(const IGtGenProcedure& proc) = 0;

    /*!
     * @brief Remove the specified instruction from the original code of the kernel
     * 
     * @param[in] ins       Instruction to be removed from the original code of the kernel
     * @return true - success, false - failure
     *         In case of error, the IGtCore::LastError() function provides the specific failure reason
     */
    virtual bool RemoveInstruction(const IGtIns& ins) = 0;

    /*!
     * @brief Set the profiling filter that controls which version of the kernel code,
     * instrumented or original, will run on the HW slice/EU/thread.
     * 
     * The profiling filter comprises a number of bit masks associated with HW components.
     * Each bit in the mask represents a HW component with the corresponding ID.
     * Instrumented code will be executed on HW components whose corresponding bits in all masks are set on.
     * 
     * @param[in]    sliceMask              Slice mask
     * @param[in]    dualSubsliceMask       Dual subslice mask
     * @param[in]    subsliceMask           Subslice mask
     * @param[in]    euMask                 EU imask
     * @param[in]    threadMask             HW thread mask
     *
     * @return true - success, false - failure
     */
    virtual bool SetProfileFilter(uint32_t sliceMask, uint32_t dualSubsliceMask, uint32_t subsliceMask,
                                  uint32_t euMask, uint32_t threadMask) = 0;

    /*!
     * @brief Link the kernel with the specified High-Level Instrumentation (HLI) module
     * @param[in] moduleHandle    Handle to a module in the HLI library (see @ref HliLibrary)
     * 
     * @return true - success, false - failure
     *         In case of error, the IGtCore::LastError() function provides the specific failure reason
     * @note This function may not detect all possible errors because the module linking can be postponed
     *       until after recording all instrumentations. These "late" errors can be processed by error
     *       handlers, registered by the @ref RegisterEventHandler function
     */
    virtual bool LinkHliModule(IGtHliModuleHandle moduleHandle) = 0;

protected:
    /// @internal Protected destructor
    virtual ~IGtKernelInstrument() = default;
};
GTPIN_ASSERT_PURE_ABSTRACT(IGtKernelInstrument);

/* ===================================================================================================== */
// Class GtKernelExecStage
/* ===================================================================================================== */
/*! @ingroup GTPIN_KERNEL_DISPATCH
 * @brief Enumeration of the kernel execution stages (contexts)
 * 
 * Normally, GTPin invokes two callbacks that indicate two kernel execution stages: OnKernelRun notifies
 * about DISPATCH execution stage, and OnKernelComplete reports the final, COMPLETE stage of the kernel
 * execution.
 * However, on some plaforms, the kernel dispatch process can be logically divided on two stages.
 * For example, in the L0 platform, the dispatch starts when the kernel is appended to the command list.
 * At this point, GTPin invokes OnKernelRun callback in the DISPATCH context. The dispatch process finishes
 * when runtime submits the command list for execution. At this point, GTPin may invoke second OnKernelRun
 * callback in the FINAL_DISPATCH context, if requested by the IGtKernelDispatch::ReportFinalDispatchStage
 * function.
 * On platforms at which dispatch process is not divided on stages, single OnKernelRun callback is invoked
 * in the DISPATCH context.
 */
class GtKernelExecStage
{
public:
    enum ENUM :uint32_t
    {
        NONE,               ///< Kernel is not yet dispatched for execution
        DISPATCH,           ///< Kernel has been dispatched for execution
        FINAL_DISPATCH,     ///< Final dispatch stage of the kernel(reported on selected platforms)
        COMPLETE            ///< Kernel finished execution
    } value;

public:
    constexpr GtKernelExecStage(ENUM v = NONE) : value(v) {} ///< Conversion constructor
    constexpr operator ENUM() const { return value; }        ///< Conversion operator

    constexpr bool IsNone()          const { return value == NONE; }
    constexpr bool IsDispatch()      const { return value == DISPATCH; }
    constexpr bool IsFinalDispatch() const { return value == FINAL_DISPATCH; }
    constexpr bool IsComplete()      const { return value == COMPLETE; }

};
GTPIN_ASSERT_STANDARD_LAYOUT(GtKernelExecStage);

/* ===================================================================================================== */
// Class IGtKernelDispatch
/* ===================================================================================================== */
/*! @ingroup GTPIN_KERNEL_DISPATCH
 *  @brief Interface that controls kernel execution, from submission to completion.
 * 
 * Application can submit (dispatch) the same kernel multiple times, and each such dispatch can be
 * configured to run either instrumented or original version of the kernel function.
 * The IGtKernelDispatch interface provides such functionality.\n
 * Additionally, the tool, that chooses the instrumented version of the kernel, can use this interface
 * to allocate and assign the profile buffer for the kernel instance being submitted.
 * 
 * @note The lifetime of the IGtKernelDispatch object is limited - it is created before the kernel dispatch,
 *       and destroyed after the kernel completion. The tool should not attempt to access the
 *       IGtKernelDispatch object after processing the corresponding IGtTool::OnKernelComplete callback.
 * 
 * @par Availability:
 * - IGtTool::OnKernelRun
 * - IGtTool::OnKernelComplete
 */
class IGtKernelDispatch
{
public:
    /*!
     * @brief Get IGtKernel object associated with this kernel instance
     * @return Reference to the object that represents a kernel function being compiled/instrumented
     */
    virtual const IGtKernel& Kernel() const = 0;

    /*!
     * @brief Get the current execution stage of this kernel instance
     * @return Current execution stage of this kernel instance
     */
    virtual GtKernelExecStage ExecStage() const = 0;

    /*!
     * @brief Check to see whether this kernel instance has already finished execution
     * @return true, if this kernel instance has completed execution
     */
    inline bool IsCompleted() const { return ExecStage().IsComplete(); }

    /*!
     * @brief Create the profile data storage for this kernel instance, or return pointer to
     * existing, earlier created buffer
     * @return Pointer to the profile data storage of this kernel instance
     * @pre ExecStage().IsDispatch()
     */
    virtual IGtProfileBuffer*  CreateProfileBuffer() = 0;

    /*!
     * @brief Get profile buffer associated with this kernel instance.
     * @return Pointer to the profile data storage of this kernel instance, or nullptr
     * if no profile buffer was created for this instance.
     */
    virtual IGtProfileBuffer*  GetProfileBuffer() = 0;

    /*!
     * @brief Get IGtMemoryMapper object associated with this kernel instance
     * @return Object that provides host-to-device memory mapping service
     */
    virtual IGtMemoryMapper& MemoryMapper() = 0;

    /*!
     * @brief Get execution descriptor associated with this kernel instance
     * @param[out] execDesc     Structure that receives execution descriptor
     */
    virtual void GetExecDescriptor(GtKernelExecDesc& execDesc) const = 0;

    /*!
     * @brief Configure GTPin/driver to run either instrumented or original version of the kernel function
     * @param[in] enabled       true - run instrumented version; false - original
     * @pre ExecStage().IsDispatch()
     */
    virtual void SetProfilingMode(bool enabled) = 0;

    /*!
     * @brief Check to see whether this kernel instance runs instrumented code
     * @return true/false, for the kernel instance that is configured to run instrumented/original code.
     */
    virtual bool IsProfilingEnabled() const = 0;

    /*!
     * @return Unique ID of the kernel dispatch assigned by GTPin. It can be used to match OnKernelRun and
     * OnKernelComplete callbacks associated with the same kernel dispatch. 
     */
    virtual uint64_t DispatchId() const = 0;

    /*!
     * @brief Request GTPin to invoke the IGtTool::OnKernelRun callback one more time, as close as possible to the
     * actual execution of the kernel in GPU.
     * 
     * On plaforms, where the kernel dispatch can be logically divided on stages, the tool may request
     * additional IGtTool::OnKernelRun callback to be invoked on the FINAL_DISPATCH stage of the dispatch process.
     * This additional callback can be helpful if kernel state changes during dispatch stages are important to the tool,
     * e.g. if it uses the GetAllocatedBuffers function.
     * 
     * @return true - the request is accepted; false - the platform does not support multiple dispatch stages.
     * If function returns false, no callbacks will be invoked on the FINAL_DISPATCH stage.
     * @pre ExecStage().IsDispatch()
     */
    virtual bool ReportFinalDispatchStage() = 0;

    /*!
     * @brief Get value of the specified payload argument in this kernel dispatch
     * @param[in]   argIndex    Index of an explicit argument in the kernel payload.
     *                          This must be a valid GtKernelArgument::index value of an argument descriptor,
     *                          from the Kernel().PayloadArguments() sequence.
     *                          The valid value of this parameter is less than Kernel().NumExplicitPayloadArguments().
     * 
     * @return The argument value in the form of a raw byte sequence. The returned sequence is empty if queried
     * argument does not exist, or the kernel payload is unknown to GTPin.
     * The byte span { nullptr, size } stands for the null value of the specified size. 
     * In case of error, the IGtCore::LastError() function provides the specific failure reason.
     * @pre !IsCompleted()
     * @pre argIndex < Kernel().NumExplicitPayloadArguments()
     */
    virtual ConstByteSpan GetPayloadArgumentValue(uint32_t argIndex) const = 0;

    /*!
     * @return Sequence of buffers, allocated in the global memory and available to the kernel.
     * The returned sequence does not include profile buffer of the instrumented kernel. It comprises buffers
     * available to the original kernel only.
     * In case of error, the IGtCore::LastError() function provides the specific failure reason.
     * 
     * @note Application can allocate memory buffers for the kernel at any point until the final dispatch stage.
     *       So, it is recommended to use the GetAllocatedBuffers() function in the 'late' dispatch callback, if supported
     *       by the runtime platform. 
     * @pre !IsCompleted()
     */
    virtual GtMemoryBufferConstSpan GetAllocatedBuffers() const = 0;

    /// @return Size, in bytes, of the per-thread scratch space (PTSS) available to the kernel
    virtual uint32_t ScratchSpaceSize() const = 0;

    /// @return Size, in bytes, of the local memory (SLM) available to the kernel's work group
    virtual uint32_t SlmSize() const = 0;

protected:
    /// @internal Protected destructor
    virtual ~IGtKernelDispatch() = default;
};
GTPIN_ASSERT_PURE_ABSTRACT(IGtKernelDispatch);


/* ===================================================================================================== */
// GtKernelType implementation
/* ===================================================================================================== */
const char* GtKernelType::ToString() const
{
    switch (_value)
    {
    case KERNEL_TYPE_HS:    return "HS";
    case KERNEL_TYPE_DS:    return "DS";
    case KERNEL_TYPE_VS:    return "VS";
    case KERNEL_TYPE_PS:    return "PS";
    case KERNEL_TYPE_CS:    return "CS";
    case KERNEL_TYPE_GS:    return "GS";
    case KERNEL_TYPE_MS:    return "MS";
    case KERNEL_TYPE_TS:    return "TS";
    case KERNEL_TYPE_RT:    return "RT";
    default:                return "";
    }
}

GtKernelType GtKernelType::MakeFromString(const char* typeCstr)
{
    std::string str(typeCstr);
    std::transform(str.begin(), str.end(), str.begin(), [](unsigned char c) -> unsigned char { return (unsigned char)std::toupper(c); });

    if (str == "HS") return KERNEL_TYPE_HS;
    if (str == "DS") return KERNEL_TYPE_DS;
    if (str == "VS") return KERNEL_TYPE_VS;
    if (str == "PS") return KERNEL_TYPE_PS;
    if (str == "CS") return KERNEL_TYPE_CS;
    if (str == "GS") return KERNEL_TYPE_GS;
    if (str == "MS") return KERNEL_TYPE_MS;
    if (str == "TS") return KERNEL_TYPE_TS;
    if (str == "RT") return KERNEL_TYPE_RT;
    return KERNEL_TYPE_INVALID;
}

/* ===================================================================================================== */
// GtRtKernelType implementation
/* ===================================================================================================== */
const char* GtRtKernelType::ToString() const
{
    switch (_value)
    {
    case KERNEL_RT_TYPE_PROLOGUE:          return "PROLOGUE";
    case KERNEL_RT_TYPE_RAYGEN:            return "RAYGEN";
    case KERNEL_RT_TYPE_MISS:              return "MISS";
    case KERNEL_RT_TYPE_HIT:               return "HIT";
    case KERNEL_RT_TYPE_ANYHIT:            return "ANYHIT";
    case KERNEL_RT_TYPE_INTERSECTION:      return "INTERSECTION";
    case KERNEL_RT_TYPE_CALLABLE:          return "CALLABLE";
    case KERNEL_RT_TYPE_CALLSTACKHANDLER:  return "CALLSTACKHANDLER";
    default:                return "";
    }
}

GtRtKernelType GtRtKernelType::MakeFromString(const char* typeCstr)
{
    std::string str(typeCstr);
    std::transform(str.begin(), str.end(), str.begin(), [](unsigned char c) -> unsigned char { return (unsigned char)std::toupper(c); });


    if (str == "PROLOGUE")         return KERNEL_RT_TYPE_PROLOGUE;
    if (str == "RAYGEN")           return KERNEL_RT_TYPE_RAYGEN;
    if (str == "MISS")             return KERNEL_RT_TYPE_MISS;
    if (str == "HIT")              return KERNEL_RT_TYPE_HIT;
    if (str == "ANYHIT")           return KERNEL_RT_TYPE_ANYHIT;
    if (str == "INTERSECTION")     return KERNEL_RT_TYPE_INTERSECTION;
    if (str == "CALLABLE")         return KERNEL_RT_TYPE_CALLABLE;
    if (str == "CALLSTACKHANDLER") return KERNEL_RT_TYPE_CALLSTACKHANDLER;
    return KERNEL_RT_TYPE_INVALID;
}

#pragma pack(pop)
}
#endif
