/*========================== begin_copyright_notice ============================
Copyright (C) 2024-2026 Intel Corporation

SPDX-License-Identifier: MIT
============================= end_copyright_notice ===========================*/

/*!
 * @file Implementation of the FunLatency Tool
 */
#include <fstream>
#include <sstream>
#include <iomanip>
#include <algorithm>

#include "funlatency.h"

using namespace gtpin;
using namespace std;

/* ============================================================================================= */
// Configuration
/* ============================================================================================= */
Knob<int>  knobNumThreadBuckets("num_thread_buckets", 0, "Number of thread buckets. 0 (default) - maximum thread buckets");
Knob<bool> knobUseRegInstrument("no_reg_instrument", true, "Disable register-bound instrumentation");
Knob<bool> knobSkipZeroResults("skip_zero_results", false, "Skip zero results in the Funlatency output");
Knob<bool> knobIncludeZeroExecMask("include_zero_exec_mask", false, "Include measurements of functions called with zero exec mask");

/* ============================================================================================= */
// FunLatency implementation
/* ============================================================================================= */
FunLatency* FunLatency::Instance()
{
    static FunLatency instance;
    return &instance;
}

void FunLatency::OnKernelBuild(IGtKernelInstrument& instrumentor)
{
    /*
     * Register allocation policy
     * ======================================================================================================
     * To ensure accuracy of latency measurments, the instrumentation overhead should be as low as possible.
     * It is believed that register-bound procedures incur lower overhead than memory-bound procedures.
     * Assuming that, we will try to use register-bound instrumentation for as many sites as possible, and
     * apply memory-bound instrumentation to the rest of the sites.
     *
     * To check and guarantee availability of free physical registers we pre-allocate registers
     * in the register-bound procedures created by the GeneratePostCodeRegBound() function.
     * For this purpose we use the IGtRegAllocator::ReserveVregOperands() interface.
     *
     * For memory-bound procedures, pre-allocation of registers is not needed, and even can be harmful, as
     * explained in the IGtRegAllocator description. On the other hand, excessive pre-allocation of registers
     * in the register-bound procedures may cause multiple spills/fills in memory-bound procedures. To avoid
     * this situation, we do the following:
     *      A) RESERVE registers in memory-bound procedures BEFORE generating register-bound procedures using
     *         IGtRegAllocator::Reserve()
     *      B) RELEASE registers in memory-bound procedures AFTER generating register-bound procedures using
     *         IGtRegAllocator::ReleaseReserved()
     * 
     * Function Latency measurment algorithm 
     * ======================================================================================================
     * Count the cycles it takes for each function to execute from the first instruction to the end of the function
     * by starting the timer before the first instruction in each function, and stopping it before the last
     * instruction in the function.
     * If a function calls other functions, in order to exclude the callee's total cycles from the caller's
     * cycles count, the tool will stop the timer before "call" instruction,count the cycels,and restart it 
     * after the call instruction. 
     * for example :
     *   //   fun1:                     fun1:                        //
     *   //    [...             \\       start the timer (preCode)   //
     *   //    ...          =====\\      [...                        //
     *   //    ...                \\     ...]                        //
     *   //    ...]                \\    stop the timer (postCode)   //
     *   //    call fun2           //    call fun2                   //
     *   //    [...               //     start the timer             //
     *   //    ...          =====//      [...                        //
     *   //    ...              //       ...]                        //
     *   //    ...]                      stop the timer              //
     *   //    ret                       ret                         //
     *
     *   fun1 total cycles measurment will be done in two parts .
     *   (A) count the total cycles from the function entry to the call ins
     *   (B) count the total cycles after call ins to the end of the function
     *   fun1 total cycles = A + B 
     *  
     */

    const IGtGenCoder& coder          = instrumentor.Coder();
    IGtVregFactory&    vregs          = coder.VregFactory();
    IGtRegAllocator&   ra             = coder.RegAllocator();
    bool               is64BitCounter = Use64BitCounters(coder);

    // Initialize virtual registers
    _timeReg = vregs.Make(VREG_TYPE_DWORD);
    _addrReg = vregs.MakeMsgAddrScratch();
    _dataReg = vregs.MakeMsgDataScratch(is64BitCounter ? VREG_TYPE_QWORD : VREG_TYPE_DWORD);

    // A) Reserve registers used in the memory-bound procedures
    ra.Reserve(_dataReg.VregNumber());
    ra.Reserve(_addrReg.VregNumber());
    ra.Reserve(vregs.GetProfileBufferAddrVreg().Num());

    InstrumentFunctions(instrumentor);

    // B) Release registers used in the memory-bound procedures
    ra.ReleaseReserved(_dataReg.VregNumber());
    ra.ReleaseReserved(_addrReg.VregNumber());
    ra.ReleaseReserved(vregs.GetProfileBufferAddrVreg().Num());
}

void FunLatency::InstrumentFunctions(IGtKernelInstrument& instrumentor)
{
    const IGtKernel&           kernel    = instrumentor.Kernel();
    const IGtCfg&              cfg       = instrumentor.Cfg();
    const IGtGenCoder&         coder     = instrumentor.Coder();
    const IGtGenModel&         genModel  = kernel.GenModel();
    IGtProfileBufferAllocator& allocator = instrumentor.ProfileBufferAllocator();
    
    uint32_t simdMask;
    switch (kernel.SimdWidth())
    {
    case 8:  simdMask = 0xFF; break;
    case 16: simdMask = 0xFFFF; break;
    case 32: simdMask = 0xFFFFFFFF; break;
    default: simdMask = 0; break;
    }

    uint32_t numRecords = cfg.NumFunctions();   // numRecords = number of functions

    // Insert code that stops timer at BBL's exit and stores result in the profile buffer
    GtGenProcedure postCode;
    GtProfileArray profileArray(sizeof(FunLatencyRecord), numRecords, genModel.MaxThreadBuckets());
    profileArray.Allocate(allocator);

    GtGenProcedure finiCode;                                // Procedure that stores results of register-bound measurments in the profile buffer
    bool           tryRegInstrument = knobUseRegInstrument; // true - try to apply register-bound instrumentation

    std::map<FunctionId, FunctionDescriptor> functions;

    // Iterate over the function instrument the first and the last BBl in each function.
    uint32_t idx = 0;
    for (auto func : cfg.Functions())
    {
        // Start the timer in each function entry 
        const IGtBbl& firstBbl = *func->Bbls().first();
        const IGtIns& firstIns = firstBbl.FirstIns();
        const IGtBbl& lastBbl  = *func->Bbls().last();
        const IGtIns& lastIns  = lastBbl.LastIns();

        // Insert code that starts timer at BBL's entry
        GtGenProcedure preCode;
        GeneratePreCode(preCode, coder);
        InstrumentBbl(instrumentor, firstBbl, GtIpoint::Before(), preCode);

        bool isMain  = func->IsKernelMain();
        bool isEntry = true;

        // Insert FunctionDescriptor for the function into the kernel functions map
        functions.emplace(std::piecewise_construct,
            std::forward_as_tuple(func->Id()),
            std::forward_as_tuple(func->Id(), func->Name(), isMain, firstIns.Id(), lastIns.Id(), idx));

        bool instrumentAfterCall = false; // Insert precode Instrumentation after call instruction
        for (auto bbl : func->Bbls())
        {
            if (instrumentAfterCall)
            {
                GtGenProcedure precode;
                GeneratePreCode(precode, coder);
                InstrumentBbl(instrumentor, *bbl, GtIpoint::Before(), precode);
                instrumentAfterCall = false;
            }

            const IGtIns& lastBblIns = bbl->LastIns();

            if (bbl->IsEot() || (!isMain && (lastBblIns.Opcode() == GED_OPCODE_ret)))
            {

                // Insert code that stops timer at the last BBL's exit and stores result in the profile buffer
                GtGenProcedure postCode;

                // Insert fake consumers of source registers to expose hidden latency of EOT instructions
                if (bbl->IsEot())
                {
                    coder.GenerateFakeSrcConsumers(postCode, lastBblIns);
                }

                if (tryRegInstrument)
                {
                    tryRegInstrument = GeneratePostCodeRegBound(postCode, finiCode, coder, profileArray, &functions.at(func->Id()), isEntry, true, simdMask);
                }
                if (!tryRegInstrument)
                {
                    GeneratePostCodeMemBound(postCode, coder, profileArray, &functions.at(func->Id()), true, simdMask);
                }
                InstrumentInstruction(instrumentor, lastBblIns, GtIpoint::Before(), postCode);
                idx++;
                isEntry = false;
                break;
            }
            else if (lastBblIns.Opcode() == GED_OPCODE_call || lastBblIns.Opcode() == GED_OPCODE_calla)
            {
                // Insert code that stops timer at the last BBL's exit and stores result in the profile buffer
                GtGenProcedure postCode;

                if (tryRegInstrument)
                {
                    tryRegInstrument = GeneratePostCodeRegBound(postCode, finiCode, coder, profileArray, &functions.at(func->Id()), isEntry, false, simdMask);
                }
                if (!tryRegInstrument)
                {
                    GeneratePostCodeMemBound(postCode, coder, profileArray, &functions.at(func->Id()), false, simdMask);
                }
                InstrumentInstruction(instrumentor, lastBblIns, GtIpoint::Before(), postCode);

                instrumentAfterCall = true;
                isEntry = false;
            }
        }
    }
    // Insert 'finiCode' at all exits of the kernel - before each EOT instruction
    instrumentor.InstrumentExits(finiCode);

    // Transform functions into array of instrumentation sites
    InstrumentSites sites(numRecords);
    std::transform(cfg.Functions().begin(), cfg.Functions().end(), sites.begin(), [](const IGtFunction* f) -> FunctionId { return f->Id(); });

    // Create FunLatencyKernelProfile object that represents profile of this kernel
    _kernels.emplace(kernel.Id(), FunLatencyKernelProfile(kernel, cfg, profileArray, std::move(sites), std::move(functions)));
}

void FunLatency::OnKernelRun(IGtKernelDispatch& dispatcher)
{
    bool             isProfileEnabled = false;
    const IGtKernel& kernel           = dispatcher.Kernel();
    GtKernelExecDesc execDesc; dispatcher.GetExecDescriptor(execDesc);
    if (kernel.IsInstrumented() && IsKernelExecProfileEnabled(execDesc, kernel.GpuPlatform(), kernel.Name().Get()))
    {
        auto it = _kernels.find(kernel.Id());

        if (it != _kernels.end())
        {
            IGtProfileBuffer* buffer = dispatcher.CreateProfileBuffer(); GTPIN_ASSERT(buffer);
            FunLatencyKernelProfile& kernelProfile = it->second;
            const GtProfileArray& profileArray = kernelProfile.GetProfileArray();
            if (profileArray.Initialize(*buffer))
            {
                isProfileEnabled = true;
            }
            else
            {
                GTPIN_ERROR_MSG(string("FunLatency : ") + string(kernel.Name()) + " : Failed to write into memory buffer");
            }
        }
    }
    dispatcher.SetProfilingMode(isProfileEnabled);
}

void FunLatency::OnKernelComplete(IGtKernelDispatch& dispatcher)
{
    const IGtKernel& kernel = dispatcher.Kernel();
    GtKernelExecDesc execDesc; dispatcher.GetExecDescriptor(execDesc);
    bool isProfilingEnabled = dispatcher.IsProfilingEnabled();
    if (!isProfilingEnabled || !IsKernelExecProfileEnabled(execDesc, kernel.GpuPlatform(), kernel.Name().Get()))
    {
        return; // Do nothing with unprofiled kernel dispatches
    }

    auto it = _kernels.find(kernel.Id());

    if (it != _kernels.end())
    {
        const IGtProfileBuffer* buffer = dispatcher.GetProfileBuffer(); GTPIN_ASSERT(buffer);
        FunLatencyKernelProfile& kernelProfile = it->second;
        const GtProfileArray& profileArray = kernelProfile.GetProfileArray();

        for (uint32_t recordNum = 0; recordNum != profileArray.NumRecords(); ++recordNum)
        {
            FunLatencyDispatchProfile dispatchProfile(dispatcher, kernelProfile.GetSiteId(recordNum));

            for (uint32_t threadBucket = 0; threadBucket < profileArray.NumThreadBuckets(); ++threadBucket)
            {
                FunLatencyRecord record;
                if (!profileArray.Read(*buffer, &record, recordNum, 1, threadBucket))
                {
                    GTPIN_ERROR_MSG(string("FunLatency : ") + string(kernel.Name()) + " : Failed to read from memory buffer");
                }
                else
                {
                    dispatchProfile.Accumulate(record);
                }
            }
            kernelProfile.AddDispatchProfile(dispatchProfile);
        }
    }
}

bool FunLatency::Use64BitCounters(const IGtGenCoder& coder)
{
    return coder.InstructionFactory().CanAccessAtomically(GED_DATA_TYPE_uq);
}

void FunLatency::GeneratePreCode(GtGenProcedure& proc, const IGtGenCoder& coder)
{
    coder.StartTimer(proc, _timeReg);
    if (!proc.empty()) { proc.front()->AppendAnnotation(__func__); }
}

bool FunLatency::GeneratePostCodeRegBound(GtGenProcedure& postProc, GtGenProcedure& finiProc, const IGtGenCoder& coder,
    const GtProfileArray& profileArray, FunctionDescriptor* funcDesc, bool isFuncEntry, bool updateFrequency, uint32_t simdMask)
{
    IGtInsFactory&   insF  = coder.InstructionFactory();
    IGtVregFactory&  vregs = coder.VregFactory();
    IGtRegAllocator& ra    = coder.RegAllocator();
    bool            is64BitCounter = Use64BitCounters(coder);

    GtReg freqReg;
    GtReg cycleReg;
    if (isFuncEntry)
    {
        funcDesc->freqReg = vregs.MakeCounter(VREG_TYPE_DWORD);                                         // Frequency counter
        funcDesc->cycleReg = vregs.MakeCounter(is64BitCounter ? VREG_TYPE_QWORD : VREG_TYPE_DWORD);      // Cycle counter
    }

    freqReg  = funcDesc->freqReg;
    cycleReg = funcDesc->cycleReg;

    GtReg       flagReg = FlagReg(0);
    GtPredicate pred(flagReg);

    GtReg cycleRegL = { cycleReg, sizeof(uint32_t), 0 };                                        // Low 32-bits of cycle counter
    GtReg cycleRegH = { cycleReg, sizeof(uint32_t), 1 };                                        // High 32-bits of cycle counter
    GtReg dataRegL  = { _dataReg, sizeof(uint32_t), 0 };                                        // Low 32-bits of '_dataReg'

    // Generate procedure that computes and aggregates cycles and frequency
    GtGenProcedure proc;
    coder.StopTimerExt(proc, _timeReg);

    proc += insF.MakeAdd(cycleRegL, cycleRegL, _timeReg); // Add elapsed time to lower 32-bits of of 'cycleReg'
    if (is64BitCounter)
    {
        // If cycleRegL overflowed, increment the high 32-bits of 'cycleReg'
        proc += insF.MakeCmp(GED_COND_MODIFIER_l, flagReg, cycleRegL, _timeReg);
        proc += insF.MakeAdd(cycleRegH, cycleRegH, 1).SetPredicate(pred);
    }

    GtReg tmpReg  = vregs.MakeScratch();
    GtReg tmpReg1 = _timeReg;
    if (!knobIncludeZeroExecMask)
    {
        proc += insF.MakeMov(tmpReg, ChannelEnableReg());
        proc += insF.MakeMov(tmpReg1, DispatchMaskReg());
        proc += insF.MakeAnd(tmpReg, tmpReg1, tmpReg);
        proc += insF.MakeAnd(tmpReg, tmpReg, simdMask);
        proc += insF.MakeCmp(GED_COND_MODIFIER_z, flagReg, tmpReg, 0);
    }
    if (updateFrequency)
    {
        proc += insF.MakeAdd(freqReg, freqReg, 1);  // freq++
        if (!knobIncludeZeroExecMask)
        {
            proc.back()->SetPredicate(!pred);
        }
    }

    if (!knobIncludeZeroExecMask)
    {
        proc += insF.MakeMov(cycleRegL, 0).SetPredicate(pred); // Clear the low 32-bits of 'cycleReg'
        if (is64BitCounter)
        {
            proc += insF.MakeMov(cycleRegH, 0).SetPredicate(pred); // Clear the high 32-bits of 'cycleReg'
        }
    }

    if (isFuncEntry && !ra.ReserveVregOperands(proc))
    {
        return false; // No more free registers
    }
    proc.front()->AppendAnnotation(__func__);
    postProc.MoveAfter(std::move(proc));

    // Generate 'finiProc' procedure that stores aggregated cycles and frequency counters in the profile buffer
    profileArray.ComputeAddress(coder, proc, _addrReg, funcDesc->recordNum);
    proc += insF.MakeRegMov(_dataReg, cycleReg);
    proc += insF.MakeAtomicAdd(NullReg(), _addrReg, _dataReg, (is64BitCounter ? GED_DATA_TYPE_uq : GED_DATA_TYPE_ud));

    if (updateFrequency)
    {
        profileArray.ComputeRelAddress(coder, proc, _addrReg, _addrReg, offsetof(FunLatencyRecord, freq));
        proc += insF.MakeRegMov(dataRegL, freqReg);
        proc += insF.MakeAtomicAdd(NullReg(), _addrReg, _dataReg, GED_DATA_TYPE_ud);

        proc.front()->AppendAnnotation("GenerateFiniCode");
        finiProc.MoveAfter(std::move(proc));
    }

    return true;
}

void FunLatency::GeneratePostCodeMemBound(GtGenProcedure& proc, const IGtGenCoder& coder,
    const GtProfileArray& profileArray, FunctionDescriptor* funcDesc, bool updateFrequency, uint32_t simdMask)
{
    IGtInsFactory&  insF           = coder.InstructionFactory();
    IGtVregFactory& vregs          = coder.VregFactory();
    bool            is64BitCounter = Use64BitCounters(coder);
    GtReg           dataRegL       = { _dataReg, sizeof(uint32_t), 0 };  // Low 32-bits of the data payload register

    // Generate code that computes elapsed time
    coder.StopTimerExt(proc, _timeReg);

    // cycles += _timeReg
    proc += insF.MakeMov(dataRegL, _timeReg);   // Move timer value to the low 32-bits of the data register
    if (is64BitCounter)
    {
        // Clear the high 32-bits of the data payload register
        GtReg dataRegH = { _dataReg, sizeof(uint32_t), 1 };
        proc += insF.MakeMov(dataRegH, 0);
    }

    GtReg       tmpReg  = vregs.MakeScratch();
    GtReg       tmpReg1 = _timeReg;
    GtReg       flagReg = FlagReg(0);
    GtPredicate pred(flagReg);

    if (!knobIncludeZeroExecMask)
    {
        proc += insF.MakeMov(tmpReg, ChannelEnableReg());
        proc += insF.MakeMov(tmpReg1, DispatchMaskReg());
        proc += insF.MakeAnd(tmpReg, tmpReg1, tmpReg);
        proc += insF.MakeAnd(tmpReg, tmpReg, simdMask);
        proc += insF.MakeCmp(GED_COND_MODIFIER_z, flagReg, tmpReg, 0);
    }

    // _addrReg =  address of the current thread's FunLatencyRecord in the profile buffer
    profileArray.ComputeAddress(coder, proc, _addrReg, funcDesc->recordNum);

    proc += insF.MakeAtomicAdd(NullReg(), _addrReg, _dataReg, (is64BitCounter ? GED_DATA_TYPE_uq : GED_DATA_TYPE_ud));
    if (!knobIncludeZeroExecMask)
    {
        proc.back()->SetPredicate(!pred);
    }

    // freq++
    if (updateFrequency)
    {
        profileArray.ComputeRelAddress(coder, proc, _addrReg, _addrReg, offsetof(FunLatencyRecord, freq));
        proc += insF.MakeAtomicInc(NullReg(), _addrReg, GED_DATA_TYPE_ud);
        if (!knobIncludeZeroExecMask)
        {
            proc.back()->SetPredicate(!pred);
        }
    }

    if (!proc.empty()) { proc.front()->AppendAnnotation(__func__); }
}

std::string FunLatency::ToString() const
{
    ostringstream ostr;
    ostr << "### Kernel/Shader execution-time profile generated by GTPin ###" << endl << endl;
    ostr << "Legend:" << endl;
    ostr << "NA - kernel was not instrumented." << endl << endl;
    if (!knobIncludeZeroExecMask)
    {
        ostr << "NOTE: Tool reports cycles and frequencies only for functions that were called with at least one enabled channel in execution mask." << endl << endl;
    }
    ostr << setw(20) << "Kernel Name" << setw(20) << "HashID" << setw(30) << "Function Name" << setw(15) << "Inst";
    ostr << setw(15) << "Freq." << setw(20) << "Total-latency" << setw(20) << "Total-latency(%)" << setw(20) << "Avg-Cycles"<< setw(10) << "Type";
    ostr << setw(10) << "SIMD" << setw(15) << "Platform" << setw(35) << "Execution descriptor";
    ostr << endl;
    for (const auto& kernelEntry : _kernels)
    {
        ostr << kernelEntry.second.ToString();
    }
    return ostr.str();
}

void FunLatency::DumpAsm() const
{
    for (const auto& kernelEntry : _kernels)
    {
        kernelEntry.second.DumpAsm();
    }
}

void FunLatency::Fini()
{
    string profileDir = GTPin_GetCore()->ProfileDir();
    string filePath = JoinPath(profileDir, "FunLatency.txt");

    ofstream fs(filePath);
    if (fs.is_open())
    {
        fs << ToString();
        fs.close();
    }
    else
    {
        GTPIN_WARNING("FunLatency : could not create file: " + filePath);
    }
    DumpAsm();
}

/* ============================================================================================= */
// FunLatencyDispatchProfile implementation
/* ============================================================================================= */
FunLatencyDispatchProfile::FunLatencyDispatchProfile(const IGtKernelDispatch& kernelDispatch, InstrumentSiteId id) :
    siteId(id), cycles(0), freq(0)
{
    kernelDispatch.GetExecDescriptor(kernelExecDesc);
}

void FunLatencyDispatchProfile::Accumulate(const FunLatencyRecord& record)
{
    cycles += record.cycles;
    freq += record.freq;
}

/* ============================================================================================= */
// FunctionDescriptor implementation
/* ============================================================================================= */
FunctionDescriptor::FunctionDescriptor(const FunctionId fId, const std::string fName,
    const bool fIsMain, const InsId firstIns, const InsId lastIns, const uint32_t recNum) :
    id(fId), name(fName), isMain(fIsMain), firstInsId(firstIns), lastInsId(lastIns), recordNum(recNum) {};

/* ============================================================================================= */
// FunLatencyKernelProfile implementation
/* ============================================================================================= */
FunLatencyKernelProfile::FunLatencyKernelProfile(const IGtKernel& kernel,
    const IGtCfg& cfg,
    const GtProfileArray& profileArray,
    InstrumentSites&& instrumentSites,
    FunctionMap&& functions) :
    _name(GlueString(kernel.Name())), _uniqueName(kernel.UniqueName()), _type(kernel.Type()), _platform(kernel.GpuPlatform()), _hashId(kernel.HashId()),
    _simd(kernel.SimdWidth()), _asmText(CfgAsmText(cfg)), _profileArray(profileArray), _totalKernelCycles(0),
    _instrumentSites(std::move(instrumentSites)),_functionMap(std::move(functions)) {};

FunLatencyDispatchProfile& FunLatencyKernelProfile::AddDispatchProfile(const FunLatencyDispatchProfile& dispatchProfile)
{
    _totalKernelCycles += dispatchProfile.cycles;
    _dispatchProfiles.push_back(dispatchProfile);
    return _dispatchProfiles.back();
}

InstrumentSiteId FunLatencyKernelProfile::GetSiteId(uint32_t recordNum) const
{
    GTPIN_ASSERT(recordNum < _instrumentSites.size());
    return _instrumentSites[recordNum];
}

void FunLatencyKernelProfile::DumpAsm() const
{
    DumpKernelAsmText(_name, _uniqueName, _asmText);
}

string FunLatencyKernelProfile::ToString() const
{
    ostringstream ostr;
    if (!_dispatchProfiles.empty())
    {
        for (const auto& dp : _dispatchProfiles)
        {
            if (knobSkipZeroResults && dp.freq == 0)
            {
                continue; // Skip zero results
            }

            auto func = _functionMap.find(dp.siteId)->second;
            double dispatchToExecTotal = (_totalKernelCycles ? ((100.0 * (double)dp.cycles) / _totalKernelCycles) : 0.0);
            uint64_t    avgCycles = (dp.freq ? (dp.cycles / dp.freq) : 0);
            string funcName = func.name + (func.isMain ? "(main)" : "");
            if (funcName.length() > 29)
            {
                funcName = funcName.substr(0, 29);
            }

            if (func.isMain)
            {
                ostr << endl;
            }

            if (func.isMain)
            {
                ostr << endl;
            }
            ostr << setw(20) << _name << setw(20) << hex << _hashId;
            ostr << setw(30) << funcName << setw(15) << std::string("[" + std::to_string((uint32_t)func.firstInsId) + "-" + std::to_string((uint32_t)func.lastInsId) + "]");
            ostr << setw(15) << dec << dp.freq << setw(20) << dec << dp.cycles;
            ostr << fixed << setprecision(2) << setw(20) << dispatchToExecTotal << setw(20) << avgCycles << setw(10) << _type.ToString();
            ostr << setw(10) << _simd << setw(15) << _platform.ToString() << setw(35) << dp.kernelExecDesc.ToString(_platform, ExecDescAlignedFormat());
            ostr << endl;
        }
    }
    else
    {
        ostr << setw(20) << _name << setw(20) << hex << _hashId;
        ostr << setw(30) << "NA" << setw(15) << +"NA";
        ostr << setw(15) << dec << "NA" << setw(20) << dec << "NA";
        ostr << fixed << setprecision(2) << setw(20) << "NA" << setw(20) << "NA" << setw(10) << "NA";
        ostr << setw(10) << "NA" << setw(15) << "NA" << setw(35) << "NA";
        ostr << endl;
    }

    ostr << endl;
    return ostr.str();
}

/* ============================================================================================= */
// GTPin_Entry
/* ============================================================================================= */
EXPORT_C_FUNC void GTPin_Entry(int argc, const char* argv[])
{
    ConfigureGTPin(argc, argv);
    FunLatency::Instance()->Register();
    atexit(FunLatency::OnFini);
}
