/*
 * SPDX-FileCopyrightText: Copyright (c) 2021-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 * SPDX-License-Identifier: MIT
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 */

/*!
 * Provides GH100+ specific KernelGsp HAL implementations.
 */

#include "rmconfig.h"
#include "gpu/conf_compute/conf_compute.h"
#include "spdm/rmspdmtransport.h"


#include "gpu/fsp/kern_fsp.h"
#include "gpu/gsp/kernel_gsp.h"
#include "gpu/pmu/kern_pmu.h"
#include "gpu/sec2/kernel_sec2.h"
#include "gpu/mem_sys/kern_mem_sys.h"
#include "gsp/gspifpub.h"
#include "vgpu/rpc.h"
#include "os/os.h"

#include "published/hopper/gh100/dev_falcon_v4.h"
#include "published/hopper/gh100/dev_gsp.h"
#include "published/hopper/gh100/dev_riscv_pri.h"
#include "published/hopper/gh100/dev_vm.h"
#include "published/hopper/gh100/dev_bus.h"
#include "published/hopper/gh100/dev_bus_addendum.h"

#include "gpu/nvlink/kernel_nvlink.h"

#define RISCV_BR_ADDR_ALIGNMENT                 (8)

const char*
kgspGetSignatureSectionNamePrefix_GH100
(
    OBJGPU    *pGpu,
    KernelGsp *pKernelGsp
)
{
    ConfidentialCompute *pCC = GPU_GET_CONF_COMPUTE(pGpu);
    if (pCC != NULL && pCC->getProperty(pCC, PDB_PROP_CONFCOMPUTE_CC_FEATURE_ENABLED))
        return GSP_CC_SIGNATURE_SECTION_NAME_PREFIX;
    return GSP_SIGNATURE_SECTION_NAME_PREFIX;
}


/*!
 * Helper Function for kgspResetHw_GH100
 *
 * @param[in]  pGpu    OBJGPU pointer
 * @param[in]  pVoid   unused void pointer
 *
 * @return NvBool NV_TRUE if reset FSM reports ASSERTED
 */
static NvBool
_kgspWaitForAsserted
(
    OBJGPU *pGpu,
    void   *pVoid
)
{
    return GPU_FLD_TEST_DRF_DEF(pGpu, _PGSP, _FALCON_ENGINE, _RESET_STATUS,
                                _ASSERTED);
}


/*!
 * Helper Function for kgspResetHw_GH100
 *
 * @param[in]  pGpu    OBJGPU pointer
 * @param[in]  pVoid   unused void pointer
 *
 * @return NvBool NV_TRUE if reset FSM reports DEASSERTED
 */
static NvBool
_kgspWaitForDeasserted
(
    OBJGPU *pGpu,
    void   *pVoid
)
{
    return GPU_FLD_TEST_DRF_DEF(pGpu, _PGSP, _FALCON_ENGINE, _RESET_STATUS,
                                _DEASSERTED);
}

/*!
 * Reset the GSP HW
 *
 * @return NV_OK if the GSP HW was properly reset
 */
NV_STATUS
kgspResetHw_GH100
(
    OBJGPU *pGpu,
    KernelGsp *pKernelGsp
)
{
    NV_STATUS status = NV_OK;
    RMTIMEOUT timeout;
    NvU32     timeoutUSec = pGpu->timeoutData.defaultResetFSMStateTransitionUs;

    //
    // Add a default delay for 10us. This is a worst case estimate.
    // See bug 200636529 comment 20
    // Use PTIMER instead of the default ostimer since it is much faster than the
    // former and hence does not cause sufficient delay.
    // This delay can be overwritten by regkey RmResetFsmStateTimeoutUs.
    //
    gpuSetTimeout(pGpu, timeoutUSec, &timeout, GPU_TIMEOUT_FLAGS_TMR);

    GPU_FLD_WR_DRF_DEF(pGpu, _PGSP, _FALCON_ENGINE, _RESET, _ASSERT);
    status = gpuTimeoutCondWait(pGpu, _kgspWaitForAsserted, NULL, &timeout);
    if (status != NV_OK)
    {
        NV_PRINTF(LEVEL_ERROR, "Timed out waiting for GSP falcon reset to assert\n");
        DBG_BREAKPOINT();
    }

    // Reset timeout
    gpuSetTimeout(pGpu, timeoutUSec, &timeout, GPU_TIMEOUT_FLAGS_TMR);

    GPU_FLD_WR_DRF_DEF(pGpu, _PGSP, _FALCON_ENGINE, _RESET, _DEASSERT);
    status = gpuTimeoutCondWait(pGpu, _kgspWaitForDeasserted, NULL, &timeout);
    if (status != NV_OK)
    {
        NV_PRINTF(LEVEL_ERROR, "Timed out waiting for GSP falcon reset to deassert\n");
        DBG_BREAKPOINT();
    }

    return status;
}

NV_STATUS
kgspAllocBootArgs_GH100
(
    OBJGPU    *pGpu,
    KernelGsp *pKernelGsp
)
{
    NvP64 pVa = NvP64_NULL;
    NvP64 pPriv = NvP64_NULL;
    NV_STATUS nvStatus = NV_OK;
    NvU64 flags = MEMDESC_FLAGS_NONE;

    if (confComputeForceUnprotAlloc(pGpu))
    {
        flags |= MEMDESC_FLAGS_ALLOC_IN_UNPROTECTED_MEMORY;
    }

    // Allocate GSP-FMC arguments
    NV_ASSERT_OK_OR_GOTO(nvStatus,
                          memdescCreate(&pKernelGsp->pGspFmcArgumentsDescriptor,
                                        pGpu, sizeof(GSP_FMC_BOOT_PARAMS), 0x1000,
                                        NV_TRUE, ADDR_SYSMEM, NV_MEMORY_CACHED,
                                        flags),
                          _kgspAllocBootArgs_exit_cleanup);

    memdescTagAlloc(nvStatus, NV_FB_ALLOC_RM_INTERNAL_OWNER_UNNAMED_TAG_56,
                    pKernelGsp->pGspFmcArgumentsDescriptor);
    NV_ASSERT_OK_OR_GOTO(nvStatus,
                          nvStatus,
                          _kgspAllocBootArgs_exit_cleanup);

    NV_ASSERT_OK_OR_GOTO(nvStatus,
                          memdescMap(pKernelGsp->pGspFmcArgumentsDescriptor, 0,
                                     memdescGetSize(pKernelGsp->pGspFmcArgumentsDescriptor),
                                     NV_TRUE, NV_PROTECT_READ_WRITE,
                                     &pVa, &pPriv),
                          _kgspAllocBootArgs_exit_cleanup);

    pKernelGsp->pGspFmcArgumentsCached = (GSP_FMC_BOOT_PARAMS *)NvP64_VALUE(pVa);
    pKernelGsp->pGspFmcArgumentsMappingPriv = pPriv;

    return kgspAllocBootArgs_TU102(pGpu, pKernelGsp);

_kgspAllocBootArgs_exit_cleanup:
    kgspFreeBootArgs_HAL(pGpu, pKernelGsp);
    return nvStatus;
}

void
kgspFreeBootArgs_GH100
(
    OBJGPU    *pGpu,
    KernelGsp *pKernelGsp
)
{
    kgspFreeBootArgs_TU102(pGpu, pKernelGsp);

    // release wpr meta data resources
    if (pKernelGsp->pGspFmcArgumentsCached != NULL)
    {
        memdescUnmap(pKernelGsp->pGspFmcArgumentsDescriptor,
                     NV_TRUE,
                     (void *)pKernelGsp->pGspFmcArgumentsCached,
                     pKernelGsp->pGspFmcArgumentsMappingPriv);
        pKernelGsp->pGspFmcArgumentsCached = NULL;
        pKernelGsp->pGspFmcArgumentsMappingPriv = NULL;
    }
    if (pKernelGsp->pGspFmcArgumentsDescriptor != NULL)
    {
        memdescFree(pKernelGsp->pGspFmcArgumentsDescriptor);
        memdescDestroy(pKernelGsp->pGspFmcArgumentsDescriptor);
        pKernelGsp->pGspFmcArgumentsDescriptor = NULL;
    }
}

NvBool
kgspIsWpr2Up_GH100
(
    OBJGPU    *pGpu,
    KernelGsp *pKernelGsp
)
{
    ConfidentialCompute *pCC = GPU_GET_CONF_COMPUTE(pGpu);
    if (pCC != NULL && pCC->getProperty(pCC, PDB_PROP_CONFCOMPUTE_CC_FEATURE_ENABLED))
    {
        //
        // Due to BAR0 decoupler, we may not be able to read WPR2 MMU regs.
        // Assume WPR2 is down.
        //
        return NV_FALSE;
    }

    return kgspIsWpr2Up_TU102(pGpu, pKernelGsp);
}

NV_STATUS
kgspWaitForGfwBootOk_GH100
(
    OBJGPU *pGpu,
    KernelGsp *pKernelGsp
)
{
    KernelFsp *pKernelFsp = GPU_GET_KERNEL_FSP(pGpu);

    if (pKernelFsp != NULL)
    {
        return kfspWaitForSecureBoot_HAL(pGpu, pKernelFsp);
    }

    return NV_OK;
}

/*!
 * Populate WPR meta structure. On Hopper, this consists of determining the
 * minimum sizes of various regions that Client RM provides as inputs.
 * The actual offsets are determined by secure ACR ucode and patched
 * into the GspFwWprMeta structure provided to GSP-RM, so most of the
 * structure is 0-initialized here.
 *
 * Firmware scrubs the last 256mb of FB, no memory outside of this region
 * may be used until the FW RM has scrubbed the remainder of memory.
 *
 *   ---------------------------- <- fbSize (end of FB, 1M aligned)
 *   | VGA WORKSPACE            |
 *   ---------------------------- <- vbiosReservedOffset  (64K? aligned)
 *   | (potential align. gap)   |
 *   ---------------------------- <- gspFwWprEnd + frtsSize + pmuReservedSize
 *   | PMU mem reservation      |
 *   ---------------------------- <- gspFwWprEnd (128K aligned) + frtsSize
 *   | FRTS data                |
 *   | ------------------------ | <- frtsOffset
 *   | BOOT BIN (e.g. GSP-FMC)  |
 *   ---------------------------- <- bootBinOffset
 *   | GSP FW ELF               |
 *   ---------------------------- <- gspFwOffset
 *   | GSP FW (WPR) HEAP        |
 *   ---------------------------- <- gspFwHeapOffset
 *   | ACR-placed metadata      |
 *   | (struct GspFwWprMeta)    |
 *   ---------------------------- <- gspFwWprStart (128K aligned)
 *   | WPR and LSB Header       |
 *   ---------------------------- <- WPR2 Start
 *   | GSP FW (non-WPR) HEAP    |
 *   ---------------------------- <- nonWprHeapOffset, gspFwRsvdStart
 *
 * @param       pGpu          GPU object pointer
 * @param       pKernelGsp    KernelGsp object pointer
 * @param       pGspFw        Pointer to GSP-RM fw image.
 * @param       pFbRegionInfo Pointer to fb region table to fill in.
 */
NV_STATUS
kgspPopulateWprMeta_GH100
(
    OBJGPU         *pGpu,
    KernelGsp      *pKernelGsp,
    GSP_FIRMWARE   *pGspFw
)
{
    GspFwWprMeta        *pWprMeta = pKernelGsp->pWprMeta;
    RM_RISCV_UCODE_DESC *pRiscvDesc = pKernelGsp->pGspRmBootUcodeDesc;

    ct_assert(sizeof(*pWprMeta) == 256);

    NV_ASSERT_OR_RETURN(IS_GSP_CLIENT(pGpu), NV_ERR_NOT_SUPPORTED);

    NV_ASSERT_OR_RETURN(pKernelGsp->pGspRmBootUcodeImage != NULL, NV_ERR_INVALID_STATE);
    NV_ASSERT_OR_RETURN(pKernelGsp->gspRmBootUcodeSize != 0, NV_ERR_INVALID_STATE);
    NV_ASSERT_OR_RETURN(pRiscvDesc != NULL, NV_ERR_INVALID_STATE);

    //
    // We send these to FSP as the size to reserve above FRTS.
    // The actual offsets get filled in by ACR ucode when it sets up WPR2.
    //
    pWprMeta->vgaWorkspaceSize = 128 * 1024;
    pWprMeta->pmuReservedSize = kpmuReservedMemorySizeGet(GPU_GET_KERNEL_PMU(pGpu));

    // Physical address and size of GSP-FMC ucode in system memory
    pWprMeta->sizeOfBootloader = pKernelGsp->gspRmBootUcodeSize;
    pWprMeta->sysmemAddrOfBootloader =
        memdescGetPhysAddr(pKernelGsp->pGspRmBootUcodeMemdesc, AT_GPU, 0);

    // Physical address and size of GSP-RM firmware in system memory
    pWprMeta->sizeOfRadix3Elf = pGspFw->imageSize;
    pWprMeta->sysmemAddrOfRadix3Elf =
        memdescGetPhysAddr(pKernelGsp->pGspUCodeRadix3Descriptor, AT_GPU, 0);

    // Set necessary info from the GSP-FMC desc
    pWprMeta->bootloaderCodeOffset = pRiscvDesc->monitorCodeOffset;
    pWprMeta->bootloaderDataOffset = pRiscvDesc->monitorDataOffset;
    pWprMeta->bootloaderManifestOffset = pRiscvDesc->manifestOffset;

    if (pKernelGsp->pSignatureMemdesc != NULL)
    {
        pWprMeta->sysmemAddrOfSignature = memdescGetPhysAddr(pKernelGsp->pSignatureMemdesc, AT_GPU, 0);
        pWprMeta->sizeOfSignature = memdescGetSize(pKernelGsp->pSignatureMemdesc);
    }

    // The non-WPR heap lives in front of the WPR region
    pWprMeta->nonWprHeapSize = kgspGetNonWprHeapSize(pGpu, pKernelGsp);

    //
    // The WPR heap size (gspFwHeapSize) is variable to also get any padding needed
    // in the carveout to align the WPR start. This is a minimum size request to
    // the GSP-FMC.
    //
    // We won't know the exact size of everything that comes after the heap until
    // after the GSP-FMC lays it all out during boot. At any rate, this value isn't
    // needed on Hopper+, since the GSP-FMC can scrub/unlock anything we would need
    // before GSP-RM boots, so we pass 0 to allow the heap to extend outside the
    // pre-scrubbed area at the end of FB, if needed.
    //
    pWprMeta->gspFwHeapSize = kgspGetFwHeapSize(pGpu, pKernelGsp, 0, 0);

    // Fill in the FRTS size
    pWprMeta->frtsSize = kgspGetFrtsSize_HAL(pGpu, pKernelGsp);

    // Number of VF partitions allocating sub-heaps from the WPR heap
    pWprMeta->gspFwHeapVfPartitionCount =
        pGpu->bVgpuGspPluginOffloadEnabled ? kgspVgpuNumVgpuPartitions_HAL(pGpu, pKernelGsp) : 0;

    // CrashCat queue (if allocated in sysmem)
    KernelCrashCatEngine *pKernelCrashCatEng = staticCast(pKernelGsp, KernelCrashCatEngine);
    MEMORY_DESCRIPTOR *pCrashCatQueueMemDesc = kcrashcatEngineGetQueueMemDesc(pKernelCrashCatEng);
    if (pCrashCatQueueMemDesc != NULL)
    {
        NV_ASSERT_CHECKED(memdescGetAddressSpace(pCrashCatQueueMemDesc) == ADDR_SYSMEM);
        pWprMeta->sysmemAddrOfCrashReportQueue = memdescGetPhysAddr(pCrashCatQueueMemDesc, AT_GPU, 0);
        pWprMeta->sizeOfCrashReportQueue = (NvU32)memdescGetSize(pCrashCatQueueMemDesc);
    }

    // Fill in the meta-metadata
    pWprMeta->revision = GSP_FW_WPR_META_REVISION;
    pWprMeta->magic = GSP_FW_WPR_META_MAGIC;

    if (gpuIsCCMultiGpuProtectedPcieModeEnabled(pGpu))
    {
        pWprMeta->flags |= GSP_FW_FLAGS_PPCIE_ENABLED;
    }
    else
    {
        pWprMeta->flags &= ~GSP_FW_FLAGS_PPCIE_ENABLED;
    }

    return NV_OK;
}

static GSP_DMA_TARGET _kgspMemdescToDmaTarget
(
    MEMORY_DESCRIPTOR *pMemDesc
)
{
    switch (memdescGetAddressSpace(pMemDesc))
    {
        case ADDR_SYSMEM:
            return (NV_MEMORY_CACHED == memdescGetCpuCacheAttrib(pMemDesc)) ? GSP_DMA_TARGET_COHERENT_SYSTEM
                                                                            : GSP_DMA_TARGET_NONCOHERENT_SYSTEM;
        case ADDR_FBMEM:
            return GSP_DMA_TARGET_LOCAL_FB;
        default:
            return GSP_DMA_TARGET_COUNT;
    }
}

/*!
 * @brief Creates and populates the GSP-FMC/ACR arguments for booting GSP-RM.
 */
NV_STATUS
kgspSetupGspFmcArgs_GH100
(
    OBJGPU *pGpu,
    KernelGsp *pKernelGsp,
    KernelGspBootMode bootMode
)
{
    NV_ASSERT_OR_RETURN(IS_GSP_CLIENT(pGpu), NV_ERR_NOT_SUPPORTED);
    NV_ASSERT_OR_RETURN(pKernelGsp->pGspFmcArgumentsCached != NULL, NV_ERR_INVALID_STATE);

    GSP_FMC_BOOT_PARAMS *pGspFmcBootParams = pKernelGsp->pGspFmcArgumentsCached;

    ConfidentialCompute *pCC = GPU_GET_CONF_COMPUTE(pGpu);
    if (pCC != NULL)
    {
        pGspFmcBootParams->initParams.regkeys = pCC->gspProxyRegkeys;
        pGspFmcBootParams->gspRmMemParams.flushSysmemAddrValLo = 0;
        pGspFmcBootParams->gspRmMemParams.flushSysmemAddrValHi = 0;

        if (pCC->getProperty(pConfCompute, PDB_PROP_CONFCOMPUTE_WAR_5107790_SYSMEM_FLUSH_ADDR))
        {
            KernelMemorySystem *pKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu);

            pGspFmcBootParams->gspRmMemParams.flushSysmemAddrValLo = NvU64_LO32(pKernelMemorySystem->sysmemFlushBuffer);
            pGspFmcBootParams->gspRmMemParams.flushSysmemAddrValHi = NvU64_HI32(pKernelMemorySystem->sysmemFlushBuffer);
        }
    }

    KernelNvlink *pKernelNvlink = GPU_GET_KERNEL_NVLINK(pGpu);
    if (pKernelNvlink != NULL)
    {
        pGspFmcBootParams->initParams.regkeys |= knvlinkGetGSPProxyRegkeys(pGpu, pKernelNvlink);
    }

    if (bootMode == KGSP_BOOT_MODE_SR_RESUME)
    {
        pGspFmcBootParams->bootGspRmParams.gspRmDescOffset = memdescGetPhysAddr(pKernelGsp->pSRMetaDescriptor, AT_GPU, 0);
        pGspFmcBootParams->bootGspRmParams.gspRmDescSize = sizeof(GspFwSRMeta);
        pGspFmcBootParams->bootGspRmParams.target = _kgspMemdescToDmaTarget(pKernelGsp->pSRMetaDescriptor);
    }
    else
    {
        pGspFmcBootParams->bootGspRmParams.gspRmDescOffset = memdescGetPhysAddr(pKernelGsp->pWprMetaDescriptor, AT_GPU, 0);
        pGspFmcBootParams->bootGspRmParams.gspRmDescSize = sizeof(*pKernelGsp->pWprMeta);
        pGspFmcBootParams->bootGspRmParams.target = _kgspMemdescToDmaTarget(pKernelGsp->pWprMetaDescriptor);
    }

    pGspFmcBootParams->bootGspRmParams.bIsGspRmBoot = NV_TRUE;

    pGspFmcBootParams->gspRmParams.bootArgsOffset = memdescGetPhysAddr(pKernelGsp->pLibosInitArgumentsDescriptor, AT_GPU, 0);
    pGspFmcBootParams->gspRmParams.target = _kgspMemdescToDmaTarget(pKernelGsp->pLibosInitArgumentsDescriptor);

    Spdm *pSpdm = GPU_GET_SPDM(pGpu);

    if (pSpdm != NULL && pSpdm->getProperty(pSpdm, PDB_PROP_SPDM_ENABLED))
    {
        NV_STATUS  status = NV_OK;

        // Perform required pre-GSP-RM boot setup that could not be done on Spdm object creation.
        status = spdmSetupCommunicationBuffers(pGpu, pSpdm);
        if (status != NV_OK)
        {
            NV_PRINTF(LEVEL_ERROR, "Failure when initializing SPDM messaging infrastructure. Status:0x%x\n",
                                    status);
            return status;
        }

        // Target will always be GSP_DMA_TARGET_COHERENT_SYSTEM
        pGspFmcBootParams->gspSpdmParams.target               = GSP_DMA_TARGET_COHERENT_SYSTEM;
        pGspFmcBootParams->gspSpdmParams.payloadBufferOffset  = memdescGetPhysAddr(pSpdm->pPayloadBufferMemDesc,
                                                                                   AT_GPU, 0);
        pGspFmcBootParams->gspSpdmParams.payloadBufferSize    = pSpdm->payloadBufferSize;
    }

    return NV_OK;
}

/*!
 * Determine if PRIV lockdown is released or the FMC has encountered an error.
 */
static NvBool
_kgspLockdownReleasedOrFmcError
(
    OBJGPU  *pGpu,
    void    *pVoid
)
{
    KernelGsp *pKernelGsp = reinterpretCast(pVoid, KernelGsp *);
    KernelFalcon *pKernelFalcon = staticCast(pKernelGsp, KernelFalcon);
    NvU32 hwcfg2, mailbox0;

    //
    // If lockdown has not been released, check NV_PGSP_FALCON_MAILBOX0, where the GSP-FMC
    // (namely ACR) logs error codes during boot. GSP-FMC reported errors are always fatal,
    // so there's no reason to continue polling for lockdown release. This register can be
    // accessed when GSP PRIV is locked down, but not when the GSP PRIV target mask is locked
    // to FSP - so this shouldn't be called until the latter has been lifted.
    //
    // Generally, this mailbox check could fail if the PRIV target mask is *never* locked to FSP,
    // e.g., this isn't a Chain-of-Trust (COT) boot, because we'd be able to read the original
    // boot args addresses stuffed in the mailboxes (before the GSP-FMC has read and cleared
    // them during bootstrap, while the PRIV target mask is typically locked in COT). To avoid
    // premature failure in this case, if NV_PGSP_FALCON_MAILBOX0 is non-zero, check whether the
    // boot args address is still in the mailboxes, and continue to wait, if so.
    //
    mailbox0 = kflcnRegRead_HAL(pGpu, pKernelFalcon, NV_PFALCON_FALCON_MAILBOX0);
    if (mailbox0 != 0)
    {
        //
        // Collision of a real error code with the boot args address should never happen:
        // - the boot args phys address is page-allocated, so at least 4K-aligned, if not more
        // - the GSP-FMC error codes stashed in NV_PGSP_FALCON_MAILBOX0 are effectively 8-bit
        //   and value 0 means "no error"
        //
        NvU64 physAddr = memdescGetPhysAddr(pKernelGsp->pGspFmcArgumentsDescriptor, AT_GPU, 0);
        if ((NvU64_LO32(physAddr) == mailbox0) &&
            (NvU64_HI32(physAddr) == kflcnRegRead_HAL(pGpu, pKernelFalcon, NV_PFALCON_FALCON_MAILBOX1)))
            return NV_FALSE;
    }

    hwcfg2 = kflcnRegRead_HAL(pGpu, pKernelFalcon, NV_PFALCON_FALCON_HWCFG2);

    return (FLD_TEST_DRF(_PFALCON, _FALCON_HWCFG2, _RISCV_BR_PRIV_LOCKDOWN, _UNLOCK, hwcfg2) ||
            (mailbox0 != 0));
}

/*!
 * Determine if SPDM partition has booted or the FMC has encountered an error.
 */
static NvBool
_kgspSpdmBootedOrFmcError
(
    OBJGPU  *pGpu,
    void    *pVoid
)
{
    //
    // The GSP-FMC (namely ACR) logs error codes to NV_PGSP_FALCON_MAILBOX0 during boot.
    // During normal boot, the mailboxes should be 0 (cleared by the GSP-FMC ucode at the
    // start), so consider any non-zero value here a reason to stop polling.
    //
    KernelGsp *pKernelGsp = GPU_GET_KERNEL_GSP(pGpu);
    NvU32      mailbox0;

    mailbox0 = kflcnRegRead_HAL(pGpu, (KernelFalcon *)pVoid, NV_PFALCON_FALCON_MAILBOX0);
    if (mailbox0 != 0)
    {
        //
        // Collision of a real error code with the boot args address should never happen:
        // - the boot args phys address is page-allocated, so at least 4K-aligned, if not more
        // - the GSP-FMC error codes stashed in NV_PGSP_FALCON_MAILBOX0 are effectively 8-bit
        //   and value 0 means "no error"
        //
        NvU64 physAddr = memdescGetPhysAddr(pKernelGsp->pGspFmcArgumentsDescriptor, AT_GPU, 0);
        if ((NvU64_LO32(physAddr) == mailbox0) &&
            (NvU64_HI32(physAddr) == kflcnRegRead_HAL(pGpu, (KernelFalcon *)pVoid, NV_PFALCON_FALCON_MAILBOX1)))
        {
            return NV_FALSE;
        }
    }
    return (kflcnRegRead_HAL(pGpu, (KernelFalcon *)pVoid, NV_PFALCON_FALCON_MAILBOX0) != 0);
}

static NvBool
_kgspFalconMailbox0Cleared
(
    OBJGPU  *pGpu,
    void    *pVoid
)
{
    return (kflcnRegRead_HAL(pGpu, (KernelFalcon *)pVoid, NV_PFALCON_FALCON_MAILBOX0) == 0);
}

/*!
 * Establish session with SPDM Responder on GSP.
 * GSP-RM boot is blocked until session establishment completes.
 *
 * NOTE: This function currently requires the API lock (for at least the async init
 * RPCs and libspdm synchronization) and as such does NOT support parallel init.
 */
static NV_STATUS
_kgspEstablishSpdmSession
(
    OBJGPU    *pGpu,
    KernelGsp *pKernelGsp
)
{
    NV_STATUS     status        = NV_OK;
    KernelFalcon *pKernelFalcon = staticCast(pKernelGsp, KernelFalcon);
    NvU32         requesterId   = NV_SPDM_REQUESTER_ID_NULL;
    NvBool        bCcEnabled    = NV_FALSE;
    Spdm         *pSpdm         = GPU_GET_SPDM(pGpu);

    ConfidentialCompute *pConfCompute = GPU_GET_CONF_COMPUTE(pGpu);

    if (pConfCompute->getProperty(pConfCompute, PDB_PROP_CONFCOMPUTE_ENABLED))
    {
        bCcEnabled  = NV_TRUE;
        requesterId = NV_SPDM_REQUESTER_ID_CONF_COMPUTE;
    }

    // Ensure SPDM Responder has booted before attempting to establish session.
    status = gpuTimeoutCondWait(pGpu, _kgspSpdmBootedOrFmcError, pKernelFalcon, NULL);
    if (status != NV_OK)
    {
        NV_PRINTF(LEVEL_ERROR, "Timeout waiting for SPDM Responder to initialize!\n");
        goto exit;
    }

    //
    // Check if the terminating condition of the above wait was that SPDM has booted, or
    // if the FMC produced an error code in the mailbox.
    //
    NvU32 mailbox0 = kflcnRegRead_HAL(pGpu, pKernelFalcon, NV_PFALCON_FALCON_MAILBOX0);
    if (mailbox0 != NV_SPDM_PARTITION_BOOT_SUCCESS)
    {
        NV_PRINTF(LEVEL_ERROR, "GSP-FMC reported an error prior to SPDM boot: 0x%x\n",
                  mailbox0);
        status = NV_ERR_NOT_READY;
        goto exit;
    }

    NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR,
                        spdmEstablishSession(pGpu, pSpdm, requesterId),
                        exit);

    if (bCcEnabled)
    {
        status = confComputeDeriveSessionKeys_HAL(pGpu, pConfCompute);
        if (status != NV_OK)
        {
            NV_PRINTF(LEVEL_ERROR, "CC key derivation failed.\n");
            goto exit;
        }
    }

    // Tell SPDM that it can continue with boot
    kflcnRegWrite_HAL(pGpu, pKernelFalcon, NV_PFALCON_FALCON_MAILBOX0, NV_SPDM_REQUESTER_SECRETS_DERIVED);

    //
    // Wait for SPDM to restore the original mailbox value, which should have been 0 prior
    // to SPDM boot starting. This is needed because the subsequent lockdown release check
    // will also consider any non-zero value in mailbox0 to indicate an FMC error code.
    //
    status = gpuTimeoutCondWait(pGpu, _kgspFalconMailbox0Cleared, (void *)pKernelFalcon, NULL);
    if (status != NV_OK)
    {
        NV_PRINTF(LEVEL_ERROR, "Timeout waiting for SPDM to proceed with boot!\n");
        goto exit;
    }

exit:
    if (status != NV_OK)
    {
        KernelFsp *pKernelFsp = GPU_GET_KERNEL_FSP(pGpu);

        if (pKernelFsp != NULL)
        {
            NV_PRINTF(LEVEL_ERROR, "Failed to establish session with SPDM Responder!\n");
            if (pKernelFsp->getProperty(pKernelFsp, PDB_PROP_KFSP_GSP_MODE_GSPRM))
            {
                kfspDumpDebugState_HAL(pGpu, pKernelFsp);
            }
            NV_PRINTF(LEVEL_ERROR, "NV_PGSP_FALCON_MAILBOX0 = 0x%x\n",
                    kflcnRegRead_HAL(pGpu, pKernelFalcon, NV_PFALCON_FALCON_MAILBOX0));
            NV_PRINTF(LEVEL_ERROR, "NV_PGSP_FALCON_MAILBOX1 = 0x%x\n",
                    kflcnRegRead_HAL(pGpu, pKernelFalcon, NV_PFALCON_FALCON_MAILBOX1));
        }
    }

    return status;
}

/*!
 * Determine if GSP has performed CC partition cleanup.
 * Success or failure will be determined by the value in Mailbox1
 */
static NvBool
_kgspHasCcCleanupFinished
(
    OBJGPU  *pGpu,
    void    *pVoid
)
{
    NvU32         ccCleanupStatus = 0;
    KernelFalcon *pKernelFalcon   = (KernelFalcon *)pVoid;

    ccCleanupStatus = kflcnRegRead_HAL(pGpu, pKernelFalcon, NV_PFALCON_FALCON_MAILBOX1);

    //
    // To avoid a timing issue and issues with pre-existing values in Mailbox1
    // we check for explicit success or failure statuses.
    //
    return (ccCleanupStatus == NV_SPDM_SECRET_TEARDOWN_SUCCESS ||
            ccCleanupStatus == NV_SPDM_SECRET_TEARDOWN_FAILURE);
}

static NV_STATUS
_kgspBootstrapGspFmc_GH100
(
    OBJGPU    *pGpu,
    KernelGsp *pKernelGsp
)
{
    KernelFalcon *pKernelFalcon = staticCast(pKernelGsp, KernelFalcon);
    RmPhysAddr physAddr;

    // Reset the GSP to prepare for RISCV bootstrap
    NV_ASSERT_OK_OR_RETURN(kgspResetHw_HAL(pGpu, pKernelGsp));

    // Stuff the GSP-FMC arguments into the mailbox regs
    physAddr = memdescGetPhysAddr(pKernelGsp->pGspFmcArgumentsDescriptor, AT_GPU, 0);
    kflcnRegWrite_HAL(pGpu, pKernelFalcon, NV_PFALCON_FALCON_MAILBOX0, NvU64_LO32(physAddr));
    kflcnRegWrite_HAL(pGpu, pKernelFalcon, NV_PFALCON_FALCON_MAILBOX1, NvU64_HI32(physAddr));

    // CC needs additional "regkeys" stuffed in a separate mailbox for the init partition
    ConfidentialCompute *pConfCompute =  GPU_GET_CONF_COMPUTE(pGpu);
    GPU_REG_WR32(pGpu, NV_PGSP_MAILBOX(NV_PGSP_MAILBOX_REGISTER_CC_REGKEYS), pConfCompute->gspProxyRegkeys);

    // Bootstrap the GSP-FMC by pointing the GSP's BootROM at it
    RM_RISCV_UCODE_DESC *pRiscvDesc = pKernelGsp->pGspRmBootUcodeDesc;
    RmPhysAddr fmcPhysAddr = memdescGetPhysAddr(pKernelGsp->pGspRmBootUcodeMemdesc, AT_GPU, 0);

    // Program FMC code DMAADDR
    physAddr = (fmcPhysAddr + pRiscvDesc->monitorCodeOffset) >> RISCV_BR_ADDR_ALIGNMENT;
    kflcnRiscvRegWrite_HAL(pGpu, pKernelFalcon, NV_PRISCV_RISCV_BCR_DMAADDR_FMCCODE_LO,
                           NvU64_LO32(physAddr));
    kflcnRiscvRegWrite_HAL(pGpu, pKernelFalcon, NV_PRISCV_RISCV_BCR_DMAADDR_FMCCODE_HI,
                           NvU64_HI32(physAddr));

    // Program FMC Data DMAADDR
    physAddr = (fmcPhysAddr + pRiscvDesc->monitorDataOffset) >> RISCV_BR_ADDR_ALIGNMENT;
    kflcnRiscvRegWrite_HAL(pGpu, pKernelFalcon, NV_PRISCV_RISCV_BCR_DMAADDR_FMCDATA_LO,
                           NvU64_LO32(physAddr));
    kflcnRiscvRegWrite_HAL(pGpu, pKernelFalcon, NV_PRISCV_RISCV_BCR_DMAADDR_FMCDATA_HI,
                           NvU64_HI32(physAddr));

    // Program manifest DMAADDR
    physAddr = (fmcPhysAddr + pRiscvDesc->manifestOffset) >> RISCV_BR_ADDR_ALIGNMENT;
    kflcnRiscvRegWrite_HAL(pGpu, pKernelFalcon, NV_PRISCV_RISCV_BCR_DMAADDR_PKCPARAM_LO,
                           NvU64_LO32(physAddr));
    kflcnRiscvRegWrite_HAL(pGpu, pKernelFalcon, NV_PRISCV_RISCV_BCR_DMAADDR_PKCPARAM_HI,
                           NvU64_HI32(physAddr));

    NvU32 target = NV_PRISCV_RISCV_BCR_DMACFG_TARGET_COHERENT_SYSMEM;
    NvU32 dmaCfg = DRF_NUM(_PRISCV_RISCV, _BCR_DMACFG, _TARGET, target) |
                   DRF_DEF(_PRISCV_RISCV, _BCR_DMACFG, _LOCK,   _LOCKED);

    kflcnRiscvRegWrite_HAL(pGpu, pKernelFalcon, NV_PRISCV_RISCV_BCR_DMACFG, dmaCfg);

    kflcnRiscvProgramBcr_HAL(pGpu, pKernelFalcon, NV_TRUE);

    // Start it
    kflcnRiscvRegWrite_HAL(pGpu, pKernelFalcon, NV_PRISCV_RISCV_CPUCTL,
                           DRF_DEF(_PRISCV_RISCV, _CPUCTL, _STARTCPU, _TRUE));

    return NV_OK;
}

/*!
 * Prepare to boot GSP-RM
 *
 * This routine handles the prequesites to booting GSP-RM that requires the API LOCK:
 *   - Clear ECC errors
 *   - Prepare GSP-FMC arguments
 *   - Prepare FSP or SEC2 boot commands, depending on what is applicable
 *
 * @param[in]   pGpu            GPU object pointer
 * @param[in]   pKernelGsp      GSP object pointer
 * @param[in]   bootMode        GSP boot mode
 *
 * @return NV_OK if GSP-RM RISCV preparation to boot was successful.
 *         Appropriate NV_ERR_xxx value otherwise.
 */
NV_STATUS
kgspPrepareForBootstrap_GH100
(
    OBJGPU *pGpu,
    KernelGsp *pKernelGsp,
    KernelGspBootMode bootMode
)
{
    KernelFsp *pKernelFsp = GPU_GET_KERNEL_FSP(pGpu);
    KernelSec2 *pKernelSec2 = GPU_GET_KERNEL_SEC2(pGpu);

    // Only for GSP client builds
    if (!IS_GSP_CLIENT(pGpu))
    {
        NV_PRINTF(LEVEL_ERROR, "IS_GSP_CLIENT is not set.\n");
        return NV_ERR_NOT_SUPPORTED;
    }

    // Setup the descriptors that GSP-FMC needs to boot GSP-RM
    NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
            kgspSetupGspFmcArgs_HAL(pGpu, pKernelGsp, bootMode));

    if (pKernelFsp != NULL && !pKernelFsp->getProperty(pKernelFsp, PDB_PROP_KFSP_DISABLE_GSPFMC))
    {
        pKernelFsp->setProperty(pKernelFsp, PDB_PROP_KFSP_GSP_MODE_GSPRM, NV_TRUE);
        NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
            kfspPrepareBootCommands_HAL(pGpu, pKernelFsp));
    }
    else if (pKernelSec2 != NULL && pKernelSec2->getProperty(pKernelSec2, PDB_PROP_KSEC2_BOOT_GSPFMC))
    {
        NV_PRINTF(LEVEL_INFO, "Sec2 preparing for GSPRM boot\n");
        pKernelSec2->setProperty(pKernelSec2, PDB_PROP_KSEC2_GSP_MODE_GSPRM, NV_TRUE);
        NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
            ksec2PrepareBootCommands_HAL(pGpu, pKernelSec2));
    }

    return NV_OK;
}

/*!
 * Boot GSP-RM.
 *
 * This routine handles the following:
 *   - sends FSP or SEC2 boot commands
 *   - waits for GSP-RM to complete initialization
 *
 * Note that this routine is based on flcnBootstrapRiscvOS_GA102().
 *
 * Note that this routine can be called without the API lock for
 * parllel initialization.
 *
 * @param[in]   pGpu            GPU object pointer
 * @param[in]   pKernelGsp      GSP object pointer
 * @param[in]   pGspFw          GSP_FIRMWARE image pointer
 *
 * @return NV_OK if GSP-RM RISCV boot was successful.
 *         Appropriate NV_ERR_xxx value otherwise.
 */
NV_STATUS
kgspBootstrap_GH100
(
    OBJGPU         *pGpu,
    KernelGsp      *pKernelGsp,
    KernelGspBootMode bootMode
)
{
    KernelFalcon *pKernelFalcon = staticCast(pKernelGsp, KernelFalcon);
    KernelFsp    *pKernelFsp = GPU_GET_KERNEL_FSP(pGpu);
    KernelSec2   *pKernelSec2 = GPU_GET_KERNEL_SEC2(pGpu);
    NV_STATUS     status = NV_OK;
    NvU32         mailbox0;

    // Reset the GSP-FMC error code register to prevent reporting stale data
    gpuResetGspFmcErrorCode_HAL(pGpu);

    //
    // Hopper+ GSP always boots in RISC-V mode. The GSP-FMC blocks RISCV_BCR_CTRL reads via PLM,
    // so override the setting here so KernelFalcon doesn't try to read it from HW.
    //
    kflcnSetRiscvMode(pKernelFalcon, NV_TRUE);

    if (pKernelFsp != NULL && !pKernelFsp->getProperty(pKernelFsp, PDB_PROP_KFSP_DISABLE_GSPFMC))
    {
        //
        // For resume, ACR will restore FRTS based on what was saved during suspend.
        // For GC6, FRTS will stay alive in WPR2. So, instruct FSP to skip
        // vidmem FRTS for boot modes other than normal boot.
        //
        if (bootMode != KGSP_BOOT_MODE_NORMAL)
        {
            pKernelFsp->pCotPayload->frtsVidmemOffset = 0;
            pKernelFsp->pCotPayload->frtsVidmemSize = 0;
        }

        NV_PRINTF(LEVEL_NOTICE, "Starting to boot GSP via FSP.\n");
        NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, kfspSendBootCommands_HAL(pGpu, pKernelFsp));
    }
    else if (pKernelSec2 != NULL && pKernelSec2->getProperty(pKernelSec2, PDB_PROP_KSEC2_BOOT_GSPFMC))
    {
        NV_PRINTF(LEVEL_INFO, "Starting to boot GSP via SEC2.\n");
        NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, ksec2SendBootCommands_HAL(pGpu, pKernelSec2));
    }
    else
    {
        NV_ASSERT_OK_OR_RETURN(_kgspBootstrapGspFmc_GH100(pGpu, pKernelGsp));
    }

    // Wait for target mask to be released.
    if (pKernelFsp != NULL)
    {
        status = kfspWaitForGspTargetMaskReleased_HAL(pGpu, pKernelFsp);
        if (status != NV_OK)
        {
            NV_PRINTF(LEVEL_ERROR, "Timeout waiting for GSP target mask release. "
                      "This error may be caused by several reasons: Bootrom may have failed, "
                      "GSP init code may have failed or ACR failed to release target mask. "
                      "RM does not have access to information on which of those conditions happened.\n");
            gpuReportGspFmcErrorCode_HAL(pGpu);
            if (pKernelFsp->getProperty(pKernelFsp, PDB_PROP_KFSP_GSP_MODE_GSPRM))
            {
                kfspDumpDebugState_HAL(pGpu, pKernelFsp);
            }

            return status;
        }
    }
    else if (pKernelSec2 != NULL &&
             pKernelSec2->getProperty(pKernelSec2, PDB_PROP_KSEC2_BOOT_GSPFMC) &&
             pKernelSec2->getProperty(pKernelSec2, PDB_PROP_KSEC2_GSP_MODE_GSPRM))
    {
        status = ksec2WaitForGspTargetMaskReleased_HAL(pGpu, pKernelSec2);
        if (status != NV_OK)
        {
            NV_PRINTF(LEVEL_ERROR, "SEC2 GSP Boot:Timeout waiting for GSP target mask release. "
                      "This error may be caused by several reasons: Bootrom may have failed, "
                      "GSP init code may have failed or ACR failed to release target mask. "
                      "RM does not have access to information on which of those conditions happened.\n");
            gpuReportGspFmcErrorCode_HAL(pGpu);
            return status;
        }
    }

    //
    // When enabled, SPDM session will be established before GSP-RM boots.
    // Wait until after target mask is released by ACR to minimize busy wait time.
    // NOTE: This is incompatible with parallel initialization. See function
    // description for more details.
    //
    Spdm *pSpdm = GPU_GET_SPDM(pGpu);

    if (pSpdm != NULL && pSpdm->getProperty(pSpdm, PDB_PROP_SPDM_ENABLED))
    {
        NV_ASSERT_OR_RETURN(rmapiLockIsOwner(), NV_ERR_INVALID_LOCK_STATE);
        NV_ASSERT_OK_OR_RETURN(_kgspEstablishSpdmSession(pGpu, pKernelGsp));
    }

    //
    // Send init RPCs necessary for GSP-RM booting, before creating OBJGPU.
    // This can only be performed after TARGET_MASK is released by FSP. If SPDM is
    // enabled, the init RPCs will also be encrypted since the session is established.
    // GSP-RM will keep polling until the last message is successfully sent.
    // This is not necessary for suspend/resume as GSP saves/restores context
    if (bootMode == KGSP_BOOT_MODE_NORMAL)
    {
        status = kgspSendInitRpcs(pGpu, pKernelGsp);
        if (status != NV_OK)
        {
            NV_PRINTF(LEVEL_ERROR, "kgspSendInitRpcs failed 0x%x!\n", status);
            return status;
        }
    }


    // Wait for lockdown to be released or the FMC to report an error
    status = gpuTimeoutCondWait(pGpu, _kgspLockdownReleasedOrFmcError, pKernelGsp, NULL);
    if (status != NV_OK)
    {
        NV_PRINTF(LEVEL_ERROR, "Timeout waiting for lockdown release. It's also "
                "possible that bootrom may have failed. RM may not have access to "
                "the BR status to be able to say for sure what failed.\n");
        gpuReportGspFmcErrorCode_HAL(pGpu);
        if (pKernelFsp != NULL && pKernelFsp->getProperty(pKernelFsp, PDB_PROP_KFSP_GSP_MODE_GSPRM))
        {
            kfspDumpDebugState_HAL(pGpu, pKernelFsp);
        }

        NV_PRINTF(LEVEL_ERROR, "NV_PGSP_FALCON_MAILBOX0 = 0x%x\n",
                  kflcnRegRead_HAL(pGpu, pKernelFalcon, NV_PFALCON_FALCON_MAILBOX0));
        NV_PRINTF(LEVEL_ERROR, "NV_PGSP_FALCON_MAILBOX1 = 0x%x\n",
                  kflcnRegRead_HAL(pGpu, pKernelFalcon, NV_PFALCON_FALCON_MAILBOX1));
        return status;
    }
    else if (gpuCheckGspFmcErrorCode_HAL(pGpu))
    {
        gpuReportGspFmcErrorCode_HAL(pGpu);
        return NV_ERR_NOT_READY;
    }
    else if ((mailbox0 = kflcnRegRead_HAL(pGpu, pKernelFalcon, NV_PFALCON_FALCON_MAILBOX0)) != 0)
    {
        NV_PRINTF(LEVEL_ERROR, "GSP-FMC reported an error while attempting to boot GSP: 0x%x\n",
            mailbox0);
        return NV_ERR_NOT_READY;
    }

    // Program FALCON_OS
    RM_RISCV_UCODE_DESC *pRiscvDesc = pKernelGsp->pGspRmBootUcodeDesc;
    kflcnRegWrite_HAL(pGpu, pKernelFalcon, NV_PFALCON_FALCON_OS, pRiscvDesc->appVersion);

    // Ensure the CPU is started
    if (kflcnIsRiscvActive_HAL(pGpu, pKernelFalcon))
    {
        NV_PRINTF(LEVEL_INFO, "GSP ucode loaded and RISCV started.\n");
    }
    else
    {
        NV_ASSERT_FAILED("Failed to boot GSP");
        gpuReportGspFmcErrorCode_HAL(pGpu);
        return NV_ERR_NOT_READY;
    }

    NV_PRINTF(LEVEL_INFO, "Waiting for GSP fw RM to be ready...\n");

    //
    // For normal boot, link the status queue.
    // Note: for resume or GC6 exit, GSP-RM will restore queue state.
    //
    if (bootMode == KGSP_BOOT_MODE_NORMAL)
    {
        NV_ASSERT_OK_OR_RETURN(GspStatusQueueInit(pGpu, &pKernelGsp->pRpc->pMessageQueueInfo));
    }

    NV_ASSERT_OK_OR_RETURN(kgspWaitForRmInitDone(pGpu, pKernelGsp));

    NV_PRINTF(LEVEL_INFO, "GSP FW RM ready.\n");

    return status;
}

/*!
 * Teardown remaining GSP state after GSP-RM unloads.
 *
 * @param[in]   pGpu            GPU object pointer
 * @param[in]   pKernelGsp      GSP object pointer
 * @param[in]   unloadMode      GSP unload mode
 */
NV_STATUS
kgspTeardown_GH100
(
    OBJGPU *pGpu,
    KernelGsp *pKernelGsp,
    KernelGspUnloadMode unloadMode
)
{
    // Wait for GSP to halt to allow ACR and GSP FMC to finish shutdown
    return kflcnWaitForHaltRiscv_HAL(pGpu, staticCast(pKernelGsp, KernelFalcon), GPU_TIMEOUT_DEFAULT, 0);
}

void
kgspGetGspRmBootUcodeStorage_GH100
(
    OBJGPU *pGpu,
    KernelGsp *pKernelGsp,
    BINDATA_STORAGE **ppBinStorageImage,
    BINDATA_STORAGE **ppBinStorageDesc
)
{

        ConfidentialCompute *pCC = GPU_GET_CONF_COMPUTE(pGpu);
        if (pCC != NULL && pCC->getProperty(pCC, PDB_PROP_CONFCOMPUTE_CC_FEATURE_ENABLED))
        {
            const BINDATA_ARCHIVE *pBinArchiveConcatenatedFMCDesc = kgspGetBinArchiveConcatenatedFMCDesc_HAL(pKernelGsp);
            const BINDATA_ARCHIVE *pBinArchiveConcatenatedFMC     = kgspGetBinArchiveConcatenatedFMC_HAL(pKernelGsp);

            if (kgspIsDebugModeEnabled(pGpu, pKernelGsp))
            {
                *ppBinStorageImage = (BINDATA_STORAGE *)bindataArchiveGetStorage(pBinArchiveConcatenatedFMC, BINDATA_LABEL_UCODE_IMAGE_DBG);
                *ppBinStorageDesc  = (BINDATA_STORAGE *)bindataArchiveGetStorage(pBinArchiveConcatenatedFMCDesc, BINDATA_LABEL_UCODE_DESC_DBG);
            }
            else
            {
                *ppBinStorageImage = (BINDATA_STORAGE *)bindataArchiveGetStorage(pBinArchiveConcatenatedFMC, BINDATA_LABEL_UCODE_IMAGE_PROD);
                *ppBinStorageDesc  = (BINDATA_STORAGE *)bindataArchiveGetStorage(pBinArchiveConcatenatedFMCDesc, BINDATA_LABEL_UCODE_DESC_PROD);
            }

            return;
        }

    kgspGetGspRmBootUcodeStorage_GA102(pGpu, pKernelGsp, ppBinStorageImage, ppBinStorageDesc);
}

NV_STATUS
kgspIssueNotifyOp_GH100
(
    OBJGPU *pGpu,
    KernelGsp *pKernelGsp,
    NvU32 opCode,
    NvU32 *pArgs,
    NvU32 argc
)
{
    NV_STATUS status = NV_OK;
    NotifyOpSharedSurface *pNotifyOpSharedSurface;
    PORT_ATOMIC NvU32 *pInUse;
    volatile NvU32 *pSeqAddr;
    volatile NvU32 *pStatusAddr;
    NvU32 *pOpCodeAddr;
    NvU32 *pArgsAddr;
    NvU32 *pArgcAddr;

    NvU32 seqValue;
    NvU32 i;
    RMTIMEOUT timeout;

    ct_assert(sizeof(PORT_ATOMIC NvU32) == sizeof(NvU32));
    ct_assert(NV_ALIGNOF(PORT_ATOMIC NvU32) == NV_ALIGNOF(NvU32));

    // 1. Validate the arguments.
    NV_CHECK_OR_RETURN(LEVEL_ERROR, opCode < GSP_NOTIFY_OP_OPCODE_MAX, NV_ERR_INVALID_ARGUMENT);
    NV_CHECK_OR_RETURN(LEVEL_ERROR, argc <= GSP_NOTIFY_OP_MAX_ARGUMENT_COUNT, NV_ERR_INVALID_ARGUMENT);
    NV_CHECK_OR_RETURN(LEVEL_ERROR, (argc == 0 || pArgs != NULL), NV_ERR_INVALID_ARGUMENT);

    // 2. Set up ptrs to the shared memory.
    NV_ASSERT_OR_RETURN(pKernelGsp->pNotifyOpSurf != NULL, NV_ERR_INVALID_STATE);

    pNotifyOpSharedSurface = pKernelGsp->pNotifyOpSurf;

    pInUse      = (PORT_ATOMIC NvU32 *) &(pNotifyOpSharedSurface->inUse);
    pSeqAddr    = (NvU32*) &(pNotifyOpSharedSurface->seqNum);
    pOpCodeAddr = (NvU32*) &(pNotifyOpSharedSurface->opCode);
    pStatusAddr = (NvU32*) &(pNotifyOpSharedSurface->status);
    pArgcAddr   = (NvU32*) &(pNotifyOpSharedSurface->argc);
    pArgsAddr   = (NvU32*) pNotifyOpSharedSurface->args;

    while (!portAtomicCompareAndSwapU32(pInUse, 1U, 0U))
    {
        // We're not going to sleep, but safe to sleep also means safe to spin..
        NV_ASSERT_OR_RETURN(portSyncExSafeToSleep(), NV_ERR_INVALID_STATE);
        osSpinLoop();
    }

    // 3. Read current sequence counter value.
    seqValue = *pSeqAddr;

    // 4. Populate the opcode and arguments.
    *pOpCodeAddr = opCode;
    *pArgcAddr = argc;
    for (i = 0; i < argc; i++)
    {
        pArgsAddr[i] = pArgs[i];
    }

    //
    // Issue a store fence to ensure that the parameters have made it to memory
    // before the interrupt is triggered.
    //
    portAtomicMemoryFenceStore();

    // 5. Trigger notification interrupt to GSP.
    gpuSetTimeout(pGpu, GPU_TIMEOUT_DEFAULT, &timeout, 0);
    GPU_VREG_WR32(pGpu, NV_VIRTUAL_FUNCTION_PRIV_DOORBELL, NV_DOORBELL_NOTIFY_LEAF_SERVICE_LOCKLESS_OP_HANDLE);

    // 6. Poll on the sequence number to ensure the op completed.
    while (seqValue + 1 != *pSeqAddr)
    {
        status = gpuCheckTimeout(pGpu, &timeout);

        if (status != NV_OK)
        {
            NV_PRINTF(LEVEL_ERROR, "gpuCheckTimeout failed, status = 0x%x\n", status);
            goto error_ret;
        }
        osSpinLoop();
    }

    status = *pStatusAddr;

error_ret:
    portAtomicSetU32(pInUse, 0);

    NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, status);
    return status;
}

NV_STATUS
kgspCheckGspRmCcCleanup_GH100
(
    OBJGPU    *pGpu,
    KernelGsp *pKernelGsp
)
{
    KernelFalcon *pKernelFalcon   = staticCast(pKernelGsp, KernelFalcon);
    NvU32         ccCleanupStatus = 0;
    NV_STATUS     status          = NV_OK;

    // Wait for GSP to explicitly state we have completed CC cleanup.
    status = gpuTimeoutCondWait(pGpu, _kgspHasCcCleanupFinished, (void *)pKernelFalcon, NULL);
    if (status != NV_OK)
    {
        NV_PRINTF(LEVEL_ERROR, "CC secret cleanup failed due to timeout!\n");
        goto exit;
    }

    // Now, check whether or not CC cleanup was successful.
    ccCleanupStatus = kflcnRegRead_HAL(pGpu, pKernelFalcon, NV_PFALCON_FALCON_MAILBOX1);
    if (ccCleanupStatus == NV_SPDM_SECRET_TEARDOWN_SUCCESS)
    {
        // FIPS: Make status clear to user.
        NV_PRINTF(LEVEL_ERROR, "CC secret cleanup successful!\n");
    }
    else
    {
        NV_PRINTF(LEVEL_ERROR, "CC secret cleanup failed! Status 0x%x\n",
                  ccCleanupStatus);
    }

exit:

    // Regardless of success or failure, write ack to GSP.
    kflcnRegWrite_HAL(pGpu, pKernelFalcon, NV_PFALCON_FALCON_MAILBOX1, NV_SPDM_SECRET_TEARDOWN_ACK);

    return status;
}
