/* _NVRM_COPYRIGHT_BEGIN_
 *
 * Copyright 2013-2015 by NVIDIA Corporation.  All rights reserved.  All
 * information contained herein is proprietary and confidential to NVIDIA
 * Corporation.  Any use, reproduction, or disclosure without the written
 * permission of NVIDIA Corporation is prohibited.
 *
 * _NVRM_COPYRIGHT_END_
 */

/*
 * This file sets up the communication between the UVM driver and RM. RM will
 * call the UVM driver providing to it the set of OPS it supports.  UVM will
 * then return by filling out the structure with the callbacks it supports.
 */

#define  __NO_VERSION__

#include "nv-misc.h"
#include "os-interface.h"
#include "nv-linux.h"

#if defined(NV_UVM_ENABLE)

#include "nv_uvm_interface.h"
#include "nv_gpu_ops.h"

struct UvmOpsUvmEvents   *g_pNvUvmEvents = NULL;

static nvidia_stack_t *g_sp;
static struct semaphore g_spLock;

// Use these to test g_sp usage. When DEBUG_GLOBAL_STACK, one out of every
// DEBUG_GLOBAL_STACK_THRESHOLD calls to nvUvmGetSafeStack will use g_sp.
#define DEBUG_GLOBAL_STACK 0
#define DEBUG_GLOBAL_STACK_THRESHOLD 2

static atomic_t g_debugGlobalStackCount = ATOMIC_INIT(0);

// Called at module load, not by an external client
int nv_uvm_init(void)
{
    int rc = nv_kmem_cache_alloc_stack(&g_sp);
    if (rc != 0)
        return rc;

    NV_INIT_MUTEX(&g_spLock);
    return 0;
}

void nv_uvm_exit(void)
{
    nv_kmem_cache_free_stack(g_sp);
}


// Testing code to force use of the global stack every now and then
static NvBool forceGlobalStack(void)
{
    if (DEBUG_GLOBAL_STACK)
    {
        if ((atomic_inc_return(&g_debugGlobalStackCount) %
             DEBUG_GLOBAL_STACK_THRESHOLD) == 0)
            return NV_TRUE;
    }
    return NV_FALSE;
}

// Guaranteed to always return a valid stack. It first attempts to allocate one
// from the pool. If that fails, it falls back to the global pre-allocated
// stack. This fallback will serialize.
//
// This is required so paths that free resources do not themselves require
// allocation of resources.
static nvidia_stack_t *nvUvmGetSafeStack(void)
{
    nvidia_stack_t *sp;
    if (forceGlobalStack() || nv_kmem_cache_alloc_stack(&sp) != 0)
    {
        sp = g_sp;
        down(&g_spLock);
    }
    return sp;
}

static void nvUvmFreeSafeStack(nvidia_stack_t *sp)
{
    if (sp == g_sp)
        up(&g_spLock);
    else
        nv_kmem_cache_free_stack(sp);
}

NV_STATUS nvUvmInterfaceRegisterGpu(UvmGpuUuid *gpuUuid)
{
    nvidia_stack_t *sp = NULL;
    NV_STATUS status;
    int rc;

    if (nv_kmem_cache_alloc_stack(&sp) != 0)
        return NV_ERR_NO_MEMORY;

    rc = nvidia_dev_get_uuid(gpuUuid->uuid, sp);
    switch (rc)
    {
        case 0:
            status = NV_OK;
            break;
        case -ENOMEM:
            status = NV_ERR_NO_MEMORY;
            break;
        case -ENODEV:
            status = NV_ERR_GPU_UUID_NOT_FOUND;
            break;
        default:
            status = NV_ERR_GENERIC;
            break;
    }

    nv_kmem_cache_free_stack(sp);
    return status;
}
EXPORT_SYMBOL(nvUvmInterfaceRegisterGpu);

void nvUvmInterfaceUnregisterGpu(UvmGpuUuid *gpuUuid)
{
    nvidia_stack_t *sp = nvUvmGetSafeStack();
    nvidia_dev_put_uuid(gpuUuid->uuid, sp);
    nvUvmFreeSafeStack(sp);
}
EXPORT_SYMBOL(nvUvmInterfaceUnregisterGpu);

NV_STATUS nvUvmInterfaceSessionCreate(uvmGpuSessionHandle *session)
{
    nvidia_stack_t *sp = NULL;
    NV_STATUS status;

    if (nv_kmem_cache_alloc_stack(&sp) != 0)
    {
        return NV_ERR_NO_MEMORY;
    }

    status = rm_gpu_ops_create_session(sp, (gpuSessionHandle *)session);

    nv_kmem_cache_free_stack(sp);
    return status;
}
EXPORT_SYMBOL(nvUvmInterfaceSessionCreate);

NV_STATUS nvUvmInterfaceSessionDestroy(uvmGpuSessionHandle session)
{
    nvidia_stack_t *sp = nvUvmGetSafeStack();
    NV_STATUS status;

    status = rm_gpu_ops_destroy_session(sp, (gpuSessionHandle)session);

    nvUvmFreeSafeStack(sp);
    return status;
}
EXPORT_SYMBOL(nvUvmInterfaceSessionDestroy);

NV_STATUS nvUvmInterfaceDupAddressSpace(uvmGpuSessionHandle session,
                                        NvU8 *pUuid,
                                        NvHandle hUserClient,
                                        NvHandle hUserVASpace,
                                        uvmGpuAddressSpaceHandle *vaSpace)
{
    nvidia_stack_t *sp = NULL;
    NV_STATUS status;

    if (nv_kmem_cache_alloc_stack(&sp) != 0)
    {
        return NV_ERR_NO_MEMORY;
    }

    status = rm_gpu_ops_dup_address_space(
             sp, (gpuSessionHandle)session, pUuid,
             hUserClient, hUserVASpace, (gpuAddressSpaceHandle *)vaSpace);

    nv_kmem_cache_free_stack(sp);
    return status;
}
EXPORT_SYMBOL(nvUvmInterfaceDupAddressSpace);

NV_STATUS nvUvmInterfaceAddressSpaceCreateMirrored(uvmGpuSessionHandle session,
                                             unsigned long long uuidMsb,
                                             unsigned long long uuidLsb,
                                             uvmGpuAddressSpaceHandle *vaSpace)
{
    nvidia_stack_t *sp = NULL;
    NV_STATUS status;

    if (nv_kmem_cache_alloc_stack(&sp) != 0)
    {
        return NV_ERR_NO_MEMORY;
    }

    status = rm_gpu_ops_address_space_create_mirrored(
             sp, (gpuSessionHandle)session,
             uuidMsb, uuidLsb, (gpuAddressSpaceHandle *)vaSpace);

    nv_kmem_cache_free_stack(sp);
    return status;
}
EXPORT_SYMBOL(nvUvmInterfaceAddressSpaceCreateMirrored);

NV_STATUS nvUvmInterfaceAddressSpaceCreate(uvmGpuSessionHandle session,
                                            unsigned long long uuidMsb,
                                            unsigned long long uuidLsb,
                                            uvmGpuAddressSpaceHandle *vaSpace,
                                            unsigned long long vaBase,
                                            unsigned long long vaSize)
{
    nvidia_stack_t *sp = NULL;
    NV_STATUS status;

    if (nv_kmem_cache_alloc_stack(&sp) != 0)
    {
        return NV_ERR_NO_MEMORY;
    }

    status = rm_gpu_ops_address_space_create(
             sp, (gpuSessionHandle)session,
             uuidMsb, uuidLsb, (gpuAddressSpaceHandle *)vaSpace,
             vaBase, vaSize);

    nv_kmem_cache_free_stack(sp);
    return status;
}
EXPORT_SYMBOL(nvUvmInterfaceAddressSpaceCreate);

void nvUvmInterfaceAddressSpaceDestroy(uvmGpuAddressSpaceHandle vaSpace)
{
    nvidia_stack_t *sp = nvUvmGetSafeStack();

    rm_gpu_ops_address_space_destroy(
        sp, (gpuAddressSpaceHandle)vaSpace);

    nvUvmFreeSafeStack(sp);
}
EXPORT_SYMBOL(nvUvmInterfaceAddressSpaceDestroy);

NV_STATUS nvUvmInterfaceMemoryAllocFB(uvmGpuAddressSpaceHandle vaSpace,
                    NvLength length, UvmGpuPointer * gpuPointer,
                    UvmGpuAllocInfo * allocInfo)
{
    nvidia_stack_t *sp = NULL;
    NV_STATUS status;

    if (nv_kmem_cache_alloc_stack(&sp) != 0)
    {
        return NV_ERR_NO_MEMORY;
    }

    status = rm_gpu_ops_memory_alloc_fb(
             sp, (gpuAddressSpaceHandle)vaSpace,
             (NvLength)length, (NvU64 *) gpuPointer,
             (struct gpuAllocInfo *)allocInfo);

    nv_kmem_cache_free_stack(sp);
    return status;
}
EXPORT_SYMBOL(nvUvmInterfaceMemoryAllocFB);

NV_STATUS nvUvmInterfaceMemoryAllocGpuVa(uvmGpuAddressSpaceHandle vaSpace,
                    NvLength length, UvmGpuPointer * gpuPointer,
                    UvmGpuVaAllocInfo * allocInfo)
{
    nvidia_stack_t *sp = NULL;
    NV_STATUS status;

    if (nv_kmem_cache_alloc_stack(&sp) != 0)
    {
        return NV_ERR_NO_MEMORY;
    }

    status = rm_gpu_ops_memory_alloc_gpu_va(
             sp, (gpuAddressSpaceHandle)vaSpace,
             (NvLength)length, (NvU64 *) gpuPointer,
             (struct gpuVaAllocInfo *)allocInfo);

    nv_kmem_cache_free_stack(sp);
    return status;
}
EXPORT_SYMBOL(nvUvmInterfaceMemoryAllocGpuVa);

NV_STATUS nvUvmInterfaceMemoryAllocGpuPa(uvmGpuAddressSpaceHandle vaSpace,
                    NvLength length, UvmGpuPointer * gpuPointer,
                    UvmGpuAllocInfo * allocInfo)
{
    nvidia_stack_t *sp = NULL;
    NV_STATUS status;

    if (nv_kmem_cache_alloc_stack(&sp) != 0)
    {
        return NV_ERR_NO_MEMORY;
    }

    status = rm_gpu_ops_memory_alloc_gpu_pa(
             sp, (gpuAddressSpaceHandle)vaSpace,
             (NvLength)length, (NvU64 *) gpuPointer,
             (struct gpuAllocInfo *)allocInfo);

    nv_kmem_cache_free_stack(sp);
    return status;
}
EXPORT_SYMBOL(nvUvmInterfaceMemoryAllocGpuPa);

NV_STATUS nvUvmInterfaceMemoryAllocSys(uvmGpuAddressSpaceHandle vaSpace,
                    NvLength length, UvmGpuPointer * gpuPointer,
                    UvmGpuAllocInfo * allocInfo)
{
    nvidia_stack_t *sp = NULL;
    NV_STATUS status;

    if (nv_kmem_cache_alloc_stack(&sp) != 0)
    {
        return NV_ERR_NO_MEMORY;
    }

    status = rm_gpu_ops_memory_alloc_sys(
             sp, (gpuAddressSpaceHandle)vaSpace,
             (NvLength)length, (NvU64 *) gpuPointer,
             (struct gpuAllocInfo *) allocInfo);

    nv_kmem_cache_free_stack(sp);
    return status;
}

EXPORT_SYMBOL(nvUvmInterfaceMemoryAllocSys);

NV_STATUS nvUvmInterfaceGetSurfaceMapInfo(uvmGpuAddressSpaceHandle vaSpace,
                                          NvHandle hAllocation, 
                                          UvmGpuSurfaceMappingInfo * surfaceMapInfo)
{
    nvidia_stack_t *sp = NULL;
    NV_STATUS status;

    if (nv_kmem_cache_alloc_stack(&sp) != 0)
    {
        return NV_ERR_NO_MEMORY;
    }

    status = rm_gpu_ops_get_surface_map_info(
             sp, (gpuAddressSpaceHandle)vaSpace,
             hAllocation, (struct gpuSurfaceMappingInfo *) surfaceMapInfo);

    nv_kmem_cache_free_stack(sp);
    return status;
}

EXPORT_SYMBOL(nvUvmInterfaceGetSurfaceMapInfo);

void nvUvmInterfaceMemoryFree(uvmGpuAddressSpaceHandle vaSpace,
                    UvmGpuPointer gpuPointer)
{
    nvidia_stack_t *sp = nvUvmGetSafeStack();

    rm_gpu_ops_memory_free(
    sp, (gpuAddressSpaceHandle)vaSpace,
    (NvU64) gpuPointer);

    nvUvmFreeSafeStack(sp);
}
EXPORT_SYMBOL(nvUvmInterfaceMemoryFree);

void nvUvmInterfaceMemoryFreePa(uvmGpuAddressSpaceHandle vaSpace,
                    UvmGpuPointer gpuPointer)
{
    nvidia_stack_t *sp = nvUvmGetSafeStack();

    rm_gpu_ops_memory_free_pa(
    sp, (gpuAddressSpaceHandle)vaSpace,
    (NvU64) gpuPointer);

    nvUvmFreeSafeStack(sp);
}
EXPORT_SYMBOL(nvUvmInterfaceMemoryFreePa);

void nvUvmInterfaceMemoryFreeVa(uvmGpuAddressSpaceHandle vaSpace,
                    UvmGpuPointer gpuPointer)
{
    nvidia_stack_t *sp = nvUvmGetSafeStack();

    rm_gpu_ops_memory_free_va(
    sp, (gpuAddressSpaceHandle)vaSpace,
    (NvU64) gpuPointer);

    nvUvmFreeSafeStack(sp);
}
EXPORT_SYMBOL(nvUvmInterfaceMemoryFreeVa);

NV_STATUS nvUvmInterfaceMemoryCpuMap(uvmGpuAddressSpaceHandle vaSpace,
           UvmGpuPointer gpuPointer, NvLength length, void **cpuPtr,
           NvU32 pageSize)
{
    nvidia_stack_t *sp = NULL;
    NV_STATUS status;

    if (nv_kmem_cache_alloc_stack(&sp) != 0)
    {
        return NV_ERR_NO_MEMORY;
    }

    status = rm_gpu_ops_memory_cpu_map(
             sp, (gpuAddressSpaceHandle)vaSpace,
             (NvU64) gpuPointer, (NvLength) length, cpuPtr, pageSize);

    nv_kmem_cache_free_stack(sp);
    return status;
}
EXPORT_SYMBOL(nvUvmInterfaceMemoryCpuMap);

void nvUvmInterfaceMemoryCpuUnMap(uvmGpuAddressSpaceHandle vaSpace,
                                  void *cpuPtr)
{
    nvidia_stack_t *sp = nvUvmGetSafeStack();
    rm_gpu_ops_memory_cpu_ummap(sp, (gpuAddressSpaceHandle)vaSpace, cpuPtr);
    nvUvmFreeSafeStack(sp);
}
EXPORT_SYMBOL(nvUvmInterfaceMemoryCpuUnMap);

NV_STATUS nvUvmInterfaceChannelAllocate(uvmGpuAddressSpaceHandle  vaSpace,
                     uvmGpuChannelHandle *channel,
                     UvmGpuChannelPointers * pointers)
{
    nvidia_stack_t *sp = NULL;
    NV_STATUS status;
    struct gpuChannelInfo *channelInfo = NULL;

    if (nv_kmem_cache_alloc_stack(&sp) != 0)
    {
        return NV_ERR_NO_MEMORY;
    }

    status = os_alloc_mem((void **)&channelInfo, sizeof(*channelInfo));
    if (status != NV_OK)
        goto cleanup_stack;

    status = rm_gpu_ops_channel_allocate(
             sp, (gpuAddressSpaceHandle)vaSpace, (gpuChannelHandle *)channel,
             channelInfo);

    if(status != NV_OK)
        goto cleanup_chInfo;

    pointers->GPGet = channelInfo->GPGet;
    pointers->GPPut = channelInfo->GPPut;
    pointers->gpFifoEntries = channelInfo->gpFifoEntries;
    pointers->channelClassNum = channelInfo->channelClassNum;
    pointers->numGpFifoEntries = channelInfo->numGpFifoEntries;
    pointers->errorNotifier = channelInfo->errorNotifier;
    pointers->eccErrorNotifier = channelInfo->eccErrorNotifier; 

cleanup_chInfo:
    os_free_mem(channelInfo);
cleanup_stack:
    nv_kmem_cache_free_stack(sp);

    return status;
}
EXPORT_SYMBOL(nvUvmInterfaceChannelAllocate);

void nvUvmInterfaceChannelDestroy(uvmGpuChannelHandle channel)
{
    nvidia_stack_t *sp = nvUvmGetSafeStack();
    rm_gpu_ops_channel_destroy(sp, (gpuChannelHandle)channel);
    nvUvmFreeSafeStack(sp);
}
EXPORT_SYMBOL(nvUvmInterfaceChannelDestroy);

const char* nvUvmInterfaceChannelTranslateError(unsigned info32)
{
    nvidia_stack_t *sp = NULL;
    const char *errorString;

    if (nv_kmem_cache_alloc_stack(&sp) != 0)
    {
        return NULL;
    }

    errorString = rm_gpu_ops_channel_translate_error(
                  sp, info32);

    nv_kmem_cache_free_stack(sp);
    return errorString;
}
EXPORT_SYMBOL(nvUvmInterfaceChannelTranslateError);

NV_STATUS nvUvmInterfaceCopyEngineAllocate(
                     uvmGpuChannelHandle channel,
                     unsigned indexStartingAtOne,
                     unsigned * copyEngineClassNumber,
                     uvmGpuCopyEngineHandle *copyEngine)
{
    nvidia_stack_t *sp = NULL;
    NV_STATUS status;

    if (nv_kmem_cache_alloc_stack(&sp) != 0)
    {
        return NV_ERR_NO_MEMORY;
    }

    status = rm_gpu_ops_copy_engine_allocate(
             sp, (gpuChannelHandle) channel, indexStartingAtOne,
             copyEngineClassNumber, (gpuObjectHandle *) copyEngine);


    nv_kmem_cache_free_stack(sp);
    return status;
}
EXPORT_SYMBOL(nvUvmInterfaceCopyEngineAllocate);

NV_STATUS nvUvmInterfaceQueryCaps(uvmGpuAddressSpaceHandle vaSpace,
                                  UvmGpuCaps * caps)
{
    nvidia_stack_t *sp = NULL;
    struct gpuCaps * pGpuCaps = NULL;
    NV_STATUS status;

    if (nv_kmem_cache_alloc_stack(&sp) != 0)
    {
        return NV_ERR_NO_MEMORY;
    }

    status = os_alloc_mem((void **)&pGpuCaps, sizeof(*pGpuCaps));
    if (status != NV_OK)
        goto cleanup_stack;

    status = rm_gpu_ops_query_caps(sp, (gpuAddressSpaceHandle)vaSpace,
                                   pGpuCaps);

    if (status == NV_OK)
    {
        caps->largePageSize = pGpuCaps->largePageSize;
        caps->smallPageSize = pGpuCaps->smallPageSize;
        caps->eccMask = pGpuCaps->eccMask;
        caps->eccOffset = pGpuCaps->eccOffset;
        caps->eccReadLocation = pGpuCaps->eccReadLocation;
        caps->bEccEnabled   = pGpuCaps->bEccEnabled;
        caps->pcieSpeed = pGpuCaps->pcieSpeed;
        caps->pcieWidth = pGpuCaps->pcieWidth;
    }

    os_free_mem(pGpuCaps);

cleanup_stack:
    nv_kmem_cache_free_stack(sp);
    return status;
}
EXPORT_SYMBOL(nvUvmInterfaceQueryCaps);

NV_STATUS nvUvmInterfaceGetAttachedUuids(NvU8 *pUuidList, unsigned *numGpus)
{
    nvidia_stack_t *sp = NULL;
    NV_STATUS status;

    if (nv_kmem_cache_alloc_stack(&sp) != 0)
    {
        return NV_ERR_NO_MEMORY;
    }

    status = rm_gpu_ops_get_attached_uuids(sp, pUuidList, numGpus);

    nv_kmem_cache_free_stack(sp);
    return status;
}
EXPORT_SYMBOL(nvUvmInterfaceGetAttachedUuids);

NV_STATUS nvUvmInterfaceGetGpuInfo(NvU8 *pUuid, unsigned uuidLength,
                                   UvmGpuInfo * pGpuInfo)
{
    nvidia_stack_t *sp = NULL;
    NV_STATUS status;

    if (nv_kmem_cache_alloc_stack(&sp) != 0)
    {
        return NV_ERR_NO_MEMORY;
    }

    status = rm_gpu_ops_get_gpu_info(sp, pUuid,
                                     uuidLength,
                                     (struct gpuInfo *) pGpuInfo);

    nv_kmem_cache_free_stack(sp);
    return status;
}
EXPORT_SYMBOL(nvUvmInterfaceGetGpuInfo);

NV_STATUS nvUvmInterfaceGetUvmPrivRegion(NvU64 *pUvmPrivRegionStart,
                                         NvU64 *pUvmPrivRegionLength)
{
    return rm_gpu_ops_get_uvm_priv_region(pUvmPrivRegionStart,
                                          pUvmPrivRegionLength);
}
EXPORT_SYMBOL(nvUvmInterfaceGetUvmPrivRegion);

NV_STATUS nvUvmInterfaceServiceDeviceInterruptsRM(uvmGpuChannelHandle channel)
{
    nvidia_stack_t *sp = NULL;
    NV_STATUS status;

    if (nv_kmem_cache_alloc_stack(&sp) != 0)
    {
        return NV_ERR_NO_MEMORY;
    }

    status = rm_gpu_ops_service_device_interrupts_rm(sp,
                                                    (gpuChannelHandle) channel);

    nv_kmem_cache_free_stack(sp);
    return status;
}
EXPORT_SYMBOL(nvUvmInterfaceServiceDeviceInterruptsRM);

NV_STATUS nvUvmInterfaceCheckEccErrorSlowpath(uvmGpuChannelHandle channel,
                                              NvBool *bEccDbeSet)
{
    nvidia_stack_t *sp = NULL;
    NV_STATUS status;

    if (nv_kmem_cache_alloc_stack(&sp) != 0)
    {
        return NV_ERR_NO_MEMORY;
    }

    status = rm_gpu_ops_check_ecc_error_slowpath(sp, (gpuChannelHandle) channel,
                                                 bEccDbeSet);

    nv_kmem_cache_free_stack(sp);
    return status;
}
EXPORT_SYMBOL(nvUvmInterfaceCheckEccErrorSlowpath);

NV_STATUS nvUvmInterfaceKillChannel(uvmGpuChannelHandle channel)
{
    nvidia_stack_t *sp = nvUvmGetSafeStack();
    NV_STATUS status;

    status = rm_gpu_ops_kill_channel(sp, (gpuChannelHandle) channel);

    nvUvmFreeSafeStack(sp);
    return status;
}
EXPORT_SYMBOL(nvUvmInterfaceKillChannel);

NV_STATUS nvUvmInterfaceSetPageDirectory(uvmGpuAddressSpaceHandle vaSpace,
                                         NvU64 physAddress, unsigned numEntries,
                                         NvBool bVidMemAperture)
{
    nvidia_stack_t *sp = NULL;
    NV_STATUS status;

    if (nv_kmem_cache_alloc_stack(&sp) != 0)
    {
        return NV_ERR_NO_MEMORY;
    }

    status = rm_gpu_ops_set_page_directory(sp, (gpuAddressSpaceHandle)vaSpace,
                                    physAddress, numEntries, bVidMemAperture);

    nv_kmem_cache_free_stack(sp);
    return status;
}
EXPORT_SYMBOL(nvUvmInterfaceSetPageDirectory);

NV_STATUS nvUvmInterfaceUnsetPageDirectory(uvmGpuAddressSpaceHandle vaSpace)
{
    nvidia_stack_t *sp = nvUvmGetSafeStack();
    NV_STATUS status;

    status =
           rm_gpu_ops_unset_page_directory(sp, (gpuAddressSpaceHandle)vaSpace);
    nvUvmFreeSafeStack(sp);
    return status;
}
EXPORT_SYMBOL(nvUvmInterfaceUnsetPageDirectory);

NV_STATUS nvUvmInterfaceGetGmmuFmt(uvmGpuAddressSpaceHandle vaSpace,
                                   void ** pFmt)
{
    nvidia_stack_t *sp = NULL;
    NV_STATUS status;

    if (nv_kmem_cache_alloc_stack(&sp) != 0)
    {
        return NV_ERR_NO_MEMORY;
    }

    status = rm_gpu_ops_get_gmmu_fmt(sp, (gpuAddressSpaceHandle)vaSpace, pFmt);

    nv_kmem_cache_free_stack(sp);
    return status;
}
EXPORT_SYMBOL(nvUvmInterfaceGetGmmuFmt);

NV_STATUS nvUvmInterfaceDupAllocation(NvHandle hPhysHandle,
                                      uvmGpuAddressSpaceHandle srcVaspace,
                                      NvU64 srcAddress,
                                      uvmGpuAddressSpaceHandle dstVaspace,
                                      NvU64 *dstAddress,
                                      NvBool bPhysHandleValid)
{
    nvidia_stack_t *sp = NULL;
    NV_STATUS status;

    if (nv_kmem_cache_alloc_stack(&sp) != 0)
    {
        return NV_ERR_NO_MEMORY;
    }

    status = rm_gpu_ops_dup_allocation(sp,
                                      hPhysHandle,
                                      (gpuAddressSpaceHandle)srcVaspace,
                                      srcAddress,
                                      (gpuAddressSpaceHandle)dstVaspace,
                                      (NvU64 *) dstAddress,
                                      bPhysHandleValid);

    nv_kmem_cache_free_stack(sp);
    return status;
}
EXPORT_SYMBOL(nvUvmInterfaceDupAllocation);

NV_STATUS nvUvmInterfaceDupMemory(uvmGpuAddressSpaceHandle vaspace,
                                      NvHandle hSourceClient,
                                      NvHandle hClientMem,
                                      NvHandle *hDupMemory)
{
    nvidia_stack_t *sp = NULL;
    NV_STATUS status;

    if (nv_kmem_cache_alloc_stack(&sp) != 0)
    {
        return NV_ERR_NO_MEMORY;
    }

    status = rm_gpu_ops_dup_memory(sp,
                                  (gpuAddressSpaceHandle)vaspace,
                                   hSourceClient,
                                   hClientMem,
                                   hDupMemory);
    nv_kmem_cache_free_stack(sp);
    return status;
}
EXPORT_SYMBOL(nvUvmInterfaceDupMemory);


NV_STATUS nvUvmInterfaceFreeDupedHandle(uvmGpuAddressSpaceHandle vaspace,
                                        NvHandle hPhysHandle)
{
    nvidia_stack_t *sp = nvUvmGetSafeStack();
    NV_STATUS status;

    status = rm_gpu_ops_free_duped_handle(sp,
                                         (gpuAddressSpaceHandle)vaspace,
                                         hPhysHandle);

    nvUvmFreeSafeStack(sp);
    return status;
}
EXPORT_SYMBOL(nvUvmInterfaceFreeDupedHandle);


NV_STATUS nvUvmInterfaceCheckPseudoP9(uvmGpuAddressSpaceHandle vaSpace,
                                      NvBool * isPseudoP9)
{
  nvidia_stack_t *sp = NULL;
  NV_STATUS status;

  status = rm_gpu_ops_check_pseudo_p9(sp, (gpuAddressSpaceHandle)vaSpace,
                                      isPseudoP9);

  return status;
}
EXPORT_SYMBOL(nvUvmInterfaceCheckPseudoP9);

NV_STATUS nvUvmInterfaceGetFbInfo(uvmGpuAddressSpaceHandle vaSpace,
                                  UvmGpuFbInfo * fbInfo)
{
    nvidia_stack_t *sp = NULL;
    NV_STATUS status;
    struct gpuFbInfo * pFbInfo = NULL;

    if (nv_kmem_cache_alloc_stack(&sp) != 0)
    {
        return NV_ERR_NO_MEMORY;
    }

    status = os_alloc_mem((void **)&pFbInfo, sizeof(*pFbInfo));
    if (status != NV_OK)
        goto cleanup_stack;

    status = rm_gpu_ops_get_fb_info (sp, (gpuAddressSpaceHandle)vaSpace,
                                     pFbInfo);
    if(status != NV_OK)
        goto cleanup_fbInfo;

    fbInfo->heapSize = pFbInfo->heapSize;
    fbInfo->reservedHeapSize = pFbInfo->reservedHeapSize;
    fbInfo->bZeroFb = pFbInfo->bZeroFb;

cleanup_fbInfo:
    os_free_mem(pFbInfo);
cleanup_stack:
    nv_kmem_cache_free_stack(sp);

    return status;
}
EXPORT_SYMBOL(nvUvmInterfaceGetFbInfo);

NV_STATUS nvUvmInterfaceGetGpuIds(NvU8 *pUuid, unsigned uuidLength,
    NvU32 *pDeviceId, NvU32 *pSubdeviceId)
{
    nvidia_stack_t *sp = NULL;
    NV_STATUS status;

    if (nv_kmem_cache_alloc_stack(&sp) != 0)
    {
        return NV_ERR_NO_MEMORY;
    }

    status = rm_gpu_ops_get_gpu_ids(sp, pUuid,
                                    uuidLength,
                                    pDeviceId,
                                    pSubdeviceId);

    nv_kmem_cache_free_stack(sp);
    return status;
}
EXPORT_SYMBOL(nvUvmInterfaceGetGpuIds);

NV_STATUS nvUvmInterfaceInitFaultInfo(uvmGpuAddressSpaceHandle vaSpace,
    UvmGpuFaultInfo *pFaultInfo)
{
    nvidia_stack_t *sp = NULL;
    NV_STATUS status;

    if (nv_kmem_cache_alloc_stack(&sp) != 0)
    {
        return NV_ERR_NO_MEMORY;
    }

    status = rm_gpu_ops_init_fault_info(sp,
                                       (gpuAddressSpaceHandle)vaSpace,
                                       (struct gpuFaultInfo*)pFaultInfo);

    nv_kmem_cache_free_stack(sp);
    return status;
}
EXPORT_SYMBOL(nvUvmInterfaceInitFaultInfo);

NV_STATUS nvUvmInterfaceGetPageLevelInfo(uvmGpuAddressSpaceHandle vaSpace, NvU64 vAddr,
    UvmGpuPageLevelInfo *pPageLevelInfo)
{
    nvidia_stack_t *sp = NULL;
    NV_STATUS status;
    if (nv_kmem_cache_alloc_stack(&sp) != 0)
    {
        return NV_ERR_NO_MEMORY;
    }
    status = rm_gpu_ops_get_page_level_info(sp,
                                           (gpuAddressSpaceHandle)vaSpace, vAddr,
                                           (struct gpuPageLevelInfo*)pPageLevelInfo);
    nv_kmem_cache_free_stack(sp);
    return status;
}
EXPORT_SYMBOL(nvUvmInterfaceGetPageLevelInfo);

NV_STATUS nvUvmInterfaceGetChannelPhysInfo(NvHandle hClient, NvHandle hChannel,
                                       UvmGpuChannelPhysInfo *pChannelInfo)
{
    nvidia_stack_t *sp = NULL;
    NV_STATUS status;
    if (nv_kmem_cache_alloc_stack(&sp) != 0)
    {
        return NV_ERR_NO_MEMORY;
    }
    status = rm_gpu_ops_get_channel_phys_info(sp,
                                         hClient, hChannel,
                                         (struct gpuChannelPhysInfo*)pChannelInfo);
    nv_kmem_cache_free_stack(sp);
    return status;
}
EXPORT_SYMBOL(nvUvmInterfaceGetChannelPhysInfo);

void nvUvmInterfaceFreeChannelInstPtr(void* memHandle)
{
    nvidia_stack_t *sp = nvUvmGetSafeStack();

    rm_gpu_ops_free_channel_inst_ptr(sp, memHandle);

    nvUvmFreeSafeStack(sp);
}
EXPORT_SYMBOL(nvUvmInterfaceFreeChannelInstPtr);

NV_STATUS nvUvmInterfaceDestroyFaultInfo(uvmGpuAddressSpaceHandle vaSpace,
    UvmGpuFaultInfo *pFaultInfo)
{
    nvidia_stack_t *sp = nvUvmGetSafeStack();
    NV_STATUS status;

    status = rm_gpu_ops_destroy_fault_info(sp,
                                          (gpuAddressSpaceHandle)vaSpace,
                                          (struct gpuFaultInfo*)pFaultInfo);

    nvUvmFreeSafeStack(sp);
    return status;
}
EXPORT_SYMBOL(nvUvmInterfaceDestroyFaultInfo);

NV_STATUS nvUvmInterfaceGetCtxBufferCount(uvmGpuAddressSpaceHandle vaSpace,
                                          NvU32 * bufferCount)
{
    NV_STATUS status;
    nvidia_stack_t *sp = NULL;
    if (nv_kmem_cache_alloc_stack(&sp) != 0)
    {
        return NV_ERR_NO_MEMORY;
    }

    status = rm_gpu_ops_get_ctx_buffer_count(sp, (gpuAddressSpaceHandle)vaSpace,
                                             bufferCount);

    nv_kmem_cache_free_stack(sp);
    return status;
}
EXPORT_SYMBOL(nvUvmInterfaceGetCtxBufferCount);

NV_STATUS nvUvmInterfaceGetCtxBufferInfo(uvmGpuAddressSpaceHandle vaSpace,
                                         NvHandle hCudaClient,
                                         NvHandle hChannel, NvU32 bufferCount,
                                         UvmGpuChannelCtxBufferInfo *ctxBufferInfo)
{
    NV_STATUS status;
    nvidia_stack_t *sp = NULL;
    if (nv_kmem_cache_alloc_stack(&sp) != 0)
    {
        return NV_ERR_NO_MEMORY;
    }

    status = rm_gpu_ops_get_ctx_buffer_info(sp,(gpuAddressSpaceHandle)vaSpace,
                                hCudaClient, hChannel, bufferCount,
                                (struct gpuChannelCtxBufferInfo *)ctxBufferInfo);

    nv_kmem_cache_free_stack(sp);
    return status;
}
EXPORT_SYMBOL(nvUvmInterfaceGetCtxBufferInfo);

NV_STATUS nvUvmInterfaceGetCtxBufferPhysInfo(void * bufferHandle, NvU64 pageCount,
                                             NvU64 * physAddrArray)
{
    NV_STATUS status;
    nvidia_stack_t *sp = NULL;
    if (nv_kmem_cache_alloc_stack(&sp) != 0)
    {
        return NV_ERR_NO_MEMORY;
    }

    status = rm_gpu_ops_get_ctx_buffer_phys_info(sp, bufferHandle, pageCount,
                                                 physAddrArray);

    nv_kmem_cache_free_stack(sp);
    return status;
}
EXPORT_SYMBOL(nvUvmInterfaceGetCtxBufferPhysInfo);

NV_STATUS nvUvmInterfaceValidateChannel(uvmGpuAddressSpaceHandle dupedVaSpace,
                                        NvHandle hUserVaSpace,
                                        NvHandle hUserClient,
                                        NvHandle hUserChannel)
{
    NV_STATUS status;
    nvidia_stack_t *sp = NULL;
    if (nv_kmem_cache_alloc_stack(&sp) != 0)
    {
        return NV_ERR_NO_MEMORY;
    }

    status = rm_gpu_ops_validate_channel(sp, (gpuAddressSpaceHandle)dupedVaSpace,
                                         hUserVaSpace, hUserClient, hUserChannel);

    nv_kmem_cache_free_stack(sp);
    return status;
}
EXPORT_SYMBOL(nvUvmInterfaceValidateChannel);

// this function is called by the UVM driver to register the ops
NV_STATUS nvUvmInterfaceRegisterUvmCallbacks(
                                         struct UvmOpsUvmEvents *importedUvmOps)
{
    if (!importedUvmOps)
    {
        return NV_ERR_INVALID_ARGUMENT;
    }

    g_pNvUvmEvents = importedUvmOps;

    return NV_OK;
}
EXPORT_SYMBOL(nvUvmInterfaceRegisterUvmCallbacks);

NV_STATUS nvUvmInterfaceDeRegisterUvmOps(void)
{
    g_pNvUvmEvents = NULL;

    return NV_OK;
}
EXPORT_SYMBOL(nvUvmInterfaceDeRegisterUvmOps);

NV_STATUS nvUvmInterfaceBindChannel(uvmGpuAddressSpaceHandle vaSpace,
    NvHandle hUserClient, NvHandle hUserChannel, NvU32 bufferCount,
    UvmGpuChannelBufferVa *bufferVaList)
{
    nvidia_stack_t *sp = NULL;
    NV_STATUS status;
    if (nv_kmem_cache_alloc_stack(&sp) != 0)
    {
        return NV_ERR_NO_MEMORY;
    }
    status = rm_gpu_ops_bind_channel(sp, (gpuAddressSpaceHandle)vaSpace,
        hUserClient, hUserChannel, bufferCount, (struct gpuChannelBufferVa*)bufferVaList);

    nv_kmem_cache_free_stack(sp);
    return status;
}
EXPORT_SYMBOL(nvUvmInterfaceBindChannel);

NV_STATUS nvUvmInterfaceUnbindChannel(uvmGpuAddressSpaceHandle vaSpace,
    NvHandle hUserClient,
    NvHandle hUserChannel, NvU32 bufferCount,
    NvU64 *bufferHandles)
{
    nvidia_stack_t *sp = NULL;
    NV_STATUS status;
    if (nv_kmem_cache_alloc_stack(&sp) != 0)
    {
        return NV_ERR_NO_MEMORY;
    }
    status = rm_gpu_ops_unbind_channel(sp, (gpuAddressSpaceHandle)vaSpace,
        hUserClient, hUserChannel, bufferCount, bufferHandles);

    nv_kmem_cache_free_stack(sp);
    return status;
}
EXPORT_SYMBOL(nvUvmInterfaceUnbindChannel);

void nv_uvm_notify_start_device(NvU8 *pUuid)
{
    UvmGpuUuid uvmUuid;

    memcpy(uvmUuid.uuid, pUuid, UVM_UUID_LEN);

    if(g_pNvUvmEvents && g_pNvUvmEvents->startDevice)
    {
        g_pNvUvmEvents->startDevice(&uvmUuid);
    }
    return;
}

void nv_uvm_notify_stop_device(NvU8 *pUuid)
{
    UvmGpuUuid uvmUuid;

    memcpy(uvmUuid.uuid, pUuid, UVM_UUID_LEN);

    if(g_pNvUvmEvents && g_pNvUvmEvents->stopDevice)
    {
        g_pNvUvmEvents->stopDevice(&uvmUuid);
    }
    return;
}

NV_STATUS nv_uvm_event_interrupt(NvU8 *pUuid)
{
    //
    // Since there is no API to de-register/modify the callback,
    // we can safely compare and invoke if it is non-null.
    //
    if (g_pNvUvmEvents && g_pNvUvmEvents->isrTopHalf)
        return g_pNvUvmEvents->isrTopHalf((UvmGpuUuid *)pUuid);

    //
    // NV_OK means that the interrupt was for the UVM driver, so use
    // NV_ERR_NO_INTR_PENDING to tell the caller that we didn't do anything.
    //
    return NV_ERR_NO_INTR_PENDING;
}

#endif // NV_UVM_ENABLE
