diff options
24 files changed, 2968 insertions, 3417 deletions
diff --git a/arch/arm/mach-tegra/include/linux/nvmem_ioctl.h b/arch/arm/mach-tegra/include/linux/nvmem_ioctl.h index 7b27f3ea84dd..555e19a90e9e 100644 --- a/arch/arm/mach-tegra/include/linux/nvmem_ioctl.h +++ b/arch/arm/mach-tegra/include/linux/nvmem_ioctl.h @@ -30,56 +30,85 @@ #define _MACH_TEGRA_NVMEM_IOCTL_H_ struct nvmem_create_handle { - union { - unsigned long key; // stores the Key for ClaimPreservedHandle - unsigned long id; // stores the ID for FromId - unsigned long size; // stores the size for CreateHandle - }; - uintptr_t handle; + union { + __u32 key; /* ClaimPreservedHandle */ + __u32 id; /* FromId */ + __u32 size; /* CreateHandle */ + }; + __u32 handle; }; -#define NVMEM_ALLOC_HANDLE_FLAG(_COH, _ALIGN) (((_COH)<<16) | ((_ALIGN)&0xffff)) +#define NVMEM_HEAP_SYSMEM (1ul<<31) +#define NVMEM_HEAP_IOVMM (1ul<<30) + +/* common carveout heaps */ +#define NVMEM_HEAP_CARVEOUT_IRAM (1ul<<29) +#define NVMEM_HEAP_CARVEOUT_GENERIC (1ul<<0) + +#define NVMEM_HEAP_CARVEOUT_MASK (NVMEM_HEAP_IOVMM - 1) + +#define NVMEM_HANDLE_UNCACHEABLE (0x0ul << 0) +#define NVMEM_HANDLE_WRITE_COMBINE (0x1ul << 0) +#define NVMEM_HANDLE_INNER_CACHEABLE (0x2ul << 0) +#define NVMEM_HANDLE_CACHEABLE (0x3ul << 0) + +#define NVMEM_HANDLE_SECURE (0x1ul << 2) + struct nvmem_alloc_handle { - uintptr_t handle; // hmem - unsigned int heap_mask; // bitmask of legal heaps, or 0 for default heaps - unsigned int flags; // munging of coherency and alignment into 1 word + __u32 handle; + __u32 heap_mask; + __u32 flags; + __u32 align; }; struct nvmem_map_caller { - uintptr_t handle; // hmem - void __user *addr; // user pointer - size_t offset; // offset into hmem; should be page-aligned - size_t length; // number of bytes to map; should be page-aligned - unsigned int flags; + __u32 handle; /* hmem */ + __u32 offset; /* offset into hmem; should be page-aligned */ + __u32 length; /* number of bytes to map */ + __u32 flags; + unsigned long addr; /* user pointer */ }; struct nvmem_rw_handle { - uintptr_t handle; // hmem - void __user *addr; // user pointer - size_t offset; // offset into hmem - size_t elem_size; // individual atom size - size_t hmem_stride; // delta in bytes between atoms in hmem - size_t user_stride; // delta in bytes between atoms in user - size_t count; // number of atoms to copy + unsigned long addr; /* user pointer */ + __u32 handle; /* hmem */ + __u32 offset; /* offset into hmem */ + __u32 elem_size; /* individual atom size */ + __u32 hmem_stride; /* delta in bytes between atoms in hmem */ + __u32 user_stride; /* delta in bytes between atoms in user */ + __u32 count; /* number of atoms to copy */ }; struct nvmem_pin_handle { - unsigned long __user *handles; // array of handles to pin/unpin - unsigned long __user *addr; // array of addresses to return - unsigned long count; // number of entries in handles + unsigned long handles; /* array of handles to pin/unpin */ + unsigned long addr; /* array of addresses to return */ + __u32 count; /* number of entries in handles */ +}; + +struct nvmem_handle_param { + __u32 handle; + __u32 param; + unsigned long result; }; enum { - NVMEM_CACHE_OP_WB = 0, - NVMEM_CACHE_OP_INV, - NVMEM_CACHE_OP_WB_INV, + NVMEM_HANDLE_PARAM_SIZE = 1, + NVMEM_HANDLE_PARAM_ALIGNMENT, + NVMEM_HANDLE_PARAM_BASE, + NVMEM_HANDLE_PARAM_HEAP, +}; + +enum { + NVMEM_CACHE_OP_WB = 0, + NVMEM_CACHE_OP_INV, + NVMEM_CACHE_OP_WB_INV, }; struct nvmem_cache_op { - uintptr_t handle; - void __user *addr; - size_t len; - int op; + unsigned long addr; + __u32 handle; + __u32 len; + __s32 op; }; #define NVMEM_IOC_MAGIC 'N' @@ -87,17 +116,16 @@ struct nvmem_cache_op { /* Creates a new memory handle. On input, the argument is the size of the new * handle; on return, the argument is the name of the new handle */ -#define NVMEM_IOC_CREATE _IOWR(NVMEM_IOC_MAGIC, 0, struct nvmem_create_handle) -#define NVMEM_IOC_CLAIM _IOWR(NVMEM_IOC_MAGIC, 1, struct nvmem_create_handle) -#define NVMEM_IOC_FROM_ID _IOWR(NVMEM_IOC_MAGIC, 2, struct nvmem_create_handle) - +#define NVMEM_IOC_CREATE _IOWR(NVMEM_IOC_MAGIC, 0, struct nvmem_create_handle) +#define NVMEM_IOC_CLAIM _IOWR(NVMEM_IOC_MAGIC, 1, struct nvmem_create_handle) +#define NVMEM_IOC_FROM_ID _IOWR(NVMEM_IOC_MAGIC, 2, struct nvmem_create_handle) /* Actually allocates memory for the specified handle */ #define NVMEM_IOC_ALLOC _IOW (NVMEM_IOC_MAGIC, 3, struct nvmem_alloc_handle) /* Frees a memory handle, unpinning any pinned pages and unmapping any mappings */ -#define NVMEM_IOC_FREE _IOW (NVMEM_IOC_MAGIC, 4, uintptr_t) +#define NVMEM_IOC_FREE _IO (NVMEM_IOC_MAGIC, 4) /* Maps the region of the specified handle into a user-provided virtual address * that was previously created via an mmap syscall on this fd */ @@ -108,23 +136,19 @@ struct nvmem_cache_op { #define NVMEM_IOC_WRITE _IOW (NVMEM_IOC_MAGIC, 6, struct nvmem_rw_handle) #define NVMEM_IOC_READ _IOW (NVMEM_IOC_MAGIC, 7, struct nvmem_rw_handle) -/* Pins a single memory handle and ensures a contiguous mapping exists in - * either phsyical memory or the GART. If the memory handle is backed by a - * swap device, all pages will be resident in memory before this ioctl returns. - * Pin operations may be performed recursively on memory handles. - */ -#define NVMEM_IOC_PIN _IOWR(NVMEM_IOC_MAGIC, 8, uintptr_t) +#define NVMEM_IOC_PARAM _IOWR(NVMEM_IOC_MAGIC, 8, struct nvmem_handle_param) -/* Unpins a single memory handle. If the memory handle is backed by a swap - * device, unpinning a memory handle may result in the handle being decommitted - */ -#define NVMEM_IOC_UNPIN _IOW (NVMEM_IOC_MAGIC, 9, uintptr_t) - -/* Like IOC_PIN and IOC_UNPIN, but operates on a list of memory handles */ +/* Pins a list of memory handles into IO-addressable memory (either IOVMM + * space or physical memory, depending on the allocation), and returns the + * address. Handles may be pinned recursively. */ #define NVMEM_IOC_PIN_MULT _IOWR(NVMEM_IOC_MAGIC, 10, struct nvmem_pin_handle) #define NVMEM_IOC_UNPIN_MULT _IOW (NVMEM_IOC_MAGIC, 11, struct nvmem_pin_handle) #define NVMEM_IOC_CACHE _IOW (NVMEM_IOC_MAGIC, 12, struct nvmem_cache_op) -#define NVMEM_IOC_MAXNR (_IOC_NR(NVMEM_IOC_CACHE)) +/* Returns a global ID usable to allow a remote process to create a handle + * reference to the same handle */ +#define NVMEM_IOC_GET_ID _IOWR(NVMEM_IOC_MAGIC, 13, struct nvmem_create_handle) + +#define NVMEM_IOC_MAXNR (_IOC_NR(NVMEM_IOC_GET_ID)) #endif diff --git a/arch/arm/mach-tegra/include/nvrm_heap.h b/arch/arm/mach-tegra/include/nvrm_heap.h deleted file mode 100644 index 468dc494caed..000000000000 --- a/arch/arm/mach-tegra/include/nvrm_heap.h +++ /dev/null @@ -1,219 +0,0 @@ -/* - * Copyright (c) 2007-2009 NVIDIA Corporation. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * Neither the name of the NVIDIA Corporation nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * - */ - -#ifndef NVRM_HEAP_H -#define NVRM_HEAP_H - -#include "nvrm_memmgr.h" -#include "nvassert.h" -#include "nvos.h" - - -#ifdef __cplusplus -extern "C" { -#endif /* __cplusplus */ - - -typedef NvRmPhysAddr (*NvRmHeapAlloc)(NvU32 size); -typedef void (*NvRmHeapFree)(NvRmPhysAddr); - -typedef struct NvRmPrivHeapRec -{ - NvRmHeap heap; - NvRmPhysAddr PhysicalAddress; - NvU32 length; - -} NvRmPrivHeap; - - -void -NvRmPrivPreservedMemHandleInit(NvRmDeviceHandle hRm); - -NvRmPrivHeap * -NvRmPrivHeapCarveoutInit(NvU32 length, - NvRmPhysAddr base); - -void -NvRmPrivHeapCarveoutDeinit(void); - -NvError -NvRmPrivHeapCarveoutPreAlloc(NvRmPhysAddr Address, NvU32 Length); - -NvError -NvRmPrivHeapCarveoutAlloc(NvU32 size, NvU32 align, NvRmPhysAddr *PAddr); - -void -NvRmPrivHeapCarveoutFree(NvRmPhysAddr addr); - -void * -NvRmPrivHeapCarveoutMemMap(NvRmPhysAddr base, NvU32 length, NvOsMemAttribute attribute); - -void -NvRmPrivHeapCarveoutGetInfo(NvU32 *CarveoutPhysBase, - void **pCarveout, - NvU32 *CarveoutSize); - -NvS32 -NvRmPrivHeapCarveoutMemoryUsed(void); - -NvS32 -NvRmPrivHeapCarveoutLargestFreeBlock(void); - -/** - * \Note Not necessarily same as CarveoutSize returned by - * NvRmPrivHeapCarveoutGetInfo. No dependency on - * carveout being mapped in. - */ -NvS32 -NvRmPrivHeapCarveoutTotalSize(void); - -NvRmPrivHeap * -NvRmPrivHeapIramInit(NvU32 length, - NvRmPhysAddr base); - -void -NvRmPrivHeapIramDeinit(void); - -NvError -NvRmPrivHeapIramAlloc(NvU32 size, NvU32 align, NvRmPhysAddr *PAddr); - -NvError -NvRmPrivHeapIramPreAlloc(NvRmPhysAddr Address, NvU32 Length); - -void -NvRmPrivHeapIramFree(NvRmPhysAddr addr); - -void * -NvRmPrivHeapIramMemMap(NvRmPhysAddr base, NvU32 length, NvOsMemAttribute attribute); - - -// -- GART -- - -#define GART_PAGE_SIZE (4096) -#define GART_MAX_PAGES (4096) - -/** - * Initialize the GART heap. This identifies the GART heap's base address - * and total size to the internal heap manager, so that it may allocate - * pages appropriately. - * - * @param hDevice An RM device handle. - * Size of the GART heap (bytes) and Base address of the GART heap space - * are in GartMemoryInfo substructure of hDevice - * - * @retval Pointer to the heap data structure, with updated values. - */ -NvRmPrivHeap * -NvRmPrivHeapGartInit(NvRmDeviceHandle hDevice); - -void -NvRmPrivHeapGartDeinit(void); - -/** - * Allocate GART storage space of the specified size (in units of GART_PAGE_SIZE). - * Alignment is handled internally by this API, since it must align with the - * GART_PAGE_SIZE. This API also updates the GART registers and returns the base - * address pointer of the space allocated within the GART heap. - * - * @see NvRmPrivHeapGartFree() - * - * @param hDevice An RM device handle. - * @param pPhysAddrArray Contains an array of page addresses. This array should - * be created using an NVOS call that acquires the underlying memory address - * for each page to be mapped by the GART. - * @param NumberOfPages The size (in pages, not bytes) of mapping requested. Must - * be greater than 0. - * @param PAddr Points to variable that will be updated with the base address of - * the next available GART page. - * - * @retval The address of the first available GART page of the requested size. - */ -NvError -NvRmPrivAp15HeapGartAlloc( - NvRmDeviceHandle hDevice, - NvOsPageAllocHandle hPageHandle, - NvU32 NumberOfPages, - NvRmPhysAddr *PAddr); - -NvError -NvRmPrivAp20HeapGartAlloc( - NvRmDeviceHandle hDevice, - NvOsPageAllocHandle hPageHandle, - NvU32 NumberOfPages, - NvRmPhysAddr *PAddr); - -/** - * Free the specified GART memory pages. - * - * @see NvRmPrivHeapGartAlloc() - * - * @param hDevice An RM device handle. - * @param addr Base address (GART space) of the memory page(s) to free. - * NULL address pointers are ignored. - * @param NumberOfPages The size (in pages, not bytes) of mapping to free. - * This needs to match the size indicated when allocated. - */ -void -NvRmPrivAp15HeapGartFree( - NvRmDeviceHandle hDevice, - NvRmPhysAddr addr, - NvU32 NumberOfPages); - -void -NvRmPrivAp20HeapGartFree( - NvRmDeviceHandle hDevice, - NvRmPhysAddr addr, - NvU32 NumberOfPages); - -/** - * Suspend GART. - */ -void -NvRmPrivAp15GartSuspend(NvRmDeviceHandle hDevice); - -void -NvRmPrivAp20GartSuspend(NvRmDeviceHandle hDevice); - -/** - * Resume GART. - */ -void -NvRmPrivAp15GartResume(NvRmDeviceHandle hDevice); - -void -NvRmPrivAp20GartResume(NvRmDeviceHandle hDevice); - -#ifdef __cplusplus -} -#endif /* __cplusplus */ - -#endif diff --git a/arch/arm/mach-tegra/include/nvrm_memmgr_private.h b/arch/arm/mach-tegra/include/nvrm_memmgr_private.h deleted file mode 100644 index 422b26c059d5..000000000000 --- a/arch/arm/mach-tegra/include/nvrm_memmgr_private.h +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright (c) 2007-2009 NVIDIA Corporation. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * Neither the name of the NVIDIA Corporation nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * - */ - - -#ifndef INCLUDED_NVDDK_MEMMGR_PRIVATE_H -#define INCLUDED_NVDDK_MEMMGR_PRIVATE_H - -#include "nvrm_heap.h" - -#ifdef __cplusplus -extern "C" -{ -#endif /* __cplusplus */ - -#define NVRM_HMEM_CHECK_MAGIC NV_DEBUG - -#define NV_RM_HMEM_IS_ALLOCATED(hMem) \ - (((hMem)->PhysicalAddress != NV_RM_INVALID_PHYS_ADDRESS) || \ - ((hMem)->VirtualAddress != NULL) || \ - ((hMem)->hPageHandle != NULL) ) - -typedef struct NvRmMemRec -{ - void *VirtualAddress; - NvRmDeviceHandle hRmDevice; - NvOsPageAllocHandle hPageHandle; - NvRmPhysAddr PhysicalAddress; - NvU32 size; - NvU32 alignment; - - /* Used for GART heap to keep track of the number of GART pages - * in use by this handle. - */ - NvU32 Pages; - - NvS32 refcount; - NvS32 pin_count; - - NvOsMemAttribute coherency; - NvRmHeap heap; - - NvBool mapped; - NvU8 priority; - -#if NVRM_HMEM_CHECK_MAGIC - NvU32 magic; // set to NVRM_MEM_MAGIC if valid -#endif -} NvRmMem; - -#ifdef __cplusplus -} -#endif /* __cplusplus */ - - -#endif - - - - diff --git a/arch/arm/mach-tegra/init_common.c b/arch/arm/mach-tegra/init_common.c index 50c4dcfdf764..4a2217d2af68 100755 --- a/arch/arm/mach-tegra/init_common.c +++ b/arch/arm/mach-tegra/init_common.c @@ -43,6 +43,7 @@ #include "nvrm_arm_cp.h" #include "nvrm_interrupt.h" #include "ap20/arusb.h" +#include "linux/nvmem_ioctl.h" const char *tegra_partition_list = NULL; char *tegra_boot_device = NULL; @@ -959,6 +960,12 @@ extern int __init tegra_dma_init(void); void __init tegra_common_init(void) { + if (tegra_get_module_inst_size("iram", 0)) { + nvmap_add_carveout_heap(tegra_get_module_inst_base("iram", 0), + tegra_get_module_inst_size("iram", 0) * + NvRmModuleGetNumInstances(s_hRmGlobal, NvRmPrivModuleID_Iram), + "iram", NVMEM_HEAP_CARVEOUT_IRAM); + } NV_ASSERT_SUCCESS(NvRmOpen(&s_hRmGlobal,0)); NV_ASSERT_SUCCESS(NvRmGpioOpen(s_hRmGlobal, &s_hGpioGlobal)); diff --git a/arch/arm/mach-tegra/nvos/nvos.c b/arch/arm/mach-tegra/nvos/nvos.c index 08f30ebb0d2f..d0b19f792c86 100644 --- a/arch/arm/mach-tegra/nvos/nvos.c +++ b/arch/arm/mach-tegra/nvos/nvos.c @@ -1427,7 +1427,6 @@ NvError NvOsBootArgSet(NvU32 key, void *arg, NvU32 size) return NvError_NotImplemented; } -static NvError NvOsGetCarveoutParam(NvBootArgsCarveout* arg, NvU32 size); NvError NvOsBootArgGet(NvU32 key, void *arg, NvU32 size) { const void *src; @@ -1465,8 +1464,6 @@ NvError NvOsBootArgGet(NvU32 key, void *arg, NvU32 size) src = &s_BootArgs.ChipShmooPhysArgs; size_src = sizeof(NvBootArgsChipShmooPhys); break; - case NvBootArgKey_Carveout: - return NvOsGetCarveoutParam((NvBootArgsCarveout*)arg, size); case NvBootArgKey_WarmBoot: src = &s_BootArgs.WarmbootArgs; size_src = sizeof(NvBootArgsWarmboot); @@ -1533,50 +1530,6 @@ void NvOsSetResourceAllocFileLine(void* userptr, const char* file, int line) } #endif -#define MAX_CARVEOUTS 2 - -static int gs_NumCarveouts = 0; - -typedef struct CarveoutRegionRec { - unsigned long base; - unsigned long size; -} CarveoutRegion; - -static CarveoutRegion gs_Carveouts[MAX_CARVEOUTS]; - -static NvError NvOsGetCarveoutParam(NvBootArgsCarveout* arg, NvU32 size) -{ - if (size != sizeof(*arg)) - return NvError_BadParameter; - - if (!gs_NumCarveouts) - return NvError_NotSupported; - - arg->base = gs_Carveouts[0].base; - arg->size = gs_Carveouts[0].size; - return NvSuccess; -} - -static int __init carveout_memory_setup(char *options) -{ - unsigned long start, size; - char *p = options; - - start = -1; - size = memparse(p, &p); - if (*p == '@') - start = memparse(p + 1, &p); - - if (gs_NumCarveouts < NV_ARRAY_SIZE(gs_Carveouts)) - { - gs_Carveouts[gs_NumCarveouts].base = start; - gs_Carveouts[gs_NumCarveouts++].size = size; - } - - return 0; -} -__setup("nvmem=", carveout_memory_setup); - static int __init parse_tegra_tag(const struct tag *tag) { const struct tag_nvidia_tegra *nvtag = &tag->u.tegra; diff --git a/arch/arm/mach-tegra/nvrm/core/ap15/Makefile b/arch/arm/mach-tegra/nvrm/core/ap15/Makefile index 85675a55c0fa..2c87c36b3245 100644 --- a/arch/arm/mach-tegra/nvrm/core/ap15/Makefile +++ b/arch/arm/mach-tegra/nvrm/core/ap15/Makefile @@ -12,7 +12,6 @@ ccflags-y += -Iarch/arm/mach-tegra/nvrm/core obj-y += ap15rm_interrupt_generic.o obj-y += ap15rm_hwmap.o -obj-y += ap15rm_gart.o obj-y += ap15rm_clocks.o obj-y += ap15rm_clock_config.o obj-y += ap15rm_clocks_info.o diff --git a/arch/arm/mach-tegra/nvrm/core/ap15/ap15rm_gart.c b/arch/arm/mach-tegra/nvrm/core/ap15/ap15rm_gart.c deleted file mode 100644 index 352016929d4b..000000000000 --- a/arch/arm/mach-tegra/nvrm/core/ap15/ap15rm_gart.c +++ /dev/null @@ -1,257 +0,0 @@ -/* - * Copyright (c) 2007-2009 NVIDIA Corporation. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * Neither the name of the NVIDIA Corporation nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * - */ - -#include "ap15/armc.h" -#include "nvrm_heap.h" -#include "nvrm_heap_simple.h" -#include "nvrm_hwintf.h" -#include "ap15rm_private.h" -#include "nvassert.h" -#include "nvcommon.h" -#include "nvrm_drf.h" - - -/** - * Initialize the GART entries, and enable the GART - */ - -#define GART_PAGE_SHIFT (12) -#define GART_PAGE_SIZE (4096) - -extern NvBool gs_GartInited; -extern NvRmHeapSimple gs_GartAllocator; -extern NvU32 *gs_GartSave; - -/** - * Initializes all of the TLB entries in the GART and enables GART translations - * All entries are initially marked invalid. - * - * @param hDevice The RM device handle. - */ -static NvError -NvRmPrivAp15InitGART(NvRmDeviceHandle hDevice); -static NvError -NvRmPrivAp15InitGART(NvRmDeviceHandle hDevice) -{ - NvU32 GartSize; - NvU32 GartEntries; - NvU32 GartEntry; - NvU32 reg; - NvU32 data; - - NV_ASSERT(hDevice != NULL); - - NvRmModuleGetBaseAddress( - hDevice, NvRmPrivModuleID_Gart, NULL, &GartSize); - - GartEntries = GartSize / GART_PAGE_SIZE; - - gs_GartSave = NvOsAlloc( sizeof(NvU32) * GartEntries ); - if ( NULL == gs_GartSave ) - return NvError_InsufficientMemory; - - data = NV_DRF_NUM(MC, GART_ENTRY_DATA, GART_ENTRY_DATA_PHYS_ADDR_VALID, 0); - for (GartEntry = 0; GartEntry < GartEntries; ++GartEntry) - { - // set the address - reg = NV_DRF_NUM(MC, GART_ENTRY_ADDR, GART_ENTRY_ADDR_TABLE_ADDR, GartEntry); - NV_REGW(hDevice, NvRmPrivModuleID_MemoryController, 0, MC_GART_ENTRY_ADDR_0, reg); - - // mark the entry invalid - NV_REGW(hDevice, NvRmPrivModuleID_MemoryController, 0, MC_GART_ENTRY_DATA_0, data); - } - - // now enable the GART - reg = NV_DRF_DEF(MC, GART_CONFIG, GART_ENABLE, ENABLE); - NV_REGW(hDevice, NvRmPrivModuleID_MemoryController, 0, MC_GART_CONFIG_0, reg); - return NvSuccess; -} - -NvError -NvRmPrivAp15HeapGartAlloc( - NvRmDeviceHandle hDevice, - NvOsPageAllocHandle hPageHandle, - NvU32 NumberOfPages, - NvRmPhysAddr *PAddr) -{ - NvError result = NvSuccess; - NvU32 reg; - NvU32 i, data; - NvU32 FirstGartPage; - - NV_ASSERT(hDevice); - NV_ASSERT(hPageHandle); - - result = NvRmPrivHeapSimpleAlloc( - &gs_GartAllocator, - NumberOfPages*GART_PAGE_SIZE, - GART_PAGE_SIZE, - PAddr); - - if (result != NvSuccess) - return result; - - FirstGartPage = *PAddr; - - /* Check that the GART address exists and is page aligned */ - NV_ASSERT(FirstGartPage); - NV_ASSERT((FirstGartPage & (GART_PAGE_SIZE - 1)) == 0); - - NvOsMutexLock(hDevice->mutex); - - // FIXME: Normally we would do this at init time, but it takes and - // egregious amount of csim time, so I'm defering it or the 3d guys - // will complain to me, and then to my boss, and then their boss, and then their bosses boss... - if (gs_GartInited == NV_FALSE) - { - result = NvRmPrivAp15InitGART(hDevice); - if ( NvSuccess != result ) - goto fail; - gs_GartInited = NV_TRUE; - } - - for (i = 0; i < NumberOfPages; i++) - { - data = (NvU32)NvOsPageAddress(hPageHandle, i * GART_PAGE_SIZE); - - /* Check that each physical address is page aligned */ - NV_ASSERT((data & (GART_PAGE_SIZE - 1)) == 0); - - reg = NV_DRF_NUM(MC, GART_ENTRY_ADDR, GART_ENTRY_ADDR_TABLE_ADDR, ((FirstGartPage + i*GART_PAGE_SIZE) >> GART_PAGE_SHIFT)); - NV_REGW(hDevice, NvRmPrivModuleID_MemoryController, 0, MC_GART_ENTRY_ADDR_0, reg); - - reg = - NV_DRF_NUM(MC, GART_ENTRY_DATA, GART_ENTRY_DATA_PHYS_ADDR_VALID, 1) | - NV_DRF_NUM(MC, GART_ENTRY_DATA, GART_ENTRY_DATA_PHYS_ADDR, (data >> GART_PAGE_SHIFT)); - - NV_REGW(hDevice, NvRmPrivModuleID_MemoryController, 0, MC_GART_ENTRY_DATA_0, reg); - // lame, on csim we have to read this back to make sure the GART entry is valid before we hit the mc - // with data to this address. - (void)NV_REGR(hDevice, NvRmPrivModuleID_MemoryController, 0, MC_GART_ENTRY_DATA_0); - } -fail: - NvOsMutexUnlock(hDevice->mutex); - - return result; -} - -void -NvRmPrivAp15HeapGartFree( - NvRmDeviceHandle hDevice, - NvRmPhysAddr addr, - NvU32 NumberOfPages) -{ - NvU32 i; - NvU32 reg; - NvU32 data; - - NV_ASSERT(hDevice); - - if (addr && NumberOfPages) - { - // Invalidate GART page table entries - data = NV_DRF_NUM(MC, GART_ENTRY_DATA, GART_ENTRY_DATA_PHYS_ADDR_VALID, 0); - for (i = 0; i < NumberOfPages; i++) - { - // set the address - reg = NV_DRF_NUM(MC, GART_ENTRY_ADDR, GART_ENTRY_ADDR_TABLE_ADDR, ((addr + i*GART_PAGE_SIZE) >> GART_PAGE_SHIFT)); - NV_REGW(hDevice, NvRmPrivModuleID_MemoryController, 0, MC_GART_ENTRY_ADDR_0, reg); - - // mark the entry invalid - NV_REGW(hDevice, NvRmPrivModuleID_MemoryController, 0, MC_GART_ENTRY_DATA_0, data); - } - NvRmPrivHeapSimpleFree(&gs_GartAllocator, addr); - } -} - - -void -NvRmPrivAp15GartSuspend(NvRmDeviceHandle hDevice) -{ - NvU32 reg; - NvU32 GartSize; - NvU32 GartEntries; - NvU32 GartEntry; - - NvOsMutexLock(hDevice->mutex); - if (gs_GartInited == NV_TRUE) - { - NvRmModuleGetBaseAddress( - hDevice, NvRmPrivModuleID_Gart, NULL, &GartSize); - GartEntries = GartSize / GART_PAGE_SIZE; - - for (GartEntry = 0; GartEntry < GartEntries; GartEntry++) - { - reg = NV_DRF_NUM(MC, GART_ENTRY_ADDR, GART_ENTRY_ADDR_TABLE_ADDR, - GartEntry); - NV_REGW(hDevice, NvRmPrivModuleID_MemoryController, 0, - MC_GART_ENTRY_ADDR_0, reg); - gs_GartSave[GartEntry] = NV_REGR(hDevice, - NvRmPrivModuleID_MemoryController, 0, MC_GART_ENTRY_DATA_0); - } - } - NvOsMutexUnlock(hDevice->mutex); -} - -void -NvRmPrivAp15GartResume(NvRmDeviceHandle hDevice) -{ - NvU32 reg; - NvU32 GartSize; - NvU32 GartEntries; - NvU32 GartEntry; - - NvOsMutexLock(hDevice->mutex); - if (gs_GartInited == NV_TRUE) - { - NvRmModuleGetBaseAddress( - hDevice, NvRmPrivModuleID_Gart, NULL, &GartSize); - GartEntries = GartSize / GART_PAGE_SIZE; - - for (GartEntry = 0; GartEntry < GartEntries; GartEntry++) - { - reg = NV_DRF_NUM(MC, GART_ENTRY_ADDR, GART_ENTRY_ADDR_TABLE_ADDR, - GartEntry); - NV_REGW(hDevice, NvRmPrivModuleID_MemoryController, 0, - MC_GART_ENTRY_ADDR_0, reg); - NV_REGW(hDevice, NvRmPrivModuleID_MemoryController, 0, - MC_GART_ENTRY_DATA_0, gs_GartSave[GartEntry] ); - } - - reg = NV_DRF_DEF(MC, GART_CONFIG, GART_ENABLE, ENABLE); - NV_REGW(hDevice, NvRmPrivModuleID_MemoryController, 0, - MC_GART_CONFIG_0, reg); - - } - NvOsMutexUnlock(hDevice->mutex); -} - diff --git a/arch/arm/mach-tegra/nvrm/core/ap15/ap15rm_init.c b/arch/arm/mach-tegra/nvrm/core/ap15/ap15rm_init.c index 9392cdab7679..ff532b9c70d2 100644 --- a/arch/arm/mach-tegra/nvrm/core/ap15/ap15rm_init.c +++ b/arch/arm/mach-tegra/nvrm/core/ap15/ap15rm_init.c @@ -39,7 +39,6 @@ #include "nvrm_rmctrace.h" #include "nvrm_configuration.h" #include "nvrm_chiplib.h" -#include "nvrm_heap.h" #include "nvrm_pmu_private.h" #include "nvrm_processor.h" #include "nvrm_xpc.h" @@ -143,8 +142,6 @@ NvRmOpenNew(NvRmDeviceHandle *pHandle) NvRmDevice *rm = 0; NvU32 *table = 0; - NvU32 CarveoutBaseAddr; - NvU32 CarveoutSize = 0; NvU32 BctCustomerOption = 0; NvU64 Uid = 0; @@ -279,31 +276,8 @@ NvRmOpenNew(NvRmDeviceHandle *pHandle) rm->bPreInit = NV_TRUE; - { - NvBootArgsCarveout Carveout; - if (NvOsBootArgGet(NvBootArgKey_Carveout, &Carveout, - sizeof(Carveout)) == NvSuccess) - { - CarveoutSize = Carveout.size; - CarveoutBaseAddr = (NvU32) Carveout.base; - } - else - { - CarveoutSize = NvOdmQueryCarveoutSize(); - CarveoutBaseAddr = rm->ExtMemoryInfo.base + - NvOdmQueryMemSize(NvOdmMemoryType_Sdram) - CarveoutSize; - } - } - - NvRmPrivHeapCarveoutInit(CarveoutSize, CarveoutBaseAddr); - NvRmPrivHeapIramInit(rm->IramMemoryInfo.size, rm->IramMemoryInfo.base); - NvRmPrivPreservedMemHandleInit(rm); - if (!NVOS_IS_WINDOWS_X86) { - // Initialize the GART heap (size & base address) - NvRmPrivHeapGartInit( rm ); - NvRmPrivCheckBondOut( rm ); /* bring modules out of reset */ @@ -523,15 +497,6 @@ NvRmClose(NvRmDeviceHandle handle) } - NvRmPrivHeapCarveoutDeinit(); - NvRmPrivHeapIramDeinit(); - - if (!NVOS_IS_WINDOWS_X86) - { - // De-Initialize the GART heap - NvRmPrivHeapGartDeinit(); - } - NvRmRmcClose( &handle->rmc ); /* deallocate the instance table */ @@ -594,14 +559,6 @@ NvRmPrivMemoryInfo( NvRmDeviceHandle hDevice ) inst++; } - if (!(NVCPU_IS_X86 && NVOS_IS_WINDOWS)) - { - /* Get GART memory module info */ - inst = tbl->ModInst + - (tbl->Modules)[NvRmPrivModuleID_Gart].Index; - hDevice->GartMemoryInfo.base = inst->PhysAddr; - hDevice->GartMemoryInfo.size = inst->Length; - } } NvError diff --git a/arch/arm/mach-tegra/nvrm/core/ap15/ap15rm_init_common.c b/arch/arm/mach-tegra/nvrm/core/ap15/ap15rm_init_common.c index 292cd21e67bc..fe3496ed65cd 100644 --- a/arch/arm/mach-tegra/nvrm/core/ap15/ap15rm_init_common.c +++ b/arch/arm/mach-tegra/nvrm/core/ap15/ap15rm_init_common.c @@ -39,7 +39,6 @@ #include "nvrm_rmctrace.h" #include "nvrm_configuration.h" #include "nvrm_chiplib.h" -#include "nvrm_heap.h" #include "nvrm_pmu_private.h" #include "nvrm_processor.h" #include "nvrm_structure.h" diff --git a/arch/arm/mach-tegra/nvrm/core/ap15/ap15rm_private.h b/arch/arm/mach-tegra/nvrm/core/ap15/ap15rm_private.h index e6a1bd130fbc..6d8a99037952 100644 --- a/arch/arm/mach-tegra/nvrm/core/ap15/ap15rm_private.h +++ b/arch/arm/mach-tegra/nvrm/core/ap15/ap15rm_private.h @@ -175,42 +175,6 @@ void NvRmPrivInitPinMux(NvRmDeviceHandle hDevice); /** - * Create a mapping from a list of physical sdram pages to the GART. No error - * checking is done here, so you can clobber your friend's mappings if you - * want. Every map for itself! This function programs registers and cannot - * fail. Invalid parameters will result in asserts for debug purposes. - * - * @see NvRmPrivGartAlloc() - * - * @param hDevice The RM device handle. - * @param pPhysAddrArray Points to an array of physical page addresses. Each - * entry represents the base address for a 4KB page of memory. These entries - * do not need to be contiguous memory blocks; after all, that's why you're - * using this API. - * @param NumberOfPages Specifies the number of physical address entries. A - * value of 0 has no effect. - * @param FirstGartPage Specifies the base address of the first available GART - * page. This value should be obtained via a call to NvRmPrivGartAlloc(). - */ -void -NvRmPrivCreateGARTMap( - NvRmDeviceHandle hDevice, - NvU32 *pPhysAddrArray, - NvU32 NumberOfPages, - NvU32 FirstGartPage); - -/** - * Suspend GART. - */ -void NvRmPrivGartSuspend( NvRmDeviceHandle hDevice ); - -/** - * Resume GART. - */ - -void NvRmPrivGartResume(NvRmDeviceHandle hDevice); - -/** * Initializes the clock manager. * * @param hRmDevice The RM device handle @@ -229,12 +193,6 @@ NvRmPrivClocksInit(NvRmDeviceHandle hRmDevice); void NvRmPrivClocksDeinit(NvRmDeviceHandle hRmDevice); -/** - * Increments a memory handle reference count. - */ -void -NvRmPrivMemIncrRef( NvRmMemHandle hMem ); - /*** Private Interrupt API's ***/ diff --git a/arch/arm/mach-tegra/nvrm/core/ap20/Makefile b/arch/arm/mach-tegra/nvrm/core/ap20/Makefile index 565115df45cb..ba4cd877b0fb 100644 --- a/arch/arm/mach-tegra/nvrm/core/ap20/Makefile +++ b/arch/arm/mach-tegra/nvrm/core/ap20/Makefile @@ -18,4 +18,3 @@ obj-y += ap20rm_power_dfs.o obj-y += ap20rm_pinmux_tables.o obj-y += ap20rm_fuse.o obj-y += ap20rm_clocks_info.o -obj-y += ap20rm_gart.o diff --git a/arch/arm/mach-tegra/nvrm/core/ap20/ap20rm_gart.c b/arch/arm/mach-tegra/nvrm/core/ap20/ap20rm_gart.c deleted file mode 100644 index 0cca272fdd89..000000000000 --- a/arch/arm/mach-tegra/nvrm/core/ap20/ap20rm_gart.c +++ /dev/null @@ -1,257 +0,0 @@ -/* - * Copyright (c) 2009 NVIDIA Corporation. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * Neither the name of the NVIDIA Corporation nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * - */ - -#include "ap20/armc.h" -#include "nvrm_heap.h" -#include "nvrm_heap_simple.h" -#include "nvrm_hwintf.h" -#include "ap15/ap15rm_private.h" -#include "nvassert.h" -#include "nvcommon.h" -#include "nvrm_drf.h" - - -/** - * Initialize the GART entries, and enable the GART - */ - -#define GART_PAGE_SHIFT (12) -#define GART_PAGE_SIZE (4096) - -extern NvBool gs_GartInited; -extern NvRmHeapSimple gs_GartAllocator; -extern NvU32 *gs_GartSave; - -/** - * Initializes all of the TLB entries in the GART and enables GART translations - * All entries are initially marked invalid. - * - * @param hDevice The RM device handle. - */ -static NvError -NvRmPrivAp20InitGART(NvRmDeviceHandle hDevice); -static NvError -NvRmPrivAp20InitGART(NvRmDeviceHandle hDevice) -{ - NvU32 GartSize; - NvU32 GartEntries; - NvU32 GartEntry; - NvU32 reg; - NvU32 data; - - NV_ASSERT(hDevice != NULL); - - NvRmModuleGetBaseAddress( - hDevice, NvRmPrivModuleID_Gart, NULL, &GartSize); - - GartEntries = GartSize / GART_PAGE_SIZE; - - gs_GartSave = NvOsAlloc( sizeof(NvU32) * GartEntries ); - if ( NULL == gs_GartSave ) - return NvError_InsufficientMemory; - - data = NV_DRF_NUM(MC, GART_ENTRY_DATA, GART_ENTRY_DATA_PHYS_ADDR_VALID, 0); - for (GartEntry = 0; GartEntry < GartEntries; ++GartEntry) - { - // set the address - reg = NV_DRF_NUM(MC, GART_ENTRY_ADDR, GART_ENTRY_ADDR_TABLE_ADDR, GartEntry); - NV_REGW(hDevice, NvRmPrivModuleID_MemoryController, 0, MC_GART_ENTRY_ADDR_0, reg); - - // mark the entry invalid - NV_REGW(hDevice, NvRmPrivModuleID_MemoryController, 0, MC_GART_ENTRY_DATA_0, data); - } - - // now enable the GART - reg = NV_DRF_DEF(MC, GART_CONFIG, GART_ENABLE, ENABLE); - NV_REGW(hDevice, NvRmPrivModuleID_MemoryController, 0, MC_GART_CONFIG_0, reg); - return NvSuccess; -} - -NvError -NvRmPrivAp20HeapGartAlloc( - NvRmDeviceHandle hDevice, - NvOsPageAllocHandle hPageHandle, - NvU32 NumberOfPages, - NvRmPhysAddr *PAddr) -{ - NvError result = NvSuccess; - NvU32 reg; - NvU32 i, data; - NvU32 FirstGartPage; - - NV_ASSERT(hDevice); - NV_ASSERT(hPageHandle); - - result = NvRmPrivHeapSimpleAlloc( - &gs_GartAllocator, - NumberOfPages*GART_PAGE_SIZE, - GART_PAGE_SIZE, - PAddr); - - if (result != NvSuccess) - return result; - - FirstGartPage = *PAddr; - - /* Check that the GART address exists and is page aligned */ - NV_ASSERT(FirstGartPage); - NV_ASSERT((FirstGartPage & (GART_PAGE_SIZE - 1)) == 0); - - NvOsMutexLock(hDevice->mutex); - - // FIXME: Normally we would do this at init time, but it takes and - // egregious amount of csim time, so I'm defering it or the 3d guys - // will complain to me, and then to my boss, and then their boss, and then their bosses boss... - if (gs_GartInited == NV_FALSE) - { - result = NvRmPrivAp20InitGART(hDevice); - if ( NvSuccess != result ) - goto fail; - gs_GartInited = NV_TRUE; - } - - for (i = 0; i < NumberOfPages; i++) - { - data = (NvU32)NvOsPageAddress(hPageHandle, i * GART_PAGE_SIZE); - - /* Check that each physical address is page aligned */ - NV_ASSERT((data & (GART_PAGE_SIZE - 1)) == 0); - - reg = NV_DRF_NUM(MC, GART_ENTRY_ADDR, GART_ENTRY_ADDR_TABLE_ADDR, ((FirstGartPage + i*GART_PAGE_SIZE) >> GART_PAGE_SHIFT)); - NV_REGW(hDevice, NvRmPrivModuleID_MemoryController, 0, MC_GART_ENTRY_ADDR_0, reg); - - reg = - NV_DRF_NUM(MC, GART_ENTRY_DATA, GART_ENTRY_DATA_PHYS_ADDR_VALID, 1) | - NV_DRF_NUM(MC, GART_ENTRY_DATA, GART_ENTRY_DATA_PHYS_ADDR, (data >> GART_PAGE_SHIFT)); - - NV_REGW(hDevice, NvRmPrivModuleID_MemoryController, 0, MC_GART_ENTRY_DATA_0, reg); - // lame, on csim we have to read this back to make sure the GART entry is valid before we hit the mc - // with data to this address. - (void)NV_REGR(hDevice, NvRmPrivModuleID_MemoryController, 0, MC_GART_ENTRY_DATA_0); - } -fail: - NvOsMutexUnlock(hDevice->mutex); - - return result; -} - -void -NvRmPrivAp20HeapGartFree( - NvRmDeviceHandle hDevice, - NvRmPhysAddr addr, - NvU32 NumberOfPages) -{ - NvU32 i; - NvU32 reg; - NvU32 data; - - NV_ASSERT(hDevice); - - if (addr && NumberOfPages) - { - // Invalidate GART page table entries - data = NV_DRF_NUM(MC, GART_ENTRY_DATA, GART_ENTRY_DATA_PHYS_ADDR_VALID, 0); - for (i = 0; i < NumberOfPages; i++) - { - // set the address - reg = NV_DRF_NUM(MC, GART_ENTRY_ADDR, GART_ENTRY_ADDR_TABLE_ADDR, ((addr + i*GART_PAGE_SIZE) >> GART_PAGE_SHIFT)); - NV_REGW(hDevice, NvRmPrivModuleID_MemoryController, 0, MC_GART_ENTRY_ADDR_0, reg); - - // mark the entry invalid - NV_REGW(hDevice, NvRmPrivModuleID_MemoryController, 0, MC_GART_ENTRY_DATA_0, data); - } - NvRmPrivHeapSimpleFree(&gs_GartAllocator, addr); - } -} - - -void -NvRmPrivAp20GartSuspend(NvRmDeviceHandle hDevice) -{ - NvU32 reg; - NvU32 GartSize; - NvU32 GartEntries; - NvU32 GartEntry; - - NvOsMutexLock(hDevice->mutex); - if (gs_GartInited == NV_TRUE) - { - NvRmModuleGetBaseAddress( - hDevice, NvRmPrivModuleID_Gart, NULL, &GartSize); - GartEntries = GartSize / GART_PAGE_SIZE; - - for (GartEntry = 0; GartEntry < GartEntries; GartEntry++) - { - reg = NV_DRF_NUM(MC, GART_ENTRY_ADDR, GART_ENTRY_ADDR_TABLE_ADDR, - GartEntry); - NV_REGW(hDevice, NvRmPrivModuleID_MemoryController, 0, - MC_GART_ENTRY_ADDR_0, reg); - gs_GartSave[GartEntry] = NV_REGR(hDevice, - NvRmPrivModuleID_MemoryController, 0, MC_GART_ENTRY_DATA_0); - } - } - NvOsMutexUnlock(hDevice->mutex); -} - -void -NvRmPrivAp20GartResume(NvRmDeviceHandle hDevice) -{ - NvU32 reg; - NvU32 GartSize; - NvU32 GartEntries; - NvU32 GartEntry; - - NvOsMutexLock(hDevice->mutex); - if (gs_GartInited == NV_TRUE) - { - NvRmModuleGetBaseAddress( - hDevice, NvRmPrivModuleID_Gart, NULL, &GartSize); - GartEntries = GartSize / GART_PAGE_SIZE; - - for (GartEntry = 0; GartEntry < GartEntries; GartEntry++) - { - reg = NV_DRF_NUM(MC, GART_ENTRY_ADDR, GART_ENTRY_ADDR_TABLE_ADDR, - GartEntry); - NV_REGW(hDevice, NvRmPrivModuleID_MemoryController, 0, - MC_GART_ENTRY_ADDR_0, reg); - NV_REGW(hDevice, NvRmPrivModuleID_MemoryController, 0, - MC_GART_ENTRY_DATA_0, gs_GartSave[GartEntry] ); - } - - reg = NV_DRF_DEF(MC, GART_CONFIG, GART_ENABLE, ENABLE); - NV_REGW(hDevice, NvRmPrivModuleID_MemoryController, 0, - MC_GART_CONFIG_0, reg); - - } - NvOsMutexUnlock(hDevice->mutex); -} - diff --git a/arch/arm/mach-tegra/nvrm/core/common/Makefile b/arch/arm/mach-tegra/nvrm/core/common/Makefile index 55c7f3042dbd..9adfebb4cf81 100644 --- a/arch/arm/mach-tegra/nvrm/core/common/Makefile +++ b/arch/arm/mach-tegra/nvrm/core/common/Makefile @@ -12,10 +12,6 @@ ccflags-y += -Iarch/arm/mach-tegra/nvrm/core/common ccflags-y += -Iarch/arm/mach-tegra/nvrm/core obj-y += nvrm_pinmux.o -obj-y += nvrm_heap_simple.o -obj-y += nvrm_memmgr.o -obj-y += nvrm_heap_carveout.o -obj-y += nvrm_heap_iram.o obj-y += nvrm_keylist.o obj-y += nvrm_configuration.o obj-y += nvrm_pmu.o diff --git a/arch/arm/mach-tegra/nvrm/core/common/nvrm_heap_carveout.c b/arch/arm/mach-tegra/nvrm/core/common/nvrm_heap_carveout.c deleted file mode 100644 index e5751a8168c8..000000000000 --- a/arch/arm/mach-tegra/nvrm/core/common/nvrm_heap_carveout.c +++ /dev/null @@ -1,188 +0,0 @@ -/* - * Copyright (c) 2007-2009 NVIDIA Corporation. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * Neither the name of the NVIDIA Corporation nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * - */ - -#include "nvcommon.h" -#include "nvos.h" -#include "nvrm_memmgr.h" -#include "nvrm_heap.h" -#include "nvrm_heap_simple.h" -#include "nvrm_hardware_access.h" - -static NvRmPrivHeap gs_CarveoutHeap; -static NvRmPhysAddr gs_CarveoutBaseAddr; -static void *gs_CarveoutVaddr; -static NvBool gs_CarveoutGloballyMapped; - -static NvRmHeapSimple gs_CarveoutAllocator; - - -NvError NvRmPrivHeapCarveoutAlloc( - NvU32 size, - NvU32 align, - NvRmPhysAddr *PAddr) -{ - return NvRmPrivHeapSimpleAlloc(&gs_CarveoutAllocator, size, align, PAddr); -} - -NvError NvRmPrivHeapCarveoutPreAlloc(NvRmPhysAddr Address, NvU32 Length) -{ - return NvRmPrivHeapSimplePreAlloc(&gs_CarveoutAllocator, Address, Length); -} - -void NvRmPrivHeapCarveoutFree(NvRmPhysAddr addr) -{ - NvRmPrivHeapSimpleFree(&gs_CarveoutAllocator, addr); -} - -NvS32 NvRmPrivHeapCarveoutMemoryUsed(void) -{ - return NvRmPrivHeapSimpleMemoryUsed(&gs_CarveoutAllocator); -} - -NvS32 NvRmPrivHeapCarveoutLargestFreeBlock(void) -{ - return NvRmPrivHeapSimpleLargestFreeBlock(&gs_CarveoutAllocator); -} - -NvS32 NvRmPrivHeapCarveoutTotalSize(void) -{ - return gs_CarveoutHeap.length; -} - - -void *NvRmPrivHeapCarveoutMemMap( - NvRmPhysAddr base, - NvU32 length, - NvOsMemAttribute attribute) -{ - NvU32 StartOffset = base - gs_CarveoutBaseAddr; - NvU32 EndOffset = StartOffset + length - 1; - - if (!gs_CarveoutVaddr) - return NULL; - - NV_ASSERT(length != 0); - - // sanity checking - if (StartOffset < gs_CarveoutHeap.length && - EndOffset < gs_CarveoutHeap.length) - { - NvUPtr uptr = (NvUPtr)gs_CarveoutVaddr; - return (void *)(uptr + StartOffset); - } - - NV_ASSERT(!"Attempt to map something that is not part of the carveout"); - return NULL; -} - - -NvRmPrivHeap *NvRmPrivHeapCarveoutInit(NvU32 length, NvRmPhysAddr base) -{ - NvError err; - NvBool bGloballyMapped = NV_FALSE; - void *vAddr = NULL; - -#if !(NVOS_IS_LINUX && !NVCPU_IS_X86) - /* try to map the memory, if we can't map it then bail out */ - err = NvRmPhysicalMemMap(base, length, - NVOS_MEM_READ_WRITE | NVOS_MEM_GLOBAL_ADDR, - NvOsMemAttribute_Uncached, &vAddr); - - if (err == NvSuccess) - { - bGloballyMapped = NV_TRUE; - } - else - { - // try again to map carveout, but with global flag gone - err = NvRmPhysicalMemMap(base, length, NVOS_MEM_READ_WRITE, - NvOsMemAttribute_Uncached, &vAddr); - - if (err != NvSuccess) - return NULL; - } -#endif - - err = NvRmPrivHeapSimple_HeapAlloc(base, length, &gs_CarveoutAllocator); - - if (err != NvSuccess) - { - if (vAddr) - NvRmPhysicalMemUnmap(vAddr, length); - return NULL; - } - - gs_CarveoutHeap.heap = NvRmHeap_ExternalCarveOut; - gs_CarveoutHeap.length = length; - gs_CarveoutHeap.PhysicalAddress = base; - gs_CarveoutBaseAddr = base; - gs_CarveoutVaddr = vAddr; - gs_CarveoutGloballyMapped = bGloballyMapped; - - return &gs_CarveoutHeap; -} - - -void NvRmPrivHeapCarveoutDeinit(void) -{ - // deinit the carveout allocator - if (gs_CarveoutVaddr) - { - NvRmPhysicalMemUnmap(gs_CarveoutVaddr, gs_CarveoutHeap.length); - gs_CarveoutVaddr = NULL; - } - - NvRmPrivHeapSimple_HeapFree(&gs_CarveoutAllocator); - NvOsMemset(&gs_CarveoutHeap, 0, sizeof(gs_CarveoutHeap)); - NvOsMemset(&gs_CarveoutAllocator, 0, sizeof(gs_CarveoutAllocator)); -} - - -void NvRmPrivHeapCarveoutGetInfo( - NvU32 *CarveoutPhysBase, - void **pCarveout, - NvU32 *CarveoutSize) -{ - if (gs_CarveoutGloballyMapped) - { - *CarveoutPhysBase = gs_CarveoutHeap.PhysicalAddress; - *pCarveout = gs_CarveoutVaddr; - *CarveoutSize = gs_CarveoutHeap.length; - } - else - { - *CarveoutPhysBase = 0; - *pCarveout = NULL; - *CarveoutSize = 0; - } -} - diff --git a/arch/arm/mach-tegra/nvrm/core/common/nvrm_heap_iram.c b/arch/arm/mach-tegra/nvrm/core/common/nvrm_heap_iram.c deleted file mode 100644 index 6f1b0cce7514..000000000000 --- a/arch/arm/mach-tegra/nvrm/core/common/nvrm_heap_iram.c +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Copyright (c) 2007-2009 NVIDIA Corporation. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * Neither the name of the NVIDIA Corporation nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * - */ - -#include "nvrm_heap.h" -#include "nvrm_heap_simple.h" -#include "nvrm_hardware_access.h" - - -static NvRmPrivHeap gs_IramHeap; -static NvUPtr gs_IramBaseAddr; -static void *gs_IramVaddr; -static NvRmHeapSimple gs_IramAllocator; - -NvError NvRmPrivHeapIramAlloc(NvU32 size, NvU32 align, NvRmPhysAddr *PAddr) -{ - NvError err; - err = NvRmPrivHeapSimpleAlloc(&gs_IramAllocator, size, align, PAddr); - return err; -} - -NvError NvRmPrivHeapIramPreAlloc(NvRmPhysAddr Address, NvU32 Length) -{ - return NvRmPrivHeapSimplePreAlloc(&gs_IramAllocator, Address, Length); -} - -void NvRmPrivHeapIramFree(NvRmPhysAddr addr) -{ - NvRmPrivHeapSimpleFree(&gs_IramAllocator, addr); -} - -NvRmPrivHeap *NvRmPrivHeapIramInit(NvU32 length, NvRmPhysAddr base) -{ - void *vAddr = NULL; - NvError err; - -#if !(NVOS_IS_LINUX && !NVCPU_IS_X86) - /* try to map the memory, if we can't map it then bail out */ - err = NvRmPhysicalMemMap(base, length, NVOS_MEM_READ_WRITE, - NvOsMemAttribute_Uncached, &vAddr); - if (err != NvSuccess) - return NULL; -#endif - - err = NvRmPrivHeapSimple_HeapAlloc(base, length, &gs_IramAllocator); - - if (err != NvSuccess) - { - if (vAddr) - NvRmPhysicalMemUnmap(vAddr, length); - return NULL; - } - - gs_IramHeap.heap = NvRmHeap_IRam; - gs_IramHeap.length = length; - gs_IramHeap.PhysicalAddress = base; - gs_IramBaseAddr = (NvUPtr)base; - gs_IramVaddr = vAddr; - - return &gs_IramHeap; -} - -void NvRmPrivHeapIramDeinit(void) -{ - // deinit the carveout allocator - if (gs_IramVaddr) - { - NvRmPhysicalMemUnmap(gs_IramVaddr, gs_IramHeap.length); - gs_IramVaddr = NULL; - } - - NvRmPrivHeapSimple_HeapFree(&gs_IramAllocator); - NvOsMemset(&gs_IramHeap, 0, sizeof(gs_IramHeap)); - NvOsMemset(&gs_IramAllocator, 0, sizeof(gs_IramAllocator)); -} - -void *NvRmPrivHeapIramMemMap( - NvRmPhysAddr base, - NvU32 length, - NvOsMemAttribute attribute) -{ - NvU32 StartOffset = base - gs_IramBaseAddr; - NvU32 EndOffset = StartOffset + length - 1; - - NV_ASSERT(length != 0); - - if (!gs_IramVaddr) - return NULL; - - // sanity checking - if (StartOffset < gs_IramHeap.length && - EndOffset < gs_IramHeap.length) - { - NvUPtr uptr = (NvUPtr)gs_IramVaddr; - return (void *)(uptr + StartOffset); - } - - NV_ASSERT(!"Attempt to map something that is not part of the iram"); - return NULL; -} diff --git a/arch/arm/mach-tegra/nvrm/core/common/nvrm_heap_simple.c b/arch/arm/mach-tegra/nvrm/core/common/nvrm_heap_simple.c deleted file mode 100644 index 7029cb13e560..000000000000 --- a/arch/arm/mach-tegra/nvrm/core/common/nvrm_heap_simple.c +++ /dev/null @@ -1,555 +0,0 @@ -/* - * Copyright (c) 2007-2009 NVIDIA Corporation. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * Neither the name of the NVIDIA Corporation nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * - */ - -/* This is an extremely simplistic memory allocator where the bookkeeping data - * is kept out of band from the allocated memory. It's intended to be used - * for framebuffer/carveout type allocations. - * - * Implementation is a simple first fit. Book-keeping is kept as a singly - * linked list, which means allocation time is O(n) w.r.t. the total number of - * allocations. - */ - - -#include "nvrm_heap_simple.h" -#include "nvassert.h" -#include "nvos.h" - -#define INITIAL_GROW_SIZE 8 -#define MAX_GROW_SIZE 512 -#define INVALID_INDEX (NvU32)-1 - -#define DEBUG_HEAP 0 - -#if DEBUG_HEAP - -static void -SanityCheckHeap(NvRmHeapSimple *pHeap) -{ - NvU32 index; - - // first clear all the touched bits - for (index = 0; index < pHeap->ArraySize; ++index) - { - pHeap->RawBlockArray[index].touched = 0; - } - - - // Walk the BlockArray - index = pHeap->BlockIndex; - for (;;) - { - if (index == INVALID_INDEX) - break; - - // make sure we're not off in the weeds. - NV_ASSERT( index < pHeap->ArraySize ); - NV_ASSERT( pHeap->RawBlockArray[index].touched == 0); - - pHeap->RawBlockArray[index].touched = 1; - index = pHeap->RawBlockArray[index].NextIndex; - } - - - // Walk the SpareArray - index = pHeap->SpareBlockIndex; - for (;;) - { - if (index == INVALID_INDEX) - break; - - // make sure we're not off in the weeds. - NV_ASSERT( index < pHeap->ArraySize ); - NV_ASSERT( pHeap->RawBlockArray[index].touched == 0); - - pHeap->RawBlockArray[index].touched = 2; - index = pHeap->RawBlockArray[index].NextIndex; - } - - - // check that all blocks get touched. - for (index = 0; index < pHeap->ArraySize; ++index) - { - NV_ASSERT(pHeap->RawBlockArray[index].touched != 0); - } -} - -#else -# define SanityCheckHeap(a) -#endif - - - -NvError NvRmPrivHeapSimple_HeapAlloc(NvRmPhysAddr base, NvU32 size, NvRmHeapSimple *pNewHeap) -{ - int i; - NvError err = NvError_InsufficientMemory; - - NV_ASSERT(pNewHeap != NULL); - NV_ASSERT(size > 0); - - NvOsMemset(pNewHeap, 0, sizeof(*pNewHeap)); - - pNewHeap->base = base; - pNewHeap->size = size; - pNewHeap->ArraySize = INITIAL_GROW_SIZE; - - pNewHeap->RawBlockArray = NvOsAlloc(sizeof(NvRmHeapSimpleBlock) * INITIAL_GROW_SIZE); - if (!pNewHeap->RawBlockArray) - { - err = NvError_InsufficientMemory; - goto fail; - } - NvOsMemset(pNewHeap->RawBlockArray, 0, sizeof(NvRmHeapSimpleBlock) * INITIAL_GROW_SIZE); - - - // setup all of the pointers (indices, whatever) - for (i = 0; i < INITIAL_GROW_SIZE; ++i) - { - pNewHeap->RawBlockArray[i].NextIndex = i + 1; - } - pNewHeap->RawBlockArray[i-1].NextIndex = INVALID_INDEX; - - pNewHeap->BlockIndex = 0; - pNewHeap->SpareBlockIndex = 1; - - pNewHeap->RawBlockArray[pNewHeap->BlockIndex].IsFree = NV_TRUE; - pNewHeap->RawBlockArray[pNewHeap->BlockIndex].PhysAddr = base; - pNewHeap->RawBlockArray[pNewHeap->BlockIndex].size = size; - pNewHeap->RawBlockArray[pNewHeap->BlockIndex].NextIndex = INVALID_INDEX; - - err = NvOsMutexCreate(&pNewHeap->mutex); - if (err) - goto fail; - - SanityCheckHeap(pNewHeap); - return NvSuccess; - -fail: - NvOsFree(pNewHeap->RawBlockArray); - return err; -} - - -/** - * Frees up a heap structure, and all items that were associated with this heap - * - * @param pHeap Pointer to the heap structure returned from NvRmPrivHeapSimpleHeapAlloc - */ - -void NvRmPrivHeapSimple_HeapFree(NvRmHeapSimple *pHeap) -{ - if (pHeap) - { - SanityCheckHeap(pHeap); - NvOsMutexDestroy(pHeap->mutex); - NvOsFree(pHeap->RawBlockArray); - } -} - -static NvError NvRmPrivHeapSimpleGrowBlockArray( - NvRmHeapSimple *pHeap) -{ - NvU32 SpareIndex = pHeap->SpareBlockIndex; - NvU32 NumFree = 0; - const NvU32 MinFree = 2; - - while (SpareIndex!=INVALID_INDEX && NumFree<MinFree) - { - NumFree++; - SpareIndex = pHeap->RawBlockArray[SpareIndex].NextIndex; - } - - if (NumFree < MinFree) - { - NvU32 i; - NvU32 NewArraySize; - NvU32 GrowSize; - NvRmHeapSimpleBlock *NewBlockArray; - - GrowSize = pHeap->ArraySize + (pHeap->ArraySize >> 1); - GrowSize = NV_MIN(GrowSize, MAX_GROW_SIZE); - - // Grow by 8 ensures that we have at least 2 blocks and also is a little - // more efficient than growing by 1 each time. - NewArraySize = pHeap->ArraySize + GrowSize; - NewBlockArray = NvOsAlloc( sizeof(NvRmHeapSimpleBlock)*NewArraySize); - if (!NewBlockArray) - { - SanityCheckHeap(pHeap); - return NvError_InsufficientMemory; - } - NvOsMemset(NewBlockArray, 0, sizeof(NvRmHeapSimpleBlock)*NewArraySize); - - NvOsMemcpy(NewBlockArray, pHeap->RawBlockArray, sizeof(NvRmHeapSimpleBlock)*pHeap->ArraySize); - - // setup the NextIndex in the new part of the array - for (i = pHeap->ArraySize; i < NewArraySize; i++) - { - NewBlockArray[i].NextIndex = i + 1; - } - - // Point the last element of the new array to the old SpareBlockList - NewBlockArray[NewArraySize - 1].NextIndex = pHeap->SpareBlockIndex; - NvOsFree(pHeap->RawBlockArray); - - // Update all our information - pHeap->RawBlockArray = NewBlockArray; - pHeap->SpareBlockIndex = pHeap->ArraySize; - pHeap->ArraySize = NewArraySize; - } - - return NvSuccess; -} - -NvError NvRmPrivHeapSimplePreAlloc( - NvRmHeapSimple *pHeap, - NvRmPhysAddr Address, - NvU32 Length) -{ - NvRmHeapSimpleBlock *pBlock; - NvU32 BlockIndex; - - NV_ASSERT(pHeap!=NULL); - - // All preallocated blocks must start at a minimum of a 32B alignment - if ((Address & 31) || (Length & 31)) - return NvError_NotSupported; - - NvOsMutexLock(pHeap->mutex); - - if (NvRmPrivHeapSimpleGrowBlockArray(pHeap)!=NvSuccess) - { - NvOsMutexUnlock(pHeap->mutex); - return NvError_InsufficientMemory; - } - - // Iteratively search through all the blocks for the block whose - // physical address region contains the requested pre-allocated - // region, and which isn't already allocated. - for (BlockIndex = pHeap->BlockIndex; BlockIndex!=INVALID_INDEX; - BlockIndex = pHeap->RawBlockArray[BlockIndex].NextIndex) - { - pBlock = &pHeap->RawBlockArray[BlockIndex]; - - if (pBlock->PhysAddr<=Address && - (pBlock->PhysAddr+pBlock->size) >= (Address+Length) && - pBlock->IsFree) - { - // If the free region starts before the preallocated region, - // split the free region into two blocks. - if (pBlock->PhysAddr < Address) - { - NvRmHeapSimpleBlock *NewBlock; - NvU32 NewBlockIndex; - - // Grab a block off the spare list and link it into place - NewBlockIndex = pHeap->SpareBlockIndex; - NewBlock = &pHeap->RawBlockArray[NewBlockIndex]; - pHeap->SpareBlockIndex = NewBlock->NextIndex; - - NewBlock->NextIndex = pBlock->NextIndex; - pBlock->NextIndex = NewBlockIndex; - - // Set up the new block - NewBlock->IsFree = NV_TRUE; - NewBlock->PhysAddr = Address; - NewBlock->size = pBlock->size; - - // Shrink the current block to - pBlock->size = (Address - pBlock->PhysAddr); - NewBlock->size -= pBlock->size; - - // Advance to the block we are actually going to allocate out of - pBlock = NewBlock; - } - - if ((pBlock->PhysAddr + pBlock->size) > (Address + Length)) - { - NvRmHeapSimpleBlock *NewBlock; - NvU32 NewBlockIndex; - - NewBlockIndex = pHeap->SpareBlockIndex; - NewBlock = &pHeap->RawBlockArray[NewBlockIndex]; - pHeap->SpareBlockIndex = NewBlock->NextIndex; - - NewBlock->NextIndex = pBlock->NextIndex; - pBlock->NextIndex = NewBlockIndex; - - NewBlock->IsFree = NV_TRUE; - NewBlock->PhysAddr = (pBlock->PhysAddr + Length); - NewBlock->size = (pBlock->size - Length); - - pBlock->size = Length; - } - - NV_ASSERT(pBlock->PhysAddr == Address && - pBlock->size == Length); - - pBlock->IsFree = NV_FALSE; - SanityCheckHeap(pHeap); - - NvOsMutexUnlock(pHeap->mutex); - return NvSuccess; - } - } - - SanityCheckHeap(pHeap); - NvOsMutexUnlock(pHeap->mutex); - return NvError_InsufficientMemory; -} - - -NvError NvRmPrivHeapSimpleAlloc( - NvRmHeapSimple *pHeap, - NvU32 size, - NvU32 align, - NvRmPhysAddr *pPAddr) -{ - NvRmHeapSimpleBlock *pBlock; - NvU32 BlockIndex; - - // Must align to a power of two - // Alignment offset should be less than the total alignment - NV_ASSERT(!(align & (align-1))); - - NV_ASSERT(pHeap != NULL); - - NvOsMutexLock(pHeap->mutex); - - if (NvRmPrivHeapSimpleGrowBlockArray(pHeap)!=NvSuccess) - { - NvOsMutexUnlock(pHeap->mutex); - return NvError_InsufficientMemory; - } - - // Scan through the list of blocks - for (BlockIndex = pHeap->BlockIndex; BlockIndex != INVALID_INDEX; BlockIndex = pHeap->RawBlockArray[BlockIndex].NextIndex) - { - NvRmPhysAddr NewOffset; - NvU32 ExtraAlignSpace; - - pBlock = &pHeap->RawBlockArray[BlockIndex]; - - // Skip blocks that are not free - if (!pBlock->IsFree) - { - continue; - } - - // Compute location where this allocation would start in this block, based - // on the alignment and range requested - NewOffset = pBlock->PhysAddr; - - NewOffset = (NewOffset + align-1) & ~(align-1); - NV_ASSERT(NewOffset >= pBlock->PhysAddr); - ExtraAlignSpace = NewOffset - pBlock->PhysAddr; - - // Is the block too small to fit this allocation, including the extra space - // required for alignment? - if (pBlock->size < (size + ExtraAlignSpace) ) - continue; - - // Do we need to split this block in two to start the allocation at the proper - // alignment? - if (ExtraAlignSpace > 0) - { - NvRmHeapSimpleBlock *NewBlock; - NvU32 NewBlockIndex; - - // Grab a block off the spare list and link it into place - NewBlockIndex = pHeap->SpareBlockIndex; - NewBlock = &pHeap->RawBlockArray[NewBlockIndex]; - pHeap->SpareBlockIndex = NewBlock->NextIndex; - - NewBlock->NextIndex = pBlock->NextIndex; - pBlock->NextIndex = NewBlockIndex; - - // Set up the new block - NewBlock->IsFree = NV_TRUE; - NewBlock->PhysAddr = pBlock->PhysAddr + ExtraAlignSpace; - NewBlock->size = pBlock->size - ExtraAlignSpace; - - // Shrink the current block to match this allocation - pBlock->size = ExtraAlignSpace; - - // Advance to the block we are actually going to allocate out of - pBlock = NewBlock; - } - - // Do we need to split this block into two? - if (pBlock->size > size) - { - NvRmHeapSimpleBlock *NewBlock; - NvU32 NewBlockIndex; - - // Grab a block off the spare list and link it into place - NewBlockIndex = pHeap->SpareBlockIndex; - NewBlock = &pHeap->RawBlockArray[NewBlockIndex]; - pHeap->SpareBlockIndex = NewBlock->NextIndex; - NewBlock->NextIndex = pBlock->NextIndex; - pBlock->NextIndex = NewBlockIndex; - - // Set up the new block - NewBlock->IsFree = NV_TRUE; - NewBlock->PhysAddr = pBlock->PhysAddr + size; - NewBlock->size = pBlock->size - size; - - // Shrink the current block to match this allocation - pBlock->size = size; - } - - NV_ASSERT(pBlock->size == size); - pBlock->IsFree = NV_FALSE; - - *pPAddr = pBlock->PhysAddr; - SanityCheckHeap(pHeap); - - NvOsMutexUnlock(pHeap->mutex); - return NvSuccess; - } - - SanityCheckHeap(pHeap); - NvOsMutexUnlock(pHeap->mutex); - return NvError_InsufficientMemory; -} - - - - -void NvRmPrivHeapSimpleFree(NvRmHeapSimple *pHeap, NvRmPhysAddr PhysAddr) -{ - NvRmHeapSimpleBlock *pBlock = NULL; - NvRmHeapSimpleBlock *pNext = NULL; - NvRmHeapSimpleBlock *pPrev = NULL; - - NvU32 BlockIndex; - NvU32 PrevIndex = INVALID_INDEX; - NvU32 NextIndex = INVALID_INDEX; - - - NV_ASSERT(pHeap != NULL); - - NvOsMutexLock(pHeap->mutex); - - // Find the block we're being asked to free - BlockIndex = pHeap->BlockIndex; - pBlock = &pHeap->RawBlockArray[BlockIndex]; - while (BlockIndex != INVALID_INDEX && (pBlock->PhysAddr != PhysAddr)) - { - PrevIndex = BlockIndex; - BlockIndex = pBlock->NextIndex; - pBlock = &pHeap->RawBlockArray[BlockIndex]; - } - - // The block we're being asked to free didn't exist or was already free - if (BlockIndex == INVALID_INDEX || pBlock->IsFree) - { - SanityCheckHeap(pHeap); - NvOsMutexUnlock(pHeap->mutex); - return; - } - - // This block is now a free block - pBlock->IsFree = NV_TRUE; - - // If next block is free, merge the two into one block - NextIndex = pBlock->NextIndex; - pNext = &pHeap->RawBlockArray[NextIndex]; - if (NextIndex != INVALID_INDEX && pNext->IsFree) - { - pBlock->size += pNext->size; - pBlock->NextIndex = pNext->NextIndex; - - pNext->NextIndex = pHeap->SpareBlockIndex; - pHeap->SpareBlockIndex = NextIndex; - } - - // If previous block is free, merge the two into one block - pPrev = &pHeap->RawBlockArray[PrevIndex]; - if (PrevIndex != INVALID_INDEX && pPrev->IsFree) - { - pPrev->size += pBlock->size; - pPrev->NextIndex = pBlock->NextIndex; - - pBlock->NextIndex = pHeap->SpareBlockIndex; - pHeap->SpareBlockIndex = BlockIndex; - } - SanityCheckHeap(pHeap); - NvOsMutexUnlock(pHeap->mutex); -} - -NvS32 NvRmPrivHeapSimpleMemoryUsed(NvRmHeapSimple* pHeap) -{ - NvS32 Index; - NvS32 MemUsed = 0; - - NV_ASSERT(pHeap != NULL); - - NvOsMutexLock(pHeap->mutex); - - for (Index = pHeap->BlockIndex; Index != INVALID_INDEX; ) - { - NvRmHeapSimpleBlock* Block = &pHeap->RawBlockArray[Index]; - - if (!Block->IsFree) - MemUsed += Block->size; - Index = Block->NextIndex; - } - - NvOsMutexUnlock(pHeap->mutex); - - return MemUsed; -} - -NvS32 NvRmPrivHeapSimpleLargestFreeBlock(NvRmHeapSimple* pHeap) -{ - NvS32 Index; - NvS32 MaxFree = 0; - - NV_ASSERT(pHeap != NULL); - - NvOsMutexLock(pHeap->mutex); - - for (Index = pHeap->BlockIndex; Index != INVALID_INDEX; ) - { - NvRmHeapSimpleBlock* Block = &pHeap->RawBlockArray[Index]; - int size = (int)Block->size; - - if (Block->IsFree && size > MaxFree) - MaxFree = size; - Index = Block->NextIndex; - } - - NvOsMutexUnlock(pHeap->mutex); - return MaxFree; -} diff --git a/arch/arm/mach-tegra/nvrm/core/common/nvrm_heap_simple.h b/arch/arm/mach-tegra/nvrm/core/common/nvrm_heap_simple.h deleted file mode 100644 index 588b87ad515b..000000000000 --- a/arch/arm/mach-tegra/nvrm/core/common/nvrm_heap_simple.h +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright (c) 2007-2009 NVIDIA Corporation. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * Neither the name of the NVIDIA Corporation nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * - */ - -#ifndef NVRM_HEAP_SIMPLE_H -#define NVRM_HEAP_SIMPLE_H - -#include "nvcommon.h" -#include "nvrm_init.h" -#include "nvos.h" - - -typedef struct NvRmHeapSimpleBlockRec NvRmHeapSimpleBlock; -struct NvRmHeapSimpleBlockRec -{ - NvBool IsFree; - NvRmPhysAddr PhysAddr; - NvU32 size; - - NvU32 NextIndex; - - // debug info - NvU32 touched; -}; - - -typedef struct NvRmHeapSimpleRec -{ - NvRmPhysAddr base; - NvU32 size; - NvU32 ArraySize; - - NvRmHeapSimpleBlock *RawBlockArray; - - NvU32 BlockIndex; - NvU32 SpareBlockIndex; - - NvOsMutexHandle mutex; -} NvRmHeapSimple; - -NvError NvRmPrivHeapSimple_HeapAlloc(NvRmPhysAddr Base, NvU32 Size, NvRmHeapSimple *pNewHeap); -void NvRmPrivHeapSimple_HeapFree(NvRmHeapSimple *); - -NvError NvRmPrivHeapSimpleAlloc(NvRmHeapSimple *, NvU32 size, NvU32 align, NvRmPhysAddr *paddr); - -NvError NvRmPrivHeapSimplePreAlloc( - NvRmHeapSimple *, - NvRmPhysAddr Address, - NvU32 Length); - -void NvRmPrivHeapSimpleFree(NvRmHeapSimple *, NvRmPhysAddr paddr); - -NvS32 NvRmPrivHeapSimpleMemoryUsed(NvRmHeapSimple* pHeap); - -NvS32 NvRmPrivHeapSimpleLargestFreeBlock(NvRmHeapSimple* pHeap); - -#endif // INCLUDED_HEAP_H diff --git a/arch/arm/mach-tegra/nvrm/core/common/nvrm_memmgr.c b/arch/arm/mach-tegra/nvrm/core/common/nvrm_memmgr.c deleted file mode 100644 index 08997f589a60..000000000000 --- a/arch/arm/mach-tegra/nvrm/core/common/nvrm_memmgr.c +++ /dev/null @@ -1,1226 +0,0 @@ -/* - * Copyright (c) 2007-2009 NVIDIA Corporation. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * Neither the name of the NVIDIA Corporation nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * - */ - -#include "nvrm_memmgr.h" -#include "nvrm_memmgr_private.h" -#include "nvrm_heap_simple.h" -#include "ap15/ap15rm_private.h" -#include "nvos.h" -#include "nvbootargs.h" -#include "nvrm_chiplib.h" - -/* FIXME: temporary hack to force all Linux allocations to be page-aligned */ -#if NVOS_IS_LINUX -#define NVRM_ALLOC_MIN_ALIGN 4096 -#else -#define NVRM_ALLOC_MIN_ALIGN 4 -#endif - -#define NVRM_CHECK_PIN 0 - -#define NVRM_MEM_MAGIC 0xdead9812 -#define NVRM_HMEM_CHECK(hMem) \ - do { \ - if (NVRM_HMEM_CHECK_MAGIC) { \ - NV_ASSERT(((NvU32)(hMem)&1)==0); \ - if (((NvU32)(hMem)&1)) { \ - (hMem) = idtomem(hMem); \ - } \ - NV_ASSERT((hMem)->magic == NVRM_MEM_MAGIC); \ - } \ - } while(0) - -static NvRmMemHandle idtomem(NvRmMemHandle hMem) -{ - NvOsDebugPrintf("RMMEM id->mem %08x\n",(int)hMem); - return (NvRmMemHandle)((NvU32)hMem&~1UL); -} - -#if NVRM_MEM_TRACE -#undef NvRmMemHandleCreate -#undef NvRmMemHandleFree -#undef NvRmMemGetId -#undef NvRmMemHandleFromId -#endif - - -/* GART related */ -NvBool gs_GartInited = NV_FALSE; -NvRmHeapSimple gs_GartAllocator; -NvU32 *gs_GartSave = NULL; -static NvRmPrivHeap gs_GartHeap; -static NvUPtr gs_GartBaseAddr; - -static NvError (*s_HeapGartAlloc)( NvRmDeviceHandle hDevice, - NvOsPageAllocHandle hPageHandle, - NvU32 NumberOfPages, NvRmPhysAddr *PAddr); -static void (*s_HeapGartFree)( NvRmDeviceHandle hDevice, - NvRmPhysAddr addr, NvU32 NumberOfPages); -static void (*s_GartSuspend)( NvRmDeviceHandle hDevice ) = NULL; -static void (*s_GartResume)( NvRmDeviceHandle hDevice ) = NULL; - - -static NvU32 gs_NextPreservedMemHandleKey; -static NvRmMemHandle gs_PreservedHandles[NV_BOOTARGS_MAX_PRESERVED_MEMHANDLES]; - -/* - * Notes: - * - * 1) The allocation of the handles should fall back to a block allocator - * that allocates say 1024 at a time to reduce heap fragmentation. - * - */ - -NvError NvRmMemHandleCreate( - NvRmDeviceHandle hRmDevice, - NvRmMemHandle *phMem, - NvU32 size) -{ - NvRmMemHandle pNewHandle = NULL; - NvError err = NvSuccess; - -#if NVCPU_IS_X86 - pNewHandle = NvOsAlloc(sizeof(*pNewHandle)+4); - pNewHandle = (NvRmMemHandle)(((NvU32)pNewHandle+3)&~3UL); -#else - pNewHandle = NvOsAlloc(sizeof(*pNewHandle)); -#endif - if (!pNewHandle) - { - err = NvError_InsufficientMemory; - goto exit_gracefully; - } - - NV_ASSERT(((NvU32)pNewHandle & 1) == 0); - - NvOsMemset(pNewHandle, 0, sizeof(*pNewHandle)); - pNewHandle->size = size; - pNewHandle->hRmDevice = hRmDevice; - pNewHandle->PhysicalAddress = NV_RM_INVALID_PHYS_ADDRESS; - pNewHandle->VirtualAddress = NULL; - pNewHandle->refcount = 1; - pNewHandle->pin_count = 0; - pNewHandle->coherency = NvOsMemAttribute_Uncached; -#if NVRM_HMEM_CHECK_MAGIC - pNewHandle->magic = NVRM_MEM_MAGIC; -#endif - - *phMem = pNewHandle; - -exit_gracefully: - if (err != NvSuccess) - NvOsFree(pNewHandle); - - return err; -} - -void NvRmPrivMemIncrRef(NvRmMemHandle hMem) -{ - NV_ASSERT(hMem); - NvOsAtomicExchangeAdd32(&hMem->refcount, 1); -} - -/* Attempt to use the pre-mapped carveout or iram aperture on Windows CE */ -#if !(NVOS_IS_LINUX && !NVCPU_IS_X86) -static void *NvRmMemMapGlobalHeap( - NvRmPhysAddr base, - NvU32 len, - NvRmHeap heap, - NvOsMemAttribute coherency) -{ - if (coherency == NvOsMemAttribute_WriteBack) - return NULL; - - if (heap == NvRmHeap_ExternalCarveOut) - return NvRmPrivHeapCarveoutMemMap(base, len, coherency); - else if (heap == NvRmHeap_IRam) - return NvRmPrivHeapIramMemMap(base, len, coherency); - - return NULL; -} -#else -#define NvRmMemMapGlobalHeap(base,len,heap,coherency) NULL -#endif - -static void NvRmPrivMemFree(NvRmMemHandle hMem) -{ - if (!hMem) - return; - - NVRM_HMEM_CHECK(hMem); - - if (!NV_RM_HMEM_IS_ALLOCATED(hMem)) - return; - - if(NVCPU_IS_X86 && !NvRmIsSimulation()) - { - NvOsFree(hMem->VirtualAddress); - hMem->VirtualAddress = NULL; - } - - if (!NvRmMemMapGlobalHeap(hMem->PhysicalAddress, 4, hMem->heap, - hMem->coherency) && hMem->VirtualAddress) { - NvRmMemUnmap(hMem, hMem->VirtualAddress, hMem->size); - hMem->VirtualAddress = NULL; - } - - switch (hMem->heap) - { - case NvRmHeap_ExternalCarveOut: - NvRmPrivHeapCarveoutFree(hMem->PhysicalAddress); - break; - case NvRmHeap_IRam: - NvRmPrivHeapIramFree(hMem->PhysicalAddress); - break; - case NvRmHeap_GART: - (*s_HeapGartFree)(hMem->hRmDevice, hMem->PhysicalAddress, - hMem->Pages); - NvOsPageFree(hMem->hPageHandle); - break; - case NvRmHeap_External: - NvOsPageFree(hMem->hPageHandle); - break; - default: - break; - } - - hMem->PhysicalAddress = NV_RM_INVALID_PHYS_ADDRESS; - hMem->VirtualAddress = NULL; - hMem->heap = 0; -#if NVRM_HMEM_CHECK_MAGIC - hMem->magic = 0; -#endif -} - -void NvRmMemHandleFree(NvRmMemHandle hMem) -{ - NvS32 old; - NvOsMutexHandle mutex; - - if( !hMem ) - { - return; - } - - NVRM_HMEM_CHECK(hMem); - old = NvOsAtomicExchangeAdd32(&hMem->refcount, -1); - if(old > 1) - { - return; - } - - NV_ASSERT(old != 0); - - mutex = hMem->hRmDevice->MemMgrMutex; - NvOsMutexLock(mutex); - - NvRmPrivMemFree(hMem); - NV_ASSERT(hMem->mapped == NV_FALSE); - if (hMem->mapped == NV_TRUE) - { - NvRmMemUnmap(hMem, hMem->VirtualAddress, hMem->size); - } - -#if NVRM_HMEM_CHECK_MAGIC - hMem->magic = 0; -#endif - - NvOsFree(hMem); - - NvOsMutexUnlock( mutex ); -} - -#define ERRATA_398959(ChipId) \ - ((ChipId).Id == 0x15 && (ChipId).Major == 1 && (ChipId).Minor == 1) - - -NvError NvRmMemAlloc( - NvRmMemHandle hMem, - const NvRmHeap *Heaps, - NvU32 NumHeaps, - NvU32 Alignment, - NvOsMemAttribute Coherency) -{ - // Default heap list does not include GART due to AP15 hardware bug. GART - // will be re-added to default heap list on AP20 and beyond. - NvRmHeap DefaultHeaps[3]; - NvU32 i; - NvError err; - - - NV_ASSERT(hMem && (!NumHeaps || Heaps)); - NVRM_HMEM_CHECK(hMem); - - /* FIXME: Windows should support full caching for memory handles. - * But not yet. - */ - if (Coherency == NvOsMemAttribute_Uncached) - Coherency = NvOsMemAttribute_WriteCombined; - - if (NV_RM_HMEM_IS_ALLOCATED(hMem)) - return NvError_AlreadyAllocated; - - if(NVCPU_IS_X86 && !NvRmIsSimulation()) - { - hMem->VirtualAddress = NvOsAlloc(hMem->size); - if(hMem->VirtualAddress) - { - if (Heaps) - { - hMem->heap = Heaps[0]; - } - - return NvSuccess; - } - return NvError_InsufficientMemory; - } - - NvOsMutexLock(hMem->hRmDevice->MemMgrMutex); - - if (hMem->size <= NVCPU_MIN_PAGE_SIZE && - (!NumHeaps || Heaps[0] != NvRmHeap_IRam)) - { - DefaultHeaps[0] = NvRmHeap_External; - DefaultHeaps[1] = NvRmHeap_ExternalCarveOut; - Heaps = DefaultHeaps; - NumHeaps = 2; - } - else if (!NumHeaps) - { - DefaultHeaps[0] = NvRmHeap_ExternalCarveOut; - DefaultHeaps[1] = NvRmHeap_External; - NumHeaps = 2; - if (!ERRATA_398959(hMem->hRmDevice->ChipId)) - DefaultHeaps[NumHeaps++] = NvRmHeap_GART; - Heaps = DefaultHeaps; - } - - // 4 is the minimum alignment for any heap. - if (Alignment < NVRM_ALLOC_MIN_ALIGN) - Alignment = NVRM_ALLOC_MIN_ALIGN; - - for (i=0, err=NvError_InsufficientMemory; - i<NumHeaps && err!=NvSuccess; i++) - { - if (Alignment > NVCPU_MIN_PAGE_SIZE && - (Heaps[i]==NvRmHeap_External || Heaps[i]==NvRmHeap_GART)) - { - NV_ASSERT(!"Invalid alignment request to GART / External heap"); - continue; - } - - switch (Heaps[i]) - { - case NvRmHeap_ExternalCarveOut: - err = NvRmPrivHeapCarveoutAlloc(hMem->size, - Alignment, &hMem->PhysicalAddress); - break; - case NvRmHeap_IRam: - err = NvRmPrivHeapIramAlloc(hMem->size, - Alignment, &hMem->PhysicalAddress); - break; - case NvRmHeap_External: - err = NvOsPageAlloc(hMem->size, Coherency, - NvOsPageFlags_Contiguous, NVOS_MEM_READ_WRITE, - &hMem->hPageHandle); - break; - case NvRmHeap_GART: - err = NvOsPageAlloc(hMem->size, Coherency, - NvOsPageFlags_NonContiguous, NVOS_MEM_READ_WRITE, - &hMem->hPageHandle); - - if (err != NvSuccess) - break; - - hMem->Pages = (hMem->size+(GART_PAGE_SIZE-1))/GART_PAGE_SIZE; - - err = (*s_HeapGartAlloc)(hMem->hRmDevice, - hMem->hPageHandle, hMem->Pages, &hMem->PhysicalAddress); - - if (err == NvSuccess) - break; - - hMem->Pages = 0; - NvOsPageFree(hMem->hPageHandle); - hMem->hPageHandle = NULL; - break; - - default: - NV_ASSERT(!"Invalid heap in heaps array"); - } - - if (err==NvSuccess) - break; - } - - NvOsMutexUnlock(hMem->hRmDevice->MemMgrMutex); - - if (err == NvSuccess) - { - hMem->alignment = Alignment; - hMem->heap = Heaps[i]; - hMem->coherency = Coherency; - - /* Don't cache virtual mappings for cacheable handles in the RM, - * since there isn't a good way to ensure proper coherency */ - if (Coherency != NvOsMemAttribute_WriteBack) - { - NvRmMemMap(hMem, 0, hMem->size, NVOS_MEM_READ_WRITE, - &hMem->VirtualAddress); - } - } - - return err; -} - -NvU32 NvRmMemPin(NvRmMemHandle hMem) -{ - NvS32 old; - - NV_ASSERT(hMem); - NVRM_HMEM_CHECK(hMem); - - old = NvOsAtomicExchangeAdd32(&hMem->pin_count, 1); - - NV_ASSERT(old != -1); - - // FIXME: finish implementation - - if (NVCPU_IS_X86 && !NvRmIsSimulation()) - return 0xFFFFFFFF; - - switch (hMem->heap) - { - case NvRmHeap_External: - return (NvU32)NvOsPageAddress(hMem->hPageHandle, 0); - case NvRmHeap_ExternalCarveOut: - case NvRmHeap_GART: - case NvRmHeap_IRam: - return hMem->PhysicalAddress; - default: - NV_ASSERT(!"Unknown heap"); - return 0xFFFFFFFF; - } -} - -void NvRmMemPinMult(NvRmMemHandle *hMems, NvU32 *Addrs, NvU32 Count) -{ - NvU32 i; - for( i = 0; i < Count; i++ ) - { - Addrs[i] = NvRmMemPin( hMems[i] ); - } -} - -void NvRmMemUnpin(NvRmMemHandle hMem) -{ - NvS32 old; - - if( !hMem ) - { - return; - } - - NVRM_HMEM_CHECK(hMem); - - old = NvOsAtomicExchangeAdd32(&hMem->pin_count, -1); - NV_ASSERT(old != 0); -} - -void NvRmMemUnpinMult(NvRmMemHandle *hMems, NvU32 Count) -{ - NvU32 i; - for(i = 0; i < Count; i++) - { - NvRmMemUnpin(hMems[i]); - } -} - -NvU32 NvRmMemGetAddress(NvRmMemHandle hMem, NvU32 Offset) -{ - NV_ASSERT(hMem != NULL); - NV_ASSERT(Offset < hMem->size); - NVRM_HMEM_CHECK(hMem); - -#if NVRM_CHECK_PIN - NV_ASSERT( hMem->pin_count ); -#endif - - if(NVCPU_IS_X86 && !NvRmIsSimulation()) - { - return (NvU32)-1; - } - - switch (hMem->heap) - { - case NvRmHeap_External: - return (NvU32)NvOsPageAddress(hMem->hPageHandle, Offset); - - case NvRmHeap_ExternalCarveOut: - case NvRmHeap_GART: - case NvRmHeap_IRam: - return (hMem->PhysicalAddress + Offset); - - default: - NV_ASSERT(!"Unknown heap"); - break; - } - - return (NvU32)-1; -} - - - - -NvError NvRmMemMap( - NvRmMemHandle hMem, - NvU32 Offset, - NvU32 Size, - NvU32 Flags, - void **pVirtAddr) -{ - NV_ASSERT(Offset + Size <= hMem->size); - NVRM_HMEM_CHECK(hMem); - - if (!hMem->VirtualAddress) - hMem->VirtualAddress = NvRmMemMapGlobalHeap( - hMem->PhysicalAddress+Offset, Size, hMem->heap, hMem->coherency); - - if (NvRmIsSimulation()) - return NvError_InsufficientMemory; - - if (hMem->VirtualAddress) - { - *pVirtAddr = (NvU8 *)hMem->VirtualAddress + Offset; - return NvSuccess; - } - - switch (hMem->heap) - { - case NvRmHeap_ExternalCarveOut: - case NvRmHeap_IRam: -#if !(NVOS_IS_LINUX && !NVCPU_IS_X86) - case NvRmHeap_GART: -#endif - return NvOsPhysicalMemMap(hMem->PhysicalAddress + Offset, - Size, hMem->coherency, Flags, pVirtAddr); - case NvRmHeap_External: - return NvOsPageMap(hMem->hPageHandle, Offset, Size, pVirtAddr); - default: - *pVirtAddr = NULL; - return NvError_NotSupported; - } -} - - -void NvRmMemUnmap(NvRmMemHandle hMem, void *pVirtAddr, NvU32 length) -{ - if (!hMem || !pVirtAddr || !length) { - return; - } - - NVRM_HMEM_CHECK(hMem); - - // No mappings ever get created in these cases - if (NvRmIsSimulation() || NVCPU_IS_X86) - return; - - /* Don't unmap from the global heap on CE */ - if (hMem->VirtualAddress && - NvRmMemMapGlobalHeap(hMem->PhysicalAddress, 4, hMem->heap, - hMem->coherency)) - { - return; - } - - /* Only unmap entire allocations; leaked mappings will be cleaned up - * when the handle is freed - */ - if (pVirtAddr != hMem->VirtualAddress || length != hMem->size) - return; - - hMem->VirtualAddress = NULL; - - switch (hMem->heap) - { - case NvRmHeap_External: - NvOsPageUnmap(hMem->hPageHandle, pVirtAddr, length); - break; - case NvRmHeap_ExternalCarveOut: - case NvRmHeap_IRam: -#if NVOS_IS_WINDOWS - case NvRmHeap_GART: -#endif - NvOsPhysicalMemUnmap(pVirtAddr, length); - break; - default: - break; - } -} - -NvU8 NvRmMemRd08(NvRmMemHandle hMem, NvU32 Offset) -{ - void *vaddr; - - NV_ASSERT(hMem->VirtualAddress != NULL); - if (!hMem->VirtualAddress) - return 0; - - vaddr = (NvU8 *)hMem->VirtualAddress + Offset; - NV_ASSERT(Offset + 1 <= hMem->size); - NVRM_HMEM_CHECK(hMem); - return NV_READ8(vaddr); -} - -NvU16 NvRmMemRd16(NvRmMemHandle hMem, NvU32 Offset) -{ - void *vaddr; - - NV_ASSERT(hMem->VirtualAddress != NULL); - if (!hMem->VirtualAddress) - return 0; - - vaddr = (NvU8 *)hMem->VirtualAddress + Offset; - NV_ASSERT(Offset + 2 <= hMem->size); - NVRM_HMEM_CHECK(hMem); - return NV_READ16(vaddr); -} - -NvU32 NvRmMemRd32(NvRmMemHandle hMem, NvU32 Offset) -{ - void *vaddr; - - NV_ASSERT(hMem->VirtualAddress != NULL); - if (!hMem->VirtualAddress) - return 0; - - vaddr = (NvU8 *)hMem->VirtualAddress + Offset; - NV_ASSERT(Offset + 4 <= hMem->size); - NVRM_HMEM_CHECK(hMem); - return NV_READ32(vaddr); -} - -void NvRmMemWr08(NvRmMemHandle hMem, NvU32 Offset, NvU8 Data) -{ - void *vaddr; - - NV_ASSERT(hMem->VirtualAddress != NULL); - if (!hMem->VirtualAddress) - return; - - vaddr = (NvU8 *)hMem->VirtualAddress + Offset; - NV_ASSERT(Offset + 1 <= hMem->size); - NVRM_HMEM_CHECK(hMem); - NVRM_RMC_TRACE((&hMem->hRmDevice->rmc, "MemoryWrite8 0x%x 0x%x\n", - hMem->PhysicalAddress + Offset, Data)); - NV_WRITE08(vaddr, Data); -} - -void NvRmMemWr16(NvRmMemHandle hMem, NvU32 Offset, NvU16 Data) -{ - void *vaddr; - - NV_ASSERT(hMem->VirtualAddress != NULL); - if (!hMem->VirtualAddress) - return; - - vaddr = (NvU8 *)hMem->VirtualAddress + Offset; - NV_ASSERT(Offset + 2 <= hMem->size); - NVRM_HMEM_CHECK(hMem); - NVRM_RMC_TRACE((&hMem->hRmDevice->rmc, "MemoryWrite16 0x%x 0x%x\n", - hMem->PhysicalAddress + Offset, Data)); - NV_WRITE16(vaddr, Data); -} - -void NvRmMemWr32(NvRmMemHandle hMem, NvU32 Offset, NvU32 Data) -{ - void *vaddr; - - NV_ASSERT(hMem->VirtualAddress != NULL); - if (!hMem->VirtualAddress) - return; - - vaddr = (NvU8 *)hMem->VirtualAddress + Offset; - NV_ASSERT(Offset + 4 <= hMem->size); - NVRM_HMEM_CHECK(hMem); - NVRM_RMC_TRACE((&hMem->hRmDevice->rmc, "MemoryWrite32 0x%x 0x%x\n", - hMem->PhysicalAddress + Offset, Data)); - NV_WRITE32(vaddr, Data); -} - -void NvRmMemRead(NvRmMemHandle hMem, NvU32 Offset, void *pDst, NvU32 Size) -{ - void *vaddr = (NvU8 *)hMem->VirtualAddress + Offset; - NV_ASSERT(Offset + Size <= hMem->size); - NVRM_HMEM_CHECK(hMem); - NV_READ(pDst, vaddr, Size); -} - -void NvRmMemWrite( - NvRmMemHandle hMem, - NvU32 Offset, - const void *pSrc, - NvU32 Size) -{ - void *vaddr = (NvU8 *)hMem->VirtualAddress + Offset; -#if NV_DEF_RMC_TRACE - NvU32 i; -#endif - - NV_ASSERT(Offset + Size <= hMem->size); - NVRM_HMEM_CHECK(hMem); - -#if NV_DEF_RMC_TRACE - for (i = 0; i < Size; i++) - { - NvU8 Data = ((const NvU8 *)pSrc)[i]; - NVRM_RMC_TRACE((&hMem->hRmDevice->rmc, "MemoryWrite8 0x%x 0x%x\n", - hMem->PhysicalAddress + i, Data)); - } -#endif - - NV_WRITE(vaddr, pSrc, Size); -} - -void NvRmMemReadStrided( - NvRmMemHandle hMem, - NvU32 Offset, - NvU32 SrcStride, - void *pDst, - NvU32 DstStride, - NvU32 ElementSize, - NvU32 Count) -{ - if ((ElementSize == SrcStride) && (ElementSize == DstStride)) - { - NvRmMemRead(hMem, Offset, pDst, ElementSize * Count); - } - else - { - while (Count--) - { - NvRmMemRead(hMem, Offset, pDst, ElementSize); - Offset += SrcStride; - pDst = (NvU8 *)pDst + DstStride; - } - } -} - -void NvRmMemWriteStrided( - NvRmMemHandle hMem, - NvU32 Offset, - NvU32 DstStride, - const void *pSrc, - NvU32 SrcStride, - NvU32 ElementSize, - NvU32 Count) -{ - if ((ElementSize == SrcStride) && (ElementSize == DstStride)) - { - NvRmMemWrite(hMem, Offset, pSrc, ElementSize * Count); - } - else - { - while (Count--) - { - NvRmMemWrite(hMem, Offset, pSrc, ElementSize); - Offset += DstStride; - pSrc = (const NvU8 *)pSrc + SrcStride; - } - } -} - -void NvRmMemMove( - NvRmMemHandle dstHMem, - NvU32 dstOffset, - NvRmMemHandle srcHMem, - NvU32 srcOffset, - NvU32 Size) -{ - NvU32 i; - - NV_ASSERT(dstOffset + Size <= dstHMem->size); - NV_ASSERT(srcOffset + Size <= srcHMem->size); - NVRM_HMEM_CHECK(dstHMem); - NVRM_HMEM_CHECK(srcHMem); - - if (((dstHMem->PhysicalAddress | - srcHMem->PhysicalAddress | - dstOffset | - srcOffset | - Size) & 3) == 0) - { - // everything is nicely word aligned - if (dstHMem == srcHMem && srcOffset < dstOffset) - { - for (i=Size; i; ) - { - NvU32 data; - i -= 4; - data = NvRmMemRd32(srcHMem, srcOffset+i); - NvRmMemWr32(dstHMem, dstOffset+i, data); - } - } - else - { - for (i=0; i < Size; i+=4) - { - NvU32 data = NvRmMemRd32(srcHMem, srcOffset+i); - NvRmMemWr32(dstHMem, dstOffset+i, data); - } - } - } - else - { - // fall back to writing one byte at a time - if (dstHMem == srcHMem && srcOffset < dstOffset) - { - for (i=Size; i--;) - { - NvU8 data = NvRmMemRd08(srcHMem, srcOffset+i); - NvRmMemWr08(dstHMem, dstOffset+i, data); - } - } - else - { - for (i=0; i < Size; ++i) - { - NvU8 data = NvRmMemRd08(srcHMem, srcOffset+i); - NvRmMemWr08(dstHMem, dstOffset+i, data); - } - } - } -} - -void NvRmMemCacheMaint( - NvRmMemHandle hMem, - void *pMapping, - NvU32 Size, - NvBool Writeback, - NvBool Invalidate) -{ - if (!hMem || !pMapping || !Size || !(Writeback || Invalidate)) - return; - - NVRM_HMEM_CHECK(hMem); - NV_ASSERT((NvU8*)pMapping+Size <= (NvU8*)hMem->VirtualAddress+Size); - if (Writeback && Invalidate) - NvOsDataCacheWritebackInvalidateRange(pMapping, Size); - else if (Writeback) - NvOsDataCacheWritebackRange(pMapping, Size); - else { - NV_ASSERT(!"Invalidate-only cache maintenance not supported in NvOs"); - } -} - -NvU32 NvRmMemGetSize(NvRmMemHandle hMem) -{ - NV_ASSERT(hMem); - NVRM_HMEM_CHECK(hMem); - return hMem->size; -} - -NvU32 NvRmMemGetAlignment(NvRmMemHandle hMem) -{ - NV_ASSERT(hMem); - NVRM_HMEM_CHECK(hMem); - return hMem->alignment; -} - -NvU32 NvRmMemGetCacheLineSize(void) -{ - // !!! FIXME: Currently for all our chips (ap15) - // both the L1 and L2 cache line sizes - // are 32 bytes. If this ever changes - // we need a way to figure it out on - // a chip by chip basis. - return 32; -} - -NvRmHeap NvRmMemGetHeapType(NvRmMemHandle hMem, NvU32 *BaseAddr) -{ - NV_ASSERT(hMem); - NVRM_HMEM_CHECK(hMem); - - if (hMem->heap == NvRmHeap_External) - *BaseAddr = (NvU32)NvOsPageAddress(hMem->hPageHandle, 0); - else - *BaseAddr = hMem->PhysicalAddress; - - return hMem->heap; -} - - -void *NvRmHostAlloc(size_t size) -{ - return NvOsAlloc(size); -} - -void NvRmHostFree(void *ptr) -{ - NvOsFree(ptr); -} - - -NvError NvRmMemMapIntoCallerPtr( - NvRmMemHandle hMem, - void *pCallerPtr, - NvU32 Offset, - NvU32 Size) -{ - NvError err; - NVRM_HMEM_CHECK(hMem); - - // The caller should be asking for an even number of pages. not strictly - // required, but the caller has already had to do the work to calculate the - // required number of pages so they might as well pass in a nice round - // number, which makes it easier to find bugs. - NV_ASSERT( (Size & (NVCPU_MIN_PAGE_SIZE-1)) == 0); - - // Make sure the supplied virtual address is page aligned. - NV_ASSERT( (((NvUPtr)pCallerPtr) & (NVCPU_MIN_PAGE_SIZE-1)) == 0); - - if (hMem->heap == NvRmHeap_External || - hMem->heap == NvRmHeap_GART) - { - err = NvOsPageMapIntoPtr(hMem->hPageHandle, - pCallerPtr, - Offset, - Size); - } - else if (hMem->heap == NvRmHeap_ExternalCarveOut || - hMem->heap == NvRmHeap_IRam) - { - // The caller is responsible for sending a size that - // is the correct number of pages, including this pageoffset - // at the beginning of the first page. - NvU32 PhysicalAddr = hMem->PhysicalAddress + Offset; - PhysicalAddr = PhysicalAddr & ~(NVCPU_MIN_PAGE_SIZE-1); - - err = NvOsPhysicalMemMapIntoCaller( - pCallerPtr, - PhysicalAddr, - Size, - NvOsMemAttribute_Uncached, - NVOS_MEM_WRITE | NVOS_MEM_READ); - } - else - { - return NvError_NotImplemented; - } - - return err; -} - - -NvU32 NvRmMemGetId(NvRmMemHandle hMem) -{ - NvU32 id = (NvU32)hMem; - - // !!! FIXME: Need to really create a unique id to handle the case where - // hMem is freed, and then the next allocated hMem returns the same pointer - // value. - - NVRM_HMEM_CHECK(hMem); - NV_ASSERT(((NvU32)hMem & 1) == 0); - if (!hMem || ((NvU32)hMem & 1)) - return 0; - -#if NVRM_MEM_CHECK_ID - id |= 1; -#endif - - return id; -} - -NvError NvRmMemHandleFromId(NvU32 id, NvRmMemHandle *phMem) -{ - NvRmMemHandle hMem; - // !!! FIXME: (see comment in GetId). Specifically handle the case where - // the memory handle has already been freed. - -#if NVRM_MEM_CHECK_ID - *phMem = NULL; - NV_ASSERT(id & 1); - if (!(id & 1)) - return NvError_BadParameter; -#endif - - hMem = (NvRmMemHandle)(id & ~1UL); - - NVRM_HMEM_CHECK(hMem); - - NvRmPrivMemIncrRef(hMem); - - *phMem = hMem; - return NvSuccess; -} - -NvError NvRmMemHandlePreserveHandle( - NvRmMemHandle hMem, - NvU32 *pKey) -{ - NvError e; - NvBootArgsPreservedMemHandle ArgMh; - - NV_ASSERT(hMem && pKey); - NvOsMutexLock(hMem->hRmDevice->MemMgrMutex); - if (gs_NextPreservedMemHandleKey >= - (NvU32)NvBootArgKey_PreservedMemHandle_Num) - { - e = NvError_InsufficientMemory; - goto clean; - } - - ArgMh.Address = (NvUPtr)hMem->PhysicalAddress; - ArgMh.Size = hMem->size; - - e = NvOsBootArgSet(gs_NextPreservedMemHandleKey, &ArgMh, sizeof(ArgMh)); - - if (e==NvSuccess) - { - *pKey = gs_NextPreservedMemHandleKey; - gs_NextPreservedMemHandleKey++; - } - else - { - *pKey = 0; - e = NvError_InsufficientMemory; - } - - clean: - NvOsMutexUnlock(hMem->hRmDevice->MemMgrMutex); - return e; -} - - -NvError NvRmMemHandleClaimPreservedHandle( - NvRmDeviceHandle hRm, - NvU32 Key, - NvRmMemHandle *pMem) -{ - NvU32 i; - NV_ASSERT(hRm && pMem && Key); - if (!pMem || !hRm || - Key<NvBootArgKey_PreservedMemHandle_0 || - Key>=NvBootArgKey_PreservedMemHandle_Num) - return NvError_BadParameter; - - *pMem = NULL; - - NvOsMutexLock(hRm->MemMgrMutex); - i = Key - NvBootArgKey_PreservedMemHandle_0; - *pMem = gs_PreservedHandles[i]; - gs_PreservedHandles[i] = NULL; - NvOsMutexUnlock(hRm->MemMgrMutex); - - if (*pMem) - return NvSuccess; - - return NvError_InsufficientMemory; -} - - -NvRmPrivHeap *NvRmPrivHeapGartInit(NvRmDeviceHandle hRmDevice) -{ - NvError err; - NvU32 length = hRmDevice->GartMemoryInfo.size; - NvRmPhysAddr base = hRmDevice->GartMemoryInfo.base; - NvRmModuleCapability caps[2]; - NvRmModuleCapability *pCap = NULL; - - caps[0].MajorVersion = 1; // AP15, AP16 - caps[0].MinorVersion = 0; - caps[0].EcoLevel = 0; - caps[0].Capability = &caps[0]; - - caps[1].MajorVersion = 1; // AP20/T20 - caps[1].MinorVersion = 1; - caps[1].EcoLevel = 0; - caps[1].Capability = &caps[1]; - - NV_ASSERT_SUCCESS(NvRmModuleGetCapabilities( - hRmDevice, - NvRmPrivModuleID_MemoryController, - caps, - NV_ARRAY_SIZE(caps), - (void**)&pCap)); - - err = NvRmPrivHeapSimple_HeapAlloc( - base, - length, - &gs_GartAllocator); - - if (err != NvSuccess) - return NULL; - - gs_GartHeap.heap = NvRmHeap_GART; - gs_GartHeap.length = length; - gs_GartHeap.PhysicalAddress = base; - - gs_GartBaseAddr = (NvUPtr)base; - (void)gs_GartBaseAddr; - - if ((pCap->MajorVersion == 1) && (pCap->MinorVersion == 0)) - { - s_HeapGartAlloc = NvRmPrivAp15HeapGartAlloc; - s_HeapGartFree = NvRmPrivAp15HeapGartFree; - s_GartSuspend = NvRmPrivAp15GartSuspend; - s_GartResume = NvRmPrivAp15GartResume; - } - else - { - s_HeapGartAlloc = NvRmPrivAp20HeapGartAlloc; - s_HeapGartFree = NvRmPrivAp20HeapGartFree; - s_GartSuspend = NvRmPrivAp20GartSuspend; - s_GartResume = NvRmPrivAp20GartResume; - } - - return &gs_GartHeap; -} - -void NvRmPrivHeapGartDeinit(void) -{ - // deinit the gart allocator - - NvRmPrivHeapSimple_HeapFree(&gs_GartAllocator); - NvOsMemset(&gs_GartHeap, 0, sizeof(gs_GartHeap)); - NvOsMemset(&gs_GartAllocator, 0, sizeof(gs_GartAllocator)); - NvOsFree( gs_GartSave ); - gs_GartInited = NV_FALSE; -} - -void NvRmPrivGartSuspend(NvRmDeviceHandle hDevice) -{ - NV_ASSERT(s_GartSuspend); - (*s_GartSuspend)( hDevice ); -} - -void NvRmPrivGartResume(NvRmDeviceHandle hDevice) -{ - NV_ASSERT(s_GartResume); - (*s_GartResume)( hDevice ); -} - -void NvRmPrivPreservedMemHandleInit(NvRmDeviceHandle hRm) -{ - unsigned int i; - NvBootArgsPreservedMemHandle mem; - - NvOsMemset(gs_PreservedHandles, 0, sizeof(gs_PreservedHandles)); - gs_NextPreservedMemHandleKey = (NvU32)NvBootArgKey_PreservedMemHandle_0; - - for (i=NvBootArgKey_PreservedMemHandle_0; - i<NvBootArgKey_PreservedMemHandle_Num; i++) - { - NvRmMemHandle hMem; - NvU32 j; - - if (NvOsBootArgGet(i, &mem, sizeof(mem))!=NvSuccess) - break; - - if (!mem.Address || !mem.Size) - break; - - if (NvRmMemHandleCreate(hRm, &hMem, mem.Size)!=NvSuccess) - continue; - - hMem->PhysicalAddress = mem.Address; - j = mem.Address; - hMem->alignment = 1; - while ((j & 1) == 0) - { - hMem->alignment <<= 1; - j >>= 1; - } - - if (NvRmPrivHeapCarveoutPreAlloc(mem.Address, mem.Size)==NvSuccess) - { - hMem->heap = NvRmHeap_ExternalCarveOut; - hMem->VirtualAddress = NvRmPrivHeapCarveoutMemMap(mem.Address, - mem.Size, NvOsMemAttribute_Uncached); - } - else if (NvRmPrivHeapIramPreAlloc(mem.Address, mem.Size)==NvSuccess) - { - hMem->heap = NvRmHeap_IRam; - hMem->VirtualAddress = NvRmPrivHeapIramMemMap(mem.Address, - mem.Size, NvOsMemAttribute_Uncached); - } - - if (hMem->heap) - gs_PreservedHandles[i-NvBootArgKey_PreservedMemHandle_0] = hMem; - else - NvRmMemHandleFree(hMem); - } -} - -NvError NvRmMemGetStat(NvRmMemStat Stat, NvS32* Result) -{ - /* Main point of this function is to be compatible backwards and forwards, - * i.e., breaking analysis apps is the thing to avoid. - * Minimum hassle - maximum impact. - * Performance is not that big of a deal. - * Could be extended to use NvS64 as return value. However, NvS64 is - * slightly more challenging in terms of printing etc. at the client side. - * This function should return counts as raw data as possible; conversions - * to percentages or anything like that should be left to the client. - */ - if (Stat == NvRmMemStat_TotalCarveout) - { - *Result = NvRmPrivHeapCarveoutTotalSize(); - } - else if (Stat == NvRmMemStat_UsedCarveout) - { - *Result = NvRmPrivHeapCarveoutMemoryUsed(); - } - else if (Stat == NvRmMemStat_LargestFreeCarveoutBlock) - { - *Result = NvRmPrivHeapCarveoutLargestFreeBlock(); - } - else if (Stat == NvRmMemStat_TotalGart) - { - *Result = gs_GartHeap.length; - } - else if (Stat == NvRmMemStat_UsedGart) - { - *Result = NvRmPrivHeapSimpleMemoryUsed(&gs_GartAllocator); - } - else if (Stat == NvRmMemStat_LargestFreeGartBlock) - { - *Result = NvRmPrivHeapSimpleLargestFreeBlock(&gs_GartAllocator); - } - else - { - return NvError_BadParameter; - } - return NvSuccess; -} diff --git a/arch/arm/mach-tegra/nvrm/core/common/nvrm_power.c b/arch/arm/mach-tegra/nvrm/core/common/nvrm_power.c index 480fe9482185..9bb8ae412807 100644 --- a/arch/arm/mach-tegra/nvrm/core/common/nvrm_power.c +++ b/arch/arm/mach-tegra/nvrm/core/common/nvrm_power.c @@ -1474,7 +1474,6 @@ NvError NvRmKernelPowerSuspend( NvRmDeviceHandle hRmDeviceHandle ) { - NvRmPrivGartSuspend(hRmDeviceHandle); NvRmPrivPmuInterruptMask(hRmDeviceHandle, NV_TRUE); NvRmPrivDfsSuspend(NvOdmQueryLowestSocPowerState()->LowestPowerState); @@ -1523,7 +1522,6 @@ NvError NvRmKernelPowerResume( NvRmDeviceHandle hRmDeviceHandle ) { NvRmPrivPmuInterruptMask(hRmDeviceHandle, NV_FALSE); - NvRmPrivGartResume(hRmDeviceHandle); return NvSuccess; } diff --git a/arch/arm/mach-tegra/tegra_sysmap.c b/arch/arm/mach-tegra/tegra_sysmap.c index 9e7d26e98a00..05da3c3d8b0a 100755 --- a/arch/arm/mach-tegra/tegra_sysmap.c +++ b/arch/arm/mach-tegra/tegra_sysmap.c @@ -42,6 +42,8 @@ static NvRmModuleID tegra_map_name_to_mod(const char *name, int inst) return NVRM_MODULE_ID(NvRmPrivModuleID_MemoryController, inst); else if (!strcmp(name, "gart")) return NVRM_MODULE_ID(NvRmPrivModuleID_Gart, inst); + else if (!strcmp(name, "iram")) + return NVRM_MODULE_ID(NvRmPrivModuleID_Iram, inst); return (NvRmModuleID) 0; } diff --git a/drivers/char/Kconfig b/drivers/char/Kconfig index d9405d51b154..b23ecd1dd34f 100644 --- a/drivers/char/Kconfig +++ b/drivers/char/Kconfig @@ -129,6 +129,25 @@ config DEVNVMAP is used by NVIDIA Tegra graphics and multimedia drivers for managing graphics memory. +config DEVNVMAP_PARANOID + bool "Validate all user-provided /dev/nvmap object references" + depends on DEVNVMAP + default n + help + Say Y here to enable additional process-level validations and + permissions for /dev/nvmap object references provided via ioctls. + May result in a decrease in performance. + +config DEVNVMAP_RECLAIM_UNPINNED_VM + bool "Allow /dev/nvmap to reclaim unpinned I/O virtual memory" + depends on DEVNVMAP && TEGRA_IOVMM + default n + help + Say Y here to enable /dev/nvmap to reclaim I/O virtual memory after + it has been unpinned, and re-use it for other objects. This can + allow a larger virtual I/O VM space than would normally be + supported by the hardware, at a slight cost in performance. + config SERIAL_NONSTANDARD bool "Non-standard serial port support" diff --git a/drivers/char/mem.c b/drivers/char/mem.c index 9785f1091721..f91999c7b735 100644 --- a/drivers/char/mem.c +++ b/drivers/char/mem.c @@ -903,11 +903,6 @@ static const struct file_operations kmsg_fops = { .write = kmsg_write, }; -#ifdef CONFIG_DEVNVMAP -extern const struct file_operations nvmap_fops; -extern const struct file_operations knvmap_fops; -#endif - static int memory_open(struct inode * inode, struct file * filp) { int ret = 0; @@ -957,14 +952,6 @@ static int memory_open(struct inode * inode, struct file * filp) filp->f_op = &oldmem_fops; break; #endif -#ifdef CONFIG_DEVNVMAP - case 13: - filp->f_op = &nvmap_fops; - break; - case 14: - filp->f_op = &knvmap_fops; - break; -#endif default: unlock_kernel(); return -ENXIO; @@ -1003,10 +990,6 @@ static const struct { #ifdef CONFIG_CRASH_DUMP {12,"oldmem", S_IRUSR | S_IWUSR | S_IRGRP, &oldmem_fops}, #endif -#ifdef CONFIG_DEVNVMAP - {13, "nvmap", S_IRUGO | S_IWUGO, &nvmap_fops}, - {14, "knvmap", S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP, &knvmap_fops }, -#endif }; static struct class *mem_class; diff --git a/drivers/char/nvmap.c b/drivers/char/nvmap.c index f1a3f1ff11b9..8546061016b6 100644 --- a/drivers/char/nvmap.c +++ b/drivers/char/nvmap.c @@ -1,9 +1,9 @@ /* * drivers/char/nvmap.c * - * Memory mapping driver for Tegra anonymous memory handles + * Memory manager for Tegra GPU memory handles * - * Copyright (c) 2009, NVIDIA Corporation. + * Copyright (c) 2009-2010, NVIDIA Corporation. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -20,6 +20,7 @@ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ +#include <linux/vmalloc.h> #include <linux/module.h> #include <linux/bitmap.h> #include <linux/wait.h> @@ -36,11 +37,17 @@ #include <linux/sched.h> #include <linux/io.h> #include <linux/tegra_devices.h> +#include <linux/rbtree.h> +#include <linux/proc_fs.h> +#include <linux/ctype.h> #include <asm/tlbflush.h> +#include <mach/iovmm.h> #include "linux/nvmem_ioctl.h" #include "nvcommon.h" #include "nvrm_memmgr.h" -#include "nvrm_memmgr_private.h" +#include "nvbootargs.h" + +/*#define IOVMM_FIRST*/ /* enable to force most allocations from iovmm */ static void nvmap_vma_open(struct vm_area_struct *vma); @@ -57,11 +64,25 @@ static int nvmap_mmap(struct file *filp, struct vm_area_struct *vma); static long nvmap_ioctl(struct file *filp, unsigned int cmd, unsigned long arg); -static int nvmap_cache_maint(struct file *filp, void __user *arg); +static int nvmap_ioctl_getid(struct file *filp, void __user *arg); + +static int nvmap_ioctl_get_param(struct file *filp, void __user* arg); + +static int nvmap_ioctl_alloc(struct file *filp, void __user *arg); + +static int nvmap_ioctl_free(struct file *filp, unsigned long arg); + +static int nvmap_ioctl_create(struct file *filp, + unsigned int cmd, void __user *arg); + +static int nvmap_ioctl_pinop(struct file *filp, + bool is_pin, void __user *arg); + +static int nvmap_ioctl_cache_maint(struct file *filp, void __user *arg); static int nvmap_map_into_caller_ptr(struct file *filp, void __user *arg); -static int nvmap_rw_handle(struct file *filp, int is_read, +static int nvmap_ioctl_rw_handle(struct file *filp, int is_read, void __user* arg); extern void NvRmPrivMemIncrRef(NvRmMemHandle hmem); @@ -81,9 +102,451 @@ static struct backing_dev_info nvmap_bdi = { static pte_t *nvmap_pte[NUM_NVMAP_PTES]; static unsigned long nvmap_ptebits[NVMAP_PAGES/BITS_PER_LONG]; + static DEFINE_SPINLOCK(nvmap_ptelock); static DECLARE_WAIT_QUEUE_HEAD(nvmap_ptefull); +/* used to lost the master tree of memory handles */ +static DEFINE_SPINLOCK(nvmap_handle_lock); + +/* only one task may be performing pin / unpin operations at once, to + * prevent deadlocks caused by interleaved IOVMM re-allocations */ +static DEFINE_MUTEX(nvmap_pin_lock); + +/* queue of tasks which are blocking on pin, for IOVMM room */ +static DECLARE_WAIT_QUEUE_HEAD(nvmap_pin_wait); +static struct rb_root nvmap_handles = RB_ROOT; + +static struct tegra_iovmm_client *nvmap_vm_client = NULL; + +/* first-fit linear allocator carveout heap manager */ +struct nvmap_mem_block { + unsigned long base; + size_t size; + short next; /* next absolute (address-order) block */ + short prev; /* previous absolute (address-order) block */ + short next_free; + short prev_free; +}; + +struct nvmap_carveout { + unsigned short num_blocks; + short spare_index; + short free_index; + short block_index; + spinlock_t lock; + const char *name; + struct nvmap_mem_block *blocks; +}; + +enum { + CARVEOUT_STAT_TOTAL_SIZE, + CARVEOUT_STAT_FREE_SIZE, + CARVEOUT_STAT_NUM_BLOCKS, + CARVEOUT_STAT_FREE_BLOCKS, + CARVEOUT_STAT_LARGEST_BLOCK, + CARVEOUT_STAT_LARGEST_FREE, + CARVEOUT_STAT_BASE, +}; + +static inline pgprot_t _nvmap_flag_to_pgprot(unsigned long flag, pgprot_t base) +{ + switch (flag) { + case NVMEM_HANDLE_UNCACHEABLE: + base = pgprot_noncached(base); + break; + case NVMEM_HANDLE_WRITE_COMBINE: + base = pgprot_writecombine(base); + break; + case NVMEM_HANDLE_INNER_CACHEABLE: + base = pgprot_inner_writeback(base); + break; + } + return base; +} + +static unsigned long _nvmap_carveout_blockstat(struct nvmap_carveout *co, + int stat) +{ + unsigned long val = 0; + short idx; + spin_lock(&co->lock); + + if (stat==CARVEOUT_STAT_BASE) { + if (co->block_index==-1) + val = ~0; + else + val = co->blocks[co->block_index].base; + spin_unlock(&co->lock); + return val; + } + + if (stat==CARVEOUT_STAT_TOTAL_SIZE || + stat==CARVEOUT_STAT_NUM_BLOCKS || + stat==CARVEOUT_STAT_LARGEST_BLOCK) + idx = co->block_index; + else + idx = co->free_index; + + while (idx!=-1) { + switch (stat) { + case CARVEOUT_STAT_TOTAL_SIZE: + val += co->blocks[idx].size; + idx = co->blocks[idx].next; + break; + case CARVEOUT_STAT_NUM_BLOCKS: + val++; + idx = co->blocks[idx].next; + break; + case CARVEOUT_STAT_LARGEST_BLOCK: + val = max_t(unsigned long, val, co->blocks[idx].size); + idx = co->blocks[idx].next; + break; + case CARVEOUT_STAT_FREE_SIZE: + val += co->blocks[idx].size; + idx = co->blocks[idx].next_free; + break; + case CARVEOUT_STAT_FREE_BLOCKS: + val ++; + idx = co->blocks[idx].next_free; + break; + case CARVEOUT_STAT_LARGEST_FREE: + val = max_t(unsigned long, val, co->blocks[idx].size); + idx = co->blocks[idx].next_free; + break; + } + } + + spin_unlock(&co->lock); + return val; +} + +#define co_is_free(_co, _idx) \ + ((_co)->free_index==(_idx) || ((_co)->blocks[(_idx)].prev_free!=-1)) + +static int _nvmap_init_carveout(struct nvmap_carveout *co, + const char *name, unsigned long base_address, size_t len) +{ + const unsigned int min_blocks = 16; + struct nvmap_mem_block *blocks = NULL; + int i; + + blocks = kzalloc(sizeof(*blocks)*min_blocks, GFP_KERNEL); + + if (!blocks) goto fail; + co->name = kstrdup(name, GFP_KERNEL); + if (!co->name) goto fail; + + for (i=1; i<min_blocks; i++) { + blocks[i].next = i+1; + blocks[i].prev = i-1; + blocks[i].next_free = -1; + blocks[i].prev_free = -1; + } + blocks[i-1].next = -1; + blocks[1].prev = -1; + + blocks[0].next = blocks[0].prev = -1; + blocks[0].next_free = blocks[0].prev_free = -1; + blocks[0].base = base_address; + blocks[0].size = len; + co->blocks = blocks; + co->num_blocks = min_blocks; + spin_lock_init(&co->lock); + co->block_index = 0; + co->spare_index = 1; + co->free_index = 0; + return 0; + +fail: + if (blocks) kfree(blocks); + return -ENOMEM; +} + +static int nvmap_grow_blocks(struct nvmap_carveout *co) +{ + struct nvmap_mem_block *blocks; + unsigned int i; + + if (co->num_blocks >= 1<<(8*sizeof(co->free_index)-1)) return -ENOMEM; + blocks = kzalloc(sizeof(*blocks)*(co->num_blocks*2), GFP_KERNEL); + if (!blocks) return -ENOMEM; + + memcpy(blocks, co->blocks, sizeof(*blocks)*(co->num_blocks)); + kfree(co->blocks); + co->blocks = blocks; + for (i=co->num_blocks; i<co->num_blocks*2; i++) { + blocks[i].next = i+1; + blocks[i].prev = i-1; + blocks[i].next_free = -1; + blocks[i].prev_free = -1; + } + blocks[co->num_blocks].prev = -1; + blocks[i-1].next = -1; + blocks[i].next_free = -1; + blocks[i].prev_free = -1; + co->spare_index = co->num_blocks; + co->num_blocks *= 2; + return 0; +} + +static int nvmap_get_spare(struct nvmap_carveout *co) { + int idx; + + if (co->spare_index == -1) + if (nvmap_grow_blocks(co)) + return -1; + + BUG_ON(co->spare_index == -1); + idx = co->spare_index; + co->spare_index = co->blocks[idx].next; + co->blocks[idx].next = -1; + co->blocks[idx].prev = -1; + co->blocks[idx].next_free = -1; + co->blocks[idx].prev_free = -1; + return idx; +} + +#define BLOCK(_co, _idx) ((_idx)==-1 ? NULL : &(_co)->blocks[(_idx)]) + +static void nvmap_zap_free(struct nvmap_carveout *co, int idx) +{ + struct nvmap_mem_block *block; + + block = BLOCK(co, idx); + if (block->prev_free != -1) + BLOCK(co, block->prev_free)->next_free = block->next_free; + else + co->free_index = block->next_free; + + if (block->next_free != -1) + BLOCK(co, block->next_free)->prev_free = block->prev_free; + + block->prev_free = -1; + block->next_free = -1; +} + +static void nvmap_split_block(struct nvmap_carveout *co, + int idx, size_t start, size_t size) +{ + if (BLOCK(co, idx)->base < start) { + int spare_idx = nvmap_get_spare(co); + struct nvmap_mem_block *spare = BLOCK(co, spare_idx); + struct nvmap_mem_block *block = BLOCK(co, idx); + if (spare) { + spare->size = start - block->base; + spare->base = block->base; + block->size -= (start - block->base); + block->base = start; + spare->next = idx; + spare->prev = block->prev; + block->prev = spare_idx; + if (spare->prev != -1) + co->blocks[spare->prev].next = spare_idx; + else + co->block_index = spare_idx; + spare->prev_free = -1; + spare->next_free = co->free_index; + if (co->free_index != -1) + co->blocks[co->free_index].prev_free = spare_idx; + co->free_index = spare_idx; + } else { + if (block->prev != -1) { + spare = BLOCK(co, block->prev); + spare->size += start - block->base; + block->base = start; + } + } + } + + if (BLOCK(co, idx)->size > size) { + int spare_idx = nvmap_get_spare(co); + struct nvmap_mem_block *spare = BLOCK(co, spare_idx); + struct nvmap_mem_block *block = BLOCK(co, idx); + if (spare) { + spare->base = block->base + size; + spare->size = block->size - size; + block->size = size; + spare->prev = idx; + spare->next = block->next; + block->next = spare_idx; + if (spare->next != -1) + co->blocks[spare->next].prev = spare_idx; + spare->prev_free = -1; + spare->next_free = co->free_index; + if (co->free_index != -1) + co->blocks[co->free_index].prev_free = spare_idx; + co->free_index = spare_idx; + } + } + + nvmap_zap_free(co, idx); +} + +#define next_spare next +#define prev_spare prev + +#define nvmap_insert_block(_list, _co, _idx) \ + do { \ + struct nvmap_mem_block *b = BLOCK((_co), (_idx)); \ + struct nvmap_mem_block *s = BLOCK((_co), (_co)->_list##_index);\ + if (s) s->prev_##_list = (_idx); \ + b->prev_##_list = -1; \ + b->next_##_list = (_co)->_list##_index; \ + (_co)->_list##_index = (_idx); \ + } while (0); + +static void nvmap_carveout_free(struct nvmap_carveout *co, int idx) +{ + struct nvmap_mem_block *b; + + spin_lock(&co->lock); + + b = BLOCK(co, idx); + + if (b->next!=-1 && co_is_free(co, b->next)) { + int zap = b->next; + struct nvmap_mem_block *n = BLOCK(co, zap); + b->size += n->size; + + b->next = n->next; + if (n->next != -1) co->blocks[n->next].prev = idx; + + nvmap_zap_free(co, zap); + nvmap_insert_block(spare, co, zap); + } + + if (b->prev!=-1 && co_is_free(co, b->prev)) { + int zap = b->prev; + struct nvmap_mem_block *p = BLOCK(co, zap); + + b->base = p->base; + b->size += p->size; + + b->prev = p->prev; + + if (p->prev != -1) co->blocks[p->prev].next = idx; + else co->block_index = idx; + + nvmap_zap_free(co, zap); + nvmap_insert_block(spare, co, zap); + } + + nvmap_insert_block(free, co, idx); + spin_unlock(&co->lock); +} + +static int nvmap_carveout_alloc(struct nvmap_carveout *co, + size_t align, size_t size) +{ + short idx; + + spin_lock(&co->lock); + + idx = co->free_index; + + while (idx != -1) { + struct nvmap_mem_block *b = BLOCK(co, idx); + /* try to be a bit more clever about generating block- + * droppings by comparing the results of a left-justified vs + * right-justified block split, and choosing the + * justification style which yields the largest remaining + * block */ + size_t end = b->base + b->size; + size_t ljust = (b->base + align - 1) & ~(align-1); + size_t rjust = (end - size) & ~(align-1); + size_t l_max, r_max; + + if (rjust < b->base) rjust = ljust; + l_max = max_t(size_t, ljust - b->base, end - (ljust + size)); + r_max = max_t(size_t, rjust - b->base, end - (rjust + size)); + + if (b->base + b->size >= ljust + size) { + if (l_max >= r_max) + nvmap_split_block(co, idx, ljust, size); + else + nvmap_split_block(co, idx, rjust, size); + break; + } + idx = b->next_free; + } + + spin_unlock(&co->lock); + return idx; +} + +#undef next_spare +#undef prev_spare + +struct nvmap_handle { + struct rb_node node; + atomic_t ref; + atomic_t pin; + unsigned long flags; + size_t size; + size_t orig_size; + struct task_struct *owner; + unsigned int poison; + union { + struct { + struct page **pages; + struct tegra_iovmm_area *area; + struct list_head mru_list; + bool contig; + bool dirty; /* IOVMM area allocated since last pin */ + } pgalloc; + struct { + struct nvmap_carveout *co_heap; + int block_idx; + unsigned long base; + unsigned int key; /* preserved by bootloader */ + } carveout; + }; + bool global; + bool secure; /* only allocated in IOVM space, zapped on unpin */ + bool heap_pgalloc; + bool alloc; + void *kern_map; /* used for RM memmgr backwards compat */ +}; + +/* handle_ref objects are file-descriptor-local references to nvmap_handle + * objects. they track the number of references and pins performed by + * the specific caller (since nvmap_handle objects may be global), so that + * a client which terminates without properly unwinding all handles (or + * all nested pins) can be unwound by nvmap. */ +struct nvmap_handle_ref { + struct nvmap_handle *h; + struct rb_node node; + atomic_t refs; + atomic_t pin; +}; + +struct nvmap_file_priv { + struct rb_root handle_refs; + atomic_t iovm_commit; + size_t iovm_limit; + spinlock_t ref_lock; + bool su; +}; + +struct nvmap_carveout_node { + struct device dev; + struct list_head heap_list; + unsigned int heap_bit; + struct nvmap_carveout carveout; +}; + +/* the master structure for all nvmap-managed carveouts and all handle_ref + * objects allocated inside the kernel. heaps are sorted by their heap_bit + * (highest heap_bit first) so that carveout allocation will be first + * attempted by the heap with the highest heap_bit set in the allocation's + * heap mask */ +static struct { + struct nvmap_file_priv init_data; + struct rw_semaphore list_sem; + struct list_head heaps; +} nvmap_context; + static struct vm_operations_struct nvmap_vma_ops = { .open = nvmap_vma_open, .close = nvmap_vma_close, @@ -107,20 +570,643 @@ const struct file_operations knvmap_fops = { }; struct nvmap_vma_priv { - NvRmMemHandle hmem; - size_t offs; - atomic_t ref; + struct nvmap_handle *h; + size_t offs; + atomic_t ref; }; +static struct proc_dir_entry *nvmap_procfs_root; +static struct proc_dir_entry *nvmap_procfs_proc; + +static void _nvmap_handle_free(struct nvmap_handle *h); + +#define NVMAP_CARVEOUT_ATTR_RO(_name) \ + struct device_attribute nvmap_heap_attr_##_name = \ + __ATTR(_name, S_IRUGO, _nvmap_sysfs_show_heap_##_name, NULL) + +#define NVMAP_CARVEOUT_ATTR_WO(_name, _mode) \ + struct device_attribute nvmap_heap_attr_##_name = \ + __ATTR(_name, _mode, NULL, _nvmap_sysfs_set_heap_##_name) + + +static ssize_t _nvmap_sysfs_show_heap_usage(struct device *d, + struct device_attribute *attr, char *buf) +{ + struct nvmap_carveout_node *c = container_of(d, + struct nvmap_carveout_node, dev); + return sprintf(buf, "%08x\n", c->heap_bit); +} + +static ssize_t _nvmap_sysfs_show_heap_name(struct device *d, + struct device_attribute *attr, char *buf) +{ + struct nvmap_carveout_node *c = container_of(d, + struct nvmap_carveout_node, dev); + return sprintf(buf, "%s\n", c->carveout.name); +} + +static ssize_t _nvmap_sysfs_show_heap_base(struct device *d, + struct device_attribute *attr, char *buf) +{ + struct nvmap_carveout_node *c = container_of(d, + struct nvmap_carveout_node, dev); + return sprintf(buf, "%08lx\n", + _nvmap_carveout_blockstat(&c->carveout, CARVEOUT_STAT_BASE)); +} + +static ssize_t _nvmap_sysfs_show_heap_free_size(struct device *d, + struct device_attribute *attr, char *buf) +{ + struct nvmap_carveout_node *c = container_of(d, + struct nvmap_carveout_node, dev); + return sprintf(buf, "%lu\n", + _nvmap_carveout_blockstat(&c->carveout, + CARVEOUT_STAT_FREE_SIZE)); +} + +static ssize_t _nvmap_sysfs_show_heap_free_count(struct device *d, + struct device_attribute *attr, char *buf) +{ + struct nvmap_carveout_node *c = container_of(d, + struct nvmap_carveout_node, dev); + return sprintf(buf, "%lu\n", + _nvmap_carveout_blockstat(&c->carveout, + CARVEOUT_STAT_FREE_BLOCKS)); +} + +static ssize_t _nvmap_sysfs_show_heap_free_max(struct device *d, + struct device_attribute *attr, char *buf) +{ + struct nvmap_carveout_node *c = container_of(d, + struct nvmap_carveout_node, dev); + return sprintf(buf, "%lu\n", + _nvmap_carveout_blockstat(&c->carveout, + CARVEOUT_STAT_LARGEST_FREE)); +} + +static ssize_t _nvmap_sysfs_show_heap_total_count(struct device *d, + struct device_attribute *attr, char *buf) +{ + struct nvmap_carveout_node *c = container_of(d, + struct nvmap_carveout_node, dev); + return sprintf(buf, "%lu\n", + _nvmap_carveout_blockstat(&c->carveout, + CARVEOUT_STAT_NUM_BLOCKS)); +} + +static ssize_t _nvmap_sysfs_show_heap_total_max(struct device *d, + struct device_attribute *attr, char *buf) +{ + struct nvmap_carveout_node *c = container_of(d, + struct nvmap_carveout_node, dev); + return sprintf(buf, "%lu\n", + _nvmap_carveout_blockstat(&c->carveout, + CARVEOUT_STAT_LARGEST_BLOCK)); +} + +static ssize_t _nvmap_sysfs_show_heap_total_size(struct device *d, + struct device_attribute *attr, char *buf) +{ + struct nvmap_carveout_node *c = container_of(d, + struct nvmap_carveout_node, dev); + return sprintf(buf, "%lu\n", + _nvmap_carveout_blockstat(&c->carveout, + CARVEOUT_STAT_TOTAL_SIZE)); +} + +static int nvmap_split_carveout_heap(struct nvmap_carveout *co, size_t size, + const char *name, unsigned int new_bitmask); + +static ssize_t _nvmap_sysfs_set_heap_split(struct device *d, + struct device_attribute *attr, const char * buf, size_t count) +{ + struct nvmap_carveout_node *c = container_of(d, + struct nvmap_carveout_node, dev); + char *tmp, *local = kzalloc(count+1, GFP_KERNEL); + char *sizestr = NULL, *bitmaskstr = NULL, *name = NULL; + char **format[] = { &sizestr, &bitmaskstr, &name }; + char ***f_iter = format; + unsigned int i; + unsigned long size, bitmask; + int err; + + if (!local) { + pr_err("%s: unable to read string\n", __func__); + return -ENOMEM; + } + + memcpy(local, buf, count); + tmp = local; + for (i=0, **f_iter = local; i<count && + (f_iter - format)<ARRAY_SIZE(format)-1; i++) { + if (local[i]==',') { + local[i] = '\0'; + f_iter++; + **f_iter = &local[i+1]; + } + } + + if (!sizestr || !bitmaskstr || !name) { + pr_err("%s: format error\n", __func__); + kfree(tmp); + return -EINVAL; + } + + for (local=name; !isspace(*local); local++); + + if (local==name) { + pr_err("%s: invalid name %s\n", __func__, name); + kfree(tmp); + return -EINVAL; + } + + *local=0; + + size = memparse(sizestr, &sizestr); + if (!size) { + kfree(tmp); + return -EINVAL; + } + + if (strict_strtoul(bitmaskstr, 0, &bitmask)==-EINVAL) { + kfree(tmp); + return -EINVAL; + } + + err = nvmap_split_carveout_heap(&c->carveout, size, name, bitmask); + + if (err) pr_err("%s: failed to create split heap %s\n", __func__, name); + kfree(tmp); + return err ? err : count; +} + +static NVMAP_CARVEOUT_ATTR_RO(usage); +static NVMAP_CARVEOUT_ATTR_RO(name); +static NVMAP_CARVEOUT_ATTR_RO(base); +static NVMAP_CARVEOUT_ATTR_RO(free_size); +static NVMAP_CARVEOUT_ATTR_RO(free_count); +static NVMAP_CARVEOUT_ATTR_RO(free_max); +static NVMAP_CARVEOUT_ATTR_RO(total_size); +static NVMAP_CARVEOUT_ATTR_RO(total_count); +static NVMAP_CARVEOUT_ATTR_RO(total_max); +static NVMAP_CARVEOUT_ATTR_WO(split, (S_IWUSR | S_IWGRP)); + +static struct attribute *nvmap_heap_default_attrs[] = { + &nvmap_heap_attr_usage.attr, + &nvmap_heap_attr_name.attr, + &nvmap_heap_attr_split.attr, + &nvmap_heap_attr_base.attr, + &nvmap_heap_attr_total_size.attr, + &nvmap_heap_attr_free_size.attr, + &nvmap_heap_attr_total_count.attr, + &nvmap_heap_attr_free_count.attr, + &nvmap_heap_attr_total_max.attr, + &nvmap_heap_attr_free_max.attr, + NULL +}; + +static struct attribute_group nvmap_heap_defattr_group = { + .attrs = nvmap_heap_default_attrs +}; + +static struct device *__nvmap_heap_parent_dev(void); +#define _nvmap_heap_parent_dev __nvmap_heap_parent_dev() + +/* unpinned I/O VMM areas may be reclaimed by nvmap to make room for + * new surfaces. unpinned surfaces are stored in segregated linked-lists + * sorted in most-recently-unpinned order (i.e., head insertion, head + * removal */ +#ifdef CONFIG_DEVNVMAP_RECLAIM_UNPINNED_VM +static DEFINE_SPINLOCK(nvmap_mru_vma_lock); +static const size_t nvmap_mru_cutoff[] = { + 262144, 393216, 786432, 1048576, 1572864 +}; + +static struct list_head nvmap_mru_vma_lists[ARRAY_SIZE(nvmap_mru_cutoff)]; + +static inline struct list_head *_nvmap_list(size_t size) +{ + unsigned int i; + + for (i=0; i<ARRAY_SIZE(nvmap_mru_cutoff); i++) + if (size <= nvmap_mru_cutoff[i]) return &nvmap_mru_vma_lists[i]; + + return &nvmap_mru_vma_lists[ARRAY_SIZE(nvmap_mru_cutoff)-1]; +} +#endif + +static inline struct nvmap_handle *_nvmap_handle_get(struct nvmap_handle *h) +{ + if (unlikely(h->poison)) { + pr_err("%s: %s getting poisoned handle\n", __func__, + current->group_leader->comm); + return NULL; + } else if (unlikely(atomic_inc_return(&h->ref)<=1)) { + pr_err("%s: %s getting a freed handle\n", + __func__, current->group_leader->comm); + return NULL; + } + return h; +} + +static inline void _nvmap_handle_put(struct nvmap_handle *h) +{ + int cnt = atomic_dec_return(&h->ref); + + if (unlikely(cnt<0)) { + pr_err("%s: %s put to negative references\n", + __func__, current->comm); + dump_stack(); + } else if (!cnt) _nvmap_handle_free(h); +} + +static struct nvmap_handle *_nvmap_claim_preserved( + struct task_struct *new_owner, unsigned long key) +{ + struct rb_node *n; + struct nvmap_handle *b = NULL; + + if (!key) return NULL; + + spin_lock(&nvmap_handle_lock); + n = rb_first(&nvmap_handles); + + while (n) { + b = rb_entry(n, struct nvmap_handle, node); + if (b->alloc && !b->heap_pgalloc && b->carveout.key == key) { + b->carveout.key = 0; + b->owner = new_owner; + break; + } + b = NULL; + n = rb_next(n); + } + + spin_unlock(&nvmap_handle_lock); + return b; +} + +static struct nvmap_handle *_nvmap_validate_get(unsigned long handle, bool su) +{ + struct nvmap_handle *b = NULL; + +#ifdef CONFIG_DEVNVMAP_PARANOID + struct rb_node *n; + + spin_lock(&nvmap_handle_lock); + + n = nvmap_handles.rb_node; + + while (n) { + b = rb_entry(n, struct nvmap_handle, node); + if ((unsigned long)b == handle) { + if (su || b->global || b->owner==current->group_leader) + b = _nvmap_handle_get(b); + else + b = NULL; + spin_unlock(&nvmap_handle_lock); + return b; + } + if (handle > (unsigned long)b) n = n->rb_right; + else n = n->rb_left; + } + spin_unlock(&nvmap_handle_lock); + return NULL; +#else + if (!handle) return NULL; + b = _nvmap_handle_get((struct nvmap_handle *)handle); + return b; +#endif +} + +static inline void _nvmap_insert_mru_vma(struct nvmap_handle *h) +{ +#ifdef CONFIG_DEVNVMAP_RECLAIM_UNPINNED_VM + spin_lock(&nvmap_mru_vma_lock); + list_add(&h->pgalloc.mru_list, _nvmap_list(h->pgalloc.area->iovm_length)); + spin_unlock(&nvmap_mru_vma_lock); +#endif +} + +static void _nvmap_remove_mru_vma(struct nvmap_handle *h) +{ +#ifdef CONFIG_DEVNVMAP_RECLAIM_UNPINNED_VM + spin_lock(&nvmap_mru_vma_lock); + if (!list_empty(&h->pgalloc.mru_list)) + list_del(&h->pgalloc.mru_list); + spin_unlock(&nvmap_mru_vma_lock); + INIT_LIST_HEAD(&h->pgalloc.mru_list); +#endif +} + +static struct tegra_iovmm_area *_nvmap_get_vm(struct nvmap_handle *h) +{ +#ifndef CONFIG_DEVNVMAP_RECLAIM_UNPINNED_VM + BUG_ON(!h->pgalloc.area); + BUG_ON(h->size > h->pgalloc.area->iovm_length); + BUG_ON((h->size | h->pgalloc.area->iovm_length) & ~PAGE_MASK); + return h->pgalloc.area; +#else + struct list_head *mru; + struct nvmap_handle *evict = NULL; + struct tegra_iovmm_area *vm = NULL; + unsigned int i, idx; + + spin_lock(&nvmap_mru_vma_lock); + + if (h->pgalloc.area) { + BUG_ON(list_empty(&h->pgalloc.mru_list)); + list_del(&h->pgalloc.mru_list); + INIT_LIST_HEAD(&h->pgalloc.mru_list); + spin_unlock(&nvmap_mru_vma_lock); + return h->pgalloc.area; + } + + vm = tegra_iovmm_create_vm(nvmap_vm_client, NULL, h->size, + _nvmap_flag_to_pgprot(h->flags, pgprot_kernel)); + + if (vm) { + INIT_LIST_HEAD(&h->pgalloc.mru_list); + spin_unlock(&nvmap_mru_vma_lock); + return vm; + } + /* attempt to re-use the most recently unpinned IOVMM area in the + * same size bin as the current handle. If that fails, iteratively + * evict handles (starting from the current bin) until an allocation + * succeeds or no more areas can be evicted */ + + mru = _nvmap_list(h->size); + if (!list_empty(mru)) + evict = list_first_entry(mru, struct nvmap_handle, + pgalloc.mru_list); + if (evict && evict->pgalloc.area->iovm_length >= h->size) { + list_del(&evict->pgalloc.mru_list); + vm = evict->pgalloc.area; + evict->pgalloc.area = NULL; + INIT_LIST_HEAD(&evict->pgalloc.mru_list); + spin_unlock(&nvmap_mru_vma_lock); + return vm; + } + + idx = mru - nvmap_mru_vma_lists; + + for (i=0; i<ARRAY_SIZE(nvmap_mru_vma_lists) && !vm; i++, idx++) { + if (idx >= ARRAY_SIZE(nvmap_mru_vma_lists)) + idx -= ARRAY_SIZE(nvmap_mru_vma_lists); + mru = &nvmap_mru_vma_lists[idx]; + while (!list_empty(mru) && !vm) { + evict = list_first_entry(mru, struct nvmap_handle, + pgalloc.mru_list); + + BUG_ON(atomic_read(&evict->pin)!=0); + BUG_ON(!evict->pgalloc.area); + list_del(&evict->pgalloc.mru_list); + INIT_LIST_HEAD(&evict->pgalloc.mru_list); + tegra_iovmm_free_vm(evict->pgalloc.area); + evict->pgalloc.area = NULL; + vm = tegra_iovmm_create_vm(nvmap_vm_client, + NULL, h->size, + _nvmap_flag_to_pgprot(h->flags, pgprot_kernel)); + } + } + + spin_unlock(&nvmap_mru_vma_lock); + return vm; +#endif +} + +static int _nvmap_do_cache_maint(struct nvmap_handle *h, + unsigned long start, unsigned long end, unsigned long op, bool get); + +void _nvmap_handle_free(struct nvmap_handle *h) +{ + int e; + spin_lock(&nvmap_handle_lock); + + /* if 2 contexts call _get and _put simultaneously, the reference + * count may drop to 0 and then increase to 1 before the handle + * can be freed. */ + if (atomic_read(&h->ref)>0) { + spin_unlock(&nvmap_handle_lock); + return; + } + BUG_ON(atomic_read(&h->ref)<0); + BUG_ON(atomic_read(&h->pin)!=0); + + rb_erase(&h->node, &nvmap_handles); + + spin_unlock(&nvmap_handle_lock); + + if (h->owner) put_task_struct(h->owner); + + /* remove when NvRmMemMgr compatibility is eliminated */ + if (h->kern_map) { + BUG_ON(!h->alloc); + if (h->heap_pgalloc) + vm_unmap_ram(h->kern_map, h->size>>PAGE_SHIFT); + else { + unsigned long addr = (unsigned long)h->kern_map; + addr &= ~PAGE_MASK; + iounmap((void *)addr); + } + } + + /* ensure that no stale data remains in the cache for this handle */ + e = _nvmap_do_cache_maint(h, 0, h->size, NVMEM_CACHE_OP_WB_INV, false); + + if (h->alloc && !h->heap_pgalloc) + nvmap_carveout_free(h->carveout.co_heap, h->carveout.block_idx); + else if (h->alloc) { + unsigned int i; + BUG_ON(h->size & ~PAGE_MASK); + BUG_ON(!h->pgalloc.pages); + _nvmap_remove_mru_vma(h); + if (h->pgalloc.area) tegra_iovmm_free_vm(h->pgalloc.area); + for (i=0; i<h->size>>PAGE_SHIFT; i++) { + ClearPageReserved(h->pgalloc.pages[i]); + __free_page(h->pgalloc.pages[i]); + } + if ((h->size>>PAGE_SHIFT)*sizeof(struct page*)>=PAGE_SIZE) + vfree(h->pgalloc.pages); + else + kfree(h->pgalloc.pages); + } + h->poison = 0xa5a5a5a5; + kfree(h); +} + +#define nvmap_gfp (GFP_KERNEL | __GFP_HIGHMEM | __GFP_NOWARN) + +static int _nvmap_alloc_do_coalloc(struct nvmap_handle *h, + struct nvmap_carveout *co, size_t align) +{ + int idx; + + idx = nvmap_carveout_alloc(co, align, h->size); + if (idx != -1) { + h->alloc = true; + h->heap_pgalloc = false; + h->carveout.co_heap = co; + h->carveout.block_idx = idx; + spin_lock(&co->lock); + h->carveout.base = co->blocks[idx].base; + spin_unlock(&co->lock); + } + + return (idx==-1) ? -ENOMEM : 0; +} + +/* map the backing pages for a heap_pgalloc handle into its IOVMM area */ +static void _nvmap_handle_iovmm_map(struct nvmap_handle *h) +{ + tegra_iovmm_addr_t va; + unsigned long i; + + BUG_ON(!h->heap_pgalloc || !h->pgalloc.area); + BUG_ON(h->size & ~PAGE_MASK); + WARN_ON(!h->pgalloc.dirty); + + for (va = h->pgalloc.area->iovm_start, i=0; + va < (h->pgalloc.area->iovm_start + h->size); + i++, va+=PAGE_SIZE) { + BUG_ON(!pfn_valid(page_to_pfn(h->pgalloc.pages[i]))); + tegra_iovmm_vm_insert_pfn(h->pgalloc.area, va, + page_to_pfn(h->pgalloc.pages[i])); + } + h->pgalloc.dirty = false; +} + +static int _nvmap_alloc_do_pgalloc(struct nvmap_handle *h, + bool contiguous, bool secure) +{ + unsigned int i = 0, cnt = (h->size + PAGE_SIZE - 1) >> PAGE_SHIFT; + struct page **pages; + + if (cnt*sizeof(*pages)>=PAGE_SIZE) + pages = vmalloc(cnt*sizeof(*pages)); + else + pages = kzalloc(sizeof(*pages)*cnt, GFP_KERNEL); + + if (!pages) return -ENOMEM; + + if (cnt==1 && !secure) contiguous = true; + + /* secure surfaces should only be allocated in discontiguous (IOVM- + * managed) space, so that the mapping can be zapped after it is + * unpinned */ + WARN_ON(secure && contiguous); + + if (contiguous) { + size_t order = get_order(h->size); + struct page *compound_page; + compound_page = alloc_pages(nvmap_gfp, order); + if (!compound_page) goto fail; + split_page(compound_page, order); + for (i=0; i<cnt; i++) + pages[i] = nth_page(compound_page, i); + for (; i<(1<<order); i++) + __free_page(nth_page(compound_page, i)); + } else { + for (i=0; i<cnt; i++) { + pages[i] = alloc_page(nvmap_gfp); + if (!pages[i]) { + pr_err("failed to allocate %u pages after %u entries\n", + cnt, i); + goto fail; + } + } + } + + h->pgalloc.area = NULL; +#ifndef CONFIG_DEVNVMAP_RECLAIM_UNPINNED_VM + if (!contiguous) { + h->pgalloc.area = tegra_iovmm_create_vm(nvmap_vm_client, + NULL, cnt << PAGE_SHIFT, + _nvmap_flag_to_pgprot(h->flags, pgprot_kernel)); + if (!h->pgalloc.area) goto fail; + h->pgalloc.dirty = true; + } +#endif + + for (i=0; i<cnt; i++) { + void *km; + SetPageReserved(pages[i]); + km = kmap(pages[i]); + if (km) __cpuc_flush_dcache_page(km); + outer_flush_range(page_to_phys(pages[i]), + page_to_phys(pages[i])+PAGE_SIZE); + kunmap(pages[i]); + } + + h->size = cnt<<PAGE_SHIFT; + h->pgalloc.pages = pages; + h->heap_pgalloc = true; + h->pgalloc.contig = contiguous; + INIT_LIST_HEAD(&h->pgalloc.mru_list); + h->alloc = true; + return 0; + +fail: + while (i--) __free_page(pages[i]); + if (pages && (cnt*sizeof(*pages)>=PAGE_SIZE)) vfree(pages); + else if (pages) kfree(pages); + return -ENOMEM; +} + +static struct nvmap_handle *_nvmap_handle_create( + struct task_struct *owner, size_t size) +{ + struct nvmap_handle *h = kzalloc(sizeof(*h), GFP_KERNEL); + struct nvmap_handle *b; + struct rb_node **p; + struct rb_node *parent = NULL; + + if (!h) return NULL; + atomic_set(&h->ref, 1); + atomic_set(&h->pin, 0); + h->owner = owner; + h->size = h->orig_size = size; + h->flags = NVMEM_HANDLE_WRITE_COMBINE; + + spin_lock(&nvmap_handle_lock); + p = &nvmap_handles.rb_node; + while (*p) { + parent = *p; + b = rb_entry(parent, struct nvmap_handle, node); + if (h > b) p = &parent->rb_right; + else p = &parent->rb_left; + } + rb_link_node(&h->node, parent, p); + rb_insert_color(&h->node, &nvmap_handles); + spin_unlock(&nvmap_handle_lock); + if (owner) get_task_struct(owner); + return h; +} + +/* nvmap pte manager */ + +static void _nvmap_set_pte_at(unsigned long addr, unsigned long pfn, + pgprot_t prot) +{ + u32 off; + int idx; + pte_t *pte; + + BUG_ON(!addr); + idx = NVMAP_PTE_INDEX(addr); + off = NVMAP_PTE_OFFSET(addr) & (PTRS_PER_PTE-1); + + pte = nvmap_pte[idx] + off; + set_pte_ext(pte, pfn_pte(pfn, prot), 0); + flush_tlb_kernel_page(addr); +} + static int _nvmap_map_pte(unsigned long pfn, pgprot_t prot, void **vaddr) { static unsigned int last_bit = 0; unsigned long bit; - pte_t *pte; unsigned long addr; unsigned long flags; - u32 off; - int idx; spin_lock_irqsave(&nvmap_ptelock, flags); @@ -141,12 +1227,7 @@ static int _nvmap_map_pte(unsigned long pfn, pgprot_t prot, void **vaddr) addr = NVMAP_BASE + bit*PAGE_SIZE; - idx = NVMAP_PTE_INDEX(addr); - off = NVMAP_PTE_OFFSET(addr) & (PTRS_PER_PTE-1); - - pte = nvmap_pte[idx] + off; - set_pte_ext(pte, pfn_pte(pfn, prot), 0); - flush_tlb_kernel_page(addr); + _nvmap_set_pte_at(addr, pfn, prot); *vaddr = (void *)addr; return 0; } @@ -195,33 +1276,20 @@ static void nvmap_vma_close(struct vm_area_struct *vma) { struct nvmap_vma_priv *priv = vma->vm_private_data; if (priv && !atomic_dec_return(&priv->ref)) { - NvRmMemHandle hmem = priv->hmem; - if (hmem) { - if (hmem->coherency==NvOsMemAttribute_WriteBack) - dmac_clean_all(); - NvRmMemHandleFree(priv->hmem); - } + if (priv->h) _nvmap_handle_put(priv->h); kfree(priv); } vma->vm_private_data = NULL; } -extern struct page *NvOsPageGetPage(NvOsPageAllocHandle, size_t); -#define nvmap_range(x,y) (x), (x)+(y) - -#define is_same_page(a, b) \ - ((unsigned long)(a)>>PAGE_SHIFT == (unsigned long)(b)>>PAGE_SHIFT) - static int nvmap_vma_fault(struct vm_area_struct *vma, struct vm_fault *vmf) { struct nvmap_vma_priv *priv; - struct page *page; - unsigned long pfn; unsigned long offs; offs = (unsigned long)(vmf->virtual_address - vma->vm_start); priv = vma->vm_private_data; - if (!priv || !priv->hmem) + if (!priv || !priv->h || !priv->h->alloc) return VM_FAULT_SIGBUS; offs += priv->offs; @@ -229,27 +1297,22 @@ static int nvmap_vma_fault(struct vm_area_struct *vma, struct vm_fault *vmf) * offset from the original VMA */ offs += (vma->vm_pgoff << PAGE_SHIFT); - if (offs >= priv->hmem->size) + if (offs >= priv->h->size) return VM_FAULT_SIGBUS; - switch (priv->hmem->heap) { - case NvRmHeap_ExternalCarveOut: - case NvRmHeap_IRam: - pfn = ((priv->hmem->PhysicalAddress+offs) >> PAGE_SHIFT); + if (!priv->h->heap_pgalloc) { + unsigned long pfn; + BUG_ON(priv->h->carveout.base & ~PAGE_MASK); + pfn = ((priv->h->carveout.base + offs) >> PAGE_SHIFT); vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn); return VM_FAULT_NOPAGE; - - case NvRmHeap_GART: - case NvRmHeap_External: - if (!priv->hmem->hPageHandle) - return VM_FAULT_SIGBUS; - page = NvOsPageGetPage(priv->hmem->hPageHandle, offs); + } else { + struct page *page; + offs >>= PAGE_SHIFT; + page = priv->h->pgalloc.pages[offs]; if (page) get_page(page); vmf->page = page; return (page) ? 0 : VM_FAULT_SIGBUS; - - default: - return VM_FAULT_SIGBUS; } } @@ -274,17 +1337,44 @@ static long nvmap_ioctl(struct file *filp, return -EFAULT; switch (cmd) { + case NVMEM_IOC_CREATE: + case NVMEM_IOC_CLAIM: + case NVMEM_IOC_FROM_ID: + err = nvmap_ioctl_create(filp, cmd, uarg); + break; + + case NVMEM_IOC_GET_ID: + err = nvmap_ioctl_getid(filp, uarg); + break; + + case NVMEM_IOC_PARAM: + err = nvmap_ioctl_get_param(filp, uarg); + break; + + case NVMEM_IOC_UNPIN_MULT: + case NVMEM_IOC_PIN_MULT: + err = nvmap_ioctl_pinop(filp, cmd==NVMEM_IOC_PIN_MULT, uarg); + break; + + case NVMEM_IOC_ALLOC: + err = nvmap_ioctl_alloc(filp, uarg); + break; + + case NVMEM_IOC_FREE: + err = nvmap_ioctl_free(filp, arg); + break; + case NVMEM_IOC_MMAP: err = nvmap_map_into_caller_ptr(filp, uarg); break; case NVMEM_IOC_WRITE: case NVMEM_IOC_READ: - err = nvmap_rw_handle(filp, cmd==NVMEM_IOC_READ, uarg); + err = nvmap_ioctl_rw_handle(filp, cmd==NVMEM_IOC_READ, uarg); break; case NVMEM_IOC_CACHE: - err = nvmap_cache_maint(filp, uarg); + err = nvmap_ioctl_cache_maint(filp, uarg); break; default: @@ -293,46 +1383,746 @@ static long nvmap_ioctl(struct file *filp, return err; } -static int nvmap_release(struct inode *inode, struct file *file) +/* must be called with the ref_lock held - given a user-space handle ID + * ref, validate that the handle_ref object may be used by the caller */ +struct nvmap_handle_ref *_nvmap_ref_lookup_locked( + struct nvmap_file_priv *priv, unsigned long ref) { + struct rb_node *n = priv->handle_refs.rb_node; + + while (n) { + struct nvmap_handle_ref *r; + r = rb_entry(n, struct nvmap_handle_ref, node); + if ((unsigned long)r->h == ref) return r; + else if (ref > (unsigned long)r->h) n = n->rb_right; + else n = n->rb_left; + } + + return NULL; +} + +/* must be called inside nvmap_pin_lock, to ensure that an entire stream + * of pins will complete without competition from a second stream. returns + * 0 if the pin was successful, -ENOMEM on failure */ +static int _nvmap_handle_pin_locked(struct nvmap_handle *h) +{ + struct tegra_iovmm_area *area; + BUG_ON(!h->alloc); + + h = _nvmap_handle_get(h); + if (!h) return -ENOMEM; + + if (atomic_inc_return(&h->pin)==1) { + if (h->heap_pgalloc && !h->pgalloc.contig) { + area = _nvmap_get_vm(h); + if (!area) { + /* no race here, inside the pin mutex */ + atomic_dec(&h->pin); + _nvmap_handle_put(h); + return -ENOMEM; + } + if (area != h->pgalloc.area) + h->pgalloc.dirty = true; + h->pgalloc.area = area; + } + } + return 0; +} + +/* doesn't need to be called inside nvmap_pin_lock, since this will only + * expand the available VM area */ +static int _nvmap_handle_unpin(struct nvmap_handle *h) +{ + int ret = 0; + + if (atomic_read(&h->pin)==0) { + pr_err("%s: %s attempting to unpin an unpinned handle\n", + __func__, current->comm); + dump_stack(); + return 0; + } + + BUG_ON(!h->alloc || atomic_read(&h->pin)==0); + if (!atomic_dec_return(&h->pin)) { + if (h->heap_pgalloc && h->pgalloc.area) { + /* if a secure handle is clean (i.e., mapped into + * IOVMM, it needs to be zapped on unpin. */ + if (h->secure && !h->pgalloc.dirty) { + tegra_iovmm_zap_vm(h->pgalloc.area); + h->pgalloc.dirty = true; + } + _nvmap_insert_mru_vma(h); + ret=1; + } + } + _nvmap_handle_put(h); + return ret; +} + +/* pin a list of handles, mapping IOVMM areas if needed. may sleep, if + * a handle's IOVMM area has been reclaimed and insufficient IOVMM space + * is available to complete the list pin. no intervening pin operations + * will interrupt this, and no validation is performed on the handles + * that are provided. */ +static int _nvmap_handle_pin_fast(unsigned int nr, struct nvmap_handle **h) +{ + unsigned int i; + int ret = 0; + + mutex_lock(&nvmap_pin_lock); + for (i=0; i<nr && !ret; i++) { + ret = wait_event_interruptible(nvmap_pin_wait, + !_nvmap_handle_pin_locked(h[i])); + } + mutex_unlock(&nvmap_pin_lock); + + if (ret) { + int do_wake = 0; + while (i--) do_wake |= _nvmap_handle_unpin(h[i]); + if (do_wake) wake_up(&nvmap_pin_wait); + return -EINTR; + } else { + for (i=0; i<nr; i++) + if (h[i]->heap_pgalloc && h[i]->pgalloc.dirty) + _nvmap_handle_iovmm_map(h[i]); + } + + return 0; +} + +static int _nvmap_do_global_unpin(unsigned long ref) +{ + struct nvmap_handle *h; + int w; + + h = _nvmap_validate_get(ref, true); + if (unlikely(!h)) { + pr_err("%s: %s attempting to unpin non-existent handle\n", + __func__, current->group_leader->comm); + return 0; + } + + pr_err("%s: %s unpinning %s's %uB %s handle without local context\n", + __func__, current->group_leader->comm, + (h->owner) ? h->owner->comm : "kernel", h->orig_size, + (h->heap_pgalloc && !h->pgalloc.contig) ? "iovmm" : + (h->heap_pgalloc) ? "sysmem" : "carveout"); + + w = _nvmap_handle_unpin(h); + _nvmap_handle_put(h); + return w; +} + +static void _nvmap_do_unpin(struct nvmap_file_priv *priv, + unsigned int nr, unsigned long *refs) +{ + struct nvmap_handle_ref *r; + unsigned int i; + int do_wake = 0; + + spin_lock(&priv->ref_lock); + for (i=0; i<nr; i++) { + if (!refs[i]) continue; + r = _nvmap_ref_lookup_locked(priv, refs[i]); + if (unlikely(!r)) { + if (priv->su) + do_wake |= _nvmap_do_global_unpin(refs[i]); + else + pr_err("%s: %s unpinning invalid handle\n", + __func__, current->comm); + } else if (unlikely(!atomic_add_unless(&r->pin, -1, 0))) + pr_err("%s: %s unpinning unpinned handle\n", + __func__, current->comm); + else + do_wake |= _nvmap_handle_unpin(r->h); + } + spin_unlock(&priv->ref_lock); + if (do_wake) wake_up(&nvmap_pin_wait); +} + +/* pins a list of handle_ref objects; same conditions apply as to + * _nvmap_handle_pin, but also bumps the pin count of each handle_ref. */ +static int _nvmap_do_pin(struct nvmap_file_priv *priv, + unsigned int nr, unsigned long *refs) +{ + int ret = 0; + unsigned int i; + struct nvmap_handle **h = (struct nvmap_handle **)refs; + struct nvmap_handle_ref *r; + + /* to optimize for the common case (client provided valid handle + * references and the pin succeeds), increment the handle_ref pin + * count during validation. in error cases, the tree will need to + * be re-walked, since the handle_ref is discarded so that an + * allocation isn't required. if a handle_ref is not found, + * locally validate that the caller has permission to pin the handle; + * handle_refs are not created in this case, so it is possible that + * if the caller crashes after pinning a global handle, the handle + * will be permanently leaked. */ + spin_lock(&priv->ref_lock); + for (i=0; i<nr && !ret; i++) { + r = _nvmap_ref_lookup_locked(priv, refs[i]); + if (!r && (!(priv->su || h[i]->global || + current->group_leader == h[i]->owner))) + ret = -EPERM; + else if (r) atomic_inc(&r->pin); + else { + pr_err("%s: %s pinning %s's %uB handle without " + "local context\n", __func__, + current->group_leader->comm, + h[i]->owner->comm, h[i]->orig_size); + } + } + + while (ret && i--) { + r = _nvmap_ref_lookup_locked(priv, refs[i]); + if (r) atomic_dec(&r->pin); + } + spin_unlock(&priv->ref_lock); + + if (ret) return ret; + + mutex_lock(&nvmap_pin_lock); + for (i=0; i<nr && !ret; i++) { + ret = wait_event_interruptible(nvmap_pin_wait, + !_nvmap_handle_pin_locked(h[i])); + } + mutex_unlock(&nvmap_pin_lock); + + if (ret) { + int do_wake = 0; + spin_lock(&priv->ref_lock); + while (i--) { + r = _nvmap_ref_lookup_locked(priv, refs[i]); + do_wake |= _nvmap_handle_unpin(r->h); + if (r) atomic_dec(&r->pin); + } + spin_unlock(&priv->ref_lock); + if (do_wake) wake_up(&nvmap_pin_wait); + return -EINTR; + } else { + for (i=0; i<nr; i++) { + if (h[i]->heap_pgalloc && h[i]->pgalloc.dirty) + _nvmap_handle_iovmm_map(h[i]); + } + } + + return 0; +} + +static int nvmap_ioctl_pinop(struct file *filp, + bool is_pin, void __user *arg) +{ + struct nvmem_pin_handle op; + struct nvmap_handle *h; + unsigned long on_stack[16]; + unsigned long *refs; + unsigned long __user *output; + unsigned int i; + int err; + + err = copy_from_user(&op, arg, sizeof(op)); + if (err) return err; + + if (!op.count) return -EINVAL; + + if (op.count > 1) { + size_t bytes = op.count * sizeof(unsigned long *); + if (!access_ok(VERIFY_READ, (void *)op.handles, bytes)) + return -EPERM; + if (is_pin && !access_ok(VERIFY_WRITE, (void *)op.addr, bytes)) + return -EPERM; + + if (op.count <= ARRAY_SIZE(on_stack)) refs = on_stack; + else refs = kzalloc(bytes, GFP_KERNEL); + + if (!refs) return -ENOMEM; + err = copy_from_user(refs, (void*)op.handles, bytes); + if (err) goto out; + } else { + refs = on_stack; + on_stack[0] = (unsigned long)op.handles; + } + + if (is_pin) + err = _nvmap_do_pin(filp->private_data, op.count, refs); + else + _nvmap_do_unpin(filp->private_data, op.count, refs); + + /* skip the output stage on unpin */ + if (err || !is_pin) goto out; + + /* it is guaranteed that if _nvmap_do_pin returns 0 that + * all of the handle_ref objects are valid, so dereferencing directly + * here is safe */ + if (op.count > 1) + output = (unsigned long __user *)op.addr; + else { + struct nvmem_pin_handle __user *tmp = arg; + output = (unsigned long __user *)&(tmp->addr); + } + + for (i=0; i<op.count; i++) { + unsigned long addr; + h = (struct nvmap_handle *)refs[i]; + if (h->heap_pgalloc && h->pgalloc.contig) + addr = page_to_phys(h->pgalloc.pages[0]); + else if (h->heap_pgalloc) + addr = h->pgalloc.area->iovm_start; + else + addr = h->carveout.base; + + __put_user(addr, &output[i]); + } + +out: + if (refs != on_stack) kfree(refs); + return err; +} + +static int nvmap_release(struct inode *inode, struct file *filp) +{ + struct nvmap_file_priv *priv = filp->private_data; + struct rb_node *n; + struct nvmap_handle_ref *r; + int refs; + int do_wake = 0; + int pins; + + if (!priv) return 0; + + while ((n = rb_first(&priv->handle_refs))) { + r = rb_entry(n, struct nvmap_handle_ref, node); + rb_erase(&r->node, &priv->handle_refs); + pins = atomic_read(&r->pin); + atomic_set(&r->pin, 0); + while (pins--) do_wake |= _nvmap_handle_unpin(r->h); + refs = atomic_read(&r->refs); + if (r->h->alloc && r->h->heap_pgalloc && !r->h->pgalloc.contig) + atomic_sub(r->h->size, &priv->iovm_commit); + while (refs--) _nvmap_handle_put(r->h); + kfree(r); + } + if (do_wake) wake_up(&nvmap_pin_wait); + kfree(priv); return 0; } static int nvmap_open(struct inode *inode, struct file *filp) { /* eliminate read, write and llseek support on this node */ + struct nvmap_file_priv *priv; int ret; + /* nvmap doesn't track total number of pinned references, so its + * IOVMM client is always locked. */ + if (!nvmap_vm_client) { + mutex_lock(&nvmap_pin_lock); + if (!nvmap_vm_client) { + nvmap_vm_client = tegra_iovmm_alloc_client("gpu", NULL); + if (nvmap_vm_client) + tegra_iovmm_client_lock(nvmap_vm_client); + } + mutex_unlock(&nvmap_pin_lock); + } + ret = nonseekable_open(inode, filp); if (unlikely(ret)) return ret; + priv = kzalloc(sizeof(*priv), GFP_KERNEL); + if (!priv) return -ENOMEM; + priv->handle_refs = RB_ROOT; + priv->su = (filp->f_op == &knvmap_fops); + + atomic_set(&priv->iovm_commit, 0); + + if (nvmap_vm_client) + priv->iovm_limit = tegra_iovmm_get_vm_size(nvmap_vm_client); +#ifdef CONFIG_DEVNVMAP_RECLAIM_UNPINNED_VM + /* to prevent fragmentation-caused deadlocks, derate the size of + * the IOVM space to 75% */ + priv->iovm_limit >>= 2; + priv->iovm_limit *= 3; +#endif + + spin_lock_init(&priv->ref_lock); + filp->f_mapping->backing_dev_info = &nvmap_bdi; - filp->private_data = NULL; + filp->private_data = priv; + return 0; +} + +static int nvmap_ioctl_getid(struct file *filp, void __user *arg) +{ + struct nvmem_create_handle op; + struct nvmap_handle *h = NULL; + int err; + + err = copy_from_user(&op, arg, sizeof(op)); + if (err) return err; + + if (!op.handle) return -EINVAL; + + h = _nvmap_validate_get((unsigned long)op.handle, + filp->f_op==&knvmap_fops); + + if (h) { + op.id = (__u32)h; + /* when the owner of a handle gets its ID, this is treated + * as a granting of the handle for use by other processes. + * however, the super-user is not capable of promoting a + * handle to global status if it was created in another + * process. */ + if (current->group_leader == h->owner) h->global = true; + + /* getid is not supposed to result in a ref count increase */ + _nvmap_handle_put(h); + + return copy_to_user(arg, &op, sizeof(op)); + } + return -EPERM; +} + +/* attempts to allocate from either contiguous system memory or IOVMM space */ +static int _nvmap_do_page_alloc(struct nvmap_file_priv *priv, + struct nvmap_handle *h, unsigned int heap_mask, + size_t align, bool secure) +{ + int ret = -ENOMEM; + size_t page_size = (h->size + PAGE_SIZE - 1) & ~(PAGE_SIZE-1); +#ifdef IOVMM_FIRST + unsigned int fallback[] = { NVMEM_HEAP_IOVMM, NVMEM_HEAP_SYSMEM, 0 }; +#else + unsigned int fallback[] = { NVMEM_HEAP_SYSMEM, NVMEM_HEAP_IOVMM, 0 }; +#endif + unsigned int *m = fallback; + + /* secure allocations must not be performed from sysmem */ + if (secure) heap_mask &= ~NVMEM_HEAP_SYSMEM; + + if (align > PAGE_SIZE) return -EINVAL; + + + while (*m && ret) { + if (heap_mask & NVMEM_HEAP_SYSMEM & *m) + ret = _nvmap_alloc_do_pgalloc(h, true, secure); + + else if (heap_mask & NVMEM_HEAP_IOVMM & *m) { + /* increment the committed IOVM space prior to + * allocation, to avoid race conditions with other + * threads simultaneously allocating. this is + * conservative, but guaranteed to work */ + + int oc; + oc = atomic_add_return(page_size, &priv->iovm_commit); + + if (oc <= priv->iovm_limit) + ret = _nvmap_alloc_do_pgalloc(h, false, secure); + else + ret = -ENOMEM; + /* on failure, or when do_pgalloc promotes a non- + * contiguous request into a contiguous request, + * release the commited iovm space */ + if (ret || h->pgalloc.contig) + atomic_sub(page_size, &priv->iovm_commit); + } + m++; + } + return ret; +} + +/* attempts to allocate from the carveout heaps */ +static int _nvmap_do_carveout_alloc(struct nvmap_handle *h, + unsigned int heap_mask, size_t align) +{ + int ret = -ENOMEM; + struct nvmap_carveout_node *n; + + down_read(&nvmap_context.list_sem); + list_for_each_entry(n, &nvmap_context.heaps, heap_list) { + if (heap_mask & n->heap_bit) + ret = _nvmap_alloc_do_coalloc(h, &n->carveout, align); + if (!ret) break; + } + up_read(&nvmap_context.list_sem); + return ret; +} + +static int _nvmap_do_alloc(struct nvmap_file_priv *priv, + unsigned long href, unsigned int heap_mask, size_t align, + unsigned int flags, bool secure, bool carveout_first) +{ + int ret = -ENOMEM; + struct nvmap_handle_ref *r; + struct nvmap_handle *h; + + if (!href) return -EINVAL; + + spin_lock(&priv->ref_lock); + r = _nvmap_ref_lookup_locked(priv, href); + spin_unlock(&priv->ref_lock); + + if (!r) return -EPERM; + + h = r->h; + if (h->alloc) return 0; + h->flags = flags; + + align = max_t(size_t, align, L1_CACHE_BYTES); + + if (secure) heap_mask &= ~NVMEM_HEAP_CARVEOUT_MASK; + + if (carveout_first || (heap_mask & NVMEM_HEAP_CARVEOUT_IRAM)) { + ret = _nvmap_do_carveout_alloc(h, heap_mask, align); + if (ret) ret = _nvmap_do_page_alloc(priv, h, + heap_mask, align, secure); + } else { + ret = _nvmap_do_page_alloc(priv, h, heap_mask, align, secure); + if (ret) ret = _nvmap_do_carveout_alloc(h, heap_mask, align); + } + + BUG_ON((!ret && !h->alloc) || (ret && h->alloc)); + return ret; +} + +static int nvmap_ioctl_alloc(struct file *filp, void __user *arg) +{ + struct nvmem_alloc_handle op; + struct nvmap_file_priv *priv = filp->private_data; + bool secure = false; +#ifdef IOVMM_FIRST + bool carveout_first = false; +#else + bool carveout_first = true; +#endif + int err; + + err = copy_from_user(&op, arg, sizeof(op)); + if (err) return err; + + if (op.align & (op.align-1)) return -EINVAL; + + /* user-space handles are aligned to page boundaries, to prevent + * data leakage. */ + op.align = max_t(size_t, op.align, PAGE_SIZE); + + if (op.flags & NVMEM_HANDLE_SECURE) secure = true; + + /* TODO: implement a way to specify carveout-first vs + * carveout-second */ + return _nvmap_do_alloc(priv, op.handle, op.heap_mask, + op.align, (op.flags & 0x3), secure, carveout_first); +} + +static int _nvmap_do_free(struct nvmap_file_priv *priv, unsigned long href) +{ + struct nvmap_handle_ref *r; + struct nvmap_handle *h; + int do_wake = 0; + if (!href) return 0; + + spin_lock(&priv->ref_lock); + r = _nvmap_ref_lookup_locked(priv, href); + + if (!r) { + spin_unlock(&priv->ref_lock); + pr_err("%s attempting to free unrealized handle\n", + current->group_leader->comm); + return -EPERM; + } + + h = r->h; + + if (!atomic_dec_return(&r->refs)) { + int pins = atomic_read(&r->pin); + rb_erase(&r->node, &priv->handle_refs); + spin_unlock(&priv->ref_lock); + if (pins) pr_err("%s: %s freeing %s's pinned %s %s %uB handle\n", + __func__, current->comm, + (r->h->owner) ? r->h->owner->comm : "kernel", + (r->h->global) ? "global" : "private", + (r->h->alloc && r->h->heap_pgalloc)?"page-alloc" : + (r->h->alloc) ? "carveout" : "unallocated", + r->h->orig_size); + while (pins--) do_wake |= _nvmap_handle_unpin(r->h); + kfree(r); + if (h->alloc && h->heap_pgalloc && !h->pgalloc.contig) + atomic_sub(h->size, &priv->iovm_commit); + if (do_wake) wake_up(&nvmap_pin_wait); + } else + spin_unlock(&priv->ref_lock); + + BUG_ON(!atomic_read(&h->ref)); + _nvmap_handle_put(h); return 0; } +static int nvmap_ioctl_free(struct file *filp, unsigned long arg) +{ + return _nvmap_do_free(filp->private_data, arg); +} + +/* given a size, pre-existing handle ID, or a preserved handle key, create + * a handle and a reference to the handle in the per-context data */ +static int _nvmap_do_create(struct nvmap_file_priv *priv, + unsigned int cmd, unsigned long key, bool su, + struct nvmap_handle_ref **ref) +{ + struct nvmap_handle_ref *r = NULL; + struct nvmap_handle *h = NULL; + struct rb_node **p, *parent = NULL; + + if (cmd == NVMEM_IOC_FROM_ID) { + /* only ugly corner case to handle with from ID: + * + * normally, if the handle that is being duplicated is IOVMM- + * backed, the handle should fail to duplicate if duping it + * would over-commit IOVMM space. however, if the handle is + * already duplicated in the client process (or the client + * is duplicating a handle it created originally), IOVMM space + * should not be doubly-reserved. + */ + h = _nvmap_validate_get(key, priv->su); + + if (!h) { + pr_err("%s: %s duplicate handle failed\n", __func__, + current->group_leader->comm); + return -EPERM; + } + + if (!h->alloc) { + pr_err("%s: attempting to clone unallocated " + "handle\n", __func__); + _nvmap_handle_put(h); + h = NULL; + return -EINVAL; + } + + spin_lock(&priv->ref_lock); + r = _nvmap_ref_lookup_locked(priv, (unsigned long)h); + spin_unlock(&priv->ref_lock); + if (r) { + atomic_inc(&r->refs); + *ref = r; + return 0; + } + + /* verify that adding this handle to the process' access list + * won't exceed the IOVM limit */ + if (h->heap_pgalloc && !h->pgalloc.contig) { + int oc = atomic_add_return(h->size, &priv->iovm_commit); + if (oc > priv->iovm_limit) { + atomic_sub(h->size, &priv->iovm_commit); + _nvmap_handle_put(h); + h = NULL; + pr_err("%s: %s duplicating handle would " + "over-commit iovmm space (%d / %dB\n", + __func__, current->group_leader->comm, + oc, priv->iovm_limit); + return -ENOMEM; + } + } + } else if (cmd == NVMEM_IOC_CREATE) { + h = _nvmap_handle_create(current->group_leader, key); + if (!h) return -ENOMEM; + } else { + h = _nvmap_claim_preserved(current->group_leader, key); + if (!h) return -EINVAL; + } + + BUG_ON(!h); + + /* if the client does something strange, like calling CreateFromId + * when it was the original creator, avoid creating two handle refs + * for the same handle */ + spin_lock(&priv->ref_lock); + r = kzalloc(sizeof(*r), GFP_KERNEL); + if (!r) { + spin_unlock(&priv->ref_lock); + if (h) _nvmap_handle_put(h); + return -ENOMEM; + } + + atomic_set(&r->refs, 1); + r->h = h; + atomic_set(&r->pin, 0); + + p = &priv->handle_refs.rb_node; + while (*p) { + struct nvmap_handle_ref *l; + parent = *p; + l = rb_entry(parent, struct nvmap_handle_ref, node); + if (r->h > l->h) p = &parent->rb_right; + else p = &parent->rb_left; + } + rb_link_node(&r->node, parent, p); + rb_insert_color(&r->node, &priv->handle_refs); + + spin_unlock(&priv->ref_lock); + *ref = r; + return 0; +} + +static int nvmap_ioctl_create(struct file *filp, + unsigned int cmd, void __user *arg) +{ + struct nvmem_create_handle op; + struct nvmap_handle_ref *r = NULL; + struct nvmap_file_priv *priv = filp->private_data; + unsigned long key; + int err = 0; + + err = copy_from_user(&op, arg, sizeof(op)); + if (err) return err; + + if (!priv) return -ENODEV; + + /* user-space-created handles are expanded to be page-aligned, + * so that mmap() will not accidentally leak a different allocation */ + if (cmd==NVMEM_IOC_CREATE) + key = (op.size + PAGE_SIZE - 1) & ~(PAGE_SIZE-1); + else if (cmd==NVMEM_IOC_CLAIM) + key = op.key; + else if (cmd==NVMEM_IOC_FROM_ID) + key = op.id; + + err = _nvmap_do_create(priv, cmd, key, (filp->f_op==&knvmap_fops), &r); + + if (!err) { + op.handle = (uintptr_t)r->h; + /* since the size is spoofed to a page-multiple above, + * clobber the orig_size field back to the requested value for + * debugging. */ + if (cmd == NVMEM_IOC_CREATE) r->h->orig_size = op.size; + err = copy_to_user(arg, &op, sizeof(op)); + if (err) _nvmap_do_free(priv, op.handle); + } + + return err; +} + static int nvmap_map_into_caller_ptr(struct file *filp, void __user *arg) { struct nvmem_map_caller op; - struct nvmap_vma_priv *priv; + struct nvmap_vma_priv *vpriv; struct vm_area_struct *vma; - NvRmMemHandle hmem; + struct nvmap_handle *h; int err = 0; err = copy_from_user(&op, arg, sizeof(op)); - if (err) - return err; + if (err) return err; - hmem = (NvRmMemHandle)op.handle; - if (!hmem) - return -EINVAL; + if (!op.handle) return -EINVAL; + + h = _nvmap_validate_get(op.handle, (filp->f_op==&knvmap_fops)); + if (!h) return -EINVAL; down_read(¤t->mm->mmap_sem); - vma = find_vma(current->mm, (unsigned long)op.addr); + vma = find_vma(current->mm, op.addr); if (!vma || !vma->vm_private_data) { err = -ENOMEM; goto out; @@ -343,54 +2133,49 @@ static int nvmap_map_into_caller_ptr(struct file *filp, void __user *arg) goto out; } - if ((op.offset + op.length) > - ((hmem->size+PAGE_SIZE-1)&PAGE_MASK)) { + if ((op.offset + op.length) > h->size) { err = -EADDRNOTAVAIL; goto out; } - priv = vma->vm_private_data; - BUG_ON(!priv); + vpriv = vma->vm_private_data; + BUG_ON(!vpriv); /* the VMA must exactly match the requested mapping operation, and the * VMA that is targetted must have been created originally by /dev/nvmap */ - if (((void*)vma->vm_start != op.addr) || - (vma->vm_ops != &nvmap_vma_ops) || - (vma->vm_end-vma->vm_start != op.length)) { + if ((vma->vm_start != op.addr) || (vma->vm_ops != &nvmap_vma_ops) || + (vma->vm_end-vma->vm_start != op.length)) { err = -EPERM; goto out; } /* verify that each mmap() system call creates a unique VMA */ - if (priv->hmem && hmem==priv->hmem) + if (vpriv->h && h==vpriv->h) goto out; - else if (priv->hmem) { + else if (vpriv->h) { err = -EADDRNOTAVAIL; goto out; } - if (hmem->alignment & ~PAGE_MASK) { + if (!h->heap_pgalloc && (h->carveout.base & ~PAGE_MASK)) { err = -EFAULT; goto out; } - priv->hmem = hmem; - priv->offs = op.offset; + vpriv->h = h; + vpriv->offs = op.offset; /* if the hmem is not writeback-cacheable, drop back to a page mapping * which will guarantee DMA coherency */ - if (hmem->coherency == NvOsMemAttribute_WriteBack) - vma->vm_page_prot = pgprot_inner_writeback(vma->vm_page_prot); - else - vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); - - NvRmPrivMemIncrRef(hmem); + vma->vm_page_prot = _nvmap_flag_to_pgprot(h->flags, + vma->vm_page_prot); - out: +out: up_read(¤t->mm->mmap_sem); + if (err) _nvmap_handle_put(h); return err; } /* Initially, the nvmap mmap system call is used to allocate an inaccessible @@ -412,7 +2197,7 @@ static int nvmap_mmap(struct file *filp, struct vm_area_struct *vma) return -ENOMEM; priv->offs = 0; - priv->hmem = NULL; + priv->h = NULL; atomic_set(&priv->ref, 1); vma->vm_flags |= VM_SHARED; @@ -423,205 +2208,442 @@ static int nvmap_mmap(struct file *filp, struct vm_area_struct *vma) return 0; } -static int nvmap_cache_maint(struct file *filp, void __user *arg) +/* perform cache maintenance on a handle; caller's handle must be pre- + * validated. */ +static int _nvmap_do_cache_maint(struct nvmap_handle *h, + unsigned long start, unsigned long end, unsigned long op, bool get) +{ + pgprot_t prot; + void *addr = NULL; + void (*inner_maint)(const void*, const void*); + void (*outer_maint)(unsigned long, unsigned long); + int err = 0; + + if (get) h = _nvmap_handle_get(h); + + if (!h) return -EINVAL; + + /* don't waste time on cache maintenance if the handle isn't cached */ + if (h->flags == NVMEM_HANDLE_UNCACHEABLE || + h->flags == NVMEM_HANDLE_WRITE_COMBINE) + goto out; + + if (op == NVMEM_CACHE_OP_WB) { + inner_maint = smp_dma_clean_range; + if (h->flags == NVMEM_HANDLE_CACHEABLE) + outer_maint = outer_clean_range; + else + outer_maint = NULL; + } else if (op == NVMEM_CACHE_OP_WB_INV) { + inner_maint = smp_dma_flush_range; + if (h->flags == NVMEM_HANDLE_CACHEABLE) + outer_maint = outer_flush_range; + else + outer_maint = NULL; + } else { + inner_maint = smp_dma_inv_range; + if (h->flags == NVMEM_HANDLE_CACHEABLE) + outer_maint = outer_inv_range; + else + outer_maint = NULL; + } + + prot = _nvmap_flag_to_pgprot(h->flags, pgprot_kernel); + + /* for any write-back operation, it is safe to writeback the entire + * cache rather than just the requested region. for large regions, it + * is faster to do this than to iterate over every line. + * only implemented for L1-only cacheable handles currently */ + if (h->flags == NVMEM_HANDLE_INNER_CACHEABLE && + end-start >= PAGE_SIZE*3 && op != NVMEM_CACHE_OP_INV) { + if (op==NVMEM_CACHE_OP_WB) dmac_clean_all(); + else dmac_flush_all(); + goto out; + } + + while (start < end) { + struct page *page = NULL; + unsigned long phys; + void *src; + size_t count; + + if (h->heap_pgalloc) { + page = h->pgalloc.pages[start>>PAGE_SHIFT]; + BUG_ON(!page); + get_page(page); + phys = page_to_phys(page) + (start & ~PAGE_MASK); + } else { + phys = h->carveout.base + start; + } + + if (!addr) { + err = nvmap_map_pte(__phys_to_pfn(phys), prot, &addr); + if (err) { + if (page) put_page(page); + break; + } + } else { + _nvmap_set_pte_at((unsigned long)addr, + __phys_to_pfn(phys), prot); + } + + src = addr + (phys & ~PAGE_MASK); + count = min_t(size_t, end-start, PAGE_SIZE-(phys&~PAGE_MASK)); + + inner_maint(src, src+count); + if (outer_maint) outer_maint(phys, phys+count); + start += count; + if (page) put_page(page); + } + +out: + if (h->flags == NVMEM_HANDLE_INNER_CACHEABLE) outer_sync(); + if (addr) nvmap_unmap_pte(addr); + if (get) _nvmap_handle_put(h); + return err; +} + +static int nvmap_ioctl_cache_maint(struct file *filp, void __user *arg) { struct nvmem_cache_op op; int err = 0; struct vm_area_struct *vma; - struct nvmap_vma_priv *priv; - size_t offs; - size_t end; - unsigned long count; - pgprot_t prot = pgprot_inner_writeback(pgprot_kernel); + struct nvmap_vma_priv *vpriv; + unsigned long start; + unsigned long end; err = copy_from_user(&op, arg, sizeof(op)); if (err) return err; if (!op.handle || !op.addr || op.op<NVMEM_CACHE_OP_WB || - op.op>NVMEM_CACHE_OP_WB_INV) + op.op>NVMEM_CACHE_OP_WB_INV) return -EINVAL; vma = find_vma(current->active_mm, (unsigned long)op.addr); if (!vma || vma->vm_ops!=&nvmap_vma_ops || - (unsigned long)op.addr + op.len > vma->vm_end) { - err = -EADDRNOTAVAIL; - goto out; - } + (unsigned long)op.addr + op.len > vma->vm_end) + return -EADDRNOTAVAIL; - priv = (struct nvmap_vma_priv *)vma->vm_private_data; + vpriv = (struct nvmap_vma_priv *)vma->vm_private_data; - if (priv->hmem != (NvRmMemHandle)op.handle) { - err = -EFAULT; - goto out; - } + if ((unsigned long)vpriv->h != op.handle) + return -EFAULT; - /* don't waste time on cache maintenance if the handle isn't cached */ - if (priv->hmem->coherency != NvOsMemAttribute_WriteBack) - goto out; + start = (unsigned long)op.addr - vma->vm_start; + end = start + op.len; - offs = (unsigned long)op.addr - vma->vm_start; - end = offs + op.len; + return _nvmap_do_cache_maint(vpriv->h, start, end, op.op, true); +} - /* for any write-back operation, it is safe to writeback the entire - * cache rather than one line at a time. for large regions, it - * is faster to do this than to iterate over every line. */ - if (end-offs >= PAGE_SIZE*3 && op.op != NVMEM_CACHE_OP_INV) { - if (op.op==NVMEM_CACHE_OP_WB) dmac_clean_all(); - else dmac_flush_all(); - goto out; +/* copies a single element from the pre-get()'ed handle h, returns + * the number of bytes copied, and the address in the nvmap mapping range + * which was used (to eliminate re-allocation when copying multiple + * elements */ +static ssize_t _nvmap_do_one_rw_handle(struct nvmap_handle *h, int is_read, + int is_user, unsigned long start, unsigned long rw_addr, + unsigned long bytes, void **nvmap_addr) +{ + pgprot_t prot = _nvmap_flag_to_pgprot(h->flags, pgprot_kernel); + unsigned long end = start + bytes; + unsigned long orig_start = start; + + if (is_user) { + if (is_read && !access_ok(VERIFY_WRITE, (void*)rw_addr, bytes)) + return -EPERM; + if (!is_read && !access_ok(VERIFY_READ, (void*)rw_addr, bytes)) + return -EPERM; } - while (offs < end) { + while (start < end) { struct page *page = NULL; - void *addr = NULL, *src; + unsigned long phys; + size_t count; + void *src; - if (priv->hmem->hPageHandle) { - struct page *page; - page = NvOsPageGetPage(priv->hmem->hPageHandle, offs); + if (h->heap_pgalloc) { + page = h->pgalloc.pages[start >> PAGE_SHIFT]; + BUG_ON(!page); get_page(page); - err = nvmap_map_pte(page_to_pfn(page), prot, &addr); + phys = page_to_phys(page) + (start & ~PAGE_MASK); } else { - unsigned long phys = priv->hmem->PhysicalAddress + offs; - err = nvmap_map_pte(phys>>PAGE_SHIFT, prot, &addr); + phys = h->carveout.base + start; } - if (err) { - if (page) put_page(page); - break; + if (!*nvmap_addr) { + int err = nvmap_map_pte(__phys_to_pfn(phys), + prot, nvmap_addr); + if (err) { + if (page) put_page(page); + count = start - orig_start; + return (count) ? count : err; + } + } else { + _nvmap_set_pte_at((unsigned long)*nvmap_addr, + __phys_to_pfn(phys), prot); + } - src = addr + (offs & ~PAGE_MASK); - count = min_t(size_t, end-offs, PAGE_SIZE-(offs & ~PAGE_MASK)); + src = *nvmap_addr + (phys & ~PAGE_MASK); + count = min_t(size_t, end-start, PAGE_SIZE-(phys&~PAGE_MASK)); - switch (op.op) { - case NVMEM_CACHE_OP_WB: - smp_dma_clean_range(src, src+count); - break; - case NVMEM_CACHE_OP_INV: - smp_dma_inv_range(src, src+count); - break; - case NVMEM_CACHE_OP_WB_INV: - smp_dma_flush_range(src, src+count); - break; - } - offs += count; - nvmap_unmap_pte(addr); + if (is_user && is_read) + copy_to_user((void*)rw_addr, src, count); + else if (is_user) + copy_from_user(src, (void*)rw_addr, count); + else if (is_read) + memcpy((void*)rw_addr, src, count); + else + memcpy(src, (void*)rw_addr, count); + + rw_addr += count; + start += count; if (page) put_page(page); } - out: - return err; + return (ssize_t)start - orig_start; } -static int nvmap_do_rw_handle(NvRmMemHandle hmem, int is_read, - unsigned long l_hmem_offs, unsigned long l_user_addr, - unsigned long bytes, pgprot_t prot) +static ssize_t _nvmap_do_rw_handle(struct nvmap_handle *h, int is_read, + int is_user, unsigned long h_offs, unsigned long sys_addr, + unsigned long h_stride, unsigned long sys_stride, + unsigned long elem_size, unsigned long count) { - void *addr = NULL, *dest; - struct page *page = NULL; - int ret = 0; - - if (hmem->hPageHandle) { - page = NvOsPageGetPage(hmem->hPageHandle, l_hmem_offs); - get_page(page); - ret = nvmap_map_pte(page_to_pfn(page), prot, &addr); - } else { - unsigned long phys = hmem->PhysicalAddress + l_hmem_offs; - ret = nvmap_map_pte(phys>>PAGE_SHIFT, prot, &addr); + ssize_t bytes_copied = 0; + void *addr = NULL; + + h = _nvmap_handle_get(h); + if (!h) return -EINVAL; + + if (elem_size == h_stride && + elem_size == sys_stride) { + elem_size *= count; + h_stride = elem_size; + sys_stride = elem_size; + count = 1; } - dest = addr + (l_hmem_offs & ~PAGE_MASK); - - if (is_read && !access_ok(VERIFY_WRITE, (void *)l_user_addr, bytes)) - ret = -EPERM; - - if (!is_read && !access_ok(VERIFY_READ, (void *)l_user_addr, bytes)) - ret = -EPERM; - - if (!ret) { - if (is_read) copy_to_user((void *)l_user_addr, dest, bytes); - else copy_from_user(dest, (void*)l_user_addr, bytes); + while (count--) { + size_t ret = _nvmap_do_one_rw_handle(h, is_read, + is_user, h_offs, sys_addr, elem_size, &addr); + if (ret < 0) { + if (!bytes_copied) bytes_copied = ret; + break; + } + bytes_copied += ret; + if (ret < elem_size) break; + sys_addr += elem_size; + h_offs += elem_size; } if (addr) nvmap_unmap_pte(addr); - if (page) put_page(page); - return ret; + _nvmap_handle_put(h); + return bytes_copied; } -static int nvmap_rw_handle(struct file *filp, int is_read, - void __user* arg) +static int nvmap_ioctl_rw_handle(struct file *filp, + int is_read, void __user* arg) { + struct nvmem_rw_handle __user *uarg = arg; struct nvmem_rw_handle op; - NvRmMemHandle hmem; - uintptr_t user_addr, hmem_offs; + struct nvmap_handle *h; + ssize_t copied; int err = 0; - pgprot_t prot; err = copy_from_user(&op, arg, sizeof(op)); - if (err) - return err; + if (err) return err; if (!op.handle || !op.addr || !op.count || !op.elem_size) return -EINVAL; - hmem = (NvRmMemHandle)op.handle; + h = _nvmap_validate_get(op.handle, (filp->f_op == &knvmap_fops)); + if (!h) return -EINVAL; /* -EPERM? */ - if (op.elem_size == op.hmem_stride && - op.elem_size == op.user_stride) { - op.elem_size *= op.count; - op.hmem_stride = op.elem_size; - op.user_stride = op.elem_size; - op.count = 1; - } + copied = _nvmap_do_rw_handle(h, is_read, 1, op.offset, + (unsigned long)op.addr, op.hmem_stride, + op.user_stride, op.elem_size, op.count); - user_addr = (uintptr_t)op.addr; - hmem_offs = (uintptr_t)op.offset; + if (copied < 0) { err = copied; copied = 0; } + else if (copied < (op.count*op.elem_size)) err = -EINTR; - if (hmem->coherency==NvOsMemAttribute_WriteBack) - prot = pgprot_inner_writeback(pgprot_kernel); - else if (hmem->coherency==NvOsMemAttribute_WriteCombined) - prot = pgprot_writecombine(pgprot_kernel); - else - prot = pgprot_noncached(pgprot_kernel); + __put_user(copied, &uarg->count); - while (op.count--) { - unsigned long remain; - unsigned long l_hmem_offs = hmem_offs; - unsigned long l_user_addr = user_addr; + _nvmap_handle_put(h); - remain = op.elem_size; + return err; +} - while (remain && !err) { - unsigned long bytes; +static unsigned int _nvmap_do_get_param(struct nvmap_handle *h, + unsigned int param) +{ + if (param==NVMEM_HANDLE_PARAM_SIZE) + return h->orig_size; - bytes = min(remain, PAGE_SIZE-(l_hmem_offs&~PAGE_MASK)); - bytes = min(bytes, PAGE_SIZE-(l_user_addr&~PAGE_MASK)); + else if (param==NVMEM_HANDLE_PARAM_ALIGNMENT) { + unsigned int i = 0; - err = nvmap_do_rw_handle(hmem, is_read, l_hmem_offs, - l_user_addr, bytes, prot); + if (!h->alloc) return 0; - if (!err) { - remain -= bytes; - l_hmem_offs += bytes; - l_user_addr += bytes; - } + if (h->heap_pgalloc) return PAGE_SIZE; + else { + while (!(i & h->carveout.base)) i<<=1; } + return i; + } else if (param==NVMEM_HANDLE_PARAM_BASE) { + + WARN_ON(!h->alloc || !atomic_read(&h->pin)); + + if (!h->alloc || !atomic_read(&h->pin)) return ~0ul; + + if (!h->heap_pgalloc) + return h->carveout.base; + + if (h->pgalloc.contig) + return page_to_phys(h->pgalloc.pages[0]); - user_addr += op.user_stride; - hmem_offs += op.hmem_stride; + if (h->pgalloc.area) + return h->pgalloc.area->iovm_start; + + return ~0ul; + } else if (param==NVMEM_HANDLE_PARAM_HEAP) { + + if (!h->alloc) return 0; + + if (!h->heap_pgalloc) { + /* FIXME: hard-coded physical address */ + if ((h->carveout.base & 0xf0000000ul)==0x40000000ul) + return NVMEM_HEAP_CARVEOUT_IRAM; + else + return NVMEM_HEAP_CARVEOUT_GENERIC; + } + + if (!h->pgalloc.contig) + return NVMEM_HEAP_IOVMM; + + return NVMEM_HEAP_SYSMEM; } - dmb(); + return 0; +} + +static int nvmap_ioctl_get_param(struct file *filp, void __user* arg) +{ + struct nvmem_handle_param op; + struct nvmap_handle *h; + int err; + + err = copy_from_user(&op, arg, sizeof(op)); + if (err) return err; + + if (op.param < NVMEM_HANDLE_PARAM_SIZE || + op.param > NVMEM_HANDLE_PARAM_HEAP) + return -EINVAL; + + h = _nvmap_validate_get(op.handle, (filp->f_op==&knvmap_fops)); + if (!h) return -EINVAL; + + op.result = _nvmap_do_get_param(h, op.param); + err = copy_to_user(arg, &op, sizeof(op)); + + _nvmap_handle_put(h); return err; } -static int __init nvmap_pte_init(void) +static struct miscdevice misc_nvmap_dev = { + .minor = MISC_DYNAMIC_MINOR, + .name = "nvmap", + .fops = &nvmap_fops +}; + +static struct miscdevice misc_knvmap_dev = { + .minor = MISC_DYNAMIC_MINOR, + .name = "knvmap", + .fops = &knvmap_fops +}; + +static struct device *__nvmap_heap_parent_dev(void) +{ + return misc_nvmap_dev.this_device; +} + +/* creates the sysfs attribute files for a carveout heap; if called + * before fs initialization, silently returns. + */ +static void _nvmap_create_heap_attrs(struct nvmap_carveout_node *n) +{ + if (!_nvmap_heap_parent_dev) return; + dev_set_name(&n->dev, "heap-%s", n->carveout.name); + n->dev.parent = _nvmap_heap_parent_dev; + n->dev.driver = NULL; + n->dev.release = NULL; + if (device_register(&n->dev)) { + pr_err("%s: failed to create heap-%s device\n", + __func__, n->carveout.name); + return; + } + if (sysfs_create_group(&n->dev.kobj, &nvmap_heap_defattr_group)) + pr_err("%s: failed to create attribute group for heap-%s " + "device\n", __func__, n->carveout.name); +} + +static int __init nvmap_dev_init(void) +{ + struct nvmap_carveout_node *n; + + if (misc_register(&misc_nvmap_dev)) + pr_err("%s error registering %s\n", __func__, + misc_nvmap_dev.name); + + if (misc_register(&misc_knvmap_dev)) + pr_err("%s error registering %s\n", __func__, + misc_knvmap_dev.name); + + /* create sysfs attribute entries for all the heaps which were + * created prior to nvmap_dev_init */ + down_read(&nvmap_context.list_sem); + list_for_each_entry(n, &nvmap_context.heaps, heap_list) { + _nvmap_create_heap_attrs(n); + } + up_read(&nvmap_context.list_sem); + + nvmap_procfs_root = proc_mkdir("nvmap", NULL); + if (nvmap_procfs_root) { + nvmap_procfs_proc = proc_mkdir("proc", nvmap_procfs_root); + } + return 0; +} +fs_initcall(nvmap_dev_init); + +/* initialization of core data structures split out to earlier in the + * init sequence, to allow kernel drivers access to nvmap before devfs + * is initialized */ +#define NR_CARVEOUTS 2 +static unsigned int nvmap_carveout_cmds = 0; +static unsigned long nvmap_carveout_cmd_base[NR_CARVEOUTS]; +static unsigned long nvmap_carveout_cmd_size[NR_CARVEOUTS]; + +static int __init nvmap_core_init(void) { u32 base = NVMAP_BASE; pgd_t *pgd; pmd_t *pmd; pte_t *pte; - int i = 0; + unsigned int i; + + init_rwsem(&nvmap_context.list_sem); + nvmap_context.init_data.handle_refs = RB_ROOT; + atomic_set(&nvmap_context.init_data.iovm_commit, 0); + /* no IOVMM allocations for kernel-created handles */ + spin_lock_init(&nvmap_context.init_data.ref_lock); + nvmap_context.init_data.su = true; + nvmap_context.init_data.iovm_limit = 0; + INIT_LIST_HEAD(&nvmap_context.heaps); + +#ifdef CONFIG_DEVNVMAP_RECLAIM_UNPINNED_VM + for (i=0; i<ARRAY_SIZE(nvmap_mru_cutoff); i++) + INIT_LIST_HEAD(&nvmap_mru_vma_lists[i]); +#endif + + i = 0; do { pgd = pgd_offset(&init_mm, base); pmd = pmd_alloc(&init_mm, pgd, base); @@ -638,7 +2660,641 @@ static int __init nvmap_pte_init(void) base += (1<<PGDIR_SHIFT); } while (base < NVMAP_END); + for (i=0; i<nvmap_carveout_cmds; i++) { + char tmp[16]; + snprintf(tmp, sizeof(tmp), "generic-%u", i); + nvmap_add_carveout_heap(nvmap_carveout_cmd_base[i], + nvmap_carveout_cmd_size[i], tmp, 0x1); + } + return 0; } -core_initcall(nvmap_pte_init); +core_initcall(nvmap_core_init); + +static int __init nvmap_heap_arg(char *options) +{ + unsigned long start, size; + char *p = options; + + start = -1; + size = memparse(p, &p); + if (*p == '@') + start = memparse(p + 1, &p); + + if (nvmap_carveout_cmds < ARRAY_SIZE(nvmap_carveout_cmd_size)) { + nvmap_carveout_cmd_base[nvmap_carveout_cmds] = start; + nvmap_carveout_cmd_size[nvmap_carveout_cmds] = size; + nvmap_carveout_cmds++; + } + return 0; +} +__setup("nvmem=", nvmap_heap_arg); + +static int _nvmap_try_create_preserved(struct nvmap_carveout *co, + struct nvmap_handle *h, unsigned long base, + size_t size, unsigned int key) +{ + unsigned long end = base + size; + short idx; + + h->carveout.base = ~0; + h->carveout.key = key; + h->carveout.co_heap = NULL; + + spin_lock(&co->lock); + idx = co->free_index; + while (idx != -1) { + struct nvmap_mem_block *b = BLOCK(co, idx); + unsigned long blk_end = b->base + b->size; + if (b->base <= base && blk_end >= end) { + nvmap_split_block(co, idx, base, size); + h->carveout.block_idx = idx; + h->carveout.base = co->blocks[idx].base; + h->carveout.co_heap = co; + h->alloc = true; + break; + } + idx = b->next_free; + } + spin_unlock(&co->lock); + return (h->carveout.co_heap == NULL) ? -ENXIO : 0; +} + +static void _nvmap_create_nvos_preserved(struct nvmap_carveout *co) +{ + unsigned int i, key; + NvBootArgsPreservedMemHandle mem; + static int was_created[NvBootArgKey_PreservedMemHandle_Num - + NvBootArgKey_PreservedMemHandle_0] = { 0 }; + + for (i=0, key=NvBootArgKey_PreservedMemHandle_0; + i<ARRAY_SIZE(was_created); i++, key++) { + struct nvmap_handle *h; + + if (was_created[i]) continue; + + if (NvOsBootArgGet(key, &mem, sizeof(mem))!=NvSuccess) continue; + if (!mem.Address || !mem.Size) continue; + + h = _nvmap_handle_create(NULL, mem.Size); + if (!h) continue; + + if (!_nvmap_try_create_preserved(co, h, mem.Address, + mem.Size, key)) + was_created[i] = 1; + else + _nvmap_handle_put(h); + } +} + +int nvmap_add_carveout_heap(unsigned long base, size_t size, + const char *name, unsigned int bitmask) +{ + struct nvmap_carveout_node *n; + struct nvmap_carveout_node *l; + + + n = kzalloc(sizeof(*n), GFP_KERNEL); + if (!n) return -ENOMEM; + + BUG_ON(bitmask & ~NVMEM_HEAP_CARVEOUT_MASK); + n->heap_bit = bitmask; + + if (_nvmap_init_carveout(&n->carveout, name, base, size)) { + kfree(n); + return -ENOMEM; + } + + down_write(&nvmap_context.list_sem); + + /* called inside the list_sem lock to ensure that the was_created + * array is protected against simultaneous access */ + _nvmap_create_nvos_preserved(&n->carveout); + _nvmap_create_heap_attrs(n); + + list_for_each_entry(l, &nvmap_context.heaps, heap_list) { + if (n->heap_bit > l->heap_bit) { + list_add_tail(&n->heap_list, &l->heap_list); + up_write(&nvmap_context.list_sem); + return 0; + } + } + list_add_tail(&n->heap_list, &nvmap_context.heaps); + up_write(&nvmap_context.list_sem); + return 0; +} + +int nvmap_create_preserved_handle(unsigned long base, size_t size, + unsigned int key) +{ + struct nvmap_carveout_node *i; + struct nvmap_handle *h; + + h = _nvmap_handle_create(NULL, size); + if (!h) return -ENOMEM; + + down_read(&nvmap_context.list_sem); + list_for_each_entry(i, &nvmap_context.heaps, heap_list) { + struct nvmap_carveout *co = &i->carveout; + if (!_nvmap_try_create_preserved(co, h, base, size, key)) + break; + } + up_read(&nvmap_context.list_sem); + + /* the base may not be correct if block splitting fails */ + if (!h->carveout.co_heap || h->carveout.base != base) { + _nvmap_handle_put(h); + return -ENOMEM; + } + + return 0; +} + +/* attempts to create a new carveout heap with a new usage bitmask by + * taking an allocation from a previous carveout with a different bitmask */ +static int nvmap_split_carveout_heap(struct nvmap_carveout *co, size_t size, + const char *name, unsigned int new_bitmask) +{ + struct nvmap_carveout_node *i, *n; + int idx = -1; + unsigned int blkbase, blksize; + + + n = kzalloc(sizeof(*n), GFP_KERNEL); + if (!n) return -ENOMEM; + n->heap_bit = new_bitmask; + + /* align split carveouts to 1M */ + idx = nvmap_carveout_alloc(co, SZ_1M, size); + if (idx != -1) { + /* take the spin lock to avoid race conditions with + * intervening allocations triggering grow_block operations */ + spin_lock(&co->lock); + blkbase = co->blocks[idx].base; + blksize = co->blocks[idx].size; + spin_unlock(&co->lock); + + if (_nvmap_init_carveout(&n->carveout,name, blkbase, blksize)) { + nvmap_carveout_free(&i->carveout, idx); + idx = -1; + } else { + spin_lock(&co->lock); + if (co->blocks[idx].prev) { + co->blocks[co->blocks[idx].prev].next = + co->blocks[idx].next; + } + if (co->blocks[idx].next) { + co->blocks[co->blocks[idx].next].prev = + co->blocks[idx].prev; + } + if (co->block_index==idx) + co->block_index = co->blocks[idx].next; + co->blocks[idx].next_free = -1; + co->blocks[idx].prev_free = -1; + co->blocks[idx].next = co->spare_index; + if (co->spare_index!=-1) + co->blocks[co->spare_index].prev = idx; + co->spare_index = idx; + spin_unlock(&co->lock); + } + } + + if (idx==-1) { + kfree(n); + return -ENOMEM; + } + + down_write(&nvmap_context.list_sem); + _nvmap_create_heap_attrs(n); + list_for_each_entry(i, &nvmap_context.heaps, heap_list) { + if (n->heap_bit > i->heap_bit) { + list_add_tail(&n->heap_list, &i->heap_list); + up_write(&nvmap_context.list_sem); + return 0; + } + } + list_add_tail(&n->heap_list, &nvmap_context.heaps); + up_write(&nvmap_context.list_sem); + return 0; +} + +/* NvRmMemMgr APIs implemented on top of nvmap */ + +#include <linux/freezer.h> + +NvU32 NvRmMemGetAddress(NvRmMemHandle hMem, NvU32 Offset) +{ + struct nvmap_handle *h = (struct nvmap_handle *)hMem; + unsigned long addr; + + if (unlikely(!atomic_read(&h->pin) || !h->alloc)) return ~0ul; + if (unlikely(Offset >= h->orig_size)) return ~0UL; + + if (h->heap_pgalloc && h->pgalloc.contig) + addr = page_to_phys(h->pgalloc.pages[0]); + else if (h->heap_pgalloc) { + BUG_ON(!h->pgalloc.area); + BUG_ON(!atomic_read(&h->pin)); + addr = h->pgalloc.area->iovm_start; + } else + addr = h->carveout.base; + + return (NvU32)addr+Offset; + +} + +void NvRmMemPinMult(NvRmMemHandle *hMems, NvU32 *addrs, NvU32 Count) +{ + struct nvmap_handle **h = (struct nvmap_handle **)hMems; + unsigned int i; + int ret; + + do { + ret = _nvmap_handle_pin_fast(Count, h); + if (ret && !try_to_freeze()) { + pr_err("%s: failed to pin handles\n", __func__); + dump_stack(); + } + } while (ret); + + for (i=0; i<Count; i++) { + addrs[i] = NvRmMemGetAddress(hMems[i], 0); + BUG_ON(addrs[i]==~0ul); + } +} + +void NvRmMemUnpinMult(NvRmMemHandle *hMems, NvU32 Count) +{ + int do_wake = 0; + unsigned int i; + + for (i=0; i<Count; i++) { + struct nvmap_handle *h = (struct nvmap_handle *)hMems[i]; + BUG_ON(atomic_read(&h->pin)==0); + do_wake |= _nvmap_handle_unpin(h); + } + + if (do_wake) wake_up(&nvmap_pin_wait); +} + +NvU32 NvRmMemPin(NvRmMemHandle hMem) +{ + NvU32 addr; + NvRmMemPinMult(&hMem, &addr, 1); + return addr; +} + +void NvRmMemUnpin(NvRmMemHandle hMem) +{ + NvRmMemUnpinMult(&hMem, 1); +} + +void NvRmMemHandleFree(NvRmMemHandle hMem) +{ + _nvmap_do_free(&nvmap_context.init_data, (unsigned long)hMem); +} + +NvError NvRmMemMap(NvRmMemHandle hMem, NvU32 Offset, NvU32 Size, + NvU32 Flags, void **pVirtAddr) +{ + struct nvmap_handle *h = (struct nvmap_handle *)hMem; + pgprot_t prot = _nvmap_flag_to_pgprot(h->flags, pgprot_kernel); + + BUG_ON(!h->alloc); + + if (Offset+Size > h->size) + return NvError_BadParameter; + + if (!h->kern_map && h->heap_pgalloc) { + BUG_ON(h->size & ~PAGE_MASK); + h->kern_map = vm_map_ram(h->pgalloc.pages, + h->size>>PAGE_SHIFT, -1, prot); + } else if (!h->kern_map) { + unsigned int size; + unsigned long addr; + + addr = h->carveout.base; + size = h->size + (addr & ~PAGE_MASK); + addr &= PAGE_MASK; + size = (size + PAGE_SIZE - 1) & PAGE_MASK; + + h->kern_map = ioremap_wc(addr, size); + if (h->kern_map) { + addr = h->carveout.base - addr; + h->kern_map += addr; + } + } + + if (h->kern_map) { + *pVirtAddr = (h->kern_map + Offset); + return NvSuccess; + } + + return NvError_InsufficientMemory; +} + +void NvRmMemUnmap(NvRmMemHandle hMem, void *pVirtAddr, NvU32 Size) +{ + return; +} + +NvU32 NvRmMemGetId(NvRmMemHandle hMem) +{ + struct nvmap_handle *h = (struct nvmap_handle *)hMem; + if (!h->owner) h->global = true; + return (NvU32)h; +} + +NvError NvRmMemHandleFromId(NvU32 id, NvRmMemHandle *hMem) +{ + struct nvmap_handle_ref *r; + + int err = _nvmap_do_create(&nvmap_context.init_data, + NVMEM_IOC_FROM_ID, id, true, &r); + + if (err || !r) return NvError_NotInitialized; + + *hMem = (NvRmMemHandle)r->h; + return NvSuccess; +} + +NvError NvRmMemHandleClaimPreservedHandle(NvRmDeviceHandle hRm, + NvU32 Key, NvRmMemHandle *hMem) +{ + struct nvmap_handle_ref *r; + + int err = _nvmap_do_create(&nvmap_context.init_data, + NVMEM_IOC_CLAIM, (unsigned long)Key, true, &r); + + if (err || !r) return NvError_NotInitialized; + + *hMem = (NvRmMemHandle)r->h; + return NvSuccess; +} + +NvError NvRmMemHandleCreate(NvRmDeviceHandle hRm, + NvRmMemHandle *hMem, NvU32 Size) +{ + struct nvmap_handle_ref *r; + int err = _nvmap_do_create(&nvmap_context.init_data, + NVMEM_IOC_CREATE, (unsigned long)Size, true, &r); + + if (err || !r) return NvError_InsufficientMemory; + *hMem = (NvRmMemHandle)r->h; + return NvSuccess; +} + +NvError NvRmMemAlloc(NvRmMemHandle hMem, const NvRmHeap *Heaps, + NvU32 NumHeaps, NvU32 Alignment, NvOsMemAttribute Coherency) +{ + unsigned int heap_mask = 0; + unsigned int flags = pgprot_kernel; + int err; + + BUG_ON(Alignment & (Alignment-1)); + + if (Coherency == NvOsMemAttribute_WriteBack) + flags = NVMEM_HANDLE_INNER_CACHEABLE; + else + flags = NVMEM_HANDLE_WRITE_COMBINE; + + if (!NumHeaps || !Heaps) + heap_mask = (NVMEM_HEAP_SYSMEM | NVMEM_HEAP_CARVEOUT_GENERIC); + else { + unsigned int i; + for (i=0; i<NumHeaps; i++) { + switch (Heaps[i]) { + case NvRmHeap_GART: + heap_mask |= NVMEM_HEAP_IOVMM; + break; + case NvRmHeap_External: + heap_mask |= NVMEM_HEAP_SYSMEM; + break; + case NvRmHeap_ExternalCarveOut: + heap_mask |= NVMEM_HEAP_CARVEOUT_GENERIC; + break; + case NvRmHeap_IRam: + heap_mask |= NVMEM_HEAP_CARVEOUT_IRAM; + break; + default: + break; + } + } + } + if (!heap_mask) return NvError_InsufficientMemory; + + err = _nvmap_do_alloc(&nvmap_context.init_data, (unsigned long)hMem, + heap_mask, (size_t)Alignment, flags, false, true); + + if (err) return NvError_InsufficientMemory; + return NvSuccess; +} + +void NvRmMemReadStrided(NvRmMemHandle hMem, NvU32 Offset, NvU32 SrcStride, + void *pDst, NvU32 DstStride, NvU32 ElementSize, NvU32 Count) +{ + ssize_t bytes = 0; + + bytes = _nvmap_do_rw_handle((struct nvmap_handle *)hMem, true, + false, Offset, (unsigned long)pDst, SrcStride, + DstStride, ElementSize, Count); + + BUG_ON(bytes != (ssize_t)(Count*ElementSize)); +} + +void NvRmMemWriteStrided(NvRmMemHandle hMem, NvU32 Offset, NvU32 DstStride, + const void *pSrc, NvU32 SrcStride, NvU32 ElementSize, NvU32 Count) +{ + ssize_t bytes = 0; + + bytes = _nvmap_do_rw_handle((struct nvmap_handle *)hMem, false, + false, Offset, (unsigned long)pSrc, DstStride, + SrcStride, ElementSize, Count); + + BUG_ON(bytes != (ssize_t)(Count*ElementSize)); +} + +NvU32 NvRmMemGetSize(NvRmMemHandle hMem) +{ + struct nvmap_handle *h = (struct nvmap_handle *)hMem; + return h->orig_size; +} + +NvRmHeap NvRmMemGetHeapType(NvRmMemHandle hMem, NvU32 *BaseAddr) +{ + struct nvmap_handle *h = (struct nvmap_handle *)hMem; + NvRmHeap heap; + + if (!h->alloc) { + *BaseAddr = ~0ul; + return (NvRmHeap)0; + } + + if (h->heap_pgalloc && !h->pgalloc.contig) + heap = NvRmHeap_GART; + else if (h->heap_pgalloc) + heap = NvRmHeap_External; + else if ((h->carveout.base & 0xf0000000ul) == 0x40000000ul) + heap = NvRmHeap_IRam; + else + heap = NvRmHeap_ExternalCarveOut; + + if (h->heap_pgalloc && h->pgalloc.contig) + *BaseAddr = (NvU32)page_to_phys(h->pgalloc.pages[0]); + else if (h->heap_pgalloc && atomic_read(&h->pin)) + *BaseAddr = h->pgalloc.area->iovm_start; + else if (h->heap_pgalloc) + *BaseAddr = ~0ul; + else + *BaseAddr = (NvU32)h->carveout.base; + + return heap; +} + +void NvRmMemCacheMaint(NvRmMemHandle hMem, void *pMapping, + NvU32 Size, NvBool Writeback, NvBool Inv) +{ + struct nvmap_handle *h = (struct nvmap_handle *)hMem; + unsigned long start; + unsigned int op; + + if (!h->kern_map || h->flags==NVMEM_HANDLE_UNCACHEABLE || + h->flags==NVMEM_HANDLE_WRITE_COMBINE) return; + + if (!Writeback && !Inv) return; + + if (Writeback && Inv) op = NVMEM_CACHE_OP_WB_INV; + else if (Writeback) op = NVMEM_CACHE_OP_WB; + else op = NVMEM_CACHE_OP_INV; + + start = (unsigned long)pMapping - (unsigned long)h->kern_map; + + _nvmap_do_cache_maint(h, start, start+Size, op, true); + return; +} + +NvU32 NvRmMemGetAlignment(NvRmMemHandle hMem) +{ + struct nvmap_handle *h = (struct nvmap_handle *)hMem; + return _nvmap_do_get_param(h, NVMEM_HANDLE_PARAM_ALIGNMENT); +} + +NvError NvRmMemGetStat(NvRmMemStat Stat, NvS32 *Result) +{ + unsigned long total_co = 0; + unsigned long free_co = 0; + unsigned long max_free = 0; + struct nvmap_carveout_node *n; + + down_read(&nvmap_context.list_sem); + list_for_each_entry(n, &nvmap_context.heaps, heap_list) { + + if (!(n->heap_bit & NVMEM_HEAP_CARVEOUT_GENERIC)) continue; + total_co += _nvmap_carveout_blockstat(&n->carveout, + CARVEOUT_STAT_TOTAL_SIZE); + free_co += _nvmap_carveout_blockstat(&n->carveout, + CARVEOUT_STAT_FREE_SIZE); + max_free = max(max_free, + _nvmap_carveout_blockstat(&n->carveout, + CARVEOUT_STAT_LARGEST_FREE)); + } + up_read(&nvmap_context.list_sem); + + if (Stat==NvRmMemStat_TotalCarveout) { + *Result = (NvU32)total_co; + return NvSuccess; + } else if (Stat==NvRmMemStat_UsedCarveout) { + *Result = (NvU32)total_co - (NvU32)free_co; + return NvSuccess; + } else if (Stat==NvRmMemStat_LargestFreeCarveoutBlock) { + *Result = (NvU32)max_free; + return NvSuccess; + } + + return NvError_BadParameter; +} + +NvU8 NvRmMemRd08(NvRmMemHandle hMem, NvU32 Offset) +{ + NvU8 val; + NvRmMemRead(hMem, Offset, &val, sizeof(val)); + return val; +} + +NvU16 NvRmMemRd16(NvRmMemHandle hMem, NvU32 Offset) +{ + NvU16 val; + NvRmMemRead(hMem, Offset, &val, sizeof(val)); + return val; +} + +NvU32 NvRmMemRd32(NvRmMemHandle hMem, NvU32 Offset) +{ + NvU32 val; + NvRmMemRead(hMem, Offset, &val, sizeof(val)); + return val; +} + +void NvRmMemWr08(NvRmMemHandle hMem, NvU32 Offset, NvU8 Data) +{ + NvRmMemWrite(hMem, Offset, &Data, sizeof(Data)); +} + +void NvRmMemWr16(NvRmMemHandle hMem, NvU32 Offset, NvU16 Data) +{ + NvRmMemWrite(hMem, Offset, &Data, sizeof(Data)); +} + +void NvRmMemWr32(NvRmMemHandle hMem, NvU32 Offset, NvU32 Data) +{ + NvRmMemWrite(hMem, Offset, &Data, sizeof(Data)); +} + +void NvRmMemRead(NvRmMemHandle hMem, NvU32 Offset, void *pDst, NvU32 Size) +{ + NvRmMemReadStrided(hMem, Offset, Size, pDst, Size, Size, 1); +} + +void NvRmMemWrite(NvRmMemHandle hMem, NvU32 Offset, + const void *pSrc, NvU32 Size) +{ + NvRmMemWriteStrided(hMem, Offset, Size, pSrc, Size, Size, 1); +} + +void NvRmMemMove(NvRmMemHandle dstHMem, NvU32 dstOffset, + NvRmMemHandle srcHMem, NvU32 srcOffset, NvU32 Size) +{ + while (Size--) { + NvU8 tmp = NvRmMemRd08(srcHMem, srcOffset); + NvRmMemWr08(dstHMem, dstOffset, tmp); + dstOffset++; + srcOffset++; + } +} + +NvU32 NvRmMemGetCacheLineSize(void) +{ + return 32; +} + +void *NvRmHostAlloc(size_t size) +{ + return NvOsAlloc(size); +} + +void NvRmHostFree(void *ptr) +{ + NvOsFree(ptr); +} + +NvError NvRmMemMapIntoCallerPtr(NvRmMemHandle hMem, void *pCallerPtr, + NvU32 Offset, NvU32 Size) +{ + return NvError_NotSupported; +} + +NvError NvRmMemHandlePreserveHandle(NvRmMemHandle hMem, NvU32 *pKey) +{ + return NvError_NotSupported; +} diff --git a/include/linux/tegra_devices.h b/include/linux/tegra_devices.h index adbfffe92f53..4f1d4c8fb267 100755 --- a/include/linux/tegra_devices.h +++ b/include/linux/tegra_devices.h @@ -84,5 +84,9 @@ struct tegra_sdio_platform_data { NvU32 StartOffset; /* start sector offset to MBR for the card */ }; +#ifdef CONFIG_DEVNVMAP +int nvmap_add_carveout_heap(unsigned long base, size_t size, + const char *name, unsigned int bitmask); +#endif #endif |