summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--drivers/char/Kconfig10
-rw-r--r--drivers/char/Makefile1
-rw-r--r--drivers/char/nvmap.c656
3 files changed, 667 insertions, 0 deletions
diff --git a/drivers/char/Kconfig b/drivers/char/Kconfig
index 438f292820af..8230d03b3706 100644
--- a/drivers/char/Kconfig
+++ b/drivers/char/Kconfig
@@ -110,6 +110,16 @@ config DEVKMEM
kind of kernel debugging operations.
When in doubt, say "N".
+config DEVNVMAP
+ bool "/dev/nvmap device support"
+ depends on ARCH_TEGRA && MACH_TEGRA_GENERIC
+ default y
+ help
+ Say Y here if you want to support the /dev/nvmap device. This device
+ is used by NVIDIA Tegra graphics and multimedia drivers for managing
+ graphics memory.
+
+
config SERIAL_NONSTANDARD
bool "Non-standard serial port support"
depends on HAS_IOMEM
diff --git a/drivers/char/Makefile b/drivers/char/Makefile
index 5ab656b57a15..9f1264860a76 100644
--- a/drivers/char/Makefile
+++ b/drivers/char/Makefile
@@ -9,6 +9,7 @@ FONTMAPFILE = cp437.uni
obj-y += mem.o random.o tty_io.o n_tty.o tty_ioctl.o tty_ldisc.o tty_buffer.o tty_port.o
+obj-$(CONFIG_DEVNVMAP) += nvmap.o
obj-$(CONFIG_LEGACY_PTYS) += pty.o
obj-$(CONFIG_UNIX98_PTYS) += pty.o
obj-y += misc.o
diff --git a/drivers/char/nvmap.c b/drivers/char/nvmap.c
new file mode 100644
index 000000000000..e4b10fdd4826
--- /dev/null
+++ b/drivers/char/nvmap.c
@@ -0,0 +1,656 @@
+/*
+ * drivers/char/nvmap.c
+ *
+ * Memory mapping driver for Tegra anonymous memory handles
+ *
+ * Copyright (c) 2009, NVIDIA Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+#include <linux/module.h>
+#include <linux/miscdevice.h>
+#include <linux/platform_device.h>
+#include <linux/mm.h>
+#include <linux/mman.h>
+#include <linux/uaccess.h>
+#include <linux/backing-dev.h>
+#include <linux/device.h>
+#include <linux/highmem.h>
+#include <linux/smp_lock.h>
+#include <linux/pagemap.h>
+#include <linux/sched.h>
+#include <linux/io.h>
+#include "../nvrm/nvrmkernel/core/ap15/ap15rm_private.h"
+#include "linux/nvmem_ioctl.h"
+#include "nvcommon.h"
+#include "nvrm_memmgr.h"
+#include "nvrm_memmgr_private.h"
+
+static void nvmap_vma_open(struct vm_area_struct *vma);
+
+static void nvmap_vma_close(struct vm_area_struct *vma);
+
+static int nvmap_vma_fault(struct vm_area_struct *vma, struct vm_fault *vmf);
+
+static int nvmap_open(struct inode *inode, struct file *filp);
+
+static int nvmap_release(struct inode *inode, struct file *file);
+
+static int nvmap_mmap(struct file *filp, struct vm_area_struct *vma);
+
+static long nvmap_ioctl(struct file *filp,
+ unsigned int cmd, unsigned long arg);
+
+static void nvmap_clean_handle(NvRmMemHandle hmem, size_t offs, size_t len);
+
+static int nvmap_cache_maint(struct file *filp, void __user *arg);
+
+static int nvmap_map_into_caller_ptr(struct file *filp, void __user *arg);
+
+static int nvmap_rw_handle(struct file *filp, int is_read,
+ void __user* arg);
+
+static struct backing_dev_info nvmap_bdi = {
+ .ra_pages = 0,
+ .capabilities = (BDI_CAP_NO_ACCT_AND_WRITEBACK |
+ BDI_CAP_READ_MAP | BDI_CAP_WRITE_MAP),
+};
+
+static struct vm_operations_struct nvmap_vma_ops = {
+ .open = nvmap_vma_open,
+ .close = nvmap_vma_close,
+ .fault = nvmap_vma_fault,
+};
+
+static struct file_operations nvmap_file_ops = {
+ .owner = THIS_MODULE,
+ .open = nvmap_open,
+ .release = nvmap_release,
+ .unlocked_ioctl = nvmap_ioctl,
+ .mmap = nvmap_mmap
+};
+
+struct nvmap_vma_priv {
+ NvRmMemHandle hmem;
+ size_t offs;
+ atomic_t ref;
+};
+
+enum {
+ NVMAP_DEV_DRAM = 0,
+ NVMAP_DEV_IRAM,
+ NVMAP_DEV_COUNT,
+};
+
+static struct {
+ struct miscdevice dev;
+ umode_t mode;
+} nvmap_device = {
+
+ .dev = {
+ .name = "nvmap",
+ .fops = &nvmap_file_ops,
+ .minor = MISC_DYNAMIC_MINOR,
+ },
+ .mode = S_IRUGO | S_IWUGO,
+};
+
+/* to ensure that the backing store for the VMA isn't freed while a fork'd
+ * reference still exists, nvmap_vma_open increments the reference count on
+ * the handle, and nvmap_vma_close decrements it. alternatively, we could
+ * disallow copying of the vma, or behave like pmem and zap the pages. FIXME.
+*/
+static void nvmap_vma_open(struct vm_area_struct *vma)
+{
+ struct nvmap_vma_priv *priv;
+
+ priv = (struct nvmap_vma_priv *)vma->vm_private_data;
+
+ BUG_ON(!priv);
+
+ atomic_inc(&priv->ref);
+}
+
+static void nvmap_vma_close(struct vm_area_struct *vma) {
+ struct nvmap_vma_priv *priv = (struct nvmap_vma_priv *)vma->vm_private_data;
+
+ if (priv && !atomic_dec_return(&priv->ref)) {
+ if (priv->hmem) {
+ size_t offs = (vma->vm_pgoff << PAGE_SHIFT) + priv->offs;
+ size_t len = vma->vm_end - vma->vm_start;
+ nvmap_clean_handle(priv->hmem, offs, len);
+ NvRmMemHandleFree(priv->hmem);
+ }
+ kfree(priv);
+ }
+ vma->vm_private_data = NULL;
+}
+
+extern struct page *NvOsPageGetPage(NvOsPageAllocHandle, size_t);
+#define nvmap_range(x,y) (x), (x)+(y)
+
+#define is_same_page(a, b) \
+ ((unsigned long)(a)>>PAGE_SHIFT == (unsigned long)(b)>>PAGE_SHIFT)
+
+static inline void nvmap_flush(void* kva, unsigned long phys, size_t len)
+{
+ BUG_ON(!is_same_page(kva, kva+len-1));
+ smp_dma_flush_range(nvmap_range(kva, len));
+ outer_flush_range(nvmap_range(phys, len));
+}
+
+static void nvmap_clean_handle(NvRmMemHandle hmem, size_t start, size_t len)
+{
+ if (hmem->coherency != NvOsMemAttribute_WriteBack)
+ return;
+
+ if (hmem->hPageHandle) {
+ size_t offs;
+ size_t end = start + len;
+ for (offs=start; offs<end; offs+=PAGE_SIZE) {
+ size_t bytes = min((size_t)end-offs, (size_t)PAGE_SIZE);
+ struct page *page = NvOsPageGetPage(hmem->hPageHandle, offs);
+ void *ptr = page_address(page) + (offs & ~PAGE_MASK);
+
+ smp_dma_clean_range(nvmap_range(ptr, bytes));
+ outer_clean_range(nvmap_range(__pa(ptr), bytes));
+ }
+ }
+ else if (hmem->VirtualAddress && hmem->PhysicalAddress) {
+ smp_dma_clean_range(nvmap_range(hmem->VirtualAddress, hmem->size));
+ outer_clean_range(nvmap_range(hmem->PhysicalAddress, hmem->size));
+ }
+}
+
+static int nvmap_vma_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
+{
+ struct nvmap_vma_priv *priv;
+ unsigned long pfn;
+ unsigned long offs = (unsigned long)(vmf->virtual_address - vma->vm_start);
+
+ priv = (struct nvmap_vma_priv *)vma->vm_private_data;
+ if (!priv || !priv->hmem)
+ return VM_FAULT_SIGBUS;
+
+ offs += priv->offs;
+ /* if the VMA was split for some reason, vm_pgoff will be the VMA's
+ * offset from the original VMA */
+ offs += (vma->vm_pgoff << PAGE_SHIFT);
+
+ if (offs >= priv->hmem->size)
+ return VM_FAULT_SIGBUS;
+
+ switch (priv->hmem->heap) {
+ case NvRmHeap_ExternalCarveOut:
+ case NvRmHeap_IRam:
+ pfn = ((priv->hmem->PhysicalAddress+offs) >> PAGE_SHIFT);
+ break;
+
+ case NvRmHeap_GART:
+ case NvRmHeap_External:
+ if (!priv->hmem->hPageHandle)
+ return VM_FAULT_SIGBUS;
+ pfn = NvOsPageAddress(priv->hmem->hPageHandle, offs) >> PAGE_SHIFT;
+ break;
+
+ default:
+ return VM_FAULT_SIGBUS;
+ }
+ vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
+ return VM_FAULT_NOPAGE;
+}
+
+static long nvmap_ioctl(struct file *filp,
+ unsigned int cmd, unsigned long arg)
+{
+ int err = 0;
+
+ if (_IOC_TYPE(cmd) != NVMEM_IOC_MAGIC)
+ return -ENOTTY;
+
+ if (_IOC_NR(cmd) > NVMEM_IOC_MAXNR)
+ return -ENOTTY;
+
+ if (_IOC_DIR(cmd) & _IOC_READ)
+ err = !access_ok(VERIFY_WRITE, (void __user *)arg, _IOC_SIZE(cmd));
+ if (_IOC_DIR(cmd) & _IOC_WRITE)
+ err = !access_ok(VERIFY_READ, (void __user *)arg, _IOC_SIZE(cmd));
+
+ if (err)
+ return -EFAULT;
+
+ switch (cmd) {
+ case NVMEM_IOC_MMAP:
+ err = nvmap_map_into_caller_ptr(filp, (void __user *)arg);
+ break;
+
+ case NVMEM_IOC_WRITE:
+ case NVMEM_IOC_READ:
+ err = nvmap_rw_handle(filp, cmd==NVMEM_IOC_READ, (void __user *)arg);
+ break;
+
+ case NVMEM_IOC_CACHE:
+ err = nvmap_cache_maint(filp, (void __user *)arg);
+ break;
+
+ default:
+ return -ENOTTY;
+ }
+ return err;
+}
+
+static int nvmap_release(struct inode *inode, struct file *file)
+{
+ return 0;
+}
+
+static int nvmap_open(struct inode *inode, struct file *filp)
+{
+ /* eliminate read, write and llseek support on this node */
+ int ret;
+
+ ret = nonseekable_open(inode, filp);
+ if (unlikely(ret))
+ return ret;
+
+ filp->f_mapping->backing_dev_info = &nvmap_bdi;
+
+ filp->private_data = NULL;
+
+ return 0;
+}
+
+static int nvmap_map_into_caller_ptr(struct file *filp, void __user *arg)
+{
+ struct nvmem_map_caller op;
+ struct nvmap_vma_priv *priv;
+ struct vm_area_struct *vma;
+ NvRmMemHandle hmem;
+ int err = 0;
+
+ err = copy_from_user(&op, arg, sizeof(op));
+ if (err)
+ return err;
+
+ hmem = (NvRmMemHandle)op.handle;
+ if (!hmem)
+ return -EINVAL;
+
+ down_read(&current->mm->mmap_sem);
+
+ vma = find_vma(current->mm, (unsigned long)op.addr);
+ if (!vma || !vma->vm_private_data) {
+ err = -ENOMEM;
+ goto out;
+ }
+
+ if (op.offset & ~PAGE_MASK) {
+ err = -EFAULT;
+ goto out;
+ }
+
+ if ((op.offset + op.length) >
+ ((hmem->size+PAGE_SIZE-1)&PAGE_MASK)) {
+ err = -EADDRNOTAVAIL;
+ goto out;
+ }
+
+ priv = (struct nvmap_vma_priv *)vma->vm_private_data;
+ BUG_ON(!priv);
+
+ /* the VMA must exactly match the requested mapping operation, and the
+ * VMA that is targetted must have been created originally by /dev/nvmap
+ */
+ if (((void*)vma->vm_start != op.addr) || (vma->vm_ops != &nvmap_vma_ops) ||
+ (vma->vm_end-vma->vm_start != op.length)) {
+ err = -EPERM;
+ goto out;
+ }
+
+ /* verify that each mmap() system call creates a unique VMA */
+
+ if (priv->hmem && hmem==priv->hmem)
+ goto out;
+ else if (priv->hmem) {
+ err = -EADDRNOTAVAIL;
+ goto out;
+ }
+
+ if (hmem->alignment & ~PAGE_MASK) {
+ err = -EFAULT;
+ goto out;
+ }
+
+ priv->hmem = hmem;
+ priv->offs = op.offset;
+
+ /* if the hmem is not writeback-cacheable, drop back to a page mapping
+ * which will guarantee DMA coherency
+ */
+ if (hmem->coherency != NvOsMemAttribute_WriteBack)
+ vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
+
+ NvRmPrivMemIncrRef(hmem);
+
+ out:
+ up_read(&current->mm->mmap_sem);
+ return err;
+}
+/* Initially, the nvmap mmap system call is used to allocate an inaccessible
+ * region of virtual-address space in the client. A subsequent
+ * NVMAP_IOC_MMAP ioctl will associate each
+ */
+static int nvmap_mmap(struct file *filp, struct vm_area_struct *vma)
+{
+ /* FIXME: drivers which do not support cow seem to be split down the
+ * middle whether to force the VM_SHARED flag, or to return an error
+ * when this flag isn't already set (i.e., MAP_PRIVATE).
+ */
+ struct nvmap_vma_priv *priv;
+
+ vma->vm_private_data = NULL;
+
+ priv = kzalloc(sizeof(*priv),GFP_KERNEL);
+ if (!priv)
+ return -ENOMEM;
+
+ priv->offs = 0;
+ priv->hmem = NULL;
+ atomic_set(&priv->ref, 1);
+
+ vma->vm_flags |= VM_SHARED;
+ vma->vm_flags |= (VM_IO | VM_DONTEXPAND | VM_PFNMAP | VM_RESERVED);
+ vma->vm_ops = &nvmap_vma_ops;
+ vma->vm_private_data = priv;
+
+ return 0;
+}
+
+static int nvmap_cache_maint(struct file *filp, void __user *arg)
+{
+ struct nvmem_cache_op op;
+ int err = 0;
+ struct vm_area_struct *vma;
+ struct nvmap_vma_priv *priv;
+ size_t offs;
+ size_t end;
+
+ err = copy_from_user(&op, arg, sizeof(op));
+ if (err)
+ return err;
+
+ if (!op.handle || !op.addr ||
+ op.op<NVMEM_CACHE_OP_WB ||
+ op.op>NVMEM_CACHE_OP_WB_INV) {
+ return -EINVAL;
+ }
+
+ vma = find_vma(current->active_mm, (unsigned long)op.addr);
+ if (!vma || vma->vm_ops!=&nvmap_vma_ops ||
+ (unsigned long)op.addr + op.len > vma->vm_end) {
+ err = -EADDRNOTAVAIL;
+ goto out;
+ }
+
+ priv = (struct nvmap_vma_priv *)vma->vm_private_data;
+
+ if (priv->hmem != (NvRmMemHandle)op.handle) {
+ err = -EFAULT;
+ goto out;
+ }
+
+ /* don't waste time on cache maintenance if the handle isn't cached */
+ if (priv->hmem->coherency != NvOsMemAttribute_WriteBack)
+ goto out;
+
+ offs = (unsigned long)op.addr - vma->vm_start;
+ end = offs + op.len;
+
+ while (offs < end) {
+ unsigned long inner_addr = 0;
+ unsigned long outer_addr;
+ size_t count;
+ int clean = 0;
+
+ /* if kernel mappings exist already for the memory handle, use
+ * the kernel's mapping for cache maintenance */
+ if (priv->hmem->VirtualAddress) {
+ inner_addr = (unsigned long)priv->hmem->VirtualAddress + offs;
+ clean = 1;
+ }
+ else {
+ /* fixme: this is overly-conservative; not all pages in the
+ * range might have be mapped, since pages are faulted on-demand.
+ * the fault handler could be extended to mark pages as accessed /
+ * dirty so that maintenance ops are only performed on pages which
+ * need it
+ */
+ if (priv->hmem->hPageHandle) {
+ struct page *page;
+ page = NvOsPageGetPage(priv->hmem->hPageHandle, offs);
+ inner_addr = (unsigned long)page_address(page);
+ inner_addr += (offs & ~PAGE_MASK);
+ clean = 1;
+ }
+ if (!inner_addr) {
+ /* this case only triggers for rm-managed memory apertures
+ * (carveout, iram) for handles which do not have a mirror
+ * mapping in the kernel.
+ *
+ * use follow_page, to ensure that we only try to clean
+ * the cache using addresses that have been mapped */
+ inner_addr = vma->vm_start + offs;
+ if (follow_page(vma, inner_addr, FOLL_WRITE))
+ clean = 1;
+ }
+ }
+
+ switch (priv->hmem->heap) {
+ case NvRmHeap_ExternalCarveOut:
+ case NvRmHeap_IRam:
+ outer_addr = priv->hmem->PhysicalAddress+offs;
+ break;
+
+ case NvRmHeap_GART:
+ case NvRmHeap_External:
+ BUG_ON(!priv->hmem->hPageHandle);
+ outer_addr = NvOsPageAddress(priv->hmem->hPageHandle, offs);
+ break;
+ default:
+ BUG();
+ }
+
+ count = PAGE_SIZE - (inner_addr & ~PAGE_MASK);
+ if (clean) {
+ switch (op.op) {
+ case NVMEM_CACHE_OP_WB:
+ smp_dma_clean_range((void*)inner_addr,(void*) inner_addr+count);
+ outer_clean_range(outer_addr, outer_addr+count);
+ break;
+ case NVMEM_CACHE_OP_INV:
+ smp_dma_inv_range((void*)inner_addr,(void*) inner_addr+count);
+ outer_inv_range(outer_addr, outer_addr+count);
+ break;
+ case NVMEM_CACHE_OP_WB_INV:
+ smp_dma_flush_range((void*)inner_addr, (void*)inner_addr+count);
+ outer_flush_range(outer_addr, outer_addr+count);
+ break;
+ }
+ }
+ offs += count;
+ }
+
+ out:
+ return err;
+}
+
+static int nvmap_rw_handle(struct file *filp, int is_read,
+ void __user* arg)
+{
+ struct nvmem_rw_handle op;
+ NvRmMemHandle hmem;
+ uintptr_t user_addr, hmem_offs;
+ int err = 0;
+
+ err = copy_from_user(&op, arg, sizeof(op));
+ if (err)
+ return err;
+
+ if (!op.handle || !op.addr || !op.count || !op.elem_size)
+ return -EINVAL;
+
+ hmem = (NvRmMemHandle)op.handle;
+
+ if (op.elem_size == op.hmem_stride &&
+ op.elem_size == op.user_stride)
+ {
+ op.elem_size *= op.count;
+ op.hmem_stride = op.elem_size;
+ op.user_stride = op.elem_size;
+ op.count = 1;
+ }
+
+ user_addr = (uintptr_t)op.addr;
+ hmem_offs = (uintptr_t)op.offset;
+
+ while (op.count--)
+ {
+ unsigned long remain;
+ unsigned long l_hmem_offs = hmem_offs;
+ unsigned long l_user_addr = user_addr;
+
+ remain = op.elem_size;
+
+ while (remain) {
+ void *hmemp;
+ unsigned long hmem_phys;
+ int needs_unmap = 1;
+ unsigned long bytes;
+
+ bytes = min(remain, PAGE_SIZE-(l_hmem_offs & ~PAGE_MASK));
+ bytes = min(bytes, PAGE_SIZE-(l_user_addr & ~PAGE_MASK));
+
+ if (hmem->VirtualAddress) {
+ hmemp = (void*)((uintptr_t)hmem->VirtualAddress + l_hmem_offs);
+ needs_unmap = 0;
+ }
+ else if (hmem->hPageHandle) {
+ unsigned long addr = l_hmem_offs & PAGE_MASK;
+
+ if (hmem->coherency == NvOsMemAttribute_WriteBack) {
+ struct page *os_page;
+ os_page = NvOsPageGetPage(hmem->hPageHandle, addr);
+ hmemp = page_address(os_page);
+ needs_unmap = 0;
+ }
+ else {
+ addr = NvOsPageAddress(hmem->hPageHandle, addr);
+ hmemp = ioremap_wc(addr, PAGE_SIZE);
+ }
+ hmemp += (l_hmem_offs & ~PAGE_MASK);
+ hmem_phys = NvOsPageAddress(hmem->hPageHandle, l_hmem_offs);
+ }
+ else {
+ uintptr_t offs = hmem->PhysicalAddress + l_hmem_offs;
+ uintptr_t base = offs & PAGE_MASK;
+
+ if (hmem->coherency == NvOsMemAttribute_WriteBack)
+ hmemp = ioremap_cached(base, PAGE_SIZE);
+ else
+ hmemp = ioremap_wc(base, PAGE_SIZE);
+
+ hmemp += (offs & ~PAGE_MASK);
+ hmem_phys = offs;
+ }
+
+ if (is_read) {
+ BUG_ON(!access_ok(VERIFY_WRITE, (void *)l_user_addr, bytes));
+ copy_to_user((void *)l_user_addr, hmemp, bytes);
+ }
+ else {
+ BUG_ON(!access_ok(VERIFY_READ, (void*)l_user_addr, bytes));
+ copy_from_user(hmemp, (void*)l_user_addr, bytes);
+ }
+
+ if (hmem->coherency == NvOsMemAttribute_WriteBack)
+ nvmap_flush(hmemp, hmem_phys, bytes);
+
+ if (needs_unmap) {
+ iounmap((void *)((uintptr_t)hmemp & PAGE_MASK));
+ }
+
+ remain -= bytes;
+ l_hmem_offs += bytes;
+ l_user_addr += bytes;
+ }
+
+ user_addr += op.user_stride;
+ hmem_offs += op.hmem_stride;
+ }
+ dmb();
+
+ return err;
+}
+
+static int nvmap_probe(struct platform_device *pdev)
+{
+ int e = 0;
+ static int id_count = 0;
+
+ /* only one nvmap device can be probed today */
+ if (id_count)
+ return -EINVAL;
+
+ e = misc_register(&nvmap_device.dev);
+
+ if (e<0)
+ nvmap_device.dev.minor = MISC_DYNAMIC_MINOR;
+ else
+ id_count++;
+
+ return e;
+}
+
+static int nvmap_remove(struct platform_device *pdev)
+{
+ if (nvmap_device.dev.minor != MISC_DYNAMIC_MINOR)
+ misc_deregister(&nvmap_device.dev);
+ nvmap_device.dev.minor = MISC_DYNAMIC_MINOR;
+
+ return 0;
+}
+
+static struct platform_driver nvmap_driver = {
+ .probe = nvmap_probe,
+ .remove = nvmap_remove,
+ .driver = { .name = "nvmap_drv" }
+};
+
+static int __init nvmap_init(void)
+{
+ int err = bdi_init(&nvmap_bdi);
+ if (err)
+ return err;
+
+ return platform_driver_register(&nvmap_driver);
+}
+
+static void __exit nvmap_deinit(void) {
+ platform_driver_unregister(&nvmap_driver);
+}
+
+module_init(nvmap_init);
+module_exit(nvmap_deinit);