diff options
author | Gary King <gking@nvidia.com> | 2009-12-14 14:54:44 -0800 |
---|---|---|
committer | Gary King <gking@nvidia.com> | 2009-12-14 14:54:44 -0800 |
commit | 5e9838b7007e6bdf16f65a017785d261fecf0c2f (patch) | |
tree | 86e62f239f80d32711420c9075273d75528abe30 | |
parent | 4c604b35038adcafe5d6cd52d8492d46711d00de (diff) |
char: add Tegra nvmap device driver
adds a driver used by NVIDIA Tegra SoCs for mapping system memory allocated
for use by graphics devices into user space, and for performing cache
maintenance operations on the mapped memory
-rw-r--r-- | drivers/char/Kconfig | 10 | ||||
-rw-r--r-- | drivers/char/Makefile | 1 | ||||
-rw-r--r-- | drivers/char/nvmap.c | 656 |
3 files changed, 667 insertions, 0 deletions
diff --git a/drivers/char/Kconfig b/drivers/char/Kconfig index 438f292820af..8230d03b3706 100644 --- a/drivers/char/Kconfig +++ b/drivers/char/Kconfig @@ -110,6 +110,16 @@ config DEVKMEM kind of kernel debugging operations. When in doubt, say "N". +config DEVNVMAP + bool "/dev/nvmap device support" + depends on ARCH_TEGRA && MACH_TEGRA_GENERIC + default y + help + Say Y here if you want to support the /dev/nvmap device. This device + is used by NVIDIA Tegra graphics and multimedia drivers for managing + graphics memory. + + config SERIAL_NONSTANDARD bool "Non-standard serial port support" depends on HAS_IOMEM diff --git a/drivers/char/Makefile b/drivers/char/Makefile index 5ab656b57a15..9f1264860a76 100644 --- a/drivers/char/Makefile +++ b/drivers/char/Makefile @@ -9,6 +9,7 @@ FONTMAPFILE = cp437.uni obj-y += mem.o random.o tty_io.o n_tty.o tty_ioctl.o tty_ldisc.o tty_buffer.o tty_port.o +obj-$(CONFIG_DEVNVMAP) += nvmap.o obj-$(CONFIG_LEGACY_PTYS) += pty.o obj-$(CONFIG_UNIX98_PTYS) += pty.o obj-y += misc.o diff --git a/drivers/char/nvmap.c b/drivers/char/nvmap.c new file mode 100644 index 000000000000..e4b10fdd4826 --- /dev/null +++ b/drivers/char/nvmap.c @@ -0,0 +1,656 @@ +/* + * drivers/char/nvmap.c + * + * Memory mapping driver for Tegra anonymous memory handles + * + * Copyright (c) 2009, NVIDIA Corporation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#include <linux/module.h> +#include <linux/miscdevice.h> +#include <linux/platform_device.h> +#include <linux/mm.h> +#include <linux/mman.h> +#include <linux/uaccess.h> +#include <linux/backing-dev.h> +#include <linux/device.h> +#include <linux/highmem.h> +#include <linux/smp_lock.h> +#include <linux/pagemap.h> +#include <linux/sched.h> +#include <linux/io.h> +#include "../nvrm/nvrmkernel/core/ap15/ap15rm_private.h" +#include "linux/nvmem_ioctl.h" +#include "nvcommon.h" +#include "nvrm_memmgr.h" +#include "nvrm_memmgr_private.h" + +static void nvmap_vma_open(struct vm_area_struct *vma); + +static void nvmap_vma_close(struct vm_area_struct *vma); + +static int nvmap_vma_fault(struct vm_area_struct *vma, struct vm_fault *vmf); + +static int nvmap_open(struct inode *inode, struct file *filp); + +static int nvmap_release(struct inode *inode, struct file *file); + +static int nvmap_mmap(struct file *filp, struct vm_area_struct *vma); + +static long nvmap_ioctl(struct file *filp, + unsigned int cmd, unsigned long arg); + +static void nvmap_clean_handle(NvRmMemHandle hmem, size_t offs, size_t len); + +static int nvmap_cache_maint(struct file *filp, void __user *arg); + +static int nvmap_map_into_caller_ptr(struct file *filp, void __user *arg); + +static int nvmap_rw_handle(struct file *filp, int is_read, + void __user* arg); + +static struct backing_dev_info nvmap_bdi = { + .ra_pages = 0, + .capabilities = (BDI_CAP_NO_ACCT_AND_WRITEBACK | + BDI_CAP_READ_MAP | BDI_CAP_WRITE_MAP), +}; + +static struct vm_operations_struct nvmap_vma_ops = { + .open = nvmap_vma_open, + .close = nvmap_vma_close, + .fault = nvmap_vma_fault, +}; + +static struct file_operations nvmap_file_ops = { + .owner = THIS_MODULE, + .open = nvmap_open, + .release = nvmap_release, + .unlocked_ioctl = nvmap_ioctl, + .mmap = nvmap_mmap +}; + +struct nvmap_vma_priv { + NvRmMemHandle hmem; + size_t offs; + atomic_t ref; +}; + +enum { + NVMAP_DEV_DRAM = 0, + NVMAP_DEV_IRAM, + NVMAP_DEV_COUNT, +}; + +static struct { + struct miscdevice dev; + umode_t mode; +} nvmap_device = { + + .dev = { + .name = "nvmap", + .fops = &nvmap_file_ops, + .minor = MISC_DYNAMIC_MINOR, + }, + .mode = S_IRUGO | S_IWUGO, +}; + +/* to ensure that the backing store for the VMA isn't freed while a fork'd + * reference still exists, nvmap_vma_open increments the reference count on + * the handle, and nvmap_vma_close decrements it. alternatively, we could + * disallow copying of the vma, or behave like pmem and zap the pages. FIXME. +*/ +static void nvmap_vma_open(struct vm_area_struct *vma) +{ + struct nvmap_vma_priv *priv; + + priv = (struct nvmap_vma_priv *)vma->vm_private_data; + + BUG_ON(!priv); + + atomic_inc(&priv->ref); +} + +static void nvmap_vma_close(struct vm_area_struct *vma) { + struct nvmap_vma_priv *priv = (struct nvmap_vma_priv *)vma->vm_private_data; + + if (priv && !atomic_dec_return(&priv->ref)) { + if (priv->hmem) { + size_t offs = (vma->vm_pgoff << PAGE_SHIFT) + priv->offs; + size_t len = vma->vm_end - vma->vm_start; + nvmap_clean_handle(priv->hmem, offs, len); + NvRmMemHandleFree(priv->hmem); + } + kfree(priv); + } + vma->vm_private_data = NULL; +} + +extern struct page *NvOsPageGetPage(NvOsPageAllocHandle, size_t); +#define nvmap_range(x,y) (x), (x)+(y) + +#define is_same_page(a, b) \ + ((unsigned long)(a)>>PAGE_SHIFT == (unsigned long)(b)>>PAGE_SHIFT) + +static inline void nvmap_flush(void* kva, unsigned long phys, size_t len) +{ + BUG_ON(!is_same_page(kva, kva+len-1)); + smp_dma_flush_range(nvmap_range(kva, len)); + outer_flush_range(nvmap_range(phys, len)); +} + +static void nvmap_clean_handle(NvRmMemHandle hmem, size_t start, size_t len) +{ + if (hmem->coherency != NvOsMemAttribute_WriteBack) + return; + + if (hmem->hPageHandle) { + size_t offs; + size_t end = start + len; + for (offs=start; offs<end; offs+=PAGE_SIZE) { + size_t bytes = min((size_t)end-offs, (size_t)PAGE_SIZE); + struct page *page = NvOsPageGetPage(hmem->hPageHandle, offs); + void *ptr = page_address(page) + (offs & ~PAGE_MASK); + + smp_dma_clean_range(nvmap_range(ptr, bytes)); + outer_clean_range(nvmap_range(__pa(ptr), bytes)); + } + } + else if (hmem->VirtualAddress && hmem->PhysicalAddress) { + smp_dma_clean_range(nvmap_range(hmem->VirtualAddress, hmem->size)); + outer_clean_range(nvmap_range(hmem->PhysicalAddress, hmem->size)); + } +} + +static int nvmap_vma_fault(struct vm_area_struct *vma, struct vm_fault *vmf) +{ + struct nvmap_vma_priv *priv; + unsigned long pfn; + unsigned long offs = (unsigned long)(vmf->virtual_address - vma->vm_start); + + priv = (struct nvmap_vma_priv *)vma->vm_private_data; + if (!priv || !priv->hmem) + return VM_FAULT_SIGBUS; + + offs += priv->offs; + /* if the VMA was split for some reason, vm_pgoff will be the VMA's + * offset from the original VMA */ + offs += (vma->vm_pgoff << PAGE_SHIFT); + + if (offs >= priv->hmem->size) + return VM_FAULT_SIGBUS; + + switch (priv->hmem->heap) { + case NvRmHeap_ExternalCarveOut: + case NvRmHeap_IRam: + pfn = ((priv->hmem->PhysicalAddress+offs) >> PAGE_SHIFT); + break; + + case NvRmHeap_GART: + case NvRmHeap_External: + if (!priv->hmem->hPageHandle) + return VM_FAULT_SIGBUS; + pfn = NvOsPageAddress(priv->hmem->hPageHandle, offs) >> PAGE_SHIFT; + break; + + default: + return VM_FAULT_SIGBUS; + } + vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn); + return VM_FAULT_NOPAGE; +} + +static long nvmap_ioctl(struct file *filp, + unsigned int cmd, unsigned long arg) +{ + int err = 0; + + if (_IOC_TYPE(cmd) != NVMEM_IOC_MAGIC) + return -ENOTTY; + + if (_IOC_NR(cmd) > NVMEM_IOC_MAXNR) + return -ENOTTY; + + if (_IOC_DIR(cmd) & _IOC_READ) + err = !access_ok(VERIFY_WRITE, (void __user *)arg, _IOC_SIZE(cmd)); + if (_IOC_DIR(cmd) & _IOC_WRITE) + err = !access_ok(VERIFY_READ, (void __user *)arg, _IOC_SIZE(cmd)); + + if (err) + return -EFAULT; + + switch (cmd) { + case NVMEM_IOC_MMAP: + err = nvmap_map_into_caller_ptr(filp, (void __user *)arg); + break; + + case NVMEM_IOC_WRITE: + case NVMEM_IOC_READ: + err = nvmap_rw_handle(filp, cmd==NVMEM_IOC_READ, (void __user *)arg); + break; + + case NVMEM_IOC_CACHE: + err = nvmap_cache_maint(filp, (void __user *)arg); + break; + + default: + return -ENOTTY; + } + return err; +} + +static int nvmap_release(struct inode *inode, struct file *file) +{ + return 0; +} + +static int nvmap_open(struct inode *inode, struct file *filp) +{ + /* eliminate read, write and llseek support on this node */ + int ret; + + ret = nonseekable_open(inode, filp); + if (unlikely(ret)) + return ret; + + filp->f_mapping->backing_dev_info = &nvmap_bdi; + + filp->private_data = NULL; + + return 0; +} + +static int nvmap_map_into_caller_ptr(struct file *filp, void __user *arg) +{ + struct nvmem_map_caller op; + struct nvmap_vma_priv *priv; + struct vm_area_struct *vma; + NvRmMemHandle hmem; + int err = 0; + + err = copy_from_user(&op, arg, sizeof(op)); + if (err) + return err; + + hmem = (NvRmMemHandle)op.handle; + if (!hmem) + return -EINVAL; + + down_read(¤t->mm->mmap_sem); + + vma = find_vma(current->mm, (unsigned long)op.addr); + if (!vma || !vma->vm_private_data) { + err = -ENOMEM; + goto out; + } + + if (op.offset & ~PAGE_MASK) { + err = -EFAULT; + goto out; + } + + if ((op.offset + op.length) > + ((hmem->size+PAGE_SIZE-1)&PAGE_MASK)) { + err = -EADDRNOTAVAIL; + goto out; + } + + priv = (struct nvmap_vma_priv *)vma->vm_private_data; + BUG_ON(!priv); + + /* the VMA must exactly match the requested mapping operation, and the + * VMA that is targetted must have been created originally by /dev/nvmap + */ + if (((void*)vma->vm_start != op.addr) || (vma->vm_ops != &nvmap_vma_ops) || + (vma->vm_end-vma->vm_start != op.length)) { + err = -EPERM; + goto out; + } + + /* verify that each mmap() system call creates a unique VMA */ + + if (priv->hmem && hmem==priv->hmem) + goto out; + else if (priv->hmem) { + err = -EADDRNOTAVAIL; + goto out; + } + + if (hmem->alignment & ~PAGE_MASK) { + err = -EFAULT; + goto out; + } + + priv->hmem = hmem; + priv->offs = op.offset; + + /* if the hmem is not writeback-cacheable, drop back to a page mapping + * which will guarantee DMA coherency + */ + if (hmem->coherency != NvOsMemAttribute_WriteBack) + vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); + + NvRmPrivMemIncrRef(hmem); + + out: + up_read(¤t->mm->mmap_sem); + return err; +} +/* Initially, the nvmap mmap system call is used to allocate an inaccessible + * region of virtual-address space in the client. A subsequent + * NVMAP_IOC_MMAP ioctl will associate each + */ +static int nvmap_mmap(struct file *filp, struct vm_area_struct *vma) +{ + /* FIXME: drivers which do not support cow seem to be split down the + * middle whether to force the VM_SHARED flag, or to return an error + * when this flag isn't already set (i.e., MAP_PRIVATE). + */ + struct nvmap_vma_priv *priv; + + vma->vm_private_data = NULL; + + priv = kzalloc(sizeof(*priv),GFP_KERNEL); + if (!priv) + return -ENOMEM; + + priv->offs = 0; + priv->hmem = NULL; + atomic_set(&priv->ref, 1); + + vma->vm_flags |= VM_SHARED; + vma->vm_flags |= (VM_IO | VM_DONTEXPAND | VM_PFNMAP | VM_RESERVED); + vma->vm_ops = &nvmap_vma_ops; + vma->vm_private_data = priv; + + return 0; +} + +static int nvmap_cache_maint(struct file *filp, void __user *arg) +{ + struct nvmem_cache_op op; + int err = 0; + struct vm_area_struct *vma; + struct nvmap_vma_priv *priv; + size_t offs; + size_t end; + + err = copy_from_user(&op, arg, sizeof(op)); + if (err) + return err; + + if (!op.handle || !op.addr || + op.op<NVMEM_CACHE_OP_WB || + op.op>NVMEM_CACHE_OP_WB_INV) { + return -EINVAL; + } + + vma = find_vma(current->active_mm, (unsigned long)op.addr); + if (!vma || vma->vm_ops!=&nvmap_vma_ops || + (unsigned long)op.addr + op.len > vma->vm_end) { + err = -EADDRNOTAVAIL; + goto out; + } + + priv = (struct nvmap_vma_priv *)vma->vm_private_data; + + if (priv->hmem != (NvRmMemHandle)op.handle) { + err = -EFAULT; + goto out; + } + + /* don't waste time on cache maintenance if the handle isn't cached */ + if (priv->hmem->coherency != NvOsMemAttribute_WriteBack) + goto out; + + offs = (unsigned long)op.addr - vma->vm_start; + end = offs + op.len; + + while (offs < end) { + unsigned long inner_addr = 0; + unsigned long outer_addr; + size_t count; + int clean = 0; + + /* if kernel mappings exist already for the memory handle, use + * the kernel's mapping for cache maintenance */ + if (priv->hmem->VirtualAddress) { + inner_addr = (unsigned long)priv->hmem->VirtualAddress + offs; + clean = 1; + } + else { + /* fixme: this is overly-conservative; not all pages in the + * range might have be mapped, since pages are faulted on-demand. + * the fault handler could be extended to mark pages as accessed / + * dirty so that maintenance ops are only performed on pages which + * need it + */ + if (priv->hmem->hPageHandle) { + struct page *page; + page = NvOsPageGetPage(priv->hmem->hPageHandle, offs); + inner_addr = (unsigned long)page_address(page); + inner_addr += (offs & ~PAGE_MASK); + clean = 1; + } + if (!inner_addr) { + /* this case only triggers for rm-managed memory apertures + * (carveout, iram) for handles which do not have a mirror + * mapping in the kernel. + * + * use follow_page, to ensure that we only try to clean + * the cache using addresses that have been mapped */ + inner_addr = vma->vm_start + offs; + if (follow_page(vma, inner_addr, FOLL_WRITE)) + clean = 1; + } + } + + switch (priv->hmem->heap) { + case NvRmHeap_ExternalCarveOut: + case NvRmHeap_IRam: + outer_addr = priv->hmem->PhysicalAddress+offs; + break; + + case NvRmHeap_GART: + case NvRmHeap_External: + BUG_ON(!priv->hmem->hPageHandle); + outer_addr = NvOsPageAddress(priv->hmem->hPageHandle, offs); + break; + default: + BUG(); + } + + count = PAGE_SIZE - (inner_addr & ~PAGE_MASK); + if (clean) { + switch (op.op) { + case NVMEM_CACHE_OP_WB: + smp_dma_clean_range((void*)inner_addr,(void*) inner_addr+count); + outer_clean_range(outer_addr, outer_addr+count); + break; + case NVMEM_CACHE_OP_INV: + smp_dma_inv_range((void*)inner_addr,(void*) inner_addr+count); + outer_inv_range(outer_addr, outer_addr+count); + break; + case NVMEM_CACHE_OP_WB_INV: + smp_dma_flush_range((void*)inner_addr, (void*)inner_addr+count); + outer_flush_range(outer_addr, outer_addr+count); + break; + } + } + offs += count; + } + + out: + return err; +} + +static int nvmap_rw_handle(struct file *filp, int is_read, + void __user* arg) +{ + struct nvmem_rw_handle op; + NvRmMemHandle hmem; + uintptr_t user_addr, hmem_offs; + int err = 0; + + err = copy_from_user(&op, arg, sizeof(op)); + if (err) + return err; + + if (!op.handle || !op.addr || !op.count || !op.elem_size) + return -EINVAL; + + hmem = (NvRmMemHandle)op.handle; + + if (op.elem_size == op.hmem_stride && + op.elem_size == op.user_stride) + { + op.elem_size *= op.count; + op.hmem_stride = op.elem_size; + op.user_stride = op.elem_size; + op.count = 1; + } + + user_addr = (uintptr_t)op.addr; + hmem_offs = (uintptr_t)op.offset; + + while (op.count--) + { + unsigned long remain; + unsigned long l_hmem_offs = hmem_offs; + unsigned long l_user_addr = user_addr; + + remain = op.elem_size; + + while (remain) { + void *hmemp; + unsigned long hmem_phys; + int needs_unmap = 1; + unsigned long bytes; + + bytes = min(remain, PAGE_SIZE-(l_hmem_offs & ~PAGE_MASK)); + bytes = min(bytes, PAGE_SIZE-(l_user_addr & ~PAGE_MASK)); + + if (hmem->VirtualAddress) { + hmemp = (void*)((uintptr_t)hmem->VirtualAddress + l_hmem_offs); + needs_unmap = 0; + } + else if (hmem->hPageHandle) { + unsigned long addr = l_hmem_offs & PAGE_MASK; + + if (hmem->coherency == NvOsMemAttribute_WriteBack) { + struct page *os_page; + os_page = NvOsPageGetPage(hmem->hPageHandle, addr); + hmemp = page_address(os_page); + needs_unmap = 0; + } + else { + addr = NvOsPageAddress(hmem->hPageHandle, addr); + hmemp = ioremap_wc(addr, PAGE_SIZE); + } + hmemp += (l_hmem_offs & ~PAGE_MASK); + hmem_phys = NvOsPageAddress(hmem->hPageHandle, l_hmem_offs); + } + else { + uintptr_t offs = hmem->PhysicalAddress + l_hmem_offs; + uintptr_t base = offs & PAGE_MASK; + + if (hmem->coherency == NvOsMemAttribute_WriteBack) + hmemp = ioremap_cached(base, PAGE_SIZE); + else + hmemp = ioremap_wc(base, PAGE_SIZE); + + hmemp += (offs & ~PAGE_MASK); + hmem_phys = offs; + } + + if (is_read) { + BUG_ON(!access_ok(VERIFY_WRITE, (void *)l_user_addr, bytes)); + copy_to_user((void *)l_user_addr, hmemp, bytes); + } + else { + BUG_ON(!access_ok(VERIFY_READ, (void*)l_user_addr, bytes)); + copy_from_user(hmemp, (void*)l_user_addr, bytes); + } + + if (hmem->coherency == NvOsMemAttribute_WriteBack) + nvmap_flush(hmemp, hmem_phys, bytes); + + if (needs_unmap) { + iounmap((void *)((uintptr_t)hmemp & PAGE_MASK)); + } + + remain -= bytes; + l_hmem_offs += bytes; + l_user_addr += bytes; + } + + user_addr += op.user_stride; + hmem_offs += op.hmem_stride; + } + dmb(); + + return err; +} + +static int nvmap_probe(struct platform_device *pdev) +{ + int e = 0; + static int id_count = 0; + + /* only one nvmap device can be probed today */ + if (id_count) + return -EINVAL; + + e = misc_register(&nvmap_device.dev); + + if (e<0) + nvmap_device.dev.minor = MISC_DYNAMIC_MINOR; + else + id_count++; + + return e; +} + +static int nvmap_remove(struct platform_device *pdev) +{ + if (nvmap_device.dev.minor != MISC_DYNAMIC_MINOR) + misc_deregister(&nvmap_device.dev); + nvmap_device.dev.minor = MISC_DYNAMIC_MINOR; + + return 0; +} + +static struct platform_driver nvmap_driver = { + .probe = nvmap_probe, + .remove = nvmap_remove, + .driver = { .name = "nvmap_drv" } +}; + +static int __init nvmap_init(void) +{ + int err = bdi_init(&nvmap_bdi); + if (err) + return err; + + return platform_driver_register(&nvmap_driver); +} + +static void __exit nvmap_deinit(void) { + platform_driver_unregister(&nvmap_driver); +} + +module_init(nvmap_init); +module_exit(nvmap_deinit); |