tegra video: add driver for host1x hardware

The graphics hardware modules on Tegra family of SOCs are accessed via
the host1x dma and synchronization engine. This driver exposes an
userspace interface for submitting command buffers to 2d, 3d, display
and mpe hardware modules and accessing the module register apertures
for exclusive use hardware modules.

Additional features of the driver include:
- interrupt-driven hardware module usage synchronization
- automatic clock management for hw modules
- hardware context switching for 3d registers

Change-Id: I693582249597fd307526ff3c7e35889d37406017
Reviewed-on: http://git-master/r/4091
Reviewed-by: Janne Hellsten <jhellsten@nvidia.com>
Tested-by: Janne Hellsten <jhellsten@nvidia.com>

25 files changed, 7784 insertions, 14 deletions

diff --git a/drivers/video/Kconfig b/drivers/video/Kconfig
index fc6750156402..a559ea047b7f 100644
--- a/drivers/video/Kconfig
+++ b/drivers/video/Kconfig
@@ -1056,19 +1056,6 @@ config FB_RIVA_BACKLIGHT
 	help
 	  Say Y here if you want to control the backlight of your display.
 
-config FB_TEGRA
-	boolean "NVIDIA Tegra dummy framebuffer driver"
-	depends on ARCH_TEGRA && FB
-	select FB_CFB_FILLRECT
-	select FB_CFB_COPYAREA
-	select FB_CFB_IMAGEBLIT
-	default FB
-	help
-	  This driver implements a dummy framebuffer device for systems based
-	  on the NVIDIA Tegra family of SoCs. Display initialization is
-	  expected to happen prior to the kernel start-up for this driver to
-	  function.
-
 config FB_I810
 	tristate "Intel 810/815 support (EXPERIMENTAL)"
 	depends on EXPERIMENTAL && FB && PCI && X86_32 && AGP_INTEL
@@ -2175,6 +2162,8 @@ config FB_BROADSHEET
 
 source "drivers/video/omap/Kconfig"
 
+source "drivers/video/tegra/Kconfig"
+
 source "drivers/video/backlight/Kconfig"
 source "drivers/video/display/Kconfig"
 
diff --git a/drivers/video/Makefile b/drivers/video/Makefile
index 927664e8bd59..769aff1399a7 100644
--- a/drivers/video/Makefile
+++ b/drivers/video/Makefile
@@ -38,7 +38,6 @@ obj-$(CONFIG_FB_PM3)		  += pm3fb.o
 obj-$(CONFIG_FB_MATROX)		  += matrox/
 obj-$(CONFIG_FB_RIVA)		  += riva/
 obj-$(CONFIG_FB_NVIDIA)		  += nvidia/
-obj-$(CONFIG_FB_TEGRA)		  += tegra-fb.o
 obj-$(CONFIG_FB_ATY)		  += aty/ macmodes.o
 obj-$(CONFIG_FB_ATY128)		  += aty/ macmodes.o
 obj-$(CONFIG_FB_RADEON)		  += aty/
@@ -129,6 +128,7 @@ obj-$(CONFIG_XEN_FBDEV_FRONTEND)  += xen-fbfront.o
 obj-$(CONFIG_FB_CARMINE)          += carminefb.o
 obj-$(CONFIG_FB_MB862XX)	  += mb862xx/
 obj-$(CONFIG_FB_MSM)              += msm/
+obj-$(CONFIG_FB_TEGRA)		  += tegra/
 
 # Platform or fallback drivers go here
 obj-$(CONFIG_FB_UVESA)            += uvesafb.o
diff --git a/drivers/video/tegra/Kconfig b/drivers/video/tegra/Kconfig
new file mode 100644
index 000000000000..8d44256bc099
--- /dev/null
+++ b/drivers/video/tegra/Kconfig
@@ -0,0 +1,54 @@
+config FB_TEGRA
+	boolean "NVIDIA Tegra graphics support"
+	depends on FB && ARCH_TEGRA
+        default FB
+	help
+	  Graphics services for NVIDIA Tegra products
+
+config FB_TEGRA_DUMMY
+	boolean "Tegra dummy framebuffer driver"
+	depends on FB_TEGRA
+	select FB_CFB_FILLRECT
+	select FB_CFB_COPYAREA
+	select FB_CFB_IMAGEBLIT
+        default FB
+	help
+	  This driver implements a dummy framebuffer device for systems based
+	  on the NVIDIA Tegra family of SoCs. Display initialization is
+	  expected to happen prior to the kernel start-up for this driver to
+	  function.
+
+config FB_TEGRA_GRHOST
+	tristate "Tegra graphics host driver"
+	depends on FB_TEGRA && TEGRA_IOVMM
+        default n
+	help
+	  Driver for the Tegra graphics host hardware.
+
+config DEVNVMAP
+	bool "/dev/nvmap device support"
+	depends on ARCH_TEGRA
+	default y
+	help
+	  Say Y here if you want to support the /dev/nvmap device. This device
+	  is used by NVIDIA Tegra graphics and multimedia drivers for managing
+	  graphics memory.
+
+config DEVNVMAP_PARANOID
+	bool "Validate all user-provided /dev/nvmap object references"
+	depends on DEVNVMAP
+	default n
+	help
+	  Say Y here to enable additional process-level validations and
+	  permissions for /dev/nvmap object references provided via ioctls.
+	  May result in a decrease in performance.
+
+config DEVNVMAP_RECLAIM_UNPINNED_VM
+	bool "Allow /dev/nvmap to reclaim unpinned I/O virtual memory"
+	depends on DEVNVMAP && TEGRA_IOVMM
+	default y
+	help
+	  Say Y here to enable /dev/nvmap to reclaim I/O virtual memory after
+	  it has been unpinned, and re-use it for other objects. This can
+	  allow a larger virtual I/O VM space than would normally be
+	  supported by the hardware, at a slight cost in performance.
diff --git a/drivers/video/tegra/Makefile b/drivers/video/tegra/Makefile
new file mode 100644
index 000000000000..80c35780c6b9
--- /dev/null
+++ b/drivers/video/tegra/Makefile
@@ -0,0 +1,3 @@
+obj-$(CONFIG_FB_TEGRA_GRHOST) += host/
+obj-$(CONFIG_FB_TEGRA_DUMMY) += tegra-fb.o
+obj-$(CONFIG_DEVNVMAP)		+= nvmap.o
diff --git a/drivers/video/tegra/host/Makefile b/drivers/video/tegra/host/Makefile
new file mode 100644
index 000000000000..92d9ba118d1c
--- /dev/null
+++ b/drivers/video/tegra/host/Makefile
@@ -0,0 +1,11 @@
+nvhost-objs = \
+	nvhost_acm.o \
+	nvhost_syncpt.o \
+	nvhost_cdma.o \
+	nvhost_cpuaccess.o \
+	nvhost_intr.o \
+	nvhost_channel.o \
+	nvhost_3dctx.o \
+	nvhost_dev.o
+
+obj-$(CONFIG_FB_TEGRA_GRHOST) += nvhost.o
\ No newline at end of file
diff --git a/drivers/video/tegra/host/nvhost_3dctx.c b/drivers/video/tegra/host/nvhost_3dctx.c
new file mode 100644
index 000000000000..c7c4c30ac9d3
--- /dev/null
+++ b/drivers/video/tegra/host/nvhost_3dctx.c
@@ -0,0 +1,490 @@
+/*
+ * drivers/video/tegra/host/nvhost_3dctx.c
+ *
+ * Tegra Graphics Host 3d hardware context
+ *
+ * Copyright (c) 2010, NVIDIA Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+#include "nvhost_hwctx.h"
+#include "nvhost_dev.h"
+
+const struct hwctx_reginfo ctxsave_regs_3d[] = {
+	HWCTX_REGINFO(0xe00, 16, DIRECT),
+	HWCTX_REGINFO(0xe10, 16, DIRECT),
+	HWCTX_REGINFO(0xe20, 1, DIRECT),
+	HWCTX_REGINFO(0xe21, 1, DIRECT),
+	HWCTX_REGINFO(0xe22, 1, DIRECT),
+	HWCTX_REGINFO(0xe25, 1, DIRECT),
+	HWCTX_REGINFO(0xe26, 1, DIRECT),
+	HWCTX_REGINFO(0xe28, 2, DIRECT),
+	HWCTX_REGINFO(0xe2a, 1, DIRECT),
+	HWCTX_REGINFO(0x1, 1, DIRECT),
+	HWCTX_REGINFO(0x2, 1, DIRECT),
+	HWCTX_REGINFO(0xc, 2, DIRECT),
+	HWCTX_REGINFO(0xe, 2, DIRECT),
+	HWCTX_REGINFO(0x10, 2, DIRECT),
+	HWCTX_REGINFO(0x12, 2, DIRECT),
+	HWCTX_REGINFO(0x14, 2, DIRECT),
+	HWCTX_REGINFO(0x100, 32, DIRECT),
+	HWCTX_REGINFO(0x120, 1, DIRECT),
+	HWCTX_REGINFO(0x121, 1, DIRECT),
+	HWCTX_REGINFO(0x124, 1, DIRECT),
+	HWCTX_REGINFO(0x125, 1, DIRECT),
+	HWCTX_REGINFO(0x200, 1, DIRECT),
+	HWCTX_REGINFO(0x201, 1, DIRECT),
+	HWCTX_REGINFO(0x202, 1, DIRECT),
+	HWCTX_REGINFO(0x203, 1, DIRECT),
+	HWCTX_REGINFO(0x204, 1, DIRECT),
+	HWCTX_REGINFO(0x207, 1024, INDIRECT),
+	HWCTX_REGINFO(0x209, 1, DIRECT),
+	HWCTX_REGINFO(0x300, 64, DIRECT),
+	HWCTX_REGINFO(0x343, 1, DIRECT),
+	HWCTX_REGINFO(0x344, 1, DIRECT),
+	HWCTX_REGINFO(0x345, 1, DIRECT),
+	HWCTX_REGINFO(0x346, 1, DIRECT),
+	HWCTX_REGINFO(0x347, 1, DIRECT),
+	HWCTX_REGINFO(0x348, 1, DIRECT),
+	HWCTX_REGINFO(0x349, 1, DIRECT),
+	HWCTX_REGINFO(0x34a, 1, DIRECT),
+	HWCTX_REGINFO(0x34b, 1, DIRECT),
+	HWCTX_REGINFO(0x34c, 1, DIRECT),
+	HWCTX_REGINFO(0x34d, 1, DIRECT),
+	HWCTX_REGINFO(0x34e, 1, DIRECT),
+	HWCTX_REGINFO(0x34f, 1, DIRECT),
+	HWCTX_REGINFO(0x350, 1, DIRECT),
+	HWCTX_REGINFO(0x351, 1, DIRECT),
+	HWCTX_REGINFO(0x352, 1, DIRECT),
+	HWCTX_REGINFO(0x353, 1, DIRECT),
+	HWCTX_REGINFO(0x354, 1, DIRECT),
+	HWCTX_REGINFO(0x355, 1, DIRECT),
+	HWCTX_REGINFO(0x356, 1, DIRECT),
+	HWCTX_REGINFO(0x357, 1, DIRECT),
+	HWCTX_REGINFO(0x358, 1, DIRECT),
+	HWCTX_REGINFO(0x359, 1, DIRECT),
+	HWCTX_REGINFO(0x35a, 1, DIRECT),
+	HWCTX_REGINFO(0x35b, 1, DIRECT),
+	HWCTX_REGINFO(0x363, 1, DIRECT),
+	HWCTX_REGINFO(0x364, 1, DIRECT),
+	HWCTX_REGINFO(0x400, 2, DIRECT),
+	HWCTX_REGINFO(0x402, 1, DIRECT),
+	HWCTX_REGINFO(0x403, 1, DIRECT),
+	HWCTX_REGINFO(0x404, 1, DIRECT),
+	HWCTX_REGINFO(0x405, 1, DIRECT),
+	HWCTX_REGINFO(0x406, 1, DIRECT),
+	HWCTX_REGINFO(0x407, 1, DIRECT),
+	HWCTX_REGINFO(0x408, 1, DIRECT),
+	HWCTX_REGINFO(0x409, 1, DIRECT),
+	HWCTX_REGINFO(0x40a, 1, DIRECT),
+	HWCTX_REGINFO(0x40b, 1, DIRECT),
+	HWCTX_REGINFO(0x40c, 1, DIRECT),
+	HWCTX_REGINFO(0x40d, 1, DIRECT),
+	HWCTX_REGINFO(0x40e, 1, DIRECT),
+	HWCTX_REGINFO(0x40f, 1, DIRECT),
+	HWCTX_REGINFO(0x411, 1, DIRECT),
+	HWCTX_REGINFO(0x500, 1, DIRECT),
+	HWCTX_REGINFO(0x501, 1, DIRECT),
+	HWCTX_REGINFO(0x502, 1, DIRECT),
+	HWCTX_REGINFO(0x503, 1, DIRECT),
+	HWCTX_REGINFO(0x520, 32, DIRECT),
+	HWCTX_REGINFO(0x540, 64, INDIRECT),
+	HWCTX_REGINFO(0x600, 0, INDIRECT_OFFSET),
+	HWCTX_REGINFO(0x602, 16, INDIRECT_DATA),
+	HWCTX_REGINFO(0x603, 128, INDIRECT),
+	HWCTX_REGINFO(0x608, 4, DIRECT),
+	HWCTX_REGINFO(0x60e, 1, DIRECT),
+	HWCTX_REGINFO(0x700, 64, INDIRECT),
+	HWCTX_REGINFO(0x710, 16, DIRECT),
+	HWCTX_REGINFO(0x720, 32, DIRECT),
+	HWCTX_REGINFO(0x740, 1, DIRECT),
+	HWCTX_REGINFO(0x741, 1, DIRECT),
+	HWCTX_REGINFO(0x800, 0, INDIRECT_OFFSET),
+	HWCTX_REGINFO(0x802, 16, INDIRECT_DATA),
+	HWCTX_REGINFO(0x803, 512, INDIRECT),
+	HWCTX_REGINFO(0x805, 64, INDIRECT),
+	HWCTX_REGINFO(0x820, 32, DIRECT),
+	HWCTX_REGINFO(0x900, 64, INDIRECT),
+	HWCTX_REGINFO(0x902, 1, DIRECT),
+	HWCTX_REGINFO(0x903, 1, DIRECT),
+	HWCTX_REGINFO(0xa02, 1, DIRECT),
+	HWCTX_REGINFO(0xa03, 1, DIRECT),
+	HWCTX_REGINFO(0xa04, 1, DIRECT),
+	HWCTX_REGINFO(0xa05, 1, DIRECT),
+	HWCTX_REGINFO(0xa06, 1, DIRECT),
+	HWCTX_REGINFO(0xa07, 1, DIRECT),
+	HWCTX_REGINFO(0xa08, 1, DIRECT),
+	HWCTX_REGINFO(0xa09, 1, DIRECT),
+	HWCTX_REGINFO(0xa0a, 1, DIRECT),
+	HWCTX_REGINFO(0xa0b, 1, DIRECT),
+	HWCTX_REGINFO(0x205, 1024, INDIRECT)
+};
+
+
+/*** restore ***/
+
+static unsigned int context_restore_size = 0;
+
+static void restore_begin(u32 *ptr, u32 waitbase)
+{
+	/* set class to host */
+	ptr[0] = nvhost_opcode_setclass(NV_HOST1X_CLASS_ID,
+					NV_CLASS_HOST_INCR_SYNCPT_BASE, 1);
+	/* increment sync point base */
+	ptr[1] = nvhost_class_host_incr_syncpt_base(waitbase, 1);
+	/* set class to 3D */
+	ptr[2] = nvhost_opcode_setclass(NV_GRAPHICS_3D_CLASS_ID, 0, 0);
+	/* program PSEQ_QUAD_ID */
+	ptr[3] = nvhost_opcode_imm(0x545, 0);
+}
+#define RESTORE_BEGIN_SIZE 4
+
+static void restore_end(u32 *ptr, u32 syncpt_id)
+{
+	/* syncpt increment to track restore gather. */
+	ptr[0] = nvhost_opcode_imm(0x0, ((1UL << 8) | (u8)(syncpt_id & 0xff)));
+}
+#define RESTORE_END_SIZE 1
+
+static void restore_direct(u32 *ptr, u32 start_reg, u32 count)
+{
+	ptr[0] = nvhost_opcode_incr(start_reg, count);
+}
+#define RESTORE_DIRECT_SIZE 1
+
+static void restore_indoffset(u32 *ptr, u32 offset_reg, u32 offset)
+{
+	ptr[0] = nvhost_opcode_imm(offset_reg, offset);
+}
+#define RESTORE_INDOFFSET_SIZE 1
+
+static void restore_inddata(u32 *ptr, u32 data_reg, u32 count)
+{
+	ptr[0] = nvhost_opcode_nonincr(data_reg, count);
+}
+#define RESTORE_INDDATA_SIZE 1
+
+static void restore_registers_from_fifo(u32 *ptr, unsigned int count,
+					struct nvhost_channel *channel,
+					unsigned int *pending)
+{
+	void __iomem *chan_regs = channel->aperture;
+	unsigned int entries = *pending;
+	while (count) {
+		unsigned int num;
+
+		while (!entries) {
+			/* query host for number of entries in fifo */
+			entries = nvhost_channel_fifostat_outfentries(
+				readl(chan_regs + HOST1X_CHANNEL_FIFOSTAT));
+			if (!entries)
+				cpu_relax();
+			/* TODO: [ahowe 2010-06-14] timeout */
+		}
+		num = min(entries, count);
+		entries -= num;
+		count -= num;
+
+		while (num & ~0x3) {
+			u32 arr[4];
+			arr[0] = readl(chan_regs + HOST1X_CHANNEL_INDDATA);
+			arr[1] = readl(chan_regs + HOST1X_CHANNEL_INDDATA);
+			arr[2] = readl(chan_regs + HOST1X_CHANNEL_INDDATA);
+			arr[3] = readl(chan_regs + HOST1X_CHANNEL_INDDATA);
+			memcpy(ptr, arr, 4*sizeof(u32));
+			ptr += 4;
+			num -= 4;
+		}
+		while (num--)
+			*ptr++ = readl(chan_regs + HOST1X_CHANNEL_INDDATA);
+	}
+	*pending = entries;
+}
+
+static void setup_restore(u32 *ptr, u32 waitbase)
+{
+	const struct hwctx_reginfo *r;
+	const struct hwctx_reginfo *rend;
+
+	restore_begin(ptr, waitbase);
+	ptr += RESTORE_BEGIN_SIZE;
+
+	r = ctxsave_regs_3d;
+	rend = ctxsave_regs_3d + ARRAY_SIZE(ctxsave_regs_3d);
+	for ( ; r != rend; ++r) {
+		u32 offset = r->offset;
+		u32 count = r->count;
+		switch (r->type) {
+		case HWCTX_REGINFO_DIRECT:
+			restore_direct(ptr, offset, count);
+			ptr += RESTORE_DIRECT_SIZE;
+			break;
+		case HWCTX_REGINFO_INDIRECT:
+			restore_indoffset(ptr, offset, 0);
+			ptr += RESTORE_INDOFFSET_SIZE;
+			restore_inddata(ptr, offset + 1, count);
+			ptr += RESTORE_INDDATA_SIZE;
+			break;
+		case HWCTX_REGINFO_INDIRECT_OFFSET:
+			restore_indoffset(ptr, offset, count);
+			ptr += RESTORE_INDOFFSET_SIZE;
+			continue; /* INDIRECT_DATA follows with real count */
+		case HWCTX_REGINFO_INDIRECT_DATA:
+			restore_inddata(ptr, offset, count);
+			ptr += RESTORE_INDDATA_SIZE;
+			break;
+		}
+		ptr += count;
+	}
+
+	restore_end(ptr, NVSYNCPT_3D);
+	wmb();
+}
+
+/*** save ***/
+
+/* the same context save command sequence is used for all contexts. */
+static struct nvmap_handle *context_save_buf = NULL;
+static u32 context_save_phys = 0;
+static u32 *context_save_ptr = NULL;
+static unsigned int context_save_size = 0;
+
+static void save_begin(u32 *ptr, u32 syncpt_id, u32 waitbase)
+{
+	/* set class to the unit to flush */
+	ptr[0] = nvhost_opcode_setclass(NV_GRAPHICS_3D_CLASS_ID, 0, 0);
+	/*
+	 * Flush pipe and signal context read thread to start reading
+	 * sync point increment
+	 */
+	ptr[1] = nvhost_opcode_imm(0, 0x100 | syncpt_id);
+	ptr[2] = nvhost_opcode_setclass(NV_HOST1X_CLASS_ID,
+					NV_CLASS_HOST_WAIT_SYNCPT_BASE, 1);
+	/* wait for base+1 */
+	ptr[3] = nvhost_class_host_wait_syncpt_base(syncpt_id, waitbase, 1);
+	ptr[4] = nvhost_opcode_setclass(NV_GRAPHICS_3D_CLASS_ID, 0, 0);
+	ptr[5] = nvhost_opcode_imm(0, syncpt_id);
+	ptr[6] = nvhost_opcode_setclass(NV_HOST1X_CLASS_ID, 0, 0);
+}
+#define SAVE_BEGIN_SIZE 7
+
+static void save_direct(u32 *ptr, u32 start_reg, u32 count)
+{
+	ptr[0] = nvhost_opcode_nonincr(NV_CLASS_HOST_INDOFF, 1);
+	ptr[1] = nvhost_class_host_indoff_reg_read(NV_HOST_MODULE_GR3D,
+						start_reg, true);
+	ptr[2] = nvhost_opcode_nonincr(NV_CLASS_HOST_INDDATA, count);
+}
+#define SAVE_DIRECT_SIZE 3
+
+static void save_indoffset(u32 *ptr, u32 offset_reg, u32 offset)
+{
+	ptr[0] = nvhost_opcode_nonincr(NV_CLASS_HOST_INDOFF, 1);
+	ptr[1] = nvhost_class_host_indoff_reg_write(NV_HOST_MODULE_GR3D,
+						offset_reg, true);
+	ptr[2] = nvhost_opcode_nonincr(NV_CLASS_HOST_INDDATA, 1);
+	ptr[3] = offset;
+}
+#define SAVE_INDOFFSET_SIZE 4
+
+static inline void save_inddata(u32 *ptr, u32 data_reg, u32 count)
+{
+	ptr[0] = nvhost_opcode_nonincr(NV_CLASS_HOST_INDOFF, 1);
+	ptr[1] = nvhost_class_host_indoff_reg_read(NV_HOST_MODULE_GR3D,
+						data_reg, false);
+	ptr[2] = nvhost_opcode_nonincr(NV_CLASS_HOST_INDDATA, count);
+}
+#define SAVE_INDDDATA_SIZE 3
+
+static void save_end(u32 *ptr, u32 syncpt_id, u32 waitbase)
+{
+	/* Wait for context read service */
+	ptr[0] = nvhost_opcode_nonincr(NV_CLASS_HOST_WAIT_SYNCPT_BASE, 1);
+	ptr[1] = nvhost_class_host_wait_syncpt_base(syncpt_id, waitbase, 3);
+	/* Increment syncpoint base */
+	ptr[2] = nvhost_opcode_nonincr(NV_CLASS_HOST_INCR_SYNCPT_BASE, 1);
+	ptr[3] = nvhost_class_host_incr_syncpt_base(waitbase, 3);
+	/* set class back to the unit */
+	ptr[4] = nvhost_opcode_setclass(NV_GRAPHICS_3D_CLASS_ID, 0, 0);
+}
+#define SAVE_END_SIZE 5
+
+static void __init setup_save(
+	u32 *ptr, unsigned int *words_save, unsigned int *words_restore,
+	u32 syncpt_id, u32 waitbase)
+{
+	const struct hwctx_reginfo *r;
+	const struct hwctx_reginfo *rend;
+	unsigned int save = SAVE_BEGIN_SIZE + SAVE_END_SIZE;
+	unsigned int restore = RESTORE_BEGIN_SIZE + RESTORE_END_SIZE;
+
+	if (ptr) {
+		save_begin(ptr, syncpt_id, waitbase);
+		ptr += SAVE_BEGIN_SIZE;
+	}
+
+	r = ctxsave_regs_3d;
+	rend = ctxsave_regs_3d + ARRAY_SIZE(ctxsave_regs_3d);
+	for ( ; r != rend; ++r) {
+		u32 offset = r->offset;
+		u32 count = r->count;
+		switch (r->type) {
+		case HWCTX_REGINFO_DIRECT:
+			if (ptr) {
+				save_direct(ptr, offset, count);
+				ptr += SAVE_DIRECT_SIZE;
+			}
+			save += SAVE_DIRECT_SIZE;
+			restore += RESTORE_DIRECT_SIZE;
+			break;
+		case HWCTX_REGINFO_INDIRECT:
+			if (ptr) {
+				save_indoffset(ptr, offset, 0);
+				ptr += SAVE_INDOFFSET_SIZE;
+				save_inddata(ptr, offset + 1, count);
+				ptr += SAVE_INDDDATA_SIZE;
+			}
+			save += SAVE_INDOFFSET_SIZE;
+			restore += RESTORE_INDOFFSET_SIZE;
+			save += SAVE_INDDDATA_SIZE;
+			restore += RESTORE_INDDATA_SIZE;
+			break;
+		case HWCTX_REGINFO_INDIRECT_OFFSET:
+			if (ptr) {
+				save_indoffset(ptr, offset, count);
+				ptr += SAVE_INDOFFSET_SIZE;
+			}
+			save += SAVE_INDOFFSET_SIZE;
+			restore += RESTORE_INDOFFSET_SIZE;
+			continue; /* INDIRECT_DATA follows with real count */
+		case HWCTX_REGINFO_INDIRECT_DATA:
+			if (ptr) {
+				save_inddata(ptr, offset, count);
+				ptr += SAVE_INDDDATA_SIZE;
+			}
+			save += SAVE_INDDDATA_SIZE;
+			restore += RESTORE_INDDATA_SIZE;
+			break;
+		}
+		if (ptr) {
+			memset(ptr, 0, count * 4);
+			ptr += count;
+		}
+		save += count;
+		restore += count;
+	}
+
+	if (ptr)
+		save_end(ptr, syncpt_id, waitbase);
+
+	if (words_save)
+		*words_save = save;
+	if (words_restore)
+		*words_restore = restore;
+	wmb();
+}
+
+/*** ctx3d ***/
+
+static int ctx3d_init(struct nvhost_hwctx *ctx)
+{
+	ctx->restore = nvmap_alloc(context_restore_size * 4, 32,
+				NVMEM_HANDLE_WRITE_COMBINE,
+				(void**)&ctx->save_cpu_data);
+	if (IS_ERR_OR_NULL(ctx->restore))
+		return PTR_ERR(ctx->restore);
+
+	setup_restore(ctx->save_cpu_data, NVWAITBASE_3D);
+	ctx->restore_phys = nvmap_pin_single(ctx->restore);
+	ctx->restore_size = context_restore_size;
+	ctx->save = context_save_buf;
+	ctx->save_phys = context_save_phys;
+	ctx->save_size = context_save_size;
+	ctx->save_incrs = 3;
+	ctx->restore_incrs = 1;
+	ctx->valid = false;
+	return 0;
+}
+
+static void ctx3d_deinit(struct nvhost_hwctx *ctx)
+{
+	nvmap_free(ctx->restore, ctx->save_cpu_data);
+}
+
+static void ctx3d_save_service(struct nvhost_hwctx *ctx)
+{
+	const struct hwctx_reginfo *r;
+	const struct hwctx_reginfo *rend;
+	unsigned int pending = 0;
+	u32 *ptr = (u32 *)ctx->save_cpu_data + RESTORE_BEGIN_SIZE;
+
+	BUG_ON(!ctx->save_cpu_data);
+
+	r = ctxsave_regs_3d;
+	rend = ctxsave_regs_3d + ARRAY_SIZE(ctxsave_regs_3d);
+	for ( ; r != rend; ++r) {
+		u32 count = r->count;
+		switch (r->type) {
+		case HWCTX_REGINFO_DIRECT:
+			ptr += RESTORE_DIRECT_SIZE;
+			break;
+		case HWCTX_REGINFO_INDIRECT:
+			ptr += RESTORE_INDOFFSET_SIZE + RESTORE_INDDATA_SIZE;
+			break;
+		case HWCTX_REGINFO_INDIRECT_OFFSET:
+			ptr += RESTORE_INDOFFSET_SIZE;
+			continue; /* INDIRECT_DATA follows with real count */
+		case HWCTX_REGINFO_INDIRECT_DATA:
+			ptr += RESTORE_INDDATA_SIZE;
+			break;
+		}
+		restore_registers_from_fifo(ptr, count, ctx->channel, &pending);
+		ptr += count;
+	}
+
+	BUG_ON((u32)((ptr + RESTORE_END_SIZE) - (u32*)ctx->save_cpu_data)
+		!= context_restore_size);
+
+	wmb();
+	nvhost_syncpt_cpu_incr(&ctx->channel->dev->syncpt, NVSYNCPT_3D);
+}
+
+
+/*** nvhost_3dctx ***/
+
+int __init nvhost_3dctx_handler_init(struct nvhost_hwctx_handler *h)
+{
+	setup_save(NULL, &context_save_size, &context_restore_size, 0, 0);
+
+	context_save_buf = nvmap_alloc(context_save_size * 4, 32,
+				NVMEM_HANDLE_WRITE_COMBINE,
+				(void**)&context_save_ptr);
+	if (IS_ERR_OR_NULL(context_save_buf))
+		return PTR_ERR(context_save_buf);
+	context_save_phys = nvmap_pin_single(context_save_buf);
+	setup_save(context_save_ptr, NULL, NULL, NVSYNCPT_3D, NVWAITBASE_3D);
+
+	h->init = ctx3d_init;
+	h->deinit = ctx3d_deinit;
+	h->save_service = ctx3d_save_service;
+	return 0;
+}
+
+/* TODO: [ahatala 2010-05-27] */
+int __init nvhost_mpectx_handler_init(struct nvhost_hwctx_handler *h)
+{
+	return 0;
+}
diff --git a/drivers/video/tegra/host/nvhost_acm.c b/drivers/video/tegra/host/nvhost_acm.c
new file mode 100644
index 000000000000..360774b72bb6
--- /dev/null
+++ b/drivers/video/tegra/host/nvhost_acm.c
@@ -0,0 +1,148 @@
+/*
+ * drivers/video/tegra/host/nvhost_acm.c
+ *
+ * Tegra Graphics Host Automatic Clock Management
+ *
+ * Copyright (c) 2010, NVIDIA Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+#include "nvhost_acm.h"
+#include <linux/string.h>
+#include <linux/sched.h>
+#include <linux/err.h>
+#include <linux/device.h>
+
+#define ACM_TIMEOUT 1*HZ
+
+void nvhost_module_busy(struct nvhost_module *mod)
+{
+	mutex_lock(&mod->lock);
+	cancel_delayed_work(&mod->powerdown);
+	if ((atomic_inc_return(&mod->refcount) == 1) && !mod->powered) {
+		int i;
+		if (mod->parent)
+			nvhost_module_busy(mod->parent);
+		for (i = 0; i < mod->num_clks; i++)
+			clk_enable(mod->clk[i]);
+		if (mod->func)
+			mod->func(mod, NVHOST_POWER_ACTION_ON);
+		mod->powered = true;
+	}
+	mutex_unlock(&mod->lock);
+}
+
+static void powerdown_handler(struct work_struct *work)
+{
+	struct nvhost_module *mod;
+	mod = container_of(to_delayed_work(work), struct nvhost_module, powerdown);
+	mutex_lock(&mod->lock);
+	BUG_ON(!mod->powered);
+	if (atomic_read(&mod->refcount) == 0) {
+		int i;
+		if (mod->func)
+			mod->func(mod, NVHOST_POWER_ACTION_OFF);
+		for (i = 0; i < mod->num_clks; i++)
+			clk_disable(mod->clk[i]);
+		mod->powered = false;
+		if (mod->parent)
+			nvhost_module_idle(mod->parent);
+	}
+	mutex_unlock(&mod->lock);
+}
+
+void nvhost_module_idle_mult(struct nvhost_module *mod, int refs)
+{
+	bool kick = false;
+
+	mutex_lock(&mod->lock);
+	if (atomic_sub_return(refs, &mod->refcount) == 0) {
+		BUG_ON(!mod->powered);
+		schedule_delayed_work(&mod->powerdown, ACM_TIMEOUT);
+		kick = true;
+	}
+	mutex_unlock(&mod->lock);
+
+	if (kick)
+		wake_up(&mod->idle);
+}
+
+static const char *get_module_clk_id(const char *module, int index)
+{
+	if (index == 1 && strcmp(module, "gr2d") == 0)
+		return "epp";
+	else if (index == 0)
+		return module;
+	return NULL;
+}
+
+int nvhost_module_init(struct nvhost_module *mod, const char *name,
+		nvhost_modulef func, struct nvhost_module *parent,
+		struct device *dev)
+{
+	int i = 0;
+
+	mod->name = name;
+
+	while (i < NVHOST_MODULE_MAX_CLOCKS) {
+		long rate;
+		mod->clk[i] = clk_get(dev, get_module_clk_id(name, i));
+		if (IS_ERR_OR_NULL(mod->clk[i]))
+			break;
+		rate = clk_round_rate(mod->clk[i], UINT_MAX);
+		if (rate < 0) {
+			pr_err("%s: can't get maximum rate for %s\n",
+				__func__, name);
+			break;
+		}
+		clk_set_rate(mod->clk[i], rate);
+		i++;
+	}
+
+	mod->num_clks = i;
+	mod->func = func;
+	mod->parent = parent;
+	mod->powered = false;
+	mutex_init(&mod->lock);
+	init_waitqueue_head(&mod->idle);
+	INIT_DELAYED_WORK(&mod->powerdown, powerdown_handler);
+
+	return 0;
+}
+
+static int is_module_idle(struct nvhost_module *mod)
+{
+	int count;
+	mutex_lock(&mod->lock);
+	count = atomic_read(&mod->refcount);
+	mutex_unlock(&mod->lock);
+	return (count == 0);
+}
+
+void nvhost_module_suspend(struct nvhost_module *mod)
+{
+	wait_event(mod->idle, is_module_idle(mod));
+	flush_delayed_work(&mod->powerdown);
+	BUG_ON(mod->powered);
+}
+
+void nvhost_module_deinit(struct nvhost_module *mod)
+{
+	int i;
+	nvhost_module_suspend(mod);
+	for (i = 0; i < mod->num_clks; i++)
+		clk_put(mod->clk[i]);
+}
diff --git a/drivers/video/tegra/host/nvhost_acm.h b/drivers/video/tegra/host/nvhost_acm.h
new file mode 100644
index 000000000000..7cf70e2be098
--- /dev/null
+++ b/drivers/video/tegra/host/nvhost_acm.h
@@ -0,0 +1,75 @@
+/*
+ * drivers/video/tegra/host/nvhost_acm.h
+ *
+ * Tegra Graphics Host Automatic Clock Management
+ *
+ * Copyright (c) 2010, NVIDIA Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+#ifndef __NVHOST_ACM_H
+#define __NVHOST_ACM_H
+
+#include <linux/workqueue.h>
+#include <linux/wait.h>
+#include <linux/mutex.h>
+#include <linux/clk.h>
+
+#define NVHOST_MODULE_MAX_CLOCKS 2
+
+struct nvhost_module;
+
+enum nvhost_power_action {
+	NVHOST_POWER_ACTION_OFF,
+	NVHOST_POWER_ACTION_ON,
+};
+
+typedef void (*nvhost_modulef)(struct nvhost_module *mod, enum nvhost_power_action action);
+
+struct nvhost_module {
+	const char *name;
+	nvhost_modulef func;
+	struct delayed_work powerdown;
+	struct clk *clk[NVHOST_MODULE_MAX_CLOCKS];
+	int num_clks;
+	struct mutex lock;
+	bool powered;
+	atomic_t refcount;
+	wait_queue_head_t idle;
+	struct nvhost_module *parent;
+};
+
+int nvhost_module_init(struct nvhost_module *mod, const char *name,
+		nvhost_modulef func, struct nvhost_module *parent,
+		struct device *dev);
+void nvhost_module_deinit(struct nvhost_module *mod);
+void nvhost_module_suspend(struct nvhost_module *mod);
+
+void nvhost_module_busy(struct nvhost_module *mod);
+void nvhost_module_idle_mult(struct nvhost_module *mod, int refs);
+
+static inline bool nvhost_module_powered(struct nvhost_module *mod)
+{
+	return mod->powered;
+}
+
+static inline void nvhost_module_idle(struct nvhost_module *mod)
+{
+	nvhost_module_idle_mult(mod, 1);
+
+}
+
+#endif
diff --git a/drivers/video/tegra/host/nvhost_cdma.c b/drivers/video/tegra/host/nvhost_cdma.c
new file mode 100644
index 000000000000..1807a1c0760b
--- /dev/null
+++ b/drivers/video/tegra/host/nvhost_cdma.c
@@ -0,0 +1,597 @@
+/*
+ * drivers/video/tegra/host/nvhost_cdma.c
+ *
+ * Tegra Graphics Host Command DMA
+ *
+ * Copyright (c) 2010, NVIDIA Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+#include "nvhost_cdma.h"
+#include "nvhost_dev.h"
+#include <asm/cacheflush.h>
+
+/*
+ * TODO:
+ *   stats
+ *     - for figuring out what to optimize further
+ *   resizable push buffer & sync queue
+ *     - some channels hardly need any, some channels (3d) could use more
+ */
+
+#define cdma_to_channel(cdma) container_of(cdma, struct nvhost_channel, cdma)
+#define cdma_to_dev(cdma) ((cdma_to_channel(cdma))->dev)
+
+/*
+ * push_buffer
+ *
+ * The push buffer is a circular array of words to be fetched by command DMA.
+ * Note that it works slightly differently to the sync queue; fence == cur
+ * means that the push buffer is full, not empty.
+ */
+
+// 8 bytes per slot. (This number does not include the final RESTART.)
+#define PUSH_BUFFER_SIZE (NVHOST_GATHER_QUEUE_SIZE * 8)
+
+static void destroy_push_buffer(struct push_buffer *pb);
+
+/**
+ * Reset to empty push buffer
+ */
+static void reset_push_buffer(struct push_buffer *pb)
+{
+	pb->fence = PUSH_BUFFER_SIZE - 8;
+	pb->cur = 0;
+}
+
+/**
+ * Init push buffer resources
+ */
+static int init_push_buffer(struct push_buffer *pb)
+{
+	pb->mem = NULL;
+	pb->mapped = NULL;
+	pb->phys = 0;
+	reset_push_buffer(pb);
+
+	/* allocate and map pushbuffer memory */
+	pb->mem = nvmap_alloc(PUSH_BUFFER_SIZE + 4, 32,
+			NVMEM_HANDLE_WRITE_COMBINE, (void**)&pb->mapped);
+	if (IS_ERR_OR_NULL(pb->mem)) {
+		pb->mem = NULL;
+		goto fail;
+	}
+
+	/* pin pushbuffer and get physical address */
+	pb->phys = nvmap_pin_single(pb->mem);
+
+	/* put the restart at the end of pushbuffer memory */
+	*(pb->mapped + (PUSH_BUFFER_SIZE >> 2)) = nvhost_opcode_restart(pb->phys);
+
+	return 0;
+
+fail:
+	destroy_push_buffer(pb);
+	return -ENOMEM;
+}
+
+/**
+ * Clean up push buffer resources
+ */
+static void destroy_push_buffer(struct push_buffer *pb)
+{
+	if (pb->mem) {
+		if (pb->phys != 0) {
+			nvmap_unpin(&pb->mem, 1);
+			pb->phys = 0;
+		}
+
+		nvmap_free(pb->mem, pb->mapped);
+		pb->mem = NULL;
+		pb->mapped = NULL;
+	}
+}
+
+/**
+ * Push two words to the push buffer
+ * Caller must ensure push buffer is not full
+ */
+static void push_to_push_buffer(struct push_buffer *pb, u32 op1, u32 op2)
+{
+	u32 cur = pb->cur;
+	u32 *p = (u32*)((u32)pb->mapped + cur);
+	BUG_ON(cur == pb->fence);
+	*(p++) = op1;
+	*(p++) = op2;
+	pb->cur = (cur + 8) & (PUSH_BUFFER_SIZE - 1);
+	/* printk("push_to_push_buffer: op1=%08x; op2=%08x; cur=%x\n", op1, op2, pb->cur); */
+}
+
+/**
+ * Pop a number of two word slots from the push buffer
+ * Caller must ensure push buffer is not empty
+ */
+static void pop_from_push_buffer(struct push_buffer *pb, unsigned int slots)
+{
+	pb->fence = (pb->fence + slots * 8) & (PUSH_BUFFER_SIZE - 1);
+}
+
+/**
+ * Return the number of two word slots free in the push buffer
+ */
+static u32 push_buffer_space(struct push_buffer *pb)
+{
+	return ((pb->fence - pb->cur) & (PUSH_BUFFER_SIZE - 1)) / 8;
+}
+
+static u32 push_buffer_putptr(struct push_buffer *pb)
+{
+	return pb->phys + pb->cur;
+}
+
+
+/* Sync Queue
+ *
+ * The sync queue is a circular buffer of u32s interpreted as:
+ *   0: SyncPointID
+ *   1: SyncPointValue
+ *   2: NumSlots (how many pushbuffer slots to free)
+ *   3: NumHandles
+ *   4..: NumHandles * nvmemhandle to unpin
+ *
+ * There's always one word unused, so (accounting for wrap):
+ *   - Write == Read => queue empty
+ *   - Write + 1 == Read => queue full
+ * The queue must not be left with less than SYNC_QUEUE_MIN_ENTRY words
+ * of space at the end of the array.
+ *
+ * We want to pass contiguous arrays of handles to NrRmMemUnpin, so arrays
+ * that would wrap at the end of the buffer will be split into two (or more)
+ * entries.
+ */
+
+/* Number of words needed to store an entry containing one handle */
+#define SYNC_QUEUE_MIN_ENTRY 5
+
+/**
+ * Reset to empty queue.
+ */
+static void reset_sync_queue(struct sync_queue *queue)
+{
+	queue->read = 0;
+	queue->write = 0;
+}
+
+/**
+ *  Find the number of handles that can be stashed in the sync queue without
+ *  waiting.
+ *  0 -> queue is full, must update to wait for some entries to be freed.
+ */
+static unsigned int sync_queue_space(struct sync_queue *queue)
+{
+	unsigned int read = queue->read;
+	unsigned int write = queue->write;
+	u32 size;
+
+	BUG_ON(read  > (NVHOST_SYNC_QUEUE_SIZE - SYNC_QUEUE_MIN_ENTRY));
+	BUG_ON(write > (NVHOST_SYNC_QUEUE_SIZE - SYNC_QUEUE_MIN_ENTRY));
+
+	/*
+	 * We can use all of the space up to the end of the buffer, unless the
+	 * read position is within that space (the read position may advance
+	 * asynchronously, but that can't take space away once we've seen it).
+	 */
+	if (read > write) {
+		size = (read - 1) - write;
+	} else {
+		size = NVHOST_SYNC_QUEUE_SIZE - write;
+
+		/*
+		 * If the read position is zero, it gets complicated. We can't
+		 * use the last word in the buffer, because that would leave
+		 * the queue empty.
+		 * But also if we use too much we would not leave enough space
+		 * for a single handle packet, and would have to wrap in
+		 * add_to_sync_queue - also leaving write == read == 0,
+		 * an empty queue.
+		 */
+		if (read == 0)
+			size -= SYNC_QUEUE_MIN_ENTRY;
+	}
+
+	/*
+	 * There must be room for an entry header and at least one handle,
+	 * otherwise we report a full queue.
+	 */
+	if (size < SYNC_QUEUE_MIN_ENTRY)
+		return 0;
+	/* Minimum entry stores one handle */
+	return (size - SYNC_QUEUE_MIN_ENTRY) + 1;
+}
+
+/**
+ * Add an entry to the sync queue.
+ */
+static void add_to_sync_queue(struct sync_queue *queue,
+			u32 sync_point_id, u32 sync_point_value,
+			u32 nr_slots,
+			struct nvmap_handle **handles, u32 nr_handles)
+{
+	u32 write = queue->write;
+	u32 *p = queue->buffer + write;
+	u32 size = 4 + nr_handles;
+
+	BUG_ON(sync_point_id == NVSYNCPT_INVALID);
+	BUG_ON(sync_queue_space(queue) < nr_handles);
+
+	write += size;
+	BUG_ON(write > NVHOST_SYNC_QUEUE_SIZE);
+
+	*p++ = sync_point_id;
+	*p++ = sync_point_value;
+	*p++ = nr_slots;
+	*p++ = nr_handles;
+	if (nr_handles)
+		memcpy(p, handles, nr_handles*sizeof(struct nvmap_handle *));
+
+	/* If there's not enough room for another entry, wrap to the start. */
+	if ((write + SYNC_QUEUE_MIN_ENTRY) > NVHOST_SYNC_QUEUE_SIZE) {
+		/*
+		 * It's an error for the read position to be zero, as that
+		 * would mean we emptied the queue while adding something.
+		 */
+		BUG_ON(queue->read == 0);
+		write = 0;
+	}
+
+	queue->write = write;
+}
+
+/**
+ * Get a pointer to the next entry in the queue, or NULL if the queue is empty.
+ * Doesn't consume the entry.
+ */
+static u32 *sync_queue_head(struct sync_queue *queue)
+{
+	u32 read = queue->read;
+	u32 write = queue->write;
+
+	BUG_ON(read  > (NVHOST_SYNC_QUEUE_SIZE - SYNC_QUEUE_MIN_ENTRY));
+	BUG_ON(write > (NVHOST_SYNC_QUEUE_SIZE - SYNC_QUEUE_MIN_ENTRY));
+
+	if (read == write)
+		return NULL;
+	return queue->buffer + read;
+}
+
+/**
+ * Advances to the next queue entry, if you want to consume it.
+ */
+static void
+dequeue_sync_queue_head(struct sync_queue *queue)
+{
+	u32 read = queue->read;
+	u32 size;
+
+	BUG_ON(read == queue->write);
+
+	size = 4 + queue->buffer[read + 3];
+
+	read += size;
+	BUG_ON(read > NVHOST_SYNC_QUEUE_SIZE);
+
+	/* If there's not enough room for another entry, wrap to the start. */
+	if ((read + SYNC_QUEUE_MIN_ENTRY) > NVHOST_SYNC_QUEUE_SIZE)
+		read = 0;
+
+	queue->read = read;
+}
+
+
+/*** Cdma internal stuff ***/
+
+/**
+ * Start channel DMA
+ */
+static void start_cdma(struct nvhost_cdma *cdma)
+{
+	void __iomem *chan_regs = cdma_to_channel(cdma)->aperture;
+
+	if (cdma->running)
+		return;
+
+	cdma->last_put = push_buffer_putptr(&cdma->push_buffer);
+
+	writel(nvhost_channel_dmactrl(true, false, false),
+		chan_regs + HOST1X_CHANNEL_DMACTRL);
+
+	/* set base, put, end pointer (all of memory) */
+	writel(0, chan_regs + HOST1X_CHANNEL_DMASTART);
+	writel(cdma->last_put, chan_regs + HOST1X_CHANNEL_DMAPUT);
+	writel(0xFFFFFFFF, chan_regs + HOST1X_CHANNEL_DMAEND);
+
+	/* reset GET */
+	writel(nvhost_channel_dmactrl(true, true, true),
+		chan_regs + HOST1X_CHANNEL_DMACTRL);
+
+	/* start the command DMA */
+	writel(nvhost_channel_dmactrl(false, false, false),
+		chan_regs + HOST1X_CHANNEL_DMACTRL);
+
+	cdma->running = true;
+}
+
+/**
+ * Kick channel DMA into action by writing its PUT offset (if it has changed)
+ */
+static void kick_cdma(struct nvhost_cdma *cdma)
+{
+	u32 put = push_buffer_putptr(&cdma->push_buffer);
+	if (put != cdma->last_put) {
+		void __iomem *chan_regs = cdma_to_channel(cdma)->aperture;
+		wmb();
+		writel(put, chan_regs + HOST1X_CHANNEL_DMAPUT);
+		cdma->last_put = put;
+	}
+}
+
+/**
+ * Return the status of the cdma's sync queue or push buffer for the given event
+ *  - sq empty: returns 1 for empty, 0 for not empty (as in "1 empty queue" :-)
+ *  - sq space: returns the number of handles that can be stored in the queue
+ *  - pb space: returns the number of free slots in the channel's push buffer
+ * Must be called with the cdma lock held.
+ */
+static unsigned int cdma_status(struct nvhost_cdma *cdma, enum cdma_event event)
+{
+	switch (event) {
+	case CDMA_EVENT_SYNC_QUEUE_EMPTY:
+		return sync_queue_head(&cdma->sync_queue) ? 0 : 1;
+	case CDMA_EVENT_SYNC_QUEUE_SPACE:
+		return sync_queue_space(&cdma->sync_queue);
+	case CDMA_EVENT_PUSH_BUFFER_SPACE:
+		return push_buffer_space(&cdma->push_buffer);
+	default:
+		return 0;
+	}
+}
+
+/**
+ * Sleep (if necessary) until the requested event happens
+ *   - CDMA_EVENT_SYNC_QUEUE_EMPTY : sync queue is completely empty.
+ *     - Returns 1
+ *   - CDMA_EVENT_SYNC_QUEUE_SPACE : there is space in the sync queue.
+ *   - CDMA_EVENT_PUSH_BUFFER_SPACE : there is space in the push buffer
+ *     - Return the amount of space (> 0)
+ * Must be called with the cdma lock held.
+ */
+static unsigned int wait_cdma(struct nvhost_cdma *cdma, enum cdma_event event)
+{
+	for (;;) {
+		unsigned int space = cdma_status(cdma, event);
+		if (space)
+			return space;
+
+		BUG_ON(cdma->event != CDMA_EVENT_NONE);
+		cdma->event = event;
+
+		mutex_unlock(&cdma->lock);
+		down(&cdma->sem);
+		mutex_lock(&cdma->lock);
+	}
+}
+
+/**
+ * For all sync queue entries that have already finished according to the
+ * current sync point registers:
+ *  - unpin & unref their mems
+ *  - pop their push buffer slots
+ *  - remove them from the sync queue
+ * This is normally called from the host code's worker thread, but can be
+ * called manually if necessary.
+ * Must be called with the cdma lock held.
+ */
+static void update_cdma(struct nvhost_cdma *cdma)
+{
+	bool signal = false;
+	struct nvhost_dev *dev = cdma_to_dev(cdma);
+
+	BUG_ON(!cdma->running);
+
+	/*
+	 * Walk the sync queue, reading the sync point registers as necessary,
+	 * to consume as many sync queue entries as possible without blocking
+	 */
+	for (;;) {
+		u32 syncpt_id, syncpt_val;
+		unsigned int nr_slots, nr_handles;
+		struct nvmap_handle **handles;
+		u32 *sync;
+
+		sync = sync_queue_head(&cdma->sync_queue);
+		if (!sync) {
+			if (cdma->event == CDMA_EVENT_SYNC_QUEUE_EMPTY)
+				signal = true;
+			break;
+		}
+
+		syncpt_id = sync[0];
+		syncpt_val = sync[1];
+
+		BUG_ON(syncpt_id == NVSYNCPT_INVALID);
+
+		/* Check whether this syncpt has completed, and bail if not */
+		if (!nvhost_syncpt_min_cmp(&dev->syncpt, syncpt_id, syncpt_val))
+			break;
+
+		nr_slots = sync[2];
+		nr_handles = sync[3];
+		handles = (struct nvmap_handle **)(sync + 4);
+
+		/* Unpin the memory */
+		if (nr_handles)
+			nvmap_unpin(handles, nr_handles);
+
+		/* Pop push buffer slots */
+		if (nr_slots) {
+			pop_from_push_buffer(&cdma->push_buffer, nr_slots);
+			if (cdma->event == CDMA_EVENT_PUSH_BUFFER_SPACE)
+				signal = true;
+		}
+
+		dequeue_sync_queue_head(&cdma->sync_queue);
+		if (cdma->event == CDMA_EVENT_SYNC_QUEUE_SPACE)
+			signal = true;
+	}
+
+	/* Wake up CdmaWait() if the requested event happened */
+	if (signal) {
+		cdma->event = CDMA_EVENT_NONE;
+		up(&cdma->sem);
+	}
+}
+
+/**
+ * Create a cdma
+ */
+int nvhost_cdma_init(struct nvhost_cdma *cdma)
+{
+	int err;
+
+	mutex_init(&cdma->lock);
+	sema_init(&cdma->sem, 0);
+	cdma->event = CDMA_EVENT_NONE;
+	cdma->running = false;
+	err = init_push_buffer(&cdma->push_buffer);
+	if (err)
+		return err;
+	reset_sync_queue(&cdma->sync_queue);
+	return 0;
+}
+
+/**
+ * Destroy a cdma
+ */
+void nvhost_cdma_deinit(struct nvhost_cdma *cdma)
+{
+	BUG_ON(cdma->running);
+	destroy_push_buffer(&cdma->push_buffer);
+}
+
+void nvhost_cdma_stop(struct nvhost_cdma *cdma)
+{
+	void __iomem *chan_regs = cdma_to_channel(cdma)->aperture;
+
+	mutex_lock(&cdma->lock);
+	if (cdma->running) {
+		wait_cdma(cdma, CDMA_EVENT_SYNC_QUEUE_EMPTY);
+		writel(nvhost_channel_dmactrl(true, false, false),
+			chan_regs + HOST1X_CHANNEL_DMACTRL);
+		cdma->running = false;
+	}
+	mutex_unlock(&cdma->lock);
+}
+
+/**
+ * Begin a cdma submit
+ */
+void nvhost_cdma_begin(struct nvhost_cdma *cdma)
+{
+	mutex_lock(&cdma->lock);
+	if (!cdma->running)
+		start_cdma(cdma);
+	cdma->slots_free = 0;
+	cdma->slots_used = 0;
+}
+
+/**
+ * Push two words into a push buffer slot
+ * Blocks as necessary if the push buffer is full.
+ */
+void nvhost_cdma_push(struct nvhost_cdma *cdma, u32 op1, u32 op2)
+{
+	u32 slots_free = cdma->slots_free;
+	if (slots_free == 0) {
+		kick_cdma(cdma);
+		slots_free = wait_cdma(cdma, CDMA_EVENT_PUSH_BUFFER_SPACE);
+	}
+	cdma->slots_free = slots_free - 1;
+	cdma->slots_used++;
+	push_to_push_buffer(&cdma->push_buffer, op1, op2);
+}
+
+/**
+ * End a cdma submit
+ * Kick off DMA, add a contiguous block of memory handles to the sync queue,
+ * and a number of slots to be freed from the pushbuffer.
+ * Blocks as necessary if the sync queue is full.
+ * The handles for a submit must all be pinned at the same time, but they
+ * can be unpinned in smaller chunks.
+ */
+void nvhost_cdma_end(struct nvhost_cdma *cdma,
+	u32 sync_point_id, u32 sync_point_value,
+	struct nvmap_handle **handles, unsigned int nr_handles)
+{
+	kick_cdma(cdma);
+
+	while (nr_handles || cdma->slots_used) {
+		unsigned int count;
+		/*
+		 * Wait until there's enough room in the
+		 * sync queue to write something.
+		 */
+		count = wait_cdma(cdma, CDMA_EVENT_SYNC_QUEUE_SPACE);
+
+		/* Add reloc entries to sync queue (as many as will fit) */
+		if (count > nr_handles)
+			count = nr_handles;
+		add_to_sync_queue(&cdma->sync_queue,
+				sync_point_id, sync_point_value,
+				cdma->slots_used, handles, count);
+		/* NumSlots only goes in the first packet */
+		cdma->slots_used = 0;
+		handles += count;
+		nr_handles -= count;
+	}
+
+	mutex_unlock(&cdma->lock);
+}
+
+/**
+ * Update cdma state according to current sync point values
+ */
+void nvhost_cdma_update(struct nvhost_cdma *cdma)
+{
+	mutex_lock(&cdma->lock);
+	update_cdma(cdma);
+	mutex_unlock(&cdma->lock);
+}
+
+/**
+ * Manually spin until all CDMA has finished. Used if an async update
+ * cannot be scheduled for any reason.
+ */
+void nvhost_cdma_flush(struct nvhost_cdma *cdma)
+{
+	mutex_lock(&cdma->lock);
+	while (sync_queue_head(&cdma->sync_queue)) {
+		update_cdma(cdma);
+		mutex_unlock(&cdma->lock);
+		schedule();
+		mutex_lock(&cdma->lock);
+	}
+	mutex_unlock(&cdma->lock);
+}
diff --git a/drivers/video/tegra/host/nvhost_cdma.h b/drivers/video/tegra/host/nvhost_cdma.h
new file mode 100644
index 000000000000..f0f40928fa4e
--- /dev/null
+++ b/drivers/video/tegra/host/nvhost_cdma.h
@@ -0,0 +1,99 @@
+/*
+ * drivers/video/tegra/host/nvhost_cdma.h
+ *
+ * Tegra Graphics Host Command DMA
+ *
+ * Copyright (c) 2010, NVIDIA Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+#ifndef __NVHOST_CDMA_H
+#define __NVHOST_CDMA_H
+
+#include <linux/sched.h>
+#include <linux/semaphore.h>
+#include <linux/nvhost.h>
+#include <linux/nvmap.h>
+
+#include "nvhost_acm.h"
+
+/*
+ * cdma
+ *
+ * This is in charge of a host command DMA channel.
+ * Sends ops to a push buffer, and takes responsibility for unpinning
+ * (& possibly freeing) of memory after those ops have completed.
+ * Producer:
+ *	begin
+ *		push - send ops to the push buffer
+ *	end - start command DMA and enqueue handles to be unpinned
+ * Consumer:
+ *	update - call to update sync queue and push buffer, unpin memory
+ */
+
+/* Size of the sync queue. If it is too small, we won't be able to queue up
+ * many command buffers. If it is too large, we waste memory. */
+#define NVHOST_SYNC_QUEUE_SIZE 8192
+
+/* Number of gathers we allow to be queued up per channel. Must be a
+   power of two. Currently sized such that pushbuffer is 4KB (512*8B). */
+#define NVHOST_GATHER_QUEUE_SIZE 512
+
+struct push_buffer {
+	struct nvmap_handle *mem; /* handle to pushbuffer memory */
+	u32 *mapped;		/* mapped pushbuffer memory */
+	u32 phys;		/* physical address of pushbuffer */
+	u32 fence;		/* index we've written */
+	u32 cur;		/* index to write to */
+};
+
+struct sync_queue {
+	unsigned int read;		    /* read position within buffer */
+	unsigned int write;		    /* write position within buffer */
+	u32 buffer[NVHOST_SYNC_QUEUE_SIZE]; /* queue data */
+};
+
+enum cdma_event {
+	CDMA_EVENT_NONE,		/* not waiting for any event */
+	CDMA_EVENT_SYNC_QUEUE_EMPTY,	/* wait for empty sync queue */
+	CDMA_EVENT_SYNC_QUEUE_SPACE,	/* wait for space in sync queue */
+	CDMA_EVENT_PUSH_BUFFER_SPACE	/* wait for space in push buffer */
+};
+
+struct nvhost_cdma {
+	struct mutex lock;		/* controls access to shared state */
+	struct semaphore sem;		/* signalled when event occurs */
+	enum cdma_event event;		/* event that sem is waiting for */
+	unsigned int slots_used;	/* pb slots used in current submit */
+	unsigned int slots_free;	/* pb slots free in current submit */
+	unsigned int last_put;		/* last value written to DMAPUT */
+	struct push_buffer push_buffer;	/* channel's push buffer */
+	struct sync_queue sync_queue;	/* channel's sync queue */
+	bool running;
+};
+
+int	nvhost_cdma_init(struct nvhost_cdma *cdma);
+void	nvhost_cdma_deinit(struct nvhost_cdma *cdma);
+void	nvhost_cdma_stop(struct nvhost_cdma *cdma);
+void	nvhost_cdma_begin(struct nvhost_cdma *cdma);
+void	nvhost_cdma_push(struct nvhost_cdma *cdma, u32 op1, u32 op2);
+void	nvhost_cdma_end(struct nvhost_cdma *cdma,
+		u32 sync_point_id, u32 sync_point_value,
+		struct nvmap_handle **handles, unsigned int nr_handles);
+void	nvhost_cdma_update(struct nvhost_cdma *cdma);
+void	nvhost_cdma_flush(struct nvhost_cdma *cdma);
+
+#endif
diff --git a/drivers/video/tegra/host/nvhost_channel.c b/drivers/video/tegra/host/nvhost_channel.c
new file mode 100644
index 000000000000..7215e597699b
--- /dev/null
+++ b/drivers/video/tegra/host/nvhost_channel.c
@@ -0,0 +1,247 @@
+/*
+ * drivers/video/tegra/host/nvhost_channel.c
+ *
+ * Tegra Graphics Host Channel
+ *
+ * Copyright (c) 2010, NVIDIA Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+#include "nvhost_channel.h"
+#include "nvhost_dev.h"
+#include "nvhost_hwctx.h"
+#include <linux/platform_device.h>
+
+#define NVMODMUTEX_2D_FULL   (1)
+#define NVMODMUTEX_2D_SIMPLE (2)
+#define NVMODMUTEX_2D_SB_A   (3)
+#define NVMODMUTEX_2D_SB_B   (4)
+#define NVMODMUTEX_3D        (5)
+#define NVMODMUTEX_DISPLAYA  (6)
+#define NVMODMUTEX_DISPLAYB  (7)
+#define NVMODMUTEX_VI        (8)
+#define NVMODMUTEX_DSI       (9)
+
+static void power_2d(struct nvhost_module *mod, enum nvhost_power_action action);
+static void power_3d(struct nvhost_module *mod, enum nvhost_power_action action);
+static void power_mpe(struct nvhost_module *mod, enum nvhost_power_action action);
+
+static const struct nvhost_channeldesc channelmap[] = {
+{
+	/* channel 0 */
+	.name	       = "display",
+	.syncpts       = BIT(NVSYNCPT_DISP0) | BIT(NVSYNCPT_DISP1) |
+			 BIT(NVSYNCPT_VBLANK0) | BIT(NVSYNCPT_VBLANK1),
+	.modulemutexes = BIT(NVMODMUTEX_DISPLAYA) | BIT(NVMODMUTEX_DISPLAYB),
+},
+{
+	/* channel 1 */
+	.name	       = "gr3d",
+	.syncpts       = BIT(NVSYNCPT_3D),
+	.waitbases     = BIT(NVWAITBASE_3D),
+	.modulemutexes = BIT(NVMODMUTEX_3D),
+	.class	       = NV_GRAPHICS_3D_CLASS_ID,
+	.power         = power_3d,
+},
+{
+	/* channel 2 */
+	.name	       = "gr2d",
+	.syncpts       = BIT(NVSYNCPT_2D_0) | BIT(NVSYNCPT_2D_1),
+	.waitbases     = BIT(NVWAITBASE_2D_0) | BIT(NVWAITBASE_2D_1),
+	.modulemutexes = BIT(NVMODMUTEX_2D_FULL) | BIT(NVMODMUTEX_2D_SIMPLE) |
+			 BIT(NVMODMUTEX_2D_SB_A) | BIT(NVMODMUTEX_2D_SB_B),
+	.power         = power_2d,
+},
+{
+	/* channel 3 */
+	.name	 = "isp",
+	.syncpts = 0,
+},
+{
+	/* channel 4 */
+	.name	       = "vi",
+	.syncpts       = BIT(NVSYNCPT_VI_ISP_0) | BIT(NVSYNCPT_VI_ISP_1) |
+		         BIT(NVSYNCPT_VI_ISP_2) | BIT(NVSYNCPT_VI_ISP_3) |
+		         BIT(NVSYNCPT_VI_ISP_4) | BIT(NVSYNCPT_VI_ISP_5),
+	.modulemutexes = BIT(NVMODMUTEX_VI),
+	.exclusive     = true,
+},
+{
+	/* channel 5 */
+	.name	       = "mpe",
+	.syncpts       = BIT(NVSYNCPT_MPE) | BIT(NVSYNCPT_MPE_EBM_EOF) |
+			 BIT(NVSYNCPT_MPE_WR_SAFE),
+	.waitbases     = BIT(NVWAITBASE_MPE),
+	.class	       = NV_VIDEO_ENCODE_MPEG_CLASS_ID,
+	.power	       = power_mpe,
+	.exclusive     = true,
+},
+{
+	/* channel 6 */
+	.name	       = "dsi",
+	.syncpts       = BIT(NVSYNCPT_DSI),
+	.modulemutexes = BIT(NVMODMUTEX_DSI),
+}};
+
+static inline void __iomem *channel_aperture(void __iomem *p, int ndx)
+{
+	ndx += NVHOST_CHANNEL_BASE;
+	p += NV_HOST1X_CHANNEL0_BASE;
+	p += ndx * NV_HOST1X_CHANNEL_MAP_SIZE_BYTES;
+	return p;
+}
+
+int __init nvhost_channel_init(struct nvhost_channel *ch,
+			struct nvhost_dev *dev, int index)
+{
+	BUILD_BUG_ON(NVHOST_NUMCHANNELS != ARRAY_SIZE(channelmap));
+
+	ch->dev = dev;
+	ch->desc = &channelmap[index];
+	ch->aperture = channel_aperture(dev->aperture, index);
+	mutex_init(&ch->reflock);
+	mutex_init(&ch->submitlock);
+
+	return nvhost_hwctx_handler_init(&ch->ctxhandler, ch->desc->name);
+}
+
+struct nvhost_channel *nvhost_getchannel(struct nvhost_channel *ch)
+{
+	int err = 0;
+	mutex_lock(&ch->reflock);
+	if (ch->refcount == 0) {
+		err = nvhost_module_init(&ch->mod, ch->desc->name,
+					ch->desc->power, &ch->dev->mod,
+					&ch->dev->pdev->dev);
+		if (!err) {
+			err = nvhost_cdma_init(&ch->cdma);
+			if (err)
+				nvhost_module_deinit(&ch->mod);
+		}
+	} else if (ch->desc->exclusive) {
+		err = -EBUSY;
+	}
+	if (!err) {
+		ch->refcount++;
+	}
+	mutex_unlock(&ch->reflock);
+
+	return err ? ERR_PTR(err) : ch;
+}
+
+void nvhost_putchannel(struct nvhost_channel *ch, struct nvhost_hwctx *ctx)
+{
+	if (ctx) {
+		mutex_lock(&ch->submitlock);
+		if (ch->cur_ctx == ctx)
+			ch->cur_ctx = NULL;
+		mutex_unlock(&ch->submitlock);
+	}
+
+	mutex_lock(&ch->reflock);
+	if (ch->refcount == 1) {
+		nvhost_module_deinit(&ch->mod);
+		/* cdma may already be stopped, that's ok */
+		nvhost_cdma_stop(&ch->cdma);
+		nvhost_cdma_deinit(&ch->cdma);
+	}
+	ch->refcount--;
+	mutex_unlock(&ch->reflock);
+}
+
+void nvhost_channel_suspend(struct nvhost_channel *ch)
+{
+	mutex_lock(&ch->reflock);
+	BUG_ON(nvhost_module_powered(&ch->mod));
+	if (ch->refcount)
+		nvhost_cdma_stop(&ch->cdma);
+	mutex_unlock(&ch->reflock);
+}
+
+void nvhost_channel_submit(
+	struct nvhost_channel *ch,
+	struct nvhost_op_pair *ops,
+	int num_pairs,
+	struct nvhost_cpuinterrupt *intrs,
+	int num_intrs,
+	struct nvmap_handle **unpins,
+	int num_unpins,
+	u32 syncpt_id,
+	u32 syncpt_val)
+{
+	int i;
+	struct nvhost_op_pair* p;
+
+	/* schedule interrupts */
+	for (i = 0; i < num_intrs; i++) {
+		nvhost_intr_add_action(&ch->dev->intr, syncpt_id, intrs[i].syncpt_val,
+				NVHOST_INTR_ACTION_CTXSAVE, intrs[i].intr_data, NULL);
+	}
+
+	/* begin a CDMA submit */
+	nvhost_cdma_begin(&ch->cdma);
+
+	/* push ops */
+	for (i = 0, p = ops; i < num_pairs; i++, p++)
+		nvhost_cdma_push(&ch->cdma, p->op1, p->op2);
+
+	/* end CDMA submit & stash pinned hMems into sync queue for later cleanup */
+	nvhost_cdma_end(&ch->cdma, syncpt_id, syncpt_val, unpins, num_unpins);
+}
+
+static void power_2d(struct nvhost_module *mod, enum nvhost_power_action action)
+{
+	/* TODO: [ahatala 2010-06-17] reimplement EPP hang war */
+	if (action == NVHOST_POWER_ACTION_OFF) {
+		/* TODO: [ahatala 2010-06-17] reset EPP */
+	}
+}
+
+static void power_3d(struct nvhost_module *mod, enum nvhost_power_action action)
+{
+	struct nvhost_channel *ch = container_of(mod, struct nvhost_channel, mod);
+
+	if (action == NVHOST_POWER_ACTION_OFF) {
+		mutex_lock(&ch->submitlock);
+		if (ch->cur_ctx) {
+			DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wq);
+			struct nvhost_op_pair save;
+			struct nvhost_cpuinterrupt ctxsw;
+			u32 syncval;
+			syncval = nvhost_syncpt_incr_max(&ch->dev->syncpt,
+							NVSYNCPT_3D,
+							ch->cur_ctx->save_incrs);
+			save.op1 = nvhost_opcode_gather(0, ch->cur_ctx->save_size);
+			save.op2 = ch->cur_ctx->save_phys;
+			ctxsw.intr_data = ch->cur_ctx;
+			ctxsw.syncpt_val = syncval - 1;
+			nvhost_channel_submit(ch, &save, 1, &ctxsw, 1, NULL, 0, NVSYNCPT_3D, syncval);
+			ch->cur_ctx->last_access_id = NVSYNCPT_3D;
+			ch->cur_ctx->last_access_value = syncval;
+			ch->cur_ctx->valid = true;
+			ch->cur_ctx = NULL;
+			nvhost_intr_add_action(&ch->dev->intr, NVSYNCPT_3D, syncval,
+					NVHOST_INTR_ACTION_WAKEUP, &wq, NULL);
+			wait_event(wq, nvhost_syncpt_min_cmp(&ch->dev->syncpt, NVSYNCPT_3D, syncval));
+			nvhost_cdma_update(&ch->cdma);
+		}
+		mutex_unlock(&ch->submitlock);
+	}
+}
+
+static void power_mpe(struct nvhost_module *mod, enum nvhost_power_action action)
+{
+}
diff --git a/drivers/video/tegra/host/nvhost_channel.h b/drivers/video/tegra/host/nvhost_channel.h
new file mode 100644
index 000000000000..a32f97266c02
--- /dev/null
+++ b/drivers/video/tegra/host/nvhost_channel.h
@@ -0,0 +1,94 @@
+/*
+ * drivers/video/tegra/host/nvhost_channel.h
+ *
+ * Tegra Graphics Host Channel
+ *
+ * Copyright (c) 2010, NVIDIA Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+#ifndef __NVHOST_CHANNEL_H
+#define __NVHOST_CHANNEL_H
+
+#include "nvhost_cdma.h"
+#include "nvhost_acm.h"
+#include "nvhost_hwctx.h"
+
+#include <linux/cdev.h>
+#include <linux/io.h>
+
+#define NVHOST_CHANNEL_BASE 0
+#define NVHOST_NUMCHANNELS (NV_HOST1X_CHANNELS - 1)
+#define NVHOST_MAX_GATHERS 512
+#define NVHOST_MAX_HANDLES 1280
+
+struct nvhost_dev;
+
+struct nvhost_channeldesc {
+	const char *name;
+	nvhost_modulef power;
+	u32 syncpts;
+	u32 waitbases;
+	u32 modulemutexes;
+	u32 class;
+	bool exclusive;
+};
+
+struct nvhost_channel {
+	int refcount;
+	struct mutex reflock;
+	struct mutex submitlock;
+	void __iomem *aperture;
+	struct nvhost_dev *dev;
+	const struct nvhost_channeldesc *desc;
+	struct nvhost_hwctx *cur_ctx;
+	struct device *node;
+	struct cdev cdev;
+	struct nvhost_hwctx_handler ctxhandler;
+	struct nvhost_module mod;
+	struct nvhost_cdma cdma;
+};
+
+struct nvhost_op_pair {
+	u32 op1;
+	u32 op2;
+};
+
+struct nvhost_cpuinterrupt {
+	u32 syncpt_val;
+	void *intr_data;
+};
+
+int nvhost_channel_init(
+	struct nvhost_channel *ch,
+	struct nvhost_dev *dev, int index);
+
+void nvhost_channel_submit(
+	struct nvhost_channel *ch,
+	struct nvhost_op_pair *ops,
+	int num_pairs,
+	struct nvhost_cpuinterrupt *intrs,
+	int num_intrs,
+	struct nvmap_handle **unpins,
+	int num_unpins,
+	u32 syncpt_id,
+	u32 syncpt_val);
+
+struct nvhost_channel *nvhost_getchannel(struct nvhost_channel *ch);
+void nvhost_putchannel(struct nvhost_channel *ch, struct nvhost_hwctx *ctx);
+void nvhost_channel_suspend(struct nvhost_channel *ch);
+
+#endif
diff --git a/drivers/video/tegra/host/nvhost_cpuaccess.c b/drivers/video/tegra/host/nvhost_cpuaccess.c
new file mode 100644
index 000000000000..efc4c07ba12d
--- /dev/null
+++ b/drivers/video/tegra/host/nvhost_cpuaccess.c
@@ -0,0 +1,139 @@
+/*
+ * drivers/video/tegra/host/nvhost_cpuaccess.c
+ *
+ * Tegra Graphics Host Cpu Register Access
+ *
+ * Copyright (c) 2010, NVIDIA Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+#include "nvhost_cpuaccess.h"
+#include "nvhost_dev.h"
+#include <linux/string.h>
+
+#define cpuaccess_to_dev(ctx) container_of(ctx, struct nvhost_dev, cpuaccess)
+
+static const char *module_resource_names[NVHOST_MODULE_NUM] = {
+	"display",
+	"display2",
+	"vi",
+	"isp",
+	"mpe"
+};
+
+int nvhost_cpuaccess_init(struct nvhost_cpuaccess *ctx,
+			struct platform_device *pdev)
+{
+	int i;
+	for (i = 0; i < NVHOST_MODULE_NUM; i++) {
+		struct resource *mem;
+		mem = platform_get_resource_byname(pdev, IORESOURCE_MEM,
+			module_resource_names[i]);
+		if (!mem) {
+			if (i == NVHOST_MODULE_DISPLAY_A ||
+			    i == NVHOST_MODULE_DISPLAY_B) {
+				ctx->reg_mem[i] = NULL;
+				ctx->regs[i] = NULL;
+				continue;
+			}
+			dev_err(&pdev->dev, "missing module memory resource\n");
+			return -ENXIO;
+		}
+		ctx->reg_mem[i] = request_mem_region(mem->start,
+						resource_size(mem), pdev->name);
+		if (!ctx->reg_mem[i]) {
+			dev_err(&pdev->dev, "failed to get module memory\n");
+			return -ENXIO;
+		}
+		ctx->regs[i] = ioremap(mem->start, resource_size(mem));
+		if (!ctx->regs[i]) {
+			dev_err(&pdev->dev, "failed to map module registers\n");
+			return -ENXIO;
+		}
+	}
+
+	return 0;
+}
+
+void nvhost_cpuaccess_deinit(struct nvhost_cpuaccess *ctx)
+{
+	int i;
+	for (i = 0; i < NVHOST_MODULE_NUM; i++) {
+		if (ctx->regs[i])
+			iounmap(ctx->regs[i]);
+		if (ctx->reg_mem[i])
+			release_resource(ctx->reg_mem[i]);
+	}
+}
+
+int nvhost_mutex_try_lock(struct nvhost_cpuaccess *ctx, unsigned int idx)
+{
+	struct nvhost_dev *dev = cpuaccess_to_dev(ctx);
+	void __iomem *sync_regs = dev->sync_aperture;
+	u32 reg;
+
+	/* mlock registers returns 0 when the lock is aquired.
+	 * writing 0 clears the lock. */
+	nvhost_module_busy(&dev->mod);
+	reg = readl(sync_regs + (HOST1X_SYNC_MLOCK_0 + idx * 4));
+	if (reg) {
+		nvhost_module_idle(&dev->mod);
+		return -ERESTARTSYS;
+	}
+	return 0;
+}
+
+void nvhost_mutex_unlock(struct nvhost_cpuaccess *ctx, unsigned int idx)
+{
+	struct nvhost_dev *dev = cpuaccess_to_dev(ctx);
+	void __iomem *sync_regs = dev->sync_aperture;
+	writel(0, sync_regs + (HOST1X_SYNC_MLOCK_0 + idx * 4));
+	nvhost_module_idle(&dev->mod);
+}
+
+void nvhost_read_module_regs(struct nvhost_cpuaccess *ctx, u32 module,
+			u32 offset, size_t size, void *values)
+{
+	struct nvhost_dev *dev = cpuaccess_to_dev(ctx);
+	void __iomem *p = ctx->regs[module] + offset;
+	u32* out = (u32*)values;
+	BUG_ON(size & 3);
+	size >>= 2;
+	nvhost_module_busy(&dev->mod);
+	while (size--) {
+		*(out++) = readl(p);
+		p += 4;
+	}
+	rmb();
+	nvhost_module_idle(&dev->mod);
+}
+
+void nvhost_write_module_regs(struct nvhost_cpuaccess *ctx, u32 module,
+			u32 offset, size_t size, const void *values)
+{
+	struct nvhost_dev *dev = cpuaccess_to_dev(ctx);
+	void __iomem *p = ctx->regs[module] + offset;
+	const u32* in = (const u32*)values;
+	BUG_ON(size & 3);
+	size >>= 2;
+	nvhost_module_busy(&dev->mod);
+	while (size--) {
+		writel(*(in++), p);
+		p += 4;
+	}
+	wmb();
+	nvhost_module_idle(&dev->mod);
+}
diff --git a/drivers/video/tegra/host/nvhost_cpuaccess.h b/drivers/video/tegra/host/nvhost_cpuaccess.h
new file mode 100644
index 000000000000..919e47c97b9e
--- /dev/null
+++ b/drivers/video/tegra/host/nvhost_cpuaccess.h
@@ -0,0 +1,71 @@
+/*
+ * drivers/video/tegra/host/nvhost_cpuaccess.h
+ *
+ * Tegra Graphics Host Cpu Register Access
+ *
+ * Copyright (c) 2010, NVIDIA Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+#ifndef __NVHOST_CPUACCESS_H
+#define __NVHOST_CPUACCESS_H
+
+#include "nvhost_hardware.h"
+#include <linux/platform_device.h>
+#include <linux/io.h>
+
+enum nvhost_module_id {
+	NVHOST_MODULE_DISPLAY_A = 0,
+	NVHOST_MODULE_DISPLAY_B,
+	NVHOST_MODULE_VI,
+	NVHOST_MODULE_ISP,
+	NVHOST_MODULE_MPE,
+#if 0
+	/* TODO: [ahatala 2010-07-02] find out if these are needed */
+	NVHOST_MODULE_FUSE,
+	NVHOST_MODULE_APB_MISC,
+	NVHOST_MODULE_CLK_RESET,
+#endif
+	NVHOST_MODULE_NUM
+};
+
+struct nvhost_cpuaccess {
+	struct resource *reg_mem[NVHOST_MODULE_NUM];
+	void __iomem *regs[NVHOST_MODULE_NUM];
+};
+
+int nvhost_cpuaccess_init(struct nvhost_cpuaccess *ctx,
+			struct platform_device *pdev);
+
+void nvhost_cpuaccess_deinit(struct nvhost_cpuaccess *ctx);
+
+int nvhost_mutex_try_lock(struct nvhost_cpuaccess *ctx, unsigned int idx);
+
+void nvhost_mutex_unlock(struct nvhost_cpuaccess *ctx, unsigned int idx);
+
+static inline bool nvhost_access_module_regs(
+	struct nvhost_cpuaccess *ctx, u32 module)
+{
+	return (module < NVHOST_MODULE_NUM) && ctx->reg_mem[module];
+}
+
+void nvhost_read_module_regs(struct nvhost_cpuaccess *ctx, u32 module,
+			u32 offset, size_t size, void *values);
+
+void nvhost_write_module_regs(struct nvhost_cpuaccess *ctx, u32 module,
+			u32 offset, size_t size, const void *values);
+
+#endif
diff --git a/drivers/video/tegra/host/nvhost_dev.c b/drivers/video/tegra/host/nvhost_dev.c
new file mode 100644
index 000000000000..6a5dbcec58ea
--- /dev/null
+++ b/drivers/video/tegra/host/nvhost_dev.c
@@ -0,0 +1,779 @@
+/*
+ * drivers/video/tegra/host/nvhost_dev.c
+ *
+ * Tegra Graphics Host Driver Entrypoint
+ *
+ * Copyright (c) 2010, NVIDIA Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+#include "nvhost_dev.h"
+
+#include <linux/nvhost.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+#include <linux/spinlock.h>
+#include <linux/fs.h>
+#include <linux/cdev.h>
+#include <linux/platform_device.h>
+#include <linux/uaccess.h>
+#include <linux/file.h>
+#include <asm/io.h>
+
+#define DRIVER_NAME "tegra_grhost"
+#define IFACE_NAME "nvhost"
+
+static int nvhost_major = NVHOST_MAJOR;
+static int nvhost_minor = NVHOST_CHANNEL_BASE;
+
+struct nvhost_channel_userctx {
+	struct nvhost_channel *ch;
+	struct nvhost_hwctx *hwctx;
+	struct file *nvmapctx;
+	u32 syncpt_id;
+	u32 syncpt_incrs;
+	u32 cmdbufs_pending;
+	u32 relocs_pending;
+	struct nvmap_handle *gather_mem;
+	struct nvhost_op_pair *gathers;
+	int num_gathers;
+	int pinarray_size;
+	struct nvmap_pinarray_elem pinarray[NVHOST_MAX_HANDLES];
+	struct nvmap_handle *unpinarray[NVHOST_MAX_HANDLES];
+	/* hw context (if needed) */
+};
+
+struct nvhost_ctrl_userctx {
+	struct nvhost_dev *dev;
+	u32 mod_locks[NV_HOST1X_NB_MLOCKS];
+};
+
+static int nvhost_channelrelease(struct inode *inode, struct file *filp)
+{
+	struct nvhost_channel_userctx *priv = filp->private_data;
+	filp->private_data = NULL;
+
+	nvhost_putchannel(priv->ch, priv->hwctx);
+	if (priv->hwctx) {
+		if (priv->hwctx->valid)
+			nvhost_syncpt_wait(&priv->ch->dev->syncpt,
+					priv->hwctx->last_access_id,
+					priv->hwctx->last_access_value);
+		priv->ch->ctxhandler.deinit(priv->hwctx);
+	}
+	if (priv->gather_mem)
+		nvmap_free(priv->gather_mem, priv->gathers);
+	if (priv->nvmapctx)
+		fput(priv->nvmapctx);
+	kfree(priv);
+	return 0;
+}
+
+static int nvhost_channelopen(struct inode *inode, struct file *filp)
+{
+	struct nvhost_channel_userctx *priv;
+	struct nvhost_channel *ch;
+	size_t hwctx_mem = 0;
+	size_t alloc_size = 0;
+
+	ch = container_of(inode->i_cdev, struct nvhost_channel, cdev);
+	ch = nvhost_getchannel(ch);
+	if (IS_ERR(ch))
+		return PTR_ERR(ch);
+
+	alloc_size += sizeof(*priv);
+	if (ch->ctxhandler.init) {
+		hwctx_mem = alloc_size;
+		alloc_size += sizeof(struct nvhost_hwctx);
+	}
+	priv = kzalloc(alloc_size, GFP_KERNEL);
+	if (!priv) {
+		nvhost_putchannel(ch, NULL);
+		return -ENOMEM;
+	}
+	filp->private_data = priv;
+	priv->ch = ch;
+	priv->gather_mem = nvmap_alloc(
+		sizeof(struct nvhost_op_pair) * NVHOST_MAX_GATHERS, 32,
+		NVMEM_HANDLE_CACHEABLE, (void**)&priv->gathers);
+	if (IS_ERR_OR_NULL(priv->gather_mem))
+		goto fail;
+	if (ch->ctxhandler.init) {
+		priv->hwctx = (struct nvhost_hwctx *)(((u8*)priv) + hwctx_mem);
+		priv->hwctx->channel = ch;
+		if (ch->ctxhandler.init(priv->hwctx) < 0) {
+			priv->hwctx = NULL;
+			goto fail;
+		}
+	}
+
+	return 0;
+fail:
+	nvhost_channelrelease(inode, filp);
+	return -ENOMEM;
+}
+
+static void add_gather(struct nvhost_channel_userctx *ctx, int idx,
+		struct nvmap_handle *mem, u32 words, u32 offset)
+{
+	struct nvmap_pinarray_elem *pin;
+	pin = &ctx->pinarray[ctx->pinarray_size++];
+	pin->patch_mem = ctx->gather_mem;
+	pin->patch_offset = (idx * sizeof(struct nvhost_op_pair)) +
+		offsetof(struct nvhost_op_pair, op2);
+	pin->pin_mem = mem;
+	pin->pin_offset = offset;
+	ctx->gathers[idx].op1 = nvhost_opcode_gather(0, words);
+}
+
+static void reset_submit(struct nvhost_channel_userctx *ctx)
+{
+	ctx->cmdbufs_pending = 0;
+	ctx->relocs_pending = 0;
+}
+
+static ssize_t nvhost_channelwrite(struct file *filp, const char __user *buf,
+				size_t count, loff_t *offp)
+{
+	struct nvhost_channel_userctx *priv = filp->private_data;
+	size_t remaining = count;
+	int err = 0;
+
+	while (remaining) {
+		size_t consumed;
+		if (!priv->relocs_pending && !priv->cmdbufs_pending) {
+			consumed = sizeof(struct nvhost_submit_hdr);
+			if (remaining < consumed)
+				break;
+			if (copy_from_user(&priv->syncpt_id, buf, consumed)) {
+				err = -EFAULT;
+				break;
+			}
+			if (!priv->cmdbufs_pending) {
+				err = -EFAULT;
+				break;
+			}
+			/* leave room for ctx switch */
+			priv->num_gathers = 2;
+			priv->pinarray_size = 0;
+		} else if (priv->cmdbufs_pending) {
+			struct nvhost_cmdbuf cmdbuf;
+			consumed = sizeof(cmdbuf);
+			if (remaining < consumed)
+				break;
+			if (copy_from_user(&cmdbuf, buf, consumed)) {
+				err = -EFAULT;
+				break;
+			}
+			add_gather(priv, priv->num_gathers++,
+				(struct nvmap_handle *)cmdbuf.mem,
+				cmdbuf.words, cmdbuf.offset);
+			priv->cmdbufs_pending--;
+		} else if (priv->relocs_pending) {
+			int numrelocs = remaining / sizeof(struct nvhost_reloc);
+			if (!numrelocs)
+				break;
+			numrelocs = min_t(int, numrelocs, priv->relocs_pending);
+			consumed = numrelocs * sizeof(struct nvhost_reloc);
+			if (copy_from_user(&priv->pinarray[priv->pinarray_size],
+						buf, consumed)) {
+				err = -EFAULT;
+				break;
+			}
+			priv->pinarray_size += numrelocs;
+			priv->relocs_pending -= numrelocs;
+		} else {
+			err = -EFAULT;
+			break;
+		}
+		remaining -= consumed;
+		buf += consumed;
+	}
+
+	if (err < 0) {
+		dev_err(&priv->ch->dev->pdev->dev, "channel write error\n");
+		reset_submit(priv);
+		return err;
+	}
+
+	return (count - remaining);
+}
+
+static int nvhost_ioctl_channel_flush(
+	struct nvhost_channel_userctx *ctx,
+	struct nvhost_get_param_args *args)
+{
+	struct nvhost_cpuinterrupt ctxsw;
+	int gather_idx = 2;
+	int num_intrs = 0;
+	u32 syncval;
+	int num_unpin;
+	int err;
+
+	if (ctx->relocs_pending || ctx->cmdbufs_pending) {
+		reset_submit(ctx);
+		dev_err(&ctx->ch->dev->pdev->dev, "channel submit out of sync\n");
+		return -EFAULT;
+	}
+	if (!ctx->nvmapctx) {
+		dev_err(&ctx->ch->dev->pdev->dev, "no nvmap context set\n");
+		return -EFAULT;
+	}
+	if (ctx->num_gathers <= 2)
+		return 0;
+
+	/* keep module powered */
+	nvhost_module_busy(&ctx->ch->mod);
+
+	/* pin mem handles and patch physical addresses */
+	err = nvmap_pin_array(ctx->nvmapctx, ctx->pinarray, ctx->pinarray_size,
+			ctx->unpinarray, &num_unpin, true);
+	if (err) {
+		dev_warn(&ctx->ch->dev->pdev->dev, "nvmap_pin_array failed: %d\n", err);
+		nvhost_module_idle(&ctx->ch->mod);
+		return err;
+	}
+
+	/* get submit lock */
+	err = mutex_lock_interruptible(&ctx->ch->submitlock);
+	if (err) {
+		nvmap_unpin(ctx->unpinarray, num_unpin);
+		nvhost_module_idle(&ctx->ch->mod);
+		return err;
+	}
+
+	/* context switch */
+	if (ctx->ch->cur_ctx != ctx->hwctx) {
+		struct nvhost_hwctx *hw = ctx->hwctx;
+		if (hw && hw->valid) {
+			gather_idx--;
+			ctx->gathers[gather_idx].op1 =
+				nvhost_opcode_gather(0, hw->restore_size);
+			ctx->gathers[gather_idx].op2 = hw->restore_phys;
+			ctx->syncpt_incrs += hw->restore_incrs;
+		}
+		hw = ctx->ch->cur_ctx;
+		if (hw) {
+			gather_idx--;
+			ctx->gathers[gather_idx].op1 =
+				nvhost_opcode_gather(0, hw->save_size);
+			ctx->gathers[gather_idx].op2 = hw->save_phys;
+			ctx->syncpt_incrs += hw->save_incrs;
+			num_intrs = 1;
+			ctxsw.syncpt_val = hw->save_incrs - 1;
+			ctxsw.intr_data = hw;
+		}
+	}
+
+	/* add a setclass for modules that require it */
+	if (gather_idx == 2 && ctx->ch->desc->class) {
+		gather_idx--;
+		ctx->gathers[gather_idx].op1 =
+			nvhost_opcode_setclass(ctx->ch->desc->class, 0, 0);
+		ctx->gathers[gather_idx].op2 = NVHOST_OPCODE_NOOP;
+	}
+
+	/* get absolute sync value */
+	if (BIT(ctx->syncpt_id) & NVSYNCPTS_CLIENT_MANAGED)
+		syncval = nvhost_syncpt_set_max(&ctx->ch->dev->syncpt,
+						ctx->syncpt_id, ctx->syncpt_incrs);
+	else
+		syncval = nvhost_syncpt_incr_max(&ctx->ch->dev->syncpt,
+						ctx->syncpt_id, ctx->syncpt_incrs);
+
+	/* patch absolute syncpt value into interrupt triggers */
+	ctxsw.syncpt_val += syncval - ctx->syncpt_incrs;
+
+	nvhost_channel_submit(ctx->ch, &ctx->gathers[gather_idx],
+			ctx->num_gathers - gather_idx, &ctxsw, num_intrs,
+			ctx->unpinarray, num_unpin, ctx->syncpt_id, syncval);
+
+	/* schedule a submit complete interrupt */
+	nvhost_intr_add_action(&ctx->ch->dev->intr, ctx->syncpt_id, syncval,
+			NVHOST_INTR_ACTION_SUBMIT_COMPLETE, ctx->ch, NULL);
+
+	/* update current context */
+	if (ctx->ch->cur_ctx != ctx->hwctx) {
+		struct nvhost_hwctx *hw = ctx->ch->cur_ctx;
+		if (hw) {
+			hw->last_access_id = ctx->syncpt_id;
+			hw->last_access_value = syncval;
+			hw->valid = true;
+		}
+		hw = ctx->hwctx;
+		hw->last_access_id = ctx->syncpt_id;
+		hw->last_access_value = syncval;
+		ctx->ch->cur_ctx = hw;
+	}
+	mutex_unlock(&ctx->ch->submitlock);
+	args->value = syncval;
+	return 0;
+}
+
+static long nvhost_channelctl(struct file *filp,
+	unsigned int cmd, unsigned long arg)
+{
+	struct nvhost_channel_userctx *priv = filp->private_data;
+	u8 buf[NVHOST_IOCTL_CHANNEL_MAX_ARG_SIZE];
+	int err = 0;
+
+	if ((_IOC_TYPE(cmd) != NVHOST_IOCTL_MAGIC) ||
+		(_IOC_NR(cmd) == 0) ||
+		(_IOC_NR(cmd) > NVHOST_IOCTL_CHANNEL_LAST))
+		return -EFAULT;
+
+	BUG_ON(_IOC_SIZE(cmd) > NVHOST_IOCTL_CHANNEL_MAX_ARG_SIZE);
+
+	if (_IOC_DIR(cmd) & _IOC_WRITE) {
+		if (copy_from_user(buf, (void __user *)arg, _IOC_SIZE(cmd)))
+			return -EFAULT;
+	}
+
+	switch (cmd) {
+	case NVHOST_IOCTL_CHANNEL_FLUSH:
+		err = nvhost_ioctl_channel_flush(priv, (void *)buf);
+		break;
+	case NVHOST_IOCTL_CHANNEL_GET_SYNCPOINTS:
+		((struct nvhost_get_param_args *)buf)->value =
+			priv->ch->desc->syncpts;
+		break;
+	case NVHOST_IOCTL_CHANNEL_GET_WAITBASES:
+		((struct nvhost_get_param_args *)buf)->value =
+			priv->ch->desc->waitbases;
+		break;
+	case NVHOST_IOCTL_CHANNEL_GET_MODMUTEXES:
+		((struct nvhost_get_param_args *)buf)->value =
+			priv->ch->desc->modulemutexes;
+		break;
+	case NVHOST_IOCTL_CHANNEL_SET_NVMAP_FD:
+	{
+		int fd = (int)((struct nvhost_set_nvmap_fd_args *)buf)->fd;
+		struct file *newctx = NULL;
+		if (fd) {
+			newctx = fget(fd);
+			if (!newctx) {
+				err = -EFAULT;
+				break;
+			}
+			err = nvmap_validate_file(newctx);
+			if (err) {
+				fput(newctx);
+				break;
+			}
+		}
+		if (priv->nvmapctx)
+			fput(priv->nvmapctx);
+		priv->nvmapctx = newctx;
+		break;
+	}
+	default:
+		err = -ENOTTY;
+		break;
+	}
+
+	if ((err == 0) && (_IOC_DIR(cmd) & _IOC_READ))
+		err = copy_to_user((void __user *)arg, buf, _IOC_SIZE(cmd));
+
+	return err;
+}
+
+static struct file_operations nvhost_channelops = {
+	.owner = THIS_MODULE,
+	.release = nvhost_channelrelease,
+	.open = nvhost_channelopen,
+	.write = nvhost_channelwrite,
+	.unlocked_ioctl = nvhost_channelctl
+};
+
+static int nvhost_ctrlrelease(struct inode *inode, struct file *filp)
+{
+	struct nvhost_ctrl_userctx *priv = filp->private_data;
+	int i;
+
+	filp->private_data = NULL;
+	if (priv->mod_locks[0])
+		nvhost_module_idle(&priv->dev->mod);
+	for (i = 1; i < NV_HOST1X_NB_MLOCKS; i++)
+		if (priv->mod_locks[i])
+			nvhost_mutex_unlock(&priv->dev->cpuaccess, i);
+	kfree(priv);
+	return 0;
+}
+
+static int nvhost_ctrlopen(struct inode *inode, struct file *filp)
+{
+	struct nvhost_dev *host = container_of(inode->i_cdev, struct nvhost_dev, cdev);
+	struct nvhost_ctrl_userctx *priv;
+
+	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	priv->dev = host;
+	filp->private_data = priv;
+	return 0;
+}
+
+static int nvhost_ioctl_ctrl_syncpt_read(
+	struct nvhost_ctrl_userctx *ctx,
+	struct nvhost_ctrl_syncpt_read_args *args)
+{
+	if (args->id >= NV_HOST1X_SYNCPT_NB_PTS)
+		return -EINVAL;
+	args->value = nvhost_syncpt_read(&ctx->dev->syncpt, args->id);
+	return 0;
+}
+
+static int nvhost_ioctl_ctrl_syncpt_incr(
+	struct nvhost_ctrl_userctx *ctx,
+	struct nvhost_ctrl_syncpt_incr_args *args)
+{
+	if (args->id >= NV_HOST1X_SYNCPT_NB_PTS)
+		return -EINVAL;
+	nvhost_syncpt_incr(&ctx->dev->syncpt, args->id);
+	return 0;
+}
+
+static int nvhost_ioctl_ctrl_syncpt_wait(
+	struct nvhost_ctrl_userctx *ctx,
+	struct nvhost_ctrl_syncpt_wait_args *args)
+{
+	u32 timeout;
+	if (args->id >= NV_HOST1X_SYNCPT_NB_PTS)
+		return -EINVAL;
+	if (args->timeout == NVHOST_NO_TIMEOUT)
+		timeout = MAX_SCHEDULE_TIMEOUT;
+	else
+		timeout = (u32)msecs_to_jiffies(args->timeout);
+
+	return nvhost_syncpt_wait_timeout(&ctx->dev->syncpt, args->id,
+					args->thresh, timeout);
+}
+
+static int nvhost_ioctl_ctrl_module_mutex(
+	struct nvhost_ctrl_userctx *ctx,
+	struct nvhost_ctrl_module_mutex_args *args)
+{
+	int err = 0;
+	if (args->id >= NV_HOST1X_SYNCPT_NB_PTS ||
+	    args->lock > 1)
+		return -EINVAL;
+
+	if (args->lock && !ctx->mod_locks[args->id]) {
+		if (args->id == 0)
+			nvhost_module_busy(&ctx->dev->mod);
+		else
+			err = nvhost_mutex_try_lock(&ctx->dev->cpuaccess, args->id);
+		if (!err)
+			ctx->mod_locks[args->id] = 1;
+	}
+	else if (!args->lock && ctx->mod_locks[args->id]) {
+		if (args->id == 0)
+			nvhost_module_idle(&ctx->dev->mod);
+		else
+			nvhost_mutex_unlock(&ctx->dev->cpuaccess, args->id);
+		ctx->mod_locks[args->id] = 0;
+	}
+	return err;
+}
+
+static int nvhost_ioctl_ctrl_module_regrdwr(
+	struct nvhost_ctrl_userctx *ctx,
+	struct nvhost_ctrl_module_regrdwr_args *args)
+{
+	u32 num_offsets = args->num_offsets;
+	u32 *offsets = args->offsets;
+	void *values = args->values;
+	u32 vals[64];
+
+	if (!nvhost_access_module_regs(&ctx->dev->cpuaccess, args->id) ||
+	    (num_offsets == 0))
+		return -EINVAL;
+
+	while (num_offsets--) {
+		u32 remaining = args->block_size;
+		u32 offs;
+		if (get_user(offs, offsets))
+			return -EFAULT;
+		offsets++;
+		while (remaining) {
+			u32 batch = min(remaining, 64*sizeof(u32));
+			if (args->write) {
+				if (copy_from_user(vals, values, batch))
+					return -EFAULT;
+				nvhost_write_module_regs(&ctx->dev->cpuaccess,
+							args->id, offs, batch, vals);
+			} else {
+				nvhost_read_module_regs(&ctx->dev->cpuaccess,
+							args->id, offs, batch, vals);
+				if (copy_to_user(values, vals, batch))
+					return -EFAULT;
+			}
+			remaining -= batch;
+			offs += batch;
+			values += batch;
+		}
+	}
+
+	return 0;
+}
+
+static long nvhost_ctrlctl(struct file *filp,
+	unsigned int cmd, unsigned long arg)
+{
+	struct nvhost_ctrl_userctx *priv = filp->private_data;
+	u8 buf[NVHOST_IOCTL_CTRL_MAX_ARG_SIZE];
+	int err = 0;
+
+	if ((_IOC_TYPE(cmd) != NVHOST_IOCTL_MAGIC) ||
+		(_IOC_NR(cmd) == 0) ||
+		(_IOC_NR(cmd) > NVHOST_IOCTL_CTRL_LAST))
+		return -EFAULT;
+
+	BUG_ON(_IOC_SIZE(cmd) > NVHOST_IOCTL_CTRL_MAX_ARG_SIZE);
+
+	if (_IOC_DIR(cmd) & _IOC_WRITE) {
+		if (copy_from_user(buf, (void __user *)arg, _IOC_SIZE(cmd)))
+			return -EFAULT;
+	}
+
+	switch (cmd) {
+	case NVHOST_IOCTL_CTRL_SYNCPT_READ:
+		err = nvhost_ioctl_ctrl_syncpt_read(priv, (void *)buf);
+		break;
+	case NVHOST_IOCTL_CTRL_SYNCPT_INCR:
+		err = nvhost_ioctl_ctrl_syncpt_incr(priv, (void *)buf);
+		break;
+	case NVHOST_IOCTL_CTRL_SYNCPT_WAIT:
+		err = nvhost_ioctl_ctrl_syncpt_wait(priv, (void *)buf);
+		break;
+	case NVHOST_IOCTL_CTRL_MODULE_MUTEX:
+		err = nvhost_ioctl_ctrl_module_mutex(priv, (void *)buf);
+		break;
+	case NVHOST_IOCTL_CTRL_MODULE_REGRDWR:
+		err = nvhost_ioctl_ctrl_module_regrdwr(priv, (void *)buf);
+		break;
+	default:
+		err = -ENOTTY;
+		break;
+	}
+
+	if ((err == 0) && (_IOC_DIR(cmd) & _IOC_READ))
+		err = copy_to_user((void __user *)arg, buf, _IOC_SIZE(cmd));
+
+	return err;
+}
+
+static struct file_operations nvhost_ctrlops = {
+	.owner = THIS_MODULE,
+	.release = nvhost_ctrlrelease,
+	.open = nvhost_ctrlopen,
+	.unlocked_ioctl = nvhost_ctrlctl
+};
+
+static void power_host(struct nvhost_module *mod, enum nvhost_power_action action)
+{
+	struct nvhost_dev *dev = container_of(mod, struct nvhost_dev, mod);
+
+	if (action == NVHOST_POWER_ACTION_ON) {
+		nvhost_intr_start(&dev->intr, clk_get_rate(mod->clk[0]));
+		nvhost_syncpt_reset(&dev->syncpt);
+	} else if (action == NVHOST_POWER_ACTION_OFF) {
+		int i;
+		for (i = 0; i < NVHOST_NUMCHANNELS; i++)
+			nvhost_channel_suspend(&dev->channels[i]);
+		nvhost_syncpt_save(&dev->syncpt);
+		nvhost_intr_stop(&dev->intr);
+	}
+}
+
+static int __init nvhost_user_init(struct nvhost_dev *host)
+{
+	int i, err, devno;
+
+	host->nvhost_class = class_create(THIS_MODULE, IFACE_NAME);
+	if (IS_ERR(host->nvhost_class)) {
+		err = PTR_ERR(host->nvhost_class);
+		dev_err(&host->pdev->dev, "failed to create class\n");
+		goto fail;
+	}
+
+	if (nvhost_major) {
+		devno = MKDEV(nvhost_major, nvhost_minor);
+		err = register_chrdev_region(devno, NVHOST_NUMCHANNELS + 1, IFACE_NAME);
+	} else {
+		err = alloc_chrdev_region(&devno, nvhost_minor,
+					NVHOST_NUMCHANNELS + 1, IFACE_NAME);
+		nvhost_major = MAJOR(devno);
+	}
+	if (err < 0) {
+		dev_err(&host->pdev->dev, "failed to reserve chrdev region\n");
+		goto fail;
+	}
+
+	for (i = 0; i < NVHOST_NUMCHANNELS; i++) {
+		struct nvhost_channel *ch = &host->channels[i];
+
+		if (!strcmp(ch->desc->name, "display") &&
+		    !nvhost_access_module_regs(&host->cpuaccess,
+						NVHOST_MODULE_DISPLAY_A))
+			continue;
+
+		cdev_init(&ch->cdev, &nvhost_channelops);
+		ch->cdev.owner = THIS_MODULE;
+
+		devno = MKDEV(nvhost_major, nvhost_minor + i);
+		err = cdev_add(&ch->cdev, devno, 1);
+		if (err < 0) {
+			dev_err(&host->pdev->dev, "failed to add chan %i cdev\n", i);
+			goto fail;
+		}
+		ch->node = device_create(host->nvhost_class, NULL, devno, NULL,
+				IFACE_NAME "-%s", ch->desc->name);
+		if (IS_ERR(ch->node)) {
+			err = PTR_ERR(ch->node);
+			dev_err(&host->pdev->dev, "failed to create chan %i device\n", i);
+			goto fail;
+		}
+	}
+
+	cdev_init(&host->cdev, &nvhost_ctrlops);
+	host->cdev.owner = THIS_MODULE;
+	devno = MKDEV(nvhost_major, nvhost_minor + NVHOST_NUMCHANNELS);
+	err = cdev_add(&host->cdev, devno, 1);
+	if (err < 0)
+		goto fail;
+	host->ctrl = device_create(host->nvhost_class, NULL, devno, NULL,
+			IFACE_NAME "-ctrl");
+	if (IS_ERR(host->ctrl)) {
+		err = PTR_ERR(host->ctrl);
+		dev_err(&host->pdev->dev, "failed to create ctrl device\n");
+		goto fail;
+	}
+
+	return 0;
+fail:
+	return err;
+}
+
+static int __init nvhost_probe(struct platform_device *pdev)
+{
+	struct nvhost_dev *host;
+	struct resource *regs, *intr0, *intr1;
+	int i, err;
+
+	regs = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	intr0 = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
+	intr1 = platform_get_resource(pdev, IORESOURCE_IRQ, 1);
+
+	if (!regs || !intr0 || !intr1) {
+		dev_err(&pdev->dev, "missing required platform resources\n");
+		return -ENXIO;
+	}
+
+	host = kzalloc(sizeof(*host), GFP_KERNEL);
+	if (!host)
+		return -ENOMEM;
+
+	host->pdev = pdev;
+
+	host->reg_mem = request_mem_region(regs->start,
+					resource_size(regs), pdev->name);
+	if (!host->reg_mem) {
+		dev_err(&pdev->dev, "failed to get host register memory\n");
+		err = -ENXIO;
+		goto fail;
+	}
+	host->aperture = ioremap(regs->start, resource_size(regs));
+	if (!host->aperture) {
+		dev_err(&pdev->dev, "failed to remap host registers\n");
+		err = -ENXIO;
+		goto fail;
+	}
+	host->sync_aperture = host->aperture +
+		(NV_HOST1X_CHANNEL0_BASE +
+			HOST1X_CHANNEL_SYNC_REG_BASE);
+
+	for (i = 0; i < NVHOST_NUMCHANNELS; i++) {
+		struct nvhost_channel *ch = &host->channels[i];
+		err = nvhost_channel_init(ch, host, i);
+		if (err < 0) {
+			dev_err(&pdev->dev, "failed to init channel %d\n", i);
+			goto fail;
+		}
+	}
+
+	err = nvhost_cpuaccess_init(&host->cpuaccess, pdev);
+	if (err) goto fail;
+	err = nvhost_intr_init(&host->intr, intr1->start, intr0->start);
+	if (err) goto fail;
+	err = nvhost_user_init(host);
+	if (err) goto fail;
+	err = nvhost_module_init(&host->mod, "host1x", power_host, NULL, &pdev->dev);
+	if (err) goto fail;
+
+	platform_set_drvdata(pdev, host);
+
+	dev_info(&pdev->dev, "initialized\n");
+	return 0;
+
+fail:
+	/* TODO: [ahatala 2010-05-04] */
+	kfree(host);
+	return err;
+}
+
+static int __exit nvhost_remove(struct platform_device *pdev)
+{
+	return 0;
+}
+
+static int nvhost_suspend(struct platform_device *pdev, pm_message_t state)
+{
+	struct nvhost_dev *host = platform_get_drvdata(pdev);
+	dev_info(&pdev->dev, "suspending\n");
+	nvhost_module_suspend(&host->mod);
+	dev_info(&pdev->dev, "suspended\n");
+	return 0;
+}
+
+static struct platform_driver nvhost_driver = {
+	.remove = __exit_p(nvhost_remove),
+	.suspend = nvhost_suspend,
+	.driver = {
+		.owner = THIS_MODULE,
+		.name = DRIVER_NAME
+	}
+};
+
+static int __init nvhost_mod_init(void)
+{
+	return platform_driver_probe(&nvhost_driver, nvhost_probe);
+}
+
+static void __exit nvhost_mod_exit(void)
+{
+	platform_driver_unregister(&nvhost_driver);
+}
+
+module_init(nvhost_mod_init);
+module_exit(nvhost_mod_exit);
+
+MODULE_AUTHOR("NVIDIA");
+MODULE_DESCRIPTION("Graphics host driver for Tegra products");
+MODULE_VERSION("1.0");
+MODULE_LICENSE("Dual BSD/GPL");
+MODULE_ALIAS("platform-nvhost");
diff --git a/drivers/video/tegra/host/nvhost_dev.h b/drivers/video/tegra/host/nvhost_dev.h
new file mode 100644
index 000000000000..385dad8d867f
--- /dev/null
+++ b/drivers/video/tegra/host/nvhost_dev.h
@@ -0,0 +1,50 @@
+/*
+ * drivers/video/tegra/host/nvhost_dev.h
+ *
+ * Tegra Graphics Host Driver Entrypoint
+ *
+ * Copyright (c) 2010, NVIDIA Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+#ifndef __NVHOST_DEV_H
+#define __NVHOST_DEV_H
+
+#include "nvhost_acm.h"
+#include "nvhost_syncpt.h"
+#include "nvhost_intr.h"
+#include "nvhost_cpuaccess.h"
+#include "nvhost_channel.h"
+#include "nvhost_hardware.h"
+
+#define NVHOST_MAJOR 0 /* dynamic */
+
+struct nvhost_dev {
+	void __iomem *aperture;
+	void __iomem *sync_aperture;
+	struct resource *reg_mem;
+	struct platform_device *pdev;
+	struct class *nvhost_class;
+	struct cdev cdev;
+	struct device *ctrl;
+	struct nvhost_syncpt syncpt;
+	struct nvhost_cpuaccess cpuaccess;
+	struct nvhost_intr intr;
+	struct nvhost_module mod;
+	struct nvhost_channel channels[NVHOST_NUMCHANNELS];
+};
+
+#endif
diff --git a/drivers/video/tegra/host/nvhost_hardware.h b/drivers/video/tegra/host/nvhost_hardware.h
new file mode 100644
index 000000000000..661f775e6be3
--- /dev/null
+++ b/drivers/video/tegra/host/nvhost_hardware.h
@@ -0,0 +1,219 @@
+/*
+ * drivers/video/tegra/host/nvhost_hardware.h
+ *
+ * Tegra Graphics Host Register Offsets
+ *
+ * Copyright (c) 2010, NVIDIA Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+#ifndef __NVHOST_HARDWARE_H
+#define __NVHOST_HARDWARE_H
+
+#include <linux/types.h>
+#include <linux/bitops.h>
+
+/* class ids */
+enum {
+	NV_HOST1X_CLASS_ID = 0x1,
+	NV_VIDEO_ENCODE_MPEG_CLASS_ID = 0x20,
+	NV_GRAPHICS_3D_CLASS_ID = 0x60
+};
+
+
+/* channel registers */
+#define NV_HOST1X_CHANNELS 8
+#define NV_HOST1X_CHANNEL0_BASE 0
+#define NV_HOST1X_CHANNEL_MAP_SIZE_BYTES 16384
+
+enum {
+	HOST1X_CHANNEL_FIFOSTAT	= 0x00,
+	HOST1X_CHANNEL_INDDATA	= 0x0c,
+	HOST1X_CHANNEL_DMASTART = 0x14,
+	HOST1X_CHANNEL_DMAPUT	= 0x18,
+	HOST1X_CHANNEL_DMAEND	= 0x20,
+	HOST1X_CHANNEL_DMACTRL	= 0x24
+};
+
+static inline unsigned nvhost_channel_fifostat_outfentries(u32 reg)
+{
+	return (reg >> 24) & 0x1f;
+}
+
+static inline u32 nvhost_channel_dmactrl(bool stop, bool get_rst, bool init_get)
+{
+	u32 v = stop ? 1 : 0;
+	if (get_rst)
+		v |= 2;
+	if (init_get)
+		v |= 4;
+	return v;
+}
+
+
+/* sync registers */
+#define NV_HOST1X_SYNCPT_NB_PTS 32
+#define NV_HOST1X_SYNCPT_NB_BASES 8
+#define NV_HOST1X_NB_MLOCKS 16
+#define HOST1X_CHANNEL_SYNC_REG_BASE 12288
+
+enum {
+	HOST1X_SYNC_INTMASK = 0x4,
+	HOST1X_SYNC_INTC0MASK = 0x8,
+	HOST1X_SYNC_HINTSTATUS = 0x20,
+	HOST1X_SYNC_HINTMASK = 0x24,
+	HOST1X_SYNC_HINTSTATUS_EXT = 0x28,
+	HOST1X_SYNC_HINTMASK_EXT = 0x2c,
+	HOST1X_SYNC_SYNCPT_THRESH_CPU0_INT_STATUS = 0x40,
+	HOST1X_SYNC_SYNCPT_THRESH_CPU1_INT_STATUS = 0x48,
+	HOST1X_SYNC_SYNCPT_THRESH_INT_DISABLE = 0x60,
+	HOST1X_SYNC_SYNCPT_THRESH_INT_ENABLE_CPU0 = 0x68,
+	HOST1X_SYNC_USEC_CLK = 0x1a4,
+	HOST1X_SYNC_CTXSW_TIMEOUT_CFG = 0x1a8,
+	HOST1X_SYNC_IP_BUSY_TIMEOUT = 0x1bc,
+	HOST1X_SYNC_IP_READ_TIMEOUT_ADDR = 0x1c0,
+	HOST1X_SYNC_IP_WRITE_TIMEOUT_ADDR = 0x1c4,
+	HOST1X_SYNC_MLOCK_0 = 0x2c0,
+	HOST1X_SYNC_MLOCK_OWNER_0 = 0x340,
+	HOST1X_SYNC_SYNCPT_0 = 0x400,
+	HOST1X_SYNC_SYNCPT_INT_THRESH_0 = 0x500,
+	HOST1X_SYNC_SYNCPT_BASE_0 = 0x600,
+	HOST1X_SYNC_SYNCPT_CPU_INCR = 0x700
+};
+
+static inline bool nvhost_sync_hintstatus_ext_ip_read_int(u32 reg)
+{
+	return (reg & BIT(30)) != 0;
+}
+
+static inline bool nvhost_sync_hintstatus_ext_ip_write_int(u32 reg)
+{
+	return (reg & BIT(31)) != 0;
+}
+
+static inline bool nvhost_sync_mlock_owner_ch_owns(u32 reg)
+{
+	return (reg & BIT(0)) != 0;
+}
+
+static inline bool nvhost_sync_mlock_owner_cpu_owns(u32 reg)
+{
+	return (reg & BIT(1)) != 0;
+}
+
+static inline unsigned int nvhost_sync_mlock_owner_owner_chid(u32 reg)
+{
+	return (reg >> 8) & 0xf;
+}
+
+
+/* host class */
+enum {
+	NV_CLASS_HOST_INCR_SYNCPT = 0x0,
+	NV_CLASS_HOST_WAIT_SYNCPT = 0x8,
+	NV_CLASS_HOST_WAIT_SYNCPT_BASE = 0x9,
+	NV_CLASS_HOST_INCR_SYNCPT_BASE = 0xc,
+	NV_CLASS_HOST_INDOFF = 0x2d,
+	NV_CLASS_HOST_INDDATA = 0x2e
+};
+
+static inline u32 nvhost_class_host_wait_syncpt_base(
+	unsigned indx, unsigned base_indx, unsigned offset)
+{
+	return (indx << 24) | (base_indx << 16) | offset;
+}
+
+static inline u32 nvhost_class_host_incr_syncpt_base(
+	unsigned base_indx, unsigned offset)
+{
+	return (base_indx << 24) | offset;
+}
+
+enum {
+	NV_HOST_MODULE_HOST1X = 0,
+	NV_HOST_MODULE_MPE = 1,
+	NV_HOST_MODULE_GR3D = 6
+};
+
+static inline u32 nvhost_class_host_indoff_reg_write(
+	unsigned mod_id, unsigned offset, bool auto_inc)
+{
+	u32 v = (0xf << 28) | (mod_id << 18) | (offset << 2);
+	if (auto_inc)
+		v |= BIT(27);
+	return v;
+}
+
+static inline u32 nvhost_class_host_indoff_reg_read(
+	unsigned mod_id, unsigned offset, bool auto_inc)
+{
+	u32 v = (mod_id << 18) | (offset << 2) | 1;
+	if (auto_inc)
+		v |= BIT(27);
+	return v;
+}
+
+
+/* cdma opcodes */
+static inline u32 nvhost_opcode_setclass(
+	unsigned class_id, unsigned offset, unsigned mask)
+{
+	return (0 << 28) | (offset << 16) | (class_id << 6) | mask;
+}
+
+static inline u32 nvhost_opcode_incr(unsigned offset, unsigned count)
+{
+	return (1 << 28) | (offset << 16) | count;
+}
+
+static inline u32 nvhost_opcode_nonincr(unsigned offset, unsigned count)
+{
+	return (2 << 28) | (offset << 16) | count;
+}
+
+static inline u32 nvhost_opcode_mask(unsigned offset, unsigned mask)
+{
+	return (3 << 28) | (offset << 16) | mask;
+}
+
+static inline u32 nvhost_opcode_imm(unsigned offset, unsigned value)
+{
+	return (4 << 28) | (offset << 16) | value;
+}
+
+static inline u32 nvhost_opcode_restart(unsigned address)
+{
+	return (5 << 28) | (address >> 4);
+}
+
+static inline u32 nvhost_opcode_gather(unsigned offset, unsigned count)
+{
+	return (6 << 28) | (offset << 16) | count;
+}
+
+static inline u32 nvhost_opcode_gather_nonincr(unsigned offset,	unsigned count)
+{
+	return (6 << 28) | (offset << 16) | BIT(15) | count;
+}
+
+static inline u32 nvhost_opcode_gather_incr(unsigned offset, unsigned count)
+{
+	return (6 << 28) | (offset << 16) | BIT(15) | BIT(14) | count;
+}
+
+#define NVHOST_OPCODE_NOOP nvhost_opcode_nonincr(0, 0)
+
+#endif /* __NVHOST_HARDWARE_H */
diff --git a/drivers/video/tegra/host/nvhost_hwctx.h b/drivers/video/tegra/host/nvhost_hwctx.h
new file mode 100644
index 000000000000..d1a8930e0ecb
--- /dev/null
+++ b/drivers/video/tegra/host/nvhost_hwctx.h
@@ -0,0 +1,86 @@
+/*
+ * drivers/video/tegra/host/nvhost_hwctx.h
+ *
+ * Tegra Graphics Host Hardware Context Interface
+ *
+ * Copyright (c) 2010, NVIDIA Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+#ifndef __NVHOST_HWCTX_H
+#define __NVHOST_HWCTX_H
+
+#include <linux/string.h>
+#include <linux/nvhost.h>
+#include <linux/nvmap.h>
+
+struct nvhost_channel;
+
+struct nvhost_hwctx {
+	struct nvhost_channel *channel;
+	u32 last_access_id;
+	u32 last_access_value;
+	bool valid;
+
+	struct nvmap_handle *save;
+	u32 save_phys;
+	u32 save_size;
+	u32 save_incrs;
+	void *save_cpu_data;
+
+	struct nvmap_handle *restore;
+	u32 restore_phys;
+	u32 restore_size;
+	u32 restore_incrs;
+};
+
+struct nvhost_hwctx_handler {
+	int (*init) (struct nvhost_hwctx *ctx);
+	void (*deinit) (struct nvhost_hwctx *ctx);
+	void (*save_service) (struct nvhost_hwctx *ctx);
+};
+
+int nvhost_3dctx_handler_init(struct nvhost_hwctx_handler *h);
+int nvhost_mpectx_handler_init(struct nvhost_hwctx_handler *h);
+
+static inline int nvhost_hwctx_handler_init(
+	struct nvhost_hwctx_handler *h,
+	const char *module)
+{
+	if (strcmp(module, "gr3d") == 0)
+		return nvhost_3dctx_handler_init(h);
+	else if (strcmp(module, "mpe") == 0)
+		return nvhost_mpectx_handler_init(h);
+
+	return 0;
+}
+
+struct hwctx_reginfo {
+	unsigned int offset:12;
+	unsigned int count:16;
+	unsigned int type:2;
+};
+
+enum {
+	HWCTX_REGINFO_DIRECT = 0,
+	HWCTX_REGINFO_INDIRECT,
+	HWCTX_REGINFO_INDIRECT_OFFSET,
+	HWCTX_REGINFO_INDIRECT_DATA
+};
+
+#define HWCTX_REGINFO(offset, count, type) {offset, count, HWCTX_REGINFO_##type}
+
+#endif
diff --git a/drivers/video/tegra/host/nvhost_intr.c b/drivers/video/tegra/host/nvhost_intr.c
new file mode 100644
index 000000000000..3d101b221b71
--- /dev/null
+++ b/drivers/video/tegra/host/nvhost_intr.c
@@ -0,0 +1,561 @@
+/*
+ * drivers/video/tegra/host/nvhost_intr.c
+ *
+ * Tegra Graphics Host Interrupt Management
+ *
+ * Copyright (c) 2010, NVIDIA Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+#include "nvhost_intr.h"
+#include "nvhost_dev.h"
+#include <linux/interrupt.h>
+#include <linux/slab.h>
+#include <linux/irq.h>
+
+#define intr_to_dev(x) container_of(x, struct nvhost_dev, intr)
+
+
+/*** HW host sync management ***/
+
+void init_host_sync(void __iomem *sync_regs)
+{
+	/* disable the ip_busy_timeout. this prevents write drops, etc.
+	 * there's no real way to recover from a hung client anyway.
+	 */
+	writel(0, sync_regs + HOST1X_SYNC_IP_BUSY_TIMEOUT);
+
+	/* increase the auto-ack timout to the maximum value. 2d will hang
+	 * otherwise on ap20.
+	 */
+	writel(0xff, sync_regs + HOST1X_SYNC_CTXSW_TIMEOUT_CFG);
+}
+
+void set_host_clocks_per_microsecond(void __iomem *sync_regs, u32 cpm)
+{
+	/* write microsecond clock register */
+	writel(cpm, sync_regs + HOST1X_SYNC_USEC_CLK);
+}
+
+static void set_syncpt_threshold(void __iomem *sync_regs, u32 id, u32 thresh)
+{
+	thresh &= 0xffff;
+	writel(thresh, sync_regs + (HOST1X_SYNC_SYNCPT_INT_THRESH_0 + id * 4));
+}
+
+static void enable_syncpt_interrupt(void __iomem *sync_regs, u32 id)
+{
+	writel(BIT(id),	sync_regs + HOST1X_SYNC_SYNCPT_THRESH_INT_ENABLE_CPU0);
+}
+
+void disable_all_syncpt_interrupts(void __iomem *sync_regs)
+{
+	/* disable interrupts for both cpu's */
+	writel(0, sync_regs + HOST1X_SYNC_SYNCPT_THRESH_INT_DISABLE);
+
+	/* clear status for both cpu's */
+	writel(0xfffffffful, sync_regs +
+		HOST1X_SYNC_SYNCPT_THRESH_CPU0_INT_STATUS);
+	writel(0xfffffffful, sync_regs +
+		HOST1X_SYNC_SYNCPT_THRESH_CPU1_INT_STATUS);
+}
+
+
+/*** Wait list management ***/
+
+struct nvhost_waitlist {
+	struct list_head list;
+	struct kref refcount;
+	u32 thresh;
+	enum nvhost_intr_action action;
+	atomic_t state;
+	void *data;
+	int count;
+};
+
+enum waitlist_state
+{
+	WLS_PENDING,
+	WLS_REMOVED,
+	WLS_CANCELLED,
+	WLS_HANDLED
+};
+
+static void waiter_release(struct kref *kref)
+{
+	kfree(container_of(kref, struct nvhost_waitlist, refcount));
+}
+
+/**
+ * add a waiter to a waiter queue, sorted by threshold
+ * returns true if it was added at the head of the queue
+ */
+static bool add_waiter_to_queue(struct nvhost_waitlist *waiter,
+				struct list_head *queue)
+{
+	struct nvhost_waitlist *pos;
+	u32 thresh = waiter->thresh;
+
+	list_for_each_entry_reverse(pos, queue, list)
+		if ((s32)(pos->thresh - thresh) <= 0) {
+			list_add(&waiter->list, &pos->list);
+			return false;
+		}
+
+	list_add(&waiter->list, queue);
+	return true;
+}
+
+/**
+ * run through a waiter queue for a single sync point ID
+ * and gather all completed waiters into lists by actions
+ */
+static void remove_completed_waiters(struct list_head *head, u32 sync,
+			struct list_head completed[NVHOST_INTR_ACTION_COUNT])
+{
+	struct list_head *dest;
+	struct nvhost_waitlist *waiter, *next, *prev;
+
+	list_for_each_entry_safe(waiter, next, head, list) {
+		if ((s32)(waiter->thresh - sync) > 0)
+			break;
+
+		dest = completed + waiter->action;
+
+		/* consolidate submit cleanups */
+		if (waiter->action == NVHOST_INTR_ACTION_SUBMIT_COMPLETE
+			&& !list_empty(dest)) {
+			prev = list_entry(dest->prev,
+					struct nvhost_waitlist, list);
+			if (prev->data == waiter->data) {
+				prev->count++;
+				dest = NULL;
+			}
+		}
+
+		/* PENDING->REMOVED or CANCELLED->HANDLED */
+		if (atomic_inc_return(&waiter->state) == WLS_HANDLED || !dest) {
+			list_del(&waiter->list);
+			kref_put(&waiter->refcount, waiter_release);
+		} else {
+			list_move_tail(&waiter->list, dest);
+		}
+	}
+}
+
+void reset_threshold_interrupt(struct list_head *head,
+		unsigned int id, void __iomem *sync_regs)
+{
+	u32 thresh = list_first_entry(head,
+				struct nvhost_waitlist, list)->thresh;
+
+	set_syncpt_threshold(sync_regs, id, thresh);
+	enable_syncpt_interrupt(sync_regs, id);
+}
+
+
+static void action_submit_complete(struct nvhost_waitlist *waiter)
+{
+	struct nvhost_channel *channel = waiter->data;
+	int nr_completed = waiter->count;
+
+	nvhost_cdma_update(&channel->cdma);
+	nvhost_module_idle_mult(&channel->mod, nr_completed);
+}
+
+static void action_ctxsave(struct nvhost_waitlist *waiter)
+{
+	struct nvhost_hwctx *hwctx = waiter->data;
+	struct nvhost_channel *channel = hwctx->channel;
+
+	channel->ctxhandler.save_service(hwctx);
+}
+
+static void action_wakeup(struct nvhost_waitlist *waiter)
+{
+	wait_queue_head_t *wq = waiter->data;
+
+	wake_up(wq);
+}
+
+static void action_wakeup_interruptible(struct nvhost_waitlist *waiter)
+{
+	wait_queue_head_t *wq = waiter->data;
+
+	wake_up_interruptible(wq);
+}
+
+typedef void (*action_handler)(struct nvhost_waitlist *waiter);
+
+static action_handler action_handlers[NVHOST_INTR_ACTION_COUNT] = {
+	action_submit_complete,
+	action_ctxsave,
+	action_wakeup,
+	action_wakeup_interruptible,
+};
+
+static void run_handlers(struct list_head completed[NVHOST_INTR_ACTION_COUNT])
+{
+	struct list_head *head = completed;
+	int i;
+
+	for (i = 0; i < NVHOST_INTR_ACTION_COUNT; ++i, ++head) {
+		action_handler handler = action_handlers[i];
+		struct nvhost_waitlist *waiter, *next;
+
+		list_for_each_entry_safe(waiter, next, head, list) {
+			list_del(&waiter->list);
+			handler(waiter);
+			atomic_set(&waiter->state, WLS_HANDLED);
+			smp_wmb();
+			kref_put(&waiter->refcount, waiter_release);
+		}
+	}
+}
+
+/**
+ * Remove & handle all waiters that have completed for the given syncpt
+ */
+int process_wait_list(struct nvhost_intr_syncpt *syncpt,
+		u32 threshold, void __iomem *sync_regs)
+{
+	struct list_head completed[NVHOST_INTR_ACTION_COUNT];
+	unsigned int i;
+	int empty;
+
+	for (i = 0; i < NVHOST_INTR_ACTION_COUNT; ++i)
+		INIT_LIST_HEAD(completed + i);
+
+	spin_lock(&syncpt->lock);
+
+	remove_completed_waiters(&syncpt->wait_head, threshold, completed);
+
+	empty = list_empty(&syncpt->wait_head);
+	if (!empty)
+		reset_threshold_interrupt(&syncpt->wait_head,
+					syncpt->id, sync_regs);
+
+	spin_unlock(&syncpt->lock);
+
+	run_handlers(completed);
+
+	return empty;
+}
+
+
+/*** host syncpt interrupt service functions ***/
+
+/**
+ * Sync point threshold interrupt service function
+ * Handles sync point threshold triggers, in interrupt context
+ */
+static irqreturn_t syncpt_thresh_isr(int irq, void *dev_id)
+{
+	struct nvhost_intr_syncpt *syncpt = dev_id;
+	unsigned int id = syncpt->id;
+	struct nvhost_intr *intr = container_of(syncpt, struct nvhost_intr,
+						syncpt[id]);
+	void __iomem *sync_regs = intr_to_dev(intr)->sync_aperture;
+
+	writel(BIT(id),
+		sync_regs + HOST1X_SYNC_SYNCPT_THRESH_INT_DISABLE);
+	writel(BIT(id),
+		sync_regs + HOST1X_SYNC_SYNCPT_THRESH_CPU0_INT_STATUS);
+
+	return IRQ_WAKE_THREAD;
+}
+
+/**
+ * Sync point threshold interrupt service thread function
+ * Handles sync point threshold triggers, in thread context
+ */
+static irqreturn_t syncpt_thresh_fn(int irq, void *dev_id)
+{
+	struct nvhost_intr_syncpt *syncpt = dev_id;
+	unsigned int id = syncpt->id;
+	struct nvhost_intr *intr = container_of(syncpt, struct nvhost_intr,
+						syncpt[id]);
+	struct nvhost_dev *dev = intr_to_dev(intr);
+
+	(void)process_wait_list(syncpt,
+			nvhost_syncpt_update_min(&dev->syncpt, id),
+			dev->sync_aperture);
+
+	return IRQ_HANDLED;
+}
+
+/**
+ * lazily request a syncpt's irq
+ */
+static int request_syncpt_irq(struct nvhost_intr_syncpt *syncpt)
+{
+	int err;
+
+	if (syncpt->irq_requested)
+		return 0;
+
+	err = request_threaded_irq(syncpt->irq,
+				syncpt_thresh_isr, syncpt_thresh_fn,
+				0, syncpt->thresh_irq_name, syncpt);
+	if (err)
+		return err;
+
+	syncpt->irq_requested = 1;
+	return 0;
+}
+
+/**
+ * free a syncpt's irq. syncpt interrupt should be disabled first.
+ */
+static void free_syncpt_irq(struct nvhost_intr_syncpt *syncpt)
+{
+	if (syncpt->irq_requested) {
+		free_irq(syncpt->irq, syncpt);
+		syncpt->irq_requested = 0;
+	}
+}
+
+
+/*** host general interrupt service functions ***/
+
+/**
+ * Host general interrupt service function
+ * Handles read / write failures
+ */
+static irqreturn_t host1x_isr(int irq, void *dev_id)
+{
+	struct nvhost_intr *intr = dev_id;
+	void __iomem *sync_regs = intr_to_dev(intr)->sync_aperture;
+	u32 stat;
+	u32 ext_stat;
+	u32 addr;
+
+	stat = readl(sync_regs + HOST1X_SYNC_HINTSTATUS);
+	ext_stat = readl(sync_regs + HOST1X_SYNC_HINTSTATUS_EXT);
+
+	if (nvhost_sync_hintstatus_ext_ip_read_int(ext_stat)) {
+		addr = readl(sync_regs + HOST1X_SYNC_IP_READ_TIMEOUT_ADDR);
+		pr_err("Host read timeout at address %x\n", addr);
+	}
+
+	if (nvhost_sync_hintstatus_ext_ip_write_int(ext_stat)) {
+		addr = readl(sync_regs + HOST1X_SYNC_IP_WRITE_TIMEOUT_ADDR);
+		pr_err("Host write timeout at address %x\n", addr);
+	}
+
+	writel(ext_stat, sync_regs + HOST1X_SYNC_HINTSTATUS_EXT);
+	writel(stat, sync_regs + HOST1X_SYNC_HINTSTATUS);
+
+	return IRQ_HANDLED;
+}
+
+static int request_host_general_irq(struct nvhost_intr *intr)
+{
+	void __iomem *sync_regs = intr_to_dev(intr)->sync_aperture;
+	int err;
+
+	if (intr->host_general_irq_requested)
+		return 0;
+
+	/* master disable for general (not syncpt) host interrupts */
+	writel(0, sync_regs + HOST1X_SYNC_INTMASK);
+
+	/* clear status & extstatus */
+	writel(0xfffffffful, sync_regs + HOST1X_SYNC_HINTSTATUS_EXT);
+	writel(0xfffffffful, sync_regs + HOST1X_SYNC_HINTSTATUS);
+
+	err = request_irq(intr->host_general_irq, host1x_isr, 0,
+			"host_status", intr);
+	if (err)
+		return err;
+
+	/* enable extra interrupt sources IP_READ_INT and IP_WRITE_INT */
+	writel(BIT(30) | BIT(31), sync_regs + HOST1X_SYNC_HINTMASK_EXT);
+
+	/* enable extra interrupt sources */
+	writel(BIT(31), sync_regs + HOST1X_SYNC_HINTMASK);
+
+	/* enable host module interrupt to CPU0 */
+	writel(BIT(0), sync_regs + HOST1X_SYNC_INTC0MASK);
+
+	/* master enable for general (not syncpt) host interrupts */
+	writel(BIT(0), sync_regs + HOST1X_SYNC_INTMASK);
+
+	intr->host_general_irq_requested = true;
+
+	return err;
+}
+
+static void free_host_general_irq(struct nvhost_intr *intr)
+{
+	if (intr->host_general_irq_requested) {
+		void __iomem *sync_regs = intr_to_dev(intr)->sync_aperture;
+
+		/* master disable for general (not syncpt) host interrupts */
+		writel(0, sync_regs + HOST1X_SYNC_INTMASK);
+
+		free_irq(intr->host_general_irq, intr);
+		intr->host_general_irq_requested = false;
+	}
+}
+
+
+/*** Main API ***/
+
+int nvhost_intr_add_action(struct nvhost_intr *intr, u32 id, u32 thresh,
+			enum nvhost_intr_action action, void *data,
+			void **ref)
+{
+	struct nvhost_waitlist *waiter;
+	struct nvhost_intr_syncpt *syncpt;
+	void __iomem *sync_regs;
+	int queue_was_empty;
+	int err;
+
+	/* create and initialize a new waiter */
+	waiter = kmalloc(sizeof(*waiter), GFP_KERNEL);
+	if (!waiter)
+		return -ENOMEM;
+	INIT_LIST_HEAD(&waiter->list);
+	kref_init(&waiter->refcount);
+	if (ref)
+		kref_get(&waiter->refcount);
+	waiter->thresh = thresh;
+	waiter->action = action;
+	atomic_set(&waiter->state, WLS_PENDING);
+	waiter->data = data;
+	waiter->count = 1;
+
+	BUG_ON(id >= NV_HOST1X_SYNCPT_NB_PTS);
+	syncpt = intr->syncpt + id;
+	sync_regs = intr_to_dev(intr)->sync_aperture;
+
+	spin_lock(&syncpt->lock);
+
+	/* lazily request irq for this sync point */
+	if (!syncpt->irq_requested) {
+		spin_unlock(&syncpt->lock);
+
+		mutex_lock(&intr->mutex);
+		err = request_syncpt_irq(syncpt);
+		mutex_unlock(&intr->mutex);
+
+		if (err) {
+			kfree(waiter);
+			return err;
+		}
+
+		spin_lock(&syncpt->lock);
+	}
+
+	queue_was_empty = list_empty(&syncpt->wait_head);
+
+	if (add_waiter_to_queue(waiter, &syncpt->wait_head)) {
+		/* added at head of list - new threshold value */
+		set_syncpt_threshold(sync_regs, id, thresh);
+
+		/* added as first waiter - enable interrupt */
+		if (queue_was_empty)
+			enable_syncpt_interrupt(sync_regs, id);
+	}
+
+	spin_unlock(&syncpt->lock);
+
+	if (ref)
+		*ref = waiter;
+	return 0;
+}
+
+void nvhost_intr_put_ref(struct nvhost_intr *intr, void *ref)
+{
+	struct nvhost_waitlist *waiter = ref;
+
+	while (atomic_cmpxchg(&waiter->state,
+				WLS_PENDING, WLS_CANCELLED) == WLS_REMOVED)
+		schedule();
+
+	kref_put(&waiter->refcount, waiter_release);
+}
+
+
+/*** Init & shutdown ***/
+
+int nvhost_intr_init(struct nvhost_intr *intr, u32 irq_gen, u32 irq_sync)
+{
+	unsigned int id;
+	struct nvhost_intr_syncpt *syncpt;
+
+	mutex_init(&intr->mutex);
+	intr->host_general_irq = irq_gen;
+	intr->host_general_irq_requested = false;
+
+	for (id = 0, syncpt = intr->syncpt;
+	     id < NV_HOST1X_SYNCPT_NB_PTS;
+	     ++id, ++syncpt) {
+		syncpt->id = id;
+		syncpt->irq = irq_sync + id;
+		syncpt->irq_requested = 0;
+		spin_lock_init(&syncpt->lock);
+		INIT_LIST_HEAD(&syncpt->wait_head);
+		snprintf(syncpt->thresh_irq_name,
+			sizeof(syncpt->thresh_irq_name),
+			"host_sp_%02d", id);
+	}
+
+	return 0;
+}
+
+void nvhost_intr_deinit(struct nvhost_intr *intr)
+{
+	nvhost_intr_stop(intr);
+}
+
+void nvhost_intr_start(struct nvhost_intr *intr, u32 hz)
+{
+	struct nvhost_dev *dev = intr_to_dev(intr);
+	void __iomem *sync_regs = dev->sync_aperture;
+
+	mutex_lock(&intr->mutex);
+
+	init_host_sync(sync_regs);
+	set_host_clocks_per_microsecond(sync_regs, (hz + 1000000 - 1)/1000000);
+
+	request_host_general_irq(intr);
+
+	mutex_unlock(&intr->mutex);
+}
+
+void nvhost_intr_stop(struct nvhost_intr *intr)
+{
+	void __iomem *sync_regs = intr_to_dev(intr)->sync_aperture;
+	unsigned int id;
+	struct nvhost_intr_syncpt *syncpt;
+
+	mutex_lock(&intr->mutex);
+
+	disable_all_syncpt_interrupts(sync_regs);
+
+	for (id = 0, syncpt = intr->syncpt;
+	     id < NV_HOST1X_SYNCPT_NB_PTS;
+	     ++id, ++syncpt) {
+		BUG_ON(!list_empty(&syncpt->wait_head));
+		free_syncpt_irq(syncpt);
+	}
+
+	free_host_general_irq(intr);
+
+	mutex_unlock(&intr->mutex);
+}
diff --git a/drivers/video/tegra/host/nvhost_intr.h b/drivers/video/tegra/host/nvhost_intr.h
new file mode 100644
index 000000000000..fb3e613d70da
--- /dev/null
+++ b/drivers/video/tegra/host/nvhost_intr.h
@@ -0,0 +1,104 @@
+/*
+ * drivers/video/tegra/host/nvhost_intr.h
+ *
+ * Tegra Graphics Host Interrupt Management
+ *
+ * Copyright (c) 2010, NVIDIA Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+#ifndef __NVHOST_INTR_H
+#define __NVHOST_INTR_H
+
+#include <linux/kthread.h>
+#include <linux/semaphore.h>
+
+#include "nvhost_hardware.h"
+
+struct nvhost_channel;
+
+enum nvhost_intr_action {
+	/**
+	 * Perform cleanup after a submit has completed.
+	 * 'data' points to a channel
+	 */
+	NVHOST_INTR_ACTION_SUBMIT_COMPLETE = 0,
+
+	/**
+	 * Save a HW context.
+	 * 'data' points to a context
+	 */
+	NVHOST_INTR_ACTION_CTXSAVE,
+
+	/**
+	 * Wake up a  task.
+	 * 'data' points to a wait_queue_head_t
+	 */
+	NVHOST_INTR_ACTION_WAKEUP,
+
+	/**
+	 * Wake up a interruptible task.
+	 * 'data' points to a wait_queue_head_t
+	 */
+	NVHOST_INTR_ACTION_WAKEUP_INTERRUPTIBLE,
+
+	NVHOST_INTR_ACTION_COUNT
+};
+
+struct nvhost_intr_syncpt {
+	u8 id;
+	u8 irq_requested;
+	u16 irq;
+	spinlock_t lock;
+	struct list_head wait_head;
+	char thresh_irq_name[12];
+};
+
+struct nvhost_intr {
+	struct nvhost_intr_syncpt syncpt[NV_HOST1X_SYNCPT_NB_PTS];
+	struct mutex mutex;
+	int host_general_irq;
+	bool host_general_irq_requested;
+};
+
+/**
+ * Schedule an action to be taken when a sync point reaches the given threshold.
+ *
+ * @id the sync point
+ * @thresh the threshold
+ * @action the action to take
+ * @data a pointer to extra data depending on action, see above
+ * @ref must be passed if cancellation is possible, else NULL
+ *
+ * This is a non-blocking api.
+ */
+int nvhost_intr_add_action(struct nvhost_intr *intr, u32 id, u32 thresh,
+			enum nvhost_intr_action action, void *data,
+			void **ref);
+
+/**
+ * Unreference an action submitted to nvhost_intr_add_action().
+ * You must call this if you passed non-NULL as ref.
+ * @ref the ref returned from nvhost_intr_add_action()
+ */
+void nvhost_intr_put_ref(struct nvhost_intr *intr, void *ref);
+
+int nvhost_intr_init(struct nvhost_intr *intr, u32 irq_gen, u32 irq_sync);
+void nvhost_intr_deinit(struct nvhost_intr *intr);
+void nvhost_intr_start(struct nvhost_intr *intr, u32 hz);
+void nvhost_intr_stop(struct nvhost_intr *intr);
+
+#endif
diff --git a/drivers/video/tegra/host/nvhost_mpectx.c b/drivers/video/tegra/host/nvhost_mpectx.c
new file mode 100644
index 000000000000..a5812e7469a3
--- /dev/null
+++ b/drivers/video/tegra/host/nvhost_mpectx.c
@@ -0,0 +1,23 @@
+/*
+ * drivers/video/tegra/host/nvhost_mpectx.c
+ *
+ * Tegra Graphics Host MPE HW Context
+ *
+ * Copyright (c) 2010, NVIDIA Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+/* Placeholder */
diff --git a/drivers/video/tegra/host/nvhost_syncpt.c b/drivers/video/tegra/host/nvhost_syncpt.c
new file mode 100644
index 000000000000..e2474df830b4
--- /dev/null
+++ b/drivers/video/tegra/host/nvhost_syncpt.c
@@ -0,0 +1,252 @@
+/*
+ * drivers/video/tegra/host/nvhost_syncpt.c
+ *
+ * Tegra Graphics Host Syncpoints
+ *
+ * Copyright (c) 2010, NVIDIA Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+#include "nvhost_syncpt.h"
+#include "nvhost_dev.h"
+
+#define client_managed(id) (BIT(id) & NVSYNCPTS_CLIENT_MANAGED)
+#define syncpt_to_dev(sp) container_of(sp, struct nvhost_dev, syncpt)
+#define SYNCPT_CHECK_PERIOD 2*HZ
+
+static bool check_max(struct nvhost_syncpt *sp, u32 id, u32 real)
+{
+	u32 max;
+	if (client_managed(id))
+		return true;
+	smp_rmb();
+	max = (u32)atomic_read(&sp->max_val[id]);
+	return ((s32)(max - real) >= 0);
+}
+
+/**
+ * Write the current syncpoint value back to hw.
+ */
+static void reset_syncpt(struct nvhost_syncpt *sp, u32 id)
+{
+	struct nvhost_dev *dev = syncpt_to_dev(sp);
+	int min;
+	smp_rmb();
+	min = atomic_read(&sp->min_val[id]);
+	writel(min, dev->sync_aperture + (HOST1X_SYNC_SYNCPT_0 + id * 4));
+}
+
+/**
+ * Write the current waitbase value back to hw.
+ */
+static void reset_syncpt_wait_base(struct nvhost_syncpt *sp, u32 id)
+{
+	struct nvhost_dev *dev = syncpt_to_dev(sp);
+	writel(sp->base_val[id],
+		dev->sync_aperture + (HOST1X_SYNC_SYNCPT_BASE_0 + id * 4));
+}
+
+/**
+ * Read waitbase value from hw.
+ */
+static void read_syncpt_wait_base(struct nvhost_syncpt *sp, u32 id)
+{
+	struct nvhost_dev *dev = syncpt_to_dev(sp);
+	sp->base_val[id] = readl(dev->sync_aperture +
+				(HOST1X_SYNC_SYNCPT_BASE_0 + id * 4));
+}
+
+/**
+ * Resets syncpoint and waitbase values to sw shadows
+ */
+void nvhost_syncpt_reset(struct nvhost_syncpt *sp)
+{
+	u32 i;
+	for (i = 0; i < NV_HOST1X_SYNCPT_NB_PTS; i++)
+		reset_syncpt(sp, i);
+	for (i = 0; i < NV_HOST1X_SYNCPT_NB_BASES; i++)
+		reset_syncpt_wait_base(sp, i);
+	wmb();
+}
+
+/**
+ * Updates sw shadow state for client managed registers
+ */
+void nvhost_syncpt_save(struct nvhost_syncpt *sp)
+{
+	u32 i;
+
+	for (i = 0; i < NV_HOST1X_SYNCPT_NB_PTS; i++) {
+		if (client_managed(i))
+			nvhost_syncpt_update_min(sp, i);
+		else
+			BUG_ON(!nvhost_syncpt_min_eq_max(sp, i));
+	}
+
+	for (i = 0; i < NV_HOST1X_SYNCPT_NB_BASES; i++)
+		read_syncpt_wait_base(sp, i);
+}
+
+/**
+ * Updates the last value read from hardware.
+ */
+u32 nvhost_syncpt_update_min(struct nvhost_syncpt *sp, u32 id)
+{
+	struct nvhost_dev *dev = syncpt_to_dev(sp);
+	void __iomem *sync_regs = dev->sync_aperture;
+	u32 old, live;
+
+	do {
+		smp_rmb();
+		old = (u32)atomic_read(&sp->min_val[id]);
+		live = readl(sync_regs + (HOST1X_SYNC_SYNCPT_0 + id * 4));
+	} while ((u32)atomic_cmpxchg(&sp->min_val[id], old, live) != old);
+
+	BUG_ON(!check_max(sp, id, live));
+
+	return live;
+}
+
+/**
+ * Get the current syncpoint value
+ */
+u32 nvhost_syncpt_read(struct nvhost_syncpt *sp, u32 id)
+{
+	u32 val;
+
+	nvhost_module_busy(&syncpt_to_dev(sp)->mod);
+	val = nvhost_syncpt_update_min(sp, id);
+	nvhost_module_idle(&syncpt_to_dev(sp)->mod);
+	return val;
+}
+
+/**
+ * Write a cpu syncpoint increment to the hardware, without touching
+ * the cache. Caller is responsible for host being powered.
+ */
+void nvhost_syncpt_cpu_incr(struct nvhost_syncpt *sp, u32 id)
+{
+	struct nvhost_dev *dev = syncpt_to_dev(sp);
+	BUG_ON(!client_managed(id) && nvhost_syncpt_min_eq_max(sp, id));
+	writel(BIT(id), dev->sync_aperture + HOST1X_SYNC_SYNCPT_CPU_INCR);
+	wmb();
+}
+
+/**
+ * Increment syncpoint value from cpu, updating cache
+ */
+void nvhost_syncpt_incr(struct nvhost_syncpt *sp, u32 id)
+{
+	nvhost_syncpt_incr_max(sp, id, 1);
+	nvhost_module_busy(&syncpt_to_dev(sp)->mod);
+	nvhost_syncpt_cpu_incr(sp, id);
+	nvhost_module_idle(&syncpt_to_dev(sp)->mod);
+}
+
+/**
+ * Main entrypoint for syncpoint value waits.
+ */
+int nvhost_syncpt_wait_timeout(struct nvhost_syncpt *sp, u32 id,
+			u32 thresh, u32 timeout)
+{
+	DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wq);
+	void *ref;
+	int err = 0;
+
+	BUG_ON(!check_max(sp, id, thresh));
+
+	/* first check cache */
+	if (nvhost_syncpt_min_cmp(sp, id, thresh))
+		return 0;
+
+	/* keep host alive */
+	nvhost_module_busy(&syncpt_to_dev(sp)->mod);
+
+	if (client_managed(id) || !nvhost_syncpt_min_eq_max(sp, id)) {
+		/* try to read from register */
+		u32 val = nvhost_syncpt_update_min(sp, id);
+		if ((s32)(val - thresh) >= 0)
+			goto done;
+	}
+
+	if (!timeout) {
+		err = -EAGAIN;
+		goto done;
+	}
+
+	/* schedule a wakeup when the syncpoint value is reached */
+	err = nvhost_intr_add_action(&(syncpt_to_dev(sp)->intr), id, thresh,
+				NVHOST_INTR_ACTION_WAKEUP_INTERRUPTIBLE, &wq, &ref);
+	if (err)
+		goto done;
+
+	/* wait for the syncpoint, or timeout, or signal */
+	while (timeout) {
+		u32 check = min_t(u32, SYNCPT_CHECK_PERIOD, timeout);
+		err = wait_event_interruptible_timeout(wq,
+						nvhost_syncpt_min_cmp(sp, id, thresh),
+						check);
+		if (err != 0)
+			break;
+		if (timeout != NVHOST_NO_TIMEOUT)
+			timeout -= SYNCPT_CHECK_PERIOD;
+		if (timeout) {
+			dev_warn(&syncpt_to_dev(sp)->pdev->dev,
+				"syncpoint id %d (%s) stuck waiting %d\n",
+				id, nvhost_syncpt_name(id), thresh);
+			nvhost_syncpt_debug(sp);
+		}
+	};
+	if (err > 0)
+		err = 0;
+	else if (err == 0)
+		err = -EAGAIN;
+	nvhost_intr_put_ref(&(syncpt_to_dev(sp)->intr), ref);
+
+done:
+	nvhost_module_idle(&syncpt_to_dev(sp)->mod);
+	return err;
+}
+
+static const char *s_syncpt_names[32] = {
+	"", "", "", "", "", "", "", "", "", "", "", "",
+	"vi_isp_0", "vi_isp_1", "vi_isp_2", "vi_isp_3", "vi_isp_4", "vi_isp_5",
+	"2d_0", "2d_1",
+	"", "",
+	"3d", "mpe", "disp0", "disp1", "vblank0", "vblank1", "mpe_ebm_eof", "mpe_wr_safe",
+	"2d_tinyblt", "dsi"
+};
+
+const char *nvhost_syncpt_name(u32 id)
+{
+	BUG_ON(id > ARRAY_SIZE(s_syncpt_names));
+	return s_syncpt_names[id];
+}
+
+void nvhost_syncpt_debug(struct nvhost_syncpt *sp)
+{
+	u32 i;
+	for (i = 0; i < NV_HOST1X_SYNCPT_NB_PTS; i++) {
+		u32 max = nvhost_syncpt_read_max(sp, i);
+		if (!max)
+			continue;
+		dev_info(&syncpt_to_dev(sp)->pdev->dev,
+			"id %d (%s) min %d max %d\n",
+			i, nvhost_syncpt_name(i),
+			nvhost_syncpt_update_min(sp, i), max);
+
+	}
+}
diff --git a/drivers/video/tegra/host/nvhost_syncpt.h b/drivers/video/tegra/host/nvhost_syncpt.h
new file mode 100644
index 000000000000..f161f2051406
--- /dev/null
+++ b/drivers/video/tegra/host/nvhost_syncpt.h
@@ -0,0 +1,150 @@
+/*
+ * drivers/video/tegra/host/nvhost_syncpt.h
+ *
+ * Tegra Graphics Host Syncpoints
+ *
+ * Copyright (c) 2010, NVIDIA Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+#ifndef __NVHOST_SYNCPT_H
+#define __NVHOST_SYNCPT_H
+
+#include <linux/kernel.h>
+#include <linux/sched.h>
+#include <asm/atomic.h>
+
+#include "nvhost_hardware.h"
+
+#define NVSYNCPT_VI_ISP_0		     (12)
+#define NVSYNCPT_VI_ISP_1		     (13)
+#define NVSYNCPT_VI_ISP_2		     (14)
+#define NVSYNCPT_VI_ISP_3		     (15)
+#define NVSYNCPT_VI_ISP_4		     (16)
+#define NVSYNCPT_VI_ISP_5		     (17)
+#define NVSYNCPT_2D_0			     (18)
+#define NVSYNCPT_2D_1			     (19)
+#define NVSYNCPT_3D			     (22)
+#define NVSYNCPT_MPE			     (23)
+#define NVSYNCPT_DISP0			     (24)
+#define NVSYNCPT_DISP1			     (25)
+#define NVSYNCPT_VBLANK0		     (26)
+#define NVSYNCPT_VBLANK1		     (27)
+#define NVSYNCPT_MPE_EBM_EOF		     (28)
+#define NVSYNCPT_MPE_WR_SAFE		     (29)
+#define NVSYNCPT_DSI			     (31)
+#define NVSYNCPT_INVALID		     (-1)
+
+/*#define NVSYNCPT_2D_CHANNEL2_0    (20) */
+/*#define NVSYNCPT_2D_CHANNEL2_1    (21) */
+/*#define NVSYNCPT_2D_TINYBLT_WAR		     (30)*/
+/*#define NVSYNCPT_2D_TINYBLT_RESTORE_CLASS_ID (30)*/
+
+/* sync points that are wholly managed by the client */
+#define NVSYNCPTS_CLIENT_MANAGED ( \
+	BIT(NVSYNCPT_DISP0) | BIT(NVSYNCPT_DISP1) | BIT(NVSYNCPT_DSI) | \
+	BIT(NVSYNCPT_VI_ISP_0) | BIT(NVSYNCPT_VI_ISP_2) | \
+	BIT(NVSYNCPT_VI_ISP_3) | BIT(NVSYNCPT_VI_ISP_4) | BIT(NVSYNCPT_VI_ISP_5) | \
+	BIT(NVSYNCPT_MPE_EBM_EOF) | BIT(NVSYNCPT_MPE_WR_SAFE) | \
+	BIT(NVSYNCPT_2D_1))
+
+#define NVWAITBASE_2D_0 (1)
+#define NVWAITBASE_2D_1 (2)
+#define NVWAITBASE_3D   (3)
+#define NVWAITBASE_MPE  (4)
+
+struct nvhost_syncpt {
+	atomic_t min_val[NV_HOST1X_SYNCPT_NB_PTS];
+	atomic_t max_val[NV_HOST1X_SYNCPT_NB_PTS];
+	u32 base_val[NV_HOST1X_SYNCPT_NB_BASES];
+};
+
+/**
+ * Updates the value sent to hardware.
+ */
+static inline u32 nvhost_syncpt_incr_max(struct nvhost_syncpt *sp,
+					u32 id, u32 incrs)
+{
+	return (u32)atomic_add_return(incrs, &sp->max_val[id]);
+}
+
+/**
+ * Updated the value sent to hardware.
+ */
+static inline u32 nvhost_syncpt_set_max(struct nvhost_syncpt *sp,
+					u32 id, u32 val)
+{
+	atomic_set(&sp->max_val[id], val);
+	smp_wmb();
+	return val;
+}
+
+static inline u32 nvhost_syncpt_read_max(struct nvhost_syncpt *sp, u32 id)
+{
+	smp_rmb();
+	return (u32)atomic_read(&sp->max_val[id]);
+}
+
+/**
+ * Returns true if syncpoint has reached threshold
+ */
+static inline bool nvhost_syncpt_min_cmp(struct nvhost_syncpt *sp,
+					u32 id, u32 thresh)
+{
+	u32 cur;
+	smp_rmb();
+	cur = (u32)atomic_read(&sp->min_val[id]);
+	return ((s32)(cur - thresh) >= 0);
+}
+
+/**
+ * Returns true if syncpoint min == max
+ */
+static inline bool nvhost_syncpt_min_eq_max(struct nvhost_syncpt *sp, u32 id)
+{
+	int min, max;
+	smp_rmb();
+	min = atomic_read(&sp->min_val[id]);
+	max = atomic_read(&sp->max_val[id]);
+	return (min == max);
+}
+
+void nvhost_syncpt_cpu_incr(struct nvhost_syncpt *sp, u32 id);
+
+u32 nvhost_syncpt_update_min(struct nvhost_syncpt *sp, u32 id);
+
+void nvhost_syncpt_save(struct nvhost_syncpt *sp);
+
+void nvhost_syncpt_reset(struct nvhost_syncpt *sp);
+
+u32 nvhost_syncpt_read(struct nvhost_syncpt *sp, u32 id);
+
+void nvhost_syncpt_incr(struct nvhost_syncpt *sp, u32 id);
+
+int nvhost_syncpt_wait_timeout(struct nvhost_syncpt *sp, u32 id, u32 thresh,
+			u32 timeout);
+
+static inline int nvhost_syncpt_wait(struct nvhost_syncpt *sp, u32 id, u32 thresh)
+{
+	return nvhost_syncpt_wait_timeout(sp, id, thresh, MAX_SCHEDULE_TIMEOUT);
+}
+
+
+const char *nvhost_syncpt_name(u32 id);
+
+void nvhost_syncpt_debug(struct nvhost_syncpt *sp);
+
+#endif
diff --git a/drivers/video/tegra/nvmap.c b/drivers/video/tegra/nvmap.c
new file mode 100644
index 000000000000..448ac8f34771
--- /dev/null
+++ b/drivers/video/tegra/nvmap.c
@@ -0,0 +1,3529 @@
+/*
+ * drivers/char/nvmap.c
+ *
+ * Memory manager for Tegra GPU memory handles
+ *
+ * Copyright (c) 2009-2010, NVIDIA Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+#define NV_DEBUG 0
+
+#include <linux/vmalloc.h>
+#include <linux/module.h>
+#include <linux/bitmap.h>
+#include <linux/wait.h>
+#include <linux/miscdevice.h>
+#include <linux/platform_device.h>
+#include <linux/mm.h>
+#include <linux/mman.h>
+#include <linux/uaccess.h>
+#include <linux/backing-dev.h>
+#include <linux/device.h>
+#include <linux/highmem.h>
+#include <linux/smp_lock.h>
+#include <linux/pagemap.h>
+#include <linux/sched.h>
+#include <linux/io.h>
+#include <linux/rbtree.h>
+#include <linux/proc_fs.h>
+#include <linux/ctype.h>
+#include <linux/nvmap.h>
+#include <asm/tlbflush.h>
+#include <mach/iovmm.h>
+#include "nvcommon.h"
+#include "nvrm_memmgr.h"
+#include "nvbootargs.h"
+
+#ifndef NVMAP_BASE
+#define NVMAP_BASE 0xFEE00000
+#define NVMAP_SIZE SZ_2M
+#endif
+
+static void nvmap_vma_open(struct vm_area_struct *vma);
+
+static void nvmap_vma_close(struct vm_area_struct *vma);
+
+static int nvmap_vma_fault(struct vm_area_struct *vma, struct vm_fault *vmf);
+
+static int nvmap_open(struct inode *inode, struct file *filp);
+
+static int nvmap_release(struct inode *inode, struct file *file);
+
+static int nvmap_mmap(struct file *filp, struct vm_area_struct *vma);
+
+static long nvmap_ioctl(struct file *filp,
+	unsigned int cmd, unsigned long arg);
+
+static int nvmap_ioctl_getid(struct file *filp, void __user *arg);
+
+static int nvmap_ioctl_get_param(struct file *filp, void __user* arg);
+
+static int nvmap_ioctl_alloc(struct file *filp, void __user *arg);
+
+static int nvmap_ioctl_free(struct file *filp, unsigned long arg);
+
+static int nvmap_ioctl_create(struct file *filp,
+	unsigned int cmd, void __user *arg);
+
+static int nvmap_ioctl_pinop(struct file *filp,
+	bool is_pin, void __user *arg);
+
+static int nvmap_ioctl_cache_maint(struct file *filp, void __user *arg);
+
+static int nvmap_map_into_caller_ptr(struct file *filp, void __user *arg);
+
+static int nvmap_ioctl_rw_handle(struct file *filp, int is_read,
+	void __user* arg);
+
+static struct backing_dev_info nvmap_bdi = {
+	.ra_pages	= 0,
+	.capabilities	= (BDI_CAP_NO_ACCT_AND_WRITEBACK |
+			   BDI_CAP_READ_MAP | BDI_CAP_WRITE_MAP),
+};
+
+#define NVMAP_PTE_OFFSET(x) (((unsigned long)(x) - NVMAP_BASE) >> PAGE_SHIFT)
+#define NVMAP_PTE_INDEX(x) (((unsigned long)(x) - NVMAP_BASE)>>PGDIR_SHIFT)
+#define NUM_NVMAP_PTES (NVMAP_SIZE >> PGDIR_SHIFT)
+#define NVMAP_END (NVMAP_BASE + NVMAP_SIZE)
+#define NVMAP_PAGES (NVMAP_SIZE >> PAGE_SHIFT)
+
+/* private nvmap_handle flag for pinning duplicate detection */
+#define NVMEM_HANDLE_VISITED (0x1ul << 31)
+
+/* Heaps to use for kernel allocs when no heap list supplied */
+#define NVMAP_KERNEL_DEFAULT_HEAPS (NVMEM_HEAP_SYSMEM | NVMEM_HEAP_CARVEOUT_GENERIC)
+
+/* Heaps for which secure allocations are allowed */
+#define NVMAP_SECURE_HEAPS (NVMEM_HEAP_CARVEOUT_IRAM | NVMEM_HEAP_IOVMM)
+
+static pte_t *nvmap_pte[NUM_NVMAP_PTES];
+static unsigned long nvmap_ptebits[NVMAP_PAGES/BITS_PER_LONG];
+
+static DEFINE_SPINLOCK(nvmap_ptelock);
+static DECLARE_WAIT_QUEUE_HEAD(nvmap_ptefull);
+
+/* used to lost the master tree of memory handles */
+static DEFINE_SPINLOCK(nvmap_handle_lock);
+
+/* only one task may be performing pin / unpin operations at once, to
+ * prevent deadlocks caused by interleaved IOVMM re-allocations */
+static DEFINE_MUTEX(nvmap_pin_lock);
+
+/* queue of tasks which are blocking on pin, for IOVMM room */
+static DECLARE_WAIT_QUEUE_HEAD(nvmap_pin_wait);
+static struct rb_root nvmap_handles = RB_ROOT;
+
+static struct tegra_iovmm_client *nvmap_vm_client = NULL;
+
+/* default heap order policy */
+static unsigned int _nvmap_heap_policy (unsigned int heaps, int numpages)
+{
+	static const unsigned int multipage_order[] = {
+		NVMEM_HEAP_CARVEOUT_MASK,
+		NVMEM_HEAP_SYSMEM,
+		NVMEM_HEAP_IOVMM,
+		0
+	};
+	static const unsigned int singlepage_order[] = {
+		NVMEM_HEAP_SYSMEM,
+		NVMEM_HEAP_CARVEOUT_MASK,
+		NVMEM_HEAP_IOVMM,
+		0
+	};
+	const unsigned int* order;
+
+	if (numpages == 1)
+		order = singlepage_order;
+	else
+		order = multipage_order;
+
+	while (*order) {
+		unsigned int h = (*order & heaps);
+		if (h) return h;
+		order++;
+	}
+	return 0;
+};
+
+/* first-fit linear allocator carveout heap manager */
+struct nvmap_mem_block {
+	unsigned long	base;
+	size_t		size;
+	short		next; /* next absolute (address-order) block */
+	short		prev; /* previous absolute (address-order) block */
+	short		next_free;
+	short		prev_free;
+};
+
+struct nvmap_carveout {
+	unsigned short		num_blocks;
+	short			spare_index;
+	short			free_index;
+	short			block_index;
+	spinlock_t		lock;
+	const char		*name;
+	struct nvmap_mem_block	*blocks;
+};
+
+enum {
+	CARVEOUT_STAT_TOTAL_SIZE,
+	CARVEOUT_STAT_FREE_SIZE,
+	CARVEOUT_STAT_NUM_BLOCKS,
+	CARVEOUT_STAT_FREE_BLOCKS,
+	CARVEOUT_STAT_LARGEST_BLOCK,
+	CARVEOUT_STAT_LARGEST_FREE,
+	CARVEOUT_STAT_BASE,
+};
+
+static inline pgprot_t _nvmap_flag_to_pgprot(unsigned long flag, pgprot_t base)
+{
+	switch (flag) {
+	case NVMEM_HANDLE_UNCACHEABLE:
+		base = pgprot_noncached(base);
+		break;
+	case NVMEM_HANDLE_WRITE_COMBINE:
+		base = pgprot_writecombine(base);
+		break;
+	case NVMEM_HANDLE_INNER_CACHEABLE:
+		base = pgprot_inner_writeback(base);
+		break;
+	}
+	return base;
+}
+
+static unsigned long _nvmap_carveout_blockstat(struct nvmap_carveout *co,
+	int stat)
+{
+	unsigned long val = 0;
+	short idx;
+	spin_lock(&co->lock);
+
+	if (stat==CARVEOUT_STAT_BASE) {
+		if (co->block_index==-1)
+			val = ~0;
+		else
+			val = co->blocks[co->block_index].base;
+		spin_unlock(&co->lock);
+		return val;
+	}
+
+	if (stat==CARVEOUT_STAT_TOTAL_SIZE ||
+	    stat==CARVEOUT_STAT_NUM_BLOCKS ||
+	    stat==CARVEOUT_STAT_LARGEST_BLOCK)
+		idx = co->block_index;
+	else
+		idx = co->free_index;
+
+	while (idx!=-1) {
+		switch (stat) {
+		case CARVEOUT_STAT_TOTAL_SIZE:
+			val += co->blocks[idx].size;
+			idx = co->blocks[idx].next;
+			break;
+		case CARVEOUT_STAT_NUM_BLOCKS:
+			val++;
+			idx = co->blocks[idx].next;
+			break;
+		case CARVEOUT_STAT_LARGEST_BLOCK:
+			val = max_t(unsigned long, val, co->blocks[idx].size);
+			idx = co->blocks[idx].next;
+			break;
+		case CARVEOUT_STAT_FREE_SIZE:
+			val += co->blocks[idx].size;
+			idx = co->blocks[idx].next_free;
+			break;
+		case CARVEOUT_STAT_FREE_BLOCKS:
+			val ++;
+			idx = co->blocks[idx].next_free;
+			break;
+		case CARVEOUT_STAT_LARGEST_FREE:
+			val = max_t(unsigned long, val, co->blocks[idx].size);
+			idx = co->blocks[idx].next_free;
+			break;
+	    }
+	}
+
+	spin_unlock(&co->lock);
+	return val;
+}
+
+#define co_is_free(_co, _idx) \
+	((_co)->free_index==(_idx) || ((_co)->blocks[(_idx)].prev_free!=-1))
+
+static int _nvmap_init_carveout(struct nvmap_carveout *co,
+	const char *name, unsigned long base_address, size_t len)
+{
+	unsigned int num_blocks;
+	struct nvmap_mem_block *blocks = NULL;
+	int i;
+
+	num_blocks = min_t(unsigned int, len/1024, 1024);
+	blocks = vmalloc(sizeof(*blocks)*num_blocks);
+
+	if (!blocks) goto fail;
+	co->name = kstrdup(name, GFP_KERNEL);
+	if (!co->name) goto fail;
+
+	for (i=1; i<num_blocks; i++) {
+		blocks[i].next = i+1;
+		blocks[i].prev = i-1;
+		blocks[i].next_free = -1;
+		blocks[i].prev_free = -1;
+	}
+	blocks[i-1].next = -1;
+	blocks[1].prev = -1;
+
+	blocks[0].next = blocks[0].prev = -1;
+	blocks[0].next_free = blocks[0].prev_free = -1;
+	blocks[0].base = base_address;
+	blocks[0].size = len;
+	co->blocks = blocks;
+	co->num_blocks = num_blocks;
+	spin_lock_init(&co->lock);
+	co->block_index = 0;
+	co->spare_index = 1;
+	co->free_index = 0;
+	return 0;
+
+fail:
+	if (blocks) kfree(blocks);
+	return -ENOMEM;
+}
+
+static int nvmap_get_spare(struct nvmap_carveout *co) {
+	int idx;
+
+	if (co->spare_index == -1)
+		return -1;
+
+	idx = co->spare_index;
+	co->spare_index = co->blocks[idx].next;
+	co->blocks[idx].next = -1;
+	co->blocks[idx].prev = -1;
+	co->blocks[idx].next_free = -1;
+	co->blocks[idx].prev_free = -1;
+	return idx;
+}
+
+#define BLOCK(_co, _idx) ((_idx)==-1 ? NULL : &(_co)->blocks[(_idx)])
+
+static void nvmap_zap_free(struct nvmap_carveout *co, int idx)
+{
+	struct nvmap_mem_block *block;
+
+	block = BLOCK(co, idx);
+	if (block->prev_free != -1)
+		BLOCK(co, block->prev_free)->next_free = block->next_free;
+	else
+		co->free_index = block->next_free;
+
+	if (block->next_free != -1)
+		BLOCK(co, block->next_free)->prev_free = block->prev_free;
+
+	block->prev_free = -1;
+	block->next_free = -1;
+}
+
+static void nvmap_split_block(struct nvmap_carveout *co,
+	int idx, size_t start, size_t size)
+{
+	if (BLOCK(co, idx)->base < start) {
+		int spare_idx = nvmap_get_spare(co);
+		struct nvmap_mem_block *spare = BLOCK(co, spare_idx);
+		struct nvmap_mem_block *block = BLOCK(co, idx);
+		if (spare) {
+			spare->size = start - block->base;
+			spare->base = block->base;
+			block->size -= (start - block->base);
+			block->base = start;
+			spare->next = idx;
+			spare->prev = block->prev;
+			block->prev = spare_idx;
+			if (spare->prev != -1)
+				co->blocks[spare->prev].next = spare_idx;
+			else
+				co->block_index = spare_idx;
+			spare->prev_free = -1;
+			spare->next_free = co->free_index;
+			if (co->free_index != -1)
+				co->blocks[co->free_index].prev_free = spare_idx;
+			co->free_index = spare_idx;
+		} else {
+			if (block->prev != -1) {
+				spare = BLOCK(co, block->prev);
+				spare->size += start - block->base;
+				block->size -= (start - block->base);
+				block->base = start;
+			}
+		}
+	}
+
+	if (BLOCK(co, idx)->size > size) {
+		int spare_idx = nvmap_get_spare(co);
+		struct nvmap_mem_block *spare = BLOCK(co, spare_idx);
+		struct nvmap_mem_block *block = BLOCK(co, idx);
+		if (spare) {
+			spare->base = block->base + size;
+			spare->size = block->size - size;
+			block->size = size;
+			spare->prev = idx;
+			spare->next = block->next;
+			block->next = spare_idx;
+			if (spare->next != -1)
+				co->blocks[spare->next].prev = spare_idx;
+			spare->prev_free = -1;
+			spare->next_free = co->free_index;
+			if (co->free_index != -1)
+				co->blocks[co->free_index].prev_free = spare_idx;
+			co->free_index = spare_idx;
+		}
+	}
+
+	nvmap_zap_free(co, idx);
+}
+
+#define next_spare next
+#define prev_spare prev
+
+#define nvmap_insert_block(_list, _co, _idx)				\
+	do {								\
+		struct nvmap_mem_block *b = BLOCK((_co), (_idx));	\
+		struct nvmap_mem_block *s = BLOCK((_co), (_co)->_list##_index);\
+		if (s) s->prev_##_list = (_idx);			\
+		b->prev_##_list = -1;					\
+		b->next_##_list = (_co)->_list##_index;			\
+		(_co)->_list##_index = (_idx);				\
+	} while (0);
+
+#define are_blocks_contiguous(_b_first, _b_next) \
+	(_b_first->base + _b_first->size == _b_next->base)
+
+static void nvmap_carveout_free(struct nvmap_carveout *co, int idx)
+{
+	struct nvmap_mem_block *b;
+
+	spin_lock(&co->lock);
+
+	b = BLOCK(co, idx);
+
+	if (b->next!=-1 &&
+		co_is_free(co, b->next) &&
+		are_blocks_contiguous(b,BLOCK(co, b->next))) {
+		int zap = b->next;
+		struct nvmap_mem_block *n = BLOCK(co, zap);
+		b->size += n->size;
+
+		b->next = n->next;
+		if (n->next != -1) co->blocks[n->next].prev = idx;
+
+		nvmap_zap_free(co, zap);
+		nvmap_insert_block(spare, co, zap);
+	}
+
+	if (b->prev!=-1 &&
+		co_is_free(co, b->prev) &&
+		are_blocks_contiguous(BLOCK(co, b->prev),b)) {
+		int zap = b->prev;
+		struct nvmap_mem_block *p = BLOCK(co, zap);
+
+		b->base = p->base;
+		b->size += p->size;
+
+		b->prev = p->prev;
+
+		if (p->prev != -1) co->blocks[p->prev].next = idx;
+		else co->block_index = idx;
+
+		nvmap_zap_free(co, zap);
+		nvmap_insert_block(spare, co, zap);
+	}
+
+	nvmap_insert_block(free, co, idx);
+	spin_unlock(&co->lock);
+}
+
+static int nvmap_carveout_alloc(struct nvmap_carveout *co,
+	size_t align, size_t size)
+{
+	short idx;
+
+	spin_lock(&co->lock);
+
+	idx = co->free_index;
+
+	while (idx != -1) {
+		struct nvmap_mem_block *b = BLOCK(co, idx);
+		/* try to be a bit more clever about generating block-
+		 * droppings by comparing the results of a left-justified vs
+		 * right-justified block split, and choosing the
+		 * justification style which yields the largest remaining
+		 * block */
+		size_t end = b->base + b->size;
+		size_t ljust = (b->base + align - 1) & ~(align-1);
+		size_t rjust = (end - size) & ~(align-1);
+		size_t l_max, r_max;
+
+		if (rjust < b->base) rjust = ljust;
+		l_max = max_t(size_t, ljust - b->base, end - (ljust + size));
+		r_max = max_t(size_t, rjust - b->base, end - (rjust + size));
+
+		if (b->base + b->size >= ljust + size) {
+			if (l_max >= r_max)
+				nvmap_split_block(co, idx, ljust, size);
+			else
+				nvmap_split_block(co, idx, rjust, size);
+			break;
+		}
+		idx = b->next_free;
+	}
+
+	spin_unlock(&co->lock);
+	return idx;
+}
+
+#undef next_spare
+#undef prev_spare
+
+#define NVDA_POISON (('n'<<24) | ('v'<<16) | ('d'<<8) | ('a'))
+
+struct nvmap_handle {
+	struct rb_node node;
+	atomic_t ref;
+	atomic_t pin;
+	unsigned long flags;
+	size_t size;
+	size_t orig_size;
+	struct task_struct *owner;
+	unsigned int poison;
+	union {
+		struct {
+			struct page **pages;
+			struct tegra_iovmm_area *area;
+			struct list_head mru_list;
+			bool contig;
+			bool dirty; /* IOVMM area allocated since last pin */
+		} pgalloc;
+		struct {
+			struct nvmap_carveout *co_heap;
+			int block_idx;
+			unsigned long base;
+			unsigned int key; /* preserved by bootloader */
+		} carveout;
+	};
+	bool global;
+	bool secure; /* only allocated in IOVM space, zapped on unpin */
+	bool heap_pgalloc;
+	bool alloc;
+	void *kern_map; /* used for RM memmgr backwards compat */
+};
+
+/* handle_ref objects are file-descriptor-local references to nvmap_handle
+ * objects. they track the number of references and pins performed by
+ * the specific caller (since nvmap_handle objects may be global), so that
+ * a client which terminates without properly unwinding all handles (or
+ * all nested pins) can be unwound by nvmap. */
+struct nvmap_handle_ref {
+	struct nvmap_handle *h;
+	struct rb_node node;
+	atomic_t refs;
+	atomic_t pin;
+};
+
+struct nvmap_file_priv {
+	struct rb_root handle_refs;
+	atomic_t iovm_commit;
+	size_t iovm_limit;
+	spinlock_t ref_lock;
+	bool su;
+};
+
+struct nvmap_carveout_node {
+	struct device		dev;
+	struct list_head	heap_list;
+	unsigned int		heap_bit;
+	struct nvmap_carveout	carveout;
+};
+
+/* the master structure for all nvmap-managed carveouts and all handle_ref
+ * objects allocated inside the kernel. heaps are sorted by their heap_bit
+ * (highest heap_bit first) so that carveout allocation will be first
+ * attempted by the heap with the highest heap_bit set in the allocation's
+ * heap mask */
+static struct {
+	struct nvmap_file_priv init_data;
+	struct rw_semaphore list_sem;
+	struct list_head heaps;
+} nvmap_context;
+
+static struct vm_operations_struct nvmap_vma_ops = {
+	.open	= nvmap_vma_open,
+	.close	= nvmap_vma_close,
+	.fault	= nvmap_vma_fault,
+};
+
+const struct file_operations nvmap_fops = {
+	.owner		= THIS_MODULE,
+	.open		= nvmap_open,
+	.release	= nvmap_release,
+	.unlocked_ioctl = nvmap_ioctl,
+	.mmap		= nvmap_mmap
+};
+
+const struct file_operations knvmap_fops = {
+	.owner		= THIS_MODULE,
+	.open		= nvmap_open,
+	.release	= nvmap_release,
+	.unlocked_ioctl = nvmap_ioctl,
+	.mmap		= nvmap_mmap
+};
+
+struct nvmap_vma_priv {
+	struct nvmap_handle	*h;
+	size_t			offs;
+	atomic_t		ref;
+};
+
+static struct proc_dir_entry *nvmap_procfs_root;
+static struct proc_dir_entry *nvmap_procfs_proc;
+
+static void _nvmap_handle_free(struct nvmap_handle *h);
+
+#define NVMAP_CARVEOUT_ATTR_RO(_name)					\
+	struct device_attribute nvmap_heap_attr_##_name =		\
+		__ATTR(_name, S_IRUGO, _nvmap_sysfs_show_heap_##_name, NULL)
+
+#define NVMAP_CARVEOUT_ATTR_WO(_name, _mode)				\
+	struct device_attribute nvmap_heap_attr_##_name =		\
+		__ATTR(_name, _mode, NULL, _nvmap_sysfs_set_heap_##_name)
+
+
+static ssize_t _nvmap_sysfs_show_heap_usage(struct device *d,
+	struct device_attribute *attr, char *buf)
+{
+	struct nvmap_carveout_node *c = container_of(d,
+		struct nvmap_carveout_node, dev);
+	return sprintf(buf, "%08x\n", c->heap_bit);
+}
+
+static ssize_t _nvmap_sysfs_show_heap_name(struct device *d,
+	struct device_attribute *attr, char *buf)
+{
+	struct nvmap_carveout_node *c = container_of(d,
+		struct nvmap_carveout_node, dev);
+	return sprintf(buf, "%s\n", c->carveout.name);
+}
+
+static ssize_t _nvmap_sysfs_show_heap_base(struct device *d,
+	struct device_attribute *attr, char *buf)
+{
+	struct nvmap_carveout_node *c = container_of(d,
+		struct nvmap_carveout_node, dev);
+	return sprintf(buf, "%08lx\n",
+		_nvmap_carveout_blockstat(&c->carveout, CARVEOUT_STAT_BASE));
+}
+
+static ssize_t _nvmap_sysfs_show_heap_free_size(struct device *d,
+	struct device_attribute *attr, char *buf)
+{
+	struct nvmap_carveout_node *c = container_of(d,
+		struct nvmap_carveout_node, dev);
+	return sprintf(buf, "%lu\n",
+		_nvmap_carveout_blockstat(&c->carveout,
+			CARVEOUT_STAT_FREE_SIZE));
+}
+
+static ssize_t _nvmap_sysfs_show_heap_free_count(struct device *d,
+	struct device_attribute *attr, char *buf)
+{
+	struct nvmap_carveout_node *c = container_of(d,
+		struct nvmap_carveout_node, dev);
+	return sprintf(buf, "%lu\n",
+		_nvmap_carveout_blockstat(&c->carveout,
+			CARVEOUT_STAT_FREE_BLOCKS));
+}
+
+static ssize_t _nvmap_sysfs_show_heap_free_max(struct device *d,
+	struct device_attribute *attr, char *buf)
+{
+	struct nvmap_carveout_node *c = container_of(d,
+		struct nvmap_carveout_node, dev);
+	return sprintf(buf, "%lu\n",
+		_nvmap_carveout_blockstat(&c->carveout,
+			CARVEOUT_STAT_LARGEST_FREE));
+}
+
+static ssize_t _nvmap_sysfs_show_heap_total_count(struct device *d,
+	struct device_attribute *attr, char *buf)
+{
+	struct nvmap_carveout_node *c = container_of(d,
+		struct nvmap_carveout_node, dev);
+	return sprintf(buf, "%lu\n",
+		_nvmap_carveout_blockstat(&c->carveout,
+			CARVEOUT_STAT_NUM_BLOCKS));
+}
+
+static ssize_t _nvmap_sysfs_show_heap_total_max(struct device *d,
+	struct device_attribute *attr, char *buf)
+{
+	struct nvmap_carveout_node *c = container_of(d,
+		struct nvmap_carveout_node, dev);
+	return sprintf(buf, "%lu\n",
+		_nvmap_carveout_blockstat(&c->carveout,
+			CARVEOUT_STAT_LARGEST_BLOCK));
+}
+
+static ssize_t _nvmap_sysfs_show_heap_total_size(struct device *d,
+	struct device_attribute *attr, char *buf)
+{
+	struct nvmap_carveout_node *c = container_of(d,
+		struct nvmap_carveout_node, dev);
+	return sprintf(buf, "%lu\n",
+		_nvmap_carveout_blockstat(&c->carveout,
+			CARVEOUT_STAT_TOTAL_SIZE));
+}
+
+static int nvmap_split_carveout_heap(struct nvmap_carveout *co, size_t size,
+	const char *name, unsigned int new_bitmask);
+
+static ssize_t _nvmap_sysfs_set_heap_split(struct device *d,
+	struct device_attribute *attr, const char * buf, size_t count)
+{
+	struct nvmap_carveout_node *c = container_of(d,
+		struct nvmap_carveout_node, dev);
+	char *tmp, *local = kzalloc(count+1, GFP_KERNEL);
+	char *sizestr = NULL, *bitmaskstr = NULL, *name = NULL;
+	char **format[] = { &sizestr, &bitmaskstr, &name };
+	char ***f_iter = format;
+	unsigned int i;
+	unsigned long size, bitmask;
+	int err;
+
+	if (!local) {
+		pr_err("%s: unable to read string\n", __func__);
+		return -ENOMEM;
+	}
+
+	memcpy(local, buf, count);
+	tmp = local;
+	for (i=0, **f_iter = local; i<count &&
+	    (f_iter - format)<ARRAY_SIZE(format)-1; i++) {
+		if (local[i]==',') {
+			local[i] = '\0';
+			f_iter++;
+			**f_iter = &local[i+1];
+		}
+	}
+
+	if (!sizestr || !bitmaskstr || !name) {
+		pr_err("%s: format error\n", __func__);
+		kfree(tmp);
+		return -EINVAL;
+	}
+
+	for (local=name; !isspace(*local); local++);
+
+	if (local==name) {
+		pr_err("%s: invalid name %s\n", __func__, name);
+		kfree(tmp);
+		return -EINVAL;
+	}
+
+	*local=0;
+
+	size = memparse(sizestr, &sizestr);
+	if (!size) {
+		kfree(tmp);
+		return -EINVAL;
+	}
+
+	if (strict_strtoul(bitmaskstr, 0, &bitmask)==-EINVAL) {
+		kfree(tmp);
+		return -EINVAL;
+	}
+
+	err = nvmap_split_carveout_heap(&c->carveout, size, name, bitmask);
+
+	if (err) pr_err("%s: failed to create split heap %s\n", __func__, name);
+	kfree(tmp);
+	return err ? err : count;
+}
+
+static NVMAP_CARVEOUT_ATTR_RO(usage);
+static NVMAP_CARVEOUT_ATTR_RO(name);
+static NVMAP_CARVEOUT_ATTR_RO(base);
+static NVMAP_CARVEOUT_ATTR_RO(free_size);
+static NVMAP_CARVEOUT_ATTR_RO(free_count);
+static NVMAP_CARVEOUT_ATTR_RO(free_max);
+static NVMAP_CARVEOUT_ATTR_RO(total_size);
+static NVMAP_CARVEOUT_ATTR_RO(total_count);
+static NVMAP_CARVEOUT_ATTR_RO(total_max);
+static NVMAP_CARVEOUT_ATTR_WO(split, (S_IWUSR | S_IWGRP));
+
+static struct attribute *nvmap_heap_default_attrs[] = {
+	&nvmap_heap_attr_usage.attr,
+	&nvmap_heap_attr_name.attr,
+	&nvmap_heap_attr_split.attr,
+	&nvmap_heap_attr_base.attr,
+	&nvmap_heap_attr_total_size.attr,
+	&nvmap_heap_attr_free_size.attr,
+	&nvmap_heap_attr_total_count.attr,
+	&nvmap_heap_attr_free_count.attr,
+	&nvmap_heap_attr_total_max.attr,
+	&nvmap_heap_attr_free_max.attr,
+	NULL
+};
+
+static struct attribute_group nvmap_heap_defattr_group = {
+	.attrs = nvmap_heap_default_attrs
+};
+
+static struct device *__nvmap_heap_parent_dev(void);
+#define _nvmap_heap_parent_dev __nvmap_heap_parent_dev()
+
+/* unpinned I/O VMM areas may be reclaimed by nvmap to make room for
+ * new surfaces. unpinned surfaces are stored in segregated linked-lists
+ * sorted in most-recently-unpinned order (i.e., head insertion, head
+ * removal */
+#ifdef CONFIG_DEVNVMAP_RECLAIM_UNPINNED_VM
+static DEFINE_SPINLOCK(nvmap_mru_vma_lock);
+static const size_t nvmap_mru_cutoff[] = {
+	262144, 393216, 786432, 1048576, 1572864
+};
+
+static struct list_head nvmap_mru_vma_lists[ARRAY_SIZE(nvmap_mru_cutoff)];
+
+static inline struct list_head *_nvmap_list(size_t size)
+{
+	unsigned int i;
+
+	for (i=0; i<ARRAY_SIZE(nvmap_mru_cutoff); i++)
+		if (size <= nvmap_mru_cutoff[i]) return &nvmap_mru_vma_lists[i];
+
+	return &nvmap_mru_vma_lists[ARRAY_SIZE(nvmap_mru_cutoff)-1];
+}
+#endif
+
+static inline struct nvmap_handle *_nvmap_handle_get(struct nvmap_handle *h)
+{
+	if (unlikely(h->poison!=NVDA_POISON)) {
+		pr_err("%s: %s getting poisoned handle\n", __func__,
+			current->group_leader->comm);
+		return NULL;
+	} else if (unlikely(atomic_inc_return(&h->ref)<=1)) {
+		pr_err("%s: %s getting a freed handle\n",
+			__func__, current->group_leader->comm);
+		return NULL;
+	}
+	return h;
+}
+
+static inline void _nvmap_handle_put(struct nvmap_handle *h)
+{
+	int cnt = atomic_dec_return(&h->ref);
+
+	if (unlikely(cnt<0)) {
+		pr_err("%s: %s put to negative references\n",
+			__func__, current->comm);
+		dump_stack();
+	} else if (!cnt) _nvmap_handle_free(h);
+}
+
+static struct nvmap_handle *_nvmap_claim_preserved(
+	struct task_struct *new_owner, unsigned long key)
+{
+	struct rb_node *n;
+	struct nvmap_handle *b = NULL;
+
+	if (!key) return NULL;
+
+	spin_lock(&nvmap_handle_lock);
+	n = rb_first(&nvmap_handles);
+
+	while (n) {
+		b = rb_entry(n, struct nvmap_handle, node);
+		if (b->alloc && !b->heap_pgalloc && b->carveout.key == key) {
+			b->carveout.key = 0;
+			b->owner = new_owner;
+			break;
+		}
+		b = NULL;
+		n = rb_next(n);
+	}
+
+	spin_unlock(&nvmap_handle_lock);
+	return b;
+}
+
+static struct nvmap_handle *_nvmap_validate_get(unsigned long handle, bool su)
+{
+	struct nvmap_handle *b = NULL;
+
+#ifdef CONFIG_DEVNVMAP_PARANOID
+	struct rb_node *n;
+
+	spin_lock(&nvmap_handle_lock);
+
+	n = nvmap_handles.rb_node;
+
+	while (n) {
+		b = rb_entry(n, struct nvmap_handle, node);
+		if ((unsigned long)b == handle) {
+			if (su || b->global || b->owner==current->group_leader)
+				b = _nvmap_handle_get(b);
+			else
+				b = NULL;
+			spin_unlock(&nvmap_handle_lock);
+			return b;
+		}
+		if (handle > (unsigned long)b) n = n->rb_right;
+		else n = n->rb_left;
+	}
+	spin_unlock(&nvmap_handle_lock);
+	return NULL;
+#else
+	if (!handle) return NULL;
+	b = _nvmap_handle_get((struct nvmap_handle *)handle);
+	return b;
+#endif
+}
+
+/*  nvmap_mru_vma_lock should be acquired by the caller before calling this */
+static inline void _nvmap_insert_mru_vma(struct nvmap_handle *h)
+{
+#ifdef CONFIG_DEVNVMAP_RECLAIM_UNPINNED_VM
+	list_add(&h->pgalloc.mru_list, _nvmap_list(h->pgalloc.area->iovm_length));
+#endif
+}
+
+static void _nvmap_remove_mru_vma(struct nvmap_handle *h)
+{
+#ifdef CONFIG_DEVNVMAP_RECLAIM_UNPINNED_VM
+	spin_lock(&nvmap_mru_vma_lock);
+	if (!list_empty(&h->pgalloc.mru_list))
+		list_del(&h->pgalloc.mru_list);
+	spin_unlock(&nvmap_mru_vma_lock);
+	INIT_LIST_HEAD(&h->pgalloc.mru_list);
+#endif
+}
+
+static struct tegra_iovmm_area *_nvmap_get_vm(struct nvmap_handle *h)
+{
+#ifndef CONFIG_DEVNVMAP_RECLAIM_UNPINNED_VM
+	BUG_ON(!h->pgalloc.area);
+	BUG_ON(h->size > h->pgalloc.area->iovm_length);
+	BUG_ON((h->size | h->pgalloc.area->iovm_length) & ~PAGE_MASK);
+	return h->pgalloc.area;
+#else
+	struct list_head *mru;
+	struct nvmap_handle *evict = NULL;
+	struct tegra_iovmm_area *vm = NULL;
+	unsigned int i, idx;
+
+	if (h->pgalloc.area) {
+		spin_lock(&nvmap_mru_vma_lock);
+		BUG_ON(list_empty(&h->pgalloc.mru_list));
+		list_del(&h->pgalloc.mru_list);
+		INIT_LIST_HEAD(&h->pgalloc.mru_list);
+		spin_unlock(&nvmap_mru_vma_lock);
+		return h->pgalloc.area;
+	}
+
+	vm = tegra_iovmm_create_vm(nvmap_vm_client, NULL, h->size,
+		_nvmap_flag_to_pgprot(h->flags, pgprot_kernel));
+
+	if (vm) {
+		INIT_LIST_HEAD(&h->pgalloc.mru_list);
+		return vm;
+	}
+	/* attempt to re-use the most recently unpinned IOVMM area in the
+	 * same size bin as the current handle. If that fails, iteratively
+	 * evict handles (starting from the current bin) until an allocation
+	 * succeeds or no more areas can be evicted */
+
+	spin_lock(&nvmap_mru_vma_lock);
+	mru = _nvmap_list(h->size);
+	if (!list_empty(mru))
+		evict = list_first_entry(mru, struct nvmap_handle,
+			pgalloc.mru_list);
+	if (evict && evict->pgalloc.area->iovm_length >= h->size) {
+		list_del(&evict->pgalloc.mru_list);
+		vm = evict->pgalloc.area;
+		evict->pgalloc.area = NULL;
+		INIT_LIST_HEAD(&evict->pgalloc.mru_list);
+		spin_unlock(&nvmap_mru_vma_lock);
+		return vm;
+	}
+
+	idx = mru - nvmap_mru_vma_lists;
+
+	for (i=0; i<ARRAY_SIZE(nvmap_mru_vma_lists) && !vm; i++, idx++) {
+		if (idx >= ARRAY_SIZE(nvmap_mru_vma_lists))
+			idx -= ARRAY_SIZE(nvmap_mru_vma_lists);
+		mru = &nvmap_mru_vma_lists[idx];
+		while (!list_empty(mru) && !vm) {
+			evict = list_first_entry(mru, struct nvmap_handle,
+				pgalloc.mru_list);
+
+			BUG_ON(atomic_add_return(0, &evict->pin)!=0);
+			BUG_ON(!evict->pgalloc.area);
+			list_del(&evict->pgalloc.mru_list);
+			INIT_LIST_HEAD(&evict->pgalloc.mru_list);
+			spin_unlock(&nvmap_mru_vma_lock);
+			tegra_iovmm_free_vm(evict->pgalloc.area);
+			evict->pgalloc.area = NULL;
+			vm = tegra_iovmm_create_vm(nvmap_vm_client,
+				NULL, h->size,
+				_nvmap_flag_to_pgprot(h->flags, pgprot_kernel));
+			spin_lock(&nvmap_mru_vma_lock);
+		}
+	}
+	spin_unlock(&nvmap_mru_vma_lock);
+	return vm;
+#endif
+}
+
+static int _nvmap_do_cache_maint(struct nvmap_handle *h,
+	unsigned long start, unsigned long end, unsigned long op, bool get);
+
+void _nvmap_handle_free(struct nvmap_handle *h)
+{
+	int e;
+	spin_lock(&nvmap_handle_lock);
+
+	/* if 2 contexts call _get and _put simultaneously, the reference
+	 * count may drop to 0 and then increase to 1 before the handle
+	 * can be freed. */
+	if (atomic_add_return(0, &h->ref)>0) {
+		spin_unlock(&nvmap_handle_lock);
+		return;
+	}
+	smp_rmb();
+	BUG_ON(atomic_read(&h->ref)<0);
+	BUG_ON(atomic_read(&h->pin)!=0);
+
+	rb_erase(&h->node, &nvmap_handles);
+
+	spin_unlock(&nvmap_handle_lock);
+
+	if (h->owner) put_task_struct(h->owner);
+
+	/* remove when NvRmMemMgr compatibility is eliminated */
+	if (h->kern_map) {
+		BUG_ON(!h->alloc);
+		if (h->heap_pgalloc)
+			vm_unmap_ram(h->kern_map, h->size>>PAGE_SHIFT);
+		else {
+			unsigned long addr = (unsigned long)h->kern_map;
+			addr &= ~PAGE_MASK;
+			iounmap((void *)addr);
+		}
+	}
+
+	/* ensure that no stale data remains in the cache for this handle */
+	e = _nvmap_do_cache_maint(h, 0, h->size, NVMEM_CACHE_OP_WB_INV, false);
+
+	if (h->alloc && !h->heap_pgalloc)
+		nvmap_carveout_free(h->carveout.co_heap, h->carveout.block_idx);
+	else if (h->alloc) {
+		unsigned int i;
+		BUG_ON(h->size & ~PAGE_MASK);
+		BUG_ON(!h->pgalloc.pages);
+		_nvmap_remove_mru_vma(h);
+		if (h->pgalloc.area) tegra_iovmm_free_vm(h->pgalloc.area);
+		for (i=0; i<h->size>>PAGE_SHIFT; i++) {
+			ClearPageReserved(h->pgalloc.pages[i]);
+			__free_page(h->pgalloc.pages[i]);
+		}
+		if ((h->size>>PAGE_SHIFT)*sizeof(struct page*)>=PAGE_SIZE)
+			vfree(h->pgalloc.pages);
+		else
+			kfree(h->pgalloc.pages);
+	}
+	h->poison = 0xa5a5a5a5;
+	kfree(h);
+}
+
+#define nvmap_gfp (GFP_KERNEL | __GFP_HIGHMEM | __GFP_NOWARN)
+
+/* map the backing pages for a heap_pgalloc handle into its IOVMM area */
+static void _nvmap_handle_iovmm_map(struct nvmap_handle *h)
+{
+	tegra_iovmm_addr_t va;
+	unsigned long i;
+
+	BUG_ON(!h->heap_pgalloc || !h->pgalloc.area);
+	BUG_ON(h->size & ~PAGE_MASK);
+	WARN_ON(!h->pgalloc.dirty);
+
+	for (va = h->pgalloc.area->iovm_start, i=0;
+	    va < (h->pgalloc.area->iovm_start + h->size);
+	    i++, va+=PAGE_SIZE) {
+		BUG_ON(!pfn_valid(page_to_pfn(h->pgalloc.pages[i])));
+		tegra_iovmm_vm_insert_pfn(h->pgalloc.area, va,
+			page_to_pfn(h->pgalloc.pages[i]));
+	}
+	h->pgalloc.dirty = false;
+}
+
+static int nvmap_pagealloc(struct nvmap_handle *h, bool contiguous)
+{
+	unsigned int i = 0, cnt = (h->size + PAGE_SIZE - 1) >> PAGE_SHIFT;
+	struct page **pages;
+
+	if (cnt*sizeof(*pages)>=PAGE_SIZE)
+		pages = vmalloc(cnt*sizeof(*pages));
+	else
+		pages = kzalloc(sizeof(*pages)*cnt, GFP_KERNEL);
+
+	if (!pages) return -ENOMEM;
+
+	if (contiguous) {
+		size_t order = get_order(h->size);
+		struct page *compound_page;
+		compound_page = alloc_pages(nvmap_gfp, order);
+		if (!compound_page) goto fail;
+		split_page(compound_page, order);
+		for (i=0; i<cnt; i++)
+			pages[i] = nth_page(compound_page, i);
+		for (; i<(1<<order); i++)
+			__free_page(nth_page(compound_page, i));
+	} else {
+		for (i=0; i<cnt; i++) {
+			pages[i] = alloc_page(nvmap_gfp);
+			if (!pages[i]) {
+			    pr_err("failed to allocate %u pages after %u entries\n",
+				   cnt, i);
+			    goto fail;
+			}
+		}
+	}
+
+	h->pgalloc.area = NULL;
+#ifndef CONFIG_DEVNVMAP_RECLAIM_UNPINNED_VM
+	if (!contiguous) {
+		h->pgalloc.area = tegra_iovmm_create_vm(nvmap_vm_client,
+			NULL, cnt << PAGE_SHIFT,
+			_nvmap_flag_to_pgprot(h->flags, pgprot_kernel));
+		if (!h->pgalloc.area) goto fail;
+		h->pgalloc.dirty = true;
+	}
+#endif
+
+	for (i=0; i<cnt; i++) {
+		void *km;
+		SetPageReserved(pages[i]);
+		km = kmap(pages[i]);
+		if (km) __cpuc_flush_dcache_area(km, PAGE_SIZE);
+		outer_flush_range(page_to_phys(pages[i]),
+			page_to_phys(pages[i])+PAGE_SIZE);
+		kunmap(pages[i]);
+	}
+
+	h->size = cnt<<PAGE_SHIFT;
+	h->pgalloc.pages = pages;
+	h->pgalloc.contig = contiguous;
+	INIT_LIST_HEAD(&h->pgalloc.mru_list);
+	return 0;
+
+fail:
+	while (i--) __free_page(pages[i]);
+	if (pages && (cnt*sizeof(*pages)>=PAGE_SIZE)) vfree(pages);
+	else if (pages) kfree(pages);
+	return -ENOMEM;
+}
+
+static struct nvmap_handle *_nvmap_handle_create(
+	struct task_struct *owner, size_t size)
+{
+	struct nvmap_handle *h = kzalloc(sizeof(*h), GFP_KERNEL);
+	struct nvmap_handle *b;
+	struct rb_node **p;
+	struct rb_node *parent = NULL;
+
+	if (!h) return NULL;
+	atomic_set(&h->ref, 1);
+	atomic_set(&h->pin, 0);
+	h->owner = owner;
+	h->size = h->orig_size = size;
+	h->flags = NVMEM_HANDLE_WRITE_COMBINE;
+	h->poison = NVDA_POISON;
+
+	spin_lock(&nvmap_handle_lock);
+	p = &nvmap_handles.rb_node;
+	while (*p) {
+		parent = *p;
+		b = rb_entry(parent, struct nvmap_handle, node);
+		if (h > b) p = &parent->rb_right;
+		else p = &parent->rb_left;
+	}
+	rb_link_node(&h->node, parent, p);
+	rb_insert_color(&h->node, &nvmap_handles);
+	spin_unlock(&nvmap_handle_lock);
+	if (owner) get_task_struct(owner);
+	return h;
+}
+
+/* nvmap pte manager */
+
+static void _nvmap_set_pte_at(unsigned long addr, unsigned long pfn,
+	pgprot_t prot)
+{
+	u32 off;
+	int idx;
+	pte_t *pte;
+
+	BUG_ON(!addr);
+	idx = NVMAP_PTE_INDEX(addr);
+	off = NVMAP_PTE_OFFSET(addr) & (PTRS_PER_PTE-1);
+
+	pte = nvmap_pte[idx] + off;
+	set_pte_ext(pte, pfn_pte(pfn, prot), 0);
+	flush_tlb_kernel_page(addr);
+}
+
+static int _nvmap_map_pte(unsigned long pfn, pgprot_t prot, void **vaddr)
+{
+	static unsigned int last_bit = 0;
+	unsigned long bit;
+	unsigned long addr;
+	unsigned long flags;
+
+	spin_lock_irqsave(&nvmap_ptelock, flags);
+
+	bit = find_next_zero_bit(nvmap_ptebits, NVMAP_PAGES, last_bit);
+	if (bit==NVMAP_PAGES) {
+		bit = find_first_zero_bit(nvmap_ptebits, last_bit);
+		if (bit == last_bit) bit = NVMAP_PAGES;
+	}
+
+	if (bit==NVMAP_PAGES) {
+		spin_unlock_irqrestore(&nvmap_ptelock, flags);
+		return -ENOMEM;
+	}
+
+	last_bit = bit;
+	set_bit(bit, nvmap_ptebits);
+	spin_unlock_irqrestore(&nvmap_ptelock, flags);
+
+	addr = NVMAP_BASE + bit*PAGE_SIZE;
+
+	_nvmap_set_pte_at(addr, pfn, prot);
+	*vaddr = (void *)addr;
+	return 0;
+}
+
+static int nvmap_map_pte(unsigned long pfn, pgprot_t prot, void **addr)
+{
+	int ret;
+	ret = wait_event_interruptible(nvmap_ptefull,
+		!_nvmap_map_pte(pfn, prot, addr));
+
+	if (ret==-ERESTARTSYS) return -EINTR;
+	return ret;
+}
+
+static void nvmap_unmap_pte(void *addr)
+{
+	unsigned long bit = NVMAP_PTE_OFFSET(addr);
+	unsigned long flags;
+
+	/* the ptes aren't cleared in this function, since the address isn't
+	 * re-used until it is allocated again by nvmap_map_pte. */
+	BUG_ON(bit >= NVMAP_PAGES);
+	spin_lock_irqsave(&nvmap_ptelock, flags);
+	clear_bit(bit, nvmap_ptebits);
+	spin_unlock_irqrestore(&nvmap_ptelock, flags);
+	wake_up(&nvmap_ptefull);
+}
+
+/* to ensure that the backing store for the VMA isn't freed while a fork'd
+ * reference still exists, nvmap_vma_open increments the reference count on
+ * the handle, and nvmap_vma_close decrements it. alternatively, we could
+ * disallow copying of the vma, or behave like pmem and zap the pages. FIXME.
+*/
+static void nvmap_vma_open(struct vm_area_struct *vma)
+{
+	struct nvmap_vma_priv *priv;
+
+	priv = vma->vm_private_data;
+
+	BUG_ON(!priv);
+
+	atomic_inc(&priv->ref);
+}
+
+static void nvmap_vma_close(struct vm_area_struct *vma) {
+	struct nvmap_vma_priv *priv = vma->vm_private_data;
+
+	if (priv && !atomic_dec_return(&priv->ref)) {
+		if (priv->h) _nvmap_handle_put(priv->h);
+		kfree(priv);
+	}
+	vma->vm_private_data = NULL;
+}
+
+static int nvmap_vma_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
+{
+	struct nvmap_vma_priv *priv;
+	unsigned long offs;
+
+	offs = (unsigned long)(vmf->virtual_address - vma->vm_start);
+	priv = vma->vm_private_data;
+	if (!priv || !priv->h || !priv->h->alloc)
+		return VM_FAULT_SIGBUS;
+
+	offs += priv->offs;
+	/* if the VMA was split for some reason, vm_pgoff will be the VMA's
+	 * offset from the original VMA */
+	offs += (vma->vm_pgoff << PAGE_SHIFT);
+
+	if (offs >= priv->h->size)
+		return VM_FAULT_SIGBUS;
+
+	if (!priv->h->heap_pgalloc) {
+		unsigned long pfn;
+		BUG_ON(priv->h->carveout.base & ~PAGE_MASK);
+		pfn = ((priv->h->carveout.base + offs) >> PAGE_SHIFT);
+		vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
+		return VM_FAULT_NOPAGE;
+	} else {
+		struct page *page;
+		offs >>= PAGE_SHIFT;
+		page = priv->h->pgalloc.pages[offs];
+		if (page) get_page(page);
+		vmf->page = page;
+		return (page) ? 0 : VM_FAULT_SIGBUS;
+	}
+}
+
+static long nvmap_ioctl(struct file *filp,
+	unsigned int cmd, unsigned long arg)
+{
+	int err = 0;
+	void __user *uarg = (void __user *)arg;
+
+	if (_IOC_TYPE(cmd) != NVMEM_IOC_MAGIC)
+		return -ENOTTY;
+
+	if (_IOC_NR(cmd) > NVMEM_IOC_MAXNR)
+		return -ENOTTY;
+
+	if (_IOC_DIR(cmd) & _IOC_READ)
+		err = !access_ok(VERIFY_WRITE, uarg, _IOC_SIZE(cmd));
+	if (_IOC_DIR(cmd) & _IOC_WRITE)
+		err = !access_ok(VERIFY_READ, uarg, _IOC_SIZE(cmd));
+
+	if (err)
+		return -EFAULT;
+
+	switch (cmd) {
+	case NVMEM_IOC_CREATE:
+	case NVMEM_IOC_CLAIM:
+	case NVMEM_IOC_FROM_ID:
+		err = nvmap_ioctl_create(filp, cmd, uarg);
+		break;
+
+	case NVMEM_IOC_GET_ID:
+		err = nvmap_ioctl_getid(filp, uarg);
+		break;
+
+	case NVMEM_IOC_PARAM:
+		err = nvmap_ioctl_get_param(filp, uarg);
+		break;
+
+	case NVMEM_IOC_UNPIN_MULT:
+	case NVMEM_IOC_PIN_MULT:
+		err = nvmap_ioctl_pinop(filp, cmd==NVMEM_IOC_PIN_MULT, uarg);
+		break;
+
+	case NVMEM_IOC_ALLOC:
+		err = nvmap_ioctl_alloc(filp, uarg);
+		break;
+
+	case NVMEM_IOC_FREE:
+		err = nvmap_ioctl_free(filp, arg);
+		break;
+
+	case NVMEM_IOC_MMAP:
+		err = nvmap_map_into_caller_ptr(filp, uarg);
+		break;
+
+	case NVMEM_IOC_WRITE:
+	case NVMEM_IOC_READ:
+		err = nvmap_ioctl_rw_handle(filp, cmd==NVMEM_IOC_READ, uarg);
+		break;
+
+	case NVMEM_IOC_CACHE:
+		err = nvmap_ioctl_cache_maint(filp, uarg);
+		break;
+
+	default:
+		return -ENOTTY;
+	}
+	return err;
+}
+
+/* must be called with the ref_lock held - given a user-space handle ID
+ * ref, validate that the handle_ref object may be used by the caller */
+struct nvmap_handle_ref *_nvmap_ref_lookup_locked(
+	struct nvmap_file_priv *priv, unsigned long ref)
+{
+	struct rb_node *n = priv->handle_refs.rb_node;
+	struct nvmap_handle *h = (struct nvmap_handle *)ref;
+
+	if (unlikely(h->poison != NVDA_POISON)) {
+		pr_err("%s: handle is poisoned\n", __func__);
+		return NULL;
+	}
+
+	while (n) {
+		struct nvmap_handle_ref *r;
+		r = rb_entry(n, struct nvmap_handle_ref, node);
+		if ((unsigned long)r->h == ref) return r;
+		else if (ref > (unsigned long)r->h) n = n->rb_right;
+		else n = n->rb_left;
+	}
+
+	return NULL;
+}
+
+/* must be called inside nvmap_pin_lock, to ensure that an entire stream
+ * of pins will complete without competition from a second stream. returns
+ * 0 if the pin was successful, -ENOMEM on failure */
+static int _nvmap_handle_pin_locked(struct nvmap_handle *h)
+{
+	struct tegra_iovmm_area *area;
+	BUG_ON(!h->alloc);
+
+	h = _nvmap_handle_get(h);
+	if (!h) return -ENOMEM;
+
+	if (atomic_inc_return(&h->pin)==1) {
+		if (h->heap_pgalloc && !h->pgalloc.contig) {
+			area = _nvmap_get_vm(h);
+			if (!area) {
+				/* no race here, inside the pin mutex */
+				atomic_dec(&h->pin);
+				_nvmap_handle_put(h);
+				return -ENOMEM;
+			}
+			if (area != h->pgalloc.area)
+				h->pgalloc.dirty = true;
+			h->pgalloc.area = area;
+		}
+	}
+	return 0;
+}
+
+/* doesn't need to be called inside nvmap_pin_lock, since this will only
+ * expand the available VM area */
+static int _nvmap_handle_unpin(struct nvmap_handle *h)
+{
+	int ret = 0;
+
+	if (atomic_add_return(0, &h->pin)==0) {
+		pr_err("%s: %s attempting to unpin an unpinned handle\n",
+			__func__, current->comm);
+		dump_stack();
+		return 0;
+	}
+
+	BUG_ON(!h->alloc);
+#ifdef CONFIG_DEVNVMAP_RECLAIM_UNPINNED_VM
+	spin_lock(&nvmap_mru_vma_lock);
+#endif
+	if (!atomic_dec_return(&h->pin)) {
+		if (h->heap_pgalloc && h->pgalloc.area) {
+			/* if a secure handle is clean (i.e., mapped into
+			 * IOVMM, it needs to be zapped on unpin. */
+			if (h->secure && !h->pgalloc.dirty) {
+				tegra_iovmm_zap_vm(h->pgalloc.area);
+				h->pgalloc.dirty = true;
+			}
+			_nvmap_insert_mru_vma(h);
+			ret=1;
+		}
+	}
+#ifdef CONFIG_DEVNVMAP_RECLAIM_UNPINNED_VM
+	spin_unlock(&nvmap_mru_vma_lock);
+#endif
+	_nvmap_handle_put(h);
+	return ret;
+}
+
+/* pin a list of handles, mapping IOVMM areas if needed. may sleep, if
+ * a handle's IOVMM area has been reclaimed and insufficient IOVMM space
+ * is available to complete the list pin. no intervening pin operations
+ * will interrupt this, and no validation is performed on the handles
+ * that are provided. */
+static int _nvmap_handle_pin_fast(unsigned int nr, struct nvmap_handle **h)
+{
+	unsigned int i;
+	int ret = 0;
+
+	mutex_lock(&nvmap_pin_lock);
+	for (i=0; i<nr && !ret; i++) {
+		ret = wait_event_interruptible(nvmap_pin_wait,
+			!_nvmap_handle_pin_locked(h[i]));
+	}
+	mutex_unlock(&nvmap_pin_lock);
+
+	if (ret) {
+		int do_wake = 0;
+		while (i--) do_wake |= _nvmap_handle_unpin(h[i]);
+		if (do_wake) wake_up(&nvmap_pin_wait);
+		return -EINTR;
+	} else {
+		for (i=0; i<nr; i++)
+			if (h[i]->heap_pgalloc && h[i]->pgalloc.dirty)
+				_nvmap_handle_iovmm_map(h[i]);
+	}
+
+	return 0;
+}
+
+static int _nvmap_do_global_unpin(unsigned long ref)
+{
+	struct nvmap_handle *h;
+	int w;
+
+	h = _nvmap_validate_get(ref, true);
+	if (unlikely(!h)) {
+		pr_err("%s: %s attempting to unpin non-existent handle\n",
+			__func__, current->group_leader->comm);
+		return 0;
+	}
+
+	pr_err("%s: %s unpinning %s's %uB %s handle without local context\n",
+		__func__, current->group_leader->comm,
+		(h->owner) ? h->owner->comm : "kernel", h->orig_size,
+		(h->heap_pgalloc && !h->pgalloc.contig) ? "iovmm" :
+		(h->heap_pgalloc) ? "sysmem" : "carveout");
+
+	w = _nvmap_handle_unpin(h);
+	_nvmap_handle_put(h);
+	return w;
+}
+
+static void _nvmap_do_unpin(struct nvmap_file_priv *priv,
+	unsigned int nr, unsigned long *refs)
+{
+	struct nvmap_handle_ref *r;
+	unsigned int i;
+	int do_wake = 0;
+
+	spin_lock(&priv->ref_lock);
+	for (i=0; i<nr; i++) {
+		if (!refs[i]) continue;
+		r = _nvmap_ref_lookup_locked(priv, refs[i]);
+		if (unlikely(!r)) {
+			if (priv->su)
+				do_wake |= _nvmap_do_global_unpin(refs[i]);
+			else
+				pr_err("%s: %s unpinning invalid handle\n",
+					__func__, current->comm);
+		} else if (unlikely(!atomic_add_unless(&r->pin, -1, 0)))
+			pr_err("%s: %s unpinning unpinned handle\n",
+				__func__, current->comm);
+		else
+			do_wake |= _nvmap_handle_unpin(r->h);
+	}
+	spin_unlock(&priv->ref_lock);
+	if (do_wake) wake_up(&nvmap_pin_wait);
+}
+
+/* pins a list of handle_ref objects; same conditions apply as to
+ * _nvmap_handle_pin, but also bumps the pin count of each handle_ref. */
+static int _nvmap_do_pin(struct nvmap_file_priv *priv,
+	unsigned int nr, unsigned long *refs)
+{
+	int ret = 0;
+	unsigned int i;
+	struct nvmap_handle **h = (struct nvmap_handle **)refs;
+	struct nvmap_handle_ref *r;
+
+	/* to optimize for the common case (client provided valid handle
+	 * references and the pin succeeds), increment the handle_ref pin
+	 * count during validation. in error cases, the tree will need to
+	 * be re-walked, since the handle_ref is discarded so that an
+	 * allocation isn't required. if a handle_ref is not found,
+	 * locally validate that the caller has permission to pin the handle;
+	 * handle_refs are not created in this case, so it is possible that
+	 * if the caller crashes after pinning a global handle, the handle
+	 * will be permanently leaked. */
+	spin_lock(&priv->ref_lock);
+	for (i=0; i<nr && !ret; i++) {
+		r = _nvmap_ref_lookup_locked(priv, refs[i]);
+		if (r) atomic_inc(&r->pin);
+		else {
+			if ((h[i]->poison != NVDA_POISON) ||
+			    (!(priv->su || h[i]->global ||
+			    current->group_leader == h[i]->owner)))
+				ret = -EPERM;
+			else {
+				pr_err("%s: %s pinning %s's %uB handle without "
+					"local context\n", __func__,
+					current->group_leader->comm,
+					h[i]->owner->comm, h[i]->orig_size);
+			}
+		}
+	}
+
+	while (ret && i--) {
+		r = _nvmap_ref_lookup_locked(priv, refs[i]);
+		if (r) atomic_dec(&r->pin);
+	}
+	spin_unlock(&priv->ref_lock);
+
+	if (ret) return ret;
+
+	mutex_lock(&nvmap_pin_lock);
+	for (i=0; i<nr && !ret; i++) {
+		ret = wait_event_interruptible(nvmap_pin_wait,
+			!_nvmap_handle_pin_locked(h[i]));
+	}
+	mutex_unlock(&nvmap_pin_lock);
+
+	if (ret) {
+		int do_wake = 0;
+		spin_lock(&priv->ref_lock);
+		while (i--) {
+			r = _nvmap_ref_lookup_locked(priv, refs[i]);
+			do_wake |= _nvmap_handle_unpin(r->h);
+			if (r) atomic_dec(&r->pin);
+		}
+		spin_unlock(&priv->ref_lock);
+		if (do_wake) wake_up(&nvmap_pin_wait);
+		return -EINTR;
+	} else {
+		for (i=0; i<nr; i++) {
+			if (h[i]->heap_pgalloc && h[i]->pgalloc.dirty)
+				_nvmap_handle_iovmm_map(h[i]);
+		}
+	}
+
+	return 0;
+}
+
+static int nvmap_ioctl_pinop(struct file *filp,
+	bool is_pin, void __user *arg)
+{
+	struct nvmem_pin_handle op;
+	struct nvmap_handle *h;
+	unsigned long on_stack[16];
+	unsigned long *refs;
+	unsigned long __user *output;
+	unsigned int i;
+	int err;
+
+	err = copy_from_user(&op, arg, sizeof(op));
+	if (err) return err;
+
+	if (!op.count) return -EINVAL;
+
+	if (op.count > 1) {
+		size_t bytes = op.count * sizeof(unsigned long *);
+		if (!access_ok(VERIFY_READ, (void *)op.handles, bytes))
+			return -EPERM;
+		if (is_pin && op.addr &&
+		    !access_ok(VERIFY_WRITE, (void *)op.addr, bytes))
+			return -EPERM;
+
+		if (op.count <= ARRAY_SIZE(on_stack)) refs = on_stack;
+		else refs = kzalloc(bytes, GFP_KERNEL);
+
+		if (!refs) return -ENOMEM;
+		err = copy_from_user(refs, (void*)op.handles, bytes);
+		if (err) goto out;
+	} else {
+		refs = on_stack;
+		on_stack[0] = (unsigned long)op.handles;
+	}
+
+	if (is_pin)
+		err = _nvmap_do_pin(filp->private_data, op.count, refs);
+	else
+		_nvmap_do_unpin(filp->private_data, op.count, refs);
+
+	/* skip the output stage on unpin */
+	if (err || !is_pin) goto out;
+
+	/* it is guaranteed that if _nvmap_do_pin returns 0 that
+	 * all of the handle_ref objects are valid, so dereferencing directly
+	 * here is safe */
+	if (op.count > 1)
+		output = (unsigned long __user *)op.addr;
+	else {
+		struct nvmem_pin_handle __user *tmp = arg;
+		output = (unsigned long __user *)&(tmp->addr);
+	}
+
+	if (!output) goto out;
+
+	for (i=0; i<op.count; i++) {
+		unsigned long addr;
+		h = (struct nvmap_handle *)refs[i];
+		if (h->heap_pgalloc && h->pgalloc.contig)
+			addr = page_to_phys(h->pgalloc.pages[0]);
+		else if (h->heap_pgalloc)
+			addr = h->pgalloc.area->iovm_start;
+		else
+			addr = h->carveout.base;
+
+		__put_user(addr, &output[i]);
+	}
+
+out:
+	if (refs != on_stack) kfree(refs);
+	return err;
+}
+
+static int nvmap_release(struct inode *inode, struct file *filp)
+{
+	struct nvmap_file_priv *priv = filp->private_data;
+	struct rb_node *n;
+	struct nvmap_handle_ref *r;
+	int refs;
+	int do_wake = 0;
+	int pins;
+
+	if (!priv) return 0;
+
+	while ((n = rb_first(&priv->handle_refs))) {
+		r = rb_entry(n, struct nvmap_handle_ref, node);
+		rb_erase(&r->node, &priv->handle_refs);
+		smp_rmb();
+		pins = atomic_read(&r->pin);
+		atomic_set(&r->pin, 0);
+		while (pins--) do_wake |= _nvmap_handle_unpin(r->h);
+		refs = atomic_read(&r->refs);
+		if (r->h->alloc && r->h->heap_pgalloc && !r->h->pgalloc.contig)
+			atomic_sub(r->h->size, &priv->iovm_commit);
+		while (refs--) _nvmap_handle_put(r->h);
+		kfree(r);
+	}
+	if (do_wake) wake_up(&nvmap_pin_wait);
+	kfree(priv);
+	return 0;
+}
+
+static int nvmap_open(struct inode *inode, struct file *filp)
+{
+	/* eliminate read, write and llseek support on this node */
+	struct nvmap_file_priv *priv;
+	int ret;
+
+	/* nvmap doesn't track total number of pinned references, so its
+	 * IOVMM client is always locked. */
+	if (!nvmap_vm_client) {
+		mutex_lock(&nvmap_pin_lock);
+		if (!nvmap_vm_client) {
+			nvmap_vm_client = tegra_iovmm_alloc_client("gpu", NULL);
+			if (nvmap_vm_client)
+				tegra_iovmm_client_lock(nvmap_vm_client);
+		}
+		mutex_unlock(&nvmap_pin_lock);
+	}
+
+	ret = nonseekable_open(inode, filp);
+	if (unlikely(ret))
+		return ret;
+
+	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
+	if (!priv) return -ENOMEM;
+	priv->handle_refs = RB_ROOT;
+	priv->su = (filp->f_op == &knvmap_fops);
+
+	atomic_set(&priv->iovm_commit, 0);
+
+	if (nvmap_vm_client)
+		priv->iovm_limit = tegra_iovmm_get_vm_size(nvmap_vm_client);
+#ifdef CONFIG_DEVNVMAP_RECLAIM_UNPINNED_VM
+	/* to prevent fragmentation-caused deadlocks, derate the size of
+	 * the IOVM space to 75% */
+	priv->iovm_limit >>= 2;
+	priv->iovm_limit *= 3;
+#endif
+
+	spin_lock_init(&priv->ref_lock);
+
+	filp->f_mapping->backing_dev_info = &nvmap_bdi;
+
+	filp->private_data = priv;
+	return 0;
+}
+
+static int nvmap_ioctl_getid(struct file *filp, void __user *arg)
+{
+	struct nvmem_create_handle op;
+	struct nvmap_handle *h = NULL;
+	int err;
+
+	err = copy_from_user(&op, arg, sizeof(op));
+	if (err) return err;
+
+	if (!op.handle) return -EINVAL;
+
+	h = _nvmap_validate_get((unsigned long)op.handle,
+		filp->f_op==&knvmap_fops);
+
+	if (h) {
+		op.id = (__u32)h;
+		/* when the owner of a handle gets its ID, this is treated
+		 * as a granting of the handle for use by other processes.
+		 * however, the super-user is not capable of promoting a
+		 * handle to global status if it was created in another
+		 * process. */
+		if (current->group_leader == h->owner) h->global = true;
+
+		/* getid is not supposed to result in a ref count increase */
+		_nvmap_handle_put(h);
+
+		return copy_to_user(arg, &op, sizeof(op));
+	}
+	return -EPERM;
+}
+
+static int _nvmap_do_alloc(struct nvmap_file_priv *priv,
+	unsigned long href, unsigned int heap_mask, size_t align,
+	unsigned int flags)
+{
+	struct nvmap_handle_ref *r;
+	struct nvmap_handle *h;
+	int numpages;
+
+	align = max_t(size_t, align, L1_CACHE_BYTES);
+
+	if (!href) return -EINVAL;
+
+	spin_lock(&priv->ref_lock);
+	r = _nvmap_ref_lookup_locked(priv, href);
+	spin_unlock(&priv->ref_lock);
+
+	if (!r) return -EPERM;
+
+	h = r->h;
+	if (h->alloc) return 0;
+
+	numpages = ((h->size + PAGE_SIZE - 1) >> PAGE_SHIFT);
+	h->secure = (flags & NVMEM_HANDLE_SECURE);
+	h->flags = (flags & 0x3);
+
+	BUG_ON(!numpages);
+
+	/* secure allocations can only be served from secure heaps */
+	if (h->secure) {
+		heap_mask &= NVMAP_SECURE_HEAPS;
+		if (!heap_mask) return -EINVAL;
+	}
+	/* can't do greater than page size alignment with page alloc */
+	if (align > PAGE_SIZE)
+		heap_mask &= NVMEM_HEAP_CARVEOUT_MASK;
+
+	while (heap_mask && !h->alloc) {
+		unsigned int heap_type = _nvmap_heap_policy(heap_mask, numpages);
+
+		if (heap_type & NVMEM_HEAP_CARVEOUT_MASK) {
+			struct nvmap_carveout_node *n;
+
+			down_read(&nvmap_context.list_sem);
+			list_for_each_entry(n, &nvmap_context.heaps, heap_list) {
+				if (heap_type & n->heap_bit) {
+					struct nvmap_carveout* co = &n->carveout;
+					int idx = nvmap_carveout_alloc(co, align, h->size);
+					if (idx != -1) {
+						h->carveout.co_heap = co;
+						h->carveout.block_idx = idx;
+						spin_lock(&co->lock);
+						h->carveout.base = co->blocks[idx].base;
+						spin_unlock(&co->lock);
+						h->heap_pgalloc = false;
+						h->alloc = true;
+						break;
+					}
+				}
+			}
+			up_read(&nvmap_context.list_sem);
+		}
+		else if (heap_type & NVMEM_HEAP_IOVMM) {
+			int ret;
+
+			BUG_ON(align > PAGE_SIZE);
+
+			/* increment the committed IOVM space prior to
+			 * allocation, to avoid race conditions with other
+			 * threads simultaneously allocating. this is
+			 * conservative, but guaranteed to work */
+			if (atomic_add_return(numpages << PAGE_SHIFT, &priv->iovm_commit)
+				< priv->iovm_limit) {
+				ret = nvmap_pagealloc(h, false);
+			}
+			else ret = -ENOMEM;
+
+			if (ret) {
+				atomic_sub(numpages << PAGE_SHIFT, &priv->iovm_commit);
+			}
+			else {
+				BUG_ON(h->pgalloc.contig);
+				h->heap_pgalloc = true;
+				h->alloc = true;
+			}
+		}
+		else if (heap_type & NVMEM_HEAP_SYSMEM) {
+			if (nvmap_pagealloc(h, true) == 0) {
+				BUG_ON(!h->pgalloc.contig);
+				h->heap_pgalloc = true;
+				h->alloc = true;
+			}
+		}
+		else break;
+
+		heap_mask &= ~heap_type;
+	}
+
+	return (h->alloc ? 0 : -ENOMEM);
+}
+
+static int nvmap_ioctl_alloc(struct file *filp, void __user *arg)
+{
+	struct nvmem_alloc_handle op;
+	struct nvmap_file_priv *priv = filp->private_data;
+	int err;
+
+	err = copy_from_user(&op, arg, sizeof(op));
+	if (err) return err;
+
+	if (op.align & (op.align-1)) return -EINVAL;
+
+	/* user-space handles are aligned to page boundaries, to prevent
+	 * data leakage. */
+	op.align = max_t(size_t, op.align, PAGE_SIZE);
+
+	return _nvmap_do_alloc(priv, op.handle, op.heap_mask, op.align, op.flags);
+}
+
+static int _nvmap_do_free(struct nvmap_file_priv *priv, unsigned long href)
+{
+	struct nvmap_handle_ref *r;
+	struct nvmap_handle *h;
+	int do_wake = 0;
+
+	if (!href) return 0;
+
+	spin_lock(&priv->ref_lock);
+	r = _nvmap_ref_lookup_locked(priv, href);
+
+	if (!r) {
+		spin_unlock(&priv->ref_lock);
+		pr_err("%s attempting to free unrealized handle\n",
+			current->group_leader->comm);
+		return -EPERM;
+	}
+
+	h = r->h;
+
+	smp_rmb();
+	if (!atomic_dec_return(&r->refs)) {
+		int pins = atomic_read(&r->pin);
+		rb_erase(&r->node, &priv->handle_refs);
+		spin_unlock(&priv->ref_lock);
+		if (pins) pr_err("%s: %s freeing %s's pinned %s %s %uB handle\n",
+			__func__, current->comm,
+			(r->h->owner) ? r->h->owner->comm : "kernel",
+			(r->h->global) ? "global" : "private",
+			(r->h->alloc && r->h->heap_pgalloc) ? "page-alloc" :
+			(r->h->alloc) ? "carveout" : "unallocated",
+			r->h->orig_size);
+		while (pins--) do_wake |= _nvmap_handle_unpin(r->h);
+		kfree(r);
+		if (h->alloc && h->heap_pgalloc && !h->pgalloc.contig)
+			atomic_sub(h->size, &priv->iovm_commit);
+		if (do_wake) wake_up(&nvmap_pin_wait);
+	} else
+		spin_unlock(&priv->ref_lock);
+
+	BUG_ON(!atomic_read(&h->ref));
+	_nvmap_handle_put(h);
+	return 0;
+}
+
+static int nvmap_ioctl_free(struct file *filp, unsigned long arg)
+{
+	return _nvmap_do_free(filp->private_data, arg);
+}
+
+/* given a size, pre-existing handle ID, or a preserved handle key, create
+ * a handle and a reference to the handle in the per-context data */
+static int _nvmap_do_create(struct nvmap_file_priv *priv,
+	unsigned int cmd, unsigned long key, bool su,
+	struct nvmap_handle_ref **ref)
+{
+	struct nvmap_handle_ref *r = NULL;
+	struct nvmap_handle *h = NULL;
+	struct rb_node **p, *parent = NULL;
+
+	if (cmd == NVMEM_IOC_FROM_ID) {
+		/* only ugly corner case to handle with from ID:
+		 *
+		 * normally, if the handle that is being duplicated is IOVMM-
+		 * backed, the handle should fail to duplicate if duping it
+		 * would over-commit IOVMM space.  however, if the handle is
+		 * already duplicated in the client process (or the client
+		 * is duplicating a handle it created originally), IOVMM space
+		 * should not be doubly-reserved.
+		 */
+		h = _nvmap_validate_get(key, priv->su);
+
+		if (!h) {
+			pr_err("%s: %s duplicate handle failed\n", __func__,
+				current->group_leader->comm);
+			return -EPERM;
+		}
+
+		if (!h->alloc) {
+			pr_err("%s: attempting to clone unallocated "
+				"handle\n", __func__);
+			_nvmap_handle_put(h);
+			h = NULL;
+			return -EINVAL;
+		}
+
+		spin_lock(&priv->ref_lock);
+		r = _nvmap_ref_lookup_locked(priv, (unsigned long)h);
+		spin_unlock(&priv->ref_lock);
+		if (r) {
+			/* if the client does something strange, like calling CreateFromId
+			 * when it was the original creator, avoid creating two handle refs
+			 * for the same handle */
+			atomic_inc(&r->refs);
+			*ref = r;
+			return 0;
+		}
+
+		/* verify that adding this handle to the process' access list
+		 * won't exceed the IOVM limit */
+		/* TODO: [ahatala 2010-04-20] let the kernel over-commit for now */
+		if (h->heap_pgalloc && !h->pgalloc.contig && !su) {
+			int oc = atomic_add_return(h->size, &priv->iovm_commit);
+			if (oc > priv->iovm_limit) {
+				atomic_sub(h->size, &priv->iovm_commit);
+				_nvmap_handle_put(h);
+				h = NULL;
+				pr_err("%s: %s duplicating handle would "
+					"over-commit iovmm space (%dB / %dB)\n",
+					__func__, current->group_leader->comm,
+					oc, priv->iovm_limit);
+				return -ENOMEM;
+			}
+		}
+	} else if (cmd == NVMEM_IOC_CREATE) {
+		h = _nvmap_handle_create(current->group_leader, key);
+		if (!h) return -ENOMEM;
+	} else {
+		h = _nvmap_claim_preserved(current->group_leader, key);
+		if (!h) return -EINVAL;
+	}
+
+	BUG_ON(!h);
+
+	r = kzalloc(sizeof(*r), GFP_KERNEL);
+	if (!r) {
+		if (h) _nvmap_handle_put(h);
+		return -ENOMEM;
+	}
+
+	atomic_set(&r->refs, 1);
+	r->h = h;
+	atomic_set(&r->pin, 0);
+
+	spin_lock(&priv->ref_lock);
+	p = &priv->handle_refs.rb_node;
+	while (*p) {
+		struct nvmap_handle_ref *l;
+		parent = *p;
+		l = rb_entry(parent, struct nvmap_handle_ref, node);
+		if (r->h > l->h) p = &parent->rb_right;
+		else p = &parent->rb_left;
+	}
+	rb_link_node(&r->node, parent, p);
+	rb_insert_color(&r->node, &priv->handle_refs);
+
+	spin_unlock(&priv->ref_lock);
+	*ref = r;
+	return 0;
+}
+
+static int nvmap_ioctl_create(struct file *filp,
+	unsigned int cmd, void __user *arg)
+{
+	struct nvmem_create_handle op;
+	struct nvmap_handle_ref *r = NULL;
+	struct nvmap_file_priv *priv = filp->private_data;
+	unsigned long key;
+	int err = 0;
+
+	err = copy_from_user(&op, arg, sizeof(op));
+	if (err) return err;
+
+	if (!priv) return -ENODEV;
+
+	/* user-space-created handles are expanded to be page-aligned,
+	 * so that mmap() will not accidentally leak a different allocation */
+	if (cmd==NVMEM_IOC_CREATE)
+		key = (op.size + PAGE_SIZE - 1) & ~(PAGE_SIZE-1);
+	else if (cmd==NVMEM_IOC_CLAIM)
+		key = op.key;
+	else if (cmd==NVMEM_IOC_FROM_ID)
+		key = op.id;
+
+	err = _nvmap_do_create(priv, cmd, key, (filp->f_op==&knvmap_fops), &r);
+
+	if (!err) {
+		op.handle = (uintptr_t)r->h;
+		/* since the size is spoofed to a page-multiple above,
+		 * clobber the orig_size field back to the requested value for
+		 * debugging. */
+		if (cmd == NVMEM_IOC_CREATE) r->h->orig_size = op.size;
+		err = copy_to_user(arg, &op, sizeof(op));
+		if (err) _nvmap_do_free(priv, op.handle);
+	}
+
+	return err;
+}
+
+static int nvmap_map_into_caller_ptr(struct file *filp, void __user *arg)
+{
+	struct nvmem_map_caller op;
+	struct nvmap_vma_priv *vpriv;
+	struct vm_area_struct *vma;
+	struct nvmap_handle *h;
+	int err = 0;
+
+	err = copy_from_user(&op, arg, sizeof(op));
+	if (err) return err;
+
+	if (!op.handle) return -EINVAL;
+
+	h = _nvmap_validate_get(op.handle, (filp->f_op==&knvmap_fops));
+	if (!h) return -EINVAL;
+
+	down_read(&current->mm->mmap_sem);
+
+	vma = find_vma(current->mm, op.addr);
+	if (!vma || !vma->vm_private_data) {
+		err = -ENOMEM;
+		goto out;
+	}
+
+	if (op.offset & ~PAGE_MASK) {
+		err = -EFAULT;
+		goto out;
+	}
+
+	if ((op.offset + op.length) > h->size) {
+		err = -EADDRNOTAVAIL;
+		goto out;
+	}
+
+	vpriv = vma->vm_private_data;
+	BUG_ON(!vpriv);
+
+	/* the VMA must exactly match the requested mapping operation, and the
+	 * VMA that is targetted must have been created originally by /dev/nvmap
+	 */
+	if ((vma->vm_start != op.addr) || (vma->vm_ops != &nvmap_vma_ops) ||
+	    (vma->vm_end-vma->vm_start != op.length)) {
+		err = -EPERM;
+		goto out;
+	}
+
+	/* verify that each mmap() system call creates a unique VMA */
+
+	if (vpriv->h && h==vpriv->h)
+		goto out;
+	else if (vpriv->h) {
+		err = -EADDRNOTAVAIL;
+		goto out;
+	}
+
+	if (!h->heap_pgalloc && (h->carveout.base & ~PAGE_MASK)) {
+		err = -EFAULT;
+		goto out;
+	}
+
+	vpriv->h = h;
+	vpriv->offs = op.offset;
+
+	/* if the hmem is not writeback-cacheable, drop back to a page mapping
+	 * which will guarantee DMA coherency
+	 */
+	vma->vm_page_prot = _nvmap_flag_to_pgprot(h->flags,
+		vma->vm_page_prot);
+
+out:
+	up_read(&current->mm->mmap_sem);
+	if (err) _nvmap_handle_put(h);
+	return err;
+}
+/* Initially, the nvmap mmap system call is used to allocate an inaccessible
+ * region of virtual-address space in the client.  A subsequent
+ * NVMAP_IOC_MMAP ioctl will associate each
+ */
+static int nvmap_mmap(struct file *filp, struct vm_area_struct *vma)
+{
+	/* FIXME: drivers which do not support cow seem to be split down the
+	 * middle whether to force the VM_SHARED flag, or to return an error
+	 * when this flag isn't already set (i.e., MAP_PRIVATE).
+	 */
+	struct nvmap_vma_priv *priv;
+
+	vma->vm_private_data = NULL;
+
+	priv = kzalloc(sizeof(*priv),GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	priv->offs = 0;
+	priv->h = NULL;
+	atomic_set(&priv->ref, 1);
+
+	vma->vm_flags |= VM_SHARED;
+	vma->vm_flags |= (VM_IO | VM_DONTEXPAND | VM_MIXEDMAP | VM_RESERVED);
+	vma->vm_ops = &nvmap_vma_ops;
+	vma->vm_private_data = priv;
+
+	return 0;
+}
+
+/* perform cache maintenance on a handle; caller's handle must be pre-
+ * validated. */
+static int _nvmap_do_cache_maint(struct nvmap_handle *h,
+	unsigned long start, unsigned long end, unsigned long op, bool get)
+{
+	pgprot_t prot;
+	void *addr = NULL;
+	void (*inner_maint)(const void*, const void*);
+	void (*outer_maint)(unsigned long, unsigned long);
+	int err = 0;
+
+	if (get) h = _nvmap_handle_get(h);
+
+	if (!h) return -EINVAL;
+
+	/* don't waste time on cache maintenance if the handle isn't cached */
+	if (h->flags == NVMEM_HANDLE_UNCACHEABLE ||
+	    h->flags == NVMEM_HANDLE_WRITE_COMBINE)
+		goto out;
+
+	if (op == NVMEM_CACHE_OP_WB) {
+		inner_maint = dmac_clean_range;
+		if (h->flags == NVMEM_HANDLE_CACHEABLE)
+			outer_maint = outer_clean_range;
+		else
+			outer_maint = NULL;
+	} else if (op == NVMEM_CACHE_OP_WB_INV) {
+		inner_maint = dmac_flush_range;
+		if (h->flags == NVMEM_HANDLE_CACHEABLE)
+			outer_maint = outer_flush_range;
+		else
+			outer_maint = NULL;
+	} else {
+		inner_maint = dmac_inv_range;
+		if (h->flags == NVMEM_HANDLE_CACHEABLE)
+			outer_maint = outer_inv_range;
+		else
+			outer_maint = NULL;
+	}
+
+	prot = _nvmap_flag_to_pgprot(h->flags, pgprot_kernel);
+
+	while (start < end) {
+		struct page *page = NULL;
+		unsigned long phys;
+		void *src;
+		size_t count;
+
+		if (h->heap_pgalloc) {
+			page = h->pgalloc.pages[start>>PAGE_SHIFT];
+			BUG_ON(!page);
+			get_page(page);
+			phys = page_to_phys(page) + (start & ~PAGE_MASK);
+		} else {
+			phys = h->carveout.base + start;
+		}
+
+		if (!addr) {
+			err = nvmap_map_pte(__phys_to_pfn(phys), prot, &addr);
+			if (err) {
+				if (page) put_page(page);
+				break;
+			}
+		} else {
+			_nvmap_set_pte_at((unsigned long)addr,
+				__phys_to_pfn(phys), prot);
+		}
+
+		src = addr + (phys & ~PAGE_MASK);
+		count = min_t(size_t, end-start, PAGE_SIZE-(phys&~PAGE_MASK));
+
+		inner_maint(src, src+count);
+		if (outer_maint) outer_maint(phys, phys+count);
+		start += count;
+		if (page) put_page(page);
+	}
+
+out:
+	if (h->flags == NVMEM_HANDLE_INNER_CACHEABLE) outer_sync();
+	if (addr) nvmap_unmap_pte(addr);
+	if (get) _nvmap_handle_put(h);
+	return err;
+}
+
+static int nvmap_ioctl_cache_maint(struct file *filp, void __user *arg)
+{
+	struct nvmem_cache_op	op;
+	int			err = 0;
+	struct vm_area_struct	*vma;
+	struct nvmap_vma_priv	*vpriv;
+	unsigned long		start;
+	unsigned long		end;
+
+	err = copy_from_user(&op, arg, sizeof(op));
+	if (err) return err;
+
+	if (!op.handle || !op.addr || op.op<NVMEM_CACHE_OP_WB ||
+	    op.op>NVMEM_CACHE_OP_WB_INV)
+		return -EINVAL;
+
+	vma = find_vma(current->active_mm, (unsigned long)op.addr);
+	if (!vma || vma->vm_ops!=&nvmap_vma_ops ||
+	    (unsigned long)op.addr + op.len > vma->vm_end)
+		return -EADDRNOTAVAIL;
+
+	vpriv = (struct nvmap_vma_priv *)vma->vm_private_data;
+
+	if ((unsigned long)vpriv->h != op.handle)
+		return -EFAULT;
+
+	start = (unsigned long)op.addr - vma->vm_start;
+	end  = start + op.len;
+
+	return _nvmap_do_cache_maint(vpriv->h, start, end, op.op, true);
+}
+
+/* copies a single element from the pre-get()'ed handle h, returns
+ * the number of bytes copied, and the address in the nvmap mapping range
+ * which was used (to eliminate re-allocation when copying multiple
+ * elements */
+static ssize_t _nvmap_do_one_rw_handle(struct nvmap_handle *h, int is_read,
+	int is_user, unsigned long start, unsigned long rw_addr,
+	unsigned long bytes, void **nvmap_addr)
+{
+	pgprot_t prot = _nvmap_flag_to_pgprot(h->flags, pgprot_kernel);
+	unsigned long end = start + bytes;
+	unsigned long orig_start = start;
+
+	if (is_user) {
+		if (is_read && !access_ok(VERIFY_WRITE, (void*)rw_addr, bytes))
+			return -EPERM;
+		if (!is_read && !access_ok(VERIFY_READ, (void*)rw_addr, bytes))
+			return -EPERM;
+	}
+
+	while (start < end) {
+		struct page *page = NULL;
+		unsigned long phys;
+		size_t count;
+		void *src;
+
+		if (h->heap_pgalloc) {
+			page = h->pgalloc.pages[start >> PAGE_SHIFT];
+			BUG_ON(!page);
+			get_page(page);
+			phys = page_to_phys(page) + (start & ~PAGE_MASK);
+		} else {
+			phys = h->carveout.base + start;
+		}
+
+		if (!*nvmap_addr) {
+			int err = nvmap_map_pte(__phys_to_pfn(phys),
+				prot, nvmap_addr);
+			if (err) {
+				if (page) put_page(page);
+				count = start - orig_start;
+				return (count) ? count : err;
+			}
+		} else {
+			_nvmap_set_pte_at((unsigned long)*nvmap_addr,
+				__phys_to_pfn(phys), prot);
+
+		}
+
+		src = *nvmap_addr + (phys & ~PAGE_MASK);
+		count = min_t(size_t, end-start, PAGE_SIZE-(phys&~PAGE_MASK));
+
+		if (is_user && is_read)
+			copy_to_user((void*)rw_addr, src, count);
+		else if (is_user)
+			copy_from_user(src, (void*)rw_addr, count);
+		else if (is_read)
+			memcpy((void*)rw_addr, src, count);
+		else
+			memcpy(src, (void*)rw_addr, count);
+
+		rw_addr += count;
+		start += count;
+		if (page) put_page(page);
+	}
+
+	return (ssize_t)start - orig_start;
+}
+
+static ssize_t _nvmap_do_rw_handle(struct nvmap_handle *h, int is_read,
+	int is_user, unsigned long h_offs, unsigned long sys_addr,
+	unsigned long h_stride, unsigned long sys_stride,
+	unsigned long elem_size, unsigned long count)
+{
+	ssize_t bytes_copied = 0;
+	void *addr = NULL;
+
+	h = _nvmap_handle_get(h);
+	if (!h) return -EINVAL;
+
+	if (elem_size == h_stride &&
+	    elem_size == sys_stride) {
+		elem_size *= count;
+		h_stride = elem_size;
+		sys_stride = elem_size;
+		count = 1;
+	}
+
+	while (count--) {
+		size_t ret = _nvmap_do_one_rw_handle(h, is_read,
+			is_user, h_offs, sys_addr, elem_size, &addr);
+		if (ret < 0) {
+			if (!bytes_copied) bytes_copied = ret;
+			break;
+		}
+		bytes_copied += ret;
+		if (ret < elem_size) break;
+		sys_addr += sys_stride;
+		h_offs += h_stride;
+	}
+
+	if (addr) nvmap_unmap_pte(addr);
+	_nvmap_handle_put(h);
+	return bytes_copied;
+}
+
+static int nvmap_ioctl_rw_handle(struct file *filp,
+	int is_read, void __user* arg)
+{
+	struct nvmem_rw_handle __user *uarg = arg;
+	struct nvmem_rw_handle op;
+	struct nvmap_handle *h;
+	ssize_t copied;
+	int err = 0;
+
+	err = copy_from_user(&op, arg, sizeof(op));
+	if (err) return err;
+
+	if (!op.handle || !op.addr || !op.count || !op.elem_size)
+		return -EINVAL;
+
+	h = _nvmap_validate_get(op.handle, (filp->f_op == &knvmap_fops));
+	if (!h) return -EINVAL; /* -EPERM? */
+
+	copied = _nvmap_do_rw_handle(h, is_read, 1, op.offset,
+		(unsigned long)op.addr, op.hmem_stride,
+		op.user_stride, op.elem_size, op.count);
+
+	if (copied < 0) { err = copied; copied = 0; }
+	else if (copied < (op.count*op.elem_size)) err = -EINTR;
+
+	__put_user(copied, &uarg->count);
+
+	_nvmap_handle_put(h);
+
+	return err;
+}
+
+static unsigned int _nvmap_do_get_param(struct nvmap_handle *h,
+	unsigned int param)
+{
+	if (param==NVMEM_HANDLE_PARAM_SIZE)
+		return h->orig_size;
+
+	else if (param==NVMEM_HANDLE_PARAM_ALIGNMENT) {
+		if (!h->alloc) return 0;
+
+		if (h->heap_pgalloc) return PAGE_SIZE;
+		else {
+			unsigned int i=1;
+			if (!h->carveout.base) return SZ_4M;
+			while (!(i & h->carveout.base)) i<<=1;
+			return i;
+		}
+	} else if (param==NVMEM_HANDLE_PARAM_BASE) {
+
+		if (!h->alloc || !atomic_add_return(0, &h->pin)){
+			WARN_ON(1);
+			return ~0ul;
+		}
+
+		if (!h->heap_pgalloc)
+			return h->carveout.base;
+
+		if (h->pgalloc.contig)
+			return page_to_phys(h->pgalloc.pages[0]);
+
+		if (h->pgalloc.area)
+			return h->pgalloc.area->iovm_start;
+
+		return ~0ul;
+	} else if (param==NVMEM_HANDLE_PARAM_HEAP) {
+
+		if (!h->alloc) return 0;
+
+		if (!h->heap_pgalloc) {
+			/* FIXME: hard-coded physical address */
+			if ((h->carveout.base & 0xf0000000ul)==0x40000000ul)
+				return NVMEM_HEAP_CARVEOUT_IRAM;
+			else
+				return NVMEM_HEAP_CARVEOUT_GENERIC;
+		}
+
+		if (!h->pgalloc.contig)
+			return NVMEM_HEAP_IOVMM;
+
+		return NVMEM_HEAP_SYSMEM;
+	}
+
+	return 0;
+}
+
+static int nvmap_ioctl_get_param(struct file *filp, void __user* arg)
+{
+	struct nvmem_handle_param op;
+	struct nvmap_handle *h;
+	int err;
+
+	err = copy_from_user(&op, arg, sizeof(op));
+	if (err) return err;
+
+	if (op.param < NVMEM_HANDLE_PARAM_SIZE ||
+	    op.param > NVMEM_HANDLE_PARAM_HEAP)
+		return -EINVAL;
+
+	h = _nvmap_validate_get(op.handle, (filp->f_op==&knvmap_fops));
+	if (!h) return -EINVAL;
+
+	op.result = _nvmap_do_get_param(h, op.param);
+	err = copy_to_user(arg, &op, sizeof(op));
+
+	_nvmap_handle_put(h);
+	return err;
+}
+
+static struct miscdevice misc_nvmap_dev = {
+	.minor = MISC_DYNAMIC_MINOR,
+	.name = "nvmap",
+	.fops = &nvmap_fops
+};
+
+static struct miscdevice misc_knvmap_dev = {
+	.minor = MISC_DYNAMIC_MINOR,
+	.name = "knvmap",
+	.fops = &knvmap_fops
+};
+
+static struct device *__nvmap_heap_parent_dev(void)
+{
+	return misc_nvmap_dev.this_device;
+}
+
+/* creates the sysfs attribute files for a carveout heap; if called
+ * before fs initialization, silently returns.
+ */
+static void _nvmap_create_heap_attrs(struct nvmap_carveout_node *n)
+{
+	if (!_nvmap_heap_parent_dev) return;
+	dev_set_name(&n->dev, "heap-%s", n->carveout.name);
+	n->dev.parent = _nvmap_heap_parent_dev;
+	n->dev.driver = NULL;
+	n->dev.release = NULL;
+	if (device_register(&n->dev)) {
+		pr_err("%s: failed to create heap-%s device\n",
+			__func__, n->carveout.name);
+		return;
+	}
+	if (sysfs_create_group(&n->dev.kobj, &nvmap_heap_defattr_group))
+		pr_err("%s: failed to create attribute group for heap-%s "
+			"device\n", __func__, n->carveout.name);
+}
+
+static int __init nvmap_dev_init(void)
+{
+	struct nvmap_carveout_node *n;
+
+	if (misc_register(&misc_nvmap_dev))
+		pr_err("%s error registering %s\n", __func__,
+			misc_nvmap_dev.name);
+
+	if (misc_register(&misc_knvmap_dev))
+		pr_err("%s error registering %s\n", __func__,
+			misc_knvmap_dev.name);
+
+	/* create sysfs attribute entries for all the heaps which were
+	 * created prior to nvmap_dev_init */
+	down_read(&nvmap_context.list_sem);
+	list_for_each_entry(n, &nvmap_context.heaps, heap_list) {
+		_nvmap_create_heap_attrs(n);
+	}
+	up_read(&nvmap_context.list_sem);
+
+	nvmap_procfs_root = proc_mkdir("nvmap", NULL);
+	if (nvmap_procfs_root) {
+		nvmap_procfs_proc = proc_mkdir("proc", nvmap_procfs_root);
+	}
+	return 0;
+}
+fs_initcall(nvmap_dev_init);
+
+/* initialization of core data structures split out to earlier in the
+ * init sequence, to allow kernel drivers access to nvmap before devfs
+ * is initialized */
+#define NR_CARVEOUTS 2
+static unsigned int nvmap_carveout_cmds = 0;
+static unsigned long nvmap_carveout_cmd_base[NR_CARVEOUTS];
+static unsigned long nvmap_carveout_cmd_size[NR_CARVEOUTS];
+
+static int __init nvmap_core_init(void)
+{
+	u32 base = NVMAP_BASE;
+	pgd_t *pgd;
+	pmd_t *pmd;
+	pte_t *pte;
+	unsigned int i;
+
+	init_rwsem(&nvmap_context.list_sem);
+	nvmap_context.init_data.handle_refs = RB_ROOT;
+	atomic_set(&nvmap_context.init_data.iovm_commit, 0);
+	/* no IOVMM allocations for kernel-created handles */
+	spin_lock_init(&nvmap_context.init_data.ref_lock);
+	nvmap_context.init_data.su = true;
+	nvmap_context.init_data.iovm_limit = 0;
+	INIT_LIST_HEAD(&nvmap_context.heaps);
+
+#ifdef CONFIG_DEVNVMAP_RECLAIM_UNPINNED_VM
+	for (i=0; i<ARRAY_SIZE(nvmap_mru_cutoff); i++)
+		INIT_LIST_HEAD(&nvmap_mru_vma_lists[i]);
+#endif
+
+	i = 0;
+	do {
+		pgd = pgd_offset(&init_mm, base);
+		pmd = pmd_alloc(&init_mm, pgd, base);
+		if (!pmd) {
+			pr_err("%s: no pmd tables\n", __func__);
+			return -ENOMEM;
+		}
+		pte = pte_alloc_kernel(pmd, base);
+		if (!pte) {
+			pr_err("%s: no pte tables\n", __func__);
+			return -ENOMEM;
+		}
+		nvmap_pte[i++] = pte;
+		base += (1<<PGDIR_SHIFT);
+	} while (base < NVMAP_END);
+
+	for (i=0; i<nvmap_carveout_cmds; i++) {
+		char tmp[16];
+		snprintf(tmp, sizeof(tmp), "generic-%u", i);
+		nvmap_add_carveout_heap(nvmap_carveout_cmd_base[i],
+			nvmap_carveout_cmd_size[i], tmp, 0x1);
+	}
+
+	return 0;
+}
+core_initcall(nvmap_core_init);
+
+static int __init nvmap_heap_arg(char *options)
+{
+	unsigned long start, size;
+	char *p = options;
+
+	start = -1;
+	size = memparse(p, &p);
+	if (*p == '@')
+		start = memparse(p + 1, &p);
+
+	if (nvmap_carveout_cmds < ARRAY_SIZE(nvmap_carveout_cmd_size)) {
+		nvmap_carveout_cmd_base[nvmap_carveout_cmds] = start;
+		nvmap_carveout_cmd_size[nvmap_carveout_cmds] = size;
+		nvmap_carveout_cmds++;
+	}
+	return 0;
+}
+__setup("nvmem=", nvmap_heap_arg);
+
+static int _nvmap_try_create_preserved(struct nvmap_carveout *co,
+	struct nvmap_handle *h, unsigned long base,
+	size_t size, unsigned int key)
+{
+	unsigned long end = base + size;
+	short idx;
+
+	h->carveout.base = ~0;
+	h->carveout.key = key;
+	h->carveout.co_heap = NULL;
+
+	spin_lock(&co->lock);
+	idx = co->free_index;
+	while (idx != -1) {
+		struct nvmap_mem_block *b = BLOCK(co, idx);
+		unsigned long blk_end = b->base + b->size;
+		if (b->base <= base && blk_end >= end) {
+			nvmap_split_block(co, idx, base, size);
+			h->carveout.block_idx = idx;
+			h->carveout.base = co->blocks[idx].base;
+			h->carveout.co_heap = co;
+			h->alloc = true;
+			break;
+		}
+		idx = b->next_free;
+	}
+	spin_unlock(&co->lock);
+
+	return (h->carveout.co_heap == NULL) ? -ENXIO : 0;
+}
+
+static void _nvmap_create_nvos_preserved(struct nvmap_carveout *co)
+{
+#ifdef CONFIG_TEGRA_NVOS
+	unsigned int i, key;
+	NvBootArgsPreservedMemHandle mem;
+	static int was_created[NvBootArgKey_PreservedMemHandle_Num -
+		NvBootArgKey_PreservedMemHandle_0] = { 0 };
+
+	for (i=0, key=NvBootArgKey_PreservedMemHandle_0;
+	     i<ARRAY_SIZE(was_created); i++, key++) {
+		struct nvmap_handle *h;
+
+		if (was_created[i]) continue;
+
+		if (NvOsBootArgGet(key, &mem, sizeof(mem))!=NvSuccess) continue;
+		if (!mem.Address || !mem.Size) continue;
+
+		h = _nvmap_handle_create(NULL, mem.Size);
+		if (!h) continue;
+
+		if (!_nvmap_try_create_preserved(co, h, mem.Address,
+		    mem.Size, key))
+			was_created[i] = 1;
+		else
+			_nvmap_handle_put(h);
+	}
+#endif
+}
+
+int nvmap_add_carveout_heap(unsigned long base, size_t size,
+	const char *name, unsigned int bitmask)
+{
+	struct nvmap_carveout_node *n;
+	struct nvmap_carveout_node *l;
+
+
+	n = kzalloc(sizeof(*n), GFP_KERNEL);
+	if (!n) return -ENOMEM;
+
+	BUG_ON(bitmask & ~NVMEM_HEAP_CARVEOUT_MASK);
+	n->heap_bit = bitmask;
+
+	if (_nvmap_init_carveout(&n->carveout, name, base, size)) {
+		kfree(n);
+		return -ENOMEM;
+	}
+
+	down_write(&nvmap_context.list_sem);
+
+	/* called inside the list_sem lock to ensure that the was_created
+	 * array is protected against simultaneous access */
+	_nvmap_create_nvos_preserved(&n->carveout);
+	_nvmap_create_heap_attrs(n);
+
+	list_for_each_entry(l, &nvmap_context.heaps, heap_list) {
+		if (n->heap_bit > l->heap_bit) {
+			list_add_tail(&n->heap_list, &l->heap_list);
+			up_write(&nvmap_context.list_sem);
+			return 0;
+		}
+	}
+	list_add_tail(&n->heap_list, &nvmap_context.heaps);
+	up_write(&nvmap_context.list_sem);
+	return 0;
+}
+
+int nvmap_create_preserved_handle(unsigned long base, size_t size,
+	unsigned int key)
+{
+	struct nvmap_carveout_node *i;
+	struct nvmap_handle *h;
+
+	h = _nvmap_handle_create(NULL, size);
+	if (!h) return -ENOMEM;
+
+	down_read(&nvmap_context.list_sem);
+	list_for_each_entry(i, &nvmap_context.heaps, heap_list) {
+		struct nvmap_carveout *co = &i->carveout;
+		if (!_nvmap_try_create_preserved(co, h, base, size, key))
+			break;
+	}
+	up_read(&nvmap_context.list_sem);
+
+	/* the base may not be correct if block splitting fails */
+	if (!h->carveout.co_heap || h->carveout.base != base) {
+		_nvmap_handle_put(h);
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+/* attempts to create a new carveout heap with a new usage bitmask by
+ * taking an allocation from a previous carveout with a different bitmask */
+static int nvmap_split_carveout_heap(struct nvmap_carveout *co, size_t size,
+	const char *name, unsigned int new_bitmask)
+{
+	struct nvmap_carveout_node *i, *n;
+	int idx = -1;
+	unsigned int blkbase, blksize;
+
+
+	n = kzalloc(sizeof(*n), GFP_KERNEL);
+	if (!n) return -ENOMEM;
+	n->heap_bit = new_bitmask;
+
+	/* align split carveouts to 1M */
+	idx = nvmap_carveout_alloc(co, SZ_1M, size);
+	if (idx != -1) {
+		/* take the spin lock to avoid race conditions with
+		 * intervening allocations triggering grow_block operations */
+		spin_lock(&co->lock);
+		blkbase = co->blocks[idx].base;
+		blksize = co->blocks[idx].size;
+		spin_unlock(&co->lock);
+
+		if (_nvmap_init_carveout(&n->carveout,name, blkbase, blksize)) {
+			nvmap_carveout_free(co, idx);
+			idx = -1;
+		} else {
+			spin_lock(&co->lock);
+			if (co->blocks[idx].prev) {
+				co->blocks[co->blocks[idx].prev].next =
+					co->blocks[idx].next;
+			}
+			if (co->blocks[idx].next) {
+				co->blocks[co->blocks[idx].next].prev =
+					co->blocks[idx].prev;
+			}
+			if (co->block_index==idx)
+				co->block_index = co->blocks[idx].next;
+			co->blocks[idx].next_free = -1;
+			co->blocks[idx].prev_free = -1;
+			co->blocks[idx].next = co->spare_index;
+			if (co->spare_index!=-1)
+				co->blocks[co->spare_index].prev = idx;
+			co->spare_index = idx;
+			spin_unlock(&co->lock);
+		}
+	}
+
+	if (idx==-1) {
+		kfree(n);
+		return -ENOMEM;
+	}
+
+	down_write(&nvmap_context.list_sem);
+	_nvmap_create_heap_attrs(n);
+	list_for_each_entry(i, &nvmap_context.heaps, heap_list) {
+		if (n->heap_bit > i->heap_bit) {
+			list_add_tail(&n->heap_list, &i->heap_list);
+			up_write(&nvmap_context.list_sem);
+			return 0;
+		}
+	}
+	list_add_tail(&n->heap_list, &nvmap_context.heaps);
+	up_write(&nvmap_context.list_sem);
+	return 0;
+}
+
+/* NvRmMemMgr APIs implemented on top of nvmap */
+
+#if defined(CONFIG_TEGRA_NVRM)
+#include <linux/freezer.h>
+
+NvU32 NvRmMemGetAddress(NvRmMemHandle hMem, NvU32 Offset)
+{
+	struct nvmap_handle *h = (struct nvmap_handle *)hMem;
+	unsigned long addr;
+
+	if (unlikely(!atomic_add_return(0, &h->pin) || !h->alloc ||
+	    Offset >= h->orig_size)) {
+		WARN_ON(1);
+		return ~0ul;
+	}
+
+	if (h->heap_pgalloc && h->pgalloc.contig)
+		addr = page_to_phys(h->pgalloc.pages[0]);
+	else if (h->heap_pgalloc) {
+		BUG_ON(!h->pgalloc.area);
+		addr = h->pgalloc.area->iovm_start;
+	} else
+		addr = h->carveout.base;
+
+	return (NvU32)addr+Offset;
+
+}
+
+void NvRmMemPinMult(NvRmMemHandle *hMems, NvU32 *addrs, NvU32 Count)
+{
+	struct nvmap_handle **h = (struct nvmap_handle **)hMems;
+	unsigned int i;
+	int ret;
+
+	do {
+		ret = _nvmap_handle_pin_fast(Count, h);
+		if (ret && !try_to_freeze()) {
+			pr_err("%s: failed to pin handles\n", __func__);
+			dump_stack();
+		}
+	} while (ret);
+
+	for (i=0; i<Count; i++) {
+		addrs[i] = NvRmMemGetAddress(hMems[i], 0);
+		BUG_ON(addrs[i]==~0ul);
+	}
+}
+
+void NvRmMemUnpinMult(NvRmMemHandle *hMems, NvU32 Count)
+{
+	int do_wake = 0;
+	unsigned int i;
+
+	for (i=0; i<Count; i++) {
+		struct nvmap_handle *h = (struct nvmap_handle *)hMems[i];
+		if (h) {
+			BUG_ON(atomic_add_return(0, &h->pin)==0);
+			do_wake |= _nvmap_handle_unpin(h);
+		}
+	}
+
+	if (do_wake) wake_up(&nvmap_pin_wait);
+}
+
+NvU32 NvRmMemPin(NvRmMemHandle hMem)
+{
+	NvU32 addr;
+	NvRmMemPinMult(&hMem, &addr, 1);
+	return addr;
+}
+
+void NvRmMemUnpin(NvRmMemHandle hMem)
+{
+	NvRmMemUnpinMult(&hMem, 1);
+}
+
+void NvRmMemHandleFree(NvRmMemHandle hMem)
+{
+	_nvmap_do_free(&nvmap_context.init_data, (unsigned long)hMem);
+}
+
+NvError NvRmMemMap(NvRmMemHandle hMem, NvU32 Offset, NvU32 Size,
+	NvU32 Flags, void **pVirtAddr)
+{
+	struct nvmap_handle *h = (struct nvmap_handle *)hMem;
+	pgprot_t prot = _nvmap_flag_to_pgprot(h->flags, pgprot_kernel);
+
+	BUG_ON(!h->alloc);
+
+	if (Offset+Size > h->size)
+		return NvError_BadParameter;
+
+	if (!h->kern_map && h->heap_pgalloc) {
+		BUG_ON(h->size & ~PAGE_MASK);
+		h->kern_map = vm_map_ram(h->pgalloc.pages,
+			h->size>>PAGE_SHIFT, -1, prot);
+	} else if (!h->kern_map) {
+		unsigned int size;
+		unsigned long addr;
+
+		addr = h->carveout.base;
+		size = h->size + (addr & ~PAGE_MASK);
+		addr &= PAGE_MASK;
+		size = (size + PAGE_SIZE - 1) & PAGE_MASK;
+
+		h->kern_map = ioremap_wc(addr, size);
+		if (h->kern_map) {
+			addr = h->carveout.base - addr;
+			h->kern_map += addr;
+		}
+	}
+
+	if (h->kern_map) {
+		*pVirtAddr = (h->kern_map + Offset);
+		return NvSuccess;
+	}
+
+	return NvError_InsufficientMemory;
+}
+
+void NvRmMemUnmap(NvRmMemHandle hMem, void *pVirtAddr, NvU32 Size)
+{
+	return;
+}
+
+NvU32 NvRmMemGetId(NvRmMemHandle hMem)
+{
+	struct nvmap_handle *h = (struct nvmap_handle *)hMem;
+	if (!h->owner) h->global = true;
+	return (NvU32)h;
+}
+
+NvError NvRmMemHandleFromId(NvU32 id, NvRmMemHandle *hMem)
+{
+	struct nvmap_handle_ref *r;
+
+	int err = _nvmap_do_create(&nvmap_context.init_data,
+		NVMEM_IOC_FROM_ID, id, true, &r);
+
+	if (err || !r) return NvError_NotInitialized;
+
+	*hMem = (NvRmMemHandle)r->h;
+	return NvSuccess;
+}
+
+NvError NvRmMemHandleClaimPreservedHandle(NvRmDeviceHandle hRm,
+	NvU32 Key, NvRmMemHandle *hMem)
+{
+	struct nvmap_handle_ref *r;
+
+	int err = _nvmap_do_create(&nvmap_context.init_data,
+		NVMEM_IOC_CLAIM, (unsigned long)Key, true, &r);
+
+	if (err || !r) return NvError_NotInitialized;
+
+	*hMem = (NvRmMemHandle)r->h;
+	return NvSuccess;
+}
+
+NvError NvRmMemHandleCreate(NvRmDeviceHandle hRm,
+	NvRmMemHandle *hMem, NvU32 Size)
+{
+	struct nvmap_handle_ref *r;
+	int err = _nvmap_do_create(&nvmap_context.init_data,
+		NVMEM_IOC_CREATE, (unsigned long)Size, true, &r);
+
+	if (err || !r) return NvError_InsufficientMemory;
+	*hMem = (NvRmMemHandle)r->h;
+	return NvSuccess;
+}
+
+NvError NvRmMemAlloc(NvRmMemHandle hMem, const NvRmHeap *Heaps,
+	NvU32 NumHeaps, NvU32 Alignment, NvOsMemAttribute Coherency)
+{
+	unsigned int flags = pgprot_kernel;
+	int err = -ENOMEM;
+
+	BUG_ON(Alignment & (Alignment-1));
+
+	if (Coherency == NvOsMemAttribute_WriteBack)
+		flags = NVMEM_HANDLE_INNER_CACHEABLE;
+	else
+		flags = NVMEM_HANDLE_WRITE_COMBINE;
+
+	if (!NumHeaps || !Heaps) {
+		err = _nvmap_do_alloc(&nvmap_context.init_data,
+				  (unsigned long)hMem, NVMAP_KERNEL_DEFAULT_HEAPS,
+				  (size_t)Alignment, flags);
+	}
+	else {
+		unsigned int i;
+		for (i = 0; i < NumHeaps; i++) {
+			unsigned int heap;
+			switch (Heaps[i]) {
+			case NvRmHeap_GART:
+				heap = NVMEM_HEAP_IOVMM;
+				break;
+			case NvRmHeap_External:
+				heap = NVMEM_HEAP_SYSMEM;
+				break;
+			case NvRmHeap_ExternalCarveOut:
+				heap = NVMEM_HEAP_CARVEOUT_GENERIC;
+				break;
+			case NvRmHeap_IRam:
+				heap = NVMEM_HEAP_CARVEOUT_IRAM;
+				break;
+			default:
+				heap = 0;
+				break;
+			}
+			if (heap) {
+				err = _nvmap_do_alloc(&nvmap_context.init_data,
+						  (unsigned long)hMem, heap,
+						  (size_t)Alignment, flags);
+				if (!err) break;
+			}
+		}
+	}
+
+	return (err ? NvError_InsufficientMemory : NvSuccess);
+}
+
+void NvRmMemReadStrided(NvRmMemHandle hMem, NvU32 Offset, NvU32 SrcStride,
+	void *pDst, NvU32 DstStride, NvU32 ElementSize, NvU32 Count)
+{
+	ssize_t bytes = 0;
+
+	bytes = _nvmap_do_rw_handle((struct nvmap_handle *)hMem, true,
+		false, Offset, (unsigned long)pDst, SrcStride,
+		DstStride, ElementSize, Count);
+
+	BUG_ON(bytes != (ssize_t)(Count*ElementSize));
+}
+
+void NvRmMemWriteStrided(NvRmMemHandle hMem, NvU32 Offset, NvU32 DstStride,
+	const void *pSrc, NvU32 SrcStride, NvU32 ElementSize, NvU32 Count)
+{
+	ssize_t bytes = 0;
+
+	bytes = _nvmap_do_rw_handle((struct nvmap_handle *)hMem, false,
+		false, Offset, (unsigned long)pSrc, DstStride,
+		SrcStride, ElementSize, Count);
+
+	BUG_ON(bytes != (ssize_t)(Count*ElementSize));
+}
+
+NvU32 NvRmMemGetSize(NvRmMemHandle hMem)
+{
+	struct nvmap_handle *h = (struct nvmap_handle *)hMem;
+	return h->orig_size;
+}
+
+NvRmHeap NvRmMemGetHeapType(NvRmMemHandle hMem, NvU32 *BaseAddr)
+{
+	struct nvmap_handle *h = (struct nvmap_handle *)hMem;
+	NvRmHeap heap;
+
+	if (!h->alloc) {
+		*BaseAddr = ~0ul;
+		return (NvRmHeap)0;
+	}
+
+	if (h->heap_pgalloc && !h->pgalloc.contig)
+		heap = NvRmHeap_GART;
+	else if (h->heap_pgalloc)
+		heap = NvRmHeap_External;
+	else if ((h->carveout.base & 0xf0000000ul) == 0x40000000ul)
+		heap = NvRmHeap_IRam;
+	else
+		heap = NvRmHeap_ExternalCarveOut;
+
+	if (h->heap_pgalloc && h->pgalloc.contig)
+		*BaseAddr = (NvU32)page_to_phys(h->pgalloc.pages[0]);
+	else if (h->heap_pgalloc && atomic_add_return(0, &h->pin))
+		*BaseAddr = h->pgalloc.area->iovm_start;
+	else if (h->heap_pgalloc)
+		*BaseAddr = ~0ul;
+	else
+		*BaseAddr = (NvU32)h->carveout.base;
+
+	return heap;
+}
+
+void NvRmMemCacheMaint(NvRmMemHandle hMem, void *pMapping,
+	NvU32 Size, NvBool Writeback, NvBool Inv)
+{
+	struct nvmap_handle *h = (struct nvmap_handle *)hMem;
+	unsigned long start;
+	unsigned int op;
+
+	if (!h->kern_map || h->flags==NVMEM_HANDLE_UNCACHEABLE ||
+		h->flags==NVMEM_HANDLE_WRITE_COMBINE) return;
+
+	if (!Writeback && !Inv) return;
+
+	if (Writeback && Inv) op = NVMEM_CACHE_OP_WB_INV;
+	else if (Writeback) op = NVMEM_CACHE_OP_WB;
+	else op = NVMEM_CACHE_OP_INV;
+
+	start = (unsigned long)pMapping - (unsigned long)h->kern_map;
+
+	_nvmap_do_cache_maint(h, start, start+Size, op, true);
+	return;
+}
+
+NvU32 NvRmMemGetAlignment(NvRmMemHandle hMem)
+{
+	struct nvmap_handle *h = (struct nvmap_handle *)hMem;
+	return _nvmap_do_get_param(h, NVMEM_HANDLE_PARAM_ALIGNMENT);
+}
+
+NvError NvRmMemGetStat(NvRmMemStat Stat, NvS32 *Result)
+{
+	unsigned long total_co = 0;
+	unsigned long free_co = 0;
+	unsigned long max_free = 0;
+	struct nvmap_carveout_node *n;
+
+	down_read(&nvmap_context.list_sem);
+	list_for_each_entry(n, &nvmap_context.heaps, heap_list) {
+
+		if (!(n->heap_bit & NVMEM_HEAP_CARVEOUT_GENERIC)) continue;
+			total_co += _nvmap_carveout_blockstat(&n->carveout,
+					CARVEOUT_STAT_TOTAL_SIZE);
+			free_co += _nvmap_carveout_blockstat(&n->carveout,
+					CARVEOUT_STAT_FREE_SIZE);
+			max_free = max(max_free,
+				_nvmap_carveout_blockstat(&n->carveout,
+				    CARVEOUT_STAT_LARGEST_FREE));
+	}
+	up_read(&nvmap_context.list_sem);
+
+	if (Stat==NvRmMemStat_TotalCarveout) {
+		*Result = (NvU32)total_co;
+		return NvSuccess;
+	} else if (Stat==NvRmMemStat_UsedCarveout) {
+		*Result = (NvU32)total_co - (NvU32)free_co;
+		return NvSuccess;
+	} else if (Stat==NvRmMemStat_LargestFreeCarveoutBlock) {
+		*Result = (NvU32)max_free;
+		return NvSuccess;
+	}
+
+	return NvError_BadParameter;
+}
+
+NvU8 NvRmMemRd08(NvRmMemHandle hMem, NvU32 Offset)
+{
+	NvU8 val;
+	NvRmMemRead(hMem, Offset, &val, sizeof(val));
+	return val;
+}
+
+NvU16 NvRmMemRd16(NvRmMemHandle hMem, NvU32 Offset)
+{
+	NvU16 val;
+	NvRmMemRead(hMem, Offset, &val, sizeof(val));
+	return val;
+}
+
+NvU32 NvRmMemRd32(NvRmMemHandle hMem, NvU32 Offset)
+{
+	NvU32 val;
+	NvRmMemRead(hMem, Offset, &val, sizeof(val));
+	return val;
+}
+
+void NvRmMemWr08(NvRmMemHandle hMem, NvU32 Offset, NvU8 Data)
+{
+	NvRmMemWrite(hMem, Offset, &Data, sizeof(Data));
+}
+
+void NvRmMemWr16(NvRmMemHandle hMem, NvU32 Offset, NvU16 Data)
+{
+	NvRmMemWrite(hMem, Offset, &Data, sizeof(Data));
+}
+
+void NvRmMemWr32(NvRmMemHandle hMem, NvU32 Offset, NvU32 Data)
+{
+	NvRmMemWrite(hMem, Offset, &Data, sizeof(Data));
+}
+
+void NvRmMemRead(NvRmMemHandle hMem, NvU32 Offset, void *pDst, NvU32 Size)
+{
+	NvRmMemReadStrided(hMem, Offset, Size, pDst, Size, Size, 1);
+}
+
+void NvRmMemWrite(NvRmMemHandle hMem, NvU32 Offset,
+	const void *pSrc, NvU32 Size)
+{
+	NvRmMemWriteStrided(hMem, Offset, Size, pSrc, Size, Size, 1);
+}
+
+void NvRmMemMove(NvRmMemHandle dstHMem, NvU32 dstOffset,
+	NvRmMemHandle srcHMem, NvU32 srcOffset, NvU32 Size)
+{
+	while (Size--) {
+		NvU8 tmp = NvRmMemRd08(srcHMem, srcOffset);
+		NvRmMemWr08(dstHMem, dstOffset, tmp);
+		dstOffset++;
+		srcOffset++;
+	}
+}
+
+NvU32 NvRmMemGetCacheLineSize(void)
+{
+	return 32;
+}
+
+void *NvRmHostAlloc(size_t size)
+{
+	return NvOsAlloc(size);
+}
+
+void NvRmHostFree(void *ptr)
+{
+	NvOsFree(ptr);
+}
+
+NvError NvRmMemMapIntoCallerPtr(NvRmMemHandle hMem, void *pCallerPtr,
+	NvU32 Offset, NvU32 Size)
+{
+	return NvError_NotSupported;
+}
+
+NvError NvRmMemHandlePreserveHandle(NvRmMemHandle hMem, NvU32 *pKey)
+{
+	return NvError_NotSupported;
+}
+
+#endif
+
+static u32 nvmap_get_physaddr(struct nvmap_handle *h)
+{
+	u32 addr;
+
+	if (h->heap_pgalloc && h->pgalloc.contig) {
+		addr = page_to_phys(h->pgalloc.pages[0]);
+	} else if (h->heap_pgalloc) {
+		BUG_ON(!h->pgalloc.area);
+		addr = h->pgalloc.area->iovm_start;
+	} else {
+		addr = h->carveout.base;
+	}
+
+	return addr;
+}
+
+struct nvmap_handle *nvmap_alloc(
+	size_t size, size_t align,
+	unsigned int flags, void **map)
+{
+	struct nvmap_handle_ref *r = NULL;
+	struct nvmap_handle *h;
+	int err;
+
+	err = _nvmap_do_create(&nvmap_context.init_data,
+			NVMEM_IOC_CREATE, (unsigned long)size, true, &r);
+	if (err || !r)
+		return ERR_PTR(err);
+	h = r->h;
+
+	err = _nvmap_do_alloc(&nvmap_context.init_data,
+			(unsigned long)h, NVMAP_KERNEL_DEFAULT_HEAPS,
+			align, flags);
+	if (err) {
+		_nvmap_do_free(&nvmap_context.init_data, (unsigned long)h);
+		return ERR_PTR(err);
+	}
+
+	if (!map)
+		return h;
+
+	if (h->heap_pgalloc) {
+		*map = vm_map_ram(h->pgalloc.pages, h->size >> PAGE_SHIFT, -1,
+				_nvmap_flag_to_pgprot(h->flags, pgprot_kernel));
+	} else {
+		size_t mapaddr = h->carveout.base;
+		size_t mapsize = h->size;
+
+		mapsize += (mapaddr & ~PAGE_MASK);
+		mapaddr &= PAGE_MASK;
+		mapsize = (mapsize + PAGE_SIZE - 1) & PAGE_MASK;
+
+		/* TODO: [ahatala 2010-06-21] honor coherency flag? */
+		*map = ioremap_wc(mapaddr, mapsize);
+		if (*map)
+			*map += (h->carveout.base - mapaddr);
+	}
+	if (!*map) {
+		_nvmap_do_free(&nvmap_context.init_data, (unsigned long)h);
+		return ERR_PTR(-ENOMEM);
+	}
+	/* TODO: [ahatala 2010-06-22] get rid of kern_map */
+	h->kern_map = *map;
+	return h;
+}
+
+void nvmap_free(struct nvmap_handle *h, void *map)
+{
+	if (map) {
+		BUG_ON(h->kern_map != map);
+
+		if (h->heap_pgalloc) {
+			vm_unmap_ram(map, h->size >> PAGE_SHIFT);
+		} else {
+			unsigned long addr = (unsigned long)map;
+			addr &= ~PAGE_MASK;
+			iounmap((void *)addr);
+		}
+		h->kern_map = NULL;
+	}
+	_nvmap_do_free(&nvmap_context.init_data, (unsigned long)h);
+}
+
+u32 nvmap_pin_single(struct nvmap_handle *h)
+{
+	int ret;
+	do {
+		ret = _nvmap_handle_pin_fast(1, &h);
+		if (ret) {
+			pr_err("%s: failed to pin handle\n", __func__);
+			dump_stack();
+		}
+	} while (ret);
+
+	return nvmap_get_physaddr(h);
+}
+
+int nvmap_pin_array(struct file *filp,
+		struct nvmap_pinarray_elem *arr, int num_elems,
+		struct nvmap_handle **unique_arr, int *num_unique, bool wait)
+{
+	struct nvmap_pinarray_elem *elem;
+	struct nvmap_file_priv *priv = filp->private_data;
+	int i, unique_idx = 0;
+	unsigned long pfn = 0;
+	void *pteaddr = NULL;
+	int ret = 0;
+
+	mutex_lock(&nvmap_pin_lock);
+
+	/* find unique handles, pin them and collect into unpin array */
+	for (elem = arr, i = num_elems; i && !ret; i--, elem++) {
+		struct nvmap_handle *to_pin = elem->pin_mem;
+		if (to_pin->poison != NVDA_POISON) {
+			pr_err("%s: handle is poisoned\n", __func__);
+			ret = -EFAULT;
+		}
+		else if (!(to_pin->flags & NVMEM_HANDLE_VISITED)) {
+			if (!priv->su && !to_pin->global) {
+				struct nvmap_handle_ref *r;
+				spin_lock(&priv->ref_lock);
+				r = _nvmap_ref_lookup_locked(priv,
+							(unsigned long)to_pin);
+				spin_unlock(&priv->ref_lock);
+				if (!r) {
+					pr_err("%s: handle access failure\n", __func__);
+					ret = -EPERM;
+					break;
+				}
+			}
+			if (wait) {
+				ret = wait_event_interruptible(
+					nvmap_pin_wait,
+					!_nvmap_handle_pin_locked(to_pin));
+			}
+			else
+				ret = _nvmap_handle_pin_locked(to_pin);
+			if (!ret) {
+				to_pin->flags |= NVMEM_HANDLE_VISITED;
+				unique_arr[unique_idx++] = to_pin;
+			}
+		}
+	}
+
+	/* clear visited flags before releasing mutex */
+	i = unique_idx;
+	while (i--)
+		unique_arr[i]->flags &= ~NVMEM_HANDLE_VISITED;
+
+	mutex_unlock(&nvmap_pin_lock);
+
+	if (!ret)
+		ret = nvmap_map_pte(pfn, pgprot_kernel, &pteaddr);
+
+	if (unlikely(ret)) {
+		int do_wake = 0;
+		i = unique_idx;
+		while (i--)
+			do_wake |= _nvmap_handle_unpin(unique_arr[i]);
+		if (do_wake)
+			wake_up(&nvmap_pin_wait);
+		return ret;
+	}
+
+	for (elem = arr, i = num_elems; i; i--, elem++) {
+		struct nvmap_handle *h_patch = elem->patch_mem;
+		struct nvmap_handle *h_pin = elem->pin_mem;
+		struct page *page = NULL;
+		u32* patch_addr;
+
+		/* commit iovmm mapping */
+		if (h_pin->heap_pgalloc && h_pin->pgalloc.dirty)
+			_nvmap_handle_iovmm_map(h_pin);
+
+		/* patch */
+		if (h_patch->kern_map) {
+			patch_addr = (u32*)((unsigned long)h_patch->kern_map +
+					elem->patch_offset);
+		} else {
+			unsigned long phys, new_pfn;
+			if (h_patch->heap_pgalloc) {
+				page = h_patch->pgalloc.pages[elem->patch_offset >> PAGE_SHIFT];
+				get_page(page);
+				phys = page_to_phys(page) + (elem->patch_offset & ~PAGE_MASK);
+			} else {
+				phys = h_patch->carveout.base + elem->patch_offset;
+			}
+			new_pfn = __phys_to_pfn(phys);
+			if (new_pfn != pfn) {
+				_nvmap_set_pte_at((unsigned long)pteaddr, new_pfn,
+						_nvmap_flag_to_pgprot(h_patch->flags, pgprot_kernel));
+				pfn = new_pfn;
+			}
+			patch_addr = (u32*)((unsigned long)pteaddr + (phys & ~PAGE_MASK));
+		}
+
+		*patch_addr = nvmap_get_physaddr(h_pin) + elem->pin_offset;
+
+		if (page)
+			put_page(page);
+	}
+	nvmap_unmap_pte(pteaddr);
+	*num_unique = unique_idx;
+	return 0;
+}
+
+void nvmap_unpin(struct nvmap_handle **h, int num_handles)
+{
+	int do_wake = 0;
+
+	while (num_handles--) {
+		BUG_ON(!*h);
+		do_wake |= _nvmap_handle_unpin(*h);
+		h++;
+	}
+
+	if (do_wake) wake_up(&nvmap_pin_wait);
+}
+
+int nvmap_validate_file(struct file *f)
+{
+	return (f->f_op==&knvmap_fops || f->f_op==&nvmap_fops) ? 0 : -EFAULT;
+}
diff --git a/drivers/video/tegra-fb.c b/drivers/video/tegra/tegra-fb.c
index 68e4506aad3e..68e4506aad3e 100644
--- a/drivers/video/tegra-fb.c
+++ b/drivers/video/tegra/tegra-fb.c