arm: tegra: nvmap: Forcing to convert CarveOut requests to IOVM

Adding a build time CONFIG option to enable forcing of conversion
of non-IRAM CarveOut memory allocation requests to IOVM requests.
Default is "y" to force the conversion.
Each forced conversion is reported to console.
Allocation alignments larger than page size for IOVM are enabled.
Single page CarveOut allocations are converted to system memory.
CarveOut memory reservation has been removed for aruba, cardhu,
and enterprise.

Original-Change-Id: I3a598431d15b92ce853b3bec97be4b583d021264
Reviewed-on: http://git-master/r/29849
Reviewed-by: Varun Colbert <vcolbert@nvidia.com>
Tested-by: Varun Colbert <vcolbert@nvidia.com>

Rebase-Id: Reaf79d5ef57d3be8f2dc572de1919e854a117114

1 files changed, 85 insertions, 43 deletions

diff --git a/arch/arm/mach-tegra/iovmm.c b/arch/arm/mach-tegra/iovmm.c
index d492811e26b8..d5b49ca9087a 100644
--- a/arch/arm/mach-tegra/iovmm.c
+++ b/arch/arm/mach-tegra/iovmm.c
@@ -3,7 +3,7 @@
  *
  * Tegra I/O VM manager
  *
- * Copyright (c) 2010, NVIDIA Corporation.
+ * Copyright (c) 2010-2011, NVIDIA Corporation.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -53,6 +53,8 @@
 
 struct tegra_iovmm_block {
 	struct tegra_iovmm_area vm_area;
+	tegra_iovmm_addr_t	start;
+	size_t			length;
 	atomic_t		ref;
 	unsigned long		flags;
 	unsigned long		poison;
@@ -94,17 +96,16 @@ static tegra_iovmm_addr_t iovmm_align_down(struct tegra_iovmm_device *dev,
 
 static void tegra_iovmm_block_stats(struct tegra_iovmm_domain *domain,
 	unsigned int *num_blocks, unsigned int *num_free,
-	tegra_iovmm_addr_t *total, tegra_iovmm_addr_t *total_free,
-	tegra_iovmm_addr_t *max_free)
+	tegra_iovmm_addr_t *total, size_t *total_free, size_t *max_free)
 {
 	struct rb_node *n;
 	struct tegra_iovmm_block *b;
 
 	*num_blocks = 0;
 	*num_free = 0;
-	*total = (tegra_iovmm_addr_t)0;
-	*total_free = (tegra_iovmm_addr_t)0;
-	*max_free = (tegra_iovmm_addr_t)0;
+	*total = 0;
+	*total_free = 0;
+	*max_free = 0;
 
 	spin_lock(&domain->block_lock);
 	n = rb_first(&domain->all_blocks);
@@ -112,12 +113,11 @@ static void tegra_iovmm_block_stats(struct tegra_iovmm_domain *domain,
 		b = rb_entry(n, struct tegra_iovmm_block, all_node);
 		n = rb_next(n);
 		(*num_blocks)++;
-		(*total) += iovmm_length(b);
+		*total += b->length;
 		if (test_bit(BK_free, &b->flags)) {
 			(*num_free)++;
-			(*total_free) += iovmm_length(b);
-			(*max_free) = max_t(tegra_iovmm_addr_t,
-				(*max_free), iovmm_length(b));
+			*total_free += b->length;
+			*max_free = max_t(size_t, *max_free, b->length);
 		}
 	}
 	spin_unlock(&domain->block_lock);
@@ -127,7 +127,7 @@ static int tegra_iovmm_read_proc(char *page, char **start, off_t off,
 	int count, int *eof, void *data)
 {
 	struct iovmm_share_group *grp;
-	tegra_iovmm_addr_t max_free, total_free, total;
+	size_t max_free, total_free, total;
 	unsigned int num, num_free;
 
 	int len = 0;
@@ -190,8 +190,8 @@ static void iovmm_free_block(struct tegra_iovmm_domain *domain,
 	if (succ) succ_free = test_bit(BK_free, &succ->flags);
 
 	if (pred_free && succ_free) {
-		iovmm_length(pred) += iovmm_length(block);
-		iovmm_length(pred) += iovmm_length(succ);
+		pred->length += block->length;
+		pred->length += succ->length;
 		rb_erase(&block->all_node, &domain->all_blocks);
 		rb_erase(&succ->all_node, &domain->all_blocks);
 		rb_erase(&succ->free_node, &domain->free_blocks);
@@ -200,13 +200,13 @@ static void iovmm_free_block(struct tegra_iovmm_domain *domain,
 		iovmm_block_put(succ);
 		block = pred;
 	} else if (pred_free) {
-		iovmm_length(pred) += iovmm_length(block);
+		pred->length += block->length;
 		rb_erase(&block->all_node, &domain->all_blocks);
 		rb_erase(&pred->free_node, &domain->free_blocks);
 		iovmm_block_put(block);
 		block = pred;
 	} else if (succ_free) {
-		iovmm_length(block) += iovmm_length(succ);
+		block->length += succ->length;
 		rb_erase(&succ->all_node, &domain->all_blocks);
 		rb_erase(&succ->free_node, &domain->free_blocks);
 		iovmm_block_put(succ);
@@ -217,7 +217,7 @@ static void iovmm_free_block(struct tegra_iovmm_domain *domain,
 		struct tegra_iovmm_block *b;
 		parent = *p;
 		b = rb_entry(parent, struct tegra_iovmm_block, free_node);
-		if (iovmm_length(block) >= iovmm_length(b))
+		if (block->length >= b->length)
 			p = &parent->rb_right;
 		else
 			p = &parent->rb_left;
@@ -232,7 +232,8 @@ static void iovmm_free_block(struct tegra_iovmm_domain *domain,
  * block will be created and inserted into the free list in its place.
  * since all free blocks are stored in two trees the new block needs to be
  * linked into both. */
-static void iovmm_split_free_block(struct tegra_iovmm_domain *domain,
+static struct tegra_iovmm_block *iovmm_split_free_block(
+	struct tegra_iovmm_domain *domain,
 	struct tegra_iovmm_block *block, unsigned long size)
 {
 	struct rb_node **p;
@@ -241,20 +242,21 @@ static void iovmm_split_free_block(struct tegra_iovmm_domain *domain,
 	struct tegra_iovmm_block *b;
 
 	rem = kmem_cache_zalloc(iovmm_cache, GFP_KERNEL);
-	if (!rem) return;
+	if (!rem)
+		return NULL;
 
 	spin_lock(&domain->block_lock);
 	p = &domain->free_blocks.rb_node;
 
-	iovmm_start(rem) = iovmm_start(block) + size;
-	iovmm_length(rem) = iovmm_length(block) - size;
+	rem->start  = block->start + size;
+	rem->length = block->length - size;
 	atomic_set(&rem->ref, 1);
-	iovmm_length(block) = size;
+	block->length = size;
 
 	while (*p) {
 		parent = *p;
 		b = rb_entry(parent, struct tegra_iovmm_block, free_node);
-		if (iovmm_length(rem) >= iovmm_length(b))
+		if (rem->length >= b->length)
 			p = &parent->rb_right;
 		else
 			p = &parent->rb_left;
@@ -268,24 +270,32 @@ static void iovmm_split_free_block(struct tegra_iovmm_domain *domain,
 	while (*p) {
 		parent = *p;
 		b = rb_entry(parent, struct tegra_iovmm_block, all_node);
-		if (iovmm_start(rem) >= iovmm_start(b))
+		if (rem->start >= b->start)
 			p = &parent->rb_right;
 		else
 			p = &parent->rb_left;
 	}
 	rb_link_node(&rem->all_node, parent, p);
 	rb_insert_color(&rem->all_node, &domain->all_blocks);
+
+	return rem;
 }
 
 static struct tegra_iovmm_block *iovmm_alloc_block(
-	struct tegra_iovmm_domain *domain, unsigned long size)
+	struct tegra_iovmm_domain *domain, size_t size, size_t align)
 {
+#define SIMALIGN(b, a)	(((b)->start%(a)) ? ((a)-((b)->start%(a))) : 0)
+#define NO_SPLIT(m)	((m) < MIN_SPLIT_BYTES(domain))
+#define DO_SPLIT(m)	((m) >= MIN_SPLIT_BYTES(domain))
+
 	struct rb_node *n;
 	struct tegra_iovmm_block *b, *best;
+	size_t simalign;
 	static int splitting = 0;
 
 	BUG_ON(!size);
 	size = iovmm_align_up(domain->dev, size);
+	align = iovmm_align_up(domain->dev, align);
 	for (;;) {
 		spin_lock(&domain->block_lock);
 		if (!splitting)
@@ -296,31 +306,60 @@ static struct tegra_iovmm_block *iovmm_alloc_block(
 	n = domain->free_blocks.rb_node;
 	best = NULL;
 	while (n) {
+		tegra_iovmm_addr_t aligned_start, block_ceil;
+
 		b = rb_entry(n, struct tegra_iovmm_block, free_node);
-		if (iovmm_length(b) < size) {
-			n = n->rb_right;
-		} else if (iovmm_length(b) == size) {
-			best = b;
-			break;
-		} else {
+		simalign = SIMALIGN(b, align);
+		aligned_start = b->start + simalign;
+		block_ceil = b->start + b->length;
+
+		if (block_ceil >= aligned_start + size) {
+			/* Block has enough size */
 			best = b;
+			if (NO_SPLIT(simalign) &&
+				NO_SPLIT(block_ceil - (aligned_start + size)))
+				break;
 			n = n->rb_left;
+		} else {
+			n = n->rb_right;
 		}
 	}
 	if (!best) {
 		spin_unlock(&domain->block_lock);
 		return NULL;
 	}
+
+	simalign = SIMALIGN(best, align);
+	if (DO_SPLIT(simalign)) {
+		splitting = 1;
+		spin_unlock(&domain->block_lock);
+
+		/* Split off misalignment */
+		b = best;
+		best = iovmm_split_free_block(domain, b, simalign);
+		if (best)
+			simalign = 0;
+		else
+			best = b;
+	}
+
+	/* Unfree designed block */
 	rb_erase(&best->free_node, &domain->free_blocks);
 	clear_bit(BK_free, &best->flags);
 	atomic_inc(&best->ref);
-	if (iovmm_length(best) >= size+MIN_SPLIT_BYTES(domain)) {
+
+	iovmm_start(best) = best->start + simalign;
+	iovmm_length(best) = size;
+
+	if (DO_SPLIT((best->start + best->length) - iovmm_end(best))) {
 		splitting = 1;
 		spin_unlock(&domain->block_lock);
-		iovmm_split_free_block(domain, best, size);
-		splitting = 0;
+
+		/* Split off excess */
+		(void)iovmm_split_free_block(domain, best, size + simalign);
 	}
 
+	splitting = 0;
 	spin_unlock(&domain->block_lock);
 
 	return best;
@@ -342,8 +381,8 @@ int tegra_iovmm_domain_init(struct tegra_iovmm_domain *domain,
 	spin_lock_init(&domain->block_lock);
 	init_rwsem(&domain->map_lock);
 	init_waitqueue_head(&domain->delay_lock);
-	iovmm_start(b) = iovmm_align_up(dev, start);
-	iovmm_length(b) = iovmm_align_down(dev, end) - iovmm_start(b);
+	b->start  = iovmm_align_up(dev, start);
+	b->length = iovmm_align_down(dev, end) - b->start;
 	set_bit(BK_free, &b->flags);
 	rb_link_node(&b->free_node, NULL, &domain->free_blocks.rb_node);
 	rb_insert_color(&b->free_node, &domain->free_blocks);
@@ -354,7 +393,7 @@ int tegra_iovmm_domain_init(struct tegra_iovmm_domain *domain,
 
 struct tegra_iovmm_area *tegra_iovmm_create_vm(
 	struct tegra_iovmm_client *client, struct tegra_iovmm_area_ops *ops,
-	unsigned long size, pgprot_t pgprot)
+	size_t size, size_t align, pgprot_t pgprot)
 {
 	struct tegra_iovmm_block *b;
 	struct tegra_iovmm_domain *domain;
@@ -363,7 +402,7 @@ struct tegra_iovmm_area *tegra_iovmm_create_vm(
 
 	domain = client->domain;
 
-	b = iovmm_alloc_block(domain, size);
+	b = iovmm_alloc_block(domain, size, align);
 	if (!b) return NULL;
 
 	b->vm_area.domain = domain;
@@ -479,8 +518,9 @@ struct tegra_iovmm_area *tegra_iovmm_find_area_get(
 
 	while (n) {
 		b = rb_entry(n, struct tegra_iovmm_block, all_node);
-		if ((iovmm_start(b) <= addr) && (iovmm_end(b) >= addr)) {
-			if (test_bit(BK_free, &b->flags)) b = NULL;
+		if (iovmm_start(b) <= addr && addr <= iovmm_end(b)) {
+			if (test_bit(BK_free, &b->flags))
+				b = NULL;
 			break;
 		}
 		if (addr > iovmm_start(b))
@@ -489,9 +529,11 @@ struct tegra_iovmm_area *tegra_iovmm_find_area_get(
 			n = n->rb_left;
 		b = NULL;
 	}
-	if (b) atomic_inc(&b->ref);
+	if (b)
+		atomic_inc(&b->ref);
 	spin_unlock(&client->domain->block_lock);
-	if (!b) return NULL;
+	if (!b)
+		return NULL;
 	return &b->vm_area;
 }
 
@@ -606,7 +648,7 @@ size_t tegra_iovmm_get_vm_size(struct tegra_iovmm_client *client)
 	while (n) {
 		b = rb_entry(n, struct tegra_iovmm_block, all_node);
 		n = rb_next(n);
-		size += iovmm_length(b);
+		size += b->length;
 	}
 	spin_unlock(&domain->block_lock);
 
@@ -732,7 +774,7 @@ int tegra_iovmm_register(struct tegra_iovmm_device *dev)
 	}
 	list_add_tail(&dev->list, &iovmm_devices);
 	mutex_unlock(&iovmm_group_list_lock);
-	printk("%s: added %s\n", __func__, dev->name);
+	pr_info("%s: added %s\n", __func__, dev->name);
 	return 0;
 }