diff options
author | Krishna Reddy <vdumpa@nvidia.com> | 2012-03-12 00:13:24 -0700 |
---|---|---|
committer | Rohan Somvanshi <rsomvanshi@nvidia.com> | 2012-03-12 08:23:25 -0700 |
commit | b880eb85f32ff21590cbc814bb060a02b21ebf54 (patch) | |
tree | 45ffd5089f5ea50e6f157338f86c26e2c424b13c /drivers/video/tegra/nvmap | |
parent | 12f180aae4c69ffa764ba6b9d70700d4fa02d0a4 (diff) |
video: tegra: nvmap: optimize uc & wc allocations.
Changing page attributes and cache maintenance reduces
performance in applications doing runtime reallocations.
Keep pool of UC & WC pages to avoid expensive
operations when doing allocations.
bug 865816
(refactored initial changes from Kirill and added shrinker
notification handling)
Change-Id: I43206efb1adc750ded672bfe074e0648f2f9490b
Signed-off-by: Krishna Reddy <vdumpa@nvidia.com>
Reviewed-on: http://git-master/r/87532
Reviewed-by: Donghan Ryu <dryu@nvidia.com>
Reviewed-by: Kirill Artamonov <kartamonov@nvidia.com>
Reviewed-by: Rohan Somvanshi <rsomvanshi@nvidia.com>
Tested-by: Rohan Somvanshi <rsomvanshi@nvidia.com>
Diffstat (limited to 'drivers/video/tegra/nvmap')
-rw-r--r-- | drivers/video/tegra/nvmap/nvmap.h | 26 | ||||
-rw-r--r-- | drivers/video/tegra/nvmap/nvmap_dev.c | 13 | ||||
-rw-r--r-- | drivers/video/tegra/nvmap/nvmap_handle.c | 246 |
3 files changed, 271 insertions, 14 deletions
diff --git a/drivers/video/tegra/nvmap/nvmap.h b/drivers/video/tegra/nvmap/nvmap.h index 63b3471ec141..f7a732c38bda 100644 --- a/drivers/video/tegra/nvmap/nvmap.h +++ b/drivers/video/tegra/nvmap/nvmap.h @@ -3,7 +3,7 @@ * * GPU memory management driver for Tegra * - * Copyright (c) 2010-2011, NVIDIA Corporation. + * Copyright (c) 2010-2012, NVIDIA Corporation. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -86,10 +86,34 @@ struct nvmap_handle { struct mutex lock; }; +#define NVMAP_DEFAULT_PAGE_POOL_SIZE 8192 +#define NVMAP_NUM_POOLS 2 +#define NVMAP_UC_POOL 0 +#define NVMAP_WC_POOL 1 + +struct nvmap_page_pool { + spinlock_t lock; + int npages; + struct page **page_array; + int max_pages; +}; + +int nvmap_page_pool_init(struct nvmap_page_pool *pool, int flags); +struct page *nvmap_page_pool_alloc(struct nvmap_page_pool *pool); +bool nvmap_page_pool_release(struct nvmap_page_pool *pool, struct page *page); +int nvmap_page_pool_get_free_count(struct nvmap_page_pool *pool); + struct nvmap_share { struct tegra_iovmm_client *iovmm; wait_queue_head_t pin_wait; struct mutex pin_lock; + union { + struct nvmap_page_pool pools[NVMAP_NUM_POOLS]; + struct { + struct nvmap_page_pool uc_pool; + struct nvmap_page_pool wc_pool; + }; + }; #ifdef CONFIG_NVMAP_RECLAIM_UNPINNED_VM struct mutex mru_lock; struct list_head *mru_lists; diff --git a/drivers/video/tegra/nvmap/nvmap_dev.c b/drivers/video/tegra/nvmap/nvmap_dev.c index b38f04c9670a..7f7dcc9bfda2 100644 --- a/drivers/video/tegra/nvmap/nvmap_dev.c +++ b/drivers/video/tegra/nvmap/nvmap_dev.c @@ -3,7 +3,7 @@ * * User-space interface to nvmap * - * Copyright (c) 2011, NVIDIA Corporation. + * Copyright (c) 2011-2012, NVIDIA Corporation. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -1182,6 +1182,11 @@ static int nvmap_probe(struct platform_device *pdev) init_waitqueue_head(&dev->iovmm_master.pin_wait); mutex_init(&dev->iovmm_master.pin_lock); + nvmap_page_pool_init(&dev->iovmm_master.uc_pool, + NVMAP_HANDLE_UNCACHEABLE); + nvmap_page_pool_init(&dev->iovmm_master.wc_pool, + NVMAP_HANDLE_WRITE_COMBINE); + dev->iovmm_master.iovmm = tegra_iovmm_alloc_client(dev_name(&pdev->dev), NULL, &(dev->dev_user)); @@ -1308,6 +1313,12 @@ static int nvmap_probe(struct platform_device *pdev) dev, &debug_iovmm_clients_fops); debugfs_create_file("allocations", 0664, iovmm_root, dev, &debug_iovmm_allocations_fops); + debugfs_create_u32("uc_page_pool_npages", + S_IRUGO|S_IWUSR, iovmm_root, + &dev->iovmm_master.uc_pool.npages); + debugfs_create_u32("wc_page_pool_npages", + S_IRUGO|S_IWUSR, iovmm_root, + &dev->iovmm_master.wc_pool.npages); } } diff --git a/drivers/video/tegra/nvmap/nvmap_handle.c b/drivers/video/tegra/nvmap/nvmap_handle.c index b83a7896eeb3..53640ac58e42 100644 --- a/drivers/video/tegra/nvmap/nvmap_handle.c +++ b/drivers/video/tegra/nvmap/nvmap_handle.c @@ -3,7 +3,7 @@ * * Handle allocation and freeing routines for nvmap * - * Copyright (c) 2009-2011, NVIDIA Corporation. + * Copyright (c) 2009-2012, NVIDIA Corporation. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -38,6 +38,7 @@ #include <linux/vmstat.h> #include <linux/swap.h> +#include <linux/shrinker.h> #include "nvmap.h" #include "nvmap_mru.h" @@ -62,6 +63,181 @@ * preserve kmalloc space, if the array of pages exceeds PAGELIST_VMALLOC_MIN, * the array is allocated using vmalloc. */ #define PAGELIST_VMALLOC_MIN (PAGE_SIZE * 2) +#define NVMAP_TEST_PAGE_POOL_SHRINKER 0 + +static struct page *nvmap_alloc_pages_exact(gfp_t gfp, size_t size); + +static int nvmap_page_pool_shrink(struct shrinker *shrinker, + struct shrink_control *sc) +{ + int shrink_pages = sc->nr_to_scan; + int wc_free_pages, uc_free_pages; + struct nvmap_share *share = nvmap_get_share_from_dev(nvmap_dev); + int wc_pages_to_free = 0, uc_pages_to_free = 0; + struct page *page; + + pr_debug("%s: sh_pages=%d", __func__, shrink_pages); + shrink_pages = shrink_pages % 2 ? shrink_pages + 1 : shrink_pages; + wc_free_pages = nvmap_page_pool_get_free_count(&share->wc_pool); + uc_free_pages = nvmap_page_pool_get_free_count(&share->uc_pool); + + if (shrink_pages == 0) + return wc_free_pages + uc_free_pages; + + if (!(sc->gfp_mask & __GFP_WAIT)) + return -1; + + if (wc_free_pages >= uc_free_pages) { + wc_pages_to_free = wc_free_pages - uc_free_pages; + if (wc_pages_to_free >= shrink_pages) + wc_pages_to_free = shrink_pages; + else { + shrink_pages -= wc_pages_to_free; + wc_pages_to_free += shrink_pages / 2; + uc_pages_to_free = shrink_pages / 2; + } + } else { + uc_pages_to_free = uc_free_pages - wc_free_pages; + if (uc_pages_to_free >= shrink_pages) + uc_pages_to_free = shrink_pages; + else { + shrink_pages -= uc_pages_to_free; + uc_pages_to_free += shrink_pages / 2; + wc_pages_to_free = shrink_pages / 2; + } + } + + while (uc_pages_to_free--) { + page = nvmap_page_pool_alloc(&share->uc_pool); + if (!page) + break; + set_pages_array_wb(&page, 1); + __free_page(page); + } + + while (wc_pages_to_free--) { + page = nvmap_page_pool_alloc(&share->wc_pool); + if (!page) + break; + set_pages_array_wb(&page, 1); + __free_page(page); + } + + wc_free_pages = nvmap_page_pool_get_free_count(&share->wc_pool); + uc_free_pages = nvmap_page_pool_get_free_count(&share->uc_pool); + pr_debug("%s: free pages=%d", __func__, wc_free_pages+uc_free_pages); + return wc_free_pages + uc_free_pages; +} + +static struct shrinker nvmap_page_pool_shrinker = { + .shrink = nvmap_page_pool_shrink, + .seeks = 1, +}; + +#if NVMAP_TEST_PAGE_POOL_SHRINKER +static int shrink_state; +static int shrink_set(const char *arg, const struct kernel_param *kp) +{ + struct shrink_control sc; + + sc.nr_to_scan = 32768 * 2 - 1; + nvmap_page_pool_shrink(NULL, &sc); + shrink_state = 1; + return 0; +} + +static int shrink_get(char *buff, const struct kernel_param *kp) +{ + return param_get_int(buff, kp); +} + +static struct kernel_param_ops shrink_ops = { + .get = shrink_get, + .set = shrink_set, +}; + +module_param_cb(shrink, &shrink_ops, &shrink_state, 0644); +#endif +int nvmap_page_pool_init(struct nvmap_page_pool *pool, int flags) +{ + struct page *page; + int i; + static int reg = 1; + struct sysinfo info; + + si_meminfo(&info); + spin_lock_init(&pool->lock); + pool->npages = 0; + /* Use 1/4th of total ram for page pools. + * 1/8th for wc and 1/8th for uc. + */ + pool->max_pages = info.totalram >> 3; + if (pool->max_pages <= 0) + pool->max_pages = NVMAP_DEFAULT_PAGE_POOL_SIZE; + pr_info("nvmap %s page pool size=%d pages", + flags == NVMAP_HANDLE_UNCACHEABLE ? "uc" : "wc", + pool->max_pages); + pool->page_array = vmalloc(sizeof(void *) * pool->max_pages); + if (!pool->page_array) + return -ENOMEM; + + if (reg) { + reg = 0; + register_shrinker(&nvmap_page_pool_shrinker); + } + + for (i = 0; i < pool->max_pages; i++) { + page = nvmap_alloc_pages_exact(GFP_NVMAP, + PAGE_SIZE); + if (!page) + return 0; + if (flags == NVMAP_HANDLE_WRITE_COMBINE) + set_pages_array_wc(&page, 1); + else if (flags == NVMAP_HANDLE_UNCACHEABLE) + set_pages_array_uc(&page, 1); + if (!nvmap_page_pool_release(pool, page)) { + set_pages_array_wb(&page, 1); + __free_page(page); + return 0; + } + } + return 0; +} + +struct page *nvmap_page_pool_alloc(struct nvmap_page_pool *pool) +{ + struct page *page = NULL; + + spin_lock(&pool->lock); + if (pool->npages > 0) + page = pool->page_array[--pool->npages]; + spin_unlock(&pool->lock); + return page; +} + +bool nvmap_page_pool_release(struct nvmap_page_pool *pool, + struct page *page) +{ + int ret = false; + + spin_lock(&pool->lock); + if (pool->npages < pool->max_pages) { + pool->page_array[pool->npages++] = page; + ret = true; + } + spin_unlock(&pool->lock); + return ret; +} + +int nvmap_page_pool_get_free_count(struct nvmap_page_pool *pool) +{ + int count; + + spin_lock(&pool->lock); + count = pool->npages; + spin_unlock(&pool->lock); + return count; +} static inline void *altalloc(size_t len) { @@ -84,10 +260,11 @@ static inline void altfree(void *ptr, size_t len) void _nvmap_handle_free(struct nvmap_handle *h) { - struct nvmap_device *dev = h->dev; - unsigned int i, nr_page; + struct nvmap_share *share = nvmap_get_share_from_dev(h->dev); + unsigned int i, nr_page, page_index = 0; + struct nvmap_page_pool *pool = NULL; - if (nvmap_handle_remove(dev, h) != 0) + if (nvmap_handle_remove(h->dev, h) != 0) return; if (!h->alloc) @@ -104,18 +281,38 @@ void _nvmap_handle_free(struct nvmap_handle *h) BUG_ON(h->size & ~PAGE_MASK); BUG_ON(!h->pgalloc.pages); - nvmap_mru_remove(nvmap_get_share_from_dev(dev), h); + nvmap_mru_remove(share, h); + + /* Add to page pools, if necessary */ + if (h->flags == NVMAP_HANDLE_WRITE_COMBINE) + pool = &share->wc_pool; + else if (h->flags == NVMAP_HANDLE_UNCACHEABLE) + pool = &share->uc_pool; + + if (pool) { + while (page_index < nr_page) { + if (!nvmap_page_pool_release(pool, + h->pgalloc.pages[page_index])) + break; + page_index++; + } + } + + if (page_index == nr_page) + goto skip_attr_restore; /* Restore page attributes. */ if (h->flags == NVMAP_HANDLE_WRITE_COMBINE || h->flags == NVMAP_HANDLE_UNCACHEABLE || h->flags == NVMAP_HANDLE_INNER_CACHEABLE) - set_pages_array_wb(h->pgalloc.pages, nr_page); + set_pages_array_wb(&h->pgalloc.pages[page_index], + nr_page - page_index); +skip_attr_restore: if (h->pgalloc.area) tegra_iovmm_free_vm(h->pgalloc.area); - for (i = 0; i < nr_page; i++) + for (i = page_index; i < nr_page; i++) __free_page(h->pgalloc.pages[i]); altfree(h->pgalloc.pages, nr_page * sizeof(struct page *)); @@ -148,9 +345,10 @@ static int handle_page_alloc(struct nvmap_client *client, struct nvmap_handle *h, bool contiguous) { size_t size = PAGE_ALIGN(h->size); + struct nvmap_share *share = nvmap_get_share_from_dev(h->dev); unsigned int nr_page = size >> PAGE_SHIFT; pgprot_t prot; - unsigned int i = 0; + unsigned int i = 0, page_index = 0; struct page **pages; pages = altalloc(nr_page * sizeof(*pages)); @@ -171,6 +369,21 @@ static int handle_page_alloc(struct nvmap_client *client, } else { for (i = 0; i < nr_page; i++) { + pages[i] = NULL; + + /* Get pages from pool if there are any */ + if (h->flags == NVMAP_HANDLE_WRITE_COMBINE) + pages[i] = nvmap_page_pool_alloc( + &share->wc_pool); + else if (h->flags == NVMAP_HANDLE_UNCACHEABLE) + pages[i] = nvmap_page_pool_alloc( + &share->uc_pool); + + if (pages[i]) { + page_index++; + continue; + } + pages[i] = nvmap_alloc_pages_exact(GFP_NVMAP, PAGE_SIZE); if (!pages[i]) @@ -188,14 +401,21 @@ static int handle_page_alloc(struct nvmap_client *client, #endif } + if (nr_page == page_index) + goto skip_attr_change; + /* Update the pages mapping in kernel page table. */ if (h->flags == NVMAP_HANDLE_WRITE_COMBINE) - set_pages_array_wc(pages, nr_page); + set_pages_array_wc(&pages[page_index], + nr_page - page_index); else if (h->flags == NVMAP_HANDLE_UNCACHEABLE) - set_pages_array_uc(pages, nr_page); + set_pages_array_uc(&pages[page_index], + nr_page - page_index); else if (h->flags == NVMAP_HANDLE_INNER_CACHEABLE) - set_pages_array_iwb(pages, nr_page); + set_pages_array_iwb(&pages[page_index], + nr_page - page_index); +skip_attr_change: h->size = size; h->pgalloc.pages = pages; h->pgalloc.contig = contiguous; @@ -203,8 +423,10 @@ static int handle_page_alloc(struct nvmap_client *client, return 0; fail: - while (i--) + while (i--) { + set_pages_array_wb(&pages[i], 1); __free_page(pages[i]); + } altfree(pages, nr_page * sizeof(*pages)); wmb(); return -ENOMEM; |