[PATCH 06/13] drm/ttm: convert page allocation to use page ptr array instead of list V4
j.glisse at gmail.com
j.glisse at gmail.com
Wed Nov 9 12:22:23 PST 2011
From: Jerome Glisse <jglisse at redhat.com>
Use the ttm_tt page ptr array for page allocation, move the list to
array unwinding into the page allocation functions.
V2 split the fix to use ttm put page as a separate fix
properly fill pages array when TTM_PAGE_FLAG_ZERO_ALLOC is not
set
V3 Added back page_count()==1 check when freeing page
V4 Rebase on top of memory accounting overhaul
Signed-off-by: Jerome Glisse <jglisse at redhat.com>
---
drivers/gpu/drm/ttm/ttm_memory.c | 47 +++++++++++------
drivers/gpu/drm/ttm/ttm_page_alloc.c | 90 ++++++++++++++++++++--------------
drivers/gpu/drm/ttm/ttm_tt.c | 68 ++++++++------------------
include/drm/ttm/ttm_memory.h | 13 +++--
include/drm/ttm/ttm_page_alloc.h | 17 +++---
5 files changed, 120 insertions(+), 115 deletions(-)
diff --git a/drivers/gpu/drm/ttm/ttm_memory.c b/drivers/gpu/drm/ttm/ttm_memory.c
index b550baf..98f6899 100644
--- a/drivers/gpu/drm/ttm/ttm_memory.c
+++ b/drivers/gpu/drm/ttm/ttm_memory.c
@@ -326,32 +326,45 @@ int ttm_mem_global_alloc(struct ttm_mem_global *glob,
}
EXPORT_SYMBOL(ttm_mem_global_alloc);
-int ttm_mem_global_alloc_page(struct ttm_mem_global *glob,
- struct page *page,
- bool no_wait)
+int ttm_mem_global_alloc_pages(struct ttm_mem_global *glob,
+ unsigned npages,
+ bool no_wait)
{
-
- if (ttm_mem_global_alloc(glob, PAGE_SIZE, no_wait))
+ if (ttm_mem_global_alloc(glob, PAGE_SIZE * npages, no_wait))
return -ENOMEM;
+ ttm_check_swapping(glob);
+ return 0;
+}
+
+void ttm_mem_global_account_pages(struct ttm_mem_global *glob,
+ struct page **pages,
+ unsigned npages)
+{
+ unsigned i;
/* check if page is dma32 */
- if (page_to_pfn(page) > 0x00100000UL) {
- spin_lock(&glob->lock);
- glob->used_mem -= PAGE_SIZE;
- glob->used_dma32_mem += PAGE_SIZE;
- spin_unlock(&glob->lock);
+ spin_lock(&glob->lock);
+ for (i = 0; i < npages; i++) {
+ if (page_to_pfn(pages[i]) > 0x00100000UL) {
+ glob->used_mem -= PAGE_SIZE;
+ glob->used_dma32_mem += PAGE_SIZE;
+ }
}
- ttm_check_swapping(glob);
- return 0;
+ spin_unlock(&glob->lock);
}
-void ttm_mem_global_free_page(struct ttm_mem_global *glob, struct page *page)
+void ttm_mem_global_free_pages(struct ttm_mem_global *glob,
+ struct page **pages, unsigned npages)
{
+ unsigned i;
+
spin_lock(&glob->lock);
- if (page_to_pfn(page) > 0x00100000UL) {
- glob->used_dma32_mem -= PAGE_SIZE;
- } else {
- glob->used_mem -= PAGE_SIZE;
+ for (i = 0; i < npages; i++) {
+ if (page_to_pfn(pages[i]) > 0x00100000UL) {
+ glob->used_dma32_mem -= PAGE_SIZE;
+ } else {
+ glob->used_mem -= PAGE_SIZE;
+ }
}
spin_unlock(&glob->lock);
}
diff --git a/drivers/gpu/drm/ttm/ttm_page_alloc.c b/drivers/gpu/drm/ttm/ttm_page_alloc.c
index 727e93d..c4f18b9 100644
--- a/drivers/gpu/drm/ttm/ttm_page_alloc.c
+++ b/drivers/gpu/drm/ttm/ttm_page_alloc.c
@@ -619,8 +619,10 @@ static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool,
* @return count of pages still required to fulfill the request.
*/
static unsigned ttm_page_pool_get_pages(struct ttm_page_pool *pool,
- struct list_head *pages, int ttm_flags,
- enum ttm_caching_state cstate, unsigned count)
+ struct list_head *pages,
+ int ttm_flags,
+ enum ttm_caching_state cstate,
+ unsigned count)
{
unsigned long irq_flags;
struct list_head *p;
@@ -664,13 +666,14 @@ out:
* On success pages list will hold count number of correctly
* cached pages.
*/
-int ttm_get_pages(struct list_head *pages, int flags,
- enum ttm_caching_state cstate, unsigned count,
- dma_addr_t *dma_address)
+int ttm_get_pages(struct page **pages, unsigned npages, int flags,
+ enum ttm_caching_state cstate, dma_addr_t *dma_address)
{
struct ttm_page_pool *pool = ttm_get_pool(flags, cstate);
struct page *p = NULL;
+ struct list_head plist;
gfp_t gfp_flags = GFP_USER;
+ unsigned count = 0;
int r;
/* set zero flag for page allocation if required */
@@ -684,94 +687,107 @@ int ttm_get_pages(struct list_head *pages, int flags,
else
gfp_flags |= GFP_HIGHUSER;
- for (r = 0; r < count; ++r) {
- p = alloc_page(gfp_flags);
- if (!p) {
-
+ for (count = 0; count < npages; ++count) {
+ pages[count] = alloc_page(gfp_flags);
+ if (pages[count] == NULL) {
printk(KERN_ERR TTM_PFX
"Unable to allocate page.");
return -ENOMEM;
}
-
- list_add(&p->lru, pages);
}
return 0;
}
-
/* combine zero flag to pool flags */
gfp_flags |= pool->gfp_flags;
/* First we take pages from the pool */
- count = ttm_page_pool_get_pages(pool, pages, flags, cstate, count);
+ INIT_LIST_HEAD(&plist);
+ npages = ttm_page_pool_get_pages(pool, &plist, flags, cstate, npages);
/* clear the pages coming from the pool if requested */
+ count = 0;
+ list_for_each_entry(p, &plist, lru) {
+ pages[count++] = p;
+ }
if (flags & TTM_PAGE_FLAG_ZERO_ALLOC) {
- list_for_each_entry(p, pages, lru) {
+ list_for_each_entry(p, &plist, lru) {
clear_page(page_address(p));
}
}
/* If pool didn't have enough pages allocate new one. */
- if (count > 0) {
+ if (npages > 0) {
/* ttm_alloc_new_pages doesn't reference pool so we can run
* multiple requests in parallel.
**/
- r = ttm_alloc_new_pages(pages, gfp_flags, flags, cstate, count);
+ INIT_LIST_HEAD(&plist);
+ r = ttm_alloc_new_pages(&plist, gfp_flags, flags, cstate, npages);
+ list_for_each_entry(p, &plist, lru) {
+ pages[count++] = p;
+ }
if (r) {
/* If there is any pages in the list put them back to
* the pool. */
printk(KERN_ERR TTM_PFX
"Failed to allocate extra pages "
"for large request.");
- ttm_put_pages(pages, 0, flags, cstate, NULL);
+ ttm_put_pages(pages, count, flags, cstate, NULL);
return r;
}
}
-
return 0;
}
/* Put all pages in pages list to correct pool to wait for reuse */
-void ttm_put_pages(struct list_head *pages, unsigned page_count, int flags,
+void ttm_put_pages(struct page **pages, unsigned npages, int flags,
enum ttm_caching_state cstate, dma_addr_t *dma_address)
{
unsigned long irq_flags;
struct ttm_page_pool *pool = ttm_get_pool(flags, cstate);
- struct page *p, *tmp;
+ unsigned i;
if (pool == NULL) {
/* No pool for this memory type so free the pages */
- list_for_each_entry_safe(p, tmp, pages, lru) {
- __free_page(p);
+ for (i = 0; i < npages; i++) {
+ if (pages[i]) {
+ if (page_count(pages[i]) != 1)
+ printk(KERN_ERR TTM_PFX
+ "Erroneous page count. "
+ "Leaking pages.\n");
+ __free_page(pages[i]);
+ pages[i] = NULL;
+ }
}
- /* Make the pages list empty */
- INIT_LIST_HEAD(pages);
return;
}
- if (page_count == 0) {
- list_for_each_entry_safe(p, tmp, pages, lru) {
- ++page_count;
- }
- }
spin_lock_irqsave(&pool->lock, irq_flags);
- list_splice_init(pages, &pool->list);
- pool->npages += page_count;
+ for (i = 0; i < npages; i++) {
+ if (pages[i]) {
+ if (page_count(pages[i]) != 1)
+ printk(KERN_ERR TTM_PFX
+ "Erroneous page count. "
+ "Leaking pages.\n");
+ list_add_tail(&pages[i]->lru, &pool->list);
+ pages[i] = NULL;
+ pool->npages++;
+ }
+ }
/* Check that we don't go over the pool limit */
- page_count = 0;
+ npages = 0;
if (pool->npages > _manager->options.max_size) {
- page_count = pool->npages - _manager->options.max_size;
+ npages = pool->npages - _manager->options.max_size;
/* free at least NUM_PAGES_TO_ALLOC number of pages
* to reduce calls to set_memory_wb */
- if (page_count < NUM_PAGES_TO_ALLOC)
- page_count = NUM_PAGES_TO_ALLOC;
+ if (npages < NUM_PAGES_TO_ALLOC)
+ npages = NUM_PAGES_TO_ALLOC;
}
spin_unlock_irqrestore(&pool->lock, irq_flags);
- if (page_count)
- ttm_page_pool_free(pool, page_count);
+ if (npages)
+ ttm_page_pool_free(pool, npages);
}
static void ttm_page_pool_init_locked(struct ttm_page_pool *pool, int flags,
diff --git a/drivers/gpu/drm/ttm/ttm_tt.c b/drivers/gpu/drm/ttm/ttm_tt.c
index e5432efa..303bbba 100644
--- a/drivers/gpu/drm/ttm/ttm_tt.c
+++ b/drivers/gpu/drm/ttm/ttm_tt.c
@@ -65,35 +65,23 @@ static void ttm_tt_free_page_directory(struct ttm_tt *ttm)
static struct page *__ttm_tt_get_page(struct ttm_tt *ttm, int index)
{
struct page *p;
- struct list_head h;
struct ttm_mem_global *mem_glob = ttm->glob->mem_glob;
int ret;
if (NULL == (p = ttm->pages[index])) {
+ ret = ttm_mem_global_alloc_pages(mem_glob, 1, false);
+ if (unlikely(ret != 0))
+ return NULL;
- INIT_LIST_HEAD(&h);
-
- ret = ttm_get_pages(&h, ttm->page_flags, ttm->caching_state, 1,
+ ret = ttm_get_pages(&ttm->pages[index], 1, ttm->page_flags,
+ ttm->caching_state,
&ttm->dma_address[index]);
-
if (ret != 0)
return NULL;
- p = list_first_entry(&h, struct page, lru);
-
- ret = ttm_mem_global_alloc_page(mem_glob, p, false);
- if (unlikely(ret != 0))
- goto out_err;
-
- ttm->pages[index] = p;
+ ttm_mem_global_account_pages(mem_glob, &ttm->pages[index], 1);
}
return p;
-out_err:
- INIT_LIST_HEAD(&h);
- list_add(&p->lru, &h);
- ttm_put_pages(&h, 1, ttm->page_flags,
- ttm->caching_state, &ttm->dma_address[index]);
- return NULL;
}
struct page *ttm_tt_get_page(struct ttm_tt *ttm, int index)
@@ -110,8 +98,7 @@ struct page *ttm_tt_get_page(struct ttm_tt *ttm, int index)
int ttm_tt_populate(struct ttm_tt *ttm)
{
- struct page *page;
- unsigned long i;
+ struct ttm_mem_global *mem_glob = ttm->glob->mem_glob;
struct ttm_backend *be;
int ret;
@@ -126,11 +113,16 @@ int ttm_tt_populate(struct ttm_tt *ttm)
be = ttm->be;
- for (i = 0; i < ttm->num_pages; ++i) {
- page = __ttm_tt_get_page(ttm, i);
- if (!page)
- return -ENOMEM;
- }
+ ret = ttm_mem_global_alloc_pages(mem_glob, ttm->num_pages, false);
+ if (unlikely(ret != 0))
+ return -ENOMEM;
+
+ ret = ttm_get_pages(ttm->pages, ttm->num_pages, ttm->page_flags,
+ ttm->caching_state, ttm->dma_address);
+ if (ret != 0)
+ return -ENOMEM;
+
+ ttm_mem_global_account_pages(mem_glob, ttm->pages, ttm->num_pages);
be->func->populate(be, ttm->num_pages, ttm->pages,
ttm->dummy_read_page, ttm->dma_address);
@@ -242,33 +234,15 @@ EXPORT_SYMBOL(ttm_tt_set_placement_caching);
static void ttm_tt_free_alloced_pages(struct ttm_tt *ttm)
{
- int i;
- unsigned count = 0;
- struct list_head h;
- struct page *cur_page;
struct ttm_backend *be = ttm->be;
-
- INIT_LIST_HEAD(&h);
+ struct ttm_mem_global *glob = ttm->glob->mem_glob;
if (be)
be->func->clear(be);
- for (i = 0; i < ttm->num_pages; ++i) {
- cur_page = ttm->pages[i];
- ttm->pages[i] = NULL;
- if (cur_page) {
- if (page_count(cur_page) != 1)
- printk(KERN_ERR TTM_PFX
- "Erroneous page count. "
- "Leaking pages.\n");
- ttm_mem_global_free_page(ttm->glob->mem_glob,
- cur_page);
- list_add(&cur_page->lru, &h);
- count++;
- }
- }
- ttm_put_pages(&h, count, ttm->page_flags, ttm->caching_state,
- ttm->dma_address);
+ ttm_mem_global_free_pages(glob, ttm->pages, ttm->num_pages);
+ ttm_put_pages(ttm->pages, ttm->num_pages, ttm->page_flags,
+ ttm->caching_state, ttm->dma_address);
ttm->state = tt_unpopulated;
}
diff --git a/include/drm/ttm/ttm_memory.h b/include/drm/ttm/ttm_memory.h
index d24792b..c0c732d 100644
--- a/include/drm/ttm/ttm_memory.h
+++ b/include/drm/ttm/ttm_memory.h
@@ -150,11 +150,14 @@ extern int ttm_mem_global_alloc(struct ttm_mem_global *glob,
bool no_wait);
extern void ttm_mem_global_free(struct ttm_mem_global *glob,
uint64_t amount);
-extern int ttm_mem_global_alloc_page(struct ttm_mem_global *glob,
- struct page *page,
- bool no_wait);
-extern void ttm_mem_global_free_page(struct ttm_mem_global *glob,
- struct page *page);
+extern int ttm_mem_global_alloc_pages(struct ttm_mem_global *glob,
+ unsigned npages,
+ bool no_wait);
+extern void ttm_mem_global_free_pages(struct ttm_mem_global *glob,
+ struct page **pages, unsigned npages);
+extern void ttm_mem_global_account_pages(struct ttm_mem_global *glob,
+ struct page **pages,
+ unsigned npages);
extern size_t ttm_round_pot(size_t size);
#endif
diff --git a/include/drm/ttm/ttm_page_alloc.h b/include/drm/ttm/ttm_page_alloc.h
index 129de12..fffb3bd 100644
--- a/include/drm/ttm/ttm_page_alloc.h
+++ b/include/drm/ttm/ttm_page_alloc.h
@@ -32,29 +32,28 @@
/**
* Get count number of pages from pool to pages list.
*
- * @pages: head of empty linked list where pages are filled.
+ * @pages: array of pages ptr
+ * @npages: number of pages to allocate.
* @flags: ttm flags for page allocation.
* @cstate: ttm caching state for the page.
- * @count: number of pages to allocate.
* @dma_address: The DMA (bus) address of pages (if TTM_PAGE_FLAG_DMA32 set).
*/
-int ttm_get_pages(struct list_head *pages,
+int ttm_get_pages(struct page **pages,
+ unsigned npages,
int flags,
enum ttm_caching_state cstate,
- unsigned count,
dma_addr_t *dma_address);
/**
* Put linked list of pages to pool.
*
- * @pages: list of pages to free.
- * @page_count: number of pages in the list. Zero can be passed for unknown
- * count.
+ * @pages: array of pages ptr
+ * @npages: number of pages to free.
* @flags: ttm flags for page allocation.
* @cstate: ttm caching state.
* @dma_address: The DMA (bus) address of pages (if TTM_PAGE_FLAG_DMA32 set).
*/
-void ttm_put_pages(struct list_head *pages,
- unsigned page_count,
+void ttm_put_pages(struct page **pages,
+ unsigned npages,
int flags,
enum ttm_caching_state cstate,
dma_addr_t *dma_address);
--
1.7.7.1
More information about the dri-devel
mailing list