[PATCH 4/4] drm/ttm: add transparent huge page support for wc or uc allocations v2

Alex Deucher alexdeucher at gmail.com
Tue Oct 10 21:47:06 UTC 2017


On Tue, Oct 10, 2017 at 4:53 AM, Christian König
<ckoenig.leichtzumerken at gmail.com> wrote:
> From: Christian König <christian.koenig at amd.com>
>
> Add a new huge page pool and try to allocate from it when it makes sense.
>
> v2: avoid compound pages for now
>
> Signed-off-by: Christian König <christian.koenig at amd.com>
> Acked-by: Alex Deucher <alexander.deucher at amd.com>

Series is:
Acked-by: Alex Deucher <alexander.deucher at amd.com>

> ---
>  drivers/gpu/drm/ttm/ttm_page_alloc.c | 136 ++++++++++++++++++++++++++++-------
>  1 file changed, 109 insertions(+), 27 deletions(-)
>
> diff --git a/drivers/gpu/drm/ttm/ttm_page_alloc.c b/drivers/gpu/drm/ttm/ttm_page_alloc.c
> index 3974732..b6f16e7ff 100644
> --- a/drivers/gpu/drm/ttm/ttm_page_alloc.c
> +++ b/drivers/gpu/drm/ttm/ttm_page_alloc.c
> @@ -95,7 +95,7 @@ struct ttm_pool_opts {
>         unsigned        small;
>  };
>
> -#define NUM_POOLS 4
> +#define NUM_POOLS 6
>
>  /**
>   * struct ttm_pool_manager - Holds memory pools for fst allocation
> @@ -122,6 +122,8 @@ struct ttm_pool_manager {
>                         struct ttm_page_pool    uc_pool;
>                         struct ttm_page_pool    wc_pool_dma32;
>                         struct ttm_page_pool    uc_pool_dma32;
> +                       struct ttm_page_pool    wc_pool_huge;
> +                       struct ttm_page_pool    uc_pool_huge;
>                 } ;
>         };
>  };
> @@ -256,8 +258,8 @@ static int set_pages_array_uc(struct page **pages, int addrinarray)
>
>  /**
>   * Select the right pool or requested caching state and ttm flags. */
> -static struct ttm_page_pool *ttm_get_pool(int flags,
> -               enum ttm_caching_state cstate)
> +static struct ttm_page_pool *ttm_get_pool(int flags, bool huge,
> +                                         enum ttm_caching_state cstate)
>  {
>         int pool_index;
>
> @@ -269,9 +271,15 @@ static struct ttm_page_pool *ttm_get_pool(int flags,
>         else
>                 pool_index = 0x1;
>
> -       if (flags & TTM_PAGE_FLAG_DMA32)
> +       if (flags & TTM_PAGE_FLAG_DMA32) {
> +               if (huge)
> +                       return NULL;
>                 pool_index |= 0x2;
>
> +       } else if (huge) {
> +               pool_index |= 0x4;
> +       }
> +
>         return &_manager->pools[pool_index];
>  }
>
> @@ -494,12 +502,14 @@ static void ttm_handle_caching_state_failure(struct list_head *pages,
>   * pages returned in pages array.
>   */
>  static int ttm_alloc_new_pages(struct list_head *pages, gfp_t gfp_flags,
> -               int ttm_flags, enum ttm_caching_state cstate, unsigned count)
> +                              int ttm_flags, enum ttm_caching_state cstate,
> +                              unsigned count, unsigned order)
>  {
>         struct page **caching_array;
>         struct page *p;
>         int r = 0;
> -       unsigned i, cpages;
> +       unsigned i, j, cpages;
> +       unsigned npages = 1 << order;
>         unsigned max_cpages = min(count,
>                         (unsigned)(PAGE_SIZE/sizeof(struct page *)));
>
> @@ -512,7 +522,7 @@ static int ttm_alloc_new_pages(struct list_head *pages, gfp_t gfp_flags,
>         }
>
>         for (i = 0, cpages = 0; i < count; ++i) {
> -               p = alloc_page(gfp_flags);
> +               p = alloc_pages(gfp_flags, order);
>
>                 if (!p) {
>                         pr_err("Unable to get page %u\n", i);
> @@ -531,14 +541,18 @@ static int ttm_alloc_new_pages(struct list_head *pages, gfp_t gfp_flags,
>                         goto out;
>                 }
>
> +               list_add(&p->lru, pages);
> +
>  #ifdef CONFIG_HIGHMEM
>                 /* gfp flags of highmem page should never be dma32 so we
>                  * we should be fine in such case
>                  */
> -               if (!PageHighMem(p))
> +               if (PageHighMem(p))
> +                       continue;
> +
>  #endif
> -               {
> -                       caching_array[cpages++] = p;
> +               for (j = 0; j < npages; ++j) {
> +                       caching_array[cpages++] = p++;
>                         if (cpages == max_cpages) {
>
>                                 r = ttm_set_pages_caching(caching_array,
> @@ -552,8 +566,6 @@ static int ttm_alloc_new_pages(struct list_head *pages, gfp_t gfp_flags,
>                                 cpages = 0;
>                         }
>                 }
> -
> -               list_add(&p->lru, pages);
>         }
>
>         if (cpages) {
> @@ -573,9 +585,9 @@ static int ttm_alloc_new_pages(struct list_head *pages, gfp_t gfp_flags,
>   * Fill the given pool if there aren't enough pages and the requested number of
>   * pages is small.
>   */
> -static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool,
> -               int ttm_flags, enum ttm_caching_state cstate, unsigned count,
> -               unsigned long *irq_flags)
> +static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool, int ttm_flags,
> +                                     enum ttm_caching_state cstate,
> +                                     unsigned count, unsigned long *irq_flags)
>  {
>         struct page *p;
>         int r;
> @@ -605,7 +617,7 @@ static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool,
>
>                 INIT_LIST_HEAD(&new_pages);
>                 r = ttm_alloc_new_pages(&new_pages, pool->gfp_flags, ttm_flags,
> -                               cstate, alloc_size);
> +                                       cstate, alloc_size, 0);
>                 spin_lock_irqsave(&pool->lock, *irq_flags);
>
>                 if (!r) {
> @@ -635,7 +647,7 @@ static int ttm_page_pool_get_pages(struct ttm_page_pool *pool,
>                                    struct list_head *pages,
>                                    int ttm_flags,
>                                    enum ttm_caching_state cstate,
> -                                  unsigned count)
> +                                  unsigned count, unsigned order)
>  {
>         unsigned long irq_flags;
>         struct list_head *p;
> @@ -643,7 +655,9 @@ static int ttm_page_pool_get_pages(struct ttm_page_pool *pool,
>         int r = 0;
>
>         spin_lock_irqsave(&pool->lock, irq_flags);
> -       ttm_page_pool_fill_locked(pool, ttm_flags, cstate, count, &irq_flags);
> +       if (!order)
> +               ttm_page_pool_fill_locked(pool, ttm_flags, cstate, count,
> +                                         &irq_flags);
>
>         if (count >= pool->npages) {
>                 /* take all pages from the pool */
> @@ -698,7 +712,7 @@ static int ttm_page_pool_get_pages(struct ttm_page_pool *pool,
>                  * multiple requests in parallel.
>                  **/
>                 r = ttm_alloc_new_pages(pages, gfp_flags, ttm_flags, cstate,
> -                                       count);
> +                                       count, order);
>         }
>
>         return r;
> @@ -708,8 +722,9 @@ static int ttm_page_pool_get_pages(struct ttm_page_pool *pool,
>  static void ttm_put_pages(struct page **pages, unsigned npages, int flags,
>                           enum ttm_caching_state cstate)
>  {
> +       struct ttm_page_pool *pool = ttm_get_pool(flags, false, cstate);
> +       struct ttm_page_pool *huge = ttm_get_pool(flags, true, cstate);
>         unsigned long irq_flags;
> -       struct ttm_page_pool *pool = ttm_get_pool(flags, cstate);
>         unsigned i;
>
>         if (pool == NULL) {
> @@ -737,8 +752,48 @@ static void ttm_put_pages(struct page **pages, unsigned npages, int flags,
>                 return;
>         }
>
> +       i = 0;
> +#ifdef CONFIG_TRANSPARENT_HUGEPAGE
> +       if (huge) {
> +               unsigned max_size, n2free;
> +
> +               spin_lock_irqsave(&huge->lock, irq_flags);
> +               while (i < npages) {
> +                       struct page *p = pages[i];
> +                       unsigned j;
> +
> +                       if (!p)
> +                               break;
> +
> +                       for (j = 0; j < HPAGE_PMD_NR; ++j)
> +                               if (p++ != pages[i + j])
> +                                   break;
> +
> +                       if (j != HPAGE_PMD_NR)
> +                               break;
> +
> +                       list_add_tail(&pages[i]->lru, &huge->list);
> +
> +                       for (j = 0; j < HPAGE_PMD_NR; ++j)
> +                               pages[i++] = NULL;
> +                       huge->npages++;
> +               }
> +
> +               /* Check that we don't go over the pool limit */
> +               max_size = _manager->options.max_size;
> +               max_size /= HPAGE_PMD_NR;
> +               if (huge->npages > max_size)
> +                       n2free = huge->npages - max_size;
> +               else
> +                       n2free = 0;
> +               spin_unlock_irqrestore(&huge->lock, irq_flags);
> +               if (n2free)
> +                       ttm_page_pool_free(huge, n2free, false);
> +       }
> +#endif
> +
>         spin_lock_irqsave(&pool->lock, irq_flags);
> -       for (i = 0; i < npages; i++) {
> +       while (i < npages) {
>                 if (pages[i]) {
>                         if (page_count(pages[i]) != 1)
>                                 pr_err("Erroneous page count. Leaking pages.\n");
> @@ -746,6 +801,7 @@ static void ttm_put_pages(struct page **pages, unsigned npages, int flags,
>                         pages[i] = NULL;
>                         pool->npages++;
>                 }
> +               ++i;
>         }
>         /* Check that we don't go over the pool limit */
>         npages = 0;
> @@ -768,7 +824,8 @@ static void ttm_put_pages(struct page **pages, unsigned npages, int flags,
>  static int ttm_get_pages(struct page **pages, unsigned npages, int flags,
>                          enum ttm_caching_state cstate)
>  {
> -       struct ttm_page_pool *pool = ttm_get_pool(flags, cstate);
> +       struct ttm_page_pool *pool = ttm_get_pool(flags, false, cstate);
> +       struct ttm_page_pool *huge = ttm_get_pool(flags, true, cstate);
>         struct list_head plist;
>         struct page *p = NULL;
>         unsigned count;
> @@ -821,11 +878,28 @@ static int ttm_get_pages(struct page **pages, unsigned npages, int flags,
>                 return 0;
>         }
>
> -       /* First we take pages from the pool */
> +       count = 0;
> +
> +#ifdef CONFIG_TRANSPARENT_HUGEPAGE
> +       if (huge && npages >= HPAGE_PMD_NR) {
> +               INIT_LIST_HEAD(&plist);
> +               ttm_page_pool_get_pages(huge, &plist, flags, cstate,
> +                                       npages / HPAGE_PMD_NR,
> +                                       HPAGE_PMD_ORDER);
> +
> +               list_for_each_entry(p, &plist, lru) {
> +                       unsigned j;
> +
> +                       for (j = 0; j < HPAGE_PMD_NR; ++j)
> +                               pages[count++] = &p[j];
> +               }
> +       }
> +#endif
> +
>         INIT_LIST_HEAD(&plist);
> -       r = ttm_page_pool_get_pages(pool, &plist, flags, cstate, npages);
> +       r = ttm_page_pool_get_pages(pool, &plist, flags, cstate,
> +                                   npages - count, 0);
>
> -       count = 0;
>         list_for_each_entry(p, &plist, lru)
>                 pages[count++] = p;
>
> @@ -872,6 +946,14 @@ int ttm_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages)
>         ttm_page_pool_init_locked(&_manager->uc_pool_dma32,
>                                   GFP_USER | GFP_DMA32, "uc dma");
>
> +       ttm_page_pool_init_locked(&_manager->wc_pool_huge,
> +                                 GFP_TRANSHUGE & ~(__GFP_MOVABLE | __GFP_COMP),
> +                                 "wc huge");
> +
> +       ttm_page_pool_init_locked(&_manager->uc_pool_huge,
> +                                 GFP_TRANSHUGE & ~(__GFP_MOVABLE | __GFP_COMP)
> +                                 , "uc huge");
> +
>         _manager->options.max_size = max_pages;
>         _manager->options.small = SMALL_ALLOCATION;
>         _manager->options.alloc_size = NUM_PAGES_TO_ALLOC;
> @@ -1041,12 +1123,12 @@ int ttm_page_alloc_debugfs(struct seq_file *m, void *data)
>                 seq_printf(m, "No pool allocator running.\n");
>                 return 0;
>         }
> -       seq_printf(m, "%6s %12s %13s %8s\n",
> +       seq_printf(m, "%7s %12s %13s %8s\n",
>                         h[0], h[1], h[2], h[3]);
>         for (i = 0; i < NUM_POOLS; ++i) {
>                 p = &_manager->pools[i];
>
> -               seq_printf(m, "%6s %12ld %13ld %8d\n",
> +               seq_printf(m, "%7s %12ld %13ld %8d\n",
>                                 p->name, p->nrefills,
>                                 p->nfrees, p->npages);
>         }
> --
> 2.7.4
>
> _______________________________________________
> amd-gfx mailing list
> amd-gfx at lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/amd-gfx


More information about the amd-gfx mailing list