[PATCH] drm/ttm: add NUMA node id to the pool

[Felix Kuehling]

There is a subsequent patch where amdgpu directly calls ttm_pool_init to 
create pools per NUMA node. That will depend on the updated function 
signature.

Regards,
   Felix


On 2023-03-31 15:17, Alex Deucher wrote:
> On Fri, Mar 31, 2023 at 2:54 AM Christian König
> <ckoenig.leichtzumerken at gmail.com> wrote:
>> Should I push this to drm-misc-next or do we take it through
>> amd-staging-drm-next?
> I think either way is fine.  We can carry it internally as needed for
> testing if you want to commit it to drm-misc-next.  I don't think
> there are any direct code dependencies, but you or Rajneesh can
> correct me if I'm wrong.
>
> Alex
>
>> Christian.
>>
>> Am 30.03.23 um 21:50 schrieb Alex Deucher:
>>> From: Rajneesh Bhardwaj <rajneesh.bhardwaj at amd.com>
>>>
>>> This allows backing ttm_tt structure with pages from different NUMA
>>> pools.
>>>
>>> Tested-by: Graham Sider <graham.sider at amd.com>
>>> Reviewed-by: Felix Kuehling <felix.kuehling at amd.com>
>>> Signed-off-by: Christian König <christian.koenig at amd.com>
>>> Signed-off-by: Rajneesh Bhardwaj <rajneesh.bhardwaj at amd.com>
>>> Signed-off-by: Alex Deucher <alexander.deucher at amd.com>
>>> ---
>>>    drivers/gpu/drm/ttm/ttm_device.c |  2 +-
>>>    drivers/gpu/drm/ttm/ttm_pool.c   | 13 ++++++++-----
>>>    include/drm/ttm/ttm_pool.h       |  4 +++-
>>>    3 files changed, 12 insertions(+), 7 deletions(-)
>>>
>>> diff --git a/drivers/gpu/drm/ttm/ttm_device.c b/drivers/gpu/drm/ttm/ttm_device.c
>>> index e7147e304637..4a8164a5320f 100644
>>> --- a/drivers/gpu/drm/ttm/ttm_device.c
>>> +++ b/drivers/gpu/drm/ttm/ttm_device.c
>>> @@ -218,7 +218,7 @@ int ttm_device_init(struct ttm_device *bdev, struct ttm_device_funcs *funcs,
>>>        bdev->funcs = funcs;
>>>
>>>        ttm_sys_man_init(bdev);
>>> -     ttm_pool_init(&bdev->pool, dev, use_dma_alloc, use_dma32);
>>> +     ttm_pool_init(&bdev->pool, dev, NUMA_NO_NODE, use_dma_alloc, use_dma32);
>>>
>>>        bdev->vma_manager = vma_manager;
>>>        INIT_DELAYED_WORK(&bdev->wq, ttm_device_delayed_workqueue);
>>> diff --git a/drivers/gpu/drm/ttm/ttm_pool.c b/drivers/gpu/drm/ttm/ttm_pool.c
>>> index 9f6764bf3b15..1068a41cded1 100644
>>> --- a/drivers/gpu/drm/ttm/ttm_pool.c
>>> +++ b/drivers/gpu/drm/ttm/ttm_pool.c
>>> @@ -92,7 +92,7 @@ static struct page *ttm_pool_alloc_page(struct ttm_pool *pool, gfp_t gfp_flags,
>>>                        __GFP_KSWAPD_RECLAIM;
>>>
>>>        if (!pool->use_dma_alloc) {
>>> -             p = alloc_pages(gfp_flags, order);
>>> +             p = alloc_pages_node(pool->nid, gfp_flags, order);
>>>                if (p)
>>>                        p->private = order;
>>>                return p;
>>> @@ -286,7 +286,7 @@ static struct ttm_pool_type *ttm_pool_select_type(struct ttm_pool *pool,
>>>                                                  enum ttm_caching caching,
>>>                                                  unsigned int order)
>>>    {
>>> -     if (pool->use_dma_alloc)
>>> +     if (pool->use_dma_alloc || pool->nid != NUMA_NO_NODE)
>>>                return &pool->caching[caching].orders[order];
>>>
>>>    #ifdef CONFIG_X86
>>> @@ -523,29 +523,32 @@ EXPORT_SYMBOL(ttm_pool_free);
>>>     *
>>>     * @pool: the pool to initialize
>>>     * @dev: device for DMA allocations and mappings
>>> + * @nid: NUMA node to use for allocations
>>>     * @use_dma_alloc: true if coherent DMA alloc should be used
>>>     * @use_dma32: true if GFP_DMA32 should be used
>>>     *
>>>     * Initialize the pool and its pool types.
>>>     */
>>>    void ttm_pool_init(struct ttm_pool *pool, struct device *dev,
>>> -                bool use_dma_alloc, bool use_dma32)
>>> +                int nid, bool use_dma_alloc, bool use_dma32)
>>>    {
>>>        unsigned int i, j;
>>>
>>>        WARN_ON(!dev && use_dma_alloc);
>>>
>>>        pool->dev = dev;
>>> +     pool->nid = nid;
>>>        pool->use_dma_alloc = use_dma_alloc;
>>>        pool->use_dma32 = use_dma32;
>>>
>>> -     if (use_dma_alloc) {
>>> +     if (use_dma_alloc || nid != NUMA_NO_NODE) {
>>>                for (i = 0; i < TTM_NUM_CACHING_TYPES; ++i)
>>>                        for (j = 0; j < MAX_ORDER; ++j)
>>>                                ttm_pool_type_init(&pool->caching[i].orders[j],
>>>                                                   pool, i, j);
>>>        }
>>>    }
>>> +EXPORT_SYMBOL(ttm_pool_init);
>>>
>>>    /**
>>>     * ttm_pool_fini - Cleanup a pool
>>> @@ -559,7 +562,7 @@ void ttm_pool_fini(struct ttm_pool *pool)
>>>    {
>>>        unsigned int i, j;
>>>
>>> -     if (pool->use_dma_alloc) {
>>> +     if (pool->use_dma_alloc || pool->nid != NUMA_NO_NODE) {
>>>                for (i = 0; i < TTM_NUM_CACHING_TYPES; ++i)
>>>                        for (j = 0; j < MAX_ORDER; ++j)
>>>                                ttm_pool_type_fini(&pool->caching[i].orders[j]);
>>> diff --git a/include/drm/ttm/ttm_pool.h b/include/drm/ttm/ttm_pool.h
>>> index ef09b23d29e3..23bd8be6d4f8 100644
>>> --- a/include/drm/ttm/ttm_pool.h
>>> +++ b/include/drm/ttm/ttm_pool.h
>>> @@ -61,12 +61,14 @@ struct ttm_pool_type {
>>>     * struct ttm_pool - Pool for all caching and orders
>>>     *
>>>     * @dev: the device we allocate pages for
>>> + * @nid: which numa node to use
>>>     * @use_dma_alloc: if coherent DMA allocations should be used
>>>     * @use_dma32: if GFP_DMA32 should be used
>>>     * @caching: pools for each caching/order
>>>     */
>>>    struct ttm_pool {
>>>        struct device *dev;
>>> +     int nid;
>>>
>>>        bool use_dma_alloc;
>>>        bool use_dma32;
>>> @@ -81,7 +83,7 @@ int ttm_pool_alloc(struct ttm_pool *pool, struct ttm_tt *tt,
>>>    void ttm_pool_free(struct ttm_pool *pool, struct ttm_tt *tt);
>>>
>>>    void ttm_pool_init(struct ttm_pool *pool, struct device *dev,
>>> -                bool use_dma_alloc, bool use_dma32);
>>> +                int nid, bool use_dma_alloc, bool use_dma32);
>>>    void ttm_pool_fini(struct ttm_pool *pool);
>>>
>>>    int ttm_pool_debugfs(struct ttm_pool *pool, struct seq_file *m);