[PATCH v2] drm/amdgpu: refine amdgpu_bo_create_kernel_at()
Christian König
christian.koenig at amd.com
Fri Jul 28 08:00:50 UTC 2023
Am 28.07.23 um 09:00 schrieb Lazar, Lijo:
>
>
> On 7/28/2023 11:34 AM, Lang Yu wrote:
>> Use amdgpu_bo_create_reserved() to create a BO in VRAM
>> domain would fail if requested VRAM size is large(>128MB)
>> on APU which usually has a default 512MB VRAM.
>>
>> That's because VRAM is framgented after several allocations.
>>
>> The approach is using amdgpu_bo_create_reserved() to
>> create a BO in CPU domain first, it will always succeed.
>>
>> v2: Don't overwrite the contents at specific offset.
>>
>> Signed-off-by: Lang Yu <Lang.Yu at amd.com>
>> ---
>> drivers/gpu/drm/amd/amdgpu/amdgpu_object.c | 24 ++++++++++++++++------
>> drivers/gpu/drm/amd/amdgpu/amdgpu_object.h | 3 ++-
>> drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 15 +++++++-------
>> drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 2 +-
>> 4 files changed, 28 insertions(+), 16 deletions(-)
>>
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
>> index ff73cc11d47e..fa30e96f27d0 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
>> @@ -358,6 +358,7 @@ int amdgpu_bo_create_kernel(struct amdgpu_device
>> *adev,
>> * @offset: offset of the BO
>> * @size: size of the BO
>> * @bo_ptr: used to initialize BOs in structures
>> + * @gpu_addr: GPU addr of the pinned BO
>> * @cpu_addr: optional CPU address mapping
>> *
>> * Creates a kernel BO at a specific offset in VRAM.
>> @@ -367,7 +368,8 @@ int amdgpu_bo_create_kernel(struct amdgpu_device
>> *adev,
>> */
>> int amdgpu_bo_create_kernel_at(struct amdgpu_device *adev,
>> uint64_t offset, uint64_t size,
>> - struct amdgpu_bo **bo_ptr, void **cpu_addr)
>> + struct amdgpu_bo **bo_ptr,
>> + u64 *gpu_addr, void **cpu_addr)
>
> A generic question (not considering other details in this patch) -
> this API is literally asking to create BO at a particular GPU VRAM
> offset. The offset goes in as the input, so why does it need to return
> a GPU offset?
Yeah, that's exactly why I said that the intention of the function was
misunderstood.
This change here doesn't seem to make much sense.
Regards,
Christian.
>
> Thanks,
> Lijo
>
>> {
>> struct ttm_operation_ctx ctx = { false, false };
>> unsigned int i;
>> @@ -377,32 +379,42 @@ int amdgpu_bo_create_kernel_at(struct
>> amdgpu_device *adev,
>> size = ALIGN(size, PAGE_SIZE);
>> r = amdgpu_bo_create_reserved(adev, size, PAGE_SIZE,
>> - AMDGPU_GEM_DOMAIN_VRAM, bo_ptr, NULL,
>> - cpu_addr);
>> + AMDGPU_GEM_DOMAIN_CPU,
>> + bo_ptr, NULL, NULL);
>> if (r)
>> return r;
>> if ((*bo_ptr) == NULL)
>> return 0;
>> + (*bo_ptr)->flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
>> + (*bo_ptr)->flags |= cpu_addr ?
>> AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED
>> + : AMDGPU_GEM_CREATE_NO_CPU_ACCESS;
>> +
>> /*
>> * Remove the original mem node and create a new one at the
>> request
>> * position.
>> */
>> - if (cpu_addr)
>> - amdgpu_bo_kunmap(*bo_ptr);
>> -
>> ttm_resource_free(&(*bo_ptr)->tbo, &(*bo_ptr)->tbo.resource);
>> for (i = 0; i < (*bo_ptr)->placement.num_placement; ++i) {
>> (*bo_ptr)->placements[i].fpfn = offset >> PAGE_SHIFT;
>> (*bo_ptr)->placements[i].lpfn = (offset + size) >> PAGE_SHIFT;
>> + (*bo_ptr)->placements[i].mem_type = TTM_PL_VRAM;
>> + (*bo_ptr)->placements[i].flags = TTM_PL_FLAG_CONTIGUOUS;
>> +
>> + if (!((*bo_ptr)->flags &
>> AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED))
>> + (*bo_ptr)->placements[i].flags |= TTM_PL_FLAG_TOPDOWN;
>> }
>> +
>> r = ttm_bo_mem_space(&(*bo_ptr)->tbo, &(*bo_ptr)->placement,
>> &(*bo_ptr)->tbo.resource, &ctx);
>> if (r)
>> goto error;
>> + if (gpu_addr)
>> + *gpu_addr = amdgpu_bo_gpu_offset(*bo_ptr);
>> +
>> if (cpu_addr) {
>> r = amdgpu_bo_kmap(*bo_ptr, cpu_addr);
>> if (r)
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
>> index 5d3440d719e4..8f5b5664a1b6 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
>> @@ -315,7 +315,8 @@ int amdgpu_bo_create_kernel(struct amdgpu_device
>> *adev,
>> u64 *gpu_addr, void **cpu_addr);
>> int amdgpu_bo_create_kernel_at(struct amdgpu_device *adev,
>> uint64_t offset, uint64_t size,
>> - struct amdgpu_bo **bo_ptr, void **cpu_addr);
>> + struct amdgpu_bo **bo_ptr,
>> + u64 *gpu_addr, void **cpu_addr);
>> int amdgpu_bo_create_user(struct amdgpu_device *adev,
>> struct amdgpu_bo_param *bp,
>> struct amdgpu_bo_user **ubo_ptr);
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
>> index 7c6dd3de1867..a210c243dac0 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
>> @@ -1619,7 +1619,7 @@ static int
>> amdgpu_ttm_fw_reserve_vram_init(struct amdgpu_device *adev)
>> adev->mman.fw_vram_usage_start_offset,
>> adev->mman.fw_vram_usage_size,
>> &adev->mman.fw_vram_usage_reserved_bo,
>> - &adev->mman.fw_vram_usage_va);
>> + NULL, &adev->mman.fw_vram_usage_va);
>> }
>> /**
>> @@ -1644,7 +1644,7 @@ static int
>> amdgpu_ttm_drv_reserve_vram_init(struct amdgpu_device *adev)
>> adev->mman.drv_vram_usage_start_offset,
>> adev->mman.drv_vram_usage_size,
>> &adev->mman.drv_vram_usage_reserved_bo,
>> - &adev->mman.drv_vram_usage_va);
>> + NULL, &adev->mman.drv_vram_usage_va);
>> }
>> /*
>> @@ -1729,8 +1729,7 @@ static int amdgpu_ttm_reserve_tmr(struct
>> amdgpu_device *adev)
>> ret = amdgpu_bo_create_kernel_at(adev,
>> ctx->c2p_train_data_offset,
>> ctx->train_data_size,
>> - &ctx->c2p_bo,
>> - NULL);
>> + &ctx->c2p_bo, NULL, NULL);
>> if (ret) {
>> DRM_ERROR("alloc c2p_bo failed(%d)!\n", ret);
>> amdgpu_ttm_training_reserve_vram_fini(adev);
>> @@ -1742,7 +1741,7 @@ static int amdgpu_ttm_reserve_tmr(struct
>> amdgpu_device *adev)
>> if (!adev->gmc.is_app_apu) {
>> ret = amdgpu_bo_create_kernel_at(
>> adev, adev->gmc.real_vram_size - reserve_size,
>> - reserve_size, &adev->mman.fw_reserved_memory, NULL);
>> + reserve_size, &adev->mman.fw_reserved_memory, NULL, NULL);
>> if (ret) {
>> DRM_ERROR("alloc tmr failed(%d)!\n", ret);
>> amdgpu_bo_free_kernel(&adev->mman.fw_reserved_memory,
>> @@ -1885,14 +1884,14 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
>> r = amdgpu_bo_create_kernel_at(adev, 0,
>> adev->mman.stolen_vga_size,
>> &adev->mman.stolen_vga_memory,
>> - NULL);
>> + NULL, NULL);
>> if (r)
>> return r;
>> r = amdgpu_bo_create_kernel_at(adev,
>> adev->mman.stolen_vga_size,
>> adev->mman.stolen_extended_size,
>> &adev->mman.stolen_extended_memory,
>> - NULL);
>> + NULL, NULL);
>> if (r)
>> return r;
>> @@ -1901,7 +1900,7 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
>> adev->mman.stolen_reserved_offset,
>> adev->mman.stolen_reserved_size,
>> &adev->mman.stolen_reserved_memory,
>> - NULL);
>> + NULL, NULL);
>> if (r)
>> return r;
>> } else {
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
>> index 41aa853a07d2..b93b42b916ce 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
>> @@ -397,7 +397,7 @@ static void amdgpu_virt_ras_reserve_bps(struct
>> amdgpu_device *adev)
>> */
>> if (amdgpu_bo_create_kernel_at(adev, bp <<
>> AMDGPU_GPU_PAGE_SHIFT,
>> AMDGPU_GPU_PAGE_SIZE,
>> - &bo, NULL))
>> + &bo, NULL, NULL))
>> DRM_DEBUG("RAS WARN: reserve vram for retired page %llx
>> fail\n", bp);
>> data->bps_bo[i] = bo;
More information about the amd-gfx
mailing list