[PATCH v3 06/12] drm/amdgpu: create kernel doorbell pages
Christian König
christian.koenig at amd.com
Wed Jun 21 09:13:51 UTC 2023
Am 21.06.23 um 11:10 schrieb Shashank Sharma:
> Hey Christian,
>
> On 21/06/2023 10:32, Christian König wrote:
>> Am 20.06.23 um 19:16 schrieb Shashank Sharma:
>>> This patch:
>>> - creates a doorbell page for graphics driver usages.
>>> - adds a few new varlables in adev->doorbell structure to
>>> keep track of kernel's doorbell-bo.
>>> - removes the adev->doorbell.ptr variable, replaces it with
>>> kernel-doorbell-bo's cpu address.
>>>
>>> V2: - Create doorbell BO directly, no wrappe functions (Alex)
>>> - no additional doorbell structure (Alex, Christian)
>>> - Use doorbell_cpu_ptr, remove ioremap (Christian, Alex)
>>> - Allocate one extra page of doorbells for MES (Alex)
>>>
>>> Cc: Alex Deucher <alexander.deucher at amd.com>
>>> Cc: Christian Koenig <christian.koenig at amd.com>
>>> Signed-off-by: Shashank Sharma <shashank.sharma at amd.com>
>>> ---
>>> drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell.h | 8 ++-
>>> .../gpu/drm/amd/amdgpu/amdgpu_doorbell_mgr.c | 56
>>> ++++++++++++++-----
>>> drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h | 3 +
>>> drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 7 +++
>>> 4 files changed, 60 insertions(+), 14 deletions(-)
>>>
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell.h
>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell.h
>>> index 783e2b8b086c..002899edb9d7 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell.h
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell.h
>>> @@ -31,10 +31,15 @@ struct amdgpu_doorbell {
>>> /* doorbell mmio */
>>> resource_size_t base;
>>> resource_size_t size;
>>> - u32 __iomem *ptr;
>>> /* Number of doorbells reserved for amdgpu kernel driver */
>>> u32 num_kernel_doorbells;
>>> +
>>> + /* Kernel doorbells */
>>> + struct amdgpu_bo *kernel_doorbells;
>>> +
>>> + /* For CPU access of doorbells */
>>> + uint32_t *cpu_addr;
>>> };
>>> /* Reserved doorbells for amdgpu (including multimedia).
>>> @@ -311,6 +316,7 @@ void amdgpu_mm_wdoorbell64(struct amdgpu_device
>>> *adev, u32 index, u64 v);
>>> */
>>> int amdgpu_doorbell_init(struct amdgpu_device *adev);
>>> void amdgpu_doorbell_fini(struct amdgpu_device *adev);
>>> +int amdgpu_doorbell_create_kernel_doorbells(struct amdgpu_device
>>> *adev);
>>> #define RDOORBELL32(index) amdgpu_mm_rdoorbell(adev, (index))
>>> #define WDOORBELL32(index, v) amdgpu_mm_wdoorbell(adev, (index), (v))
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell_mgr.c
>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell_mgr.c
>>> index eb113e38f5e9..118f4bed32fd 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell_mgr.c
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell_mgr.c
>>> @@ -39,7 +39,7 @@ u32 amdgpu_mm_rdoorbell(struct amdgpu_device
>>> *adev, u32 index)
>>> return 0;
>>> if (index < adev->doorbell.num_kernel_doorbells)
>>> - return readl(adev->doorbell.ptr + index);
>>> + return readl(adev->doorbell.cpu_addr + index);
>>> DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n",
>>> index);
>>> return 0;
>>> @@ -61,7 +61,7 @@ void amdgpu_mm_wdoorbell(struct amdgpu_device
>>> *adev, u32 index, u32 v)
>>> return;
>>> if (index < adev->doorbell.num_kernel_doorbells)
>>> - writel(v, adev->doorbell.ptr + index);
>>> + writel(v, adev->doorbell.cpu_addr + index);
>>> else
>>> DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n",
>>> index);
>>> }
>>> @@ -81,7 +81,7 @@ u64 amdgpu_mm_rdoorbell64(struct amdgpu_device
>>> *adev, u32 index)
>>> return 0;
>>> if (index < adev->doorbell.num_kernel_doorbells)
>>> - return atomic64_read((atomic64_t *)(adev->doorbell.ptr +
>>> index));
>>> + return atomic64_read((atomic64_t *)(adev->doorbell.cpu_addr
>>> + index));
>>> DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n",
>>> index);
>>> return 0;
>>> @@ -103,11 +103,47 @@ void amdgpu_mm_wdoorbell64(struct
>>> amdgpu_device *adev, u32 index, u64 v)
>>> return;
>>> if (index < adev->doorbell.num_kernel_doorbells)
>>> - atomic64_set((atomic64_t *)(adev->doorbell.ptr + index), v);
>>> + atomic64_set((atomic64_t *)(adev->doorbell.cpu_addr +
>>> index), v);
>>> else
>>> DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n",
>>> index);
>>> }
>>> +/**
>>> + * amdgpu_doorbell_create_kernel_doorbells - Create kernel
>>> doorbells for graphics
>>> + *
>>> + * @adev: amdgpu_device pointer
>>> + *
>>> + * Creates doorbells for graphics driver usages.
>>> + * returns 0 on success, error otherwise.
>>> + */
>>> +int amdgpu_doorbell_create_kernel_doorbells(struct amdgpu_device
>>> *adev)
>>> +{
>>> + int r;
>>> + int size;
>>> +
>>> + /* Reserve first num_kernel_doorbells (page-aligned) for kernel
>>> ops */
>>> + size = ALIGN(adev->doorbell.num_kernel_doorbells * sizeof(u32),
>>> PAGE_SIZE);
>>> +
>>> + /* Allocate an extra page for MES kernel usages (ring test) */
>>> + adev->mes.db_start_dw_offset = size / sizeof(u32);
>>> + size += PAGE_SIZE;
>>
>> This is initialized but not used, so I suspect it will temporary
>> break the MES test? Or is the MES just implicitly using the correct
>> offset somehow?
>
> I am not sure if I understand this comment properly, can you please
> elaborate ?
>
> If you are talking about 'size', its just used below while creating
> the doorbell BO.
>
> If you mean adev->mes.db_start_dw_offset, its being used in
> amdgpu_mes_kernel_doorbell_get/free and amdgpu_mes_doorbell_init
I was talking about adev->mes.db_start_dw_offset. But using this in
amdgpu_mes_kernel_doorbell_get/free comes in a later patch, doesn't it?
If yes, doesn't that break the MES test?
Christian.
>
> So both of the variables are being used.
>
> - Shashank
>
>>
>> Apart from that it looks good to me,
>> Christian.
>>
>>> +
>>> + r = amdgpu_bo_create_kernel(adev,
>>> + size,
>>> + PAGE_SIZE,
>>> + AMDGPU_GEM_DOMAIN_DOORBELL,
>>> + &adev->doorbell.kernel_doorbells,
>>> + NULL,
>>> + (void **)&adev->doorbell.cpu_addr);
>>> + if (r) {
>>> + DRM_ERROR("Failed to allocate kernel doorbells, err=%d\n", r);
>>> + return r;
>>> + }
>>> +
>>> + adev->doorbell.num_kernel_doorbells = size / sizeof(u32);
>>> + return 0;
>>> +}
>>> +
>>> /*
>>> * GPU doorbell aperture helpers function.
>>> */
>>> @@ -127,7 +163,6 @@ int amdgpu_doorbell_init(struct amdgpu_device
>>> *adev)
>>> adev->doorbell.base = 0;
>>> adev->doorbell.size = 0;
>>> adev->doorbell.num_kernel_doorbells = 0;
>>> - adev->doorbell.ptr = NULL;
>>> return 0;
>>> }
>>> @@ -156,12 +191,6 @@ int amdgpu_doorbell_init(struct amdgpu_device
>>> *adev)
>>> if (adev->asic_type >= CHIP_VEGA10)
>>> adev->doorbell.num_kernel_doorbells += 0x400;
>>> - adev->doorbell.ptr = ioremap(adev->doorbell.base,
>>> - adev->doorbell.num_kernel_doorbells *
>>> - sizeof(u32));
>>> - if (adev->doorbell.ptr == NULL)
>>> - return -ENOMEM;
>>> -
>>> return 0;
>>> }
>>> @@ -174,6 +203,7 @@ int amdgpu_doorbell_init(struct amdgpu_device
>>> *adev)
>>> */
>>> void amdgpu_doorbell_fini(struct amdgpu_device *adev)
>>> {
>>> - iounmap(adev->doorbell.ptr);
>>> - adev->doorbell.ptr = NULL;
>>> + amdgpu_bo_free_kernel(&adev->doorbell.kernel_doorbells,
>>> + NULL,
>>> + (void **)&adev->doorbell.cpu_addr);
>>> }
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
>>> index 547ec35691fa..a403418d5eac 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
>>> @@ -128,6 +128,9 @@ struct amdgpu_mes {
>>> int (*kiq_hw_init)(struct
>>> amdgpu_device *adev);
>>> int (*kiq_hw_fini)(struct
>>> amdgpu_device *adev);
>>> + /* MES doorbells */
>>> + uint32_t db_start_dw_offset;
>>> +
>>> /* ip specific functions */
>>> const struct amdgpu_mes_funcs *funcs;
>>> };
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
>>> index 6d1587bbda52..c48e97646541 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
>>> @@ -1854,6 +1854,13 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
>>> return r;
>>> }
>>> + /* Create a boorbell page for kernel usages */
>>> + r = amdgpu_doorbell_create_kernel_doorbells(adev);
>>> + if (r) {
>>> + DRM_ERROR("Failed to initialize kernel doorbells.\n");
>>> + return r;
>>> + }
>>> +
>>> /* Initialize preemptible memory pool */
>>> r = amdgpu_preempt_mgr_init(adev);
>>> if (r) {
>>
More information about the amd-gfx
mailing list