[PATCH 03/16] drm/amdgpu: create a new file for doorbell manager
Shashank Sharma
shashank.sharma at amd.com
Thu Mar 30 13:42:22 UTC 2023
On 30/03/2023 15:29, Luben Tuikov wrote:
> Hi Shashank,
>
> Inline:
>
> On 2023-03-30 07:09, Christian König wrote:
>> Am 29.03.23 um 17:47 schrieb Shashank Sharma:
>>> From: Shashank Sharma <contactshashanksharma at gmail.com>
>>>
>>> This patch:
>>> - creates a new file for doorbell management.
>>> - moves doorbell code from amdgpu_device.c to this file.
>>>
>>> Cc: Alex Deucher <alexander.deucher at amd.com>
>>> Cc: Christian Koenig <christian.koenig at amd.com>
>>> Signed-off-by: Shashank Sharma <shashank.sharma at amd.com>
>>> ---
>>> drivers/gpu/drm/amd/amdgpu/Makefile | 2 +-
>>> drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 164 ---------------
>>> drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell.h | 22 +++
>>> .../gpu/drm/amd/amdgpu/amdgpu_doorbell_mgr.c | 186 ++++++++++++++++++
>>> 4 files changed, 209 insertions(+), 165 deletions(-)
>>> create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell_mgr.c
>>>
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile
>>> index 798d0e9a60b7..204665f20319 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/Makefile
>>> +++ b/drivers/gpu/drm/amd/amdgpu/Makefile
>>> @@ -41,7 +41,7 @@ ccflags-y := -I$(FULL_AMD_PATH)/include/asic_reg \
>>> amdgpu-y := amdgpu_drv.o
>>>
>>> # add KMS driver
>>> -amdgpu-y += amdgpu_device.o amdgpu_kms.o \
>>> +amdgpu-y += amdgpu_device.o amdgpu_doorbell_mgr.o amdgpu_kms.o \
>>> amdgpu_atombios.o atombios_crtc.o amdgpu_connectors.o \
>>> atom.o amdgpu_fence.o amdgpu_ttm.o amdgpu_object.o amdgpu_gart.o \
>>> amdgpu_encoders.o amdgpu_display.o amdgpu_i2c.o \
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>> index 57ee1c4a81e9..7f8fcac4f18b 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>> @@ -579,94 +579,6 @@ void amdgpu_mm_wreg_mmio_rlc(struct amdgpu_device *adev,
>>> }
>>> }
>>>
>>> -/**
>>> - * amdgpu_mm_rdoorbell - read a doorbell dword
>>> - *
>>> - * @adev: amdgpu_device pointer
>>> - * @index: doorbell index
>>> - *
>>> - * Returns the value in the doorbell aperture at the
>>> - * requested doorbell index (CIK).
>>> - */
>>> -u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index)
>>> -{
>>> - if (amdgpu_device_skip_hw_access(adev))
>>> - return 0;
>>> -
>>> - if (index < adev->doorbell.num_kernel_doorbells) {
>>> - return readl(adev->doorbell.ptr + index);
>>> - } else {
>>> - DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
>>> - return 0;
>>> - }
>>> -}
>>> -
>>> -/**
>>> - * amdgpu_mm_wdoorbell - write a doorbell dword
>>> - *
>>> - * @adev: amdgpu_device pointer
>>> - * @index: doorbell index
>>> - * @v: value to write
>>> - *
>>> - * Writes @v to the doorbell aperture at the
>>> - * requested doorbell index (CIK).
>>> - */
>>> -void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v)
>>> -{
>>> - if (amdgpu_device_skip_hw_access(adev))
>>> - return;
>>> -
>>> - if (index < adev->doorbell.num_kernel_doorbells) {
>>> - writel(v, adev->doorbell.ptr + index);
>>> - } else {
>>> - DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
>>> - }
>>> -}
>>> -
>>> -/**
>>> - * amdgpu_mm_rdoorbell64 - read a doorbell Qword
>>> - *
>>> - * @adev: amdgpu_device pointer
>>> - * @index: doorbell index
>>> - *
>>> - * Returns the value in the doorbell aperture at the
>>> - * requested doorbell index (VEGA10+).
>>> - */
>>> -u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index)
>>> -{
>>> - if (amdgpu_device_skip_hw_access(adev))
>>> - return 0;
>>> -
>>> - if (index < adev->doorbell.num_kernel_doorbells) {
>>> - return atomic64_read((atomic64_t *)(adev->doorbell.ptr + index));
>>> - } else {
>>> - DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
>>> - return 0;
>>> - }
>>> -}
>>> -
>>> -/**
>>> - * amdgpu_mm_wdoorbell64 - write a doorbell Qword
>>> - *
>>> - * @adev: amdgpu_device pointer
>>> - * @index: doorbell index
>>> - * @v: value to write
>>> - *
>>> - * Writes @v to the doorbell aperture at the
>>> - * requested doorbell index (VEGA10+).
>>> - */
>>> -void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v)
>>> -{
>>> - if (amdgpu_device_skip_hw_access(adev))
>>> - return;
>>> -
>>> - if (index < adev->doorbell.num_kernel_doorbells) {
>>> - atomic64_set((atomic64_t *)(adev->doorbell.ptr + index), v);
>>> - } else {
>>> - DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
>>> - }
>>> -}
>>> -
>>> /**
>>> * amdgpu_device_indirect_rreg - read an indirect register
>>> *
>>> @@ -1016,82 +928,6 @@ int amdgpu_device_pci_reset(struct amdgpu_device *adev)
>>> return pci_reset_function(adev->pdev);
>>> }
>>>
>>> -/*
>>> - * GPU doorbell aperture helpers function.
>>> - */
>>> -/**
>>> - * amdgpu_device_doorbell_init - Init doorbell driver information.
>>> - *
>>> - * @adev: amdgpu_device pointer
>>> - *
>>> - * Init doorbell driver information (CIK)
>>> - * Returns 0 on success, error on failure.
>>> - */
>>> -static int amdgpu_device_doorbell_init(struct amdgpu_device *adev)
>>> -{
>>> -
>>> - /* No doorbell on SI hardware generation */
>>> - if (adev->asic_type < CHIP_BONAIRE) {
>>> - adev->doorbell.base = 0;
>>> - adev->doorbell.size = 0;
>>> - adev->doorbell.num_kernel_doorbells = 0;
>>> - adev->doorbell.ptr = NULL;
>>> - return 0;
>>> - }
>>> -
>>> - if (pci_resource_flags(adev->pdev, 2) & IORESOURCE_UNSET)
>>> - return -EINVAL;
>>> -
>>> - amdgpu_asic_init_doorbell_index(adev);
>>> -
>>> - /* doorbell bar mapping */
>>> - adev->doorbell.base = pci_resource_start(adev->pdev, 2);
>>> - adev->doorbell.size = pci_resource_len(adev->pdev, 2);
>>> -
>>> - if (adev->enable_mes) {
>>> - adev->doorbell.num_kernel_doorbells =
>>> - adev->doorbell.size / sizeof(u32);
>>> - } else {
>>> - adev->doorbell.num_kernel_doorbells =
>>> - min_t(u32, adev->doorbell.size / sizeof(u32),
>>> - adev->doorbell_index.max_assignment+1);
>>> - if (adev->doorbell.num_kernel_doorbells == 0)
>>> - return -EINVAL;
>>> -
>>> - /* For Vega, reserve and map two pages on doorbell BAR since SDMA
>>> - * paging queue doorbell use the second page. The
>>> - * AMDGPU_DOORBELL64_MAX_ASSIGNMENT definition assumes all the
>>> - * doorbells are in the first page. So with paging queue enabled,
>>> - * the max num_kernel_doorbells should + 1 page (0x400 in dword)
>>> - */
>>> - if (adev->asic_type >= CHIP_VEGA10)
>>> - adev->doorbell.num_kernel_doorbells += 0x400;
>>> - }
>>> -
>>> - adev->doorbell.ptr = ioremap(adev->doorbell.base,
>>> - adev->doorbell.num_kernel_doorbells *
>>> - sizeof(u32));
>>> - if (adev->doorbell.ptr == NULL)
>>> - return -ENOMEM;
>>> -
>>> - return 0;
>>> -}
>>> -
>>> -/**
>>> - * amdgpu_device_doorbell_fini - Tear down doorbell driver information.
>>> - *
>>> - * @adev: amdgpu_device pointer
>>> - *
>>> - * Tear down doorbell driver information (CIK)
>>> - */
>>> -static void amdgpu_device_doorbell_fini(struct amdgpu_device *adev)
>>> -{
>>> - iounmap(adev->doorbell.ptr);
>>> - adev->doorbell.ptr = NULL;
>>> -}
>>> -
>>> -
>>> -
>>> /*
>>> * amdgpu_device_wb_*()
>>> * Writeback is the method by which the GPU updates special pages in memory
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell.h
>>> index 6064943a1b53..f9c3b77bf65d 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell.h
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell.h
>>> @@ -306,6 +306,28 @@ void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v);
>>> u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index);
>>> void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v);
>>>
>>> +/*
>>> + * GPU doorbell aperture helpers function.
>>> + */
>>> +/**
>>> + * amdgpu_device_doorbell_init - Init doorbell driver information.
>>> + *
>>> + * @adev: amdgpu_device pointer
>>> + *
>>> + * Init doorbell driver information (CIK)
>>> + * Returns 0 on success, error on failure.
>>> + */
>>> +int amdgpu_device_doorbell_init(struct amdgpu_device *adev);
>>> +
>>> +/**
>>> + * amdgpu_device_doorbell_fini - Tear down doorbell driver information.
>>> + *
>>> + * @adev: amdgpu_device pointer
>>> + *
>>> + * Tear down doorbell driver information (CIK)
>>> + */
>>> +void amdgpu_device_doorbell_fini(struct amdgpu_device *adev);
>> I might be wrong, but having kerneldoc on both the declaration and the
>> implementation is forbidden I think.
>>
>> We usually have it on the implementation side only.
> Yes, that is correct--implementation side only as that's where
> the actual code can be found.
Noted, I will remove the extra doc.
- Shashank
>> Christian.
>>
>>> +
>>> #define RDOORBELL32(index) amdgpu_mm_rdoorbell(adev, (index))
>>> #define WDOORBELL32(index, v) amdgpu_mm_wdoorbell(adev, (index), (v))
>>> #define RDOORBELL64(index) amdgpu_mm_rdoorbell64(adev, (index))
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell_mgr.c
>>> new file mode 100644
>>> index 000000000000..2206926ba289
>>> --- /dev/null
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_doorbell_mgr.c
>>> @@ -0,0 +1,186 @@
>>> +/*
> Add an SPDX licence identifier as the very first line:
> //SPDX-License-Identifier: <value>
>
> with some appropriate value, maybe "GPL-2.0-or-later" (no quotes when added).
>
> Regards,
> Luben
>
>>> + * Copyright 2023 Advanced Micro Devices, Inc.
>>> + *
>>> + * Permission is hereby granted, free of charge, to any person obtaining a
>>> + * copy of this software and associated documentation files (the "Software"),
>>> + * to deal in the Software without restriction, including without limitation
>>> + * the rights to use, copy, modify, merge, publish, distribute, sublicense,
>>> + * and/or sell copies of the Software, and to permit persons to whom the
>>> + * Software is furnished to do so, subject to the following conditions:
>>> + *
>>> + * The above copyright notice and this permission notice shall be included in
>>> + * all copies or substantial portions of the Software.
>>> + *
>>> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
>>> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
>>> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
>>> + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
>>> + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
>>> + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
>>> + * OTHER DEALINGS IN THE SOFTWARE.
>>> + *
>>> + */
>>> +
>>> +#include "amdgpu.h"
>>> +
>>> +/**
>>> + * amdgpu_mm_rdoorbell - read a doorbell dword
>>> + *
>>> + * @adev: amdgpu_device pointer
>>> + * @index: doorbell index
>>> + *
>>> + * Returns the value in the doorbell aperture at the
>>> + * requested doorbell index (CIK).
>>> + */
>>> +u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index)
>>> +{
>>> + if (amdgpu_device_skip_hw_access(adev))
>>> + return 0;
>>> +
>>> + if (index < adev->doorbell.num_kernel_doorbells) {
>>> + return readl(adev->doorbell.ptr + index);
>>> + } else {
>>> + DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
>>> + return 0;
>>> + }
>>> +}
>>> +
>>> +/**
>>> + * amdgpu_mm_wdoorbell - write a doorbell dword
>>> + *
>>> + * @adev: amdgpu_device pointer
>>> + * @index: doorbell index
>>> + * @v: value to write
>>> + *
>>> + * Writes @v to the doorbell aperture at the
>>> + * requested doorbell index (CIK).
>>> + */
>>> +void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v)
>>> +{
>>> + if (amdgpu_device_skip_hw_access(adev))
>>> + return;
>>> +
>>> + if (index < adev->doorbell.num_kernel_doorbells) {
>>> + writel(v, adev->doorbell.ptr + index);
>>> + } else {
>>> + DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
>>> + }
>>> +}
>>> +
>>> +/**
>>> + * amdgpu_mm_rdoorbell64 - read a doorbell Qword
>>> + *
>>> + * @adev: amdgpu_device pointer
>>> + * @index: doorbell index
>>> + *
>>> + * Returns the value in the doorbell aperture at the
>>> + * requested doorbell index (VEGA10+).
>>> + */
>>> +u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index)
>>> +{
>>> + if (amdgpu_device_skip_hw_access(adev))
>>> + return 0;
>>> +
>>> + if (index < adev->doorbell.num_kernel_doorbells) {
>>> + return atomic64_read((atomic64_t *)(adev->doorbell.ptr + index));
>>> + } else {
>>> + DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
>>> + return 0;
>>> + }
>>> +}
>>> +
>>> +/**
>>> + * amdgpu_mm_wdoorbell64 - write a doorbell Qword
>>> + *
>>> + * @adev: amdgpu_device pointer
>>> + * @index: doorbell index
>>> + * @v: value to write
>>> + *
>>> + * Writes @v to the doorbell aperture at the
>>> + * requested doorbell index (VEGA10+).
>>> + */
>>> +void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v)
>>> +{
>>> + if (amdgpu_device_skip_hw_access(adev))
>>> + return;
>>> +
>>> + if (index < adev->doorbell.num_kernel_doorbells) {
>>> + atomic64_set((atomic64_t *)(adev->doorbell.ptr + index), v);
>>> + } else {
>>> + DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
>>> + }
>>> +}
>>> +
>>> +/*
>>> + * GPU doorbell aperture helpers function.
>>> + */
>>> +/**
>>> + * amdgpu_device_doorbell_init - Init doorbell driver information.
>>> + *
>>> + * @adev: amdgpu_device pointer
>>> + *
>>> + * Init doorbell driver information (CIK)
>>> + * Returns 0 on success, error on failure.
>>> + */
>>> +int amdgpu_device_doorbell_init(struct amdgpu_device *adev)
>>> +{
>>> +
>>> + /* No doorbell on SI hardware generation */
>>> + if (adev->asic_type < CHIP_BONAIRE) {
>>> + adev->doorbell.base = 0;
>>> + adev->doorbell.size = 0;
>>> + adev->doorbell.num_kernel_doorbells = 0;
>>> + adev->doorbell.ptr = NULL;
>>> + return 0;
>>> + }
>>> +
>>> + if (pci_resource_flags(adev->pdev, 2) & IORESOURCE_UNSET)
>>> + return -EINVAL;
>>> +
>>> + amdgpu_asic_init_doorbell_index(adev);
>>> +
>>> + /* doorbell bar mapping */
>>> + adev->doorbell.base = pci_resource_start(adev->pdev, 2);
>>> + adev->doorbell.size = pci_resource_len(adev->pdev, 2);
>>> +
>>> + if (adev->enable_mes) {
>>> + adev->doorbell.num_kernel_doorbells =
>>> + adev->doorbell.size / sizeof(u32);
>>> + } else {
>>> + adev->doorbell.num_kernel_doorbells =
>>> + min_t(u32, adev->doorbell.size / sizeof(u32),
>>> + adev->doorbell_index.max_assignment+1);
>>> + if (adev->doorbell.num_kernel_doorbells == 0)
>>> + return -EINVAL;
>>> +
>>> + /* For Vega, reserve and map two pages on doorbell BAR since SDMA
>>> + * paging queue doorbell use the second page. The
>>> + * AMDGPU_DOORBELL64_MAX_ASSIGNMENT definition assumes all the
>>> + * doorbells are in the first page. So with paging queue enabled,
>>> + * the max num_kernel_doorbells should + 1 page (0x400 in dword)
>>> + */
>>> + if (adev->asic_type >= CHIP_VEGA10)
>>> + adev->doorbell.num_kernel_doorbells += 0x400;
>>> + }
>>> +
>>> + adev->doorbell.ptr = ioremap(adev->doorbell.base,
>>> + adev->doorbell.num_kernel_doorbells *
>>> + sizeof(u32));
>>> + if (adev->doorbell.ptr == NULL)
>>> + return -ENOMEM;
>>> +
>>> + return 0;
>>> +}
>>> +
>>> +/**
>>> + * amdgpu_device_doorbell_fini - Tear down doorbell driver information.
>>> + *
>>> + * @adev: amdgpu_device pointer
>>> + *
>>> + * Tear down doorbell driver information (CIK)
>>> + */
>>> +void amdgpu_device_doorbell_fini(struct amdgpu_device *adev)
>>> +{
>>> + iounmap(adev->doorbell.ptr);
>>> + adev->doorbell.ptr = NULL;
>>> +}
More information about the amd-gfx
mailing list