[PATCH v6 8/9] drm/amdgpu: generate doorbell index for userqueue

[Alex Deucher]

On Fri, Sep 8, 2023 at 11:55 PM Shashank Sharma <shashank.sharma at amd.com> wrote:
>
> The userspace sends us the doorbell object and the relative doobell
> index in the object to be used for the usermode queue, but the FW
> expects the absolute doorbell index on the PCI BAR in the MQD. This
> patch adds a function to convert this relative doorbell index to
> absolute doorbell index.
>
> This patch is dependent on the doorbell manager series which is
> expected to be merged soon:
> Link: https://patchwork.freedesktop.org/series/115802/
>
> V5: Fix the db object reference leak (Christian)
> V6: Pin the doorbell bo in userqueue_create() function, and unpin it
>     in userqueue destoy (Christian)
>
> Cc: Alex Deucher <alexander.deucher at amd.com>
> Cc: Christian Koenig <christian.koenig at amd.com>
> Signed-off-by: Shashank Sharma <shashank.sharma at amd.com>

Reviewed-by: Alex Deucher <alexander.deucher at amd.com>

> ---
>  drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.c | 40 +++++++++++++++++++
>  drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c        |  1 +
>  .../gpu/drm/amd/include/amdgpu_userqueue.h    |  1 +
>  3 files changed, 42 insertions(+)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.c
> index 03fc8e89eafb..a311d4949bb8 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.c
> @@ -32,6 +32,35 @@ amdgpu_userqueue_find(struct amdgpu_userq_mgr *uq_mgr, int qid)
>         return idr_find(&uq_mgr->userq_idr, qid);
>  }
>
> +static uint64_t
> +amdgpu_userqueue_get_doorbell_index(struct amdgpu_userq_mgr *uq_mgr,
> +                                    struct amdgpu_usermode_queue *queue,
> +                                    struct drm_file *filp,
> +                                    uint32_t doorbell_offset)
> +{
> +       uint64_t index;
> +       struct drm_gem_object *gobj;
> +
> +       gobj = drm_gem_object_lookup(filp, queue->doorbell_handle);
> +       if (gobj == NULL) {
> +               DRM_ERROR("Can't find GEM object for doorbell\n");
> +               return -EINVAL;
> +       }
> +
> +       queue->db_bo = amdgpu_bo_ref(gem_to_amdgpu_bo(gobj));
> +       drm_gem_object_put(gobj);
> +
> +       /* Pin the BO before generating the index, unpin in queue destroy */
> +       if (amdgpu_bo_pin(queue->db_bo, AMDGPU_GEM_DOMAIN_DOORBELL)) {
> +               DRM_ERROR("[Usermode queues] Failed to pin doorbell object\n");
> +               return -EINVAL;
> +       }
> +
> +       index = amdgpu_doorbell_index_on_bar(uq_mgr->adev, queue->db_bo, doorbell_offset);
> +       DRM_DEBUG_DRIVER("[Usermode queues] doorbell index=%lld\n", index);
> +       return index;
> +}
> +
>  static int
>  amdgpu_userqueue_destroy(struct drm_file *filp, int queue_id)
>  {
> @@ -50,6 +79,8 @@ amdgpu_userqueue_destroy(struct drm_file *filp, int queue_id)
>         }
>         uq_funcs = uq_mgr->userq_funcs[queue->queue_type];
>         uq_funcs->mqd_destroy(uq_mgr, queue);
> +       amdgpu_bo_unpin(queue->db_bo);
> +       amdgpu_bo_unref(&queue->db_bo);
>         idr_remove(&uq_mgr->userq_idr, queue_id);
>         kfree(queue);
>
> @@ -64,6 +95,7 @@ amdgpu_userqueue_create(struct drm_file *filp, union drm_amdgpu_userq *args)
>         struct amdgpu_userq_mgr *uq_mgr = &fpriv->userq_mgr;
>         const struct amdgpu_userq_funcs *uq_funcs;
>         struct amdgpu_usermode_queue *queue;
> +       uint64_t index;
>         int qid, r = 0;
>
>         /* Usermode queues are only supported for GFX/SDMA engines as of now */
> @@ -93,6 +125,14 @@ amdgpu_userqueue_create(struct drm_file *filp, union drm_amdgpu_userq *args)
>         queue->flags = args->in.flags;
>         queue->vm = &fpriv->vm;
>
> +       /* Convert relative doorbell offset into absolute doorbell index */
> +       index = amdgpu_userqueue_get_doorbell_index(uq_mgr, queue, filp, args->in.doorbell_offset);
> +       if (index == (uint64_t)-EINVAL) {
> +               DRM_ERROR("Failed to get doorbell for queue\n");
> +               goto unlock;
> +       }
> +       queue->doorbell_index = index;
> +
>         r = uq_funcs->mqd_create(uq_mgr, &args->in, queue);
>         if (r) {
>                 DRM_ERROR("Failed to create Queue\n");
> diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
> index c0eb622dfc37..d855df9b7b37 100644
> --- a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
> @@ -6660,6 +6660,7 @@ static int gfx_v11_0_userq_mqd_create(struct amdgpu_userq_mgr *uq_mgr,
>         userq_props.queue_size = mqd_user.queue_size;
>         userq_props.hqd_base_gpu_addr = mqd_user.queue_va;
>         userq_props.mqd_gpu_addr = queue->mqd.gpu_addr;
> +       userq_props.doorbell_index = queue->doorbell_index;
>         userq_props.use_doorbell = true;
>
>         r = mqd_gfx_generic->init_mqd(adev, (void *)queue->mqd.cpu_ptr, &userq_props);
> diff --git a/drivers/gpu/drm/amd/include/amdgpu_userqueue.h b/drivers/gpu/drm/amd/include/amdgpu_userqueue.h
> index ae155de62560..b5600cff086e 100644
> --- a/drivers/gpu/drm/amd/include/amdgpu_userqueue.h
> +++ b/drivers/gpu/drm/amd/include/amdgpu_userqueue.h
> @@ -44,6 +44,7 @@ struct amdgpu_usermode_queue {
>         struct amdgpu_mqd_prop  *userq_prop;
>         struct amdgpu_userq_mgr *userq_mgr;
>         struct amdgpu_vm        *vm;
> +       struct amdgpu_bo        *db_bo;
>         struct amdgpu_userq_obj mqd;
>         struct amdgpu_userq_obj fw_obj;
>  };
> --
> 2.42.0
>