[PATCH v4 2/3] drm/amdgpu: Add a max ibs per submission limit.

[Christian König]

Am 13.04.23 um 16:22 schrieb Bas Nieuwenhuizen:
> And ensure each ring supports that many submissions. This makes
> sure that we don't get surprises after the submission has been
> scheduled where the ring allocation actually gets rejected.
>
> My calculations on the existing limits:
> COMPUTE v10: 128
> COMPUTE v11: 128
> COMPUTE v6: 157
> COMPUTE v7: 133
> COMPUTE v8: 130
> COMPUTE v9: 125
> GFX v10: 208
> GFX v11: 213
> GFX v6: 154 (doubling this in the previous patch)
> GFX v7: 226
> GFX v8: 213
> GFX v9: 208
> GFX v9 (SW): 208
> SDMA CIK: 87
> SDMA SI: 97
> SDMA v2.4: 74
> SDMA v3.0: 74
> SDMA v4.0: 72
> SDMA v5.0: 51
> SDMA v6.0: 52
> UVD ENC v6.0: 98
> UVD ENC v7.0: 92
> UVD v3.1: 124
> UVD v4.2: 124
> UVD v5.0: 83
> UVD v6.0  (VM): 55
> UVD v7.0: 51
> VCE v2.0: 126
> VCE v3.0 (VM): 98
> VCE v4.0: 93
> VCN DEC v1.0: 49
> VCN DEC v2.0: 51
> VCN DEC v3.0: 51
> VCN ENC v1.0: 58
> VCN ENC v2.0: 93
> VCN ENC v3.0: 93
> VCN ENC v4.0: 93
> VCN JPEG v1.0: 17
> VCN JPEG v2.0: 16
> VCN JPEG v2.5: 17
> VCN JPEG v3.0: 17
> VCN JPEG v4.0: 17
>
> Link: https://gitlab.freedesktop.org/drm/amd/-/issues/2498
> Signed-off-by: Bas Nieuwenhuizen <bas at basnieuwenhuizen.nl>

Reviewed-by: Christian König <christian.koenig at amd.com>

> ---
>   drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c   |  3 +++
>   drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c | 29 ++++++++++++++++++++++++
>   drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h |  1 +
>   3 files changed, 33 insertions(+)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> index 7af3041ccd0e..8362738974c8 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> @@ -110,6 +110,9 @@ static int amdgpu_cs_p1_ib(struct amdgpu_cs_parser *p,
>   	if (r < 0)
>   		return r;
>   
> +	if (num_ibs[r] >= amdgpu_ring_max_ibs(chunk_ib->ip_type))
> +		return -EINVAL;
> +
>   	++(num_ibs[r]);
>   	p->gang_leader_idx = r;
>   	return 0;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
> index dc474b809604..f676c236b657 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
> @@ -49,6 +49,26 @@
>    * them until the pointers are equal again.
>    */
>   
> +/**
> + * amdgpu_ring_max_ibs - Return max IBs that fit in a single submission.
> + *
> + * @type: ring type for which to return the limit.
> + */
> +unsigned int amdgpu_ring_max_ibs(enum amdgpu_ring_type type)
> +{
> +	switch (type) {
> +	case AMDGPU_RING_TYPE_GFX:
> +		/* Need to keep at least 192 on GFX7+ for old radv. */
> +		return 192;
> +	case AMDGPU_RING_TYPE_COMPUTE:
> +		return 125;
> +	case AMDGPU_RING_TYPE_VCN_JPEG:
> +		return 16;
> +	default:
> +		return 49;
> +	}
> +}
> +
>   /**
>    * amdgpu_ring_alloc - allocate space on the ring buffer
>    *
> @@ -182,6 +202,7 @@ int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring,
>   	int sched_hw_submission = amdgpu_sched_hw_submission;
>   	u32 *num_sched;
>   	u32 hw_ip;
> +	unsigned int max_ibs_dw;
>   
>   	/* Set the hw submission limit higher for KIQ because
>   	 * it's used for a number of gfx/compute tasks by both
> @@ -290,6 +311,14 @@ int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring,
>   		return r;
>   	}
>   
> +	max_ibs_dw = ring->funcs->emit_frame_size +
> +		     amdgpu_ring_max_ibs(ring->funcs->type) * ring->funcs->emit_ib_size;
> +	max_ibs_dw = (max_ibs_dw + ring->funcs->align_mask) & ~ring->funcs->align_mask;
> +
> +	if (WARN_ON(max_ibs_dw > max_dw)) {
> +		max_dw = max_ibs_dw;
> +	}
> +
>   	ring->ring_size = roundup_pow_of_two(max_dw * 4 * sched_hw_submission);
>   
>   	ring->buf_mask = (ring->ring_size / 4) - 1;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> index 3989e755a5b4..e6e672727529 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> @@ -317,6 +317,7 @@ struct amdgpu_ring {
>   #define amdgpu_ring_patch_cond_exec(r,o) (r)->funcs->patch_cond_exec((r),(o))
>   #define amdgpu_ring_preempt_ib(r) (r)->funcs->preempt_ib(r)
>   
> +unsigned int amdgpu_ring_max_ibs(enum amdgpu_ring_type type);
>   int amdgpu_ring_alloc(struct amdgpu_ring *ring, unsigned ndw);
>   void amdgpu_ring_ib_begin(struct amdgpu_ring *ring);
>   void amdgpu_ring_ib_end(struct amdgpu_ring *ring);