[PATCH 2/2] drm/amd/amdgpu : Use the MES INV_TLBS API for tlb invalidation on gfx12
Alex Deucher
alexdeucher at gmail.com
Wed Aug 6 18:05:41 UTC 2025
On Wed, Aug 6, 2025 at 2:02 PM Liu, Shaoyun <Shaoyun.Liu at amd.com> wrote:
>
> [AMD Official Use Only - AMD Internal Distribution Only]
>
> -----Original Message-----
> From: amd-gfx <amd-gfx-bounces at lists.freedesktop.org> On Behalf Of Alex Deucher
> Sent: Wednesday, August 6, 2025 1:45 PM
> To: Liu, Shaoyun <Shaoyun.Liu at amd.com>
> Cc: amd-gfx at lists.freedesktop.org
> Subject: Re: [PATCH 2/2] drm/amd/amdgpu : Use the MES INV_TLBS API for tlb invalidation on gfx12
>
> On Wed, Jul 30, 2025 at 10:33 AM Shaoyun Liu <shaoyun.liu at amd.com> wrote:
> >
> > From MES version 0x81, it provide the new API INV_TLBS that support
> > invalidate tlbs with PASID.
> >
> > Signed-off-by: Shaoyun Liu <shaoyun.liu at amd.com>
> > ---> drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h | 9 +++++++++
> > drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c | 15 +++++++++++++++
> > drivers/gpu/drm/amd/amdgpu/mes_v12_0.c | 24 ++++++++++++++++++++++++
> > 3 files changed, 48 insertions(+)
> >
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
> > index c0d2c195fe2e..f4c40f1aecd2 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
> > @@ -280,6 +280,12 @@ struct mes_reset_queue_input {
> > bool is_kq;
> > };
> >
> > +struct mes_inv_tlbs_pasid_input {
> > + uint16_t pasid;
> > + uint8_t hub_id;
> > + uint8_t flush_type;
> > +};
> > +
> > enum mes_misc_opcode {
> > MES_MISC_OP_WRITE_REG,
> > MES_MISC_OP_READ_REG,
> > @@ -367,6 +373,9 @@ struct amdgpu_mes_funcs {
> >
> > int (*reset_hw_queue)(struct amdgpu_mes *mes,
> > struct mes_reset_queue_input *input);
> > +
> > + int (*invalidate_tlbs_pasid)(struct amdgpu_mes *mes,
> > + struct mes_inv_tlbs_pasid_input *input);
> > };
> >
> > #define amdgpu_mes_kiq_hw_init(adev) (adev)->mes.kiq_hw_init((adev))
> > diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c
> > b/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c
> > index feb92e107af8..323ec04094ed 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c
> > +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c
> > @@ -339,6 +339,21 @@ static void gmc_v12_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev,
> > uint16_t queried;
> > int vmid, i;
> >
> > + if (adev->enable_uni_mes && adev->mes.ring[0].sched.ready &&
>
>
> maybe specify the pipe index explicitly? E.g.,,
> adev->mes.ring[AMDGPU_MES_SCHED_PIPE].sched.ready
> I always forget which index is KIQ and which is SCHED.
>
> [shaoyunl] ok .
>
> > + (adev->mes.sched_version & AMDGPU_MES_VERSION_MASK) >=
> > + 0x81) {
> > +
> > + struct mes_inv_tlbs_pasid_input input = {0};
> > + input.pasid = pasid;
> > + input.flush_type = flush_type;
> > + adev->mes.funcs->invalidate_tlbs_pasid(&adev->mes, &input);
> > + if (all_hub) {
> > + /* hub_id = 1 means for mm_hub*/
> > + input.hub_id = 1;
> > + adev->mes.funcs->invalidate_tlbs_pasid(&adev->mes, &input);
> > + }
> > + return;
> > + }
> > +
> > for (vmid = 1; vmid < 16; vmid++) {
> > bool valid;
> >
> > diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c
> > b/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c
> > index 6b222630f3fa..2e9191fffaf1 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c
> > +++ b/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c
> > @@ -108,6 +108,7 @@ static const char *mes_v12_0_opcodes[] = {
> > "SET_SE_MODE",
> > "SET_GANG_SUBMIT",
> > "SET_HW_RSRC_1",
> > + "INVALIDATE_TLBS",
> > };
> >
> > static const char *mes_v12_0_misc_opcodes[] = { @@ -879,6 +880,28 @@
> > static int mes_v12_0_reset_hw_queue(struct amdgpu_mes *mes,
> > offsetof(union MESAPI__RESET, api_status)); }
> >
> > +static int mes_v12_0_inv_tlbs_pasid(struct amdgpu_mes *mes,
> > + struct mes_inv_tlbs_pasid_input
> > +*input) {
> > + union MESAPI__INV_TLBS mes_inv_tlbs;
> > +
> > + memset(&mes_inv_tlbs, 0, sizeof(mes_inv_tlbs));
> > +
> > + mes_inv_tlbs.header.type = MES_API_TYPE_SCHEDULER;
> > + mes_inv_tlbs.header.opcode = MES_SCH_API_INV_TLBS;
> > + mes_inv_tlbs.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
> > +
> > + mes_inv_tlbs.invalidate_tlbs.inv_sel = 0;
> > + mes_inv_tlbs.invalidate_tlbs.flush_type = input->flush_type;
> > + mes_inv_tlbs.invalidate_tlbs.inv_sel_id = input->pasid;
> > + mes_inv_tlbs.invalidate_tlbs.hub_id = (uint32_t)input->hub_id;
> > +
> > + return mes_v12_0_submit_pkt_and_poll_completion(mes,
> > + AMDGPU_MES_KIQ_PIPE,
>
> Should this be AMDGPU_MES_KIQ_PIPE or AMDGPU_MES_SCHED_PIPE? This seems to differ from the check above in gmc_v12_0_flush_gpu_tlb_pasid().
>
> Alex
>
> [Shaoyun.liu] What I want is submit to the kiq pipe when unified MES is ready , the above check is for MES is ready or not. We would like all none running-list related operation been submitted to KIQ so it won't create any unnecessary delay or impact on process schedule decision . It's like original driver usage for KIQ . The sched pipe (HIQ) will be used for all user queues.
>
In that case, should we check if both pipes are ready in
gmc_v12_0_flush_gpu_tlb_pasid()?
Alex
>
> > + &mes_inv_tlbs, sizeof(mes_inv_tlbs),
> > + offsetof(union MESAPI__INV_TLBS, api_status));
> > +
> > +}
> > +
> > static const struct amdgpu_mes_funcs mes_v12_0_funcs = {
> > .add_hw_queue = mes_v12_0_add_hw_queue,
> > .remove_hw_queue = mes_v12_0_remove_hw_queue, @@ -888,6 +911,7
> > @@ static const struct amdgpu_mes_funcs mes_v12_0_funcs = {
> > .resume_gang = mes_v12_0_resume_gang,
> > .misc_op = mes_v12_0_misc_op,
> > .reset_hw_queue = mes_v12_0_reset_hw_queue,
> > + .invalidate_tlbs_pasid = mes_v12_0_inv_tlbs_pasid,
> > };
> >
> > static int mes_v12_0_allocate_ucode_buffer(struct amdgpu_device
> > *adev,
> > --
> > 2.34.1
> >
More information about the amd-gfx
mailing list