[PATCH] tests/intel/xe_tlb: Add test to check TLB invalidation

[Ch, Sai Gowtham]

>-----Original Message-----
>From: Nirmoy Das <nirmoy.das at linux.intel.com>
>Sent: Friday, September 27, 2024 3:29 PM
>To: Ch, Sai Gowtham <sai.gowtham.ch at intel.com>; igt-dev at lists.freedesktop.org;
>Das, Nirmoy <nirmoy.das at intel.com>
>Subject: Re: [PATCH] tests/intel/xe_tlb: Add test to check TLB invalidation
>
>
>On 9/27/2024 6:35 AM, sai.gowtham.ch at intel.com wrote:
>> From: Sai Gowtham Ch <sai.gowtham.ch at intel.com>
>>
>> Test validates TLB invalidation by binding different buffer objects
>> with the same vma and submitting workload simultaneously, Ideally
>> expecting gpu to handle pages by invalidating and avoding page faults.
>>
>> Cc: Nirmoy Das <nirmoy.das at intel.com>
>> Signed-off-by: Sai Gowtham Ch <sai.gowtham.ch at intel.com>
>> ---
>>  tests/intel/xe_tlb.c | 154 +++++++++++++++++++++++++++++++++++++++++++
>>  tests/meson.build    |   1 +
>>  2 files changed, 155 insertions(+)
>>  create mode 100644 tests/intel/xe_tlb.c
>>
>> diff --git a/tests/intel/xe_tlb.c b/tests/intel/xe_tlb.c new file mode
>> 100644 index 000000000..ef44a1069
>> --- /dev/null
>> +++ b/tests/intel/xe_tlb.c
>> @@ -0,0 +1,154 @@
>> +/* SPDX-License-Identifier: MIT */
>> +/*
>> +* Copyright © 2024 Intel Corporation
>> +*
>> +* Authors:
>> +*    Sai Gowtham Ch <sai.gowtham.ch at intel.com>
>> +*/
>> +#include "igt.h"
>> +#include "lib/igt_syncobj.h"
>> +#include "xe/xe_ioctl.h"
>> +#include "xe/xe_query.h"
>> +#include "xe_drm.h"
>> +#include "igt_debugfs.h"
>> +
>> +/**
>> + * TEST: Check Translation Lookaside Buffer Invalidation.
>> + * Category: Software building block
>> + * Mega feature: General Core features
>> + * Sub-category: CMD submission
>> + * Functionality: TLB invalidate
>> + * Test category: functionality test
>> + */
>> +struct data {
>> +	uint32_t batch[16];
>> +	uint32_t data;
>> +	uint64_t addr;
>> +};
>> +
>> +static void store_dword_batch(struct data *data, uint64_t addr, int
>> +value) {
>> +	int b;
>> +	uint64_t batch_offset = (char *)&(data->batch) - (char *)data;
>> +	uint64_t batch_addr = addr + batch_offset;
>> +	uint64_t sdi_offset = (char *)&(data->data) - (char *)data;
>> +	uint64_t sdi_addr = addr + sdi_offset;
>> +
>> +	b = 0;
>> +	data->batch[b++] = MI_STORE_DWORD_IMM_GEN4;
>> +	data->batch[b++] = sdi_addr;
>> +	data->batch[b++] = sdi_addr >> 32;
>> +	data->batch[b++] = value;
>> +	data->batch[b++] = MI_BATCH_BUFFER_END;
>> +	igt_assert(b <= ARRAY_SIZE(data->batch));
>> +
>> +	data->addr = batch_addr;
>> +}
>> +
>I would move this to lib/igt_store.c  or use the one we have there.
>> +/**
>> + * SUBTEST: basic-tlb
>> + * Description: Check Translation Lookaside Buffer Invalidation.
>> + */
>> +static void tlb_invalidation(int fd, struct
>> +drm_xe_engine_class_instance *eci) {
>> +	struct drm_xe_sync sync[2] = {
>> +		{ .type = DRM_XE_SYNC_TYPE_SYNCOBJ, .flags =
>DRM_XE_SYNC_FLAG_SIGNAL, },
>> +		{ .type = DRM_XE_SYNC_TYPE_SYNCOBJ, .flags =
>DRM_XE_SYNC_FLAG_SIGNAL, }
>> +	};
>> +	struct drm_xe_exec exec = {
>> +		.num_batch_buffer = 1,
>> +		.num_syncs = 2,
>> +		.syncs = to_user_pointer(&sync),
>> +	};
>> +	struct data *data1;
>> +	struct data *data2;
>> +	uint32_t vm;
>> +	uint32_t exec_queue;
>> +	uint32_t bind_engine;
>> +	uint32_t syncobj;
>> +	size_t bo_size;
>> +	int value1 = 0x123456;
>> +	int value2 = 0x123465;
>> +	uint64_t addr = 0x100000;
>> +	uint32_t bo1, bo2;
>> +	char tlb_pre[4096], tlb_pos[4096];
>> +	char path[256];
>> +
>> +	syncobj = syncobj_create(fd, 0);
>> +	sync[0].handle = syncobj_create(fd, 0);
>> +	sync[1].handle = syncobj;
>> +
>> +	vm = xe_vm_create(fd, 0, 0);
>> +	bo_size = sizeof(*data1);
>> +	bo_size = xe_bb_size(fd, bo_size);
>> +	bo1 = xe_bo_create(fd, vm, bo_size,
>> +				   vram_if_possible(fd, eci->gt_id),
>> +
>DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM);
>> +	bo2 = xe_bo_create(fd, vm, bo_size,
>> +				   vram_if_possible(fd, eci->gt_id),
>> +
>DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM);
>> +
>> +	sprintf(path, "/sys/kernel/debug/dri/0/gt%d/stats", eci->gt_id);
>
>This should go to lib as well.
I feel checking tlb invalidation counts more generic to this test, and we can move this 
to lib as an enhancement in the future if needed. 

If you think other wise can move them to libs. 
>
>> +	igt_debugfs_dump(fd, path);
>> +	igt_debugfs_read(fd, path, tlb_pre);
>> +	igt_info("tlb invalidations counts before submitting
>> +worakloads\n%s\n", tlb_pre);
>
>We need to store this and compare before and after value. There should a fix
>number of tlb inval for a simple job submission
>
>so may be it is worth to add a check like:
>
>igt_assert_gt(num_inval, min_expected). Otherwise looks good to me.
Sure can be done.

Thanks,
Gowtham
>
>
>Regards,
>
>Nirmoy
>
>> +
>> +	exec_queue = xe_exec_queue_create(fd, vm, eci, 0);
>> +	bind_engine = xe_bind_exec_queue_create(fd, vm, 0);
>> +	xe_vm_bind_async(fd, vm, bind_engine, bo1, 0, addr, bo_size, sync, 1);
>> +	data1 = xe_bo_map(fd, bo1, bo_size);
>> +
>> +	store_dword_batch(data1, addr, value1);
>> +	exec.exec_queue_id = exec_queue;
>> +	exec.address = data1->addr;
>> +	sync[0].flags &= ~DRM_XE_SYNC_FLAG_SIGNAL;
>> +	sync[1].flags |= DRM_XE_SYNC_FLAG_SIGNAL;
>> +	xe_exec(fd, &exec);
>> +	igt_assert(syncobj_wait(fd, &syncobj, 1, INT64_MAX, 0, NULL));
>> +	xe_vm_bind_async(fd, vm, bind_engine, bo2, 0, addr, bo_size, sync, 1);
>> +	data2 = xe_bo_map(fd, bo2, bo_size);
>> +
>> +	store_dword_batch(data2, addr, value2);
>> +	exec.exec_queue_id = exec_queue;
>> +	exec.address = data2->addr;
>> +	sync[0].flags &= ~DRM_XE_SYNC_FLAG_SIGNAL;
>> +	sync[1].flags |= DRM_XE_SYNC_FLAG_SIGNAL;
>> +	xe_exec(fd, &exec);
>> +	igt_assert(syncobj_wait(fd, &syncobj, 1, INT64_MAX, 0, NULL));
>> +
>> +	igt_debugfs_dump(fd, path);
>> +	igt_debugfs_read(fd, path, tlb_pos);
>> +	igt_info("tlb invalidations counts after submitting
>> +worakloads\n%s\n", tlb_pos);
>> +
>> +	igt_assert_eq(data1->data, value1);
>> +	igt_assert_eq(data2->data, value2);
>> +	igt_assert(strcmp(tlb_pre, tlb_pos) != 0);
>> +
>> +	syncobj_destroy(fd, sync[0].handle);
>> +	syncobj_destroy(fd, syncobj);
>> +	munmap(data1, bo_size);
>> +	munmap(data2, bo_size);
>> +	gem_close(fd, bo1);
>> +	gem_close(fd, bo2);
>> +	xe_exec_queue_destroy(fd, exec_queue);
>> +	xe_vm_destroy(fd, vm);
>> +}
>> +
>> +igt_main
>> +{
>> +	int fd;
>> +	struct drm_xe_engine *engine;
>> +
>> +	igt_fixture {
>> +		fd = drm_open_driver(DRIVER_XE);
>> +	}
>> +
>> +	igt_subtest("basic-tlb") {
>> +		engine = xe_engine(fd, 0);
>> +		tlb_invalidation(fd, &engine->instance);
>> +	}
>> +
>> +	igt_fixture {
>> +		drm_close_driver(fd);
>> +	}
>> +}
>> diff --git a/tests/meson.build b/tests/meson.build index
>> e5d8852f3..17c808ab6 100644
>> --- a/tests/meson.build
>> +++ b/tests/meson.build
>> @@ -316,6 +316,7 @@ intel_xe_progs = [
>>  	'xe_sysfs_defaults',
>>  	'xe_sysfs_preempt_timeout',
>>  	'xe_sysfs_scheduler',
>> +        'xe_tlb',
>>  ]
>>
>>  intel_xe_eudebug_progs = [