[igt-dev] [PATCH i-g-t] tests/xe: Add a test for corner cases of eviction.

Maarten Lankhorst maarten.lankhorst at linux.intel.com
Thu Jun 1 11:53:56 UTC 2023


On 2023-05-31 17:58, Kumar, Janga Rahul wrote:
>
>> -----Original Message-----
>> From: igt-dev <igt-dev-bounces at lists.freedesktop.org> On Behalf Of Maarten
>> Lankhorst
>> Sent: 29 May 2023 18:18
>> To: igt-dev at lists.freedesktop.org
>> Subject: [igt-dev] [PATCH i-g-t] tests/xe: Add a test for corner cases of eviction.
>>
>> Gitlab issues 194 and 239 show some corner cases. In particular, signal handling,
>> pt eviction and losing track of external objects on the list.
>>
>> The patch series to fix those was sent as
>> https://patchwork.freedesktop.org/series/118428/
>>
>> But because it's tricky to reproduce those issues, I had to add some custom igt's
>> to handle those.
>>
>> Signed-off-by: Maarten Lankhorst <maarten.lankhorst at linux.intel.com>
>> ---
>>  tests/meson.build     |   1 +
>>  tests/xe/xe_evicted.c | 313 ++++++++++++++++++++++++++++++++++++++++++
>>  2 files changed, 314 insertions(+)
>>  create mode 100644 tests/xe/xe_evicted.c
>>
>> diff --git a/tests/meson.build b/tests/meson.build index 6551194fe..166937b6c
>> 100644
>> --- a/tests/meson.build
>> +++ b/tests/meson.build
>> @@ -248,6 +248,7 @@ xe_progs = [
>>  	'xe_dma_buf_sync',
>>  	'xe_debugfs',
>>  	'xe_evict',
>> +	'xe_evicted',
>>  	'xe_exec_balancer',
>>  	'xe_exec_basic',
>>  	'xe_exec_compute_mode',
>> diff --git a/tests/xe/xe_evicted.c b/tests/xe/xe_evicted.c new file mode 100644
>> index 000000000..3f5e6a82c
>> --- /dev/null
>> +++ b/tests/xe/xe_evicted.c
>> @@ -0,0 +1,313 @@
>> +// SPDX-License-Identifier: MIT
>> +/*
>> + * Copyright © 2023 Intel Corporation
>> + */
>> +
>> +/**
>> + * TEST: Test various eviction and ENOMEM corner cases.
>> + * Category: Software building block
>> + * Sub-category: VM_BIND / eviction
>> + * Test category: functionality test
>> + */
>> +
>> +#include "igt.h"
>> +#include "lib/igt_syncobj.h"
>> +#include "lib/intel_reg.h"
>> +#include "xe_drm.h"
>> +
>> +#include "xe/xe_ioctl.h"
>> +#include "xe/xe_query.h"
>> +#include "xe/xe_spin.h"
>> +#include <string.h>
>> +
>> +static void evict_mem(struct xe_device *xe) {
>> +	if (xe->has_vram)
>> +		igt_debugfs_write(xe->fd, "vram0_evict", "1");
>> +	else
>> +		igt_debugfs_write(xe->fd, "gtt_evict", "1"); }
>> +
>> +static void execstress(struct xe_device *xe, uint32_t engine, uint32_t
>> +addr) {
>> +	struct timespec start, end;
>> +
>> +	igt_fork(child, 2) {
>> +		struct drm_xe_sync sync = {
>> +			.flags = DRM_XE_SYNC_SYNCOBJ |
>> DRM_XE_SYNC_SIGNAL,
>> +			.handle = syncobj_create(xe->fd, 0),
>> +		};
>> +
>> +		clock_gettime(CLOCK_MONOTONIC, &start);
>> +
>> +		do {
>> +			if (!child)
>> +				evict_mem(xe);
>> +			else
>> +				xe_exec_sync(xe->fd, engine, addr, &sync, 1);
>> +
>> +			clock_gettime(CLOCK_MONOTONIC, &end);
>> +		} while (end.tv_sec - start.tv_sec <= 3);
>> +
>> +		if (child)
>> +			igt_assert(syncobj_wait(xe->fd, &sync.handle, 1,
>> INT64_MAX, 0, NULL));
>> +		syncobj_destroy(xe->fd, sync.handle);
>> +	}
>> +
>> +	igt_waitchildren();
>> +
>> +	/* Leave function in a known state */
>> +	evict_mem(xe);
>> +}
>> +
>> +/* See issue 194, unbinding first VMA may cause BO not to be locked
>> +correctly */ static void test_multimap(struct xe_device *xe, bool
>> +external) {
>> +	uint64_t size = xe->default_alignment;
>> +	uint32_t vm = xe_vm_create(xe->fd,
>> DRM_XE_VM_CREATE_SCRATCH_PAGE, 0);
>> +	uint32_t bo = xe_bo_create(xe->fd, 0, external ? 0 : vm, size);
>> +	uint32_t engine = xe_engine_create_class(xe->fd, vm,
>> DRM_XE_ENGINE_CLASS_COPY);
>> +	uint32_t *map = xe_bo_map(xe->fd, bo, size);
>> +
>> +	*map = MI_BATCH_BUFFER_END;
>> +	munmap(map, size);
>> +
>> +	/* Create 2 mappings */
>> +	xe_vm_bind_sync(xe->fd, vm, bo, 0, 0, size);
>> +	xe_vm_bind_sync(xe->fd, vm, bo, 0, size, size);
>> +
>> +	execstress(xe, engine, size);
>> +
>> +	/* Unbind the first one, to possibly confuse stuff */
>> +	xe_vm_unbind_sync(xe->fd, vm, 0, 0, size);
>> +
>> +	execstress(xe, engine, size);
>> +
>> +	/* Teardown.. */
>> +	xe_engine_destroy(xe->fd, engine);
>> +	xe_vm_destroy(xe->fd, vm);
>> +	gem_close(xe->fd, bo);
>> +}
>> +
>> +static uint64_t avail_vram(struct xe_device **xe) {
>> +	int fd = (*xe)->fd;
>> +	int region_idx;
>> +
>> +	/* Refresh to get up-to-date values */
>> +	xe_device_put(fd);
>> +	*xe = xe_device_get(fd);
>> +
>> +	region_idx = ffs((*xe)->gts->gts[0].native_mem_regions) - 1;
>> +
>> +	return (*xe)->mem_usage->regions[region_idx].total_size -
>> +	       (*xe)->mem_usage->regions[region_idx].used;
>> +}
>> +
>> +static void test_pt_eviction(struct xe_device *xe, bool external) {
>> +	uint32_t bo, i;
>> +	uint64_t size, full_size, full_addr, addr, avail;
>> +	uint32_t vm;
>> +	uint32_t engine;
>> +	uint32_t *map;
>> +	struct drm_xe_exec exec = {};
>> +	igt_require(xe->has_vram);
>> +
>> +	vm = xe_vm_create(xe->fd, 0, 0);
>> +	engine = xe_engine_create_class(xe->fd, vm,
>> DRM_XE_ENGINE_CLASS_COPY);
>> +
>> +	/* Refresh xe_device size after creating vm + engine */
>> +	evict_mem(xe);
>> +	full_size = (avail_vram(&xe) - (2 << 20ULL)) & ~((2 << 20ULL) - 1);
>> +
>> +	bo = xe_bo_create(xe->fd, 0, external ? 0 : vm, full_size);
>> +
>> +	/* Only map what we need.. */
>> +	map = xe_bo_map(xe->fd, bo, xe->default_alignment);
>> +	*map = MI_BATCH_BUFFER_END;
>> +	munmap(map, xe->default_alignment);
>> +
>> +	/* Stuff bo at the end, so we can use start for partial bo's */
>> +	full_addr = (1ULL << (uint64_t)xe_va_bits(xe->fd)) - full_size;
>> +	xe_vm_bind_sync(xe->fd, vm, bo, 0, full_addr, full_size);
>> +
>> +	exec.engine_id = engine;
>> +	exec.num_batch_buffer = 1;
>> +	exec.address = full_addr;
>> +
>> +	/* Now map partially at 0, and see how many pagetables we can still
>> fill.. */
>> +	addr = 0;
>> +	size = xe->default_alignment;
>> +	xe_vm_bind_sync(xe->fd, vm, bo, 0, addr, size);
>> +
>> +	xe_exec_sync(xe->fd, engine, full_addr, NULL, 0);
>> +
>> +	avail = full_size >> 12;
>> +	while (1) {
>> +		uint64_t new_avail = avail_vram(&xe) >> 12;
>> +
>> +		if (new_avail >= avail)
>> +			igt_info("TTM delayed destroy may have freed some
>> memory\n");
>> +		avail = new_avail;
>> +
>> +		igt_info("%"PRIu64" available 4K pages left\n", avail);
>> +		if (!avail)
>> +			break;
>> +
>> +		if (avail == 1) {
>> +			xe_vm_bind_sync(xe->fd, vm, bo, 0, 2 << 20ULL, size);
>> +			break;
>> +		}
>> +
>> +		addr += 1 << 30ULL;
>> +
>> +		/* Remove 1 for second level PT */
>> +		avail--;
>> +
>> +		for (i = 0; i < min((uint64_t)512, avail); i++) {
>> +			uint64_t this_addr = addr + (i << 21);
>> +
>> +			xe_vm_bind_sync(xe->fd, vm, bo, 0, this_addr, size);
>> +		}
>> +	}
>> +
>> +	/* Verify that with 0 memory available, exec still completes */
>> +	xe_exec_sync(xe->fd, engine, full_addr, NULL, 0);
>> +
>> +	/* Bind 1 more to go over the edge, this should never succeed.. */
>> +	if (__xe_vm_bind(xe->fd, vm, 0, bo, 0, 4 << 20ULL, size,
>> +			 XE_VM_BIND_OP_MAP, NULL, 0, 0, 0) == 0) {
>> +
>> +		/* Boom! */
>> +		igt_assert(igt_ioctl(xe->fd, DRM_IOCTL_XE_EXEC, &exec) < 0
>> &&
>> +			   errno == ENOMEM);
>> +
>> +		xe_vm_unbind_sync(xe->fd, vm, 0, 4 << 20ULL, size);
>> +	} else {
>> +		igt_assert(errno == ENOMEM);
>> +	}
>> +
>> +	/*
>> +	 * After evicting, we may end up creating new page tables,
>> +	 * so this should fail..
>> +	 */
>> +	evict_mem(xe);
>> +	igt_assert(igt_ioctl(xe->fd, DRM_IOCTL_XE_EXEC, &exec) < 0 &&
>> +		   errno == ENOMEM);
>> +
>> +	/* Cleanup */
>> +	xe_engine_destroy(xe->fd, engine);
>> +	xe_vm_destroy(xe->fd, vm);
>> +	gem_close(xe->fd, bo);
>> +}
>> +
>> +/* Test OOM with signal handler enabled, to mimic X.org (issue #239) */
> When you are checking these OOM tests, Can the igt process get killed due to OOM killer ? how are you preventing it in this test, is this test passing on any platform.
>> +static void test_signal_oom(struct xe_device *xe, bool external) {
>> +	uint32_t bo[13], num_bo = 0, i;
>> +	uint64_t binds, size = xe->has_vram ? xe->vram_size[0] :
>> igt_get_avail_ram_mb();
>> +	uint32_t vm = xe_vm_create(xe->fd,
>> DRM_XE_VM_CREATE_SCRATCH_PAGE, 0);
>> +	uint32_t engine = xe_engine_create_class(xe->fd, vm,
>> DRM_XE_ENGINE_CLASS_COPY);
>> +	int err;
>> +	struct drm_xe_gem_create create = {
>> +		.flags = vram_if_possible(xe->fd, 0),
>> +	};
>> +	struct drm_xe_sync sync[2] = {
>> +		{
>> +			.flags = DRM_XE_SYNC_SYNCOBJ |
>> DRM_XE_SYNC_SIGNAL,
>> +			.handle = syncobj_create(xe->fd, 0),
>> +		},
>> +		{
>> +			.flags = DRM_XE_SYNC_SYNCOBJ |
>> DRM_XE_SYNC_SIGNAL,
>> +			.handle = syncobj_create(xe->fd, 0),
>> +		},
>> +	};
>> +	struct drm_xe_exec exec = {
>> +		.engine_id = engine,
>> +		.syncs = (uintptr_t)sync,
>> +		.num_syncs = ARRAY_SIZE(sync),
>> +		.address = 0,
>> +		.num_batch_buffer = 1,
>> +	};
>> +
>> +	size /= 12;
>> +	size &= ~(xe->default_alignment - 1);
>> +
>> +	create.size = size;
>> +	if (!external)
>> +		create.vm_id = vm;
>> +
>> +	/* Create as many bo's as we can without OOMs */
>> +	while (num_bo < ARRAY_SIZE(bo)) {
>> +		igt_while_interruptible(true) {
>> +			if (num_bo >= ARRAY_SIZE(bo))
>> +				continue;
>> +
>> +			create.handle = 0;
>> +			err = igt_ioctl(xe->fd, DRM_IOCTL_XE_GEM_CREATE,
>> &create);
>> +			if (!err)
>> +				bo[num_bo++] = create.handle;
>> +		}
>> +
>> +		igt_assert(err == 0 || errno == ENOMEM);
>> +		if (err) {
>> +			igt_assert(!xe->has_vram || num_bo > 9);
> 			Can you comment why you are checking with num_bo with 9 ? Instead of defining bo array size as 13, if you are trying to check the vram eviction, can you use the available System memory and VRAM ,  allowed memory limit for eviction and derive these limits based on calculation so that they can work for all platforms.
I want to be able to fill at least 100% of vram, or at least 75% of the available sysram here. If we return ENOMEM it's obviously the limit is reached. The mappable limit for sysram is 50% though, if we attempt to allocate more after ENOMEM we may end up killing the process.
>> +			break;
>> +		}
>> +	}
>> +
>> +	igt_info("Created %u bo's (total: %"PRIu64" MB)\n",
>> +		 num_bo, (num_bo * size) >> 20ULL);
>> +
>> +	binds = 0;
>> +	for (i = 0; i < num_bo; i++) {
>> +		evict_mem(xe);
>> +		igt_debug("Binding bo %i, total attempts %"PRIi64"\n", i, binds);
>> +
>> +		igt_while_interruptible(true)
>> +			xe_vm_bind(xe->fd, vm, bo[i], 0, ((binds++) << 32ULL),
>> size, sync, 1);
>> +	}
>> +
>> +	sync[0].flags &= ~DRM_XE_SYNC_SIGNAL;
>> +
>> +	/* Try a few times, why not? */
>> +	for (i = 0; i < 4; i++) {
>> +		igt_while_interruptible(true) {
>> +			err = igt_ioctl(xe->fd, DRM_IOCTL_XE_EXEC, &exec);
>> +			igt_assert(err < 0 && errno == ENOMEM);
>> +		}
>> +	}
>> +
>> +	xe_vm_destroy(xe->fd, vm);
>> +	while (num_bo--)
>> +		gem_close(xe->fd, bo[num_bo]);
>> +	syncobj_destroy(xe->fd, sync[1].handle);
>> +	syncobj_destroy(xe->fd, sync[0].handle); }
>> +
>> +igt_main
>> +{
>> +	int fd;
>> +
>> +	igt_fixture
>> +		fd = drm_open_driver(DRIVER_XE);
>> +
>> +	igt_subtest("multimap-internal")
>> +		test_multimap(xe_device_get(fd), false);
>> +
>> +	igt_subtest("multimap-external")
>> +		test_multimap(xe_device_get(fd), true);
>> +
>> +	igt_subtest("pt-eviction-internal")
>> +		test_pt_eviction(xe_device_get(fd), false);
>> +
>> +	igt_subtest("pt-eviction-external")
>> +		test_pt_eviction(xe_device_get(fd), false);
> 		For testing external , change above line to test_pt_eviction(xe_device_get(fd), true);
>> +
>> +	igt_subtest("signal-oom-internal")
>> +		test_signal_oom(xe_device_get(fd), false);
>> +
>> +	igt_subtest("signal-oom-external")
>> +		test_signal_oom(xe_device_get(fd), false);
> 		For testing external , change above line to test_signal_oom(xe_device_get(fd), true);
>
> Thanks,
> Rahul

Thanks for catching!

I'll test if it works as expected on integrated too, it may fail at the VM_BIND stage..

~Maarten



More information about the igt-dev mailing list