[PATCH i-g-t v6 18/22] tests/core_hotunplug: More thorough i915 healthcheck and recovery
Janusz Krzysztofik
janusz.krzysztofik at linux.intel.com
Mon Sep 7 13:54:49 UTC 2020
The test now assumes the i915 driver is able to identify potential
hardware or driver issues while rebinding to a device and indicate them
by marking the GPU wedged. Should that assumption occur wrong, the
health check phase of the test would happily succeed while potentially
leaving the device in an unusable state. That would not only give us
falsely positive test results but could also potentially affect
subsequently run applications. Then, we should examine health of the
exercised device more thoroughly and try harder to recover it from
potentially detected stalls.
We could use a gem_test_engine() library function which submits and
asserts successful execution of a NOP batch on each physical engine.
Unfortunately, on failure this function jumps out of an IGT test
section it is called from, while we would like to continue with
recovery steps, possibly not adding another level of test section group
nesting. Moreover, the function opens the device again and doesn't
close the extra file descriptor before the jump, while we care for
being able to close the exercised device completely before running
certain subtest operations. Then, reimplement the function locally
with those issues fixed and use it as an i915 health check. Call it
also on test startup so operations performed by the test are never
blamed for driver or hardware issues which may potentially exist and
be possible to detect on test start.
Should the i915 GPU be found unresponsive by the health check called
from a recovery section, try harder to recover it to a usable state
with a global GPU reset.
For still more effective detection of GPU hangs, use a hang detector
provided by IGT library. However, replace the library signal handler
with our own implementation that doesn't jump out of the current IGT
test section on GPU hang so we are still able to perform the reset and
retry.
v2: Skip i915 health check if a GPU hang has been already detected by a
previous health check run and not yet recovered with a GPU reset,
- take care of stopping a hang detector instance possibly left
running by a failed health check attempt.
v3: Re-run i915 health check as a first setp of i915 recovery (use full
GPU reset as a last resort),
- prefix i915 health check debug messages with step indicators,
- fix spelling error in a comment.
v4: Unbind the driver from an unhealthy device before recovery,
- drop caches on i915 health check completion.
v5: Refresh on a new patch added to the series which already unbinds
the driver form a device found unhealthy.
Signed-off-by: Janusz Krzysztofik <janusz.krzysztofik at linux.intel.com>
---
tests/core_hotunplug.c | 119 ++++++++++++++++++++++++++++++++++++-----
1 file changed, 105 insertions(+), 14 deletions(-)
diff --git a/tests/core_hotunplug.c b/tests/core_hotunplug.c
index 30e7653c1..da075a8b6 100644
--- a/tests/core_hotunplug.c
+++ b/tests/core_hotunplug.c
@@ -23,8 +23,10 @@
#include <fcntl.h>
#include <limits.h>
+#include <signal.h>
#include <stdlib.h>
#include <string.h>
+#include <sys/ioctl.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
@@ -199,10 +201,88 @@ static void cleanup(struct hotunplug *priv)
priv->fd.sysfs_dev = close_sysfs(priv->fd.sysfs_dev);
}
-static void node_healthcheck(struct hotunplug *priv, bool render)
+static bool local_i915_is_wedged(int i915)
+{
+ int err = 0;
+
+ if (ioctl(i915, DRM_IOCTL_I915_GEM_THROTTLE))
+ err = -errno;
+ return err == -EIO;
+}
+
+static bool hang_detected = false;
+
+static void local_sig_abort(int sig)
+{
+ errno = 0; /* inside a signal, last errno reporting is confusing */
+ hang_detected = true;
+}
+
+static int local_i915_healthcheck(int i915, const char *prefix)
+{
+ const uint32_t bbe = MI_BATCH_BUFFER_END;
+ struct drm_i915_gem_exec_object2 obj = { };
+ struct drm_i915_gem_execbuffer2 execbuf = {
+ .buffers_ptr = to_user_pointer(&obj),
+ .buffer_count = 1,
+ };
+ const struct intel_execution_engine2 *engine;
+
+ /* stop our hang detector possibly still running if we failed before */
+ igt_stop_hang_detector();
+
+ /* don't run again before GPU reset if hang has been already detected */
+ if (hang_detected)
+ return -EIO;
+
+ igt_debug("%srunning i915 GPU healthcheck\n", prefix);
+
+ if (local_i915_is_wedged(i915))
+ return -EIO;
+
+ obj.handle = gem_create(i915, 4096);
+ gem_write(i915, obj.handle, 0, &bbe, sizeof(bbe));
+
+ igt_fork_hang_detector(i915);
+ signal(SIGIO, local_sig_abort);
+
+ __for_each_physical_engine(i915, engine) {
+ execbuf.flags = engine->flags;
+ gem_execbuf(i915, &execbuf);
+ }
+
+ gem_sync(i915, obj.handle);
+ gem_close(i915, obj.handle);
+
+ igt_stop_hang_detector();
+ if (hang_detected)
+ return -EIO;
+
+ if (local_i915_is_wedged(i915))
+ return -EIO;
+
+ return 0;
+}
+
+static int local_i915_recover(int i915)
+{
+ hang_detected = false;
+ if (!local_i915_healthcheck(i915, "re-"))
+ return 0;
+
+ igt_debug("forcing i915 GPU reset\n");
+ igt_force_gpu_reset(i915);
+
+ hang_detected = false;
+ return local_i915_healthcheck(i915, "post-");
+}
+
+#define FLAG_RENDER (1 << 0)
+#define FLAG_RECOVER (1 << 1)
+static void node_healthcheck(struct hotunplug *priv, unsigned flags)
{
/* preserve error code potentially stored before in priv->fd.drm */
- bool closed = priv->fd.drm == -1;
+ bool closed = priv->fd.drm == -1, render = flags & FLAG_RENDER;
int fd_drm;
priv->failure = render ? "Render device reopen failure!" :
@@ -212,9 +292,19 @@ static void node_healthcheck(struct hotunplug *priv, bool render)
priv->fd.drm = fd_drm;
if (is_i915_device(fd_drm)) {
- priv->failure = "GEM failure";
- igt_require_gem(fd_drm);
- priv->failure = NULL;
+ const char *failure = NULL;
+
+ /* don't report library failed asserts as healthcheck failure */
+ priv->failure = "Unrecoverable test failure";
+
+ if (local_i915_healthcheck(fd_drm, "") &&
+ (!(flags & FLAG_RECOVER) || local_i915_recover(fd_drm)))
+ failure = "Healthcheck failure!";
+
+ gem_quiescent_gpu(fd_drm);
+
+ priv->failure = failure;
+
} else {
/* no device specific healthcheck, rely on reopen result */
priv->failure = NULL;
@@ -228,13 +318,13 @@ static void node_healthcheck(struct hotunplug *priv, bool render)
igt_fail_on_f(priv->failure, "%s\n", priv->failure);
}
-static void healthcheck(struct hotunplug *priv)
+static void healthcheck(struct hotunplug *priv, bool recover)
{
/* device name may have changed, rebuild IGT device list */
igt_devices_scan(true);
- node_healthcheck(priv, false);
- node_healthcheck(priv, true);
+ node_healthcheck(priv, recover ? FLAG_RECOVER : 0);
+ node_healthcheck(priv, FLAG_RENDER | (recover ? FLAG_RECOVER : 0));
}
static void recover(struct hotunplug *priv)
@@ -253,7 +343,7 @@ static void recover(struct hotunplug *priv)
driver_bind(priv, 60);
if (priv->failure)
- healthcheck(priv);
+ healthcheck(priv, true);
}
static void post_healthcheck(struct hotunplug *priv)
@@ -289,7 +379,7 @@ static void unbind_rebind(struct hotunplug *priv)
driver_bind(priv, 0);
- healthcheck(priv);
+ healthcheck(priv, false);
}
static void unplug_rescan(struct hotunplug *priv)
@@ -300,7 +390,7 @@ static void unplug_rescan(struct hotunplug *priv)
bus_rescan(priv, 0);
- healthcheck(priv);
+ healthcheck(priv, false);
}
static void hotunbind_lateclose(struct hotunplug *priv)
@@ -316,7 +406,7 @@ static void hotunbind_lateclose(struct hotunplug *priv)
priv->fd.drm = close_device(priv->fd.drm);
igt_assert_eq(priv->fd.drm, -1);
- healthcheck(priv);
+ healthcheck(priv, false);
}
static void hotunplug_lateclose(struct hotunplug *priv)
@@ -332,7 +422,7 @@ static void hotunplug_lateclose(struct hotunplug *priv)
priv->fd.drm = close_device(priv->fd.drm);
igt_assert_eq(priv->fd.drm, -1);
- healthcheck(priv);
+ healthcheck(priv, false);
}
/* Main */
@@ -352,7 +442,8 @@ igt_main
if (is_i915_device(fd_drm)) {
gem_quiescent_gpu(fd_drm);
- igt_require_gem(fd_drm);
+ igt_skip_on_f(local_i915_healthcheck(fd_drm, "pre-"),
+ "i915 device not healthy on test start\n");
}
/* Make sure subtests always reopen the same device */
--
2.21.1
More information about the Intel-gfx-trybot
mailing list