[igt-dev] [PATCH i-g-t 1/8] lib: Introduce dynamic subsubtests
Petri Latvala
petri.latvala at intel.com
Thu Oct 24 11:05:39 UTC 2019
Dynamic subsubtests, or subtests of subtests, are individual pieces of
tests that are not statically available all the time.
A good example of a need for a dynamic subsubtest is i915 engine
listing: A normal subtest for each engine class ("bsd"), and a dynamic
subsubtest for each instance ("bsd0", "bsd2", etc). Or a normal
subtest for an operation with a dynamic subsubtest for every engine
there is.
Another example is dynamic subsubtests for pipes: Instead of using
foreach_pipe_static, make one subtest and use foreach_pipe with
dynamic subsubtests for each pipe.
v2: Rebase and adapt to igt_describe changes
v3: Rename to igt_subtest_with_dynamic_subsubtests &
igt_dynamic_subsubtest, better docs, make igt_describe fail loudly
if it's used in an impossible context.
Signed-off-by: Petri Latvala <petri.latvala at intel.com>
---
lib/igt_core.c | 148 ++++++++++++++++++++++++++++++++++++++++++-------
lib/igt_core.h | 123 ++++++++++++++++++++++++++++++++++++++++
2 files changed, 250 insertions(+), 21 deletions(-)
diff --git a/lib/igt_core.c b/lib/igt_core.c
index 7bd5afa5..9c5c698f 100644
--- a/lib/igt_core.c
+++ b/lib/igt_core.c
@@ -263,9 +263,12 @@ bool igt_skip_crc_compare;
static bool list_subtests = false;
static bool describe_subtests = false;
static char *run_single_subtest = NULL;
+static char *run_single_dynamic_subtest = NULL;
static bool run_single_subtest_found = false;
static const char *in_subtest = NULL;
+static const char *in_dynamic_subtest = NULL;
static struct timespec subtest_time;
+static struct timespec dynamic_subtest_time;
static clockid_t igt_clock = (clockid_t)-1;
static bool in_fixture = false;
static bool test_with_subtests = false;
@@ -300,6 +303,7 @@ enum {
OPT_LIST_SUBTESTS = 500,
OPT_DESCRIBE_SUBTESTS,
OPT_RUN_SUBTEST,
+ OPT_RUN_DYNAMIC_SUBTEST,
OPT_DESCRIPTION,
OPT_DEBUG,
OPT_INTERACTIVE_DEBUG,
@@ -323,6 +327,8 @@ char *igt_frame_dump_path;
static bool stderr_needs_sentinel = false;
+static int _igt_dynamic_tests_executed = -1;
+
const char *igt_test_name(void)
{
return command_str;
@@ -354,7 +360,9 @@ static void _igt_log_buffer_dump(void)
{
uint8_t i;
- if (in_subtest)
+ if (in_dynamic_subtest)
+ fprintf(stderr, "Dynamic subtest %s failed.\n", in_dynamic_subtest);
+ else if (in_subtest)
fprintf(stderr, "Subtest %s failed.\n", in_subtest);
else
fprintf(stderr, "Test %s failed.\n", command_str);
@@ -639,6 +647,7 @@ static void print_usage(const char *help_str, bool output_on_stderr)
fprintf(f, "Usage: %s [OPTIONS]\n", command_str);
fprintf(f, " --list-subtests\n"
" --run-subtest <pattern>\n"
+ " --dynamic-subtest <pattern>\n"
" --debug[=log-domain]\n"
" --interactive-debug[=domain]\n"
" --skip-crc-compare\n"
@@ -781,6 +790,7 @@ static int common_init(int *argc, char **argv,
{"list-subtests", no_argument, NULL, OPT_LIST_SUBTESTS},
{"describe", optional_argument, NULL, OPT_DESCRIBE_SUBTESTS},
{"run-subtest", required_argument, NULL, OPT_RUN_SUBTEST},
+ {"dynamic-subtest", required_argument, NULL, OPT_RUN_DYNAMIC_SUBTEST},
{"help-description", no_argument, NULL, OPT_DESCRIPTION},
{"debug", optional_argument, NULL, OPT_DEBUG},
{"interactive-debug", optional_argument, NULL, OPT_INTERACTIVE_DEBUG},
@@ -900,6 +910,11 @@ static int common_init(int *argc, char **argv,
if (!list_subtests)
run_single_subtest = strdup(optarg);
break;
+ case OPT_RUN_DYNAMIC_SUBTEST:
+ assert(optarg);
+ if (!list_subtests)
+ run_single_dynamic_subtest = strdup(optarg);
+ break;
case OPT_DESCRIPTION:
print_test_description();
ret = -1;
@@ -1084,6 +1099,19 @@ static void __igt_print_description(const char *subtest_name, const char *file,
printf("%sNO DOCUMENTATION!\n\n", indent);
}
+static bool valid_name_for_subtest(const char *subtest_name)
+{
+ int i;
+
+ /* check the subtest name only contains a-z, A-Z, 0-9, '-' and '_' */
+ for (i = 0; subtest_name[i] != '\0'; i++)
+ if (subtest_name[i] != '_' && subtest_name[i] != '-'
+ && !isalnum(subtest_name[i]))
+ return false;
+
+ return true;
+}
+
/*
* Note: Testcases which use these helpers MUST NOT output anything to stdout
* outside of places protected by igt_run_subtest checks - the piglit
@@ -1091,18 +1119,13 @@ static void __igt_print_description(const char *subtest_name, const char *file,
*/
bool __igt_run_subtest(const char *subtest_name, const char *file, const int line)
{
- int i;
-
assert(!igt_can_fail());
- /* check the subtest name only contains a-z, A-Z, 0-9, '-' and '_' */
- for (i = 0; subtest_name[i] != '\0'; i++)
- if (subtest_name[i] != '_' && subtest_name[i] != '-'
- && !isalnum(subtest_name[i])) {
- igt_critical("Invalid subtest name \"%s\".\n",
- subtest_name);
- igt_exit();
- }
+ if (!valid_name_for_subtest(subtest_name)) {
+ igt_critical("Invalid subtest name \"%s\".\n",
+ subtest_name);
+ igt_exit();
+ }
if (run_single_subtest) {
if (uwildmat(subtest_name, run_single_subtest) == 0) {
@@ -1149,6 +1172,36 @@ bool __igt_run_subtest(const char *subtest_name, const char *file, const int lin
return (in_subtest = subtest_name);
}
+bool __igt_run_dynamic_subtest(const char *dynamic_subtest_name)
+{
+ assert(in_subtest);
+ assert(_igt_dynamic_tests_executed >= 0);
+
+ if (!valid_name_for_subtest(dynamic_subtest_name)) {
+ igt_critical("Invalid dynamic subtest name \"%s\".\n",
+ dynamic_subtest_name);
+ igt_exit();
+ }
+
+ if (run_single_dynamic_subtest &&
+ uwildmat(dynamic_subtest_name, run_single_dynamic_subtest) == 0)
+ return false;
+
+ igt_kmsg(KMSG_INFO "%s: starting dynamic subtest %s\n",
+ command_str, dynamic_subtest_name);
+ igt_info("Starting dynamic subtest: %s\n", dynamic_subtest_name);
+ fflush(stdout);
+ if (stderr_needs_sentinel)
+ fprintf(stderr, "Starting dynamic subtest: %s\n", dynamic_subtest_name);
+
+ _igt_log_buffer_reset();
+
+ _igt_dynamic_tests_executed++;
+
+ igt_gettime(&dynamic_subtest_time);
+ return (in_dynamic_subtest = dynamic_subtest_name);
+}
+
/**
* igt_subtest_name:
*
@@ -1203,26 +1256,53 @@ void __igt_subtest_group_restore(int save, int desc)
static bool skipped_one = false;
static bool succeeded_one = false;
static bool failed_one = false;
+static bool dynamic_failed_one = false;
+
+bool __igt_enter_dynamic_container(void)
+{
+ _igt_dynamic_tests_executed = 0;
+ dynamic_failed_one = false;
+
+ return true;
+}
static void exit_subtest(const char *) __attribute__((noreturn));
static void exit_subtest(const char *result)
{
struct timespec now;
+ const char *subtest_text = in_dynamic_subtest ? "Dynamic subtest" : "Subtest";
+ const char **subtest_name = in_dynamic_subtest ? &in_dynamic_subtest : &in_subtest;
+ struct timespec *thentime = in_dynamic_subtest ? &dynamic_subtest_time : &subtest_time;
+ jmp_buf *jmptarget = in_dynamic_subtest ? &igt_dynamic_subsubtest_jmpbuf : &igt_subtest_jmpbuf;
igt_gettime(&now);
- igt_info("%sSubtest %s: %s (%.3fs)%s\n",
+
+ igt_info("%s%s %s: %s (%.3fs)%s\n",
(!__igt_plain_output) ? "\x1b[1m" : "",
- in_subtest, result, igt_time_elapsed(&subtest_time, &now),
+ subtest_text, *subtest_name, result,
+ igt_time_elapsed(thentime, &now),
(!__igt_plain_output) ? "\x1b[0m" : "");
fflush(stdout);
if (stderr_needs_sentinel)
- fprintf(stderr, "Subtest %s: %s (%.3fs)\n",
- in_subtest, result, igt_time_elapsed(&subtest_time, &now));
+ fprintf(stderr, "%s %s: %s (%.3fs)\n",
+ subtest_text, *subtest_name,
+ result, igt_time_elapsed(thentime, &now));
igt_terminate_spins();
- in_subtest = NULL;
- siglongjmp(igt_subtest_jmpbuf, 1);
+ if (!in_dynamic_subtest)
+ _igt_dynamic_tests_executed = -1;
+
+ /*
+ * Don't keep the above text in the log if exiting a dynamic
+ * subsubtest, the subtest would print it again otherwise
+ */
+ if (in_dynamic_subtest)
+ _igt_log_buffer_reset();
+
+ *subtest_name = NULL;
+
+ siglongjmp(*jmptarget, 1);
}
/**
@@ -1253,6 +1333,7 @@ void igt_skip(const char *f, ...)
}
if (in_subtest) {
+ /* Doing the same even if inside a dynamic subtest */
exit_subtest("SKIP");
} else if (test_with_subtests) {
skip_subtests_henceforth = SKIP;
@@ -1309,7 +1390,22 @@ void __igt_skip_check(const char *file, const int line,
*/
void igt_success(void)
{
- succeeded_one = true;
+ if (in_subtest && !in_dynamic_subtest && _igt_dynamic_tests_executed >= 0) {
+ /*
+ * We're exiting a dynamic container, yield a result
+ * according to the dynamic tests that got
+ * executed.
+ */
+ if (dynamic_failed_one)
+ igt_fail(IGT_EXIT_FAILURE);
+
+ if (_igt_dynamic_tests_executed == 0)
+ igt_skip("No dynamic tests executed.\n");
+ }
+
+ if (!in_dynamic_subtest)
+ succeeded_one = true;
+
if (in_subtest)
exit_subtest("SUCCESS");
}
@@ -1340,10 +1436,17 @@ void igt_fail(int exitcode)
if (in_atexit_handler)
_exit(IGT_EXIT_FAILURE);
- if (!failed_one)
- igt_exitcode = exitcode;
+ if (in_dynamic_subtest) {
+ dynamic_failed_one = true;
+ } else {
+ /* Dynamic subtest containers must not fail explicitly */
+ assert(_igt_dynamic_tests_executed < 0 || dynamic_failed_one);
- failed_one = true;
+ if (!failed_one)
+ igt_exitcode = exitcode;
+
+ failed_one = true;
+ }
/* Silent exit, parent will do the yelling. */
if (test_child)
@@ -1419,6 +1522,9 @@ void igt_describe_f(const char *fmt, ...)
int ret;
va_list args;
+ if (in_subtest && _igt_dynamic_tests_executed >= 0)
+ assert(!"Documenting dynamic subsubtests is impossible. Document the subtest instead.");
+
if (!describe_subtests)
return;
diff --git a/lib/igt_core.h b/lib/igt_core.h
index e8b61128..86bc1b41 100644
--- a/lib/igt_core.h
+++ b/lib/igt_core.h
@@ -146,6 +146,7 @@ void __igt_fixture_end(void) __attribute__((noreturn));
/* subtest infrastructure */
jmp_buf igt_subtest_jmpbuf;
+jmp_buf igt_dynamic_subsubtest_jmpbuf;
typedef int (*igt_opt_handler_t)(int opt, int opt_index, void *data);
#define IGT_OPT_HANDLER_SUCCESS 0
#define IGT_OPT_HANDLER_ERROR -2
@@ -177,6 +178,8 @@ int igt_subtest_init_parse_opts(int *argc, char **argv,
igt_subtest_init_parse_opts(&argc, argv, NULL, NULL, NULL, NULL, NULL);
bool __igt_run_subtest(const char *subtest_name, const char *file, const int line);
+bool __igt_enter_dynamic_container(void);
+bool __igt_run_dynamic_subtest(const char *dynamic_subtest_name);
#define __igt_tokencat2(x, y) x ## y
/**
@@ -226,6 +229,126 @@ bool __igt_run_subtest(const char *subtest_name, const char *file, const int lin
#define igt_subtest_f(f...) \
__igt_subtest_f(igt_tokencat(__tmpchar, __LINE__), f)
+/**
+ * igt_subtest_with_dynamic_subsubtests:
+ * @name: name of the subtest
+ *
+ * This is a magic control flow block which denotes a subtest code
+ * block that contains dynamic subsubtests. The _f variant accepts a
+ * printf format string, which is useful for constructing
+ * combinatorial tests.
+ *
+ * See igt_subtest_with_dynamic_subsubtests_f() for documentation.
+ */
+#define igt_subtest_with_dynamic_subsubtests(name) for (; __igt_run_subtest((name), __FILE__, __LINE__) && \
+ __igt_enter_dynamic_container() && \
+ (sigsetjmp(igt_subtest_jmpbuf, 1) == 0); \
+ igt_success())
+#define __igt_subtest_with_dynamic_subsubtests_f(tmp, format...) \
+ for (char tmp [256]; \
+ snprintf( tmp , sizeof( tmp ), \
+ format), \
+ __igt_run_subtest(tmp, __FILE__, __LINE__ ) && \
+ __igt_enter_dynamic_container() && \
+ (sigsetjmp(igt_subtest_jmpbuf, 1) == 0); \
+ igt_success())
+
+/**
+ * igt_subtest_with_dynamic_subsubtests_f:
+ * @...: format string and optional arguments
+ *
+ * This is a magic control flow block which denotes a subtest code
+ * block that contains dynamic subsubtests. The _f variant accepts a
+ * printf format string, which is useful for constructing
+ * combinatorial tests.
+ *
+ * Dynamic subsubtests are to be used when reporting several aspects
+ * of something separately is desired, but knowing the full possible
+ * set beforehand is either too big of a set or just plain
+ * impossible. Otherwise, use normal subtests. An easy example is
+ * performing an operation separately for each KMS pipe: A subtest per
+ * pipe requires iterating through all possible pipe identifiers,
+ * checking if the pipe exists for the tested device and skipping if
+ * does not, and then performing the operation. With dynamic
+ * subsubtests instead, there would be a single subtest for the
+ * operation that loops over the pipes available, enters a dynamic
+ * subsubtest for each pipe and performs the operation for that pipe
+ * in there.
+ *
+ * The result of a subtest igt_subtest_with_dynamic_subsubtests will be
+ * * SKIP, if no dynamic subsubtests are entered
+ * * PASS, if _all_ dynamic subsubtests PASS
+ * * FAIL, if _any_ dynamic subsubtests FAIL
+ *
+ * Within a igt_subtest_with_dynamic_subsubtests block, explicit
+ * failure (e.g. igt_assert) is not allowed, only dynamic subsubtests
+ * themselves will produce test results. igt_skip()/igt_require() is
+ * allowed. Example:
+ *
+ * |[<!-- language="C" -->
+ * igt_main
+ * {
+ * igt_subtest_with_dynamic_subsubtests("engine-tests") {
+ * igt_require(is_awesome(fd)); // requires ok here
+ *
+ * for_each_engine(fd, e) {
+ * igt_dynamic_subtest_f("%s", e->name) {
+ * igt_assert(works(e)); // asserts ok here
+ * }
+ * }
+ * }
+ * }
+ * ]|
+ *
+ * Like igt_subtest_with_dynamic_subsubtests(), but also accepts a printf
+ * format string instead of a static string.
+ */
+#define igt_subtest_with_dynamic_subsubtests_f(f...) \
+ __igt_subtest_with_dynamic_subsubtests_f(igt_tokencat(__tmpchar, __LINE__), f)
+
+/**
+ * igt_dynamic_subsubtest:
+ * @name: name of the dynamic subtest
+ *
+ * This is a magic control flow block which denotes a dynamic
+ * subtest-of-a-subtest code block. Within that code block
+ * igt_skip|success will only bail out of the dynamic subtest. The _f
+ * variant accepts a printf format string, which is useful for
+ * constructing combinatorial tests.
+ *
+ * See igt_subtest_with_dynamic_subsubtests_f() for documentation on
+ * dynamic subsubtests.
+ */
+#define igt_dynamic_subsubtest(name) for (; __igt_run_dynamic_subtest((name)) && \
+ (sigsetjmp(igt_dynamic_subsubtest_jmpbuf, 1) == 0); \
+ igt_success())
+#define __igt_dynamic_subsubtest_f(tmp, format...) \
+ for (char tmp [256]; \
+ snprintf( tmp , sizeof( tmp ), \
+ format), \
+ __igt_run_dynamic_subtest( tmp ) && \
+ (sigsetjmp(igt_dynamic_subsubtest_jmpbuf, 1) == 0); \
+ igt_success())
+
+/**
+ * igt_dynamic_subsubtest_f:
+ * @...: format string and optional arguments
+ *
+ * This is a magic control flow block which denotes a dynamic
+ * subtest-of-a-subtest code block. Within that code block
+ * igt_skip|success will only bail out of the dynamic subtest. The _f
+ * variant accepts a printf format string, which is useful for
+ * constructing combinatorial tests.
+ *
+ * See igt_subtest_with_dynamic_subsubtests_f() for documentation on
+ * dynamic subsubtests.
+ *
+ * Like igt_dynamic_subsubtest(), but also accepts a printf format string
+ * instead of a static string.
+ */
+#define igt_dynamic_subsubtest_f(f...) \
+ __igt_dynamic_subsubtest_f(igt_tokencat(__tmpchar, __LINE__), f)
+
const char *igt_subtest_name(void);
bool igt_only_list_subtests(void);
--
2.19.1
More information about the igt-dev
mailing list