[igt-dev] [PATCH i-g-t v4 1/8] lib: Introduce dynamic subsubtests

Petri Latvala petri.latvala at intel.com
Mon Nov 11 10:11:35 UTC 2019


Dynamic subsubtests, or subtests of subtests, are individual pieces of
tests that are not statically available all the time.

A good example of a need for a dynamic subsubtest is i915 engine
listing: A normal subtest for each engine class ("bsd"), and a dynamic
subsubtest for each instance ("bsd0", "bsd2", etc). Or a normal
subtest for an operation with a dynamic subsubtest for every engine
there is.

Another example is dynamic subsubtests for pipes: Instead of using
foreach_pipe_static, make one subtest and use foreach_pipe with
dynamic subsubtests for each pipe.

v2: Rebase and adapt to igt_describe changes

v3: Rename to igt_subtest_with_dynamic_subsubtests &
    igt_dynamic_subsubtest, better docs, make igt_describe fail loudly
    if it's used in an impossible context.

v4: Mention dynamic blocks in the warning for longjmp in core docs.

Signed-off-by: Petri Latvala <petri.latvala at intel.com>
Reviewed-by: Arkadiusz Hiler <arkadiusz.hiler at intel.com>
---
 lib/igt_core.c | 162 ++++++++++++++++++++++++++++++++++++++++---------
 lib/igt_core.h | 123 +++++++++++++++++++++++++++++++++++++
 2 files changed, 258 insertions(+), 27 deletions(-)

diff --git a/lib/igt_core.c b/lib/igt_core.c
index 7bd5afa5..1e1b8aa1 100644
--- a/lib/igt_core.c
+++ b/lib/igt_core.c
@@ -125,12 +125,14 @@
  *   to interacting with the kernel driver must be reinitialized to avoid such
  *   issues.
  *
- * - Code blocks with magic control flow are implemented with setjmp() and
- *   longjmp(). This applies to #igt_fixture and #igt_subtest blocks and all the
- *   three variants to finish test: igt_success(), igt_skip() and igt_fail().
- *   Mostly this is of no concern, except when such a control block changes
- *   stack variables defined in the same function as the control block resides.
- *   Any store/load behaviour after a longjmp() is ill-defined for these
+ * - Code blocks with magic control flow are implemented with setjmp()
+ *   and longjmp(). This applies to #igt_fixture, #igt_subtest,
+ *   #igt_subtest_with_dynamic_subsubtests and #igt_dynamic_subsubtest
+ *   blocks and all the three variants to finish test: igt_success(),
+ *   igt_skip() and igt_fail(). Mostly this is of no concern, except
+ *   when such a control block changes stack variables defined in the
+ *   same function as the control block resides.  Any store/load
+ *   behaviour after a longjmp() is ill-defined for these
  *   variables. Avoid such code.
  *
  *   Quoting the man page for longjmp():
@@ -263,9 +265,12 @@ bool igt_skip_crc_compare;
 static bool list_subtests = false;
 static bool describe_subtests = false;
 static char *run_single_subtest = NULL;
+static char *run_single_dynamic_subtest = NULL;
 static bool run_single_subtest_found = false;
 static const char *in_subtest = NULL;
+static const char *in_dynamic_subtest = NULL;
 static struct timespec subtest_time;
+static struct timespec dynamic_subtest_time;
 static clockid_t igt_clock = (clockid_t)-1;
 static bool in_fixture = false;
 static bool test_with_subtests = false;
@@ -300,6 +305,7 @@ enum {
 	OPT_LIST_SUBTESTS = 500,
 	OPT_DESCRIBE_SUBTESTS,
 	OPT_RUN_SUBTEST,
+	OPT_RUN_DYNAMIC_SUBTEST,
 	OPT_DESCRIPTION,
 	OPT_DEBUG,
 	OPT_INTERACTIVE_DEBUG,
@@ -323,6 +329,8 @@ char *igt_frame_dump_path;
 
 static bool stderr_needs_sentinel = false;
 
+static int _igt_dynamic_tests_executed = -1;
+
 const char *igt_test_name(void)
 {
 	return command_str;
@@ -354,7 +362,9 @@ static void _igt_log_buffer_dump(void)
 {
 	uint8_t i;
 
-	if (in_subtest)
+	if (in_dynamic_subtest)
+		fprintf(stderr, "Dynamic subtest %s failed.\n", in_dynamic_subtest);
+	else if (in_subtest)
 		fprintf(stderr, "Subtest %s failed.\n", in_subtest);
 	else
 		fprintf(stderr, "Test %s failed.\n", command_str);
@@ -639,6 +649,7 @@ static void print_usage(const char *help_str, bool output_on_stderr)
 	fprintf(f, "Usage: %s [OPTIONS]\n", command_str);
 	fprintf(f, "  --list-subtests\n"
 		   "  --run-subtest <pattern>\n"
+		   "  --dynamic-subtest <pattern>\n"
 		   "  --debug[=log-domain]\n"
 		   "  --interactive-debug[=domain]\n"
 		   "  --skip-crc-compare\n"
@@ -781,6 +792,7 @@ static int common_init(int *argc, char **argv,
 		{"list-subtests",     no_argument,       NULL, OPT_LIST_SUBTESTS},
 		{"describe",          optional_argument, NULL, OPT_DESCRIBE_SUBTESTS},
 		{"run-subtest",       required_argument, NULL, OPT_RUN_SUBTEST},
+		{"dynamic-subtest",   required_argument, NULL, OPT_RUN_DYNAMIC_SUBTEST},
 		{"help-description",  no_argument,       NULL, OPT_DESCRIPTION},
 		{"debug",             optional_argument, NULL, OPT_DEBUG},
 		{"interactive-debug", optional_argument, NULL, OPT_INTERACTIVE_DEBUG},
@@ -900,6 +912,11 @@ static int common_init(int *argc, char **argv,
 			if (!list_subtests)
 				run_single_subtest = strdup(optarg);
 			break;
+		case OPT_RUN_DYNAMIC_SUBTEST:
+			assert(optarg);
+			if (!list_subtests)
+				run_single_dynamic_subtest = strdup(optarg);
+			break;
 		case OPT_DESCRIPTION:
 			print_test_description();
 			ret = -1;
@@ -1084,6 +1101,19 @@ static void __igt_print_description(const char *subtest_name, const char *file,
 		printf("%sNO DOCUMENTATION!\n\n", indent);
 }
 
+static bool valid_name_for_subtest(const char *subtest_name)
+{
+	int i;
+
+	/* check the subtest name only contains a-z, A-Z, 0-9, '-' and '_' */
+	for (i = 0; subtest_name[i] != '\0'; i++)
+		if (subtest_name[i] != '_' && subtest_name[i] != '-'
+		    && !isalnum(subtest_name[i]))
+			return false;
+
+	return true;
+}
+
 /*
  * Note: Testcases which use these helpers MUST NOT output anything to stdout
  * outside of places protected by igt_run_subtest checks - the piglit
@@ -1091,18 +1121,13 @@ static void __igt_print_description(const char *subtest_name, const char *file,
  */
 bool __igt_run_subtest(const char *subtest_name, const char *file, const int line)
 {
-	int i;
-
 	assert(!igt_can_fail());
 
-	/* check the subtest name only contains a-z, A-Z, 0-9, '-' and '_' */
-	for (i = 0; subtest_name[i] != '\0'; i++)
-		if (subtest_name[i] != '_' && subtest_name[i] != '-'
-		    && !isalnum(subtest_name[i])) {
-			igt_critical("Invalid subtest name \"%s\".\n",
-				     subtest_name);
-			igt_exit();
-		}
+	if (!valid_name_for_subtest(subtest_name)) {
+		igt_critical("Invalid subtest name \"%s\".\n",
+			     subtest_name);
+		igt_exit();
+	}
 
 	if (run_single_subtest) {
 		if (uwildmat(subtest_name, run_single_subtest) == 0) {
@@ -1149,6 +1174,36 @@ bool __igt_run_subtest(const char *subtest_name, const char *file, const int lin
 	return (in_subtest = subtest_name);
 }
 
+bool __igt_run_dynamic_subtest(const char *dynamic_subtest_name)
+{
+	assert(in_subtest);
+	assert(_igt_dynamic_tests_executed >= 0);
+
+	if (!valid_name_for_subtest(dynamic_subtest_name)) {
+			igt_critical("Invalid dynamic subtest name \"%s\".\n",
+				     dynamic_subtest_name);
+			igt_exit();
+	}
+
+	if (run_single_dynamic_subtest &&
+	    uwildmat(dynamic_subtest_name, run_single_dynamic_subtest) == 0)
+		return false;
+
+	igt_kmsg(KMSG_INFO "%s: starting dynamic subtest %s\n",
+		 command_str, dynamic_subtest_name);
+	igt_info("Starting dynamic subtest: %s\n", dynamic_subtest_name);
+	fflush(stdout);
+	if (stderr_needs_sentinel)
+		fprintf(stderr, "Starting dynamic subtest: %s\n", dynamic_subtest_name);
+
+	_igt_log_buffer_reset();
+
+	_igt_dynamic_tests_executed++;
+
+	igt_gettime(&dynamic_subtest_time);
+	return (in_dynamic_subtest = dynamic_subtest_name);
+}
+
 /**
  * igt_subtest_name:
  *
@@ -1203,26 +1258,53 @@ void __igt_subtest_group_restore(int save, int desc)
 static bool skipped_one = false;
 static bool succeeded_one = false;
 static bool failed_one = false;
+static bool dynamic_failed_one = false;
+
+bool __igt_enter_dynamic_container(void)
+{
+	_igt_dynamic_tests_executed = 0;
+	dynamic_failed_one = false;
+
+	return true;
+}
 
 static void exit_subtest(const char *) __attribute__((noreturn));
 static void exit_subtest(const char *result)
 {
 	struct timespec now;
+	const char *subtest_text = in_dynamic_subtest ? "Dynamic subtest" : "Subtest";
+	const char **subtest_name = in_dynamic_subtest ? &in_dynamic_subtest : &in_subtest;
+	struct timespec *thentime = in_dynamic_subtest ? &dynamic_subtest_time : &subtest_time;
+	jmp_buf *jmptarget = in_dynamic_subtest ? &igt_dynamic_subsubtest_jmpbuf : &igt_subtest_jmpbuf;
 
 	igt_gettime(&now);
-	igt_info("%sSubtest %s: %s (%.3fs)%s\n",
+
+	igt_info("%s%s %s: %s (%.3fs)%s\n",
 		 (!__igt_plain_output) ? "\x1b[1m" : "",
-		 in_subtest, result, igt_time_elapsed(&subtest_time, &now),
+		 subtest_text, *subtest_name, result,
+		 igt_time_elapsed(thentime, &now),
 		 (!__igt_plain_output) ? "\x1b[0m" : "");
 	fflush(stdout);
 	if (stderr_needs_sentinel)
-		fprintf(stderr, "Subtest %s: %s (%.3fs)\n",
-			in_subtest, result, igt_time_elapsed(&subtest_time, &now));
+		fprintf(stderr, "%s %s: %s (%.3fs)\n",
+			subtest_text, *subtest_name,
+			result, igt_time_elapsed(thentime, &now));
 
 	igt_terminate_spins();
 
-	in_subtest = NULL;
-	siglongjmp(igt_subtest_jmpbuf, 1);
+	if (!in_dynamic_subtest)
+		_igt_dynamic_tests_executed = -1;
+
+	/*
+	 * Don't keep the above text in the log if exiting a dynamic
+	 * subsubtest, the subtest would print it again otherwise
+	 */
+	if (in_dynamic_subtest)
+		_igt_log_buffer_reset();
+
+	*subtest_name = NULL;
+
+	siglongjmp(*jmptarget, 1);
 }
 
 /**
@@ -1253,6 +1335,7 @@ void igt_skip(const char *f, ...)
 	}
 
 	if (in_subtest) {
+		/* Doing the same even if inside a dynamic subtest */
 		exit_subtest("SKIP");
 	} else if (test_with_subtests) {
 		skip_subtests_henceforth = SKIP;
@@ -1309,7 +1392,22 @@ void __igt_skip_check(const char *file, const int line,
  */
 void igt_success(void)
 {
-	succeeded_one = true;
+	if (in_subtest && !in_dynamic_subtest && _igt_dynamic_tests_executed >= 0) {
+		/*
+		 * We're exiting a dynamic container, yield a result
+		 * according to the dynamic tests that got
+		 * executed.
+		 */
+		if (dynamic_failed_one)
+			igt_fail(IGT_EXIT_FAILURE);
+
+		if (_igt_dynamic_tests_executed == 0)
+			igt_skip("No dynamic tests executed.\n");
+	}
+
+	if (!in_dynamic_subtest)
+		succeeded_one = true;
+
 	if (in_subtest)
 		exit_subtest("SUCCESS");
 }
@@ -1340,10 +1438,17 @@ void igt_fail(int exitcode)
 	if (in_atexit_handler)
 		_exit(IGT_EXIT_FAILURE);
 
-	if (!failed_one)
-		igt_exitcode = exitcode;
+	if (in_dynamic_subtest) {
+		dynamic_failed_one = true;
+	} else {
+		/* Dynamic subtest containers must not fail explicitly */
+		assert(_igt_dynamic_tests_executed < 0 || dynamic_failed_one);
 
-	failed_one = true;
+		if (!failed_one)
+			igt_exitcode = exitcode;
+
+		failed_one = true;
+	}
 
 	/* Silent exit, parent will do the yelling. */
 	if (test_child)
@@ -1419,6 +1524,9 @@ void igt_describe_f(const char *fmt, ...)
 	int ret;
 	va_list args;
 
+	if (in_subtest && _igt_dynamic_tests_executed >= 0)
+		assert(!"Documenting dynamic subsubtests is impossible. Document the subtest instead.");
+
 	if (!describe_subtests)
 		return;
 
diff --git a/lib/igt_core.h b/lib/igt_core.h
index e8b61128..86bc1b41 100644
--- a/lib/igt_core.h
+++ b/lib/igt_core.h
@@ -146,6 +146,7 @@ void __igt_fixture_end(void) __attribute__((noreturn));
 
 /* subtest infrastructure */
 jmp_buf igt_subtest_jmpbuf;
+jmp_buf igt_dynamic_subsubtest_jmpbuf;
 typedef int (*igt_opt_handler_t)(int opt, int opt_index, void *data);
 #define IGT_OPT_HANDLER_SUCCESS 0
 #define IGT_OPT_HANDLER_ERROR -2
@@ -177,6 +178,8 @@ int igt_subtest_init_parse_opts(int *argc, char **argv,
 	igt_subtest_init_parse_opts(&argc, argv, NULL, NULL, NULL, NULL, NULL);
 
 bool __igt_run_subtest(const char *subtest_name, const char *file, const int line);
+bool __igt_enter_dynamic_container(void);
+bool __igt_run_dynamic_subtest(const char *dynamic_subtest_name);
 #define __igt_tokencat2(x, y) x ## y
 
 /**
@@ -226,6 +229,126 @@ bool __igt_run_subtest(const char *subtest_name, const char *file, const int lin
 #define igt_subtest_f(f...) \
 	__igt_subtest_f(igt_tokencat(__tmpchar, __LINE__), f)
 
+/**
+ * igt_subtest_with_dynamic_subsubtests:
+ * @name: name of the subtest
+ *
+ * This is a magic control flow block which denotes a subtest code
+ * block that contains dynamic subsubtests. The _f variant accepts a
+ * printf format string, which is useful for constructing
+ * combinatorial tests.
+ *
+ * See igt_subtest_with_dynamic_subsubtests_f() for documentation.
+ */
+#define igt_subtest_with_dynamic_subsubtests(name) for (; __igt_run_subtest((name), __FILE__, __LINE__) && \
+							 __igt_enter_dynamic_container() && \
+							 (sigsetjmp(igt_subtest_jmpbuf, 1) == 0); \
+						 igt_success())
+#define __igt_subtest_with_dynamic_subsubtests_f(tmp, format...) \
+	for (char tmp [256]; \
+	     snprintf( tmp , sizeof( tmp ), \
+		      format), \
+	       __igt_run_subtest(tmp, __FILE__, __LINE__ ) && \
+	     __igt_enter_dynamic_container() && \
+	     (sigsetjmp(igt_subtest_jmpbuf, 1) == 0); \
+	     igt_success())
+
+/**
+ * igt_subtest_with_dynamic_subsubtests_f:
+ * @...: format string and optional arguments
+ *
+ * This is a magic control flow block which denotes a subtest code
+ * block that contains dynamic subsubtests. The _f variant accepts a
+ * printf format string, which is useful for constructing
+ * combinatorial tests.
+ *
+ * Dynamic subsubtests are to be used when reporting several aspects
+ * of something separately is desired, but knowing the full possible
+ * set beforehand is either too big of a set or just plain
+ * impossible. Otherwise, use normal subtests. An easy example is
+ * performing an operation separately for each KMS pipe: A subtest per
+ * pipe requires iterating through all possible pipe identifiers,
+ * checking if the pipe exists for the tested device and skipping if
+ * does not, and then performing the operation. With dynamic
+ * subsubtests instead, there would be a single subtest for the
+ * operation that loops over the pipes available, enters a dynamic
+ * subsubtest for each pipe and performs the operation for that pipe
+ * in there.
+ *
+ * The result of a subtest igt_subtest_with_dynamic_subsubtests will be
+ * * SKIP, if no dynamic subsubtests are entered
+ * * PASS, if _all_ dynamic subsubtests PASS
+ * * FAIL, if _any_ dynamic subsubtests FAIL
+ *
+ * Within a igt_subtest_with_dynamic_subsubtests block, explicit
+ * failure (e.g. igt_assert) is not allowed, only dynamic subsubtests
+ * themselves will produce test results. igt_skip()/igt_require() is
+ * allowed. Example:
+ *
+ * |[<!-- language="C" -->
+ * igt_main
+ * {
+ *     igt_subtest_with_dynamic_subsubtests("engine-tests") {
+ *               igt_require(is_awesome(fd)); // requires ok here
+ *
+ *               for_each_engine(fd, e) {
+ *                       igt_dynamic_subtest_f("%s", e->name) {
+ *                               igt_assert(works(e)); // asserts ok here
+ *                       }
+ *               }
+ *       }
+ * }
+ * ]|
+ *
+ * Like igt_subtest_with_dynamic_subsubtests(), but also accepts a printf
+ * format string instead of a static string.
+ */
+#define igt_subtest_with_dynamic_subsubtests_f(f...) \
+	__igt_subtest_with_dynamic_subsubtests_f(igt_tokencat(__tmpchar, __LINE__), f)
+
+/**
+ * igt_dynamic_subsubtest:
+ * @name: name of the dynamic subtest
+ *
+ * This is a magic control flow block which denotes a dynamic
+ * subtest-of-a-subtest code block. Within that code block
+ * igt_skip|success will only bail out of the dynamic subtest. The _f
+ * variant accepts a printf format string, which is useful for
+ * constructing combinatorial tests.
+ *
+ * See igt_subtest_with_dynamic_subsubtests_f() for documentation on
+ * dynamic subsubtests.
+ */
+#define igt_dynamic_subsubtest(name) for (; __igt_run_dynamic_subtest((name)) && \
+					  (sigsetjmp(igt_dynamic_subsubtest_jmpbuf, 1) == 0); \
+				  igt_success())
+#define __igt_dynamic_subsubtest_f(tmp, format...) \
+	for (char tmp [256]; \
+	     snprintf( tmp , sizeof( tmp ), \
+		      format), \
+	     __igt_run_dynamic_subtest( tmp ) && \
+	     (sigsetjmp(igt_dynamic_subsubtest_jmpbuf, 1) == 0); \
+	     igt_success())
+
+/**
+ * igt_dynamic_subsubtest_f:
+ * @...: format string and optional arguments
+ *
+ * This is a magic control flow block which denotes a dynamic
+ * subtest-of-a-subtest code block. Within that code block
+ * igt_skip|success will only bail out of the dynamic subtest. The _f
+ * variant accepts a printf format string, which is useful for
+ * constructing combinatorial tests.
+ *
+ * See igt_subtest_with_dynamic_subsubtests_f() for documentation on
+ * dynamic subsubtests.
+ *
+ * Like igt_dynamic_subsubtest(), but also accepts a printf format string
+ * instead of a static string.
+ */
+#define igt_dynamic_subsubtest_f(f...) \
+	__igt_dynamic_subsubtest_f(igt_tokencat(__tmpchar, __LINE__), f)
+
 const char *igt_subtest_name(void);
 bool igt_only_list_subtests(void);
 
-- 
2.19.1



More information about the igt-dev mailing list