[igt-dev] [PATCH i-g-t 4/4] runner_tests: Test that dynamic subtest failure is handled correctly

[Petri Latvala]

See also: commit 0e6457f1bfe2 ("lib: Don't dump log buffer when
dynamic subtest failure is inherited")

This is quite an explicit top-to-bottom test that we don't get an
incorrect warn result for an innocent dynamic subtest. It is tested
here in runner_test instead of testing in lib/tests that the extra
lines are not printed, because not printing the extra lines is an
implementation detail that might change later. The issue is after all
about test results parsed by igt_runner.

Signed-off-by: Petri Latvala <petri.latvala at intel.com>
Cc: Arkadiusz Hiler <arkadiusz.hiler at intel.com>
---
 runner/meson.build    |  2 +-
 runner/runner_tests.c | 51 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 52 insertions(+), 1 deletion(-)

diff --git a/runner/meson.build b/runner/meson.build
index 404b8e04..7c2e8e0d 100644
--- a/runner/meson.build
+++ b/runner/meson.build
@@ -59,7 +59,7 @@ if jsonc.found()
 				 c_args : '-DTESTDATA_DIRECTORY="@0@"'.format(testdata_dir),
 				 link_with : runnerlib,
 				 install : false,
-				 dependencies : igt_deps)
+				 dependencies : [igt_deps, jsonc])
 	test('runner', runner_test, timeout : 300)
 
 	runner_json_test = executable('runner_json_test', runner_json_test_sources,
diff --git a/runner/runner_tests.c b/runner/runner_tests.c
index a232aa2b..6b0e6cc4 100644
--- a/runner/runner_tests.c
+++ b/runner/runner_tests.c
@@ -4,11 +4,14 @@
 #include <sys/types.h>
 #include <unistd.h>
 
+#include <json.h>
+
 #include "igt.h"
 
 #include "settings.h"
 #include "job_list.h"
 #include "executor.h"
+#include "resultgen.h"
 
 /*
  * NOTE: this test is using a lot of variables that are changed in igt_fixture,
@@ -1348,6 +1351,54 @@ igt_main
 			free(list);
 	}
 
+	igt_subtest_group {
+		struct job_list *list = malloc(sizeof(*list));
+		volatile int dirfd = -1;
+		char dirname[] = "tmpdirXXXXXX";
+
+		igt_fixture {
+			igt_require(mkdtemp(dirname) != NULL);
+			rmdir(dirname);
+
+			init_job_list(list);
+		}
+
+		igt_subtest("dynamic-subtest-failure-should-not-cause-warn") {
+			struct execute_state state;
+			struct json_object *results, *obj;
+			const char *argv[] = { "runner",
+					       "-t", "dynamic",
+					       testdatadir,
+					       dirname,
+			};
+
+			igt_assert(parse_options(ARRAY_SIZE(argv), (char**)argv, settings));
+			igt_assert(create_job_list(list, settings));
+			igt_assert(initialize_execute_state(&state, settings, list));
+			igt_assert(execute(&state, settings, list));
+
+			igt_assert_f((dirfd = open(dirname, O_DIRECTORY | O_RDONLY)) >= 0,
+				     "Execute didn't create the results directory\n");
+			igt_assert_f((results = generate_results_json(dirfd)) != NULL,
+				     "Results parsing failed\n");
+
+			obj = results;
+			igt_assert(json_object_object_get_ex(obj, "tests", &obj));
+			igt_assert(json_object_object_get_ex(obj, "igt at dynamic@dynamic-subtest at passing", &obj));
+			igt_assert(json_object_object_get_ex(obj, "result", &obj));
+
+			igt_assert_eqstr(json_object_get_string(obj), "pass");
+
+			igt_assert_eq(json_object_put(results), 1);
+		}
+
+		igt_fixture {
+			close(dirfd);
+			clear_directory(dirname);
+			free_job_list(list);
+		}
+	}
+
 	igt_subtest("file-descriptor-leakage") {
 		int i;
 
-- 
2.20.1