[igt-dev] [PATCH i-g-t 5/9] runner: Abort the run when test exits with IGT_EXIT_ABORT
Arkadiusz Hiler
arkadiusz.hiler at intel.com
Tue Feb 25 16:55:52 UTC 2020
Now that the IGT tests have a mechanism for signaling broken testing
conditions we can stop the run on the first test that has noticed it,
and possibly has triggered that state.
Traditionally run would have continued with that test failing and the
side effects would trickle down into the other tests causing a lot of
skip/fails.
v2: extra explanations, small cleanup (Petri)
Signed-off-by: Arkadiusz Hiler <arkadiusz.hiler at intel.com>
Reviewed-by: Petri Latvala <petri.latvala at intel.com>
---
runner/executor.c | 3 +
.../aborted-after-a-test/reference.json | 6 +
.../aborted-on-boot/reference.json | 6 +
.../dmesg-escapes/reference.json | 3 +
.../dmesg-results/reference.json | 5 +
.../reference.json | 3 +
.../reference.json | 3 +
.../dmesg-warn-level/reference.json | 3 +
.../reference.json | 3 +
.../dynamic-subtests/reference.json | 3 +
.../empty-result-files/reference.json | 3 +
.../reference.json | 5 +
.../json_tests_data/normal-run/reference.json | 5 +
.../reference.json | 4 +
.../notrun-results/reference.json | 5 +
.../piglit-style-dmesg/reference.json | 5 +
.../unprintable-characters/reference.json | 5 +-
.../warnings-with-dmesg-warns/reference.json | 5 +
.../json_tests_data/warnings/reference.json | 5 +
runner/resultgen.c | 31 ++
runner/runner_tests.c | 290 +++++++++++++++++-
runner/testdata/abort-dynamic.c | 46 +++
runner/testdata/abort-fixture.c | 37 +++
runner/testdata/abort-simple.c | 29 ++
runner/testdata/abort.c | 36 +++
runner/testdata/meson.build | 4 +
26 files changed, 547 insertions(+), 6 deletions(-)
create mode 100644 runner/testdata/abort-dynamic.c
create mode 100644 runner/testdata/abort-fixture.c
create mode 100644 runner/testdata/abort-simple.c
create mode 100644 runner/testdata/abort.c
diff --git a/runner/executor.c b/runner/executor.c
index 72e45b65..95c11897 100644
--- a/runner/executor.c
+++ b/runner/executor.c
@@ -1065,6 +1065,9 @@ static int monitor_output(pid_t child,
fdatasync(outputs[_F_JOURNAL]);
}
+ if (status == IGT_EXIT_ABORT)
+ aborting = true;
+
if (time_spent)
*time_spent = time;
}
diff --git a/runner/json_tests_data/aborted-after-a-test/reference.json b/runner/json_tests_data/aborted-after-a-test/reference.json
index 583242c7..0776f758 100644
--- a/runner/json_tests_data/aborted-after-a-test/reference.json
+++ b/runner/json_tests_data/aborted-after-a-test/reference.json
@@ -60,6 +60,7 @@
"dmesg-warn":0,
"skip":0,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":4,
"fail":1,
@@ -72,6 +73,7 @@
"dmesg-warn":0,
"skip":0,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":4,
"fail":1,
@@ -84,6 +86,7 @@
"dmesg-warn":0,
"skip":0,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":1,
"fail":0,
@@ -96,6 +99,7 @@
"dmesg-warn":0,
"skip":0,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":1,
"fail":0,
@@ -108,6 +112,7 @@
"dmesg-warn":0,
"skip":0,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":2,
"fail":0,
@@ -120,6 +125,7 @@
"dmesg-warn":0,
"skip":0,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":0,
"fail":1,
diff --git a/runner/json_tests_data/aborted-on-boot/reference.json b/runner/json_tests_data/aborted-on-boot/reference.json
index d354fbac..75f19466 100644
--- a/runner/json_tests_data/aborted-on-boot/reference.json
+++ b/runner/json_tests_data/aborted-on-boot/reference.json
@@ -54,6 +54,7 @@
"dmesg-warn":0,
"skip":0,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":5,
"fail":1,
@@ -66,6 +67,7 @@
"dmesg-warn":0,
"skip":0,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":5,
"fail":1,
@@ -78,6 +80,7 @@
"dmesg-warn":0,
"skip":0,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":2,
"fail":0,
@@ -90,6 +93,7 @@
"dmesg-warn":0,
"skip":0,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":1,
"fail":0,
@@ -102,6 +106,7 @@
"dmesg-warn":0,
"skip":0,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":2,
"fail":0,
@@ -114,6 +119,7 @@
"dmesg-warn":0,
"skip":0,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":0,
"fail":1,
diff --git a/runner/json_tests_data/dmesg-escapes/reference.json b/runner/json_tests_data/dmesg-escapes/reference.json
index 70d6b366..91c57310 100644
--- a/runner/json_tests_data/dmesg-escapes/reference.json
+++ b/runner/json_tests_data/dmesg-escapes/reference.json
@@ -30,6 +30,7 @@
"dmesg-warn":0,
"skip":0,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":0,
"fail":0,
@@ -42,6 +43,7 @@
"dmesg-warn":0,
"skip":0,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":0,
"fail":0,
@@ -54,6 +56,7 @@
"dmesg-warn":0,
"skip":0,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":0,
"fail":0,
diff --git a/runner/json_tests_data/dmesg-results/reference.json b/runner/json_tests_data/dmesg-results/reference.json
index 0edbae93..e9e01185 100644
--- a/runner/json_tests_data/dmesg-results/reference.json
+++ b/runner/json_tests_data/dmesg-results/reference.json
@@ -81,6 +81,7 @@
"dmesg-warn":2,
"skip":2,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":0,
"fail":0,
@@ -93,6 +94,7 @@
"dmesg-warn":2,
"skip":2,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":0,
"fail":0,
@@ -105,6 +107,7 @@
"dmesg-warn":1,
"skip":0,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":0,
"fail":0,
@@ -117,6 +120,7 @@
"dmesg-warn":1,
"skip":0,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":0,
"fail":0,
@@ -129,6 +133,7 @@
"dmesg-warn":0,
"skip":2,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":0,
"fail":0,
diff --git a/runner/json_tests_data/dmesg-warn-level-one-piglit-style/reference.json b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/reference.json
index 4ccb18ae..8d266cdf 100644
--- a/runner/json_tests_data/dmesg-warn-level-one-piglit-style/reference.json
+++ b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/reference.json
@@ -31,6 +31,7 @@
"dmesg-warn":1,
"skip":0,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":0,
"fail":0,
@@ -43,6 +44,7 @@
"dmesg-warn":1,
"skip":0,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":0,
"fail":0,
@@ -55,6 +57,7 @@
"dmesg-warn":1,
"skip":0,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":0,
"fail":0,
diff --git a/runner/json_tests_data/dmesg-warn-level-piglit-style/reference.json b/runner/json_tests_data/dmesg-warn-level-piglit-style/reference.json
index d706ee4c..4a1e8b31 100644
--- a/runner/json_tests_data/dmesg-warn-level-piglit-style/reference.json
+++ b/runner/json_tests_data/dmesg-warn-level-piglit-style/reference.json
@@ -31,6 +31,7 @@
"dmesg-warn":1,
"skip":0,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":0,
"fail":0,
@@ -43,6 +44,7 @@
"dmesg-warn":1,
"skip":0,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":0,
"fail":0,
@@ -55,6 +57,7 @@
"dmesg-warn":1,
"skip":0,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":0,
"fail":0,
diff --git a/runner/json_tests_data/dmesg-warn-level/reference.json b/runner/json_tests_data/dmesg-warn-level/reference.json
index 11cc39d9..400e9cfb 100644
--- a/runner/json_tests_data/dmesg-warn-level/reference.json
+++ b/runner/json_tests_data/dmesg-warn-level/reference.json
@@ -31,6 +31,7 @@
"dmesg-warn":1,
"skip":0,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":0,
"fail":0,
@@ -43,6 +44,7 @@
"dmesg-warn":1,
"skip":0,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":0,
"fail":0,
@@ -55,6 +57,7 @@
"dmesg-warn":1,
"skip":0,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":0,
"fail":0,
diff --git a/runner/json_tests_data/dynamic-subtest-name-in-multiple-subtests/reference.json b/runner/json_tests_data/dynamic-subtest-name-in-multiple-subtests/reference.json
index 370fce4d..514de06a 100644
--- a/runner/json_tests_data/dynamic-subtest-name-in-multiple-subtests/reference.json
+++ b/runner/json_tests_data/dynamic-subtest-name-in-multiple-subtests/reference.json
@@ -42,6 +42,7 @@
"dmesg-warn":0,
"skip":0,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":0,
"fail":0,
@@ -54,6 +55,7 @@
"dmesg-warn":0,
"skip":0,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":0,
"fail":0,
@@ -66,6 +68,7 @@
"dmesg-warn":0,
"skip":0,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":0,
"fail":0,
diff --git a/runner/json_tests_data/dynamic-subtests/reference.json b/runner/json_tests_data/dynamic-subtests/reference.json
index ca8f6cd7..c013d282 100644
--- a/runner/json_tests_data/dynamic-subtests/reference.json
+++ b/runner/json_tests_data/dynamic-subtests/reference.json
@@ -90,6 +90,7 @@
"dmesg-warn":0,
"skip":1,
"incomplete":1,
+ "abort":0,
"timeout":0,
"notrun":0,
"fail":2,
@@ -102,6 +103,7 @@
"dmesg-warn":0,
"skip":1,
"incomplete":1,
+ "abort":0,
"timeout":0,
"notrun":0,
"fail":2,
@@ -114,6 +116,7 @@
"dmesg-warn":0,
"skip":1,
"incomplete":1,
+ "abort":0,
"timeout":0,
"notrun":0,
"fail":2,
diff --git a/runner/json_tests_data/empty-result-files/reference.json b/runner/json_tests_data/empty-result-files/reference.json
index ef225601..f81ffb81 100644
--- a/runner/json_tests_data/empty-result-files/reference.json
+++ b/runner/json_tests_data/empty-result-files/reference.json
@@ -24,6 +24,7 @@
"dmesg-warn":0,
"skip":0,
"incomplete":1,
+ "abort":0,
"timeout":0,
"notrun":0,
"fail":0,
@@ -36,6 +37,7 @@
"dmesg-warn":0,
"skip":0,
"incomplete":1,
+ "abort":0,
"timeout":0,
"notrun":0,
"fail":0,
@@ -48,6 +50,7 @@
"dmesg-warn":0,
"skip":0,
"incomplete":1,
+ "abort":0,
"timeout":0,
"notrun":0,
"fail":0,
diff --git a/runner/json_tests_data/incomplete-before-any-subtests/reference.json b/runner/json_tests_data/incomplete-before-any-subtests/reference.json
index b8a76dd7..2a4bd456 100644
--- a/runner/json_tests_data/incomplete-before-any-subtests/reference.json
+++ b/runner/json_tests_data/incomplete-before-any-subtests/reference.json
@@ -49,6 +49,7 @@
"dmesg-warn":0,
"skip":0,
"incomplete":1,
+ "abort":0,
"timeout":0,
"notrun":4,
"fail":0,
@@ -61,6 +62,7 @@
"dmesg-warn":0,
"skip":0,
"incomplete":1,
+ "abort":0,
"timeout":0,
"notrun":4,
"fail":0,
@@ -73,6 +75,7 @@
"dmesg-warn":0,
"skip":0,
"incomplete":1,
+ "abort":0,
"timeout":0,
"notrun":1,
"fail":0,
@@ -85,6 +88,7 @@
"dmesg-warn":0,
"skip":0,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":1,
"fail":0,
@@ -97,6 +101,7 @@
"dmesg-warn":0,
"skip":0,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":2,
"fail":0,
diff --git a/runner/json_tests_data/normal-run/reference.json b/runner/json_tests_data/normal-run/reference.json
index 982038f9..0a00b1ca 100644
--- a/runner/json_tests_data/normal-run/reference.json
+++ b/runner/json_tests_data/normal-run/reference.json
@@ -78,6 +78,7 @@
"dmesg-warn":0,
"skip":2,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":0,
"fail":1,
@@ -90,6 +91,7 @@
"dmesg-warn":0,
"skip":2,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":0,
"fail":1,
@@ -102,6 +104,7 @@
"dmesg-warn":0,
"skip":0,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":0,
"fail":1,
@@ -114,6 +117,7 @@
"dmesg-warn":0,
"skip":0,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":0,
"fail":0,
@@ -126,6 +130,7 @@
"dmesg-warn":0,
"skip":2,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":0,
"fail":0,
diff --git a/runner/json_tests_data/notrun-results-multiple-mode/reference.json b/runner/json_tests_data/notrun-results-multiple-mode/reference.json
index 492c0a9e..3f8b7fb0 100644
--- a/runner/json_tests_data/notrun-results-multiple-mode/reference.json
+++ b/runner/json_tests_data/notrun-results-multiple-mode/reference.json
@@ -48,6 +48,7 @@
"dmesg-warn":0,
"skip":0,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":3,
"fail":0,
@@ -60,6 +61,7 @@
"dmesg-warn":0,
"skip":0,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":3,
"fail":0,
@@ -72,6 +74,7 @@
"dmesg-warn":0,
"skip":0,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":1,
"fail":0,
@@ -84,6 +87,7 @@
"dmesg-warn":0,
"skip":0,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":2,
"fail":0,
diff --git a/runner/json_tests_data/notrun-results/reference.json b/runner/json_tests_data/notrun-results/reference.json
index 49a2f693..800de38c 100644
--- a/runner/json_tests_data/notrun-results/reference.json
+++ b/runner/json_tests_data/notrun-results/reference.json
@@ -54,6 +54,7 @@
"dmesg-warn":0,
"skip":0,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":4,
"fail":0,
@@ -66,6 +67,7 @@
"dmesg-warn":0,
"skip":0,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":4,
"fail":0,
@@ -78,6 +80,7 @@
"dmesg-warn":0,
"skip":0,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":1,
"fail":0,
@@ -90,6 +93,7 @@
"dmesg-warn":0,
"skip":0,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":1,
"fail":0,
@@ -102,6 +106,7 @@
"dmesg-warn":0,
"skip":0,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":2,
"fail":0,
diff --git a/runner/json_tests_data/piglit-style-dmesg/reference.json b/runner/json_tests_data/piglit-style-dmesg/reference.json
index 45d6108e..bf5d86ee 100644
--- a/runner/json_tests_data/piglit-style-dmesg/reference.json
+++ b/runner/json_tests_data/piglit-style-dmesg/reference.json
@@ -78,6 +78,7 @@
"dmesg-warn":0,
"skip":2,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":0,
"fail":0,
@@ -90,6 +91,7 @@
"dmesg-warn":0,
"skip":2,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":0,
"fail":0,
@@ -102,6 +104,7 @@
"dmesg-warn":0,
"skip":0,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":0,
"fail":0,
@@ -114,6 +117,7 @@
"dmesg-warn":0,
"skip":0,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":0,
"fail":0,
@@ -126,6 +130,7 @@
"dmesg-warn":0,
"skip":2,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":0,
"fail":0,
diff --git a/runner/json_tests_data/unprintable-characters/reference.json b/runner/json_tests_data/unprintable-characters/reference.json
index d3add1eb..88c62c34 100644
--- a/runner/json_tests_data/unprintable-characters/reference.json
+++ b/runner/json_tests_data/unprintable-characters/reference.json
@@ -30,6 +30,7 @@
"dmesg-warn":0,
"skip":0,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":0,
"fail":0,
@@ -42,6 +43,7 @@
"dmesg-warn":0,
"skip":0,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":0,
"fail":0,
@@ -54,6 +56,7 @@
"dmesg-warn":0,
"skip":0,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":0,
"fail":0,
@@ -69,4 +72,4 @@
}
}
}
-}
\ No newline at end of file
+}
diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/reference.json b/runner/json_tests_data/warnings-with-dmesg-warns/reference.json
index fa571703..bd0bb3a3 100644
--- a/runner/json_tests_data/warnings-with-dmesg-warns/reference.json
+++ b/runner/json_tests_data/warnings-with-dmesg-warns/reference.json
@@ -79,6 +79,7 @@
"dmesg-warn":1,
"skip":2,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":0,
"fail":0,
@@ -91,6 +92,7 @@
"dmesg-warn":1,
"skip":2,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":0,
"fail":0,
@@ -103,6 +105,7 @@
"dmesg-warn":1,
"skip":0,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":0,
"fail":0,
@@ -115,6 +118,7 @@
"dmesg-warn":0,
"skip":0,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":0,
"fail":0,
@@ -127,6 +131,7 @@
"dmesg-warn":0,
"skip":2,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":0,
"fail":0,
diff --git a/runner/json_tests_data/warnings/reference.json b/runner/json_tests_data/warnings/reference.json
index 53e0c3c7..a2b79da9 100644
--- a/runner/json_tests_data/warnings/reference.json
+++ b/runner/json_tests_data/warnings/reference.json
@@ -78,6 +78,7 @@
"dmesg-warn":0,
"skip":2,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":0,
"fail":0,
@@ -90,6 +91,7 @@
"dmesg-warn":0,
"skip":2,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":0,
"fail":0,
@@ -102,6 +104,7 @@
"dmesg-warn":0,
"skip":0,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":0,
"fail":0,
@@ -114,6 +117,7 @@
"dmesg-warn":0,
"skip":0,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":0,
"fail":0,
@@ -126,6 +130,7 @@
"dmesg-warn":0,
"skip":2,
"incomplete":0,
+ "abort":0,
"timeout":0,
"notrun":0,
"fail":0,
diff --git a/runner/resultgen.c b/runner/resultgen.c
index 611d36cb..fe59aafe 100644
--- a/runner/resultgen.c
+++ b/runner/resultgen.c
@@ -645,6 +645,23 @@ static void process_dynamic_subtest_output(const char *piglit_name,
&dynresulttext, &dyntime,
dyn_result_idx < 0 ? NULL : matches.items[dyn_result_idx].where,
dynend);
+
+ /*
+ * If a dynamic subsubtest is considered incomplete we
+ * need to check parent's status first, to be sure that
+ * the binary hasn't aborted (exit code). If it has
+ * aborted then we have to attribute this status to our
+ * subsubtest.
+ */
+ if (!strcmp(dynresulttext, "incomplete")) {
+ struct json_object *parent_subtest;
+
+ if (json_object_object_get_ex(tests, piglit_name, &parent_subtest) &&
+ json_object_object_get_ex(parent_subtest, "result", &parent_subtest) &&
+ !strcmp(json_object_get_string(parent_subtest), "abort"))
+ dynresulttext = "abort";
+ }
+
set_result(current_dynamic_test, dynresulttext);
set_runtime(current_dynamic_test, dyntime);
}
@@ -1078,6 +1095,8 @@ static const char *result_from_exitcode(int exitcode)
return "pass";
case IGT_EXIT_INVALID:
return "skip";
+ case IGT_EXIT_ABORT:
+ return "abort";
case INCOMPLETE_EXITCODE:
return "incomplete";
default:
@@ -1154,6 +1173,17 @@ static void fill_from_journal(int fd,
}
}
+ if (subtests->size && exitcode == IGT_EXIT_ABORT) {
+ char *last_subtest = subtests->subs[subtests->size - 1].name;
+ char subtest_piglit_name[256];
+ struct json_object *subtest_obj;
+
+ generate_piglit_name(entry->binary, last_subtest, subtest_piglit_name, sizeof(subtest_piglit_name));
+ subtest_obj = get_or_create_json_object(tests, subtest_piglit_name);
+
+ set_result(subtest_obj, "abort");
+ }
+
if (subtests->size == 0) {
char *subtestname = NULL;
char piglit_name[256];
@@ -1322,6 +1352,7 @@ static struct json_object *get_totals_object(struct json_object *totals,
json_object_object_add(obj, "dmesg-warn", json_object_new_int(0));
json_object_object_add(obj, "skip", json_object_new_int(0));
json_object_object_add(obj, "incomplete", json_object_new_int(0));
+ json_object_object_add(obj, "abort", json_object_new_int(0));
json_object_object_add(obj, "timeout", json_object_new_int(0));
json_object_object_add(obj, "notrun", json_object_new_int(0));
json_object_object_add(obj, "fail", json_object_new_int(0));
diff --git a/runner/runner_tests.c b/runner/runner_tests.c
index 139ee9fd..60e00960 100644
--- a/runner/runner_tests.c
+++ b/runner/runner_tests.c
@@ -28,9 +28,10 @@ static const char testdatadir[] = TESTDATA_DIRECTORY;
* that test binaries without subtests should still be counted as one
* for this macro.
*/
-#define NUM_TESTDATA_SUBTESTS 6
+#define NUM_TESTDATA_SUBTESTS 15
+#define NUM_TESTDATA_ABORT_SUBTESTS 9
/* The total number of test binaries in runner/testdata/ */
-#define NUM_TESTDATA_BINARIES 4
+#define NUM_TESTDATA_BINARIES 8
static const char *igt_get_result(struct json_object *tests, const char* testname)
{
@@ -954,6 +955,7 @@ igt_main
struct execute_state state;
const char *argv[] = { "runner",
"--dry-run",
+ "-x", "^abort",
testdatadir,
dirname,
};
@@ -964,7 +966,7 @@ igt_main
igt_assert(initialize_execute_state(&state, settings, list));
igt_assert_eq(state.next, 0);
igt_assert(state.dry);
- igt_assert_eq(list->size, NUM_TESTDATA_SUBTESTS);
+ igt_assert_eq(list->size, NUM_TESTDATA_SUBTESTS - NUM_TESTDATA_ABORT_SUBTESTS);
igt_assert_f((dirfd = open(dirname, O_DIRECTORY | O_RDONLY)) >= 0,
"Dry run initialization didn't create the results directory.\n");
@@ -985,7 +987,7 @@ igt_main
igt_assert(initialize_execute_state_from_resume(dirfd, &state, settings, list));
igt_assert_eq(state.next, 0);
igt_assert(!state.dry);
- igt_assert_eq(list->size, NUM_TESTDATA_SUBTESTS);
+ igt_assert_eq(list->size, NUM_TESTDATA_SUBTESTS - NUM_TESTDATA_ABORT_SUBTESTS);
/* initialize_execute_state_from_resume() closes the dirfd */
igt_assert_f((dirfd = open(dirname, O_DIRECTORY | O_RDONLY)) >= 0,
"Dry run resume somehow deleted the results directory.\n");
@@ -1512,7 +1514,7 @@ igt_main
struct execute_state state;
struct json_object *results, *tests;
const char *argv[] = { "runner",
- "-t", "dynamic",
+ "-t", "^dynamic$",
testdatadir,
dirname,
};
@@ -1541,6 +1543,284 @@ igt_main
}
}
+ igt_subtest_group {
+ struct job_list *list = malloc(sizeof(*list));
+ volatile int dirfd = -1;
+ char dirname[] = "tmpdirXXXXXX";
+
+ igt_fixture {
+ igt_require(mkdtemp(dirname) != NULL);
+ rmdir(dirname);
+
+ init_job_list(list);
+ }
+
+ igt_subtest("execute-abort-simple") {
+ struct execute_state state;
+ struct json_object *results, *tests;
+ const char *argv[] = { "runner",
+ "-t", "^abort-simple$",
+ testdatadir,
+ dirname,
+ };
+
+ igt_assert(parse_options(ARRAY_SIZE(argv), (char**)argv, settings));
+ igt_assert(create_job_list(list, settings));
+ igt_assert(initialize_execute_state(&state, settings, list));
+ igt_assert(!execute(&state, settings, list)); /* false = signal abort */
+
+ igt_assert_f((dirfd = open(dirname, O_DIRECTORY | O_RDONLY)) >= 0,
+ "Execute didn't create the results directory\n");
+ igt_assert_f((results = generate_results_json(dirfd)) != NULL,
+ "Results parsing failed\n");
+
+ igt_assert(json_object_object_get_ex(results, "tests", &tests));
+
+ igt_assert_eqstr(igt_get_result(tests, "igt at abort-simple"), "abort");
+
+ igt_assert_eq(json_object_put(results), 1);
+ }
+
+ igt_fixture {
+ close(dirfd);
+ clear_directory(dirname);
+ free_job_list(list);
+ }
+ }
+
+ igt_subtest_group {
+ struct job_list *list = malloc(sizeof(*list));
+ volatile int dirfd = -1;
+
+ for (int multiple = 0; multiple <= 1; ++multiple) {
+ char dirname[] = "tmpdirXXXXXX";
+
+ igt_fixture {
+ igt_require(mkdtemp(dirname) != NULL);
+ rmdir(dirname);
+
+ init_job_list(list);
+ }
+
+ igt_subtest_f("execute-abort%s", multiple ? "-multiple" : "") {
+ struct execute_state state;
+ struct json_object *results, *tests;
+ const char *argv[] = { "runner",
+ "-t", "^abort$",
+ multiple ? "--multiple-mode" : "--sync",
+ testdatadir,
+ dirname,
+ };
+
+ igt_assert(parse_options(ARRAY_SIZE(argv), (char**)argv, settings));
+ igt_assert(create_job_list(list, settings));
+ igt_assert(initialize_execute_state(&state, settings, list));
+ igt_assert(!execute(&state, settings, list)); /* false = signal abort */
+
+ igt_assert_f((dirfd = open(dirname, O_DIRECTORY | O_RDONLY)) >= 0,
+ "Execute didn't create the results directory\n");
+ igt_assert_f((results = generate_results_json(dirfd)) != NULL,
+ "Results parsing failed\n");
+
+ igt_assert(json_object_object_get_ex(results, "tests", &tests));
+
+ igt_assert_eqstr(igt_get_result(tests, "igt at abort@a-subtest"), "pass");
+ igt_assert_eqstr(igt_get_result(tests, "igt at abort@b-subtest"), "abort");
+
+ if (multiple) /* no notrun injection for multiple mode */
+ igt_assert_no_result_for(tests, "igt at abort@c-subtest");
+ else
+ igt_assert_eqstr(igt_get_result(tests, "igt at abort@c-subtest"), "notrun");
+
+ igt_assert_eq(json_object_put(results), 1);
+ }
+
+ igt_fixture {
+ close(dirfd);
+ clear_directory(dirname);
+ free_job_list(list);
+ }
+ }
+ }
+
+ igt_subtest_group {
+ struct job_list *list = malloc(sizeof(*list));
+ volatile int dirfd = -1;
+
+ for (int multiple = 0; multiple <= 1; ++multiple) {
+ char dirname[] = "tmpdirXXXXXX";
+
+ igt_fixture {
+ igt_require(mkdtemp(dirname) != NULL);
+ rmdir(dirname);
+
+ init_job_list(list);
+ }
+
+ igt_subtest_f("execute-abort-fixture%s", multiple ? "-multiple" : "") {
+ struct execute_state state;
+ struct json_object *results, *tests;
+ const char *argv[] = { "runner", multiple ? "--multiple-mode" : "--sync",
+ "-t", "^abort-fixture$",
+ testdatadir,
+ dirname,
+ };
+
+ igt_assert(parse_options(ARRAY_SIZE(argv), (char**)argv, settings));
+ igt_assert(create_job_list(list, settings));
+ igt_assert(initialize_execute_state(&state, settings, list));
+ igt_assert(!execute(&state, settings, list)); /* false = signal abort */
+
+ igt_assert_f((dirfd = open(dirname, O_DIRECTORY | O_RDONLY)) >= 0,
+ "Execute didn't create the results directory\n");
+ igt_assert_f((results = generate_results_json(dirfd)) != NULL,
+ "Results parsing failed\n");
+
+ igt_assert(json_object_object_get_ex(results, "tests", &tests));
+
+ if (multiple) {
+ /*
+ * running the whole binary via -t, no
+ * way of blaming the particular subtest
+ */
+ igt_assert_eqstr(igt_get_result(tests, "igt at abort-fixture"), "abort");
+ igt_assert_no_result_for(tests, "igt at abort-fixture@a-subtest");
+ igt_assert_no_result_for(tests, "igt at abort-fixture@b-subtest");
+ } else {
+ igt_assert_eqstr(igt_get_result(tests, "igt at abort-fixture@a-subtest"), "abort");
+ igt_assert_eqstr(igt_get_result(tests, "igt at abort-fixture@b-subtest"), "notrun");
+ }
+
+ igt_assert_eq(json_object_put(results), 1);
+ }
+
+ igt_fixture {
+ close(dirfd);
+ clear_directory(dirname);
+ free_job_list(list);
+ }
+ }
+ }
+
+ igt_subtest_group {
+ struct job_list *list = malloc(sizeof(*list));
+ volatile int dirfd = -1;
+
+ for (int multiple = 0; multiple <= 1; ++multiple) {
+ char dirname[] = "tmpdirXXXXXX";
+ char filename[] = "tmplistXXXXXX";
+ const char testlisttext[] = "igt at abort-fixture@b-subtest\n"
+ "igt at abort-fixture@a-subtest\n";
+
+ igt_fixture {
+ int fd;
+ igt_require((fd = mkstemp(filename)) >= 0);
+ igt_require(write(fd, testlisttext, strlen(testlisttext)) == strlen(testlisttext));
+ close(fd);
+ igt_require(mkdtemp(dirname) != NULL);
+ rmdir(dirname);
+
+ init_job_list(list);
+ }
+
+ igt_subtest_f("execute-abort-fixture-testlist%s", multiple ? "-multiple" : "") {
+ struct execute_state state;
+ struct json_object *results, *tests;
+ const char *argv[] = { "runner", multiple ? "--multiple-mode" : "--sync",
+ "--test-list", filename,
+ testdatadir,
+ dirname,
+ };
+
+ igt_assert(parse_options(ARRAY_SIZE(argv), (char**)argv, settings));
+ igt_assert(create_job_list(list, settings));
+ igt_assert(initialize_execute_state(&state, settings, list));
+ igt_assert(!execute(&state, settings, list)); /* false = signal abort */
+
+ igt_assert_f((dirfd = open(dirname, O_DIRECTORY | O_RDONLY)) >= 0,
+ "Execute didn't create the results directory\n");
+ igt_assert_f((results = generate_results_json(dirfd)) != NULL,
+ "Results parsing failed\n");
+
+ igt_assert(json_object_object_get_ex(results, "tests", &tests));
+
+ if (multiple) /* multiple mode = no notruns */
+ igt_assert_no_result_for(tests, "igt at abort-fixture@a-subtest");
+ else
+ igt_assert_eqstr(igt_get_result(tests, "igt at abort-fixture@a-subtest"), "notrun");
+
+
+ igt_assert_eqstr(igt_get_result(tests, "igt at abort-fixture@b-subtest"), "abort");
+
+ igt_assert_eq(json_object_put(results), 1);
+ }
+
+ igt_fixture {
+ unlink(filename);
+ close(dirfd);
+ clear_directory(dirname);
+ free_job_list(list);
+ }
+ }
+ }
+
+ igt_subtest_group {
+ struct job_list *list = malloc(sizeof(*list));
+ volatile int dirfd = -1;
+
+ for (int multiple = 0; multiple <= 1; ++multiple) {
+ char dirname[] = "tmpdirXXXXXX";
+
+ igt_fixture {
+ igt_require(mkdtemp(dirname) != NULL);
+ rmdir(dirname);
+
+ init_job_list(list);
+ }
+
+ igt_subtest_f("execute-abort-dynamic%s", multiple ? "-multiple" : "") {
+ struct execute_state state;
+ struct json_object *results, *tests;
+ const char *argv[] = { "runner", multiple ? "--multiple-mode" : "--sync",
+ "-t", "^abort-dynamic$",
+ testdatadir,
+ dirname,
+ };
+
+ igt_assert(parse_options(ARRAY_SIZE(argv), (char**)argv, settings));
+ igt_assert(create_job_list(list, settings));
+ igt_assert(initialize_execute_state(&state, settings, list));
+ igt_assert(!execute(&state, settings, list)); /* false = signal abort */
+
+ igt_assert_f((dirfd = open(dirname, O_DIRECTORY | O_RDONLY)) >= 0,
+ "Execute didn't create the results directory\n");
+ igt_assert_f((results = generate_results_json(dirfd)) != NULL,
+ "Results parsing failed\n");
+
+ igt_assert(json_object_object_get_ex(results, "tests", &tests));
+
+ igt_assert_eqstr(igt_get_result(tests, "igt at abort-dynamic@a-subtest at dynamic-1"), "pass");
+ igt_assert_eqstr(igt_get_result(tests, "igt at abort-dynamic@b-subtest at dynamic-1"), "pass");
+ igt_assert_eqstr(igt_get_result(tests, "igt at abort-dynamic@b-subtest at dynamic-2"), "abort");
+
+ igt_assert_no_result_for(tests, "igt at abort-dynamic@b-subtest at dynamic-3");
+
+ if (multiple) /* multiple mode = no notruns */
+ igt_assert_no_result_for(tests, "igt at abort-dynamic@c-subtest");
+ else
+ igt_assert_eqstr(igt_get_result(tests, "igt at abort-dynamic@c-subtest"), "notrun");
+
+ igt_assert_eq(json_object_put(results), 1);
+ }
+
+ igt_fixture {
+ close(dirfd);
+ clear_directory(dirname);
+ free_job_list(list);
+ }
+ }
+ }
+
igt_subtest("file-descriptor-leakage") {
int i;
diff --git a/runner/testdata/abort-dynamic.c b/runner/testdata/abort-dynamic.c
new file mode 100644
index 00000000..66cf13a7
--- /dev/null
+++ b/runner/testdata/abort-dynamic.c
@@ -0,0 +1,46 @@
+/*
+ * Copyright © 2020 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#include "igt.h"
+
+igt_main
+{
+ igt_subtest_with_dynamic("a-subtest") {
+ igt_dynamic("dynamic-1")
+ ;
+ }
+
+ igt_subtest_with_dynamic("b-subtest") {
+ igt_dynamic("dynamic-1")
+ ;
+
+ igt_dynamic("dynamic-2")
+ igt_abort_on_f(true, "I'm out!\n");
+
+ igt_dynamic("dynamic-3")
+ ;
+ }
+
+ igt_subtest("c-subtest")
+ ;
+}
diff --git a/runner/testdata/abort-fixture.c b/runner/testdata/abort-fixture.c
new file mode 100644
index 00000000..6a1cb986
--- /dev/null
+++ b/runner/testdata/abort-fixture.c
@@ -0,0 +1,37 @@
+/*
+ * Copyright © 2020 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#include "igt.h"
+
+igt_main
+{
+ igt_fixture {
+ igt_abort_on_f(true, "I'm out!\n");
+ }
+
+ igt_subtest("a-subtest")
+ ;
+
+ igt_subtest("b-subtest")
+ ;
+}
diff --git a/runner/testdata/abort-simple.c b/runner/testdata/abort-simple.c
new file mode 100644
index 00000000..94eef72c
--- /dev/null
+++ b/runner/testdata/abort-simple.c
@@ -0,0 +1,29 @@
+/*
+ * Copyright © 2020 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#include "igt.h"
+
+igt_simple_main
+{
+ igt_abort_on_f(true, "I'm out!\n");
+}
diff --git a/runner/testdata/abort.c b/runner/testdata/abort.c
new file mode 100644
index 00000000..074946a4
--- /dev/null
+++ b/runner/testdata/abort.c
@@ -0,0 +1,36 @@
+/*
+ * Copyright © 2020 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#include "igt.h"
+
+igt_main
+{
+ igt_subtest("a-subtest")
+ ;
+
+ igt_subtest("b-subtest")
+ igt_abort_on_f(true, "I'm out!\n");
+
+ igt_subtest("c-subtest")
+ ;
+}
diff --git a/runner/testdata/meson.build b/runner/testdata/meson.build
index 631ba5b9..1f822591 100644
--- a/runner/testdata/meson.build
+++ b/runner/testdata/meson.build
@@ -3,6 +3,10 @@ testdata_progs = [ 'successtest',
'no-subtests',
'skippers',
'dynamic',
+ 'abort',
+ 'abort-dynamic',
+ 'abort-fixture',
+ 'abort-simple',
]
testdata_executables = []
--
2.24.1
More information about the igt-dev
mailing list