[igt-dev] [PATCH i-g-t 1/1] runner: Add explicit "notrun" results for tests that were not executed
Petri Latvala
petri.latvala at intel.com
Tue Nov 13 13:04:30 UTC 2018
When possible, all tests we know we were going to attempt to execute
now appear in the results as "notrun". The only known case where it's
not possible to add an explicit "notrun" is when running in
multiple-mode, because "no subtests" and "run all subtests, we didn't
list them beforehand" are represented the same.
Note: A function call is commented out in resultgen.c because it needs
the function to exist, and that's in a patch that is still in flight.
Signed-off-by: Petri Latvala <petri.latvala at intel.com>
Cc: Martin Peres <martin.peres at linux.intel.com>
Cc: Arkadiusz Hiler <arkadiusz.hiler at intel.com>
---
.../joblist.txt | 4 -
.../notrun-results-multiple-mode/0/dmesg.txt | 5 +
.../notrun-results-multiple-mode/0/err.txt | 2 +
.../0/journal.txt | 2 +
.../notrun-results-multiple-mode/0/out.txt | 3 +
.../notrun-results-multiple-mode/README.txt | 4 +
.../notrun-results-multiple-mode/endtime.txt | 1 +
.../notrun-results-multiple-mode/joblist.txt | 5 +
.../notrun-results-multiple-mode/metadata.txt | 12 ++
.../reference.json | 102 +++++++++++++++
.../starttime.txt | 1 +
.../notrun-results-multiple-mode/uname.txt | 1 +
.../notrun-results/0/dmesg.txt | 5 +
.../json_tests_data/notrun-results/0/err.txt | 2 +
.../notrun-results/0/journal.txt | 2 +
.../json_tests_data/notrun-results/0/out.txt | 3 +
.../json_tests_data/notrun-results/README.txt | 4 +
.../notrun-results/endtime.txt | 1 +
.../notrun-results/joblist.txt | 5 +
.../notrun-results/metadata.txt | 12 ++
.../notrun-results/reference.json | 120 ++++++++++++++++++
.../notrun-results/starttime.txt | 1 +
.../json_tests_data/notrun-results/uname.txt | 1 +
runner/resultgen.c | 43 ++++++-
runner/runner_json_tests.c | 2 +
25 files changed, 337 insertions(+), 6 deletions(-)
create mode 100644 runner/json_tests_data/notrun-results-multiple-mode/0/dmesg.txt
create mode 100644 runner/json_tests_data/notrun-results-multiple-mode/0/err.txt
create mode 100644 runner/json_tests_data/notrun-results-multiple-mode/0/journal.txt
create mode 100644 runner/json_tests_data/notrun-results-multiple-mode/0/out.txt
create mode 100644 runner/json_tests_data/notrun-results-multiple-mode/README.txt
create mode 100644 runner/json_tests_data/notrun-results-multiple-mode/endtime.txt
create mode 100644 runner/json_tests_data/notrun-results-multiple-mode/joblist.txt
create mode 100644 runner/json_tests_data/notrun-results-multiple-mode/metadata.txt
create mode 100644 runner/json_tests_data/notrun-results-multiple-mode/reference.json
create mode 100644 runner/json_tests_data/notrun-results-multiple-mode/starttime.txt
create mode 100644 runner/json_tests_data/notrun-results-multiple-mode/uname.txt
create mode 100644 runner/json_tests_data/notrun-results/0/dmesg.txt
create mode 100644 runner/json_tests_data/notrun-results/0/err.txt
create mode 100644 runner/json_tests_data/notrun-results/0/journal.txt
create mode 100644 runner/json_tests_data/notrun-results/0/out.txt
create mode 100644 runner/json_tests_data/notrun-results/README.txt
create mode 100644 runner/json_tests_data/notrun-results/endtime.txt
create mode 100644 runner/json_tests_data/notrun-results/joblist.txt
create mode 100644 runner/json_tests_data/notrun-results/metadata.txt
create mode 100644 runner/json_tests_data/notrun-results/reference.json
create mode 100644 runner/json_tests_data/notrun-results/starttime.txt
create mode 100644 runner/json_tests_data/notrun-results/uname.txt
diff --git a/runner/json_tests_data/incomplete-before-any-subtests/joblist.txt b/runner/json_tests_data/incomplete-before-any-subtests/joblist.txt
index 31ef8413..81f914a7 100644
--- a/runner/json_tests_data/incomplete-before-any-subtests/joblist.txt
+++ b/runner/json_tests_data/incomplete-before-any-subtests/joblist.txt
@@ -1,5 +1 @@
successtest first-subtest
-successtest second-subtest
-no-subtests
-skippers skip-one
-skippers skip-two
diff --git a/runner/json_tests_data/notrun-results-multiple-mode/0/dmesg.txt b/runner/json_tests_data/notrun-results-multiple-mode/0/dmesg.txt
new file mode 100644
index 00000000..a189e704
--- /dev/null
+++ b/runner/json_tests_data/notrun-results-multiple-mode/0/dmesg.txt
@@ -0,0 +1,5 @@
+6,951,3216186095083,-;Console: switching to colour dummy device 80x25
+14,952,3216186095097,-;[IGT] successtest: executing
+14,953,3216186101115,-;[IGT] successtest: starting subtest first-subtest
+14,954,3216186101160,-;[IGT] successtest: exiting, ret=0
+6,955,3216186101299,-;Console: switching to colour frame buffer device 240x75
diff --git a/runner/json_tests_data/notrun-results-multiple-mode/0/err.txt b/runner/json_tests_data/notrun-results-multiple-mode/0/err.txt
new file mode 100644
index 00000000..5dc78057
--- /dev/null
+++ b/runner/json_tests_data/notrun-results-multiple-mode/0/err.txt
@@ -0,0 +1,2 @@
+Starting subtest: first-subtest
+Subtest first-subtest: SUCCESS (0.000s)
diff --git a/runner/json_tests_data/notrun-results-multiple-mode/0/journal.txt b/runner/json_tests_data/notrun-results-multiple-mode/0/journal.txt
new file mode 100644
index 00000000..86a30e07
--- /dev/null
+++ b/runner/json_tests_data/notrun-results-multiple-mode/0/journal.txt
@@ -0,0 +1,2 @@
+first-subtest
+exit:0 (0.014s)
diff --git a/runner/json_tests_data/notrun-results-multiple-mode/0/out.txt b/runner/json_tests_data/notrun-results-multiple-mode/0/out.txt
new file mode 100644
index 00000000..5946bf31
--- /dev/null
+++ b/runner/json_tests_data/notrun-results-multiple-mode/0/out.txt
@@ -0,0 +1,3 @@
+IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
+Starting subtest: first-subtest
+Subtest first-subtest: SUCCESS (0.000s)
diff --git a/runner/json_tests_data/notrun-results-multiple-mode/README.txt b/runner/json_tests_data/notrun-results-multiple-mode/README.txt
new file mode 100644
index 00000000..fedb6cf5
--- /dev/null
+++ b/runner/json_tests_data/notrun-results-multiple-mode/README.txt
@@ -0,0 +1,4 @@
+Tests that are not executed are expected to be present in results.json
+when possible. If run with --multiple-mode, we can't distinguish
+tests without subtests from tests where we attempt to execute all
+subtests.
diff --git a/runner/json_tests_data/notrun-results-multiple-mode/endtime.txt b/runner/json_tests_data/notrun-results-multiple-mode/endtime.txt
new file mode 100644
index 00000000..635f6ae9
--- /dev/null
+++ b/runner/json_tests_data/notrun-results-multiple-mode/endtime.txt
@@ -0,0 +1 @@
+1539953735.172373
diff --git a/runner/json_tests_data/notrun-results-multiple-mode/joblist.txt b/runner/json_tests_data/notrun-results-multiple-mode/joblist.txt
new file mode 100644
index 00000000..31ef8413
--- /dev/null
+++ b/runner/json_tests_data/notrun-results-multiple-mode/joblist.txt
@@ -0,0 +1,5 @@
+successtest first-subtest
+successtest second-subtest
+no-subtests
+skippers skip-one
+skippers skip-two
diff --git a/runner/json_tests_data/notrun-results-multiple-mode/metadata.txt b/runner/json_tests_data/notrun-results-multiple-mode/metadata.txt
new file mode 100644
index 00000000..4f027f72
--- /dev/null
+++ b/runner/json_tests_data/notrun-results-multiple-mode/metadata.txt
@@ -0,0 +1,12 @@
+abort_on_error : 0
+name : normal-run
+dry_run : 0
+sync : 0
+log_level : 0
+overwrite : 0
+multiple_mode : 1
+inactivity_timeout : 0
+use_watchdog : 0
+piglit_style_dmesg : 0
+test_root : /path/does/not/exist
+results_path : /path/does/not/exist
diff --git a/runner/json_tests_data/notrun-results-multiple-mode/reference.json b/runner/json_tests_data/notrun-results-multiple-mode/reference.json
new file mode 100644
index 00000000..b953b4a2
--- /dev/null
+++ b/runner/json_tests_data/notrun-results-multiple-mode/reference.json
@@ -0,0 +1,102 @@
+{
+ "__type__":"TestrunResult",
+ "results_version":9,
+ "name":"normal-run",
+ "uname":"Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64",
+ "time_elapsed":{
+ "__type__":"TimeAttribute",
+ "start":1539953735.1110389,
+ "end":1539953735.1723731
+ },
+ "tests":{
+ "igt at successtest@first-subtest":{
+ "out":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n",
+ "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+ "result":"pass",
+ "time":{
+ "__type__":"TimeAttribute",
+ "start":0,
+ "end":0
+ },
+ "err":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n",
+ "dmesg":"<6> [3216186.095083] Console: switching to colour dummy device 80x25\n<6> [3216186.095097] [IGT] successtest: executing\n<6> [3216186.101115] [IGT] successtest: starting subtest first-subtest\n<6> [3216186.101160] [IGT] successtest: exiting, ret=0\n<6> [3216186.101299] Console: switching to colour frame buffer device 240x75\n"
+ },
+ "igt at successtest@second-subtest":{
+ "result":"notrun",
+ "out":"",
+ "err":"",
+ "dmesg":""
+ },
+ "igt at skippers@skip-one":{
+ "result":"notrun",
+ "out":"",
+ "err":"",
+ "dmesg":""
+ },
+ "igt at skippers@skip-two":{
+ "result":"notrun",
+ "out":"",
+ "err":"",
+ "dmesg":""
+ }
+ },
+ "totals":{
+ "":{
+ "crash":0,
+ "pass":1,
+ "dmesg-fail":0,
+ "dmesg-warn":0,
+ "skip":0,
+ "incomplete":0,
+ "timeout":0,
+ "notrun":3,
+ "fail":0,
+ "warn":0
+ },
+ "root":{
+ "crash":0,
+ "pass":1,
+ "dmesg-fail":0,
+ "dmesg-warn":0,
+ "skip":0,
+ "incomplete":0,
+ "timeout":0,
+ "notrun":3,
+ "fail":0,
+ "warn":0
+ },
+ "igt at successtest":{
+ "crash":0,
+ "pass":1,
+ "dmesg-fail":0,
+ "dmesg-warn":0,
+ "skip":0,
+ "incomplete":0,
+ "timeout":0,
+ "notrun":1,
+ "fail":0,
+ "warn":0
+ },
+ "igt at skippers":{
+ "crash":0,
+ "pass":0,
+ "dmesg-fail":0,
+ "dmesg-warn":0,
+ "skip":0,
+ "incomplete":0,
+ "timeout":0,
+ "notrun":2,
+ "fail":0,
+ "warn":0
+ }
+ },
+ "runtimes":{
+ "igt at successtest":{
+ "time":{
+ "__type__":"TimeAttribute",
+ "start":0,
+ "end":0.014
+ }
+ },
+ }
+}
diff --git a/runner/json_tests_data/notrun-results-multiple-mode/starttime.txt b/runner/json_tests_data/notrun-results-multiple-mode/starttime.txt
new file mode 100644
index 00000000..ae038f18
--- /dev/null
+++ b/runner/json_tests_data/notrun-results-multiple-mode/starttime.txt
@@ -0,0 +1 @@
+1539953735.111039
diff --git a/runner/json_tests_data/notrun-results-multiple-mode/uname.txt b/runner/json_tests_data/notrun-results-multiple-mode/uname.txt
new file mode 100644
index 00000000..a7aef6f7
--- /dev/null
+++ b/runner/json_tests_data/notrun-results-multiple-mode/uname.txt
@@ -0,0 +1 @@
+Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64
diff --git a/runner/json_tests_data/notrun-results/0/dmesg.txt b/runner/json_tests_data/notrun-results/0/dmesg.txt
new file mode 100644
index 00000000..a189e704
--- /dev/null
+++ b/runner/json_tests_data/notrun-results/0/dmesg.txt
@@ -0,0 +1,5 @@
+6,951,3216186095083,-;Console: switching to colour dummy device 80x25
+14,952,3216186095097,-;[IGT] successtest: executing
+14,953,3216186101115,-;[IGT] successtest: starting subtest first-subtest
+14,954,3216186101160,-;[IGT] successtest: exiting, ret=0
+6,955,3216186101299,-;Console: switching to colour frame buffer device 240x75
diff --git a/runner/json_tests_data/notrun-results/0/err.txt b/runner/json_tests_data/notrun-results/0/err.txt
new file mode 100644
index 00000000..5dc78057
--- /dev/null
+++ b/runner/json_tests_data/notrun-results/0/err.txt
@@ -0,0 +1,2 @@
+Starting subtest: first-subtest
+Subtest first-subtest: SUCCESS (0.000s)
diff --git a/runner/json_tests_data/notrun-results/0/journal.txt b/runner/json_tests_data/notrun-results/0/journal.txt
new file mode 100644
index 00000000..86a30e07
--- /dev/null
+++ b/runner/json_tests_data/notrun-results/0/journal.txt
@@ -0,0 +1,2 @@
+first-subtest
+exit:0 (0.014s)
diff --git a/runner/json_tests_data/notrun-results/0/out.txt b/runner/json_tests_data/notrun-results/0/out.txt
new file mode 100644
index 00000000..5946bf31
--- /dev/null
+++ b/runner/json_tests_data/notrun-results/0/out.txt
@@ -0,0 +1,3 @@
+IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
+Starting subtest: first-subtest
+Subtest first-subtest: SUCCESS (0.000s)
diff --git a/runner/json_tests_data/notrun-results/README.txt b/runner/json_tests_data/notrun-results/README.txt
new file mode 100644
index 00000000..4355aebd
--- /dev/null
+++ b/runner/json_tests_data/notrun-results/README.txt
@@ -0,0 +1,4 @@
+Tests that are not executed are expected to be present in results.json
+when possible. If run without --multiple-mode, we can distinguish
+tests without subtests from tests where we attempt to execute all
+subtests.
diff --git a/runner/json_tests_data/notrun-results/endtime.txt b/runner/json_tests_data/notrun-results/endtime.txt
new file mode 100644
index 00000000..635f6ae9
--- /dev/null
+++ b/runner/json_tests_data/notrun-results/endtime.txt
@@ -0,0 +1 @@
+1539953735.172373
diff --git a/runner/json_tests_data/notrun-results/joblist.txt b/runner/json_tests_data/notrun-results/joblist.txt
new file mode 100644
index 00000000..31ef8413
--- /dev/null
+++ b/runner/json_tests_data/notrun-results/joblist.txt
@@ -0,0 +1,5 @@
+successtest first-subtest
+successtest second-subtest
+no-subtests
+skippers skip-one
+skippers skip-two
diff --git a/runner/json_tests_data/notrun-results/metadata.txt b/runner/json_tests_data/notrun-results/metadata.txt
new file mode 100644
index 00000000..1316560d
--- /dev/null
+++ b/runner/json_tests_data/notrun-results/metadata.txt
@@ -0,0 +1,12 @@
+abort_on_error : 0
+name : normal-run
+dry_run : 0
+sync : 0
+log_level : 0
+overwrite : 0
+multiple_mode : 0
+inactivity_timeout : 0
+use_watchdog : 0
+piglit_style_dmesg : 0
+test_root : /path/does/not/exist
+results_path : /path/does/not/exist
diff --git a/runner/json_tests_data/notrun-results/reference.json b/runner/json_tests_data/notrun-results/reference.json
new file mode 100644
index 00000000..4c9a5ce2
--- /dev/null
+++ b/runner/json_tests_data/notrun-results/reference.json
@@ -0,0 +1,120 @@
+{
+ "__type__":"TestrunResult",
+ "results_version":9,
+ "name":"normal-run",
+ "uname":"Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64",
+ "time_elapsed":{
+ "__type__":"TimeAttribute",
+ "start":1539953735.1110389,
+ "end":1539953735.1723731
+ },
+ "tests":{
+ "igt at successtest@first-subtest":{
+ "out":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n",
+ "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+ "result":"pass",
+ "time":{
+ "__type__":"TimeAttribute",
+ "start":0,
+ "end":0
+ },
+ "err":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n",
+ "dmesg":"<6> [3216186.095083] Console: switching to colour dummy device 80x25\n<6> [3216186.095097] [IGT] successtest: executing\n<6> [3216186.101115] [IGT] successtest: starting subtest first-subtest\n<6> [3216186.101160] [IGT] successtest: exiting, ret=0\n<6> [3216186.101299] Console: switching to colour frame buffer device 240x75\n"
+ },
+ "igt at successtest@second-subtest":{
+ "result":"notrun",
+ "out":"",
+ "err":"",
+ "dmesg":""
+ },
+ "igt at no-subtests":{
+ "result":"notrun",
+ "out":"",
+ "err":"",
+ "dmesg":""
+ },
+ "igt at skippers@skip-one":{
+ "result":"notrun",
+ "out":"",
+ "err":"",
+ "dmesg":""
+ },
+ "igt at skippers@skip-two":{
+ "result":"notrun",
+ "out":"",
+ "err":"",
+ "dmesg":""
+ }
+ },
+ "totals":{
+ "":{
+ "crash":0,
+ "pass":1,
+ "dmesg-fail":0,
+ "dmesg-warn":0,
+ "skip":0,
+ "incomplete":0,
+ "timeout":0,
+ "notrun":4,
+ "fail":0,
+ "warn":0
+ },
+ "root":{
+ "crash":0,
+ "pass":1,
+ "dmesg-fail":0,
+ "dmesg-warn":0,
+ "skip":0,
+ "incomplete":0,
+ "timeout":0,
+ "notrun":4,
+ "fail":0,
+ "warn":0
+ },
+ "igt at successtest":{
+ "crash":0,
+ "pass":1,
+ "dmesg-fail":0,
+ "dmesg-warn":0,
+ "skip":0,
+ "incomplete":0,
+ "timeout":0,
+ "notrun":1,
+ "fail":0,
+ "warn":0
+ },
+ "igt at no-subtests":{
+ "crash":0,
+ "pass":0,
+ "dmesg-fail":0,
+ "dmesg-warn":0,
+ "skip":0,
+ "incomplete":0,
+ "timeout":0,
+ "notrun":1,
+ "fail":0,
+ "warn":0
+ },
+ "igt at skippers":{
+ "crash":0,
+ "pass":0,
+ "dmesg-fail":0,
+ "dmesg-warn":0,
+ "skip":0,
+ "incomplete":0,
+ "timeout":0,
+ "notrun":2,
+ "fail":0,
+ "warn":0
+ }
+ },
+ "runtimes":{
+ "igt at successtest":{
+ "time":{
+ "__type__":"TimeAttribute",
+ "start":0,
+ "end":0.014
+ }
+ },
+ }
+}
diff --git a/runner/json_tests_data/notrun-results/starttime.txt b/runner/json_tests_data/notrun-results/starttime.txt
new file mode 100644
index 00000000..ae038f18
--- /dev/null
+++ b/runner/json_tests_data/notrun-results/starttime.txt
@@ -0,0 +1 @@
+1539953735.111039
diff --git a/runner/json_tests_data/notrun-results/uname.txt b/runner/json_tests_data/notrun-results/uname.txt
new file mode 100644
index 00000000..a7aef6f7
--- /dev/null
+++ b/runner/json_tests_data/notrun-results/uname.txt
@@ -0,0 +1 @@
+Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64
diff --git a/runner/resultgen.c b/runner/resultgen.c
index a62e400e..b7e41ec1 100644
--- a/runner/resultgen.c
+++ b/runner/resultgen.c
@@ -977,6 +977,45 @@ static bool parse_test_directory(int dirfd,
return true;
}
+static void try_add_notrun_results(const struct job_list_entry *entry,
+ const struct settings *settings,
+ struct results *results)
+{
+ struct subtests subtests = {};
+ struct json_object *current_test;
+ size_t i;
+
+ if (entry->subtest_count == 0) {
+ char piglit_name[256];
+
+ /* We cannot distinguish no-subtests from run-all-subtests in multiple-mode */
+ if (settings->multiple_mode)
+ return;
+ generate_piglit_name(entry->binary, NULL, piglit_name, sizeof(piglit_name));
+ current_test = get_or_create_json_object(results->tests, piglit_name);
+ json_object_object_add(current_test, "out", json_object_new_string(""));
+ json_object_object_add(current_test, "err", json_object_new_string(""));
+ json_object_object_add(current_test, "dmesg", json_object_new_string(""));
+ json_object_object_add(current_test, "result", json_object_new_string("notrun"));
+ }
+
+ for (i = 0; i < entry->subtest_count; i++) {
+ char piglit_name[256];
+
+ generate_piglit_name(entry->binary, entry->subtests[i], piglit_name, sizeof(piglit_name));
+ current_test = get_or_create_json_object(results->tests, piglit_name);
+ json_object_object_add(current_test, "out", json_object_new_string(""));
+ json_object_object_add(current_test, "err", json_object_new_string(""));
+ json_object_object_add(current_test, "dmesg", json_object_new_string(""));
+ json_object_object_add(current_test, "result", json_object_new_string("notrun"));
+ add_subtest(&subtests, strdup(entry->subtests[i]));
+ }
+
+ add_to_totals(entry->binary, &subtests, results);
+ // This function is appearing from another patch that is currently in review.
+ // free_subtests(&subtests);
+}
+
static void create_result_root_nodes(struct json_object *root,
struct results *results)
{
@@ -1066,8 +1105,8 @@ struct json_object *generate_results_json(int dirfd)
snprintf(name, 16, "%zd", i);
if ((testdirfd = openat(dirfd, name, O_DIRECTORY | O_RDONLY)) < 0) {
- fprintf(stderr, "Warning: Cannot open result directory %s\n", name);
- break;
+ try_add_notrun_results(&job_list.entries[i], &settings, &results);
+ continue;
}
if (!parse_test_directory(testdirfd, &job_list.entries[i], &settings, &results)) {
diff --git a/runner/runner_json_tests.c b/runner/runner_json_tests.c
index 758700d4..4584ab5f 100644
--- a/runner/runner_json_tests.c
+++ b/runner/runner_json_tests.c
@@ -154,6 +154,8 @@ static const char *dirnames[] = {
"piglit-style-dmesg",
"incomplete-before-any-subtests",
"dmesg-results",
+ "notrun-results",
+ "notrun-results-multiple-mode",
};
igt_main
--
2.18.0
More information about the igt-dev
mailing list