[igt-dev] [PATCH i-g-t 2/2] runner: Unit tests for json generation

Petri Latvala petri.latvala at intel.com
Tue Oct 23 12:20:24 UTC 2018


Test the results.json generation with a top-down approach: With a
directory of test run intermediary logs, check that the resulting json
would match a reference json file.

Signed-off-by: Petri Latvala <petri.latvala at intel.com>
Cc: Arkadiusz Hiler <arkadiusz.hiler at intel.com>
---
 runner/.gitignore                             |   1 +
 runner/Makefile.am                            |  14 +-
 runner/json_tests_data/README.txt             |  10 +
 .../json_tests_data/dmesg-results/0/dmesg.txt |   6 +
 .../json_tests_data/dmesg-results/0/err.txt   |   2 +
 .../dmesg-results/0/journal.txt               |   2 +
 .../json_tests_data/dmesg-results/0/out.txt   |   3 +
 .../json_tests_data/dmesg-results/1/dmesg.txt |   6 +
 .../json_tests_data/dmesg-results/1/err.txt   |   2 +
 .../dmesg-results/1/journal.txt               |   2 +
 .../json_tests_data/dmesg-results/1/out.txt   |   3 +
 .../json_tests_data/dmesg-results/2/dmesg.txt |   5 +
 .../json_tests_data/dmesg-results/2/err.txt   |   0
 .../dmesg-results/2/journal.txt               |   1 +
 .../json_tests_data/dmesg-results/2/out.txt   |   2 +
 .../json_tests_data/dmesg-results/3/dmesg.txt |   5 +
 .../json_tests_data/dmesg-results/3/err.txt   |   1 +
 .../dmesg-results/3/journal.txt               |   2 +
 .../json_tests_data/dmesg-results/3/out.txt   |   6 +
 .../json_tests_data/dmesg-results/4/dmesg.txt |   5 +
 .../json_tests_data/dmesg-results/4/err.txt   |   1 +
 .../dmesg-results/4/journal.txt               |   2 +
 .../json_tests_data/dmesg-results/4/out.txt   |   6 +
 .../json_tests_data/dmesg-results/README.txt  |   3 +
 .../json_tests_data/dmesg-results/endtime.txt |   1 +
 .../json_tests_data/dmesg-results/joblist.txt |   5 +
 .../dmesg-results/metadata.txt                |  12 ++
 .../dmesg-results/reference.json              | 161 +++++++++++++++++
 .../dmesg-results/starttime.txt               |   1 +
 .../json_tests_data/dmesg-results/uname.txt   |   1 +
 .../0/dmesg.txt                               |   2 +
 .../incomplete-before-any-subtests/0/err.txt  |   0
 .../0/journal.txt                             |   0
 .../incomplete-before-any-subtests/0/out.txt  |   1 +
 .../incomplete-before-any-subtests/README.txt |   4 +
 .../endtime.txt                               |   1 +
 .../joblist.txt                               |   5 +
 .../metadata.txt                              |  12 ++
 .../reference.json                            |  60 ++++++
 .../starttime.txt                             |   1 +
 .../incomplete-before-any-subtests/uname.txt  |   1 +
 runner/json_tests_data/normal-run/0/dmesg.txt |   5 +
 runner/json_tests_data/normal-run/0/err.txt   |   2 +
 .../json_tests_data/normal-run/0/journal.txt  |   2 +
 runner/json_tests_data/normal-run/0/out.txt   |   3 +
 runner/json_tests_data/normal-run/1/dmesg.txt |   5 +
 runner/json_tests_data/normal-run/1/err.txt   |   2 +
 .../json_tests_data/normal-run/1/journal.txt  |   2 +
 runner/json_tests_data/normal-run/1/out.txt   |   3 +
 runner/json_tests_data/normal-run/2/dmesg.txt |   4 +
 runner/json_tests_data/normal-run/2/err.txt   |   0
 .../json_tests_data/normal-run/2/journal.txt  |   1 +
 runner/json_tests_data/normal-run/2/out.txt   |   2 +
 runner/json_tests_data/normal-run/3/dmesg.txt |   4 +
 runner/json_tests_data/normal-run/3/err.txt   |   1 +
 .../json_tests_data/normal-run/3/journal.txt  |   2 +
 runner/json_tests_data/normal-run/3/out.txt   |   6 +
 runner/json_tests_data/normal-run/4/dmesg.txt |   4 +
 runner/json_tests_data/normal-run/4/err.txt   |   1 +
 .../json_tests_data/normal-run/4/journal.txt  |   2 +
 runner/json_tests_data/normal-run/4/out.txt   |   6 +
 runner/json_tests_data/normal-run/README.txt  |   1 +
 runner/json_tests_data/normal-run/endtime.txt |   1 +
 runner/json_tests_data/normal-run/joblist.txt |   5 +
 .../json_tests_data/normal-run/metadata.txt   |  12 ++
 .../json_tests_data/normal-run/reference.json | 158 ++++++++++++++++
 .../json_tests_data/normal-run/starttime.txt  |   1 +
 runner/json_tests_data/normal-run/uname.txt   |   1 +
 .../piglit-style-dmesg/0/dmesg.txt            |   6 +
 .../piglit-style-dmesg/0/err.txt              |   3 +
 .../piglit-style-dmesg/0/journal.txt          |   2 +
 .../piglit-style-dmesg/0/out.txt              |   3 +
 .../piglit-style-dmesg/1/dmesg.txt            |   5 +
 .../piglit-style-dmesg/1/err.txt              |   2 +
 .../piglit-style-dmesg/1/journal.txt          |   2 +
 .../piglit-style-dmesg/1/out.txt              |   3 +
 .../piglit-style-dmesg/2/dmesg.txt            |   4 +
 .../piglit-style-dmesg/2/err.txt              |   0
 .../piglit-style-dmesg/2/journal.txt          |   1 +
 .../piglit-style-dmesg/2/out.txt              |   2 +
 .../piglit-style-dmesg/3/dmesg.txt            |   4 +
 .../piglit-style-dmesg/3/err.txt              |   1 +
 .../piglit-style-dmesg/3/journal.txt          |   2 +
 .../piglit-style-dmesg/3/out.txt              |   6 +
 .../piglit-style-dmesg/4/dmesg.txt            |   4 +
 .../piglit-style-dmesg/4/err.txt              |   1 +
 .../piglit-style-dmesg/4/journal.txt          |   2 +
 .../piglit-style-dmesg/4/out.txt              |   6 +
 .../piglit-style-dmesg/README.txt             |   2 +
 .../piglit-style-dmesg/endtime.txt            |   1 +
 .../piglit-style-dmesg/joblist.txt            |   5 +
 .../piglit-style-dmesg/metadata.txt           |  12 ++
 .../piglit-style-dmesg/reference.json         | 158 ++++++++++++++++
 .../piglit-style-dmesg/starttime.txt          |   1 +
 .../piglit-style-dmesg/uname.txt              |   1 +
 .../warnings-with-dmesg-warns/0/dmesg.txt     |   6 +
 .../warnings-with-dmesg-warns/0/err.txt       |   3 +
 .../warnings-with-dmesg-warns/0/journal.txt   |   2 +
 .../warnings-with-dmesg-warns/0/out.txt       |   3 +
 .../warnings-with-dmesg-warns/1/dmesg.txt     |   5 +
 .../warnings-with-dmesg-warns/1/err.txt       |   2 +
 .../warnings-with-dmesg-warns/1/journal.txt   |   2 +
 .../warnings-with-dmesg-warns/1/out.txt       |   3 +
 .../warnings-with-dmesg-warns/2/dmesg.txt     |   4 +
 .../warnings-with-dmesg-warns/2/err.txt       |   0
 .../warnings-with-dmesg-warns/2/journal.txt   |   1 +
 .../warnings-with-dmesg-warns/2/out.txt       |   2 +
 .../warnings-with-dmesg-warns/3/dmesg.txt     |   4 +
 .../warnings-with-dmesg-warns/3/err.txt       |   1 +
 .../warnings-with-dmesg-warns/3/journal.txt   |   2 +
 .../warnings-with-dmesg-warns/3/out.txt       |   6 +
 .../warnings-with-dmesg-warns/4/dmesg.txt     |   4 +
 .../warnings-with-dmesg-warns/4/err.txt       |   1 +
 .../warnings-with-dmesg-warns/4/journal.txt   |   2 +
 .../warnings-with-dmesg-warns/4/out.txt       |   6 +
 .../warnings-with-dmesg-warns/README.txt      |   3 +
 .../warnings-with-dmesg-warns/endtime.txt     |   1 +
 .../warnings-with-dmesg-warns/joblist.txt     |   5 +
 .../warnings-with-dmesg-warns/metadata.txt    |  12 ++
 .../warnings-with-dmesg-warns/reference.json  | 159 ++++++++++++++++
 .../warnings-with-dmesg-warns/starttime.txt   |   1 +
 .../warnings-with-dmesg-warns/uname.txt       |   1 +
 runner/json_tests_data/warnings/0/dmesg.txt   |   5 +
 runner/json_tests_data/warnings/0/err.txt     |   3 +
 runner/json_tests_data/warnings/0/journal.txt |   2 +
 runner/json_tests_data/warnings/0/out.txt     |   3 +
 runner/json_tests_data/warnings/1/dmesg.txt   |   5 +
 runner/json_tests_data/warnings/1/err.txt     |   2 +
 runner/json_tests_data/warnings/1/journal.txt |   2 +
 runner/json_tests_data/warnings/1/out.txt     |   3 +
 runner/json_tests_data/warnings/2/dmesg.txt   |   4 +
 runner/json_tests_data/warnings/2/err.txt     |   0
 runner/json_tests_data/warnings/2/journal.txt |   1 +
 runner/json_tests_data/warnings/2/out.txt     |   2 +
 runner/json_tests_data/warnings/3/dmesg.txt   |   4 +
 runner/json_tests_data/warnings/3/err.txt     |   1 +
 runner/json_tests_data/warnings/3/journal.txt |   2 +
 runner/json_tests_data/warnings/3/out.txt     |   6 +
 runner/json_tests_data/warnings/4/dmesg.txt   |   4 +
 runner/json_tests_data/warnings/4/err.txt     |   1 +
 runner/json_tests_data/warnings/4/journal.txt |   2 +
 runner/json_tests_data/warnings/4/out.txt     |   6 +
 runner/json_tests_data/warnings/README.txt    |   2 +
 runner/json_tests_data/warnings/endtime.txt   |   1 +
 runner/json_tests_data/warnings/joblist.txt   |   5 +
 runner/json_tests_data/warnings/metadata.txt  |  12 ++
 .../json_tests_data/warnings/reference.json   | 158 ++++++++++++++++
 runner/json_tests_data/warnings/starttime.txt |   1 +
 runner/json_tests_data/warnings/uname.txt     |   1 +
 runner/meson.build                            |   8 +
 runner/runner_json_tests.c                    | 171 ++++++++++++++++++
 151 files changed, 1489 insertions(+), 2 deletions(-)
 create mode 100644 runner/json_tests_data/README.txt
 create mode 100644 runner/json_tests_data/dmesg-results/0/dmesg.txt
 create mode 100644 runner/json_tests_data/dmesg-results/0/err.txt
 create mode 100644 runner/json_tests_data/dmesg-results/0/journal.txt
 create mode 100644 runner/json_tests_data/dmesg-results/0/out.txt
 create mode 100644 runner/json_tests_data/dmesg-results/1/dmesg.txt
 create mode 100644 runner/json_tests_data/dmesg-results/1/err.txt
 create mode 100644 runner/json_tests_data/dmesg-results/1/journal.txt
 create mode 100644 runner/json_tests_data/dmesg-results/1/out.txt
 create mode 100644 runner/json_tests_data/dmesg-results/2/dmesg.txt
 create mode 100644 runner/json_tests_data/dmesg-results/2/err.txt
 create mode 100644 runner/json_tests_data/dmesg-results/2/journal.txt
 create mode 100644 runner/json_tests_data/dmesg-results/2/out.txt
 create mode 100644 runner/json_tests_data/dmesg-results/3/dmesg.txt
 create mode 100644 runner/json_tests_data/dmesg-results/3/err.txt
 create mode 100644 runner/json_tests_data/dmesg-results/3/journal.txt
 create mode 100644 runner/json_tests_data/dmesg-results/3/out.txt
 create mode 100644 runner/json_tests_data/dmesg-results/4/dmesg.txt
 create mode 100644 runner/json_tests_data/dmesg-results/4/err.txt
 create mode 100644 runner/json_tests_data/dmesg-results/4/journal.txt
 create mode 100644 runner/json_tests_data/dmesg-results/4/out.txt
 create mode 100644 runner/json_tests_data/dmesg-results/README.txt
 create mode 100644 runner/json_tests_data/dmesg-results/endtime.txt
 create mode 100644 runner/json_tests_data/dmesg-results/joblist.txt
 create mode 100644 runner/json_tests_data/dmesg-results/metadata.txt
 create mode 100644 runner/json_tests_data/dmesg-results/reference.json
 create mode 100644 runner/json_tests_data/dmesg-results/starttime.txt
 create mode 100644 runner/json_tests_data/dmesg-results/uname.txt
 create mode 100644 runner/json_tests_data/incomplete-before-any-subtests/0/dmesg.txt
 create mode 100644 runner/json_tests_data/incomplete-before-any-subtests/0/err.txt
 create mode 100644 runner/json_tests_data/incomplete-before-any-subtests/0/journal.txt
 create mode 100644 runner/json_tests_data/incomplete-before-any-subtests/0/out.txt
 create mode 100644 runner/json_tests_data/incomplete-before-any-subtests/README.txt
 create mode 100644 runner/json_tests_data/incomplete-before-any-subtests/endtime.txt
 create mode 100644 runner/json_tests_data/incomplete-before-any-subtests/joblist.txt
 create mode 100644 runner/json_tests_data/incomplete-before-any-subtests/metadata.txt
 create mode 100644 runner/json_tests_data/incomplete-before-any-subtests/reference.json
 create mode 100644 runner/json_tests_data/incomplete-before-any-subtests/starttime.txt
 create mode 100644 runner/json_tests_data/incomplete-before-any-subtests/uname.txt
 create mode 100644 runner/json_tests_data/normal-run/0/dmesg.txt
 create mode 100644 runner/json_tests_data/normal-run/0/err.txt
 create mode 100644 runner/json_tests_data/normal-run/0/journal.txt
 create mode 100644 runner/json_tests_data/normal-run/0/out.txt
 create mode 100644 runner/json_tests_data/normal-run/1/dmesg.txt
 create mode 100644 runner/json_tests_data/normal-run/1/err.txt
 create mode 100644 runner/json_tests_data/normal-run/1/journal.txt
 create mode 100644 runner/json_tests_data/normal-run/1/out.txt
 create mode 100644 runner/json_tests_data/normal-run/2/dmesg.txt
 create mode 100644 runner/json_tests_data/normal-run/2/err.txt
 create mode 100644 runner/json_tests_data/normal-run/2/journal.txt
 create mode 100644 runner/json_tests_data/normal-run/2/out.txt
 create mode 100644 runner/json_tests_data/normal-run/3/dmesg.txt
 create mode 100644 runner/json_tests_data/normal-run/3/err.txt
 create mode 100644 runner/json_tests_data/normal-run/3/journal.txt
 create mode 100644 runner/json_tests_data/normal-run/3/out.txt
 create mode 100644 runner/json_tests_data/normal-run/4/dmesg.txt
 create mode 100644 runner/json_tests_data/normal-run/4/err.txt
 create mode 100644 runner/json_tests_data/normal-run/4/journal.txt
 create mode 100644 runner/json_tests_data/normal-run/4/out.txt
 create mode 100644 runner/json_tests_data/normal-run/README.txt
 create mode 100644 runner/json_tests_data/normal-run/endtime.txt
 create mode 100644 runner/json_tests_data/normal-run/joblist.txt
 create mode 100644 runner/json_tests_data/normal-run/metadata.txt
 create mode 100644 runner/json_tests_data/normal-run/reference.json
 create mode 100644 runner/json_tests_data/normal-run/starttime.txt
 create mode 100644 runner/json_tests_data/normal-run/uname.txt
 create mode 100644 runner/json_tests_data/piglit-style-dmesg/0/dmesg.txt
 create mode 100644 runner/json_tests_data/piglit-style-dmesg/0/err.txt
 create mode 100644 runner/json_tests_data/piglit-style-dmesg/0/journal.txt
 create mode 100644 runner/json_tests_data/piglit-style-dmesg/0/out.txt
 create mode 100644 runner/json_tests_data/piglit-style-dmesg/1/dmesg.txt
 create mode 100644 runner/json_tests_data/piglit-style-dmesg/1/err.txt
 create mode 100644 runner/json_tests_data/piglit-style-dmesg/1/journal.txt
 create mode 100644 runner/json_tests_data/piglit-style-dmesg/1/out.txt
 create mode 100644 runner/json_tests_data/piglit-style-dmesg/2/dmesg.txt
 create mode 100644 runner/json_tests_data/piglit-style-dmesg/2/err.txt
 create mode 100644 runner/json_tests_data/piglit-style-dmesg/2/journal.txt
 create mode 100644 runner/json_tests_data/piglit-style-dmesg/2/out.txt
 create mode 100644 runner/json_tests_data/piglit-style-dmesg/3/dmesg.txt
 create mode 100644 runner/json_tests_data/piglit-style-dmesg/3/err.txt
 create mode 100644 runner/json_tests_data/piglit-style-dmesg/3/journal.txt
 create mode 100644 runner/json_tests_data/piglit-style-dmesg/3/out.txt
 create mode 100644 runner/json_tests_data/piglit-style-dmesg/4/dmesg.txt
 create mode 100644 runner/json_tests_data/piglit-style-dmesg/4/err.txt
 create mode 100644 runner/json_tests_data/piglit-style-dmesg/4/journal.txt
 create mode 100644 runner/json_tests_data/piglit-style-dmesg/4/out.txt
 create mode 100644 runner/json_tests_data/piglit-style-dmesg/README.txt
 create mode 100644 runner/json_tests_data/piglit-style-dmesg/endtime.txt
 create mode 100644 runner/json_tests_data/piglit-style-dmesg/joblist.txt
 create mode 100644 runner/json_tests_data/piglit-style-dmesg/metadata.txt
 create mode 100644 runner/json_tests_data/piglit-style-dmesg/reference.json
 create mode 100644 runner/json_tests_data/piglit-style-dmesg/starttime.txt
 create mode 100644 runner/json_tests_data/piglit-style-dmesg/uname.txt
 create mode 100644 runner/json_tests_data/warnings-with-dmesg-warns/0/dmesg.txt
 create mode 100644 runner/json_tests_data/warnings-with-dmesg-warns/0/err.txt
 create mode 100644 runner/json_tests_data/warnings-with-dmesg-warns/0/journal.txt
 create mode 100644 runner/json_tests_data/warnings-with-dmesg-warns/0/out.txt
 create mode 100644 runner/json_tests_data/warnings-with-dmesg-warns/1/dmesg.txt
 create mode 100644 runner/json_tests_data/warnings-with-dmesg-warns/1/err.txt
 create mode 100644 runner/json_tests_data/warnings-with-dmesg-warns/1/journal.txt
 create mode 100644 runner/json_tests_data/warnings-with-dmesg-warns/1/out.txt
 create mode 100644 runner/json_tests_data/warnings-with-dmesg-warns/2/dmesg.txt
 create mode 100644 runner/json_tests_data/warnings-with-dmesg-warns/2/err.txt
 create mode 100644 runner/json_tests_data/warnings-with-dmesg-warns/2/journal.txt
 create mode 100644 runner/json_tests_data/warnings-with-dmesg-warns/2/out.txt
 create mode 100644 runner/json_tests_data/warnings-with-dmesg-warns/3/dmesg.txt
 create mode 100644 runner/json_tests_data/warnings-with-dmesg-warns/3/err.txt
 create mode 100644 runner/json_tests_data/warnings-with-dmesg-warns/3/journal.txt
 create mode 100644 runner/json_tests_data/warnings-with-dmesg-warns/3/out.txt
 create mode 100644 runner/json_tests_data/warnings-with-dmesg-warns/4/dmesg.txt
 create mode 100644 runner/json_tests_data/warnings-with-dmesg-warns/4/err.txt
 create mode 100644 runner/json_tests_data/warnings-with-dmesg-warns/4/journal.txt
 create mode 100644 runner/json_tests_data/warnings-with-dmesg-warns/4/out.txt
 create mode 100644 runner/json_tests_data/warnings-with-dmesg-warns/README.txt
 create mode 100644 runner/json_tests_data/warnings-with-dmesg-warns/endtime.txt
 create mode 100644 runner/json_tests_data/warnings-with-dmesg-warns/joblist.txt
 create mode 100644 runner/json_tests_data/warnings-with-dmesg-warns/metadata.txt
 create mode 100644 runner/json_tests_data/warnings-with-dmesg-warns/reference.json
 create mode 100644 runner/json_tests_data/warnings-with-dmesg-warns/starttime.txt
 create mode 100644 runner/json_tests_data/warnings-with-dmesg-warns/uname.txt
 create mode 100644 runner/json_tests_data/warnings/0/dmesg.txt
 create mode 100644 runner/json_tests_data/warnings/0/err.txt
 create mode 100644 runner/json_tests_data/warnings/0/journal.txt
 create mode 100644 runner/json_tests_data/warnings/0/out.txt
 create mode 100644 runner/json_tests_data/warnings/1/dmesg.txt
 create mode 100644 runner/json_tests_data/warnings/1/err.txt
 create mode 100644 runner/json_tests_data/warnings/1/journal.txt
 create mode 100644 runner/json_tests_data/warnings/1/out.txt
 create mode 100644 runner/json_tests_data/warnings/2/dmesg.txt
 create mode 100644 runner/json_tests_data/warnings/2/err.txt
 create mode 100644 runner/json_tests_data/warnings/2/journal.txt
 create mode 100644 runner/json_tests_data/warnings/2/out.txt
 create mode 100644 runner/json_tests_data/warnings/3/dmesg.txt
 create mode 100644 runner/json_tests_data/warnings/3/err.txt
 create mode 100644 runner/json_tests_data/warnings/3/journal.txt
 create mode 100644 runner/json_tests_data/warnings/3/out.txt
 create mode 100644 runner/json_tests_data/warnings/4/dmesg.txt
 create mode 100644 runner/json_tests_data/warnings/4/err.txt
 create mode 100644 runner/json_tests_data/warnings/4/journal.txt
 create mode 100644 runner/json_tests_data/warnings/4/out.txt
 create mode 100644 runner/json_tests_data/warnings/README.txt
 create mode 100644 runner/json_tests_data/warnings/endtime.txt
 create mode 100644 runner/json_tests_data/warnings/joblist.txt
 create mode 100644 runner/json_tests_data/warnings/metadata.txt
 create mode 100644 runner/json_tests_data/warnings/reference.json
 create mode 100644 runner/json_tests_data/warnings/starttime.txt
 create mode 100644 runner/json_tests_data/warnings/uname.txt
 create mode 100644 runner/runner_json_tests.c

diff --git a/runner/.gitignore b/runner/.gitignore
index c5030e66..3812da23 100644
--- a/runner/.gitignore
+++ b/runner/.gitignore
@@ -2,3 +2,4 @@ igt_runner
 igt_resume
 igt_results
 runner_test
+runner_json_test
diff --git a/runner/Makefile.am b/runner/Makefile.am
index ddcbf86a..b55cf797 100644
--- a/runner/Makefile.am
+++ b/runner/Makefile.am
@@ -30,8 +30,9 @@ AM_CFLAGS = $(JSONC_CFLAGS) \
 	-I$(srcdir)/../lib \
 	-D_GNU_SOURCE
 
-TESTS = runner_test
-check_PROGRAMS = runner_test
+TESTS = runner_test runner_json_test
+check_PROGRAMS = runner_test runner_json_test
+
 runner_test_SOURCES = runner_tests.c
 runner_test_CFLAGS = -DTESTDATA_DIRECTORY=\"$(abs_builddir)/testdata\" \
 	-I$(top_srcdir)/include/drm-uapi \
@@ -40,4 +41,13 @@ runner_test_CFLAGS = -DTESTDATA_DIRECTORY=\"$(abs_builddir)/testdata\" \
 	-I$(srcdir)/../lib \
 	-D_GNU_SOURCE
 
+runner_json_test_SOURCES = runner_json_tests.c
+runner_json_test_CFLAGS = -DJSON_TESTS_DIRECTORY=\"$(abs_builddir)/json_tests_data\" \
+	$(JSONC_CFLAGS) \
+	-I$(top_srcdir)/include/drm-uapi \
+	$(DRM_CFLAGS) $(CAIRO_CFLAGS) $(LIBUDEV_CFLAGS) \
+	-I$(srcdir)/.. \
+	-I$(srcdir)/../lib \
+	-D_GNU_SOURCE
+
 endif
diff --git a/runner/json_tests_data/README.txt b/runner/json_tests_data/README.txt
new file mode 100644
index 00000000..206d7630
--- /dev/null
+++ b/runner/json_tests_data/README.txt
@@ -0,0 +1,10 @@
+This directory is for testing igt_runner's results.json creation. Each
+subdirectory is a run-result directory, referred to from
+runner_json_tests.c, with test run intermediary files without any
+results.json. A semi-automatically crafted reference.json file in each
+respective directory is compared with igt_runner's generated json and
+needs to match semantically; indentation and such doesn't matter, data
+has to be equal.
+
+Each directory also contains a README file explaining what the
+particular directory aims to test.
diff --git a/runner/json_tests_data/dmesg-results/0/dmesg.txt b/runner/json_tests_data/dmesg-results/0/dmesg.txt
new file mode 100644
index 00000000..e50f75db
--- /dev/null
+++ b/runner/json_tests_data/dmesg-results/0/dmesg.txt
@@ -0,0 +1,6 @@
+6,951,3216186095083,-;Console: switching to colour dummy device 80x25
+14,952,3216186095097,-;[IGT] successtest: executing
+14,953,3216186101115,-;[IGT] successtest: starting subtest first-subtest
+3,954,3216186101159,-;Warning from kernel
+14,955,3216186101160,-;[IGT] successtest: exiting, ret=0
+6,956,3216186101299,-;Console: switching to colour frame buffer device 240x75
diff --git a/runner/json_tests_data/dmesg-results/0/err.txt b/runner/json_tests_data/dmesg-results/0/err.txt
new file mode 100644
index 00000000..5dc78057
--- /dev/null
+++ b/runner/json_tests_data/dmesg-results/0/err.txt
@@ -0,0 +1,2 @@
+Starting subtest: first-subtest
+Subtest first-subtest: SUCCESS (0.000s)
diff --git a/runner/json_tests_data/dmesg-results/0/journal.txt b/runner/json_tests_data/dmesg-results/0/journal.txt
new file mode 100644
index 00000000..86a30e07
--- /dev/null
+++ b/runner/json_tests_data/dmesg-results/0/journal.txt
@@ -0,0 +1,2 @@
+first-subtest
+exit:0 (0.014s)
diff --git a/runner/json_tests_data/dmesg-results/0/out.txt b/runner/json_tests_data/dmesg-results/0/out.txt
new file mode 100644
index 00000000..5946bf31
--- /dev/null
+++ b/runner/json_tests_data/dmesg-results/0/out.txt
@@ -0,0 +1,3 @@
+IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
+Starting subtest: first-subtest
+Subtest first-subtest: SUCCESS (0.000s)
diff --git a/runner/json_tests_data/dmesg-results/1/dmesg.txt b/runner/json_tests_data/dmesg-results/1/dmesg.txt
new file mode 100644
index 00000000..6c816cf2
--- /dev/null
+++ b/runner/json_tests_data/dmesg-results/1/dmesg.txt
@@ -0,0 +1,6 @@
+6,956,3216186111837,-;Console: switching to colour dummy device 80x25
+14,957,3216186111851,-;[IGT] successtest: executing
+14,958,3216186114762,-;[IGT] successtest: starting subtest second-subtest
+3,954,3216186101159,-;Warning from kernel
+14,959,3216186114814,-;[IGT] successtest: exiting, ret=0
+6,960,3216186114933,-;Console: switching to colour frame buffer device 240x75
diff --git a/runner/json_tests_data/dmesg-results/1/err.txt b/runner/json_tests_data/dmesg-results/1/err.txt
new file mode 100644
index 00000000..0f64562f
--- /dev/null
+++ b/runner/json_tests_data/dmesg-results/1/err.txt
@@ -0,0 +1,2 @@
+Starting subtest: second-subtest
+Subtest second-subtest: FAIL (0.000s)
diff --git a/runner/json_tests_data/dmesg-results/1/journal.txt b/runner/json_tests_data/dmesg-results/1/journal.txt
new file mode 100644
index 00000000..99f57815
--- /dev/null
+++ b/runner/json_tests_data/dmesg-results/1/journal.txt
@@ -0,0 +1,2 @@
+second-subtest
+exit:0 (0.013s)
diff --git a/runner/json_tests_data/dmesg-results/1/out.txt b/runner/json_tests_data/dmesg-results/1/out.txt
new file mode 100644
index 00000000..d2778f3c
--- /dev/null
+++ b/runner/json_tests_data/dmesg-results/1/out.txt
@@ -0,0 +1,3 @@
+IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
+Starting subtest: second-subtest
+Subtest second-subtest: FAIL (0.000s)
diff --git a/runner/json_tests_data/dmesg-results/2/dmesg.txt b/runner/json_tests_data/dmesg-results/2/dmesg.txt
new file mode 100644
index 00000000..913979a8
--- /dev/null
+++ b/runner/json_tests_data/dmesg-results/2/dmesg.txt
@@ -0,0 +1,5 @@
+6,961,3216186123400,-;Console: switching to colour dummy device 80x25
+14,962,3216186123414,-;[IGT] no-subtests: executing
+3,954,3216186101159,-;Warning from kernel
+14,963,3216186125204,-;[IGT] no-subtests: exiting, ret=0
+6,964,3216186125374,-;Console: switching to colour frame buffer device 240x75
diff --git a/runner/json_tests_data/dmesg-results/2/err.txt b/runner/json_tests_data/dmesg-results/2/err.txt
new file mode 100644
index 00000000..e69de29b
diff --git a/runner/json_tests_data/dmesg-results/2/journal.txt b/runner/json_tests_data/dmesg-results/2/journal.txt
new file mode 100644
index 00000000..7151877f
--- /dev/null
+++ b/runner/json_tests_data/dmesg-results/2/journal.txt
@@ -0,0 +1 @@
+exit:0 (0.010s)
diff --git a/runner/json_tests_data/dmesg-results/2/out.txt b/runner/json_tests_data/dmesg-results/2/out.txt
new file mode 100644
index 00000000..695b67c2
--- /dev/null
+++ b/runner/json_tests_data/dmesg-results/2/out.txt
@@ -0,0 +1,2 @@
+IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
+SUCCESS (0.000s)
diff --git a/runner/json_tests_data/dmesg-results/3/dmesg.txt b/runner/json_tests_data/dmesg-results/3/dmesg.txt
new file mode 100644
index 00000000..a61e83e8
--- /dev/null
+++ b/runner/json_tests_data/dmesg-results/3/dmesg.txt
@@ -0,0 +1,5 @@
+6,965,3216186135188,-;Console: switching to colour dummy device 80x25
+14,966,3216186135212,-;[IGT] skippers: executing
+3,954,3216186101159,-;Warning from kernel
+14,967,3216186137075,-;[IGT] skippers: exiting, ret=77
+6,968,3216186137206,-;Console: switching to colour frame buffer device 240x75
diff --git a/runner/json_tests_data/dmesg-results/3/err.txt b/runner/json_tests_data/dmesg-results/3/err.txt
new file mode 100644
index 00000000..59b73d09
--- /dev/null
+++ b/runner/json_tests_data/dmesg-results/3/err.txt
@@ -0,0 +1 @@
+Subtest skip-one: SKIP
diff --git a/runner/json_tests_data/dmesg-results/3/journal.txt b/runner/json_tests_data/dmesg-results/3/journal.txt
new file mode 100644
index 00000000..afab8ba6
--- /dev/null
+++ b/runner/json_tests_data/dmesg-results/3/journal.txt
@@ -0,0 +1,2 @@
+skip-one
+exit:77 (0.011s)
diff --git a/runner/json_tests_data/dmesg-results/3/out.txt b/runner/json_tests_data/dmesg-results/3/out.txt
new file mode 100644
index 00000000..96284e78
--- /dev/null
+++ b/runner/json_tests_data/dmesg-results/3/out.txt
@@ -0,0 +1,6 @@
+IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
+Test requirement not met in function __real_main3, file ../runner/testdata/skippers.c:6:
+Test requirement: false
+Skipping from fixture
+Last errno: 2, No such file or directory
+Subtest skip-one: SKIP
diff --git a/runner/json_tests_data/dmesg-results/4/dmesg.txt b/runner/json_tests_data/dmesg-results/4/dmesg.txt
new file mode 100644
index 00000000..fcb43d57
--- /dev/null
+++ b/runner/json_tests_data/dmesg-results/4/dmesg.txt
@@ -0,0 +1,5 @@
+6,969,3216186145899,-;Console: switching to colour dummy device 80x25
+14,970,3216186145912,-;[IGT] skippers: executing
+3,954,3216186101159,-;Warning from kernel
+14,971,3216186147754,-;[IGT] skippers: exiting, ret=77
+6,972,3216186147894,-;Console: switching to colour frame buffer device 240x75
diff --git a/runner/json_tests_data/dmesg-results/4/err.txt b/runner/json_tests_data/dmesg-results/4/err.txt
new file mode 100644
index 00000000..2251da1e
--- /dev/null
+++ b/runner/json_tests_data/dmesg-results/4/err.txt
@@ -0,0 +1 @@
+Subtest skip-two: SKIP
diff --git a/runner/json_tests_data/dmesg-results/4/journal.txt b/runner/json_tests_data/dmesg-results/4/journal.txt
new file mode 100644
index 00000000..a9dba132
--- /dev/null
+++ b/runner/json_tests_data/dmesg-results/4/journal.txt
@@ -0,0 +1,2 @@
+skip-two
+exit:77 (0.010s)
diff --git a/runner/json_tests_data/dmesg-results/4/out.txt b/runner/json_tests_data/dmesg-results/4/out.txt
new file mode 100644
index 00000000..2024db8f
--- /dev/null
+++ b/runner/json_tests_data/dmesg-results/4/out.txt
@@ -0,0 +1,6 @@
+IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
+Test requirement not met in function __real_main3, file ../runner/testdata/skippers.c:6:
+Test requirement: false
+Skipping from fixture
+Last errno: 2, No such file or directory
+Subtest skip-two: SKIP
diff --git a/runner/json_tests_data/dmesg-results/README.txt b/runner/json_tests_data/dmesg-results/README.txt
new file mode 100644
index 00000000..d928a526
--- /dev/null
+++ b/runner/json_tests_data/dmesg-results/README.txt
@@ -0,0 +1,3 @@
+Certain types of kernel log messages cause the result to change from
+'pass' to 'dmesg-warn', and from 'fail' to 'dmesg-fail'. 'skip' should
+stay 'skip'.
diff --git a/runner/json_tests_data/dmesg-results/endtime.txt b/runner/json_tests_data/dmesg-results/endtime.txt
new file mode 100644
index 00000000..635f6ae9
--- /dev/null
+++ b/runner/json_tests_data/dmesg-results/endtime.txt
@@ -0,0 +1 @@
+1539953735.172373
diff --git a/runner/json_tests_data/dmesg-results/joblist.txt b/runner/json_tests_data/dmesg-results/joblist.txt
new file mode 100644
index 00000000..31ef8413
--- /dev/null
+++ b/runner/json_tests_data/dmesg-results/joblist.txt
@@ -0,0 +1,5 @@
+successtest first-subtest
+successtest second-subtest
+no-subtests
+skippers skip-one
+skippers skip-two
diff --git a/runner/json_tests_data/dmesg-results/metadata.txt b/runner/json_tests_data/dmesg-results/metadata.txt
new file mode 100644
index 00000000..1316560d
--- /dev/null
+++ b/runner/json_tests_data/dmesg-results/metadata.txt
@@ -0,0 +1,12 @@
+abort_on_error : 0
+name : normal-run
+dry_run : 0
+sync : 0
+log_level : 0
+overwrite : 0
+multiple_mode : 0
+inactivity_timeout : 0
+use_watchdog : 0
+piglit_style_dmesg : 0
+test_root : /path/does/not/exist
+results_path : /path/does/not/exist
diff --git a/runner/json_tests_data/dmesg-results/reference.json b/runner/json_tests_data/dmesg-results/reference.json
new file mode 100644
index 00000000..811b63eb
--- /dev/null
+++ b/runner/json_tests_data/dmesg-results/reference.json
@@ -0,0 +1,161 @@
+{
+  "__type__":"TestrunResult",
+  "results_version":9,
+  "name":"normal-run",
+  "uname":"Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64",
+  "time_elapsed":{
+    "__type__":"TimeAttribute",
+    "start":1539953735.1110389,
+    "end":1539953735.1723731
+  },
+  "tests":{
+    "igt at successtest@first-subtest":{
+      "out":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n",
+      "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+      "result":"dmesg-warn",
+      "time":{
+        "__type__":"TimeAttribute",
+        "start":0,
+        "end":0
+      },
+      "err":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n",
+	"dmesg":"<6> [3216186.095083] Console: switching to colour dummy device 80x25\n<6> [3216186.095097] [IGT] successtest: executing\n<6> [3216186.101115] [IGT] successtest: starting subtest first-subtest\n<3> [3216186.101159] Warning from kernel\n<6> [3216186.101160] [IGT] successtest: exiting, ret=0\n<6> [3216186.101299] Console: switching to colour frame buffer device 240x75\n",
+	"dmesg-warnings":"<3> [3216186.101159] Warning from kernel\n"
+    },
+    "igt at successtest@second-subtest":{
+      "out":"Starting subtest: second-subtest\nSubtest second-subtest: FAIL (0.000s)\n",
+      "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+      "result":"dmesg-fail",
+      "time":{
+        "__type__":"TimeAttribute",
+        "start":0,
+        "end":0
+      },
+      "err":"Starting subtest: second-subtest\nSubtest second-subtest: FAIL (0.000s)\n",
+	"dmesg":"<6> [3216186.111837] Console: switching to colour dummy device 80x25\n<6> [3216186.111851] [IGT] successtest: executing\n<6> [3216186.114762] [IGT] successtest: starting subtest second-subtest\n<3> [3216186.101159] Warning from kernel\n<6> [3216186.114814] [IGT] successtest: exiting, ret=0\n<6> [3216186.114933] Console: switching to colour frame buffer device 240x75\n",
+	"dmesg-warnings":"<3> [3216186.101159] Warning from kernel\n"
+    },
+    "igt at no-subtests":{
+      "time":{
+        "__type__":"TimeAttribute",
+        "start":0,
+        "end":0.01
+      },
+      "result":"dmesg-warn",
+      "out":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)\nSUCCESS (0.000s)\n",
+      "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+      "err":"",
+	"dmesg":"<6> [3216186.123400] Console: switching to colour dummy device 80x25\n<6> [3216186.123414] [IGT] no-subtests: executing\n<3> [3216186.101159] Warning from kernel\n<6> [3216186.125204] [IGT] no-subtests: exiting, ret=0\n<6> [3216186.125374] Console: switching to colour frame buffer device 240x75\n",
+	"dmesg-warnings":"<3> [3216186.101159] Warning from kernel\n"
+    },
+    "igt at skippers@skip-one":{
+      "out":"Test requirement not met in function __real_main3, file ..\/runner\/testdata\/skippers.c:6:\nTest requirement: false\nSkipping from fixture\nLast errno: 2, No such file or directory\nSubtest skip-one: SKIP\n",
+      "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+      "result":"skip",
+      "time":{
+        "__type__":"TimeAttribute",
+        "start":0,
+        "end":0
+      },
+      "err":"Subtest skip-one: SKIP\n",
+      "dmesg":"<6> [3216186.135188] Console: switching to colour dummy device 80x25\n<6> [3216186.135212] [IGT] skippers: executing\n<3> [3216186.101159] Warning from kernel\n<6> [3216186.137075] [IGT] skippers: exiting, ret=77\n<6> [3216186.137206] Console: switching to colour frame buffer device 240x75\n"
+    },
+    "igt at skippers@skip-two":{
+      "out":"Test requirement not met in function __real_main3, file ..\/runner\/testdata\/skippers.c:6:\nTest requirement: false\nSkipping from fixture\nLast errno: 2, No such file or directory\nSubtest skip-two: SKIP\n",
+      "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+      "result":"skip",
+      "time":{
+        "__type__":"TimeAttribute",
+        "start":0,
+        "end":0
+      },
+      "err":"Subtest skip-two: SKIP\n",
+      "dmesg":"<6> [3216186.145899] Console: switching to colour dummy device 80x25\n<6> [3216186.145912] [IGT] skippers: executing\n<3> [3216186.101159] Warning from kernel\n<6> [3216186.147754] [IGT] skippers: exiting, ret=77\n<6> [3216186.147894] Console: switching to colour frame buffer device 240x75\n"
+    }
+  },
+  "totals":{
+    "":{
+      "crash":0,
+      "pass":0,
+      "dmesg-fail":1,
+      "dmesg-warn":2,
+      "skip":2,
+      "incomplete":0,
+      "timeout":0,
+      "notrun":0,
+      "fail":0,
+      "warn":0
+    },
+    "root":{
+      "crash":0,
+      "pass":0,
+      "dmesg-fail":1,
+      "dmesg-warn":2,
+      "skip":2,
+      "incomplete":0,
+      "timeout":0,
+      "notrun":0,
+      "fail":0,
+      "warn":0
+    },
+    "igt at successtest":{
+      "crash":0,
+      "pass":0,
+      "dmesg-fail":1,
+      "dmesg-warn":1,
+      "skip":0,
+      "incomplete":0,
+      "timeout":0,
+      "notrun":0,
+      "fail":0,
+      "warn":0
+    },
+    "igt at no-subtests":{
+      "crash":0,
+      "pass":0,
+      "dmesg-fail":0,
+      "dmesg-warn":1,
+      "skip":0,
+      "incomplete":0,
+      "timeout":0,
+      "notrun":0,
+      "fail":0,
+      "warn":0
+    },
+    "igt at skippers":{
+      "crash":0,
+      "pass":0,
+      "dmesg-fail":0,
+      "dmesg-warn":0,
+      "skip":2,
+      "incomplete":0,
+      "timeout":0,
+      "notrun":0,
+      "fail":0,
+      "warn":0
+    }
+  },
+  "runtimes":{
+    "igt at successtest":{
+      "time":{
+        "__type__":"TimeAttribute",
+        "start":0,
+        "end":0.027
+      }
+    },
+    "igt at no-subtests":{
+      "time":{
+        "__type__":"TimeAttribute",
+        "start":0,
+        "end":0.01
+      }
+    },
+    "igt at skippers":{
+      "time":{
+        "__type__":"TimeAttribute",
+        "start":0,
+        "end":0.020999999999999998
+      }
+    }
+  }
+}
diff --git a/runner/json_tests_data/dmesg-results/starttime.txt b/runner/json_tests_data/dmesg-results/starttime.txt
new file mode 100644
index 00000000..ae038f18
--- /dev/null
+++ b/runner/json_tests_data/dmesg-results/starttime.txt
@@ -0,0 +1 @@
+1539953735.111039
diff --git a/runner/json_tests_data/dmesg-results/uname.txt b/runner/json_tests_data/dmesg-results/uname.txt
new file mode 100644
index 00000000..a7aef6f7
--- /dev/null
+++ b/runner/json_tests_data/dmesg-results/uname.txt
@@ -0,0 +1 @@
+Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64
diff --git a/runner/json_tests_data/incomplete-before-any-subtests/0/dmesg.txt b/runner/json_tests_data/incomplete-before-any-subtests/0/dmesg.txt
new file mode 100644
index 00000000..7c6834d5
--- /dev/null
+++ b/runner/json_tests_data/incomplete-before-any-subtests/0/dmesg.txt
@@ -0,0 +1,2 @@
+6,951,3216186095083,-;Console: switching to colour dummy device 80x25
+14,952,3216186095097,-;[IGT] successtest: executing
diff --git a/runner/json_tests_data/incomplete-before-any-subtests/0/err.txt b/runner/json_tests_data/incomplete-before-any-subtests/0/err.txt
new file mode 100644
index 00000000..e69de29b
diff --git a/runner/json_tests_data/incomplete-before-any-subtests/0/journal.txt b/runner/json_tests_data/incomplete-before-any-subtests/0/journal.txt
new file mode 100644
index 00000000..e69de29b
diff --git a/runner/json_tests_data/incomplete-before-any-subtests/0/out.txt b/runner/json_tests_data/incomplete-before-any-subtests/0/out.txt
new file mode 100644
index 00000000..ebaa6300
--- /dev/null
+++ b/runner/json_tests_data/incomplete-before-any-subtests/0/out.txt
@@ -0,0 +1 @@
+IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
diff --git a/runner/json_tests_data/incomplete-before-any-subtests/README.txt b/runner/json_tests_data/incomplete-before-any-subtests/README.txt
new file mode 100644
index 00000000..fb64f581
--- /dev/null
+++ b/runner/json_tests_data/incomplete-before-any-subtests/README.txt
@@ -0,0 +1,4 @@
+When a test hangs before it even enters a subtest, we don't have a
+journal entry for that subtest. This test checks that the result still
+gets attributed to a subtest, since we know from the job_list that a
+subtest should be executed.
diff --git a/runner/json_tests_data/incomplete-before-any-subtests/endtime.txt b/runner/json_tests_data/incomplete-before-any-subtests/endtime.txt
new file mode 100644
index 00000000..635f6ae9
--- /dev/null
+++ b/runner/json_tests_data/incomplete-before-any-subtests/endtime.txt
@@ -0,0 +1 @@
+1539953735.172373
diff --git a/runner/json_tests_data/incomplete-before-any-subtests/joblist.txt b/runner/json_tests_data/incomplete-before-any-subtests/joblist.txt
new file mode 100644
index 00000000..31ef8413
--- /dev/null
+++ b/runner/json_tests_data/incomplete-before-any-subtests/joblist.txt
@@ -0,0 +1,5 @@
+successtest first-subtest
+successtest second-subtest
+no-subtests
+skippers skip-one
+skippers skip-two
diff --git a/runner/json_tests_data/incomplete-before-any-subtests/metadata.txt b/runner/json_tests_data/incomplete-before-any-subtests/metadata.txt
new file mode 100644
index 00000000..1316560d
--- /dev/null
+++ b/runner/json_tests_data/incomplete-before-any-subtests/metadata.txt
@@ -0,0 +1,12 @@
+abort_on_error : 0
+name : normal-run
+dry_run : 0
+sync : 0
+log_level : 0
+overwrite : 0
+multiple_mode : 0
+inactivity_timeout : 0
+use_watchdog : 0
+piglit_style_dmesg : 0
+test_root : /path/does/not/exist
+results_path : /path/does/not/exist
diff --git a/runner/json_tests_data/incomplete-before-any-subtests/reference.json b/runner/json_tests_data/incomplete-before-any-subtests/reference.json
new file mode 100644
index 00000000..3eb6c9c1
--- /dev/null
+++ b/runner/json_tests_data/incomplete-before-any-subtests/reference.json
@@ -0,0 +1,60 @@
+{
+  "__type__":"TestrunResult",
+  "results_version":9,
+  "name":"normal-run",
+  "uname":"Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64",
+  "time_elapsed":{
+    "__type__":"TimeAttribute",
+    "start":1539953735.1110389,
+    "end":1539953735.1723731
+  },
+  "tests":{
+    "igt at successtest@first-subtest":{
+      "out":"",
+      "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+      "result":"incomplete",
+      "err":"",
+      "dmesg":"<6> [3216186.095083] Console: switching to colour dummy device 80x25\n<6> [3216186.095097] [IGT] successtest: executing\n"
+    }
+  },
+  "totals":{
+    "":{
+      "crash":0,
+      "pass":0,
+      "dmesg-fail":0,
+      "dmesg-warn":0,
+      "skip":0,
+      "incomplete":1,
+      "timeout":0,
+      "notrun":0,
+      "fail":0,
+      "warn":0
+    },
+    "root":{
+      "crash":0,
+      "pass":0,
+      "dmesg-fail":0,
+      "dmesg-warn":0,
+      "skip":0,
+      "incomplete":1,
+      "timeout":0,
+      "notrun":0,
+      "fail":0,
+      "warn":0
+    },
+    "igt at successtest":{
+      "crash":0,
+      "pass":0,
+      "dmesg-fail":0,
+      "dmesg-warn":0,
+      "skip":0,
+      "incomplete":1,
+      "timeout":0,
+      "notrun":0,
+      "fail":0,
+      "warn":0
+    },
+  },
+  "runtimes":{
+  }
+}
diff --git a/runner/json_tests_data/incomplete-before-any-subtests/starttime.txt b/runner/json_tests_data/incomplete-before-any-subtests/starttime.txt
new file mode 100644
index 00000000..ae038f18
--- /dev/null
+++ b/runner/json_tests_data/incomplete-before-any-subtests/starttime.txt
@@ -0,0 +1 @@
+1539953735.111039
diff --git a/runner/json_tests_data/incomplete-before-any-subtests/uname.txt b/runner/json_tests_data/incomplete-before-any-subtests/uname.txt
new file mode 100644
index 00000000..a7aef6f7
--- /dev/null
+++ b/runner/json_tests_data/incomplete-before-any-subtests/uname.txt
@@ -0,0 +1 @@
+Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64
diff --git a/runner/json_tests_data/normal-run/0/dmesg.txt b/runner/json_tests_data/normal-run/0/dmesg.txt
new file mode 100644
index 00000000..a189e704
--- /dev/null
+++ b/runner/json_tests_data/normal-run/0/dmesg.txt
@@ -0,0 +1,5 @@
+6,951,3216186095083,-;Console: switching to colour dummy device 80x25
+14,952,3216186095097,-;[IGT] successtest: executing
+14,953,3216186101115,-;[IGT] successtest: starting subtest first-subtest
+14,954,3216186101160,-;[IGT] successtest: exiting, ret=0
+6,955,3216186101299,-;Console: switching to colour frame buffer device 240x75
diff --git a/runner/json_tests_data/normal-run/0/err.txt b/runner/json_tests_data/normal-run/0/err.txt
new file mode 100644
index 00000000..5dc78057
--- /dev/null
+++ b/runner/json_tests_data/normal-run/0/err.txt
@@ -0,0 +1,2 @@
+Starting subtest: first-subtest
+Subtest first-subtest: SUCCESS (0.000s)
diff --git a/runner/json_tests_data/normal-run/0/journal.txt b/runner/json_tests_data/normal-run/0/journal.txt
new file mode 100644
index 00000000..86a30e07
--- /dev/null
+++ b/runner/json_tests_data/normal-run/0/journal.txt
@@ -0,0 +1,2 @@
+first-subtest
+exit:0 (0.014s)
diff --git a/runner/json_tests_data/normal-run/0/out.txt b/runner/json_tests_data/normal-run/0/out.txt
new file mode 100644
index 00000000..5946bf31
--- /dev/null
+++ b/runner/json_tests_data/normal-run/0/out.txt
@@ -0,0 +1,3 @@
+IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
+Starting subtest: first-subtest
+Subtest first-subtest: SUCCESS (0.000s)
diff --git a/runner/json_tests_data/normal-run/1/dmesg.txt b/runner/json_tests_data/normal-run/1/dmesg.txt
new file mode 100644
index 00000000..59b7cee3
--- /dev/null
+++ b/runner/json_tests_data/normal-run/1/dmesg.txt
@@ -0,0 +1,5 @@
+6,956,3216186111837,-;Console: switching to colour dummy device 80x25
+14,957,3216186111851,-;[IGT] successtest: executing
+14,958,3216186114762,-;[IGT] successtest: starting subtest second-subtest
+14,959,3216186114814,-;[IGT] successtest: exiting, ret=0
+6,960,3216186114933,-;Console: switching to colour frame buffer device 240x75
diff --git a/runner/json_tests_data/normal-run/1/err.txt b/runner/json_tests_data/normal-run/1/err.txt
new file mode 100644
index 00000000..0f64562f
--- /dev/null
+++ b/runner/json_tests_data/normal-run/1/err.txt
@@ -0,0 +1,2 @@
+Starting subtest: second-subtest
+Subtest second-subtest: FAIL (0.000s)
diff --git a/runner/json_tests_data/normal-run/1/journal.txt b/runner/json_tests_data/normal-run/1/journal.txt
new file mode 100644
index 00000000..99f57815
--- /dev/null
+++ b/runner/json_tests_data/normal-run/1/journal.txt
@@ -0,0 +1,2 @@
+second-subtest
+exit:0 (0.013s)
diff --git a/runner/json_tests_data/normal-run/1/out.txt b/runner/json_tests_data/normal-run/1/out.txt
new file mode 100644
index 00000000..d2778f3c
--- /dev/null
+++ b/runner/json_tests_data/normal-run/1/out.txt
@@ -0,0 +1,3 @@
+IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
+Starting subtest: second-subtest
+Subtest second-subtest: FAIL (0.000s)
diff --git a/runner/json_tests_data/normal-run/2/dmesg.txt b/runner/json_tests_data/normal-run/2/dmesg.txt
new file mode 100644
index 00000000..998b4797
--- /dev/null
+++ b/runner/json_tests_data/normal-run/2/dmesg.txt
@@ -0,0 +1,4 @@
+6,961,3216186123400,-;Console: switching to colour dummy device 80x25
+14,962,3216186123414,-;[IGT] no-subtests: executing
+14,963,3216186125204,-;[IGT] no-subtests: exiting, ret=0
+6,964,3216186125374,-;Console: switching to colour frame buffer device 240x75
diff --git a/runner/json_tests_data/normal-run/2/err.txt b/runner/json_tests_data/normal-run/2/err.txt
new file mode 100644
index 00000000..e69de29b
diff --git a/runner/json_tests_data/normal-run/2/journal.txt b/runner/json_tests_data/normal-run/2/journal.txt
new file mode 100644
index 00000000..7151877f
--- /dev/null
+++ b/runner/json_tests_data/normal-run/2/journal.txt
@@ -0,0 +1 @@
+exit:0 (0.010s)
diff --git a/runner/json_tests_data/normal-run/2/out.txt b/runner/json_tests_data/normal-run/2/out.txt
new file mode 100644
index 00000000..695b67c2
--- /dev/null
+++ b/runner/json_tests_data/normal-run/2/out.txt
@@ -0,0 +1,2 @@
+IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
+SUCCESS (0.000s)
diff --git a/runner/json_tests_data/normal-run/3/dmesg.txt b/runner/json_tests_data/normal-run/3/dmesg.txt
new file mode 100644
index 00000000..21e75031
--- /dev/null
+++ b/runner/json_tests_data/normal-run/3/dmesg.txt
@@ -0,0 +1,4 @@
+6,965,3216186135188,-;Console: switching to colour dummy device 80x25
+14,966,3216186135212,-;[IGT] skippers: executing
+14,967,3216186137075,-;[IGT] skippers: exiting, ret=77
+6,968,3216186137206,-;Console: switching to colour frame buffer device 240x75
diff --git a/runner/json_tests_data/normal-run/3/err.txt b/runner/json_tests_data/normal-run/3/err.txt
new file mode 100644
index 00000000..59b73d09
--- /dev/null
+++ b/runner/json_tests_data/normal-run/3/err.txt
@@ -0,0 +1 @@
+Subtest skip-one: SKIP
diff --git a/runner/json_tests_data/normal-run/3/journal.txt b/runner/json_tests_data/normal-run/3/journal.txt
new file mode 100644
index 00000000..afab8ba6
--- /dev/null
+++ b/runner/json_tests_data/normal-run/3/journal.txt
@@ -0,0 +1,2 @@
+skip-one
+exit:77 (0.011s)
diff --git a/runner/json_tests_data/normal-run/3/out.txt b/runner/json_tests_data/normal-run/3/out.txt
new file mode 100644
index 00000000..96284e78
--- /dev/null
+++ b/runner/json_tests_data/normal-run/3/out.txt
@@ -0,0 +1,6 @@
+IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
+Test requirement not met in function __real_main3, file ../runner/testdata/skippers.c:6:
+Test requirement: false
+Skipping from fixture
+Last errno: 2, No such file or directory
+Subtest skip-one: SKIP
diff --git a/runner/json_tests_data/normal-run/4/dmesg.txt b/runner/json_tests_data/normal-run/4/dmesg.txt
new file mode 100644
index 00000000..737bc692
--- /dev/null
+++ b/runner/json_tests_data/normal-run/4/dmesg.txt
@@ -0,0 +1,4 @@
+6,969,3216186145899,-;Console: switching to colour dummy device 80x25
+14,970,3216186145912,-;[IGT] skippers: executing
+14,971,3216186147754,-;[IGT] skippers: exiting, ret=77
+6,972,3216186147894,-;Console: switching to colour frame buffer device 240x75
diff --git a/runner/json_tests_data/normal-run/4/err.txt b/runner/json_tests_data/normal-run/4/err.txt
new file mode 100644
index 00000000..2251da1e
--- /dev/null
+++ b/runner/json_tests_data/normal-run/4/err.txt
@@ -0,0 +1 @@
+Subtest skip-two: SKIP
diff --git a/runner/json_tests_data/normal-run/4/journal.txt b/runner/json_tests_data/normal-run/4/journal.txt
new file mode 100644
index 00000000..a9dba132
--- /dev/null
+++ b/runner/json_tests_data/normal-run/4/journal.txt
@@ -0,0 +1,2 @@
+skip-two
+exit:77 (0.010s)
diff --git a/runner/json_tests_data/normal-run/4/out.txt b/runner/json_tests_data/normal-run/4/out.txt
new file mode 100644
index 00000000..2024db8f
--- /dev/null
+++ b/runner/json_tests_data/normal-run/4/out.txt
@@ -0,0 +1,6 @@
+IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
+Test requirement not met in function __real_main3, file ../runner/testdata/skippers.c:6:
+Test requirement: false
+Skipping from fixture
+Last errno: 2, No such file or directory
+Subtest skip-two: SKIP
diff --git a/runner/json_tests_data/normal-run/README.txt b/runner/json_tests_data/normal-run/README.txt
new file mode 100644
index 00000000..64270e88
--- /dev/null
+++ b/runner/json_tests_data/normal-run/README.txt
@@ -0,0 +1 @@
+A normal test run with pass, fail and skip results.
diff --git a/runner/json_tests_data/normal-run/endtime.txt b/runner/json_tests_data/normal-run/endtime.txt
new file mode 100644
index 00000000..635f6ae9
--- /dev/null
+++ b/runner/json_tests_data/normal-run/endtime.txt
@@ -0,0 +1 @@
+1539953735.172373
diff --git a/runner/json_tests_data/normal-run/joblist.txt b/runner/json_tests_data/normal-run/joblist.txt
new file mode 100644
index 00000000..31ef8413
--- /dev/null
+++ b/runner/json_tests_data/normal-run/joblist.txt
@@ -0,0 +1,5 @@
+successtest first-subtest
+successtest second-subtest
+no-subtests
+skippers skip-one
+skippers skip-two
diff --git a/runner/json_tests_data/normal-run/metadata.txt b/runner/json_tests_data/normal-run/metadata.txt
new file mode 100644
index 00000000..1316560d
--- /dev/null
+++ b/runner/json_tests_data/normal-run/metadata.txt
@@ -0,0 +1,12 @@
+abort_on_error : 0
+name : normal-run
+dry_run : 0
+sync : 0
+log_level : 0
+overwrite : 0
+multiple_mode : 0
+inactivity_timeout : 0
+use_watchdog : 0
+piglit_style_dmesg : 0
+test_root : /path/does/not/exist
+results_path : /path/does/not/exist
diff --git a/runner/json_tests_data/normal-run/reference.json b/runner/json_tests_data/normal-run/reference.json
new file mode 100644
index 00000000..0d28b2c2
--- /dev/null
+++ b/runner/json_tests_data/normal-run/reference.json
@@ -0,0 +1,158 @@
+{
+  "__type__":"TestrunResult",
+  "results_version":9,
+  "name":"normal-run",
+  "uname":"Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64",
+  "time_elapsed":{
+    "__type__":"TimeAttribute",
+    "start":1539953735.1110389,
+    "end":1539953735.1723731
+  },
+  "tests":{
+    "igt at successtest@first-subtest":{
+      "out":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n",
+      "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+      "result":"pass",
+      "time":{
+        "__type__":"TimeAttribute",
+        "start":0,
+        "end":0
+      },
+      "err":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n",
+      "dmesg":"<6> [3216186.095083] Console: switching to colour dummy device 80x25\n<6> [3216186.095097] [IGT] successtest: executing\n<6> [3216186.101115] [IGT] successtest: starting subtest first-subtest\n<6> [3216186.101160] [IGT] successtest: exiting, ret=0\n<6> [3216186.101299] Console: switching to colour frame buffer device 240x75\n"
+    },
+    "igt at successtest@second-subtest":{
+      "out":"Starting subtest: second-subtest\nSubtest second-subtest: FAIL (0.000s)\n",
+      "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+      "result":"fail",
+      "time":{
+        "__type__":"TimeAttribute",
+        "start":0,
+        "end":0
+      },
+      "err":"Starting subtest: second-subtest\nSubtest second-subtest: FAIL (0.000s)\n",
+      "dmesg":"<6> [3216186.111837] Console: switching to colour dummy device 80x25\n<6> [3216186.111851] [IGT] successtest: executing\n<6> [3216186.114762] [IGT] successtest: starting subtest second-subtest\n<6> [3216186.114814] [IGT] successtest: exiting, ret=0\n<6> [3216186.114933] Console: switching to colour frame buffer device 240x75\n"
+    },
+    "igt at no-subtests":{
+      "time":{
+        "__type__":"TimeAttribute",
+        "start":0,
+        "end":0.01
+      },
+      "result":"pass",
+      "out":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)\nSUCCESS (0.000s)\n",
+      "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+      "err":"",
+      "dmesg":"<6> [3216186.123400] Console: switching to colour dummy device 80x25\n<6> [3216186.123414] [IGT] no-subtests: executing\n<6> [3216186.125204] [IGT] no-subtests: exiting, ret=0\n<6> [3216186.125374] Console: switching to colour frame buffer device 240x75\n"
+    },
+    "igt at skippers@skip-one":{
+      "out":"Test requirement not met in function __real_main3, file ..\/runner\/testdata\/skippers.c:6:\nTest requirement: false\nSkipping from fixture\nLast errno: 2, No such file or directory\nSubtest skip-one: SKIP\n",
+      "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+      "result":"skip",
+      "time":{
+        "__type__":"TimeAttribute",
+        "start":0,
+        "end":0
+      },
+      "err":"Subtest skip-one: SKIP\n",
+      "dmesg":"<6> [3216186.135188] Console: switching to colour dummy device 80x25\n<6> [3216186.135212] [IGT] skippers: executing\n<6> [3216186.137075] [IGT] skippers: exiting, ret=77\n<6> [3216186.137206] Console: switching to colour frame buffer device 240x75\n"
+    },
+    "igt at skippers@skip-two":{
+      "out":"Test requirement not met in function __real_main3, file ..\/runner\/testdata\/skippers.c:6:\nTest requirement: false\nSkipping from fixture\nLast errno: 2, No such file or directory\nSubtest skip-two: SKIP\n",
+      "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+      "result":"skip",
+      "time":{
+        "__type__":"TimeAttribute",
+        "start":0,
+        "end":0
+      },
+      "err":"Subtest skip-two: SKIP\n",
+      "dmesg":"<6> [3216186.145899] Console: switching to colour dummy device 80x25\n<6> [3216186.145912] [IGT] skippers: executing\n<6> [3216186.147754] [IGT] skippers: exiting, ret=77\n<6> [3216186.147894] Console: switching to colour frame buffer device 240x75\n"
+    }
+  },
+  "totals":{
+    "":{
+      "crash":0,
+      "pass":2,
+      "dmesg-fail":0,
+      "dmesg-warn":0,
+      "skip":2,
+      "incomplete":0,
+      "timeout":0,
+      "notrun":0,
+      "fail":1,
+      "warn":0
+    },
+    "root":{
+      "crash":0,
+      "pass":2,
+      "dmesg-fail":0,
+      "dmesg-warn":0,
+      "skip":2,
+      "incomplete":0,
+      "timeout":0,
+      "notrun":0,
+      "fail":1,
+      "warn":0
+    },
+    "igt at successtest":{
+      "crash":0,
+      "pass":1,
+      "dmesg-fail":0,
+      "dmesg-warn":0,
+      "skip":0,
+      "incomplete":0,
+      "timeout":0,
+      "notrun":0,
+      "fail":1,
+      "warn":0
+    },
+    "igt at no-subtests":{
+      "crash":0,
+      "pass":1,
+      "dmesg-fail":0,
+      "dmesg-warn":0,
+      "skip":0,
+      "incomplete":0,
+      "timeout":0,
+      "notrun":0,
+      "fail":0,
+      "warn":0
+    },
+    "igt at skippers":{
+      "crash":0,
+      "pass":0,
+      "dmesg-fail":0,
+      "dmesg-warn":0,
+      "skip":2,
+      "incomplete":0,
+      "timeout":0,
+      "notrun":0,
+      "fail":0,
+      "warn":0
+    }
+  },
+  "runtimes":{
+    "igt at successtest":{
+      "time":{
+        "__type__":"TimeAttribute",
+        "start":0,
+        "end":0.027
+      }
+    },
+    "igt at no-subtests":{
+      "time":{
+        "__type__":"TimeAttribute",
+        "start":0,
+        "end":0.01
+      }
+    },
+    "igt at skippers":{
+      "time":{
+        "__type__":"TimeAttribute",
+        "start":0,
+        "end":0.020999999999999998
+      }
+    }
+  }
+}
diff --git a/runner/json_tests_data/normal-run/starttime.txt b/runner/json_tests_data/normal-run/starttime.txt
new file mode 100644
index 00000000..ae038f18
--- /dev/null
+++ b/runner/json_tests_data/normal-run/starttime.txt
@@ -0,0 +1 @@
+1539953735.111039
diff --git a/runner/json_tests_data/normal-run/uname.txt b/runner/json_tests_data/normal-run/uname.txt
new file mode 100644
index 00000000..a7aef6f7
--- /dev/null
+++ b/runner/json_tests_data/normal-run/uname.txt
@@ -0,0 +1 @@
+Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64
diff --git a/runner/json_tests_data/piglit-style-dmesg/0/dmesg.txt b/runner/json_tests_data/piglit-style-dmesg/0/dmesg.txt
new file mode 100644
index 00000000..e50f75db
--- /dev/null
+++ b/runner/json_tests_data/piglit-style-dmesg/0/dmesg.txt
@@ -0,0 +1,6 @@
+6,951,3216186095083,-;Console: switching to colour dummy device 80x25
+14,952,3216186095097,-;[IGT] successtest: executing
+14,953,3216186101115,-;[IGT] successtest: starting subtest first-subtest
+3,954,3216186101159,-;Warning from kernel
+14,955,3216186101160,-;[IGT] successtest: exiting, ret=0
+6,956,3216186101299,-;Console: switching to colour frame buffer device 240x75
diff --git a/runner/json_tests_data/piglit-style-dmesg/0/err.txt b/runner/json_tests_data/piglit-style-dmesg/0/err.txt
new file mode 100644
index 00000000..e18c00e9
--- /dev/null
+++ b/runner/json_tests_data/piglit-style-dmesg/0/err.txt
@@ -0,0 +1,3 @@
+Starting subtest: first-subtest
+This is a warning line
+Subtest first-subtest: SUCCESS (0.000s)
diff --git a/runner/json_tests_data/piglit-style-dmesg/0/journal.txt b/runner/json_tests_data/piglit-style-dmesg/0/journal.txt
new file mode 100644
index 00000000..86a30e07
--- /dev/null
+++ b/runner/json_tests_data/piglit-style-dmesg/0/journal.txt
@@ -0,0 +1,2 @@
+first-subtest
+exit:0 (0.014s)
diff --git a/runner/json_tests_data/piglit-style-dmesg/0/out.txt b/runner/json_tests_data/piglit-style-dmesg/0/out.txt
new file mode 100644
index 00000000..5946bf31
--- /dev/null
+++ b/runner/json_tests_data/piglit-style-dmesg/0/out.txt
@@ -0,0 +1,3 @@
+IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
+Starting subtest: first-subtest
+Subtest first-subtest: SUCCESS (0.000s)
diff --git a/runner/json_tests_data/piglit-style-dmesg/1/dmesg.txt b/runner/json_tests_data/piglit-style-dmesg/1/dmesg.txt
new file mode 100644
index 00000000..59b7cee3
--- /dev/null
+++ b/runner/json_tests_data/piglit-style-dmesg/1/dmesg.txt
@@ -0,0 +1,5 @@
+6,956,3216186111837,-;Console: switching to colour dummy device 80x25
+14,957,3216186111851,-;[IGT] successtest: executing
+14,958,3216186114762,-;[IGT] successtest: starting subtest second-subtest
+14,959,3216186114814,-;[IGT] successtest: exiting, ret=0
+6,960,3216186114933,-;Console: switching to colour frame buffer device 240x75
diff --git a/runner/json_tests_data/piglit-style-dmesg/1/err.txt b/runner/json_tests_data/piglit-style-dmesg/1/err.txt
new file mode 100644
index 00000000..57be9f7b
--- /dev/null
+++ b/runner/json_tests_data/piglit-style-dmesg/1/err.txt
@@ -0,0 +1,2 @@
+Starting subtest: second-subtest
+Subtest second-subtest: SUCCESS (0.000s)
diff --git a/runner/json_tests_data/piglit-style-dmesg/1/journal.txt b/runner/json_tests_data/piglit-style-dmesg/1/journal.txt
new file mode 100644
index 00000000..99f57815
--- /dev/null
+++ b/runner/json_tests_data/piglit-style-dmesg/1/journal.txt
@@ -0,0 +1,2 @@
+second-subtest
+exit:0 (0.013s)
diff --git a/runner/json_tests_data/piglit-style-dmesg/1/out.txt b/runner/json_tests_data/piglit-style-dmesg/1/out.txt
new file mode 100644
index 00000000..24b37244
--- /dev/null
+++ b/runner/json_tests_data/piglit-style-dmesg/1/out.txt
@@ -0,0 +1,3 @@
+IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
+Starting subtest: second-subtest
+Subtest second-subtest: SUCCESS (0.000s)
diff --git a/runner/json_tests_data/piglit-style-dmesg/2/dmesg.txt b/runner/json_tests_data/piglit-style-dmesg/2/dmesg.txt
new file mode 100644
index 00000000..998b4797
--- /dev/null
+++ b/runner/json_tests_data/piglit-style-dmesg/2/dmesg.txt
@@ -0,0 +1,4 @@
+6,961,3216186123400,-;Console: switching to colour dummy device 80x25
+14,962,3216186123414,-;[IGT] no-subtests: executing
+14,963,3216186125204,-;[IGT] no-subtests: exiting, ret=0
+6,964,3216186125374,-;Console: switching to colour frame buffer device 240x75
diff --git a/runner/json_tests_data/piglit-style-dmesg/2/err.txt b/runner/json_tests_data/piglit-style-dmesg/2/err.txt
new file mode 100644
index 00000000..e69de29b
diff --git a/runner/json_tests_data/piglit-style-dmesg/2/journal.txt b/runner/json_tests_data/piglit-style-dmesg/2/journal.txt
new file mode 100644
index 00000000..7151877f
--- /dev/null
+++ b/runner/json_tests_data/piglit-style-dmesg/2/journal.txt
@@ -0,0 +1 @@
+exit:0 (0.010s)
diff --git a/runner/json_tests_data/piglit-style-dmesg/2/out.txt b/runner/json_tests_data/piglit-style-dmesg/2/out.txt
new file mode 100644
index 00000000..695b67c2
--- /dev/null
+++ b/runner/json_tests_data/piglit-style-dmesg/2/out.txt
@@ -0,0 +1,2 @@
+IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
+SUCCESS (0.000s)
diff --git a/runner/json_tests_data/piglit-style-dmesg/3/dmesg.txt b/runner/json_tests_data/piglit-style-dmesg/3/dmesg.txt
new file mode 100644
index 00000000..21e75031
--- /dev/null
+++ b/runner/json_tests_data/piglit-style-dmesg/3/dmesg.txt
@@ -0,0 +1,4 @@
+6,965,3216186135188,-;Console: switching to colour dummy device 80x25
+14,966,3216186135212,-;[IGT] skippers: executing
+14,967,3216186137075,-;[IGT] skippers: exiting, ret=77
+6,968,3216186137206,-;Console: switching to colour frame buffer device 240x75
diff --git a/runner/json_tests_data/piglit-style-dmesg/3/err.txt b/runner/json_tests_data/piglit-style-dmesg/3/err.txt
new file mode 100644
index 00000000..59b73d09
--- /dev/null
+++ b/runner/json_tests_data/piglit-style-dmesg/3/err.txt
@@ -0,0 +1 @@
+Subtest skip-one: SKIP
diff --git a/runner/json_tests_data/piglit-style-dmesg/3/journal.txt b/runner/json_tests_data/piglit-style-dmesg/3/journal.txt
new file mode 100644
index 00000000..afab8ba6
--- /dev/null
+++ b/runner/json_tests_data/piglit-style-dmesg/3/journal.txt
@@ -0,0 +1,2 @@
+skip-one
+exit:77 (0.011s)
diff --git a/runner/json_tests_data/piglit-style-dmesg/3/out.txt b/runner/json_tests_data/piglit-style-dmesg/3/out.txt
new file mode 100644
index 00000000..96284e78
--- /dev/null
+++ b/runner/json_tests_data/piglit-style-dmesg/3/out.txt
@@ -0,0 +1,6 @@
+IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
+Test requirement not met in function __real_main3, file ../runner/testdata/skippers.c:6:
+Test requirement: false
+Skipping from fixture
+Last errno: 2, No such file or directory
+Subtest skip-one: SKIP
diff --git a/runner/json_tests_data/piglit-style-dmesg/4/dmesg.txt b/runner/json_tests_data/piglit-style-dmesg/4/dmesg.txt
new file mode 100644
index 00000000..737bc692
--- /dev/null
+++ b/runner/json_tests_data/piglit-style-dmesg/4/dmesg.txt
@@ -0,0 +1,4 @@
+6,969,3216186145899,-;Console: switching to colour dummy device 80x25
+14,970,3216186145912,-;[IGT] skippers: executing
+14,971,3216186147754,-;[IGT] skippers: exiting, ret=77
+6,972,3216186147894,-;Console: switching to colour frame buffer device 240x75
diff --git a/runner/json_tests_data/piglit-style-dmesg/4/err.txt b/runner/json_tests_data/piglit-style-dmesg/4/err.txt
new file mode 100644
index 00000000..2251da1e
--- /dev/null
+++ b/runner/json_tests_data/piglit-style-dmesg/4/err.txt
@@ -0,0 +1 @@
+Subtest skip-two: SKIP
diff --git a/runner/json_tests_data/piglit-style-dmesg/4/journal.txt b/runner/json_tests_data/piglit-style-dmesg/4/journal.txt
new file mode 100644
index 00000000..a9dba132
--- /dev/null
+++ b/runner/json_tests_data/piglit-style-dmesg/4/journal.txt
@@ -0,0 +1,2 @@
+skip-two
+exit:77 (0.010s)
diff --git a/runner/json_tests_data/piglit-style-dmesg/4/out.txt b/runner/json_tests_data/piglit-style-dmesg/4/out.txt
new file mode 100644
index 00000000..2024db8f
--- /dev/null
+++ b/runner/json_tests_data/piglit-style-dmesg/4/out.txt
@@ -0,0 +1,6 @@
+IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
+Test requirement not met in function __real_main3, file ../runner/testdata/skippers.c:6:
+Test requirement: false
+Skipping from fixture
+Last errno: 2, No such file or directory
+Subtest skip-two: SKIP
diff --git a/runner/json_tests_data/piglit-style-dmesg/README.txt b/runner/json_tests_data/piglit-style-dmesg/README.txt
new file mode 100644
index 00000000..3130337b
--- /dev/null
+++ b/runner/json_tests_data/piglit-style-dmesg/README.txt
@@ -0,0 +1,2 @@
+When --piglit-style-dmesg is used, only a particular pattern of kernel
+log messages trigger a result change to dmesg-warn/dmesg-fail.
diff --git a/runner/json_tests_data/piglit-style-dmesg/endtime.txt b/runner/json_tests_data/piglit-style-dmesg/endtime.txt
new file mode 100644
index 00000000..635f6ae9
--- /dev/null
+++ b/runner/json_tests_data/piglit-style-dmesg/endtime.txt
@@ -0,0 +1 @@
+1539953735.172373
diff --git a/runner/json_tests_data/piglit-style-dmesg/joblist.txt b/runner/json_tests_data/piglit-style-dmesg/joblist.txt
new file mode 100644
index 00000000..31ef8413
--- /dev/null
+++ b/runner/json_tests_data/piglit-style-dmesg/joblist.txt
@@ -0,0 +1,5 @@
+successtest first-subtest
+successtest second-subtest
+no-subtests
+skippers skip-one
+skippers skip-two
diff --git a/runner/json_tests_data/piglit-style-dmesg/metadata.txt b/runner/json_tests_data/piglit-style-dmesg/metadata.txt
new file mode 100644
index 00000000..7f1372c1
--- /dev/null
+++ b/runner/json_tests_data/piglit-style-dmesg/metadata.txt
@@ -0,0 +1,12 @@
+abort_on_error : 0
+name : normal-run
+dry_run : 0
+sync : 0
+log_level : 0
+overwrite : 0
+multiple_mode : 0
+inactivity_timeout : 0
+use_watchdog : 0
+piglit_style_dmesg : 1
+test_root : /path/does/not/exist
+results_path : /path/does/not/exist
diff --git a/runner/json_tests_data/piglit-style-dmesg/reference.json b/runner/json_tests_data/piglit-style-dmesg/reference.json
new file mode 100644
index 00000000..2257b3c7
--- /dev/null
+++ b/runner/json_tests_data/piglit-style-dmesg/reference.json
@@ -0,0 +1,158 @@
+{
+  "__type__":"TestrunResult",
+  "results_version":9,
+  "name":"normal-run",
+  "uname":"Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64",
+  "time_elapsed":{
+    "__type__":"TimeAttribute",
+    "start":1539953735.1110389,
+    "end":1539953735.1723731
+  },
+  "tests":{
+    "igt at successtest@first-subtest":{
+      "out":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n",
+      "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+      "result":"warn",
+      "time":{
+        "__type__":"TimeAttribute",
+        "start":0,
+        "end":0
+      },
+      "err":"Starting subtest: first-subtest\nThis is a warning line\nSubtest first-subtest: SUCCESS (0.000s)\n",
+	"dmesg":"<6> [3216186.095083] Console: switching to colour dummy device 80x25\n<6> [3216186.095097] [IGT] successtest: executing\n<6> [3216186.101115] [IGT] successtest: starting subtest first-subtest\n<3> [3216186.101159] Warning from kernel\n<6> [3216186.101160] [IGT] successtest: exiting, ret=0\n<6> [3216186.101299] Console: switching to colour frame buffer device 240x75\n"
+    },
+    "igt at successtest@second-subtest":{
+      "out":"Starting subtest: second-subtest\nSubtest second-subtest: SUCCESS (0.000s)\n",
+      "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+      "result":"pass",
+      "time":{
+        "__type__":"TimeAttribute",
+        "start":0,
+        "end":0
+      },
+      "err":"Starting subtest: second-subtest\nSubtest second-subtest: SUCCESS (0.000s)\n",
+      "dmesg":"<6> [3216186.111837] Console: switching to colour dummy device 80x25\n<6> [3216186.111851] [IGT] successtest: executing\n<6> [3216186.114762] [IGT] successtest: starting subtest second-subtest\n<6> [3216186.114814] [IGT] successtest: exiting, ret=0\n<6> [3216186.114933] Console: switching to colour frame buffer device 240x75\n"
+    },
+    "igt at no-subtests":{
+      "time":{
+        "__type__":"TimeAttribute",
+        "start":0,
+        "end":0.01
+      },
+      "result":"pass",
+      "out":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)\nSUCCESS (0.000s)\n",
+      "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+      "err":"",
+      "dmesg":"<6> [3216186.123400] Console: switching to colour dummy device 80x25\n<6> [3216186.123414] [IGT] no-subtests: executing\n<6> [3216186.125204] [IGT] no-subtests: exiting, ret=0\n<6> [3216186.125374] Console: switching to colour frame buffer device 240x75\n"
+    },
+    "igt at skippers@skip-one":{
+      "out":"Test requirement not met in function __real_main3, file ..\/runner\/testdata\/skippers.c:6:\nTest requirement: false\nSkipping from fixture\nLast errno: 2, No such file or directory\nSubtest skip-one: SKIP\n",
+      "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+      "result":"skip",
+      "time":{
+        "__type__":"TimeAttribute",
+        "start":0,
+        "end":0
+      },
+      "err":"Subtest skip-one: SKIP\n",
+      "dmesg":"<6> [3216186.135188] Console: switching to colour dummy device 80x25\n<6> [3216186.135212] [IGT] skippers: executing\n<6> [3216186.137075] [IGT] skippers: exiting, ret=77\n<6> [3216186.137206] Console: switching to colour frame buffer device 240x75\n"
+    },
+    "igt at skippers@skip-two":{
+      "out":"Test requirement not met in function __real_main3, file ..\/runner\/testdata\/skippers.c:6:\nTest requirement: false\nSkipping from fixture\nLast errno: 2, No such file or directory\nSubtest skip-two: SKIP\n",
+      "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+      "result":"skip",
+      "time":{
+        "__type__":"TimeAttribute",
+        "start":0,
+        "end":0
+      },
+      "err":"Subtest skip-two: SKIP\n",
+      "dmesg":"<6> [3216186.145899] Console: switching to colour dummy device 80x25\n<6> [3216186.145912] [IGT] skippers: executing\n<6> [3216186.147754] [IGT] skippers: exiting, ret=77\n<6> [3216186.147894] Console: switching to colour frame buffer device 240x75\n"
+    }
+  },
+  "totals":{
+    "":{
+      "crash":0,
+      "pass":2,
+      "dmesg-fail":0,
+      "dmesg-warn":0,
+      "skip":2,
+      "incomplete":0,
+      "timeout":0,
+      "notrun":0,
+      "fail":0,
+      "warn":1
+    },
+    "root":{
+      "crash":0,
+      "pass":2,
+      "dmesg-fail":0,
+      "dmesg-warn":0,
+      "skip":2,
+      "incomplete":0,
+      "timeout":0,
+      "notrun":0,
+      "fail":0,
+      "warn":1
+    },
+    "igt at successtest":{
+      "crash":0,
+      "pass":1,
+      "dmesg-fail":0,
+      "dmesg-warn":0,
+      "skip":0,
+      "incomplete":0,
+      "timeout":0,
+      "notrun":0,
+      "fail":0,
+      "warn":1
+    },
+    "igt at no-subtests":{
+      "crash":0,
+      "pass":1,
+      "dmesg-fail":0,
+      "dmesg-warn":0,
+      "skip":0,
+      "incomplete":0,
+      "timeout":0,
+      "notrun":0,
+      "fail":0,
+      "warn":0
+    },
+    "igt at skippers":{
+      "crash":0,
+      "pass":0,
+      "dmesg-fail":0,
+      "dmesg-warn":0,
+      "skip":2,
+      "incomplete":0,
+      "timeout":0,
+      "notrun":0,
+      "fail":0,
+      "warn":0
+    }
+  },
+  "runtimes":{
+    "igt at successtest":{
+      "time":{
+        "__type__":"TimeAttribute",
+        "start":0,
+        "end":0.027
+      }
+    },
+    "igt at no-subtests":{
+      "time":{
+        "__type__":"TimeAttribute",
+        "start":0,
+        "end":0.01
+      }
+    },
+    "igt at skippers":{
+      "time":{
+        "__type__":"TimeAttribute",
+        "start":0,
+        "end":0.020999999999999998
+      }
+    }
+  }
+}
diff --git a/runner/json_tests_data/piglit-style-dmesg/starttime.txt b/runner/json_tests_data/piglit-style-dmesg/starttime.txt
new file mode 100644
index 00000000..ae038f18
--- /dev/null
+++ b/runner/json_tests_data/piglit-style-dmesg/starttime.txt
@@ -0,0 +1 @@
+1539953735.111039
diff --git a/runner/json_tests_data/piglit-style-dmesg/uname.txt b/runner/json_tests_data/piglit-style-dmesg/uname.txt
new file mode 100644
index 00000000..a7aef6f7
--- /dev/null
+++ b/runner/json_tests_data/piglit-style-dmesg/uname.txt
@@ -0,0 +1 @@
+Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64
diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/0/dmesg.txt b/runner/json_tests_data/warnings-with-dmesg-warns/0/dmesg.txt
new file mode 100644
index 00000000..e50f75db
--- /dev/null
+++ b/runner/json_tests_data/warnings-with-dmesg-warns/0/dmesg.txt
@@ -0,0 +1,6 @@
+6,951,3216186095083,-;Console: switching to colour dummy device 80x25
+14,952,3216186095097,-;[IGT] successtest: executing
+14,953,3216186101115,-;[IGT] successtest: starting subtest first-subtest
+3,954,3216186101159,-;Warning from kernel
+14,955,3216186101160,-;[IGT] successtest: exiting, ret=0
+6,956,3216186101299,-;Console: switching to colour frame buffer device 240x75
diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/0/err.txt b/runner/json_tests_data/warnings-with-dmesg-warns/0/err.txt
new file mode 100644
index 00000000..e18c00e9
--- /dev/null
+++ b/runner/json_tests_data/warnings-with-dmesg-warns/0/err.txt
@@ -0,0 +1,3 @@
+Starting subtest: first-subtest
+This is a warning line
+Subtest first-subtest: SUCCESS (0.000s)
diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/0/journal.txt b/runner/json_tests_data/warnings-with-dmesg-warns/0/journal.txt
new file mode 100644
index 00000000..86a30e07
--- /dev/null
+++ b/runner/json_tests_data/warnings-with-dmesg-warns/0/journal.txt
@@ -0,0 +1,2 @@
+first-subtest
+exit:0 (0.014s)
diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/0/out.txt b/runner/json_tests_data/warnings-with-dmesg-warns/0/out.txt
new file mode 100644
index 00000000..5946bf31
--- /dev/null
+++ b/runner/json_tests_data/warnings-with-dmesg-warns/0/out.txt
@@ -0,0 +1,3 @@
+IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
+Starting subtest: first-subtest
+Subtest first-subtest: SUCCESS (0.000s)
diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/1/dmesg.txt b/runner/json_tests_data/warnings-with-dmesg-warns/1/dmesg.txt
new file mode 100644
index 00000000..59b7cee3
--- /dev/null
+++ b/runner/json_tests_data/warnings-with-dmesg-warns/1/dmesg.txt
@@ -0,0 +1,5 @@
+6,956,3216186111837,-;Console: switching to colour dummy device 80x25
+14,957,3216186111851,-;[IGT] successtest: executing
+14,958,3216186114762,-;[IGT] successtest: starting subtest second-subtest
+14,959,3216186114814,-;[IGT] successtest: exiting, ret=0
+6,960,3216186114933,-;Console: switching to colour frame buffer device 240x75
diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/1/err.txt b/runner/json_tests_data/warnings-with-dmesg-warns/1/err.txt
new file mode 100644
index 00000000..57be9f7b
--- /dev/null
+++ b/runner/json_tests_data/warnings-with-dmesg-warns/1/err.txt
@@ -0,0 +1,2 @@
+Starting subtest: second-subtest
+Subtest second-subtest: SUCCESS (0.000s)
diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/1/journal.txt b/runner/json_tests_data/warnings-with-dmesg-warns/1/journal.txt
new file mode 100644
index 00000000..99f57815
--- /dev/null
+++ b/runner/json_tests_data/warnings-with-dmesg-warns/1/journal.txt
@@ -0,0 +1,2 @@
+second-subtest
+exit:0 (0.013s)
diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/1/out.txt b/runner/json_tests_data/warnings-with-dmesg-warns/1/out.txt
new file mode 100644
index 00000000..24b37244
--- /dev/null
+++ b/runner/json_tests_data/warnings-with-dmesg-warns/1/out.txt
@@ -0,0 +1,3 @@
+IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
+Starting subtest: second-subtest
+Subtest second-subtest: SUCCESS (0.000s)
diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/2/dmesg.txt b/runner/json_tests_data/warnings-with-dmesg-warns/2/dmesg.txt
new file mode 100644
index 00000000..998b4797
--- /dev/null
+++ b/runner/json_tests_data/warnings-with-dmesg-warns/2/dmesg.txt
@@ -0,0 +1,4 @@
+6,961,3216186123400,-;Console: switching to colour dummy device 80x25
+14,962,3216186123414,-;[IGT] no-subtests: executing
+14,963,3216186125204,-;[IGT] no-subtests: exiting, ret=0
+6,964,3216186125374,-;Console: switching to colour frame buffer device 240x75
diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/2/err.txt b/runner/json_tests_data/warnings-with-dmesg-warns/2/err.txt
new file mode 100644
index 00000000..e69de29b
diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/2/journal.txt b/runner/json_tests_data/warnings-with-dmesg-warns/2/journal.txt
new file mode 100644
index 00000000..7151877f
--- /dev/null
+++ b/runner/json_tests_data/warnings-with-dmesg-warns/2/journal.txt
@@ -0,0 +1 @@
+exit:0 (0.010s)
diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/2/out.txt b/runner/json_tests_data/warnings-with-dmesg-warns/2/out.txt
new file mode 100644
index 00000000..695b67c2
--- /dev/null
+++ b/runner/json_tests_data/warnings-with-dmesg-warns/2/out.txt
@@ -0,0 +1,2 @@
+IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
+SUCCESS (0.000s)
diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/3/dmesg.txt b/runner/json_tests_data/warnings-with-dmesg-warns/3/dmesg.txt
new file mode 100644
index 00000000..21e75031
--- /dev/null
+++ b/runner/json_tests_data/warnings-with-dmesg-warns/3/dmesg.txt
@@ -0,0 +1,4 @@
+6,965,3216186135188,-;Console: switching to colour dummy device 80x25
+14,966,3216186135212,-;[IGT] skippers: executing
+14,967,3216186137075,-;[IGT] skippers: exiting, ret=77
+6,968,3216186137206,-;Console: switching to colour frame buffer device 240x75
diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/3/err.txt b/runner/json_tests_data/warnings-with-dmesg-warns/3/err.txt
new file mode 100644
index 00000000..59b73d09
--- /dev/null
+++ b/runner/json_tests_data/warnings-with-dmesg-warns/3/err.txt
@@ -0,0 +1 @@
+Subtest skip-one: SKIP
diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/3/journal.txt b/runner/json_tests_data/warnings-with-dmesg-warns/3/journal.txt
new file mode 100644
index 00000000..afab8ba6
--- /dev/null
+++ b/runner/json_tests_data/warnings-with-dmesg-warns/3/journal.txt
@@ -0,0 +1,2 @@
+skip-one
+exit:77 (0.011s)
diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/3/out.txt b/runner/json_tests_data/warnings-with-dmesg-warns/3/out.txt
new file mode 100644
index 00000000..96284e78
--- /dev/null
+++ b/runner/json_tests_data/warnings-with-dmesg-warns/3/out.txt
@@ -0,0 +1,6 @@
+IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
+Test requirement not met in function __real_main3, file ../runner/testdata/skippers.c:6:
+Test requirement: false
+Skipping from fixture
+Last errno: 2, No such file or directory
+Subtest skip-one: SKIP
diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/4/dmesg.txt b/runner/json_tests_data/warnings-with-dmesg-warns/4/dmesg.txt
new file mode 100644
index 00000000..737bc692
--- /dev/null
+++ b/runner/json_tests_data/warnings-with-dmesg-warns/4/dmesg.txt
@@ -0,0 +1,4 @@
+6,969,3216186145899,-;Console: switching to colour dummy device 80x25
+14,970,3216186145912,-;[IGT] skippers: executing
+14,971,3216186147754,-;[IGT] skippers: exiting, ret=77
+6,972,3216186147894,-;Console: switching to colour frame buffer device 240x75
diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/4/err.txt b/runner/json_tests_data/warnings-with-dmesg-warns/4/err.txt
new file mode 100644
index 00000000..2251da1e
--- /dev/null
+++ b/runner/json_tests_data/warnings-with-dmesg-warns/4/err.txt
@@ -0,0 +1 @@
+Subtest skip-two: SKIP
diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/4/journal.txt b/runner/json_tests_data/warnings-with-dmesg-warns/4/journal.txt
new file mode 100644
index 00000000..a9dba132
--- /dev/null
+++ b/runner/json_tests_data/warnings-with-dmesg-warns/4/journal.txt
@@ -0,0 +1,2 @@
+skip-two
+exit:77 (0.010s)
diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/4/out.txt b/runner/json_tests_data/warnings-with-dmesg-warns/4/out.txt
new file mode 100644
index 00000000..2024db8f
--- /dev/null
+++ b/runner/json_tests_data/warnings-with-dmesg-warns/4/out.txt
@@ -0,0 +1,6 @@
+IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
+Test requirement not met in function __real_main3, file ../runner/testdata/skippers.c:6:
+Test requirement: false
+Skipping from fixture
+Last errno: 2, No such file or directory
+Subtest skip-two: SKIP
diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/README.txt b/runner/json_tests_data/warnings-with-dmesg-warns/README.txt
new file mode 100644
index 00000000..7ef9ee73
--- /dev/null
+++ b/runner/json_tests_data/warnings-with-dmesg-warns/README.txt
@@ -0,0 +1,3 @@
+A test that has output in stderr should result in 'warn' instead of
+'pass'. But if there is a message in the kernel logs that would make
+the result 'dmesg-warn' instead, that takes priority over 'warn'.
diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/endtime.txt b/runner/json_tests_data/warnings-with-dmesg-warns/endtime.txt
new file mode 100644
index 00000000..635f6ae9
--- /dev/null
+++ b/runner/json_tests_data/warnings-with-dmesg-warns/endtime.txt
@@ -0,0 +1 @@
+1539953735.172373
diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/joblist.txt b/runner/json_tests_data/warnings-with-dmesg-warns/joblist.txt
new file mode 100644
index 00000000..31ef8413
--- /dev/null
+++ b/runner/json_tests_data/warnings-with-dmesg-warns/joblist.txt
@@ -0,0 +1,5 @@
+successtest first-subtest
+successtest second-subtest
+no-subtests
+skippers skip-one
+skippers skip-two
diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/metadata.txt b/runner/json_tests_data/warnings-with-dmesg-warns/metadata.txt
new file mode 100644
index 00000000..1316560d
--- /dev/null
+++ b/runner/json_tests_data/warnings-with-dmesg-warns/metadata.txt
@@ -0,0 +1,12 @@
+abort_on_error : 0
+name : normal-run
+dry_run : 0
+sync : 0
+log_level : 0
+overwrite : 0
+multiple_mode : 0
+inactivity_timeout : 0
+use_watchdog : 0
+piglit_style_dmesg : 0
+test_root : /path/does/not/exist
+results_path : /path/does/not/exist
diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/reference.json b/runner/json_tests_data/warnings-with-dmesg-warns/reference.json
new file mode 100644
index 00000000..9e027efb
--- /dev/null
+++ b/runner/json_tests_data/warnings-with-dmesg-warns/reference.json
@@ -0,0 +1,159 @@
+{
+  "__type__":"TestrunResult",
+  "results_version":9,
+  "name":"normal-run",
+  "uname":"Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64",
+  "time_elapsed":{
+    "__type__":"TimeAttribute",
+    "start":1539953735.1110389,
+    "end":1539953735.1723731
+  },
+  "tests":{
+    "igt at successtest@first-subtest":{
+      "out":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n",
+      "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+      "result":"dmesg-warn",
+      "time":{
+        "__type__":"TimeAttribute",
+        "start":0,
+        "end":0
+      },
+      "err":"Starting subtest: first-subtest\nThis is a warning line\nSubtest first-subtest: SUCCESS (0.000s)\n",
+	"dmesg":"<6> [3216186.095083] Console: switching to colour dummy device 80x25\n<6> [3216186.095097] [IGT] successtest: executing\n<6> [3216186.101115] [IGT] successtest: starting subtest first-subtest\n<3> [3216186.101159] Warning from kernel\n<6> [3216186.101160] [IGT] successtest: exiting, ret=0\n<6> [3216186.101299] Console: switching to colour frame buffer device 240x75\n",
+	"dmesg-warnings":"<3> [3216186.101159] Warning from kernel\n"
+    },
+    "igt at successtest@second-subtest":{
+      "out":"Starting subtest: second-subtest\nSubtest second-subtest: SUCCESS (0.000s)\n",
+      "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+      "result":"pass",
+      "time":{
+        "__type__":"TimeAttribute",
+        "start":0,
+        "end":0
+      },
+      "err":"Starting subtest: second-subtest\nSubtest second-subtest: SUCCESS (0.000s)\n",
+      "dmesg":"<6> [3216186.111837] Console: switching to colour dummy device 80x25\n<6> [3216186.111851] [IGT] successtest: executing\n<6> [3216186.114762] [IGT] successtest: starting subtest second-subtest\n<6> [3216186.114814] [IGT] successtest: exiting, ret=0\n<6> [3216186.114933] Console: switching to colour frame buffer device 240x75\n"
+    },
+    "igt at no-subtests":{
+      "time":{
+        "__type__":"TimeAttribute",
+        "start":0,
+        "end":0.01
+      },
+      "result":"pass",
+      "out":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)\nSUCCESS (0.000s)\n",
+      "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+      "err":"",
+      "dmesg":"<6> [3216186.123400] Console: switching to colour dummy device 80x25\n<6> [3216186.123414] [IGT] no-subtests: executing\n<6> [3216186.125204] [IGT] no-subtests: exiting, ret=0\n<6> [3216186.125374] Console: switching to colour frame buffer device 240x75\n"
+    },
+    "igt at skippers@skip-one":{
+      "out":"Test requirement not met in function __real_main3, file ..\/runner\/testdata\/skippers.c:6:\nTest requirement: false\nSkipping from fixture\nLast errno: 2, No such file or directory\nSubtest skip-one: SKIP\n",
+      "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+      "result":"skip",
+      "time":{
+        "__type__":"TimeAttribute",
+        "start":0,
+        "end":0
+      },
+      "err":"Subtest skip-one: SKIP\n",
+      "dmesg":"<6> [3216186.135188] Console: switching to colour dummy device 80x25\n<6> [3216186.135212] [IGT] skippers: executing\n<6> [3216186.137075] [IGT] skippers: exiting, ret=77\n<6> [3216186.137206] Console: switching to colour frame buffer device 240x75\n"
+    },
+    "igt at skippers@skip-two":{
+      "out":"Test requirement not met in function __real_main3, file ..\/runner\/testdata\/skippers.c:6:\nTest requirement: false\nSkipping from fixture\nLast errno: 2, No such file or directory\nSubtest skip-two: SKIP\n",
+      "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+      "result":"skip",
+      "time":{
+        "__type__":"TimeAttribute",
+        "start":0,
+        "end":0
+      },
+      "err":"Subtest skip-two: SKIP\n",
+      "dmesg":"<6> [3216186.145899] Console: switching to colour dummy device 80x25\n<6> [3216186.145912] [IGT] skippers: executing\n<6> [3216186.147754] [IGT] skippers: exiting, ret=77\n<6> [3216186.147894] Console: switching to colour frame buffer device 240x75\n"
+    }
+  },
+  "totals":{
+    "":{
+      "crash":0,
+      "pass":2,
+      "dmesg-fail":0,
+      "dmesg-warn":1,
+      "skip":2,
+      "incomplete":0,
+      "timeout":0,
+      "notrun":0,
+      "fail":0,
+      "warn":0
+    },
+    "root":{
+      "crash":0,
+      "pass":2,
+      "dmesg-fail":0,
+      "dmesg-warn":1,
+      "skip":2,
+      "incomplete":0,
+      "timeout":0,
+      "notrun":0,
+      "fail":0,
+      "warn":0
+    },
+    "igt at successtest":{
+      "crash":0,
+      "pass":1,
+      "dmesg-fail":0,
+      "dmesg-warn":1,
+      "skip":0,
+      "incomplete":0,
+      "timeout":0,
+      "notrun":0,
+      "fail":0,
+      "warn":0
+    },
+    "igt at no-subtests":{
+      "crash":0,
+      "pass":1,
+      "dmesg-fail":0,
+      "dmesg-warn":0,
+      "skip":0,
+      "incomplete":0,
+      "timeout":0,
+      "notrun":0,
+      "fail":0,
+      "warn":0
+    },
+    "igt at skippers":{
+      "crash":0,
+      "pass":0,
+      "dmesg-fail":0,
+      "dmesg-warn":0,
+      "skip":2,
+      "incomplete":0,
+      "timeout":0,
+      "notrun":0,
+      "fail":0,
+      "warn":0
+    }
+  },
+  "runtimes":{
+    "igt at successtest":{
+      "time":{
+        "__type__":"TimeAttribute",
+        "start":0,
+        "end":0.027
+      }
+    },
+    "igt at no-subtests":{
+      "time":{
+        "__type__":"TimeAttribute",
+        "start":0,
+        "end":0.01
+      }
+    },
+    "igt at skippers":{
+      "time":{
+        "__type__":"TimeAttribute",
+        "start":0,
+        "end":0.020999999999999998
+      }
+    }
+  }
+}
diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/starttime.txt b/runner/json_tests_data/warnings-with-dmesg-warns/starttime.txt
new file mode 100644
index 00000000..ae038f18
--- /dev/null
+++ b/runner/json_tests_data/warnings-with-dmesg-warns/starttime.txt
@@ -0,0 +1 @@
+1539953735.111039
diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/uname.txt b/runner/json_tests_data/warnings-with-dmesg-warns/uname.txt
new file mode 100644
index 00000000..a7aef6f7
--- /dev/null
+++ b/runner/json_tests_data/warnings-with-dmesg-warns/uname.txt
@@ -0,0 +1 @@
+Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64
diff --git a/runner/json_tests_data/warnings/0/dmesg.txt b/runner/json_tests_data/warnings/0/dmesg.txt
new file mode 100644
index 00000000..a189e704
--- /dev/null
+++ b/runner/json_tests_data/warnings/0/dmesg.txt
@@ -0,0 +1,5 @@
+6,951,3216186095083,-;Console: switching to colour dummy device 80x25
+14,952,3216186095097,-;[IGT] successtest: executing
+14,953,3216186101115,-;[IGT] successtest: starting subtest first-subtest
+14,954,3216186101160,-;[IGT] successtest: exiting, ret=0
+6,955,3216186101299,-;Console: switching to colour frame buffer device 240x75
diff --git a/runner/json_tests_data/warnings/0/err.txt b/runner/json_tests_data/warnings/0/err.txt
new file mode 100644
index 00000000..e18c00e9
--- /dev/null
+++ b/runner/json_tests_data/warnings/0/err.txt
@@ -0,0 +1,3 @@
+Starting subtest: first-subtest
+This is a warning line
+Subtest first-subtest: SUCCESS (0.000s)
diff --git a/runner/json_tests_data/warnings/0/journal.txt b/runner/json_tests_data/warnings/0/journal.txt
new file mode 100644
index 00000000..86a30e07
--- /dev/null
+++ b/runner/json_tests_data/warnings/0/journal.txt
@@ -0,0 +1,2 @@
+first-subtest
+exit:0 (0.014s)
diff --git a/runner/json_tests_data/warnings/0/out.txt b/runner/json_tests_data/warnings/0/out.txt
new file mode 100644
index 00000000..5946bf31
--- /dev/null
+++ b/runner/json_tests_data/warnings/0/out.txt
@@ -0,0 +1,3 @@
+IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
+Starting subtest: first-subtest
+Subtest first-subtest: SUCCESS (0.000s)
diff --git a/runner/json_tests_data/warnings/1/dmesg.txt b/runner/json_tests_data/warnings/1/dmesg.txt
new file mode 100644
index 00000000..59b7cee3
--- /dev/null
+++ b/runner/json_tests_data/warnings/1/dmesg.txt
@@ -0,0 +1,5 @@
+6,956,3216186111837,-;Console: switching to colour dummy device 80x25
+14,957,3216186111851,-;[IGT] successtest: executing
+14,958,3216186114762,-;[IGT] successtest: starting subtest second-subtest
+14,959,3216186114814,-;[IGT] successtest: exiting, ret=0
+6,960,3216186114933,-;Console: switching to colour frame buffer device 240x75
diff --git a/runner/json_tests_data/warnings/1/err.txt b/runner/json_tests_data/warnings/1/err.txt
new file mode 100644
index 00000000..57be9f7b
--- /dev/null
+++ b/runner/json_tests_data/warnings/1/err.txt
@@ -0,0 +1,2 @@
+Starting subtest: second-subtest
+Subtest second-subtest: SUCCESS (0.000s)
diff --git a/runner/json_tests_data/warnings/1/journal.txt b/runner/json_tests_data/warnings/1/journal.txt
new file mode 100644
index 00000000..99f57815
--- /dev/null
+++ b/runner/json_tests_data/warnings/1/journal.txt
@@ -0,0 +1,2 @@
+second-subtest
+exit:0 (0.013s)
diff --git a/runner/json_tests_data/warnings/1/out.txt b/runner/json_tests_data/warnings/1/out.txt
new file mode 100644
index 00000000..24b37244
--- /dev/null
+++ b/runner/json_tests_data/warnings/1/out.txt
@@ -0,0 +1,3 @@
+IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
+Starting subtest: second-subtest
+Subtest second-subtest: SUCCESS (0.000s)
diff --git a/runner/json_tests_data/warnings/2/dmesg.txt b/runner/json_tests_data/warnings/2/dmesg.txt
new file mode 100644
index 00000000..998b4797
--- /dev/null
+++ b/runner/json_tests_data/warnings/2/dmesg.txt
@@ -0,0 +1,4 @@
+6,961,3216186123400,-;Console: switching to colour dummy device 80x25
+14,962,3216186123414,-;[IGT] no-subtests: executing
+14,963,3216186125204,-;[IGT] no-subtests: exiting, ret=0
+6,964,3216186125374,-;Console: switching to colour frame buffer device 240x75
diff --git a/runner/json_tests_data/warnings/2/err.txt b/runner/json_tests_data/warnings/2/err.txt
new file mode 100644
index 00000000..e69de29b
diff --git a/runner/json_tests_data/warnings/2/journal.txt b/runner/json_tests_data/warnings/2/journal.txt
new file mode 100644
index 00000000..7151877f
--- /dev/null
+++ b/runner/json_tests_data/warnings/2/journal.txt
@@ -0,0 +1 @@
+exit:0 (0.010s)
diff --git a/runner/json_tests_data/warnings/2/out.txt b/runner/json_tests_data/warnings/2/out.txt
new file mode 100644
index 00000000..695b67c2
--- /dev/null
+++ b/runner/json_tests_data/warnings/2/out.txt
@@ -0,0 +1,2 @@
+IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
+SUCCESS (0.000s)
diff --git a/runner/json_tests_data/warnings/3/dmesg.txt b/runner/json_tests_data/warnings/3/dmesg.txt
new file mode 100644
index 00000000..21e75031
--- /dev/null
+++ b/runner/json_tests_data/warnings/3/dmesg.txt
@@ -0,0 +1,4 @@
+6,965,3216186135188,-;Console: switching to colour dummy device 80x25
+14,966,3216186135212,-;[IGT] skippers: executing
+14,967,3216186137075,-;[IGT] skippers: exiting, ret=77
+6,968,3216186137206,-;Console: switching to colour frame buffer device 240x75
diff --git a/runner/json_tests_data/warnings/3/err.txt b/runner/json_tests_data/warnings/3/err.txt
new file mode 100644
index 00000000..59b73d09
--- /dev/null
+++ b/runner/json_tests_data/warnings/3/err.txt
@@ -0,0 +1 @@
+Subtest skip-one: SKIP
diff --git a/runner/json_tests_data/warnings/3/journal.txt b/runner/json_tests_data/warnings/3/journal.txt
new file mode 100644
index 00000000..afab8ba6
--- /dev/null
+++ b/runner/json_tests_data/warnings/3/journal.txt
@@ -0,0 +1,2 @@
+skip-one
+exit:77 (0.011s)
diff --git a/runner/json_tests_data/warnings/3/out.txt b/runner/json_tests_data/warnings/3/out.txt
new file mode 100644
index 00000000..96284e78
--- /dev/null
+++ b/runner/json_tests_data/warnings/3/out.txt
@@ -0,0 +1,6 @@
+IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
+Test requirement not met in function __real_main3, file ../runner/testdata/skippers.c:6:
+Test requirement: false
+Skipping from fixture
+Last errno: 2, No such file or directory
+Subtest skip-one: SKIP
diff --git a/runner/json_tests_data/warnings/4/dmesg.txt b/runner/json_tests_data/warnings/4/dmesg.txt
new file mode 100644
index 00000000..737bc692
--- /dev/null
+++ b/runner/json_tests_data/warnings/4/dmesg.txt
@@ -0,0 +1,4 @@
+6,969,3216186145899,-;Console: switching to colour dummy device 80x25
+14,970,3216186145912,-;[IGT] skippers: executing
+14,971,3216186147754,-;[IGT] skippers: exiting, ret=77
+6,972,3216186147894,-;Console: switching to colour frame buffer device 240x75
diff --git a/runner/json_tests_data/warnings/4/err.txt b/runner/json_tests_data/warnings/4/err.txt
new file mode 100644
index 00000000..2251da1e
--- /dev/null
+++ b/runner/json_tests_data/warnings/4/err.txt
@@ -0,0 +1 @@
+Subtest skip-two: SKIP
diff --git a/runner/json_tests_data/warnings/4/journal.txt b/runner/json_tests_data/warnings/4/journal.txt
new file mode 100644
index 00000000..a9dba132
--- /dev/null
+++ b/runner/json_tests_data/warnings/4/journal.txt
@@ -0,0 +1,2 @@
+skip-two
+exit:77 (0.010s)
diff --git a/runner/json_tests_data/warnings/4/out.txt b/runner/json_tests_data/warnings/4/out.txt
new file mode 100644
index 00000000..2024db8f
--- /dev/null
+++ b/runner/json_tests_data/warnings/4/out.txt
@@ -0,0 +1,6 @@
+IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
+Test requirement not met in function __real_main3, file ../runner/testdata/skippers.c:6:
+Test requirement: false
+Skipping from fixture
+Last errno: 2, No such file or directory
+Subtest skip-two: SKIP
diff --git a/runner/json_tests_data/warnings/README.txt b/runner/json_tests_data/warnings/README.txt
new file mode 100644
index 00000000..c776add0
--- /dev/null
+++ b/runner/json_tests_data/warnings/README.txt
@@ -0,0 +1,2 @@
+Any output in stderr should result in the result being 'warn' instead
+of 'pass'.
diff --git a/runner/json_tests_data/warnings/endtime.txt b/runner/json_tests_data/warnings/endtime.txt
new file mode 100644
index 00000000..635f6ae9
--- /dev/null
+++ b/runner/json_tests_data/warnings/endtime.txt
@@ -0,0 +1 @@
+1539953735.172373
diff --git a/runner/json_tests_data/warnings/joblist.txt b/runner/json_tests_data/warnings/joblist.txt
new file mode 100644
index 00000000..31ef8413
--- /dev/null
+++ b/runner/json_tests_data/warnings/joblist.txt
@@ -0,0 +1,5 @@
+successtest first-subtest
+successtest second-subtest
+no-subtests
+skippers skip-one
+skippers skip-two
diff --git a/runner/json_tests_data/warnings/metadata.txt b/runner/json_tests_data/warnings/metadata.txt
new file mode 100644
index 00000000..1316560d
--- /dev/null
+++ b/runner/json_tests_data/warnings/metadata.txt
@@ -0,0 +1,12 @@
+abort_on_error : 0
+name : normal-run
+dry_run : 0
+sync : 0
+log_level : 0
+overwrite : 0
+multiple_mode : 0
+inactivity_timeout : 0
+use_watchdog : 0
+piglit_style_dmesg : 0
+test_root : /path/does/not/exist
+results_path : /path/does/not/exist
diff --git a/runner/json_tests_data/warnings/reference.json b/runner/json_tests_data/warnings/reference.json
new file mode 100644
index 00000000..8d5edbe1
--- /dev/null
+++ b/runner/json_tests_data/warnings/reference.json
@@ -0,0 +1,158 @@
+{
+  "__type__":"TestrunResult",
+  "results_version":9,
+  "name":"normal-run",
+  "uname":"Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64",
+  "time_elapsed":{
+    "__type__":"TimeAttribute",
+    "start":1539953735.1110389,
+    "end":1539953735.1723731
+  },
+  "tests":{
+    "igt at successtest@first-subtest":{
+      "out":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n",
+      "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+      "result":"warn",
+      "time":{
+        "__type__":"TimeAttribute",
+        "start":0,
+        "end":0
+      },
+      "err":"Starting subtest: first-subtest\nThis is a warning line\nSubtest first-subtest: SUCCESS (0.000s)\n",
+      "dmesg":"<6> [3216186.095083] Console: switching to colour dummy device 80x25\n<6> [3216186.095097] [IGT] successtest: executing\n<6> [3216186.101115] [IGT] successtest: starting subtest first-subtest\n<6> [3216186.101160] [IGT] successtest: exiting, ret=0\n<6> [3216186.101299] Console: switching to colour frame buffer device 240x75\n"
+    },
+    "igt at successtest@second-subtest":{
+      "out":"Starting subtest: second-subtest\nSubtest second-subtest: SUCCESS (0.000s)\n",
+      "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+      "result":"pass",
+      "time":{
+        "__type__":"TimeAttribute",
+        "start":0,
+        "end":0
+      },
+      "err":"Starting subtest: second-subtest\nSubtest second-subtest: SUCCESS (0.000s)\n",
+      "dmesg":"<6> [3216186.111837] Console: switching to colour dummy device 80x25\n<6> [3216186.111851] [IGT] successtest: executing\n<6> [3216186.114762] [IGT] successtest: starting subtest second-subtest\n<6> [3216186.114814] [IGT] successtest: exiting, ret=0\n<6> [3216186.114933] Console: switching to colour frame buffer device 240x75\n"
+    },
+    "igt at no-subtests":{
+      "time":{
+        "__type__":"TimeAttribute",
+        "start":0,
+        "end":0.01
+      },
+      "result":"pass",
+      "out":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)\nSUCCESS (0.000s)\n",
+      "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+      "err":"",
+      "dmesg":"<6> [3216186.123400] Console: switching to colour dummy device 80x25\n<6> [3216186.123414] [IGT] no-subtests: executing\n<6> [3216186.125204] [IGT] no-subtests: exiting, ret=0\n<6> [3216186.125374] Console: switching to colour frame buffer device 240x75\n"
+    },
+    "igt at skippers@skip-one":{
+      "out":"Test requirement not met in function __real_main3, file ..\/runner\/testdata\/skippers.c:6:\nTest requirement: false\nSkipping from fixture\nLast errno: 2, No such file or directory\nSubtest skip-one: SKIP\n",
+      "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+      "result":"skip",
+      "time":{
+        "__type__":"TimeAttribute",
+        "start":0,
+        "end":0
+      },
+      "err":"Subtest skip-one: SKIP\n",
+      "dmesg":"<6> [3216186.135188] Console: switching to colour dummy device 80x25\n<6> [3216186.135212] [IGT] skippers: executing\n<6> [3216186.137075] [IGT] skippers: exiting, ret=77\n<6> [3216186.137206] Console: switching to colour frame buffer device 240x75\n"
+    },
+    "igt at skippers@skip-two":{
+      "out":"Test requirement not met in function __real_main3, file ..\/runner\/testdata\/skippers.c:6:\nTest requirement: false\nSkipping from fixture\nLast errno: 2, No such file or directory\nSubtest skip-two: SKIP\n",
+      "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+      "result":"skip",
+      "time":{
+        "__type__":"TimeAttribute",
+        "start":0,
+        "end":0
+      },
+      "err":"Subtest skip-two: SKIP\n",
+      "dmesg":"<6> [3216186.145899] Console: switching to colour dummy device 80x25\n<6> [3216186.145912] [IGT] skippers: executing\n<6> [3216186.147754] [IGT] skippers: exiting, ret=77\n<6> [3216186.147894] Console: switching to colour frame buffer device 240x75\n"
+    }
+  },
+  "totals":{
+    "":{
+      "crash":0,
+      "pass":2,
+      "dmesg-fail":0,
+      "dmesg-warn":0,
+      "skip":2,
+      "incomplete":0,
+      "timeout":0,
+      "notrun":0,
+      "fail":0,
+      "warn":1
+    },
+    "root":{
+      "crash":0,
+      "pass":2,
+      "dmesg-fail":0,
+      "dmesg-warn":0,
+      "skip":2,
+      "incomplete":0,
+      "timeout":0,
+      "notrun":0,
+      "fail":0,
+      "warn":1
+    },
+    "igt at successtest":{
+      "crash":0,
+      "pass":1,
+      "dmesg-fail":0,
+      "dmesg-warn":0,
+      "skip":0,
+      "incomplete":0,
+      "timeout":0,
+      "notrun":0,
+      "fail":0,
+      "warn":1
+    },
+    "igt at no-subtests":{
+      "crash":0,
+      "pass":1,
+      "dmesg-fail":0,
+      "dmesg-warn":0,
+      "skip":0,
+      "incomplete":0,
+      "timeout":0,
+      "notrun":0,
+      "fail":0,
+      "warn":0
+    },
+    "igt at skippers":{
+      "crash":0,
+      "pass":0,
+      "dmesg-fail":0,
+      "dmesg-warn":0,
+      "skip":2,
+      "incomplete":0,
+      "timeout":0,
+      "notrun":0,
+      "fail":0,
+      "warn":0
+    }
+  },
+  "runtimes":{
+    "igt at successtest":{
+      "time":{
+        "__type__":"TimeAttribute",
+        "start":0,
+        "end":0.027
+      }
+    },
+    "igt at no-subtests":{
+      "time":{
+        "__type__":"TimeAttribute",
+        "start":0,
+        "end":0.01
+      }
+    },
+    "igt at skippers":{
+      "time":{
+        "__type__":"TimeAttribute",
+        "start":0,
+        "end":0.020999999999999998
+      }
+    }
+  }
+}
diff --git a/runner/json_tests_data/warnings/starttime.txt b/runner/json_tests_data/warnings/starttime.txt
new file mode 100644
index 00000000..ae038f18
--- /dev/null
+++ b/runner/json_tests_data/warnings/starttime.txt
@@ -0,0 +1 @@
+1539953735.111039
diff --git a/runner/json_tests_data/warnings/uname.txt b/runner/json_tests_data/warnings/uname.txt
new file mode 100644
index 00000000..a7aef6f7
--- /dev/null
+++ b/runner/json_tests_data/warnings/uname.txt
@@ -0,0 +1 @@
+Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64
diff --git a/runner/meson.build b/runner/meson.build
index 6c4b4862..de6e6f1c 100644
--- a/runner/meson.build
+++ b/runner/meson.build
@@ -10,6 +10,7 @@ runner_sources = [ 'runner.c' ]
 resume_sources = [ 'resume.c' ]
 results_sources = [ 'results.c' ]
 runner_test_sources = [ 'runner_tests.c' ]
+runner_json_test_sources = [ 'runner_json_tests.c' ]
 
 if _build_runner and jsonc.found()
 	subdir('testdata')
@@ -46,6 +47,13 @@ if _build_runner and jsonc.found()
 				 dependencies : igt_deps)
 	test('runner', runner_test)
 
+	runner_json_test = executable('runner_json_test', runner_json_test_sources,
+				      c_args : '-DJSON_TESTS_DIRECTORY="@0@"'.format(join_paths(meson.current_source_dir(), 'json_tests_data')),
+				      link_with : runnerlib,
+				      install : false,
+				      dependencies : [igt_deps, jsonc])
+	test('runner_json', runner_json_test)
+
 	build_info += 'Build test runner: Yes'
 else
 	build_info += 'Build test runner: No'
diff --git a/runner/runner_json_tests.c b/runner/runner_json_tests.c
new file mode 100644
index 00000000..758700d4
--- /dev/null
+++ b/runner/runner_json_tests.c
@@ -0,0 +1,171 @@
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+
+#include <json.h>
+
+#include "igt.h"
+#include "resultgen.h"
+
+static char testdatadir[] = JSON_TESTS_DIRECTORY;
+
+static struct json_object *read_json(int fd)
+{
+	struct json_object *obj;
+	struct json_tokener *tok = json_tokener_new();
+	enum json_tokener_error err;
+	char buf[512];
+	ssize_t s;
+
+	do {
+		s = read(fd, buf, sizeof(buf));
+		obj = json_tokener_parse_ex(tok, buf, s);
+	} while ((err = json_tokener_get_error(tok)) == json_tokener_continue);
+
+	igt_assert_eq(err, json_tokener_success);
+
+	json_tokener_free(tok);
+	return obj;
+}
+
+static void compare(struct json_object *one,
+		    struct json_object *two);
+
+static void compare_objects(struct json_object *one,
+			    struct json_object *two)
+{
+	json_object_iter iter;
+	struct json_object *subobj;
+
+	json_object_object_foreachC(one, iter) {
+		igt_debug("Key %s\n", iter.key);
+
+		igt_assert(json_object_object_get_ex(two, iter.key, &subobj));
+
+		compare(iter.val, subobj);
+	}
+}
+
+static void compare_arrays(struct json_object *one,
+			   struct json_object *two)
+{
+	size_t i;
+
+	for (i = 0; i < json_object_array_length(one); i++) {
+		igt_debug("Array index %zd\n", i);
+		compare(json_object_array_get_idx(one, i),
+			json_object_array_get_idx(two, i));
+	}
+}
+
+static bool compatible_types(struct json_object *one,
+			     struct json_object *two)
+{
+	/*
+	 * A double of value 0.0 gets written as "0", which gets read
+	 * as an int.
+	 */
+	json_type onetype = json_object_get_type(one);
+	json_type twotype = json_object_get_type(two);
+
+	switch (onetype) {
+	case json_type_boolean:
+	case json_type_string:
+	case json_type_object:
+	case json_type_array:
+	case json_type_null:
+		return onetype == twotype;
+		break;
+	case json_type_double:
+	case json_type_int:
+		return twotype == json_type_double || twotype == json_type_int;
+		break;
+	}
+
+	igt_assert(!"Cannot be reached");
+	return false;
+}
+
+static void compare(struct json_object *one,
+		    struct json_object *two)
+{
+	igt_assert(compatible_types(one, two));
+
+	switch (json_object_get_type(one)) {
+	case json_type_boolean:
+		igt_assert_eq(json_object_get_boolean(one), json_object_get_boolean(two));
+		break;
+	case json_type_double:
+	case json_type_int:
+		/*
+		 * A double of value 0.0 gets written as "0", which
+		 * gets read as an int. Both yield 0.0 with
+		 * json_object_get_double(). Comparing doubles with ==
+		 * considered crazy but it's good enough.
+		 */
+		igt_assert(json_object_get_double(one) == json_object_get_double(two));
+		break;
+	case json_type_string:
+		igt_assert(!strcmp(json_object_get_string(one), json_object_get_string(two)));
+		break;
+	case json_type_object:
+		igt_assert_eq(json_object_object_length(one), json_object_object_length(two));
+		compare_objects(one, two);
+		break;
+	case json_type_array:
+		igt_assert_eq(json_object_array_length(one), json_object_array_length(two));
+		compare_arrays(one, two);
+		break;
+	case json_type_null:
+		break;
+	default:
+		igt_assert(!"Cannot be reached");
+	}
+}
+
+static void run_results_and_compare(int dirfd, const char *dirname)
+{
+	int testdirfd = openat(dirfd, dirname, O_RDONLY | O_DIRECTORY);
+	int reference;
+	struct json_object *resultsobj, *referenceobj;
+
+	igt_assert_fd(testdirfd);
+
+	igt_assert((resultsobj = generate_results_json(testdirfd)) != NULL);
+
+	reference = openat(testdirfd, "reference.json", O_RDONLY);
+	close(testdirfd);
+
+	igt_assert_fd(reference);
+	referenceobj = read_json(reference);
+	close(reference);
+	igt_assert(referenceobj != NULL);
+
+	igt_debug("Root object\n");
+	compare(resultsobj, referenceobj);
+	igt_assert_eq(json_object_put(resultsobj), 1);
+	igt_assert_eq(json_object_put(referenceobj), 1);
+}
+
+static const char *dirnames[] = {
+	"normal-run",
+	"warnings",
+	"warnings-with-dmesg-warns",
+	"piglit-style-dmesg",
+	"incomplete-before-any-subtests",
+	"dmesg-results",
+};
+
+igt_main
+{
+	int dirfd = open(testdatadir, O_RDONLY | O_DIRECTORY);
+	size_t i;
+
+	igt_assert_fd(dirfd);
+
+	for (i = 0; i < ARRAY_SIZE(dirnames); i++) {
+		igt_subtest(dirnames[i]) {
+			run_results_and_compare(dirfd, dirnames[i]);
+		}
+	}
+}
-- 
2.18.0



More information about the igt-dev mailing list