[Piglit] [PATCH 2/4] piglt: Add support for subtests v2
Tom Stellard
tom at stellard.net
Thu Dec 6 14:14:48 PST 2012
From: Tom Stellard <thomas.stellard at amd.com>
Subtests allow a test to produce more than one result. This is useful
for tests where it may be convenient to combine several somewhat
unrelated tests into a single program. For example, a cl test like:
kernel void test(TYPE *out, TYPE *in) {
out[0] = some_builtin_function(in[0], in[1]);
}
That uses only one kernel, but recompiles it several times with different
definitions of the macro 'TYPE' in order to test the builtin with all
the possible types.
To take advantage of the subtest feature, programs should output one
PIGLIT line per subtest as in the following example:
PIGLIT:subtest {'testA' : 'pass'}
PIGLIT:subtest {'testB' : 'fail'}
Where testA and testB are the name of the subtests.
In the result summary, this will be displayed as:
TestName 1/2
testA pass
testB fail
v2:
- Print one line for each subtest rather than printing them all
together.
---
framework/core.py | 7 ++++-
framework/exectest.py | 8 +++++-
framework/summary.py | 67 +++++++++++++++++++++++++++++++++++++++++-------
tests/util/piglit-util.c | 8 ++++++
tests/util/piglit-util.h | 1 +
5 files changed, 80 insertions(+), 11 deletions(-)
diff --git a/framework/core.py b/framework/core.py
index 5a97dba..344d7b9 100644
--- a/framework/core.py
+++ b/framework/core.py
@@ -460,7 +460,12 @@ class Test:
status(result['result'])
- json_writer.write_dict_item(path, result)
+ if 'subtest' in result and len(result['subtest'].keys()) > 1:
+ for test in result['subtest'].keys():
+ result['result'] = result['subtest'][test]
+ json_writer.write_dict_item(path + '/' + test, result)
+ else:
+ json_writer.write_dict_item(path, result)
else:
status("dry-run")
diff --git a/framework/exectest.py b/framework/exectest.py
index aa89c66..e96b5ba 100644
--- a/framework/exectest.py
+++ b/framework/exectest.py
@@ -213,7 +213,13 @@ class PlainExecTest(ExecTest):
if len(outpiglit) > 0:
try:
- results.update(eval(''.join(outpiglit), {}))
+ for piglit in outpiglit:
+ if piglit.startswith('subtest'):
+ if not results.has_key('subtest'):
+ results['subtest'] = {}
+ results['subtest'].update(eval(piglit[7:]))
+ else:
+ results.update(eval(piglit))
out = '\n'.join(filter(lambda s: not s.startswith('PIGLIT:'), outlines))
except:
results['result'] = 'fail'
diff --git a/framework/summary.py b/framework/summary.py
index a193351..91e0870 100644
--- a/framework/summary.py
+++ b/framework/summary.py
@@ -106,6 +106,11 @@ results is an array of TestResult instances, one per testrun
##### GroupSummary: Summarize a group of tests
#############################################################################
class GroupSummary:
+ def createDummyGroup(self, result, test_name):
+ new_group = core.GroupResult()
+ new_group[' ' + test_name + '(All Tests)'] = result[test_name]
+ result[test_name] = new_group
+
def __init__(self, summary, path, name, results):
"""\
summary is the root summary object
@@ -141,6 +146,25 @@ results is an array of GroupResult instances, one per testrun
childpath = self.path + '/' + childpath
if isinstance(result[name], core.GroupResult):
+ # This loop checks to make sure that all results
+ # with the same 'name' are of the same type.
+ # This is necessary to handle the case where
+ # a testname in an earlier result (an earlier
+ # result in this case means a result that
+ # comes before the current result in self.results)
+ # denotes a test group but in a later
+ # result it denotes a single test case, for example:
+ #
+ # result 0:
+ # test/group/a PASS
+ # test/group/b PASS
+ # test/group/c PASS
+ # result 1:
+ # test/group PASS
+ for r in self.results:
+ if r.has_key(name) and not isinstance(r[name], core.GroupResult):
+ self.createDummyGroup(r, name)
+
childresults = [r.get(name, core.GroupResult())
for r in self.results]
@@ -151,15 +175,40 @@ results is an array of GroupResult instances, one per testrun
childresults
)
else:
- childresults = [r.get(name, core.TestResult({ 'result': 'skip' }))
- for r in self.results]
-
- self.children[name] = TestSummary(
- summary,
- childpath,
- name,
- childresults
- )
+ # We need to check and handle the reversed case
+ # described in the above comment e.g.:
+ # result 0:
+ # test/group PASS
+ # result 1:
+ # test/group/a PASS
+ # test/group/b PASS
+ # test/group/c PASS
+ need_group = 0
+ for r in self.results:
+ if r.has_key(name) and not isinstance(r[name], core.TestResult):
+ need_group = 1
+ if need_group:
+ for r in self.results:
+ if r.has_key(name) and isinstance(r[name], core.TestResult):
+ self.createDummyGroup(r, name)
+ childresults = [r.get(name, core.GroupResult())
+ for r in self.results]
+
+ self.children[name] = GroupSummary(
+ summary,
+ childpath,
+ name,
+ childresults
+ )
+ else:
+ childresults = [r.get(name, core.TestResult({ 'result': 'skip' }))
+ for r in self.results]
+ self.children[name] = TestSummary(
+ summary,
+ childpath,
+ name,
+ childresults
+ )
for j in range(len(self.results)):
self.results[j].passvector.add(childresults[j].passvector)
diff --git a/tests/util/piglit-util.c b/tests/util/piglit-util.c
index 98e0800..2c11062 100644
--- a/tests/util/piglit-util.c
+++ b/tests/util/piglit-util.c
@@ -195,6 +195,14 @@ piglit_report_result(enum piglit_result result)
}
}
+void
+piglit_report_subtest_result(const char *name, enum piglit_result result)
+{
+ const char *result_str = piglit_result_to_string(result);
+ printf("PIGLIT:subtest {'%s' : '%s'}\n", name, result_str);
+ fflush(stdout);
+}
+
#ifndef HAVE_STRCHRNUL
char *strchrnul(const char *s, int c)
{
diff --git a/tests/util/piglit-util.h b/tests/util/piglit-util.h
index 2bbc767..600bf92 100644
--- a/tests/util/piglit-util.h
+++ b/tests/util/piglit-util.h
@@ -135,6 +135,7 @@ int piglit_find_line(const char *program, int position);
void piglit_merge_result(enum piglit_result *all, enum piglit_result subtest);
const char * piglit_result_to_string(enum piglit_result result);
void piglit_report_result(enum piglit_result result);
+void piglit_report_subtest_result(const char *name, enum piglit_result result);
#ifndef HAVE_STRCHRNUL
char *strchrnul(const char *s, int c);
--
1.7.11.4
More information about the Piglit
mailing list