[Piglit] [PATCH 1/4] piglt: Add support for subtests
Aaron Watry
awatry at gmail.com
Wed Dec 5 18:02:40 PST 2012
My python is a bit rusty, but it looks sane to me.
The series is:
Tested By: Aaron Watry <awatry at gmail.com>
The C and .cl changes all get a
Reviewed By: Aaron Watry <awatry at gmail.com>
--Aaron
On Wed, Dec 5, 2012 at 12:00 PM, Tom Stellard <tom at stellard.net> wrote:
> From: Tom Stellard <thomas.stellard at amd.com>
>
> Subtests allow a test to produce more than one result. This is useful
> for tests where it may be convenient to combine several somewhat
> unrelated tests into a single program. For example, a cl test like:
>
> kernel void test(TYPE *out, TYPE *in) {
> out[0] = some_builtin_function(in[0], in[1]);
> }
>
> That uses only one kernel, but recompiles it several times with different
> definitions of the macro 'TYPE' in order to test the builtin with all
> the possible types.
>
> To take advantage of the subtest feature, programs should output an
> additional PIGLIT line similar to the following example:
>
> PIGLIT: {'subtests' : {'testA' : 'pass', 'testB : 'fail'}}
>
> Where testA and testB are the name of the subtests.
>
> In the result summary, this will be displayed as:
>
> TestName 1/2
> testA pass
> testB fail
> ---
> framework/core.py | 7 +++++-
> framework/exectest.py | 3 ++-
> framework/summary.py | 68 ++++++++++++++++++++++++++++++++++++++++++++-------
> 3 files changed, 67 insertions(+), 11 deletions(-)
>
> diff --git a/framework/core.py b/framework/core.py
> index 5a97dba..5e11323 100644
> --- a/framework/core.py
> +++ b/framework/core.py
> @@ -460,7 +460,12 @@ class Test:
>
> status(result['result'])
>
> - json_writer.write_dict_item(path, result)
> + if 'subtests' in result and len(result['subtests'].keys()) > 1:
> + for test in result['subtests'].keys():
> + result['result'] = result['subtests'][test]
> + json_writer.write_dict_item(path + '/' + test, result)
> + else:
> + json_writer.write_dict_item(path, result)
> else:
> status("dry-run")
>
> diff --git a/framework/exectest.py b/framework/exectest.py
> index aa89c66..3004ca8 100644
> --- a/framework/exectest.py
> +++ b/framework/exectest.py
> @@ -213,7 +213,8 @@ class PlainExecTest(ExecTest):
>
> if len(outpiglit) > 0:
> try:
> - results.update(eval(''.join(outpiglit), {}))
> + for piglit in outpiglit:
> + results.update(eval(''.join(piglit), {}))
> out = '\n'.join(filter(lambda s: not s.startswith('PIGLIT:'), outlines))
> except:
> results['result'] = 'fail'
> diff --git a/framework/summary.py b/framework/summary.py
> index a193351..802ecd5 100644
> --- a/framework/summary.py
> +++ b/framework/summary.py
> @@ -106,6 +106,11 @@ results is an array of TestResult instances, one per testrun
> ##### GroupSummary: Summarize a group of tests
> #############################################################################
> class GroupSummary:
> + def createDummyGroup(self, result, test_name):
> + new_group = core.GroupResult()
> + new_group[' ' + test_name + '(All Tests)'] = result[test_name]
> + result[test_name] = new_group
> +
> def __init__(self, summary, path, name, results):
> """\
> summary is the root summary object
> @@ -132,6 +137,7 @@ results is an array of GroupResult instances, one per testrun
>
> # Collect, create and annotate children
> for result in self.results:
> + #result is a GroupResult instance
> for name in result:
> if name in self.children:
> continue
> @@ -141,6 +147,25 @@ results is an array of GroupResult instances, one per testrun
> childpath = self.path + '/' + childpath
>
> if isinstance(result[name], core.GroupResult):
> + # This loop checks to make sure that all results
> + # with the same 'name' are of the same type.
> + # This is necessary to handle the case where
> + # a testname in an earlier result (an earlier
> + # result in this case means a result that
> + # comes before the current result in self.results)
> + # denotes a test group but in a later
> + # result it denotes a single test case, for example:
> + #
> + # result 0:
> + # test/group/a PASS
> + # test/group/b PASS
> + # test/group/c PASS
> + # result 1:
> + # test/group PASS
> + for r in self.results:
> + if r.has_key(name) and not isinstance(r[name], core.GroupResult):
> + self.createDummyGroup(r, name)
> +
> childresults = [r.get(name, core.GroupResult())
> for r in self.results]
>
> @@ -151,15 +176,40 @@ results is an array of GroupResult instances, one per testrun
> childresults
> )
> else:
> - childresults = [r.get(name, core.TestResult({ 'result': 'skip' }))
> - for r in self.results]
> -
> - self.children[name] = TestSummary(
> - summary,
> - childpath,
> - name,
> - childresults
> - )
> + # We need to check and handle the reversed case
> + # described in the above comment e.g.:
> + # result 0:
> + # test/group PASS
> + # result 1:
> + # test/group/a PASS
> + # test/group/b PASS
> + # test/group/c PASS
> + need_group = 0
> + for r in self.results:
> + if r.has_key(name) and not isinstance(r[name], core.TestResult):
> + need_group = 1
> + if need_group:
> + for r in self.results:
> + if r.has_key(name) and isinstance(r[name], core.TestResult):
> + self.createDummyGroup(r, name)
> + childresults = [r.get(name, core.GroupResult())
> + for r in self.results]
> +
> + self.children[name] = GroupSummary(
> + summary,
> + childpath,
> + name,
> + childresults
> + )
> + else:
> + childresults = [r.get(name, core.TestResult({ 'result': 'skip' }))
> + for r in self.results]
> + self.children[name] = TestSummary(
> + summary,
> + childpath,
> + name,
> + childresults
> + )
>
> for j in range(len(self.results)):
> self.results[j].passvector.add(childresults[j].passvector)
> --
> 1.7.11.4
>
More information about the Piglit
mailing list