[Piglit] [PATCH 1/4] piglt: Add support for subtests

[Tom Stellard]

From: Tom Stellard <thomas.stellard at amd.com>

Subtests allow a test to produce more than one result.  This is useful
for tests where it may be convenient to combine several somewhat
unrelated tests into a single program.  For example, a cl test like:

kernel void test(TYPE *out, TYPE *in) {
	out[0] = some_builtin_function(in[0], in[1]);
}

That uses only one kernel, but recompiles it several times with different
definitions of the macro 'TYPE' in order to test the builtin with all
the possible types.

To take advantage of the subtest feature, programs should output an
additional PIGLIT line similar to the following example:

PIGLIT: {'subtests' : {'testA' : 'pass', 'testB : 'fail'}}

Where testA and testB are the name of the subtests.

In the result summary, this will be displayed as:

TestName                            1/2
      testA                         pass
      testB                         fail
---
 framework/core.py     |  7 +++++-
 framework/exectest.py |  3 ++-
 framework/summary.py  | 68 ++++++++++++++++++++++++++++++++++++++++++++-------
 3 files changed, 67 insertions(+), 11 deletions(-)

diff --git a/framework/core.py b/framework/core.py
index 5a97dba..5e11323 100644
--- a/framework/core.py
+++ b/framework/core.py
@@ -460,7 +460,12 @@ class Test:
 
 			status(result['result'])
 
-			json_writer.write_dict_item(path, result)
+			if 'subtests' in result and len(result['subtests'].keys()) > 1:
+				for test in result['subtests'].keys():
+					result['result'] = result['subtests'][test]
+					json_writer.write_dict_item(path + '/' + test, result)
+			else:
+				json_writer.write_dict_item(path, result)
 		else:
 			status("dry-run")
 
diff --git a/framework/exectest.py b/framework/exectest.py
index aa89c66..3004ca8 100644
--- a/framework/exectest.py
+++ b/framework/exectest.py
@@ -213,7 +213,8 @@ class PlainExecTest(ExecTest):
 
 		if len(outpiglit) > 0:
 			try:
-				results.update(eval(''.join(outpiglit), {}))
+				for piglit in outpiglit:
+					results.update(eval(''.join(piglit), {}))
 				out = '\n'.join(filter(lambda s: not s.startswith('PIGLIT:'), outlines))
 			except:
 				results['result'] = 'fail'
diff --git a/framework/summary.py b/framework/summary.py
index a193351..802ecd5 100644
--- a/framework/summary.py
+++ b/framework/summary.py
@@ -106,6 +106,11 @@ results is an array of TestResult instances, one per testrun
 ##### GroupSummary: Summarize a group of tests
 #############################################################################
 class GroupSummary:
+	def createDummyGroup(self, result, test_name):
+		new_group = core.GroupResult()
+		new_group[' ' + test_name + '(All Tests)'] = result[test_name]
+		result[test_name] = new_group
+
 	def __init__(self, summary, path, name, results):
 		"""\
 summary is the root summary object
@@ -132,6 +137,7 @@ results is an array of GroupResult instances, one per testrun
 
 		# Collect, create and annotate children
 		for result in self.results:
+			#result is a GroupResult instance
 			for name in result:
 				if name in self.children:
 					continue
@@ -141,6 +147,25 @@ results is an array of GroupResult instances, one per testrun
 					childpath = self.path + '/' + childpath
 
 				if isinstance(result[name], core.GroupResult):
+					# This loop checks to make sure that all results
+					# with the same 'name' are of the same type.
+					# This is necessary to handle the case where
+					# a testname in an earlier result (an earlier
+					# result in this case means a result that
+					# comes before the current result in self.results)
+					# denotes a test group but in a later
+					# result it denotes a single test case, for example:
+					#
+					# result 0:
+					#	test/group/a PASS
+					#	test/group/b PASS
+					#	test/group/c PASS
+					# result 1:
+					#	test/group PASS
+					for r in self.results:
+						if r.has_key(name) and not isinstance(r[name], core.GroupResult):
+							self.createDummyGroup(r, name)
+
 					childresults = [r.get(name, core.GroupResult())
 							for r in self.results]
 
@@ -151,15 +176,40 @@ results is an array of GroupResult instances, one per testrun
 						childresults
 					)
 				else:
-					childresults = [r.get(name, core.TestResult({ 'result': 'skip' }))
-							for r in self.results]
-
-					self.children[name] = TestSummary(
-						summary,
-						childpath,
-						name,
-						childresults
-					)
+					# We need to check and handle the reversed case
+					# described in the above comment e.g.:
+					# result 0:
+					#	test/group PASS
+					# result 1:
+					#	test/group/a PASS
+					#	test/group/b PASS
+					#	test/group/c PASS
+					need_group = 0
+					for r in self.results:
+						if r.has_key(name) and not isinstance(r[name], core.TestResult):
+							need_group = 1
+					if need_group:
+						for r in self.results:
+							if r.has_key(name) and isinstance(r[name], core.TestResult):
+								self.createDummyGroup(r, name)
+						childresults = [r.get(name, core.GroupResult())
+								for r in self.results]
+
+						self.children[name] = GroupSummary(
+							summary,
+							childpath,
+							name,
+							childresults
+						)
+					else:
+						childresults = [r.get(name, core.TestResult({ 'result': 'skip' }))
+								for r in self.results]
+						self.children[name] = TestSummary(
+							summary,
+							childpath,
+							name,
+							childresults
+						)
 
 				for j in range(len(self.results)):
 					self.results[j].passvector.add(childresults[j].passvector)
-- 
1.7.11.4