[Piglit] [RFC 07/10] framework: add a dEQP runner that can run without process isolation
Nicolai Hähnle
nhaehnle at gmail.com
Wed Oct 11 10:26:56 UTC 2017
From: Nicolai Hähnle <nicolai.haehnle at amd.com>
---
framework/test/deqp.py | 202 ++++++++++++++++++++++++++++++++++---------------
1 file changed, 141 insertions(+), 61 deletions(-)
diff --git a/framework/test/deqp.py b/framework/test/deqp.py
index 8627feabb..f62d731b1 100644
--- a/framework/test/deqp.py
+++ b/framework/test/deqp.py
@@ -15,34 +15,37 @@
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
from __future__ import (
absolute_import, division, print_function, unicode_literals
)
import abc
+import collections
import os
+import re
import subprocess
try:
from lxml import etree as et
except ImportError:
from xml.etree import cElementTree as et
import six
from six.moves import range
from framework import core, grouptools, exceptions
from framework import options
from framework.profile import TestProfile
-from framework.test.base import Test, is_crash_returncode, TestRunError
+from framework.results import TestResult
+from framework.test.base import Test, is_crash_returncode, TestRunError, TestRunner
__all__ = [
'DEQPBaseTest',
'gen_caselist_txt',
'get_option',
'iter_deqp_test_cases',
'make_profile',
]
@@ -75,24 +78,25 @@ def select_source(bin_, filename, mustpass, extra_args):
if options.OPTIONS.deqp_mustpass:
return gen_mustpass_tests(mustpass)
else:
return iter_deqp_test_cases(
gen_caselist_txt(bin_, filename, extra_args))
def make_profile(test_list, test_class):
"""Create a TestProfile instance."""
profile = TestProfile()
+ runner = DEQPTestRunner()
for testname in test_list:
# deqp uses '.' as the testgroup separator.
piglit_name = testname.replace('.', grouptools.SEPARATOR)
- profile.test_list[piglit_name] = test_class(testname)
+ profile.test_list[piglit_name] = test_class(testname, runner)
return profile
def gen_mustpass_tests(mp_list):
"""Return a testlist from the mustpass list."""
root = et.parse(mp_list).getroot()
group = []
def gen(base):
@@ -146,83 +150,159 @@ def iter_deqp_test_cases(case_file):
continue
elif line.startswith('TEST:'):
yield line[len('TEST:'):].strip()
else:
raise exceptions.PiglitFatalError(
'deqp: {}:{}: ill-formed line'.format(case_file, i))
@six.add_metaclass(abc.ABCMeta)
class DEQPBaseTest(Test):
- __RESULT_MAP = {
- "Pass": "pass",
- "Fail": "fail",
- "QualityWarning": "warn",
- "InternalError": "fail",
- "Crash": "crash",
- "NotSupported": "skip",
- "ResourceError": "crash",
- }
-
@abc.abstractproperty
def deqp_bin(self):
"""The path to the exectuable."""
@abc.abstractproperty
def extra_args(self):
"""Extra arguments to be passed to the each test instance.
Needs to return a list, since self.command uses the '+' operator, which
only works to join two lists together.
"""
return _EXTRA_ARGS
- def __init__(self, case_name):
- command = [self.deqp_bin, '--deqp-case=' + case_name]
-
- super(DEQPBaseTest, self).__init__(command)
-
- # dEQP's working directory must be the same as that of the executable,
- # otherwise it cannot find its data files (2014-12-07).
- # This must be called after super or super will overwrite it
- self.cwd = os.path.dirname(self.deqp_bin)
-
- @Test.command.getter
- def command(self):
- """Return the command plus any extra arguments."""
- command = super(DEQPBaseTest, self).command
- return command + self.extra_args
-
- def __find_map(self, result):
- """Run over the lines and set the result."""
- # splitting this into a separate function allows us to return cleanly,
- # otherwise this requires some break/else/continue madness
- for line in result.out.split('\n'):
- line = line.lstrip()
- for k, v in six.iteritems(self.__RESULT_MAP):
- if line.startswith(k):
- result.result = v
- return
-
- def interpret_result(self, result):
- if is_crash_returncode(result.returncode):
- result.result = 'crash'
- elif result.returncode != 0:
- result.result = 'fail'
- else:
- self.__find_map(result)
-
- # We failed to parse the test output. Fallback to 'fail'.
- if result.result == 'notrun':
- result.result = 'fail'
-
- # def _run_command(self, *args, **kwargs):
- # """Rerun the command if X11 connection failure happens."""
- # for _ in range(5):
- # super(DEQPBaseTest, self)._run_command(*args, **kwargs)
- # x_err_msg = "FATAL ERROR: Failed to open display"
- # if x_err_msg in self.result.err or x_err_msg in self.result.out:
- # continue
- # return
-
- # raise TestRunError('Failed to connect to X server 5 times', 'fail')
+ def __init__(self, case_name, runner):
+ # Use only the case name as command. The real command line will be
+ # built by the runner.
+ super(DEQPBaseTest, self).__init__([case_name], runner=runner)
+
+
+class DEQPTestRunner(TestRunner):
+ """Runner for dEQP tests. Supports running multiple tests at a time.
+ """
+ __RESULT_MAP = {
+ "Pass": "pass",
+ "Fail": "fail",
+ "QualityWarning": "warn",
+ "InternalError": "fail",
+ "Crash": "crash",
+ "NotSupported": "skip",
+ "ResourceError": "crash",
+ }
+
+ RE_test_case = re.compile(r'Test case \'(.*)\'..')
+ RE_result = re.compile(r' (' + '|'.join(six.iterkeys(__RESULT_MAP)) + r') \(.*\)')
+
+ @TestRunner.max_tests.getter
+ def max_tests(self):
+ """Limit the number of tests so that the progress indicator is still useful
+ in typical runs and we can also still force concurrency. Plus, this is
+ likely to behave better in the case of system hangs.
+ """
+ return 100
+
+ def _build_case_trie(self, case_names):
+ """Turn a list of case names into the trie format expected by dEQP."""
+ def make_trie():
+ return collections.defaultdict(make_trie)
+ root = make_trie()
+
+ for case_name in case_names:
+ node = root
+ for name in case_name.split('.'):
+ node = node[name]
+
+ def format_trie(trie):
+ if len(trie) == 0:
+ return ''
+ return '{' + ','.join(k + format_trie(v) for k, v in six.iteritems(trie)) + '}'
+
+ return format_trie(root)
+
+ def _run_tests(self, results, tests):
+ prog = None
+ extra_args = None
+ case_names = {}
+
+ for test in tests:
+ assert isinstance(test, DEQPBaseTest)
+
+ test_prog = test.deqp_bin
+ test_extra_args = test.extra_args
+
+ if prog is None:
+ prog = test_prog
+ elif prog != test_prog:
+ raise exceptions.PiglitInternalError(
+ 'dEQP binaries must match for tests to be run in the same command!\n')
+
+ if extra_args is None:
+ extra_args = test_extra_args
+ elif len(extra_args) != len(test_extra_args) or \
+ any(a != b for a, b in zip(extra_args, test_extra_args)):
+ raise exceptions.PiglitInternalError(
+ 'Extra arguments must match for tests to be run in the same command!\n')
+
+ case_names[test.command[0]] = test
+
+ case_name_trie = self._build_case_trie(six.iterkeys(case_names))
+ self._run_command(
+ results,
+ [prog, '--deqp-caselist=' + case_name_trie] + extra_args,
+ # dEQP's working directory must be the same as that of the executable,
+ # otherwise it cannot find its data files (2014-12-07).
+ cwd=os.path.dirname(prog))
+
+ test = None
+ out_all = []
+ out = out_all
+ for each in results.out.split('\n'):
+ out.append(each)
+
+ m = self.RE_test_case.search(each)
+ if m is not None:
+ if test is not None:
+ out.append('PIGLIT: Failed to parse test result of {}'.format(test.name))
+ results.set_result(test.name, 'fail')
+ out_all += out
+
+ test = case_names[m.group(1)]
+ out = []
+ else:
+ m = self.RE_result.search(each)
+ if m is not None:
+ if m.group(1) not in self.__RESULT_MAP:
+ out.append('PIGLIT: Unknown result status: {}'.format(m.group(1)))
+ status = 'fail'
+ else:
+ status = self.__RESULT_MAP[m.group(1)]
+
+ if test is not None:
+ result = TestResult(status)
+ result.root = test.name
+ result.returncode = 0
+ result.out = '\n'.join(out)
+
+ test.interpret_result(result)
+ results.set_result(test.name, result.result)
+ out_all.append(result.out)
+
+ test = None
+ out = out_all
+ else:
+ out.append('PIGLIT: Unexpected result status: {}'.format(m.group(1)))
+
+ if is_crash_returncode(results.returncode):
+ if test is not None:
+ results.set_result(test.name, 'crash')
+ out_all += out
+ test = None
+ else:
+ results.result = 'crash'
+ elif test is not None:
+ out.append('PIGLIT: Failed to parse test result of {}'.format(test.name))
+ results.set_result(test.name, 'fail')
+ out_all += out
+ test = None
+
+ results.out = '\n'.join(out_all)
--
2.11.0
More information about the Piglit
mailing list