[Piglit] [PATCH 5/11] framework: Add an initial implementaiton of a DEQPGroupTest
Dylan Baker
dylan at pnwbakers.com
Fri Aug 26 18:31:12 UTC 2016
This patch adds a new DEQP base class that uses the ReducedProcessMixin
to provide a "group at a time" mode, in which a group leaf of the DEQP
test hierarchy is run in a single process. It doesn't remove the
existing "test at a time" mode, where a single test is runin a single
process. It also provides two classes built on top of that class meant
to be used by concrete implementations. One that uses asterisks and one
that uses trie lists. Each have their advantages. The asterisks are
simpler and much shorter, the trie lists can represent groups that have
both tests and groups, but can grow larger than dEQP can parse.
The implementation details are such that some shared bits are split out
into a DEQPBaseTest, and a DEQPSingleTest and DEQPGroupTest implement
the two modes. This renames the old DEQPBaseTest to DEQPSingleTest,
which requires touching all of the deqp suites to change their parent
class.
Signed-off-by: Dylan Baker <dylanx.c.baker at intel.com>
---
framework/test/deqp.py | 238 +++++++++++++++++++++++----
tests/cts_gl.py | 2 +-
tests/cts_gles.py | 2 +-
tests/deqp_egl.py | 2 +-
tests/deqp_gles2.py | 2 +-
tests/deqp_gles3.py | 2 +-
tests/deqp_gles31.py | 2 +-
tests/deqp_vk.py | 2 +-
tox.ini | 1 +-
unittests/framework/test/test_deqp.py | 203 ++++++++++++++++++++---
10 files changed, 396 insertions(+), 60 deletions(-)
diff --git a/framework/test/deqp.py b/framework/test/deqp.py
index b85cf08..25dd077 100644
--- a/framework/test/deqp.py
+++ b/framework/test/deqp.py
@@ -36,8 +36,9 @@ from six.moves import range
from framework import core, grouptools, exceptions
from framework import options
+from framework import status
+from framework.test import base
from framework.profile import TestProfile
-from framework.test.base import Test, is_crash_returncode, TestRunError
__all__ = [
'DEQPBaseTest',
@@ -72,6 +73,14 @@ _EXTRA_ARGS = get_option('PIGLIT_DEQP_EXTRA_ARGS',
default='').split()
+class DEQPUnsupportedMode(exceptions.PiglitInternalError):
+ """An exception raised when dEQP/piglit doesn't support the given mode.
+
+ Primarily this is meant to be used in the deqp_gles* modules, where
+ group-at-a-time can actually make them slower.
+ """
+
+
def select_source(bin_, filename, mustpass, extra_args):
"""Return either the mustpass list or the generated list."""
if options.OPTIONS.deqp_mustpass:
@@ -97,8 +106,8 @@ def gen_mustpass_tests(mp_list):
root = et.parse(mp_list).getroot()
group = []
- def gen(base):
- for elem in base:
+ def gen(root):
+ for elem in root:
if elem.tag == 'Test':
yield '{}.{}'.format('.'.join(group), elem.get('name'))
else:
@@ -152,25 +161,54 @@ def iter_deqp_test_cases(case_file):
'deqp: {}:{}: ill-formed line'.format(case_file, i))
- at six.add_metaclass(abc.ABCMeta)
-class DEQPBaseTest(Test):
- """Base test class for dEQP suites.
+def format_trie_list(classname, testnames):
+ """Create a trie list from classname and testnames.
+
+ dEQP doesn't accept multiple --deqp-case/-n flags, so we need this. It's
+ called Trie, and it's ugly as heck. Trie is like
+ {group1{group2{test1,test2}}} (as a simple example. Every other method
+ involves creating files, so it's really the only viable method.
+
+ This string looks like gobly-gook, but basicaly in python formatted
+ strings a '{{' becomes a '{' (since {} has special meaning).
+ """
+ return '--deqp-caselist={{{0}{{{1}}}{2}'.format(
+ '{'.join(classname), ','.join(testnames), '}' * len(classname))
+
+
+class _DEQPCase(object):
+ """Mixin to provide the --deqp-case argument."""
+
+ def __init__(self, case_name, *args, **kwargs):
+ super(_DEQPCase, self).__init__('--deqp-case=' + case_name, *args, **kwargs)
- Each particular dEQP implementation will need to override the two abstract
- properties (the easiest way to do so is as a class attribute), otherwise
- not other changes are required.
+
+ at six.add_metaclass(abc.ABCMeta)
+class DEQPBaseTest(base.Test):
+ """A shared base class that provides some shared methods and attributes for
+ use by both the Single and Group classes.
"""
- __RESULT_MAP = {
- "Pass": "pass",
- "Fail": "fail",
- "QualityWarning": "warn",
- "InternalError": "fail",
- "Crash": "crash",
- "NotSupported": "skip",
- "ResourceError": "crash",
+ _RESULT_MAP = {
+ "Pass": status.PASS,
+ "Fail": status.FAIL,
+ "QualityWarning": status.WARN,
+ "InternalError": status.FAIL,
+ "Crash": status.CRASH,
+ "NotSupported": status.SKIP,
+ "ResourceError": status.CRASH,
}
+ def __init__(self, command):
+ command = [self.deqp_bin, command]
+
+ super(DEQPBaseTest, self).__init__(command)
+
+ # dEQP's working directory must be the same as that of the executable,
+ # otherwise it cannot find its data files (2014-12-07).
+ # This must be called after super or super will overwrite it
+ self.cwd = os.path.dirname(self.deqp_bin)
+
@abc.abstractproperty
def deqp_bin(self):
"""The path to the exectuable."""
@@ -185,37 +223,33 @@ class DEQPBaseTest(Test):
"""
return _EXTRA_ARGS
- def __init__(self, case_name):
- command = [self.deqp_bin, '--deqp-case=' + case_name]
-
- super(DEQPBaseTest, self).__init__(command)
-
- # dEQP's working directory must be the same as that of the executable,
- # otherwise it cannot find its data files (2014-12-07).
- # This must be called after super or super will overwrite it
- self.cwd = os.path.dirname(self.deqp_bin)
-
# The error of the getter is a known bug in pylint
# https://github.com/PyCQA/pylint/issues/844
- @Test.command.getter # pylint: disable=no-member
+ @base.Test.command.getter # pylint: disable=no-member
def command(self):
"""Return the command plus any extra arguments."""
command = super(DEQPBaseTest, self).command
return command + self.extra_args
+
+# Pylint can't figure out the six magic.
+ at six.add_metaclass(abc.ABCMeta) # pylint: disable=abstract-method
+class DEQPSingleTest(_DEQPCase, DEQPBaseTest):
+ """Base test class for dEQP tests to run a single test per process."""
+
def __find_map(self):
"""Run over the lines and set the result."""
# splitting this into a separate function allows us to return cleanly,
# otherwise this requires some break/else/continue madness
for line in self.result.out.split('\n'):
line = line.lstrip()
- for k, v in six.iteritems(self.__RESULT_MAP):
+ for k, v in six.iteritems(self._RESULT_MAP):
if line.startswith(k):
self.result.result = v
return
def interpret_result(self):
- if is_crash_returncode(self.result.returncode):
+ if base.is_crash_returncode(self.result.returncode):
self.result.result = 'crash'
elif self.result.returncode != 0:
self.result.result = 'fail'
@@ -229,8 +263,148 @@ class DEQPBaseTest(Test):
def _run_command(self, *args, **kwargs):
"""Rerun the command if X11 connection failure happens."""
for _ in range(5):
- super(DEQPBaseTest, self)._run_command(*args, **kwargs)
+ super(DEQPSingleTest, self)._run_command(*args, **kwargs)
if "FATAL ERROR: Failed to open display" not in self.result.err:
return
- raise TestRunError('Failed to connect to X server 5 times', 'fail')
+ raise base.TestRunError('Failed to connect to X server 5 times', 'fail')
+
+
+# Pylint can't figure out the six magic.
+ at six.add_metaclass(abc.ABCMeta) # pylint: disable=abstract-method
+class DEQPGroupTest(base.ReducedProcessMixin, DEQPBaseTest):
+ """A class for running DEQP in a group at a time mode.
+
+ With this class (in contrast to DEQPBaseTest), multiple tests will be run
+ in a single process ("group-at-a-time"), this reduces reliablity somewhat,
+ but vastly reduces runtime, which may be an acceptable tradoff.
+
+ Each test is stored as a subtest, and the stdout and stderr is combined.
+
+ This class is aware of which tests should be run by the process, and if the
+ process ends in a non 0 status it will attempt to resume the tests.
+ """
+
+ def __init__(self, command, case_name, subtests):
+ super(DEQPGroupTest, self).__init__(command, subtests=subtests)
+
+ self.__casename = case_name
+
+ def _populate_subtests(self):
+ # Only add the subtest name, not the group name
+ self.result.subtests.update(
+ {self._subtest_name(x): status.NOTRUN for x in self._expected})
+
+ def interpret_result(self):
+ # We failed to parse the test output. Fallback to 'fail'.
+ if base.is_crash_returncode(self.result.returncode):
+ self.result.result = status.CRASH
+ elif self.result.returncode != 0:
+ self.result.result = status.FAIL
+
+ current = None
+
+ for line in self.result.out.split('\n'):
+ if self._is_subtest(line):
+ # There is one case where it is valid for current to be not
+ # None, and that's when a case crashed, and the resume happens
+ # the case will have no status, but it should have been marked
+ # crash in that case, so if it's still NOTRUN then something is
+ # wrong
+ assert (current is None or
+ self.result.subtests[current] is not status.NOTRUN), \
+ 'Two test cases opened?'
+
+ # Get only the test name, not the group name
+ current = self._subtest_name(line[:-3])
+ assert current in self.result.subtests
+
+ elif line.startswith(' '):
+ try:
+ res = self._RESULT_MAP[line.lstrip().split(' ', 1)[0]]
+ except KeyError:
+ # In general there are only two cases that that will match
+ # the elif and those are during the first and last lines
+ # of the output. In some dEQP based suites that number is
+ # higher (the GLES CTS for example). However the number is
+ # small enough that simply continue-ing on the exception
+ # is more efficient to try/except than to write an if
+ # statement that covers all of the potential cases.
+ continue
+
+ assert current is not None, 'Result without case?'
+
+ self.result.subtests[current] = res
+
+ # That case is closed, reset current
+ current = None
+
+ def __calculate_resume(self, current):
+ self.result.out += 'Splitting command to avoid too-long argument\n\n'
+ val = ((current // 1000) + 1) * 1000
+ if current % 1000 == 999:
+ val += 1000
+ return val
+
+ def _resume(self, current):
+ # The command needs to be the deqp binary, eachsubtest, and then the
+ # extra args
+ command = [self.command[0]]
+
+ command.append(format_trie_list(
+ self.__casename.split('.'),
+ (self._get_test(s) for s in
+ self._expected[current:self.__calculate_resume(current)])))
+ assert len(command[-1]) < 65000, len(command[-1])
+ command.extend(self.extra_args)
+ return command
+
+ def _is_subtest(self, line):
+ return line.startswith("Test case '") and line.endswith("'..")
+
+ @staticmethod
+ def _get_test(test):
+ return test.rsplit('.', 1)[1]
+
+ def _subtest_name(self, test):
+ return self._get_test(test).lower()
+
+
+# Pylint can't figure out the six magic.
+class DEQPGroupAsteriskTest(_DEQPCase, DEQPGroupTest): # pylint: disable=abstract-method
+ """A class that uses .* to construct case names.
+
+ This class uses an asterisk append to the end of the group name
+ ("my.group.*") to run multiple cases at a time. This has the advantage of
+ being very succinct, and for the Vulkan CTS avoids the problem of building
+ a "trie" command so long that the binary cannot parse it. But for other
+ suites like the OpenGL CTS this isn't workable because it has groups that
+ contain both tests and groups other groups.
+ """
+
+ def __init__(self, case_name, subtests):
+ # This is not exactly correct since if someone were to turn on the
+ # group at a time mode and try to use the mustpass list it would fail,
+ # but piglit doesn't support that configuration because it's slower than
+ # test at a time, and removes the protections of process isolation.
+ super(DEQPGroupAsteriskTest, self).__init__(
+ case_name + '.*', case_name, subtests=subtests)
+
+
+# Pylint can't figure out the six magic.
+class DEQPGroupTrieTest(DEQPGroupTest): # pylint: disable=abstract-method
+ """A class that uses trie lists to construct the testcase arguments.
+
+ This class uses trie lists to construct the command for the test. This has
+ the advantage of working correctly for groups that contain both tests and
+ groups, but has the disadvantage of being more verbose, and lists can be
+ constructed that are so large the dEQP binary can't parse them.
+ """
+
+ def __init__(self, case_name, subtests):
+ super(DEQPGroupTrieTest, self).__init__(
+ format_trie_list(
+ case_name.split('.'),
+ (self._get_test(s) for s in subtests)),
+ case_name,
+ subtests=subtests)
diff --git a/tests/cts_gl.py b/tests/cts_gl.py
index d650ffd..5b03e4a 100644
--- a/tests/cts_gl.py
+++ b/tests/cts_gl.py
@@ -57,7 +57,7 @@ _EXTRA_ARGS = deqp.get_option('PIGLIT_CTS_GL_EXTRA_ARGS', ('cts_gl', 'extra_args
default='').split()
-class DEQPCTSTest(deqp.DEQPBaseTest):
+class DEQPCTSTest(deqp.DEQPSingleTest):
deqp_bin = _CTS_BIN
@property
diff --git a/tests/cts_gles.py b/tests/cts_gles.py
index 64de79d..a6a7ec8 100644
--- a/tests/cts_gles.py
+++ b/tests/cts_gles.py
@@ -58,7 +58,7 @@ _EXTRA_ARGS = deqp.get_option('PIGLIT_CTS_GLES_EXTRA_ARGS', ('cts_gles', 'extra_
default='').split()
-class DEQPCTSTest(deqp.DEQPBaseTest):
+class DEQPCTSTest(deqp.DEQPSingleTest):
deqp_bin = _CTS_BIN
@property
diff --git a/tests/deqp_egl.py b/tests/deqp_egl.py
index 7422c82..2b8dfac 100644
--- a/tests/deqp_egl.py
+++ b/tests/deqp_egl.py
@@ -38,7 +38,7 @@ _EXTRA_ARGS = deqp.get_option('PIGLIT_DEQP_EGL_EXTRA_ARGS',
default='').split()
-class DEQPEGLTest(deqp.DEQPBaseTest):
+class DEQPEGLTest(deqp.DEQPSingleTest):
deqp_bin = _EGL_BIN
@property
diff --git a/tests/deqp_gles2.py b/tests/deqp_gles2.py
index 518a4e0..1544740 100644
--- a/tests/deqp_gles2.py
+++ b/tests/deqp_gles2.py
@@ -43,7 +43,7 @@ _EXTRA_ARGS = deqp.get_option('PIGLIT_DEQP_GLES2_EXTRA_ARGS',
default='').split()
-class DEQPGLES2Test(deqp.DEQPBaseTest):
+class DEQPGLES2Test(deqp.DEQPSingleTest):
deqp_bin = _DEQP_GLES2_BIN
@property
diff --git a/tests/deqp_gles3.py b/tests/deqp_gles3.py
index 210c6e4..4151319 100644
--- a/tests/deqp_gles3.py
+++ b/tests/deqp_gles3.py
@@ -79,7 +79,7 @@ _EXTRA_ARGS = deqp.get_option('PIGLIT_DEQP_GLES3_EXTRA_ARGS',
default='').split()
-class DEQPGLES3Test(deqp.DEQPBaseTest):
+class DEQPGLES3Test(deqp.DEQPSingleTest):
deqp_bin = _DEQP_GLES3_BIN
@property
diff --git a/tests/deqp_gles31.py b/tests/deqp_gles31.py
index 7021e89..4011f55 100644
--- a/tests/deqp_gles31.py
+++ b/tests/deqp_gles31.py
@@ -43,7 +43,7 @@ _EXTRA_ARGS = deqp.get_option('PIGLIT_DEQP_GLES31_EXTRA_ARGS',
default='').split()
-class DEQPGLES31Test(deqp.DEQPBaseTest):
+class DEQPGLES31Test(deqp.DEQPSingleTest):
deqp_bin = _DEQP_GLES31_BIN
@property
diff --git a/tests/deqp_vk.py b/tests/deqp_vk.py
index 4df9e3b..60182c8 100644
--- a/tests/deqp_vk.py
+++ b/tests/deqp_vk.py
@@ -46,7 +46,7 @@ _DEQP_ASSERT = re.compile(
r'deqp-vk: external/vulkancts/.*: Assertion `.*\' failed.')
-class DEQPVKTest(deqp.DEQPBaseTest):
+class DEQPVKTest(deqp.DEQPSingleTest):
"""Test representation for Khronos Vulkan CTS."""
timeout = 60
deqp_bin = _DEQP_VK_BIN
diff --git a/tox.ini b/tox.ini
index 12cdddb..f77d6b0 100644
--- a/tox.ini
+++ b/tox.ini
@@ -9,6 +9,7 @@ python_paths = framework/ generated_tests/
passenv=
HOME
USERPROFILE
+ DISPLAY
setenv =
USERNAME = foo
deps =
diff --git a/unittests/framework/test/test_deqp.py b/unittests/framework/test/test_deqp.py
index 1eb84b1..58504fd 100644
--- a/unittests/framework/test/test_deqp.py
+++ b/unittests/framework/test/test_deqp.py
@@ -46,7 +46,17 @@ from framework.test import deqp
# pylint:disable=invalid-name,no-self-use
-class _DEQPTestTest(deqp.DEQPBaseTest):
+class _DEQPTestTest(deqp.DEQPSingleTest):
+ deqp_bin = 'deqp.bin'
+ extra_args = ['extra']
+
+
+class _DEQPGroupAsteriskTest(deqp.DEQPGroupAsteriskTest):
+ deqp_bin = 'deqp.bin'
+ extra_args = ['extra']
+
+
+class _DEQPGroupTrieTest(deqp.DEQPGroupTrieTest):
deqp_bin = 'deqp.bin'
extra_args = ['extra']
@@ -146,15 +156,27 @@ class TestIterDeqpTestCases(object):
self._do_test('this will fail', None, tmpdir)
-class TestDEQPBaseTest(object):
- """Test the DEQPBaseTest class."""
+class TestFormatTrieList(object):
+ """Tests for the format_trie_list function."""
+
+ def test_basic(self):
+ classname = ['foo', 'bar']
+ testnames = ['a', 'b']
+ expected = '--deqp-caselist={foo{bar{a,b}}}'
+ actual = deqp.format_trie_list(classname, testnames)
+ assert actual == expected
+
+
+class TestDEQPSingleTest(object):
+ """Test the DEQPSingleTest class."""
@classmethod
def setup_class(cls):
cls.test = _DEQPTestTest('a.deqp.test')
- def test_command_adds_extra_args(self):
- assert self.test.command[-1] == 'extra'
+ def test_command(self):
+ assert self.test.command == \
+ ['deqp.bin', '--deqp-case=a.deqp.test', 'extra']
class TestInterpretResultReturncodes(object):
"""Test the interpret_result method's returncode handling."""
@@ -164,7 +186,7 @@ class TestDEQPBaseTest(object):
cls.test = _DEQPTestTest('a.deqp.test')
def test_crash(self):
- """deqp.DEQPBaseTest.interpret_result: if returncode is < 0 stauts
+ """deqp.DEQPSingleTest.interpret_result: if returncode is < 0 stauts
is crash.
"""
self.test.result.returncode = -9
@@ -172,7 +194,7 @@ class TestDEQPBaseTest(object):
assert self.test.result.result is status.CRASH
def test_returncode_fail(self):
- """deqp.DEQPBaseTest.interpret_result: if returncode is > 0 result
+ """deqp.DEQPSingleTest.interpret_result: if returncode is > 0 result
is fail.
"""
self.test.result.returncode = 1
@@ -180,7 +202,7 @@ class TestDEQPBaseTest(object):
assert self.test.result.result is status.FAIL
def test_fallthrough(self):
- """deqp.DEQPBaseTest.interpret_result: if no case is hit set to
+ """deqp.DEQPSingleTest.interpret_result: if no case is hit set to
fail.
"""
self.test.result.returncode = 0
@@ -189,7 +211,7 @@ class TestDEQPBaseTest(object):
assert self.test.result.result is status.FAIL
def test_windows_returncode_3(self, mocker):
- """deqp.DEQPBaseTest.interpret_result: on windows returncode 3 is
+ """deqp.DEQPSingleTest.interpret_result: on windows returncode 3 is
crash.
"""
mocker.patch('framework.test.base.sys.platform', 'win32')
@@ -197,8 +219,8 @@ class TestDEQPBaseTest(object):
self.test.interpret_result()
assert self.test.result.result is status.CRASH
- class TestDEQPBaseTestIntepretResultOutput(object):
- """Tests for DEQPBaseTest.__find_map."""
+ class TestIntepretResultOutput(object):
+ """Tests for DEQPSingleTest.__find_map."""
inst = None
__OUT = textwrap.dedent("""\
@@ -240,31 +262,31 @@ class TestDEQPBaseTest(object):
self.inst.result.returncode = 0
def test_fail(self):
- """test.deqp.DEQPBaseTest.interpret_result: when Fail in result the
- result is 'fail'.
+ """test.deqp.DEQPSingleTest.interpret_result: when Fail in result
+ the result is 'fail'.
"""
self.inst.result.out = self.__gen_stdout('Fail')
self.inst.interpret_result()
assert self.inst.result.result is status.FAIL
def test_pass(self):
- """test.deqp.DEQPBaseTest.interpret_result: when Pass in result the
- result is 'Pass'.
+ """test.deqp.DEQPsingleTest.interpret_result: when Pass in result
+ the result is 'Pass'.
"""
self.inst.result.out = self.__gen_stdout('Pass')
self.inst.interpret_result()
assert self.inst.result.result is status.PASS
def test_warn(self):
- """test.deqp.DEQPBaseTest.interpret_result: when QualityWarning in
- result the result is 'warn'.
+ """test.deqp.DEQPSingleTest.interpret_result: when QualityWarning
+ in result the result is 'warn'.
"""
self.inst.result.out = self.__gen_stdout('QualityWarning')
self.inst.interpret_result()
assert self.inst.result.result is status.WARN
def test_error(self):
- """test.deqp.DEQPBaseTest.interpret_result: when InternalError in
+ """test.deqp.DEQPSingleTest.interpret_result: when InternalError in
result the result is 'fail'.
"""
self.inst.result.out = self.__gen_stdout('InternalError')
@@ -272,7 +294,7 @@ class TestDEQPBaseTest(object):
assert self.inst.result.result is status.FAIL
def test_crash(self):
- """test.deqp.DEQPBaseTest.interpret_result: when InternalError in
+ """test.deqp.DEQPSingleTest.interpret_result: when InternalError in
result the result is 'crash'.
"""
self.inst.result.out = self.__gen_stdout('Crash')
@@ -280,7 +302,7 @@ class TestDEQPBaseTest(object):
assert self.inst.result.result is status.CRASH
def test_skip(self):
- """test.deqp.DEQPBaseTest.interpret_result: when NotSupported in
+ """test.deqp.DEQPSingleTest.interpret_result: when NotSupported in
result the result is 'skip'.
"""
self.inst.result.out = self.__gen_stdout('NotSupported')
@@ -288,7 +310,7 @@ class TestDEQPBaseTest(object):
assert self.inst.result.result is status.SKIP
def test_resourceerror(self):
- """test.deqp.DEQPBaseTest.interpret_result: when ResourceError in
+ """test.deqp.DEQPSingleTest.interpret_result: when ResourceError in
result the result is 'crash'.
"""
self.inst.result.out = self.__gen_stdout('ResourceError')
@@ -296,6 +318,145 @@ class TestDEQPBaseTest(object):
assert self.inst.result.result is status.CRASH
+class TestDEQPGroupTest(object):
+ """Tests for the DEQPGroupTest class."""
+
+ def test_asterisk_command(self):
+ test = _DEQPGroupAsteriskTest('foo.bar.aux', ['foo.bar.aux.a'])
+ assert test.command == \
+ ['deqp.bin', "--deqp-case=foo.bar.aux.*", 'extra']
+
+ def test_trie_command(self):
+ test = _DEQPGroupTrieTest('foo.bar.aux',
+ ['foo.bar.aux.a', 'foo.bar.aux.b'])
+ assert test.command == \
+ ['deqp.bin', "--deqp-caselist={foo{bar{aux{a,b}}}}", 'extra']
+
+ def test_subtests_prepopulated(self):
+ subtests = {'foo.bar.aux.oh', 'foo.bar.aux.mok', 'foo.bar.aux.go'}
+ expected = {'oh', 'mok', 'go'}
+ test = _DEQPGroupAsteriskTest('foo.bar.aux', subtests)
+ assert set(test.result.subtests.keys()) == expected
+ assert set(test.result.subtests.values()) == {status.NOTRUN}
+
+ class TestInterpretResult(object):
+ """Tests for DEQPGroupTest.interpret_result."""
+
+ def test_subtests_assigned(self):
+ """Each subtest should be updated, not of them should still be
+ status.NOTRUN, and no new ones should be added.
+ """
+ test = _DEQPGroupAsteriskTest(
+ 'dEQP-VK.api.smoke',
+ ['dEQP-VK.api.smoke.create_sampler',
+ 'dEQP-VK.api.smoke.create_shader',
+ 'dEQP-VK.api.smoke.triangle',
+ 'dEQP-VK.api.smoke.asm_triangle',
+ 'dEQP-VK.api.smoke.asm_triangle_no_opname',
+ 'dEQP-VK.api.smoke.unused_resolve_attachment',
+ ])
+ test.result.returncode = 0
+ test.result.out = textwrap.dedent("""\
+ dEQP Core git-fc145f51987c742ecaa162c2ef540a5f47482547 (0xfc145f51) starting..
+ target implementation = 'Null'
+
+ Test case 'dEQP-VK.api.smoke.create_sampler'..
+ Test case duration in microseconds = 4 us
+ Pass (Creating sampler succeeded)
+
+ Test case 'dEQP-VK.api.smoke.create_shader'..
+ Vertex shader compile time = 8.565000 ms
+ Link time = 0.003000 ms
+ Test case duration in microseconds = 8907 us
+ Pass (Creating shader module succeeded)
+
+ Test case 'dEQP-VK.api.smoke.triangle'..
+ Fragment shader compile time = 0.268000 ms
+ Link time = 0.002000 ms
+ Vertex shader compile time = 0.256000 ms
+ Link time = 0.003000 ms
+ Test case duration in microseconds = 15645 us
+ Fail (Image comparison failed)
+
+ Test case 'dEQP-VK.api.smoke.asm_triangle'..
+ SpirV assembly time = 0.063000 ms
+ SpirV assembly time = 0.047000 ms
+ Test case duration in microseconds = 15078 us
+ Fail (Image comparison failed)
+
+ Test case 'dEQP-VK.api.smoke.asm_triangle_no_opname'..
+ SpirV assembly time = 0.038000 ms
+ SpirV assembly time = 0.064000 ms
+ Test case duration in microseconds = 14866 us
+ Fail (Image comparison failed)
+
+ Test case 'dEQP-VK.api.smoke.unused_resolve_attachment'..
+ Fragment shader compile time = 0.266000 ms
+ Link time = 0.002000 ms
+ Vertex shader compile time = 0.251000 ms
+ Link time = 0.002000 ms
+ Test case duration in microseconds = 15412 us
+ Fail (Image comparison failed)
+
+ DONE!
+
+ Test run totals:
+ Passed: 2/6 (33.3%)
+ Failed: 4/6 (66.7%)
+ Not supported: 0/6 (0.0%)
+ Warnings: 0/6 (0.0%)
+ """)
+ test.interpret_result()
+
+ assert test.result.subtests == {
+ 'create_sampler': status.PASS,
+ 'create_shader': status.PASS,
+ 'triangle': status.FAIL,
+ 'asm_triangle': status.FAIL,
+ 'asm_triangle_no_opname': status.FAIL,
+ 'unused_resolve_attachment': status.FAIL,
+ }
+
+ class TestResume(object):
+ """Tests for the _resume method."""
+
+ def test_basic(self):
+ test = _DEQPGroupAsteriskTest(
+ 'foo.bar', ['foo.bar.a', 'foo.bar.b', 'foo.bar.c'])
+ actual = test._resume(1)
+ expected = ['deqp.bin', '--deqp-caselist={foo{bar{b,c}}}', 'extra']
+ assert actual == expected
+
+ def test_large(self):
+ """Test that a very large set of tests doesn't exceed the trielist
+ limit
+ """
+ test = _DEQPGroupAsteriskTest(
+ 'foo.bar', ['foo.bar.{}'.format(a) for a in range(2000)])
+ actual = test._resume(1)
+ expected = [
+ 'deqp.bin',
+ '--deqp-caselist={{foo{{bar{{{0}}}}}}}'.format(
+ # The 0 test is failed and shouldn't be rerun
+ ','.join(six.text_type(s) for s in range(1, 1000))),
+ 'extra']
+ assert actual == expected
+
+ def test_large_split(self):
+ """Test that the splitting works correctly when test 999 fails
+ """
+ test = _DEQPGroupAsteriskTest(
+ 'foo.bar', ['foo.bar.{}'.format(a) for a in range(2000)])
+ actual = test._resume(999)
+ expected = [
+ 'deqp.bin',
+ '--deqp-caselist={{foo{{bar{{{0}}}}}}}'.format(
+ # The 0 test is failed and shouldn't be rerun
+ ','.join(six.text_type(s) for s in range(999, 2000))),
+ 'extra']
+ assert actual == expected
+
+
class TestGenMustpassTests(object):
"""Tests for the gen_mustpass_tests function."""
--
git-series 0.8.7
More information about the Piglit
mailing list