[Piglit] [PATCH v2] framework: handle crash codes like piglit native tests.
baker.dylan.c at gmail.com
baker.dylan.c at gmail.com
Thu Nov 12 16:43:47 PST 2015
From: Dylan Baker <baker.dylan.c at gmail.com>
This changes the behavior of the dEQP integration such that a status
that is < 0 on unix or (< 0 || == 3) on windows will be a crash rather
than a fail. A status > 0 (except 3 on windows) will still be marked
fail.
This makes use of the helper function from framework/test/base.py
rather than calling super() (which would still call the helper) because
we don't want to get the warn status that we would also inherit.
v2: - Update tests to use actual deqp ouput.
- Fix status loop, v1 had a bug that would cause the loop to not
exit when it needed to, but it would pass the simplified tests.
cc: Jason Ekstrand <jason at jlekstrand.net>
Signed-off-by: Dylan Baker <dylanx.c.baker at intel.com>
---
framework/test/base.py | 5 +-
framework/test/deqp.py | 23 +++--
framework/tests/deqp_tests.py | 191 ++++++++++++++++++++++++++++++++++--------
3 files changed, 174 insertions(+), 45 deletions(-)
diff --git a/framework/test/base.py b/framework/test/base.py
index bf00396..e130ec5 100644
--- a/framework/test/base.py
+++ b/framework/test/base.py
@@ -47,6 +47,7 @@ __all__ = [
'TestRunError',
'ValgrindMixin',
'WindowResizeMixin',
+ 'is_crash_returncode',
]
@@ -112,7 +113,7 @@ class ProcessTimeout(threading.Thread):
return self.status
-def _is_crash_returncode(returncode):
+def is_crash_returncode(returncode):
"""Determine whether the given process return code correspond to a
crash.
"""
@@ -204,7 +205,7 @@ class Test(object):
def interpret_result(self):
"""Convert the raw output of the test into a form piglit understands.
"""
- if _is_crash_returncode(self.result.returncode):
+ if is_crash_returncode(self.result.returncode):
# check if the process was terminated by the timeout
if self.timeout > 0 and self.__proc_timeout.join() > 0:
self.result.result = 'timeout'
diff --git a/framework/test/deqp.py b/framework/test/deqp.py
index 8290faf..5c84131 100644
--- a/framework/test/deqp.py
+++ b/framework/test/deqp.py
@@ -24,7 +24,8 @@ import subprocess
# Piglit modules
from framework import core, grouptools, exceptions
-from framework.profile import Test, TestProfile
+from framework.profile import TestProfile
+from framework.test.base import Test, is_crash_returncode
__all__ = [
'DEQPBaseTest',
@@ -141,11 +142,10 @@ class DEQPBaseTest(Test):
command = super(DEQPBaseTest, self).command
return command + self.extra_args
- def interpret_result(self):
- if self.result.returncode != 0:
- self.result.result = 'fail'
- return
-
+ def __find_map(self):
+ """Run over the lines and set the result."""
+ # splitting this into a separate function allows us to return cleanly,
+ # otherwise this requires some break/else/continue madness
for line in self.result.out.split('\n'):
line = line.lstrip()
for k, v in self.__RESULT_MAP.iteritems():
@@ -153,5 +153,14 @@ class DEQPBaseTest(Test):
self.result.result = v
return
+ def interpret_result(self):
+ if is_crash_returncode(self.result.returncode):
+ self.result.result = 'crash'
+ elif self.result.returncode != 0:
+ self.result.result = 'fail'
+ else:
+ self.__find_map()
+
# We failed to parse the test output. Fallback to 'fail'.
- self.result.result = 'fail'
+ if self.result.result == 'notrun':
+ self.result.result = 'fail'
diff --git a/framework/tests/deqp_tests.py b/framework/tests/deqp_tests.py
index d9327ec..d95390d 100644
--- a/framework/tests/deqp_tests.py
+++ b/framework/tests/deqp_tests.py
@@ -25,6 +25,9 @@ tests
"""
+from __future__ import absolute_import, division, print_function
+
+import mock
import nose.tools as nt
from framework import profile, grouptools, exceptions
@@ -137,42 +140,158 @@ def test_DEQPBaseTest_command():
nt.eq_(test.command[-1], 'extra')
-def test_DEQPBaseTest_interpret_result_returncode():
- """deqp.DEQPBaseTest.interpret_result: if returncode is not 0 result is fail
- """
- test = _DEQPTestTest('a.deqp.test')
- test.result.returncode = 1
- test.interpret_result()
-
- nt.eq_(test.result.result, 'fail')
+class TestDEQPBaseTestInterpretResult(object):
+ """Tests for DEQPBaseTest.interpret_result.
+ This specifically tests the part that searches stdout.
-def test_DEQPBaseTest_interpret_result_fallthrough():
- """deqp.DEQPBaseTest.interpret_result: if no case is hit set to fail
"""
- test = _DEQPTestTest('a.deqp.test')
- test.result.returncode = 0
- test.result.out = ''
- test.interpret_result()
-
- nt.eq_(test.result.result, 'fail')
-
-
- at utils.nose_generator
-def test_DEQPBaseTest_interpret_result_status():
- """generate tests for each status possiblility."""
- def test(status, expected):
- inst = _DEQPTestTest('a.deqp.test')
- inst.result.returncode = 0
- inst.result.out = status
- inst.interpret_result()
- nt.eq_(inst.result.result, expected)
-
- desc = ('deqp.DEQPBaseTest.interpret_result: '
- 'when "{}" in stdout status is set to "{}"')
-
- _map = deqp.DEQPBaseTest._DEQPBaseTest__RESULT_MAP.iteritems() # pylint: disable=no-member,protected-access
-
- for status, expected in _map:
- test.description = desc.format(status, expected)
- yield test, status, expected
+ def __init__(self):
+ self.test = None
+
+ def setup(self):
+ self.test = _DEQPTestTest('a.deqp.test')
+
+ def test_crash(self):
+ """deqp.DEQPBaseTest.interpret_result: if returncode is < 0 stauts is crash"""
+ self.test.result.returncode = -9
+ self.test.interpret_result()
+ nt.eq_(self.test.result.result, 'crash')
+
+ def test_returncode_fail(self):
+ """deqp.DEQPBaseTest.interpret_result: if returncode is > 0 result is fail"""
+ self.test.result.returncode = 1
+ self.test.interpret_result()
+ nt.eq_(self.test.result.result, 'fail')
+
+ def test_fallthrough(self):
+ """deqp.DEQPBaseTest.interpret_result: if no case is hit set to fail"""
+ self.test.result.returncode = 0
+ self.test.result.out = ''
+ self.test.interpret_result()
+ nt.eq_(self.test.result.result, 'fail')
+
+ def test_windows_returncode_3(self):
+ """deqp.DEQPBaseTest.interpret_result: on windows returncode 3 is crash"""
+ self.test.result.returncode = 3
+ with mock.patch('framework.test.base.sys.platform', 'win32'):
+ self.test.interpret_result()
+ nt.eq_(self.test.result.result, 'crash')
+
+
+class TestDEQPBaseTestIntepretResultStatus(object):
+ """Tests for DEQPBaseTest.__find_map."""
+ def __init__(self):
+ self.inst = None
+
+ @staticmethod
+ def __make_out(status):
+ out = (
+ "dEQP Core 2014.x (0xcafebabe) starting..\n"
+ "arget implementation = 'DRM'\n"
+ "\n"
+ "Test case 'dEQP-GLES2.functional.shaders.conversions.vector_to_vector.vec3_to_ivec3_fragment'..\n"
+ "Vertex shader compile time = 0.129000 ms\n"
+ "Fragment shader compile time = 0.264000 ms\n"
+ "Link time = 0.814000 ms\n"
+ "Test case duration in microseconds = 487155 us\n"
+ "{stat} ({stat})\n"
+ "\n"
+ "DONE!\n"
+ "\n"
+ "Test run totals:\n"
+ "Passed: {pass_}/1 (100.0%)\n"
+ "Failed: {fail}/1 (0.0%)\n"
+ "Not supported: {supp}/1 (0.0%)\n"
+ "Warnings: {warn}/1 (0.0%)\n"
+ )
+
+ if status == 'Fail':
+ return out.format(
+ stat=status,
+ pass_=0,
+ fail=1,
+ supp=0,
+ warn=0,
+ )
+ elif status == 'NotSupported':
+ return out.format(
+ stat=status,
+ pass_=0,
+ fail=0,
+ supp=0,
+ warn=0,
+ )
+ elif status == 'Pass':
+ return out.format(
+ stat=status,
+ pass_=1,
+ fail=0,
+ supp=0,
+ warn=0,
+ )
+ elif status == 'QualityWarning':
+ return out.format(
+ stat=status,
+ pass_=0,
+ fail=0,
+ supp=0,
+ warn=1,
+ )
+ elif status == 'InternalError':
+ return out.format(
+ stat=status,
+ pass_=0,
+ fail=0,
+ supp=1,
+ warn=0,
+ )
+ elif status == 'Crash':
+ return out.format(
+ stat=status,
+ pass_=0,
+ fail=1,
+ supp=0,
+ warn=0,
+ )
+ raise Exception('Unreachable')
+
+ def setup(self):
+ self.inst = _DEQPTestTest('a.deqp.test')
+ self.inst.result.returncode = 0
+
+ def test_fail(self):
+ """test.deqp.DEQPBaseTest.interpret_result: when Fail in result the result is 'fail'"""
+ self.inst.result.out = self.__make_out('Fail')
+ self.inst.interpret_result()
+ nt.eq_(self.inst.result.result, 'fail')
+
+ def test_pass(self):
+ """test.deqp.DEQPBaseTest.interpret_result: when Pass in result the result is 'Pass'"""
+ self.inst.result.out = self.__make_out('Pass')
+ self.inst.interpret_result()
+ nt.eq_(self.inst.result.result, 'pass')
+
+ def test_warn(self):
+ """test.deqp.DEQPBaseTest.interpret_result: when QualityWarning in result the result is 'warn'"""
+ self.inst.result.out = self.__make_out('QualityWarning')
+ self.inst.interpret_result()
+ nt.eq_(self.inst.result.result, 'warn')
+
+ def test_error(self):
+ """test.deqp.DEQPBaseTest.interpret_result: when InternalError in result the result is 'fail'"""
+ self.inst.result.out = self.__make_out('InternalError')
+ self.inst.interpret_result()
+ nt.eq_(self.inst.result.result, 'fail')
+
+ def test_crash(self):
+ """test.deqp.DEQPBaseTest.interpret_result: when InternalError in result the result is 'crash'"""
+ self.inst.result.out = self.__make_out('Crash')
+ self.inst.interpret_result()
+ nt.eq_(self.inst.result.result, 'crash')
+
+ def test_skip(self):
+ """test.deqp.DEQPBaseTest.interpret_result: when NotSupported in result the result is 'skip'"""
+ self.inst.result.out = self.__make_out('NotSupported')
+ self.inst.interpret_result()
+ nt.eq_(self.inst.result.result, 'skip')
--
2.6.2
More information about the Piglit
mailing list