[Piglit] [PATCH v2] framework: handle crash codes like piglit native tests.
Jason Ekstrand
jason at jlekstrand.net
Sat Nov 14 21:34:36 PST 2015
On Thu, Nov 12, 2015 at 4:43 PM, <baker.dylan.c at gmail.com> wrote:
> From: Dylan Baker <baker.dylan.c at gmail.com>
>
> This changes the behavior of the dEQP integration such that a status
> that is < 0 on unix or (< 0 || == 3) on windows will be a crash rather
> than a fail. A status > 0 (except 3 on windows) will still be marked
> fail.
>
> This makes use of the helper function from framework/test/base.py
> rather than calling super() (which would still call the helper) because
> we don't want to get the warn status that we would also inherit.
>
> v2: - Update tests to use actual deqp ouput.
> - Fix status loop, v1 had a bug that would cause the loop to not
> exit when it needed to, but it would pass the simplified tests.
Seems to work as advertised this time. Thanks!
Tested-by: Jason Ekstrand <jason.ekstrand at intel.com>
> cc: Jason Ekstrand <jason at jlekstrand.net>
> Signed-off-by: Dylan Baker <dylanx.c.baker at intel.com>
> ---
> framework/test/base.py | 5 +-
> framework/test/deqp.py | 23 +++--
> framework/tests/deqp_tests.py | 191 ++++++++++++++++++++++++++++++++++--------
> 3 files changed, 174 insertions(+), 45 deletions(-)
>
> diff --git a/framework/test/base.py b/framework/test/base.py
> index bf00396..e130ec5 100644
> --- a/framework/test/base.py
> +++ b/framework/test/base.py
> @@ -47,6 +47,7 @@ __all__ = [
> 'TestRunError',
> 'ValgrindMixin',
> 'WindowResizeMixin',
> + 'is_crash_returncode',
> ]
>
>
> @@ -112,7 +113,7 @@ class ProcessTimeout(threading.Thread):
> return self.status
>
>
> -def _is_crash_returncode(returncode):
> +def is_crash_returncode(returncode):
> """Determine whether the given process return code correspond to a
> crash.
> """
> @@ -204,7 +205,7 @@ class Test(object):
> def interpret_result(self):
> """Convert the raw output of the test into a form piglit understands.
> """
> - if _is_crash_returncode(self.result.returncode):
> + if is_crash_returncode(self.result.returncode):
> # check if the process was terminated by the timeout
> if self.timeout > 0 and self.__proc_timeout.join() > 0:
> self.result.result = 'timeout'
> diff --git a/framework/test/deqp.py b/framework/test/deqp.py
> index 8290faf..5c84131 100644
> --- a/framework/test/deqp.py
> +++ b/framework/test/deqp.py
> @@ -24,7 +24,8 @@ import subprocess
>
> # Piglit modules
> from framework import core, grouptools, exceptions
> -from framework.profile import Test, TestProfile
> +from framework.profile import TestProfile
> +from framework.test.base import Test, is_crash_returncode
>
> __all__ = [
> 'DEQPBaseTest',
> @@ -141,11 +142,10 @@ class DEQPBaseTest(Test):
> command = super(DEQPBaseTest, self).command
> return command + self.extra_args
>
> - def interpret_result(self):
> - if self.result.returncode != 0:
> - self.result.result = 'fail'
> - return
> -
> + def __find_map(self):
> + """Run over the lines and set the result."""
> + # splitting this into a separate function allows us to return cleanly,
> + # otherwise this requires some break/else/continue madness
> for line in self.result.out.split('\n'):
> line = line.lstrip()
> for k, v in self.__RESULT_MAP.iteritems():
> @@ -153,5 +153,14 @@ class DEQPBaseTest(Test):
> self.result.result = v
> return
>
> + def interpret_result(self):
> + if is_crash_returncode(self.result.returncode):
> + self.result.result = 'crash'
> + elif self.result.returncode != 0:
> + self.result.result = 'fail'
> + else:
> + self.__find_map()
> +
> # We failed to parse the test output. Fallback to 'fail'.
> - self.result.result = 'fail'
> + if self.result.result == 'notrun':
> + self.result.result = 'fail'
> diff --git a/framework/tests/deqp_tests.py b/framework/tests/deqp_tests.py
> index d9327ec..d95390d 100644
> --- a/framework/tests/deqp_tests.py
> +++ b/framework/tests/deqp_tests.py
> @@ -25,6 +25,9 @@ tests
>
> """
>
> +from __future__ import absolute_import, division, print_function
> +
> +import mock
> import nose.tools as nt
>
> from framework import profile, grouptools, exceptions
> @@ -137,42 +140,158 @@ def test_DEQPBaseTest_command():
> nt.eq_(test.command[-1], 'extra')
>
>
> -def test_DEQPBaseTest_interpret_result_returncode():
> - """deqp.DEQPBaseTest.interpret_result: if returncode is not 0 result is fail
> - """
> - test = _DEQPTestTest('a.deqp.test')
> - test.result.returncode = 1
> - test.interpret_result()
> -
> - nt.eq_(test.result.result, 'fail')
> +class TestDEQPBaseTestInterpretResult(object):
> + """Tests for DEQPBaseTest.interpret_result.
>
> + This specifically tests the part that searches stdout.
>
> -def test_DEQPBaseTest_interpret_result_fallthrough():
> - """deqp.DEQPBaseTest.interpret_result: if no case is hit set to fail
> """
> - test = _DEQPTestTest('a.deqp.test')
> - test.result.returncode = 0
> - test.result.out = ''
> - test.interpret_result()
> -
> - nt.eq_(test.result.result, 'fail')
> -
> -
> - at utils.nose_generator
> -def test_DEQPBaseTest_interpret_result_status():
> - """generate tests for each status possiblility."""
> - def test(status, expected):
> - inst = _DEQPTestTest('a.deqp.test')
> - inst.result.returncode = 0
> - inst.result.out = status
> - inst.interpret_result()
> - nt.eq_(inst.result.result, expected)
> -
> - desc = ('deqp.DEQPBaseTest.interpret_result: '
> - 'when "{}" in stdout status is set to "{}"')
> -
> - _map = deqp.DEQPBaseTest._DEQPBaseTest__RESULT_MAP.iteritems() # pylint: disable=no-member,protected-access
> -
> - for status, expected in _map:
> - test.description = desc.format(status, expected)
> - yield test, status, expected
> + def __init__(self):
> + self.test = None
> +
> + def setup(self):
> + self.test = _DEQPTestTest('a.deqp.test')
> +
> + def test_crash(self):
> + """deqp.DEQPBaseTest.interpret_result: if returncode is < 0 stauts is crash"""
> + self.test.result.returncode = -9
> + self.test.interpret_result()
> + nt.eq_(self.test.result.result, 'crash')
> +
> + def test_returncode_fail(self):
> + """deqp.DEQPBaseTest.interpret_result: if returncode is > 0 result is fail"""
> + self.test.result.returncode = 1
> + self.test.interpret_result()
> + nt.eq_(self.test.result.result, 'fail')
> +
> + def test_fallthrough(self):
> + """deqp.DEQPBaseTest.interpret_result: if no case is hit set to fail"""
> + self.test.result.returncode = 0
> + self.test.result.out = ''
> + self.test.interpret_result()
> + nt.eq_(self.test.result.result, 'fail')
> +
> + def test_windows_returncode_3(self):
> + """deqp.DEQPBaseTest.interpret_result: on windows returncode 3 is crash"""
> + self.test.result.returncode = 3
> + with mock.patch('framework.test.base.sys.platform', 'win32'):
> + self.test.interpret_result()
> + nt.eq_(self.test.result.result, 'crash')
> +
> +
> +class TestDEQPBaseTestIntepretResultStatus(object):
> + """Tests for DEQPBaseTest.__find_map."""
> + def __init__(self):
> + self.inst = None
> +
> + @staticmethod
> + def __make_out(status):
Did you seriously just name a function "make out"? How about
"generate_test_output" or "dummy_deqp_output" or something like that?
> + out = (
> + "dEQP Core 2014.x (0xcafebabe) starting..\n"
> + "arget implementation = 'DRM'\n"
> + "\n"
> + "Test case 'dEQP-GLES2.functional.shaders.conversions.vector_to_vector.vec3_to_ivec3_fragment'..\n"
> + "Vertex shader compile time = 0.129000 ms\n"
> + "Fragment shader compile time = 0.264000 ms\n"
> + "Link time = 0.814000 ms\n"
> + "Test case duration in microseconds = 487155 us\n"
> + "{stat} ({stat})\n"
> + "\n"
> + "DONE!\n"
> + "\n"
> + "Test run totals:\n"
> + "Passed: {pass_}/1 (100.0%)\n"
> + "Failed: {fail}/1 (0.0%)\n"
> + "Not supported: {supp}/1 (0.0%)\n"
> + "Warnings: {warn}/1 (0.0%)\n"
> + )
> +
> + if status == 'Fail':
> + return out.format(
> + stat=status,
> + pass_=0,
> + fail=1,
> + supp=0,
> + warn=0,
> + )
> + elif status == 'NotSupported':
> + return out.format(
> + stat=status,
> + pass_=0,
> + fail=0,
> + supp=0,
> + warn=0,
> + )
> + elif status == 'Pass':
> + return out.format(
> + stat=status,
> + pass_=1,
> + fail=0,
> + supp=0,
> + warn=0,
> + )
> + elif status == 'QualityWarning':
> + return out.format(
> + stat=status,
> + pass_=0,
> + fail=0,
> + supp=0,
> + warn=1,
> + )
> + elif status == 'InternalError':
> + return out.format(
> + stat=status,
> + pass_=0,
> + fail=0,
> + supp=1,
> + warn=0,
> + )
> + elif status == 'Crash':
> + return out.format(
> + stat=status,
> + pass_=0,
> + fail=1,
> + supp=0,
> + warn=0,
> + )
> + raise Exception('Unreachable')
> +
> + def setup(self):
> + self.inst = _DEQPTestTest('a.deqp.test')
> + self.inst.result.returncode = 0
> +
> + def test_fail(self):
> + """test.deqp.DEQPBaseTest.interpret_result: when Fail in result the result is 'fail'"""
> + self.inst.result.out = self.__make_out('Fail')
> + self.inst.interpret_result()
> + nt.eq_(self.inst.result.result, 'fail')
> +
> + def test_pass(self):
> + """test.deqp.DEQPBaseTest.interpret_result: when Pass in result the result is 'Pass'"""
> + self.inst.result.out = self.__make_out('Pass')
> + self.inst.interpret_result()
> + nt.eq_(self.inst.result.result, 'pass')
> +
> + def test_warn(self):
> + """test.deqp.DEQPBaseTest.interpret_result: when QualityWarning in result the result is 'warn'"""
> + self.inst.result.out = self.__make_out('QualityWarning')
> + self.inst.interpret_result()
> + nt.eq_(self.inst.result.result, 'warn')
> +
> + def test_error(self):
> + """test.deqp.DEQPBaseTest.interpret_result: when InternalError in result the result is 'fail'"""
> + self.inst.result.out = self.__make_out('InternalError')
> + self.inst.interpret_result()
> + nt.eq_(self.inst.result.result, 'fail')
> +
> + def test_crash(self):
> + """test.deqp.DEQPBaseTest.interpret_result: when InternalError in result the result is 'crash'"""
> + self.inst.result.out = self.__make_out('Crash')
> + self.inst.interpret_result()
> + nt.eq_(self.inst.result.result, 'crash')
> +
> + def test_skip(self):
> + """test.deqp.DEQPBaseTest.interpret_result: when NotSupported in result the result is 'skip'"""
> + self.inst.result.out = self.__make_out('NotSupported')
> + self.inst.interpret_result()
> + nt.eq_(self.inst.result.result, 'skip')
> --
> 2.6.2
>
More information about the Piglit
mailing list