[Piglit] [PATCH] framework: handle crash codes like piglit native tests

[baker.dylan.c at gmail.com]

From: Dylan Baker <baker.dylan.c at gmail.com>

This changes the behavior of the dEQP integration such that a status
that is < 0 on unix or (< 0 || == 3) on windows will be a crash rather
than a fail. A status > 0 (except 3 on windows) will still be marked
fail.

This makes use of the helper function from framework/test/base.py
rather than calling super() (which would still call the helper) because
we don't want to get the warn status that we would also inherit.

cc: Jason Ekstrand <jason at jlekstrand.net>
Signed-off-by: Dylan Baker <dylanx.c.baker at intel.com>
---
 framework/test/base.py        |  5 ++--
 framework/test/deqp.py        | 24 +++++++++++---------
 framework/tests/deqp_tests.py | 53 ++++++++++++++++++++++++++++++-------------
 3 files changed, 53 insertions(+), 29 deletions(-)

diff --git a/framework/test/base.py b/framework/test/base.py
index bf00396..e130ec5 100644
--- a/framework/test/base.py
+++ b/framework/test/base.py
@@ -47,6 +47,7 @@ __all__ = [
     'TestRunError',
     'ValgrindMixin',
     'WindowResizeMixin',
+    'is_crash_returncode',
 ]
 
 
@@ -112,7 +113,7 @@ class ProcessTimeout(threading.Thread):
         return self.status
 
 
-def _is_crash_returncode(returncode):
+def is_crash_returncode(returncode):
     """Determine whether the given process return code correspond to a
     crash.
     """
@@ -204,7 +205,7 @@ class Test(object):
     def interpret_result(self):
         """Convert the raw output of the test into a form piglit understands.
         """
-        if _is_crash_returncode(self.result.returncode):
+        if is_crash_returncode(self.result.returncode):
             # check if the process was terminated by the timeout
             if self.timeout > 0 and self.__proc_timeout.join() > 0:
                 self.result.result = 'timeout'
diff --git a/framework/test/deqp.py b/framework/test/deqp.py
index 8290faf..0107e2b 100644
--- a/framework/test/deqp.py
+++ b/framework/test/deqp.py
@@ -24,7 +24,8 @@ import subprocess
 
 # Piglit modules
 from framework import core, grouptools, exceptions
-from framework.profile import Test, TestProfile
+from framework.profile import TestProfile
+from framework.test.base import Test, is_crash_returncode
 
 __all__ = [
     'DEQPBaseTest',
@@ -142,16 +143,17 @@ class DEQPBaseTest(Test):
         return command + self.extra_args
 
     def interpret_result(self):
-        if self.result.returncode != 0:
+        if is_crash_returncode(self.result.returncode):
+            self.result.result = 'crash'
+        elif self.result.returncode != 0:
             self.result.result = 'fail'
-            return
-
-        for line in self.result.out.split('\n'):
-            line = line.lstrip()
-            for k, v in self.__RESULT_MAP.iteritems():
-                if line.startswith(k):
-                    self.result.result = v
-                    return
+        else:
+            for line in self.result.out.split('\n'):
+                line = line.lstrip()
+                for k, v in self.__RESULT_MAP.iteritems():
+                    if line.startswith(k):
+                        self.result.result = v
 
         # We failed to parse the test output. Fallback to 'fail'.
-        self.result.result = 'fail'
+        if self.result.result == 'notrun':
+            self.result.result = 'fail'
diff --git a/framework/tests/deqp_tests.py b/framework/tests/deqp_tests.py
index d9327ec..6acf7e9 100644
--- a/framework/tests/deqp_tests.py
+++ b/framework/tests/deqp_tests.py
@@ -25,6 +25,9 @@ tests
 
 """
 
+from __future__ import absolute_import, division, print_function
+
+import mock
 import nose.tools as nt
 
 from framework import profile, grouptools, exceptions
@@ -137,25 +140,43 @@ def test_DEQPBaseTest_command():
     nt.eq_(test.command[-1], 'extra')
 
 
-def test_DEQPBaseTest_interpret_result_returncode():
-    """deqp.DEQPBaseTest.interpret_result: if returncode is not 0 result is fail
-    """
-    test = _DEQPTestTest('a.deqp.test')
-    test.result.returncode = 1
-    test.interpret_result()
-
-    nt.eq_(test.result.result, 'fail')
+class TestDEQPBaseTestInterpretResult(object):
+    """Tests for DEQPBaseTest.interpret_result.
 
+    This specifically tests the part that searches stdout.
 
-def test_DEQPBaseTest_interpret_result_fallthrough():
-    """deqp.DEQPBaseTest.interpret_result: if no case is hit set to fail
     """
-    test = _DEQPTestTest('a.deqp.test')
-    test.result.returncode = 0
-    test.result.out = ''
-    test.interpret_result()
-
-    nt.eq_(test.result.result, 'fail')
+    def __init__(self):
+        self.test = None
+
+    def setup(self):
+        self.test = _DEQPTestTest('a.deqp.test')
+
+    def test_crash(self):
+        """deqp.DEQPBaseTest.interpret_result: if returncode is < 0 stauts is crash"""
+        self.test.result.returncode = -9
+        self.test.interpret_result()
+        nt.eq_(self.test.result.result, 'crash')
+
+    def test_returncode_fail(self):
+        """deqp.DEQPBaseTest.interpret_result: if returncode is > 0 result is fail"""
+        self.test.result.returncode = 1
+        self.test.interpret_result()
+        nt.eq_(self.test.result.result, 'fail')
+
+    def test_fallthrough(self):
+        """deqp.DEQPBaseTest.interpret_result: if no case is hit set to fail"""
+        self.test.result.returncode = 0
+        self.test.result.out = ''
+        self.test.interpret_result()
+        nt.eq_(self.test.result.result, 'fail')
+
+    def test_windows_returncode_3(self):
+        """deqp.DEQPBaseTest.interpret_result: on windows returncode 3 is crash"""
+        self.test.result.returncode = 3
+        with mock.patch('framework.test.base.sys.platform', 'win32'):
+            self.test.interpret_result()
+        nt.eq_(self.test.result.result, 'crash')
 
 
 @utils.nose_generator
-- 
2.6.2