[Piglit] [PATCH 5/5] deqp: Allow rerun statuses to be overwritten.

[Dylan Baker]

This adds an environment variable and a piglit.conf option to set the
statuses to rerun in test at a time mode.

Signed-off-by: Dylan Baker <dylanx.c.baker at intel.com>
---
 framework/test/deqp.py  | 22 +++++++++++++++-------
 piglit.conf.example     | 10 ++++++++++
 unittests/deqp_tests.py | 13 +++++++++++++
 3 files changed, 38 insertions(+), 7 deletions(-)

diff --git a/framework/test/deqp.py b/framework/test/deqp.py
index e7b52a4..19db355 100644
--- a/framework/test/deqp.py
+++ b/framework/test/deqp.py
@@ -62,7 +62,9 @@ _EXTRA_ARGS = get_option('PIGLIT_DEQP_EXTRA_ARGS',
                          ('deqp', 'extra_args'),
                          default='').split()
 
-_RERUN = ['crash', 'incomplete', 'timeout', 'notrun', 'skip']
+_RERUN = get_option('PIGLIT_DEQP_RERUN',
+                    ('deqp', 'group_rerun'),
+                    default='crash notrun incomplete skip timeout').split()
 
 
 def _gen_caselist_txt(bin_, caselist, extra_args):
@@ -114,10 +116,15 @@ def _iter_test_groups(case_file):
     slice_group = slice(len('GROUP: '), None)
     slice_test = slice(len('TEST: '), None)
 
-    group = ''
+    group = '.'
     tests = []
     with open(case_file, 'r') as caselist_file:
-        for i, line in enumerate(_iterate_file(caselist_file)):
+        iter_ = _iterate_file(caselist_file)
+        # Wind passed the very base group, which would otherwise require
+        # special handling.
+        next(iter_)
+
+        for i, line in enumerate(iter_):
             if line.startswith('GROUP:'):
                 new = line[slice_group].strip()
 
@@ -227,8 +234,9 @@ class DEQPProfile(TestProfile):
         super(DEQPProfile, self).run(logger, backend)
 
         if OPTIONS.deqp_group_rerun and self._rerun:
-            print('\nRerunning failed tests in single mode. '
-                  '(run with --deqp-no-group-rerun to disable)\n')
+            print('\nRerunning tests in single mode with statuses: "{}". '
+                  '(run with --deqp-no-group-rerun to disable)\n'.format(
+                      ', '.join(_RERUN)))
 
             log = LogManager(logger, len(self._rerun))
             self._run(log, backend,
@@ -334,7 +342,7 @@ class DEQPGroupTest(DEQPBaseTest):
     __finder = re.compile(r'^  (Warnings|Not supported|Failed|Passed):\s+\d/(?P<total>\d+).*')
 
     def __init__(self, case_name, individual_cases, **kwargs):
-        super(DEQPGroupTest, self).__init__(case_name + '*', **kwargs)
+        super(DEQPGroupTest, self).__init__(case_name + '.*', **kwargs)
         self._individual_cases = individual_cases
 
     def interpret_result(self):
@@ -410,7 +418,7 @@ class DEQPGroupTest(DEQPBaseTest):
                         '{}\ncurrent line: {}'.format(self.result.out, l))
 
         # If group_rerun (the default) and the status is crash rerun
-        if OPTIONS.deqp_group_rerun and self.result.result == 'crash':
+        if OPTIONS.deqp_group_rerun and self.result.result in _RERUN:
             self.rerun.extend(self._individual_cases)
         # We failed to parse the test output. Fallback to 'fail'.
         elif self.result.result == 'notrun':
diff --git a/piglit.conf.example b/piglit.conf.example
index 944d5d9..46260a8 100644
--- a/piglit.conf.example
+++ b/piglit.conf.example
@@ -36,6 +36,14 @@ testB
 [deqp]
 ; Options that affect all deqp based suites
 ;extra_args=--deqp-visibility=hidden
+;
+; Option that sets the statuses to rerun individually for. Should be a space
+; separated list, and will have split(' ') called on it.
+; This can be overwritten by PIGLIT_DEQP_RERUN enviornment variable.
+;
+; Non-exaustive options: fail dmesg-fail warn dmesg-warn
+; Default=crash notrun incomplete skip timeout
+;group_rerun=crash notrun incomplete skip timeout
 
 [deqp-gles2]
 ; Path to the deqp-gles2 executable
@@ -152,3 +160,5 @@ run_test=./%(test_name)s
 [expected-crashes]
 ; Like expected-failures, but specifies that a test is expected to
 ; crash.
+
+; vim: filetype=dosini
diff --git a/unittests/deqp_tests.py b/unittests/deqp_tests.py
index f3e62b0..b718db2 100644
--- a/unittests/deqp_tests.py
+++ b/unittests/deqp_tests.py
@@ -502,6 +502,7 @@ def test_DEQPGroupTest_interpret_result_nonzero():
 def test_iter_test_groups():
     """deqp._test_test_groups: Returns expected values"""
     text = textwrap.dedent("""\
+        GROUP: dEQP-GLES2
         GROUP: dEQP-GLES2.info
         TEST: dEQP-GLES2.info.vendor
         TEST: dEQP-GLES2.info.renderer
@@ -509,6 +510,8 @@ def test_iter_test_groups():
         TEST: dEQP-GLES2.info.shading_language_version
         TEST: dEQP-GLES2.info.extensions
         TEST: dEQP-GLES2.info.render_target
+        GROUP: dEQP-GLES2.cap
+        TEST: dEQP-GLES2.cap.testy
         GROUP: dEQP-GLES2.capability
         GROUP: dEQP-GLES2.capability.limits
         TEST: dEQP-GLES2.capability.limits.vertex_attribs
@@ -526,6 +529,11 @@ def test_iter_test_groups():
         GROUP: dEQP-GLES2.capability.extensions
         GROUP: dEQP-GLES2.capability.extensions.uncompressed_texture_formats
         TEST: dEQP-GLES2.capability.extensions.uncompressed_texture_formats.foo
+        GROUP: dEQP-GLES2.egl
+        GROUP: dEQP-GLES2.egl.egl_image
+        TEST: dEQP-GLES2.egl.egl_image.egl_image
+        GROUP: dEQP-GLES2.egl.egl_image_external
+        TEST: dEQP-GLES2.egl.egl_image_external.TestGetBinding
     """)
 
     expected = [
@@ -536,6 +544,8 @@ def test_iter_test_groups():
           'dEQP-GLES2.info.shading_language_version',
           'dEQP-GLES2.info.extensions',
           'dEQP-GLES2.info.render_target',]),
+        ('dEQP-GLES2.cap',
+         ['dEQP-GLES2.cap.testy']),
         ('dEQP-GLES2.capability.limits',
          ['dEQP-GLES2.capability.limits.vertex_attribs',
           'dEQP-GLES2.capability.limits.varying_vectors',
@@ -551,6 +561,9 @@ def test_iter_test_groups():
          ['dEQP-GLES2.capability.limits_lower.minimum_size']),
         ('dEQP-GLES2.capability.extensions.uncompressed_texture_formats',
          ['dEQP-GLES2.capability.extensions.uncompressed_texture_formats.foo']),
+        ('dEQP-GLES2.egl.egl_image', ['dEQP-GLES2.egl.egl_image.egl_image']),
+        ('dEQP-GLES2.egl.egl_image_external',
+         ['dEQP-GLES2.egl.egl_image_external.TestGetBinding']),
     ]
 
     with mock.patch('framework.test.deqp.open', create=True,
-- 
2.7.4