[Piglit] [PATCH v2] framework: Add mustpass option for the Vulkan CTS test runner

[Alejandro Piñeiro]

This option was added mostly to being able to run the mustpass
case list included on Vulkan CTS repository. As that file is in
txt format, we assume that the case list will be in txt format.

That is not the case for the other deqp-xxx profiles, that uses
a xml format. In any case, it is properly documented on the
piglit.conf.example

v2: allow to receive a txt file with just the name of the test,
    without any formatting
---

This new version is somewhat more intrusive. It allows to receive
a txt file with just the name of the tests. But doing that
it removed the ill-format-line check on deqp.make_profile.

The advantage of this is that we don't need to transform the
mustpass file that cames with the vulkan cts repository.

FWIW, deqp-vk also accepts that format. So with this change,
piglit just allows to receive the testcase list in both txt formats.

 framework/test/deqp.py | 14 ++++++++++----
 piglit.conf.example    |  7 +++++++
 tests/deqp_vk.py       | 10 +++++++---
 3 files changed, 24 insertions(+), 7 deletions(-)

diff --git a/framework/test/deqp.py b/framework/test/deqp.py
index 5b53efd..a63bea7 100644
--- a/framework/test/deqp.py
+++ b/framework/test/deqp.py
@@ -78,6 +78,14 @@ def select_source(bin_, filename, mustpass, extra_args):
         return iter_deqp_test_cases(
             gen_caselist_txt(bin_, filename, extra_args))
 
+def select_source_txt(bin_, filename, mustpass, extra_args):
+    """Return either the mustpass list in txt format or the generated list."""
+    if options.OPTIONS.deqp_mustpass:
+        return iter_deqp_test_cases(mustpass)
+    else:
+        return iter_deqp_test_cases(
+            gen_caselist_txt(bin_, filename, extra_args))
+
 
 def make_profile(test_list, test_class):
     """Create a TestProfile instance."""
@@ -146,10 +154,8 @@ def iter_deqp_test_cases(case_file):
                 continue
             elif line.startswith('TEST:'):
                 yield line[len('TEST:'):].strip()
-            else:
-                raise exceptions.PiglitFatalError(
-                    'deqp: {}:{}: ill-formed line'.format(case_file, i))
-
+            else: # We assume that the line is just the name of the test
+                yield line
 
 @six.add_metaclass(abc.ABCMeta)
 class DEQPBaseTest(Test):
diff --git a/piglit.conf.example b/piglit.conf.example
index e1a91aa..46d1ed6 100644
--- a/piglit.conf.example
+++ b/piglit.conf.example
@@ -99,6 +99,13 @@ testB
 ; Can be overwritten by PIGLIT_DEQP_VK_BIN environment variable
 ;bin=/home/knuth/Vulkan-CTS/external/vulkancts/modules/vulkan/deqp-vk
 
+; Path to the test case list for deqp-vk. You can also set this with
+; the environment variable PIGLIT_DEQP_VK_MUSTPASS. Piglit will run
+; the subset of dEQP-VK tests if this option is set. Note that in this
+; case the mustpass file is in txt format, in opposite to other
+; deqp-xxx mustpass list where they are on xml format.
+;mustpasslist=/home/knuth/Vulkan-CTS/external/vulkancts/mustpass/1.0.1/vk-default.txt
+
 ; Space-separated list of extra command line arguments for deqp-vk. The
 ; option is not required. The environment variable PIGLIT_DEQP_VK_EXTRA_ARGS
 ; overrides the value set here.
diff --git a/tests/deqp_vk.py b/tests/deqp_vk.py
index 4df9e3b..b802ff1 100644
--- a/tests/deqp_vk.py
+++ b/tests/deqp_vk.py
@@ -30,6 +30,7 @@ from __future__ import (
 import re
 
 from framework.test import deqp
+from framework.options import OPTIONS
 
 __all__ = ['profile']
 
@@ -38,6 +39,10 @@ _DEQP_VK_BIN = deqp.get_option('PIGLIT_DEQP_VK_BIN',
                                ('deqp-vk', 'bin'),
                                required=True)
 
+_DEQP_MUSTPASS = deqp.get_option('PIGLIT_DEQP_VK_MUSTPASS',
+                                 ('deqp-vk', 'mustpasslist'),
+                                 required=OPTIONS.deqp_mustpass)
+
 _EXTRA_ARGS = deqp.get_option('PIGLIT_DEQP_VK_EXTRA_ARGS',
                               ('deqp-vk', 'extra_args'),
                               default='').split()
@@ -72,7 +77,6 @@ class DEQPVKTest(deqp.DEQPBaseTest):
 
 
 profile = deqp.make_profile(  # pylint: disable=invalid-name
-    deqp.iter_deqp_test_cases(
-        deqp.gen_caselist_txt(_DEQP_VK_BIN, 'dEQP-VK-cases.txt',
-                              _EXTRA_ARGS)),
+    deqp.select_source_txt(_DEQP_VK_BIN, 'dEQP-VK-cases.txt',
+                           _DEQP_MUSTPASS, _EXTRA_ARGS),
     DEQPVKTest)
-- 
2.9.3