[Piglit] [PATCH v3] framework: Add --ignore-missing option

[Arkadiusz Hiler]

Currently, if a test from provided testlist fails to be discovered by
the framework, piglit blows up with an exception.

This is both good - for consistency/early errors - and bad - for
handling some CI/automation scenarios (e.g autobisecting the tests).

So let's keep the current default, but allow some flexibility with the
new option that reports the missing tests as 'notrun'.

v2:
 - change switch name to 'ignore', as skip is too suggestive
 - use DummyTest to get 'notrun' result instead of warnings

v3: don't use OPTIONS

bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=99649
Cc: Dylan Baker <dylan at pnwbakers.com>
Cc: Tomi Sarvela <tomi.p.sarvela at intel.com>
Cc: Martin Peres <martin.peres at intel.com>
Signed-off-by: Arkadiusz Hiler <arkadiusz.hiler at intel.com>
---
 framework/profile.py      | 10 +++++++---
 framework/programs/run.py | 12 ++++++++++++
 framework/test/base.py    | 12 ++++++++++++
 3 files changed, 31 insertions(+), 3 deletions(-)

diff --git a/framework/profile.py b/framework/profile.py
index a625318..cf0e298 100644
--- a/framework/profile.py
+++ b/framework/profile.py
@@ -42,11 +42,11 @@ import re
 
 import six
 
-from framework import grouptools, exceptions
+from framework import grouptools, exceptions, status
 from framework.dmesg import get_dmesg
 from framework.log import LogManager
 from framework.monitoring import Monitoring
-from framework.test.base import Test
+from framework.test.base import Test, DummyTest
 
 __all__ = [
     'RegexFilter',
@@ -285,6 +285,7 @@ class TestProfile(object):
         self.options = {
             'dmesg': get_dmesg(False),
             'monitor': Monitoring(False),
+            'ignore_missing': False,
         }
 
     def setup(self):
@@ -314,7 +315,10 @@ class TestProfile(object):
         if self.forced_test_list:
             opts = collections.OrderedDict()
             for n in self.forced_test_list:
-                opts[n] = self.test_list[n]
+                if self.options['ignore_missing'] and n not in self.test_list:
+                    opts[n] = DummyTest(name=n, result=status.NOTRUN)
+                else:
+                    opts[n] = self.test_list[n]
         else:
             opts = self.test_list  # pylint: disable=redefined-variable-type
 
diff --git a/framework/programs/run.py b/framework/programs/run.py
index 6444cfe..4524f17 100644
--- a/framework/programs/run.py
+++ b/framework/programs/run.py
@@ -208,6 +208,10 @@ def _run_parser(input_):
                              'isolation. This allows, but does not require, '
                              'tests to run multiple tests per process. '
                              'This value can also be set in piglit.conf.')
+    parser.add_argument("--ignore-missing",
+                        dest="ignore_missing",
+                        action="store_true",
+                        help="missing tests are considered as 'notrun'")
     parser.add_argument("test_profile",
                         metavar="<Profile path(s)>",
                         nargs='+',
@@ -234,6 +238,7 @@ def _create_metadata(args, name, forced_test_list):
     if args.platform:
         opts['platform'] = args.platform
     opts['forced_test_list'] = forced_test_list
+    opts['ignore_missing'] = args.ignore_missing
 
     metadata = {'options': opts}
     metadata['name'] = name
@@ -346,6 +351,10 @@ def run(input_):
         for p in profiles:
             p.options['monitor'] = monitoring.Monitoring(args.monitored)
 
+    if args.ignore_missing:
+        for p in profiles:
+            p.options['ignore_missing'] = args.ignore_missing
+
     for p in profiles:
         if args.exclude_tests:
             p.filters.append(profile.RegexFilter(args.exclude_tests,
@@ -422,6 +431,9 @@ def resume(input_):
             p.options['monitor'] = monitoring.Monitoring(
                 results.options['monitoring'])
 
+        if results.options['ignore_missing']:
+            p.options['ignore_missing'] = results.options['ignore_missing']
+
         if exclude_tests:
             p.filters.append(lambda n, _: n not in exclude_tests)
         if results.options['exclude_filter']:
diff --git a/framework/test/base.py b/framework/test/base.py
index d73dee9..e74ea3d 100644
--- a/framework/test/base.py
+++ b/framework/test/base.py
@@ -381,6 +381,18 @@ class Test(object):
         return not self == other
 
 
+class DummyTest(Test):
+    def __init__(self, name, result):
+        super(DummyTest, self).__init__([name])
+        self.result.result = result
+
+    def execute(self, path, log, options):
+        pass
+
+    def interpret_result(self):
+        pass
+
+
 class WindowResizeMixin(object):
     """ Mixin class that deals with spurious window resizes
 
-- 
2.9.4