[Piglit] [PATCH v2] framework: Add --ignore-missing option

[Arkadiusz Hiler]

Currently, if a test from provided testlist fails to be discovered by
the framework, piglit blows up with an exception.

This is both good - for consistency/early errors - and bad - for
handling some CI/automation scenarios (e.g autobisecting the tests).

So let's keep the current default, but allow some flexibility with the
new option that reports the missing tests as 'notrun'.

v2:
 - change switch name to 'ignore', as skip is too suggestive
 - use DummyTest to get 'notrun' result instead of warnings

bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=99649
Cc: Dylan Baker <dylan at pnwbakers.com>
Cc: Tomi Sarvela <tomi.p.sarvela at intel.com>
Cc: Martin Peres <martin.peres at intel.com>
Tested-by: Petri Latvala <petri.latvala at intel.com>
Signed-off-by: Arkadiusz Hiler <arkadiusz.hiler at intel.com>
---
 framework/profile.py      | 10 +++++++---
 framework/programs/run.py |  6 ++++++
 framework/test/base.py    | 12 ++++++++++++
 3 files changed, 25 insertions(+), 3 deletions(-)

diff --git a/framework/profile.py b/framework/profile.py
index a625318..bb6c39e 100644
--- a/framework/profile.py
+++ b/framework/profile.py
@@ -42,11 +42,12 @@ import re
 
 import six
 
-from framework import grouptools, exceptions
+from framework import grouptools, exceptions, status
 from framework.dmesg import get_dmesg
 from framework.log import LogManager
 from framework.monitoring import Monitoring
-from framework.test.base import Test
+from framework.test.base import Test, DummyTest
+from framework.options import OPTIONS
 
 __all__ = [
     'RegexFilter',
@@ -314,7 +315,10 @@ class TestProfile(object):
         if self.forced_test_list:
             opts = collections.OrderedDict()
             for n in self.forced_test_list:
-                opts[n] = self.test_list[n]
+                if OPTIONS.ignore_missing and n not in self.test_list:
+                    opts[n] = DummyTest(name=n, result=status.NOTRUN)
+                else:
+                    opts[n] = self.test_list[n]
         else:
             opts = self.test_list  # pylint: disable=redefined-variable-type
 
diff --git a/framework/programs/run.py b/framework/programs/run.py
index 6444cfe..d30e6a4 100644
--- a/framework/programs/run.py
+++ b/framework/programs/run.py
@@ -208,6 +208,10 @@ def _run_parser(input_):
                              'isolation. This allows, but does not require, '
                              'tests to run multiple tests per process. '
                              'This value can also be set in piglit.conf.')
+    parser.add_argument("--ignore-missing",
+                        dest="ignore_missing",
+                        action="store_true",
+                        help="missing tests are considered as 'notrun'")
     parser.add_argument("test_profile",
                         metavar="<Profile path(s)>",
                         nargs='+',
@@ -291,6 +295,7 @@ def run(input_):
     options.OPTIONS.sync = args.sync
     options.OPTIONS.deqp_mustpass = args.deqp_mustpass
     options.OPTIONS.process_isolation = args.process_isolation
+    options.OPTIONS.ignore_missing = args.ignore_missing
 
     # Set the platform to pass to waffle
     options.OPTIONS.env['PIGLIT_PLATFORM'] = args.platform
@@ -389,6 +394,7 @@ def resume(input_):
     options.OPTIONS.sync = results.options['sync']
     options.OPTIONS.deqp_mustpass = results.options['deqp_mustpass']
     options.OPTIONS.proces_isolation = results.options['process_isolation']
+    options.OPTIONS.ignore_missing = results.options['ignore_missing']
 
     core.get_config(args.config_file)
 
diff --git a/framework/test/base.py b/framework/test/base.py
index d73dee9..e74ea3d 100644
--- a/framework/test/base.py
+++ b/framework/test/base.py
@@ -381,6 +381,18 @@ class Test(object):
         return not self == other
 
 
+class DummyTest(Test):
+    def __init__(self, name, result):
+        super(DummyTest, self).__init__([name])
+        self.result.result = result
+
+    def execute(self, path, log, options):
+        pass
+
+    def interpret_result(self):
+        pass
+
+
 class WindowResizeMixin(object):
     """ Mixin class that deals with spurious window resizes
 
-- 
2.9.4