[Piglit] [PATCH v3] framework: Add --ignore-missing option

[Dylan Baker]

Quoting Arkadiusz Hiler (2017-08-14 05:09:01)
> Currently, if a test from provided testlist fails to be discovered by
> the framework, piglit blows up with an exception.
> 
> This is both good - for consistency/early errors - and bad - for
> handling some CI/automation scenarios (e.g autobisecting the tests).
> 
> So let's keep the current default, but allow some flexibility with the
> new option that reports the missing tests as 'notrun'.
> 
> v2:
>  - change switch name to 'ignore', as skip is too suggestive
>  - use DummyTest to get 'notrun' result instead of warnings
> 
> v3: don't use OPTIONS
> 
> bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=99649
> Cc: Dylan Baker <dylan at pnwbakers.com>
> Cc: Tomi Sarvela <tomi.p.sarvela at intel.com>
> Cc: Martin Peres <martin.peres at intel.com>
> Signed-off-by: Arkadiusz Hiler <arkadiusz.hiler at intel.com>
> ---
>  framework/profile.py      | 10 +++++++---
>  framework/programs/run.py | 12 ++++++++++++
>  framework/test/base.py    | 12 ++++++++++++
>  3 files changed, 31 insertions(+), 3 deletions(-)
> 
> diff --git a/framework/profile.py b/framework/profile.py
> index a625318..cf0e298 100644
> --- a/framework/profile.py
> +++ b/framework/profile.py
> @@ -42,11 +42,11 @@ import re
>  
>  import six
>  
> -from framework import grouptools, exceptions
> +from framework import grouptools, exceptions, status
>  from framework.dmesg import get_dmesg
>  from framework.log import LogManager
>  from framework.monitoring import Monitoring
> -from framework.test.base import Test
> +from framework.test.base import Test, DummyTest
>  
>  __all__ = [
>      'RegexFilter',
> @@ -285,6 +285,7 @@ class TestProfile(object):
>          self.options = {
>              'dmesg': get_dmesg(False),
>              'monitor': Monitoring(False),
> +            'ignore_missing': False,
>          }
>  
>      def setup(self):
> @@ -314,7 +315,10 @@ class TestProfile(object):
>          if self.forced_test_list:
>              opts = collections.OrderedDict()
>              for n in self.forced_test_list:
> -                opts[n] = self.test_list[n]
> +                if self.options['ignore_missing'] and n not in self.test_list:
> +                    opts[n] = DummyTest(name=n, result=status.NOTRUN)

name and result are not keyword arguments, they're positional, so please write
it like this instead:
opts[n] = DummyTest(n, status.NOTRUN)

As long as this passes the unit test suite with the above changed,
Reviewed-by: Dylan Baker <dylan at pnwbakers.com>

Thanks for fixing that for me.

> +                else:
> +                    opts[n] = self.test_list[n]
>          else:
>              opts = self.test_list  # pylint: disable=redefined-variable-type
>  
> diff --git a/framework/programs/run.py b/framework/programs/run.py
> index 6444cfe..4524f17 100644
> --- a/framework/programs/run.py
> +++ b/framework/programs/run.py
> @@ -208,6 +208,10 @@ def _run_parser(input_):
>                               'isolation. This allows, but does not require, '
>                               'tests to run multiple tests per process. '
>                               'This value can also be set in piglit.conf.')
> +    parser.add_argument("--ignore-missing",
> +                        dest="ignore_missing",
> +                        action="store_true",
> +                        help="missing tests are considered as 'notrun'")
>      parser.add_argument("test_profile",
>                          metavar="<Profile path(s)>",
>                          nargs='+',
> @@ -234,6 +238,7 @@ def _create_metadata(args, name, forced_test_list):
>      if args.platform:
>          opts['platform'] = args.platform
>      opts['forced_test_list'] = forced_test_list
> +    opts['ignore_missing'] = args.ignore_missing
>  
>      metadata = {'options': opts}
>      metadata['name'] = name
> @@ -346,6 +351,10 @@ def run(input_):
>          for p in profiles:
>              p.options['monitor'] = monitoring.Monitoring(args.monitored)
>  
> +    if args.ignore_missing:
> +        for p in profiles:
> +            p.options['ignore_missing'] = args.ignore_missing
> +
>      for p in profiles:
>          if args.exclude_tests:
>              p.filters.append(profile.RegexFilter(args.exclude_tests,
> @@ -422,6 +431,9 @@ def resume(input_):
>              p.options['monitor'] = monitoring.Monitoring(
>                  results.options['monitoring'])
>  
> +        if results.options['ignore_missing']:
> +            p.options['ignore_missing'] = results.options['ignore_missing']
> +
>          if exclude_tests:
>              p.filters.append(lambda n, _: n not in exclude_tests)
>          if results.options['exclude_filter']:
> diff --git a/framework/test/base.py b/framework/test/base.py
> index d73dee9..e74ea3d 100644
> --- a/framework/test/base.py
> +++ b/framework/test/base.py
> @@ -381,6 +381,18 @@ class Test(object):
>          return not self == other
>  
>  
> +class DummyTest(Test):
> +    def __init__(self, name, result):
> +        super(DummyTest, self).__init__([name])
> +        self.result.result = result
> +
> +    def execute(self, path, log, options):
> +        pass
> +
> +    def interpret_result(self):
> +        pass
> +
> +
>  class WindowResizeMixin(object):
>      """ Mixin class that deals with spurious window resizes
>  
> -- 
> 2.9.4
> 
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 488 bytes
Desc: signature
URL: <https://lists.freedesktop.org/archives/piglit/attachments/20170814/e4fcdd92/attachment.sig>