[Piglit] [PATCH 2/4] framework/log.py: Make self.__summary keys explicitly

[Ilia Mirkin]

On Mon, Feb 17, 2014 at 2:45 PM, Dylan Baker <baker.dylan.c at gmail.com> wrote:
> It would be an unwelcome surprise if some test returns 'fails' or 'pas',
> so rather than allowing such a thing to happen, assert that the result
> is actually viable.
>
> Signed-off-by: Dylan Baker <baker.dylan.c at gmail.com>
> ---
>  framework/core.py            |  2 +-
>  framework/log.py             | 10 ++++--
>  framework/tests/log_tests.py | 74 +++++++++++++++++++++++++++++++++++++++++---
>  3 files changed, 77 insertions(+), 9 deletions(-)
>
> diff --git a/framework/core.py b/framework/core.py
> index fc59e3c..f6ae80f 100644
> --- a/framework/core.py
> +++ b/framework/core.py
> @@ -435,7 +435,6 @@ class Test(object):
>          '''
>
>          log_current = log.get_current()
> -        test_result = None
>          # Run the test
>          if env.execute:
>              try:
> @@ -471,6 +470,7 @@ class Test(object):
>              else:
>                  json_writer.write_dict_item(path, result)
>          else:
> +            test_result = 'dry-run'
>              log.log()
>          log.mark_complete(log_current, test_result)
>
> diff --git a/framework/log.py b/framework/log.py
> index 25ecdf5..7cdc940 100644
> --- a/framework/log.py
> +++ b/framework/log.py
> @@ -21,6 +21,7 @@
>  #
>
>  import sys
> +import collections
>  from .threads import synchronized_self
>
>
> @@ -37,7 +38,9 @@ class Log(object):
>          self.__running = []
>          self.__generator = (x for x in xrange(self.__total))
>          self.__pad = len(str(self.__total))
> -        self.__summary = {}
> +        self.__summary_keys = set(['pass', 'fail', 'warn', 'crash', 'skip',
> +                                   'dmesg-warn', 'dmesg-fail', 'dry-run'])
> +        self.__summary = collections.defaultdict(lambda: 0)
>
>      def _summary(self):
>          return ", ".join("{0}: {1}".format(k, self.__summary[k])
> @@ -67,8 +70,8 @@ class Log(object):
>          # increment the number of completed tests
>          self.__complete += 1
>
> -        if result:
> -            self.__summary[result] = self.__summary.get(result, 0) + 1
> +        assert result in self.__summary_keys
> +        self.__summary[result] += 1
>
>      @synchronized_self
>      def log(self):
> @@ -80,6 +83,7 @@ class Log(object):
>          """
>          sys.stdout.write("{0} {1} {2}\r".format(
>              self._percent(), self._summary(), self._running()))
> +
>          # Need to flush explicitly, otherwise it all gets buffered without a
>          # newline.
>          sys.stdout.flush()
> diff --git a/framework/tests/log_tests.py b/framework/tests/log_tests.py
> index 5f0640f..09d99e4 100644
> --- a/framework/tests/log_tests.py
> +++ b/framework/tests/log_tests.py
> @@ -20,10 +20,14 @@
>
>  """ Module provides tests for log.py module """
>
> +import sys
> +import itertools
>  from types import *  # This is a special * safe module
>  import nose.tools as nt
>  from framework.log import Log
>
> +valid_statuses = ('pass', 'fail', 'crash', 'warn', 'dmesg-warn',
> +                  'dmesg-fail', 'skip', 'dry-run')
>
>  def test_initialize_log():
>      """ Test that Log initializes with """
> @@ -63,11 +67,6 @@ def check_mark_complete_increment_summary(stat):
>
>  def test_mark_complete_increment_summary():
>      """ Generator that creates tests for self.__summary """
> -
> -
> -    valid_statuses = ('pass', 'fail', 'crash', 'warn', 'dmesg-warn',
> -                      'dmesg-fail', 'skip')
> -
>      yieldable = check_mark_complete_increment_summary
>
>      for stat in valid_statuses:
> @@ -83,3 +82,68 @@ def test_mark_complete_removes_complete():
>      log.mark_complete(ret, 'pass')
>      nt.assert_not_in(ret, log._Log__running,
>                       msg="Running tests not removed from running list")
> +
> +
> + at nt.raises(AssertionError)
> +def check_mark_complete_incrment_summary_bad():
> +    log = Log(100)
> +    ret = log.get_current()
> +    log.mark_complete(ret, 'fails')
> +
> +
> +def check_output(stat):
> +    """ Test that dry output returns the expected values """
> +    log = Log(100)
> +
> +    for x in stat:
> +        ret = log.get_current()
> +        log.mark_complete(ret, x)
> +
> +    # Print to stdout, and then capture it, the capture absolutely needs to
> +    # imediatly after log.log()
> +    log.log()
> +    output = sys.stdout.getvalue().strip()  # sys.stdout is a StringIO in nose
> +
> +    # Create a count of the statuses in alphabetical order. What we need to end
> +    # up with is something like: 'crash': 2, 'skip': 1
> +    s = []
> +    for x in sorted(set(stat)):
> +        s.append("{0}: {1}".format(x, stat.count(x)))
> +
> +    expected = '[{0}/100] {1} Running Test(s):'.format(
> +            str(len(stat)).zfill(3),
> +            ", ".join(s))

This is a dangerous sort of test. Any change in the output will make
it fail, and I can pretty much guarantee that people won't run the
tests. I wonder if it'd be better to test for the bits you expect,
e.g. n/y and that each status: val is in there?

> +
> +    print expected
> +    nt.assert_equal(output, expected,
> +                    msg="Did not return execpted output: " + expected)

expected. (dyslexics, untie!)

> +
> +
> +def gen_test_output():
> +    """ Generate a set of tests for checking __summary output
> +
> +    This generates severl hundered tests
> +
> +    """
> +    yieldable = check_output
> +
> +    # test dry run seperately, we don't want to test it mixed in with other
> +    # statuses, since it shouldn't be mixed
> +    yieldable.description = "Test output for status combinations: dry-run"
> +    yield yieldable, ['dry-run']
> +
> +    yieldable.description = "Test output for status combinations: dry-run dry-run"
> +    yield yieldable, ['dry-run', 'dry-run']
> +
> +    yieldable.description = "Test output for status combinations: dry-run dry-run dry-run"
> +    yield yieldable, ['dry-run', 'dry-run', 'dry-run']
> +
> +    # All combinations except dry-run up to 3 in length. We could generate
> +    # more, but it quickly becomes overwhelming, and the usefulness is
> +    # uncertain
> +    for r in xrange(1, 4):
> +        for stat in itertools.product(valid_statuses[:-1],
> +                                      repeat=r):
> +            yieldable.description = "Test output for status: " + \
> +                    " ".join([str(x) for x in stat])

The [ ] are unnecessary. Generator expressions have been around since
2.4, iirc. (With the additional benefit that x won't leak out into the
function scope... not that one would really care.)

> +            yield yieldable, stat
> --
> 1.8.5.5
>