[Piglit] [PATCH] framework: keep track of running stats and display them

[Ilia Mirkin]

On Sat, Feb 15, 2014 at 4:45 AM, Dylan Baker <baker.dylan.c at gmail.com> wrote:
> On Saturday, February 15, 2014 02:28:11 AM Ilia Mirkin wrote:
>> This will display a line that looks like:
>> [00076/11064] fail: 1, pass: 55, skip: 19 Running Test(s): 00075
>>
>> Signed-off-by: Ilia Mirkin <imirkin at alum.mit.edu>
>> ---
>>  framework/core.py | 11 +++++++----
>>  framework/log.py  | 19 +++++++++++++++----
>>  2 files changed, 22 insertions(+), 8 deletions(-)
>>
>> diff --git a/framework/core.py b/framework/core.py
>> index ac25917..4bcaa82 100644
>> --- a/framework/core.py
>> +++ b/framework/core.py
>> @@ -435,6 +435,7 @@ class Test(object):
>>          '''
>>
>>          log_current = log.get_current()
>> +        counts = {}
>>          # Run the test
>>          if env.execute:
>>              try:
>> @@ -462,15 +463,17 @@ class Test(object):
>>                  result['traceback'] = \
>>                      "".join(traceback.format_tb(sys.exc_info()[2]))
>>
>> -            if 'subtest' in result and len(result['subtest'].keys()) > 1:
>> -                for test in result['subtest'].keys():
>> -                    result['result'] = result['subtest'][test]
>> +            if 'subtest' in result and len(result['subtest']) > 1:
>> +                for test in result['subtest']:
>> +                    res = result['result'] = result['subtest'][test]
>> +                    counts[res] = counts.get(res, 0) + 1
>>                      json_writer.write_dict_item(os.path.join(path, test),
>> result) else:
>> +                counts[result['result']] = 1
>>                  json_writer.write_dict_item(path, result)
>>          else:
>>              log.log()
>> -        log.mark_complete(log_current)
>> +        log.mark_complete(log_current, counts)
>>
>>
>>  class Group(dict):
>> diff --git a/framework/log.py b/framework/log.py
>> index 01c3a32..ba045da 100644
>> --- a/framework/log.py
>> +++ b/framework/log.py
>> @@ -37,22 +37,30 @@ class Log(object):
>>          self.__running = []
>>          self.__generator = (x for x in xrange(self.__total))
>>          self.__pad = len(str(self.__total))
>> +        self.__summary = {}
>> +
>> +    def _summary(self):
>> +        return ", ".join("{0}: {1}".format(k, self.__summary[k])
>> +                         for k in sorted(self.__summary))
>>
>>      def _running(self):
>>          return "Running Test(s): {}".format(
>>              " ".join([str(x).zfill(self.__pad) for x in self.__running]))
>>
>>      def _percent(self):
>> -        return "[{0}/{1}]".format(str(self.__complete).zfill(self.__pad),
>> -                                  str(self.__total).zfill(self.__pad))
>> +        return "[{0}/{1}] {2}".format(
>> +            str(self.__complete).zfill(self.__pad),
>> +            str(self.__total).zfill(self.__pad),
>> +            self._summary())
>>
>>      @synchronized_self
>> -    def mark_complete(self, value):
>> +    def mark_complete(self, value, counts):
>>          """ Used to mark a test as complete in the log
>>
>>          Arguments:
>>          value -- the test number to mark complete
>> -
>> +        counts -- a map from result to count of tests with that result
>> +
>>          """
>>          # Mark running complete
>>          assert value in self.__running
>> @@ -61,6 +69,9 @@ class Log(object):
>>          # increment the number of completed tests
>>          self.__complete += 1
>>
>> +        for k, v in counts.iteritems():
>> +            self.__summary[k] = self.__summary.get(k, 0) + v
>> +
>>      @synchronized_self
>>      def log(self):
>>          """ Print to the screen
>
> Keeping track of subtests is going to make all of the accounting in log
> inaccurate, correct? Since subtests have never been displayed by the logger
> (not this one or the previous one), counting them into your total will end up
> with a count greater than the total, or am I totally missing something?

Not missing anything. I can redo this to just add 1/test, rather than
counting the subtests. Of course if you can add in your head _that_
fast... oh, and piglit-summary-html, at least, shows data by subtest.

  -ilia