[Piglit] [PATCH 2/2] file backends write an incomplete status before running tests
Dylan Baker
baker.dylan.c at gmail.com
Wed Apr 15 17:07:30 PDT 2015
This sets the backends (both the json and junit) to write out an
incomplete status before starting the test. (for junit this is a 'fail'
with a message of 'incomplete', junit is a fairly limited format). When
the test is completed this incomplete status is overwritten with the
final status.
This patch also adds a new mode to the 'summary console' option, '-i',
which will list all tests with an incomplete status.
This will allow a developer to see what tests failed when a system
crashed, or what tests were running when they aborted piglit.
Signed-off-by: Dylan Baker <dylanx.c.baker at intel.com>
---
framework/backends/abstract.py | 60 ++++++++++++++++++++++++++++---
framework/backends/json.py | 12 +++----
framework/backends/junit.py | 36 ++++++++++---------
framework/profile.py | 5 +--
framework/programs/run.py | 7 ++--
framework/programs/summary.py | 17 ++++++---
framework/status.py | 2 +-
framework/summary.py | 40 +++++++++++++--------
framework/tests/json_backend_tests.py | 62 +++++++++++++++++++++++++++------
framework/tests/json_tests.py | 3 +-
framework/tests/junit_backends_tests.py | 34 +++++++-----------
11 files changed, 193 insertions(+), 85 deletions(-)
diff --git a/framework/backends/abstract.py b/framework/backends/abstract.py
index 928ba9a..47186f2 100644
--- a/framework/backends/abstract.py
+++ b/framework/backends/abstract.py
@@ -28,7 +28,12 @@ This module provides mixins and base classes for backend modules.
from __future__ import print_function, absolute_import
import os
import abc
+import shutil
import itertools
+import contextlib
+
+from framework.results import TestResult
+from framework.status import INCOMPLETE
class Backend(object):
@@ -100,11 +105,16 @@ class Backend(object):
"""
@abc.abstractmethod
- def write_test(self, name, data):
+ def write_test(self, name):
""" Write a test into the backend store
This method writes an actual test into the backend store.
+ Should be a context manager, used with the with statement. It should
+ first write an incomplete status value, then yield and object that will
+ overwrite that value with the final value. That object needs to take a
+ 'data' paramter whic is a result.TestResult object.
+
Arguments:
name -- the name of the test to be written
data -- a TestResult object representing the test data
@@ -133,15 +143,55 @@ class FileBackend(Backend):
"""
def __init__(self, dest, file_start_count=0, file_fsync=False, **kwargs):
self._dest = dest
- self._counter = itertools.count(file_start_count)
- self._file_sync = file_fsync
+ self.__counter = itertools.count(file_start_count)
+ self.__file_sync = file_fsync
- def _fsync(self, file_):
+ __INCOMPLETE = TestResult({'result': INCOMPLETE})
+
+ def __fsync(self, file_):
""" Sync the file to disk
If self._file_sync is truthy this will sync self._file to disk
"""
file_.flush()
- if self._file_sync:
+ if self.__file_sync:
os.fsync(file_.fileno())
+
+ @abc.abstractmethod
+ def _write(self, f, name, data):
+ """Method that writes a TestResult into a result file."""
+
+ @abc.abstractproperty
+ def _file_extension(self):
+ """The file extension of the backend."""
+
+ @contextlib.contextmanager
+ def write_test(self, name):
+ """Write a test.
+
+ When this context manager is opened it will first write a placeholder
+ file with the status incomplete.
+
+ When it is called to write the finall result it will create a temporary
+ file, write to that file, then move that file over the original,
+ incomplete status file. This helps to make the operation atomic, as
+ long as the filesystem continues running and the result was valid in
+ the original file it will be valid at the end
+
+ """
+ def finish(val):
+ tfile = file_ + '.tmp'
+ with open(tfile, 'w') as f:
+ self._write(f, name, val)
+ self.__fsync(f)
+ shutil.move(tfile, file_)
+
+ file_ = os.path.join(self._dest, 'tests', '{}.{}'.format(
+ self.__counter.next(), self._file_extension))
+
+ with open(file_, 'w') as f:
+ self._write(f, name, self.__INCOMPLETE)
+ self.__fsync(f)
+
+ yield finish
diff --git a/framework/backends/json.py b/framework/backends/json.py
index affd64e..9f138a6 100644
--- a/framework/backends/json.py
+++ b/framework/backends/json.py
@@ -74,6 +74,8 @@ class JSONBackend(FileBackend):
a file it just ignores it, making the result atomic.
"""
+ _file_extension = 'json'
+
def initialize(self, metadata):
""" Write boilerplate json code
@@ -141,13 +143,9 @@ class JSONBackend(FileBackend):
os.unlink(os.path.join(self._dest, 'metadata.json'))
shutil.rmtree(os.path.join(self._dest, 'tests'))
- def write_test(self, name, data):
- """ Write a test into the JSON tests dictionary """
- t = os.path.join(self._dest, 'tests',
- '{}.json'.format(self._counter.next()))
- with open(t, 'w') as f:
- json.dump({name: data}, f, default=piglit_encoder)
- self._fsync(f)
+ @staticmethod
+ def _write(f, name, data):
+ json.dump({name: data}, f, default=piglit_encoder)
def load_results(filename):
diff --git a/framework/backends/junit.py b/framework/backends/junit.py
index 3602f9e..21cb991 100644
--- a/framework/backends/junit.py
+++ b/framework/backends/junit.py
@@ -47,6 +47,7 @@ class JUnitBackend(FileBackend):
https://svn.jenkins-ci.org/trunk/hudson/dtkit/dtkit-format/dtkit-junit-model/src/main/resources/com/thalesgroup/dtkit/junit/model/xsd/junit-7.xsd
"""
+ _file_extension = 'xml'
def __init__(self, dest, junit_suffix='', **options):
super(JUnitBackend, self).__init__(dest, **options)
@@ -106,7 +107,7 @@ class JUnitBackend(FileBackend):
shutil.rmtree(os.path.join(self._dest, 'tests'))
- def write_test(self, name, data):
+ def _write(self, f, name, data):
def calculate_result():
"""Set the result."""
@@ -184,27 +185,28 @@ class JUnitBackend(FileBackend):
# Create the root element
element = etree.Element('testcase', name=full_test_name,
classname=classname,
- time=str(data['time']),
+ # Incomplete will not have a time.
+ time=str(data.get('time')),
status=str(data['result']))
- # Add stdout
- out = etree.SubElement(element, 'system-out')
- out.text = data['out']
+ # If this is an incomplete status then none of these values will be
+ # available, nor
+ if data['result'] != 'incomplete':
+ # Add stdout
+ out = etree.SubElement(element, 'system-out')
+ out.text = data['out']
- # Prepend command line to stdout
- out.text = data['command'] + '\n' + out.text
+ # Prepend command line to stdout
+ out.text = data['command'] + '\n' + out.text
- # Add stderr
- err = etree.SubElement(element, 'system-err')
- err.text = data['err']
+ # Add stderr
+ err = etree.SubElement(element, 'system-err')
+ err.text = data['err']
+ calculate_result()
+ else:
+ etree.SubElement(element, 'failure', message='Incomplete run.')
- calculate_result()
-
- t = os.path.join(self._dest, 'tests',
- '{}.xml'.format(self._counter.next()))
- with open(t, 'w') as f:
- f.write(etree.tostring(element))
- self._fsync(f)
+ f.write(etree.tostring(element))
def _load(results_file):
diff --git a/framework/profile.py b/framework/profile.py
index d5d140c..46a979c 100644
--- a/framework/profile.py
+++ b/framework/profile.py
@@ -276,8 +276,9 @@ class TestProfile(object):
"""
name, test = pair
- test.execute(name, log.get(), self.dmesg)
- backend.write_test(name, test.result)
+ with backend.write_test(name) as w:
+ test.execute(name, log.get(), self.dmesg)
+ w(test.result)
def run_threads(pool, testlist):
""" Open a pool, close it, and join it """
diff --git a/framework/programs/run.py b/framework/programs/run.py
index ca94596..6053074 100644
--- a/framework/programs/run.py
+++ b/framework/programs/run.py
@@ -339,8 +339,11 @@ def resume(input_):
file_start_count=len(results.tests) + 1)
# Specifically do not initialize again, everything initialize does is done.
- for name in results.tests.iterkeys():
- opts.exclude_tests.add(name)
+ # Don't re-run tests that have already completed, incomplete status tests
+ # have obviously not completed.
+ for name, result in results.tests.iteritems():
+ if result['result'] != 'incomplete':
+ opts.exclude_tests.add(name)
profile = framework.profile.merge_test_profiles(results.options['profile'])
profile.results_dir = args.results_path
diff --git a/framework/programs/summary.py b/framework/programs/summary.py
index b012a48..c839930 100644
--- a/framework/programs/summary.py
+++ b/framework/programs/summary.py
@@ -108,13 +108,22 @@ def console(input_):
# and then call for only summary
excGroup1 = parser.add_mutually_exclusive_group()
excGroup1.add_argument("-d", "--diff",
- action="store_true",
+ action="store_const",
+ const="diff",
+ dest='mode',
help="Only display the differences between multiple "
"result files")
excGroup1.add_argument("-s", "--summary",
- action="store_true",
+ action="store_const",
+ const="summary",
+ dest='mode',
help="Only display the summary, not the individual "
"test results")
+ excGroup1.add_argument("-i", "--incomplete",
+ action="store_const",
+ const="incomplete",
+ dest='mode',
+ help="Only display tests that are incomplete.")
parser.add_argument("-l", "--list",
action="store",
help="Use test results from a list file")
@@ -127,7 +136,7 @@ def console(input_):
# Throw an error if -d/--diff is called, but only one results file is
# provided
- if args.diff and len(args.results) < 2:
+ if args.mode == 'diff' and len(args.results) < 2:
parser.error('-d/--diff cannot be specified unless two or more '
'results files are specified')
@@ -137,7 +146,7 @@ def console(input_):
# Generate the output
output = summary.Summary(args.results)
- output.generate_text(args.diff, args.summary)
+ output.generate_text(args.mode)
def csv(input_):
diff --git a/framework/status.py b/framework/status.py
index 90f69fb..8d649f2 100644
--- a/framework/status.py
+++ b/framework/status.py
@@ -183,7 +183,7 @@ class Status(object):
raise TypeError("Cannot compare type: {}".format(type(other)))
def __ne__(self, other):
- return self.fraction != other.fraction or int(self) != int(other)
+ return not self == other
def __ge__(self, other):
return self.fraction[1] > other.fraction[1] or (
diff --git a/framework/summary.py b/framework/summary.py
index 9b30b5e..66f100a 100644
--- a/framework/summary.py
+++ b/framework/summary.py
@@ -310,7 +310,7 @@ class Summary:
self.totals = {}
self.tests = {'all': set(), 'changes': set(), 'problems': set(),
'skipped': set(), 'regressions': set(), 'fixes': set(),
- 'enabled': set(), 'disabled': set()}
+ 'enabled': set(), 'disabled': set(), 'incomplete': set()}
def fgh(test, result):
""" Helper for updating the fractions and status lists """
@@ -411,6 +411,9 @@ class Summary:
if so.SKIP in status:
self.tests['skipped'].add(test)
+ if so.INCOMPLETE in status:
+ self.tests['incomplete'].add(test)
+
# find fixes, regressions, and changes
for i in xrange(len(status) - 1):
first = status[i]
@@ -437,7 +440,7 @@ class Summary:
"""
self.totals = {'pass': 0, 'fail': 0, 'crash': 0, 'skip': 0,
'timeout': 0, 'warn': 0, 'dmesg-warn': 0,
- 'dmesg-fail': 0}
+ 'dmesg-fail': 0, 'incomplete': 0,}
for test in results.tests.itervalues():
self.totals[str(test['result'])] += 1
@@ -555,22 +558,30 @@ class Summary:
else:
out.write(empty_status.render(page=page, pages=pages))
- def generate_text(self, diff, summary):
+ def generate_text(self, mode):
""" Write summary information to the console """
+ assert mode in ['summary', 'diff', 'incomplete']
self.__find_totals(self.results[-1])
# Print the name of the test and the status from each test run
- if not summary:
- if diff:
- for test in self.tests['changes']:
- print("%(test)s: %(statuses)s" % {'test': test, 'statuses':
- ' '.join([str(i.tests.get(test, {'result': so.SKIP})
- ['result']) for i in self.results])})
- else:
- for test in self.tests['all']:
- print("%(test)s: %(statuses)s" % {'test': test, 'statuses':
- ' '.join([str(i.tests.get(test, {'result': so.SKIP})
- ['result']) for i in self.results])})
+ if mode == 'diff':
+ for test in self.tests['changes']:
+ print("{test}: {statuses}".format(
+ test=test,
+ statuses=' '.join(str(i.tests.get(test, {'result': so.SKIP})
+ ['result']) for i in self.results)))
+ elif mode == 'incomplete':
+ for test in self.tests['incomplete']:
+ print("{test}: {statuses}".format(
+ test=test,
+ statuses=' '.join(str(i.tests.get(test, {'result': so.SKIP})
+ ['result']) for i in self.results)))
+ elif mode != 'summary':
+ for test in self.tests['all']:
+ print("{test}: {statuses}".format(
+ test=test,
+ statuses=' '.join(str(i.tests.get(test, {'result': so.SKIP})
+ ['result']) for i in self.results)))
# Print the summary
print("summary:\n"
@@ -580,6 +591,7 @@ class Summary:
" skip: {skip}\n"
" timeout: {timeout}\n"
" warn: {warn}\n"
+ " incomplete: {incomplete}\n"
" dmesg-warn: {dmesg-warn}\n"
" dmesg-fail: {dmesg-fail}".format(**self.totals))
if self.tests['changes']:
diff --git a/framework/tests/json_backend_tests.py b/framework/tests/json_backend_tests.py
index 45626e8..3c251ae 100644
--- a/framework/tests/json_backend_tests.py
+++ b/framework/tests/json_backend_tests.py
@@ -87,7 +87,8 @@ class TestJSONTestMethod(utils.StaticDirectory):
super(TestJSONTestMethod, cls).setup_class()
test = backends.json.JSONBackend(cls.tdir)
test.initialize(BACKEND_INITIAL_META)
- test.write_test(cls.test_name, cls.result)
+ with test.write_test(cls.test_name) as t:
+ t(cls.result)
def test_write_test(self):
""" JSONBackend.write_test() adds tests to a 'tests' directory """
@@ -122,7 +123,8 @@ class TestJSONTestFinalize(utils.StaticDirectory):
super(TestJSONTestFinalize, cls).setup_class()
test = backends.json.JSONBackend(cls.tdir)
test.initialize(BACKEND_INITIAL_META)
- test.write_test(cls.test_name, cls.result)
+ with test.write_test(cls.test_name) as t:
+ t(cls.result)
test.finalize()
def test_remove_metadata(self):
@@ -205,9 +207,12 @@ def test_resume_load():
with utils.tempdir() as f:
backend = backends.json.JSONBackend(f)
backend.initialize(BACKEND_INITIAL_META)
- backend.write_test("group1/test1", {'result': 'fail'})
- backend.write_test("group1/test2", {'result': 'pass'})
- backend.write_test("group2/test3", {'result': 'fail'})
+ with backend.write_test("group1/test1") as t:
+ t({'result': 'fail'})
+ with backend.write_test("group1/test2") as t:
+ t({'result': 'pass'})
+ with backend.write_test("group2/test3") as t:
+ t({'result': 'fail'})
backends.json._resume(f)
@@ -217,9 +222,12 @@ def test_resume_load_valid():
with utils.tempdir() as f:
backend = backends.json.JSONBackend(f)
backend.initialize(BACKEND_INITIAL_META)
- backend.write_test("group1/test1", {'result': 'fail'})
- backend.write_test("group1/test2", {'result': 'pass'})
- backend.write_test("group2/test3", {'result': 'fail'})
+ with backend.write_test("group1/test1") as t:
+ t({'result': 'fail'})
+ with backend.write_test("group1/test2") as t:
+ t({'result': 'pass'})
+ with backend.write_test("group2/test3") as t:
+ t({'result': 'fail'})
test = backends.json._resume(f)
@@ -234,9 +242,12 @@ def test_resume_load_invalid():
with utils.tempdir() as f:
backend = backends.json.JSONBackend(f)
backend.initialize(BACKEND_INITIAL_META)
- backend.write_test("group1/test1", {'result': 'fail'})
- backend.write_test("group1/test2", {'result': 'pass'})
- backend.write_test("group2/test3", {'result': 'fail'})
+ with backend.write_test("group1/test1") as t:
+ t({'result': 'fail'})
+ with backend.write_test("group1/test2") as t:
+ t({'result': 'pass'})
+ with backend.write_test("group2/test3") as t:
+ t({'result': 'fail'})
with open(os.path.join(f, 'tests', 'x.json'), 'w') as w:
w.write('foo')
@@ -248,6 +259,35 @@ def test_resume_load_invalid():
)
+def test_resume_load_incomplete():
+ """backends.json._resume: loads incomplete results.
+
+ Because resume, aggregate, and summary all use the function called _resume
+ we can't remove incomplete tests here. It's probably worth doing a refactor
+ to split some code out and allow this to be done in the resume path.
+
+ """
+ with utils.tempdir() as f:
+ backend = backends.json.JSONBackend(f)
+ backend.initialize(BACKEND_INITIAL_META)
+ with backend.write_test("group1/test1") as t:
+ t({'result': 'fail'})
+ with backend.write_test("group1/test2") as t:
+ t({'result': 'pass'})
+ with backend.write_test("group2/test3") as t:
+ t({'result': 'crash'})
+ with backend.write_test("group2/test4") as t:
+ t({'result': 'incomplete'})
+
+ test = backends.json._resume(f)
+
+ nt.assert_set_equal(
+ set(test.tests.keys()),
+ set(['group1/test1', 'group1/test2', 'group2/test3',
+ 'group2/test4']),
+ )
+
+
@utils.no_error
def test_load_results_folder_as_main():
""" Test that load_results takes a folder with a file named main in it """
diff --git a/framework/tests/json_tests.py b/framework/tests/json_tests.py
index 70a501a..d6ca0bf 100644
--- a/framework/tests/json_tests.py
+++ b/framework/tests/json_tests.py
@@ -63,7 +63,8 @@ class TestJsonOutput(utils.StaticDirectory):
backend = JSONBackend(cls.tdir, file_fsync=True)
backend.initialize(_create_metadata(args, 'test', core.Options()))
- backend.write_test('result', {'result': 'pass'})
+ with backend.write_test('result') as t:
+ t({'result': 'pass'})
backend.finalize({'time_elapsed': 1.22})
with open(os.path.join(cls.tdir, 'results.json'), 'r') as f:
cls.json = json.load(f)
diff --git a/framework/tests/junit_backends_tests.py b/framework/tests/junit_backends_tests.py
index f42f707..249bf49 100644
--- a/framework/tests/junit_backends_tests.py
+++ b/framework/tests/junit_backends_tests.py
@@ -83,9 +83,8 @@ class TestJUnitSingleTest(TestJunitNoTests):
cls.test_file = os.path.join(cls.tdir, 'results.xml')
test = backends.junit.JUnitBackend(cls.tdir)
test.initialize(BACKEND_INITIAL_META)
- test.write_test(
- grouptools.join('a', 'test', 'group', 'test1'),
- results.TestResult({
+ with test.write_test(grouptools.join('a', 'test', 'group', 'test1')) as t:
+ t(results.TestResult({
'time': 1.2345,
'result': 'pass',
'out': 'this is stdout',
@@ -115,26 +114,22 @@ class TestJUnitMultiTest(TestJUnitSingleTest):
cls.test_file = os.path.join(cls.tdir, 'results.xml')
test = backends.junit.JUnitBackend(cls.tdir)
test.initialize(BACKEND_INITIAL_META)
- test.write_test(
- grouptools.join('a', 'test', 'group', 'test1'),
- results.TestResult({
+ with test.write_test(grouptools.join('a', 'test', 'group', 'test1')) as t:
+ t(results.TestResult({
'time': 1.2345,
'result': 'pass',
'out': 'this is stdout',
'err': 'this is stderr',
'command': 'foo',
- })
- )
- test.write_test(
- 'a/different/test/group/test2',
- results.TestResult({
+ }))
+ with test.write_test('a/different/test/group/test2') as t:
+ t(results.TestResult({
'time': 1.2345,
'result': 'fail',
'out': 'this is stdout',
'err': 'this is stderr',
'command': 'foo',
- })
- )
+ }))
test.finalize()
def test_xml_well_formed(self):
@@ -152,16 +147,14 @@ def test_junit_replace():
with utils.tempdir() as tdir:
test = backends.junit.JUnitBackend(tdir)
test.initialize(BACKEND_INITIAL_META)
- test.write_test(
- grouptools.join('a', 'test', 'group', 'test1'),
- results.TestResult({
+ with test.write_test(grouptools.join('a', 'test', 'group', 'test1')) as t:
+ t(results.TestResult({
'time': 1.2345,
'result': 'pass',
'out': 'this is stdout',
'err': 'this is stderr',
'command': 'foo',
- })
- )
+ }))
test.finalize()
test_value = etree.parse(os.path.join(tdir, 'results.xml')).getroot()
@@ -175,9 +168,8 @@ def test_junit_skips_bad_tests():
with utils.tempdir() as tdir:
test = backends.junit.JUnitBackend(tdir)
test.initialize(BACKEND_INITIAL_META)
- test.write_test(
- grouptools.join('a', 'test', 'group', 'test1'),
- results.TestResult({
+ with test.write_test(grouptools.join('a', 'test', 'group', 'test1')) as t:
+ t(results.TestResult({
'time': 1.2345,
'result': 'pass',
'out': 'this is stdout',
--
2.3.5
More information about the Piglit
mailing list