[Piglit] [PATCH 3/4] framework/backends/junit.py: Add a writer class that handles subtests
Dylan Baker
dylan at pnwbakers.com
Fri Aug 5 00:02:41 UTC 2016
This class handles subtests by making a test with subtests into a
testsuite element, and then makes each subtest a testcase element.
---
framework/backends/junit.py | 128 +++++++++++++++++-----
unittests/framework/backends/test_junit.py | 97 +++++++++++++++++-
2 files changed, 197 insertions(+), 28 deletions(-)
diff --git a/framework/backends/junit.py b/framework/backends/junit.py
index d1b77d0..414b1df 100644
--- a/framework/backends/junit.py
+++ b/framework/backends/junit.py
@@ -32,7 +32,7 @@ except ImportError:
import six
-from framework import grouptools, results, status, exceptions
+from framework import grouptools, results, exceptions
from framework.core import PIGLIT_CONFIG
from .abstract import FileBackend
from .register import Registry
@@ -108,7 +108,11 @@ class JUnitWriter(object):
def _make_result(element, result, expected_result):
"""Adds the skipped, failure, or error element."""
res = None
- if result in ['fail', 'dmesg-warn', 'dmesg-fail']:
+
+ if result == 'incomplete':
+ res = etree.SubElement(element, 'failure',
+ message='Incomplete run.')
+ elif result in ['fail', 'dmesg-warn', 'dmesg-fail']:
if expected_result == "failure":
res = etree.SubElement(element, 'skipped',
message='expected failure')
@@ -139,9 +143,18 @@ class JUnitWriter(object):
if res is not None:
res.attrib['type'] = six.text_type(result)
- def __call__(self, f, name, data):
- classname, testname = self._make_names(name)
+ def _make_root(self, testname, classname, data):
+ """Creates and returns the root element."""
+ element = etree.Element('testcase',
+ name=self._make_full_test_name(testname),
+ classname=classname,
+ # Incomplete will not have a time.
+ time=str(data.time.total),
+ status=str(data.result))
+
+ return element
+ def _make_full_test_name(self, testname):
# Jenkins will display special pages when the test has certain names.
# https://jenkins-ci.org/issue/18062
# https://jenkins-ci.org/issue/19810
@@ -151,44 +164,99 @@ class JUnitWriter(object):
if full_test_name in _JUNIT_SPECIAL_NAMES:
testname += '_'
full_test_name = testname + self._test_suffix
+ return full_test_name
- # Create the root element
- element = etree.Element('testcase', name=full_test_name,
- classname=classname,
- # Incomplete will not have a time.
- time=str(data.time.total),
- status=str(data.result))
+ def _expected_result(self, name):
+ """Get the expected result of the test."""
+ name = name.replace("=", ".").replace(":", ".")
+ expected_result = "pass"
+
+ if name in self._expected_failures:
+ expected_result = "failure"
+ # a test can either fail or crash, but not both
+ assert name not in self._expected_crashes
+
+ if name in self._expected_crashes:
+ expected_result = "error"
+
+ return expected_result
+
+ def __call__(self, f, name, data):
+ classname, testname = self._make_names(name)
+ element = self._make_root(testname, classname, data)
+ expected_result = self._expected_result(
+ '{}.{}'.format(classname, testname).lower())
# If this is an incomplete status then none of these values will be
# available, nor
if data.result != 'incomplete':
- expected_result = "pass"
+ self._set_xml_err(element, data, expected_result)
+
+ # Add stdout
+ out = etree.SubElement(element, 'system-out')
+ out.text = data.out
- # replace special characters and make case insensitive
- lname = (classname + "." + testname).lower()
- lname = lname.replace("=", ".")
- lname = lname.replace(":", ".")
+ # Prepend command line to stdout
+ out.text = data.command + '\n' + out.text
- if lname in self._expected_failures:
- expected_result = "failure"
- # a test can either fail or crash, but not both
- assert lname not in self._expected_crashes
+ self._make_result(element, data.result, expected_result)
- if lname in self._expected_crashes:
- expected_result = "error"
+ f.write(six.text_type(etree.tostring(element).decode('utf-8')))
- self._set_xml_err(element, data, expected_result)
+
+class JUnitSubtestWriter(JUnitWriter):
+ """A JUnitWriter derived class that treats subtest at testsuites.
+
+ This class will turn a piglit test with subtests into a testsuite element
+ with each subtest as a testcase element. This subclass is needed because
+ not all JUnit readers (like the JUnit plugin for Jenkins) handle nested
+ testsuites correctly.
+ """
+
+ def _make_root(self, testname, classname, data):
+ if data.subtests:
+ element = etree.Element('testsuite',
+ name='{}.{}'.format(classname, testname),
+ time=str(data.time.total),
+ tests=six.text_type(len(data.subtests)))
+ for test, result in six.iteritems(data.subtests):
+ etree.SubElement(element,
+ 'testcase',
+ name=self._make_full_test_name(test),
+ status=six.text_type(result))
+
+ else:
+ element = super(JUnitSubtestWriter, self)._make_root(
+ testname, classname, data)
+ return element
+
+ def __call__(self, f, name, data):
+ classname, testname = self._make_names(name)
+ element = self._make_root(testname, classname, data)
+
+ # If this is an incomplete status then none of these values will be
+ # available, nor
+ if data.result != 'incomplete':
+ self._set_xml_err(element, data, 'pass')
# Add stdout
out = etree.SubElement(element, 'system-out')
out.text = data.out
-
# Prepend command line to stdout
out.text = data.command + '\n' + out.text
- self._make_result(element, data.result, expected_result)
+ for subname, result in six.iteritems(data.subtests):
+ subtest = self._make_full_test_name(subname)
+ # replace special characters and make case insensitive
+ elem = element.find('.//testcase[@name="{}"]'.format(subtest))
+ assert elem is not None
+ self._make_result(elem, result,
+ self._expected_result('{}.{}.{}'.format(
+ classname, testname, subtest).lower()))
else:
- etree.SubElement(element, 'failure', message='Incomplete run.')
+ self._make_result(element, data.result,
+ self._expected_result('{}.{}'.format(
+ classname, testname)))
f.write(six.text_type(etree.tostring(element).decode('utf-8')))
@@ -203,7 +271,7 @@ class JUnitBackend(FileBackend):
_file_extension = 'xml'
_write = None # this silences the abstract-not-subclassed warning
- def __init__(self, dest, junit_suffix='', **options):
+ def __init__(self, dest, junit_suffix='', junit_subtests=False, **options):
super(JUnitBackend, self).__init__(dest, **options)
# make dictionaries of all test names expected to crash/fail
@@ -218,8 +286,12 @@ class JUnitBackend(FileBackend):
for fail, _ in PIGLIT_CONFIG.items("expected-crashes"):
expected_crashes[fail.lower()] = True
- self._write = JUnitWriter(junit_suffix, expected_failures,
- expected_crashes)
+ if not junit_subtests:
+ self._write = JUnitWriter(
+ junit_suffix, expected_failures, expected_crashes)
+ else:
+ self._write = JUnitSubtestWriter( # pylint: disable=redefined-variable-type
+ junit_suffix, expected_failures, expected_crashes)
def initialize(self, metadata):
""" Do nothing
diff --git a/unittests/framework/backends/test_junit.py b/unittests/framework/backends/test_junit.py
index 8f09dac..fbd04b8 100644
--- a/unittests/framework/backends/test_junit.py
+++ b/unittests/framework/backends/test_junit.py
@@ -254,3 +254,100 @@ class TestJUnitWriter(object):
schema = etree.XMLSchema(file=JUNIT_SCHEMA) # pylint: disable=no-member
with open(test_file, 'r') as f:
assert schema.validate(etree.parse(f))
+
+
+class TestJUnitSubtestWriter(object):
+ """Tests for the JUnitWriter class."""
+
+ def test_junit_replace(self, tmpdir):
+ """backends.junit.JUnitBackend.write_test: grouptools.SEPARATOR is
+ replaced with '.'.
+ """
+ result = results.TestResult()
+ result.time.end = 1.2345
+ result.out = 'this is stdout'
+ result.err = 'this is stderr'
+ result.command = 'foo'
+ result.subtests['foo'] = 'pass'
+ result.subtests['bar'] = 'fail'
+
+ test = backends.junit.JUnitBackend(six.text_type(tmpdir),
+ junit_subtests=True)
+ test.initialize(shared.INITIAL_METADATA)
+ with test.write_test(grouptools.join('a', 'group', 'test1')) as t:
+ t(result)
+ test.finalize()
+
+ test_value = etree.parse(six.text_type(tmpdir.join('results.xml')))
+ test_value = test_value.getroot()
+
+ assert test_value.find('.//testsuite/testsuite').attrib['name'] == \
+ 'piglit.a.group.test1'
+
+ def test_junit_replace_suffix(self, tmpdir):
+ """backends.junit.JUnitBackend.write_test: grouptools.SEPARATOR is
+ replaced with '.'.
+ """
+ result = results.TestResult()
+ result.time.end = 1.2345
+ result.out = 'this is stdout'
+ result.err = 'this is stderr'
+ result.command = 'foo'
+ result.subtests['foo'] = 'pass'
+ result.subtests['bar'] = 'fail'
+
+ test = backends.junit.JUnitBackend(six.text_type(tmpdir),
+ junit_subtests=True,
+ junit_suffix='.foo')
+ test.initialize(shared.INITIAL_METADATA)
+ with test.write_test(grouptools.join('a', 'group', 'test1')) as t:
+ t(result)
+ test.finalize()
+
+ test_value = etree.parse(six.text_type(tmpdir.join('results.xml')))
+ test_value = test_value.getroot()
+
+ suite = test_value.find('.//testsuite/testsuite')
+ assert suite.attrib['name'] == 'piglit.a.group.test1'
+ assert suite.find('.//testcase[@name="{}"]'.format('foo.foo')) is not None
+
+ class TestValid(object):
+ @pytest.fixture
+ def test_file(self, tmpdir):
+ tmpdir.mkdir('foo')
+ p = tmpdir.join('foo')
+
+ result = results.TestResult()
+ result.time.end = 1.2345
+ result.out = 'this is stdout'
+ result.err = 'this is stderr'
+ result.command = 'foo'
+ result.pid = 1034
+ result.subtests['foo'] = 'pass'
+ result.subtests['bar'] = 'fail'
+
+ test = backends.junit.JUnitBackend(six.text_type(p),
+ junit_subtests=True)
+ test.initialize(shared.INITIAL_METADATA)
+ with test.write_test(grouptools.join('a', 'group', 'test1')) as t:
+ t(result)
+
+ result.result = 'fail'
+ with test.write_test(grouptools.join('a', 'test', 'test1')) as t:
+ t(result)
+ test.finalize()
+
+ return six.text_type(p.join('results.xml'))
+
+ def test_xml_well_formed(self, test_file):
+ """backends.junit.JUnitBackend.write_test: produces well formed xml."""
+ etree.parse(test_file)
+
+ @pytest.mark.skipif(etree.__name__ != 'lxml.etree',
+ reason="This test requires lxml")
+ def test_xml_valid(self, test_file):
+ """backends.junit.JUnitBackend.write_test: produces valid JUnit xml."""
+ # This XMLSchema class is unique to lxml
+ schema = etree.XMLSchema(file=JUNIT_SCHEMA) # pylint: disable=no-member
+ with open(test_file, 'r') as f:
+ assert schema.validate(etree.parse(f))
--
git-series 0.8.7
More information about the Piglit
mailing list