[Piglit] [PATCH 17/19] framework: Add a JUnit backend
Dylan Baker
baker.dylan.c at gmail.com
Thu Aug 28 15:35:45 PDT 2014
This adds a mostly compliant JUnit backend. It currently passes all of
its tests and jenkins happily consumes the xml we produce.
This needs to have some refactoring done in profile to make it 100%
compliant with the junit-7.xsd from upstream jenkins, (although I would
be comfortable for now pushing with the known non-optimal behavior if
Jenkins will accept it), because JUnit expects to be given the number of
tests for the initial metadata block, but we have no way to calculate
that number until after the run has started. This is because of the
flattening pass in profile that flattens the nested directory structure
into a flat dictionary.
There are two options to solve this problem:
1) Flatten all.py and other modules. This is a lot of work and I have
many work-in-progress branches to do just hat
2) Push the pass out to a public method and call it ahead of time. This
seems really hacky to me, and I'd rather not do something that ugly.
Currently this patch just passes 0 for the test count unconditionally,
jenkins does not seem to have a problem with this.
This includes JUnit.xsd from the jenkins svn repository for piglit
framework unit testing. This is only used in the piglit python framework
unit tests.
Signed-off-by: Dylan Baker <dylanx.c.baker at intel.com>
---
framework/programs/run.py | 4 ++
framework/results.py | 65 ++++++++++++++++++++++-
framework/tests/results_tests.py | 91 ++++++++++++++++++++++++++++++++
framework/tests/schema/junit-7.xsd | 104 +++++++++++++++++++++++++++++++++++++
framework/tests/utils.py | 22 ++++++++
5 files changed, 285 insertions(+), 1 deletion(-)
create mode 100644 framework/tests/schema/junit-7.xsd
diff --git a/framework/programs/run.py b/framework/programs/run.py
index 8b7045d..82fc797 100644
--- a/framework/programs/run.py
+++ b/framework/programs/run.py
@@ -221,6 +221,10 @@ def run(input_):
options['platform'] = args.platform
options['name'] = results.name
options['env'] = core.collect_system_info()
+ # FIXME: this should be the actual count, but profile needs to be
+ # refactored to make that possible because of the flattening pass that is
+ # part of profile.run
+ options['test_count'] = 0
# Begin json.
backend = framework.results.get_backend(args.backend)(
diff --git a/framework/results.py b/framework/results.py
index 741adc9..eabeb4d 100644
--- a/framework/results.py
+++ b/framework/results.py
@@ -26,11 +26,16 @@ import os
import sys
import abc
import threading
+import posixpath
from cStringIO import StringIO
try:
import simplejson as json
except ImportError:
import json
+try:
+ from lxml import etree
+except ImportError:
+ import xml.etree.cElementTree as etree
import framework.status as status
@@ -43,7 +48,7 @@ __all__ = [
]
# A list of available backends
-BACKENDS = ['json']
+BACKENDS = ['json', 'junit']
# The current version of the JSON results
CURRENT_JSON_VERSION = 1
@@ -349,6 +354,63 @@ class JSONBackend(Backend):
self._write_dict_item(name, data)
+class JUnitBackend(Backend):
+ """ Backend that produces ANT JUnit XML
+
+ Based on the following schema:
+ https://svn.jenkins-ci.org/trunk/hudson/dtkit/dtkit-format/dtkit-junit-model/src/main/resources/com/thalesgroup/dtkit/junit/model/xsd/junit-7.xsd
+
+ """
+ # TODO: add fsync support
+
+ def __init__(self, dest, metadata, **options):
+ self._file = open(os.path.join(dest, 'results.xml'), 'w')
+
+ # Write initial headers and other data that etree cannot write for us
+ self._file.write('<?xml version="1.0" encoding="UTF-8" ?>\n')
+ self._file.write('<testsuites>\n')
+ self._file.write(
+ '<testsuite name="piglit" tests="{}">\n'.format(
+ metadata['test_count']))
+
+ def finalize(self, metadata=None):
+ self._file.write('</testsuite>\n')
+ self._file.write('</testsuites>\n')
+ self._file.close()
+
+ def write_test(self, name, data):
+ # Split the name of the test and the group (what junit refers to as
+ # classname), and replace piglits '/' seperated groups with '.', after
+ # replacing any '.' with '_' (so we don't get false groups)
+ classname, testname = posixpath.split(name)
+ assert classname
+ assert testname
+ classname = classname.replace('.', '_').replace('/', '.')
+ element = etree.Element('testcase', name=testname, classname=classname,
+ time=str(data['time']),
+ status=str(data['result']))
+
+ # Add stdout
+ out = etree.SubElement(element, 'system-out')
+ out.text = data['out']
+
+ # Add stderr
+ err = etree.SubElement(element, 'system-err')
+ err.text = data['err']
+
+ # Add relavent result value, if the result is pass then it doesn't need
+ # one of these statuses
+ if data['result'] == 'skip':
+ etree.SubElement(element, 'skipped')
+ elif data['result'] in ['warn', 'fail', 'dmesg-warn', 'dmesg-fail']:
+ etree.SubElement(element, 'failure')
+ elif data['result'] == 'crash':
+ etree.SubElement(element, 'error')
+
+ self._file.write(etree.tostring(element))
+ self._file.write('\n')
+
+
class TestResult(dict):
def __init__(self, *args):
super(TestResult, self).__init__(*args)
@@ -587,6 +649,7 @@ def get_backend(backend):
""" Returns a BackendInstance based on the string passed """
backends = {
'json': JSONBackend,
+ 'junit': JUnitBackend,
}
# Be sure that we're exporting the same list of backends that we actually
diff --git a/framework/tests/results_tests.py b/framework/tests/results_tests.py
index f11a86c..df89126 100644
--- a/framework/tests/results_tests.py
+++ b/framework/tests/results_tests.py
@@ -23,6 +23,10 @@
import os
import json
+try:
+ from lxml import etree
+except ImportError:
+ import xml.etree.cElementTree as etree
import nose.tools as nt
import framework.tests.utils as utils
import framework.results as results
@@ -32,8 +36,11 @@ import framework.status as status
BACKEND_INITIAL_META = {
'name': 'name',
'env': {},
+ 'test_count': 0,
}
+JUNIT_SCHEMA = 'framework/tests/schema/junit-7.xsd'
+
def check_initialize(target):
""" Check that a class initializes without error """
@@ -181,3 +188,87 @@ def test_get_backend():
for name, inst in backends.iteritems():
check.description = 'get_backend({0}) returns {0} backend'.format(name)
yield check, name, inst
+
+
+class TestJunitNoTests(utils.StaticDirectory):
+ @classmethod
+ def setup_class(cls):
+ super(TestJunitNoTests, cls).setup_class()
+ test = results.JUnitBackend(cls.tdir, BACKEND_INITIAL_META)
+ test.finalize()
+ cls.test_file = os.path.join(cls.tdir, 'results.xml')
+
+ def test_xml_well_formed(self):
+ """ JUnitBackend.__init__ and finalize produce well formed xml
+
+ While it will produce valid XML, it cannot produc valid JUnit, since
+ JUnit requires at least one test to be valid
+
+ """
+ try:
+ etree.parse(self.test_file)
+ except Exception as e:
+ raise AssertionError(e)
+
+
+class TestJUnitSingleTest(TestJunitNoTests):
+ @classmethod
+ def setup_class(cls):
+ super(TestJUnitSingleTest, cls).setup_class()
+ cls.test_file = os.path.join(cls.tdir, 'results.xml')
+ test = results.JUnitBackend(cls.tdir, BACKEND_INITIAL_META)
+ test.write_test(
+ 'a/test/group/test1',
+ results.TestResult({
+ 'time': 1.2345,
+ 'result': 'pass',
+ 'out': 'this is stdout',
+ 'err': 'this is stderr',
+ })
+ )
+ test.finalize()
+
+ def test_xml_well_formed(self):
+ """ JUnitBackend.write_test() (once) produces well formed xml """
+ super(TestJUnitSingleTest, self).test_xml_well_formed()
+
+ def test_xml_valid(self):
+ """ JUnitBackend.write_test() (once) produces valid xml """
+ schema = etree.XMLSchema(file=JUNIT_SCHEMA)
+ with open(self.test_file, 'r') as f:
+ assert schema.validate(etree.parse(f)), 'xml is not valid'
+
+
+class TestJUnitMultiTest(TestJUnitSingleTest):
+ @classmethod
+ def setup_class(cls):
+ super(TestJUnitMultiTest, cls).setup_class()
+ cls.test_file = os.path.join(cls.tdir, 'results.xml')
+ test = results.JUnitBackend(cls.tdir, BACKEND_INITIAL_META)
+ test.write_test(
+ 'a/test/group/test1',
+ results.TestResult({
+ 'time': 1.2345,
+ 'result': 'pass',
+ 'out': 'this is stdout',
+ 'err': 'this is stderr',
+ })
+ )
+ test.write_test(
+ 'a/different/test/group/test2',
+ results.TestResult({
+ 'time': 1.2345,
+ 'result': 'fail',
+ 'out': 'this is stdout',
+ 'err': 'this is stderr',
+ })
+ )
+ test.finalize()
+
+ def test_xml_well_formed(self):
+ """ JUnitBackend.write_test() (twice) produces well formed xml """
+ super(TestJUnitMultiTest, self).test_xml_well_formed()
+
+ def test_xml_valid(self):
+ """ JUnitBackend.write_test() (twice) produces valid xml """
+ super(TestJUnitMultiTest, self).test_xml_valid()
diff --git a/framework/tests/schema/junit-7.xsd b/framework/tests/schema/junit-7.xsd
new file mode 100644
index 0000000..bc07b52
--- /dev/null
+++ b/framework/tests/schema/junit-7.xsd
@@ -0,0 +1,104 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema">
+
+ <xs:element name="failure">
+ <xs:complexType mixed="true">
+ <xs:attribute name="type" type="xs:string" use="optional"/>
+ <xs:attribute name="message" type="xs:string" use="optional"/>
+ </xs:complexType>
+ </xs:element>
+
+ <xs:element name="error">
+ <xs:complexType mixed="true">
+ <xs:attribute name="type" type="xs:string" use="optional"/>
+ <xs:attribute name="message" type="xs:string" use="optional"/>
+ </xs:complexType>
+ </xs:element>
+
+ <xs:element name="skipped">
+ <xs:complexType mixed="true">
+ <xs:attribute name="type" type="xs:string" use="optional"/>
+ <xs:attribute name="message" type="xs:string" use="optional"/>
+ </xs:complexType>
+ </xs:element>
+
+ <xs:element name="properties">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element ref="property" minOccurs="0" maxOccurs="unbounded"/>
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+
+ <xs:element name="property">
+ <xs:complexType>
+ <xs:attribute name="name" type="xs:string" use="required"/>
+ <xs:attribute name="value" type="xs:string" use="required"/>
+ </xs:complexType>
+ </xs:element>
+
+ <xs:element name="system-err" type="xs:string"/>
+ <xs:element name="system-out" type="xs:string"/>
+
+ <xs:element name="testcase">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:choice minOccurs="0" maxOccurs="unbounded">
+ <xs:element ref="skipped"/>
+ <xs:element ref="error"/>
+ <xs:element ref="failure"/>
+ <xs:element ref="system-out"/>
+ <xs:element ref="system-err"/>
+ </xs:choice>
+ </xs:sequence>
+ <xs:attribute name="name" type="xs:string" use="required"/>
+ <xs:attribute name="assertions" type="xs:string" use="optional"/>
+ <xs:attribute name="time" type="xs:string" use="optional"/>
+ <xs:attribute name="classname" type="xs:string" use="optional"/>
+ <xs:attribute name="status" type="xs:string" use="optional"/>
+ <xs:attribute name="class" type="xs:string" use="optional"/>
+ <xs:attribute name="file" type="xs:string" use="optional"/>
+ <xs:attribute name="line" type="xs:string" use="optional"/>
+ </xs:complexType>
+ </xs:element>
+
+ <xs:element name="testsuite">
+ <xs:complexType>
+ <xs:choice minOccurs="0" maxOccurs="unbounded">
+ <xs:element ref="testsuite"/>
+ <xs:element ref="properties"/>
+ <xs:element ref="testcase"/>
+ <xs:element ref="system-out"/>
+ <xs:element ref="system-err"/>
+ </xs:choice>
+ <xs:attribute name="name" type="xs:string" use="optional"/>
+ <xs:attribute name="tests" type="xs:string" use="required"/>
+ <xs:attribute name="failures" type="xs:string" use="optional"/>
+ <xs:attribute name="errors" type="xs:string" use="optional"/>
+ <xs:attribute name="time" type="xs:string" use="optional"/>
+ <xs:attribute name="disabled" type="xs:string" use="optional"/>
+ <xs:attribute name="skipped" type="xs:string" use="optional"/>
+ <xs:attribute name="timestamp" type="xs:string" use="optional"/>
+ <xs:attribute name="hostname" type="xs:string" use="optional"/>
+ <xs:attribute name="id" type="xs:string" use="optional"/>
+ <xs:attribute name="package" type="xs:string" use="optional"/>
+ <xs:attribute name="assertions" type="xs:string" use="optional"/>
+ <xs:attribute name="file" type="xs:string" use="optional"/>
+ </xs:complexType>
+ </xs:element>
+
+ <xs:element name="testsuites">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element ref="testsuite" minOccurs="0" maxOccurs="unbounded"/>
+ </xs:sequence>
+ <xs:attribute name="name" type="xs:string" use="optional"/>
+ <xs:attribute name="time" type="xs:string" use="optional"/>
+ <xs:attribute name="tests" type="xs:string" use="optional"/>
+ <xs:attribute name="failures" type="xs:string" use="optional"/>
+ <xs:attribute name="disabled" type="xs:string" use="optional"/>
+ <xs:attribute name="errors" type="xs:string" use="optional"/>
+ </xs:complexType>
+ </xs:element>
+
+</xs:schema>
\ No newline at end of file
diff --git a/framework/tests/utils.py b/framework/tests/utils.py
index b2fe86e..2694fab 100644
--- a/framework/tests/utils.py
+++ b/framework/tests/utils.py
@@ -230,3 +230,25 @@ class TestWithEnvClean(object):
# reversed order to make any sense
for call, args in reversed(self._teardown_calls):
call(*args)
+
+
+class StaticDirectory(object):
+ """ Helper class providing shared files creation and cleanup
+
+ One should override the setup_class method in a child class, call super(),
+ and then add files to cls.dir.
+
+ Tests in this class should NOT modify the contents of tidr, if you want
+ that functionality you want a different class
+
+ """
+ @classmethod
+ def setup_class(cls):
+ """ Create a temperary directory that will be removed in teardown_class
+ """
+ cls.tdir = tempfile.mkdtemp()
+
+ @classmethod
+ def teardown_class(cls):
+ """ Remove the temporary directory """
+ shutil.rmtree(cls.tdir)
--
2.1.0
More information about the Piglit
mailing list