[Piglit] [RFC 8/9] backends: implemented loading support for junit.
Dylan Baker
baker.dylan.c at gmail.com
Mon Apr 6 14:30:18 PDT 2015
This allows junit results to be loaded like any other reuslts, including
for use with the piglit summary generator.
Signed-off-by: Dylan Baker <dylanx.c.baker at intel.com>
---
framework/backends/junit.py | 70 +++++++++++++++-
framework/tests/junit_backends_tests.py | 140 +++++++++++++++++++++++++++++++-
framework/tests/utils.py | 5 +-
3 files changed, 208 insertions(+), 7 deletions(-)
diff --git a/framework/backends/junit.py b/framework/backends/junit.py
index 5ec4e8e..3602f9e 100644
--- a/framework/backends/junit.py
+++ b/framework/backends/junit.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2014 Intel Corporation
+# Copyright (c) 2014, 2015 Intel Corporation
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
@@ -29,7 +29,7 @@ try:
except ImportError:
import xml.etree.cElementTree as etree
-import framework.grouptools as grouptools
+from framework import grouptools, results, status
from framework.core import PIGLIT_CONFIG
from .abstract import FileBackend
from .register import Registry
@@ -207,9 +207,73 @@ class JUnitBackend(FileBackend):
self._fsync(f)
+def _load(results_file):
+ """Load a junit results instance and return a TestrunResult.
+
+ It's worth noting that junit is not as descriptive as piglit's own json
+ format, so some data structures will be empty compared to json.
+
+ This tries to not make too many assumptions about the strucuter of the
+ JUnit document.
+
+ """
+ run_result = results.TestrunResult()
+
+ splitpath = os.path.splitext(results_file)[0].split(os.path.sep)
+ if splitpath[-1] != 'results':
+ run_result.name = splitpath[-1]
+ elif len(splitpath) > 1:
+ run_result.name = splitpath[-2]
+ else:
+ run_result.name = 'junit result'
+
+ tree = etree.parse(results_file).getroot().find('.//testsuite[@name="piglit"]')
+ for test in tree.iterfind('testcase'):
+ result = results.TestResult()
+ # Take the class name minus the 'piglit.' element, replace junit's '.'
+ # separator with piglit's separator, and join the group and test names
+ name = test.attrib['classname'].split('.', 1)[1]
+ name = name.replace('.', grouptools.SEPARATOR)
+ name = grouptools.join(name, test.attrib['name'])
+
+ # Remove the trailing _ if they were added (such as to api and search)
+ if name.endswith('_'):
+ name = name[:-1]
+
+ result['result'] = status.status_lookup(test.attrib['status'])
+ result['time'] = float(test.attrib['time'])
+ result['err'] = test.find('system-err').text
+
+ # The command is prepended to system-out, so we need to separate those
+ # into two separate elements
+ out = test.find('system-out').text.split('\n')
+ result['command'] = out[0]
+ result['out'] = '\n'.join(out[1:])
+
+ run_result.tests[name] = result
+
+ return run_result
+
+
+def load(results_dir):
+ """Searches for a results file and returns a TestrunResult.
+
+ wraps _load and searches for the result file.
+
+ """
+ if not os.path.isdir(results_dir):
+ return _load(results_dir)
+ elif os.path.exists(os.path.join(results_dir, 'tests')):
+ raise NotImplementedError('resume support of junit not implemented')
+ elif os.path.exists(os.path.join(results_dir, 'results.xml')):
+ return _load(os.path.join(results_dir, 'results.xml'))
+ else:
+ raise Exception("No results found")
+
+
REGISTRY = Registry(
extensions=['.xml'],
backend=JUnitBackend,
- load=None,
+ load=load,
meta=lambda x: x, # The venerable no-op function
)
diff --git a/framework/tests/junit_backends_tests.py b/framework/tests/junit_backends_tests.py
index 9afef75..d283119 100644
--- a/framework/tests/junit_backends_tests.py
+++ b/framework/tests/junit_backends_tests.py
@@ -31,14 +31,26 @@ except ImportError:
import xml.etree.cElementTree as etree
import nose.tools as nt
-from framework import results, backends, grouptools
+from framework import results, backends, grouptools, status
import framework.tests.utils as utils
from .backends_tests import BACKEND_INITIAL_META
JUNIT_SCHEMA = 'framework/tests/schema/junit-7.xsd'
-doc_formatter = utils.DocFormatter({'seperator': grouptools.SEPARATOR})
+doc_formatter = utils.DocFormatter({'separator': grouptools.SEPARATOR})
+
+_XML = """\
+<?xml version='1.0' encoding='utf-8'?>
+ <testsuites>
+ <testsuite name="piglit" tests="1">
+ <testcase classname="piglit.foo.bar" name="a-test" status="pass" time="1.12345">
+ <system-out>this/is/a/command\nThis is stdout</system-out>
+ <system-err>this is stderr</system-err>
+ </testcase>
+ </testsuite>
+ </testsuites>
+"""
class TestJunitNoTests(utils.StaticDirectory):
@@ -133,7 +145,7 @@ class TestJUnitMultiTest(TestJUnitSingleTest):
@doc_formatter
def test_junit_replace():
- """JUnitBackend.write_test: '{seperator}' is replaced with '.'"""
+ """JUnitBackend.write_test: '{separator}' is replaced with '.'"""
with utils.tempdir() as tdir:
test = backends.junit.JUnitBackend(tdir)
test.initialize(BACKEND_INITIAL_META)
@@ -177,3 +189,125 @@ def test_junit_skips_bad_tests():
test.finalize()
except etree.ParseError as e:
raise AssertionError(e)
+
+
+class TestJUnitLoad(utils.StaticDirectory):
+ """Methods that test loading JUnit results."""
+ __instance = None
+
+ @classmethod
+ def setup_class(cls):
+ super(TestJUnitLoad, cls).setup_class()
+ cls.xml_file = os.path.join(cls.tdir, 'results.xml')
+
+ with open(cls.xml_file, 'w') as f:
+ f.write(_XML)
+
+ cls.testname = grouptools.join('foo', 'bar', 'a-test')
+
+ @classmethod
+ def xml(cls):
+ if cls.__instance is None:
+ cls.__instance = backends.junit._load(cls.xml_file)
+ return cls.__instance
+
+ @utils.no_error
+ def test_no_errors(self):
+ """backends.junit._load: Raises no errors for valid junit."""
+ self.xml()
+
+ def test_return_testrunresult(self):
+ """backends.junit._load: returns a TestrunResult instance"""
+ nt.assert_is_instance(self.xml(), results.TestrunResult)
+
+ @doc_formatter
+ def test_replace_sep(self):
+ """backends.junit._load: replaces '.' with '{separator}'"""
+ nt.assert_in(self.testname, self.xml().tests)
+
+ def test_testresult_instance(self):
+ """backends.junit._load: replaces result with TestResult instance."""
+ nt.assert_is_instance(self.xml().tests[self.testname], results.TestResult)
+
+ def test_status_instance(self):
+ """backends.junit._load: a status is found and loaded."""
+ nt.assert_is_instance(self.xml().tests[self.testname]['result'],
+ status.Status)
+
+ def test_time(self):
+ """backends.junit._load: Time is loaded correctly."""
+ time = self.xml().tests[self.testname]['time']
+ nt.assert_is_instance(time, float)
+ nt.assert_equal(time, 1.12345)
+
+ def test_command(self):
+ """backends.junit._load: command is loaded correctly."""
+ test = self.xml().tests[self.testname]['command']
+ nt.assert_equal(test, 'this/is/a/command')
+
+ def test_out(self):
+ """backends.junit._load: stdout is loaded correctly."""
+ test = self.xml().tests[self.testname]['out']
+ nt.assert_equal(test, 'This is stdout')
+
+ def test_err(self):
+ """backends.junit._load: stderr is loaded correctly."""
+ test = self.xml().tests[self.testname]['err']
+ nt.assert_equal(test, 'this is stderr')
+
+ @utils.no_error
+ def test_load_file(self):
+ """backends.junit.load: Loads a file directly."""
+ backends.junit.REGISTRY.load(self.xml_file)
+
+ @utils.no_error
+ def test_load_dir(self):
+ """backends.junit.load: Loads a directory."""
+ backends.junit.REGISTRY.load(self.tdir)
+
+
+def test_load_file_name():
+ """backend.junit._load: uses the filename for name if filename != 'results'
+ """
+ with utils.tempdir() as tdir:
+ filename = os.path.join(tdir, 'foobar.xml')
+ with open(filename, 'w') as f:
+ f.write(_XML)
+
+ test = backends.junit.REGISTRY.load(filename)
+ nt.assert_equal(test.name, 'foobar')
+
+
+def test_load_folder_name():
+ """backend.junit._load: uses the foldername if the result is 'results'
+ """
+ with utils.tempdir() as tdir:
+ os.mkdir(os.path.join(tdir, 'a cool test'))
+ filename = os.path.join(tdir, 'a cool test', 'results.xml')
+ with open(filename, 'w') as f:
+ f.write(_XML)
+
+ test = backends.junit.REGISTRY.load(filename)
+ nt.assert_equal(test.name, 'a cool test')
+
+
+def test_load_default_name():
+ """backend.junit._load: uses 'junit result' for name as fallback"""
+ curdir = os.getcwd()
+
+ # This try/finally ensures that no matter what the directory will be changed
+ # back, this is necessary to ensure that it doesn't spoil the environment
+ # for other tests.
+ try:
+ with utils.tempdir() as tdir:
+ os.chdir(tdir)
+
+ filename = 'results.xml'
+ with open(filename, 'w') as f:
+ f.write(_XML)
+
+ test = backends.junit.REGISTRY.load(filename)
+ finally:
+ os.chdir(curdir)
+
+ nt.assert_equal(test.name, 'junit result')
diff --git a/framework/tests/utils.py b/framework/tests/utils.py
index f59f6eb..ce10cdd 100644
--- a/framework/tests/utils.py
+++ b/framework/tests/utils.py
@@ -25,6 +25,9 @@ in a single place.
"""
+# TODO: add a chdir decorator that gets the current dir, runs the test in a
+# try and returns to the start dir in the finally
+
from __future__ import print_function, absolute_import
import os
import sys
@@ -299,7 +302,7 @@ def no_error(func):
try:
func(*args, **kwargs)
except Exception as e:
- raise TestFailure(e.message)
+ raise TestFailure(*e.args)
return test_wrapper
--
2.3.5
More information about the Piglit
mailing list