[Piglit] [Patch v3 1/5] json_tests.py: Add unit tests for json backend
Dylan Baker
baker.dylan.c at gmail.com
Thu Oct 30 16:32:38 PDT 2014
Refactor run.py a little to allow testing the Backend.initialize() call.
This adds tests for the output of the json piglit format. This currently
has 4 failing tests exposing two separate bugs:
1) the logger verbosity level is not stored and thus cannot be used by
resume
2) lspci, uname, and glxinfo are not written into the root of the json
output.
This also adds a unit test utility function for skipping unit tests when
the OS is not the required OS.
Signed-off-by: Dylan Baker <dylanx.c.baker at intel.com>
---
framework/programs/run.py | 25 ++++---
framework/tests/json_tests.py | 151 ++++++++++++++++++++++++++++++++++++++++++
framework/tests/utils.py | 7 ++
3 files changed, 172 insertions(+), 11 deletions(-)
create mode 100644 framework/tests/json_tests.py
diff --git a/framework/programs/run.py b/framework/programs/run.py
index e96c0f0..573a193 100644
--- a/framework/programs/run.py
+++ b/framework/programs/run.py
@@ -191,6 +191,19 @@ def _run_parser(input_):
return parser.parse_args(unparsed)
+def _create_metadata(args, name, opts):
+ """Create and return a metadata dict for Backend.initialize()."""
+ options = {'profile': args.test_profile}
+ for key, value in opts:
+ options[key] = value
+ if args.platform:
+ options['platform'] = args.platform
+ options['name'] = name
+ options['env'] = core.collect_system_info()
+
+ return options
+
+
def _disable_windows_exception_messages():
"""Disable Windows error message boxes for this and all child processes."""
if sys.platform == 'win32':
@@ -250,21 +263,11 @@ def run(input_):
else:
results.name = path.basename(args.results_path)
- # Create a dictionary to pass to initialize json, it needs the contents of
- # the env dictionary and profile and platform information
- options = {'profile': args.test_profile}
- for key, value in opts:
- options[key] = value
- if args.platform:
- options['platform'] = args.platform
- options['name'] = results.name
- options['env'] = core.collect_system_info()
-
backend = backends.get_backend(args.backend)(
args.results_path,
file_fsync=opts.sync,
junit_suffix=args.junit_suffix)
- backend.initialize(options)
+ backend.initialize(_create_metadata(args, results.name, opts))
profile = framework.profile.merge_test_profiles(args.test_profile)
profile.results_dir = args.results_path
diff --git a/framework/tests/json_tests.py b/framework/tests/json_tests.py
new file mode 100644
index 0000000..f0daef9
--- /dev/null
+++ b/framework/tests/json_tests.py
@@ -0,0 +1,151 @@
+# Copyright (c) 2014 Intel Corporation
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+"""Tests for the json outpt format.
+
+This module drills the ouput of the json backend with a series of tests
+designed to catch changes in the json output. This is a rather volatile set of
+tests and they will change with each version of the json output.
+
+"""
+
+from __future__ import absolute_import
+import os
+
+import nose.tools as nt
+try:
+ import simplejson as json
+except ImportError:
+ import json
+
+import framework.core as core
+import framework.tests.utils as utils
+from framework.backends.json_ import JSONBackend
+from framework.programs.run import _create_metadata
+
+
+# Helpers
+class Namespace(object):
+ """Simple namespace object for replicating and argparse namespace."""
+ pass
+
+
+# Tests
+# pylint: disable=too-many-public-methods
+class TestJsonOutput(utils.StaticDirectory):
+ """Class for testing JSON output."""
+ @classmethod
+ def setup_class(cls):
+ super(TestJsonOutput, cls).setup_class()
+
+ args = Namespace()
+ # pylint: disable=attribute-defined-outside-init
+ args.test_profile = ['fake.py']
+ args.platform = 'gbm'
+ args.log_level = 'verbose'
+
+ backend = JSONBackend(cls.tdir, file_fsync=True)
+ backend.initialize(_create_metadata(args, 'test', core.Options()))
+ backend.write_test('result', {'result': 'pass'})
+ backend.finalize({'time_elapsed': 1.22})
+ with open(os.path.join(cls.tdir, 'results.json'), 'r') as f:
+ cls.json = json.load(f)
+
+ def test_root_results_version(self):
+ """JSON: result_version is a root key."""
+ nt.assert_in('results_version', self.json)
+
+ def test_root_name(self):
+ """JSON: name is a root key."""
+ nt.assert_in('name', self.json)
+
+ def test_root_options(self):
+ """JSON: options is a root key."""
+ nt.assert_in('options', self.json)
+
+ def test_root_tests(self):
+ """JSON: tests is a root key."""
+ nt.assert_in('tests', self.json)
+
+ def test_root_lspci(self):
+ """JSON: lspci is a root key."""
+ utils.platform_check('linux')
+ utils.binary_check('lspci')
+ nt.assert_in('lspci', self.json)
+
+ def test_root_uname(self):
+ """JSON: uname is a root key."""
+ utils.platform_check('linux')
+ utils.binary_check('uname')
+ nt.assert_in('uname', self.json)
+
+ def test_root_glxinfo(self):
+ """JSON: glxinfo is a root key."""
+ utils.platform_check('linux')
+ utils.binary_check('glxinfo')
+ nt.assert_in('glxinfo', self.json)
+
+ def test_root_time_elapsed(self):
+ """JSON: time_elapsed is a root key."""
+ nt.assert_in('time_elapsed', self.json)
+
+ def test_options_profile(self):
+ """JSON: profile is an options key."""
+ nt.assert_in('profile', self.json['options'])
+
+ def test_options_dmesg(self):
+ """JSON: dmesg is an options key."""
+ nt.assert_in('dmesg', self.json['options'])
+
+ def test_options_execute(self):
+ """JSON: execute is an options key."""
+ nt.assert_in('execute', self.json['options'])
+
+ def test_options_log_level(self):
+ """JSON: log_level is an options key."""
+ nt.assert_in('log_level', self.json['options'])
+
+ def test_options_platform(self):
+ """JSON: platform is an options key."""
+ nt.assert_in('platform', self.json['options'])
+
+ def test_options_sync(self):
+ """JSON: sync is an options key."""
+ nt.assert_in('sync', self.json['options'])
+
+ def test_options_valgrind(self):
+ """JSON: valgrind is an options key."""
+ nt.assert_in('valgrind', self.json['options'])
+
+ def test_options_concurrent(self):
+ """JSON: concurrent is an options key."""
+ nt.assert_in('concurrent', self.json['options'])
+
+ def test_options_filter(self):
+ """JSON: filter is an options key."""
+ nt.assert_in('filter', self.json['options'])
+
+ def test_options_exclude_tests(self):
+ """JSON: exclude_tests is an options key."""
+ nt.assert_in('exclude_tests', self.json['options'])
+
+ def test_options_exclude_filter(self):
+ """JSON: exclude_filter is an options key."""
+ nt.assert_in('exclude_filter', self.json['options'])
diff --git a/framework/tests/utils.py b/framework/tests/utils.py
index 3b13a1f..f67825f 100644
--- a/framework/tests/utils.py
+++ b/framework/tests/utils.py
@@ -26,6 +26,7 @@ in a single place.
"""
import os
+import sys
import shutil
import tempfile
import functools
@@ -263,3 +264,9 @@ def binary_check(bin_):
subprocess.check_call(['which', bin_], stdout=null, stderr=null)
except subprocess.CalledProcessError:
raise SkipTest('Binary {} not available'.format(bin_))
+
+
+def platform_check(plat):
+ """If the platform is not in the list specified skip the test."""
+ if not sys.platform.startswith(plat):
+ raise SkipTest('Platform {} is not supported'.format(sys.platform))
--
2.1.2
More information about the Piglit
mailing list