[Piglit] [PATCH 1/2] JSON: move info utilities out of root
Dylan Baker
dylan at pnwbakers.com
Wed Jun 27 18:08:52 UTC 2018
And into a generic sub element in the output. This makes information
like glxinfo and clinfo not a hard requirement and makes the format
easier to generate from swineherd.
---
framework/backends/json.py | 16 ++-
framework/programs/run.py | 3 +-
framework/results.py | 10 +-
framework/summary/html_.py | 8 +-
.../framework/backends/schema/piglit-10.json | 125 ++++++++++++++++++
unittests/framework/backends/shared.py | 22 ++-
.../framework/backends/test_json_update.py | 80 +++++++++++
unittests/framework/test_results.py | 46 +++----
8 files changed, 257 insertions(+), 53 deletions(-)
create mode 100644 unittests/framework/backends/schema/piglit-10.json
diff --git a/framework/backends/json.py b/framework/backends/json.py
index 882169e09..46ea6f7bc 100644
--- a/framework/backends/json.py
+++ b/framework/backends/json.py
@@ -53,7 +53,7 @@ __all__ = [
]
# The current version of the JSON results
-CURRENT_JSON_VERSION = 9
+CURRENT_JSON_VERSION = 10
# The minimum JSON format supported
MINIMUM_SUPPORTED_VERSION = 7
@@ -322,6 +322,7 @@ def _update_results(results, filepath):
updates = {
7: _update_seven_to_eight,
8: _update_eight_to_nine,
+ 9: _update_nine_to_ten,
}
while results['results_version'] < CURRENT_JSON_VERSION:
@@ -400,6 +401,19 @@ def _update_eight_to_nine(result):
return result
+def _update_nine_to_ten(result):
+ result['info'] = {}
+ result['info']['system'] = {}
+ for e in ['glxinfo', 'wglinfo', 'clinfo', 'lspci', 'uname']:
+ r = result.pop(e)
+ if r:
+ result['info']['system'][e] = r
+
+ result['results_version'] = 10
+
+ return result
+
+
REGISTRY = Registry(
extensions=['.json'],
backend=JSONBackend,
diff --git a/framework/programs/run.py b/framework/programs/run.py
index afb7eb78d..da978fdaf 100644
--- a/framework/programs/run.py
+++ b/framework/programs/run.py
@@ -250,7 +250,8 @@ def _create_metadata(args, name, forced_test_list):
metadata = {'options': opts}
metadata['name'] = name
- metadata.update(core.collect_system_info())
+ metadata['info'] = {}
+ metadata['info']['system'] = core.collect_system_info()
return metadata
diff --git a/framework/results.py b/framework/results.py
index b936298c3..31cb8dff3 100644
--- a/framework/results.py
+++ b/framework/results.py
@@ -298,13 +298,8 @@ class Totals(dict):
class TestrunResult(object):
"""The result of a single piglit run."""
def __init__(self):
- self.name = None
- self.uname = None
+ self.info = {}
self.options = {}
- self.glxinfo = None
- self.wglinfo = None
- self.clinfo = None
- self.lspci = None
self.time_elapsed = TimeAttribute()
self.tests = collections.OrderedDict()
self.totals = collections.defaultdict(Totals)
@@ -372,8 +367,7 @@ class TestrunResult(object):
"""
res = cls()
- for name in ['name', 'uname', 'options', 'glxinfo', 'wglinfo', 'lspci',
- 'results_version', 'clinfo']:
+ for name in ['name', 'info', 'options', 'results_version']:
value = dict_.get(name)
if value:
setattr(res, name, value)
diff --git a/framework/summary/html_.py b/framework/summary/html_.py
index 14dd76fa8..945230bc6 100644
--- a/framework/summary/html_.py
+++ b/framework/summary/html_.py
@@ -100,10 +100,10 @@ def _make_testrun_info(results, destination, exclude=None):
totals=each.totals['root'],
time=each.time_elapsed.delta,
options=each.options,
- uname=each.uname,
- glxinfo=each.glxinfo,
- clinfo=each.clinfo,
- lspci=each.lspci))
+ uname=each.info['system'].get('uname'),
+ glxinfo=each.info['system'].get('glxinfo'),
+ clinfo=each.info['system'].get('clinfo'),
+ lspci=each.info['system'].get('lspci')))
# Then build the individual test results
for key, value in six.iteritems(each.tests):
diff --git a/unittests/framework/backends/schema/piglit-10.json b/unittests/framework/backends/schema/piglit-10.json
new file mode 100644
index 000000000..e400b9733
--- /dev/null
+++ b/unittests/framework/backends/schema/piglit-10.json
@@ -0,0 +1,125 @@
+{
+ "$schema": "http://json-schema.org/draft-04/schema#",
+ "title": "TestrunResult",
+ "description": "The collection of all results",
+ "type": "object",
+ "properties": {
+ "__type__": { "type": "string" },
+ "name": { "type": "string" },
+ "results_version": { "type": "number" },
+ "time_elapsed": { "$ref": "#/definitions/timeAttribute" },
+ "info": {
+ "type": "object",
+ "description": "Relavent information about the system running tests.",
+ "additionalProperties": {
+ "type": "object",
+ "additionalProperties": { "type": "string" }
+ }
+ },
+ "options": {
+ "descrption": "The options that were invoked with this run. These are implementation specific and not required.",
+ "type": "object",
+ "properties": {
+ "exclude_tests": {
+ "type": "array",
+ "items": { "type": "string" },
+ "uniqueItems": true
+ },
+ "include_filter": {
+ "type": "array",
+ "items": { "type": "string" }
+ },
+ "exclude_filter": {
+ "type": "array",
+ "items": { "type": "string" }
+ },
+ "sync": { "type": "boolean" },
+ "valgrind": { "type": "boolean" },
+ "monitored": { "type": "boolean" },
+ "dmesg": { "type": "boolean" },
+ "execute": { "type": "boolean" },
+ "concurrent": { "enum": ["none", "all", "some"] },
+ "platform": { "type": "string" },
+ "log_level": { "type": "string" },
+ "env": {
+ "description": "Environment variables that must be specified",
+ "type": "object",
+ "additionalProperties": { "type": "string" }
+ },
+ "profile": {
+ "type": "array",
+ "items": { "type": "string" }
+ }
+ }
+ },
+ "totals": {
+ "type": "object",
+ "description": "A calculation of the group totals.",
+ "additionalProperties": {
+ "type": "object",
+ "properties": {
+ "crash": { "type": "number" },
+ "dmesg-fail": { "type": "number" },
+ "dmesg-warn": { "type": "number" },
+ "fail": { "type": "number" },
+ "incomplete": { "type": "number" },
+ "notrun": { "type": "number" },
+ "pass": { "type": "number" },
+ "skip": { "type": "number" },
+ "timeout": { "type": "number" },
+ "warn": { "type": "number" }
+ },
+ "additionalProperties": false,
+ "required": [ "crash", "dmesg-fail", "dmesg-warn", "fail", "incomplete", "notrun", "pass", "skip", "timeout", "warn" ]
+ }
+ },
+ "tests": {
+ "type": "object",
+ "additionalProperties": {
+ "type": "object",
+ "properties": {
+ "__type__": { "type": "string" },
+ "err": { "type": "string" },
+ "exception": { "type": ["string", "null"] },
+ "result": {
+ "type": "string",
+ "enum": [ "pass", "fail", "crash", "warn", "incomplete", "notrun", "skip", "dmesg-warn", "dmesg-fail" ]
+ },
+ "environment": { "type": "string" },
+ "command": { "type": "string" },
+ "traceback": { "type": ["string", "null"] },
+ "out": { "type": "string" },
+ "dmesg": { "type": "string" },
+ "pid": {
+ "type": "array",
+ "items": { "type": "number" }
+ },
+ "returncode": { "type": [ "number", "null" ] },
+ "time": { "$ref": "#/definitions/timeAttribute" },
+ "subtests": {
+ "type": "object",
+ "properties": { "__type__": { "type": "string" } },
+ "additionalProperties": { "type": "string" },
+ "required": [ "__type__" ]
+ }
+ },
+ "additionalProperties": false
+ }
+ }
+ },
+ "additionalProperties": false,
+ "required": [ "__type__", "name", "results_version", "time_elapsed", "tests", "info" ],
+ "definitions": {
+ "timeAttribute": {
+ "type": "object",
+ "description": "An element containing a start and end time",
+ "properties": {
+ "__type__": { "type": "string" },
+ "start": { "type": "number" },
+ "end": { "type": "number" }
+ },
+ "additionalProperties": false,
+ "required": [ "__type__", "start", "end" ]
+ }
+ }
+}
diff --git a/unittests/framework/backends/shared.py b/unittests/framework/backends/shared.py
index d9f5790af..59b30c9aa 100644
--- a/unittests/framework/backends/shared.py
+++ b/unittests/framework/backends/shared.py
@@ -31,24 +31,21 @@ from framework.options import OPTIONS
INITIAL_METADATA = {
'name': 'name',
'options': dict(OPTIONS),
- 'clinfo': None,
- 'glxinfo': None,
- 'wglinfo': None,
- 'lspci': None,
- 'uname': None,
+ 'info': {
+ 'system': {}
+ }
}
# This is current JSON data, in raw form with only the minimum required
# changes. This does not contain piglit specifc objects, only strings, floats,
# ints, and Nones (instead of JSON's null)
JSON = {
- "results_version": 9,
+ "results_version": 10,
"time_elapsed": {
"start": 1469638791.2351687,
"__type__": "TimeAttribute",
"end": 1469638791.4387212
},
- "wglinfo": None,
"tests": {
"spec@!opengl 1.0 at gl-1.0-readpixsanity": {
"dmesg": "",
@@ -75,11 +72,6 @@ JSON = {
" PIGLIT_PLATFORM=\"gbm\"")
}
},
- # pylint: disable=line-too-long
- "lspci": "00:00.0 Host bridge...",
- "clinfo": None,
- "uname": "uname",
- # pylint: enable=line-too-long
"options": {
"dmesg": False,
"concurrent": "some",
@@ -102,7 +94,11 @@ JSON = {
},
"name": "foo",
"__type__": "TestrunResult",
- "glxinfo": None,
+ "info": {
+ "system": {
+ "lspci": "00:00.0 Host bridge...",
+ },
+ },
"totals": {
"spec": {
'__type__': 'Totals',
diff --git a/unittests/framework/backends/test_json_update.py b/unittests/framework/backends/test_json_update.py
index 29ca19a50..dca6f9d77 100644
--- a/unittests/framework/backends/test_json_update.py
+++ b/unittests/framework/backends/test_json_update.py
@@ -224,3 +224,83 @@ class TestV8toV9(object):
jsonschema.validate(
json.loads(json.dumps(result, default=backends.json.piglit_encoder)),
schema)
+
+
+class TestV9toV10(object):
+ """Tests for Version 8 to version 9."""
+
+ data = {
+ "results_version": 9,
+ "name": "test",
+ "options": {
+ "profile": ['quick'],
+ "dmesg": False,
+ "verbose": False,
+ "platform": "gbm",
+ "sync": False,
+ "valgrind": False,
+ "filter": [],
+ "concurrent": "all",
+ "test_count": 0,
+ "exclude_tests": [],
+ "exclude_filter": [],
+ "env": {},
+ },
+ "lspci": "stuff",
+ "uname": "more stuff",
+ "glxinfo": "and stuff",
+ "wglinfo": "stuff",
+ "clinfo": "stuff",
+ "tests": {
+ 'a at test': {
+ "time": {
+ 'start': 1.2,
+ 'end': 1.8,
+ '__type__': 'TimeAttribute'
+ },
+ 'dmesg': '',
+ 'result': 'fail',
+ '__type__': 'TestResult',
+ 'command': '/a/command',
+ 'traceback': None,
+ 'out': '',
+ 'environment': 'A=variable',
+ 'returncode': 0,
+ 'err': '',
+ 'pid': [5],
+ 'subtests': {
+ '__type__': 'Subtests',
+ },
+ 'exception': None,
+ },
+ },
+ "time_elapsed": {
+ 'start': 1.2,
+ 'end': 1.8,
+ '__type__': 'TimeAttribute'
+ },
+ '__type__': 'TestrunResult',
+ }
+
+ @pytest.fixture
+ def result(self, tmpdir):
+ p = tmpdir.join('result.json')
+ p.write(json.dumps(self.data, default=backends.json.piglit_encoder))
+ with p.open('r') as f:
+ return backends.json._update_nine_to_ten(backends.json._load(f))
+
+ @pytest.mark.parametrize("key", ['glxinfo', 'wglinfo', 'clinfo', 'uname', 'lspci'])
+ def test(self, key, result):
+ assert key not in result, 'Root key/value not removed'
+ assert key in result['info']['system'], 'Key not added to info/system'
+ assert result['info']['system'][key] == self.data[key], \
+ 'Value not set properly.'
+
+ def test_valid(self, result):
+ with open(os.path.join(os.path.dirname(__file__), 'schema',
+ 'piglit-10.json'),
+ 'r') as f:
+ schema = json.load(f)
+ jsonschema.validate(
+ json.loads(json.dumps(result, default=backends.json.piglit_encoder)),
+ schema)
diff --git a/unittests/framework/test_results.py b/unittests/framework/test_results.py
index 52257e03c..cd7c2423d 100644
--- a/unittests/framework/test_results.py
+++ b/unittests/framework/test_results.py
@@ -467,13 +467,17 @@ class TestTestrunResult(object):
def setup_class(cls):
"""Setup values used by all tests."""
test = results.TestrunResult()
+ test.info = {
+ 'system': {
+ 'uname': 'this is uname',
+ 'glxinfo': 'glxinfo',
+ 'clinfo': 'clinfo',
+ 'wglinfo': 'wglinfo',
+ 'lspci': 'this is lspci',
+ }
+ }
test.name = 'name'
- test.uname = 'this is uname'
test.options = {'some': 'option'}
- test.glxinfo = 'glxinfo'
- test.clinfo = 'clinfo'
- test.wglinfo = 'wglinfo'
- test.lspci = 'this is lspci'
test.time_elapsed.end = 1.23
test.tests = {'a test': results.TestResult('pass')}
@@ -483,30 +487,21 @@ class TestTestrunResult(object):
"""name is properly encoded."""
assert self.test['name'] == 'name'
- def test_uname(self):
- """uname is properly encoded."""
- assert self.test['uname'] == 'this is uname'
+ def test_info(self):
+ assert self.test['info'] == {
+ 'system': {
+ 'uname': 'this is uname',
+ 'glxinfo': 'glxinfo',
+ 'clinfo': 'clinfo',
+ 'wglinfo': 'wglinfo',
+ 'lspci': 'this is lspci',
+ }
+ }
def test_options(self):
"""options is properly encoded."""
assert dict(self.test['options']) == {'some': 'option'}
- def test_glxinfo(self):
- """glxinfo is properly encoded."""
- assert self.test['glxinfo'] == 'glxinfo'
-
- def test_wglinfo(self):
- """wglinfo is properly encoded."""
- assert self.test['wglinfo'] == 'wglinfo'
-
- def test_clinfo(self):
- """clinfo is properly encoded."""
- assert self.test['clinfo'] == 'clinfo'
-
- def test_lspci(self):
- """lspci is properly encoded."""
- assert self.test['lspci'] == 'this is lspci'
-
def test_time(self):
"""time_elapsed is properly encoded."""
assert self.test['time_elapsed'].end == 1.23
@@ -527,8 +522,7 @@ class TestTestrunResult(object):
return results.TestrunResult.from_dict(shared.JSON)
@pytest.mark.parametrize("attrib", [
- 'name', 'uname', 'glxinfo', 'wglinfo', 'lspci', 'results_version',
- 'clinfo', 'options',
+ 'name', 'results_version', 'info', 'options',
])
def test_attribs_restored(self, attrib, inst):
"""tests for basic attributes."""
--
2.17.1
More information about the Piglit
mailing list