[Piglit] [PATCH v3 27/28] framework: Embed images in results.
Dylan Baker
dylan at pnwbakers.com
Mon Oct 31 17:50:23 UTC 2016
This helps to make the results format more portable, since it no longer
requires a separate image directory in a hardcoded location.
Signed-off-by: Dylan Baker <dylanx.c.baker at intel.com>
---
framework/backends/json.py | 36 ++-
framework/results.py | 7 +-
templates/test_result.mako | 8 +-
tests/xts.py | 24 +-
unittests/framework/backends/images/LICENSE | 6 +-
unittests/framework/backends/images/openlogo-nd-25.png | Bin 0 -> 409 bytes
unittests/framework/backends/images/ref.png | 1 +-
unittests/framework/backends/images/render.png | 1 +-
unittests/framework/backends/schema/piglit-10.json | 135 ++++++++++-
unittests/framework/backends/shared.py | 2 +-
unittests/framework/backends/test_json_update.py | 101 +++++++-
11 files changed, 301 insertions(+), 20 deletions(-)
create mode 100644 unittests/framework/backends/images/LICENSE
create mode 100644 unittests/framework/backends/images/openlogo-nd-25.png
create mode 120000 unittests/framework/backends/images/ref.png
create mode 120000 unittests/framework/backends/images/render.png
create mode 100644 unittests/framework/backends/schema/piglit-10.json
diff --git a/framework/backends/json.py b/framework/backends/json.py
index 17002ed..bf64ad9 100644
--- a/framework/backends/json.py
+++ b/framework/backends/json.py
@@ -23,10 +23,10 @@
from __future__ import (
absolute_import, division, print_function, unicode_literals
)
+import base64
import collections
import functools
import os
-import posixpath
import shutil
import sys
@@ -53,7 +53,7 @@ __all__ = [
]
# The current version of the JSON results
-CURRENT_JSON_VERSION = 9
+CURRENT_JSON_VERSION = 10
# The minimum JSON format supported
MINIMUM_SUPPORTED_VERSION = 7
@@ -312,6 +312,7 @@ def _update_results(results, filepath):
updates = {
7: _update_seven_to_eight,
8: _update_eight_to_nine,
+ 9: _update_nine_to_ten,
}
while results['results_version'] < CURRENT_JSON_VERSION:
@@ -387,6 +388,37 @@ def _update_eight_to_nine(result):
return result
+def _update_nine_to_ten(result):
+ """Update json results from version 8 to 9.
+
+ This changes the PID feild of the TestResult object to alist of Integers or
+ null rather than a single integer or null.
+
+ """
+ for test in six.itervalues(result['tests']):
+ if 'images' in test:
+ new_list = []
+
+ for image in test['images']:
+ new = {'description': image['image_desc']}
+
+ if 'image_ref' in image:
+ with open(image['image_ref'], 'rb') as f:
+ new['reference'] = base64.b64encode(f.read()).decode('utf-8')
+
+ if 'image_render' in image:
+ with open(image['image_render'], 'rb') as f:
+ new['rendered'] = base64.b64encode(f.read()).decode('utf-8')
+
+ new_list.append(new)
+
+ test['images'] = new_list
+
+ result['results_version'] = 10
+
+ return result
+
+
REGISTRY = Registry(
extensions=['.json'],
backend=JSONBackend,
diff --git a/framework/results.py b/framework/results.py
index 3c2bb2a..fc1011d 100644
--- a/framework/results.py
+++ b/framework/results.py
@@ -207,6 +207,11 @@ class TestResult(object):
'dmesg': self.dmesg,
'pid': self.pid,
}
+
+ # Images are not required, only add them if they are not empty
+ if self.images:
+ obj['images'] = self.images
+
return obj
@classmethod
@@ -225,7 +230,7 @@ class TestResult(object):
inst = cls()
for each in ['returncode', 'command', 'exception', 'environment',
- 'traceback', 'dmesg', 'pid', 'result']:
+ 'traceback', 'dmesg', 'pid', 'result', 'images']:
if each in dict_:
setattr(inst, each, dict_[each])
diff --git a/templates/test_result.mako b/templates/test_result.mako
index ff08797..ab2b2b7 100644
--- a/templates/test_result.mako
+++ b/templates/test_result.mako
@@ -38,11 +38,11 @@
<td>reference</td>
<td>rendered</td>
</tr>
- % for image in images:
+ % for image in value.images:
<tr>
- <td>${image['image_desc']}</td>
- <td><img src="file://${image['image_ref']}" /></td>
- <td><img src="file://${image['image_render']}" /></td>
+ <td>${image['description']}</td>
+ <td><img alt="Reference Image" src="data:image/png;base64,${image['reference']}" /></td>
+ <td><img alt="Rendered Image" src="data:image/png;base64,${image['rendered']}" /></td>
</tr>
% endfor
</table>
diff --git a/tests/xts.py b/tests/xts.py
index 715ecfa..d2f1e30 100644
--- a/tests/xts.py
+++ b/tests/xts.py
@@ -26,6 +26,7 @@
from __future__ import (
absolute_import, division, print_function, unicode_literals
)
+import base64
import os
import re
import subprocess
@@ -50,14 +51,6 @@ class XTSProfile(TestProfile): # pylint: disable=too-few-public-methods
"""
XTSTest.RESULTS_PATH = self.results_dir
- try:
- os.mkdir(os.path.join(self.results_dir, 'images'))
- except OSError as e:
- # If the exception is not 'directory already exists', raise the
- # exception
- if e.errno != 17:
- raise
-
class XTSTest(Test): # pylint: disable=too-few-public-methods
""" X Test Suite class
@@ -123,7 +116,9 @@ class XTSTest(Test): # pylint: disable=too-few-public-methods
try:
out = subprocess.check_output(command, cwd=self.cwd)
except OSError:
- images.append({'image_desc': 'image processing failed'})
+ images.append({'description': 'image processing failed',
+ 'reference': None,
+ 'rendered': None})
continue
# Each Err*.err log contains a rendered image, and a reference
@@ -139,13 +134,18 @@ class XTSTest(Test): # pylint: disable=too-few-public-methods
self.RESULTS_PATH, 'images', '{1}-{2}-render.png'.format(
self.testname, match.group(1)))
+ with open(ref_path, 'rb') as f:
+ ref = base64.b64encode(f.read()).decode('utf-8')
+ with open(render_path, 'rb') as f:
+ render = base64.b64encode(f.read()).decode('utf-8')
+
split = out.splitlines()
os.rename(os.path.join(self.cwd, split[0]), render_path)
os.rename(os.path.join(self.cwd, split[1]), ref_path)
- images.append({'image_desc': desc,
- 'image_ref': ref_path,
- 'image_render': render_path})
+ images.append({'description': desc,
+ 'reference': ref,
+ 'rendered': render})
return images
diff --git a/unittests/framework/backends/images/LICENSE b/unittests/framework/backends/images/LICENSE
new file mode 100644
index 0000000..96286c0
--- /dev/null
+++ b/unittests/framework/backends/images/LICENSE
@@ -0,0 +1,6 @@
+The Debian Open Use Logo(s) are Copyright (c) 1999 Software in the Public
+Interest, Inc., and are released under the terms of the GNU Lesser General
+Public License, version 3 or any later version, or, at your option, of the
+Creative Commons Attribution-ShareAlike 3.0 Unported License.
+
+openlogo-nd-25.png and it's links are covered by this license.
diff --git a/unittests/framework/backends/images/openlogo-nd-25.png b/unittests/framework/backends/images/openlogo-nd-25.png
new file mode 100644
index 0000000..66f0ba0
Binary files /dev/null and b/unittests/framework/backends/images/openlogo-nd-25.png differ
diff --git a/unittests/framework/backends/images/ref.png b/unittests/framework/backends/images/ref.png
new file mode 120000
index 0000000..c7a1526
--- /dev/null
+++ b/unittests/framework/backends/images/ref.png
@@ -0,0 +1 @@
+openlogo-nd-25.png
\ No newline at end of file
diff --git a/unittests/framework/backends/images/render.png b/unittests/framework/backends/images/render.png
new file mode 120000
index 0000000..c7a1526
--- /dev/null
+++ b/unittests/framework/backends/images/render.png
@@ -0,0 +1 @@
+openlogo-nd-25.png
\ No newline at end of file
diff --git a/unittests/framework/backends/schema/piglit-10.json b/unittests/framework/backends/schema/piglit-10.json
new file mode 100644
index 0000000..440db14
--- /dev/null
+++ b/unittests/framework/backends/schema/piglit-10.json
@@ -0,0 +1,135 @@
+{
+ "$schema": "http://json-schema.org/draft-04/schema#",
+ "title": "TestrunResult",
+ "description": "The collection of all results",
+ "type": "object",
+ "properties": {
+ "__type__": { "type": "string" },
+ "clinfo": { "type": ["string", "null"] },
+ "glxinfo": { "type": ["string", "null"] },
+ "lspci": { "type": ["string", "null"] },
+ "wglinfo": { "type": ["string", "null"] },
+ "name": { "type": "string" },
+ "results_version": { "type": "number" },
+ "uname": { "type": [ "string", "null" ] },
+ "time_elapsed": { "$ref": "#/definitions/timeAttribute" },
+ "options": {
+ "descrption": "The options that were invoked with this run. These are implementation specific and not required.",
+ "type": "object",
+ "properties": {
+ "exclude_tests": {
+ "type": "array",
+ "items": { "type": "string" },
+ "uniqueItems": true
+ },
+ "include_filter": {
+ "type": "array",
+ "items": { "type": "string" }
+ },
+ "exclude_filter": {
+ "type": "array",
+ "items": { "type": "string" }
+ },
+ "sync": { "type": "boolean" },
+ "valgrind": { "type": "boolean" },
+ "monitored": { "type": "boolean" },
+ "dmesg": { "type": "boolean" },
+ "execute": { "type": "boolean" },
+ "concurrent": { "enum": ["none", "all", "some"] },
+ "platform": { "type": "string" },
+ "log_level": { "type": "string" },
+ "env": {
+ "description": "Environment variables that must be specified",
+ "type": "object",
+ "additionalProperties": { "type": "string" }
+ },
+ "profile": {
+ "type": "array",
+ "items": { "type": "string" }
+ }
+ }
+ },
+ "totals": {
+ "type": "object",
+ "description": "A calculation of the group totals.",
+ "additionalProperties": {
+ "type": "object",
+ "properties": {
+ "crash": { "type": "number" },
+ "dmesg-fail": { "type": "number" },
+ "dmesg-warn": { "type": "number" },
+ "fail": { "type": "number" },
+ "incomplete": { "type": "number" },
+ "notrun": { "type": "number" },
+ "pass": { "type": "number" },
+ "skip": { "type": "number" },
+ "timeout": { "type": "number" },
+ "warn": { "type": "number" }
+ },
+ "additionalProperties": false,
+ "required": [ "crash", "dmesg-fail", "dmesg-warn", "fail", "incomplete", "notrun", "pass", "skip", "timeout", "warn" ]
+ }
+ },
+ "tests": {
+ "type": "object",
+ "additionalProperties": {
+ "type": "object",
+ "properties": {
+ "__type__": { "type": "string" },
+ "err": { "type": "string" },
+ "exception": { "type": ["string", "null"] },
+ "result": {
+ "type": "string",
+ "enum": [ "pass", "fail", "crash", "warn", "incomplete", "notrun", "skip", "dmesg-warn", "dmesg-fail" ]
+ },
+ "environment": { "type": "string" },
+ "command": { "type": "string" },
+ "traceback": { "type": ["string", "null"] },
+ "out": { "type": "string" },
+ "dmesg": { "type": "string" },
+ "pid": {
+ "type": "array",
+ "items": { "type": "number" }
+ },
+ "returncode": { "type": [ "number", "null" ] },
+ "time": { "$ref": "#/definitions/timeAttribute" },
+ "subtests": {
+ "type": "object",
+ "properties": { "__type__": { "type": "string" } },
+ "additionalProperties": { "type": "string" },
+ "required": [ "__type__" ]
+ },
+ "images": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "description": { "type": "string" },
+ "reference": { "type": "string" },
+ "rendered": { "type": "string" }
+ },
+ "required": [ "description" ],
+ "additionalProperties": false
+ }
+ }
+ },
+ "additionalProperties": false
+ }
+ }
+ },
+ "additionalProperties": false,
+ "required": [ "__type__", "clinfo", "glxinfo", "lspci", "wglinfo", "name", "results_version", "uname", "time_elapsed", "tests" ],
+ "definitions": {
+ "timeAttribute": {
+ "type": "object",
+ "description": "An element containing a start and end time",
+ "properties": {
+ "__type__": { "type": "string" },
+ "start": { "type": "number" },
+ "end": { "type": "number" }
+ },
+ "additionalProperties": false,
+ "required": [ "__type__", "start", "end" ]
+ }
+ }
+}
diff --git a/unittests/framework/backends/shared.py b/unittests/framework/backends/shared.py
index d9f5790..5232053 100644
--- a/unittests/framework/backends/shared.py
+++ b/unittests/framework/backends/shared.py
@@ -42,7 +42,7 @@ INITIAL_METADATA = {
# changes. This does not contain piglit specifc objects, only strings, floats,
# ints, and Nones (instead of JSON's null)
JSON = {
- "results_version": 9,
+ "results_version": 10,
"time_elapsed": {
"start": 1469638791.2351687,
"__type__": "TimeAttribute",
diff --git a/unittests/framework/backends/test_json_update.py b/unittests/framework/backends/test_json_update.py
index c8e3ee6..b09413d 100644
--- a/unittests/framework/backends/test_json_update.py
+++ b/unittests/framework/backends/test_json_update.py
@@ -23,6 +23,7 @@
from __future__ import (
absolute_import, division, print_function, unicode_literals
)
+import base64
import os
try:
import simplejson as json
@@ -201,3 +202,103 @@ class TestV8toV9(object):
jsonschema.validate(
json.loads(json.dumps(result, default=backends.json.piglit_encoder)),
schema)
+
+
+class TestV9toV10(object):
+ """Tests for Version 9 to version 10."""
+
+ ref = os.path.join(os.path.dirname(__file__), 'images', 'ref.png')
+ render = os.path.join(os.path.dirname(__file__), 'images', 'render.png')
+
+ data = {
+ "results_version": 9,
+ "name": "test",
+ "options": {
+ "profile": ['quick'],
+ "dmesg": False,
+ "verbose": False,
+ "platform": "gbm",
+ "sync": False,
+ "valgrind": False,
+ "filter": [],
+ "concurrent": "all",
+ "test_count": 0,
+ "exclude_tests": [],
+ "exclude_filter": [],
+ "env": {},
+ },
+ "lspci": "stuff",
+ "uname": "more stuff",
+ "glxinfo": "and stuff",
+ "wglinfo": "stuff",
+ "clinfo": "stuff",
+ "tests": {
+ 'a at test': {
+ "time": {
+ 'start': 1.2,
+ 'end': 1.8,
+ '__type__': 'TimeAttribute'
+ },
+ 'dmesg': '',
+ 'result': 'fail',
+ '__type__': 'TestResult',
+ 'command': '/a/command',
+ 'traceback': None,
+ 'out': '',
+ 'environment': 'A=variable',
+ 'returncode': 0,
+ 'err': '',
+ 'pid': [5],
+ 'subtests': {
+ '__type__': 'Subtests',
+ },
+ 'exception': None,
+ 'images': [
+ {
+ 'image_desc': "Test image",
+ 'image_ref': os.path.join(os.path.dirname(__file__),
+ 'images', 'ref.png'),
+ 'image_render': os.path.join(os.path.dirname(__file__),
+ 'images', 'render.png'),
+ },
+ ],
+ }
+ },
+ "time_elapsed": {
+ 'start': 1.2,
+ 'end': 1.8,
+ '__type__': 'TimeAttribute'
+ },
+ '__type__': 'TestrunResult',
+ }
+
+ @pytest.fixture
+ def result(self, tmpdir):
+ p = tmpdir.join('result.json')
+ p.write(json.dumps(self.data, default=backends.json.piglit_encoder))
+ with p.open('r') as f:
+ return backends.json._update_nine_to_ten(backends.json._load(f))
+
+ def test_images(self, result):
+ """Converts the images attribute correctly."""
+ test = result['tests']['a at test']['images'][0]
+
+ with open(self.ref, 'rb') as f:
+ ref = base64.b64encode(f.read()).decode('utf-8')
+ assert test['reference'] == ref
+
+ with open(self.render, 'rb') as f:
+ rend = base64.b64encode(f.read()).decode('utf-8')
+ assert test['rendered'] == rend
+
+ assert test['description'] == \
+ self.data['tests']['a at test']['images'][0]['image_desc']
+
+ def test_valid(self, result):
+ with open(os.path.join(os.path.dirname(__file__), 'schema',
+ 'piglit-10.json'),
+ 'r') as f:
+ schema = json.load(f)
+ jsonschema.validate(
+ json.loads(json.dumps(result, default=backends.json.piglit_encoder)),
+ schema)
--
git-series 0.8.10
More information about the Piglit
mailing list