[Piglit] [PATCH 2/3] framework: Drop support for JSON formats < 7
Dylan Baker
dylan at pnwbakers.com
Tue Oct 11 00:06:53 UTC 2016
Most of these are pretty old at this point (over a year), and it seems
unlikely anyone still has results in those formats. The biggest problem
with them is that some of them (especially the early ones) are complex
transformations with a lot of validation needed, and some of the middle
ones rely on the class representations to do the transformations, which
is what this series is trying to change, since the tight coupling makes
changes to the schema difficult, or nearly impossible in other cases.
If there is compelling reason I could push support back a bit more, but
I really want to drop some of the versions we support since it
simplifies the code considerably.
This also drops support for the older "main" file, and only supports
results.json and results.json.<compression>
---
framework/backends/json.py | 264 +-------
unittests/framework/backends/test_json.py | 9 -
unittests/framework/backends/test_json_update.py | 738 -----------------------
3 files changed, 9 insertions(+), 1002 deletions(-)
diff --git a/framework/backends/json.py b/framework/backends/json.py
index cf7f119..ff91721 100644
--- a/framework/backends/json.py
+++ b/framework/backends/json.py
@@ -55,6 +55,9 @@ __all__ = [
# The current version of the JSON results
CURRENT_JSON_VERSION = 9
+# The minimum JSON format supported
+MINIMUM_SUPPORTED_VERSION = 7
+
# The level to indent a final file
INDENT = 4
@@ -242,9 +245,7 @@ def load_results(filename, compression_):
else:
# Look for a compressed result first, then a bare result, finally for
# an old main file
- for name in ['results.json.{}'.format(compression_),
- 'results.json',
- 'main']:
+ for name in ['results.json.{}'.format(compression_), 'results.json']:
if os.path.exists(os.path.join(filename, name)):
filepath = os.path.join(filename, name)
break
@@ -331,13 +332,6 @@ def _update_results(results, filepath):
# Python lacks a switch statement, the workaround is to use a
# dictionary
updates = {
- 0: _update_zero_to_one,
- 1: _update_one_to_two,
- 2: _update_two_to_three,
- 3: _update_three_to_four,
- 4: _update_four_to_five,
- 5: _update_five_to_six,
- 6: _update_six_to_seven,
7: _update_seven_to_eight,
8: _update_eight_to_nine,
}
@@ -347,10 +341,11 @@ def _update_results(results, filepath):
return results
- # If there is no version, then set it to 0, this will trigger a full
- # update.
- if not hasattr(results, 'results_version'):
- results.results_version = 0
+ if results.results_version < MINIMUM_SUPPORTED_VERSION:
+ raise exceptions.PiglitFatalError(
+ 'Unsupported version "{}", '
+ 'minimum supported version is "{}"'.format(
+ results.results_version, MINIMUM_SUPPORTED_VERSION))
# If the results version is the current version there is no need to
# update, just return the results
@@ -376,247 +371,6 @@ def _write(results, file_):
json.dump(results, f, default=piglit_encoder, indent=INDENT)
-def _update_zero_to_one(result):
- """ Update version zero results to version 1 results
-
- Changes from version 0 to version 1
-
- - dmesg is sometimes stored as a list, sometimes stored as a string. In
- version 1 it is always stored as a string
- - in version 0 subtests are sometimes stored as duplicates, sometimes stored
- only with a single entry, in version 1 tests with subtests are only
- recorded once, always.
- - Version 0 can have an info entry, or returncode, out, and err entries,
- Version 1 will only have the latter
- - version 0 results are called 'main', while version 1 results are called
- 'results.json' (This is not handled internally, it's either handled by
- update_results() which will write the file back to disk, or needs to be
- handled manually by the user)
-
- """
- updated_results = {}
- remove = set()
-
- for name, test in six.iteritems(result.tests):
- assert not isinstance(test, results.TestResult), \
- 'Test was erroniaously turned into a TestResult'
-
- # fix dmesg errors if any
- if isinstance(test.get('dmesg'), list):
- test['dmesg'] = '\n'.join(test['dmesg'])
-
- # If a test as an info attribute, we want to remove it, if it doesn't
- # have a returncode, out, or attribute we'll want to get those out of
- # info first
- #
- # This expects that the order of info is roughly returncode, errors,
- # output, *extra it can handle having extra information in the middle,
- if (None in [test.get('out'), test.get('err'),
- test.get('returncode')] and test.get('info')):
-
- # This attempts to split everything before Errors: as a returncode,
- # and everything before Output: as Errors, and everything else as
- # output. This may result in extra info being put in out, this is
- # actually fine since out is only parsed by humans.
- returncode, split = test['info'].split('\n\nErrors:')
- err, out = split.split('\n\nOutput:')
-
- # returncode can be 0, and 0 is falsy, so ensure it is actually
- # None
- if test.get('returncode') is None:
- # In some cases the returncode might not be set (like the test
- # skipped), in that case it will be None, so set it
- # appropriately
- try:
- test['returncode'] = int(
- returncode[len('returncode: '):].strip())
- except ValueError:
- test['returncode'] = None
- if not test.get('err'):
- test['err'] = err.strip()
- if not test.get('out'):
- test['out'] = out.strip()
-
- # Remove the unused info key
- if test.get('info'):
- del test['info']
-
- # If there is more than one subtest written in version 0 results that
- # entry will be a complete copy of the original entry with '/{name}'
- # appended. This loop looks for tests with subtests, removes the
- # duplicate entries, and creates a new entry in update_results for the
- # single full tests.
- #
- # this must be the last thing done in this loop, or there will be pain
- if test.get('subtest'):
- for sub in six.iterkeys(test['subtest']):
- # adding the leading / ensures that we get exactly what we
- # expect, since endswith does a character by character match, if
- # the subtest name is duplicated it won't match, and if there
- # are more trailing characters it will not match
- #
- # We expect duplicate names like this:
- # "group1/groupA/test1/subtest 1": <thing>,
- # "group1/groupA/test1/subtest 2": <thing>,
- # "group1/groupA/test1/subtest 3": <thing>,
- # "group1/groupA/test1/subtest 4": <thing>,
- # "group1/groupA/test1/subtest 5": <thing>,
- # "group1/groupA/test1/subtest 6": <thing>,
- # "group1/groupA/test1/subtest 7": <thing>,
- # but what we want is groupg1/groupA/test1 and none of the
- # subtest as keys in the dictionary at all
- if name.endswith('/{0}'.format(sub)):
- testname = name[:-(len(sub) + 1)] # remove leading /
- assert testname[-1] != '/'
-
- remove.add(name)
- break
- else:
- # This handles two cases, first that the results have only
- # single entries for each test, regardless of subtests (new
- # style), or that the test onhly as a single subtest and thus
- # was recorded correctly
- testname = name
-
- if testname not in updated_results:
- updated_results[testname] = test
-
- for name in remove:
- del result.tests[name]
- result.tests.update(updated_results)
-
- # set the results version
- result.results_version = 1
-
- return result
-
-
-def _update_one_to_two(results):
- """Update version 1 results to version 2.
-
- Version two results are actually identical to version one results, however,
- there was an error in version 1 at the end causing metadata in the options
- dictionary to be incorrect. Version 2 corrects that.
-
- Namely uname, glxinfo, wglinfo, and lspci were put in the options['env']
- instead of in the root.
-
- """
- if 'env' in results.options:
- env = results.options['env']
- if env.get('glxinfo'):
- results.glxinfo = env['glxinfo']
- if env.get('lspci'):
- results.lspci = env['lspci']
- if env.get('uname'):
- results.uname = env['uname']
- if env.get('wglinfo'):
- results.wglinfo = env['wglinfo']
- del results.options['env']
-
- results.results_version = 2
-
- return results
-
-
-def _update_two_to_three(results):
- """Lower key names."""
- for key, value in results.tests.items():
- lowered = key.lower()
- if not key == lowered:
- results.tests[lowered] = value
- del results.tests[key]
-
- results.results_version = 3
-
- return results
-
-
-def _update_three_to_four(results):
- """Update results v3 to v4.
-
- This update requires renaming a few tests. The complete lists can be found
- in framework/data/results_v3_to_v4.json, a json file containing a list of
- lists (They would be tuples if json has tuples), the first element being
- the original name, and the second being a new name to update to
-
- """
- mapped_updates = [
- ("spec/arb_texture_rg/fs-shadow2d-red-01",
- "spec/arb_texture_rg/execution/fs-shadow2d-red-01"),
- ("spec/arb_texture_rg/fs-shadow2d-red-02",
- "spec/arb_texture_rg/execution/fs-shadow2d-red-02"),
- ("spec/arb_texture_rg/fs-shadow2d-red-03",
- "spec/arb_texture_rg/execution/fs-shadow2d-red-03"),
- ("spec/arb_draw_instanced/draw-non-instanced",
- "spec/arb_draw_instanced/execution/draw-non-instanced"),
- ("spec/arb_draw_instanced/instance-array-dereference",
- "spec/arb_draw_instanced/execution/instance-array-dereference"),
- ]
-
- for original, new in mapped_updates:
- if original in results.tests:
- results.tests[new] = results.tests[original]
- del results.tests[original]
-
- # This needs to use posixpath rather than grouptools because version 4 uses
- # / as a separator, but grouptools isn't guaranteed to do so
- for test, result in results.tests.items():
- if posixpath.dirname(test) == 'glslparsertest':
- group = posixpath.join('glslparsertest/shaders',
- posixpath.basename(test))
- results.tests[group] = result
- del results.tests[test]
-
- results.results_version = 4
-
- return results
-
-
-def _update_four_to_five(results):
- """Updates json results from version 4 to version 5."""
- new_tests = {}
-
- for name, test in six.iteritems(results.tests):
- new_tests[name.replace('/', '@').replace('\\', '@')] = test
-
- results.tests = new_tests
- results.results_version = 5
-
- return results
-
-
-def _update_five_to_six(result):
- """Updates json results from version 5 to 6.
-
- This uses a special field to for marking TestResult instances, rather than
- just checking for fields we expect.
-
- """
- new_tests = {}
-
- for name, test in six.iteritems(result.tests):
- new_tests[name] = results.TestResult.from_dict(test)
-
- result.tests = new_tests
- result.results_version = 6
-
- return result
-
-
-def _update_six_to_seven(result):
- """Update json results from version 6 to 7.
-
- Version 7 results always contain the totals member.
-
- """
- if not result.totals:
- result.calculate_group_totals()
- result.results_version = 7
-
- return result
-
-
def _update_seven_to_eight(result):
"""Update json results from version 7 to 8.
diff --git a/unittests/framework/backends/test_json.py b/unittests/framework/backends/test_json.py
index 0c49aff..056e2c1 100644
--- a/unittests/framework/backends/test_json.py
+++ b/unittests/framework/backends/test_json.py
@@ -234,15 +234,6 @@ class TestResume(object):
class TestLoadResults(object):
"""Tests for the load_results function."""
- def test_folder_with_main(self, tmpdir):
- """backends.json.load_results: takes a folder with a file named main in
- it.
- """
- p = tmpdir.join('main')
- with p.open('w') as f:
- f.write(json.dumps(shared.JSON))
- backends.json.load_results(six.text_type(tmpdir), 'none')
-
def test_folder_with_results_json(self, tmpdir):
"""backends.json.load_results: takes a folder with a file named
results.json.
diff --git a/unittests/framework/backends/test_json_update.py b/unittests/framework/backends/test_json_update.py
index 501a895..db45c1b 100644
--- a/unittests/framework/backends/test_json_update.py
+++ b/unittests/framework/backends/test_json_update.py
@@ -24,7 +24,6 @@ from __future__ import (
absolute_import, division, print_function, unicode_literals
)
import os
-import copy
try:
import simplejson as json
except ImportError:
@@ -36,13 +35,10 @@ except ImportError:
import jsonschema
import pytest
-import six
from framework import backends
from framework import results
-from . import shared
-
# pylint: disable=protected-access,no-self-use
@@ -53,740 +49,6 @@ def setup_module():
yield
-class TestV0toV1(object):
- """Tests for version 0 -> version 1 of json results format."""
-
- # NOTE: It is very important to NOT use grouptools in this class.
- # The grouptools module changes and is updated from time to time, but this
- # file must remain static. It tests the update from one static version to
- # another static version.
- data = {
- 'options': {
- 'profile': "tests/fake.py",
- 'filter': [],
- 'exclude_filter': [],
- },
- 'name': 'fake-tests',
- 'lspci': 'fake',
- 'glxinfo': 'fake',
- 'tests': {
- 'sometest': {
- 'result': 'pass',
- 'time': 0.01,
- 'dmesg': ['this', 'is', 'dmesg'],
- 'info': 'Returncode: 1\n\nErrors:stderr\n\nOutput: stdout\n',
- },
- 'group1/groupA/test/subtest 1': {
- 'info': 'Returncode: 1\n\nErrors:stderr\n\nOutput:stdout\n',
- 'subtest': {
- 'subtest 1': 'pass',
- 'subtest 2': 'pass'
- },
- 'returncode': 0,
- 'command': 'this is a command',
- 'result': 'pass',
- 'time': 0.1
- },
- 'group1/groupA/test/subtest 2': {
- 'info': 'Returncode: 1\n\nErrors:stderr\n\nOutput:stdout\n',
- 'subtest': {
- 'subtest 1': 'pass',
- 'subtest 2': 'pass'
- },
- 'returncode': 0,
- 'command': 'this is a command',
- 'result': 'pass',
- 'time': 0.1
- },
- 'single/test/thing': {
- 'info': 'Returncode: 1\n\nErrors:stderr\n\nOutput:stdout\n',
- 'subtest': {
- 'subtest 1': 'pass',
- },
- 'returncode': 0,
- 'command': 'this is a command',
- 'result': 'pass',
- 'time': 0.1
- },
- 'group2/groupA/test/subtest 1/depth': {
- 'info': 'Returncode: 1\n\nErrors:stderr\n\nOutput:stdout\n',
- 'subtest': {
- 'subtest 1/depth': 'pass',
- 'subtest 2/float': 'pass'
- },
- 'returncode': 0,
- 'command': 'this is a command',
- 'result': 'pass',
- 'time': 0.1
- },
- 'group2/groupA/test/subtest 2/float': {
- 'info': 'Returncode: 1\n\nErrors:stderr\n\nOutput:stdout\n',
- 'subtest': {
- 'subtest 1/depth': 'pass',
- 'subtest 2/float': 'pass'
- },
- 'returncode': 0,
- 'command': 'this is a command',
- 'result': 'pass',
- 'time': 0.1
- },
- 'group3/groupA/test': {
- 'info': 'Returncode: 1\n\nErrors:stderr\n\nOutput:stdout\n',
- 'subtest': {
- 'subtest 1': 'pass',
- 'subtest 2': 'pass',
- 'subtest 3': 'pass',
- },
- 'returncode': 0,
- 'command': 'this is a command',
- 'result': 'pass',
- 'time': 0.1
- }
- }
- }
-
- @pytest.fixture
- def result(self, tmpdir):
- p = tmpdir.join('results.json')
- p.write(json.dumps(self.data))
- with p.open('r') as f:
- return backends.json._update_zero_to_one(backends.json._load(f))
-
- def test_dmesg(self, result):
- """backends.json.update_results (0 -> 1): dmesg is converted from a
- list to a string.
- """
- assert result.tests['sometest']['dmesg'] == 'this\nis\ndmesg'
-
- def test_subtests_remove_duplicates(self, result):
- """backends.json.update_results (0 -> 1): Removes duplicate entries"""
- assert 'group1/groupA/test/subtest 1' not in result.tests
- assert 'group1/groupA/test/subtest 2' not in result.tests
-
- def test_subtests_add_test(self, result):
- """backends.json.update_results (0 -> 1): Add an entry for the actual
- test.
- """
- assert result.tests.get('group1/groupA/test')
-
- def test_subtests_is_testresult(self, result):
- """backends.json.update_results (0 -> 1): The result of the new test is
- a dict Instance.
- """
- assert isinstance(result.tests['group1/groupA/test'], dict)
-
- def test_info_delete(self, result):
- """backends.json.update_results (0 -> 1): Remove the info name from
- results.
- """
- for value in six.itervalues(result.tests):
- assert 'info' not in value
-
- def test_returncode_from_info(self, result):
- """backends.json.update_results (0 -> 1): Use the returncode from info
- if there is no returncode.
- """
- assert result.tests['sometest']['returncode'] == 1
-
- def test_returncode_no_override(self, result):
- """backends.json.update_results (0 -> 1): Do not clobber returncode
- with info.
-
- The returncode from info should not overwrite an existing returcnode
- attribute, this test only tests that the value in info isn't used when
- there is a value in returncode already.
- """
- assert result.tests['group1/groupA/test']['returncode'] != 1
-
- def test_err_from_info(self, result):
- """backends.json.update_results (0 -> 1): add an err attribute from
- info.
- """
- assert result.tests['group1/groupA/test']['err'] == 'stderr'
-
- def test_out_from_info(self, result):
- """backends.json.update_results (0 -> 1): add an out attribute from
- info.
- """
- assert result.tests['group1/groupA/test']['out'] == 'stdout'
-
- def test_set_version(self, result):
- """backends.json.update_results (0 -> 1): Set the version to 1"""
- assert result.results_version == 1
-
- def test_dont_break_single_subtest(self, result):
- """backends.json.update_results (0 -> 1): Don't break single subtest
- entries.
-
- A test with a single subtest was written correctly before, don't break
- it by removing the name of the test. ex:
- test/foo/bar: {
- ...
- subtest: {
- 1x1: pass
- }
- }
-
- should remain test/foo/bar since bar is the name of the test not a
- subtest
- """
- assert result.tests.get('single/test/thing')
-
- def test_subtests_with_slash(self, result):
- """backends.json.update_results (0 -> 1): Subtest names with /'s are
- handled correctly.
- """
- expected = 'group2/groupA/test/subtest 1'
- assert expected not in result.tests
-
- def test_handle_fixed_subtests(self, result):
- """backends.json.update_results (0 -> 1): Correctly handle new single
- entry subtests correctly.
- """
- assert 'group3/groupA/test' in result.tests
-
- def test_load_results_unversioned(self, tmpdir):
- """backends.json.load_results: Loads unversioned results and updates .
- correctly.
-
- This test pickes on attribute (dmesg) to test, with the assumption taht
- if the other tests work then once the update path starts it runs
- correctly.
- """
- p = tmpdir.join('results.json')
- p.write(json.dumps(self.data, default=backends.json.piglit_encoder))
- result = backends.json.load_results(six.text_type(p), 'none')
-
- assert result.tests['sometest'].dmesg == 'this\nis\ndmesg'
-
- def test_load_results_v0(self, tmpdir):
- """backends.json.load_results: Loads results v0 and updates correctly.
-
- This test pickes on attribute (dmesg) to test, with the assumption taht
- if the other tests work then once the update path starts it runs
- correctly.
- """
- data = copy.deepcopy(self.data)
- data['results_version'] = 0
-
- p = tmpdir.join('results.json')
- p.write(json.dumps(data, default=backends.json.piglit_encoder))
- result = backends.json.load_results(six.text_type(p), 'none')
-
- assert result.tests['sometest'].dmesg == 'this\nis\ndmesg'
-
- def test_info_split(self, tmpdir):
- """backends.json.update_results (0 -> 1): info can split into any
- number of elements.
- """
- data = copy.copy(self.data)
- data['tests']['sometest']['info'] = \
- 'Returncode: 1\n\nErrors:stderr\n\nOutput: stdout\n\nmore\n\nstuff'
-
- p = tmpdir.join('results.json')
- p.write(json.dumps(data, default=backends.json.piglit_encoder))
- with p.open('r') as f:
- backends.json._update_zero_to_one(backends.json._load(f))
-
- def test_load_results(self, tmpdir):
- """backends.json.update_results (1 -> current): load_results properly
- updates.
- """
- p = tmpdir.join('results.json')
- p.write(json.dumps(self.data, default=backends.json.piglit_encoder))
- result = backends.json.load_results(six.text_type(p), 'none')
- assert result.results_version == backends.json.CURRENT_JSON_VERSION # pylint: disable=no-member
-
-
-class TestV1toV2(object):
- """Tests version 1 to version 2."""
-
- class TestWithChanges(object):
- """Test V1 to V2 of results."""
- data = {
- "results_version": 1,
- "name": "test",
- "options": {
- "profile": ['quick'],
- "dmesg": False,
- "verbose": False,
- "platform": "gbm",
- "sync": False,
- "valgrind": False,
- "filter": [],
- "concurrent": "all",
- "test_count": 0,
- "exclude_tests": [],
- "exclude_filter": [],
- "env": {
- "lspci": "stuff",
- "uname": "more stuff",
- "glxinfo": "and stuff",
- "wglinfo": "stuff"
- }
- },
- "tests": {
- "test/is/a/test": {
- "returncode": 0,
- "err": None,
- "environment": None,
- "command": "foo",
- "result": "skip",
- "time": 0.123,
- "out": None,
- }
- }
- }
-
- @pytest.fixture
- def result(self, tmpdir):
- p = tmpdir.join('result.json')
- p.write(json.dumps(self.data, default=backends.json.piglit_encoder))
- with p.open('r') as f:
- return backends.json._update_one_to_two(backends.json._load(f))
-
- def test_version_is_two(self, result):
- """backends.json.update_results (1 -> 2): The result version is updated
- to 2.
- """
- assert result.results_version == 2
-
- def test_no_env(self, result):
- """backends.json.update_results (1 -> 2): Removes options['env']."""
- assert 'env' not in result.options
-
- def test_glxinfo(self, result):
- """backends.json.update_results (1 -> 2): puts glxinfo in the root."""
- assert result.glxinfo == 'and stuff'
-
- def test_lspci(self, result):
- """backends.json.update_results (1 -> 2): puts lspci in the root."""
- assert result.lspci == 'stuff'
-
- def test_uname(self, result):
- """backends.json.update_results (1 -> 2): puts uname in the root."""
- assert result.uname == 'more stuff'
-
- def test_wglinfo(self, result):
- """backends.json.update_results (1 -> 2): puts wglinfo in the root."""
- assert result.wglinfo == 'stuff'
-
- class TestWithoutChanges(object):
- """Test a version 1 to 2 update when version 1 was correct"""
- data = {
- "results_version": 1,
- "name": "test",
- "lspci": "stuff",
- "uname": "more stuff",
- "glxinfo": "and stuff",
- "wglinfo": "stuff",
- "options": {
- "profile": ['quick'],
- "dmesg": False,
- "verbose": False,
- "platform": "gbm",
- "sync": False,
- "valgrind": False,
- "filter": [],
- "concurrent": "all",
- "test_count": 0,
- "exclude_tests": [],
- "exclude_filter": [],
- },
- "tests": {
- "test/is/a/test": {
- "returncode": 0,
- "err": None,
- "environment": None,
- "command": "foo",
- "result": "skip",
- "time": 0.123,
- "out": None,
- }
- }
- }
-
- @pytest.fixture
- def result(self, tmpdir):
- p = tmpdir.join('result.json')
- p.write(json.dumps(self.data, default=backends.json.piglit_encoder))
- with p.open('r') as f:
- return backends.json._update_one_to_two(backends.json._load(f))
-
- def test_version_is_two(self, result):
- """backends.json.update_results (1 -> 2) no change: The result version
- is updated to 2.
- """
- assert result.results_version == 2
-
- def test_glxinfo(self, result):
- """backends.json.update_results (1 -> 2) no change: doesn't clobber
- glxinfo.
- """
- assert result.glxinfo == 'and stuff'
-
- def test_lspci(self, result):
- """backends.json.update_results (1 -> 2) no change: doesn't clobber
- lspci.
- """
- assert result.lspci == 'stuff'
-
- def test_uname(self, result):
- """backends.json.update_results (1 -> 2) no change: doesn't clobber
- uname.
- """
- assert result.uname == 'more stuff'
-
- def test_wglinfo(self, result):
- """backends.json.update_results (1 -> 2) no change: doesn't clobber
- wglinfo.
- """
- assert result.wglinfo == 'stuff'
-
-
-class TestV2toV3(object):
- """Tests for version 2 -> version 3 of json results"""
- # NOTE: do NOT use grouptools in this class, see v0 tests for explanation
- data = {
- "results_version": 2,
- "name": "test",
- "options": {
- "profile": ['quick'],
- "dmesg": False,
- "verbose": False,
- "platform": "gbm",
- "sync": False,
- "valgrind": False,
- "filter": [],
- "concurrent": "all",
- "test_count": 0,
- "exclude_tests": [],
- "exclude_filter": [],
- "env": {
- "lspci": "stuff",
- "uname": "more stuff",
- "glxinfo": "and stuff",
- "wglinfo": "stuff"
- }
- },
- "tests": {
- "test/is/a/test": {
- "returncode": 0,
- "err": None,
- "environment": None,
- "command": "foo",
- "result": "skip",
- "time": 0.123,
- "out": None,
- },
- "Test/Is/SomE/Other1/Test": {
- "returncode": 0,
- "err": None,
- "environment": None,
- "command": "foo",
- "result": "skip",
- "time": 0.123,
- "out": None,
- }
- }
- }
-
- @pytest.fixture
- def result(self, tmpdir):
- p = tmpdir.join('result.json')
- p.write(json.dumps(self.data, default=backends.json.piglit_encoder))
- with p.open('r') as f:
- return backends.json._update_two_to_three(backends.json._load(f))
-
- def test_unchanged(self, result):
- """backends.json.update_results (2 -> 3): results with no caps are not
- mangled.
- """
- assert 'test/is/a/test' in result.tests
-
- def test_lower(self, result):
- """backends.json.update_results (2 -> 3): results with caps are
- lowered.
- """
- assert 'test/is/some/other1/test' in result.tests
-
- def test_removed(self, result):
- """backends.json.update_results (2 -> 3): results with caps are
- removed.
- """
- assert 'Test/Is/SomE/Other1/Test' not in result.tests
-
-
-class TestV3toV4(object):
- """Tests for version 3 to version 4."""
-
- # NOTE: do NOT use grouptools in this module, see v0 tests for explanation
- test_data = {
- 'returncode': 0,
- 'err': None,
- 'environment': None,
- 'command': 'foo',
- 'result': 'skip',
- 'time': 0.123,
- 'out': None,
- }
-
- data = {
- "results_version": 3,
- "name": "test",
- "options": {
- "profile": ['quick'],
- "dmesg": False,
- "verbose": False,
- "platform": "gbm",
- "sync": False,
- "valgrind": False,
- "filter": [],
- "concurrent": "all",
- "test_count": 0,
- "exclude_tests": [],
- "exclude_filter": [],
- "env": {
- "lspci": "stuff",
- "uname": "more stuff",
- "glxinfo": "and stuff",
- "wglinfo": "stuff"
- }
- },
- "tests": {
- "spec/arb_texture_rg/fs-shadow2d-red-01": test_data,
- "spec/arb_texture_rg/fs-shadow2d-red-02": test_data,
- "spec/arb_texture_rg/fs-shadow2d-red-03": test_data,
- "spec/arb_draw_instanced/draw-non-instanced": test_data,
- "spec/arb_draw_instanced/instance-array-dereference": test_data,
- "glslparsertest/foo": test_data,
- }
- }
-
- old = list(data['tests'].keys())
- new = [
- "spec/arb_texture_rg/execution/fs-shadow2d-red-01",
- "spec/arb_texture_rg/execution/fs-shadow2d-red-02",
- "spec/arb_texture_rg/execution/fs-shadow2d-red-03",
- "spec/arb_draw_instanced/execution/draw-non-instanced",
- "spec/arb_draw_instanced/execution/instance-array-dereference",
- ]
-
- @pytest.fixture
- def result(self, tmpdir):
- p = tmpdir.join('result.json')
- p.write(json.dumps(self.data, default=backends.json.piglit_encoder))
- with p.open('r') as f:
- return backends.json._update_three_to_four(backends.json._load(f))
-
- def test_old_removed(self, result):
- """backends.json.update_results (3 -> 4): All old test names are
- removed.
- """
- for old in self.old:
- assert old not in result.tests
-
- def test_new_added(self, result):
- """backends.json.update_results (3 -> 4): All new test names are added.
- """
- for new in self.new:
- assert new in result.tests
-
- def test_new_has_data(self, result):
- """backends.json.update_results (3 -> 4): All new tests have expected
- data.
- """
- for new in self.new:
- assert result.tests[new] == self.test_data
-
- def test_missing(self, tmpdir):
- """backends.json.update_results (3 -> 4): updates successfully when
- tests to rename are not present.
- """
- data = copy.copy(self.data)
- del data['tests']['spec/arb_draw_instanced/instance-array-dereference']
-
- p = tmpdir.join('result.json')
- p.write(json.dumps(self.data, default=backends.json.piglit_encoder))
-
- with p.open('r') as f:
- backends.json._update_three_to_four(backends.json._load(f))
-
-
-class TestV4toV5(object):
- test_data = {
- 'returncode': 0,
- 'err': None,
- 'environment': None,
- 'command': 'foo',
- 'result': 'skip',
- 'time': 0.123,
- 'out': None,
- }
-
- data = {
- "results_version": 4,
- "name": "test",
- "options": {
- "profile": ['quick'],
- "dmesg": False,
- "verbose": False,
- "platform": "gbm",
- "sync": False,
- "valgrind": False,
- "filter": [],
- "concurrent": "all",
- "test_count": 0,
- "exclude_tests": [],
- "exclude_filter": [],
- "env": {
- "lspci": "stuff",
- "uname": "more stuff",
- "glxinfo": "and stuff",
- "wglinfo": "stuff"
- }
- },
- "tests": {
- "a/test/group/of/great/length": test_data,
- "has\\windows": test_data,
- }
- }
-
- old = list(data['tests'].keys())
- new = [
- 'a at test@group at of@great at length',
- 'has at windows',
- ]
-
- @pytest.fixture
- def result(self, tmpdir):
- p = tmpdir.join('result.json')
- p.write(json.dumps(self.data, default=backends.json.piglit_encoder))
- with p.open('r') as f:
- return backends.json._update_four_to_five(backends.json._load(f))
-
- def test_posix_removed(self, result):
- """backends.json.update_results (4 -> 5): / is replaced with @."""
- assert 'a/test/of/great/length' not in result.tests
-
- def test_win_removed(self, result):
- """backends.json.update_results (4 -> 5): \\ is replaced with @."""
- assert 'has\\windows' not in result.tests
-
- def test_new_added(self, result):
- """backends.json.update_results (4 -> 5): All new test names are added.
- """
- for new in self.new:
- assert new in result.tests
-
- def test_new_has_data(self, result):
- """backends.json.update_results (4 -> 5): All new tests have expected
- data.
- """
- for new in self.new:
- assert result.tests[new] == self.test_data
-
-
-class TestV5toV6(object):
- test_data = {
- 'returncode': 0,
- 'err': '',
- 'environment': None,
- 'command': 'foo',
- 'result': 'skip',
- 'time': 0.123,
- 'out': '',
- }
-
- data = {
- "results_version": 4,
- "name": "test",
- "options": {
- "profile": ['quick'],
- "dmesg": False,
- "verbose": False,
- "platform": "gbm",
- "sync": False,
- "valgrind": False,
- "filter": [],
- "concurrent": "all",
- "test_count": 0,
- "exclude_tests": [],
- "exclude_filter": [],
- "env": {
- "lspci": "stuff",
- "uname": "more stuff",
- "glxinfo": "and stuff",
- "wglinfo": "stuff"
- }
- },
- "tests": {
- 'a at test': test_data,
- }
- }
-
- @pytest.fixture
- def result(self, tmpdir):
- p = tmpdir.join('result.json')
- p.write(json.dumps(self.data, default=backends.json.piglit_encoder))
- with p.open('r') as f:
- return backends.json._update_five_to_six(backends.json._load(f))
-
- def test_result_is_testresult_instance(self, result):
- """backends.json.update_results (5 -> 6): A test result is converted to
- a TestResult instance.
- """
- assert isinstance(result.tests['a at test'], results.TestResult)
-
-
-class TestV6toV7(object):
- """Tests for version 6 to version 7."""
-
- data = {
- "results_version": 6,
- "name": "test",
- "options": {
- "profile": ['quick'],
- "dmesg": False,
- "verbose": False,
- "platform": "gbm",
- "sync": False,
- "valgrind": False,
- "filter": [],
- "concurrent": "all",
- "test_count": 0,
- "exclude_tests": [],
- "exclude_filter": [],
- "env": {
- "lspci": "stuff",
- "uname": "more stuff",
- "glxinfo": "and stuff",
- "wglinfo": "stuff"
- }
- },
- "tests": {
- 'a at test': results.TestResult('pass'),
- 'a at nother@test': results.TestResult('fail'),
- 'a at nother@thing': results.TestResult('crash'),
- }
- }
-
- @pytest.fixture
- def result(self, tmpdir):
- p = tmpdir.join('result.json')
- p.write(json.dumps(self.data, default=backends.json.piglit_encoder))
- with p.open('r') as f:
- return backends.json._update_six_to_seven(backends.json._load(f))
-
- def test_is_TestrunResult(self, result):
- """backends.json.update_results (6 -> 7): makes TestrunResult."""
- assert isinstance(result, results.TestrunResult)
-
- def test_totals(self, result):
- """backends.json.update_results (6 -> 7): Totals are populated."""
- assert result.totals != {}
-
-
class TestV7toV8(object):
"""Tests for Version 7 to version 8."""
--
2.10.0
More information about the Piglit
mailing list