[Piglit] [Patch v3 12/13] framework: Update results to use versioned numbers
Dylan Baker
baker.dylan.c at gmail.com
Fri Jun 20 12:50:38 PDT 2014
This patch updates our json to version 1. Changes from version 0 to
version 1 are as follows:
- renamed 'main' to 'results.json'
- dmesg must be a string (It was stored a list in some version 0
results)
- subtests are never stored as duplicate entries, a single instance of
the test is recorded, (In version 0 both are possible)
- there is no info entry in version 1, err, out, and returncode are
always split into seperate entries
This patch adds support to the results module for handling updates to
the results in a sane way. It does this by adding a result_version
attribute to the TestrunResult (which is stored as json), and
implementing the ability to incrementally update results between
versions.
It does this automatically on load, non-destructively, moving the old
results to results.json.old, but does write the updated results to disk,
making the cost of this update a one time cost.
v2: - Handle non-writable directories and files, this also fixes using
file-descriptors as inputs
Signed-off-by: Dylan Baker <baker.dylan.c at gmail.com>
---
framework/programs/run.py | 4 +-
framework/results.py | 131 +++++++++++++++++++++++++++++---
framework/tests/results_tests.py | 157 +++++++++++++++++++++++++++++++++++++--
framework/tests/utils.py | 2 +
4 files changed, 278 insertions(+), 16 deletions(-)
diff --git a/framework/programs/run.py b/framework/programs/run.py
index 9255f64..ff13974 100644
--- a/framework/programs/run.py
+++ b/framework/programs/run.py
@@ -157,7 +157,7 @@ def run(input_):
results.name = path.basename(args.results_path)
# Begin json.
- result_filepath = path.join(args.results_path, 'main')
+ result_filepath = path.join(args.results_path, 'results.json')
result_file = open(result_filepath, 'w')
json_writer = framework.results.JSONWriter(result_file)
@@ -212,7 +212,7 @@ def resume(input_):
if results.options.get('platform'):
env.env['PIGLIT_PLATFORM'] = results.options['platform']
- results_path = path.join(args.results_path, "main")
+ results_path = path.join(args.results_path, 'results.json')
json_writer = framework.results.JSONWriter(open(results_path, 'w+'))
json_writer.initialize_json(results.options, results.name,
core.collect_system_info())
diff --git a/framework/results.py b/framework/results.py
index 1edc423..48d4fc9 100644
--- a/framework/results.py
+++ b/framework/results.py
@@ -23,6 +23,7 @@
from __future__ import print_function
import os
+import sys
from cStringIO import StringIO
try:
import simplejson as json
@@ -39,6 +40,9 @@ __all__ = [
'load_results',
]
+# The current version of the JSON results
+CURRENT_JSON_VERSION = 1
+
def _piglit_encoder(obj):
""" Encoder for piglit that can transform additional classes into json
@@ -119,7 +123,7 @@ class JSONWriter(object):
self.__is_collection_empty = []
def initialize_json(self, options, name, env):
- """ Write boilerplate json code
+ """ Write boilerplate json code
This writes all of the json except the actuall tests.
@@ -132,6 +136,7 @@ class JSONWriter(object):
"""
self.open_dict()
+ self.write_dict_item('results_version', CURRENT_JSON_VERSION)
self.write_dict_item('name', name)
self.write_dict_key('options')
@@ -229,6 +234,7 @@ class TestrunResult(object):
'wglinfo',
'glxinfo',
'lspci',
+ 'results_version',
'time_elapsed']
self.name = None
self.uname = None
@@ -338,12 +344,119 @@ def load_results(filename):
"main"
"""
+ # This will load any file or file-like thing. That would include pipes and
+ # file descriptors
+ if not os.path.isdir(filename):
+ filepath = filename
+ else:
+ # If there are both old and new results in a directory pick the new
+ # ones first
+ if os.path.exists(os.path.join(filename, 'results.json')):
+ filepath = os.path.join(filename, 'results.json')
+ # Version 0 results are called 'main'
+ elif os.path.exists(os.path.join(filename, 'main')):
+ filepath = os.path.join(filename, 'main')
+ else:
+ raise Exception("No results found")
+
+ with open(filepath, 'r') as f:
+ testrun = TestrunResult(f)
+
+ return update_results(testrun, filepath)
+
+
+def update_results(results, filepath):
+ """ Update results to the lastest version
+
+ This function is a wraper for other update_* functions, providing
+ incremental updates from one version to another.
+
+ """
+ # If the results version is the current version there is no need to
+ # update, just return the results
+ if getattr(results, 'results_version', 0) == CURRENT_JSON_VERSION:
+ return results
+
+ # If there is no version then the results are version 0, and need to be
+ # updated to version 1
+ if getattr(results, 'results_version', False):
+ results = _update_zero_to_one(results)
+
+ # Move the old results, and write the current results
+ filedir = os.path.dirname(filepath)
try:
- with open(filename, 'r') as resultsfile:
- testrun = TestrunResult(resultsfile)
- except IOError:
- with open(os.path.join(filename, "main"), 'r') as resultsfile:
- testrun = TestrunResult(resultsfile)
-
- assert testrun.name is not None
- return testrun
+ os.rename(filepath, os.path.join(filedir, 'results.json.old'))
+ results.write(os.path.join(filedir, 'results.json'))
+ except OSError:
+ print("WARNING: Could not write updated results {}".format(filepath),
+ file=sys.stderr)
+
+ return results
+
+
+def _update_zero_to_one(results):
+ """ Update version zero results to version 1 results
+
+ Changes from version 0 to version 1
+
+ - dmesg is sometimes stored as a list, sometimes stored as a string. In
+ version 1 it is always stored as a string
+ - in version 0 subtests are somtimes stored as duplicates, sometimes stored
+ only with a single entry, in version 1 tests with subtests are only
+ recorded once, always.
+ - Version 0 can have an info entry, or returncode, out, and err entries,
+ Version 1 will only have the latter
+ - version 0 results are called 'main', while version 1 results are called
+ 'results.json' (This is not handled internally, it's either handled by
+ update_results() which will write the file back to disk, or needs to be
+ handled manually by the user)
+
+ """
+ updated_results = {}
+ remove = set()
+
+ for name, test in results.tests.iteritems():
+ # fix dmesg errors if any
+ if isinstance(test.get('dmesg'), list):
+ test['dmesg'] = '\n'.join(test['dmesg'])
+
+ # If a test as an info attribute, we want to remove it, if it doesn't
+ # have a returncode, out, or attribute we'll want to get those out of
+ # info first
+ if (None in [test.get('out'), test.get('err'), test.get('returncode')]
+ and test.get('info')):
+ code, err, out = test['info'].split('\n\n')
+
+ # returncode can be 0, and 0 is falsy, so ensure it is actually
+ # None
+ if test.get('returncode') is None:
+ test['returncode'] = int(code.split()[1].strip())
+ if not test.get('out'):
+ test['out'] = out.split()[1]
+ if not test.get('err'):
+ test['err'] = err.split()[1]
+
+ # Remove the unused info key
+ if test.get('info'):
+ del test['info']
+
+ # Walk through the list of tests, if any of them contain duplicate
+ # subtests, add those subtests to the remove set, and then write a
+ # single test result to the update_results dictionary, which will be
+ # merged into results
+ #
+ # this must be the last thing done in this loop, or there will be pain
+ if test.get('subtest'):
+ remove.add(name)
+ testname = os.path.dirname(name)
+ if testname not in updated_results.iterkeys():
+ updated_results[testname] = test
+
+ for name in remove:
+ del results.tests[name]
+ results.tests.update(updated_results)
+
+ # set the results version
+ results.results_version = 1
+
+ return results
diff --git a/framework/tests/results_tests.py b/framework/tests/results_tests.py
index b31c505..f5c4ddf 100644
--- a/framework/tests/results_tests.py
+++ b/framework/tests/results_tests.py
@@ -24,6 +24,7 @@
import os
import tempfile
import json
+import copy
import nose.tools as nt
import framework.tests.utils as utils
import framework.results as results
@@ -67,21 +68,28 @@ def test_initialize_jsonwriter():
assert isinstance(func, results.JSONWriter)
-def test_load_results_folder():
+def test_load_results_folder_as_main():
""" Test that load_results takes a folder with a file named main in it """
with utils.tempdir() as tdir:
with open(os.path.join(tdir, 'main'), 'w') as tfile:
tfile.write(json.dumps(utils.JSON_DATA))
- results_ = results.load_results(tdir)
- assert results_
+ results.load_results(tdir)
+
+
+def test_load_results_folder():
+ """ Test that load_results takes a folder with a file named results.json """
+ with utils.tempdir() as tdir:
+ with open(os.path.join(tdir, 'results.json'), 'w') as tfile:
+ tfile.write(json.dumps(utils.JSON_DATA))
+
+ results.load_results(tdir)
def test_load_results_file():
""" Test that load_results takes a file """
with utils.resultfile() as tfile:
- results_ = results.load_results(tfile.name)
- assert results_
+ results.load_results(tfile.name)
def test_testresult_to_status():
@@ -107,3 +115,142 @@ def test_testrunresult_write():
new = results.load_results(os.path.join(tdir, 'results.json'))
nt.assert_dict_equal(result.__dict__, new.__dict__)
+
+
+ at nt.istest
+def generate_zero_to_one():
+ """ Generate tests for version 0 to version 1 conversion """
+ data = copy.deepcopy(utils.JSON_DATA)
+ data['tests']['sometest']['dmesg'] = ['this', 'is', 'dmesg']
+ data['tests']['sometest']['info'] = \
+ 'Returncode: 1\n\nErrors: stderr\n\nOutput: stdout\n'
+ data['tests'].update({
+ 'group1/groupA/test/subtest 1': {
+ 'info': 'Returncode: 1\n\nErrors: stderr\n\nOutput: stdout\n',
+ 'subtest': {
+ 'subtest 1': 'pass',
+ 'subtest 2': 'pass'
+ },
+ 'returncode': 0,
+ 'command': 'this is a command',
+ 'result': 'pass',
+ 'time': 0.1
+ },
+ 'group1/groupA/test/subtest 2': {
+ 'info': 'Returncode: 1\n\nErrors: stderr\n\nOutput: stdout\n',
+ 'subtest': {
+ 'subtest 1': 'pass',
+ 'subtest 2': 'pass'
+ },
+ 'returncode': 0,
+ 'command': 'this is a command',
+ 'result': 'pass',
+ 'time': 0.1
+ }
+ })
+
+ with utils.with_tempfile(json.dumps(data)) as f:
+ res = results._update_zero_to_one(results.load_results(f))
+
+ zero_to_one_dmesg.description = \
+ "version 1: dmesg is converted from a list to a string"
+ yield zero_to_one_dmesg, res
+
+ zero_to_one_subtests_remove_duplicates.description = \
+ "Version 1: Removes duplicate entrieds"
+ yield zero_to_one_subtests_remove_duplicates, res
+
+ zero_to_one_subtests_add_test.description = \
+ "Version 1: Add an entry for the actual test"
+ yield zero_to_one_subtests_add_test, res
+
+ zero_to_one_subtests_test_is_testresult.description = \
+ "Version 1: The result of the new test is a TestResult Instance"
+ yield zero_to_one_subtests_test_is_testresult, res
+
+ zero_to_one_info_delete.description = \
+ "Version 1: Info should be removed"
+ yield zero_to_one_info_delete, res
+
+ zero_to_one_returncode_from_info.description = \
+ "Version 1: Use the returncode from info if there is no returncode"
+ yield zero_to_one_returncode_from_info, res
+
+ zero_to_one_returncode_no_override.description = \
+ "Version 1: Do not clobber returncode with info"
+ yield zero_to_one_returncode_no_override, res
+
+ zero_to_one_err_from_info.description = \
+ "Version 1: add an err attribute from info"
+ yield zero_to_one_err_from_info, res
+
+ zero_to_one_out_from_info.description = \
+ "Version 1: add an out attribute from info"
+ yield zero_to_one_out_from_info, res
+
+ zero_to_one_set_version.description = \
+ "Version 1: Set results_version to 1"
+ yield zero_to_one_set_version, res
+
+
+def zero_to_one_dmesg(result):
+ """ version 1: dmesg is converted from a list to a string """
+ assert result.tests['sometest']['dmesg'] == 'this\nis\ndmesg'
+
+
+ at nt.nottest
+def zero_to_one_subtests_remove_duplicates(result):
+ """ Version 1: Removes duplicate entrieds"""
+ assert 'group1/groupA/test/subtest 1' not in result.tests
+ assert 'group1/groupA/test/subtest 2' not in result.tests
+
+
+ at nt.nottest
+def zero_to_one_subtests_add_test(result):
+ """ Add an entry for the actual test """
+ assert result.tests.get('group1/groupA/test')
+
+
+ at nt.nottest
+def zero_to_one_subtests_test_is_testresult(result):
+ """ The result of the new test is a TestResult Instance """
+ assert isinstance(
+ result.tests['group1/groupA/test'],
+ results.TestResult)
+
+
+def zero_to_one_info_delete(result):
+ """ Remove the info name from results """
+ for value in result.tests.itervalues():
+ assert 'info' not in value
+
+
+def zero_to_one_returncode_from_info(result):
+ """ Info returncode should be added to returncode if there isn't one """
+ assert result.tests['sometest']['returncode'] == 1
+
+
+def zero_to_one_returncode_no_override(result):
+ """ The returncode from info should not overwrite an existing returcnode
+ attribute
+
+ This test only tests that the value in info isn't used when there is a
+ value in returncode already
+
+ """
+ assert result.tests['group1/groupA/test']['returncode'] != 1
+
+
+def zero_to_one_err_from_info(result):
+ """ generate err from info """
+ assert result.tests['group1/groupA/test']['err'] == 'stderr'
+
+
+def zero_to_one_out_from_info(result):
+ """ generate out from info """
+ assert result.tests['group1/groupA/test']['out'] == 'stdout'
+
+
+def zero_to_one_set_version(result):
+ """ Set the version to 1 """
+ assert result.results_version == 1
diff --git a/framework/tests/utils.py b/framework/tests/utils.py
index f337b1e..252ced6 100644
--- a/framework/tests/utils.py
+++ b/framework/tests/utils.py
@@ -33,6 +33,7 @@ try:
import simplejson as json
except ImportError:
import json
+import framework.results
__all__ = [
@@ -49,6 +50,7 @@ JSON_DATA = {
"filter": [],
"exclude_filter": []
},
+ "results_version": framework.results.CURRENT_JSON_VERSION,
"name": "fake-tests",
"lspci": "fake",
"glxinfo": "fake",
--
2.0.0
More information about the Piglit
mailing list