[Piglit] [PATCH 7/8] framework: Update results to use versioned numbers
Ilia Mirkin
imirkin at alum.mit.edu
Thu Jun 5 15:45:52 PDT 2014
On Thu, Jun 5, 2014 at 6:26 PM, Dylan Baker <baker.dylan.c at gmail.com> wrote:
> This patch updates our json to version 1. Changes from version 0 to
> version 1 are as follows:
>
> - renamed 'main' to 'results.json'
> - dmesg must be a string (It was stored a list in some version 0
> results)
> - subtests are never stored as duplicate entries, a single instance of
> the test is recorded, (In version 0 both are possible)
> - there is no info entry in version 1, err, out, and returncode are
> always split into seperate entries
>
> This patch adds support to the results module for handling updates to
> the results in a sane way. It does this by adding a result_version
> attribute to the TestrunResult (which is stored as json), and
> implementing the ability to incrementally update results between
> versions.
>
> It does this automatically on load, non-destructively, moving the old
> results to results.json.old, but does write the updated results to disk,
> making the cost of this update a one time cost.
FYI, the way I run piglit-summary right now is:
~/src/piglit/piglit-summary-html.py -e pass -e skip tmp <(xzcat
nve7-2014-04-29-mupuf-gs5.xz) <(xzcat nve7-2014-06-05-mupuf-gs5.xz )
--overwrite
(or something along those lines.)
Is your approach going to work with this? (Obviously it can't _update_
the data, but will it still not break?)
>
> Signed-off-by: Dylan Baker <baker.dylan.c at gmail.com>
> ---
> framework/programs/run.py | 4 +-
> framework/results.py | 126 ++++++++++++++++++++++++++++---
> framework/tests/results_tests.py | 158 +++++++++++++++++++++++++++++++++++++--
> framework/tests/utils.py | 2 +
> 4 files changed, 273 insertions(+), 17 deletions(-)
>
> diff --git a/framework/programs/run.py b/framework/programs/run.py
> index d61c065..6f352ab 100644
> --- a/framework/programs/run.py
> +++ b/framework/programs/run.py
> @@ -157,7 +157,7 @@ def run(input_):
> results.name = path.basename(args.results_path)
>
> # Begin json.
> - result_filepath = path.join(args.results_path, 'main')
> + result_filepath = path.join(args.results_path, 'results.json')
> result_file = open(result_filepath, 'w')
> json_writer = framework.results.JSONWriter(result_file)
>
> @@ -216,7 +216,7 @@ def resume(input_):
> except KeyError:
> pass
>
> - results_path = path.join(args.results_path, "main")
> + results_path = path.join(args.results_path, 'results.json')
> json_writer = framework.results.JSONWriter(open(results_path, 'w+'))
> json_writer.initialize_json(results.options, results.name,
> env.collectData())
> diff --git a/framework/results.py b/framework/results.py
> index 017c4a4..f26b7cb 100644
> --- a/framework/results.py
> +++ b/framework/results.py
> @@ -39,6 +39,9 @@ __all__ = [
> 'load_results',
> ]
>
> +# The current version of the JSON results
> +CURRENT_JSON_VERSION = 1
> +
>
> def _piglit_encoder(obj):
> """ Encoder for piglit that can transform additional classes into json
> @@ -119,7 +122,7 @@ class JSONWriter(object):
> self.__is_collection_empty = []
>
> def initialize_json(self, options, name, env):
> - """ Write boilerplate json code
> + """ Write boilerplate json code
>
> This writes all of the json except the actuall tests.
>
> @@ -132,6 +135,7 @@ class JSONWriter(object):
>
> """
> self.open_dict()
> + self.write_dict_item('results_version', CURRENT_JSON_VERSION)
> self.write_dict_item('name', name)
>
> self.write_dict_key('options')
> @@ -229,6 +233,7 @@ class TestrunResult(object):
> 'wglinfo',
> 'glxinfo',
> 'lspci',
> + 'results_version',
> 'time_elapsed']
> self.name = None
> self.uname = None
> @@ -337,12 +342,113 @@ def load_results(filename):
> "main"
>
> """
> - try:
> - with open(filename, 'r') as resultsfile:
> - testrun = TestrunResult(resultsfile)
> - except IOError:
> - with open(os.path.join(filename, "main"), 'r') as resultsfile:
> - testrun = TestrunResult(resultsfile)
> -
> - assert testrun.name is not None
> - return testrun
> + if os.path.isfile(filename):
> + filepath = filename
> + else:
> + # If there are both old and new results in a directory pick the new
> + # ones first
> + if os.path.exists(os.path.join(filename, 'results.json')):
> + filepath = os.path.join(filename, 'results.json')
> + # Version 0 results are called 'main'
> + elif os.path.exists(os.path.join(filename, 'main')):
> + filepath = os.path.join(filename, 'main')
> + else:
> + raise Exception("No results found")
> +
> + with open(filepath, 'r') as f:
> + testrun = TestrunResult(f)
> +
> + return update_results(testrun, filepath)
> +
> +
> +def update_results(results, filepath):
> + """ Update results to the lastest version
> +
> + This function is a wraper for other update_* functions, providing
> + incremental updates from one version to another.
> +
> + """
> + # If the results version is the current version there is no need to
> + # update, just return the results
> + if getattr(results, 'results_version', 0) == CURRENT_JSON_VERSION:
> + return results
> +
> + # If there is no version then the results are version 0, and need to be
> + # updated to version 1
> + if getattr(results, 'results_version', False):
> + results = _update_zero_to_one(results)
> +
> + # Move the old results, and write the current results
> + filedir = os.path.dirname(filepath)
> + os.rename(filepath, os.path.join(filedir, 'results.json.old'))
> + results.write(os.path.join(filedir, 'results.json'))
> +
> + return results
> +
> +
> +def _update_zero_to_one(results):
> + """ Update version zero results to version 1 results
> +
> + Changes from version 0 to version 1
> +
> + - dmesg is sometimes stored as a list, sometimes stored as a string. In
> + version 1 it is always stored as a string
> + - in version 0 subtests are somtimes stored as duplicates, sometimes stored
> + only with a single entry, in version 1 tests with subtests are only
> + recorded once, always.
> + - Version 0 can have an info entry, or returncode, out, and err entries,
> + Version 1 will only have the latter
> + - version 0 results are called 'main', while version 1 results are called
> + 'results.json' (This is not handled internally, it's either handled by
> + update_results() which will write the file back to disk, or needs to be
> + handled manually by the user)
> +
> + """
> + updated_results = {}
> + remove = set()
> +
> + for name, test in results.tests.iteritems():
> + # fix dmesg errors if any
> + if isinstance(test.get('dmesg'), list):
> + test['dmesg'] = '\n'.join(test['dmesg'])
> +
> + # If a test as an info attribute, we want to remove it, if it doesn't
> + # have a returncode, out, or attribute we'll want to get those out of
> + # info first
> + if (None in [test.get('out'), test.get('err'), test.get('returncode')]
> + and test.get('info')):
> + code, err, out = test['info'].split('\n\n')
> +
> + # returncode can be 0, and 0 is falsy, so ensure it is actually
> + # None
> + if test.get('returncode') is None:
> + test['returncode'] = int(code.split()[1].strip())
> + if not test.get('out'):
> + test['out'] = out.split()[1]
> + if not test.get('err'):
> + test['err'] = err.split()[1]
> +
> + # Remove the unused info key
> + if test.get('info'):
> + del test['info']
> +
> + # Walk through the list of tests, if any of them contain duplicate
> + # subtests, add those subtests to the remove set, and then write a
> + # single test result to the update_results dictionary, which will be
> + # merged into results
> + #
> + # this must be the last thing done in this loop, or there will be pain
> + if test.get('subtest'):
> + remove.add(name)
> + testname = os.path.dirname(name)
> + if testname not in updated_results.iterkeys():
> + updated_results[testname] = test
> +
> + for name in remove:
> + del results.tests[name]
> + results.tests.update(updated_results)
> +
> + # set the results version
> + results.results_version = 1
> +
> + return results
> diff --git a/framework/tests/results_tests.py b/framework/tests/results_tests.py
> index 64fb679..afd71f2 100644
> --- a/framework/tests/results_tests.py
> +++ b/framework/tests/results_tests.py
> @@ -24,6 +24,8 @@
> import os
> import tempfile
> import json
> +import copy
> +import nose.tools as nt
> import framework.tests.utils as utils
> import framework.results as results
> import framework.status as status
> @@ -66,21 +68,28 @@ def test_initialize_jsonwriter():
> assert isinstance(func, results.JSONWriter)
>
>
> -def test_load_results_folder():
> +def test_load_results_folder_as_main():
> """ Test that load_results takes a folder with a file named main in it """
> with utils.tempdir() as tdir:
> with open(os.path.join(tdir, 'main'), 'w') as tfile:
> tfile.write(json.dumps(utils.JSON_DATA))
>
> - results_ = results.load_results(tdir)
> - assert results_
> + results.load_results(tdir)
> +
> +
> +def test_load_results_folder():
> + """ Test that load_results takes a folder with a file named results.json """
> + with utils.tempdir() as tdir:
> + with open(os.path.join(tdir, 'results.json'), 'w') as tfile:
> + tfile.write(json.dumps(utils.JSON_DATA))
> +
> + results.load_results(tdir)
>
>
> def test_load_results_file():
> """ Test that load_results takes a file """
> with utils.resultfile() as tfile:
> - results_ = results.load_results(tfile.name)
> - assert results_
> + results.load_results(tfile.name)
>
>
> def test_testresult_to_status():
> @@ -106,3 +115,142 @@ def test_testrunresult_write():
> new = results.load_results(os.path.join(tdir, 'results.json'))
>
> assert result.__dict__ == new.__dict__
> +
> +
> + at nt.istest
> +def generate_zero_to_one():
> + """ Generate tests for version 0 to version 1 conversion """
> + data = copy.deepcopy(utils.JSON_DATA)
> + data['tests']['sometest']['dmesg'] = ['this', 'is', 'dmesg']
> + data['tests']['sometest']['info'] = \
> + 'Returncode: 1\n\nErrors: stderr\n\nOutput: stdout\n'
> + data['tests'].update({
> + 'group1/groupA/test/subtest 1': {
> + 'info': 'Returncode: 1\n\nErrors: stderr\n\nOutput: stdout\n',
> + 'subtest': {
> + 'subtest 1': 'pass',
> + 'subtest 2': 'pass'
> + },
> + 'returncode': 0,
> + 'command': 'this is a command',
> + 'result': 'pass',
> + 'time': 0.1
> + },
> + 'group1/groupA/test/subtest 2': {
> + 'info': 'Returncode: 1\n\nErrors: stderr\n\nOutput: stdout\n',
> + 'subtest': {
> + 'subtest 1': 'pass',
> + 'subtest 2': 'pass'
> + },
> + 'returncode': 0,
> + 'command': 'this is a command',
> + 'result': 'pass',
> + 'time': 0.1
> + }
> + })
> +
> + with utils.with_tempfile(json.dumps(data)) as f:
> + res = results._update_zero_to_one(results.load_results(f))
> +
> + zero_to_one_dmesg.description = \
> + "version 1: dmesg is converted from a list to a string"
> + yield zero_to_one_dmesg, res
> +
> + zero_to_one_subtests_remove_duplicates.description = \
> + "Version 1: Removes duplicate entrieds"
> + yield zero_to_one_subtests_remove_duplicates, res
> +
> + zero_to_one_subtests_add_test.description = \
> + "Version 1: Add an entry for the actual test"
> + yield zero_to_one_subtests_add_test, res
> +
> + zero_to_one_subtests_test_is_testresult.description = \
> + "Version 1: The result of the new test is a TestResult Instance"
> + yield zero_to_one_subtests_test_is_testresult, res
> +
> + zero_to_one_info_delete.description = \
> + "Version 1: Info should be removed"
> + yield zero_to_one_info_delete, res
> +
> + zero_to_one_returncode_from_info.description = \
> + "Version 1: Use the returncode from info if there is no returncode"
> + yield zero_to_one_returncode_from_info, res
> +
> + zero_to_one_returncode_no_override.description = \
> + "Version 1: Do not clobber returncode with info"
> + yield zero_to_one_returncode_no_override, res
> +
> + zero_to_one_err_from_info.description = \
> + "Version 1: add an err attribute from info"
> + yield zero_to_one_err_from_info, res
> +
> + zero_to_one_out_from_info.description = \
> + "Version 1: add an out attribute from info"
> + yield zero_to_one_out_from_info, res
> +
> + zero_to_one_set_version.description = \
> + "Version 1: Set results_version to 1"
> + yield zero_to_one_set_version, res
> +
> +
> +def zero_to_one_dmesg(result):
> + """ version 1: dmesg is converted from a list to a string """
> + assert result.tests['sometest']['dmesg'] == 'this\nis\ndmesg'
> +
> +
> + at nt.nottest
> +def zero_to_one_subtests_remove_duplicates(result):
> + """ Version 1: Removes duplicate entrieds"""
> + assert 'group1/groupA/test/subtest 1' not in result.tests
> + assert 'group1/groupA/test/subtest 2' not in result.tests
> +
> +
> + at nt.nottest
> +def zero_to_one_subtests_add_test(result):
> + """ Add an entry for the actual test """
> + assert result.tests.get('group1/groupA/test')
> +
> +
> + at nt.nottest
> +def zero_to_one_subtests_test_is_testresult(result):
> + """ The result of the new test is a TestResult Instance """
> + assert isinstance(
> + result.tests['group1/groupA/test'],
> + results.TestResult)
> +
> +
> +def zero_to_one_info_delete(result):
> + """ Remove the info name from results """
> + for value in result.tests.itervalues():
> + assert 'info' not in value
> +
> +
> +def zero_to_one_returncode_from_info(result):
> + """ Info returncode should be added to returncode if there isn't one """
> + assert result.tests['sometest']['returncode'] == 1
> +
> +
> +def zero_to_one_returncode_no_override(result):
> + """ The returncode from info should not overwrite an existing returcnode
> + attribute
> +
> + This test only tests that the value in info isn't used when there is a
> + value in returncode already
> +
> + """
> + assert result.tests['group1/groupA/test']['returncode'] != 1
> +
> +
> +def zero_to_one_err_from_info(result):
> + """ generate err from info """
> + assert result.tests['group1/groupA/test']['err'] == 'stderr'
> +
> +
> +def zero_to_one_out_from_info(result):
> + """ generate out from info """
> + assert result.tests['group1/groupA/test']['out'] == 'stdout'
> +
> +
> +def zero_to_one_set_version(result):
> + """ Set the version to 1 """
> + assert result.results_version == 1
> diff --git a/framework/tests/utils.py b/framework/tests/utils.py
> index f337b1e..252ced6 100644
> --- a/framework/tests/utils.py
> +++ b/framework/tests/utils.py
> @@ -33,6 +33,7 @@ try:
> import simplejson as json
> except ImportError:
> import json
> +import framework.results
>
>
> __all__ = [
> @@ -49,6 +50,7 @@ JSON_DATA = {
> "filter": [],
> "exclude_filter": []
> },
> + "results_version": framework.results.CURRENT_JSON_VERSION,
> "name": "fake-tests",
> "lspci": "fake",
> "glxinfo": "fake",
> --
> 2.0.0
>
> _______________________________________________
> Piglit mailing list
> Piglit at lists.freedesktop.org
> http://lists.freedesktop.org/mailman/listinfo/piglit
More information about the Piglit
mailing list