[Piglit] [PATCH 7/8] framework: Update results to use versioned numbers
Ilia Mirkin
imirkin at alum.mit.edu
Thu Jun 5 15:55:16 PDT 2014
On Thu, Jun 5, 2014 at 6:50 PM, Dylan Baker <baker.dylan.c at gmail.com> wrote:
> On Thursday, June 05, 2014 06:45:52 PM Ilia Mirkin wrote:
>> On Thu, Jun 5, 2014 at 6:26 PM, Dylan Baker <baker.dylan.c at gmail.com> wrote:
>> > This patch updates our json to version 1. Changes from version 0 to
>> > version 1 are as follows:
>> >
>> > - renamed 'main' to 'results.json'
>> > - dmesg must be a string (It was stored a list in some version 0
>> >
>> > results)
>> >
>> > - subtests are never stored as duplicate entries, a single instance of
>> >
>> > the test is recorded, (In version 0 both are possible)
>> >
>> > - there is no info entry in version 1, err, out, and returncode are
>> >
>> > always split into seperate entries
>> >
>> > This patch adds support to the results module for handling updates to
>> > the results in a sane way. It does this by adding a result_version
>> > attribute to the TestrunResult (which is stored as json), and
>> > implementing the ability to incrementally update results between
>> > versions.
>> >
>> > It does this automatically on load, non-destructively, moving the old
>> > results to results.json.old, but does write the updated results to disk,
>> > making the cost of this update a one time cost.
>>
>> FYI, the way I run piglit-summary right now is:
>>
>> ~/src/piglit/piglit-summary-html.py -e pass -e skip tmp <(xzcat
>> nve7-2014-04-29-mupuf-gs5.xz) <(xzcat nve7-2014-06-05-mupuf-gs5.xz )
>> --overwrite
>>
>> (or something along those lines.)
>>
>> Is your approach going to work with this? (Obviously it can't _update_
>> the data, but will it still not break?)
>
> Honestly I'm not sure, it's not something that I or anyone at Intel does
> AFAIK, I can give it a test, but I doubt it, since it relies on being able to
> move the old file and write a new one.
>
> Also, I've got another series after this that just compresses/decompresses the
> results for you (although we only get bz2 or gz since we're using python2,
> xz/lzma support was added to python3 only)
Could you detect the situation when it's a pipe (or other "weird"
file) and just update the results in-memory?
>
>>
>> > Signed-off-by: Dylan Baker <baker.dylan.c at gmail.com>
>> > ---
>> >
>> > framework/programs/run.py | 4 +-
>> > framework/results.py | 126 ++++++++++++++++++++++++++++---
>> > framework/tests/results_tests.py | 158
>> > +++++++++++++++++++++++++++++++++++++-- framework/tests/utils.py
>> > | 2 +
>> > 4 files changed, 273 insertions(+), 17 deletions(-)
>> >
>> > diff --git a/framework/programs/run.py b/framework/programs/run.py
>> > index d61c065..6f352ab 100644
>> > --- a/framework/programs/run.py
>> > +++ b/framework/programs/run.py
>> >
>> > @@ -157,7 +157,7 @@ def run(input_):
>> > results.name = path.basename(args.results_path)
>> >
>> > # Begin json.
>> >
>> > - result_filepath = path.join(args.results_path, 'main')
>> > + result_filepath = path.join(args.results_path, 'results.json')
>> >
>> > result_file = open(result_filepath, 'w')
>> > json_writer = framework.results.JSONWriter(result_file)
>> >
>> > @@ -216,7 +216,7 @@ def resume(input_):
>> > except KeyError:
>> > pass
>> >
>> > - results_path = path.join(args.results_path, "main")
>> > + results_path = path.join(args.results_path, 'results.json')
>> >
>> > json_writer = framework.results.JSONWriter(open(results_path, 'w+'))
>> > json_writer.initialize_json(results.options, results.name,
>> >
>> > env.collectData())
>> >
>> > diff --git a/framework/results.py b/framework/results.py
>> > index 017c4a4..f26b7cb 100644
>> > --- a/framework/results.py
>> > +++ b/framework/results.py
>> > @@ -39,6 +39,9 @@ __all__ = [
>> >
>> > 'load_results',
>> >
>> > ]
>> >
>> > +# The current version of the JSON results
>> > +CURRENT_JSON_VERSION = 1
>> > +
>> >
>> > def _piglit_encoder(obj):
>> > """ Encoder for piglit that can transform additional classes into
>> > json
>> >
>> > @@ -119,7 +122,7 @@ class JSONWriter(object):
>> > self.__is_collection_empty = []
>> >
>> > def initialize_json(self, options, name, env):
>> > - """ Write boilerplate json code
>> > + """ Write boilerplate json code
>> >
>> > This writes all of the json except the actuall tests.
>> >
>> > @@ -132,6 +135,7 @@ class JSONWriter(object):
>> > """
>> > self.open_dict()
>> >
>> > + self.write_dict_item('results_version', CURRENT_JSON_VERSION)
>> >
>> > self.write_dict_item('name', name)
>> >
>> > self.write_dict_key('options')
>> >
>> > @@ -229,6 +233,7 @@ class TestrunResult(object):
>> > 'wglinfo',
>> > 'glxinfo',
>> > 'lspci',
>> >
>> > + 'results_version',
>> >
>> > 'time_elapsed']
>> >
>> > self.name = None
>> > self.uname = None
>> >
>> > @@ -337,12 +342,113 @@ def load_results(filename):
>> > "main"
>> >
>> > """
>> >
>> > - try:
>> > - with open(filename, 'r') as resultsfile:
>> > - testrun = TestrunResult(resultsfile)
>> > - except IOError:
>> > - with open(os.path.join(filename, "main"), 'r') as resultsfile:
>> > - testrun = TestrunResult(resultsfile)
>> > -
>> > - assert testrun.name is not None
>> > - return testrun
>> > + if os.path.isfile(filename):
>> > + filepath = filename
>> > + else:
>> > + # If there are both old and new results in a directory pick the
>> > new + # ones first
>> > + if os.path.exists(os.path.join(filename, 'results.json')):
>> > + filepath = os.path.join(filename, 'results.json')
>> > + # Version 0 results are called 'main'
>> > + elif os.path.exists(os.path.join(filename, 'main')):
>> > + filepath = os.path.join(filename, 'main')
>> > + else:
>> > + raise Exception("No results found")
>> > +
>> > + with open(filepath, 'r') as f:
>> > + testrun = TestrunResult(f)
>> > +
>> > + return update_results(testrun, filepath)
>> > +
>> > +
>> > +def update_results(results, filepath):
>> > + """ Update results to the lastest version
>> > +
>> > + This function is a wraper for other update_* functions, providing
>> > + incremental updates from one version to another.
>> > +
>> > + """
>> > + # If the results version is the current version there is no need to
>> > + # update, just return the results
>> > + if getattr(results, 'results_version', 0) == CURRENT_JSON_VERSION:
>> > + return results
>> > +
>> > + # If there is no version then the results are version 0, and need to
>> > be + # updated to version 1
>> > + if getattr(results, 'results_version', False):
>> > + results = _update_zero_to_one(results)
>> > +
>> > + # Move the old results, and write the current results
>> > + filedir = os.path.dirname(filepath)
>> > + os.rename(filepath, os.path.join(filedir, 'results.json.old'))
>> > + results.write(os.path.join(filedir, 'results.json'))
>> > +
>> > + return results
>> > +
>> > +
>> > +def _update_zero_to_one(results):
>> > + """ Update version zero results to version 1 results
>> > +
>> > + Changes from version 0 to version 1
>> > +
>> > + - dmesg is sometimes stored as a list, sometimes stored as a string.
>> > In + version 1 it is always stored as a string
>> > + - in version 0 subtests are somtimes stored as duplicates, sometimes
>> > stored + only with a single entry, in version 1 tests with subtests
>> > are only + recorded once, always.
>> > + - Version 0 can have an info entry, or returncode, out, and err
>> > entries, + Version 1 will only have the latter
>> > + - version 0 results are called 'main', while version 1 results are
>> > called + 'results.json' (This is not handled internally, it's either
>> > handled by + update_results() which will write the file back to
>> > disk, or needs to be + handled manually by the user)
>> > +
>> > + """
>> > + updated_results = {}
>> > + remove = set()
>> > +
>> > + for name, test in results.tests.iteritems():
>> > + # fix dmesg errors if any
>> > + if isinstance(test.get('dmesg'), list):
>> > + test['dmesg'] = '\n'.join(test['dmesg'])
>> > +
>> > + # If a test as an info attribute, we want to remove it, if it
>> > doesn't + # have a returncode, out, or attribute we'll want to get
>> > those out of + # info first
>> > + if (None in [test.get('out'), test.get('err'),
>> > test.get('returncode')] + and test.get('info')):
>> > + code, err, out = test['info'].split('\n\n')
>> > +
>> > + # returncode can be 0, and 0 is falsy, so ensure it is
>> > actually + # None
>> > + if test.get('returncode') is None:
>> > + test['returncode'] = int(code.split()[1].strip())
>> > + if not test.get('out'):
>> > + test['out'] = out.split()[1]
>> > + if not test.get('err'):
>> > + test['err'] = err.split()[1]
>> > +
>> > + # Remove the unused info key
>> > + if test.get('info'):
>> > + del test['info']
>> > +
>> > + # Walk through the list of tests, if any of them contain
>> > duplicate
>> > + # subtests, add those subtests to the remove set, and then write
>> > a
>> > + # single test result to the update_results dictionary, which will
>> > be + # merged into results
>> > + #
>> > + # this must be the last thing done in this loop, or there will be
>> > pain + if test.get('subtest'):
>> > + remove.add(name)
>> > + testname = os.path.dirname(name)
>> > + if testname not in updated_results.iterkeys():
>> > + updated_results[testname] = test
>> > +
>> > + for name in remove:
>> > + del results.tests[name]
>> > + results.tests.update(updated_results)
>> > +
>> > + # set the results version
>> > + results.results_version = 1
>> > +
>> > + return results
>> > diff --git a/framework/tests/results_tests.py
>> > b/framework/tests/results_tests.py index 64fb679..afd71f2 100644
>> > --- a/framework/tests/results_tests.py
>> > +++ b/framework/tests/results_tests.py
>> > @@ -24,6 +24,8 @@
>> >
>> > import os
>> > import tempfile
>> > import json
>> >
>> > +import copy
>> > +import nose.tools as nt
>> >
>> > import framework.tests.utils as utils
>> > import framework.results as results
>> > import framework.status as status
>> >
>> > @@ -66,21 +68,28 @@ def test_initialize_jsonwriter():
>> > assert isinstance(func, results.JSONWriter)
>> >
>> > -def test_load_results_folder():
>> >
>> > +def test_load_results_folder_as_main():
>> > """ Test that load_results takes a folder with a file named main in
>> > it """
>> >
>> > with utils.tempdir() as tdir:
>> > with open(os.path.join(tdir, 'main'), 'w') as tfile:
>> > tfile.write(json.dumps(utils.JSON_DATA))
>> >
>> > - results_ = results.load_results(tdir)
>> > - assert results_
>> > + results.load_results(tdir)
>> > +
>> > +
>> > +def test_load_results_folder():
>> > + """ Test that load_results takes a folder with a file named
>> > results.json """ + with utils.tempdir() as tdir:
>> > + with open(os.path.join(tdir, 'results.json'), 'w') as tfile:
>> > + tfile.write(json.dumps(utils.JSON_DATA))
>> > +
>> > + results.load_results(tdir)
>> >
>> > def test_load_results_file():
>> > """ Test that load_results takes a file """
>> >
>> > with utils.resultfile() as tfile:
>> > - results_ = results.load_results(tfile.name)
>> > - assert results_
>> > + results.load_results(tfile.name)
>> >
>> > def test_testresult_to_status():
>> > @@ -106,3 +115,142 @@ def test_testrunresult_write():
>> > new = results.load_results(os.path.join(tdir,
>> > 'results.json'))
>> >
>> > assert result.__dict__ == new.__dict__
>> >
>> > +
>> > +
>> > + at nt.istest
>> > +def generate_zero_to_one():
>> > + """ Generate tests for version 0 to version 1 conversion """
>> > + data = copy.deepcopy(utils.JSON_DATA)
>> > + data['tests']['sometest']['dmesg'] = ['this', 'is', 'dmesg']
>> > + data['tests']['sometest']['info'] = \
>> > + 'Returncode: 1\n\nErrors: stderr\n\nOutput: stdout\n'
>> > + data['tests'].update({
>> > + 'group1/groupA/test/subtest 1': {
>> > + 'info': 'Returncode: 1\n\nErrors: stderr\n\nOutput:
>> > stdout\n',
>> > + 'subtest': {
>> > + 'subtest 1': 'pass',
>> > + 'subtest 2': 'pass'
>> > + },
>> > + 'returncode': 0,
>> > + 'command': 'this is a command',
>> > + 'result': 'pass',
>> > + 'time': 0.1
>> > + },
>> > + 'group1/groupA/test/subtest 2': {
>> > + 'info': 'Returncode: 1\n\nErrors: stderr\n\nOutput:
>> > stdout\n',
>> > + 'subtest': {
>> > + 'subtest 1': 'pass',
>> > + 'subtest 2': 'pass'
>> > + },
>> > + 'returncode': 0,
>> > + 'command': 'this is a command',
>> > + 'result': 'pass',
>> > + 'time': 0.1
>> > + }
>> > + })
>> > +
>> > + with utils.with_tempfile(json.dumps(data)) as f:
>> > + res = results._update_zero_to_one(results.load_results(f))
>> > +
>> > + zero_to_one_dmesg.description = \
>> > + "version 1: dmesg is converted from a list to a string"
>> > + yield zero_to_one_dmesg, res
>> > +
>> > + zero_to_one_subtests_remove_duplicates.description = \
>> > + "Version 1: Removes duplicate entrieds"
>> > + yield zero_to_one_subtests_remove_duplicates, res
>> > +
>> > + zero_to_one_subtests_add_test.description = \
>> > + "Version 1: Add an entry for the actual test"
>> > + yield zero_to_one_subtests_add_test, res
>> > +
>> > + zero_to_one_subtests_test_is_testresult.description = \
>> > + "Version 1: The result of the new test is a TestResult Instance"
>> > + yield zero_to_one_subtests_test_is_testresult, res
>> > +
>> > + zero_to_one_info_delete.description = \
>> > + "Version 1: Info should be removed"
>> > + yield zero_to_one_info_delete, res
>> > +
>> > + zero_to_one_returncode_from_info.description = \
>> > + "Version 1: Use the returncode from info if there is no
>> > returncode" + yield zero_to_one_returncode_from_info, res
>> > +
>> > + zero_to_one_returncode_no_override.description = \
>> > + "Version 1: Do not clobber returncode with info"
>> > + yield zero_to_one_returncode_no_override, res
>> > +
>> > + zero_to_one_err_from_info.description = \
>> > + "Version 1: add an err attribute from info"
>> > + yield zero_to_one_err_from_info, res
>> > +
>> > + zero_to_one_out_from_info.description = \
>> > + "Version 1: add an out attribute from info"
>> > + yield zero_to_one_out_from_info, res
>> > +
>> > + zero_to_one_set_version.description = \
>> > + "Version 1: Set results_version to 1"
>> > + yield zero_to_one_set_version, res
>> > +
>> > +
>> > +def zero_to_one_dmesg(result):
>> > + """ version 1: dmesg is converted from a list to a string """
>> > + assert result.tests['sometest']['dmesg'] == 'this\nis\ndmesg'
>> > +
>> > +
>> > + at nt.nottest
>> > +def zero_to_one_subtests_remove_duplicates(result):
>> > + """ Version 1: Removes duplicate entrieds"""
>> > + assert 'group1/groupA/test/subtest 1' not in result.tests
>> > + assert 'group1/groupA/test/subtest 2' not in result.tests
>> > +
>> > +
>> > + at nt.nottest
>> > +def zero_to_one_subtests_add_test(result):
>> > + """ Add an entry for the actual test """
>> > + assert result.tests.get('group1/groupA/test')
>> > +
>> > +
>> > + at nt.nottest
>> > +def zero_to_one_subtests_test_is_testresult(result):
>> > + """ The result of the new test is a TestResult Instance """
>> > + assert isinstance(
>> > + result.tests['group1/groupA/test'],
>> > + results.TestResult)
>> > +
>> > +
>> > +def zero_to_one_info_delete(result):
>> > + """ Remove the info name from results """
>> > + for value in result.tests.itervalues():
>> > + assert 'info' not in value
>> > +
>> > +
>> > +def zero_to_one_returncode_from_info(result):
>> > + """ Info returncode should be added to returncode if there isn't one
>> > """ + assert result.tests['sometest']['returncode'] == 1
>> > +
>> > +
>> > +def zero_to_one_returncode_no_override(result):
>> > + """ The returncode from info should not overwrite an existing
>> > returcnode + attribute
>> > +
>> > + This test only tests that the value in info isn't used when there is
>> > a
>> > + value in returncode already
>> > +
>> > + """
>> > + assert result.tests['group1/groupA/test']['returncode'] != 1
>> > +
>> > +
>> > +def zero_to_one_err_from_info(result):
>> > + """ generate err from info """
>> > + assert result.tests['group1/groupA/test']['err'] == 'stderr'
>> > +
>> > +
>> > +def zero_to_one_out_from_info(result):
>> > + """ generate out from info """
>> > + assert result.tests['group1/groupA/test']['out'] == 'stdout'
>> > +
>> > +
>> > +def zero_to_one_set_version(result):
>> > + """ Set the version to 1 """
>> > + assert result.results_version == 1
>> > diff --git a/framework/tests/utils.py b/framework/tests/utils.py
>> > index f337b1e..252ced6 100644
>> > --- a/framework/tests/utils.py
>> > +++ b/framework/tests/utils.py
>> >
>> > @@ -33,6 +33,7 @@ try:
>> > import simplejson as json
>> >
>> > except ImportError:
>> > import json
>> >
>> > +import framework.results
>> >
>> > __all__ = [
>> >
>> > @@ -49,6 +50,7 @@ JSON_DATA = {
>> >
>> > "filter": [],
>> > "exclude_filter": []
>> >
>> > },
>> >
>> > + "results_version": framework.results.CURRENT_JSON_VERSION,
>> >
>> > "name": "fake-tests",
>> > "lspci": "fake",
>> > "glxinfo": "fake",
>> >
>> > --
>> > 2.0.0
>> >
>> > _______________________________________________
>> > Piglit mailing list
>> > Piglit at lists.freedesktop.org
>> > http://lists.freedesktop.org/mailman/listinfo/piglit
More information about the Piglit
mailing list