[Piglit] [PATCH v3] Add support for feature readiness.
Gabriel Feceoru
gabriel.feceoru at intel.com
Fri Nov 6 06:48:33 PST 2015
On 06.11.2015 00:20, Dylan Baker wrote:
> On Tue, Nov 03, 2015 at 05:50:41PM +0200, Gabriel Feceoru wrote:
>> This adds a new "summary feature" command to piglit which creates a HTML table
>> with the feature x DUT status (useful for multiple features, multiple DUTs).
>> Another use case is the feature status for subsequent test results (depending
>> on the meaning of that test result - DUT or build)
>> A feature readiness is defined by the piglit regexp which selects the tests
>> relevant for that feature and the acceptance percentage threshold (pass rate).
>>
>> It requires an input json file containing the list of features, in the
>> following format (this is just an example):
>> {
>> "glsl" : {
>> "include_tests" : "glsl",
>> "exclude_tests" : "",
>> "target_rate" : 90
>> },
>> "arb" : {
>> "include_tests" : "arb_gpu",
>> "exclude_tests" : "",
>> "target_rate" : 10
>> }
>>
>> }
>>
>> v3:
>> Changed json rate to int instead of string
>> Applied other review comments
>>
>> v2:
>> Apply review comments (Dylan, Thomas)
>>
>> Fixed 2nd round of review comments
>>
>> Signed-off-by: Gabriel Feceoru <gabriel.feceoru at intel.com>
>> ---
>> framework/programs/summary.py | 37 ++++++++++++++++
>> framework/summary/__init__.py | 2 +-
>> framework/summary/feature.py | 99 +++++++++++++++++++++++++++++++++++++++++++
>> framework/summary/html_.py | 23 +++++++++-
>> piglit | 4 ++
>> templates/feature.mako | 73 +++++++++++++++++++++++++++++++
>> 6 files changed, 236 insertions(+), 2 deletions(-)
>> create mode 100644 framework/summary/feature.py
>> create mode 100644 templates/feature.mako
>>
>> diff --git a/framework/programs/summary.py b/framework/programs/summary.py
>> index 8711ee5..b23f1ef 100644
>> --- a/framework/programs/summary.py
>> +++ b/framework/programs/summary.py
>> @@ -35,6 +35,7 @@ __all__ = [
>> 'console',
>> 'csv',
>> 'html',
>> + 'feature'
>> ]
>>
>>
>> @@ -224,3 +225,39 @@ def aggregate(input_):
>>
>> print("Aggregated file written to: {}.{}".format(
>> outfile, backends.compression.get_mode()))
>> +
>> +
>> + at exceptions.handler
>> +def feature(input_):
>> + parser = argparse.ArgumentParser()
>> + parser.add_argument("-o", "--overwrite",
>> + action="store_true",
>> + help="Overwrite existing directories")
>> + parser.add_argument("featureFile",
>> + metavar="<Feature json file>",
>> + help="Json file containing the features description")
>> + parser.add_argument("summaryDir",
>> + metavar="<Summary Directory>",
>> + help="Directory to put HTML files in")
>> + parser.add_argument("resultsFiles",
>> + metavar="<Results Files>",
>> + nargs="*",
>> + help="Results files to include in HTML")
>> + args = parser.parse_args(input_)
>> +
>> + # If args.list and args.resultsFiles are empty, then raise an error
>> + if not args.featureFile and not args.resultsFiles:
>> + raise parser.error("Missing required option -l or <resultsFiles>")
>> +
>> + # If args.list and args.resultsFiles are empty, then raise an error
>> + if not args.resultsFiles or not path.exists(args.featureFile):
>> + raise parser.error("Missing json file")
>> +
>> + # if overwrite is requested delete the output directory
>> + if path.exists(args.summaryDir) and args.overwrite:
>> + shutil.rmtree(args.summaryDir)
>> +
>> + # If the requested directory doesn't exist, create it or throw an error
>> + core.checkDir(args.summaryDir, not args.overwrite)
>> +
>> + summary.feat(args.resultsFiles, args.summaryDir, args.featureFile)
>> diff --git a/framework/summary/__init__.py b/framework/summary/__init__.py
>> index 8a1ff8a..0f0f144 100644
>> --- a/framework/summary/__init__.py
>> +++ b/framework/summary/__init__.py
>> @@ -25,5 +25,5 @@
>> # public parts here, so that we have a nice interface to work with.
>>
>> from __future__ import absolute_import, division, print_function
>> -from .html_ import html
>> +from .html_ import html, feat
>> from .console_ import console
>> diff --git a/framework/summary/feature.py b/framework/summary/feature.py
>> new file mode 100644
>> index 0000000..2fb8f50
>> --- /dev/null
>> +++ b/framework/summary/feature.py
>> @@ -0,0 +1,99 @@
>> +# Copyright (c) 2015 Intel Corporation
>> +
>> +# Permission is hereby granted, free of charge, to any person
>> +# obtaining a copy of this software and associated documentation
>> +# files (the "Software"), to deal in the Software without
>> +# restriction, including without limitation the rights to use,
>> +# copy, modify, merge, publish, distribute, sublicense, and/or
>> +# sell copies of the Software, and to permit persons to whom the
>> +# Software is furnished to do so, subject to the following
>> +# conditions:
>> +#
>> +# This permission notice shall be included in all copies or
>> +# substantial portions of the Software.
>> +#
>> +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
>> +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
>> +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
>> +# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHOR(S) BE
>> +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
>> +# AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
>> +# OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
>> +# DEALINGS IN THE SOFTWARE.
>> +
>> +from __future__ import print_function, division, absolute_import
>> +
>> +import copy
>> +
>> +try:
>> + import simplejson as json
>> +except ImportError:
>> + import json
>> +
>> +from framework import core, exceptions, profile, status
>> +
>> +
>> +class FeatResults(object): # pylint: disable=too-few-public-methods
>> + """Container object for results.
>> +
>> + Has the results, feature profiles and feature computed results.
>> +
>> + """
>> + def __init__(self, results, json_file):
>> +
>> + with open(json_file) as data:
>> + feature_data = json.load(data)
>> +
>> + self.feat_fractions = {}
>> + self.feat_status = {}
>> + self.features = set()
>> + self.results = results
>> +
>> + profiles = {}
>> +
>> + # we expect all the result sets to be for the same profile
>> + profile_orig = profile.load_test_profile(results[0].options['profile'][0])
>> +
>> + for feature in feature_data:
>> + self.features.add(feature)
>> +
>> + incl_str = feature_data[feature]["include_tests"]
>> + excl_str = feature_data[feature]["exclude_tests"]
>> +
>> + include_filter = [incl_str] if incl_str and not incl_str.isspace() else []
>> + exclude_filter = [excl_str] if excl_str and not excl_str.isspace() else []
>> +
>> + opts = core.Options(include_filter=include_filter,
>> + exclude_filter=exclude_filter)
>> +
>> + profiles[feature] = copy.deepcopy(profile_orig)
>> +
>> + # An empty list will raise PiglitFatalError exception
>> + # But for reporting we need to handle this situation
>> + try:
>> + profiles[feature]._prepare_test_list(opts)
>> + except exceptions.PiglitFatalError:
>> + pass
>> +
>> + for results in self.results:
>> + self.feat_fractions[results.name] = {}
>> + self.feat_status[results.name] = {}
>> +
>> + for feature in feature_data:
>> + result_set = set(results.tests)
>> + profile_set = set(profiles[feature].test_list)
>> +
>> + common_set = profile_set & result_set
>> + passed_list = [x for x in common_set if results.tests[x].result == status.PASS]
>> +
>> + total = len(common_set)
>> + passed = len(passed_list)
>> +
>> + self.feat_fractions[results.name][feature] = (passed, total)
>> + if total == 0:
>> + self.feat_status[results.name][feature] = status.NOTRUN
>> + else:
>> + if 100 * passed // total >= feature_data[feature]["target_rate"]:
>> + self.feat_status[results.name][feature] = status.PASS
>> + else:
>> + self.feat_status[results.name][feature] = status.FAIL
>> diff --git a/framework/summary/html_.py b/framework/summary/html_.py
>> index 12172b7..60d7f5e 100644
>> --- a/framework/summary/html_.py
>> +++ b/framework/summary/html_.py
>> @@ -37,9 +37,11 @@ from mako.lookup import TemplateLookup
>> from framework import backends, exceptions
>>
>> from .common import Results, escape_filename, escape_pathname
>> +from .feature import FeatResults
>>
>> __all__ = [
>> 'html',
>> + 'feat'
>> ]
>>
>> _TEMP_DIR = os.path.join(
>> @@ -62,8 +64,9 @@ def _copy_static_files(destination):
>> os.path.join(destination, "result.css"))
>>
>>
>> -def _make_testrun_info(results, destination, exclude):
>> +def _make_testrun_info(results, destination, exclude=None):
>> """Create the pages for each results file."""
>> + exclude = exclude or {}
>> result_css = os.path.join(destination, "result.css")
>> index = os.path.join(destination, "index.html")
>>
>> @@ -146,6 +149,14 @@ def _make_comparison_pages(results, destination, exclude):
>> page=page, pages=pages))
>>
>>
>> +def _make_feature_info(results, destination):
>> + """Create the feature readiness page."""
>> +
>> + with open(os.path.join(destination, "feature.html"), 'w') as out:
>> + out.write(_TEMPLATES.get_template('feature.mako').render(
>> + results=results))
>> +
>> +
>> def html(results, destination, exclude):
>> """
>> Produce HTML summaries.
>> @@ -161,3 +172,13 @@ def html(results, destination, exclude):
>> _copy_static_files(destination)
>> _make_testrun_info(results, destination, exclude)
>> _make_comparison_pages(results, destination, exclude)
>> +
>> +
>> +def feat(results, destination, feat_desc):
>> + """Produce HTML feature readiness summary."""
>> +
>> + feat_res = FeatResults([backends.load(i) for i in results], feat_desc)
>> +
>> + _copy_static_files(destination)
>> + _make_testrun_info(feat_res, destination)
>> + _make_feature_info(feat_res, destination)
>> diff --git a/piglit b/piglit
>> index 5ae43e9..c2a2e2b 100755
>> --- a/piglit
>> +++ b/piglit
>> @@ -139,6 +139,10 @@ def main():
>> add_help=False,
>> help="Aggregate incomplete piglit run.")
>> aggregate.set_defaults(func=summary.aggregate)
>> + feature = summary_parser.add_parser('feature',
>> + add_help=False,
>> + help="generate feature readiness html report.")
>> + feature.set_defaults(func=summary.feature)
>>
>> # Parse the known arguments (piglit run or piglit summary html for
>> # example), and then pass the arguments that this parser doesn't know about
>> diff --git a/templates/feature.mako b/templates/feature.mako
>> new file mode 100644
>> index 0000000..ac9bc86
>> --- /dev/null
>> +++ b/templates/feature.mako
>> @@ -0,0 +1,73 @@
>> +<%!
>> + import posixpath # this must be posixpath, since we want /'s not \'s
>> + import re
>> +
>> +
>> + def feat_result(result):
>> + """Percentage result string"""
>> + return '{}/{}'.format(result[0], result[1])
>> +
>> +
>> + def escape_filename(key):
>> + """Avoid reserved characters in filenames."""
>> + return re.sub(r'[<>:"|?*#]', '_', key)
>> +
>> +
>> + def escape_pathname(key):
>> + """ Remove / and \\ from names """
>> + return re.sub(r'[/\\]', '_', key)
>> +
>> +
>> + def normalize_href(href):
>> + """Force backward slashes in URLs."""
>> + return href.replace('\\', '/')
>> +%>
>> +
>> +<?xml version="1.0" encoding="UTF-8"?>
>> +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
>> + "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
>> +<html xmlns="http://www.w3.org/1999/xhtml">
>> + <head>
>> + <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
>> + <title>Result summary</title>
>> + <link rel="stylesheet" href="index.css" type="text/css" />
>> + </head>
>> + <body>
>> + <h1>Feature readiness</h1>
>> + <table>
>> + <colgroup>
>> + ## Name Column
>> + <col />
>> +
>> + ## Status columns
>> + ## Create an additional column for each summary
>> + % for _ in xrange(len(results.results)):
>> + <col />
>> + % endfor
>> + </colgroup>
>> + <tr>
>> + <th/>
>> + % for res in results.results:
>> + <th class="head"><b>${res.name}</b><br />\
>> + (<a href="${normalize_href(posixpath.join(escape_pathname(res.name), 'index.html'))}">info</a>)</th>
>> + % endfor
>> + </tr>
>> + % for feature in results.features:
>> + <tr>
>> + ## Add the left most column, the feature name
>> + <td>
>> + <div class="group">
>> + <b>${feature}</b>
>> + </div>
>> + </td>
>> + ## add the feature totals
>> + % for res in results.results:
>> + <td class="${results.feat_status[res.name][feature]}">
>> + <b>${feat_result(results.feat_fractions[res.name][feature])}</b>
>> + </td>
>> + % endfor
>> + </tr>
>> + % endfor
>> + </table>
>> + </body>
>> +</html>
>> --
>> 1.9.1
>>
>> _______________________________________________
>> Piglit mailing list
>> Piglit at lists.freedesktop.org
>> http://lists.freedesktop.org/mailman/listinfo/piglit
>
>
> I haven't run this version, but I did run the last version. The changes
> look good.
> Reviewed-by: Dylan Baker <baker.dylan.c at gmail.com>
>
> I'll commit this tomorrow if anyone doesn't object by then.
>
Thank you for the review and help.
Gabriel.
More information about the Piglit
mailing list