[Piglit] [PATCH v3] Add support for feature readiness.
Dylan Baker
baker.dylan.c at gmail.com
Thu Nov 5 14:20:50 PST 2015
On Tue, Nov 03, 2015 at 05:50:41PM +0200, Gabriel Feceoru wrote:
> This adds a new "summary feature" command to piglit which creates a HTML table
> with the feature x DUT status (useful for multiple features, multiple DUTs).
> Another use case is the feature status for subsequent test results (depending
> on the meaning of that test result - DUT or build)
> A feature readiness is defined by the piglit regexp which selects the tests
> relevant for that feature and the acceptance percentage threshold (pass rate).
>
> It requires an input json file containing the list of features, in the
> following format (this is just an example):
> {
> "glsl" : {
> "include_tests" : "glsl",
> "exclude_tests" : "",
> "target_rate" : 90
> },
> "arb" : {
> "include_tests" : "arb_gpu",
> "exclude_tests" : "",
> "target_rate" : 10
> }
>
> }
>
> v3:
> Changed json rate to int instead of string
> Applied other review comments
>
> v2:
> Apply review comments (Dylan, Thomas)
>
> Fixed 2nd round of review comments
>
> Signed-off-by: Gabriel Feceoru <gabriel.feceoru at intel.com>
> ---
> framework/programs/summary.py | 37 ++++++++++++++++
> framework/summary/__init__.py | 2 +-
> framework/summary/feature.py | 99 +++++++++++++++++++++++++++++++++++++++++++
> framework/summary/html_.py | 23 +++++++++-
> piglit | 4 ++
> templates/feature.mako | 73 +++++++++++++++++++++++++++++++
> 6 files changed, 236 insertions(+), 2 deletions(-)
> create mode 100644 framework/summary/feature.py
> create mode 100644 templates/feature.mako
>
> diff --git a/framework/programs/summary.py b/framework/programs/summary.py
> index 8711ee5..b23f1ef 100644
> --- a/framework/programs/summary.py
> +++ b/framework/programs/summary.py
> @@ -35,6 +35,7 @@ __all__ = [
> 'console',
> 'csv',
> 'html',
> + 'feature'
> ]
>
>
> @@ -224,3 +225,39 @@ def aggregate(input_):
>
> print("Aggregated file written to: {}.{}".format(
> outfile, backends.compression.get_mode()))
> +
> +
> + at exceptions.handler
> +def feature(input_):
> + parser = argparse.ArgumentParser()
> + parser.add_argument("-o", "--overwrite",
> + action="store_true",
> + help="Overwrite existing directories")
> + parser.add_argument("featureFile",
> + metavar="<Feature json file>",
> + help="Json file containing the features description")
> + parser.add_argument("summaryDir",
> + metavar="<Summary Directory>",
> + help="Directory to put HTML files in")
> + parser.add_argument("resultsFiles",
> + metavar="<Results Files>",
> + nargs="*",
> + help="Results files to include in HTML")
> + args = parser.parse_args(input_)
> +
> + # If args.list and args.resultsFiles are empty, then raise an error
> + if not args.featureFile and not args.resultsFiles:
> + raise parser.error("Missing required option -l or <resultsFiles>")
> +
> + # If args.list and args.resultsFiles are empty, then raise an error
> + if not args.resultsFiles or not path.exists(args.featureFile):
> + raise parser.error("Missing json file")
> +
> + # if overwrite is requested delete the output directory
> + if path.exists(args.summaryDir) and args.overwrite:
> + shutil.rmtree(args.summaryDir)
> +
> + # If the requested directory doesn't exist, create it or throw an error
> + core.checkDir(args.summaryDir, not args.overwrite)
> +
> + summary.feat(args.resultsFiles, args.summaryDir, args.featureFile)
> diff --git a/framework/summary/__init__.py b/framework/summary/__init__.py
> index 8a1ff8a..0f0f144 100644
> --- a/framework/summary/__init__.py
> +++ b/framework/summary/__init__.py
> @@ -25,5 +25,5 @@
> # public parts here, so that we have a nice interface to work with.
>
> from __future__ import absolute_import, division, print_function
> -from .html_ import html
> +from .html_ import html, feat
> from .console_ import console
> diff --git a/framework/summary/feature.py b/framework/summary/feature.py
> new file mode 100644
> index 0000000..2fb8f50
> --- /dev/null
> +++ b/framework/summary/feature.py
> @@ -0,0 +1,99 @@
> +# Copyright (c) 2015 Intel Corporation
> +
> +# Permission is hereby granted, free of charge, to any person
> +# obtaining a copy of this software and associated documentation
> +# files (the "Software"), to deal in the Software without
> +# restriction, including without limitation the rights to use,
> +# copy, modify, merge, publish, distribute, sublicense, and/or
> +# sell copies of the Software, and to permit persons to whom the
> +# Software is furnished to do so, subject to the following
> +# conditions:
> +#
> +# This permission notice shall be included in all copies or
> +# substantial portions of the Software.
> +#
> +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
> +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
> +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
> +# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHOR(S) BE
> +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
> +# AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
> +# OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
> +# DEALINGS IN THE SOFTWARE.
> +
> +from __future__ import print_function, division, absolute_import
> +
> +import copy
> +
> +try:
> + import simplejson as json
> +except ImportError:
> + import json
> +
> +from framework import core, exceptions, profile, status
> +
> +
> +class FeatResults(object): # pylint: disable=too-few-public-methods
> + """Container object for results.
> +
> + Has the results, feature profiles and feature computed results.
> +
> + """
> + def __init__(self, results, json_file):
> +
> + with open(json_file) as data:
> + feature_data = json.load(data)
> +
> + self.feat_fractions = {}
> + self.feat_status = {}
> + self.features = set()
> + self.results = results
> +
> + profiles = {}
> +
> + # we expect all the result sets to be for the same profile
> + profile_orig = profile.load_test_profile(results[0].options['profile'][0])
> +
> + for feature in feature_data:
> + self.features.add(feature)
> +
> + incl_str = feature_data[feature]["include_tests"]
> + excl_str = feature_data[feature]["exclude_tests"]
> +
> + include_filter = [incl_str] if incl_str and not incl_str.isspace() else []
> + exclude_filter = [excl_str] if excl_str and not excl_str.isspace() else []
> +
> + opts = core.Options(include_filter=include_filter,
> + exclude_filter=exclude_filter)
> +
> + profiles[feature] = copy.deepcopy(profile_orig)
> +
> + # An empty list will raise PiglitFatalError exception
> + # But for reporting we need to handle this situation
> + try:
> + profiles[feature]._prepare_test_list(opts)
> + except exceptions.PiglitFatalError:
> + pass
> +
> + for results in self.results:
> + self.feat_fractions[results.name] = {}
> + self.feat_status[results.name] = {}
> +
> + for feature in feature_data:
> + result_set = set(results.tests)
> + profile_set = set(profiles[feature].test_list)
> +
> + common_set = profile_set & result_set
> + passed_list = [x for x in common_set if results.tests[x].result == status.PASS]
> +
> + total = len(common_set)
> + passed = len(passed_list)
> +
> + self.feat_fractions[results.name][feature] = (passed, total)
> + if total == 0:
> + self.feat_status[results.name][feature] = status.NOTRUN
> + else:
> + if 100 * passed // total >= feature_data[feature]["target_rate"]:
> + self.feat_status[results.name][feature] = status.PASS
> + else:
> + self.feat_status[results.name][feature] = status.FAIL
> diff --git a/framework/summary/html_.py b/framework/summary/html_.py
> index 12172b7..60d7f5e 100644
> --- a/framework/summary/html_.py
> +++ b/framework/summary/html_.py
> @@ -37,9 +37,11 @@ from mako.lookup import TemplateLookup
> from framework import backends, exceptions
>
> from .common import Results, escape_filename, escape_pathname
> +from .feature import FeatResults
>
> __all__ = [
> 'html',
> + 'feat'
> ]
>
> _TEMP_DIR = os.path.join(
> @@ -62,8 +64,9 @@ def _copy_static_files(destination):
> os.path.join(destination, "result.css"))
>
>
> -def _make_testrun_info(results, destination, exclude):
> +def _make_testrun_info(results, destination, exclude=None):
> """Create the pages for each results file."""
> + exclude = exclude or {}
> result_css = os.path.join(destination, "result.css")
> index = os.path.join(destination, "index.html")
>
> @@ -146,6 +149,14 @@ def _make_comparison_pages(results, destination, exclude):
> page=page, pages=pages))
>
>
> +def _make_feature_info(results, destination):
> + """Create the feature readiness page."""
> +
> + with open(os.path.join(destination, "feature.html"), 'w') as out:
> + out.write(_TEMPLATES.get_template('feature.mako').render(
> + results=results))
> +
> +
> def html(results, destination, exclude):
> """
> Produce HTML summaries.
> @@ -161,3 +172,13 @@ def html(results, destination, exclude):
> _copy_static_files(destination)
> _make_testrun_info(results, destination, exclude)
> _make_comparison_pages(results, destination, exclude)
> +
> +
> +def feat(results, destination, feat_desc):
> + """Produce HTML feature readiness summary."""
> +
> + feat_res = FeatResults([backends.load(i) for i in results], feat_desc)
> +
> + _copy_static_files(destination)
> + _make_testrun_info(feat_res, destination)
> + _make_feature_info(feat_res, destination)
> diff --git a/piglit b/piglit
> index 5ae43e9..c2a2e2b 100755
> --- a/piglit
> +++ b/piglit
> @@ -139,6 +139,10 @@ def main():
> add_help=False,
> help="Aggregate incomplete piglit run.")
> aggregate.set_defaults(func=summary.aggregate)
> + feature = summary_parser.add_parser('feature',
> + add_help=False,
> + help="generate feature readiness html report.")
> + feature.set_defaults(func=summary.feature)
>
> # Parse the known arguments (piglit run or piglit summary html for
> # example), and then pass the arguments that this parser doesn't know about
> diff --git a/templates/feature.mako b/templates/feature.mako
> new file mode 100644
> index 0000000..ac9bc86
> --- /dev/null
> +++ b/templates/feature.mako
> @@ -0,0 +1,73 @@
> +<%!
> + import posixpath # this must be posixpath, since we want /'s not \'s
> + import re
> +
> +
> + def feat_result(result):
> + """Percentage result string"""
> + return '{}/{}'.format(result[0], result[1])
> +
> +
> + def escape_filename(key):
> + """Avoid reserved characters in filenames."""
> + return re.sub(r'[<>:"|?*#]', '_', key)
> +
> +
> + def escape_pathname(key):
> + """ Remove / and \\ from names """
> + return re.sub(r'[/\\]', '_', key)
> +
> +
> + def normalize_href(href):
> + """Force backward slashes in URLs."""
> + return href.replace('\\', '/')
> +%>
> +
> +<?xml version="1.0" encoding="UTF-8"?>
> +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
> + "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
> +<html xmlns="http://www.w3.org/1999/xhtml">
> + <head>
> + <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
> + <title>Result summary</title>
> + <link rel="stylesheet" href="index.css" type="text/css" />
> + </head>
> + <body>
> + <h1>Feature readiness</h1>
> + <table>
> + <colgroup>
> + ## Name Column
> + <col />
> +
> + ## Status columns
> + ## Create an additional column for each summary
> + % for _ in xrange(len(results.results)):
> + <col />
> + % endfor
> + </colgroup>
> + <tr>
> + <th/>
> + % for res in results.results:
> + <th class="head"><b>${res.name}</b><br />\
> + (<a href="${normalize_href(posixpath.join(escape_pathname(res.name), 'index.html'))}">info</a>)</th>
> + % endfor
> + </tr>
> + % for feature in results.features:
> + <tr>
> + ## Add the left most column, the feature name
> + <td>
> + <div class="group">
> + <b>${feature}</b>
> + </div>
> + </td>
> + ## add the feature totals
> + % for res in results.results:
> + <td class="${results.feat_status[res.name][feature]}">
> + <b>${feat_result(results.feat_fractions[res.name][feature])}</b>
> + </td>
> + % endfor
> + </tr>
> + % endfor
> + </table>
> + </body>
> +</html>
> --
> 1.9.1
>
> _______________________________________________
> Piglit mailing list
> Piglit at lists.freedesktop.org
> http://lists.freedesktop.org/mailman/listinfo/piglit
I haven't run this version, but I did run the last version. The changes
look good.
Reviewed-by: Dylan Baker <baker.dylan.c at gmail.com>
I'll commit this tomorrow if anyone doesn't object by then.
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 473 bytes
Desc: not available
URL: <http://lists.freedesktop.org/archives/piglit/attachments/20151105/1a814409/attachment.sig>
More information about the Piglit
mailing list