[Piglit] [PATCH v3] Add support for feature readiness.
Gabriel Feceoru
gabriel.feceoru at intel.com
Tue Nov 3 07:50:41 PST 2015
This adds a new "summary feature" command to piglit which creates a HTML table
with the feature x DUT status (useful for multiple features, multiple DUTs).
Another use case is the feature status for subsequent test results (depending
on the meaning of that test result - DUT or build)
A feature readiness is defined by the piglit regexp which selects the tests
relevant for that feature and the acceptance percentage threshold (pass rate).
It requires an input json file containing the list of features, in the
following format (this is just an example):
{
"glsl" : {
"include_tests" : "glsl",
"exclude_tests" : "",
"target_rate" : 90
},
"arb" : {
"include_tests" : "arb_gpu",
"exclude_tests" : "",
"target_rate" : 10
}
}
v3:
Changed json rate to int instead of string
Applied other review comments
v2:
Apply review comments (Dylan, Thomas)
Fixed 2nd round of review comments
Signed-off-by: Gabriel Feceoru <gabriel.feceoru at intel.com>
---
framework/programs/summary.py | 37 ++++++++++++++++
framework/summary/__init__.py | 2 +-
framework/summary/feature.py | 99 +++++++++++++++++++++++++++++++++++++++++++
framework/summary/html_.py | 23 +++++++++-
piglit | 4 ++
templates/feature.mako | 73 +++++++++++++++++++++++++++++++
6 files changed, 236 insertions(+), 2 deletions(-)
create mode 100644 framework/summary/feature.py
create mode 100644 templates/feature.mako
diff --git a/framework/programs/summary.py b/framework/programs/summary.py
index 8711ee5..b23f1ef 100644
--- a/framework/programs/summary.py
+++ b/framework/programs/summary.py
@@ -35,6 +35,7 @@ __all__ = [
'console',
'csv',
'html',
+ 'feature'
]
@@ -224,3 +225,39 @@ def aggregate(input_):
print("Aggregated file written to: {}.{}".format(
outfile, backends.compression.get_mode()))
+
+
+ at exceptions.handler
+def feature(input_):
+ parser = argparse.ArgumentParser()
+ parser.add_argument("-o", "--overwrite",
+ action="store_true",
+ help="Overwrite existing directories")
+ parser.add_argument("featureFile",
+ metavar="<Feature json file>",
+ help="Json file containing the features description")
+ parser.add_argument("summaryDir",
+ metavar="<Summary Directory>",
+ help="Directory to put HTML files in")
+ parser.add_argument("resultsFiles",
+ metavar="<Results Files>",
+ nargs="*",
+ help="Results files to include in HTML")
+ args = parser.parse_args(input_)
+
+ # If args.list and args.resultsFiles are empty, then raise an error
+ if not args.featureFile and not args.resultsFiles:
+ raise parser.error("Missing required option -l or <resultsFiles>")
+
+ # If args.list and args.resultsFiles are empty, then raise an error
+ if not args.resultsFiles or not path.exists(args.featureFile):
+ raise parser.error("Missing json file")
+
+ # if overwrite is requested delete the output directory
+ if path.exists(args.summaryDir) and args.overwrite:
+ shutil.rmtree(args.summaryDir)
+
+ # If the requested directory doesn't exist, create it or throw an error
+ core.checkDir(args.summaryDir, not args.overwrite)
+
+ summary.feat(args.resultsFiles, args.summaryDir, args.featureFile)
diff --git a/framework/summary/__init__.py b/framework/summary/__init__.py
index 8a1ff8a..0f0f144 100644
--- a/framework/summary/__init__.py
+++ b/framework/summary/__init__.py
@@ -25,5 +25,5 @@
# public parts here, so that we have a nice interface to work with.
from __future__ import absolute_import, division, print_function
-from .html_ import html
+from .html_ import html, feat
from .console_ import console
diff --git a/framework/summary/feature.py b/framework/summary/feature.py
new file mode 100644
index 0000000..2fb8f50
--- /dev/null
+++ b/framework/summary/feature.py
@@ -0,0 +1,99 @@
+# Copyright (c) 2015 Intel Corporation
+
+# Permission is hereby granted, free of charge, to any person
+# obtaining a copy of this software and associated documentation
+# files (the "Software"), to deal in the Software without
+# restriction, including without limitation the rights to use,
+# copy, modify, merge, publish, distribute, sublicense, and/or
+# sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following
+# conditions:
+#
+# This permission notice shall be included in all copies or
+# substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHOR(S) BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+# AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
+# OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+from __future__ import print_function, division, absolute_import
+
+import copy
+
+try:
+ import simplejson as json
+except ImportError:
+ import json
+
+from framework import core, exceptions, profile, status
+
+
+class FeatResults(object): # pylint: disable=too-few-public-methods
+ """Container object for results.
+
+ Has the results, feature profiles and feature computed results.
+
+ """
+ def __init__(self, results, json_file):
+
+ with open(json_file) as data:
+ feature_data = json.load(data)
+
+ self.feat_fractions = {}
+ self.feat_status = {}
+ self.features = set()
+ self.results = results
+
+ profiles = {}
+
+ # we expect all the result sets to be for the same profile
+ profile_orig = profile.load_test_profile(results[0].options['profile'][0])
+
+ for feature in feature_data:
+ self.features.add(feature)
+
+ incl_str = feature_data[feature]["include_tests"]
+ excl_str = feature_data[feature]["exclude_tests"]
+
+ include_filter = [incl_str] if incl_str and not incl_str.isspace() else []
+ exclude_filter = [excl_str] if excl_str and not excl_str.isspace() else []
+
+ opts = core.Options(include_filter=include_filter,
+ exclude_filter=exclude_filter)
+
+ profiles[feature] = copy.deepcopy(profile_orig)
+
+ # An empty list will raise PiglitFatalError exception
+ # But for reporting we need to handle this situation
+ try:
+ profiles[feature]._prepare_test_list(opts)
+ except exceptions.PiglitFatalError:
+ pass
+
+ for results in self.results:
+ self.feat_fractions[results.name] = {}
+ self.feat_status[results.name] = {}
+
+ for feature in feature_data:
+ result_set = set(results.tests)
+ profile_set = set(profiles[feature].test_list)
+
+ common_set = profile_set & result_set
+ passed_list = [x for x in common_set if results.tests[x].result == status.PASS]
+
+ total = len(common_set)
+ passed = len(passed_list)
+
+ self.feat_fractions[results.name][feature] = (passed, total)
+ if total == 0:
+ self.feat_status[results.name][feature] = status.NOTRUN
+ else:
+ if 100 * passed // total >= feature_data[feature]["target_rate"]:
+ self.feat_status[results.name][feature] = status.PASS
+ else:
+ self.feat_status[results.name][feature] = status.FAIL
diff --git a/framework/summary/html_.py b/framework/summary/html_.py
index 12172b7..60d7f5e 100644
--- a/framework/summary/html_.py
+++ b/framework/summary/html_.py
@@ -37,9 +37,11 @@ from mako.lookup import TemplateLookup
from framework import backends, exceptions
from .common import Results, escape_filename, escape_pathname
+from .feature import FeatResults
__all__ = [
'html',
+ 'feat'
]
_TEMP_DIR = os.path.join(
@@ -62,8 +64,9 @@ def _copy_static_files(destination):
os.path.join(destination, "result.css"))
-def _make_testrun_info(results, destination, exclude):
+def _make_testrun_info(results, destination, exclude=None):
"""Create the pages for each results file."""
+ exclude = exclude or {}
result_css = os.path.join(destination, "result.css")
index = os.path.join(destination, "index.html")
@@ -146,6 +149,14 @@ def _make_comparison_pages(results, destination, exclude):
page=page, pages=pages))
+def _make_feature_info(results, destination):
+ """Create the feature readiness page."""
+
+ with open(os.path.join(destination, "feature.html"), 'w') as out:
+ out.write(_TEMPLATES.get_template('feature.mako').render(
+ results=results))
+
+
def html(results, destination, exclude):
"""
Produce HTML summaries.
@@ -161,3 +172,13 @@ def html(results, destination, exclude):
_copy_static_files(destination)
_make_testrun_info(results, destination, exclude)
_make_comparison_pages(results, destination, exclude)
+
+
+def feat(results, destination, feat_desc):
+ """Produce HTML feature readiness summary."""
+
+ feat_res = FeatResults([backends.load(i) for i in results], feat_desc)
+
+ _copy_static_files(destination)
+ _make_testrun_info(feat_res, destination)
+ _make_feature_info(feat_res, destination)
diff --git a/piglit b/piglit
index 5ae43e9..c2a2e2b 100755
--- a/piglit
+++ b/piglit
@@ -139,6 +139,10 @@ def main():
add_help=False,
help="Aggregate incomplete piglit run.")
aggregate.set_defaults(func=summary.aggregate)
+ feature = summary_parser.add_parser('feature',
+ add_help=False,
+ help="generate feature readiness html report.")
+ feature.set_defaults(func=summary.feature)
# Parse the known arguments (piglit run or piglit summary html for
# example), and then pass the arguments that this parser doesn't know about
diff --git a/templates/feature.mako b/templates/feature.mako
new file mode 100644
index 0000000..ac9bc86
--- /dev/null
+++ b/templates/feature.mako
@@ -0,0 +1,73 @@
+<%!
+ import posixpath # this must be posixpath, since we want /'s not \'s
+ import re
+
+
+ def feat_result(result):
+ """Percentage result string"""
+ return '{}/{}'.format(result[0], result[1])
+
+
+ def escape_filename(key):
+ """Avoid reserved characters in filenames."""
+ return re.sub(r'[<>:"|?*#]', '_', key)
+
+
+ def escape_pathname(key):
+ """ Remove / and \\ from names """
+ return re.sub(r'[/\\]', '_', key)
+
+
+ def normalize_href(href):
+ """Force backward slashes in URLs."""
+ return href.replace('\\', '/')
+%>
+
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+ <head>
+ <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
+ <title>Result summary</title>
+ <link rel="stylesheet" href="index.css" type="text/css" />
+ </head>
+ <body>
+ <h1>Feature readiness</h1>
+ <table>
+ <colgroup>
+ ## Name Column
+ <col />
+
+ ## Status columns
+ ## Create an additional column for each summary
+ % for _ in xrange(len(results.results)):
+ <col />
+ % endfor
+ </colgroup>
+ <tr>
+ <th/>
+ % for res in results.results:
+ <th class="head"><b>${res.name}</b><br />\
+ (<a href="${normalize_href(posixpath.join(escape_pathname(res.name), 'index.html'))}">info</a>)</th>
+ % endfor
+ </tr>
+ % for feature in results.features:
+ <tr>
+ ## Add the left most column, the feature name
+ <td>
+ <div class="group">
+ <b>${feature}</b>
+ </div>
+ </td>
+ ## add the feature totals
+ % for res in results.results:
+ <td class="${results.feat_status[res.name][feature]}">
+ <b>${feat_result(results.feat_fractions[res.name][feature])}</b>
+ </td>
+ % endfor
+ </tr>
+ % endfor
+ </table>
+ </body>
+</html>
--
1.9.1
More information about the Piglit
mailing list