[Piglit] [NOMERGE 1/7] utils/check.py: a small utility for checking missing tests.

[baker.dylan.c at gmail.com]

From: Dylan Baker <baker.dylan.c at gmail.com>

This adds a utility that parses all.py and then checks the binaries in
bin/ to see which ones are called.

A JSON file called "check_ignore.json" is provided to whitelist binaries
that shouldn't be in all.py for some reason

Future TODO:
- Add support for cl.py
---

I still don't think this is ready to land, so I'm sending it out so
others can see it, but I think it still needs some work before I'd
consider submitting it to master.

 utils/__init__.py       |   0
 utils/check.py          | 126 ++++++++++++++++++++++++++++++++++++++++++++++++
 utils/check_ignore.json |   3 ++
 3 files changed, 129 insertions(+)
 create mode 100644 utils/__init__.py
 create mode 100755 utils/check.py
 create mode 100644 utils/check_ignore.json

diff --git a/utils/__init__.py b/utils/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/utils/check.py b/utils/check.py
new file mode 100755
index 0000000..c902a5a
--- /dev/null
+++ b/utils/check.py
@@ -0,0 +1,126 @@
+#!/usr/bin/python2
+
+# Copyright (c) 2015 Intel Corporation
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+"""Small script that checks that all binaries in bin are in all.py."""
+
+from __future__ import print_function, absolute_import, division
+import os
+import stat
+import sys
+
+try:
+    import simplejson as json
+except ImportError:
+    import json  # pylint: disable=wrong-import-order
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+from tests.all import profile
+
+DIR = os.path.abspath(os.path.join(os.getcwd(), 'bin'))
+
+# A JSON formatted list containing binaries to ignore when looking at called
+# binaries. These are generally not meant to go in all.py for some reason.
+IGNORED = os.path.join(os.path.dirname(__file__), 'check_ignore.json')
+
+
+class NotBuiltError(Exception):
+    pass
+
+
+def _is_executable(filename):
+    """Check if a filepath is an executable.
+
+    Even if the executable bit is set, it could still be a shell script, or a
+    .c file, or something else that has the bit set incorrectly.
+
+    This checks for the exectuable bit, but then also checks that the file
+    extension is either empty or is 'exe' (covering both windows and *nix
+    systems).
+
+    """
+    if stat.S_IXUSR & os.stat(filename)[stat.ST_MODE]:
+        if os.path.splitext(filename)[1] not in ['', 'exe']:
+            return False
+        else:
+            return True
+    return False
+
+
+def make_called_binary_set():
+    """Create a set of binaries that are actually called in the profile."""
+    bins = set()
+    for test in profile.test_list.itervalues():
+        bins.add(os.path.basename(test.command[0]))
+    return bins
+
+
+def make_actual_binary_set():
+    """Create a set of binaries that are in the path."""
+    if not os.path.exists(DIR):
+        raise NotBuiltError
+
+    bins = set()
+    for dirpath, _, filenames in os.walk(DIR):
+        for filename in filenames:
+            filename = os.path.join(dirpath, filename)
+            if _is_executable(filename):
+                bins.add(os.path.basename(filename))
+
+    with open(IGNORED, 'r') as f:
+        ignored = set(json.load(f))
+
+    return set.difference(bins, ignored)
+
+
+def compare_sets():
+    """Find the binaries that are missing from called."""
+    called = make_called_binary_set()
+    actual = make_actual_binary_set()
+
+    missing = set()
+    for x in actual:
+        if x not in called:
+            missing.add(x)
+
+    return missing
+
+
+def main():
+    """Main function."""
+    try:
+        missing = compare_sets()
+    except NotBuiltError:
+        print('Error: directory "bin" doesn\'t exist', file=sys.stderr)
+        sys.exit(2)
+
+    if missing:
+        print('The following binaries are not referenced in all.py:')
+        print('\n'.join(missing))
+        sys.exit(1)
+    else:
+        print('All binaires are called in all.py, congratulations!')
+
+    sys.exit(0)
+
+
+if __name__ == '__main__':
+    main()
diff --git a/utils/check_ignore.json b/utils/check_ignore.json
new file mode 100644
index 0000000..60b3f4d
--- /dev/null
+++ b/utils/check_ignore.json
@@ -0,0 +1,3 @@
+[
+    "arb_robustness_draw-vbo-bounds"
+]
-- 
2.6.4