[Piglit] [PATCH 5/5] framework: Make testlist option impose ordering

[Dylan Baker]

The --test-list argument hasn't been used a lot because it isn't that
useful. Currently it is just a list of values passed into the same
function that handles the -t/--include-tests switch. That isn't very
very useful since -t can be given multiple times, and it is regex so
it's easy enough to build some complex regular expressions.

This patch changes the behavior of test to an authoritative list of
tests that will be run in a defined order (or as close as possible when
using concurrency). It does not exclude the -t and -x options, and can
be run with them, retaining ordering as much as possible. This means
that passing a testlist will remove any tests from the profile that are
not in the list (it also enforces that all of the tests exist), and runs
them in the order of the testlist file.

Signed-off-by: Dylan Baker <dylanx.c.baker at intel.com>
---
 framework/profile.py      | 15 +++++++++++++++
 framework/programs/run.py | 12 ++++++------
 2 files changed, 21 insertions(+), 6 deletions(-)

diff --git a/framework/profile.py b/framework/profile.py
index 9a117a2..9bdc720 100644
--- a/framework/profile.py
+++ b/framework/profile.py
@@ -167,6 +167,13 @@ class TestDict(collections.MutableMapping):
             if not callable((k, v)):
                 del self[k]
 
+    def reorder(self, order):
+        """Reorder the TestDict to match the order of the provided list."""
+        new = collections.OrderedDict()
+        for k in order:
+            new[k] = self.__container[k]
+        self.__container = new
+
 
 class TestProfile(object):
     """ Class that holds a list of tests for execution
@@ -188,6 +195,7 @@ class TestProfile(object):
     """
     def __init__(self):
         self.test_list = TestDict()
+        self.forced_test_list = []
         self.filters = []
         # Sets a default of a Dummy
         self._dmesg = None
@@ -239,6 +247,13 @@ class TestProfile(object):
                     return False
             return True
 
+        if self.forced_test_list:
+            # Remove all tests not in the test list, then reorder the tests to
+            # match the testlist. This still allows additional filters to be
+            # run afterwards.
+            self.test_list.filter(lambda i: i[0] in self.forced_test_list)
+            self.test_list.reorder(self.forced_test_list)
+
         # Filter out unwanted tests
         self.test_list.filter(check_all)
 
diff --git a/framework/programs/run.py b/framework/programs/run.py
index be40b97..dff9f49 100644
--- a/framework/programs/run.py
+++ b/framework/programs/run.py
@@ -159,6 +159,7 @@ def _run_parser(input_):
                             default='quiet',
                             help="Set the logger verbosity level")
     parser.add_argument("--test-list",
+                        type=os.path.abspath,
                         help="A file containing a list of tests to run")
     parser.add_argument('-o', '--overwrite',
                         dest='overwrite',
@@ -227,12 +228,6 @@ def run(input_):
     if args.dmesg:
         args.concurrency = "none"
 
-    # build up the include filter based on test_list
-    if args.test_list:
-        with open(args.test_list) as test_list:
-            for line in test_list.readlines():
-                args.include_tests.append(line.rstrip())
-
     # Pass arguments into Options
     options.OPTIONS.concurrent = args.concurrency
     options.OPTIONS.exclude_filter = args.exclude_tests
@@ -279,6 +274,11 @@ def run(input_):
 
     profile = framework.profile.merge_test_profiles(args.test_profile)
     profile.results_dir = args.results_path
+    # If a test list is provided then set the forced_test_list value.
+    if args.test_list:
+        with open(args.test_list) as test_list:
+            # Strip newlines
+            profile.forced_test_list = list([t.strip() for t in test_list])
 
     results.time_elapsed.start = time.time()
     # Set the dmesg type
-- 
2.8.2