[Piglit] [PATCH 1/5] Status: Skip and NotRun should not show up in fixes and regressions

[Dylan Baker]

This adds tests to demonstrate the above status is true. It also updates
the docstrings in status.py to reflect this change.

Signed-off-by: Dylan Baker <baker.dylan.c at gmail.com>
---
 framework/status.py             | 16 ++++------------
 framework/tests/status_tests.py | 32 +++++++++++++++++++++++++++++---
 2 files changed, 33 insertions(+), 15 deletions(-)

diff --git a/framework/status.py b/framework/status.py
index e34e4af..d5e6b63 100644
--- a/framework/status.py
+++ b/framework/status.py
@@ -35,8 +35,6 @@ A tuple representing :
 
 Status ordering from best to worst:
 
-NotRun
-skip
 pass
 dmesg-warn
 warn
@@ -45,20 +43,14 @@ fail
 crash
 timeout
 
-(NotRun, pass, skip) are considered equivalent for regression testing.
-
-The motivation is if you accidentally expose a feature that doesn't work,
-you'll get skip->fail, which is a regression. If you disable the feature,
-you'll get fail->skip, which is a fix.
-
-NotRun->fail should also be considered a regression for you not to miss
-new failing tests.
+SKIP and NOTRUN are not factored into regressions and fixes, they are counted
+seperately.
 
 The formula for determining regressions is:
-  max(old_status, pass) < new_status
+  max(PASS, old_status) < new_status
 
 The formula for determining fixes is:
-  old_status > max(new_status, pass)
+  max(PASS, old_status) > new_status
 
 """
 
diff --git a/framework/tests/status_tests.py b/framework/tests/status_tests.py
index 9280bd5..8ba11aa 100644
--- a/framework/tests/status_tests.py
+++ b/framework/tests/status_tests.py
@@ -26,12 +26,12 @@ etc
 """
 
 import itertools
+import nose.tools as nt
 import framework.status as status
 
 # Statuses from worst to last. NotRun is intentionally not in this list and
 # tested separately because of upcoming features for it
-STATUSES = ["notrun", "pass", "dmesg-warn", "warn", "dmesg-fail", "fail",
-            "crash"]
+STATUSES = ["pass", "dmesg-warn", "warn", "dmesg-fail", "fail", "crash"]
 
 # Create lists of fixes and regressions programmatically based on the STATUSES
 # list. This means less code, and easier expansion changes.
@@ -58,7 +58,7 @@ def test_gen_lookup():
     """ Generator that attempts to do a lookup on all statuses """
     yieldable = check_lookup
 
-    for stat in STATUSES + ['skip']:
+    for stat in STATUSES + ['skip', 'notrun']:
         yieldable.description = "Lookup: {}".format(stat)
         yield yieldable, stat
 
@@ -114,3 +114,29 @@ def test_is_change():
         yieldable.description = ("Test that {0} -> {1} is a "
                                  "change".format(new, old))
         yield yieldable, new, old
+
+
+def check_not_change(new, old):
+    """ Check that a status doesn't count as a change 
+
+    This checks that new < old and old < new do not return true. This is meant
+    for checking skip and notrun, which we don't want to show up as regressions
+    and fixes, but to go in their own special catagories.
+
+    """
+    nt.assert_false(new < old,
+                    msg="{new} -> {old}, is a change "
+                        "but shouldn't be".format(**locals()))
+    nt.assert_false(new > old,
+                    msg="{new} <- {old}, is a change "
+                        "but shouldn't be".format(**locals()))
+
+
+def test_not_change():
+    """ Skip and NotRun should not count as changes """
+    yieldable = check_not_change
+
+    for nochange, stat in itertools.product(['skip', 'notrun'], STATUSES):
+        yieldable.description = "{0} -> {1} should not be a change".format(
+                nochange, stat)
+        yield yieldable, status.status_lookup(nochange), status.status_lookup(stat)
-- 
1.9.0