gst-qa-system: Make reruns silently replace non-reruns where possible

[Edward Hervey]

Module: gst-qa-system
Branch: master
Commit: 1870728f918964c73a05fc87dccf64a5d1f5a84b
URL:    http://cgit.freedesktop.org/gstreamer/gst-qa-system/commit/?id=1870728f918964c73a05fc87dccf64a5d1f5a84b

Author: David Laban <david.laban at collabora.co.uk>
Date:   Thu Sep 15 15:12:52 2011 +0100

Make reruns silently replace non-reruns where possible

This makes it easier to compare results from gst-media-test --no-reruns
against results gathered with reruns enabled.

---

 bin/insanity-dumpresults-json |   28 ++++++++++++++++++++++++++++
 1 files changed, 28 insertions(+), 0 deletions(-)

diff --git a/bin/insanity-dumpresults-json b/bin/insanity-dumpresults-json
index 63c7f69..98ae464 100755
--- a/bin/insanity-dumpresults-json
+++ b/bin/insanity-dumpresults-json
@@ -70,6 +70,9 @@ def getTestName(db, testid):
 
     if parentid:
         parent_name = getTestName(db, parentid)
+        # strip the rerun mechanism from results names.
+        if "GstMediaTestScenario" in parent_name:
+            parent_name = ""
     else:
         parent_name = ""
 
@@ -99,11 +102,17 @@ def getTestInfo(db, testid):
 
     if resperc == 100:
         data["result"] = "pass"
+    elif dict(checks).get("no-unexpected-failures", 0):
+        data["result"] = "expected-failure"
     else:
         data["result"] = "fail"
 
     data["test_case_id"] = getTestName(db, testid)
 
+    # strip the rerun mechanism from results unless something is wrong.
+    if ttype == "GstMediaTestScenario" and \
+            dict(checks)["similar-results"]:
+        return {}
 
     monitors = getMonitorsInfo(db, testid)
 
@@ -140,12 +149,31 @@ def getTestRun(db, testrunid, failedonly=False, hidescenarios=False):
                                   failedonly=failedonly)
 
     test_results = []
+    tests_by_name = {}
     for testid in tests:
         data = getTestInfo(db, testid)
         if not data:
             continue
+        name = data["test_case_id"]
+        tests_by_name[name] = data
         test_results.append(data)
 
+    for name, data in tests_by_name.items():
+        if "rerun." in name:
+            canonical_name = name.replace("rerun.", "")
+            if canonical_name in tests_by_name:
+                if tests_by_name[canonical_name]["attributes"]["success-percentage"] == \
+                        data["attributes"]["success-percentage"]:
+                    test_results.remove(tests_by_name[canonical_name])
+                    data["test_case_id"] = canonical_name
+                    tests_by_name[canonical_name] = data
+                else:
+                    warning("%s and %s have different results. " \
+                            "Leaving both in report.", canonical_name, name)
+            else:
+                warning("%s is a rerun, but %s doesn't exist",
+                        name, canonical_name)
+
 
     test_results.sort(key=lambda r: r["test_case_id"])
     return test_results