Mesa (master): tracie: Produce JUnit XML results
GitLab Mirror
gitlab-mirror at kemper.freedesktop.org
Tue Sep 8 08:24:34 UTC 2020
Module: Mesa
Branch: master
Commit: d24ffd870615f0918c19c35b428251ba86bb49d7
URL: http://cgit.freedesktop.org/mesa/mesa/commit/?id=d24ffd870615f0918c19c35b428251ba86bb49d7
Author: Alexandros Frantzis <alexandros.frantzis at collabora.com>
Date: Thu Sep 3 16:29:05 2020 +0300
tracie: Produce JUnit XML results
Write tracie results to the 'results/junit.xml' file using the JUnit XML
format. Among other uses, this file can be picked up by gitlab to
display more useful information to the user.
Signed-off-by: Alexandros Frantzis <alexandros.frantzis at collabora.com>
Reviewed-by: Eric Anholt <eric at anholt.net>
Reviewed-by: Pierre-Eric Pelloux-Prayer <pierre-eric.pelloux-prayer at amd.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/6598>
---
.gitlab-ci/tracie/tests/test.py | 36 +++++++++++++++++++++++++++++
.gitlab-ci/tracie/tracie.py | 50 ++++++++++++++++++++++++++++++++++++-----
2 files changed, 81 insertions(+), 5 deletions(-)
diff --git a/.gitlab-ci/tracie/tests/test.py b/.gitlab-ci/tracie/tests/test.py
index ff4780cbe3b..ac531a3aef6 100644
--- a/.gitlab-ci/tracie/tests/test.py
+++ b/.gitlab-ci/tracie/tests/test.py
@@ -2,6 +2,7 @@ import logging
import pytest
import re
import shutil
+import xml.etree.ElementTree as ET
from os import environ, chdir
from os.path import dirname, exists, realpath
@@ -10,6 +11,7 @@ import tracie
RESULTS_YAML = "results/results.yml"
+JUNIT_XML = "results/junit.xml"
TRACE_LOG_TEST1 = "results/trace1/test/gl-test-device/magenta.testtrace.log"
TRACE_LOG_TEST2 = "results/trace2/test/vk-test-device/olive.testtrace.log"
TRACE_PNG_TEST1 = "results/trace1/test/gl-test-device/magenta.testtrace-0.png"
@@ -215,3 +217,37 @@ def test_tracie_stores_images_on_request():
assert run_tracie()
assert exists(TRACE_PNG_TEST1)
assert exists(TRACE_PNG_TEST2)
+
+def test_tracie_writes_junit_xml():
+ assert run_tracie()
+ junit_xml = ET.parse(JUNIT_XML)
+ assert junit_xml.getroot().tag == 'testsuites'
+ testsuites = junit_xml.findall("./testsuite")
+ testcases_gl = junit_xml.findall("./testsuite[@name='traces.yml:gl-test-device']/testcase")
+ testcases_vk = junit_xml.findall("./testsuite[@name='traces.yml:vk-test-device']/testcase")
+
+ assert len(testsuites) == 2
+ assert len(testcases_gl) == 1
+ assert len(testcases_vk) == 1
+ assert testcases_gl[0].get("name") == "trace1/magenta.testtrace"
+ assert testcases_gl[0].get("classname") == "traces.yml:gl-test-device"
+ assert testcases_vk[0].get("name") == "trace2/olive.testtrace"
+ assert testcases_vk[0].get("classname") == "traces.yml:vk-test-device"
+
+def test_tracie_writes_dashboard_url_in_junit_xml_failure_tag():
+ filename = "./tests/traces.yml"
+ content = read_from(filename)
+ content = content.replace("5efda83854befe0155ff8517a58d5b51",
+ "8e0a801367e1714463475a824dab363b")
+ write_to(content, filename)
+
+ assert not run_tracie()
+
+ junit_xml = ET.parse(JUNIT_XML)
+ failures_gl = junit_xml.findall("./testsuite[@name='traces.yml:gl-test-device']/testcase/failure")
+ failures_vk = junit_xml.findall("./testsuite[@name='traces.yml:vk-test-device']/testcase/failure")
+
+ assert len(failures_gl) == 0
+ assert len(failures_vk) == 1
+ dashboard_url = "https://tracie.freedesktop.org/dashboard/imagediff/test-project/42/trace2/olive.testtrace"
+ assert dashboard_url in failures_vk[0].text
diff --git a/.gitlab-ci/tracie/tracie.py b/.gitlab-ci/tracie/tracie.py
index 170400d9d54..30468078868 100644
--- a/.gitlab-ci/tracie/tracie.py
+++ b/.gitlab-ci/tracie/tracie.py
@@ -138,6 +138,12 @@ def ensure_reference_image(file_name, checksum):
return
upload_to_minio(file_name, resource, 'image/png')
+def image_diff_url(trace_path):
+ return "%s/imagediff/%s/%s/%s" % (DASHBOARD_URL,
+ os.environ.get('CI_PROJECT_PATH'),
+ os.environ.get('CI_JOB_ID'),
+ trace_path)
+
def gitlab_check_trace(project_url, device_name, trace, expectation):
gitlab_ensure_trace(project_url, trace)
@@ -158,11 +164,7 @@ def gitlab_check_trace(project_url, device_name, trace, expectation):
(trace['path'], expectation['checksum'], checksum))
print("[check_image] For more information see "
"https://gitlab.freedesktop.org/mesa/mesa/blob/master/.gitlab-ci/tracie/README.md")
- image_diff_url = "%s/imagediff/%s/%s/%s" % (DASHBOARD_URL,
- os.environ['CI_PROJECT_PATH'],
- os.environ['CI_JOB_ID'],
- trace['path'])
- print("[check_image] %s" % image_diff_url)
+ print("[check_image] %s" % image_diff_url(trace['path']))
ok = False
trace_dir = os.path.split(trace['path'])[0]
@@ -185,6 +187,39 @@ def gitlab_check_trace(project_url, device_name, trace, expectation):
return ok, result
+def write_junit_xml(junit_xml_path, traces_filename, device_name, results):
+ tests = len(results)
+ failures = sum(1 for r in results.values() if r["actual"] != r["expected"])
+
+ try:
+ testsuites = ET.parse(junit_xml_path).getroot()
+ except:
+ test_name = os.environ.get('CI_PROJECT_PATH') + "/" + \
+ os.environ.get('CI_PIPELINE_ID') + "/" + \
+ os.environ.get('CI_JOB_ID')
+ testsuites = ET.Element('testsuites', name=test_name)
+
+ testsuites.set('tests', str(int(testsuites.get('tests', 0)) + tests))
+ testsuites.set('failures', str(int(testsuites.get('failures', 0)) + failures))
+
+ testsuite_name = os.path.basename(traces_filename) + ":" + device_name
+
+ testsuite = ET.SubElement(testsuites, 'testsuite',
+ name=testsuite_name,
+ tests=str(tests), failures=str(failures))
+
+ for (path, result) in results.items():
+ testcase = ET.SubElement(testsuite, 'testcase', name=path,
+ classname=testsuite_name)
+ if result["actual"] != result["expected"]:
+ failure = ET.SubElement(testcase, 'failure')
+ failure.text = \
+ ("Images differ (expected: %s, actual: %s).\n" + \
+ "To view the image differences visit:\n%s") % \
+ (result["expected"], result["actual"], image_diff_url(path))
+
+ ET.ElementTree(testsuites).write(junit_xml_path)
+
def run(filename, device_name):
with open(filename, 'r') as f:
@@ -210,8 +245,13 @@ def run(filename, device_name):
os.makedirs(RESULTS_PATH, exist_ok=True)
with open(os.path.join(RESULTS_PATH, 'results.yml'), 'w') as f:
yaml.safe_dump(results, f, default_flow_style=False)
+
+ junit_xml_path = os.path.join(RESULTS_PATH, "junit.xml")
+ write_junit_xml(junit_xml_path, filename, device_name, results)
+
if os.environ.get('TRACIE_UPLOAD_TO_MINIO', '0') == '1':
upload_artifact(os.path.join(RESULTS_PATH, 'results.yml'), 'traces/results.yml', 'text/yaml')
+ upload_artifact(junit_xml_path, 'traces/junit.xml', 'text/xml')
return all_ok
More information about the mesa-commit
mailing list