Mesa (main): ci/lava: Parse all test cases from 0_mesa suite
GitLab Mirror
gitlab-mirror at kemper.freedesktop.org
Thu Apr 28 07:34:16 UTC 2022
Module: Mesa
Branch: main
Commit: 18d80f25ee242ec21a4f772d107551e8dadae955
URL: http://cgit.freedesktop.org/mesa/mesa/commit/?id=18d80f25ee242ec21a4f772d107551e8dadae955
Author: Guilherme Gallo <guilherme.gallo at collabora.com>
Date: Wed Mar 23 22:10:51 2022 -0300
ci/lava: Parse all test cases from 0_mesa suite
LAVA can filter which test suite to show the results from, let's list
all testcases possible in the mesa test suite, to be able to divide more
complex jobs into test_cases.
Another advantage is that the test case can vary its name.
Signed-off-by: Guilherme Gallo <guilherme.gallo at collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/15938>
---
.gitlab-ci/lava/lava_job_submitter.py | 40 ++++++++-----
.gitlab-ci/tests/test_lava_job_submitter.py | 91 +++++++++++++++++++++--------
2 files changed, 95 insertions(+), 36 deletions(-)
diff --git a/.gitlab-ci/lava/lava_job_submitter.py b/.gitlab-ci/lava/lava_job_submitter.py
index 05fca6b7079..fc1c1aaa146 100755
--- a/.gitlab-ci/lava/lava_job_submitter.py
+++ b/.gitlab-ci/lava/lava_job_submitter.py
@@ -267,7 +267,7 @@ class LAVAJob():
fatal_err(f"Could not get LAVA job logs. Reason: {mesa_exception}")
-def get_job_results(proxy, job_id, test_suite, test_case):
+def get_job_results(proxy, job_id, test_suite):
# Look for infrastructure errors and retry if we see them.
results_yaml = _call_proxy(proxy.results.get_testjob_results_yaml, job_id)
results = yaml.load(results_yaml, Loader=loader(False))
@@ -275,19 +275,33 @@ def get_job_results(proxy, job_id, test_suite, test_case):
metadata = res["metadata"]
if "result" not in metadata or metadata["result"] != "fail":
continue
- if 'error_type' in metadata and metadata['error_type'] == "Infrastructure":
- raise MesaCIException("LAVA job {} failed with Infrastructure Error. Retry.".format(job_id))
- if 'case' in metadata and metadata['case'] == "validate":
- raise MesaCIException("LAVA job {} failed validation (possible download error). Retry.".format(job_id))
+ if "error_type" in metadata and metadata["error_type"] == "Infrastructure":
+ raise MesaCIException(
+ f"LAVA job {job_id} failed with Infrastructure Error. Retry."
+ )
+ if "case" in metadata and metadata["case"] == "validate":
+ raise MesaCIException(
+ f"LAVA job {job_id} failed validation (possible download error). Retry."
+ )
- results_yaml = _call_proxy(proxy.results.get_testcase_results_yaml, job_id, test_suite, test_case)
- results = yaml.load(results_yaml, Loader=loader(False))
+ results_yaml = _call_proxy(
+ proxy.results.get_testsuite_results_yaml, job_id, test_suite
+ )
+ results: list = yaml.load(results_yaml, Loader=loader(False))
if not results:
- raise MesaCIException("LAVA: no result for test_suite '{}', test_case '{}'".format(test_suite, test_case))
-
- print_log("LAVA: result for test_suite '{}', test_case '{}': {}".format(test_suite, test_case, results[0]['result']))
- if results[0]['result'] != 'pass':
- return False
+ raise MesaCIException(
+ f"LAVA: no result for test_suite '{test_suite}'"
+ )
+
+ for metadata in results:
+ test_case = metadata["name"]
+ result = metadata["metadata"]["result"]
+ print_log(
+ f"LAVA: result for test_suite '{test_suite}', "
+ f"test_case '{test_case}': {result}"
+ )
+ if result != "pass":
+ return False
return True
@@ -353,7 +367,7 @@ def follow_job_execution(job):
print(line)
show_job_data(job)
- return get_job_results(job.proxy, job.job_id, "0_mesa", "mesa")
+ return get_job_results(job.proxy, job.job_id, "0_mesa")
def retriable_follow_job(proxy, job_definition):
diff --git a/.gitlab-ci/tests/test_lava_job_submitter.py b/.gitlab-ci/tests/test_lava_job_submitter.py
index 43896db20fe..bd21958c075 100644
--- a/.gitlab-ci/tests/test_lava_job_submitter.py
+++ b/.gitlab-ci/tests/test_lava_job_submitter.py
@@ -26,19 +26,20 @@ import xmlrpc.client
from contextlib import nullcontext as does_not_raise
from datetime import datetime
from itertools import cycle, repeat
-from typing import Iterable, Union, Generator, Tuple
+from typing import Generator, Iterable, Tuple, Union
from unittest.mock import MagicMock, patch
import pytest
import yaml
from freezegun import freeze_time
from lava.lava_job_submitter import (
- NUMBER_OF_RETRIES_TIMEOUT_DETECTION,
DEVICE_HANGING_TIMEOUT_SEC,
+ NUMBER_OF_RETRIES_TIMEOUT_DETECTION,
+ LAVAJob,
follow_job_execution,
+ get_job_results,
hide_sensitive_data,
retriable_follow_job,
- LAVAJob
)
NUMBER_OF_MAX_ATTEMPTS = NUMBER_OF_RETRIES_TIMEOUT_DETECTION + 1
@@ -51,35 +52,37 @@ def jobs_logs_response(finished=False, msg=None, lvl="target") -> Tuple[bool, st
return finished, yaml.safe_dump(logs)
-def result_get_testjob_results_response() -> str:
- result = {"result": "test"}
- results = [{"metadata": result}]
-
- return yaml.safe_dump(results)
-
+RESULT_GET_TESTJOB_RESULTS = [{"metadata": {"result": "test"}}]
-def result_get_testcase_results_response() -> str:
- result = {"result": "pass"}
- test_cases = [result]
- return yaml.safe_dump(test_cases)
+def generate_testsuite_result(name="test-mesa-ci", result="pass", metadata_extra = None, extra = None):
+ if metadata_extra is None:
+ metadata_extra = {}
+ if extra is None:
+ extra = {}
+ return {"metadata": {"result": result, **metadata_extra}, "name": name}
@pytest.fixture
def mock_proxy():
- def create_proxy_mock(**kwargs):
+ def create_proxy_mock(
+ job_results=RESULT_GET_TESTJOB_RESULTS,
+ testsuite_results=[generate_testsuite_result()],
+ **kwargs
+ ):
proxy_mock = MagicMock()
proxy_submit_mock = proxy_mock.scheduler.jobs.submit
proxy_submit_mock.return_value = "1234"
proxy_results_mock = proxy_mock.results.get_testjob_results_yaml
- proxy_results_mock.return_value = result_get_testjob_results_response()
+ proxy_results_mock.return_value = yaml.safe_dump(job_results)
- proxy_test_cases_mock = proxy_mock.results.get_testcase_results_yaml
- proxy_test_cases_mock.return_value = result_get_testcase_results_response()
+ proxy_test_suites_mock = proxy_mock.results.get_testsuite_results_yaml
+ proxy_test_suites_mock.return_value = yaml.safe_dump(testsuite_results)
proxy_logs_mock = proxy_mock.scheduler.jobs.logs
proxy_logs_mock.return_value = jobs_logs_response()
+
for key, value in kwargs.items():
setattr(proxy_logs_mock, key, value)
@@ -156,56 +159,93 @@ NETWORK_EXCEPTION = xmlrpc.client.ProtocolError("", 0, "test", {})
XMLRPC_FAULT = xmlrpc.client.Fault(0, "test")
PROXY_SCENARIOS = {
- "finish case": (generate_n_logs(1), does_not_raise(), True),
+ "finish case": (generate_n_logs(1), does_not_raise(), True, {}),
"works at last retry": (
generate_n_logs(n=NUMBER_OF_MAX_ATTEMPTS, tick_fn=[ DEVICE_HANGING_TIMEOUT_SEC + 1 ] * NUMBER_OF_RETRIES_TIMEOUT_DETECTION + [1]),
does_not_raise(),
True,
+ {},
),
"timed out more times than retry attempts": (
generate_n_logs(n=4, tick_fn=DEVICE_HANGING_TIMEOUT_SEC + 1),
pytest.raises(SystemExit),
False,
+ {},
),
"long log case, no silence": (
generate_n_logs(n=1000, tick_fn=0),
does_not_raise(),
True,
+ {},
+ ),
+ "no retries, testsuite succeed": (
+ generate_n_logs(n=1, tick_fn=0),
+ does_not_raise(),
+ True,
+ {
+ "testsuite_results": [
+ generate_testsuite_result(result="pass")
+ ]
+ },
+ ),
+ "no retries, but testsuite fails": (
+ generate_n_logs(n=1, tick_fn=0),
+ does_not_raise(),
+ False,
+ {
+ "testsuite_results": [
+ generate_testsuite_result(result="fail")
+ ]
+ },
+ ),
+ "no retries, one testsuite fails": (
+ generate_n_logs(n=1, tick_fn=0),
+ does_not_raise(),
+ False,
+ {
+ "testsuite_results": [
+ generate_testsuite_result(result="fail"),
+ generate_testsuite_result(result="pass")
+ ]
+ },
),
"very long silence": (
generate_n_logs(n=NUMBER_OF_MAX_ATTEMPTS + 1, tick_fn=100000),
pytest.raises(SystemExit),
False,
+ {},
),
# If a protocol error happens, _call_proxy will retry without affecting timeouts
"unstable connection, ProtocolError followed by final message": (
(NETWORK_EXCEPTION, jobs_logs_response(finished=True)),
does_not_raise(),
True,
+ {},
),
# After an arbitrary number of retries, _call_proxy should call sys.exit
"unreachable case, subsequent ProtocolErrors": (
repeat(NETWORK_EXCEPTION),
pytest.raises(SystemExit),
False,
+ {},
),
- "XMLRPC Fault": ([XMLRPC_FAULT], pytest.raises(SystemExit, match="1"), False),
+ "XMLRPC Fault": ([XMLRPC_FAULT], pytest.raises(SystemExit, match="1"), False, {}),
}
@patch("time.sleep", return_value=None) # mock sleep to make test faster
@pytest.mark.parametrize(
- "side_effect, expectation, has_finished",
+ "side_effect, expectation, job_result, proxy_args",
PROXY_SCENARIOS.values(),
ids=PROXY_SCENARIOS.keys(),
)
def test_retriable_follow_job(
- mock_sleep, side_effect, expectation, has_finished, mock_proxy
+ mock_sleep, side_effect, expectation, job_result, proxy_args, mock_proxy
):
with expectation:
- proxy = mock_proxy(side_effect=side_effect)
+ proxy = mock_proxy(side_effect=side_effect, **proxy_args)
result = retriable_follow_job(proxy, "")
- assert has_finished == result
+ assert job_result == result
WAIT_FOR_JOB_SCENARIOS = {
@@ -271,3 +311,8 @@ def test_hide_sensitive_data(input, expectation, tag):
result = yaml.safe_load(yaml_result)
assert result == expectation
+
+
+def test_get_job_results(mock_proxy):
+ proxy = mock_proxy()
+ get_job_results(proxy, 1, "0_mesa")
\ No newline at end of file
More information about the mesa-commit
mailing list