Mesa (main): gitlab-ci: add python script to submit lava jobs
GitLab Mirror
gitlab-mirror at kemper.freedesktop.org
Tue Jun 1 13:26:56 UTC 2021
Module: Mesa
Branch: main
Commit: 08dba3a76b95de80dfb4701711b00b4dc6618953
URL: http://cgit.freedesktop.org/mesa/mesa/commit/?id=08dba3a76b95de80dfb4701711b00b4dc6618953
Author: Gustavo Padovan <gustavo.padovan at collabora.com>
Date: Fri May 28 21:38:54 2021 -0300
gitlab-ci: add python script to submit lava jobs
Covert the job submission process to a python script for more
robustness and control. allowing easier manipulation of job data.
As a result, it adds retry logic to deal with Infrastructure Errors in LAVA.
_call_proxy() is equipped with a robust retry logic, which I have been
using already in the past few weeks in stress testing to run hundreds
of jobs.
Signed-off-by: Gustavo Padovan <gustavo.padovan at collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/11079>
---
.gitlab-ci/generate_lava.py | 49 ----------
.gitlab-ci/lava-gitlab-ci.yml | 11 +--
.gitlab-ci/lava_job_submitter.py | 193 +++++++++++++++++++++++++++++++++++++++
.gitlab-ci/prepare-artifacts.sh | 2 +-
4 files changed, 196 insertions(+), 59 deletions(-)
diff --git a/.gitlab-ci/generate_lava.py b/.gitlab-ci/generate_lava.py
deleted file mode 100755
index 66b898841f4..00000000000
--- a/.gitlab-ci/generate_lava.py
+++ /dev/null
@@ -1,49 +0,0 @@
-#!/usr/bin/env python3
-
-from jinja2 import Environment, FileSystemLoader
-import argparse
-import os
-import datetime
-
-parser = argparse.ArgumentParser()
-parser.add_argument("--template")
-parser.add_argument("--pipeline-info")
-parser.add_argument("--base-artifacts-url")
-parser.add_argument("--mesa-url")
-parser.add_argument("--device-type")
-parser.add_argument("--dtb", nargs='?', default="")
-parser.add_argument("--kernel-image-name")
-parser.add_argument("--kernel-image-type", nargs='?', default="")
-parser.add_argument("--gpu-version")
-parser.add_argument("--boot-method")
-parser.add_argument("--lava-tags", nargs='?', default="")
-parser.add_argument("--env-vars", nargs='?', default="")
-parser.add_argument("--deqp-version")
-parser.add_argument("--ci-node-index")
-parser.add_argument("--ci-node-total")
-parser.add_argument("--job-type")
-args = parser.parse_args()
-
-env = Environment(loader = FileSystemLoader(os.path.dirname(args.template)), trim_blocks=True, lstrip_blocks=True)
-template = env.get_template(os.path.basename(args.template))
-
-env_vars = "%s CI_NODE_INDEX=%s CI_NODE_TOTAL=%s" % (args.env_vars, args.ci_node_index, args.ci_node_total)
-
-values = {}
-values['pipeline_info'] = args.pipeline_info
-values['base_artifacts_url'] = args.base_artifacts_url
-values['mesa_url'] = args.mesa_url
-values['device_type'] = args.device_type
-values['dtb'] = args.dtb
-values['kernel_image_name'] = args.kernel_image_name
-values['kernel_image_type'] = args.kernel_image_type
-values['gpu_version'] = args.gpu_version
-values['boot_method'] = args.boot_method
-values['tags'] = args.lava_tags
-values['env_vars'] = env_vars
-values['deqp_version'] = args.deqp_version
-
-f = open(os.path.splitext(os.path.basename(args.template))[0], "w")
-f.write(template.render(values))
-f.close()
-
diff --git a/.gitlab-ci/lava-gitlab-ci.yml b/.gitlab-ci/lava-gitlab-ci.yml
index e919a50bdbd..c36581b642f 100644
--- a/.gitlab-ci/lava-gitlab-ci.yml
+++ b/.gitlab-ci/lava-gitlab-ci.yml
@@ -21,7 +21,7 @@
- rm -rf results
- mkdir -p results
- >
- artifacts/generate_lava.py \
+ artifacts/lava_job_submitter.py \
--template artifacts/lava.yml.jinja2 \
--pipeline-info "$CI_PIPELINE_URL on $CI_COMMIT_REF_NAME ${CI_NODE_INDEX}/${CI_NODE_TOTAL}" \
--base-artifacts-url ${ARTIFACTS_URL} \
@@ -36,15 +36,8 @@
--boot-method ${BOOT_METHOD} \
--lava-tags "${LAVA_TAGS}" \
--ci-node-index "${CI_NODE_INDEX}" \
- --ci-node-total "${CI_NODE_TOTAL}"
- - lava_job_id=`lavacli jobs submit lava.yml` || lavacli jobs submit lava.yml
- - echo $lava_job_id
+ --ci-node-total "${CI_NODE_TOTAL}" | tee results/lava.log
- cp lava.yml results/.
- - lavacli jobs logs $lava_job_id | tee results/lava-$lava_job_id.log
- - lavacli jobs show $lava_job_id
- - result=`lavacli results $lava_job_id 0_mesa mesa | head -1`
- - echo $result
- - '[[ "$result" == "pass" ]]'
artifacts:
name: "mesa_${CI_JOB_NAME}"
when: always
diff --git a/.gitlab-ci/lava_job_submitter.py b/.gitlab-ci/lava_job_submitter.py
new file mode 100755
index 00000000000..2bd0b7322c1
--- /dev/null
+++ b/.gitlab-ci/lava_job_submitter.py
@@ -0,0 +1,193 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2020, 2021 Collabora Limited
+# Author: Gustavo Padovan <gustavo.padovan at collabora.com>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice (including the next
+# paragraph) shall be included in all copies or substantial portions of the
+# Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+"""Send a job to LAVA, track it and collect log back"""
+
+import argparse
+import jinja2
+import lavacli
+import os
+import sys
+import time
+import urllib.parse
+import xmlrpc
+import yaml
+
+from datetime import datetime
+from lavacli.utils import loader
+
+
+def log_msg(msg):
+ return "{}: {}".format(datetime.now(), msg)
+
+
+def print_log(msg):
+ print(log_msg(msg))
+
+
+def generate_lava_yaml(args):
+ env = jinja2.Environment(loader = jinja2.FileSystemLoader(os.path.dirname(args.template)), trim_blocks=True, lstrip_blocks=True)
+ template = env.get_template(os.path.basename(args.template))
+
+ env_vars = "%s CI_NODE_INDEX=%s CI_NODE_TOTAL=%s" % (args.env_vars, args.ci_node_index, args.ci_node_total)
+
+ values = {}
+ values['pipeline_info'] = args.pipeline_info
+ values['base_artifacts_url'] = args.base_artifacts_url
+ values['mesa_url'] = args.mesa_url
+ values['device_type'] = args.device_type
+ values['dtb'] = args.dtb
+ values['kernel_image_name'] = args.kernel_image_name
+ values['kernel_image_type'] = args.kernel_image_type
+ values['gpu_version'] = args.gpu_version
+ values['boot_method'] = args.boot_method
+ values['tags'] = args.lava_tags
+ values['env_vars'] = env_vars
+ values['deqp_version'] = args.deqp_version
+
+ f = open(os.path.splitext(os.path.basename(args.template))[0], "w")
+ yaml = template.render(values)
+ f.write(yaml)
+ f.close()
+
+ return yaml
+
+
+def setup_lava_proxy():
+ config = lavacli.load_config("default")
+ uri, usr, tok = (config.get(key) for key in ("uri", "username", "token"))
+ uri_obj = urllib.parse.urlparse(uri)
+ uri_str = "{}://{}:{}@{}{}".format(uri_obj.scheme, usr, tok, uri_obj.netloc, uri_obj.path)
+ transport = lavacli.RequestsTransport(
+ uri_obj.scheme,
+ config.get("proxy"),
+ config.get("timeout", 120.0),
+ config.get("verify_ssl_cert", True),
+ )
+ proxy = xmlrpc.client.ServerProxy(
+ uri_str, allow_none=True, transport=transport)
+
+ print_log("Proxy for {} created.".format(config['uri']))
+
+ return proxy
+
+
+def _call_proxy(fn, *args):
+ retries = 60
+ for n in range(1, retries + 1):
+ try:
+ return fn(*args)
+ except xmlrpc.client.ProtocolError as err:
+ if n == retries:
+ sys.exit(log_msg("A protocol error occurred (Err {} {})".format(err.errcode, err.errmsg)))
+ else:
+ time.sleep(15)
+ pass
+ except xmlrpc.client.Fault as err:
+ sys.exit(log_msg("FATAL: Fault: {} (code: {})".format(err.faultString, err.faultCode)))
+
+
+def get_job_results(proxy, job_id, test_suite, test_case):
+ results_yaml = _call_proxy(proxy.results.get_testjob_results_yaml, job_id)
+ results = yaml.load(results_yaml, Loader=loader(False))
+
+ metadata = results[0]['metadata']
+ if metadata['result'] == 'fail' and 'error_type' in metadata and metadata['error_type'] == "Infrastructure":
+ print_log("LAVA job {} failed with Infrastructure Error. Retry.".format(job_id))
+ return False
+
+ results_yaml = _call_proxy(proxy.results.get_testcase_results_yaml, job_id, test_suite, test_case)
+ results = yaml.load(results_yaml, Loader=loader(False))
+ if results:
+ print_log("LAVA: result for test_suite '{}', test_case '{}': {}".format(test_suite, test_case, results[0]['result']))
+ else:
+ print_log("LAVA: no result for test_suite '{}', test_case '{}'".format(test_suite, test_case))
+
+ return True
+
+
+def follow_job_execution(proxy, job_id):
+ line_count = 0
+ finished = False
+ while not finished:
+ (finished, data) = _call_proxy(proxy.scheduler.jobs.logs, job_id, line_count)
+ logs = yaml.load(str(data), Loader=loader(False))
+ if logs:
+ for line in logs:
+ print("{} {}".format(line["dt"], line["msg"]))
+
+ line_count += len(logs)
+
+
+def show_job_data(proxy, job_id):
+ show = _call_proxy(proxy.scheduler.jobs.show, job_id)
+ for field, value in show.items():
+ print("{}\t: {}".format(field, value))
+
+
+def submit_job(proxy, job_file):
+ return _call_proxy(proxy.scheduler.jobs.submit, job_file)
+
+
+def main(args):
+ proxy = setup_lava_proxy()
+
+ yaml_file = generate_lava_yaml(args)
+
+ while True:
+ job_id = submit_job(proxy, yaml_file)
+
+ print_log("LAVA job id: {}".format(job_id))
+
+ follow_job_execution(proxy, job_id)
+
+ show_job_data(proxy, job_id)
+
+ if get_job_results(proxy, job_id, "0_mesa", "mesa") == True:
+ break
+
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser("LAVA job submitter")
+
+ parser.add_argument("--template")
+ parser.add_argument("--pipeline-info")
+ parser.add_argument("--base-artifacts-url")
+ parser.add_argument("--mesa-url")
+ parser.add_argument("--device-type")
+ parser.add_argument("--dtb", nargs='?', default="")
+ parser.add_argument("--kernel-image-name")
+ parser.add_argument("--kernel-image-type", nargs='?', default="")
+ parser.add_argument("--gpu-version")
+ parser.add_argument("--boot-method")
+ parser.add_argument("--lava-tags", nargs='?', default="")
+ parser.add_argument("--env-vars", nargs='?', default="")
+ parser.add_argument("--deqp-version")
+ parser.add_argument("--ci-node-index")
+ parser.add_argument("--ci-node-total")
+ parser.add_argument("--job-type")
+
+ parser.set_defaults(func=main)
+ args = parser.parse_args()
+ args.func(args)
diff --git a/.gitlab-ci/prepare-artifacts.sh b/.gitlab-ci/prepare-artifacts.sh
index a16f6eae6e9..9acbd023c58 100755
--- a/.gitlab-ci/prepare-artifacts.sh
+++ b/.gitlab-ci/prepare-artifacts.sh
@@ -45,7 +45,7 @@ tar -cf artifacts/install.tar install
if [ -n "$MINIO_ARTIFACT_NAME" ]; then
# Pass needed files to the test stage
- cp $CI_PROJECT_DIR/.gitlab-ci/generate_lava.py artifacts/.
+ cp $CI_PROJECT_DIR/.gitlab-ci/lava_job_submitter.py artifacts/.
cp $CI_PROJECT_DIR/.gitlab-ci/lava.yml.jinja2 artifacts/.
MINIO_ARTIFACT_NAME="$MINIO_ARTIFACT_NAME.tar.gz"
More information about the mesa-commit
mailing list