[igt-dev] [PATCH v3 11/12] ci: Add files from Mesa required to submit jobs to LAVA
Tomeu Vizoso
tomeu.vizoso at collabora.com
Fri Mar 11 14:12:59 UTC 2022
Mostly to submit jobs to LAVA and to setup the device after boot.
Signed-off-by: Tomeu Vizoso <tomeu.vizoso at collabora.com>
---
ci/ci-common/capture-devcoredump.sh | 14 +
ci/ci-common/generate-env.sh | 112 ++++++++
ci/ci-common/init-stage1.sh | 22 ++
ci/ci-common/init-stage2.sh | 78 ++++++
ci/lava/lava-submit.sh | 38 +++
ci/lava/lava_job_submitter.py | 380 ++++++++++++++++++++++++++++
6 files changed, 644 insertions(+)
create mode 100755 ci/ci-common/capture-devcoredump.sh
create mode 100755 ci/ci-common/generate-env.sh
create mode 100755 ci/ci-common/init-stage1.sh
create mode 100755 ci/ci-common/init-stage2.sh
create mode 100755 ci/lava/lava-submit.sh
create mode 100755 ci/lava/lava_job_submitter.py
diff --git a/ci/ci-common/capture-devcoredump.sh b/ci/ci-common/capture-devcoredump.sh
new file mode 100755
index 000000000000..ae370538eaeb
--- /dev/null
+++ b/ci/ci-common/capture-devcoredump.sh
@@ -0,0 +1,14 @@
+#!/bin/sh
+
+while true; do
+ devcds=`find /sys/devices/virtual/devcoredump/ -name data 2>/dev/null`
+ for i in $devcds; do
+ echo "Found a devcoredump at $i."
+ if cp $i /results/first.devcore; then
+ echo 1 > $i
+ echo "Saved to the job artifacts at /first.devcore"
+ exit 0
+ fi
+ done
+ sleep 10
+done
diff --git a/ci/ci-common/generate-env.sh b/ci/ci-common/generate-env.sh
new file mode 100755
index 000000000000..dc2b7febefb8
--- /dev/null
+++ b/ci/ci-common/generate-env.sh
@@ -0,0 +1,112 @@
+#!/bin/bash
+
+for var in \
+ ASAN_OPTIONS \
+ BASE_SYSTEM_FORK_HOST_PREFIX \
+ BASE_SYSTEM_MAINLINE_HOST_PREFIX \
+ CI_COMMIT_BRANCH \
+ CI_COMMIT_REF_NAME \
+ CI_COMMIT_TITLE \
+ CI_JOB_ID \
+ CI_JOB_JWT_FILE \
+ CI_JOB_NAME \
+ CI_JOB_URL \
+ CI_MERGE_REQUEST_SOURCE_BRANCH_NAME \
+ CI_MERGE_REQUEST_TITLE \
+ CI_NODE_INDEX \
+ CI_NODE_TOTAL \
+ CI_PAGES_DOMAIN \
+ CI_PIPELINE_ID \
+ CI_PIPELINE_URL \
+ CI_PROJECT_DIR \
+ CI_PROJECT_NAME \
+ CI_PROJECT_PATH \
+ CI_PROJECT_ROOT_NAMESPACE \
+ CI_RUNNER_DESCRIPTION \
+ CI_SERVER_URL \
+ CROSVM_GALLIUM_DRIVER \
+ CROSVM_GPU_ARGS \
+ CROSVM_TEST_SCRIPT \
+ DEQP_BIN_DIR \
+ DEQP_CASELIST_FILTER \
+ DEQP_CASELIST_INV_FILTER \
+ DEQP_CONFIG \
+ DEQP_EXPECTED_RENDERER \
+ DEQP_FRACTION \
+ DEQP_HEIGHT \
+ DEQP_RESULTS_DIR \
+ DEQP_RUNNER_OPTIONS \
+ DEQP_SUITE \
+ DEQP_TEMP_DIR \
+ DEQP_VARIANT \
+ DEQP_VER \
+ DEQP_WIDTH \
+ DEVICE_NAME \
+ DRIVER_NAME \
+ EGL_PLATFORM \
+ ETNA_MESA_DEBUG \
+ FDO_CI_CONCURRENT \
+ FDO_UPSTREAM_REPO \
+ FD_MESA_DEBUG \
+ FLAKES_CHANNEL \
+ GALLIUM_DRIVER \
+ GALLIVM_PERF \
+ GPU_VERSION \
+ GTEST \
+ GTEST_FAILS \
+ GTEST_FRACTION \
+ GTEST_RESULTS_DIR \
+ GTEST_RUNNER_OPTIONS \
+ GTEST_SKIPS \
+ HWCI_FREQ_MAX \
+ HWCI_KERNEL_MODULES \
+ HWCI_START_XORG \
+ HWCI_TEST_SCRIPT \
+ IGT_FORCE_DRIVER \
+ IR3_SHADER_DEBUG \
+ JOB_ARTIFACTS_BASE \
+ JOB_RESULTS_PATH \
+ JOB_ROOTFS_OVERLAY_PATH \
+ LD_LIBRARY_PATH \
+ LP_NUM_THREADS \
+ MESA_BASE_TAG \
+ MESA_BUILD_PATH \
+ MESA_DEBUG \
+ MESA_GLES_VERSION_OVERRIDE \
+ MESA_GLSL_VERSION_OVERRIDE \
+ MESA_GL_VERSION_OVERRIDE \
+ MESA_IMAGE \
+ MESA_IMAGE_PATH \
+ MESA_IMAGE_TAG \
+ MESA_TEMPLATES_COMMIT \
+ MESA_VK_IGNORE_CONFORMANCE_WARNING \
+ MINIO_HOST \
+ NIR_DEBUG \
+ PAN_I_WANT_A_BROKEN_VULKAN_DRIVER \
+ PAN_MESA_DEBUG \
+ PIGLIT_FRACTION \
+ PIGLIT_NO_WINDOW \
+ PIGLIT_OPTIONS \
+ PIGLIT_PLATFORM \
+ PIGLIT_PROFILES \
+ PIGLIT_REPLAY_ARTIFACTS_BASE_URL \
+ PIGLIT_REPLAY_DESCRIPTION_FILE \
+ PIGLIT_REPLAY_DEVICE_NAME \
+ PIGLIT_REPLAY_EXTRA_ARGS \
+ PIGLIT_REPLAY_REFERENCE_IMAGES_BASE \
+ PIGLIT_REPLAY_SUBCOMMAND \
+ PIGLIT_RESULTS \
+ PIGLIT_TESTS \
+ PIPELINE_ARTIFACTS_BASE \
+ SKQP_ASSETS_DIR \
+ SKQP_BACKENDS \
+ TU_DEBUG \
+ VIRGL_HOST_API \
+ VK_CPU \
+ VK_DRIVER \
+ VK_ICD_FILENAMES \
+ ; do
+ if [ -n "${!var+x}" ]; then
+ echo "export $var=${!var at Q}"
+ fi
+done
diff --git a/ci/ci-common/init-stage1.sh b/ci/ci-common/init-stage1.sh
new file mode 100755
index 000000000000..648c37a2f903
--- /dev/null
+++ b/ci/ci-common/init-stage1.sh
@@ -0,0 +1,22 @@
+#!/bin/sh
+
+# Very early init, used to make sure devices and network are set up and
+# reachable.
+
+set -ex
+
+cd /
+
+mount -t proc none /proc
+mount -t sysfs none /sys
+mount -t devtmpfs none /dev || echo possibly already mounted
+mkdir -p /dev/pts
+mount -t devpts devpts /dev/pts
+mount -t tmpfs tmpfs /tmp
+
+echo "nameserver 8.8.8.8" > /etc/resolv.conf
+[ -z "$NFS_SERVER_IP" ] || echo "$NFS_SERVER_IP caching-proxy" >> /etc/hosts
+
+# Set the time so we can validate certificates before we fetch anything;
+# however as not all DUTs have network, make this non-fatal.
+for i in 1 2 3; do sntp -sS pool.ntp.org && break || sleep 2; done || true
diff --git a/ci/ci-common/init-stage2.sh b/ci/ci-common/init-stage2.sh
new file mode 100755
index 000000000000..aff36635e59b
--- /dev/null
+++ b/ci/ci-common/init-stage2.sh
@@ -0,0 +1,78 @@
+#!/bin/sh
+
+# Second-stage init, used to set up devices and our job environment before
+# running tests.
+
+. /set-job-env-vars.sh
+
+set -ex
+
+# Set up any devices required by the jobs
+[ -z "$HWCI_KERNEL_MODULES" ] || (echo -n $HWCI_KERNEL_MODULES | xargs -d, -n1 /usr/sbin/modprobe)
+
+# Fix prefix confusion: the build installs to $CI_PROJECT_DIR, but we expect
+# it in /install
+ln -sf $CI_PROJECT_DIR/build /install
+export LD_LIBRARY_PATH=/install/lib
+export LIBGL_DRIVERS_PATH=/install/lib/dri
+
+# Store Mesa's disk cache under /tmp, rather than sending it out over NFS.
+export XDG_CACHE_HOME=/tmp
+
+# Make sure Python can find all our imports
+export PYTHONPATH=$(python3 -c "import sys;print(\":\".join(sys.path))")
+
+if [ "$HWCI_FREQ_MAX" = "true" ]; then
+ # Ensure initialization of the DRM device (needed by MSM)
+ head -0 /dev/dri/renderD128
+
+ # Disable GPU frequency scaling
+ DEVFREQ_GOVERNOR=`find /sys/devices -name governor | grep gpu || true`
+ test -z "$DEVFREQ_GOVERNOR" || echo performance > $DEVFREQ_GOVERNOR || true
+
+ # Disable CPU frequency scaling
+ echo performance | tee -a /sys/devices/system/cpu/cpufreq/policy*/scaling_governor || true
+
+ # Disable GPU runtime power management
+ GPU_AUTOSUSPEND=`find /sys/devices -name autosuspend_delay_ms | grep gpu | head -1`
+ test -z "$GPU_AUTOSUSPEND" || echo -1 > $GPU_AUTOSUSPEND || true
+fi
+
+# Start a little daemon to capture the first devcoredump we encounter. (They
+# expire after 5 minutes, so we poll for them).
+./capture-devcoredump.sh &
+
+# If we want Xorg to be running for the test, then we start it up before the
+# HWCI_TEST_SCRIPT because we need to use xinit to start X (otherwise
+# without using -displayfd you can race with Xorg's startup), but xinit will eat
+# your client's return code
+if [ -n "$HWCI_START_XORG" ]; then
+ echo "touch /xorg-started; sleep 100000" > /xorg-script
+ env \
+ xinit /bin/sh /xorg-script -- /usr/bin/Xorg -noreset -s 0 -dpms -logfile /Xorg.0.log &
+
+ # Wait for xorg to be ready for connections.
+ for i in 1 2 3 4 5; do
+ if [ -e /xorg-started ]; then
+ break
+ fi
+ sleep 5
+ done
+ export DISPLAY=:0
+fi
+
+RESULT=fail
+if sh $HWCI_TEST_SCRIPT; then
+ RESULT=pass
+ rm -rf results/trace/$PIGLIT_REPLAY_DEVICE_NAME
+fi
+
+# upload artifacts
+MINIO=$(cat /proc/cmdline | tr ' ' '\n' | grep minio_results | cut -d '=' -f 2 || true)
+if [ -n "$MINIO" ]; then
+ tar -czf results.tar.gz results/;
+ ci-fairy minio login --token-file "${CI_JOB_JWT_FILE}";
+ ci-fairy minio cp results.tar.gz minio://"$MINIO"/results.tar.gz;
+fi
+
+echo "hwci: test: $RESULT"
diff --git a/ci/lava/lava-submit.sh b/ci/lava/lava-submit.sh
new file mode 100755
index 000000000000..fac1c42bc296
--- /dev/null
+++ b/ci/lava/lava-submit.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+
+set -e
+set -x
+
+SCRIPTS_DIR=`dirname "$0"`/..
+
+rm -rf results
+mkdir -p results/job-rootfs-overlay/
+
+cp $SCRIPTS_DIR/ci-common/capture-devcoredump.sh results/job-rootfs-overlay/
+cp $SCRIPTS_DIR/ci-common/init-*.sh results/job-rootfs-overlay/
+$SCRIPTS_DIR/ci-common/generate-env.sh > results/job-rootfs-overlay/set-job-env-vars.sh
+
+tar zcf job-rootfs-overlay.tar.gz -C results/job-rootfs-overlay/ .
+ci-fairy minio login --token-file "${CI_JOB_JWT_FILE}"
+ci-fairy minio cp job-rootfs-overlay.tar.gz "minio://${JOB_ROOTFS_OVERLAY_PATH}"
+
+touch results/lava.log
+tail -f results/lava.log &
+$SCRIPTS_DIR/lava/lava_job_submitter.py \
+ --dump-yaml \
+ --pipeline-info "$CI_JOB_NAME: $CI_PIPELINE_URL on $CI_COMMIT_REF_NAME ${CI_NODE_INDEX}/${CI_NODE_TOTAL}" \
+ --base-system-url-prefix "https://${BASE_SYSTEM_MAINLINE_HOST_PATH}" \
+ --build-url "${FDO_HTTP_CACHE_URI:-}https://${BUILD_PATH}" \
+ --job-rootfs-overlay-url "${FDO_HTTP_CACHE_URI:-}https://${JOB_ROOTFS_OVERLAY_PATH}" \
+ --job-artifacts-base ${JOB_ARTIFACTS_BASE} \
+ --job-timeout ${JOB_TIMEOUT:-30} \
+ --first-stage-init $SCRIPTS_DIR/ci-common/init-stage1.sh \
+ --ci-project-dir ${CI_PROJECT_DIR} \
+ --device-type ${DEVICE_TYPE} \
+ --dtb ${DTB} \
+ --jwt-file "${CI_JOB_JWT_FILE}" \
+ --kernel-image-name ${KERNEL_IMAGE_NAME} \
+ --kernel-image-type "${KERNEL_IMAGE_TYPE}" \
+ --boot-method ${BOOT_METHOD} \
+ --visibility-group ${VISIBILITY_GROUP} \
+ --lava-tags "${LAVA_TAGS}" >> results/lava.log
diff --git a/ci/lava/lava_job_submitter.py b/ci/lava/lava_job_submitter.py
new file mode 100755
index 000000000000..5ea8eddf088d
--- /dev/null
+++ b/ci/lava/lava_job_submitter.py
@@ -0,0 +1,380 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2020, 2021 Collabora Limited
+# Author: Gustavo Padovan <gustavo.padovan at collabora.com>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice (including the next
+# paragraph) shall be included in all copies or substantial portions of the
+# Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+"""Send a job to LAVA, track it and collect log back"""
+
+import argparse
+import pathlib
+import sys
+import time
+import traceback
+import urllib.parse
+import xmlrpc
+
+from datetime import datetime, timedelta
+from os import getenv
+
+import lavacli
+import yaml
+from lavacli.utils import loader
+
+# Timeout in seconds to decide if the device from the dispatched LAVA job has
+# hung or not due to the lack of new log output.
+DEVICE_HANGING_TIMEOUT_SEC = int(getenv("LAVA_DEVICE_HANGING_TIMEOUT_SEC", 5*60))
+
+# How many seconds the script should wait before try a new polling iteration to
+# check if the dispatched LAVA job is running or waiting in the job queue.
+WAIT_FOR_DEVICE_POLLING_TIME_SEC = int(getenv("LAVA_WAIT_FOR_DEVICE_POLLING_TIME_SEC", 10))
+
+# How many seconds to wait between log output LAVA RPC calls.
+LOG_POLLING_TIME_SEC = int(getenv("LAVA_LOG_POLLING_TIME_SEC", 5))
+
+# How many retries should be made when a timeout happen.
+NUMBER_OF_RETRIES_TIMEOUT_DETECTION = int(getenv("LAVA_NUMBER_OF_RETRIES_TIMEOUT_DETECTION", 2))
+
+
+def print_log(msg):
+ print("{}: {}".format(datetime.now(), msg))
+
+def fatal_err(msg):
+ print_log(msg)
+ sys.exit(1)
+
+
+def hide_sensitive_data(yaml_data, hide_tag="HIDEME"):
+ return "".join(line for line in yaml_data.splitlines(True) if hide_tag not in line)
+
+
+def generate_lava_yaml(args):
+ # General metadata and permissions, plus also inexplicably kernel arguments
+ values = {
+ 'job_name': args.pipeline_info,
+ 'device_type': args.device_type,
+ 'visibility': { 'group': [ args.visibility_group ] },
+ 'priority': 75,
+ 'context': {
+ 'extra_nfsroot_args': ' init=/init rootwait minio_results={}'.format(args.job_artifacts_base)
+ },
+ 'timeouts': {
+ 'job': {
+ 'minutes': args.job_timeout
+ }
+ },
+ }
+
+ if args.lava_tags:
+ values['tags'] = args.lava_tags.split(',')
+
+ # URLs to our kernel rootfs to boot from, both generated by the base
+ # container build
+ deploy = {
+ 'timeout': { 'minutes': 10 },
+ 'to': 'tftp',
+ 'os': 'oe',
+ 'kernel': {
+ 'url': '{}/{}'.format(args.base_system_url_prefix, args.kernel_image_name),
+ },
+ 'nfsrootfs': {
+ 'url': '{}/lava-rootfs.tgz'.format(args.base_system_url_prefix),
+ 'compression': 'gz',
+ }
+ }
+ if args.kernel_image_type:
+ deploy['kernel']['type'] = args.kernel_image_type
+ if args.dtb:
+ deploy['dtb'] = {
+ 'url': '{}/{}.dtb'.format(args.base_system_url_prefix, args.dtb)
+ }
+
+ # always boot over NFS
+ boot = {
+ 'timeout': { 'minutes': 25 },
+ 'method': args.boot_method,
+ 'commands': 'nfs',
+ 'prompts': ['lava-shell:'],
+ }
+
+ # skeleton test definition: only declaring each job as a single 'test'
+ # since LAVA's test parsing is not useful to us
+ test = {
+ 'timeout': { 'minutes': args.job_timeout },
+ 'failure_retry': 1,
+ 'definitions': [ {
+ 'name': 'test',
+ 'from': 'inline',
+ 'path': 'inline/test.yaml',
+ 'repository': {
+ 'metadata': {
+ 'name': 'test',
+ 'description': 'Test plan',
+ 'os': [ 'oe' ],
+ 'scope': [ 'functional' ],
+ 'format': 'Lava-Test Test Definition 1.0',
+ },
+ 'parse': {
+ 'pattern': r'hwci: (?P<test_case_id>\S*):\s+(?P<result>(pass|fail))'
+ },
+ 'run': {
+ },
+ },
+ } ],
+ }
+
+ # job execution script:
+ # - inline .gitlab-ci/common/init-stage1.sh
+ # - fetch and unpack per-pipeline build artifacts from build job
+ # - fetch and unpack per-job environment from lava-submit.sh
+ # - exec .gitlab-ci/common/init-stage2.sh
+ init_lines = []
+
+ with open(args.first_stage_init, 'r') as init_sh:
+ init_lines += [ x.rstrip() for x in init_sh if not x.startswith('#') and x.rstrip() ]
+
+ with open(args.jwt_file) as jwt_file:
+ init_lines += [
+ "set +x",
+ f'echo -n "{jwt_file.read()}" > "{args.jwt_file}" # HIDEME',
+ "set -x",
+ ]
+
+ init_lines += [
+ 'mkdir -p {}'.format(args.ci_project_dir),
+ 'wget -S --progress=dot:giga -O- {} | tar -xz -C {}'.format(args.build_url, args.ci_project_dir),
+ 'wget -S --progress=dot:giga -O- {} | tar -xz -C /'.format(args.job_rootfs_overlay_url),
+ f'echo "export CI_JOB_JWT_FILE={args.jwt_file}" >> /set-job-env-vars.sh',
+ 'exec /init-stage2.sh',
+ ]
+ test['definitions'][0]['repository']['run']['steps'] = init_lines
+
+ values['actions'] = [
+ { 'deploy': deploy },
+ { 'boot': boot },
+ { 'test': test },
+ ]
+
+ return yaml.dump(values, width=10000000)
+
+
+def setup_lava_proxy():
+ config = lavacli.load_config("default")
+ uri, usr, tok = (config.get(key) for key in ("uri", "username", "token"))
+ uri_obj = urllib.parse.urlparse(uri)
+ uri_str = "{}://{}:{}@{}{}".format(uri_obj.scheme, usr, tok, uri_obj.netloc, uri_obj.path)
+ transport = lavacli.RequestsTransport(
+ uri_obj.scheme,
+ config.get("proxy"),
+ config.get("timeout", 120.0),
+ config.get("verify_ssl_cert", True),
+ )
+ proxy = xmlrpc.client.ServerProxy(
+ uri_str, allow_none=True, transport=transport)
+
+ print_log("Proxy for {} created.".format(config['uri']))
+
+ return proxy
+
+
+def _call_proxy(fn, *args):
+ retries = 60
+ for n in range(1, retries + 1):
+ try:
+ return fn(*args)
+ except xmlrpc.client.ProtocolError as err:
+ if n == retries:
+ traceback.print_exc()
+ fatal_err("A protocol error occurred (Err {} {})".format(err.errcode, err.errmsg))
+ else:
+ time.sleep(15)
+ except xmlrpc.client.Fault as err:
+ traceback.print_exc()
+ fatal_err("FATAL: Fault: {} (code: {})".format(err.faultString, err.faultCode))
+
+
+def get_job_results(proxy, job_id, test_suite, test_case):
+ # Look for infrastructure errors and retry if we see them.
+ results_yaml = _call_proxy(proxy.results.get_testjob_results_yaml, job_id)
+ results = yaml.load(results_yaml, Loader=loader(False))
+ for res in results:
+ metadata = res["metadata"]
+ if "result" not in metadata or metadata["result"] != "fail":
+ continue
+ if 'error_type' in metadata and metadata['error_type'] == "Infrastructure":
+ print_log("LAVA job {} failed with Infrastructure Error. Retry.".format(job_id))
+ return False
+ if 'case' in metadata and metadata['case'] == "validate":
+ print_log("LAVA job {} failed validation (possible download error). Retry.".format(job_id))
+ return False
+
+ results_yaml = _call_proxy(proxy.results.get_testcase_results_yaml, job_id, test_suite, test_case)
+ results = yaml.load(results_yaml, Loader=loader(False))
+ if not results:
+ fatal_err("LAVA: no result for test_suite '{}', test_case '{}'".format(test_suite, test_case))
+
+ print_log("LAVA: result for test_suite '{}', test_case '{}': {}".format(test_suite, test_case, results[0]['result']))
+ if results[0]['result'] != 'pass':
+ fatal_err("FAIL")
+
+ return True
+
+def wait_until_job_is_started(proxy, job_id):
+ print_log(f"Waiting for job {job_id} to start.")
+ current_state = "Submitted"
+ waiting_states = ["Submitted", "Scheduling", "Scheduled"]
+ while current_state in waiting_states:
+ job_state = _call_proxy(proxy.scheduler.job_state, job_id)
+ current_state = job_state["job_state"]
+
+ time.sleep(WAIT_FOR_DEVICE_POLLING_TIME_SEC)
+ print_log(f"Job {job_id} started.")
+
+def follow_job_execution(proxy, job_id):
+ line_count = 0
+ finished = False
+ last_time_logs = datetime.now()
+ while not finished:
+ (finished, data) = _call_proxy(proxy.scheduler.jobs.logs, job_id, line_count)
+ if logs := yaml.load(str(data), Loader=loader(False)):
+ # Reset the timeout
+ last_time_logs = datetime.now()
+ for line in logs:
+ print("{} {}".format(line["dt"], line["msg"]))
+
+ line_count += len(logs)
+
+ else:
+ time_limit = timedelta(seconds=DEVICE_HANGING_TIMEOUT_SEC)
+ if datetime.now() - last_time_logs > time_limit:
+ print_log("LAVA job {} doesn't advance (machine got hung?). Retry.".format(job_id))
+ return False
+
+ # `proxy.scheduler.jobs.logs` does not block, even when there is no
+ # new log to be fetched. To avoid dosing the LAVA dispatcher
+ # machine, let's add a sleep to save them some stamina.
+ time.sleep(LOG_POLLING_TIME_SEC)
+
+ return True
+
+def show_job_data(proxy, job_id):
+ show = _call_proxy(proxy.scheduler.jobs.show, job_id)
+ for field, value in show.items():
+ print("{}\t: {}".format(field, value))
+
+
+def validate_job(proxy, job_file):
+ try:
+ return _call_proxy(proxy.scheduler.jobs.validate, job_file, True)
+ except:
+ return False
+
+def submit_job(proxy, job_file):
+ return _call_proxy(proxy.scheduler.jobs.submit, job_file)
+
+
+def retriable_follow_job(proxy, yaml_file):
+ retry_count = NUMBER_OF_RETRIES_TIMEOUT_DETECTION
+
+ while retry_count >= 0:
+ job_id = submit_job(proxy, yaml_file)
+
+ print_log("LAVA job id: {}".format(job_id))
+
+ wait_until_job_is_started(proxy, job_id)
+
+ if not follow_job_execution(proxy, job_id):
+ print_log(f"Job {job_id} has timed out. Cancelling it.")
+ # Cancel the job as it is considered unreachable by GitLab CI.
+ proxy.scheduler.jobs.cancel(job_id)
+
+ retry_count -= 1
+ continue
+
+ show_job_data(proxy, job_id)
+
+ if get_job_results(proxy, job_id, "0_test", "test") == True:
+ break
+ else:
+ # The script attempted all the retries. The job seemed to fail.
+ return False
+
+ return True
+
+
+def main(args):
+ proxy = setup_lava_proxy()
+
+ yaml_file = generate_lava_yaml(args)
+
+ if args.dump_yaml:
+ print(hide_sensitive_data(generate_lava_yaml(args)))
+
+ if args.validate_only:
+ ret = validate_job(proxy, yaml_file)
+ if not ret:
+ fatal_err("Error in LAVA job definition")
+ print("LAVA job definition validated successfully")
+ return
+
+ if not retriable_follow_job(proxy, yaml_file):
+ fatal_err(
+ "Job failed after it exceeded the number of"
+ f"{NUMBER_OF_RETRIES_TIMEOUT_DETECTION} retries."
+ )
+
+
+def create_parser():
+ parser = argparse.ArgumentParser("LAVA job submitter")
+
+ parser.add_argument("--pipeline-info")
+ parser.add_argument("--base-system-url-prefix")
+ parser.add_argument("--build-url")
+ parser.add_argument("--job-rootfs-overlay-url")
+ parser.add_argument("--job-artifacts-base")
+ parser.add_argument("--job-timeout", type=int)
+ parser.add_argument("--first-stage-init")
+ parser.add_argument("--ci-project-dir")
+ parser.add_argument("--device-type")
+ parser.add_argument("--dtb", nargs='?', default="")
+ parser.add_argument("--kernel-image-name")
+ parser.add_argument("--kernel-image-type", nargs='?', default="")
+ parser.add_argument("--boot-method")
+ parser.add_argument("--lava-tags", nargs='?', default="")
+ parser.add_argument("--jwt-file", type=pathlib.Path)
+ parser.add_argument("--validate-only", action='store_true')
+ parser.add_argument("--dump-yaml", action='store_true')
+ parser.add_argument("--visibility-group")
+
+ return parser
+
+if __name__ == "__main__":
+ # given that we proxy from DUT -> LAVA dispatcher -> LAVA primary -> us ->
+ # GitLab runner -> GitLab primary -> user, safe to say we don't need any
+ # more buffering
+ sys.stdout.reconfigure(line_buffering=True)
+ sys.stderr.reconfigure(line_buffering=True)
+
+ parser = create_parser()
+
+ parser.set_defaults(func=main)
+ args = parser.parse_args()
+ args.func(args)
--
2.31.1
More information about the igt-dev
mailing list