Mesa (main): ci/lava: Reduce LAVA boot phase timeout to 3 minutes

[GitLab Mirror]

Module: Mesa
Branch: main
Commit: c64e3d92df7d9021d84312735c15bfc681cefcc5
URL:    http://cgit.freedesktop.org/mesa/mesa/commit/?id=c64e3d92df7d9021d84312735c15bfc681cefcc5

Author: Guilherme Gallo <guilherme.gallo at collabora.com>
Date:   Fri Mar 25 09:55:09 2022 -0300

ci/lava: Reduce LAVA boot phase timeout to 3 minutes

A normal boot on LAVA device would take less than 1 minute. Sometimes
the depthcharge freezes and LAVA waits until the current timeout (25
minutes) to kick in and fail the job.
With this lower timeout value, the job could fail faster and eventually
LAVA will be able to retry it.

Furthermore, set a default timeout for all actions to fix an undesired
behavior with some LAVA job subactions, such as depthcharge-action.

Signed-off-by: Guilherme Gallo <guilherme.gallo at collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/15938>

---

 .gitlab-ci/lava/lava_job_submitter.py | 21 ++++++++++++++-------
 1 file changed, 14 insertions(+), 7 deletions(-)

diff --git a/.gitlab-ci/lava/lava_job_submitter.py b/.gitlab-ci/lava/lava_job_submitter.py
index f5ccf8f7887..f840145c172 100755
--- a/.gitlab-ci/lava/lava_job_submitter.py
+++ b/.gitlab-ci/lava/lava_job_submitter.py
@@ -54,6 +54,9 @@ LOG_POLLING_TIME_SEC = int(getenv("LAVA_LOG_POLLING_TIME_SEC", 5))
 # How many retries should be made when a timeout happen.
 NUMBER_OF_RETRIES_TIMEOUT_DETECTION = int(getenv("LAVA_NUMBER_OF_RETRIES_TIMEOUT_DETECTION", 2))
 
+# How many attempts should be made when a timeout happen during LAVA device boot.
+NUMBER_OF_ATTEMPTS_LAVA_BOOT = int(getenv("LAVA_NUMBER_OF_ATTEMPTS_LAVA_BOOT", 3))
+
 
 def print_log(msg):
     print("{}: {}".format(datetime.now(), msg))
@@ -77,9 +80,13 @@ def generate_lava_yaml(args):
         'context': {
             'extra_nfsroot_args': ' init=/init rootwait usbcore.quirks=0bda:8153:k'
         },
-        'timeouts': {
-            'job': {
-                'minutes': args.job_timeout
+        "timeouts": {
+            "job": {"minutes": args.job_timeout},
+            "action": {"minutes": 3},
+            "actions": {
+                "depthcharge-action": {
+                    "minutes": 3 * NUMBER_OF_ATTEMPTS_LAVA_BOOT,
+                }
             }
         },
     }
@@ -110,10 +117,10 @@ def generate_lava_yaml(args):
 
     # always boot over NFS
     boot = {
-      'timeout': { 'minutes': 25 },
-      'method': args.boot_method,
-      'commands': 'nfs',
-      'prompts': ['lava-shell:'],
+        "failure_retry": NUMBER_OF_ATTEMPTS_LAVA_BOOT,
+        "method": args.boot_method,
+        "commands": "nfs",
+        "prompts": ["lava-shell:"],
     }
 
     # skeleton test definition: only declaring each job as a single 'test'