Mesa (main): ci/lava: Clean up variable naming, document them

GitLab Mirror gitlab-mirror at kemper.freedesktop.org
Fri Jun 11 12:47:32 UTC 2021 

Module: Mesa
Branch: main
Commit: 0fd2320c945bc9278a9f87e5d63214c22e232ba2
URL:    http://cgit.freedesktop.org/mesa/mesa/commit/?id=0fd2320c945bc9278a9f87e5d63214c22e232ba2

Author: Daniel Stone <daniels at collabora.com>
Date:   Thu Jun 10 14:53:57 2021 +0100

ci/lava: Clean up variable naming, document them

Our variable names haven't aged very well. Rename them to make them more
clear and straightforward, especially when we bring in a third rootfs
element to download.

Signed-off-by: Daniel Stone <daniels at collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/11309>

---

 .gitlab-ci/lava/lava-gitlab-ci.yml    | 14 +++++++++++---
 .gitlab-ci/lava/lava-submit.sh        | 13 +++++++------
 .gitlab-ci/lava/lava.yml.jinja2       |  8 ++++----
 .gitlab-ci/lava/lava_job_submitter.py |  8 ++++----
 4 files changed, 26 insertions(+), 17 deletions(-)

diff --git a/.gitlab-ci/lava/lava-gitlab-ci.yml b/.gitlab-ci/lava/lava-gitlab-ci.yml
index 96f4cc4c639..3d01e11f6c7 100644
--- a/.gitlab-ci/lava/lava-gitlab-ci.yml
+++ b/.gitlab-ci/lava/lava-gitlab-ci.yml
@@ -8,8 +8,16 @@
     ENV_VARS: "DEQP_PARALLEL=6"
     FIXED_ENV_VARS: "CI_PIPELINE_ID=${CI_PIPELINE_ID} CI_JOB_ID=${CI_JOB_ID} CI_PAGES_DOMAIN=${CI_PAGES_DOMAIN} CI_PROJECT_NAME=${CI_PROJECT_NAME} CI_PROJECT_DIR=${CI_PROJECT_DIR} CI_PROJECT_PATH=${CI_PROJECT_PATH} CI_PROJECT_ROOT_NAMESPACE=${CI_PROJECT_ROOT_NAMESPACE} CI_SERVER_URL=${CI_SERVER_URL} DRIVER_NAME=${DRIVER_NAME} FDO_UPSTREAM_REPO=${FDO_UPSTREAM_REPO} PIGLIT_NO_WINDOW=1 PIGLIT_REPLAY_UPLOAD_TO_MINIO=1 MINIO_HOST=${MINIO_HOST} LAVA_TEST_SCRIPT=${LAVA_TEST_SCRIPT} VK_DRIVER=${VK_DRIVER} FLAKES_CHANNEL=${FLAKES_CHANNEL}"
     DEQP_VERSION: gles2
-    ARTIFACTS_PREFIX: "https://${MINIO_HOST}/mesa-lava"
-    MESA_URL: "http://caching-proxy/cache/?uri=https://${MINIO_HOST}/artifacts/${CI_PROJECT_PATH}/${CI_PIPELINE_ID}/mesa-${ARCH}.tar.gz"
+    # proxy used to cache data locally
+    FDO_HTTP_CACHE_URI: "http://caching-proxy/cache/?uri="
+    # base system generated by the container build job, shared between many pipelines
+    BASE_SYSTEM_HOST_PREFIX: "${MINIO_HOST}/mesa-lava"
+    BASE_SYSTEM_MAINLINE_HOST_PATH: "${BASE_SYSTEM_HOST_PREFIX}/${FDO_UPSTREAM_REPO}/${DISTRIBUTION_TAG}/${ARCH}"
+    BASE_SYSTEM_FORK_HOST_PATH: "${BASE_SYSTEM_HOST_PREFIX}/${CI_PROJECT_PATH}/${DISTRIBUTION_TAG}/${ARCH}"
+    # per-pipeline/job build artifacts
+    PIPELINE_ARTIFACTS_BASE: "${MINIO_HOST}/artifacts/${CI_PROJECT_PATH}/${CI_PIPELINE_ID}"
+    MESA_BUILD_PATH: "${PIPELINE_ARTIFACTS_BASE}/mesa-${ARCH}.tar.gz"
+    JOB_RESULTS_PATH: "${PIPELINE_ARTIFACTS_BASE}/${CI_JOB_ID}/results.tar.gz"
   script:
     - ./artifacts/lava/lava-submit.sh
   artifacts:
@@ -20,7 +28,7 @@
     exclude:
       - results/*.shader_cache
   after_script:
-    - wget -q "https://minio-packet.freedesktop.org/artifacts/${CI_PROJECT_PATH}/${CI_PIPELINE_ID}/${CI_JOB_ID}/results.tar.gz" -O- | tar -xz
+    - wget -q "https://${JOB_RESULTS_PATH}" -O- | tar -xz
 
 .lava-test:armhf:
   variables:
diff --git a/.gitlab-ci/lava/lava-submit.sh b/.gitlab-ci/lava/lava-submit.sh
index f3e6717985d..fd83c598aae 100755
--- a/.gitlab-ci/lava/lava-submit.sh
+++ b/.gitlab-ci/lava/lava-submit.sh
@@ -6,18 +6,19 @@ set -x
 rm -rf results
 mkdir -p results
 
-# Try to use the kernel and rootfs built in mainline first, to save cycles
-if wget -q --method=HEAD "${ARTIFACTS_PREFIX}/${FDO_UPSTREAM_REPO}/${DISTRIBUTION_TAG}/${ARCH}/done"; then
-	ARTIFACTS_URL="${ARTIFACTS_PREFIX}/${FDO_UPSTREAM_REPO}/${DISTRIBUTION_TAG}/${ARCH}"
+# Try to use the kernel and rootfs built in mainline first, so we're more
+# likely to hit cache
+if wget -q --method=HEAD "https://${BASE_SYSTEM_MAINLINE_HOST_PATH}/done"; then
+	BASE_SYSTEM_HOST_PATH="${BASE_SYSTEM_MAINLINE_HOST_PATH}"
 else
-	ARTIFACTS_URL="${ARTIFACTS_PREFIX}/${CI_PROJECT_PATH}/${DISTRIBUTION_TAG}/${ARCH}"
+	BASE_SYSTEM_HOST_PATH="${BASE_SYSTEM_FORK_HOST_PATH}"
 fi
 
 artifacts/lava/lava_job_submitter.py \
 	--template artifacts/lava/lava.yml.jinja2 \
 	--pipeline-info "$CI_PIPELINE_URL on $CI_COMMIT_REF_NAME ${CI_NODE_INDEX}/${CI_NODE_TOTAL}" \
-	--base-artifacts-url ${ARTIFACTS_URL} \
-	--mesa-url ${MESA_URL} \
+	--base-system-url-prefix "https://${BASE_SYSTEM_HOST_PATH}" \
+	--mesa-build-url "${FDO_HTTP_CACHE_URI:-}https://${MESA_BUILD_PATH}" \
 	--device-type ${DEVICE_TYPE} \
 	--dtb ${DTB} \
 	--env-vars "${ENV_VARS} ${FIXED_ENV_VARS}" \
diff --git a/.gitlab-ci/lava/lava.yml.jinja2 b/.gitlab-ci/lava/lava.yml.jinja2
index f33988ae433..f55539acc0b 100644
--- a/.gitlab-ci/lava/lava.yml.jinja2
+++ b/.gitlab-ci/lava/lava.yml.jinja2
@@ -22,16 +22,16 @@ actions:
       minutes: 10
     to: tftp
     kernel:
-      url: {{ base_artifacts_url }}/{{ kernel_image_name }}
+      url: {{ base_system_url_prefix }}/{{ kernel_image_name }}
 {% if kernel_image_type %}
       {{ kernel_image_type }}
 {% endif %}
     nfsrootfs:
-      url: {{ base_artifacts_url }}/lava-rootfs.tgz
+      url: {{ base_system_url_prefix }}/lava-rootfs.tgz
       compression: gz
 {% if dtb %}
     dtb:
-      url: {{ base_artifacts_url }}/{{ dtb }}.dtb
+      url: {{ base_system_url_prefix }}/{{ dtb }}.dtb
 {% endif %}
     os: oe
 - boot:
@@ -82,7 +82,7 @@ actions:
           - echo "$NFS_SERVER_IP caching-proxy" >> /etc/hosts
           - for i in 1 2 3; do sntp -sS pool.ntp.org && break || sleep 2; done
 
-          - wget -S --progress=dot:giga -O- {{ mesa_url }} | tar -xz
+          - wget -S --progress=dot:giga -O- {{ mesa_build_url }} | tar -xz
           - mkdir -p $CI_PROJECT_DIR
           - ln -sf /install $CI_PROJECT_DIR/install
 
diff --git a/.gitlab-ci/lava/lava_job_submitter.py b/.gitlab-ci/lava/lava_job_submitter.py
index 5fee2d728ad..9ed098ce96a 100755
--- a/.gitlab-ci/lava/lava_job_submitter.py
+++ b/.gitlab-ci/lava/lava_job_submitter.py
@@ -55,8 +55,8 @@ def generate_lava_yaml(args):
 
     values = {}
     values['pipeline_info'] = args.pipeline_info
-    values['base_artifacts_url'] = args.base_artifacts_url
-    values['mesa_url'] = args.mesa_url
+    values['base_system_url_prefix'] = args.base_system_url_prefix
+    values['mesa_build_url'] = args.mesa_build_url
     values['device_type'] = args.device_type
     values['dtb'] = args.dtb
     values['kernel_image_name'] = args.kernel_image_name
@@ -201,8 +201,8 @@ if __name__ == '__main__':
 
     parser.add_argument("--template")
     parser.add_argument("--pipeline-info")
-    parser.add_argument("--base-artifacts-url")
-    parser.add_argument("--mesa-url")
+    parser.add_argument("--base-system-url-prefix")
+    parser.add_argument("--mesa-build-url")
     parser.add_argument("--device-type")
     parser.add_argument("--dtb", nargs='?', default="")
     parser.add_argument("--kernel-image-name")

More information about the mesa-commit mailing list