Mesa (master): ci: make piglit runner less noisy and show a better failure message
GitLab Mirror
gitlab-mirror at kemper.freedesktop.org
Tue Jan 26 10:33:33 UTC 2021
Module: Mesa
Branch: master
Commit: 8da467f4fef7bf54f2929ac195813c046cad83a6
URL: http://cgit.freedesktop.org/mesa/mesa/commit/?id=8da467f4fef7bf54f2929ac195813c046cad83a6
Author: Andres Gomez <agomez at igalia.com>
Date: Thu Jan 21 22:24:51 2021 +0200
ci: make piglit runner less noisy and show a better failure message
v2:
- Do not silence piglit run (Michel).
Signed-off-by: Andres Gomez <agomez at igalia.com>
Reviewed-by: Eric Anholt <eric at anholt.net> [v1]
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/8632>
---
.gitlab-ci/lava-gitlab-ci.yml | 2 +-
.gitlab-ci/piglit/run.sh | 84 +++++++++++++++++++++++++++----------------
2 files changed, 55 insertions(+), 31 deletions(-)
diff --git a/.gitlab-ci/lava-gitlab-ci.yml b/.gitlab-ci/lava-gitlab-ci.yml
index befd9243ef9..82489491b73 100644
--- a/.gitlab-ci/lava-gitlab-ci.yml
+++ b/.gitlab-ci/lava-gitlab-ci.yml
@@ -46,7 +46,7 @@ kernel+rootfs_armhf:
variables:
GIT_STRATEGY: none # testing doesn't build anything from source
ENV_VARS: "DEQP_PARALLEL=6"
- FIXED_ENV_VARS: "CI_PIPELINE_ID=${CI_PIPELINE_ID} CI_JOB_ID=${CI_JOB_ID} CI_PROJECT_PATH=${CI_PROJECT_PATH} CI_JOB_JWT=${CI_JOB_JWT} CI_SERVER_URL=${CI_SERVER_URL} DRIVER_NAME=${DRIVER_NAME} FDO_UPSTREAM_REPO=${FDO_UPSTREAM_REPO} PIGLIT_NO_WINDOW=1 PIGLIT_REPLAY_UPLOAD_TO_MINIO=1 MINIO_HOST=${MINIO_HOST}"
+ FIXED_ENV_VARS: "CI_PIPELINE_ID=${CI_PIPELINE_ID} CI_JOB_ID=${CI_JOB_ID} CI_JOB_URL=${CI_JOB_URL} CI_PROJECT_PATH=${CI_PROJECT_PATH} CI_JOB_JWT=${CI_JOB_JWT} CI_SERVER_URL=${CI_SERVER_URL} DRIVER_NAME=${DRIVER_NAME} FDO_UPSTREAM_REPO=${FDO_UPSTREAM_REPO} PIGLIT_NO_WINDOW=1 PIGLIT_REPLAY_UPLOAD_TO_MINIO=1 MINIO_HOST=${MINIO_HOST}"
DEQP_VERSION: gles2
ARTIFACTS_PREFIX: "https://${MINIO_HOST}/mesa-lava/"
MESA_URL: "https://${MINIO_HOST}/artifacts/${CI_PROJECT_PATH}/${CI_PIPELINE_ID}/mesa-${ARCH}.tar.gz"
diff --git a/.gitlab-ci/piglit/run.sh b/.gitlab-ci/piglit/run.sh
index d59c3833c47..631f38d624e 100755
--- a/.gitlab-ci/piglit/run.sh
+++ b/.gitlab-ci/piglit/run.sh
@@ -128,6 +128,54 @@ if [ "$RUN_CMD_WRAPPER" ]; then
RUN_CMD="set +e; $RUN_CMD_WRAPPER "$(/usr/bin/printf "%q" "$RUN_CMD")"; set -e"
fi
+print_red() {
+ RED='\033[0;31m'
+ NC='\033[0m' # No Color
+ printf "${RED}"
+ "$@"
+ printf "${NC}"
+}
+
+# wrapper to supress +x to avoid spamming the log
+quiet() {
+ set +x
+ "$@"
+ set -x
+}
+
+replay_minio_upload_images() {
+ find "$RESULTS/$__PREFIX" -type f -name "*.png" -printf "%P\n" \
+ | while read -r line; do
+
+ __TRACE="${line%-*-*}"
+ if grep -q "^$__PREFIX/$__TRACE: pass$" ".gitlab-ci/piglit/$PIGLIT_RESULTS.txt.orig"; then
+ if [ "x$CI_PROJECT_PATH" != "x$FDO_UPSTREAM_REPO" ]; then
+ continue
+ fi
+ __MINIO_PATH="$PIGLIT_REPLAY_REFERENCE_IMAGES_BASE_URL"
+ __DESTINATION_FILE_PATH="${line##*-}"
+ if ci-fairy minio ls "minio://${MINIO_HOST}${__MINIO_PATH}/${__DESTINATION_FILE_PATH}" 2>/dev/null; then
+ continue
+ fi
+ else
+ __MINIO_PATH="$PIGLIT_REPLAY_ARTIFACTS_BASE_URL"
+ __DESTINATION_FILE_PATH="$__MINIO_TRACES_PREFIX/${line##*-}"
+ # Adding to the JUnit the direct link to the diff page in
+ # the dashboard
+ __PIGLIT_TESTCASE_CLASSNAME="piglit\.trace\.$PIGLIT_REPLAY_DEVICE_NAME\.$(dirname $__TRACE | sed 's%/%\\.%g;s@%@\\%@')"
+ __PIGLIT_TESTCASE_NAME="$(basename $__TRACE | sed 's%\.%_%g;s@%@\\%@')"
+ __DASHBOARD_URL="https://tracie.freedesktop.org/dashboard/imagediff/${CI_PROJECT_PATH}/${CI_JOB_ID}/${__TRACE}"
+ sed '\%<testcase classname="'"${__PIGLIT_TESTCASE_CLASSNAME}"'" name="'"${__PIGLIT_TESTCASE_NAME}"'" status="fail"%,\%</system-out><failure type="fail"/></testcase>%{s%</system-out><failure type="fail"/></testcase>%</system-out><failure type="fail">To view the image differences visit: '"${__DASHBOARD_URL}"'</failure></testcase>%}' \
+ -i "$RESULTS"/junit.xml
+ fi
+
+ ci-fairy minio cp "$RESULTS/$__PREFIX/$line" \
+ "minio://${MINIO_HOST}${__MINIO_PATH}/${__DESTINATION_FILE_PATH}"
+ done
+}
+
+FAILURE_MESSAGE=$(printf "%s" "Unexpected change in results:")
+
eval $RUN_CMD
if [ $? -ne 0 ]; then
@@ -136,6 +184,7 @@ fi
if [ ${PIGLIT_JUNIT_RESULTS:-0} -eq 1 ]; then
./piglit summary aggregate "$RESULTS" -o junit.xml
+ FAILURE_MESSAGE=$(printf "${FAILURE_MESSAGE}\n%s" "Check the JUnit report for failures at: ${CI_JOB_URL}/artifacts/file/results/junit.xml")
fi
PIGLIT_RESULTS="${PIGLIT_RESULTS:-$PIGLIT_PROFILES}"
@@ -157,34 +206,7 @@ if [ "x$PIGLIT_PROFILES" = "xreplay" ] \
ci-fairy minio cp "$RESULTS"/results.json.bz2 \
"minio://${MINIO_HOST}${__MINIO_PATH}/${__MINIO_TRACES_PREFIX}/results.json.bz2"
- find "$RESULTS/$__PREFIX" -type f -name "*.png" -printf "%P\n" \
- | while read -r line; do
-
- __TRACE="${line%-*-*}"
- if grep -q "^$__PREFIX/$__TRACE: pass$" ".gitlab-ci/piglit/$PIGLIT_RESULTS.txt.orig"; then
- if [ "x$CI_PROJECT_PATH" != "x$FDO_UPSTREAM_REPO" ]; then
- continue
- fi
- __MINIO_PATH="$PIGLIT_REPLAY_REFERENCE_IMAGES_BASE_URL"
- __DESTINATION_FILE_PATH="${line##*-}"
- if ci-fairy minio ls "minio://${MINIO_HOST}${__MINIO_PATH}/${__DESTINATION_FILE_PATH}" 2>/dev/null; then
- continue
- fi
- else
- __MINIO_PATH="$PIGLIT_REPLAY_ARTIFACTS_BASE_URL"
- __DESTINATION_FILE_PATH="$__MINIO_TRACES_PREFIX/${line##*-}"
- # Adding to the JUnit the direct link to the diff page in
- # the dashboard
- __PIGLIT_TESTCASE_CLASSNAME="piglit\.trace\.$PIGLIT_REPLAY_DEVICE_NAME\.$(dirname $__TRACE | sed 's%/%\\.%g;s@%@\\%@')"
- __PIGLIT_TESTCASE_NAME="$(basename $__TRACE | sed 's%\.%_%g;s@%@\\%@')"
- __DASHBOARD_URL="https://tracie.freedesktop.org/dashboard/imagediff/${CI_PROJECT_PATH}/${CI_JOB_ID}/${__TRACE}"
- sed '\%<testcase classname="'"${__PIGLIT_TESTCASE_CLASSNAME}"'" name="'"${__PIGLIT_TESTCASE_NAME}"'" status="fail"%,\%</system-out><failure type="fail"/></testcase>%{s%</system-out><failure type="fail"/></testcase>%</system-out><failure type="fail">To view the image differences visit: '"${__DASHBOARD_URL}"'</failure></testcase>%}' \
- -i "$RESULTS"/junit.xml
- fi
-
- ci-fairy minio cp "$RESULTS/$__PREFIX/$line" \
- "minio://${MINIO_HOST}${__MINIO_PATH}/${__DESTINATION_FILE_PATH}"
- done
+ quiet replay_minio_upload_images
ci-fairy minio cp "$RESULTS"/junit.xml \
"minio://${MINIO_HOST}${__MINIO_PATH}/${__MINIO_TRACES_PREFIX}/junit.xml"
@@ -206,8 +228,10 @@ if [ ${PIGLIT_HTML_SUMMARY:-1} -eq 1 ]; then
find "$OLDPWD"/summary -type f -name "*.html" -print0 \
| xargs -0 sed -i 's%<img src="file://%<img src="https://'"${MINIO_HOST}${PIGLIT_REPLAY_REFERENCE_IMAGES_BASE_URL}"'/%g'
fi
+
+ FAILURE_MESSAGE=$(printf "${FAILURE_MESSAGE}\n%s" "Check the HTML summary for problems at: ${CI_JOB_URL}/artifacts/file/summary/problems.html")
fi
-printf "%s\n" "Unexpected change in results:"
-diff -u ".gitlab-ci/piglit/$PIGLIT_RESULTS.txt.baseline" $RESULTSFILE
+quiet print_red printf "%s\n" "$FAILURE_MESSAGE"
+quiet print_red diff -u ".gitlab-ci/piglit/$PIGLIT_RESULTS.txt.baseline" $RESULTSFILE
exit 1
More information about the mesa-commit
mailing list