Mesa (master): gitlab-ci: Upload tracie artifacts to MinIO
GitLab Mirror
gitlab-mirror at kemper.freedesktop.org
Tue Jul 21 09:36:30 UTC 2020
Module: Mesa
Branch: master
Commit: a1947f059fcf118c879752f72e11f6217eb5f5a0
URL: http://cgit.freedesktop.org/mesa/mesa/commit/?id=a1947f059fcf118c879752f72e11f6217eb5f5a0
Author: Tomeu Vizoso <tomeu.vizoso at collabora.com>
Date: Thu Jul 9 22:29:39 2020 +0200
gitlab-ci: Upload tracie artifacts to MinIO
Upload failed images and the results.yml file to MinIO, to facilitate
debugging.
Also, fix version checking when git is installed as Mesa is going to
output a different renderer string if git is installed.
Signed-off-by: Tomeu Vizoso <tomeu.vizoso at collabora.com>
Reviewed-by: Daniel Stone <daniels at collabora.com>
Reviewed-By: Rohan Garg <rohan.garg at collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/5472>
---
.gitlab-ci.yml | 18 +++++++--------
.gitlab-ci/container/x86_build-base.sh | 5 +++++
.gitlab-ci/container/x86_build.sh | 1 -
.gitlab-ci/lava-gitlab-ci.yml | 3 ++-
.gitlab-ci/piglit/run.sh | 8 ++++++-
.gitlab-ci/prepare-artifacts.sh | 8 ++++---
.gitlab-ci/tracie-runner-gl.sh | 5 ++++-
.gitlab-ci/tracie-runner-vk.sh | 3 +++
.gitlab-ci/tracie/README.md | 22 +++++++++---------
.gitlab-ci/tracie/tracie.py | 41 ++++++++++++++++++++++++++++++++++
10 files changed, 87 insertions(+), 27 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index d407b388b24..6ffe2bee5ed 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -203,7 +203,7 @@ x86_build-base:
- .fdo.container-build at debian
- .container
variables:
- FDO_DISTRIBUTION_TAG: &x86_build-base "2020-06-12"
+ FDO_DISTRIBUTION_TAG: &x86_build-base "2020-07-21-tracie"
.use-x86_build-base:
extends:
@@ -221,7 +221,7 @@ x86_build:
extends:
- .use-x86_build-base
variables:
- FDO_DISTRIBUTION_TAG: &x86_build "2020-07-20-jflags"
+ FDO_DISTRIBUTION_TAG: &x86_build "2020-07-21-tracie"
.use-x86_build:
variables:
@@ -235,7 +235,7 @@ i386_build:
extends:
- .use-x86_build-base
variables:
- FDO_DISTRIBUTION_TAG: &i386_build "2020-07-20-jflags"
+ FDO_DISTRIBUTION_TAG: &i386_build "2020-07-21-tracie"
.use-i386_build:
variables:
@@ -249,7 +249,7 @@ ppc64el_build:
extends:
- .use-x86_build-base
variables:
- FDO_DISTRIBUTION_TAG: &ppc64el_build "2020-07-20-jflags"
+ FDO_DISTRIBUTION_TAG: &ppc64el_build "2020-07-21-tracie"
.use-ppc64el_build:
variables:
@@ -263,7 +263,7 @@ s390x_build:
extends:
- .use-x86_build-base
variables:
- FDO_DISTRIBUTION_TAG: &s390x_build "2020-06-29-jflags"
+ FDO_DISTRIBUTION_TAG: &s390x_build "2020-07-21-tracie"
.use-s390x_build:
variables:
@@ -276,7 +276,7 @@ s390x_build:
x86_test-base:
extends: x86_build-base
variables:
- FDO_DISTRIBUTION_TAG: &x86_test-base "2020-06-02"
+ FDO_DISTRIBUTION_TAG: &x86_test-base "2020-07-21-tracie"
.use-x86_test-base:
extends:
@@ -293,19 +293,19 @@ x86_test-base:
x86_test-gl:
extends: .use-x86_test-base
variables:
- FDO_DISTRIBUTION_TAG: &x86_test-gl "2020-07-15-virgl"
+ FDO_DISTRIBUTION_TAG: &x86_test-gl "2020-07-21-tracie"
# Debian 10 based x86 test image for VK
x86_test-vk:
extends: .use-x86_test-base
variables:
- FDO_DISTRIBUTION_TAG: &x86_test-vk "2020-06-18-vkcts"
+ FDO_DISTRIBUTION_TAG: &x86_test-vk "2020-07-21-tracie"
# Debian 9 based x86 build image (old LLVM)
x86_build_old:
extends: x86_build-base
variables:
- FDO_DISTRIBUTION_TAG: &x86_build_old "2020-06-29-jflags"
+ FDO_DISTRIBUTION_TAG: &x86_build_old "2020-07-21-tracie"
FDO_DISTRIBUTION_VERSION: stretch-slim
.use-x86_build_old:
diff --git a/.gitlab-ci/container/x86_build-base.sh b/.gitlab-ci/container/x86_build-base.sh
index c904f34ebab..748fea8652b 100644
--- a/.gitlab-ci/container/x86_build-base.sh
+++ b/.gitlab-ci/container/x86_build-base.sh
@@ -8,6 +8,8 @@ export DEBIAN_FRONTEND=noninteractive
apt-get install -y \
ca-certificates \
gnupg \
+ python3-pip \
+ python3-setuptools \
unzip \
wget
@@ -30,6 +32,7 @@ apt-get install -y --no-remove \
g++ \
g++-mingw-w64-x86-64 \
gcc \
+ git \
libclang-9-dev \
libclc-dev \
libelf-dev \
@@ -73,6 +76,8 @@ apt-get install -y --no-remove -t buster-backports \
libllvm8 \
meson
+# Needed for ci-fairy, this revision is able to upload files to MinIO
+pip3 install git+http://gitlab.freedesktop.org/freedesktop/ci-templates@6f5af7e5574509726c79109e3c147cee95e81366
# for the vulkan overlay layer
wget https://github.com/KhronosGroup/glslang/releases/download/master-tot/glslang-master-linux-Release.zip
diff --git a/.gitlab-ci/container/x86_build.sh b/.gitlab-ci/container/x86_build.sh
index 9f616538e36..b2220a61faa 100644
--- a/.gitlab-ci/container/x86_build.sh
+++ b/.gitlab-ci/container/x86_build.sh
@@ -12,7 +12,6 @@ STABLE_EPHEMERAL=" \
autotools-dev \
bzip2 \
cmake \
- git \
gnupg \
libgbm-dev \
libtool \
diff --git a/.gitlab-ci/lava-gitlab-ci.yml b/.gitlab-ci/lava-gitlab-ci.yml
index dbbf1ea7c09..9bd6f1fe392 100644
--- a/.gitlab-ci/lava-gitlab-ci.yml
+++ b/.gitlab-ci/lava-gitlab-ci.yml
@@ -33,6 +33,7 @@ kernel+rootfs_armhf:
variables:
GIT_STRATEGY: none # testing doesn't build anything from source
ENV_VARS: "DEQP_PARALLEL=6"
+ FIXED_ENV_VARS: "CI_PIPELINE_ID=${CI_PIPELINE_ID} CI_PROJECT_PATH=${CI_PROJECT_PATH} TRACIE_NO_UNIT_TESTS=1 TRACIE_UPLOAD_TO_MINIO=1"
DEQP_VERSION: gles2
ARTIFACTS_PREFIX: "https://minio-packet.freedesktop.org/mesa-lava/"
MESA_URL: "https://minio-packet.freedesktop.org/artifacts/${CI_PROJECT_PATH}/${CI_PIPELINE_ID}/mesa-${ARCH}.tar.gz"
@@ -51,7 +52,7 @@ kernel+rootfs_armhf:
--base-artifacts-url ${ARTIFACTS_URL} \
--mesa-url ${MESA_URL} \
--device-type ${DEVICE_TYPE} \
- --env-vars "${ENV_VARS}" \
+ --env-vars "${ENV_VARS} ${FIXED_ENV_VARS}" \
--deqp-version ${DEQP_VERSION} \
--kernel-image-name ${KERNEL_IMAGE_NAME} \
--kernel-image-type "${KERNEL_IMAGE_TYPE}" \
diff --git a/.gitlab-ci/piglit/run.sh b/.gitlab-ci/piglit/run.sh
index c1f50b9751f..32d2fcd51db 100755
--- a/.gitlab-ci/piglit/run.sh
+++ b/.gitlab-ci/piglit/run.sh
@@ -8,10 +8,16 @@ VERSION=`cat install/VERSION`
cd /piglit
PIGLIT_OPTIONS=$(echo $PIGLIT_OPTIONS | head -n 1)
+set +e
xvfb-run --server-args="-noreset" sh -c \
"export LD_LIBRARY_PATH=$OLDPWD/install/lib;
- wflinfo --platform glx --api gl --profile core | grep \"Mesa $VERSION\\\$\" &&
+ wflinfo --platform glx --api gl --profile core | tee /tmp/version.txt | grep \"Mesa $VERSION\\\$\" &&
./piglit run -j${FDO_CI_CONCURRENT:-4} $PIGLIT_OPTIONS $PIGLIT_PROFILES $OLDPWD/results"
+retVal=$?
+if [ $retVal -ne 0 ]; then
+ echo "Found $(cat /tmp/version.txt), expected $VERSION"
+fi
+set -e
PIGLIT_RESULTS=${PIGLIT_RESULTS:-$PIGLIT_PROFILES}
mkdir -p .gitlab-ci/piglit
diff --git a/.gitlab-ci/prepare-artifacts.sh b/.gitlab-ci/prepare-artifacts.sh
index fb7cc6306d9..40e7bbb5046 100755
--- a/.gitlab-ci/prepare-artifacts.sh
+++ b/.gitlab-ci/prepare-artifacts.sh
@@ -22,7 +22,7 @@ find install -name \*.so -exec $STRIP {} \;
# Test runs don't pull down the git tree, so put the dEQP helper
# script and associated bits there.
-cp VERSION install/
+echo "$(cat VERSION) (git-$(git rev-parse HEAD | cut -b -10))" >> install/VERSION
cp -Rp .gitlab-ci/bare-metal install/
cp -Rp .gitlab-ci/deqp* install/
cp -Rp .gitlab-ci/piglit install/
@@ -41,12 +41,15 @@ cp -Rp .gitlab-ci/deqp-runner.sh install/
cp -Rp .gitlab-ci/deqp-*-fails.txt install/
cp -Rp .gitlab-ci/deqp-*-skips.txt install/
+ci-fairy minio login $CI_JOB_JWT
+# These credentials will be used for uploading artifacts from test jobs
+cp .minio_credentials install/
+
# Tar up the install dir so that symlinks and hardlinks aren't each
# packed separately in the zip file.
mkdir -p artifacts/
tar -cf artifacts/install.tar install
-# If the container has LAVA stuff, prepare the artifacts for LAVA jobs
if [ -n "$UPLOAD_FOR_LAVA" ]; then
# Pass needed files to the test stage
cp $CI_PROJECT_DIR/.gitlab-ci/generate_lava.py artifacts/.
@@ -54,7 +57,6 @@ if [ -n "$UPLOAD_FOR_LAVA" ]; then
cp $CI_PROJECT_DIR/.gitlab-ci/lava-tracie.yml.jinja2 artifacts/.
gzip -c artifacts/install.tar > mesa-${DEBIAN_ARCH}.tar.gz
- ci-fairy minio login $CI_JOB_JWT
MINIO_PATH=minio-packet.freedesktop.org/artifacts/${CI_PROJECT_PATH}/${CI_PIPELINE_ID}
ci-fairy minio cp mesa-${DEBIAN_ARCH}.tar.gz minio://${MINIO_PATH}/mesa-${DEBIAN_ARCH}.tar.gz
fi
diff --git a/.gitlab-ci/tracie-runner-gl.sh b/.gitlab-ci/tracie-runner-gl.sh
index 6eb5baadb26..5c864b9ec1b 100755
--- a/.gitlab-ci/tracie-runner-gl.sh
+++ b/.gitlab-ci/tracie-runner-gl.sh
@@ -31,9 +31,12 @@ export PAGER=cat
RESULTS=`pwd`/results
mkdir -p $RESULTS
+# For artifact uploads to MinIO
+cp install/.minio_credentials .
+
# Perform a self-test to ensure tracie is working properly.
if [ -z "$TRACIE_NO_UNIT_TESTS" ]; then
- python3 -m pytest -v --pyargs $INSTALL/tracie/tests/test.py
+ TRACIE_UPLOAD_TO_MINIO=0 python3 -m pytest -v --pyargs $INSTALL/tracie/tests/test.py
fi
if [ "$GALLIUM_DRIVER" = "virpipe" ]; then
diff --git a/.gitlab-ci/tracie-runner-vk.sh b/.gitlab-ci/tracie-runner-vk.sh
index f046b7c9b40..da34bd4d8d4 100755
--- a/.gitlab-ci/tracie-runner-vk.sh
+++ b/.gitlab-ci/tracie-runner-vk.sh
@@ -20,6 +20,9 @@ export WINEESYNC=1
export DXVK_LOG_LEVEL="none"
export DXVK_STATE_CACHE=0
+# For artifact uploads to MinIO
+cp install/.minio_credentials .
+
# Perform a self-test to ensure tracie is working properly.
python3 -m pytest -v --pyargs $INSTALL/tracie/tests/test.py
diff --git a/.gitlab-ci/tracie/README.md b/.gitlab-ci/tracie/README.md
index 014bafc92e5..b0526564b46 100644
--- a/.gitlab-ci/tracie/README.md
+++ b/.gitlab-ci/tracie/README.md
@@ -5,15 +5,13 @@ Home of the Mesa trace testing effort.
### Traces definition file
-The trace definition file contains information about the GitLab
-project and git commit to get the traces from, and a list of the
-traces to run along with their expected image checksums on each
-device. An example:
+The trace definition file contains information about the traces to run along
+with their expected image checksums on each device, and optionally from where to
+download them. An example:
```yaml
traces-db:
- gitlab-project-url: https://gitlab.freedesktop.org/gfx-ci/tracie/traces-db
- commit: master
+ download-url: https://minio-packet.freedesktop.org/mesa-tracie-public/
traces:
- path: glmark2/jellyfish.rdc
@@ -40,12 +38,10 @@ with the [image_checksum.py](.gitlab-ci/tracie/image_checksum.py) script.
Alternatively, an arbitrary checksum can be used, and during replay (see below)
the scripts will report the mismatch and expected checksum.
-### Trace-db GitLab projects
+### Trace-db download urls
-The trace-db GitLab projects are assumed to have git repositories
-using LFS for their trace files. This is so that trace files can be
-potentially checked out and replayed individually, thus reducing
-storage requirements during CI runs.
+The trace-db:download-url property contains an HTTPS url from which traces can
+be downloaded, by appending traces:path properties to it.
### Enabling trace testing on a new device
@@ -94,6 +90,10 @@ applications. Traces for proprietary games and application are typically not
redistributable, unless specific redistribution rights have been granted by the
publisher.
+Trace files in a given repository are expected to be immutable once committed
+for the first time, so any changes need to be accompanied by a change in the
+file name (eg. by appending a _v2 suffix to the file).
+
### Replaying traces
Mesa traces CI uses a set of scripts to replay traces and check the output
diff --git a/.gitlab-ci/tracie/tracie.py b/.gitlab-ci/tracie/tracie.py
index b09bf4a9014..02797c0c887 100644
--- a/.gitlab-ci/tracie/tracie.py
+++ b/.gitlab-ci/tracie/tracie.py
@@ -1,7 +1,11 @@
import argparse
+import base64
+import datetime
import enum
import glob
import hashlib
+import hmac
+import json
import os
import requests
import sys
@@ -10,6 +14,7 @@ import time
import yaml
import shutil
+from email.utils import formatdate
from pathlib import Path
from PIL import Image
from urllib import parse
@@ -55,6 +60,38 @@ def gitlab_ensure_trace(project_url, trace):
open(trace_path, "wb").write(r.content)
print("took %ds." % (time.time() - download_time), flush=True)
+def sign_with_hmac(key, message):
+ key = key.encode("UTF-8")
+ message = message.encode("UTF-8")
+
+ signature = hmac.new(key, message, hashlib.sha1).digest()
+
+ return base64.encodebytes(signature).strip().decode()
+
+def upload_artifact(file_name, content_type):
+ with open('.minio_credentials', 'r') as f:
+ credentials = json.load(f)["minio-packet.freedesktop.org"]
+ minio_key = credentials["AccessKeyId"]
+ minio_secret = credentials["SecretAccessKey"]
+ minio_token = credentials["SessionToken"]
+
+ resource = '/artifacts/%s/%s/%s' % (os.environ['CI_PROJECT_PATH'], os.environ['CI_PIPELINE_ID'], os.path.basename(file_name))
+ date = formatdate(timeval=None, localtime=False, usegmt=True)
+ url = 'https://minio-packet.freedesktop.org%s' % (resource)
+ to_sign = "PUT\n\n%s\n%s\nx-amz-security-token:%s\n%s" % (content_type, date, minio_token, resource)
+ signature = sign_with_hmac(minio_secret, to_sign)
+
+ with open(file_name, 'rb') as data:
+ headers = {'Host': 'minio-packet.freedesktop.org',
+ 'Date': date,
+ 'Content-Type': content_type,
+ 'Authorization': 'AWS %s:%s' % (minio_key, signature),
+ 'x-amz-security-token': minio_token}
+ print("Uploading artifact to %s" % url);
+ r = requests.put(url, headers=headers, data=data)
+ #print(r.text)
+ r.raise_for_status()
+
def gitlab_check_trace(project_url, device_name, trace, expectation):
gitlab_ensure_trace(project_url, trace)
@@ -82,6 +119,8 @@ def gitlab_check_trace(project_url, device_name, trace, expectation):
results_path = os.path.join(RESULTS_PATH, dir_in_results)
os.makedirs(results_path, exist_ok=True)
shutil.move(log_file, os.path.join(results_path, os.path.split(log_file)[1]))
+ if not ok and os.environ.get('TRACIE_UPLOAD_TO_MINIO', '0') == '1':
+ upload_artifact(image_file, 'image/png')
if not ok or os.environ.get('TRACIE_STORE_IMAGES', '0') == '1':
image_name = os.path.split(image_file)[1]
shutil.move(image_file, os.path.join(results_path, image_name))
@@ -116,6 +155,8 @@ def run(filename, device_name):
os.makedirs(RESULTS_PATH, exist_ok=True)
with open(os.path.join(RESULTS_PATH, 'results.yml'), 'w') as f:
yaml.safe_dump(results, f, default_flow_style=False)
+ if os.environ.get('TRACIE_UPLOAD_TO_MINIO', '0') == '1':
+ upload_artifact(os.path.join(RESULTS_PATH, 'results.yml'), 'text/yaml')
return all_ok
More information about the mesa-commit
mailing list