Mesa (master): ci/deqp: Upgrade the runner, enable junit output.
GitLab Mirror
gitlab-mirror at kemper.freedesktop.org
Wed Dec 23 21:17:28 UTC 2020
Module: Mesa
Branch: master
Commit: 302b4f772445cbfd2dd3f3ad69e4e293f5768fc9
URL: http://cgit.freedesktop.org/mesa/mesa/commit/?id=302b4f772445cbfd2dd3f3ad69e4e293f5768fc9
Author: Eric Anholt <eric at anholt.net>
Date: Fri Dec 18 16:47:51 2020 -0800
ci/deqp: Upgrade the runner, enable junit output.
I moved QPA-to-XML conversion to the runner, so Mesa CI (and developers!)
don't need to do quite so much in bash. I also made it clean up caselist
.qpa files since nobody ever wants them and we deleted them anyway. This
cleans up a ton of the job log output.
Additionally, I added a subcommend to turn the .csv into a junit output
that we can expose to gitlab. Now, the pipeline's status page will report
the failed testcases, and the "detail" button will give you a link to the
.XML to view for the failure. (We don't report all testcases because it's
too much load for the gitlab server). Note that this will 404 for the
LAVA runners for now, as they don't retain artifacts in gitlab (the plan
is to eventually have them minio upload the artifacts).
This uprev also includes a deqp output parsing fix, resulting in us
catching a couple more failures in some drivers.
Reviewed-by: Christian Gmeiner <christian.gmeiner at gmail.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/8206>
---
.gitlab-ci.yml | 13 +++++--
.gitlab-ci/build-deqp-runner.sh | 2 +-
.gitlab-ci/deqp-freedreno-a530-fails.txt | 1 +
.gitlab-ci/deqp-freedreno-a630-fails.txt | 11 ++++++
.gitlab-ci/deqp-freedreno-a630-skips.txt | 1 +
.gitlab-ci/deqp-lvp-fails.txt | 1 +
.gitlab-ci/deqp-runner.sh | 63 ++++++++++----------------------
.gitlab-ci/deqp-virgl-gl-fails.txt | 2 +-
.gitlab-ci/lava-gitlab-ci.yml | 2 +-
9 files changed, 45 insertions(+), 51 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 2d6c4db49a7..35300f2c5b1 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -362,13 +362,13 @@ x86_test-base:
x86_test-gl:
extends: .use-x86_test-base
variables:
- MESA_IMAGE_TAG: &x86_test-gl "2020-12-18-piglit-replayer"
+ MESA_IMAGE_TAG: &x86_test-gl "2020-12-22-runner"
# Debian 10 based x86 test image for VK
x86_test-vk:
extends: .use-x86_test-base
variables:
- MESA_IMAGE_TAG: &x86_test-vk "2020-12-19-build-piglit"
+ MESA_IMAGE_TAG: &x86_test-vk "2020-12-22-runner"
# Debian 10 based ARM build image
arm_build:
@@ -408,7 +408,7 @@ arm64_test:
extends:
- .use-arm_test-base
variables:
- MESA_IMAGE_TAG: &arm64_test "2020-12-19-build-piglit"
+ MESA_IMAGE_TAG: &arm64_test "2020-12-22-runner"
.use-arm64_test:
variables:
@@ -422,7 +422,7 @@ armhf_test:
extends:
- .use-arm_test-base
variables:
- MESA_IMAGE_TAG: &armhf_test "2020-12-19-build-piglit"
+ MESA_IMAGE_TAG: &armhf_test "2020-12-22-runner"
.use-armhf_test:
variables:
@@ -1179,6 +1179,9 @@ radv-raven-traces:
.deqp-test:
script:
- ./install/deqp-runner.sh
+ artifacts:
+ reports:
+ junit: results/junit.xml
.deqp-test-vk:
extends:
@@ -1412,6 +1415,8 @@ arm64_a630_vk_sysmem:
- serial*.txt
exclude:
- results/*.shader_cache
+ reports:
+ junit: results/junit.xml
.freedreno-test:
extends:
diff --git a/.gitlab-ci/build-deqp-runner.sh b/.gitlab-ci/build-deqp-runner.sh
index 99915182dce..c8b8298da49 100644
--- a/.gitlab-ci/build-deqp-runner.sh
+++ b/.gitlab-ci/build-deqp-runner.sh
@@ -4,6 +4,6 @@ set -ex
cargo install deqp-runner \
-j ${FDO_CI_CONCURRENT:-4} \
- --version 0.1.5 \
+ --version 0.4.0 \
--root /usr/local \
$EXTRA_CARGO_ARGS
diff --git a/.gitlab-ci/deqp-freedreno-a530-fails.txt b/.gitlab-ci/deqp-freedreno-a530-fails.txt
index 9b86f26f0e8..13e8d0d12ff 100644
--- a/.gitlab-ci/deqp-freedreno-a530-fails.txt
+++ b/.gitlab-ci/deqp-freedreno-a530-fails.txt
@@ -39,6 +39,7 @@ dEQP-GLES3.functional.transform_feedback.random.interleaved.lines.3,Fail
dEQP-GLES3.functional.transform_feedback.random.separate.points.3,Fail
dEQP-GLES3.functional.transform_feedback.random_full_array_capture.separate.triangles.3,Fail
dEQP-GLES31.functional.image_load_store.cube.format_reinterpret.r32i_rgba8,Fail
+dEQP-GLES31.functional.image_load_store.cube.format_reinterpret.rgba32f_rgba32ui,Fail
dEQP-GLES31.functional.image_load_store.cube.format_reinterpret.rgba8_snorm_r32ui,Fail
dEQP-GLES31.functional.image_load_store.cube.format_reinterpret.rgba8i_r32f,Fail
dEQP-GLES31.functional.image_load_store.cube.load_store.r32f_single_layer,Fail
diff --git a/.gitlab-ci/deqp-freedreno-a630-fails.txt b/.gitlab-ci/deqp-freedreno-a630-fails.txt
index c7d0899436e..42b142076cb 100644
--- a/.gitlab-ci/deqp-freedreno-a630-fails.txt
+++ b/.gitlab-ci/deqp-freedreno-a630-fails.txt
@@ -1,8 +1,14 @@
KHR-GL30.transform_feedback.api_errors_test,Fail
+KHR-GL30.transform_feedback.capture_vertex_interleaved_test,Fail
+KHR-GL30.transform_feedback.capture_vertex_separate_test,Fail
+KHR-GL30.transform_feedback.discard_vertex_test,Fail
KHR-GL30.transform_feedback.draw_xfb_feedbackk_test,Crash
KHR-GL30.transform_feedback.draw_xfb_instanced_test,Crash
KHR-GL30.transform_feedback.draw_xfb_stream_instanced_test,Crash
KHR-GL30.transform_feedback.draw_xfb_test,Crash
+KHR-GL30.transform_feedback.get_xfb_varying,Fail
+KHR-GL30.transform_feedback.query_vertex_interleaved_test,Fail
+KHR-GL30.transform_feedback.query_vertex_separate_test,Fail
dEQP-VK.api.object_management.single_alloc_callbacks.compute_pipeline,Fail
dEQP-VK.compute.indirect_dispatch.gen_in_compute.multiple_groups_multiple_invocations,Fail
dEQP-VK.compute.indirect_dispatch.upload_buffer.multiple_groups,Fail
@@ -29,6 +35,7 @@ dEQP-VK.multiview.renderpass2.masks.max_multi_view_view_count,Fail
dEQP-VK.pipeline.extended_dynamic_state.after_pipelines.depth_compare_greater_equal_greater,Fail
dEQP-VK.pipeline.extended_dynamic_state.before_draw.depth_compare_always_greater,Fail
dEQP-VK.pipeline.multisample.alpha_to_coverage_unused_attachment.samples_4.alpha_invisible,Fail
+dEQP-VK.pipeline.push_descriptor.compute.binding3_numcalls2_sampler,Crash
dEQP-VK.renderpass2.depth_stencil_resolve.image_2d_16_64_6.samples_2.d24_unorm_s8_uint.depth_zero,Fail
dEQP-VK.renderpass2.depth_stencil_resolve.image_2d_16_64_6.samples_4.x8_d24_unorm_pack32.depth_zero,Fail
dEQP-VK.renderpass2.suballocation.attachment_allocation.input_output.7,Fail
@@ -38,11 +45,15 @@ dEQP-VK.spirv_assembly.instruction.graphics.opquantize.negative_too_small_tesse,
dEQP-VK.spirv_assembly.instruction.graphics.opquantize.round_to_inf_tesse,Fail
dEQP-VK.spirv_assembly.instruction.graphics.opquantize.spec_const_carry_to_exponent_tesse,Fail
dEQP-VK.spirv_assembly.instruction.graphics.opquantize.spec_const_negative_round_up_or_round_down_tesse,Fail
+dEQP-VK.tessellation.invariance.inner_triangle_set.triangles_equal_spacing,Fail
dEQP-VK.tessellation.invariance.outer_edge_division.triangles_fractional_even_spacing,Fail
dEQP-VK.tessellation.invariance.outer_edge_index_independence.triangles_equal_spacing_ccw,Fail
+dEQP-VK.tessellation.invariance.outer_edge_index_independence.triangles_fractional_even_spacing_cw,Fail
+dEQP-VK.tessellation.invariance.outer_edge_index_independence.quads_fractional_even_spacing_ccw,Fail
dEQP-VK.tessellation.invariance.outer_edge_symmetry.isolines_equal_spacing_cw,Fail
dEQP-VK.tessellation.invariance.outer_edge_symmetry.quads_fractional_odd_spacing_ccw,Fail
dEQP-VK.tessellation.invariance.outer_edge_symmetry.triangles_fractional_odd_spacing_cw,Fail
+dEQP-VK.tessellation.invariance.outer_triangle_set.quads_fractional_odd_spacing,Fail
dEQP-VK.tessellation.invariance.primitive_set.isolines_fractional_odd_spacing_ccw,Fail
dEQP-VK.tessellation.invariance.primitive_set.quads_fractional_odd_spacing_cw,Fail
dEQP-VK.tessellation.invariance.primitive_set.triangles_fractional_even_spacing_ccw,Fail
diff --git a/.gitlab-ci/deqp-freedreno-a630-skips.txt b/.gitlab-ci/deqp-freedreno-a630-skips.txt
index 588b657f356..07deb2b5405 100644
--- a/.gitlab-ci/deqp-freedreno-a630-skips.txt
+++ b/.gitlab-ci/deqp-freedreno-a630-skips.txt
@@ -18,6 +18,7 @@ dEQP-VK.spirv_assembly.instruction.graphics.spirv_ids_abuse.lots_ids_tesse
dEQP-VK.tessellation.invariance.outer_edge_division.quads_fractional_odd_spacing
# Timeout (VK-GL-CTS 1.2.5.0)
+dEQP-VK.geometry.layered.cube_array.36_36_12.readback
dEQP-VK.geometry.layered.cube_array.64_64_12.readback
# Crashes likely caused by https://gitlab.khronos.org/Tracker/vk-gl-cts/-/issues/2701
diff --git a/.gitlab-ci/deqp-lvp-fails.txt b/.gitlab-ci/deqp-lvp-fails.txt
index 14bb70d2992..0150b2520b6 100644
--- a/.gitlab-ci/deqp-lvp-fails.txt
+++ b/.gitlab-ci/deqp-lvp-fails.txt
@@ -4,6 +4,7 @@ dEQP-VK.api.copy_and_blit.core.blit_image.all_formats.color.2d.r8g8_unorm.r16g16
dEQP-VK.api.copy_and_blit.core.blit_image.all_formats.color.2d.r8g8b8a8_unorm.a2b10g10r10_unorm_pack32.optimal_optimal_linear,Fail
dEQP-VK.api.copy_and_blit.core.blit_image.all_formats.color.2d.r8g8b8a8_unorm.r16g16_unorm.general_optimal_linear,Fail
dEQP-VK.api.copy_and_blit.core.blit_image.all_formats.color.2d.r8g8b8a8_unorm.r16g16b16a16_unorm.linear_general_linear,Fail
+dEQP-VK.api.copy_and_blit.core.blit_image.all_formats.color.2d.r8_unorm.r16g16b16a16_unorm.general_optimal_linear,Fail
dEQP-VK.glsl.builtin.precision.pow.highp.vec3,Fail
dEQP-VK.glsl.texture_functions.query.texturequerylod.sampler1d_fixed_fragment,Fail
dEQP-VK.glsl.texture_functions.query.texturequerylod.sampler2darray_fixed_fragment,Fail
diff --git a/.gitlab-ci/deqp-runner.sh b/.gitlab-ci/deqp-runner.sh
index 42f6f730e46..6d04fdbb90a 100755
--- a/.gitlab-ci/deqp-runner.sh
+++ b/.gitlab-ci/deqp-runner.sh
@@ -120,6 +120,7 @@ run_cts() {
--deqp $deqp \
--output $RESULTS \
--caselist $caselist \
+ --testlog-to-xml /deqp/executor/testlog-to-xml \
$JOB \
$SUMMARY_LIMIT \
$DEQP_RUNNER_OPTIONS \
@@ -170,31 +171,6 @@ report_flakes() {
}
-# Generate junit results
-generate_junit() {
- results=$1
- echo "<?xml version=\"1.0\" encoding=\"utf-8\"?>"
- echo "<testsuites>"
- echo "<testsuite name=\"$DEQP_VER-$CI_NODE_INDEX\">"
- while read line; do
- testcase=${line%,*}
- result=${line#*,}
- # avoid counting Skip's in the # of tests:
- if [ "$result" = "Skip" ]; then
- continue;
- fi
- echo "<testcase name=\"$testcase\">"
- if [ "$result" != "Pass" ]; then
- echo "<failure type=\"$result\">"
- echo "$result: See $CI_JOB_URL/artifacts/results/$testcase.xml"
- echo "</failure>"
- fi
- echo "</testcase>"
- done < $results
- echo "</testsuite>"
- echo "</testsuites>"
-}
-
parse_renderer() {
RENDERER=`grep -A1 TestCaseResult.\*info.renderer $RESULTS/deqp-info.qpa | grep '<Text' | sed 's|.*<Text>||g' | sed 's|</Text>||g'`
VERSION=`grep -A1 TestCaseResult.\*info.version $RESULTS/deqp-info.qpa | grep '<Text' | sed 's|.*<Text>||g' | sed 's|</Text>||g'`
@@ -266,25 +242,24 @@ DEQP_EXITCODE=$?
echo "System load: $(cut -d' ' -f1-3 < /proc/loadavg)"
echo "# of CPU cores: $(cat /proc/cpuinfo | grep processor | wc -l)"
-# junit is disabled, because it overloads gitlab.freedesktop.org to parse it.
-# quiet generate_junit $RESULTS_CSV > $RESULTS/results.xml
-
-# Turn up to the first 50 individual test QPA files from failures or flakes into
-# XML results you can view from the browser.
-qpas=`find $RESULTS -name \*.qpa -a ! -name deqp-info.qpa`
-if [ -n "$qpas" ]; then
- shard_qpas=`echo "$qpas" | grep dEQP- | head -n 50`
- for qpa in $shard_qpas; do
- xml=`echo $qpa | sed 's|\.qpa|.xml|'`
- /deqp/executor/testlog-to-xml $qpa $xml
- done
-
- cp /deqp/testlog.css "$RESULTS/"
- cp /deqp/testlog.xsl "$RESULTS/"
-
- # Remove all the QPA files (extracted or not) now that we have the XML we want.
- echo $qpas | xargs rm -f
-fi
+# Remove all but the first 50 individual XML files uploaded as artifacts, to
+# save fd.o space when you break everything.
+find $RESULTS -name \*.xml | \
+ sort -n |
+ sed -n '1,+49!p' | \
+ xargs rm -f
+
+# If any QPA XMLs are there, then include the XSL/CSS in our artifacts.
+find $RESULTS -name \*.xml \
+ -exec cp /deqp/testlog.css /deqp/testlog.xsl "$RESULTS/" ";" \
+ -quit
+
+deqp-runner junit \
+ --testsuite $DEQP_VER \
+ --results $RESULTS/failures.csv \
+ --output $RESULTS/junit.xml \
+ --limit 50 \
+ --template "See https://$CI_PROJECT_NAMESPACE.pages.freedesktop.org/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/results/{{testcase}}.xml"
# Report the flakes to the IRC channel for monitoring (if configured):
quiet report_flakes $RESULTS_CSV
diff --git a/.gitlab-ci/deqp-virgl-gl-fails.txt b/.gitlab-ci/deqp-virgl-gl-fails.txt
index eeebd7eca15..0aa551ba4f4 100644
--- a/.gitlab-ci/deqp-virgl-gl-fails.txt
+++ b/.gitlab-ci/deqp-virgl-gl-fails.txt
@@ -4678,7 +4678,7 @@ KHR-GL30.transform_feedback.query_vertex_separate_test,Fail
KHR-GL31.transform_feedback.capture_vertex_interleaved_test,Fail
KHR-GL31.transform_feedback.capture_vertex_separate_test,Fail
KHR-GL31.transform_feedback.discard_vertex_test,Fail
-KHR-GL31.transform_feedback.draw_xfb_instanced_test,Fail
+KHR-GL31.transform_feedback.draw_xfb_instanced_test,Crash
KHR-GL31.transform_feedback.draw_xfb_stream_instanced_test,Crash
KHR-GL31.transform_feedback.query_vertex_interleaved_test,Fail
KHR-GL31.transform_feedback.query_vertex_separate_test,Fail
diff --git a/.gitlab-ci/lava-gitlab-ci.yml b/.gitlab-ci/lava-gitlab-ci.yml
index 0d7a76582c4..45a92e788c6 100644
--- a/.gitlab-ci/lava-gitlab-ci.yml
+++ b/.gitlab-ci/lava-gitlab-ci.yml
@@ -1,5 +1,5 @@
variables:
- DISTRIBUTION_TAG: "2020-12-19-build-piglit"
+ DISTRIBUTION_TAG: "2020-12-22-runner"
.kernel+rootfs:
stage: container-2
More information about the mesa-commit
mailing list