Mesa (main): ci: Add known-flake handling for the IRC flake reports
GitLab Mirror
gitlab-mirror at kemper.freedesktop.org
Tue Jun 1 22:19:44 UTC 2021
Module: Mesa
Branch: main
Commit: e414718aef22130d10b5ba4ed13ca45fe2608785
URL: http://cgit.freedesktop.org/mesa/mesa/commit/?id=e414718aef22130d10b5ba4ed13ca45fe2608785
Author: Emma Anholt <emma at anholt.net>
Date: Wed May 26 10:45:33 2021 -0700
ci: Add known-flake handling for the IRC flake reports
Now, flakes that aren't in the *-flakes.txt get a "NEW" in their report so
I can watch for them.
The bash was unwieldy and made debugging hard, so I switched to python.
Reviewed-by: Alyssa Rosenzweig <alyssa.rosenzweig at collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/11020>
---
.gitlab-ci/deqp-runner.sh | 63 ++++------------
.gitlab-ci/piglit/piglit-runner.sh | 73 ++++--------------
.gitlab-ci/prepare-artifacts.sh | 1 +
.gitlab-ci/report-flakes.py | 151 +++++++++++++++++++++++++++++++++++++
4 files changed, 184 insertions(+), 104 deletions(-)
diff --git a/.gitlab-ci/deqp-runner.sh b/.gitlab-ci/deqp-runner.sh
index 8fc64f20827..35ba6f4b89e 100755
--- a/.gitlab-ci/deqp-runner.sh
+++ b/.gitlab-ci/deqp-runner.sh
@@ -91,9 +91,8 @@ if [ -e "$INSTALL/deqp-$GPU_VERSION-fails.txt" ]; then
DEQP_RUNNER_OPTIONS="$DEQP_RUNNER_OPTIONS --baseline $INSTALL/deqp-$GPU_VERSION-fails.txt"
fi
-if [ -e "$INSTALL/deqp-$GPU_VERSION-flakes.txt" ]; then
- DEQP_RUNNER_OPTIONS="$DEQP_RUNNER_OPTIONS --flakes $INSTALL/deqp-$GPU_VERSION-flakes.txt"
-fi
+# Default to an empty known flakes file if it doesn't exist.
+touch $INSTALL/deqp-$GPU_VERSION-flakes.txt
if [ -e "$INSTALL/deqp-$GPU_VERSION-skips.txt" ]; then
DEQP_RUNNER_OPTIONS="$DEQP_RUNNER_OPTIONS --skips $INSTALL/deqp-$GPU_VERSION-skips.txt"
@@ -125,6 +124,7 @@ run_cts() {
--deqp $deqp \
--output $RESULTS \
--caselist $caselist \
+ --flakes $INSTALL/deqp-$GPU_VERSION-flakes.txt \
--testlog-to-xml /deqp/executor/testlog-to-xml \
$JOB \
$SUMMARY_LIMIT \
@@ -133,49 +133,6 @@ run_cts() {
$DEQP_OPTIONS
}
-report_flakes() {
- flakes=`grep ",Flake" $1 | sed 's|,Flake.*||g'`
- if [ -z "$flakes" ]; then
- return 0
- fi
-
- if [ -z "$FLAKES_CHANNEL" ]; then
- return 0
- fi
-
- # The nick needs to be something unique so that multiple runners
- # connecting at the same time don't race for one nick and get blocked.
- # freenode has a 16-char limit on nicks (9 is the IETF standard, but
- # various servers extend that). So, trim off the common prefixes of the
- # runner name, and append the job ID so that software runners with more
- # than one concurrent job (think swrast) don't collide. For freedreno,
- # that gives us a nick as long as db410c-N-JJJJJJJJ, and it'll be a while
- # before we make it to 9-digit jobs (we're at 7 so far).
- runner=`echo $CI_RUNNER_DESCRIPTION | sed 's|mesa-||' | sed 's|google-freedreno-||g'`
- bot="$runner-$CI_JOB_ID"
- channel="$FLAKES_CHANNEL"
- (
- echo NICK $bot
- echo USER $bot unused unused :Gitlab CI Notifier
- sleep 10
- echo "JOIN $channel"
- sleep 1
- desc="Flakes detected in job: $CI_JOB_URL on $CI_RUNNER_DESCRIPTION"
- if [ -n "$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME" ]; then
- desc="$desc on branch $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME ($CI_MERGE_REQUEST_TITLE)"
- elif [ -n "$CI_COMMIT_BRANCH" ]; then
- desc="$desc on branch $CI_COMMIT_BRANCH ($CI_COMMIT_TITLE)"
- fi
- echo "PRIVMSG $channel :$desc"
- for flake in $flakes; do
- echo "PRIVMSG $channel :$flake"
- done
- echo "PRIVMSG $channel :See $CI_JOB_URL/artifacts/browse/results/"
- echo "QUIT"
- ) | nc irc.freenode.net 6667 > /dev/null
-
-}
-
parse_renderer() {
RENDERER=`grep -A1 TestCaseResult.\*info.renderer $RESULTS/deqp-info.qpa | grep '<Text' | sed 's|.*<Text>||g' | sed 's|</Text>||g'`
VERSION=`grep -A1 TestCaseResult.\*info.version $RESULTS/deqp-info.qpa | grep '<Text' | sed 's|.*<Text>||g' | sed 's|</Text>||g'`
@@ -281,6 +238,18 @@ deqp-runner junit \
--template "See https://$CI_PROJECT_ROOT_NAMESPACE.pages.freedesktop.org/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/results/{{testcase}}.xml"
# Report the flakes to the IRC channel for monitoring (if configured):
-quiet report_flakes $RESULTS_CSV
+if [ -n "$FLAKES_CHANNEL" ]; then
+ python3 $INSTALL/report-flakes.py \
+ --host irc.freenode.net \
+ --port 6667 \
+ --results $RESULTS_CSV \
+ --known-flakes $INSTALL/deqp-$GPU_VERSION-flakes.txt \
+ --channel "$FLAKES_CHANNEL" \
+ --runner "$CI_RUNNER_DESCRIPTION" \
+ --job "$CI_JOB_ID" \
+ --url "$CI_JOB_URL" \
+ --branch "${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:-$CI_COMMIT_BRANCH}" \
+ --branch-title "${CI_MERGE_REQUEST_TITLE:-$CI_COMMIT_TITLE}"
+fi
exit $DEQP_EXITCODE
diff --git a/.gitlab-ci/piglit/piglit-runner.sh b/.gitlab-ci/piglit/piglit-runner.sh
index 384a617a84c..c531b2fb030 100755
--- a/.gitlab-ci/piglit/piglit-runner.sh
+++ b/.gitlab-ci/piglit/piglit-runner.sh
@@ -32,9 +32,8 @@ if [ -e "$INSTALL/piglit-$GPU_VERSION-fails.txt" ]; then
PIGLIT_RUNNER_OPTIONS="$PIGLIT_RUNNER_OPTIONS --baseline $INSTALL/piglit-$GPU_VERSION-fails.txt"
fi
-if [ -e "$INSTALL/piglit-$GPU_VERSION-flakes.txt" ]; then
- PIGLIT_RUNNER_OPTIONS="$PIGLIT_RUNNER_OPTIONS --flakes $INSTALL/piglit-$GPU_VERSION-flakes.txt"
-fi
+# Default to an empty known flakes file if it doesn't exist.
+touch $INSTALL/piglit-$GPU_VERSION-flakes.txt
if [ -e "$INSTALL/piglit-$GPU_VERSION-skips.txt" ]; then
PIGLIT_RUNNER_OPTIONS="$PIGLIT_RUNNER_OPTIONS --skips $INSTALL/piglit-$GPU_VERSION-skips.txt"
@@ -50,59 +49,6 @@ else
PIGLIT_RUNNER_OPTIONS="$PIGLIT_RUNNER_OPTIONS --jobs 4"
fi
-report_flakes() {
- # Replace spaces in test names with _ to make the channel reporting not
- # split it across lines, even though it makes it so you can't copy and
- # paste from IRC into your flakes list.
- flakes=`grep ",Flake" $1 | sed 's|,Flake.*||g' | sed 's| |_|g'`
- if [ -z "$flakes" ]; then
- return 0
- fi
-
- if [ -z "$FLAKES_CHANNEL" ]; then
- return 0
- fi
-
- # The nick needs to be something unique so that multiple runners
- # connecting at the same time don't race for one nick and get blocked.
- # freenode has a 16-char limit on nicks (9 is the IETF standard, but
- # various servers extend that). So, trim off the common prefixes of the
- # runner name, and append the job ID so that software runners with more
- # than one concurrent job (think swrast) don't collide. For freedreno,
- # that gives us a nick as long as db410c-N-JJJJJJJJ, and it'll be a while
- # before we make it to 9-digit jobs (we're at 7 so far).
- runner=`echo $CI_RUNNER_DESCRIPTION | sed 's|mesa-||' | sed 's|google-freedreno-||g'`
- bot="$runner-$CI_JOB_ID"
- channel="$FLAKES_CHANNEL"
- (
- echo NICK $bot
- echo USER $bot unused unused :Gitlab CI Notifier
- sleep 10
- echo "JOIN $channel"
- sleep 1
- desc="Flakes detected in job: $CI_JOB_URL on $CI_RUNNER_DESCRIPTION"
- if [ -n "$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME" ]; then
- desc="$desc on branch $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME ($CI_MERGE_REQUEST_TITLE)"
- elif [ -n "$CI_COMMIT_BRANCH" ]; then
- desc="$desc on branch $CI_COMMIT_BRANCH ($CI_COMMIT_TITLE)"
- fi
- echo "PRIVMSG $channel :$desc"
- for flake in $flakes; do
- echo "PRIVMSG $channel :$flake"
- done
- echo "PRIVMSG $channel :See $CI_JOB_URL/artifacts/browse/results/"
- echo "QUIT"
- ) | nc irc.freenode.net 6667 > /dev/null
-
-}
-
-# wrapper to supress +x to avoid spamming the log
-quiet() {
- set +x
- "$@"
- set -x
-}
-
RESULTS_CSV=$RESULTS/results.csv
FAILURES_CSV=$RESULTS/failures.csv
@@ -112,6 +58,7 @@ export LD_PRELOAD=$TEST_LD_PRELOAD
run \
--piglit-folder /piglit \
--output $RESULTS \
+ --flakes $INSTALL/piglit-$GPU_VERSION-flakes.txt \
--profile $PIGLIT_PROFILES \
--process-isolation \
$PIGLIT_RUNNER_OPTIONS \
@@ -129,6 +76,18 @@ deqp-runner junit \
--template "See https://$CI_PROJECT_ROOT_NAMESPACE.pages.freedesktop.org/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/results/{{testcase}}.xml"
# Report the flakes to the IRC channel for monitoring (if configured):
-quiet report_flakes $RESULTS_CSV
+if [ -n "$FLAKES_CHANNEL" ]; then
+ python3 $INSTALL/report-flakes.py \
+ --host irc.freenode.net \
+ --port 6667 \
+ --results $RESULTS_CSV \
+ --known-flakes $INSTALL/piglit-$GPU_VERSION-flakes.txt \
+ --channel "$FLAKES_CHANNEL" \
+ --runner "$CI_RUNNER_DESCRIPTION" \
+ --job "$CI_JOB_ID" \
+ --url "$CI_JOB_URL" \
+ --branch "${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:-$CI_COMMIT_BRANCH}" \
+ --branch-title "${CI_MERGE_REQUEST_TITLE:-$CI_COMMIT_TITLE}"
+fi
exit $PIGLIT_EXITCODE
diff --git a/.gitlab-ci/prepare-artifacts.sh b/.gitlab-ci/prepare-artifacts.sh
index 9acbd023c58..005cfe590bd 100755
--- a/.gitlab-ci/prepare-artifacts.sh
+++ b/.gitlab-ci/prepare-artifacts.sh
@@ -33,6 +33,7 @@ cp -Rp .gitlab-ci/fossils install/
cp -Rp .gitlab-ci/fossilize-runner.sh install/
cp -Rp .gitlab-ci/deqp-runner.sh install/
cp -Rp .gitlab-ci/deqp-*.txt install/
+cp -Rp .gitlab-ci/report-flakes.py install/
cp -Rp .gitlab-ci/vkd3d-proton install/
find . -path \*/ci/\*.txt \
-o -path \*/ci/\*traces\*.yml \
diff --git a/.gitlab-ci/report-flakes.py b/.gitlab-ci/report-flakes.py
new file mode 100644
index 00000000000..ed7009c9bd6
--- /dev/null
+++ b/.gitlab-ci/report-flakes.py
@@ -0,0 +1,151 @@
+#!/usr/bin/env python3
+#
+# Copyright © 2021 Google LLC
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice (including the next
+# paragraph) shall be included in all copies or substantial portions of the
+# Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+
+import argparse
+import io
+import re
+import socket
+import time
+
+
+class Connection:
+ def __init__(self, host, port, verbose):
+ self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ self.s.connect((host, port))
+ self.s.setblocking(0)
+ self.verbose = verbose
+
+ def send_line(self, line):
+ if self.verbose:
+ print(f"IRC: sending {line}")
+ self.s.sendall((line + '\n').encode())
+
+ def wait(self, secs):
+ for i in range(secs):
+ if self.verbose:
+ while True:
+ try:
+ data = self.s.recv(1024)
+ except io.BlockingIOError:
+ break
+ if data == "":
+ break
+ for line in data.decode().split('\n'):
+ print(f"IRC: received {line}")
+ time.sleep(1)
+
+ def quit(self):
+ self.send_line("QUIT")
+ self.s.shutdown(socket.SHUT_WR)
+ self.s.close()
+
+
+def read_flakes(results):
+ flakes = []
+ csv = re.compile("(.*),(.*),(.*)")
+ for line in open(results, 'r').readlines():
+ match = csv.match(line)
+ if match.group(2) == "Flake":
+ flakes.append(match.group(1))
+ return flakes
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--host', type=str,
+ help='IRC server hostname', required=True)
+ parser.add_argument('--port', type=int,
+ help='IRC server port', required=True)
+ parser.add_argument('--results', type=str,
+ help='results.csv file from deqp-runner or piglit-runner', required=True)
+ parser.add_argument('--known-flakes', type=str,
+ help='*-flakes.txt file passed to deqp-runner or piglit-runner', required=True)
+ parser.add_argument('--channel', type=str,
+ help='Known flakes report channel', required=True)
+ parser.add_argument('--url', type=str,
+ help='$CI_JOB_URL', required=True)
+ parser.add_argument('--runner', type=str,
+ help='$CI_RUNNER_DESCRIPTION', required=True)
+ parser.add_argument('--branch', type=str,
+ help='optional branch name')
+ parser.add_argument('--branch-title', type=str,
+ help='optional branch title')
+ parser.add_argument('--job', type=str,
+ help='$CI_JOB_ID', required=True)
+ parser.add_argument('--verbose', "-v", action="store_true",
+ help='log IRC interactions')
+ args = parser.parse_args()
+
+ flakes = read_flakes(args.results)
+ if not flakes:
+ exit(0)
+
+ known_flakes = []
+ for line in open(args.known_flakes).readlines():
+ line = line.strip()
+ if not line or line.startswith("#"):
+ continue
+ known_flakes.append(re.compile(line))
+
+ irc = Connection(args.host, args.port, args.verbose)
+
+ # The nick needs to be something unique so that multiple runners
+ # connecting at the same time don't race for one nick and get blocked.
+ # freenode has a 16-char limit on nicks (9 is the IETF standard, but
+ # various servers extend that). So, trim off the common prefixes of the
+ # runner name, and append the job ID so that software runners with more
+ # than one concurrent job (think swrast) don't collide. For freedreno,
+ # that gives us a nick as long as db410c-N-JJJJJJJJ, and it'll be a while
+ # before we make it to 9-digit jobs (we're at 7 so far).
+ nick = args.runner
+ nick = nick.replace('mesa-', '')
+ nick = nick.replace('google-freedreno-', '')
+ nick += f'-{args.job}'
+ irc.send_line(f"NICK {nick}")
+ irc.send_line(f"USER {nick} unused unused: Gitlab CI Notifier")
+ irc.wait(10)
+ irc.send_line(f"JOIN {args.channel}")
+ irc.wait(1)
+
+ branchinfo = ""
+ if args.branch:
+ branchinfo = f" on branch {args.branch} ({args.branch_title})"
+ irc.send_line(
+ f"PRIVMSG {args.channel} :Flakes detected in job {args.url} on {args.runner}{branchinfo}:")
+
+ for flake in flakes:
+ status = "NEW "
+ for known in known_flakes:
+ if known.match(flake):
+ status = ""
+ break
+
+ irc.send_line(f"PRIVMSG {args.channel} :{status}{flake}")
+
+ irc.send_line(
+ f"PRIVMSG {args.channel} :See {args.url}/artifacts/browse/results/")
+
+ irc.quit()
+
+
+if __name__ == '__main__':
+ main()
More information about the mesa-commit
mailing list