[PATCH wayland v2 3/3] tests: add scanner tests

[Pekka Paalanen]

From: Pekka Paalanen <pekka.paalanen at collabora.co.uk>

Add tests that ensure that wayland-scanner output for a given input does
not change unexpectedly. This makes it very easy to review
wayland-scanner patches.

Before, when patches were proposed for wayland-scanner, I had to
build wayland without the patches, save the generated files into a
temporary directory, apply the patches, build again, and diff the old
vs. new generated file.

No more. Now whenever someone makes intentional changes to
wayland-scanner's output, he will also have to patch the example output
files to match. That means that reviewers see the diff of the generated
files straight from the patch itself. Verifying the diff is true is as
easy as 'make check'.

The tests use separate example XML files instead of wayland.xml
directly, so that wayland.xml can be updated without fixing scanner
tests, avoiding the churn.

example.xml starts as a copy of wayland.xml. If wayland.xml starts using
new wayland-scanner features, they should be copied into example.xml
again to be covered by the tests.

This patch relies on the previous patch to actually add all the data
files for input and reference output.

The scanner output is fed through sed to remove parts that are allowed
to vary: the scanner version string.

v2: no need for scanner-test.sh to depend on the test data

Task: https://phabricator.freedesktop.org/T3313
Cc: Emilio Pozuelo Monfort <emilio.pozuelo at collabora.co.uk>
Cc: Yong Bakos <ybakos at humanoriented.com>
Signed-off-by: Pekka Paalanen <pekka.paalanen at collabora.co.uk>
---
 .gitignore            |  1 +
 Makefile.am           | 29 ++++++++++++++++++++++++++++-
 tests/scanner-test.sh | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 80 insertions(+), 1 deletion(-)
 create mode 100755 tests/scanner-test.sh

diff --git a/.gitignore b/.gitignore
index 33e809c..8da9861 100644
--- a/.gitignore
+++ b/.gitignore
@@ -38,6 +38,7 @@ ctags
 /missing
 /stamp-h1
 /test-driver
+/tests/output/
 Makefile
 Makefile.in
 exec-fd-leak-checker
diff --git a/Makefile.am b/Makefile.am
index d35231c..be372da 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -168,7 +168,15 @@ if ENABLE_CPP_TEST
 built_test_programs += cpp-compile-test
 endif
 
-TESTS = $(built_test_programs)
+AM_TESTS_ENVIRONMENT =							\
+	export WAYLAND_SCANNER='$(top_builddir)/wayland-scanner'	\
+	TEST_DATA_DIR='$(top_srcdir)/tests/data'			\
+	TEST_OUTPUT_DIR='$(top_builddir)/tests/output'			\
+	SED=$(SED)							\
+	;
+
+TESTS = $(built_test_programs)			\
+	tests/scanner-test.sh
 
 check_PROGRAMS =				\
 	$(built_test_programs)			\
@@ -245,4 +253,23 @@ os_wrappers_test_LDADD = libtest-runner.la
 
 exec_fd_leak_checker_SOURCES = tests/exec-fd-leak-checker.c
 exec_fd_leak_checker_LDADD = libtest-runner.la
+
+EXTRA_DIST += tests/scanner-test.sh			\
+	tests/data/example.xml				\
+	tests/data/example-client.h			\
+	tests/data/example-server.h			\
+	tests/data/example-code.c			\
+	tests/data/small.xml				\
+	tests/data/small-code.c				\
+	tests/data/small-client.h			\
+	tests/data/small-server.h			\
+	tests/data/small-code-core.c			\
+	tests/data/small-client-core.h			\
+	tests/data/small-server-core.h
+
+tests/scanner-test.sh: $(top_builddir)/wayland-scanner
+
+clean-local:
+	-rm -rf tests/output
+
 endif #ENABLE_LIBRARIES
diff --git a/tests/scanner-test.sh b/tests/scanner-test.sh
new file mode 100755
index 0000000..7854b86
--- /dev/null
+++ b/tests/scanner-test.sh
@@ -0,0 +1,51 @@
+#!/bin/sh
+
+echo "srcdir: $srcdir"
+echo "scanner: $WAYLAND_SCANNER"
+echo "test_data_dir: $TEST_DATA_DIR"
+echo "test_output_dir: $TEST_OUTPUT_DIR"
+echo "pwd: $PWD"
+echo "sed: $SED"
+
+RETCODE=0
+
+hard_fail() {
+	echo "$@" "ERROR"
+	exit 99
+}
+
+fail() {
+	echo "$@" "FAIL"
+	RETCODE=1
+}
+
+mkdir -p "$TEST_OUTPUT_DIR" || hard_fail "setup"
+
+generate_and_compare() {
+	echo
+	echo "Testing $1 generation: $2 -> $3"
+
+	"$WAYLAND_SCANNER" $1 < "$TEST_DATA_DIR/$2" > "$TEST_OUTPUT_DIR/$3" || \
+		hard_fail "$2 -> $3"
+
+	"$SED" -i -e 's/Generated by wayland-scanner [0-9.]*/SCANNER TEST/' \
+		"$TEST_OUTPUT_DIR/$3" || hard_fail "$2 -> $3"
+
+	diff -q "$TEST_DATA_DIR/$3" "$TEST_OUTPUT_DIR/$3" && \
+		echo "$2 -> $3 PASS" || \
+		fail "$2 -> $3"
+}
+
+generate_and_compare "code" "example.xml" "example-code.c"
+generate_and_compare "client-header" "example.xml" "example-client.h"
+generate_and_compare "server-header" "example.xml" "example-server.h"
+
+generate_and_compare "code" "small.xml" "small-code.c"
+generate_and_compare "client-header" "small.xml" "small-client.h"
+generate_and_compare "server-header" "small.xml" "small-server.h"
+
+generate_and_compare "-c code" "small.xml" "small-code-core.c"
+generate_and_compare "-c client-header" "small.xml" "small-client-core.h"
+generate_and_compare "-c server-header" "small.xml" "small-server-core.h"
+
+exit $RETCODE
-- 
2.7.3