libinput parallel test suite attempts and results

[Peter Hutterer]

This is mostly just a comment on a failed attempt, in case someone is
tempted to try it or has any feedback otherwise.

The libinput tests suite has a couple of binaries (test-touchpad,
test-pointer, etc.). The .NOTPARALLEL directive makes sure they're run
serialised during make check. Because of the various timeouts we test again,
a test run now takes nearly 3 minutes to complete. Which is annoying.

Most tests could in theory run parallel because we only ever listen to one
or two devices per test and ignore the rest. Alas, I have yet to get a
single parallel run to succeed (and even that's with only running
path-backend tests in parallel, forget about the udev tests altogether).

The Makefile.am changes are trivial, see below, but even at make -j2 usually
one or two tests fail (the wheel-only one most often since it needs custom
udev rules), at higher -j levels you can forget about it altogether.

So without some larger rewrite and figuring out how uinput + udev becomes
reliable when you're hammering hundreds of devices per second against it the
test suite will have to stay serial. 

Any ideas welcome though.

Cheers,
   Peter

diff --git a/test/Makefile.am b/test/Makefile.am
index 46959fd..c2f0fde 100644
--- a/test/Makefile.am
+++ b/test/Makefile.am
@@ -38,18 +38,31 @@ liblitest_la_SOURCES = \
 	litest-vmware-virtual-usb-mouse.c \
 	litest.c
 liblitest_la_LIBADD = $(top_builddir)/src/libinput-util.la
+liblitest_la_CFLAGS = $(AM_CFLAGS)
+if HAVE_LIBUNWIND
+liblitest_la_LIBADD += $(LIBUNWIND_LIBS) -ldl
+liblitest_la_CFLAGS += $(LIBUNWIND_CFLAGS)
+endif
+
+parallel_tests = \
+	touchpad.test \
+	device.test \
+	pointer.test \
+	touch.test \
+	trackpoint.test \
+	path.test \
+	log.test \
+	misc.test \
+	keyboard.test \
+	litest-selftest.test
+
+serial_tests = \
+	udev.test
 
 run_tests = \
-	test-touchpad \
-	test-device \
-	test-pointer \
-	test-touch \
-	test-trackpoint \
-	test-udev \
-	test-path \
-	test-log \
-	test-misc \
-	test-keyboard
+	$(parallel_tests) \
+	$(serial_tests)
+
 build_tests = \
 	test-build-cxx \
 	test-build-linker \
@@ -60,47 +73,52 @@ noinst_PROGRAMS = $(build_tests) $(run_tests)
 noinst_SCRIPTS = symbols-leak-test
 TESTS = $(run_tests) symbols-leak-test
 
-.NOTPARALLEL:
+udev_test_SOURCES = udev.c
+udev_test_LDADD = $(TEST_LIBS)
+udev_test_LDFLAGS = -no-install
 
-test_udev_SOURCES = udev.c
-test_udev_LDADD = $(TEST_LIBS)
-test_udev_LDFLAGS = -no-install
+path_test_SOURCES = path.c
+path_test_LDADD = $(TEST_LIBS)
+path_test_LDFLAGS = -no-install
 
-test_path_SOURCES = path.c
-test_path_LDADD = $(TEST_LIBS)
-test_path_LDFLAGS = -no-install
+pointer_test_SOURCES = pointer.c
+pointer_test_LDADD = $(TEST_LIBS)
+pointer_test_LDFLAGS = -no-install
 
-test_pointer_SOURCES = pointer.c
-test_pointer_LDADD = $(TEST_LIBS)
-test_pointer_LDFLAGS = -no-install
+touch_test_SOURCES = touch.c
+touch_test_LDADD = $(TEST_LIBS)
+touch_test_LDFLAGS = -no-install
 
-test_touch_SOURCES = touch.c
-test_touch_LDADD = $(TEST_LIBS)
-test_touch_LDFLAGS = -no-install
+log_test_SOURCES = log.c
+log_test_LDADD = $(TEST_LIBS)
+log_test_LDFLAGS = -no-install
 
-test_log_SOURCES = log.c
-test_log_LDADD = $(TEST_LIBS)
-test_log_LDFLAGS = -no-install
+touchpad_test_SOURCES = touchpad.c
+touchpad_test_LDADD = $(TEST_LIBS)
+touchpad_test_LDFLAGS = -no-install
 
-test_touchpad_SOURCES = touchpad.c
-test_touchpad_LDADD = $(TEST_LIBS)
-test_touchpad_LDFLAGS = -no-install
+trackpoint_test_SOURCES = trackpoint.c
+trackpoint_test_LDADD = $(TEST_LIBS)
+trackpoint_test_LDFLAGS = -no-install
 
-test_trackpoint_SOURCES = trackpoint.c
-test_trackpoint_LDADD = $(TEST_LIBS)
-test_trackpoint_LDFLAGS = -no-install
+misc_test_SOURCES = misc.c
+misc_test_LDADD = $(TEST_LIBS)
+misc_test_LDFLAGS = -no-install
 
-test_misc_SOURCES = misc.c
-test_misc_LDADD = $(TEST_LIBS)
-test_misc_LDFLAGS = -no-install
+keyboard_test_SOURCES = keyboard.c
+keyboard_test_LDADD = $(TEST_LIBS)
+keyboard_test_LDFLAGS = -no-install
 
-test_keyboard_SOURCES = keyboard.c
-test_keyboard_LDADD = $(TEST_LIBS)
-test_keyboard_LDFLAGS = -no-install
+device_test_SOURCES = device.c
+device_test_LDADD = $(TEST_LIBS)
+device_test_LDFLAGS = -no-install
 
-test_device_SOURCES = device.c
-test_device_LDADD = $(TEST_LIBS)
-test_device_LDFLAGS = -no-install
+litest_selftest_test_SOURCES = litest-selftest.c litest.c litest-int.h litest.h
+litest_selftest_test_CFLAGS = -DLITEST_DISABLE_BACKTRACE_LOGGING
+litest_selftest_test_LDADD = $(TEST_LIBS)
+litest_selftest_test_LDFLAGS = -no-install
+
+$(serial_tests:.test=.log): | $(parallel_tests:.test=.log)
 
 # build-test only
 test_build_pedantic_c99_SOURCES = build-pedantic.c
@@ -126,8 +144,8 @@ VALGRIND_FLAGS=--leak-check=full \
 	       --error-exitcode=3 \
 	       --suppressions=$(srcdir)/valgrind.suppressions
 
-valgrind:
-	$(MAKE) check-TESTS LOG_COMPILER="$(VALGRIND)" LOG_FLAGS="$(VALGRIND_FLAGS)" CK_FORK=no
+valgrind: | $(serial_tests:.test=.log) $(parallel_tests:.test=.log)
+	$(MAKE) check-TESTS LOG_COMPILER="$(VALGRIND)" LOG_FLAGS="$(VALGRIND_FLAGS)" CK_FORK=no USING_VALGRIND=yes
 
 check: valgrind