libinput parallel test suite attempts and results

[Bryce Harrington]

On Tue, May 19, 2015 at 12:04:34PM +1000, Peter Hutterer wrote:
> This is mostly just a comment on a failed attempt, in case someone is
> tempted to try it or has any feedback otherwise.
> 
> The libinput tests suite has a couple of binaries (test-touchpad,
> test-pointer, etc.). The .NOTPARALLEL directive makes sure they're run
> serialised during make check. Because of the various timeouts we test again,
> a test run now takes nearly 3 minutes to complete. Which is annoying.

Could you do something like swap in a synthetic clock where they're
doing time dependent stuff, that you can advance at will to trigger the
timeouts?
 
> Most tests could in theory run parallel because we only ever listen to one
> or two devices per test and ignore the rest. Alas, I have yet to get a
> single parallel run to succeed (and even that's with only running
> path-backend tests in parallel, forget about the udev tests altogether).
> 
> The Makefile.am changes are trivial, see below, but even at make -j2 usually
> one or two tests fail (the wheel-only one most often since it needs custom
> udev rules), at higher -j levels you can forget about it altogether.
> 
> So without some larger rewrite and figuring out how uinput + udev becomes
> reliable when you're hammering hundreds of devices per second against it the
> test suite will have to stay serial. 
> 
> Any ideas welcome though.
> 
> Cheers,
>    Peter
> 
> diff --git a/test/Makefile.am b/test/Makefile.am
> index 46959fd..c2f0fde 100644
> --- a/test/Makefile.am
> +++ b/test/Makefile.am
> @@ -38,18 +38,31 @@ liblitest_la_SOURCES = \
>  	litest-vmware-virtual-usb-mouse.c \
>  	litest.c
>  liblitest_la_LIBADD = $(top_builddir)/src/libinput-util.la
> +liblitest_la_CFLAGS = $(AM_CFLAGS)
> +if HAVE_LIBUNWIND
> +liblitest_la_LIBADD += $(LIBUNWIND_LIBS) -ldl
> +liblitest_la_CFLAGS += $(LIBUNWIND_CFLAGS)
> +endif
> +
> +parallel_tests = \
> +	touchpad.test \
> +	device.test \
> +	pointer.test \
> +	touch.test \
> +	trackpoint.test \
> +	path.test \
> +	log.test \
> +	misc.test \
> +	keyboard.test \
> +	litest-selftest.test
> +
> +serial_tests = \
> +	udev.test
>  
>  run_tests = \
> -	test-touchpad \
> -	test-device \
> -	test-pointer \
> -	test-touch \
> -	test-trackpoint \
> -	test-udev \
> -	test-path \
> -	test-log \
> -	test-misc \
> -	test-keyboard
> +	$(parallel_tests) \
> +	$(serial_tests)
> +
>  build_tests = \
>  	test-build-cxx \
>  	test-build-linker \
> @@ -60,47 +73,52 @@ noinst_PROGRAMS = $(build_tests) $(run_tests)
>  noinst_SCRIPTS = symbols-leak-test
>  TESTS = $(run_tests) symbols-leak-test
>  
> -.NOTPARALLEL:
> +udev_test_SOURCES = udev.c
> +udev_test_LDADD = $(TEST_LIBS)
> +udev_test_LDFLAGS = -no-install
>  
> -test_udev_SOURCES = udev.c
> -test_udev_LDADD = $(TEST_LIBS)
> -test_udev_LDFLAGS = -no-install
> +path_test_SOURCES = path.c
> +path_test_LDADD = $(TEST_LIBS)
> +path_test_LDFLAGS = -no-install
>  
> -test_path_SOURCES = path.c
> -test_path_LDADD = $(TEST_LIBS)
> -test_path_LDFLAGS = -no-install
> +pointer_test_SOURCES = pointer.c
> +pointer_test_LDADD = $(TEST_LIBS)
> +pointer_test_LDFLAGS = -no-install
>  
> -test_pointer_SOURCES = pointer.c
> -test_pointer_LDADD = $(TEST_LIBS)
> -test_pointer_LDFLAGS = -no-install
> +touch_test_SOURCES = touch.c
> +touch_test_LDADD = $(TEST_LIBS)
> +touch_test_LDFLAGS = -no-install
>  
> -test_touch_SOURCES = touch.c
> -test_touch_LDADD = $(TEST_LIBS)
> -test_touch_LDFLAGS = -no-install
> +log_test_SOURCES = log.c
> +log_test_LDADD = $(TEST_LIBS)
> +log_test_LDFLAGS = -no-install
>  
> -test_log_SOURCES = log.c
> -test_log_LDADD = $(TEST_LIBS)
> -test_log_LDFLAGS = -no-install
> +touchpad_test_SOURCES = touchpad.c
> +touchpad_test_LDADD = $(TEST_LIBS)
> +touchpad_test_LDFLAGS = -no-install
>  
> -test_touchpad_SOURCES = touchpad.c
> -test_touchpad_LDADD = $(TEST_LIBS)
> -test_touchpad_LDFLAGS = -no-install
> +trackpoint_test_SOURCES = trackpoint.c
> +trackpoint_test_LDADD = $(TEST_LIBS)
> +trackpoint_test_LDFLAGS = -no-install
>  
> -test_trackpoint_SOURCES = trackpoint.c
> -test_trackpoint_LDADD = $(TEST_LIBS)
> -test_trackpoint_LDFLAGS = -no-install
> +misc_test_SOURCES = misc.c
> +misc_test_LDADD = $(TEST_LIBS)
> +misc_test_LDFLAGS = -no-install
>  
> -test_misc_SOURCES = misc.c
> -test_misc_LDADD = $(TEST_LIBS)
> -test_misc_LDFLAGS = -no-install
> +keyboard_test_SOURCES = keyboard.c
> +keyboard_test_LDADD = $(TEST_LIBS)
> +keyboard_test_LDFLAGS = -no-install
>  
> -test_keyboard_SOURCES = keyboard.c
> -test_keyboard_LDADD = $(TEST_LIBS)
> -test_keyboard_LDFLAGS = -no-install
> +device_test_SOURCES = device.c
> +device_test_LDADD = $(TEST_LIBS)
> +device_test_LDFLAGS = -no-install
>  
> -test_device_SOURCES = device.c
> -test_device_LDADD = $(TEST_LIBS)
> -test_device_LDFLAGS = -no-install
> +litest_selftest_test_SOURCES = litest-selftest.c litest.c litest-int.h litest.h
> +litest_selftest_test_CFLAGS = -DLITEST_DISABLE_BACKTRACE_LOGGING
> +litest_selftest_test_LDADD = $(TEST_LIBS)
> +litest_selftest_test_LDFLAGS = -no-install
> +
> +$(serial_tests:.test=.log): | $(parallel_tests:.test=.log)
>  
>  # build-test only
>  test_build_pedantic_c99_SOURCES = build-pedantic.c
> @@ -126,8 +144,8 @@ VALGRIND_FLAGS=--leak-check=full \
>  	       --error-exitcode=3 \
>  	       --suppressions=$(srcdir)/valgrind.suppressions
>  
> -valgrind:
> -	$(MAKE) check-TESTS LOG_COMPILER="$(VALGRIND)" LOG_FLAGS="$(VALGRIND_FLAGS)" CK_FORK=no
> +valgrind: | $(serial_tests:.test=.log) $(parallel_tests:.test=.log)
> +	$(MAKE) check-TESTS LOG_COMPILER="$(VALGRIND)" LOG_FLAGS="$(VALGRIND_FLAGS)" CK_FORK=no USING_VALGRIND=yes
>  
>  check: valgrind
>  
> 
> 
> _______________________________________________
> wayland-devel mailing list
> wayland-devel at lists.freedesktop.org
> http://lists.freedesktop.org/mailman/listinfo/wayland-devel