libinput parallel test suite attempts and results

[Peter Hutterer]

On Mon, May 18, 2015 at 07:27:01PM -0700, Bryce Harrington wrote:
> On Tue, May 19, 2015 at 12:04:34PM +1000, Peter Hutterer wrote:
> > This is mostly just a comment on a failed attempt, in case someone is
> > tempted to try it or has any feedback otherwise.
> > 
> > The libinput tests suite has a couple of binaries (test-touchpad,
> > test-pointer, etc.). The .NOTPARALLEL directive makes sure they're run
> > serialised during make check. Because of the various timeouts we test again,
> > a test run now takes nearly 3 minutes to complete. Which is annoying.
> 
> Could you do something like swap in a synthetic clock where they're
> doing time dependent stuff, that you can advance at will to trigger the
> timeouts?

hmm, interesting idea. theoretically yes, but it could be awkward with
events that are sent through the kernel and thus have the timestamp that the
kernel applies. Short of filtering this on read somehow, I don't know how to
deal with that.

I think it may also solve the wrong problem, even without timeouts the
creation of devices, waiting for udev, etc. takes time. Though now that I
write this, I guess I should run gprof first before I start optimising.

Cheers,
   Peter

> > Most tests could in theory run parallel because we only ever listen to one
> > or two devices per test and ignore the rest. Alas, I have yet to get a
> > single parallel run to succeed (and even that's with only running
> > path-backend tests in parallel, forget about the udev tests altogether).
> > 
> > The Makefile.am changes are trivial, see below, but even at make -j2 usually
> > one or two tests fail (the wheel-only one most often since it needs custom
> > udev rules), at higher -j levels you can forget about it altogether.
> > 
> > So without some larger rewrite and figuring out how uinput + udev becomes
> > reliable when you're hammering hundreds of devices per second against it the
> > test suite will have to stay serial. 
> > 
> > Any ideas welcome though.
> > 
> > Cheers,
> >    Peter
> > 
> > diff --git a/test/Makefile.am b/test/Makefile.am
> > index 46959fd..c2f0fde 100644
> > --- a/test/Makefile.am
> > +++ b/test/Makefile.am
> > @@ -38,18 +38,31 @@ liblitest_la_SOURCES = \
> >  	litest-vmware-virtual-usb-mouse.c \
> >  	litest.c
> >  liblitest_la_LIBADD = $(top_builddir)/src/libinput-util.la
> > +liblitest_la_CFLAGS = $(AM_CFLAGS)
> > +if HAVE_LIBUNWIND
> > +liblitest_la_LIBADD += $(LIBUNWIND_LIBS) -ldl
> > +liblitest_la_CFLAGS += $(LIBUNWIND_CFLAGS)
> > +endif
> > +
> > +parallel_tests = \
> > +	touchpad.test \
> > +	device.test \
> > +	pointer.test \
> > +	touch.test \
> > +	trackpoint.test \
> > +	path.test \
> > +	log.test \
> > +	misc.test \
> > +	keyboard.test \
> > +	litest-selftest.test
> > +
> > +serial_tests = \
> > +	udev.test
> >  
> >  run_tests = \
> > -	test-touchpad \
> > -	test-device \
> > -	test-pointer \
> > -	test-touch \
> > -	test-trackpoint \
> > -	test-udev \
> > -	test-path \
> > -	test-log \
> > -	test-misc \
> > -	test-keyboard
> > +	$(parallel_tests) \
> > +	$(serial_tests)
> > +
> >  build_tests = \
> >  	test-build-cxx \
> >  	test-build-linker \
> > @@ -60,47 +73,52 @@ noinst_PROGRAMS = $(build_tests) $(run_tests)
> >  noinst_SCRIPTS = symbols-leak-test
> >  TESTS = $(run_tests) symbols-leak-test
> >  
> > -.NOTPARALLEL:
> > +udev_test_SOURCES = udev.c
> > +udev_test_LDADD = $(TEST_LIBS)
> > +udev_test_LDFLAGS = -no-install
> >  
> > -test_udev_SOURCES = udev.c
> > -test_udev_LDADD = $(TEST_LIBS)
> > -test_udev_LDFLAGS = -no-install
> > +path_test_SOURCES = path.c
> > +path_test_LDADD = $(TEST_LIBS)
> > +path_test_LDFLAGS = -no-install
> >  
> > -test_path_SOURCES = path.c
> > -test_path_LDADD = $(TEST_LIBS)
> > -test_path_LDFLAGS = -no-install
> > +pointer_test_SOURCES = pointer.c
> > +pointer_test_LDADD = $(TEST_LIBS)
> > +pointer_test_LDFLAGS = -no-install
> >  
> > -test_pointer_SOURCES = pointer.c
> > -test_pointer_LDADD = $(TEST_LIBS)
> > -test_pointer_LDFLAGS = -no-install
> > +touch_test_SOURCES = touch.c
> > +touch_test_LDADD = $(TEST_LIBS)
> > +touch_test_LDFLAGS = -no-install
> >  
> > -test_touch_SOURCES = touch.c
> > -test_touch_LDADD = $(TEST_LIBS)
> > -test_touch_LDFLAGS = -no-install
> > +log_test_SOURCES = log.c
> > +log_test_LDADD = $(TEST_LIBS)
> > +log_test_LDFLAGS = -no-install
> >  
> > -test_log_SOURCES = log.c
> > -test_log_LDADD = $(TEST_LIBS)
> > -test_log_LDFLAGS = -no-install
> > +touchpad_test_SOURCES = touchpad.c
> > +touchpad_test_LDADD = $(TEST_LIBS)
> > +touchpad_test_LDFLAGS = -no-install
> >  
> > -test_touchpad_SOURCES = touchpad.c
> > -test_touchpad_LDADD = $(TEST_LIBS)
> > -test_touchpad_LDFLAGS = -no-install
> > +trackpoint_test_SOURCES = trackpoint.c
> > +trackpoint_test_LDADD = $(TEST_LIBS)
> > +trackpoint_test_LDFLAGS = -no-install
> >  
> > -test_trackpoint_SOURCES = trackpoint.c
> > -test_trackpoint_LDADD = $(TEST_LIBS)
> > -test_trackpoint_LDFLAGS = -no-install
> > +misc_test_SOURCES = misc.c
> > +misc_test_LDADD = $(TEST_LIBS)
> > +misc_test_LDFLAGS = -no-install
> >  
> > -test_misc_SOURCES = misc.c
> > -test_misc_LDADD = $(TEST_LIBS)
> > -test_misc_LDFLAGS = -no-install
> > +keyboard_test_SOURCES = keyboard.c
> > +keyboard_test_LDADD = $(TEST_LIBS)
> > +keyboard_test_LDFLAGS = -no-install
> >  
> > -test_keyboard_SOURCES = keyboard.c
> > -test_keyboard_LDADD = $(TEST_LIBS)
> > -test_keyboard_LDFLAGS = -no-install
> > +device_test_SOURCES = device.c
> > +device_test_LDADD = $(TEST_LIBS)
> > +device_test_LDFLAGS = -no-install
> >  
> > -test_device_SOURCES = device.c
> > -test_device_LDADD = $(TEST_LIBS)
> > -test_device_LDFLAGS = -no-install
> > +litest_selftest_test_SOURCES = litest-selftest.c litest.c litest-int.h litest.h
> > +litest_selftest_test_CFLAGS = -DLITEST_DISABLE_BACKTRACE_LOGGING
> > +litest_selftest_test_LDADD = $(TEST_LIBS)
> > +litest_selftest_test_LDFLAGS = -no-install
> > +
> > +$(serial_tests:.test=.log): | $(parallel_tests:.test=.log)
> >  
> >  # build-test only
> >  test_build_pedantic_c99_SOURCES = build-pedantic.c
> > @@ -126,8 +144,8 @@ VALGRIND_FLAGS=--leak-check=full \
> >  	       --error-exitcode=3 \
> >  	       --suppressions=$(srcdir)/valgrind.suppressions
> >  
> > -valgrind:
> > -	$(MAKE) check-TESTS LOG_COMPILER="$(VALGRIND)" LOG_FLAGS="$(VALGRIND_FLAGS)" CK_FORK=no
> > +valgrind: | $(serial_tests:.test=.log) $(parallel_tests:.test=.log)
> > +	$(MAKE) check-TESTS LOG_COMPILER="$(VALGRIND)" LOG_FLAGS="$(VALGRIND_FLAGS)" CK_FORK=no USING_VALGRIND=yes
> >  
> >  check: valgrind
> >  
> > 
> > 
> > _______________________________________________
> > wayland-devel mailing list
> > wayland-devel at lists.freedesktop.org
> > http://lists.freedesktop.org/mailman/listinfo/wayland-devel