[PATCH weston 6/6] tests: Properly report skipped tests

[Emilio Pozuelo Monfort]

On 06/02/14 12:30, Emilio Pozuelo Monfort wrote:
> From: Emilio Pozuelo Monfort <emilio.pozuelo at collabora.co.uk>
> 
> We were calling exit(0) when tests were skipped, which counted
> them as passed instead of skipped. Fix this by properly exiting
> with 77 (which is what automake expects for skipped tests) from
> the tests themselves, then returning 77 again from weston-test-runner
> if all the tests were skipped. Finally the weston-test.so module
> catches weston-test-runner's exit code and uses it as an exit code,
> which is what automake will see and use.
> 
> Signed-off-by: Emilio Pozuelo Monfort <emilio.pozuelo at collabora.co.uk>
> ---
>  tests/weston-test-client-helper.c |  8 +++++---
>  tests/weston-test-runner.c        | 41 ++++++++++++++++++++++++++++-----------
>  tests/weston-test.c               |  6 ++++++
>  3 files changed, 41 insertions(+), 14 deletions(-)
> 
> diff --git a/tests/weston-test-client-helper.c b/tests/weston-test-client-helper.c
> index 399aa44..186b395 100644
> --- a/tests/weston-test-client-helper.c
> +++ b/tests/weston-test-client-helper.c
> @@ -505,9 +505,11 @@ skip(const char *fmt, ...)
>  	vfprintf(stderr, fmt, argp);
>  	va_end(argp);
>  
> -	/* automake tests uses exit code 77, but we don't have a good
> -	 * way to make weston exit with that from here. */
> -	exit(0);
> +	/* automake tests uses exit code 77. weston-test-runner will see
> +	 * this and use it, and then weston-test's sigchld handler (in the
> +	 * weston process) will use that as an exit status, which is what
> +	 * automake will see in the end. */
> +	exit(77);
>  }
>  
>  static void
> diff --git a/tests/weston-test-runner.c b/tests/weston-test-runner.c
> index 4274b39..6307566 100644
> --- a/tests/weston-test-runner.c
> +++ b/tests/weston-test-runner.c
> @@ -32,6 +32,8 @@
>  #include <signal.h>
>  #include "weston-test-runner.h"
>  
> +#define SKIP 77
> +
>  extern const struct weston_test __start_test_section, __stop_test_section;
>  
>  static const struct weston_test *
> @@ -67,6 +69,7 @@ static int
>  exec_and_report_test(const struct weston_test *t, void *test_data, int iteration)
>  {
>  	int success = 0;
> +	int skip = 0;
>  	int hardfail = 0;
>  	siginfo_t info;
>  
> @@ -91,6 +94,8 @@ exec_and_report_test(const struct weston_test *t, void *test_data, int iteration
>  		fprintf(stderr, "exit status %d", info.si_status);
>  		if (info.si_status == EXIT_SUCCESS)
>  			success = 1;
> +		else if (info.si_status == SKIP)
> +			skip = 1;
>  		break;
>  	case CLD_KILLED:
>  	case CLD_DUMPED:
> @@ -106,7 +111,10 @@ exec_and_report_test(const struct weston_test *t, void *test_data, int iteration
>  	if (success && !hardfail) {
>  		fprintf(stderr, ", pass.\n");
>  		return 1;
> -	} else { 
> +	} else if (skip) {
> +		fprintf(stderr, ", skip.\n");
> +		return SKIP;
> +	} else {
>  		fprintf(stderr, ", fail.\n");
>  		return 0;
>  	}
> @@ -114,14 +122,17 @@ exec_and_report_test(const struct weston_test *t, void *test_data, int iteration
>  
>  /* Returns number of tests and number of pass / fail in param args */
>  static int
> -iterate_test(const struct weston_test *t, int *passed)
> +iterate_test(const struct weston_test *t, int *passed, int *skipped)
>  {
> -	int i;
> +	int ret, i;
>  	void *current_test_data = (void *) t->table_data;
>  	for (i = 0; i < t->n_elements; ++i, current_test_data += t->element_size)
>  	{
> -		if (exec_and_report_test(t, current_test_data, i))
> +		ret = exec_and_report_test(t, current_test_data, i);
> +		if (ret == 0)
>  			++(*passed);

Logic here is inversed, the check should be ret == 1. I'll send a fixed patch
tomorrow.

Emilio

> +		else if (ret == SKIP)
> +			++(*skipped);
>  	}
>  
>  	return t->n_elements;
> @@ -132,6 +143,7 @@ int main(int argc, char *argv[])
>  	const struct weston_test *t;
>  	int total = 0;
>  	int pass = 0;
> +	int skip = 0;
>  
>  	if (argc == 2) {
>  		const char *testname = argv[1];
> @@ -149,19 +161,26 @@ int main(int argc, char *argv[])
>  			exit(EXIT_FAILURE);
>  		}
>  
> -		int number_passed_in_test = 0;
> -		total += iterate_test(t, &number_passed_in_test);
> +		int number_passed_in_test = 0, number_skipped_in_test = 0;
> +		total += iterate_test(t, &number_passed_in_test, &number_skipped_in_test);
>  		pass += number_passed_in_test;
> +		skip += number_skipped_in_test;
>  	} else {
>  		for (t = &__start_test_section; t < &__stop_test_section; t++) {
> -			int number_passed_in_test = 0;
> -			total += iterate_test(t, &number_passed_in_test);
> +			int number_passed_in_test = 0, number_skipped_in_test = 0;
> +			total += iterate_test(t, &number_passed_in_test, &number_skipped_in_test);
>  			pass += number_passed_in_test;
> +			skip += number_skipped_in_test;
>  		}
>  	}
>  
> -	fprintf(stderr, "%d tests, %d pass, %d fail\n",
> -		total, pass, total - pass);
> +	fprintf(stderr, "%d tests, %d pass, %d skip, %d fail\n",
> +		total, pass, skip, total - pass - skip);
> +
> +	if (skip == total)
> +		return SKIP;
> +	else if (pass + skip == total)
> +		return EXIT_SUCCESS;
>  
> -	return pass == total ? EXIT_SUCCESS : EXIT_FAILURE;
> +	return EXIT_FAILURE;
>  }
> diff --git a/tests/weston-test.c b/tests/weston-test.c
> index 844059d..35ccaa4 100644
> --- a/tests/weston-test.c
> +++ b/tests/weston-test.c
> @@ -54,6 +54,12 @@ test_client_sigchld(struct weston_process *process, int status)
>  	struct weston_test *test =
>  		container_of(process, struct weston_test, process);
>  
> +	/* Chain up from weston-test-runner's exit code so that automake
> +	 * knows the exit status and can report e.g. skipped tests. */
> +	if (WIFEXITED(status) && WEXITSTATUS(status) != 0)
> +		exit(WEXITSTATUS(status));
> +
> +	/* In case the child aborted or segfaulted... */
>  	assert(status == 0);
>  
>  	wl_display_terminate(test->compositor->wl_display);
>