[PATCH weston 6/6] tests: Properly report skipped tests

[Emilio Pozuelo Monfort]

From: Emilio Pozuelo Monfort <emilio.pozuelo at collabora.co.uk>

We were calling exit(0) when tests were skipped, which counted
them as passed instead of skipped. Fix this by properly exiting
with 77 (which is what automake expects for skipped tests) from
the tests themselves, then returning 77 again from weston-test-runner
if all the tests were skipped. Finally the weston-test.so module
catches weston-test-runner's exit code and uses it as an exit code,
which is what automake will see and use.

Signed-off-by: Emilio Pozuelo Monfort <emilio.pozuelo at collabora.co.uk>
---
 tests/weston-test-client-helper.c |  8 +++++---
 tests/weston-test-runner.c        | 41 ++++++++++++++++++++++++++++-----------
 tests/weston-test.c               |  6 ++++++
 3 files changed, 41 insertions(+), 14 deletions(-)

diff --git a/tests/weston-test-client-helper.c b/tests/weston-test-client-helper.c
index 399aa44..186b395 100644
--- a/tests/weston-test-client-helper.c
+++ b/tests/weston-test-client-helper.c
@@ -505,9 +505,11 @@ skip(const char *fmt, ...)
 	vfprintf(stderr, fmt, argp);
 	va_end(argp);
 
-	/* automake tests uses exit code 77, but we don't have a good
-	 * way to make weston exit with that from here. */
-	exit(0);
+	/* automake tests uses exit code 77. weston-test-runner will see
+	 * this and use it, and then weston-test's sigchld handler (in the
+	 * weston process) will use that as an exit status, which is what
+	 * automake will see in the end. */
+	exit(77);
 }
 
 static void
diff --git a/tests/weston-test-runner.c b/tests/weston-test-runner.c
index 4274b39..6307566 100644
--- a/tests/weston-test-runner.c
+++ b/tests/weston-test-runner.c
@@ -32,6 +32,8 @@
 #include <signal.h>
 #include "weston-test-runner.h"
 
+#define SKIP 77
+
 extern const struct weston_test __start_test_section, __stop_test_section;
 
 static const struct weston_test *
@@ -67,6 +69,7 @@ static int
 exec_and_report_test(const struct weston_test *t, void *test_data, int iteration)
 {
 	int success = 0;
+	int skip = 0;
 	int hardfail = 0;
 	siginfo_t info;
 
@@ -91,6 +94,8 @@ exec_and_report_test(const struct weston_test *t, void *test_data, int iteration
 		fprintf(stderr, "exit status %d", info.si_status);
 		if (info.si_status == EXIT_SUCCESS)
 			success = 1;
+		else if (info.si_status == SKIP)
+			skip = 1;
 		break;
 	case CLD_KILLED:
 	case CLD_DUMPED:
@@ -106,7 +111,10 @@ exec_and_report_test(const struct weston_test *t, void *test_data, int iteration
 	if (success && !hardfail) {
 		fprintf(stderr, ", pass.\n");
 		return 1;
-	} else { 
+	} else if (skip) {
+		fprintf(stderr, ", skip.\n");
+		return SKIP;
+	} else {
 		fprintf(stderr, ", fail.\n");
 		return 0;
 	}
@@ -114,14 +122,17 @@ exec_and_report_test(const struct weston_test *t, void *test_data, int iteration
 
 /* Returns number of tests and number of pass / fail in param args */
 static int
-iterate_test(const struct weston_test *t, int *passed)
+iterate_test(const struct weston_test *t, int *passed, int *skipped)
 {
-	int i;
+	int ret, i;
 	void *current_test_data = (void *) t->table_data;
 	for (i = 0; i < t->n_elements; ++i, current_test_data += t->element_size)
 	{
-		if (exec_and_report_test(t, current_test_data, i))
+		ret = exec_and_report_test(t, current_test_data, i);
+		if (ret == 0)
 			++(*passed);
+		else if (ret == SKIP)
+			++(*skipped);
 	}
 
 	return t->n_elements;
@@ -132,6 +143,7 @@ int main(int argc, char *argv[])
 	const struct weston_test *t;
 	int total = 0;
 	int pass = 0;
+	int skip = 0;
 
 	if (argc == 2) {
 		const char *testname = argv[1];
@@ -149,19 +161,26 @@ int main(int argc, char *argv[])
 			exit(EXIT_FAILURE);
 		}
 
-		int number_passed_in_test = 0;
-		total += iterate_test(t, &number_passed_in_test);
+		int number_passed_in_test = 0, number_skipped_in_test = 0;
+		total += iterate_test(t, &number_passed_in_test, &number_skipped_in_test);
 		pass += number_passed_in_test;
+		skip += number_skipped_in_test;
 	} else {
 		for (t = &__start_test_section; t < &__stop_test_section; t++) {
-			int number_passed_in_test = 0;
-			total += iterate_test(t, &number_passed_in_test);
+			int number_passed_in_test = 0, number_skipped_in_test = 0;
+			total += iterate_test(t, &number_passed_in_test, &number_skipped_in_test);
 			pass += number_passed_in_test;
+			skip += number_skipped_in_test;
 		}
 	}
 
-	fprintf(stderr, "%d tests, %d pass, %d fail\n",
-		total, pass, total - pass);
+	fprintf(stderr, "%d tests, %d pass, %d skip, %d fail\n",
+		total, pass, skip, total - pass - skip);
+
+	if (skip == total)
+		return SKIP;
+	else if (pass + skip == total)
+		return EXIT_SUCCESS;
 
-	return pass == total ? EXIT_SUCCESS : EXIT_FAILURE;
+	return EXIT_FAILURE;
 }
diff --git a/tests/weston-test.c b/tests/weston-test.c
index 844059d..35ccaa4 100644
--- a/tests/weston-test.c
+++ b/tests/weston-test.c
@@ -54,6 +54,12 @@ test_client_sigchld(struct weston_process *process, int status)
 	struct weston_test *test =
 		container_of(process, struct weston_test, process);
 
+	/* Chain up from weston-test-runner's exit code so that automake
+	 * knows the exit status and can report e.g. skipped tests. */
+	if (WIFEXITED(status) && WEXITSTATUS(status) != 0)
+		exit(WEXITSTATUS(status));
+
+	/* In case the child aborted or segfaulted... */
 	assert(status == 0);
 
 	wl_display_terminate(test->compositor->wl_display);
-- 
1.8.5.3