[igt-dev] [PATCH v4] runner: add --list-all and --blacklist

Oleg Vasilev oleg.vasilev at intel.com
Tue Jun 18 09:07:25 UTC 2019


Currently, runner already collects all subtest names into job_list.
--list-all allows to output it to stdout.

--blacklist option takes a filename as an argument and adds all regexes
from that file to the exclusion list.

v2:
 - Update exclude/include regex matches for tests without
	subtests to be matched with pigtit-like name (Petri)
 - Replace relative paths with those formatted with testdatadir (Petri)
 - Minor codestyle changes

v3:
 - Print test names in lowercase (Petri)

v4:
 - Replaced custom str_to_lower with generate_piglit_name (Petri)
 - Minor codestyle changes

Cc: Petri Latvala <petri.latvala at intel.com>
Signed-off-by: Oleg Vasilev <oleg.vasilev at intel.com>
---
 runner/job_list.c                   |  34 +++++++-
 runner/job_list.h                   |   1 +
 runner/runner.c                     |   5 ++
 runner/runner_tests.c               |  18 ++++-
 runner/settings.c                   | 121 +++++++++++++++++++++++-----
 runner/settings.h                   |   1 +
 runner/testdata/meson.build         |   5 ++
 runner/testdata/test-blacklist.txt  |   2 +
 runner/testdata/test-blacklist2.txt |   2 +
 9 files changed, 165 insertions(+), 24 deletions(-)
 create mode 100644 runner/testdata/test-blacklist.txt
 create mode 100644 runner/testdata/test-blacklist2.txt

diff --git a/runner/job_list.c b/runner/job_list.c
index 4a16742f..5283fd72 100644
--- a/runner/job_list.c
+++ b/runner/job_list.c
@@ -111,11 +111,17 @@ static void add_subtests(struct job_list *job_list, struct settings *settings,
 		fprintf(stderr, "popen error when executing %s: %s\n", binary, strerror(errno));
 	} else if (WIFEXITED(s)) {
 		if (WEXITSTATUS(s) == IGT_EXIT_INVALID) {
+			char piglitname[256];
+
+			generate_piglit_name(binary, NULL,
+					     piglitname, sizeof(piglitname));
 			/* No subtests on this one */
-			if (exclude && exclude->size && matches_any(binary, exclude)) {
+			if (exclude && exclude->size &&
+			    matches_any(piglitname, exclude)) {
 				return;
 			}
-			if (!include || !include->size || matches_any(binary, include)) {
+			if (!include || !include->size ||
+			    matches_any(piglitname, include)) {
 				add_job_list_entry(job_list, strdup(binary), NULL, 0);
 				return;
 			}
@@ -291,6 +297,30 @@ static bool job_list_from_test_list(struct job_list *job_list,
 	return any;
 }
 
+void list_all_tests(struct job_list *lst)
+{
+	char piglit_name[256];
+
+	for (size_t test_idx = 0; test_idx < lst->size; ++test_idx) {
+		struct job_list_entry *current_entry = lst->entries + test_idx;
+		char *binary = current_entry->binary;
+
+		if (current_entry->subtest_count == 0) {
+			generate_piglit_name(binary, NULL,
+					     piglit_name, sizeof(piglit_name));
+			printf("%s\n", piglit_name);
+			continue;
+		}
+		for (size_t subtest_idx = 0;
+		    subtest_idx < current_entry->subtest_count;
+		    ++subtest_idx) {
+			generate_piglit_name(binary, current_entry->subtests[subtest_idx],
+					     piglit_name, sizeof(piglit_name));
+			printf("%s\n", piglit_name);
+		}
+	}
+}
+
 static char *lowercase(const char *str)
 {
 	char *ret = malloc(strlen(str) + 1);
diff --git a/runner/job_list.h b/runner/job_list.h
index f8bbbddc..39c9b863 100644
--- a/runner/job_list.h
+++ b/runner/job_list.h
@@ -36,5 +36,6 @@ bool create_job_list(struct job_list *job_list, struct settings *settings);
 
 bool serialize_job_list(struct job_list *job_list, struct settings *settings);
 bool read_job_list(struct job_list *job_list, int dirfd);
+void list_all_tests(struct job_list *lst);
 
 #endif
diff --git a/runner/runner.c b/runner/runner.c
index e1a6ccba..4855ad64 100644
--- a/runner/runner.c
+++ b/runner/runner.c
@@ -24,6 +24,11 @@ int main(int argc, char **argv)
 		return 1;
 	}
 
+	if (settings.list_all) {
+		list_all_tests(&job_list);
+		return 0;
+	}
+
 	if (!initialize_execute_state(&state, &settings, &job_list)) {
 		return 1;
 	}
diff --git a/runner/runner_tests.c b/runner/runner_tests.c
index c09cda70..39d4a078 100644
--- a/runner/runner_tests.c
+++ b/runner/runner_tests.c
@@ -378,6 +378,7 @@ igt_main
 	}
 
 	igt_subtest("parse-all-settings") {
+		char blacklist_name[PATH_MAX], blacklist2_name[PATH_MAX];
 		const char *argv[] = { "runner",
 				       "-n", "foo",
 				       "--abort-on-monitored-error=taint,lockdep",
@@ -388,6 +389,8 @@ igt_main
 				       "-t", "pattern2",
 				       "-x", "xpattern1",
 				       "-x", "xpattern2",
+				       "-b", blacklist_name,
+				       "--blacklist", blacklist2_name,
 				       "-s",
 				       "-l", "verbose",
 				       "--overwrite",
@@ -401,6 +404,9 @@ igt_main
 				       "path-to-results",
 		};
 
+		sprintf(blacklist_name, "%s/test-blacklist.txt", testdatadir);
+		sprintf(blacklist2_name, "%s/test-blacklist2.txt", testdatadir);
+
 		igt_assert(parse_options(ARRAY_SIZE(argv), (char**)argv, settings));
 
 		igt_assert_eq(settings->abort_mask, ABORT_TAINT | ABORT_LOCKDEP);
@@ -410,9 +416,11 @@ igt_main
 		igt_assert_eq(settings->include_regexes.size, 2);
 		igt_assert_eqstr(settings->include_regexes.regex_strings[0], "pattern1");
 		igt_assert_eqstr(settings->include_regexes.regex_strings[1], "pattern2");
-		igt_assert_eq(settings->exclude_regexes.size, 2);
+		igt_assert_eq(settings->exclude_regexes.size, 4);
 		igt_assert_eqstr(settings->exclude_regexes.regex_strings[0], "xpattern1");
 		igt_assert_eqstr(settings->exclude_regexes.regex_strings[1], "xpattern2");
+		igt_assert_eqstr(settings->exclude_regexes.regex_strings[2], "xpattern3"); /* From blacklist */
+		igt_assert_eqstr(settings->exclude_regexes.regex_strings[3], "xpattern4"); /* From blacklist2 */
 		igt_assert(settings->sync);
 		igt_assert_eq(settings->log_level, LOG_LEVEL_VERBOSE);
 		igt_assert(settings->overwrite);
@@ -426,6 +434,14 @@ igt_main
 		igt_assert(settings->piglit_style_dmesg);
 		igt_assert_eq(settings->dmesg_warn_level, 3);
 	}
+	igt_subtest("parse-list-all") {
+		const char *argv[] = { "runner",
+				       "--list-all",
+				       "test-root-dir"};
+
+		igt_assert(parse_options(ARRAY_SIZE(argv), (char**)argv, settings));
+		igt_assert_eq(settings->list_all, 1);
+	}
 
 	igt_subtest("dmesg-warn-level-inferred") {
 		const char *argv[] = { "runner",
diff --git a/runner/settings.c b/runner/settings.c
index ad38ae8d..9920e1a6 100644
--- a/runner/settings.c
+++ b/runner/settings.c
@@ -1,5 +1,6 @@
 #include "settings.h"
 
+#include <ctype.h>
 #include <errno.h>
 #include <fcntl.h>
 #include <getopt.h>
@@ -30,6 +31,8 @@ enum {
 	OPT_MULTIPLE = 'm',
 	OPT_TIMEOUT = 'c',
 	OPT_WATCHDOG = 'g',
+	OPT_BLACKLIST = 'b',
+	OPT_LIST_ALL = 'L',
 };
 
 static struct {
@@ -117,7 +120,8 @@ static bool parse_abort_conditions(struct settings *settings, const char *optarg
 }
 
 static const char *usage_str =
-	"usage: runner [options] [test_root] results-path\n\n"
+	"usage: runner [options] [test_root] results-path\n"
+	"   or: runner --list-all [options] [test_root]\n\n"
 	"Options:\n"
 	" Piglit compatible:\n"
 	"  -h, --help            Show this help message and exit\n"
@@ -172,6 +176,10 @@ static const char *usage_str =
 	"                        (longer) filter list means the test result should\n"
 	"                        change. KERN_NOTICE dmesg level is treated as warn,\n"
 	"                        unless overridden with --dmesg-warn-level.\n"
+	"  -b, --blacklist FILENAME\n"
+	"                        Exclude all test matching to regexes from FILENAME\n"
+	"                        (can be used more than once)\n"
+	"  -L, --list-all        List all matching subtests instead of running\n"
 	"  [test_root]           Directory that contains the IGT tests. The environment\n"
 	"                        variable IGT_TEST_ROOT will be used if set, overriding\n"
 	"                        this option if given.\n"
@@ -213,6 +221,51 @@ static bool add_regex(struct regex_list *list, char *new)
 	return true;
 }
 
+static bool parse_blacklist(struct regex_list *exclude_regexes,
+			    char *blacklist_filename)
+{
+	FILE *f;
+	char *line = NULL;
+	size_t line_len = 0;
+	bool status;
+
+	if ((f = fopen(blacklist_filename, "r")) == NULL) {
+		fprintf(stderr, "Cannot open blacklist file %s\n", blacklist_filename);
+		return false;
+	}
+	while (1) {
+		size_t str_size = 0, idx = 0;
+
+		if (getline(&line, &line_len, f) == -1) {
+			if (errno == EINTR)
+				continue;
+			else
+				break;
+		}
+
+		while (true) {
+			if (line[idx] == '\n' ||
+			    line[idx] == '\0' ||
+			    line[idx] == '#')   /* # starts a comment */
+				break;
+			if (!isspace(line[idx]))
+				str_size = idx + 1;
+			idx++;
+		}
+		if (str_size > 0) {
+			char *test_regex = strndup(line, str_size);
+
+			status = add_regex(exclude_regexes, test_regex);
+			if (!status)
+				break;
+		}
+	}
+
+	free(line);
+	fclose(f);
+	return status;
+}
+
 static void free_regexes(struct regex_list *regexes)
 {
 	size_t i;
@@ -272,6 +325,8 @@ bool parse_options(int argc, char **argv,
 		{"use-watchdog", no_argument, NULL, OPT_WATCHDOG},
 		{"piglit-style-dmesg", no_argument, NULL, OPT_PIGLIT_DMESG},
 		{"dmesg-warn-level", required_argument, NULL, OPT_DMESG_WARN_LEVEL},
+		{"blacklist", required_argument, NULL, OPT_BLACKLIST},
+		{"list-all", no_argument, NULL, OPT_LIST_ALL},
 		{ 0, 0, 0, 0},
 	};
 
@@ -281,7 +336,8 @@ bool parse_options(int argc, char **argv,
 
 	settings->dmesg_warn_level = -1;
 
-	while ((c = getopt_long(argc, argv, "hn:dt:x:sl:om", long_options, NULL)) != -1) {
+	while ((c = getopt_long(argc, argv, "hn:dt:x:sl:omb:L",
+				long_options, NULL)) != -1) {
 		switch (c) {
 		case OPT_HELP:
 			usage(NULL, stdout);
@@ -342,6 +398,14 @@ bool parse_options(int argc, char **argv,
 		case OPT_DMESG_WARN_LEVEL:
 			settings->dmesg_warn_level = atoi(optarg);
 			break;
+		case OPT_BLACKLIST:
+			if (!parse_blacklist(&settings->exclude_regexes,
+					     absolute_path(optarg)))
+				goto error;
+			break;
+		case OPT_LIST_ALL:
+			settings->list_all = true;
+			break;
 		case '?':
 			usage(NULL, stderr);
 			goto error;
@@ -354,20 +418,40 @@ bool parse_options(int argc, char **argv,
 	if (settings->dmesg_warn_level < 0)
 		settings->dmesg_warn_level = 4; /* KERN_WARN */
 
-	switch (argc - optind) {
-	case 2:
-		settings->test_root = absolute_path(argv[optind]);
-		++optind;
-		/* fallthrough */
-	case 1:
-		settings->results_path = absolute_path(argv[optind]);
-		break;
-	case 0:
-		usage("Results-path missing", stderr);
-		goto error;
-	default:
-		usage("Extra arguments after results-path", stderr);
-		goto error;
+	if (settings->list_all) { /* --list-all doesn't require results path */
+		switch (argc - optind) {
+		case 1:
+			settings->test_root = absolute_path(argv[optind]);
+			++optind;
+			/* fallthrough */
+		case 0:
+			break;
+		default:
+			usage("Too many arguments for --list-all", stderr);
+			goto error;
+		}
+	} else {
+		switch (argc - optind) {
+		case 2:
+			settings->test_root = absolute_path(argv[optind]);
+			++optind;
+			/* fallthrough */
+		case 1:
+			settings->results_path = absolute_path(argv[optind]);
+			break;
+		case 0:
+			usage("Results-path missing", stderr);
+			goto error;
+		default:
+			usage("Extra arguments after results-path", stderr);
+			goto error;
+		}
+		if (!settings->name) {
+			char *name = strdup(settings->results_path);
+
+			settings->name = strdup(basename(name));
+			free(name);
+		}
 	}
 
 	if ((env_test_root = getenv("IGT_TEST_ROOT")) != NULL) {
@@ -380,11 +464,6 @@ bool parse_options(int argc, char **argv,
 		goto error;
 	}
 
-	if (!settings->name) {
-		char *name = strdup(settings->results_path);
-		settings->name = strdup(basename(name));
-		free(name);
-	}
 
 	return true;
 
diff --git a/runner/settings.h b/runner/settings.h
index 0a1ee08d..6dcfa8c5 100644
--- a/runner/settings.h
+++ b/runner/settings.h
@@ -43,6 +43,7 @@ struct settings {
 	char *results_path;
 	bool piglit_style_dmesg;
 	int dmesg_warn_level;
+	bool list_all;
 };
 
 /**
diff --git a/runner/testdata/meson.build b/runner/testdata/meson.build
index 011eff8e..2456f82a 100644
--- a/runner/testdata/meson.build
+++ b/runner/testdata/meson.build
@@ -12,6 +12,11 @@ foreach prog : testdata_progs
 					   install : false)
 endforeach
 
+configure_file(input : 'test-blacklist.txt',
+	       output : 'test-blacklist.txt', copy : true)
+configure_file(input : 'test-blacklist2.txt',
+	       output : 'test-blacklist2.txt', copy : true)
+
 testdata_list = custom_target('testdata_testlist',
 			      output : 'test-list.txt',
 			      command : [ gen_testlist, '@OUTPUT@', testdata_progs ],
diff --git a/runner/testdata/test-blacklist.txt b/runner/testdata/test-blacklist.txt
new file mode 100644
index 00000000..6b09ae5c
--- /dev/null
+++ b/runner/testdata/test-blacklist.txt
@@ -0,0 +1,2 @@
+xpattern3    # Comment 1
+# Comment 2
diff --git a/runner/testdata/test-blacklist2.txt b/runner/testdata/test-blacklist2.txt
new file mode 100644
index 00000000..d0f6e612
--- /dev/null
+++ b/runner/testdata/test-blacklist2.txt
@@ -0,0 +1,2 @@
+
+xpattern4
-- 
2.21.0



More information about the igt-dev mailing list