[igt-dev] [PATCH v2] runner: add --list-all and --blacklist
Oleg Vasilev
oleg.vasilev at intel.com
Mon Jun 17 11:49:37 UTC 2019
Currently, runner already collects all subtest names into job_list.
--list-all allows to output it to stdout.
--blacklist option takes a filename as an argument and adds all regexes
from that file to the exclusion list.
v2:
- Update exclude/include regex matches for tests without
subtests to be matched with pigtit-like name (Petri)
- Replace relative paths with those formatted with testdatadir (Petri)
- Minor codestyle changes
Cc: Petri Latvala <petri.latvala at intel.com>
Signed-off-by: Oleg Vasilev <oleg.vasilev at intel.com>
---
runner/job_list.c | 27 ++++++-
runner/job_list.h | 1 +
runner/runner.c | 5 ++
runner/runner_tests.c | 18 ++++-
runner/settings.c | 117 +++++++++++++++++++++++-----
runner/settings.h | 1 +
runner/testdata/meson.build | 5 ++
runner/testdata/test-blacklist.txt | 2 +
runner/testdata/test-blacklist2.txt | 2 +
9 files changed, 154 insertions(+), 24 deletions(-)
create mode 100644 runner/testdata/test-blacklist.txt
create mode 100644 runner/testdata/test-blacklist2.txt
diff --git a/runner/job_list.c b/runner/job_list.c
index 4a16742f..e78e0358 100644
--- a/runner/job_list.c
+++ b/runner/job_list.c
@@ -111,11 +111,16 @@ static void add_subtests(struct job_list *job_list, struct settings *settings,
fprintf(stderr, "popen error when executing %s: %s\n", binary, strerror(errno));
} else if (WIFEXITED(s)) {
if (WEXITSTATUS(s) == IGT_EXIT_INVALID) {
+ char piglitname[256];
+ generate_piglit_name(binary, NULL,
+ piglitname, sizeof(piglitname));
+
/* No subtests on this one */
- if (exclude && exclude->size && matches_any(binary, exclude)) {
+ if (exclude && exclude->size &&
+ matches_any(piglitname, exclude)) {
return;
}
- if (!include || !include->size || matches_any(binary, include)) {
+ if (!include || !include->size || matches_any(piglitname, include)) {
add_job_list_entry(job_list, strdup(binary), NULL, 0);
return;
}
@@ -291,6 +296,24 @@ static bool job_list_from_test_list(struct job_list *job_list,
return any;
}
+void list_all_tests(struct job_list* lst)
+{
+ for (size_t test_idx = 0; test_idx < lst->size; ++test_idx){
+ struct job_list_entry* current_entry = lst->entries+test_idx;
+ if (current_entry->subtest_count==0) {
+ printf("igt@%s\n", current_entry->binary);
+ continue;
+ }
+ for (size_t subtest_idx = 0;
+ subtest_idx < current_entry->subtest_count;
+ ++subtest_idx) {
+ char *subtest_name = current_entry->subtests[subtest_idx];
+ printf("igt@%s@%s\n", current_entry->binary, subtest_name);
+ }
+ }
+}
+
+
static char *lowercase(const char *str)
{
char *ret = malloc(strlen(str) + 1);
diff --git a/runner/job_list.h b/runner/job_list.h
index f8bbbddc..cee4bff6 100644
--- a/runner/job_list.h
+++ b/runner/job_list.h
@@ -36,5 +36,6 @@ bool create_job_list(struct job_list *job_list, struct settings *settings);
bool serialize_job_list(struct job_list *job_list, struct settings *settings);
bool read_job_list(struct job_list *job_list, int dirfd);
+void list_all_tests(struct job_list* lst);
#endif
diff --git a/runner/runner.c b/runner/runner.c
index e1a6ccba..4855ad64 100644
--- a/runner/runner.c
+++ b/runner/runner.c
@@ -24,6 +24,11 @@ int main(int argc, char **argv)
return 1;
}
+ if (settings.list_all) {
+ list_all_tests(&job_list);
+ return 0;
+ }
+
if (!initialize_execute_state(&state, &settings, &job_list)) {
return 1;
}
diff --git a/runner/runner_tests.c b/runner/runner_tests.c
index c09cda70..39d4a078 100644
--- a/runner/runner_tests.c
+++ b/runner/runner_tests.c
@@ -378,6 +378,7 @@ igt_main
}
igt_subtest("parse-all-settings") {
+ char blacklist_name[PATH_MAX], blacklist2_name[PATH_MAX];
const char *argv[] = { "runner",
"-n", "foo",
"--abort-on-monitored-error=taint,lockdep",
@@ -388,6 +389,8 @@ igt_main
"-t", "pattern2",
"-x", "xpattern1",
"-x", "xpattern2",
+ "-b", blacklist_name,
+ "--blacklist", blacklist2_name,
"-s",
"-l", "verbose",
"--overwrite",
@@ -401,6 +404,9 @@ igt_main
"path-to-results",
};
+ sprintf(blacklist_name, "%s/test-blacklist.txt", testdatadir);
+ sprintf(blacklist2_name, "%s/test-blacklist2.txt", testdatadir);
+
igt_assert(parse_options(ARRAY_SIZE(argv), (char**)argv, settings));
igt_assert_eq(settings->abort_mask, ABORT_TAINT | ABORT_LOCKDEP);
@@ -410,9 +416,11 @@ igt_main
igt_assert_eq(settings->include_regexes.size, 2);
igt_assert_eqstr(settings->include_regexes.regex_strings[0], "pattern1");
igt_assert_eqstr(settings->include_regexes.regex_strings[1], "pattern2");
- igt_assert_eq(settings->exclude_regexes.size, 2);
+ igt_assert_eq(settings->exclude_regexes.size, 4);
igt_assert_eqstr(settings->exclude_regexes.regex_strings[0], "xpattern1");
igt_assert_eqstr(settings->exclude_regexes.regex_strings[1], "xpattern2");
+ igt_assert_eqstr(settings->exclude_regexes.regex_strings[2], "xpattern3"); /* From blacklist */
+ igt_assert_eqstr(settings->exclude_regexes.regex_strings[3], "xpattern4"); /* From blacklist2 */
igt_assert(settings->sync);
igt_assert_eq(settings->log_level, LOG_LEVEL_VERBOSE);
igt_assert(settings->overwrite);
@@ -426,6 +434,14 @@ igt_main
igt_assert(settings->piglit_style_dmesg);
igt_assert_eq(settings->dmesg_warn_level, 3);
}
+ igt_subtest("parse-list-all") {
+ const char *argv[] = { "runner",
+ "--list-all",
+ "test-root-dir"};
+
+ igt_assert(parse_options(ARRAY_SIZE(argv), (char**)argv, settings));
+ igt_assert_eq(settings->list_all, 1);
+ }
igt_subtest("dmesg-warn-level-inferred") {
const char *argv[] = { "runner",
diff --git a/runner/settings.c b/runner/settings.c
index ad38ae8d..f4b4af90 100644
--- a/runner/settings.c
+++ b/runner/settings.c
@@ -1,5 +1,6 @@
#include "settings.h"
+#include <ctype.h>
#include <errno.h>
#include <fcntl.h>
#include <getopt.h>
@@ -30,6 +31,8 @@ enum {
OPT_MULTIPLE = 'm',
OPT_TIMEOUT = 'c',
OPT_WATCHDOG = 'g',
+ OPT_BLACKLIST = 'b',
+ OPT_LIST_ALL = 'L',
};
static struct {
@@ -117,7 +120,8 @@ static bool parse_abort_conditions(struct settings *settings, const char *optarg
}
static const char *usage_str =
- "usage: runner [options] [test_root] results-path\n\n"
+ "usage: runner [options] [test_root] results-path\n"
+ " or: runner --list-all [options] [test_root] \n\n"
"Options:\n"
" Piglit compatible:\n"
" -h, --help Show this help message and exit\n"
@@ -172,6 +176,10 @@ static const char *usage_str =
" (longer) filter list means the test result should\n"
" change. KERN_NOTICE dmesg level is treated as warn,\n"
" unless overridden with --dmesg-warn-level.\n"
+ " -b, --blacklist FILENAME\n"
+ " Exclude all test matching to regexes from FILENAME\n"
+ " (can be used more than once)\n"
+ " -L, --list-all List all matching subtests instead of running\n"
" [test_root] Directory that contains the IGT tests. The environment\n"
" variable IGT_TEST_ROOT will be used if set, overriding\n"
" this option if given.\n"
@@ -213,6 +221,50 @@ static bool add_regex(struct regex_list *list, char *new)
return true;
}
+static bool parse_blacklist(struct regex_list *exclude_regexes, char *blacklist_filename)
+{
+ FILE *f;
+ char *line = NULL;
+ size_t line_len = 0;
+ bool status;
+
+ if ((f = fopen(blacklist_filename, "r")) == NULL) {
+ fprintf(stderr, "Cannot open blacklist file %s\n", blacklist_filename);
+ return false;
+ }
+ while (1) {
+ size_t str_size = 0, idx = 0;
+
+ if (getline(&line, &line_len, f) == -1) {
+ if (errno == EINTR)
+ continue;
+ else
+ break;
+ }
+
+ while (true) {
+ if(line[idx]=='\n' ||
+ line[idx]=='#' || /* # starts a comment */
+ line[idx]=='\0')
+ break;
+ if (!isspace(line[idx]))
+ str_size = idx+1;
+ idx++;
+ }
+ if (str_size > 0) {
+ char * test_regex = strndup(line, str_size);
+ status = add_regex(exclude_regexes, test_regex);
+ if(!status){
+ break;
+ }
+ }
+ }
+
+ free(line);
+ fclose(f);
+ return status;
+}
+
static void free_regexes(struct regex_list *regexes)
{
size_t i;
@@ -272,6 +324,8 @@ bool parse_options(int argc, char **argv,
{"use-watchdog", no_argument, NULL, OPT_WATCHDOG},
{"piglit-style-dmesg", no_argument, NULL, OPT_PIGLIT_DMESG},
{"dmesg-warn-level", required_argument, NULL, OPT_DMESG_WARN_LEVEL},
+ {"blacklist", required_argument, NULL, OPT_BLACKLIST},
+ {"list-all", no_argument, NULL, OPT_LIST_ALL},
{ 0, 0, 0, 0},
};
@@ -281,7 +335,7 @@ bool parse_options(int argc, char **argv,
settings->dmesg_warn_level = -1;
- while ((c = getopt_long(argc, argv, "hn:dt:x:sl:om", long_options, NULL)) != -1) {
+ while ((c = getopt_long(argc, argv, "hn:dt:x:sl:omb:L", long_options, NULL)) != -1) {
switch (c) {
case OPT_HELP:
usage(NULL, stdout);
@@ -342,6 +396,13 @@ bool parse_options(int argc, char **argv,
case OPT_DMESG_WARN_LEVEL:
settings->dmesg_warn_level = atoi(optarg);
break;
+ case OPT_BLACKLIST:
+ if (!parse_blacklist(&settings->exclude_regexes, absolute_path(optarg)))
+ goto error;
+ break;
+ case OPT_LIST_ALL:
+ settings->list_all = true;
+ break;
case '?':
usage(NULL, stderr);
goto error;
@@ -354,20 +415,39 @@ bool parse_options(int argc, char **argv,
if (settings->dmesg_warn_level < 0)
settings->dmesg_warn_level = 4; /* KERN_WARN */
- switch (argc - optind) {
- case 2:
- settings->test_root = absolute_path(argv[optind]);
- ++optind;
- /* fallthrough */
- case 1:
- settings->results_path = absolute_path(argv[optind]);
- break;
- case 0:
- usage("Results-path missing", stderr);
- goto error;
- default:
- usage("Extra arguments after results-path", stderr);
- goto error;
+ if (settings->list_all) { /* --list-all doesn't require results path */
+ switch (argc - optind) {
+ case 1:
+ settings->test_root = absolute_path(argv[optind]);
+ ++optind;
+ /* fallthrough */
+ case 0:
+ break;
+ default:
+ usage("Too many arguments for --list-all", stderr);
+ goto error;
+ }
+ } else {
+ switch (argc - optind) {
+ case 2:
+ settings->test_root = absolute_path(argv[optind]);
+ ++optind;
+ /* fallthrough */
+ case 1:
+ settings->results_path = absolute_path(argv[optind]);
+ break;
+ case 0:
+ usage("Results-path missing", stderr);
+ goto error;
+ default:
+ usage("Extra arguments after results-path", stderr);
+ goto error;
+ }
+ if (!settings->name) {
+ char *name = strdup(settings->results_path);
+ settings->name = strdup(basename(name));
+ free(name);
+ }
}
if ((env_test_root = getenv("IGT_TEST_ROOT")) != NULL) {
@@ -380,11 +460,6 @@ bool parse_options(int argc, char **argv,
goto error;
}
- if (!settings->name) {
- char *name = strdup(settings->results_path);
- settings->name = strdup(basename(name));
- free(name);
- }
return true;
diff --git a/runner/settings.h b/runner/settings.h
index 0a1ee08d..6dcfa8c5 100644
--- a/runner/settings.h
+++ b/runner/settings.h
@@ -43,6 +43,7 @@ struct settings {
char *results_path;
bool piglit_style_dmesg;
int dmesg_warn_level;
+ bool list_all;
};
/**
diff --git a/runner/testdata/meson.build b/runner/testdata/meson.build
index 011eff8e..2456f82a 100644
--- a/runner/testdata/meson.build
+++ b/runner/testdata/meson.build
@@ -12,6 +12,11 @@ foreach prog : testdata_progs
install : false)
endforeach
+configure_file(input : 'test-blacklist.txt',
+ output : 'test-blacklist.txt', copy : true)
+configure_file(input : 'test-blacklist2.txt',
+ output : 'test-blacklist2.txt', copy : true)
+
testdata_list = custom_target('testdata_testlist',
output : 'test-list.txt',
command : [ gen_testlist, '@OUTPUT@', testdata_progs ],
diff --git a/runner/testdata/test-blacklist.txt b/runner/testdata/test-blacklist.txt
new file mode 100644
index 00000000..6b09ae5c
--- /dev/null
+++ b/runner/testdata/test-blacklist.txt
@@ -0,0 +1,2 @@
+xpattern3 # Comment 1
+# Comment 2
diff --git a/runner/testdata/test-blacklist2.txt b/runner/testdata/test-blacklist2.txt
new file mode 100644
index 00000000..d0f6e612
--- /dev/null
+++ b/runner/testdata/test-blacklist2.txt
@@ -0,0 +1,2 @@
+
+xpattern4
--
2.21.0
More information about the igt-dev
mailing list