[igt-dev] [PATCH i-g-t] runner: Implement --dry-run

[Martin Peres]

On 22/01/2019 12:47, Petri Latvala wrote:
> Actually implement --dry-run to not execute tests. With dry-run
> active, attempting to execute will figure out the list of things to
> execute, serialize them along with settings, and stop. This will be
> useful for CI that wants to post-mortem on failed test rounds to
> generate a list of tests that should have been executed and produce
> json result files (full of 'notrun') for proper statistics.
> 
> Signed-off-by: Petri Latvala <petri.latvala at intel.com>
> Cc: Andi Shyti <andi.shyti at intel.com>
> Cc: Martin Peres <martin.peres at linux.intel.com>
> Cc: Arkadiusz Hiler <arkadiusz.hiler at intel.com>
> Cc: Tomi Sarvela <tomi.p.sarvela at intel.com>

Looks clean and very well tested! Thanks!

Reviewed-by: Martin Peres <martin.peres at linux.intel.com>

> ---
>  runner/executor.c     |  7 +++++
>  runner/executor.h     |  1 +
>  runner/runner_tests.c | 70 +++++++++++++++++++++++++++++++++++++++++++
>  3 files changed, 78 insertions(+)
> 
> diff --git a/runner/executor.c b/runner/executor.c
> index 54c530b7..3f1eb010 100644
> --- a/runner/executor.c
> +++ b/runner/executor.c
> @@ -1153,6 +1153,8 @@ bool initialize_execute_state(struct execute_state *state,
>  
>  	init_time_left(state, settings);
>  
> +	state->dry = settings->dry_run;
> +
>  	return true;
>  }
>  
> @@ -1204,6 +1206,11 @@ bool execute(struct execute_state *state,
>  	double time_spent = 0.0;
>  	bool status = true;
>  
> +	if (state->dry) {
> +		printf("Dry run, not executing. Invoke igt_resume if you want to execute.\n");
> +		return false;
> +	}
> +
>  	if ((resdirfd = open(settings->results_path, O_DIRECTORY | O_RDONLY)) < 0) {
>  		/* Initialize state should have done this */
>  		fprintf(stderr, "Error: Failure opening results path %s\n",
> diff --git a/runner/executor.h b/runner/executor.h
> index 12883f15..6c83e649 100644
> --- a/runner/executor.h
> +++ b/runner/executor.h
> @@ -14,6 +14,7 @@ struct execute_state
>  	 */
>  	double time_left;
>  	double resuming;
> +	bool dry;
>  };
>  
>  enum {
> diff --git a/runner/runner_tests.c b/runner/runner_tests.c
> index 6213b8e7..0965612a 100644
> --- a/runner/runner_tests.c
> +++ b/runner/runner_tests.c
> @@ -742,6 +742,76 @@ igt_main
>  		}
>  	}
>  
> +	igt_subtest_group {
> +		char dirname[] = "tmpdirXXXXXX";
> +		struct job_list list;
> +		int dirfd = -1, subdirfd = -1, fd = -1;
> +
> +		igt_fixture {
> +			init_job_list(&list);
> +			igt_require(mkdtemp(dirname) != NULL);
> +			rmdir(dirname);
> +		}
> +
> +		igt_subtest("dry-run-option") {
> +			struct execute_state state;
> +			const char *argv[] = { "runner",
> +					       "--dry-run",
> +					       testdatadir,
> +					       dirname,
> +			};
> +
> +			igt_assert(parse_options(ARRAY_SIZE(argv), (char**)argv, &settings));
> +			igt_assert(create_job_list(&list, &settings));
> +
> +			igt_assert(initialize_execute_state(&state, &settings, &list));
> +			igt_assert_eq(state.next, 0);
> +			igt_assert(state.dry);
> +			igt_assert_eq(list.size, 5);
> +
> +			igt_assert_f((dirfd = open(dirname, O_DIRECTORY | O_RDONLY)) >= 0,
> +				     "Dry run initialization didn't create the results directory.\n");
> +
> +			/* Execute from just initialize_execute_state should fail */
> +			igt_assert(!execute(&state, &settings, &list));
> +			igt_assert(openat(dirfd, "0", O_DIRECTORY | O_RDONLY) < 0);
> +			igt_assert_f((fd = openat(dirfd, "metadata.txt", O_RDONLY)) >= 0,
> +				     "Dry run initialization didn't serialize settings.\n");
> +			close(fd);
> +			igt_assert_f((fd = openat(dirfd, "joblist.txt", O_RDONLY)) >= 0,
> +				     "Dry run initialization didn't serialize the job list.\n");
> +			close(fd);
> +			igt_assert_f((fd = openat(dirfd, "uname.txt", O_RDONLY)) < 0,
> +				     "Dry run initialization created uname.txt.\n");
> +
> +			igt_assert(initialize_execute_state_from_resume(dirfd, &state, &settings, &list));
> +			igt_assert_eq(state.next, 0);
> +			igt_assert(!state.dry);
> +			igt_assert_eq(list.size, 5);
> +			/* initialize_execute_state_from_resume() closes the dirfd */
> +			igt_assert_f((dirfd = open(dirname, O_DIRECTORY | O_RDONLY)) >= 0,
> +				     "Dry run resume somehow deleted the results directory.\n");
> +
> +			/* Execute from resume should work */
> +			igt_assert(execute(&state, &settings, &list));
> +			igt_assert_f((fd = openat(dirfd, "uname.txt", O_RDONLY)) >= 0,
> +				     "Dry run resume didn't create uname.txt.\n");
> +			close(fd);
> +			igt_assert_f((subdirfd = openat(dirfd, "0", O_DIRECTORY | O_RDONLY)) >= 0,
> +				     "Dry run resume didn't create result directory.\n");
> +			igt_assert_f((fd = openat(subdirfd, "journal.txt", O_RDONLY)) >= 0,
> +				     "Dry run resume didn't create a journal.\n");
> +		}
> +
> +		igt_fixture {
> +			close(fd);
> +			close(dirfd);
> +			close(subdirfd);
> +			clear_directory(dirname);
> +			free_job_list(&list);
> +		}
> +	}
> +
>  	igt_subtest_group {
>  		char dirname[] = "tmpdirXXXXXX";
>  		struct job_list list;
>