[PATCH v1 weston 09/11] tests: Add support for comparing output against reference images

[Pekka Paalanen]

On Wed, 19 Nov 2014 15:06:24 -0800
Bryce Harrington <bryce at osg.samsung.com> wrote:

> Steal Cairo's files_equal() routine to do byte comparison of the rendered
> files.  Note that since the clock time will change run to run we can
> only compare against the first frame (which will be black).

Not even the first frame is actually guaranteed to be black, I think.
It's just luck at this point.

> Signed-off-by: Bryce Harrington <bryce at osg.samsung.com>
> ---
>  tests/fadein-test.c               | 31 +++++++++++++++++++++++++++----
>  tests/weston-test-client-helper.c | 33 +++++++++++++++++++++++++++++++++
>  tests/weston-test-client-helper.h |  5 +++++
>  3 files changed, 65 insertions(+), 4 deletions(-)
> 
> diff --git a/tests/fadein-test.c b/tests/fadein-test.c
> index a6c284f..965bb0a 100644
> --- a/tests/fadein-test.c
> +++ b/tests/fadein-test.c
> @@ -24,27 +24,40 @@
>  
>  #include <unistd.h>
>  #include <stdio.h>
> +#include <stdbool.h>
>  
>  #include "weston-test-client-helper.h"
>  
>  char *server_parameters="--use-pixman --width=320 --height=240";
>  
>  static char*
> -output_filename(const char* basename) {
> +output_filename(const char* basename, int head) {
>  	static const char *path = "./";
>  	char *filename;
>  
> -        if (asprintf(&filename, "%s%s", path, basename) < 0)
> +        if (asprintf(&filename, "%s%s-%d.png", path, basename, head) < 0)
>  		filename = NULL;
>  
>  	return filename;
>  }
>  
> +static char*
> +reference_filename(const char* basename, int head) {
> +        static const char *path = "./tests/reference/";
> +        char *filename;
> +
> +        if (asprintf(&filename, "%s%s-%d.png", path, basename, head) < 0)
> +                filename = NULL;
> +
> +        return filename;
> +}
> +
>  TEST(fadein)
>  {
>  	struct client *client;
>  	char basename[32];
>  	char *out_path;
> +	char *ref_path;
>  	int i;
>  
>  	client = client_create(100, 100, 100, 100);
> @@ -52,11 +65,21 @@ TEST(fadein)
>  
>  	for (i = 0; i < 6; i++) {
>  		snprintf(basename, sizeof basename, "fadein-%02d", i);
> -		out_path = output_filename(basename);
> +		// FIXME: Iterate over all heads
> +		out_path = output_filename(basename, 0);
> +		ref_path = reference_filename(basename, 0);
>  
> -		wl_test_record_screenshot(client->test->wl_test, out_path);
> +		// FIXME: Would be preferred to pass in out_path rather than basename here...
> +		wl_test_record_screenshot(client->test->wl_test, basename);
>  		client_roundtrip(client);
> +		if (i == 0) {
> +			if (files_equal(out_path, ref_path))
> +				printf("%s is correct\n", out_path);
> +			else
> +				printf("%s doesn't match reference %s\n", out_path, ref_path);
> +		}
>  		free (out_path);
> +		free (ref_path);
>  
>  		usleep(250000);
>  	}
> diff --git a/tests/weston-test-client-helper.c b/tests/weston-test-client-helper.c
> index 79097fa..72834e4 100644
> --- a/tests/weston-test-client-helper.c
> +++ b/tests/weston-test-client-helper.c
> @@ -22,6 +22,7 @@
>  
>  #include <config.h>
>  
> +#include <stdbool.h>
>  #include <stdlib.h>
>  #include <stdio.h>
>  #include <string.h>
> @@ -624,3 +625,35 @@ client_create(int x, int y, int width, int height)
>  
>  	return client;
>  }
> +
> +bool
> +files_equal(const char *test_filename, const char* ref_filename)
> +{
> +        FILE *test, *ref;
> +        int t, p;
> +
> +        if (test_filename == NULL || ref_filename == NULL)
> +                return false;
> +
> +        test = fopen (test_filename, "rb");
> +        if (test == NULL)
> +                return false;
> +
> +        ref = fopen (ref_filename, "rb");
> +        if (ref == NULL) {
> +                fclose (test);
> +                return false;
> +        }
> +
> +        do {
> +                t = getc (test);
> +                p = getc (ref);
> +                if (t != p)
> +                        break;
> +        } while (t != EOF && p != EOF);
> +
> +        fclose (test);
> +        fclose (ref);
> +
> +        return t == p;  /* both EOF */
> +}

Byte by byte comparison on the raw PNG data, I didn't expect that. :-)

Well, this needs a rewrite with wl_buffer screenshots, and there we
naturally get a chance to apply some fuzz for matching as needed.


Thanks,
pq

> diff --git a/tests/weston-test-client-helper.h b/tests/weston-test-client-helper.h
> index 684afc6..20e08b1 100644
> --- a/tests/weston-test-client-helper.h
> +++ b/tests/weston-test-client-helper.h
> @@ -26,6 +26,8 @@
>  #include "config.h"
>  
>  #include <assert.h>
> +#include <stdbool.h>
> +
>  #include "weston-test-runner.h"
>  #include "wayland-test-client-protocol.h"
>  
> @@ -135,4 +137,7 @@ void
>  expect_protocol_error(struct client *client,
>  		      const struct wl_interface *intf, uint32_t code);
>  
> +bool
> +files_equal(const char *file_1, const char *file_2);
> +
>  #endif