[Cogl] [PATCH 3/3] tests: Differentiate between known failures and missing requirements
Robert Bragg
robert at sixbynine.org
Mon Nov 19 07:48:24 PST 2012
This looks good to me:
Reviewed-by: Robert Bragg <robert at linux.intel.com>
thanks,
- Robert
On Fri, Nov 9, 2012 at 4:56 PM, Neil Roberts <neil at linux.intel.com> wrote:
> Previously when make test is run it would say ‘fail’ in lower case
> letters for both tests that are known bugs we need to fix and for
> drivers that can't run the test. This makes it too easy to lose track
> of bugs.
>
> To fix this, the ADD_TEST macro has now been changed to take two sets
> of flags instead of just one. The first specifies the requirements for
> the test to run at all. The second specifies the set of flags required
> to run without any known failures. The table in the test report now
> says ‘n/a’ instead of ‘fail’ for tests that don't match the feature
> requirements.
> ---
> tests/conform/run-tests.sh | 16 +++++-
> tests/conform/test-conform-main.c | 93 ++++++++++++++++----------------
> tests/conform/test-utils.c | 108
> +++++++++++++++++++++-----------------
> tests/conform/test-utils.h | 3 +-
> 4 files changed, 126 insertions(+), 94 deletions(-)
>
> diff --git a/tests/conform/run-tests.sh b/tests/conform/run-tests.sh
> index 266f1c6..11e740a 100755
> --- a/tests/conform/run-tests.sh
> +++ b/tests/conform/run-tests.sh
> @@ -10,7 +10,8 @@ trap "" SIGFPE
> trap "" SIGSEGV
>
> EXIT=0
> -WARNING="WARNING: Missing required feature";
> +MISSING_FEATURE="WARNING: Missing required feature";
> +KNOWN_FAILURE="WARNING: Test is known to fail";
>
> if test -f ./test-conformance; then
> TEST_CONFORMANCE=./test-conformance
> @@ -20,6 +21,7 @@ fi
>
> echo "Key:"
> echo "ok = Test passed"
> +echo "n/a = Driver is missing a feature required for the test"
> echo "FAIL = Unexpected failure"
> echo "fail = Test failed, but it was an expected failure"
> echo "PASS! = Unexpected pass"
> @@ -38,6 +40,10 @@ get_status()
> 400)
> echo -n 'PASS!';;
>
> + # Special value to indicate the test is missing a required feature
> + 500)
> + echo -n "n/a";;
> +
> 0)
> echo -n "ok";;
>
> @@ -52,7 +58,13 @@ run_test()
> TMP=$?
> var_name=$2_result
> eval $var_name=$TMP
> - if grep -q "$WARNING" .log; then
> + if grep -q "$MISSING_FEATURE" .log; then
> + if test $TMP -ne 0; then
> + eval $var_name=500
> + else
> + eval $var_name=400
> + fi
> + elif grep -q "$KNOWN_FAILURE" .log; then
> if test $TMP -ne 0; then
> eval $var_name=300
> else
> diff --git a/tests/conform/test-conform-main.c
> b/tests/conform/test-conform-main.c
> index 4e2c8bb..b510543 100644
> --- a/tests/conform/test-conform-main.c
> +++ b/tests/conform/test-conform-main.c
> @@ -10,16 +10,17 @@
> #include "test-utils.h"
>
> /* A bit of sugar for adding new conformance tests */
> -#define ADD_TEST(FUNC, REQUIREMENTS) G_STMT_START { \
> - extern void FUNC (void); \
> - if (strcmp (#FUNC, argv[1]) == 0) \
> - { \
> - test_utils_init (REQUIREMENTS); \
> - FUNC (); \
> - test_utils_fini (); \
> - exit (0); \
> - } \
> -} G_STMT_END
> +#define ADD_TEST(FUNC, REQUIREMENTS, KNOWN_FAIL_REQUIREMENTS) \
> + G_STMT_START { \
> + extern void FUNC (void); \
> + if (strcmp (#FUNC, argv[1]) == 0) \
> + { \
> + test_utils_init (REQUIREMENTS, KNOWN_FAIL_REQUIREMENTS); \
> + FUNC (); \
> + test_utils_fini (); \
> + exit (0); \
> + } \
> + } G_STMT_END
>
> #define UNPORTED_TEST(FUNC)
>
> @@ -51,66 +52,70 @@ main (int argc, char **argv)
> UNPORTED_TEST (test_object);
> UNPORTED_TEST (test_fixed);
> UNPORTED_TEST (test_materials);
> - ADD_TEST (test_pipeline_user_matrix, 0);
> - ADD_TEST (test_blend_strings, 0);
> - ADD_TEST (test_premult, 0);
> + ADD_TEST (test_pipeline_user_matrix, 0, 0);
> + ADD_TEST (test_blend_strings, 0, 0);
> + ADD_TEST (test_premult, 0, 0);
> UNPORTED_TEST (test_readpixels);
> - ADD_TEST (test_path, 0);
> - ADD_TEST (test_depth_test, 0);
> - ADD_TEST (test_color_mask, 0);
> - ADD_TEST (test_backface_culling, TEST_REQUIREMENT_NPOT);
> - ADD_TEST (test_layer_remove, 0);
> + ADD_TEST (test_path, 0, 0);
> + ADD_TEST (test_depth_test, 0, 0);
> + ADD_TEST (test_color_mask, 0, 0);
> + ADD_TEST (test_backface_culling, 0, TEST_REQUIREMENT_NPOT);
> + ADD_TEST (test_layer_remove, 0, 0);
>
> - ADD_TEST (test_sparse_pipeline, 0);
> + ADD_TEST (test_sparse_pipeline, 0, 0);
>
> - ADD_TEST (test_npot_texture, TEST_REQUIREMENT_NPOT);
> + ADD_TEST (test_npot_texture, 0, TEST_REQUIREMENT_NPOT);
> UNPORTED_TEST (test_multitexture);
> UNPORTED_TEST (test_texture_mipmaps);
> - ADD_TEST (test_sub_texture, 0);
> - ADD_TEST (test_pixel_buffer, 0);
> + ADD_TEST (test_sub_texture, 0, 0);
> + ADD_TEST (test_pixel_buffer, 0, 0);
> UNPORTED_TEST (test_texture_rectangle);
> - ADD_TEST (test_texture_3d, TEST_REQUIREMENT_TEXTURE_3D);
> - ADD_TEST (test_wrap_modes, 0);
> + ADD_TEST (test_texture_3d, TEST_REQUIREMENT_TEXTURE_3D, 0);
> + ADD_TEST (test_wrap_modes, 0, 0);
> UNPORTED_TEST (test_texture_pixmap_x11);
> UNPORTED_TEST (test_texture_get_set_data);
> - ADD_TEST (test_atlas_migration, 0);
> - ADD_TEST (test_read_texture_formats, 0);
> - ADD_TEST (test_write_texture_formats, 0);
> + ADD_TEST (test_atlas_migration, 0, 0);
> + ADD_TEST (test_read_texture_formats, 0, 0);
> + ADD_TEST (test_write_texture_formats, 0, 0);
>
> UNPORTED_TEST (test_vertex_buffer_contiguous);
> UNPORTED_TEST (test_vertex_buffer_interleved);
> UNPORTED_TEST (test_vertex_buffer_mutability);
>
> - ADD_TEST (test_primitive, 0);
> + ADD_TEST (test_primitive, 0, 0);
>
> - ADD_TEST (test_just_vertex_shader, TEST_REQUIREMENT_GLSL);
> - ADD_TEST (test_pipeline_uniforms, TEST_REQUIREMENT_GLSL);
> - ADD_TEST (test_snippets, TEST_REQUIREMENT_GLSL);
> - ADD_TEST (test_custom_attributes, TEST_REQUIREMENT_GLSL);
> + ADD_TEST (test_just_vertex_shader, TEST_REQUIREMENT_GLSL, 0);
> + ADD_TEST (test_pipeline_uniforms, TEST_REQUIREMENT_GLSL, 0);
> + ADD_TEST (test_snippets, TEST_REQUIREMENT_GLSL, 0);
> + ADD_TEST (test_custom_attributes, TEST_REQUIREMENT_GLSL, 0);
>
> - ADD_TEST (test_bitmask, 0);
> + ADD_TEST (test_bitmask, 0, 0);
>
> - ADD_TEST (test_offscreen, 0);
> + ADD_TEST (test_offscreen, 0, 0);
>
> - ADD_TEST (test_point_size, 0);
> + ADD_TEST (test_point_size, 0, 0);
> ADD_TEST (test_point_sprite,
> - TEST_REQUIREMENT_POINT_SPRITE);
> + TEST_REQUIREMENT_POINT_SPRITE,
> + 0);
> ADD_TEST (test_point_sprite_orientation,
> - TEST_KNOWN_FAILURE | TEST_REQUIREMENT_POINT_SPRITE);
> + TEST_REQUIREMENT_POINT_SPRITE,
> + TEST_KNOWN_FAILURE);
>
> - ADD_TEST (test_version, 0);
> + ADD_TEST (test_version, 0, 0);
>
> - ADD_TEST (test_alpha_test, 0);
> + ADD_TEST (test_alpha_test, 0, 0);
>
> - ADD_TEST (test_map_buffer_range, TEST_REQUIREMENT_MAP_WRITE);
> + ADD_TEST (test_map_buffer_range, TEST_REQUIREMENT_MAP_WRITE, 0);
>
> UNPORTED_TEST (test_viewport);
>
> - ADD_TEST (test_gles2_context, TEST_REQUIREMENT_GLES2_CONTEXT);
> - ADD_TEST (test_gles2_context_fbo, TEST_REQUIREMENT_GLES2_CONTEXT);
> - ADD_TEST (test_gles2_context_copy_tex_image,
> TEST_REQUIREMENT_GLES2_CONTEXT);
> + ADD_TEST (test_gles2_context, TEST_REQUIREMENT_GLES2_CONTEXT, 0);
> + ADD_TEST (test_gles2_context_fbo, TEST_REQUIREMENT_GLES2_CONTEXT, 0);
> + ADD_TEST (test_gles2_context_copy_tex_image,
> + TEST_REQUIREMENT_GLES2_CONTEXT,
> + 0);
>
> - ADD_TEST (test_euler_quaternion, 0);
> + ADD_TEST (test_euler_quaternion, 0, 0);
>
> g_printerr ("Unknown test name \"%s\"\n", argv[1]);
>
> diff --git a/tests/conform/test-utils.c b/tests/conform/test-utils.c
> index 4ee0856..2173cd0 100644
> --- a/tests/conform/test-utils.c
> +++ b/tests/conform/test-utils.c
> @@ -11,94 +11,106 @@ static CoglBool cogl_test_is_verbose;
> CoglContext *test_ctx;
> CoglFramebuffer *test_fb;
>
> -void
> -test_utils_init (TestFlags flags)
> +static CoglBool
> +check_flags (TestFlags flags,
> + CoglRenderer *renderer)
> {
> - static int counter = 0;
> - CoglError *error = NULL;
> - CoglOnscreen *onscreen = NULL;
> - CoglDisplay *display;
> - CoglRenderer *renderer;
> - CoglBool missing_requirement = FALSE;
> -
> - if (counter != 0)
> - g_critical ("We don't support running more than one test at a time\n"
> - "in a single test run due to the state leakage that can\n"
> - "cause subsequent tests to fail.\n"
> - "\n"
> - "If you want to run all the tests you should run\n"
> - "$ make test-report");
> - counter++;
> -
> - if (g_getenv ("COGL_TEST_VERBOSE") || g_getenv ("V"))
> - cogl_test_is_verbose = TRUE;
> -
> - if (g_getenv ("G_DEBUG"))
> - {
> - char *debug = g_strconcat (g_getenv ("G_DEBUG"), ",fatal-warnings",
> NULL);
> - g_setenv ("G_DEBUG", debug, TRUE);
> - g_free (debug);
> - }
> - else
> - g_setenv ("G_DEBUG", "fatal-warnings", TRUE);
> -
> - g_setenv ("COGL_X11_SYNC", "1", 0);
> -
> - test_ctx = cogl_context_new (NULL, &error);
> - if (!test_ctx)
> - g_critical ("Failed to create a CoglContext: %s", error->message);
> -
> - display = cogl_context_get_display (test_ctx);
> - renderer = cogl_display_get_renderer (display);
> -
> if (flags & TEST_REQUIREMENT_GL &&
> cogl_renderer_get_driver (renderer) != COGL_DRIVER_GL &&
> cogl_renderer_get_driver (renderer) != COGL_DRIVER_GL3)
> {
> - missing_requirement = TRUE;
> + return FALSE;
> }
>
> if (flags & TEST_REQUIREMENT_NPOT &&
> !cogl_has_feature (test_ctx, COGL_FEATURE_ID_TEXTURE_NPOT))
> {
> - missing_requirement = TRUE;
> + return FALSE;
> }
>
> if (flags & TEST_REQUIREMENT_TEXTURE_3D &&
> !cogl_has_feature (test_ctx, COGL_FEATURE_ID_TEXTURE_3D))
> {
> - missing_requirement = TRUE;
> + return FALSE;
> }
>
> if (flags & TEST_REQUIREMENT_POINT_SPRITE &&
> !cogl_has_feature (test_ctx, COGL_FEATURE_ID_POINT_SPRITE))
> {
> - missing_requirement = TRUE;
> + return FALSE;
> }
>
> if (flags & TEST_REQUIREMENT_GLES2_CONTEXT &&
> !cogl_has_feature (test_ctx, COGL_FEATURE_ID_GLES2_CONTEXT))
> {
> - missing_requirement = TRUE;
> + return FALSE;
> }
>
> if (flags & TEST_REQUIREMENT_MAP_WRITE &&
> !cogl_has_feature (test_ctx, COGL_FEATURE_ID_MAP_BUFFER_FOR_WRITE))
> {
> - missing_requirement = TRUE;
> + return FALSE;
> }
>
> if (flags & TEST_REQUIREMENT_GLSL &&
> !cogl_has_feature (test_ctx, COGL_FEATURE_ID_GLSL))
> {
> - missing_requirement = TRUE;
> + return FALSE;
> }
>
> if (flags & TEST_KNOWN_FAILURE)
> {
> - missing_requirement = TRUE;
> + return FALSE;
> }
>
> + return TRUE;
> +}
> +
> +void
> +test_utils_init (TestFlags requirement_flags,
> + TestFlags known_failure_flags)
> +{
> + static int counter = 0;
> + CoglError *error = NULL;
> + CoglOnscreen *onscreen = NULL;
> + CoglDisplay *display;
> + CoglRenderer *renderer;
> + CoglBool missing_requirement;
> + CoglBool known_failure;
> +
> + if (counter != 0)
> + g_critical ("We don't support running more than one test at a time\n"
> + "in a single test run due to the state leakage that can\n"
> + "cause subsequent tests to fail.\n"
> + "\n"
> + "If you want to run all the tests you should run\n"
> + "$ make test-report");
> + counter++;
> +
> + if (g_getenv ("COGL_TEST_VERBOSE") || g_getenv ("V"))
> + cogl_test_is_verbose = TRUE;
> +
> + if (g_getenv ("G_DEBUG"))
> + {
> + char *debug = g_strconcat (g_getenv ("G_DEBUG"), ",fatal-warnings",
> NULL);
> + g_setenv ("G_DEBUG", debug, TRUE);
> + g_free (debug);
> + }
> + else
> + g_setenv ("G_DEBUG", "fatal-warnings", TRUE);
> +
> + g_setenv ("COGL_X11_SYNC", "1", 0);
> +
> + test_ctx = cogl_context_new (NULL, &error);
> + if (!test_ctx)
> + g_critical ("Failed to create a CoglContext: %s", error->message);
> +
> + display = cogl_context_get_display (test_ctx);
> + renderer = cogl_display_get_renderer (display);
> +
> + missing_requirement = !check_flags (requirement_flags, renderer);
> + known_failure = !check_flags (known_failure_flags, renderer);
> +
> if (getenv ("COGL_TEST_ONSCREEN"))
> {
> onscreen = cogl_onscreen_new (test_ctx, 640, 480);
> @@ -132,6 +144,8 @@ test_utils_init (TestFlags flags)
>
> if (missing_requirement)
> g_print ("WARNING: Missing required feature[s] for this test\n");
> + else if (known_failure)
> + g_print ("WARNING: Test is known to fail\n");
> }
>
> void
> diff --git a/tests/conform/test-utils.h b/tests/conform/test-utils.h
> index acd4fad..265dc32 100644
> --- a/tests/conform/test-utils.h
> +++ b/tests/conform/test-utils.h
> @@ -23,7 +23,8 @@ extern CoglContext *test_ctx;
> extern CoglFramebuffer *test_fb;
>
> void
> -test_utils_init (TestFlags flags);
> +test_utils_init (TestFlags requirement_flags,
> + TestFlags known_failure_flags);
>
> void
> test_utils_fini (void);
> --
> 1.7.11.3.g3c3efa5
>
> _______________________________________________
> Cogl mailing list
> Cogl at lists.freedesktop.org
> http://lists.freedesktop.org/mailman/listinfo/cogl
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.freedesktop.org/archives/cogl/attachments/20121119/543fbc8f/attachment-0001.html>
More information about the Cogl
mailing list