[Piglit] [PATCH] tests: Add integration with Khronos CTS OpenGL runner
Mark Janes
mark.a.janes at intel.com
Thu Jun 2 20:24:46 UTC 2016
Dylan Baker <dylan at pnwbakers.com> writes:
> Quoting Juan A. Suarez Romero (2016-05-18 06:34:07)
>> This adds support for running Khronos' deqp-based conformance suite for
>> OpenGL with piglit.
>> ---
>> piglit.conf.example | 10 ++++++++++
>> tests/{cts.py => cts_gl45.py} | 43 +++++++++++++++++++++++++----------------
>> tests/{cts.py => cts_gles31.py} | 2 +-
>
> Since these run all of the tests I think I'd prefer cts_gles and cts_gl,
> but that's a bit of a nit-pick.
>
>> 3 files changed, 37 insertions(+), 18 deletions(-)
>> copy tests/{cts.py => cts_gl45.py} (56%)
>> rename tests/{cts.py => cts_gles31.py} (97%)
>>
>> diff --git a/piglit.conf.example b/piglit.conf.example
>> index 944d5d9..3cce52a 100644
>> --- a/piglit.conf.example
>> +++ b/piglit.conf.example
>> @@ -95,6 +95,16 @@ testB
>> ; overrides the value set here.
>> ;extra_args=--deqp-visibility hidden
>>
>> +[cts_gl]
>> +; path to the cts opengl executable
>> +; can be overwritten by PIGLIT_CTS_GL_BIN environment variable
>> +;bin=/home/knuth/cts/cts/glcts
>> +
>> +; Space-separated list of extra command line arguments for cts. The
>> +; option is not required. The environment variable PIGLIT_CTS_GL_EXTRA_ARGS
>> +; overrides the value set here.
>> +;extra_args=--deqp-visibility hidden
>> +
>> ; Section for specific oclconform test. One of these sections is required for
>> ; each test list in the oclconform section and must be called:
>> ; oclconform-$testname
>> diff --git a/tests/cts.py b/tests/cts_gl45.py
>> similarity index 56%
>> copy from tests/cts.py
>> copy to tests/cts_gl45.py
>> index 0e64e1b..607b79b 100644
>> --- a/tests/cts.py
>> +++ b/tests/cts_gl45.py
>> @@ -20,25 +20,24 @@
>>
>> """Piglit integration for Khronos CTS tests.
>>
>> -By default this will run GLES2, GLES3, GLES31, and GLESEXT test cases. Those
>> -desiring to run only a subset of them should consider using the -t or -x
>> -options to include or exclude tests.
>> +By default this will run GL30, GL31, GL32, GL33, GL40, GL41, GL42, GL43, GL44
>> +and GL45 test cases. Those desiring to run only a subset of them should consider
>> +using the -t or -x options to include or exclude tests.
>>
>> For example:
>> -./piglit run cts -c foo -t ES3- would run only ES3 tests (note the dash to
>> -exclude ES31 tests)
>> +./piglit run cts_gl45 -c foo -t GL30- would run only GL30 tests
>>
>> This integration requires some configuration in piglit.conf, or the use of
>> environment variables.
>>
>> In piglit.conf one should set the following:
>> -[cts]:bin -- Path to the glcts binary
>> -[cts]:extra_args -- any extra arguments to be passed to cts (optional)
>> +[cts_gl]:bin -- Path to the glcts binary
>> +[cts_gl]:extra_args -- any extra arguments to be passed to cts (optional)
>
>>
>> Alternatively (or in addition, since environment variables have precedence),
>> one could set:
>> -PIGLIT_CTS_BIN -- environment equivalent of [cts]:bin
>> -PIGLIT_CTS_EXTRA_ARGS -- environment equivalent of [cts]:extra_args
>> +PIGLIT_CTS_GL_BIN -- environment equivalent of [cts_gl]:bin
>> +PIGLIT_CTS_GL_EXTRA_ARGS -- environment equivalent of [cts_gl]:extra_args
>>
>> """
>>
>> @@ -51,9 +50,9 @@ from framework.test import deqp
>>
>> __all__ = ['profile']
>>
>> -_CTS_BIN = deqp.get_option('PIGLIT_CTS_BIN', ('cts', 'bin'))
>> +_CTS_BIN = deqp.get_option('PIGLIT_CTS_GL_BIN', ('cts_gl', 'bin'))
>>
>> -_EXTRA_ARGS = deqp.get_option('PIGLIT_CTS_EXTRA_ARGS', ('cts', 'extra_args'),
>> +_EXTRA_ARGS = deqp.get_option('PIGLIT_CTS_GL_EXTRA_ARGS', ('cts_gl', 'extra_args'),
>> default='').split()
>>
>>
>> @@ -65,18 +64,28 @@ class DEQPCTSTest(deqp.DEQPBaseTest):
>> return super(DEQPCTSTest, self).extra_args + \
>> [x for x in _EXTRA_ARGS if not x.startswith('--deqp-case')]
>>
>> -
>> # Add all of the suites by default, users can use filters to remove them.
>> profile = deqp.make_profile( # pylint: disable=invalid-name
>> itertools.chain(
>> deqp.iter_deqp_test_cases(
>> - deqp.gen_caselist_txt(_CTS_BIN, 'ES2-CTS-cases.txt', _EXTRA_ARGS)),
>> + deqp.gen_caselist_txt(_CTS_BIN, 'GL30-CTS-cases.txt', _EXTRA_ARGS)),
>> + deqp.iter_deqp_test_cases(
>> + deqp.gen_caselist_txt(_CTS_BIN, 'GL31-CTS-cases.txt', _EXTRA_ARGS)),
>> + deqp.iter_deqp_test_cases(
>> + deqp.gen_caselist_txt(_CTS_BIN, 'GL32-CTS-cases.txt', _EXTRA_ARGS)),
>> + deqp.iter_deqp_test_cases(
>> + deqp.gen_caselist_txt(_CTS_BIN, 'GL33-CTS-cases.txt', _EXTRA_ARGS)),
>> + deqp.iter_deqp_test_cases(
>> + deqp.gen_caselist_txt(_CTS_BIN, 'GL40-CTS-cases.txt', _EXTRA_ARGS)),
>> + deqp.iter_deqp_test_cases(
>> + deqp.gen_caselist_txt(_CTS_BIN, 'GL41-CTS-cases.txt', _EXTRA_ARGS)),
>> + deqp.iter_deqp_test_cases(
>> + deqp.gen_caselist_txt(_CTS_BIN, 'GL42-CTS-cases.txt', _EXTRA_ARGS)),
>> deqp.iter_deqp_test_cases(
>> - deqp.gen_caselist_txt(_CTS_BIN, 'ES3-CTS-cases.txt', _EXTRA_ARGS)),
>> + deqp.gen_caselist_txt(_CTS_BIN, 'GL43-CTS-cases.txt', _EXTRA_ARGS)),
>> deqp.iter_deqp_test_cases(
>> - deqp.gen_caselist_txt(_CTS_BIN, 'ES31-CTS-cases.txt', _EXTRA_ARGS)),
>> + deqp.gen_caselist_txt(_CTS_BIN, 'GL44-CTS-cases.txt', _EXTRA_ARGS)),
>> deqp.iter_deqp_test_cases(
>> - deqp.gen_caselist_txt(_CTS_BIN, 'ESEXT-CTS-cases.txt',
>> - _EXTRA_ARGS)),
>> + deqp.gen_caselist_txt(_CTS_BIN, 'GL45-CTS-cases.txt', _EXTRA_ARGS)),
>> ),
>> DEQPCTSTest)
>> diff --git a/tests/cts.py b/tests/cts_gles31.py
>> similarity index 97%
>> rename from tests/cts.py
>> rename to tests/cts_gles31.py
>> index 0e64e1b..ad28e58 100644
>> --- a/tests/cts.py
>> +++ b/tests/cts_gles31.py
>> @@ -25,7 +25,7 @@ desiring to run only a subset of them should consider using the -t or -x
>> options to include or exclude tests.
>>
>> For example:
>> -./piglit run cts -c foo -t ES3- would run only ES3 tests (note the dash to
>> +./piglit run cts_gles31 -c foo -t ES3- would run only ES3 tests (note the dash to
>> exclude ES31 tests)
>>
>> This integration requires some configuration in piglit.conf, or the use of
>> --
>> 2.5.5
>>
>> _______________________________________________
>> Piglit mailing list
>> Piglit at lists.freedesktop.org
>> https://lists.freedesktop.org/mailman/listinfo/piglit
>
> Other than the one nit this looks good. Please be sure to let Mark Janes
> (cc'd) or myself know when you push this, as we'll need to update our CI
> since this will change some profile names.
Whoops! This was pushed on May 23, at which point our CI stopped
running the CTS. And now we have a handful of regressions to bisect.
> Reviewed-by: Dylan Baker <dylan at pnwbakers.com>
More information about the Piglit
mailing list