[Piglit] [PATCH 3/5] deqp: Convert dEQP to use a custom profile.
Dylan Baker
baker.dylan.c at gmail.com
Thu Mar 24 18:43:03 UTC 2016
This replaces the make_profile function with a subclass of TestProfile.
It's a little cleaner and easier to understand, and much better
documented than make_profile.
It will be further extended in later patches in this series to do more
than just override the constructor.
Signed-off-by: Dylan Baker <dylanx.c.baker at intel.com>
---
framework/test/deqp.py | 96 ++++++++++++++++++++++++++++++++++---------------
tests/cts.py | 42 ++++++++++++++--------
tests/deqp_gles2.py | 11 +++---
tests/deqp_gles3.py | 12 ++++---
tests/deqp_gles31.py | 11 +++---
tests/deqp_vk.py | 11 +++---
unittests/deqp_tests.py | 63 +++++++++++++++++++++-----------
7 files changed, 162 insertions(+), 84 deletions(-)
diff --git a/framework/test/deqp.py b/framework/test/deqp.py
index 1ed594a..de6aab4 100644
--- a/framework/test/deqp.py
+++ b/framework/test/deqp.py
@@ -22,6 +22,7 @@ from __future__ import (
absolute_import, division, print_function, unicode_literals
)
import abc
+import functools
import os
import re
import subprocess
@@ -36,10 +37,8 @@ from framework.options import OPTIONS
__all__ = [
'DEQPBaseTest',
'DEQPGroupTest',
- 'gen_caselist_txt',
+ 'DEQPProfile',
'get_option',
- 'iter_deqp_test_cases',
- 'make_profile',
]
@@ -63,25 +62,7 @@ _EXTRA_ARGS = get_option('PIGLIT_DEQP_EXTRA_ARGS',
default='').split()
-def make_profile(test_list, single_class=None, multi_class=None):
- """Create a TestProfile instance."""
- if OPTIONS.deqp_mode == 'group':
- assert multi_class is not None
- _class = multi_class
- elif OPTIONS.deqp_mode == 'test':
- assert single_class is not None
- _class = single_class
-
- profile = TestProfile()
- for testname in test_list:
- # deqp uses '.' as the testgroup separator.
- piglit_name = testname.replace('.', grouptools.SEPARATOR)
- profile.test_list[piglit_name] = _class(testname)
-
- return profile
-
-
-def gen_caselist_txt(bin_, caselist, extra_args):
+def _gen_caselist_txt(bin_, caselist, extra_args):
"""Generate a caselist.txt and return its path.
Extra args should be a list of extra arguments to pass to deqp.
@@ -106,7 +87,8 @@ def gen_caselist_txt(bin_, caselist, extra_args):
subprocess.check_call(
[bin_, '--deqp-runmode=txt-caselist'] + extra_args, cwd=basedir,
stdout=d, stderr=d)
- assert os.path.exists(caselist_path)
+ assert os.path.exists(caselist_path), \
+ 'Could not find {}'.format(caselist_path)
return caselist_path
@@ -119,7 +101,7 @@ def _iterate_file(file_):
yield line
-def _iter_deqp_test_groups(case_file):
+def _iter_test_groups(case_file):
"""Iterate over original dEQP testcase groups.
This generator yields the name of each leaf group (that is, a group which
@@ -141,7 +123,7 @@ def _iter_deqp_test_groups(case_file):
'deqp: {}:{}: ill-formed line'.format(case_file, i))
-def _iter_deqp_test_single(case_file):
+def _iter_test_single(case_file):
"""Iterate over original dEQP testcase names."""
with open(case_file, 'r') as caselist_file:
for i, line in enumerate(_iterate_file(caselist_file)):
@@ -154,12 +136,70 @@ def _iter_deqp_test_single(case_file):
'deqp: {}:{}: ill-formed line'.format(case_file, i))
-def iter_deqp_test_cases(case_file):
+def _iter_test_cases(case_file):
"""Wrapper that sets the iterator based on the mode."""
if OPTIONS.deqp_mode == 'group':
- return _iter_deqp_test_groups(case_file)
+ return _iter_test_groups(case_file)
elif OPTIONS.deqp_mode == 'test':
- return _iter_deqp_test_single(case_file)
+ return _iter_test_single(case_file)
+
+
+def _pop(kwargs, name, default=None):
+ """A guard function for DEQPProfile."""
+ try:
+ return kwargs.pop(name)
+ except KeyError:
+ if default is not None:
+ return default
+ raise TypeError('Required keyword argument {} was not set'.format(name))
+
+
+class DEQPProfile(TestProfile):
+ """A profile specifically for dEQP tests.
+
+ This profile provides much of the necessary setup bits for dEQP
+ integration, including generating a valid test list.
+
+ All arguments not listed will be passed to the parent class.
+
+ Keyword Arguments:
+ single_class -- the class to use for 'test at a time' mode. Required.
+ multi_class -- the class to use for 'group at a time' mode. Required.
+ filter_ -- A function that filters that test cases. By default this is a
+ no-op function.
+ bin_ -- the dEQP binary to use. Required.
+ extra_args -- the extra arguments to pass to the dEQP binary for each test.
+ Default: [].
+ filename -- the name of the txt file containing all of the test and group
+ information that dEQP generates. Required.
+
+ """
+
+ def __init__(self, *args, **kwargs):
+ # PYTHON3: This could all be done with explict keyword arguments
+ # instead of this...madness
+ pop = functools.partial(_pop, kwargs)
+
+ single_class = pop('single_class')
+ multi_class = pop('multi_class')
+ bin_ = pop('bin_')
+ filename = pop('filename')
+ extra_args = pop('extra_args', default=[])
+ filter_ = pop('filter_', default=lambda x: x)
+
+ super(DEQPProfile, self).__init__(*args, **kwargs)
+
+ iter_ = _iter_test_cases(filter_(
+ _gen_caselist_txt(bin_, filename, extra_args)))
+
+ if OPTIONS.deqp_mode == 'group':
+ class_ = multi_class
+ elif OPTIONS.deqp_mode == 'test':
+ class_ = single_class
+
+ for testname in iter_:
+ piglit_name = testname.replace('.', grouptools.SEPARATOR)
+ self.test_list[piglit_name] = class_(testname)
@six.add_metaclass(abc.ABCMeta)
diff --git a/tests/cts.py b/tests/cts.py
index e0b05d1..afbb842 100644
--- a/tests/cts.py
+++ b/tests/cts.py
@@ -45,9 +45,10 @@ PIGLIT_CTS_EXTRA_ARGS -- environment equivalent of [cts]:extra_args
from __future__ import (
absolute_import, division, print_function, unicode_literals
)
-import itertools
+import functools
from framework.test import deqp
+from framework import profile
__all__ = ['profile']
@@ -76,17 +77,28 @@ class DEQPCTSGroupTest(_Mixin, deqp.DEQPGroupTest):
pass
-# Add all of the suites by default, users can use filters to remove them.
-profile = deqp.make_profile( # pylint: disable=invalid-name
- itertools.chain(
- deqp.iter_deqp_test_cases(
- deqp.gen_caselist_txt(_CTS_BIN, 'ES2-CTS-cases.txt', _EXTRA_ARGS)),
- deqp.iter_deqp_test_cases(
- deqp.gen_caselist_txt(_CTS_BIN, 'ES3-CTS-cases.txt', _EXTRA_ARGS)),
- deqp.iter_deqp_test_cases(
- deqp.gen_caselist_txt(_CTS_BIN, 'ES31-CTS-cases.txt', _EXTRA_ARGS)),
- deqp.iter_deqp_test_cases(
- deqp.gen_caselist_txt(_CTS_BIN, 'ESEXT-CTS-cases.txt',
- _EXTRA_ARGS)),
- ),
- single_class=DEQPCTSTest, multi_class=DEQPCTSGroupTest)
+def _make_profile():
+ """Make a single profile for the GLES CTS.
+
+ The GLES CTS is wierd. It's 4 distinct test suites, that need to be run as
+ one. The profile mechanism for dEQP integration doesn't really support
+ this, so instead what is done is that 4 profiles are created, then merged.
+
+ This can easily be trimmed using the standard test filters from the command
+ line.
+
+ """
+ partial = functools.partial(deqp.DEQPProfile,
+ single_class=DEQPCTSTest,
+ multi_class=DEQPCTSGroupTest,
+ bin_=_CTS_BIN,
+ extra_args=_EXTRA_ARGS)
+ profile_ = partial(filename='ES2-CTS-cases.txt')
+ profile_.update(partial(filename='ES3-CTS-cases.txt'))
+ profile_.update(partial(filename='ES31-CTS-cases.txt'))
+ profile_.update(partial(filename='ESEXT-CTS-cases.txt'))
+
+ return profile_
+
+
+profile = _make_profile() # pylint: disable=invalid-name
diff --git a/tests/deqp_gles2.py b/tests/deqp_gles2.py
index 5bf3672..56279d8 100644
--- a/tests/deqp_gles2.py
+++ b/tests/deqp_gles2.py
@@ -57,8 +57,9 @@ class DEQPGLES2GroupTest(_Mixin, deqp.DEQPGroupTest):
pass
-profile = deqp.make_profile( # pylint: disable=invalid-name
- deqp.iter_deqp_test_cases(
- deqp.gen_caselist_txt(_DEQP_GLES2_BIN, 'dEQP-GLES2-cases.txt',
- _EXTRA_ARGS)),
- single_class=DEQPGLES2Test, multi_class=DEQPGLES2GroupTest)
+profile = deqp.DEQPProfile( # pylint: disable=invalid-name
+ single_class=DEQPGLES2Test,
+ multi_class=DEQPGLES2GroupTest,
+ bin_=_DEQP_GLES2_BIN,
+ filename='dEQP-GLES2-cases.txt',
+ extra_args=_EXTRA_ARGS)
diff --git a/tests/deqp_gles3.py b/tests/deqp_gles3.py
index c50033b..506ef09 100644
--- a/tests/deqp_gles3.py
+++ b/tests/deqp_gles3.py
@@ -99,8 +99,10 @@ class DEQPGLES3GroupTest(_Mixin, deqp.DEQPGroupTest):
pass
-profile = deqp.make_profile( # pylint: disable=invalid-name
- deqp.iter_deqp_test_cases(filter_mustpass(
- deqp.gen_caselist_txt(_DEQP_GLES3_EXE, 'dEQP-GLES3-cases.txt',
- _EXTRA_ARGS))),
- single_class=DEQPGLES3Test, multi_class=DEQPGLES3GroupTest)
+profile = deqp.DEQPProfile( # pylint: disable=invalid-name
+ single_class=DEQPGLES3Test,
+ multi_class=DEQPGLES3GroupTest,
+ bin_=_DEQP_GLES3_EXE,
+ filename='dEQP-GLES3-cases.txt',
+ filter_=filter_mustpass,
+ extra_args=_EXTRA_ARGS)
diff --git a/tests/deqp_gles31.py b/tests/deqp_gles31.py
index 666dcab..8691939 100644
--- a/tests/deqp_gles31.py
+++ b/tests/deqp_gles31.py
@@ -57,8 +57,9 @@ class DEQPGLES31GroupTest(_Mixin, deqp.DEQPGroupTest):
pass
-profile = deqp.make_profile( # pylint: disable=invalid-name
- deqp.iter_deqp_test_cases(
- deqp.gen_caselist_txt(_DEQP_GLES31_BIN, 'dEQP-GLES31-cases.txt',
- _EXTRA_ARGS)),
- single_class=DEQPGLES31Test, multi_class=DEQPGLES31GroupTest)
+profile = deqp.DEQPProfile( # pylint: disable=invalid-name
+ single_class=DEQPGLES31Test,
+ multi_class=DEQPGLES31GroupTest,
+ bin_=_DEQP_GLES31_BIN,
+ filename='dEQP-GLES31-cases.txt',
+ extra_args=_EXTRA_ARGS)
diff --git a/tests/deqp_vk.py b/tests/deqp_vk.py
index 9a31c46..1cfce6e 100644
--- a/tests/deqp_vk.py
+++ b/tests/deqp_vk.py
@@ -77,8 +77,9 @@ class DEQPVKGroupTest(_Mixin, deqp.DEQPGroupTest):
pass
-profile = deqp.make_profile( # pylint: disable=invalid-name
- deqp.iter_deqp_test_cases(
- deqp.gen_caselist_txt(_DEQP_VK_BIN, 'dEQP-VK-cases.txt',
- _EXTRA_ARGS)),
- single_class=DEQPVKTest, multi_class=DEQPVKGroupTest)
+profile = deqp.DEQPProfile( # pylint: disable=invalid-name
+ single_class=DEQPVKTest,
+ multi_class=DEQPVKGroupTest,
+ bin_=_DEQP_VK_BIN,
+ filename='dEQP-VK-cases.txt',
+ extra_args=_EXTRA_ARGS)
diff --git a/unittests/deqp_tests.py b/unittests/deqp_tests.py
index f3b243e..2d347a8 100644
--- a/unittests/deqp_tests.py
+++ b/unittests/deqp_tests.py
@@ -28,10 +28,12 @@ tests
from __future__ import (
absolute_import, division, print_function, unicode_literals
)
+import copy
import sys
import textwrap
import nose.tools as nt
+import six
# There is a bug in mock < 1.2 or python 3.4 that we'd like to avoid, otherwise
# some tests will skip.
@@ -105,34 +107,17 @@ def test_get_option_conf_no_option():
None)
-class TestMakeProfile(object):
- """Test deqp.make_profile."""
- @classmethod
- def setup_class(cls):
- cls.profile = deqp.make_profile(['this.is.a.deqp.test'], _DEQPTestTest)
-
- def test_returns_profile(self):
- """deqp.make_profile: returns a TestProfile"""
- nt.assert_is_instance(self.profile, profile.TestProfile)
-
- @doc_formatter
- def test_grouptools(self):
- """deqp.make_profile: replaces '.' with '{separator}'"""
- nt.assert_in(grouptools.join('this', 'is', 'a', 'deqp', 'test'),
- self.profile.test_list)
-
-
def test_iter_deqp_test_cases_test():
"""deqp.iter_deqp_test_cases: correctly detects a TEST: line"""
with utils.tempfile('TEST: a.deqp.test') as tfile:
- gen = deqp.iter_deqp_test_cases(tfile)
+ gen = deqp._iter_test_cases(tfile)
nt.eq_('a.deqp.test', next(gen))
def test_iter_deqp_test_cases_group():
"""deqp.iter_deqp_test_casesgen_caselist_txt: correctly detects a GROUP: line"""
with utils.tempfile('GROUP: a group\nTEST: a.deqp.test') as tfile:
- gen = deqp.iter_deqp_test_cases(tfile)
+ gen = deqp._iter_test_cases(tfile)
nt.eq_('a.deqp.test', next(gen))
@@ -141,7 +126,7 @@ def test_iter_deqp_test_cases_bad():
"""deqp.iter_deqp_test_casesgen_caselist_txt: PiglitFatalException is raised if line is not TEST: or GROUP:
"""
with utils.tempfile('this will fail') as tfile:
- gen = deqp.iter_deqp_test_cases(tfile)
+ gen = deqp._iter_test_cases(tfile)
nt.eq_('a.deqp.test', next(gen))
@@ -552,6 +537,42 @@ def test_iter_deqp_test_groups():
with mock.patch('framework.test.deqp.open', create=True,
new=mock.mock_open(read_data=text)):
- actual = list(deqp._iter_deqp_test_groups(None))
+ actual = list(deqp._iter_test_groups(None))
nt.assert_list_equal(actual, expected)
+
+
+ at utils.nose_generator
+def test_DEQPProfile_init_required():
+ """Generate tests for DEQPPRofile's constructor required arguments.
+
+ Basically this builds a valid list of arguments, and then erases them one
+ by one looking for an exception.
+
+ """
+ description = 'deqp.DEQPProfile: requires kwarg {}'
+
+ kwargs = {
+ 'single_class': None,
+ 'multi_class': None,
+ 'bin_': None,
+ 'filename': None,
+ }
+
+ def test(arg):
+ """The actual test."""
+ test_args = copy.copy(kwargs)
+ del test_args[arg]
+
+ with nt.assert_raises(TypeError) as e:
+ deqp.DEQPProfile(**test_args)
+
+ # Ensure that we're getting the right error.
+ # An initial version of the test "passed" because of a typo in
+ # DEQPProfile
+ nt.eq_(six.text_type(e.exception),
+ 'Required keyword argument {} was not set'.format(arg))
+
+ for arg in six.iterkeys(kwargs):
+ test.description = description.format(arg)
+ yield test, arg
--
2.7.4
More information about the Piglit
mailing list