[Piglit] [PATCH 6/11] framework: Implement boilerplate to generate a group-at-a-time profile
Dylan Baker
dylan at pnwbakers.com
Fri Aug 26 18:31:13 UTC 2016
This patches adds all of the boilerplate code that is required to get
group-at-a-time working. It adds the command line option, and all of the
switches to choose between creating a profile full of single tests, or
tests that run group at a time.
Signed-off-by: Dylan Baker <dylanx.c.baker at intel.com>
---
framework/options.py | 3 +-
framework/programs/run.py | 10 +-
framework/test/deqp.py | 91 +++++++++++--
unittests/framework/test/test_deqp.py | 196 +++++++++++++++++++++------
4 files changed, 245 insertions(+), 55 deletions(-)
diff --git a/framework/options.py b/framework/options.py
index 94a8084..acdfd16 100644
--- a/framework/options.py
+++ b/framework/options.py
@@ -181,6 +181,8 @@ class _Options(object): # pylint: disable=too-many-instance-attributes
monitored -- True if monitoring is desired. This forces concurrency off
env -- environment variables set for each test before run
deqp_mustpass -- True to enable the use of the deqp mustpass list feature.
+ deqp_mode -- 'single' for one test per processes or 'group' for one leaf of
+ tests per process
"""
include_filter = _ReListDescriptor('_include_filter', type_=_FilterReList)
@@ -197,6 +199,7 @@ class _Options(object): # pylint: disable=too-many-instance-attributes
self.monitored = False
self.sync = False
self.deqp_mustpass = False
+ self.deqp_mode = 'single'
# env is used to set some base environment variables that are not going
# to change across runs, without sending them to os.environ which is
diff --git a/framework/programs/run.py b/framework/programs/run.py
index 7c8660c..89709e7 100644
--- a/framework/programs/run.py
+++ b/framework/programs/run.py
@@ -178,6 +178,14 @@ def _run_parser(input_):
help='Run only the tests in the deqp mustpass list '
'when running a deqp gles{2,3,31} profile, '
'otherwise run all tests.')
+ parser.add_argument('--deqp-mode',
+ action='store',
+ choices=['single', 'group'],
+ default='single',
+ help='Whether to run a single test per process, or to '
+ 'run a leaf of the test hierarchy as a process. '
+ 'This makes a trade-off between speed and '
+ 'reliability')
parser.add_argument("test_profile",
metavar="<Profile path(s)>",
nargs='+',
@@ -259,6 +267,7 @@ def run(input_):
options.OPTIONS.monitored = args.monitored
options.OPTIONS.sync = args.sync
options.OPTIONS.deqp_mustpass = args.deqp_mustpass
+ options.OPTIONS.deqp_mode = args.deqp_mode
# Set the platform to pass to waffle
options.OPTIONS.env['PIGLIT_PLATFORM'] = args.platform
@@ -346,6 +355,7 @@ def resume(input_):
options.OPTIONS.monitored = results.options['monitored']
options.OPTIONS.sync = results.options['sync']
options.OPTIONS.deqp_mustpass = results.options['deqp_mustpass']
+ options.OPTIONS.deqp_mode = results.options['deqp_mode']
core.get_config(args.config_file)
diff --git a/framework/test/deqp.py b/framework/test/deqp.py
index 25dd077..5a2e11f 100644
--- a/framework/test/deqp.py
+++ b/framework/test/deqp.py
@@ -90,13 +90,22 @@ def select_source(bin_, filename, mustpass, extra_args):
gen_caselist_txt(bin_, filename, extra_args))
-def make_profile(test_list, test_class):
+def make_profile(test_list, single=None, group=None):
"""Create a TestProfile instance."""
+ if options.OPTIONS.deqp_mode == "single":
+ if isinstance(single, DEQPUnsupportedMode):
+ raise exceptions.PiglitFatalError(single)
+ test_class = single
+ else:
+ if isinstance(group, DEQPUnsupportedMode):
+ raise exceptions.PiglitFatalError(group)
+ test_class = group
+
profile = TestProfile()
for testname in test_list:
# deqp uses '.' as the testgroup separator.
- piglit_name = testname.replace('.', grouptools.SEPARATOR)
- profile.test_list[piglit_name] = test_class(testname)
+ piglit_name = testname[0].replace('.', grouptools.SEPARATOR)
+ profile.test_list[piglit_name] = test_class(*testname)
return profile
@@ -104,17 +113,34 @@ def make_profile(test_list, test_class):
def gen_mustpass_tests(mp_list):
"""Return a testlist from the mustpass list."""
root = et.parse(mp_list).getroot()
- group = []
+ cur_group = []
- def gen(root):
+ def single(root):
for elem in root:
if elem.tag == 'Test':
- yield '{}.{}'.format('.'.join(group), elem.get('name'))
+ yield ('{}.{}'.format('.'.join(cur_group), elem.get('name')), )
else:
- group.append(elem.get('name'))
- for test in gen(elem):
+ cur_group.append(elem.get('name'))
+ for test in single(elem):
+ yield test
+ del cur_group[-1]
+
+ def group(root):
+ for elem in root:
+ if elem.tag == 'TestCase':
+ case = '{}.{}'.format('.'.join(cur_group), elem.get('name'))
+ yield (case, ['{}.{}'.format(case, t.get('name'))
+ for t in elem.findall('.//Test')])
+ elif elem.tag == 'TestSuite':
+ cur_group.append(elem.get('name'))
+ for test in group(elem):
yield test
- del group[-1]
+ del cur_group[-1]
+
+ if options.OPTIONS.deqp_mode == 'single':
+ gen = single
+ else:
+ gen = group
for test in gen(root):
yield test
@@ -150,15 +176,54 @@ def gen_caselist_txt(bin_, caselist, extra_args):
def iter_deqp_test_cases(case_file):
"""Iterate over original dEQP testcase names."""
- with open(case_file, 'r') as caselist_file:
- for i, line in enumerate(caselist_file):
+ def single(f):
+ """Iterate over the txt file, and yield each test instance."""
+ for i, line in enumerate(f):
if line.startswith('GROUP:'):
continue
elif line.startswith('TEST:'):
- yield line[len('TEST:'):].strip()
+ # The group mode yields a tuple, so the single mode needs to as
+ # well.
+ yield (line[len('TEST:'):].strip(), )
+ else:
+ raise exceptions.PiglitFatalError(
+ 'deqp: {}:{}: ill-formed line'.format(f.name, i))
+
+ def group(f):
+ """Iterate over the txt file, and yield each group and its members.
+
+ The group must contain actual members to be yielded.
+ """
+ group = ''
+ tests = []
+
+ for i, line in enumerate(f):
+ if line.startswith('GROUP:'):
+ new = line[len('GROUP:'):].strip()
+ if group != new and tests:
+ yield (group, tests)
+ tests = []
+ group = new
+ elif line.startswith('TEST:'):
+ tests.append(line[len('TEST:'):].strip())
else:
raise exceptions.PiglitFatalError(
- 'deqp: {}:{}: ill-formed line'.format(case_file, i))
+ 'deqp: {}:{}: ill-formed line'.format(f.name, i))
+ # If we get to the end of the file and we have new tests (the would
+ # have been cleared if there weren't any.
+ if tests:
+ yield (group, tests)
+
+ adder = None
+ if options.OPTIONS.deqp_mode == 'single':
+ adder = single
+ elif options.OPTIONS.deqp_mode == 'group':
+ adder = group
+ assert adder is not None
+
+ with open(case_file, 'r') as f:
+ for x in adder(f):
+ yield x
def format_trie_list(classname, testnames):
diff --git a/unittests/framework/test/test_deqp.py b/unittests/framework/test/test_deqp.py
index 58504fd..6e1a73d 100644
--- a/unittests/framework/test/test_deqp.py
+++ b/unittests/framework/test/test_deqp.py
@@ -39,11 +39,12 @@ import six
from framework import exceptions
from framework import grouptools
+from framework import options
from framework import profile
from framework import status
from framework.test import deqp
-# pylint:disable=invalid-name,no-self-use
+# pylint:disable=no-self-use,protected-access
class _DEQPTestTest(deqp.DEQPSingleTest):
@@ -116,44 +117,123 @@ class TestGetOptions(object):
class TestMakeProfile(object):
"""Test deqp.make_profile."""
- @classmethod
- def setup_class(cls):
- cls.profile = deqp.make_profile(['this.is.a.deqp.test'], _DEQPTestTest)
+ class TestSingle(object):
+ """Tests for the single mode."""
- def test_returns_profile(self):
- """deqp.make_profile: returns a TestProfile."""
- assert isinstance(self.profile, profile.TestProfile)
+ @classmethod
+ def setup_class(cls):
+ with mock.patch('framework.test.deqp.options.OPTIONS',
+ new=options._Options()) as mocked:
+ mocked.deqp_mode = 'single'
+ cls.profile = deqp.make_profile([('this.is.a.deqp.test', )],
+ single=_DEQPTestTest)
- def test_replaces_separator(self):
- """deqp.make_profile: replaces '.' with grouptools.separator"""
- expected = grouptools.join('this', 'is', 'a', 'deqp', 'test')
- assert expected in self.profile.test_list
+ def test_returns_profile(self):
+ """deqp.make_profile: returns a TestProfile."""
+ assert isinstance(self.profile, profile.TestProfile)
+ def test_replaces_separator(self):
+ """deqp.make_profile: replaces '.' with grouptools.separator"""
+ expected = grouptools.join('this', 'is', 'a', 'deqp', 'test')
+ assert expected in self.profile.test_list
-class TestIterDeqpTestCases(object):
- """Tests for iter_deqp_test_cases."""
+ class TestGroup(object):
+ """Tests for group mode."""
- def _do_test(self, write, expect, tmpdir):
- """Run the acutal test."""
- p = tmpdir.join('foo')
- p.write(write)
- gen = deqp.iter_deqp_test_cases(six.text_type(p))
- assert next(gen) == expect
+ @classmethod
+ def setup_class(cls):
+ with mock.patch('framework.test.deqp.options.OPTIONS',
+ new=options._Options()) as mocked:
+ mocked.deqp_mode = 'group'
+ cls.profile = deqp.make_profile(
+ [('this.is.a.deqp', ['this.is.a.deqp.test',
+ 'this.is.a.deqp.thing'])],
+ group=_DEQPGroupTrieTest)
- def test_test_cases(self, tmpdir):
- """Correctly detects a test line."""
- self._do_test('TEST: a.deqp.test', 'a.deqp.test', tmpdir)
+ def test_returns_profile(self):
+ """deqp.make_profile: returns a TestProfile."""
+ assert isinstance(self.profile, profile.TestProfile)
- def test_test_group(self, tmpdir):
- """Correctly detects a group line."""
- self._do_test('GROUP: a group\nTEST: a.deqp.test', 'a.deqp.test',
- tmpdir)
+ def test_replaces_separator(self):
+ """deqp.make_profile: replaces '.' with grouptools.separator"""
+ expected = grouptools.join('this', 'is', 'a', 'deqp')
+ assert expected in self.profile.test_list
- def test_bad_entry(self, tmpdir):
- """A PiglitFatalException is raised if a line is not a TEST or GROUP.
- """
- with pytest.raises(exceptions.PiglitFatalError):
- self._do_test('this will fail', None, tmpdir)
+
+class TestIterDeqpTestCases(object):
+ """Tests for iter_deqp_test_cases."""
+
+ class TestSingle(object):
+ """Tests for the single mode."""
+
+ @pytest.yield_fixture(autouse=True, scope='class')
+ def group_setup(self):
+ with mock.patch('framework.test.deqp.options.OPTIONS',
+ new=options._Options()) as mocked:
+ mocked.deqp_mode = 'single'
+ yield
+
+ def _do_test(self, write, expect, tmpdir):
+ """Run the acutal test."""
+ p = tmpdir.join('foo')
+ p.write(write)
+ gen = deqp.iter_deqp_test_cases(six.text_type(p))
+ actual = next(gen)
+ assert actual == expect
+
+ def test_test_cases(self, tmpdir):
+ """Correctly detects a test line."""
+ self._do_test('TEST: a.deqp.test', ('a.deqp.test', ), tmpdir)
+
+ def test_test_group(self, tmpdir):
+ """Correctly detects a group line."""
+ self._do_test('GROUP: a group\nTEST: a.deqp.test',
+ ('a.deqp.test', ), tmpdir)
+
+ def test_bad_entry(self, tmpdir):
+ """A PiglitFatalException is raised if a line is not a TEST or
+ GROUP.
+ """
+ with pytest.raises(exceptions.PiglitFatalError):
+ self._do_test('this will fail', None, tmpdir)
+
+ class TestGroup(object):
+ """Tests for the group mode."""
+
+ @pytest.yield_fixture(autouse=True, scope='class')
+ def group_setup(self):
+ with mock.patch('framework.test.deqp.options.OPTIONS',
+ new=options._Options()) as mocked:
+ mocked.deqp_mode = 'group'
+ yield
+
+ def _do_test(self, write, expect, tmpdir):
+ """Run the acutal test."""
+ p = tmpdir.join('foo')
+ p.write(write)
+ gen = deqp.iter_deqp_test_cases(six.text_type(p))
+ actual = next(gen)
+ assert actual == expect
+
+ def test_test_cases(self, tmpdir):
+ """Correctly detects a test line."""
+ self._do_test(
+ textwrap.dedent("""\
+ GROUP: a
+ GROUP: a.deqp
+ TEST: a.deqp.test
+ TEST: a.deqp.failure
+ TEST: a.deqp.foo
+ """),
+ ('a.deqp', ['a.deqp.test', 'a.deqp.failure', 'a.deqp.foo']),
+ tmpdir)
+
+ def test_bad_entry(self, tmpdir):
+ """A PiglitFatalException is raised if a line is not a TEST or
+ GROUP.
+ """
+ with pytest.raises(exceptions.PiglitFatalError):
+ self._do_test('this will fail', None, tmpdir)
class TestFormatTrieList(object):
@@ -460,7 +540,7 @@ class TestDEQPGroupTest(object):
class TestGenMustpassTests(object):
"""Tests for the gen_mustpass_tests function."""
- _xml = textwrap.dedent("""\
+ xml = textwrap.dedent("""\
<?xml version="1.0" encoding="UTF-8"?>
<TestPackage name="dEQP-piglit-test" appPackageName="com.freedesktop.org.piglit.deqp" testType="deqpTest" xmlns:deqp="http://drawelements.com/deqp" deqp:glesVersion="196608">
<TestSuite name="dEQP.piglit">
@@ -478,13 +558,45 @@ class TestGenMustpassTests(object):
</TestPackage>
""")
- def test_basic(self, tmpdir):
- p = tmpdir.join('foo.xml')
- p.write(self._xml)
- tests = set(deqp.gen_mustpass_tests(six.text_type(p)))
- assert tests == {
- 'dEQP.piglit.group1.test1',
- 'dEQP.piglit.group1.test2',
- 'dEQP.piglit.nested.group2.test3',
- 'dEQP.piglit.nested.group2.test4',
- }
+ class TestSingle(object):
+ """Tests for single mode."""
+
+ @pytest.yield_fixture(autouse=True, scope='class')
+ def group_setup(self):
+ with mock.patch('framework.test.deqp.options.OPTIONS',
+ new=options._Options()) as mocked:
+ mocked.deqp_mode = 'single'
+ yield
+
+ def test_basic(self, tmpdir):
+ p = tmpdir.join('foo.xml')
+ p.write(TestGenMustpassTests.xml)
+ tests = set(deqp.gen_mustpass_tests(six.text_type(p)))
+ assert tests == {
+ ('dEQP.piglit.group1.test1', ),
+ ('dEQP.piglit.group1.test2', ),
+ ('dEQP.piglit.nested.group2.test3', ),
+ ('dEQP.piglit.nested.group2.test4', ),
+ }
+
+ class TestGroup(object):
+ """Tests for groupmode."""
+
+ @pytest.yield_fixture(autouse=True, scope='class')
+ def group_setup(self):
+ with mock.patch('framework.test.deqp.options.OPTIONS',
+ new=options._Options()) as mocked:
+ mocked.deqp_mode = 'group'
+ yield
+
+ def test_basic(self, tmpdir):
+ p = tmpdir.join('foo.xml')
+ p.write(TestGenMustpassTests.xml)
+ tests = list(deqp.gen_mustpass_tests(six.text_type(p)))
+ assert tests == [
+ ('dEQP.piglit.group1',
+ ['dEQP.piglit.group1.test1', 'dEQP.piglit.group1.test2']),
+ ('dEQP.piglit.nested.group2',
+ ['dEQP.piglit.nested.group2.test3',
+ 'dEQP.piglit.nested.group2.test4']),
+ ]
--
git-series 0.8.7
More information about the Piglit
mailing list