[Piglit] [PATCH 1/5] deqp: Add a group run mode for dEQP.
Dylan Baker
baker.dylan.c at gmail.com
Thu Mar 24 18:43:01 UTC 2016
Currently dEQP integration works by running one test at a time. This
provides good process isolation and stability, but at the cost of a good
deal of overhead to start each test. dEQP was not designed to be run one
test at a time, for all of it's tests. This overhead can obviously be
overcome by invoking dEQP directly which overcomes the performance
problem, but that has it's own problems.
This patch attempts to strike a balance between performance and
reliability by adding a new mode that runs groups of tests in a single
process. This works by finding each leaf node in the dEQP group
structure and running that as a single test process, and uses subtests
to report the result.
For piglit summary tools this is indistinguishable (although the
internal JSON representation is different), except for some metadata
(run time, stdout and stderr, etc), but the regressions/fixes/ etc will
be the same.
In my tests this reduces run time of the test suite between ~30% and
~50%.
deqp-vk can run in group mode, but doesn't have the skip conditionals.
That will be addressed in a follow up patch. There is also the issue
that the JUnit backend doesn't handle subtests. This is an ongoing
problem to solve.
Signed-off-by: Dylan Baker <dylanx.c.baker at intel.com>
---
framework/options.py | 3 +
framework/programs/run.py | 9 ++
framework/test/deqp.py | 155 +++++++++++++++++++++++++++++-
tests/cts.py | 16 +++-
tests/deqp_gles2.py | 18 +++-
tests/deqp_gles3.py | 19 ++--
tests/deqp_gles31.py | 17 +++-
tests/deqp_vk.py | 21 +++--
tox.ini | 2 +-
unittests/deqp_tests.py | 235 ++++++++++++++++++++++++++++++++++++++++++++--
unittests/utils.py | 37 ++++++++
11 files changed, 495 insertions(+), 37 deletions(-)
diff --git a/framework/options.py b/framework/options.py
index cf49520..52d90be 100644
--- a/framework/options.py
+++ b/framework/options.py
@@ -179,6 +179,8 @@ class _Options(object): # pylint: disable=too-many-instance-attributes
valgrind -- True if valgrind is to be used
dmesg -- True if dmesg checking is desired. This forces concurrency off
env -- environment variables set for each test before run
+ deqp_mode -- either 'test' or 'group. Controls deqp integration mode, to
+ either run test at a time or group at a time.
"""
include_filter = _ReListDescriptor('_include_filter', type_=_FilterReList)
@@ -193,6 +195,7 @@ class _Options(object): # pylint: disable=too-many-instance-attributes
self.valgrind = False
self.dmesg = False
self.sync = False
+ self.deqp_mode = 'test'
# env is used to set some base environment variables that are not going
# to change across runs, without sending them to os.environ which is
diff --git a/framework/programs/run.py b/framework/programs/run.py
index 581b350..5d171c1 100644
--- a/framework/programs/run.py
+++ b/framework/programs/run.py
@@ -164,6 +164,13 @@ def _run_parser(input_):
dest='overwrite',
action='store_true',
help='If the results_path already exists, delete it')
+ parser.add_argument('--deqp-test-mode',
+ action='store',
+ choices=['test', 'group'],
+ default='test',
+ dest='deqp_mode',
+ help='Run DEQP integration in either "group-at-a-time"'
+ ' or "test-at-a-time" mode.')
parser.add_argument("test_profile",
metavar="<Profile path(s)>",
nargs='+',
@@ -241,6 +248,7 @@ def run(input_):
options.OPTIONS.valgrind = args.valgrind
options.OPTIONS.dmesg = args.dmesg
options.OPTIONS.sync = args.sync
+ options.OPTIONS.deqp_mode = args.deqp_mode
# Set the platform to pass to waffle
options.OPTIONS.env['PIGLIT_PLATFORM'] = args.platform
@@ -320,6 +328,7 @@ def resume(input_):
options.OPTIONS.valgrind = results.options['valgrind']
options.OPTIONS.dmesg = results.options['dmesg']
options.OPTIONS.sync = results.options['sync']
+ options.OPTIONS.deqp_mode = results.options['deqp_mode']
core.get_config(args.config_file)
diff --git a/framework/test/deqp.py b/framework/test/deqp.py
index 80ce156..e848aeb 100644
--- a/framework/test/deqp.py
+++ b/framework/test/deqp.py
@@ -23,16 +23,19 @@ from __future__ import (
)
import abc
import os
+import re
import subprocess
import six
-from framework import core, grouptools, exceptions
+from framework import core, grouptools, exceptions, status
from framework.profile import TestProfile
from framework.test.base import Test, is_crash_returncode
+from framework.options import OPTIONS
__all__ = [
'DEQPBaseTest',
+ 'DEQPGroupTest',
'gen_caselist_txt',
'get_option',
'iter_deqp_test_cases',
@@ -60,13 +63,20 @@ _EXTRA_ARGS = get_option('PIGLIT_DEQP_EXTRA_ARGS',
default='').split()
-def make_profile(test_list, test_class):
+def make_profile(test_list, single_class=None, multi_class=None):
"""Create a TestProfile instance."""
+ if OPTIONS.deqp_mode == 'group':
+ assert multi_class is not None
+ _class = multi_class
+ elif OPTIONS.deqp_mode == 'test':
+ assert single_class is not None
+ _class = single_class
+
profile = TestProfile()
for testname in test_list:
# deqp uses '.' as the testgroup separator.
piglit_name = testname.replace('.', grouptools.SEPARATOR)
- profile.test_list[piglit_name] = test_class(testname)
+ profile.test_list[piglit_name] = _class(testname)
return profile
@@ -100,10 +110,41 @@ def gen_caselist_txt(bin_, caselist, extra_args):
return caselist_path
-def iter_deqp_test_cases(case_file):
+def _iterate_file(file_):
+ """Lazily iterate a file line-by-line."""
+ while True:
+ line = file_.readline()
+ if not line:
+ raise StopIteration
+ yield line
+
+
+def _iter_deqp_test_groups(case_file):
+ """Iterate over original dEQP testcase groups.
+
+ This generator yields the name of each leaf group (that is, a group which
+ contains only tests.)
+
+ """
+ slice_ = slice(len('GROUP: '), None)
+ group = ''
+ with open(case_file, 'r') as caselist_file:
+ for i, line in enumerate(_iterate_file(caselist_file)):
+ if line.startswith('GROUP:'):
+ group = line[slice_]
+ elif line.startswith('TEST:'):
+ if group != '':
+ yield group.rstrip()
+ group = ''
+ else:
+ raise exceptions.PiglitFatalError(
+ 'deqp: {}:{}: ill-formed line'.format(case_file, i))
+
+
+def _iter_deqp_test_single(case_file):
"""Iterate over original dEQP testcase names."""
with open(case_file, 'r') as caselist_file:
- for i, line in enumerate(caselist_file):
+ for i, line in enumerate(_iterate_file(caselist_file)):
if line.startswith('GROUP:'):
continue
elif line.startswith('TEST:'):
@@ -113,6 +154,14 @@ def iter_deqp_test_cases(case_file):
'deqp: {}:{}: ill-formed line'.format(case_file, i))
+def iter_deqp_test_cases(case_file):
+ """Wrapper that sets the iterator based on the mode."""
+ if OPTIONS.deqp_mode == 'group':
+ return _iter_deqp_test_groups(case_file)
+ elif OPTIONS.deqp_mode == 'test':
+ return _iter_deqp_test_single(case_file)
+
+
@six.add_metaclass(abc.ABCMeta)
class DEQPBaseTest(Test):
__RESULT_MAP = {
@@ -177,3 +226,99 @@ class DEQPBaseTest(Test):
# We failed to parse the test output. Fallback to 'fail'.
if self.result.result == 'notrun':
self.result.result = 'fail'
+
+
+class DEQPGroupTest(DEQPBaseTest):
+ timeout = 300 # 5 minutes
+ __name_slicer = slice(len("Test case '"), -len("'.."))
+ __finder = re.compile(r'^ (Warnings|Not supported|Failed|Passed):\s+\d/(?P<total>\d+).*')
+
+ # This a very hot path, a small speed optimization can be had by shortening
+ # this match to just one character
+ _RESULT_MAP = {
+ "P": status.PASS, # Pass
+ "F": status.FAIL, # Fail
+ "Q": status.WARN, # QualityWarnings
+ "I": status.FAIL, # InternalError
+ "C": status.CRASH, # Crash
+ "N": status.SKIP, # NotSupported
+ "R": status.CRASH, # ResourceError
+ }
+
+ def __init__(self, case_name, **kwargs):
+ super(DEQPGroupTest, self).__init__(case_name + '*', **kwargs)
+
+ def interpret_result(self):
+ """Group based result interpretation.
+
+ This method is used to find names of subtests and their results and put
+ them together.
+
+ It provides a block keyword argument, this should be a callable taking
+ the the line being processed as output. It may process the line, and
+ can raise an Exception descending from PiglitException to mark
+ conditions.
+
+ """
+ # This function is ugly and complicated. But it can be pretty easily
+ # understood as an extension of DEQPBaseTest.inrepret_result. In this
+ # case though there are multiple results, each being treated as a
+ # subtest. This function must not only find the result of each subtest,
+ # but the name as well.
+
+ # If the returncode is non-0 don't bother, call it crash and move on,
+ # since there will almost certinaly be an exception raised.
+ if self.result.returncode != 0:
+ self.result.result = 'crash'
+ return
+
+ # Strip the first 3 lines, and the last 8 lines, which aren't useful
+ # for this pass
+ lines = self.result.out.rstrip().split('\n')[3:]
+ cur = ''
+ total = None
+ for each in reversed(lines):
+ m = self.__finder.match(each)
+ if m:
+ total = int(m.group('total'))
+ break
+ assert total is not None, 'Could not calculate total test count'
+
+ lines = (l for l in lines[:-8])
+
+ # Walk over standard out line by line, looking for 'Test case' (to get
+ # the name of the test) and then for a result. Track each line, which
+ # is used to both know when to stop walking and for error reporting.
+ while len(self.result.subtests) < total:
+ for l in lines:
+ if l.startswith('Test case'):
+ name = l[self.__name_slicer].rsplit('.', 1)[1].lower()
+ break
+ else:
+ raise exceptions.PiglitInternalError(
+ 'Expected "Test case", but didn\'t find it in:\n'
+ '{}\ncurrent line: {}'.format(self.result.out, l))
+
+ for l in lines:
+ # If there is an info block fast forward through it by calling
+ # next on the generator until it is passed.
+ if l.startswith('INFO'):
+ cur = ''
+ while not (cur.startswith('INFO') and cur.endswith('----')):
+ cur = next(lines)
+
+ elif l.startswith(' '):
+ try:
+ self.result.subtests[name] = self._RESULT_MAP[l[2]]
+ except KeyError:
+ raise exceptions.PiglitInternalError(
+ 'Unknown status {}'.format(l[2:].split()[0]))
+ break
+ else:
+ raise exceptions.PiglitInternalError(
+ 'Expected " (Pass,Fail,...)", but didn\'t find it in:\n'
+ '{}\ncurrent line: {}'.format(self.result.out, l))
+
+ # We failed to parse the test output. Fallback to 'fail'.
+ if self.result.result == 'notrun':
+ self.result.result = 'fail'
diff --git a/tests/cts.py b/tests/cts.py
index 0e64e1b..e0b05d1 100644
--- a/tests/cts.py
+++ b/tests/cts.py
@@ -57,15 +57,25 @@ _EXTRA_ARGS = deqp.get_option('PIGLIT_CTS_EXTRA_ARGS', ('cts', 'extra_args'),
default='').split()
-class DEQPCTSTest(deqp.DEQPBaseTest):
+class _Mixin(object):
deqp_bin = _CTS_BIN
@property
def extra_args(self):
- return super(DEQPCTSTest, self).extra_args + \
+ return super(_Mixin, self).extra_args + \
[x for x in _EXTRA_ARGS if not x.startswith('--deqp-case')]
+class DEQPCTSTest(_Mixin, deqp.DEQPBaseTest):
+ """Class for running GLES CTS in test at a time mode."""
+ pass
+
+
+class DEQPCTSGroupTest(_Mixin, deqp.DEQPGroupTest):
+ """Class for running GLES CTS in group at a time mode."""
+ pass
+
+
# Add all of the suites by default, users can use filters to remove them.
profile = deqp.make_profile( # pylint: disable=invalid-name
itertools.chain(
@@ -79,4 +89,4 @@ profile = deqp.make_profile( # pylint: disable=invalid-name
deqp.gen_caselist_txt(_CTS_BIN, 'ESEXT-CTS-cases.txt',
_EXTRA_ARGS)),
),
- DEQPCTSTest)
+ single_class=DEQPCTSTest, multi_class=DEQPCTSGroupTest)
diff --git a/tests/deqp_gles2.py b/tests/deqp_gles2.py
index ab897bd..5bf3672 100644
--- a/tests/deqp_gles2.py
+++ b/tests/deqp_gles2.py
@@ -1,4 +1,4 @@
-# Copyright 2015 Intel Corporation
+# Copyright (c) 2015-2016 Intel Corporation
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
@@ -37,18 +37,28 @@ _EXTRA_ARGS = deqp.get_option('PIGLIT_DEQP_GLES2_EXTRA_ARGS',
default='').split()
-class DEQPGLES2Test(deqp.DEQPBaseTest):
+class _Mixin(object):
+ """Mixin class that provides shared methods for dEQP-GLES2."""
deqp_bin = _DEQP_GLES2_BIN
@property
def extra_args(self):
- return super(DEQPGLES2Test, self).extra_args + \
+ return super(_Mixin, self).extra_args + \
[x for x in _EXTRA_ARGS if not x.startswith('--deqp-case')]
+class DEQPGLES2Test(_Mixin, deqp.DEQPBaseTest):
+ """Class for running dEQP GLES2 in test at a time mode."""
+ pass
+
+
+class DEQPGLES2GroupTest(_Mixin, deqp.DEQPGroupTest):
+ """Class for running dEQP GLES2 in group at a time mode."""
+ pass
+
profile = deqp.make_profile( # pylint: disable=invalid-name
deqp.iter_deqp_test_cases(
deqp.gen_caselist_txt(_DEQP_GLES2_BIN, 'dEQP-GLES2-cases.txt',
_EXTRA_ARGS)),
- DEQPGLES2Test)
+ single_class=DEQPGLES2Test, multi_class=DEQPGLES2GroupTest)
diff --git a/tests/deqp_gles3.py b/tests/deqp_gles3.py
index 783407d..c50033b 100644
--- a/tests/deqp_gles3.py
+++ b/tests/deqp_gles3.py
@@ -1,4 +1,4 @@
-# Copyright 2014, 2015 Intel Corporation
+# Copyright (c) 2014-2016 Intel Corporation
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
@@ -79,21 +79,28 @@ def filter_mustpass(caselist_path):
return caselist_path
-class DEQPGLES3Test(deqp.DEQPBaseTest):
+class _Mixin(object):
+ """Mixin class that provides shared methods for dEQP-GLES3."""
deqp_bin = _DEQP_GLES3_EXE
@property
def extra_args(self):
- return super(DEQPGLES3Test, self).extra_args + \
+ return super(_Mixin, self).extra_args + \
[x for x in _EXTRA_ARGS if not x.startswith('--deqp-case')]
- def __init__(self, *args, **kwargs):
- super(DEQPGLES3Test, self).__init__(*args, **kwargs)
+class DEQPGLES3Test(_Mixin, deqp.DEQPBaseTest):
+ """Class for running dEQP GLES3 in group at a time mode."""
+ pass
+
+
+class DEQPGLES3GroupTest(_Mixin, deqp.DEQPGroupTest):
+ """Class for running dEQP GLES3 in group at a time mode."""
+ pass
profile = deqp.make_profile( # pylint: disable=invalid-name
deqp.iter_deqp_test_cases(filter_mustpass(
deqp.gen_caselist_txt(_DEQP_GLES3_EXE, 'dEQP-GLES3-cases.txt',
_EXTRA_ARGS))),
- DEQPGLES3Test)
+ single_class=DEQPGLES3Test, multi_class=DEQPGLES3GroupTest)
diff --git a/tests/deqp_gles31.py b/tests/deqp_gles31.py
index f516a84..666dcab 100644
--- a/tests/deqp_gles31.py
+++ b/tests/deqp_gles31.py
@@ -1,4 +1,4 @@
-# Copyright 2015 Intel Corporation
+# Copyright (c) 2015-2016 Intel Corporation
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
@@ -37,7 +37,8 @@ _EXTRA_ARGS = deqp.get_option('PIGLIT_DEQP_GLES31_EXTRA_ARGS',
default='').split()
-class DEQPGLES31Test(deqp.DEQPBaseTest):
+class _Mixin(object):
+ """Mixin class that provides shared methods for dEQP-GLES31."""
deqp_bin = _DEQP_GLES31_BIN
@property
@@ -46,8 +47,18 @@ class DEQPGLES31Test(deqp.DEQPBaseTest):
[x for x in _EXTRA_ARGS if not x.startswith('--deqp-case')]
+class DEQPGLES31Test(_Mixin, deqp.DEQPBaseTest):
+ """Class for running dEQP GLES31 in group at a time mode."""
+ pass
+
+
+class DEQPGLES31GroupTest(_Mixin, deqp.DEQPGroupTest):
+ """Class for running dEQP GLES31 in group at a time mode."""
+ pass
+
+
profile = deqp.make_profile( # pylint: disable=invalid-name
deqp.iter_deqp_test_cases(
deqp.gen_caselist_txt(_DEQP_GLES31_BIN, 'dEQP-GLES31-cases.txt',
_EXTRA_ARGS)),
- DEQPGLES31Test)
+ single_class=DEQPGLES31Test, multi_class=DEQPGLES31GroupTest)
diff --git a/tests/deqp_vk.py b/tests/deqp_vk.py
index f968179..9a31c46 100644
--- a/tests/deqp_vk.py
+++ b/tests/deqp_vk.py
@@ -30,6 +30,7 @@ from __future__ import (
import re
from framework.test import deqp
+from framework import exceptions
__all__ = ['profile']
@@ -45,17 +46,18 @@ _DEQP_ASSERT = re.compile(
r'deqp-vk: external/vulkancts/.*: Assertion `.*\' failed.')
-class DEQPVKTest(deqp.DEQPBaseTest):
- """Test representation for Khronos Vulkan CTS."""
- timeout = 60
+class _Mixin(object):
+ """Mixin class that provides shared methods for dEQP-VK."""
deqp_bin = _DEQP_VK_BIN
@property
def extra_args(self):
- return super(DEQPVKTest, self).extra_args + \
+ return super(_Mixin, self).extra_args + \
[x for x in _EXTRA_ARGS if not x.startswith('--deqp-case')]
- def __init__(self, *args, **kwargs):
- super(DEQPVKTest, self).__init__(*args, **kwargs)
+
+class DEQPVKTest(_Mixin, deqp.DEQPBaseTest):
+ """Test representation for Khronos Vulkan CTS."""
+ timeout = 60
def interpret_result(self):
if 'Failed to compile shader at vkGlslToSpirV' in self.result.out:
@@ -70,8 +72,13 @@ class DEQPVKTest(deqp.DEQPBaseTest):
super(DEQPVKTest, self).interpret_result()
+class DEQPVKGroupTest(_Mixin, deqp.DEQPGroupTest):
+ """Test representation for Khronos Vulkacn CTS in group mode."""
+ pass
+
+
profile = deqp.make_profile( # pylint: disable=invalid-name
deqp.iter_deqp_test_cases(
deqp.gen_caselist_txt(_DEQP_VK_BIN, 'dEQP-VK-cases.txt',
_EXTRA_ARGS)),
- DEQPVKTest)
+ single_class=DEQPVKTest, multi_class=DEQPVKGroupTest)
diff --git a/tox.ini b/tox.ini
index a1556fa..9a428f0 100644
--- a/tox.ini
+++ b/tox.ini
@@ -12,7 +12,7 @@ deps =
accel: simplejson
accel: lxml
py27-accel,py{33,34,35}: psutil
- py27-{accel,noaccel}: mock==1.0.1
+ py{27,33}-{accel,noaccel}: mock==1.0.1
py27-accel: backports.lzma
py27-accel: subprocess32
py{27,33,34}: mako==0.8.0
diff --git a/unittests/deqp_tests.py b/unittests/deqp_tests.py
index 7119fed..c7cc5e4 100644
--- a/unittests/deqp_tests.py
+++ b/unittests/deqp_tests.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2015 Intel Corporation
+# Copyright (c) 2015-2016 Intel Corporation
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
@@ -28,21 +28,26 @@ tests
from __future__ import (
absolute_import, division, print_function, unicode_literals
)
+import sys
import textwrap
-try:
- from unittest import mock
-except ImportError:
- import mock
-
import nose.tools as nt
-import six
+
+# There is a bug in mock < 1.2 or python 3.4 that we'd like to avoid, otherwise
+# some tests will skip.
+if sys.version_info[0:2] >= (3, 4):
+ from unittest import mock
+else:
+ try:
+ import mock
+ except ImportError:
+ from unittest import mock
from framework import profile, grouptools, exceptions
from framework.test import deqp
from . import utils
-# pylint:disable=line-too-long,invalid-name
+# pylint:disable=line-too-long,invalid-name,protected-access
doc_formatter = utils.DocFormatter({'separator': grouptools.SEPARATOR})
@@ -52,6 +57,11 @@ class _DEQPTestTest(deqp.DEQPBaseTest):
extra_args = ['extra']
+class _DEQPGroupTest(deqp.DEQPGroupTest):
+ deqp_bin = 'deqp.bin'
+ extra_args = ['extra']
+
+
@utils.set_piglit_conf(('deqp_test', 'test_env', 'from conf'))
@utils.set_env(_PIGLIT_TEST_ENV='from env')
def test_get_option_env():
@@ -270,3 +280,212 @@ class TestDEQPBaseTestIntepretResultStatus(object):
self.inst.result.out = self.__gen_stdout('ResourceError')
self.inst.interpret_result()
nt.eq_(self.inst.result.result, 'crash')
+
+
+class TestDEQPGroupTest_interpret_result(object):
+ __out = textwrap.dedent("""
+ dEQP Core unknown (0xcafebabe) starting..
+ target implementation = 'X11 GLX'
+
+ Test case 'dEQP-GLES3.functional.fragment_out.random.0'..
+ Vertex shader compile time = 4.134000 ms
+ Fragment shader compile time = 0.345000 ms
+ Link time = 2.442000 ms
+ Test case duration in microseconds = 10164 us
+ Fail (After program setup: glGetError() returned GL_INVALID_FRAMEBUFFER_OPERATION at es3fFragmentOutputTests.cpp:706)
+
+ Test case 'dEQP-GLES3.functional.fragment_out.random.1'..
+ Vertex shader compile time = 0.352000 ms
+ Fragment shader compile time = 0.276000 ms
+ Link time = 2.625000 ms
+ Test case duration in microseconds = 3894 us
+ Pass (After program setup: glGetError() returned GL_INVALID_FRAMEBUFFER_OPERATION at es3fFragmentOutputTests.cpp:706)
+
+ DONE!
+
+ Test run totals:
+ Passed: 2/2 (100.0%)
+ Failed: 0/2 (0.0%)
+ Not supported: 0/2 (0.0%)
+ Warnings: 0/2 (0.0%)
+ """)
+
+ @classmethod
+ def setup_class(cls):
+ cls.test = _DEQPGroupTest('foo')
+ cls.test.result.returncode = 0
+ cls.test.result.out = cls.__out
+ cls.test.interpret_result()
+
+ def test_name(self):
+ """test.deqp.DEQPGroupTest: set's name properly"""
+ nt.assert_set_equal({'0', '1'}, set(self.test.result.subtests.keys()))
+
+ def test_status(self):
+ """test.deqp.DEQPGroupTest: set's status properly"""
+ nt.assert_dict_equal(
+ {'0': 'fail', '1': 'pass'},
+ dict(self.test.result.subtests))
+
+
+def test_DEQPGroupTest_interpret_result_cts():
+ """test.deqp.DEQPGroupTest.interpret_result: Handles CTS shader dumps."""
+ # The following is just something that looks kind of like a CTS shader, the
+ # point is that the layout doesn't trip up the intepret_result method
+ out = textwrap.dedent("""\
+ dEQP Core GL-CTS-2.0 (0x0052484b) starting..
+ target implementation = 'intel-gbm'
+
+ Test case 'A.Test.case.1'..
+ INFO:a test-------------------------------- BEGIN ---------------------------------
+ INFO:a test
+
+ [VERTEX SHADER]
+
+ #version foobar
+ #ifdef something
+ in something
+ INFO:mo stuff:
+
+ [FRAGMENT SHADER]
+
+ #version 300 es
+ precision highp int;
+
+ struct S {
+ vec4 foo;
+ vec2 fo[2];
+ };
+ layout(std140) uniform UB0 {
+ S x;
+ S y[2];
+ } ub0;
+ INFO:even more stuff:
+
+ [VERTEX SHADER]
+
+ #version 300 es
+ bool something () {
+ if (thing) { do! }
+ INFO:and even more stuff:
+
+ [FRAGMENT SHADER]
+
+ #version 300 es
+ precision highp int;
+
+ INFO:a test:OK
+ INFO:a test:--------------------------------- END ----------------------------------
+ Pass (Pass)
+
+ Test case 'A.Test.case.2'..
+ INFO:a test-------------------------------- BEGIN ---------------------------------
+ INFO:a test
+
+ [VERTEX SHADER]
+
+ #version foobar
+ #ifdef something
+ in something
+ INFO:mo stuff:
+
+ [FRAGMENT SHADER]
+
+ #version 300 es
+ precision highp int;
+
+ struct S {
+ vec4 boo;
+ vec2 bo[2];
+ };
+ layout(std140) uniform UB0 {
+ S x;
+ S y[2];
+ } ub0;
+ INFO:even more stuff:
+
+ [VERTEX SHADER]
+
+ #version 300 es
+ bool something () {
+ if (thing) { do! }
+ INFO:and even more stuff:
+
+ [FRAGMENT SHADER]
+
+ #version 300 es
+ precision highp int;
+
+ INFO:a test:OK
+ INFO:a test:--------------------------------- END ----------------------------------
+ Pass (Pass)
+
+ DONE!
+
+ Test run totals:
+ Passed: 2/2 (100.00%)
+ Failed: 0/2 (0.00%)
+ Not supported: 0/2 (0.00%)
+ Warnings: 0/2 (0.00%)
+ """)
+
+ test = _DEQPGroupTest('foo')
+ test.result.returncode = 0
+ test.result.out = out
+ test.interpret_result()
+ nt.eq_(test.result.subtests['1'], 'pass')
+ nt.eq_(test.result.subtests['2'], 'pass')
+
+
+def test_DEQPGroupTest_interpret_result_nonzero():
+ """test.deqp.DEQPGroupTest.interpret_results: if returncode is nonzero test is crash"""
+ test = _DEQPGroupTest('foo')
+ test.result.returncode = -6
+ test.interpret_result()
+ nt.eq_(test.result.result, 'crash')
+
+
+ at utils.skip(not (sys.version_info[0:2] >= (3, 4) or
+ float(mock.__version__[:3]) >= 1.2),
+ 'Test requires that mock.mock_open provides readline method.')
+def test_iter_deqp_test_groups():
+ """deqp._test_deqp_test_groups: Returns expected values"""
+ text = textwrap.dedent("""\
+ GROUP: dEQP-GLES2.info
+ TEST: dEQP-GLES2.info.vendor
+ TEST: dEQP-GLES2.info.renderer
+ TEST: dEQP-GLES2.info.version
+ TEST: dEQP-GLES2.info.shading_language_version
+ TEST: dEQP-GLES2.info.extensions
+ TEST: dEQP-GLES2.info.render_target
+ GROUP: dEQP-GLES2.capability
+ GROUP: dEQP-GLES2.capability.limits
+ TEST: dEQP-GLES2.capability.limits.vertex_attribs
+ TEST: dEQP-GLES2.capability.limits.varying_vectors
+ TEST: dEQP-GLES2.capability.limits.vertex_uniform_vectors
+ TEST: dEQP-GLES2.capability.limits.fragment_uniform_vectors
+ TEST: dEQP-GLES2.capability.limits.texture_image_units
+ TEST: dEQP-GLES2.capability.limits.vertex_texture_image_units
+ TEST: dEQP-GLES2.capability.limits.combined_texture_image_units
+ TEST: dEQP-GLES2.capability.limits.texture_2d_size
+ TEST: dEQP-GLES2.capability.limits.texture_cube_size
+ TEST: dEQP-GLES2.capability.limits.renderbuffer_size
+ GROUP: dEQP-GLES2.capability.limits_lower
+ TEST: dEQP-GLES2.capability.limits_lower.minimum_size
+ GROUP: dEQP-GLES2.capability.extensions
+ GROUP: dEQP-GLES2.capability.extensions.uncompressed_texture_formats
+ TEST: dEQP-GLES2.capability.extensions.uncompressed_texture_formats.foo
+ """)
+
+ expected = [
+ 'dEQP-GLES2.info',
+ 'dEQP-GLES2.capability.limits',
+ 'dEQP-GLES2.capability.limits_lower',
+ 'dEQP-GLES2.capability.extensions.uncompressed_texture_formats',
+ ]
+
+ with mock.patch('framework.test.deqp.open', create=True,
+ new=mock.mock_open(read_data=text)):
+ actual = list(deqp._iter_deqp_test_groups(None))
+
+ nt.assert_list_equal(actual, expected)
diff --git a/unittests/utils.py b/unittests/utils.py
index 178d24b..eb33631 100644
--- a/unittests/utils.py
+++ b/unittests/utils.py
@@ -38,6 +38,10 @@ import subprocess
import errno
import importlib
from contextlib import contextmanager
+try:
+ from unittest import mock
+except ImportError:
+ import mock
try:
import simplejson as json
@@ -537,3 +541,36 @@ def unset_compression():
os.environ['PIGLIT_COMPRESSION'] = _SAVED_COMPRESSION
else:
del os.environ['PIGLIT_COMPRESSION']
+
+
+def skip(condition, msg='Test skipped.'):
+ """Decorator that will skip if the condition evaluates to Truthy.
+
+ Arguments:
+ condition -- A value or condition. If Truthy the test will skip.
+
+ Keyword Arugments:
+ msg -- a message to print if the test skips. default: 'Test skipped.'
+
+ >>> @skip(True)
+ ... def mytest():
+ ... pass
+ >>> mytest()
+ Traceback (most recent call last):
+ ...
+ unittest.case.SkipTest: Test skipped.
+
+ >>> @skip(False)
+ ... def mytest():
+ ... pass
+ >>> mytest()
+
+ """
+ def _wrapper(func):
+ @functools.wraps(func)
+ def _inner(*args, **kwargs):
+ if condition:
+ raise SkipTest(msg)
+ return func(*args, **kwargs)
+ return _inner
+ return _wrapper
--
2.7.4
More information about the Piglit
mailing list