[Piglit] [PATCH 1/5] framework/test/deqp.py: Move generic parts of deqp into core
Dylan Baker
baker.dylan.c at gmail.com
Thu May 21 14:18:39 PDT 2015
DEQP provides a large number of test profiles, but currently we can only
use the gles3 profile. It would be useful to get at the gles31 profile
and the gles2 profile at least, if not also the egl profile.
This also adds unit tests for the shared parts of deqp.
Signed-off-by: Dylan Baker <dylanx.c.baker at intel.com>
---
framework/test/deqp.py | 150 ++++++++++++++++++++++++++++++++++++
framework/tests/deqp_tests.py | 171 ++++++++++++++++++++++++++++++++++++++++++
framework/tests/utils.py | 94 +++++++++++++++++++++--
3 files changed, 407 insertions(+), 8 deletions(-)
create mode 100644 framework/test/deqp.py
create mode 100644 framework/tests/deqp_tests.py
diff --git a/framework/test/deqp.py b/framework/test/deqp.py
new file mode 100644
index 0000000..a63611d
--- /dev/null
+++ b/framework/test/deqp.py
@@ -0,0 +1,150 @@
+# Copyright 2014, 2015 Intel Corporation
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import abc
+import os
+import subprocess
+
+# Piglit modules
+from framework import core, grouptools, exceptions
+from framework.profile import Test, TestProfile
+
+__all__ = [
+ 'DEQPBaseTest',
+ 'gen_caselist_txt',
+ 'get_option',
+ 'iter_deqp_test_cases',
+ 'make_profile',
+]
+
+
+def make_profile(test_list, test_class):
+ """Create a TestProfile instance."""
+ profile = TestProfile()
+ for testname in test_list:
+ # deqp uses '.' as the testgroup separator.
+ piglit_name = testname.replace('.', grouptools.SEPARATOR)
+ profile.test_list[piglit_name] = test_class(testname)
+
+ return profile
+
+
+def get_option(env_varname, config_option):
+ """Query the given environment variable and then piglit.conf for the option.
+
+ Return None if the option is unset.
+
+ """
+ opt = os.environ.get(env_varname, None)
+ if opt is not None:
+ return opt
+
+ opt = core.PIGLIT_CONFIG.safe_get(config_option[0], config_option[1])
+
+ return opt
+
+
+def gen_caselist_txt(bin_, caselist):
+ """Generate a caselist.txt and return its path."""
+ # dEQP is stupid (2014-12-07):
+ # 1. To generate the caselist file, dEQP requires that the process's
+ # current directory must be that same as that of the executable.
+ # Otherwise, it fails to find its data files.
+ # 2. dEQP creates the caselist file in the process's current directory
+ # and provides no option to change its location.
+ # 3. dEQP creates a GL context when generating the caselist. Therefore,
+ # the caselist must be generated on the test target rather than the
+ # build host. In other words, when the build host and test target
+ # differ then we cannot pre-generate the caselist on the build host:
+ # we must *dynamically* generate it during the testrun.
+ basedir = os.path.dirname(bin_)
+ caselist_path = os.path.join(basedir, caselist)
+ # TODO: need to catch some exceptions here...
+ subprocess.check_call(
+ [bin_, '--deqp-runmode=txt-caselist'], cwd=basedir)
+ assert os.path.exists(caselist_path)
+ return caselist_path
+
+
+def iter_deqp_test_cases(case_file):
+ """Iterate over original dEQP testcase names."""
+ with open(case_file, 'r') as caselist_file:
+ for i, line in enumerate(caselist_file):
+ if line.startswith('GROUP:'):
+ continue
+ elif line.startswith('TEST:'):
+ yield line[len('TEST:'):].strip()
+ else:
+ raise exceptions.PiglitFatalError(
+ 'deqp: {}:{}: ill-formed line'.format(case_file, i))
+
+
+class DEQPBaseTest(Test):
+ __metaclass__ = abc.ABCMeta
+ __RESULT_MAP = {"Pass": "pass",
+ "Fail": "fail",
+ "QualityWarning": "warn",
+ "InternalError": "fail",
+ "Crash": "crash",
+ "NotSupported": "skip"}
+
+ @abc.abstractproperty
+ def deqp_bin(self):
+ """The path to the exectuable."""
+
+ @abc.abstractproperty
+ def extra_args(self):
+ """Extra arguments to be passed to the each test instance.
+
+ Needs to return a list, since self.command uses the '+' operator, which
+ only works to join two lists together.
+
+ """
+
+ def __init__(self, case_name):
+ command = [self.deqp_bin, '--deqp-case=' + case_name]
+
+ super(DEQPBaseTest, self).__init__(command)
+
+ # dEQP's working directory must be the same as that of the executable,
+ # otherwise it cannot find its data files (2014-12-07).
+ # This must be called after super or super will overwrite it
+ self.cwd = os.path.dirname(self.deqp_bin)
+
+ @Test.command.getter
+ def command(self):
+ """Return the command plus any extra arguments."""
+ command = super(DEQPBaseTest, self).command
+ return command + self.extra_args
+
+ def interpret_result(self):
+ if self.result['returncode'] != 0:
+ self.result['result'] = 'fail'
+ return
+
+ for line in self.result['out'].split('\n'):
+ line = line.lstrip()
+ for k, v in self.__RESULT_MAP.iteritems():
+ if line.startswith(k):
+ self.result['result'] = v
+ return
+
+ # We failed to parse the test output. Fallback to 'fail'.
+ self.result['result'] = 'fail'
diff --git a/framework/tests/deqp_tests.py b/framework/tests/deqp_tests.py
new file mode 100644
index 0000000..d472c75
--- /dev/null
+++ b/framework/tests/deqp_tests.py
@@ -0,0 +1,171 @@
+# Copyright (c) 2015 Intel Corporation
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+"""Tests for the dEQP integration in framework.
+
+This tests the core framework shared code, and not the individual packages in
+tests
+
+"""
+
+import nose.tools as nt
+
+from framework import profile, grouptools, exceptions
+from framework.core import PIGLIT_CONFIG
+from framework.test import deqp
+from framework.tests import utils
+
+# pylint:disable=line-too-long,invalid-name
+
+doc_formatter = utils.DocFormatter({'separator': grouptools.SEPARATOR})
+
+
+class _DEQPTestTest(deqp.DEQPBaseTest):
+ deqp_bin = 'deqp.bin'
+ extra_args = ['extra']
+
+
+ at utils.set_piglit_conf(('deqp_test', 'test_env', 'from conf'))
+ at utils.set_env(_PIGLIT_TEST_ENV='from env')
+def test_get_option_env():
+ """deqp.get_option: if env is set it overrides piglit.conf"""
+ nt.eq_(deqp.get_option('_PIGLIT_TEST_ENV', ('deqp_test', 'test_env')),
+ 'from env')
+
+
+ at utils.set_piglit_conf(('deqp_test', 'test_env', 'from conf'))
+ at utils.set_env(_PIGLIT_TEST_ENV=None)
+def test_get_option_conf():
+ """deqp.get_option: if env is not set a value is taken from piglit.conf"""
+ nt.eq_(deqp.get_option('_PIGLIT_TEST_ENV', ('deqp_test', 'test_env')),
+ 'from conf')
+
+
+ at utils.set_env(_PIGLIT_TEST_ENV=None)
+def test_get_option_conf_no_section():
+ """deqp.get_option: if a no_section error is raised and env is unset None is return
+ """
+ assert not PIGLIT_CONFIG.has_section('deqp_test')
+ nt.eq_(deqp.get_option('_PIGLIT_TEST_ENV', ('deqp_test', 'test_env')), None)
+
+
+# The first argument ensures the sectio exists
+ at utils.set_piglit_conf(('deqp_test', 'test_env', 'from conf'),
+ ('deqp_test', 'not_exists', None))
+ at utils.set_env(_PIGLIT_TEST_ENV=None)
+def test_get_option_conf_no_option():
+ """deqp.get_option: if a no_option error is raised and env is unset None is return
+ """
+ nt.eq_(deqp.get_option('_PIGLIT_TEST_ENV', ('deqp_test', 'not_exists')),
+ None)
+
+
+class TestMakeProfile(object):
+ """Test deqp.make_profile."""
+ @classmethod
+ def setup_class(cls):
+ cls.profile = deqp.make_profile(['this.is.a.deqp.test'], _DEQPTestTest)
+
+ def test_returns_profile(self):
+ """deqp.make_profile: returns a TestProfile"""
+ nt.assert_is_instance(self.profile, profile.TestProfile)
+
+ @doc_formatter
+ def test_grouptools(self):
+ """deqp.make_profile: replaces '.' with '{separator}'"""
+ nt.assert_in(grouptools.join('this', 'is', 'a', 'deqp', 'test'),
+ self.profile.test_list)
+
+
+def test_iter_deqp_test_cases_test():
+ """deqp.iter_deqp_test_cases: correctly detects a TEST: line"""
+ with utils.tempfile('TEST: a.deqp.test') as tfile:
+ gen = deqp.iter_deqp_test_cases(tfile)
+ nt.eq_('a.deqp.test', next(gen))
+
+
+def test_iter_deqp_test_cases_group():
+ """deqp.iter_deqp_test_casesgen_caselist_txt: correctly detects a GROUP: line"""
+ with utils.tempfile('GROUP: a group\nTEST: a.deqp.test') as tfile:
+ gen = deqp.iter_deqp_test_cases(tfile)
+ nt.eq_('a.deqp.test', next(gen))
+
+
+ at nt.raises(exceptions.PiglitFatalError)
+def test_iter_deqp_test_cases_bad():
+ """deqp.iter_deqp_test_casesgen_caselist_txt: PiglitFatalException is raised if line is not TEST: or GROUP:
+ """
+ with utils.tempfile('this will fail') as tfile:
+ gen = deqp.iter_deqp_test_cases(tfile)
+ nt.eq_('a.deqp.test', next(gen))
+
+
+ at utils.no_error
+def test_DEQPBaseTest_initialize():
+ """deqp.DEQPBaseTest: can be initialized (with abstract methods overwritten)
+ """
+ _DEQPTestTest('a.deqp.test')
+
+
+def test_DEQPBaseTest_command():
+ """deqp.DEQPBaseTest.command: cls.extra_args are added to self.command"""
+ test = _DEQPTestTest('a.deqp.test')
+ nt.eq_(test.command[-1], 'extra')
+
+
+def test_DEQPBaseTest_interpret_result_returncode():
+ """deqp.DEQPBaseTest.interpret_result: if returncode is not 0 result is fail
+ """
+ test = _DEQPTestTest('a.deqp.test')
+ test.result['returncode'] = 1
+ test.interpret_result()
+
+ nt.eq_(test.result['result'], 'fail')
+
+
+def test_DEQPBaseTest_interpret_result_fallthrough():
+ """deqp.DEQPBaseTest.interpret_result: if no case is hit set to fail
+ """
+ test = _DEQPTestTest('a.deqp.test')
+ test.result['returncode'] = 0
+ test.result['out'] = ''
+ test.interpret_result()
+
+ nt.eq_(test.result['result'], 'fail')
+
+
+ at utils.nose_generator
+def test_DEQPBaseTest_interpret_result_status():
+ """generate tests for each status possiblility."""
+ def test(status, expected):
+ inst = _DEQPTestTest('a.deqp.test')
+ inst.result['returncode'] = 0
+ inst.result['out'] = status
+ inst.interpret_result()
+ nt.eq_(inst.result['result'], expected)
+
+ desc = ('deqp.DEQPBaseTest.interpret_result: '
+ 'when "{}" in stdout status is set to "{}"')
+
+ _map = deqp.DEQPBaseTest._DEQPBaseTest__RESULT_MAP.iteritems() # pylint: disable=no-member,protected-access
+
+ for status, expected in _map:
+ test.description = desc.format(status, expected)
+ yield test, status, expected
diff --git a/framework/tests/utils.py b/framework/tests/utils.py
index 9fd1cc8..a970984 100644
--- a/framework/tests/utils.py
+++ b/framework/tests/utils.py
@@ -40,7 +40,7 @@ except ImportError:
import json
from nose.plugins.skip import SkipTest
-from framework import test, backends, results
+from framework import test, backends, results, core
__all__ = [
@@ -50,6 +50,8 @@ __all__ = [
'JSON_DATA'
]
+core.get_config()
+
class _Tree(dict):
"""Private helper to make JSON_DATA easier to work with."""
@@ -362,16 +364,92 @@ def capture_stderr(func):
return _inner
-def not_raises(exception):
- """Decorator for tests that should not raise one of the follow exceptions.
+def set_env(**envargs):
+ """Decorator that sets environment variables and then unsets them.
+
+ If an value is set to None that key will be deleted from os.environ
+
"""
- def _decorator(function):
- @functools.wraps(function)
+
+ def _decorator(func):
+ """The actual decorator."""
+
+ @functools.wraps(func)
def _inner(*args, **kwargs):
+ """The returned function."""
+ backup = {}
+ for key, value in envargs.iteritems():
+ backup[key] = os.environ.get(key, "__DONOTRESTORE__")
+ if value is not None:
+ os.environ[key] = value
+ elif key in os.environ:
+ del os.environ[key]
+
try:
- function(*args, **kwargs)
- except exception as e:
- raise TestFailure(e)
+ func(*args, **kwargs)
+ finally:
+ for key, value in backup.iteritems():
+ if value == "__DONOTRESTORE__" and key in os.environ:
+ del os.environ[key]
+ else:
+ os.environ[key] = value
+
+ return _inner
+
+ return _decorator
+
+
+def set_piglit_conf(*values):
+ """Decorator that sets and then usets values from core.PIGLIT_CONF.
+
+ This decorator takes arguments for sections and options to overwrite in
+ piglit.conf. It will first backup any options to be overwritten, and store
+ any options that don't exist. Then it will set those options, run the test,
+ and finally restore any options overwritten, and delete any new options
+ added. If value is set to NoneType the option will be removed.
+
+ Arguments:
+ Values -- tuples containing a section, option, and value in the form:
+ (<section>, <option>, <value>)
+
+ """
+ def _decorator(func):
+ """The actual decorator."""
+
+ @functools.wraps(func)
+ def _inner(*args, **kwargs):
+ """The function returned by the decorator."""
+ backup = set()
+ remove = set()
+
+ for section, key, value in values:
+ get = core.PIGLIT_CONFIG.safe_get(section, key)
+ # If there is a value, save that value to restore it, if there
+ # is not a value AND if there is a value to set (IE: we're not
+ # clearing a value if it exsists), the add it to remove
+ if get is not None:
+ backup.add((section, key, get))
+ elif value is not None:
+ remove.add((section, key))
+
+ # set any new values, and remove any values that are set to
+ # None
+ if value is not None:
+ if not core.PIGLIT_CONFIG.has_section(section):
+ core.PIGLIT_CONFIG.add_section(section)
+ core.PIGLIT_CONFIG.set(section, key, value)
+ elif (core.PIGLIT_CONFIG.has_section(section) and
+ core.PIGLIT_CONFIG.has_option(section, key)):
+ core.PIGLIT_CONFIG.remove_option(section, key)
+
+ try:
+ func(*args, **kwargs)
+ finally:
+ # Restore all values
+ for section, key, value in backup:
+ core.PIGLIT_CONFIG.set(section, key, value)
+ for section, key in remove:
+ core.PIGLIT_CONFIG.remove_option(section, key)
return _inner
--
2.4.1
More information about the Piglit
mailing list