[Piglit] [PATCH 21/49] unittests: port deqp tests to pytest
Dylan Baker
dylan at pnwbakers.com
Fri Jul 29 18:39:07 UTC 2016
Signed-off-by: Dylan Baker <dylanx.c.baker at intel.com>
---
unittests/deqp_tests.py | 282 --------------------------------
unittests/framework/test/test_deqp.py | 296 ++++++++++++++++++++++++++++++++++
2 files changed, 296 insertions(+), 282 deletions(-)
delete mode 100644 unittests/deqp_tests.py
create mode 100644 unittests/framework/test/test_deqp.py
diff --git a/unittests/deqp_tests.py b/unittests/deqp_tests.py
deleted file mode 100644
index e3cd5d5..0000000
--- a/unittests/deqp_tests.py
+++ /dev/null
@@ -1,282 +0,0 @@
-# Copyright (c) 2015 Intel Corporation
-
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-# SOFTWARE.
-
-"""Tests for the dEQP integration in framework.
-
-This tests the core framework shared code, and not the individual packages in
-tests
-
-"""
-
-from __future__ import (
- absolute_import, division, print_function, unicode_literals
-)
-import textwrap
-
-try:
- from unittest import mock
-except ImportError:
- import mock
-
-import nose.tools as nt
-import six
-
-from framework import profile, grouptools, exceptions
-from framework.test import deqp
-from . import utils
-
-# pylint:disable=line-too-long,invalid-name
-
-doc_formatter = utils.nose.DocFormatter({'separator': grouptools.SEPARATOR})
-
-
-class _DEQPTestTest(deqp.DEQPBaseTest):
- deqp_bin = 'deqp.bin'
- extra_args = ['extra']
-
-
- at utils.piglit.set_piglit_conf(('deqp_test', 'test_env', 'from conf'))
- at mock.patch.dict('os.environ', {'_PIGLIT_TEST_ENV': 'from env'})
-def test_get_option_env():
- """deqp.get_option: if env is set it overrides piglit.conf"""
- nt.eq_(deqp.get_option('_PIGLIT_TEST_ENV', ('deqp_test', 'test_env')),
- 'from env')
-
-
- at utils.piglit.set_piglit_conf(('deqp_test', 'test_env', 'from conf'))
- at mock.patch.dict('os.environ', {}, True)
-def test_get_option_conf():
- """deqp.get_option: if env is not set a value is taken from piglit.conf"""
- nt.eq_(deqp.get_option('_PIGLIT_TEST_ENV', ('deqp_test', 'test_env')),
- 'from conf')
-
-
- at mock.patch.dict('os.environ', {}, True)
-def test_get_option_default():
- """deqp.get_option: default value is returned when env and conf are unset
- """
- nt.eq_(deqp.get_option('_PIGLIT_TEST_ENV', ('deqp_test', 'test_env'),
- 'foobar'),
- 'foobar')
-
-
- at mock.patch.dict('os.environ', {}, True)
-def test_get_option_conf_no_section():
- """deqp.get_option: if a no_section error is raised and env is unset None is return
- """
- nt.eq_(deqp.get_option('_PIGLIT_TEST_ENV', ('deqp_test', 'test_env')), None)
-
-
-# The first argument ensures the sectio exists
- at utils.piglit.set_piglit_conf(('deqp_test', 'test_env', 'from conf'),
- ('deqp_test', 'not_exists', None))
- at mock.patch.dict('os.environ', {}, True)
-def test_get_option_conf_no_option():
- """deqp.get_option: if a no_option error is raised and env is unset None is return
- """
- nt.eq_(deqp.get_option('_PIGLIT_TEST_ENV', ('deqp_test', 'not_exists')),
- None)
-
-
- at mock.patch.dict('os.environ', {}, True)
- at nt.raises(exceptions.PiglitFatalError)
-def test_get_option_required():
- """deqp.get_option: dies if a required option cannot be retrieved."""
- deqp.get_option('NOT_REAL', ('fake', 'fake'), default='', required=True)
-
-
-
-class TestMakeProfile(object):
- """Test deqp.make_profile."""
- @classmethod
- def setup_class(cls):
- cls.profile = deqp.make_profile(['this.is.a.deqp.test'], _DEQPTestTest)
-
- def test_returns_profile(self):
- """deqp.make_profile: returns a TestProfile"""
- nt.assert_is_instance(self.profile, profile.TestProfile)
-
- @doc_formatter
- def test_grouptools(self):
- """deqp.make_profile: replaces '.' with '{separator}'"""
- nt.assert_in(grouptools.join('this', 'is', 'a', 'deqp', 'test'),
- self.profile.test_list)
-
-
-def test_iter_deqp_test_cases_test():
- """deqp.iter_deqp_test_cases: correctly detects a TEST: line"""
- with utils.nose.tempfile('TEST: a.deqp.test') as tfile:
- gen = deqp.iter_deqp_test_cases(tfile)
- nt.eq_('a.deqp.test', next(gen))
- gen.close()
-
-
-def test_iter_deqp_test_cases_group():
- """deqp.iter_deqp_test_casesgen_caselist_txt: correctly detects a GROUP: line"""
- with utils.nose.tempfile('GROUP: a group\nTEST: a.deqp.test') as tfile:
- gen = deqp.iter_deqp_test_cases(tfile)
- nt.eq_('a.deqp.test', next(gen))
- gen.close()
-
-
- at nt.raises(exceptions.PiglitFatalError)
-def test_iter_deqp_test_cases_bad():
- """deqp.iter_deqp_test_casesgen_caselist_txt: PiglitFatalException is raised if line is not TEST: or GROUP:
- """
- with utils.nose.tempfile('this will fail') as tfile:
- gen = deqp.iter_deqp_test_cases(tfile)
- nt.eq_('a.deqp.test', next(gen))
-
-
- at utils.nose.no_error
-def test_DEQPBaseTest_initialize():
- """deqp.DEQPBaseTest: can be initialized (with abstract methods overwritten)
- """
- _DEQPTestTest('a.deqp.test')
-
-
-def test_DEQPBaseTest_command():
- """deqp.DEQPBaseTest.command: cls.extra_args are added to self.command"""
- test = _DEQPTestTest('a.deqp.test')
- nt.eq_(test.command[-1], 'extra')
-
-
-class TestDEQPBaseTestInterpretResult(object):
- """Tests for DEQPBaseTest.interpret_result.
-
- This specifically tests the part that searches stdout.
-
- """
- def __init__(self):
- self.test = None
-
- def setup(self):
- self.test = _DEQPTestTest('a.deqp.test')
-
- def test_crash(self):
- """deqp.DEQPBaseTest.interpret_result: if returncode is < 0 stauts is crash"""
- self.test.result.returncode = -9
- self.test.interpret_result()
- nt.eq_(self.test.result.result, 'crash')
-
- def test_returncode_fail(self):
- """deqp.DEQPBaseTest.interpret_result: if returncode is > 0 result is fail"""
- self.test.result.returncode = 1
- self.test.interpret_result()
- nt.eq_(self.test.result.result, 'fail')
-
- def test_fallthrough(self):
- """deqp.DEQPBaseTest.interpret_result: if no case is hit set to fail"""
- self.test.result.returncode = 0
- self.test.result.out = ''
- self.test.interpret_result()
- nt.eq_(self.test.result.result, 'fail')
-
- def test_windows_returncode_3(self):
- """deqp.DEQPBaseTest.interpret_result: on windows returncode 3 is crash"""
- self.test.result.returncode = 3
- with mock.patch('framework.test.base.sys.platform', 'win32'):
- self.test.interpret_result()
- nt.eq_(self.test.result.result, 'crash')
-
-
-class TestDEQPBaseTestIntepretResultStatus(object):
- """Tests for DEQPBaseTest.__find_map."""
- def __init__(self):
- self.inst = None
-
- __OUT = textwrap.dedent("""\
- dEQP Core 2014.x (0xcafebabe) starting..
- target implementation = 'DRM'
-
- Test case 'dEQP-GLES2.functional.shaders.conversions.vector_to_vector.vec3_to_ivec3_fragment'..
- Vertex shader compile time = 0.129000 ms
- Fragment shader compile time = 0.264000 ms
- Link time = 0.814000 ms
- Test case duration in microseconds = 487155 us
- {stat} ({stat})
-
- DONE!
-
- Test run totals:
- Passed: {pass_}/1 (100.0%)
- Failed: {fail}/1 (0.0%)
- Not supported: {supp}/1 (0.0%)
- Warnings: {warn}/1 (0.0%)
- Test run was ABORTED!
- """)
-
- def __gen_stdout(self, status):
- assert status in ['Fail', 'NotSupported', 'Pass', 'QualityWarning',
- 'InternalError', 'Crash', 'ResourceError']
-
- return self.__OUT.format(
- stat=status,
- pass_=1 if status == 'Pass' else 0,
- fail=1 if status in ['Crash', 'Fail', 'ResourceError'] else 0,
- supp=1 if status == 'InternalError' else 0,
- warn=1 if status == 'QualityWarning' else 0,
- )
-
- def setup(self):
- self.inst = _DEQPTestTest('a.deqp.test')
- self.inst.result.returncode = 0
-
- def test_fail(self):
- """test.deqp.DEQPBaseTest.interpret_result: when Fail in result the result is 'fail'"""
- self.inst.result.out = self.__gen_stdout('Fail')
- self.inst.interpret_result()
- nt.eq_(self.inst.result.result, 'fail')
-
- def test_pass(self):
- """test.deqp.DEQPBaseTest.interpret_result: when Pass in result the result is 'Pass'"""
- self.inst.result.out = self.__gen_stdout('Pass')
- self.inst.interpret_result()
- nt.eq_(self.inst.result.result, 'pass')
-
- def test_warn(self):
- """test.deqp.DEQPBaseTest.interpret_result: when QualityWarning in result the result is 'warn'"""
- self.inst.result.out = self.__gen_stdout('QualityWarning')
- self.inst.interpret_result()
- nt.eq_(self.inst.result.result, 'warn')
-
- def test_error(self):
- """test.deqp.DEQPBaseTest.interpret_result: when InternalError in result the result is 'fail'"""
- self.inst.result.out = self.__gen_stdout('InternalError')
- self.inst.interpret_result()
- nt.eq_(self.inst.result.result, 'fail')
-
- def test_crash(self):
- """test.deqp.DEQPBaseTest.interpret_result: when InternalError in result the result is 'crash'"""
- self.inst.result.out = self.__gen_stdout('Crash')
- self.inst.interpret_result()
- nt.eq_(self.inst.result.result, 'crash')
-
- def test_skip(self):
- """test.deqp.DEQPBaseTest.interpret_result: when NotSupported in result the result is 'skip'"""
- self.inst.result.out = self.__gen_stdout('NotSupported')
- self.inst.interpret_result()
- nt.eq_(self.inst.result.result, 'skip')
-
- def test_resourceerror(self):
- """test.deqp.DEQPBaseTest.interpret_result: when ResourceError in result the result is 'crash'"""
- self.inst.result.out = self.__gen_stdout('ResourceError')
- self.inst.interpret_result()
- nt.eq_(self.inst.result.result, 'crash')
diff --git a/unittests/framework/test/test_deqp.py b/unittests/framework/test/test_deqp.py
new file mode 100644
index 0000000..8e7579a
--- /dev/null
+++ b/unittests/framework/test/test_deqp.py
@@ -0,0 +1,296 @@
+# Copyright (c) 2015-2016 Intel Corporation
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+"""Tests for the dEQP integration in framework.
+
+This tests the core framework shared code, and not the individual packages in
+tests
+
+"""
+
+from __future__ import (
+ absolute_import, division, print_function, unicode_literals
+)
+import textwrap
+try:
+ from unittest import mock
+except ImportError:
+ import mock
+
+import pytest
+import six
+
+from framework import exceptions
+from framework import grouptools
+from framework import profile
+from framework import status
+from framework.test import deqp
+
+# pylint:disable=invalid-name,no-self-use
+
+
+class _DEQPTestTest(deqp.DEQPBaseTest):
+ deqp_bin = 'deqp.bin'
+ extra_args = ['extra']
+
+
+class TestGetOptions(object):
+ """Tests for the get_option function."""
+
+ @pytest.fixture
+ def env(self, mocker):
+ """Create a mocked os.environ."""
+ return mocker.patch('framework.test.deqp.os.environ', {})
+
+ @pytest.fixture
+ def conf(self, mocker):
+ """Create a mocked piglit.conf."""
+ return mocker.patch('framework.core.PIGLIT_CONFIG.safe_get',
+ mocker.Mock(return_value=None))
+
+ def test_from_default(self):
+ """deqp.get_option: if env is set it overrides piglit.conf."""
+ # The mock means that only the first value matters
+ actual = deqp.get_option('foo', ('foo', 'foo'),
+ default=mock.sentinel.good)
+
+ assert actual is mock.sentinel.good
+
+ def test_from_conf(self, conf):
+ """deqp.get_option: if env is not set a value is taken from
+ piglit.conf.
+ """
+ conf.return_value = mock.sentinel
+
+ # The mock means that these values don't actually matter
+ actual = deqp.get_option('foo', ('foo', 'foo'))
+
+ assert actual is mock.sentinel
+
+ def test_from_env(self, env, conf):
+ """deqp.get_option: if env is set it overrides piglit.conf."""
+ conf.return_value = mock.sentinel.bad
+ env['TEST'] = mock.sentinel.good
+
+ # The mock means that only the first value matters
+ actual = deqp.get_option('TEST', ('foo', 'foo'))
+
+ assert actual is mock.sentinel.good
+
+ def test_get_option_required(self, mocker):
+ """deqp.get_option: dies if a required option cannot be retrieved."""
+ mocker.patch('framework.test.deqp.os.environ', {}, True)
+
+ with pytest.raises(exceptions.PiglitFatalError):
+ deqp.get_option('NOT_REAL', ('fake', 'fake'), default='',
+ required=True)
+
+
+class TestMakeProfile(object):
+ """Test deqp.make_profile."""
+
+ @classmethod
+ def setup_class(cls):
+ cls.profile = deqp.make_profile(['this.is.a.deqp.test'], _DEQPTestTest)
+
+ def test_returns_profile(self):
+ """deqp.make_profile: returns a TestProfile."""
+ assert isinstance(self.profile, profile.TestProfile)
+
+ def test_replaces_separator(self):
+ """deqp.make_profile: replaces '.' with grouptools.separator"""
+ expected = grouptools.join('this', 'is', 'a', 'deqp', 'test')
+ assert expected in self.profile.test_list
+
+
+class TestIterDeqpTestCases(object):
+ """Tests for iter_deqp_test_cases."""
+
+ def _do_test(self, write, expect, tmpdir):
+ """Run the acutal test."""
+ p = tmpdir.join('foo')
+ p.write(write)
+ gen = deqp.iter_deqp_test_cases(six.text_type(p))
+ assert next(gen) == expect
+
+ def test_test_cases(self, tmpdir):
+ """Correctly detects a test line."""
+ self._do_test('TEST: a.deqp.test', 'a.deqp.test', tmpdir)
+
+ def test_test_group(self, tmpdir):
+ """Correctly detects a group line."""
+ self._do_test('GROUP: a group\nTEST: a.deqp.test', 'a.deqp.test',
+ tmpdir)
+
+ def test_bad_entry(self, tmpdir):
+ """A PiglitFatalException is raised if a line is not a TEST or GROUP.
+ """
+ with pytest.raises(exceptions.PiglitFatalError):
+ self._do_test('this will fail', None, tmpdir)
+
+
+class TestDEQPBaseTest(object):
+ """Test the DEQPBaseTest class."""
+
+ @classmethod
+ def setup_class(cls):
+ cls.test = _DEQPTestTest('a.deqp.test')
+
+ def test_command_adds_extra_args(self):
+ assert self.test.command[-1] == 'extra'
+
+ class TestInterpretResultReturncodes(object):
+ """Test the interpret_result method's returncode handling."""
+
+ @classmethod
+ def setup_class(cls):
+ cls.test = _DEQPTestTest('a.deqp.test')
+
+ def test_crash(self):
+ """deqp.DEQPBaseTest.interpret_result: if returncode is < 0 stauts
+ is crash.
+ """
+ self.test.result.returncode = -9
+ self.test.interpret_result()
+ assert self.test.result.result is status.CRASH
+
+ def test_returncode_fail(self):
+ """deqp.DEQPBaseTest.interpret_result: if returncode is > 0 result
+ is fail.
+ """
+ self.test.result.returncode = 1
+ self.test.interpret_result()
+ assert self.test.result.result is status.FAIL
+
+ def test_fallthrough(self):
+ """deqp.DEQPBaseTest.interpret_result: if no case is hit set to
+ fail.
+ """
+ self.test.result.returncode = 0
+ self.test.result.out = ''
+ self.test.interpret_result()
+ assert self.test.result.result is status.FAIL
+
+ def test_windows_returncode_3(self, mocker):
+ """deqp.DEQPBaseTest.interpret_result: on windows returncode 3 is
+ crash.
+ """
+ mocker.patch('framework.test.base.sys.platform', 'win32')
+ self.test.result.returncode = 3
+ self.test.interpret_result()
+ assert self.test.result.result is status.CRASH
+
+ class TestDEQPBaseTestIntepretResultOutput(object):
+ """Tests for DEQPBaseTest.__find_map."""
+
+ inst = None
+ __OUT = textwrap.dedent("""\
+ dEQP Core 2014.x (0xcafebabe) starting..
+ target implementation = 'DRM'
+
+ Test case 'dEQP-GLES2.functional.shaders.conversions.vector_to_vector.vec3_to_ivec3_fragment'..
+ Vertex shader compile time = 0.129000 ms
+ Fragment shader compile time = 0.264000 ms
+ Link time = 0.814000 ms
+ Test case duration in microseconds = 487155 us
+ {stat} ({stat})
+
+ DONE!
+
+ Test run totals:
+ Passed: {pass_}/1 (100.0%)
+ Failed: {fail}/1 (0.0%)
+ Not supported: {supp}/1 (0.0%)
+ Warnings: {warn}/1 (0.0%)
+ Test run was ABORTED!
+ """)
+
+ def __gen_stdout(self, stat):
+ """Make a string that looks like DEQP output."""
+ assert stat in ['Fail', 'NotSupported', 'Pass', 'QualityWarning',
+ 'InternalError', 'Crash', 'ResourceError']
+
+ return self.__OUT.format(
+ stat=stat,
+ pass_=1 if stat == 'Pass' else 0,
+ fail=1 if stat in ['Crash', 'Fail', 'ResourceError'] else 0,
+ supp=1 if stat == 'InternalError' else 0,
+ warn=1 if stat == 'QualityWarning' else 0,
+ )
+
+ def setup(self):
+ self.inst = _DEQPTestTest('a.deqp.test')
+ self.inst.result.returncode = 0
+
+ def test_fail(self):
+ """test.deqp.DEQPBaseTest.interpret_result: when Fail in result the
+ result is 'fail'.
+ """
+ self.inst.result.out = self.__gen_stdout('Fail')
+ self.inst.interpret_result()
+ assert self.inst.result.result is status.FAIL
+
+ def test_pass(self):
+ """test.deqp.DEQPBaseTest.interpret_result: when Pass in result the
+ result is 'Pass'.
+ """
+ self.inst.result.out = self.__gen_stdout('Pass')
+ self.inst.interpret_result()
+ assert self.inst.result.result is status.PASS
+
+ def test_warn(self):
+ """test.deqp.DEQPBaseTest.interpret_result: when QualityWarning in
+ result the result is 'warn'.
+ """
+ self.inst.result.out = self.__gen_stdout('QualityWarning')
+ self.inst.interpret_result()
+ assert self.inst.result.result is status.WARN
+
+ def test_error(self):
+ """test.deqp.DEQPBaseTest.interpret_result: when InternalError in
+ result the result is 'fail'.
+ """
+ self.inst.result.out = self.__gen_stdout('InternalError')
+ self.inst.interpret_result()
+ assert self.inst.result.result is status.FAIL
+
+ def test_crash(self):
+ """test.deqp.DEQPBaseTest.interpret_result: when InternalError in
+ result the result is 'crash'.
+ """
+ self.inst.result.out = self.__gen_stdout('Crash')
+ self.inst.interpret_result()
+ assert self.inst.result.result is status.CRASH
+
+ def test_skip(self):
+ """test.deqp.DEQPBaseTest.interpret_result: when NotSupported in
+ result the result is 'skip'.
+ """
+ self.inst.result.out = self.__gen_stdout('NotSupported')
+ self.inst.interpret_result()
+ assert self.inst.result.result is status.SKIP
+
+ def test_resourceerror(self):
+ """test.deqp.DEQPBaseTest.interpret_result: when ResourceError in
+ result the result is 'crash'.
+ """
+ self.inst.result.out = self.__gen_stdout('ResourceError')
+ self.inst.interpret_result()
+ assert self.inst.result.result is status.CRASH
--
2.9.0
More information about the Piglit
mailing list