[Piglit] [Patch v5 2/4] Adds unit tests for the dmesg module
Dylan Baker
baker.dylan.c at gmail.com
Tue Feb 4 09:35:56 PST 2014
This adds a few tests for the dmesg class.
It also adds a test hook into core.Test. This is a little ugly, but I
just couldn't come up with any other solution that didn't involve
making the dmesg code more invasive or harder to read.
v2: - Adds tests to simulate dmesg 'wrapping'
v3: - Replace assert with nose.test assert helpers. These have the
advantage of providing the option to add a message explaining what
when wrong.
- Adds test for LinuxDmesg's enforcement of timestamps
v4: - Remove timestamp error check test. It was polluting the entire
environment's LinuxDmesg.DMESG_COMMAND in some cases
- Refactor a few tests as a result of refactoring in framework/dmesg
- Make the error message in the dmesg_wrap tests more helpful
- Reduce test_dmesg_performance from .025 to .015 seconds.
Signed-off-by: Dylan Baker <baker.dylan.c at gmail.com>
---
framework/core.py | 5 +
framework/tests/dmesg_tests.py | 296 +++++++++++++++++++++++++++++++++++++++++
2 files changed, 301 insertions(+)
create mode 100644 framework/tests/dmesg_tests.py
diff --git a/framework/core.py b/framework/core.py
index a193a2d..da2a716 100644
--- a/framework/core.py
+++ b/framework/core.py
@@ -417,6 +417,10 @@ class Test(object):
self.runConcurrent = runConcurrent
self.skip_test = False
+ # This is a hook for doing some testing on execute right before
+ # self.run is called.
+ self._test_hook_execute_run = lambda: None
+
def run(self):
raise NotImplementedError
@@ -436,6 +440,7 @@ class Test(object):
try:
status("running")
time_start = time.time()
+ self._test_hook_execute_run()
result = self.run(env)
time_end = time.time()
if 'time' not in result:
diff --git a/framework/tests/dmesg_tests.py b/framework/tests/dmesg_tests.py
new file mode 100644
index 0000000..287cf74
--- /dev/null
+++ b/framework/tests/dmesg_tests.py
@@ -0,0 +1,296 @@
+# Copyright (c) 2014 Intel Corporation
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+""" Provides tests for the dmesg class """
+
+import os
+import sys
+import subprocess
+import nose.tools as nt
+from nose.plugins.skip import SkipTest
+from framework.dmesg import DummyDmesg, LinuxDmesg, get_dmesg, DmesgError
+from framework.core import TestResult, PiglitJSONEncoder, Environment
+from framework.exectest import PlainExecTest
+from framework.gleantest import GleanTest
+from framework.shader_test import ShaderTest
+from framework.glsl_parser_test import GLSLParserTest
+
+
+def _get_dmesg():
+ """ checks to ensure dmesg is not DummyDmesg, raises skip if it is
+
+ If we are on a non-posix system we will get a dummy dmesg, go ahead and
+ skip in that case
+ """
+ test = get_dmesg()
+ if isinstance(test, DummyDmesg):
+ raise SkipTest("A DummyDmesg was returned, testing dmesg impossible.")
+ return test
+
+
+def _write_dev_kmesg():
+ """ Try to write to /dev/kmesg, skip if not possible
+
+ Writing to the dmesg ringbuffer at /dev/kmesg ensures that we varies
+ functionality in the LinuxDmesg class will go down the
+ dmesg-has-new-entries path.
+
+ If we anything goes wrong here just skip.
+ """
+ err = subprocess.call(['sudo', 'sh', '-c', 'echo "piglit dmesg test" > /dev/kmsg'])
+ if err != 0:
+ raise SkipTest("Writing to the ringbuffer failed")
+
+
+def test_get_dmesg_dummy():
+ """ Test that get_dmesg function returns a Dummy when asked """
+ dummy = get_dmesg(not_dummy=False)
+ nt.assert_is(type(dummy), DummyDmesg,
+ msg="Error: get_dmesg should have returned DummyDmesg, "
+ "but it actually returned {}".format(type(dummy)))
+
+
+def test_get_dmesg_linux():
+ """ Test that get_dmesg() returns a LinuxDmesg instance when asked """
+ if not sys.platform.startswith('linux'):
+ raise SkipTest("Cannot test a LinuxDmesg on a non linux system")
+ posix = _get_dmesg()
+ nt.assert_is(type(posix), LinuxDmesg,
+ msg="Error: get_dmesg should have returned LinuxDmesg, "
+ "but it actually returned {}".format(type(posix)))
+
+
+def test_update_dmesg():
+ """ Tests that update_dmesg actually updates
+
+ This will skip on non-Posix systems, since there is no way to actually test
+ it.
+
+ Because this test needs to write into the dmesg ringbuffer to assure that
+ the ringbuffer has changed and that our class successfully catches that
+ change it requires root access, gained by sudo. In the event that it cannot
+ get sudo it will skip.
+
+ """
+ dmesg = _get_dmesg()
+ _write_dev_kmesg()
+
+ dmesg.update_dmesg()
+ nt.assert_is_not_none(dmesg._new_messages,
+ msg="LinuxDmesg does not return updates, even when "
+ "dmesg has been updated.")
+
+
+def test_dmesg_wrap_partial():
+ """ Test that dmesg still works after dmesg wraps partially
+
+ We can overwrite the DMESG_COMMAND class variable to emluate dmesg being
+ filled up and overflowing. What this test does is starts with a string that
+ looks like this: "a\nb\nc\n" (this is used to emluate the contents of
+ dmesg), we then replace that with "b\nc\nd\n", and ensure that the update
+ of dmesg contains only 'd', becasue 'd' is the only new value in the
+ updated dmesg.
+
+ """
+ # We don't want weird side effects of changing DMESG_COMMAND globally, so
+ # instead we set it as a class instance and manually clear the
+ # _last_messages attribute
+ dmesg = LinuxDmesg()
+ dmesg.DMESG_COMMAND = ['echo', 'a\nb\nc\n']
+ dmesg.update_dmesg()
+
+ # Update the DMESG_COMMAND to add d\n and remove a\n, this simluates the
+ # wrap
+ dmesg.DMESG_COMMAND = ['echo', 'b\nc\nd\n']
+ dmesg.update_dmesg()
+
+ nt.assert_items_equal(dmesg._new_messages, ['d'],
+ msg="_new_messages should be equal to ['d'], but is "
+ "{} instead.".format(dmesg._new_messages))
+
+
+def test_dmesg_wrap_complete():
+ """ Test that dmesg still works after dmesg wraps completely
+
+ just like the partial version, but with nothingin common.
+
+ """
+ # We don't want weird side effects of changing DMESG_COMMAND globally, so
+ # instead we set it as a class instance and manually clear the
+ # _last_messages attribute
+ dmesg = LinuxDmesg()
+ dmesg.DMESG_COMMAND = ['echo', 'a\nb\nc\n']
+ dmesg.update_dmesg()
+
+ # Udamte the DMESG_COMMAND to add d\n and remove a\n, this simluates the
+ # wrap
+ dmesg.DMESG_COMMAND = ['echo', '1\n2\n3\n']
+ dmesg.update_dmesg()
+
+ nt.assert_items_equal(dmesg._new_messages, ['1', '2', '3'],
+ msg="_new_messages should be equal to "
+ "['1', '2', '3'], but is {} instead".format(
+ dmesg._new_messages))
+
+
+def test_update_result_replace():
+ """ Generates tests for update_result """
+ dmesg = _get_dmesg()
+
+ for res in ['pass', 'fail', 'crash', 'warn', 'skip', 'notrun']:
+ result = TestResult()
+ result['result'] = res
+ result['subtest'] = {}
+ result['subtest']['test'] = res
+
+ _write_dev_kmesg()
+ new_result = dmesg.update_result(result)
+
+ # Create a yieldable and set the description for useful per-test names
+ yieldable = check_update_result
+ yieldable.description = "Test update_result: {0}".format(res)
+ yield yieldable, new_result['result'], res
+
+ yieldable.description = "Test update_result subtest: {0}".format(res)
+ yield yieldable, new_result['subtest']['test'], res
+
+
+def check_update_result(result, status):
+ """ Tests that update result replaces results correctly
+
+ Dmesg.update_results() should take a TestResult instance and replace the
+ result instance with a dmesg-statuses when appropriate. Appropriate
+ instances to be replaced are: pass, warn, fail.
+
+ """
+ if status == "pass":
+ nt.assert_equal(result, 'dmesg-warn',
+ msg="pass should be replaced with dmesg-warn")
+ elif status in ['warn', 'fail']:
+ nt.assert_equal(result, 'dmesg-fail',
+ msg="{} should be replaced with "
+ "dmesg-fail".format(status))
+ else:
+ nt.assert_equal(result, status,
+ msg="{} should not have changed, but it "
+ "did.".format(result))
+
+
+def test_update_result_add_dmesg():
+ """ Tests update_result's addition of dmesg attribute """
+ dmesg = _get_dmesg()
+
+ result = TestResult()
+ result['result'] = 'pass'
+
+ _write_dev_kmesg()
+ result = dmesg.update_result(result)
+
+ nt.assert_in('dmesg', result,
+ msg="result does not have dmesg member but should")
+ return result
+
+
+def test_json_serialize_updated_result():
+ """ Test that a TestResult that has been updated is json serializable """
+ encoder = PiglitJSONEncoder()
+ result = test_update_result_add_dmesg()
+ encoded = encoder.encode(result)
+
+
+def test_testclasses_dmesg():
+ """ Generator that creates tests for """
+ env = Environment()
+
+ lists = [(PlainExecTest, ['attribs', '-auto', '-fbo'], 'PlainExecTest'),
+ (GleanTest, 'basic', "GleanTest"),
+ (ShaderTest,
+ ['tests/shaders/loopfunc.shader_test', '-auto', '-fbo'],
+ 'ShaderTest'),
+ (GLSLParserTest, 'tests/glslparsertest/shaders/main1.vert',
+ 'GLSLParserTest')]
+
+ for tclass, tfile, desc in lists:
+ yieldable = check_classes_dmesg
+ yieldable.description = "Test dmesg in {}".format(desc)
+ yield yieldable, tclass, tfile, env
+
+
+def check_classes_dmesg(test_class, test_args, env):
+ """ Do the actual check on the provided test class for dmesg """
+ if not os.path.exists('bin'):
+ raise SkipTest("This tests requires a working, built version of "
+ "piglit")
+
+ class DummyJsonWriter():
+ """ A very simple dummy for json writer """
+ def __init__(self):
+ pass
+
+ def write_dict_item(self, path, result):
+ self.result = result
+
+ def _write_dmesg():
+ """ Small helper to write dmesg """
+ subprocess.call(['sudo', 'sh', '-c', 'echo "piglit test" > /dev/kmsg'])
+
+ dmesg = _get_dmesg()
+
+ # Create the test and then write to dmesg to ensure that it actually works
+ test = test_class(test_args)
+ test._test_hook_execute_run = _write_dmesg
+
+ json = DummyJsonWriter()
+
+ test.execute(env, None, json, dmesg)
+
+ nt.assert_in(json.result['result'], ['dmesg-warn', 'dmesg-fail'],
+ msg="{0} did not update status with dmesg".format(type(test)))
+
+
+ at nt.timed(.015)
+def test_dmesg_performance():
+ """ Test that update_dmesg returns in less than .015 seconds on large loads
+
+ One of the largest problems with dmesg is that there are 16000+ tests
+ already, and more will be added. Since dmesg runs serially minimizing the
+ amount of time spend on setup and teardown of tests is paramount, even at 1
+ second the results will add 4 minutes to the test run with 16000 tests.
+
+ For the purpose of this test we won't actually read dmesg, it's not nice to
+ keep trashing dmesg, and this test needs to see that a large list, say 7000
+ items, won't take forever.
+
+ This was tested on an ivybridge i3-3217U where ~.013 was a common value,
+ most modern systems with a desktop Intel or AMD processor should be able to
+ hit this number. Systems with very low power CPUs may not be able to hit
+ this number.
+
+ """
+ dmesg = LinuxDmesg()
+ # 7000 is an arbitrary big number
+ length = 7000 - len(dmesg._new_messages)
+ # make new messages really big
+ dmesg._new_messages = ["piglit test {}".format(x) for x in xrange(length)] + dmesg._new_messages
+
+ result = TestResult()
+ result['result'] = 'pass'
+
+ result = dmesg.update_result(result)
--
1.8.5.3
More information about the Piglit
mailing list