[Piglit] [PATCH 4/5] Converts python unittest testcases to nose test cases
Dylan Baker
baker.dylan.c at gmail.com
Thu Jan 23 10:25:10 PST 2014
This patch converts all of the status_tests.py into nose style tests.
Nose makes everything we bent over backwards for with unittest easy,
reducing code in both amount and complexity.
Signed-off-by: Dylan Baker <baker.dylan.c at gmail.com>
---
framework/tests/helpers.py | 105 ----------------
framework/tests/status.py | 262 ----------------------------------------
framework/tests/status_tests.py | 88 ++++++++++++++
piglit-framework-tests.py | 47 -------
4 files changed, 88 insertions(+), 414 deletions(-)
delete mode 100644 framework/tests/helpers.py
delete mode 100644 framework/tests/status.py
create mode 100644 framework/tests/status_tests.py
delete mode 100755 piglit-framework-tests.py
diff --git a/framework/tests/helpers.py b/framework/tests/helpers.py
deleted file mode 100644
index 2fd7c3d..0000000
--- a/framework/tests/helpers.py
+++ /dev/null
@@ -1,105 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright (c) 2013 Intel Corporation
-#
-# Permission is hereby granted, free of charge, to any person obtaining a
-# copy of this software and associated documentation files (the "Software"),
-# to deal in the Software without restriction, including without limitation
-# the rights to use, copy, modify, merge, publish, distribute, sublicense,
-# and/or sell copies of the Software, and to permit persons to whom the
-# Software is furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice (including the next
-# paragraph) shall be included in all copies or substantial portions of the
-# Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
-# IN THE SOFTWARE.
-
-
-import sys
-
-import framework.core as core
-
-
-def test_iterations(*parameters):
- """ Magic that allows a single method to create a whole bunch of functions.
-
- This is desirable because of the way unittest works: Each method gets a
- name in the output, and each method stops on the first error. This makes
- using a loop useless, if 10/20 iterations should fail, the first failure
- stops the loop. The solution other than using a decorator is to create a
- test for each iteration, which could result in hundreds of tests.
-
- """
-
- def decorator(method, parameters=parameters):
- def tuplify(x):
- if not isinstance(x, tuple):
- return (x, )
- return x
-
- for parameter in map(tuplify, parameters):
- name_for_parameter = "{0}({1})".format(method.__name__,
- ", ".join(map(repr, parameter)))
- frame = sys._getframe(1) # pylint: disable-msg=W0212
- frame.f_locals[name_for_parameter] = \
- lambda self, m=method, p=parameter: m(self, *p)
- return None
- return decorator
-
-
-def create_testresult(name, lspci="fake lspci", glxinfo="fake glxinfo",
- tests={}):
- """ Helper function that returns a complete TestResults file
-
- This takes one required argument
- :name: This must be set, it names the run. If two runs have the same name
- there can be problems in the summary code.
-
- This function also takes three optional arguments
- :lspci: This is the lspci information in the file. Default="fake lspci"
- :glxinfo: glxinfo in the file. Default="fake glxinfo"
- :tests: A dictionary of tests
-
- """
- assert(isinstance(tests, dict))
-
- return core.TestResult({"options": {"profile": "fake",
- "filter": [],
- "exclude_filter": []},
- "name": "{0}".format(name),
- "lspci": "{0}".format(lspci),
- "glxinfo": "{0}".format(glxinfo),
- "time_elapsed": 10.23456789,
- "tests": tests})
-
-
-def create_test(name, result, info="fake info", returncode=0, time=0.123456789,
- command="fake command"):
- """ Helper function that takes input and returns a dictionary of results
- for a single tests
-
- Takes two required arguments:
- :name: The name of the tests
- :result: The result the test returned
-
- Additionally takes four optional arguments:
- :info: The info entry. Default="fake info"
- :returncode: The returncode of the test. Default=0
- :time: The amount of time the test ran. Default=0.123456789
- :command: The command that was executed. Default="fake command"
-
- """
- #FIXME: This should be able to create other entries for failed tests
-
- return {name: {"info": info,
- "returncode": returncode,
- "time": time,
- "command": command,
- "result": result}}
diff --git a/framework/tests/status.py b/framework/tests/status.py
deleted file mode 100644
index 3630abe..0000000
--- a/framework/tests/status.py
+++ /dev/null
@@ -1,262 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright (c) 2013 Intel Corporation
-#
-# Permission is hereby granted, free of charge, to any person obtaining a
-# copy of this software and associated documentation files (the "Software"),
-# to deal in the Software without restriction, including without limitation
-# the rights to use, copy, modify, merge, publish, distribute, sublicense,
-# and/or sell copies of the Software, and to permit persons to whom the
-# Software is furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice (including the next
-# paragraph) shall be included in all copies or substantial portions of the
-# Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
-# IN THE SOFTWARE.
-
-
-import os
-import os.path as path
-import unittest
-import itertools
-import tempfile
-
-try:
- import simplejson as json
-except ImportError:
- import json
-import framework.summary as summary
-from helpers import test_iterations, create_testresult, create_test
-
-
-""" Status ordering from best to worst:
-
-See ../summary.py.
-
-"""
-
-# These lists normally would reside within a class, however, these are meant to
-# be fed to decorators, which donot have access to class variables.
-
-# List of possible statuses.
-# 'notrun' must be last or all kinds of things might break.
-STATUSES = ['pass', 'fail', 'skip', 'warn', 'crash', 'dmesg-warn',
- 'dmesg-fail', 'notrun']
-
-# list of possible regressions
-REGRESSIONS = [("pass", "warn"),
- ("pass", "dmesg-warn"),
- ("pass", "fail"),
- ("pass", "dmesg-fail"),
- ("pass", "crash"),
- ("dmesg-warn", "warn"),
- ("dmesg-warn", "dmesg-fail"),
- ("dmesg-warn", "fail"),
- ("dmesg-warn", "crash"),
- ("warn", "fail"),
- ("warn", "crash"),
- ("warn", "dmesg-fail"),
- ("dmesg-fail", "crash"),
- ("dmesg-fail", "fail"),
- ("fail", "crash"),
- ("skip", "crash"),
- ("skip", "fail"),
- ("skip", "dmesg-fail"),
- ("skip", "warn"),
- ("skip", "dmesg-warn"),
- ("notrun", "crash"),
- ("notrun", "fail"),
- ("notrun", "dmesg-fail"),
- ("notrun", "warn"),
- ("notrun", "dmesg-warn")]
-
-
-# List of possible fixes
-FIXES = [("crash", "fail"),
- ("crash", "dmesg-fail"),
- ("crash", "warn"),
- ("crash", "dmesg-warn"),
- ("crash", "pass"),
- ("crash", "skip"),
- ("crash", "notrun"),
- ("fail", "dmesg-fail"),
- ("fail", "warn"),
- ("fail", "dmesg-warn"),
- ("fail", "pass"),
- ("fail", "skip"),
- ("fail", "notrun"),
- ("dmesg-fail", "warn"),
- ("dmesg-fail", "dmesg-warn"),
- ("dmesg-fail", "pass"),
- ("dmesg-fail", "skip"),
- ("dmesg-fail", "notrun"),
- ("warn", "dmesg-warn"),
- ("warn", "pass"),
- ("warn", "skip"),
- ("warn", "notrun"),
- ("dmesg-warn", "pass"),
- ("dmesg-warn", "skip"),
- ("dmesg-warn", "notrun")]
-
-
-# List of statuses that should be problems.
-PROBLEMS = ["crash", "fail", "warn", "dmesg-warn", "dmesg-fail"]
-
-
-class SummaryTestBase(unittest.TestCase):
- """ Abstract base class for testing the Summary module"""
- tmpdir = tempfile.mkdtemp("piglit")
- files = []
-
- @classmethod
- def tearDownClass(cls):
- """ Remove any folders which were created and are empty """
- for branch in cls.files:
- os.remove(branch) # Delete the fie itself
- while branch != '/':
- branch = path.dirname(branch)
- try:
- os.rmdir(branch)
- except OSError as exception:
- # If there is an OSError the directory is not empty, so
- # every additional attempt to call os.rmdir will fail.
- if exception.errno == 39:
- break
-
- @classmethod
- def _makedirs(cls, dirs):
- """ Create one more more folders, with some error checking
-
- This catches a "Directory Exists" error (OSError.errno == 17), and
- passes that; but any other error will be raised
-
- """
- try:
- os.makedirs(dirs)
- except OSError as exception:
- # os.mkdir (and by extension os.makedirs) quite annoyingly throws
- # an error.
-
- # if the directory exists if the error was not a "Directory Exists"
- # Error, go ahead and raise it.
- if exception.errno != 17:
- raise exception
-
- # you cannot pass in an if clause, since that would pass the if,
- # not the except
- pass
-
- def _generate_summary(self, files):
- return summary.Summary([path.join(self.tmpdir, i) for i in files])
-
-
-class StatusTest(SummaryTestBase):
- """ Test status comparisons in the Summary module.
-
- This test creates summary objects with one or two results that it
- generates. These results have a single test in them forming known status
- pairs. These pairs are explicitly a fix, regression, or change. The code
- then creates a summary and ensures that the status has been added to the
- proper field, and not to any additional fields.
-
- """
-
- @classmethod
- def setUpClass(cls):
- """ Create all of the necissary files for running the test
-
- Create a set of temprorary result files, and add them to the files
- attribute. This attribute will be deleted by the tearDownClass() method
- after the tests have run
-
- """
- cls._makedirs(cls.tmpdir)
-
- # Create temporary files in the /tmp/piglit directory
- for result in STATUSES[:-1]: # notrun cannot be generated here
- tmpfile = path.join(cls.tmpdir, result, "main")
- cls._makedirs(path.dirname(tmpfile))
-
- with open(tmpfile, 'w') as f:
- json.dump(create_testresult(name=result,
- tests=create_test("test", result)),
- f, indent=4)
-
- # Add the file to the list of files to be removed in tearDownClass
- cls.files.append(tmpfile)
-
- # Create an additional file to test a Not Run status. This needs to be
- # generated separately since the test result should not include the
- # test that will have the not run status.
- tmpfile = path.join(cls.tmpdir, "notrun", "main")
- cls._makedirs(path.dirname(tmpfile))
-
- with open(tmpfile, 'w') as f:
- # If there are not tests buildDictionary fails
- json.dump(create_testresult(name="notrun",
- tests=create_test("diff", "pass")),
- f, indent=4)
-
- cls.files.append(tmpfile)
-
- @test_iterations(*REGRESSIONS)
- def test_is_regression(self, x, y):
- """ Only explicitly defined regressions should be treated as such
-
- Test that all combinations that are not explicitly a regression are not
- treated as regressions
-
- """
- result = self._generate_summary([x, y])
- self.assertEqual(
- len(result.tests['regressions']), 1,
- "{0} -> {1} is not a regression but should be".format(x, y))
-
- @test_iterations(*FIXES)
- def test_is_fix(self, x, y):
- """ Statuses that are explicitly defined as fixes should be fixes
-
- Test that ensures that all tests in the FIXES list are treated as fixes
-
- """
- result = self._generate_summary([x, y])
- self.assertEqual(
- len(result.tests['fixes']), 1,
- "{0} -> {1} is not a fix but should be".format(x, y))
-
- #XXX: is "notrun" -> not(notrun) a change?
- @test_iterations(*[i for i in itertools.product(STATUSES[:-1], STATUSES[:-1])
- if i[0] != i[1]])
- def test_is_change(self, x, y):
- """ Check that all changes are treated as changes
-
- Given two statuses if status1 != status2 then that should be a change
-
- """
- result = self._generate_summary([x, y])
- self.assertEqual(
- len(result.tests['changes']), 1,
- "{0} -> {1} is a not a change but should be".format(x, y))
-
- @test_iterations(*PROBLEMS)
- def test_is_problem(self, x):
- """ Only statuses in the PROBLEMS list should be added to problems """
- result = self._generate_summary([x])
- self.assertEqual(
- len(result.tests['problems']), 1,
- "{0} is not a problem but should be".format(x))
-
- @test_iterations("skip")
- def test_is_skip(self, x):
- """ Ensure that skip is being added to the skip list """
- result = self._generate_summary([x])
- self.assertEqual(
- len(result.tests['skipped']), 1,
- "Skip is not being added to the list of skips")
diff --git a/framework/tests/status_tests.py b/framework/tests/status_tests.py
new file mode 100644
index 0000000..a15bec1
--- /dev/null
+++ b/framework/tests/status_tests.py
@@ -0,0 +1,88 @@
+# Copyright (c) 2014 Intel Corperation
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+""" Tests for the Status module
+
+Note: see framework/status.py for the authoritative list of fixes, regression,
+etc
+
+"""
+
+import itertools
+import framework.status as status
+
+# Statuses from worst to last. NotRun is intentionally not in this list and
+# tested separately because of upcoming features for it
+STATUSES = ["notrun", "pass", "dmesg-warn", "warn", "dmesg-fail", "fail",
+ "crash", "timeout"]
+
+# Create lists of fixes and regressions programmatically based on the STATUSES
+# list. This means less code, and easier expansion changes.
+REGRESSIONS = itertools.combinations(STATUSES, 2)
+FIXES = itertools.combinations(reversed(STATUSES), 2)
+
+# all statuses except pass are problems
+PROBLEMS = STATUSES[1:]
+
+
+def is_regression(x, y):
+ # Test for regressions
+ assert status.status_lookup(x) < status.status_lookup(y)
+
+
+def is_fix(x, y):
+ # Test for fix
+ assert status.status_lookup(x) > status.status_lookup(y)
+
+
+def is_equivalent(x, y):
+ # Test if status is equivalent. Note that this does not mean 'same', two
+ # statuses could be equivalent in terms of fixes and regressions, but that
+ # doesn't require that they are the same status
+ assert status.status_lookup(x) == status.status_lookup(y)
+
+
+def is_not_equivalent(x, y):
+ # Test that status is not equivalent.
+ assert status.status_lookup(x) != status.status_lookup(y)
+
+
+def test_is_regression():
+ # Generate all tests for regressions
+ for x, y in REGRESSIONS:
+ yield is_regression, x, y
+
+
+def test_is_fix():
+ # Generates all tests for fixes
+ for x, y in FIXES:
+ yield is_fix, x, y
+
+
+def test_is_equivalent():
+ # test the assertion that NotRun, Pass and Skip should be considered
+ # equivalent for regression testing.
+ for x, y in itertools.izip(STATUSES, STATUSES):
+ yield is_equivalent, x, y
+
+
+def test_is_change():
+ for x, y in itertools.permutations(STATUSES, 2):
+ yield is_not_equivalent, x, y
diff --git a/piglit-framework-tests.py b/piglit-framework-tests.py
deleted file mode 100755
index aa4d9e6..0000000
--- a/piglit-framework-tests.py
+++ /dev/null
@@ -1,47 +0,0 @@
-#!/usr/bin/env python
-#
-# Copyright (c) 2013 Intel Corporation
-#
-# Permission is hereby granted, free of charge, to any person obtaining a
-# copy of this software and associated documentation files (the "Software"),
-# to deal in the Software without restriction, including without limitation
-# the rights to use, copy, modify, merge, publish, distribute, sublicense,
-# and/or sell copies of the Software, and to permit persons to whom the
-# Software is furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice (including the next
-# paragraph) shall be included in all copies or substantial portions of the
-# Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
-# IN THE SOFTWARE.
-
-import argparse
-import unittest
-
-import framework.tests.status
-
-# Create a dictionary of all of tests. Do this before the parser so we can use
-# it as a list of optional arguments for the parser
-tests = {"status": unittest.TestLoader().loadTestsFromModule(framework.tests.status)}
-
-parser = argparse.ArgumentParser()
-parser.add_argument("tests",
- action="append",
- choices=tests.keys(),
- help="Testing profiles for the framework")
-parser.add_argument("-v", "--verbose",
- action="store",
- choices=['0', '1', '2'],
- default='1',
- help="Set the level of verbosity to run tests at")
-args = parser.parse_args()
-
-# Run the tests
-map(unittest.TextTestRunner(verbosity=int(args.verbose)).run,
- [tests[x] for x in args.tests])
--
1.8.5.3
More information about the Piglit
mailing list