[Piglit] [PATCH 24/44] compression.py: use the proper read and write modes for python versions

baker.dylan.c at gmail.com baker.dylan.c at gmail.com
Wed Jan 27 16:06:32 PST 2016


From: Dylan Baker <baker.dylan.c at gmail.com>

In python 2 the write mode should be 'w', and the read 'r', but in
python 3 they need to be 'wt', and 'rt' respectively. Python 3 also has
an lzma module built in, which vastly simplifies supporting xz.

Signed-off-by: Dylan Baker <dylanx.c.baker at intel.com>
---
 framework/backends/compression.py     | 216 +++++++++++++++++++---------------
 unittests/compressed_backend_tests.py |   8 +-
 2 files changed, 123 insertions(+), 101 deletions(-)

diff --git a/framework/backends/compression.py b/framework/backends/compression.py
index a1b9ade..863693e 100644
--- a/framework/backends/compression.py
+++ b/framework/backends/compression.py
@@ -72,109 +72,129 @@ class UnsupportedCompressor(exceptions.PiglitInternalError):
         return u'unsupported compression method {}'.format(self.__method)
 
 
-# TODO: in python3 the bz2 module has an open function
-COMPRESSION_SUFFIXES = ['.gz', '.bz2']
 
 DEFAULT = 'bz2'
 
-COMPRESSORS = {
-    'bz2': functools.partial(bz2.BZ2File, mode='w'),
-    'gz': functools.partial(gzip.open, mode='w'),
-    'none': functools.partial(open, mode='w'),
-}
-
-DECOMPRESSORS = {
-    'bz2': functools.partial(bz2.BZ2File, mode='r'),
-    'gz': functools.partial(gzip.open, mode='r'),
-    'none': functools.partial(open, mode='r'),
-}
-
-# TODO: in python3 there is builtin xz support, and doesn't need this madness
-# First try to use backports.lzma, that's the easiest solution. If that fails
-# then go to trying the shell. If that fails then piglit won't have xz support,
-# and will raise an error if xz is used
-try:
-    import backports.lzma
-
-    COMPRESSORS['xz'] = functools.partial(backports.lzma.open, mode='w')
-    DECOMPRESSORS['xz'] = functools.partial(backports.lzma.open, mode='r')
-    COMPRESSION_SUFFIXES += ['.xz']
-except ImportError:
+if six.PY2:
+    COMPRESSION_SUFFIXES = ['.gz', '.bz2']
+    COMPRESSORS = {
+        'bz2': functools.partial(bz2.BZ2File, mode='w'),
+        'gz': functools.partial(gzip.open, mode='w'),
+        'none': functools.partial(open, mode='w'),
+    }
+
+    DECOMPRESSORS = {
+        'bz2': functools.partial(bz2.BZ2File, mode='r'),
+        'gz': functools.partial(gzip.open, mode='r'),
+        'none': functools.partial(open, mode='r'),
+    }
+
+    # First try to use backports.lzma, that's the easiest solution. If that
+    # fails then go to trying the shell. If that fails then piglit won't have
+    # xz support, and will raise an error if xz is used
     try:
-        with open(os.devnull, 'w') as d:
-            subprocess.check_call(['xz', '--help'], stdout=d, stderr=d)
-    except OSError:
-        pass
-    else:
-
-        @contextlib.contextmanager
-        def _compress_xz(filename):
-            """Emulates an open function in write mode for xz.
-
-            Python 2.x doesn't support xz, but it's dang useful. This
-            function calls out to the shell and tries to use xz from the
-            environment to get xz compression.
-
-            This obviously won't work without a working xz binary.
-
-            This function tries to emulate the default values of the lzma
-            module in python3 as much as possible
-
-            """
-            if filename.endswith('.xz'):
-                filename = filename[:-3]
-
-            with open(filename, 'w') as f:
-                yield f
-
-            try:
-                with open(os.devnull, 'w') as null:
-                    subprocess.check_call(
-                        ['xz', '--compress', '-9', '--force', filename],
-                        stderr=null)
-            except OSError as e:
-                if e.errno == errno.ENOENT:
-                    raise exceptions.PiglitFatalError(
-                        'No xz binary available')
-                raise
-
-        @contextlib.contextmanager
-        def _decompress_xz(filename):
-            """Eumlates an option function in read mode for xz.
-
-            See the comment in _compress_xz for more information.
-
-            This function tries to emulate the lzma module as much as
-            possible
-
-            """
-            if not filename.endswith('.xz'):
-                filename = '{}.xz'.format(filename)
-
-            try:
-                with open(os.devnull, 'w') as null:
-                    string = subprocess.check_output(
-                        ['xz', '--decompress', '--stdout', filename],
-                        stderr=null)
-            except OSError as e:
-                if e.errno == errno.ENOENT:
-                    raise exceptions.PiglitFatalError(
-                        'No xz binary available')
-                raise
-
-            # We need a file-like object, so the contents must be placed in
-            # a StringIO object.
-            io = StringIO()
-            io.write(string)
-            io.seek(0)
-
-            yield io
-
-            io.close()
-
-        COMPRESSORS['xz'] = _compress_xz
-        DECOMPRESSORS['xz'] = _decompress_xz
+        import backports.lzma  # pylint: disable=wrong-import-position
+
+        COMPRESSORS['xz'] = functools.partial(backports.lzma.open, mode='w')
+        DECOMPRESSORS['xz'] = functools.partial(backports.lzma.open, mode='r')
         COMPRESSION_SUFFIXES += ['.xz']
+    except ImportError:
+        try:
+            with open(os.devnull, 'w') as d:
+                subprocess.check_call(['xz', '--help'], stdout=d, stderr=d)
+        except OSError:
+            pass
+        else:
+
+            @contextlib.contextmanager
+            def _compress_xz(filename):
+                """Emulates an open function in write mode for xz.
+
+                Python 2.x doesn't support xz, but it's dang useful. This
+                function calls out to the shell and tries to use xz from the
+                environment to get xz compression.
+
+                This obviously won't work without a working xz binary.
+
+                This function tries to emulate the default values of the lzma
+                module in python3 as much as possible
+
+                """
+                if filename.endswith('.xz'):
+                    filename = filename[:-3]
+
+                with open(filename, 'w') as f:
+                    yield f
+
+                try:
+                    with open(os.devnull, 'w') as null:
+                        subprocess.check_call(
+                            ['xz', '--compress', '-9', '--force', filename],
+                            stderr=null)
+                except OSError as e:
+                    if e.errno == errno.ENOENT:
+                        raise exceptions.PiglitFatalError(
+                            'No xz binary available')
+                    raise
+
+            @contextlib.contextmanager
+            def _decompress_xz(filename):
+                """Eumlates an option function in read mode for xz.
+
+                See the comment in _compress_xz for more information.
+
+                This function tries to emulate the lzma module as much as
+                possible
+
+                """
+                if not filename.endswith('.xz'):
+                    filename = '{}.xz'.format(filename)
+
+                try:
+                    with open(os.devnull, 'w') as null:
+                        string = subprocess.check_output(
+                            ['xz', '--decompress', '--stdout', filename],
+                            stderr=null)
+                except OSError as e:
+                    if e.errno == errno.ENOENT:
+                        raise exceptions.PiglitFatalError(
+                            'No xz binary available')
+                    raise
+
+                # We need a file-like object, so the contents must be placed in
+                # a StringIO object.
+                io = StringIO()
+                io.write(string)
+                io.seek(0)
+
+                yield io
+
+                io.close()
+
+            COMPRESSORS['xz'] = _compress_xz
+            DECOMPRESSORS['xz'] = _decompress_xz
+            COMPRESSION_SUFFIXES += ['.xz']
+else:
+    # In the case of python 3 this all just works, no monkeying around with
+    # imports and fallbacks. just import the right modules and go
+
+    import lzma  # pylint: disable=wrong-import-position,wrong-import-order
+
+    COMPRESSION_SUFFIXES = ['.gz', '.bz2', '.xz']
+
+    COMPRESSORS = {
+        'bz2': functools.partial(bz2.open, mode='wt'),
+        'gz': functools.partial(gzip.open, mode='wt'),
+        'none': functools.partial(open, mode='w'),
+        'xz': functools.partial(lzma.open, mode='wt'),
+    }
+
+    DECOMPRESSORS = {
+        'bz2': functools.partial(bz2.open, mode='rt'),
+        'gz': functools.partial(gzip.open, mode='rt'),
+        'none': functools.partial(open, mode='r'),
+        'xz': functools.partial(lzma.open, mode='rt'),
+    }
 
 
 def get_mode():
diff --git a/unittests/compressed_backend_tests.py b/unittests/compressed_backend_tests.py
index 19eea24..270b9b4 100644
--- a/unittests/compressed_backend_tests.py
+++ b/unittests/compressed_backend_tests.py
@@ -33,9 +33,10 @@ import functools
 
 import nose.tools as nt
 from nose.plugins.skip import SkipTest
+import six
 
-from framework import results
 from . import utils
+from framework import results
 from framework.backends import compression, abstract
 
 # pylint: disable=line-too-long,protected-access
@@ -236,8 +237,9 @@ def test_update_piglit_conf():
 @utils.test_in_tempdir
 def test_xz_shell_override():
     """framework.backends.compression: the xz shell utility path can overwrite"""
-    # TODO: this test will not be required by python3, where the builtin lzma
-    # module replaces all of this.
+    if six.PY3:
+        raise SkipTest('Test is irrelvent on python 3')
+
     try:
         import backports.lzma  # pylint: disable=unused-variable
     except ImportError:
-- 
2.7.0



More information about the Piglit mailing list