[Piglit] [PATCH v4 6/6] framework: add support for xz compression via backports.lzma

[Dylan Baker]

This adds the option of using a python module, rather than calling out
to the shell for xz support.

v4: - add this patch

Signed-off-by: Dylan Baker <dylanx.c.baker at intel.com>
---
 README                            |   3 +
 framework/backends/compression.py | 129 ++++++++++++++++++++------------------
 2 files changed, 71 insertions(+), 61 deletions(-)

diff --git a/README b/README
index 1968221..773a350 100644
--- a/README
+++ b/README
@@ -45,6 +45,9 @@ Optionally, you can install the following:
   - lxml. An accelerated python xml library using libxml2 (http://lxml.de/)
   - simplejson. A fast C based implementation of the python json library.
     (https://simplejson.readthedocs.org/en/latest/)
+  - backports.lzma. A packport of python3's lzma module to python2,
+    this enables fast native xz (de)compression in piglit for results files
+    (https://github.com/peterjc/backports.lzma)
 
 Now configure the build system:
 
diff --git a/framework/backends/compression.py b/framework/backends/compression.py
index ef97c95..4bb4b0f 100644
--- a/framework/backends/compression.py
+++ b/framework/backends/compression.py
@@ -73,80 +73,87 @@ DECOMPRESSORS = {
 # TODO: in python3 there is builtin xz support, and doesn't need this madness
 # If there is an xz binary then try calling out to that
 try:
-    with open(os.devnull, 'w') as d:
-        subprocess.check_call(['xz'], stderr=d)
-except subprocess.CalledProcessError as e:
-    if e.returncode == 1:
-        import contextlib
-        try:
-            import cStringIO as StringIO
-        except ImportError:
-            import StringIO
+    import backports.lzma
+
+    COMPRESSORS['xz'] = functools.partial(backports.lzma.open, mode='w')
+    DECOMPRESSORS['xz'] = functools.partial(backports.lzma.open, mode='r')
+    COMPRESSION_SUFFIXES += ['.xz']
+except ImportError:
+    try:
+        with open(os.devnull, 'w') as d:
+            subprocess.check_call(['xz'], stderr=d)
+    except subprocess.CalledProcessError as e:
+        if e.returncode == 1:
+            import contextlib
+            try:
+                import cStringIO as StringIO
+            except ImportError:
+                import StringIO
 
-        @contextlib.contextmanager
-        def _compress_xz(filename):
-            """Emulates an open function in write mode for xz.
+            @contextlib.contextmanager
+            def _compress_xz(filename):
+                """Emulates an open function in write mode for xz.
 
-            Python 2.x doesn't support xz, but it's dang useful. This function calls
-            out to the shell and tries to use xz from the environment to get xz
-            compression.
+                Python 2.x doesn't support xz, but it's dang useful. This function calls
+                out to the shell and tries to use xz from the environment to get xz
+                compression.
 
-            This obviously won't work without a working xz binary.
+                This obviously won't work without a working xz binary.
 
-            This function tries to emulate the default values of the lzma module in
-            python3 as much as possible
+                This function tries to emulate the default values of the lzma module in
+                python3 as much as possible
 
-            """
-            if filename.endswith('.xz'):
-                filename = filename[:-2]
+                """
+                if filename.endswith('.xz'):
+                    filename = filename[:-2]
 
-            with open(filename, 'w') as f:
-                yield f
+                with open(filename, 'w') as f:
+                    yield f
 
-            try:
-                subprocess.check_call(['xz', '--compress', '-9', filename])
-            except OSError as e:
-                if e.errno == errno.ENOENT:
-                    raise exceptions.PiglitFatalError('No xz binary available')
-                raise
+                try:
+                    subprocess.check_call(['xz', '--compress', '-9', filename])
+                except OSError as e:
+                    if e.errno == errno.ENOENT:
+                        raise exceptions.PiglitFatalError('No xz binary available')
+                    raise
 
-        @contextlib.contextmanager
-        def _decompress_xz(filename):
-            """Eumlates an option function in read mode for xz.
+            @contextlib.contextmanager
+            def _decompress_xz(filename):
+                """Eumlates an option function in read mode for xz.
 
-            See the comment in _compress_xz for more information.
+                See the comment in _compress_xz for more information.
 
-            This module tries to emulate the lzma module as much as possible
+                This module tries to emulate the lzma module as much as possible
 
-            """
-            if not filename.endswith('.xz'):
-                filename = '{}.xz'.format(filename)
+                """
+                if not filename.endswith('.xz'):
+                    filename = '{}.xz'.format(filename)
 
-            print('decompress: ' + filename)
+                print('decompress: ' + filename)
 
-            try:
-                string = subprocess.check_output(
-                    ['xz', '--decompress', '--stdout', filename])
-            except OSError as e:
-                if e.errno == errno.ENOENT:
-                    raise exceptions.PiglitFatalError('No xz binary available')
-                raise
-
-            # We need a file-like object, so the contents must be placed in a StringIO
-            # object.
-            io = StringIO.StringIO()
-            io.write(string)
-            io.seek(0)
-
-            yield io
-
-            io.close()
-
-        COMPRESSORS['xz'] = _compress_xz
-        DECOMPRESSORS['xz'] = _decompress_xz
-        COMPRESSION_SUFFIXES += ['.xz']
-except OSError:
-    pass
+                try:
+                    string = subprocess.check_output(
+                        ['xz', '--decompress', '--stdout', filename])
+                except OSError as e:
+                    if e.errno == errno.ENOENT:
+                        raise exceptions.PiglitFatalError('No xz binary available')
+                    raise
+
+                # We need a file-like object, so the contents must be placed in a StringIO
+                # object.
+                io = StringIO.StringIO()
+                io.write(string)
+                io.seek(0)
+
+                yield io
+
+                io.close()
+
+            COMPRESSORS['xz'] = _compress_xz
+            DECOMPRESSORS['xz'] = _decompress_xz
+            COMPRESSION_SUFFIXES += ['.xz']
+    except OSError:
+        pass
 
 
 def _set_mode():
-- 
2.4.5