[Piglit] [PATCH v4 3/6] framework: add support for gzip compressionn

[Dylan Baker]

This adds support to compress results with gzip compression. This
reduces the size of json results significantly (from 21M to 1.6M when
running the quick profile (which is about 7% of the uncompressed size).

v4: - add additional tests

Signed-off-by: Dylan Baker <dylanx.c.baker at intel.com>
---
 framework/backends/compression.py           |  9 ++-
 framework/tests/compressed_backend_tests.py | 89 ++++++++++++++++++++++++++++-
 piglit.conf.example                         |  6 +-
 3 files changed, 97 insertions(+), 7 deletions(-)

diff --git a/framework/backends/compression.py b/framework/backends/compression.py
index 37383e4..44b8b9e 100644
--- a/framework/backends/compression.py
+++ b/framework/backends/compression.py
@@ -43,21 +43,24 @@ they're passing unicode and not bytes.
 
 from __future__ import print_function, absolute_import, division
 import functools
+import gzip
 import os
 
 from framework import exceptions
 from framework.core import PIGLIT_CONFIG
 
-COMPRESSION_SUFFIXES = []
+COMPRESSION_SUFFIXES = ['.gz']
 
-DEFAULT = 'none'
+DEFAULT = 'gz'
 
 COMPRESSORS = {
+    'gz': functools.partial(gzip.open, mode='w'),
     'none': functools.partial(open, mode='w'),
 }
 
 DECOMPRESSORS = {
-    'none': functools.partial(open, mode='r')
+    'gz': functools.partial(gzip.open, mode='r'),
+    'none': functools.partial(open, mode='r'),
 }
 
 
diff --git a/framework/tests/compressed_backend_tests.py b/framework/tests/compressed_backend_tests.py
index eb40f0a..897a154 100644
--- a/framework/tests/compressed_backend_tests.py
+++ b/framework/tests/compressed_backend_tests.py
@@ -32,10 +32,31 @@ import functools
 import nose.tools as nt
 
 from framework.tests import utils
-from framework.backends import compression
+from framework.backends import compression, abstract
 
 # pylint: disable=line-too-long,protected-access
 
+# Helpers
+
+
+class _TestBackend(abstract.FileBackend):
+    """A class for testing backend compression."""
+    _file_extension = 'test_extension'
+
+    def initialize(self, *args, **kwargs):  # pylint: disable=unused-argument
+        os.mkdir(os.path.join(self._dest, 'tests'))
+
+    def finalize(self, *args, **kwargs): # pylint: disable=unused-argument
+        tests = os.path.join(self._dest, 'tests')
+        with self._write_final(os.path.join(self._dest, 'results.txt')) as f:
+            for file_ in os.listdir(tests):
+                with open(os.path.join(tests, file_), 'r') as t:
+                    f.write(t.read())
+
+    @staticmethod
+    def _write(f, name, data):  # pylint: disable=arguments-differ
+        f.write('{}: {}'.format(name, data))
+
 
 def _add_compression(value):
     """Decorator that temporarily adds support for a compression method."""
@@ -60,6 +81,31 @@ def _add_compression(value):
     return _wrapper
 
 
+def _set_compression_mode(mode):
+    """Change the compression mode for one test."""
+
+    def _wrapper(func):
+        """The actual decorator."""
+
+        @functools.wraps(func)
+        @utils.set_env(PIGLIT_COMPRESSION=mode)
+        def _inner(*args, **kwargs):
+            """The called function."""
+            restore = compression.MODE
+            compression.MODE = compression._set_mode()
+            compression.COMPRESSOR = compression.COMPRESSORS[compression.MODE]
+
+            try:
+                func(*args, **kwargs)
+            finally:
+                compression.MODE = restore
+                compression.COMPRESSOR = compression.COMPRESSORS[compression.MODE]
+
+        return _inner
+
+    return _wrapper
+
+
 def _test_compressor(mode):
     """Helper to simplify testing compressors."""
     func = compression.COMPRESSORS[mode]
@@ -83,6 +129,29 @@ def _test_decompressor(mode):
             nt.eq_(f.read(), 'foo')
 
 
+def _test_extension():
+    """Create an final file and return the extension."""
+    with utils.tempdir() as d:
+        obj = _TestBackend(d)
+        obj.initialize()
+        with obj.write_test('foo') as t:
+            t({'result': 'foo'})
+
+        obj.finalize()
+
+        for each in os.listdir(d):
+            if each.startswith('results.txt'):
+                ext = os.path.splitext(each)[1]
+                break
+        else:
+            raise utils.TestFailure('No results file generated')
+
+    return ext
+
+
+# Tests
+
+
 @utils.no_error
 def test_compress_none():
     """framework.backends.compression: can compress to 'none'"""
@@ -94,6 +163,7 @@ def test_decompress_none():
     _test_decompressor('none')
 
 
+
 @_add_compression('foobar')
 @utils.set_env(PIGLIT_COMPRESSION='foobar')
 def test_set_mode_env():
@@ -114,3 +184,20 @@ def test_set_mode_piglit_conf():
 def test_set_mode_default():
     """framework.backends.compression._set_mode: uses DEFAULT if env and piglit.conf are unset"""
     nt.eq_(compression._set_mode(), compression.DEFAULT)
+
+
+ at utils.no_error
+def test_compress_gz():
+    """framework.backends.compression: can compress to 'gz'"""
+    _test_compressor('gz')
+
+
+def test_decompress_gz():
+    """framework.backends.compression: can decompress from 'gz'"""
+    _test_decompressor('gz')
+
+
+ at _set_compression_mode('gz')
+def test_gz_output():
+    """framework.backends: when using gz compression a gz file is created"""
+    nt.eq_(_test_extension(), '.gz')
diff --git a/piglit.conf.example b/piglit.conf.example
index 49bc8b6..baf8108 100644
--- a/piglit.conf.example
+++ b/piglit.conf.example
@@ -111,9 +111,9 @@ run_test=./%(test_name)s
 ;backend=json
 
 ; Set the default compression method to use,
-; May be one of: 'none'
-; Default: none (Note that this may change in the future)
-;compression=none
+; May be one of: 'none', 'gz'
+; Default: 'gz'
+;compression=gz
 
 [expected-failures]
 ; Provide a list of test names that are expected to fail.  These tests
-- 
2.4.5