[Piglit] [PATCH 1/2] framework: Add ability to set a compression method on file backends

Dylan Baker baker.dylan.c at gmail.com
Fri May 29 14:43:32 PDT 2015


This creates a framework for compressing results as part of the
FileBackend class. This allows for the simple addition of compression
formats (gz is easily possible in python2, xz in python3), but does not
actually implmenet any compression.

This patch implements a framework, and tests for that framework.

Signed-off-by: Dylan Baker <dylanx.c.baker at intel.com>
---
 framework/backends/__init__.py        | 34 ++++++++++++------
 framework/backends/abstract.py        | 45 +++++++++++++++++++----
 framework/backends/json.py            | 12 +++----
 framework/tests/json_backend_tests.py | 68 +++++++++++++++++++++++++++++++++--
 framework/tests/utils.py              | 27 +++++++++-----
 piglit.conf.example                   |  5 +++
 6 files changed, 155 insertions(+), 36 deletions(-)

diff --git a/framework/backends/__init__.py b/framework/backends/__init__.py
index e2f199a..8af0326 100644
--- a/framework/backends/__init__.py
+++ b/framework/backends/__init__.py
@@ -55,6 +55,8 @@ __all__ = [
     'set_meta',
 ]
 
+_COMPRESSION_SUFFIXES = []
+
 
 class BackendError(Exception):
     pass
@@ -117,27 +119,37 @@ def load(file_path):
     then return the TestrunResult instance.
 
     """
+    def get_extension(file_path):
+        """Helper function to get the extension string."""
+        name, extension = os.path.splitext(file_path)
+        # If we hit a compressed suffix, get an additional suffix to test with.
+        # i.e: Use .json.gz rather that .gz
+        if extension in _COMPRESSION_SUFFIXES:
+            extension = os.path.splitext(name)[1] + extension
+        elif not extension:
+            extension = ''
+
+        return extension
+
     extension = None
 
     if os.path.isfile(file_path):
-        extension = os.path.splitext(file_path)[1]
-        if not extension:
-            extension = ''
+        extension = get_extension(file_path)
     else:
-        for file in os.listdir(file_path):
-            if file.startswith('result'):
-                extension = os.path.splitext(file)[1]
+        for file_ in os.listdir(file_path):
+            if file_.startswith('result'):
+                extension = get_extension(file_)
                 break
-            elif file == 'main':
+            elif file_ == 'main':
                 extension = ''
                 break
-    tests = os.path.join(file_path, 'tests')
     if extension is None:
+        tests = os.path.join(file_path, 'tests')
         if os.path.exists(tests):
-            extension = os.path.splitext(os.listdir(tests)[0])[1]
+            extension = get_extension(os.listdir(tests)[0])
         else:
-            # At this point we have failed to find any sort of backend, just except
-            # and die
+            # At this point we have failed to find any sort of backend, just
+            # except and die
             raise BackendError("No backend found for any file in {}".format(
                 file_path))
 
diff --git a/framework/backends/abstract.py b/framework/backends/abstract.py
index 47186f2..38c0b39 100644
--- a/framework/backends/abstract.py
+++ b/framework/backends/abstract.py
@@ -26,15 +26,45 @@ This module provides mixins and base classes for backend modules.
 """
 
 from __future__ import print_function, absolute_import
-import os
 import abc
-import shutil
-import itertools
 import contextlib
+import functools
+import itertools
+import os
+import shutil
 
+from framework.core import PIGLIT_CONFIG
 from framework.results import TestResult
 from framework.status import INCOMPLETE
 
+_DEFAULT_COMPRESSION_MODE = 'none'
+_COMPRESSORS = {
+    'none': functools.partial(open, mode='w'),
+}
+
+
+ at contextlib.contextmanager
+def write_compressed(filename):
+    """Write a the final result using desired compression.
+
+    This helper function reads the piglit.conf to decide whether to use
+    compression, and what type of compression to use.
+
+    Currently it implements no compression
+
+    """
+    method = (os.environ.get('PIGLIT_COMPRESSION') or
+              PIGLIT_CONFIG.safe_get('core', 'compression') or
+              _DEFAULT_COMPRESSION_MODE)
+    assert method in _COMPRESSORS, \
+        'unsupported compression method {}'.format(method)
+
+    if method != 'none':
+        filename = '{}.{}'.format(filename, method)
+
+    with _COMPRESSORS[method](filename) as f:
+        yield f
+
 
 class Backend(object):
     """ Abstract base class for summary backends
@@ -143,8 +173,9 @@ class FileBackend(Backend):
     """
     def __init__(self, dest, file_start_count=0, file_fsync=False, **kwargs):
         self._dest = dest
-        self.__counter = itertools.count(file_start_count)
-        self.__file_sync = file_fsync
+        self._counter = itertools.count(file_start_count)
+        self._file_sync = file_fsync
+        self._write_final = write_compressed
 
     __INCOMPLETE = TestResult({'result': INCOMPLETE})
 
@@ -155,7 +186,7 @@ class FileBackend(Backend):
 
         """
         file_.flush()
-        if self.__file_sync:
+        if self._file_sync:
             os.fsync(file_.fileno())
 
     @abc.abstractmethod
@@ -188,7 +219,7 @@ class FileBackend(Backend):
             shutil.move(tfile, file_)
 
         file_ = os.path.join(self._dest, 'tests', '{}.{}'.format(
-            self.__counter.next(), self._file_extension))
+            self._counter.next(), self._file_extension))
 
         with open(file_, 'w') as f:
             self._write(f, name, self.__INCOMPLETE)
diff --git a/framework/backends/json.py b/framework/backends/json.py
index 2fe239c..e3be2f5 100644
--- a/framework/backends/json.py
+++ b/framework/backends/json.py
@@ -32,7 +32,7 @@ except ImportError:
     import json
 
 from framework import status, results, exceptions
-from .abstract import FileBackend
+from .abstract import FileBackend, write_compressed
 from .register import Registry
 
 __all__ = [
@@ -140,10 +140,10 @@ class JSONBackend(FileBackend):
                     pass
         assert data['tests']
 
-        # write out the combined file.
-        with open(os.path.join(self._dest, 'results.json'), 'w') as f:
-            json.dump(data, f, default=piglit_encoder,
-                      indent=INDENT)
+        # write out the combined file. Use the compression writer from the
+        # FileBackend
+        with self._write_final(os.path.join(self._dest, 'results.json')) as f:
+            json.dump(data, f, default=piglit_encoder, indent=INDENT)
 
         # Delete the temporary files
         os.unlink(os.path.join(self._dest, 'metadata.json'))
@@ -310,7 +310,7 @@ def _update_results(results, filepath):
 
 def _write(results, file_):
     """WRite the values of the results out to a file."""
-    with open(file_, 'w') as f:
+    with write_compressed(file_) as f:
         json.dump({k:v for k, v in results.__dict__.iteritems()},
                   f,
                   default=piglit_encoder,
diff --git a/framework/tests/json_backend_tests.py b/framework/tests/json_backend_tests.py
index 889332a..ac460d7 100644
--- a/framework/tests/json_backend_tests.py
+++ b/framework/tests/json_backend_tests.py
@@ -24,6 +24,7 @@
 
 from __future__ import print_function, absolute_import
 import os
+import functools
 
 try:
     import simplejson as json
@@ -32,9 +33,36 @@ except ImportError:
 import nose.tools as nt
 
 from framework import results, backends, exceptions
+from framework.core import PIGLIT_CONFIG
 import framework.tests.utils as utils
 from .backends_tests import BACKEND_INITIAL_META
 
+PIGLIT_CONFIG.add_section('core')
+
+
+def _set_compression(compression):
+    """Set a specific compression level on at test."""
+    def _decorator(func):
+        """The actual decorator."""
+
+        @functools.wraps(func)
+        def _inner(*args, **kwargs):
+            """The returned function wrapper."""
+            restore = PIGLIT_CONFIG.safe_get('core', 'compression')
+            PIGLIT_CONFIG.set('core', 'compression', compression)
+
+            try:
+                func(*args, **kwargs)
+            finally:
+                if restore:
+                    PIGLIT_CONFIG.set('core', 'compression', restore)
+                else:
+                    PIGLIT_CONFIG.remove_option('core', 'compression')
+
+        return _inner
+
+    return _decorator
+
 
 def test_initialize_jsonbackend():
     """backends.json.JSONBackend: Class initializes
@@ -91,8 +119,9 @@ class TestJSONTestMethod(utils.StaticDirectory):
         nt.assert_dict_equal({self.test_name: self.result}, test)
 
 
-class TestJSONTestFinalize(utils.StaticDirectory):
+class TestJSONTestFinalizeNone(utils.StaticDirectory):
     @classmethod
+    @_set_compression('none')
     def setup_class(cls):
         cls.test_name = 'a/test/group/test1'
         cls.result = results.TestResult({
@@ -101,7 +130,7 @@ class TestJSONTestFinalize(utils.StaticDirectory):
             'out': 'this is stdout',
             'err': 'this is stderr',
         })
-        super(TestJSONTestFinalize, cls).setup_class()
+        super(TestJSONTestFinalizeNone, cls).setup_class()
         test = backends.json.JSONBackend(cls.tdir)
         test.initialize(BACKEND_INITIAL_META)
         with test.write_test(cls.test_name) as t:
@@ -117,7 +146,8 @@ class TestJSONTestFinalize(utils.StaticDirectory):
         assert not os.path.exists(os.path.join(self.tdir, 'tests'))
 
     def test_create_results(self):
-        """backends.json.JSONBackend.finalize(): creates a results.json file"""
+        """backends.json.JSONBackend.finalize(): creates a results.json file
+        """
         assert os.path.exists(os.path.join(self.tdir, 'results.json'))
 
     @utils.no_error
@@ -127,6 +157,38 @@ class TestJSONTestFinalize(utils.StaticDirectory):
             json.load(f)
 
 
+class TestJSONTestFinalizeEnvNone(utils.StaticDirectory):
+    @classmethod
+    @utils.set_env(PIGLIT_COMPRESSION='none')
+    @_set_compression('gz')
+    def setup_class(cls):
+        cls.test_name = 'a/test/group/test1'
+        cls.result = results.TestResult({
+            'time': 1.2345,
+            'result': 'pass',
+            'out': 'this is stdout',
+            'err': 'this is stderr',
+        })
+        super(TestJSONTestFinalizeEnvNone, cls).setup_class()
+        test = backends.json.JSONBackend(cls.tdir)
+        test.initialize(BACKEND_INITIAL_META)
+        with test.write_test(cls.test_name) as t:
+            t(cls.result)
+        test.finalize()
+
+    def test_create_results(self):
+        """backends.json.JSONBackend.finalize(): creates a results.json file with env
+        """
+        assert os.path.exists(os.path.join(self.tdir, 'results.json'))
+
+    @utils.no_error
+    def test_results_valid(self):
+        """backends.json.JSONBackend.finalize(): results.json is valid with env
+        """
+        with open(os.path.join(self.tdir, 'results.json'), 'r') as f:
+            json.load(f)
+
+
 def test_update_results_current():
     """backends.json.update_results(): returns early when the results_version is current"""
     data = utils.JSON_DATA.copy()
diff --git a/framework/tests/utils.py b/framework/tests/utils.py
index 9fd1cc8..a75f9fe 100644
--- a/framework/tests/utils.py
+++ b/framework/tests/utils.py
@@ -362,17 +362,26 @@ def capture_stderr(func):
     return _inner
 
 
-def not_raises(exception):
-    """Decorator for tests that should not raise one of the follow exceptions.
-    """
-    def _decorator(function):
-        @functools.wraps(function)
+def set_env(**envargs):
+    """Decorator that sets environment variables and then unsets them."""
+    def _decorator(func):
+        """The actual decorator."""
+        @functools.wraps(func)
         def _inner(*args, **kwargs):
+            """The returned function."""
+            backup = {}
+            for key, value in envargs.iteritems():
+                backup[key] = os.environ.get(key, "__DONOTRESTORE__")
+                os.environ[key] = value
+
             try:
-                function(*args, **kwargs)
-            except exception as e:
-                raise TestFailure(e)
+                func(*args, **kwargs)
+            finally:
+                for key, value in backup.iteritems():
+                    if value == "__DONOTRESTORE__":
+                        del os.environ[key]
+                    else:
+                        os.environ[key] = value
 
         return _inner
-
     return _decorator
diff --git a/piglit.conf.example b/piglit.conf.example
index c30eef4..9601ad6 100644
--- a/piglit.conf.example
+++ b/piglit.conf.example
@@ -90,6 +90,11 @@ run_test=./%(test_name)s
 ; -b/--backend
 ;backend=json
 
+; Set the default compression method to use,
+; May be one of: 'none'
+; Default: none (Note that this may change in the future)
+;compression=none
+
 [expected-failures]
 ; Provide a list of test names that are expected to fail.  These tests
 ; will be listed as passing in JUnit output when they fail.  Any
-- 
2.4.2



More information about the Piglit mailing list