[Piglit] [Patch v2] generated_tests: Catch exception in os.makedirs

Dylan Baker baker.dylan.c at gmail.com
Fri Dec 12 11:09:25 PST 2014


Currently we check for the existence of a directory, and if it doesn't
exist we create it. This would be fine, except that multiple generators
can create the same directory trees and when os.makedirs tries to create
a tree that already exists it raises and OSError exception, with the
errno set to 17 (or errno.EEXIST). Since between the time that a check
is made and the time the os.makedirs completes a second generator can
create the same directory tree we can have exceptions raised
sporadically, especially as the number of threads being used to generate
goes up.

We maintain the check for existence before trying to create the
directories because os.makedirs is slow, so not even trying is better
than trying in vain.

v2: - Use a helper function to reduce code duplication
    - use errno.EEXIST instead of magic number 17 (Mark)

Signed-off-by: Dylan Baker <dylanx.c.baker at intel.com>
---
 generated_tests/gen_builtin_packing_tests.py       |  4 +-
 generated_tests/gen_builtin_uniform_tests.py       |  5 ++-
 generated_tests/gen_builtin_uniform_tests_fp64.py  |  5 ++-
 generated_tests/gen_const_builtin_equal_tests.py   |  4 +-
 generated_tests/gen_constant_array_size_tests.py   |  4 +-
 .../gen_constant_array_size_tests_fp64.py          |  4 +-
 generated_tests/gen_interpolation_tests.py         |  4 +-
 generated_tests/gen_non-lvalue_tests.py            |  4 +-
 generated_tests/gen_outerproduct_invalid_params.py |  4 +-
 generated_tests/gen_outerproduct_tests.py          |  4 +-
 generated_tests/gen_shader_bit_encoding_tests.py   |  4 +-
 .../gen_shader_image_load_store_tests.py           |  5 ++-
 generated_tests/gen_texture_lod_tests.py           |  4 +-
 generated_tests/gen_texture_query_lod_tests.py     |  4 +-
 generated_tests/gen_uniform_initializer_tests.py   |  7 ++--
 generated_tests/genclbuiltins.py                   |  5 ++-
 generated_tests/generate-cl-store-tests.py         |  5 ++-
 .../interpolation-qualifier-built-in-variable.py   |  5 +--
 generated_tests/modules/__init__.py                |  0
 .../utils.py}                                      | 48 ++++++++++------------
 20 files changed, 64 insertions(+), 65 deletions(-)
 create mode 100644 generated_tests/modules/__init__.py
 copy generated_tests/{gen_outerproduct_invalid_params.py => modules/utils.py} (56%)

diff --git a/generated_tests/gen_builtin_packing_tests.py b/generated_tests/gen_builtin_packing_tests.py
index 52bf1a0..901f493 100644
--- a/generated_tests/gen_builtin_packing_tests.py
+++ b/generated_tests/gen_builtin_packing_tests.py
@@ -43,6 +43,7 @@ from math import copysign, fabs, fmod, frexp, isinf, isnan, modf
 from numpy import int8, int16, uint8, uint16, uint32, float32
 
 from templates import template_dir
+from modules import utils
 
 TEMPLATES = template_dir(os.path.basename(os.path.splitext(__file__)[0]))
 
@@ -1027,8 +1028,7 @@ class ShaderTest(object):
 
     def write_file(self):
         dirname = os.path.dirname(self.filename)
-        if not os.path.exists(dirname):
-            os.makedirs(dirname)
+        utils.safe_makedirs(dirname)
 
         with open(self.filename, "w") as f:
             f.write(self.__template.render(func=self.__func_info))
diff --git a/generated_tests/gen_builtin_uniform_tests.py b/generated_tests/gen_builtin_uniform_tests.py
index d89a6f3..9bb3075 100644
--- a/generated_tests/gen_builtin_uniform_tests.py
+++ b/generated_tests/gen_builtin_uniform_tests.py
@@ -52,6 +52,8 @@ import os
 import os.path
 import sys
 
+from modules import utils
+
 
 def compute_offset_and_scale(test_vectors):
     """Compute scale and offset values such that for each result in
@@ -537,8 +539,7 @@ class ShaderTest(object):
         shader_test += self.make_test()
         filename = self.filename()
         dirname = os.path.dirname(filename)
-        if not os.path.exists(dirname):
-            os.makedirs(dirname)
+        utils.safe_makedirs(dirname)
         with open(filename, 'w') as f:
             f.write(shader_test)
 
diff --git a/generated_tests/gen_builtin_uniform_tests_fp64.py b/generated_tests/gen_builtin_uniform_tests_fp64.py
index ec12650..23032bd 100644
--- a/generated_tests/gen_builtin_uniform_tests_fp64.py
+++ b/generated_tests/gen_builtin_uniform_tests_fp64.py
@@ -52,6 +52,8 @@ import os
 import os.path
 import sys
 
+from modules import utils
+
 
 def compute_offset_and_scale(test_vectors):
     """Compute scale and offset values such that for each result in
@@ -507,8 +509,7 @@ class ShaderTest(object):
         shader_test += self.make_test()
         filename = self.filename()
         dirname = os.path.dirname(filename)
-        if not os.path.exists(dirname):
-            os.makedirs(dirname)
+        utils.safe_makedirs(dirname)
         with open(filename, 'w') as f:
             f.write(shader_test)
 
diff --git a/generated_tests/gen_const_builtin_equal_tests.py b/generated_tests/gen_const_builtin_equal_tests.py
index eb2856c..47a238c 100644
--- a/generated_tests/gen_const_builtin_equal_tests.py
+++ b/generated_tests/gen_const_builtin_equal_tests.py
@@ -25,6 +25,7 @@ import re
 import os
 
 from templates import template_file
+from modules import utils
 
 TEMPLATE = template_file(os.path.basename(os.path.splitext(__file__)[0]),
                          'template.shader_test.mako')
@@ -73,8 +74,7 @@ def main():
     """ Main function """
     dirname = os.path.join('spec', 'glsl-1.20', 'execution',
                            'built-in-functions')
-    if not os.path.exists(dirname):
-        os.makedirs(dirname)
+    utils.safe_makedirs(dirname)
 
     for test_id, x in enumerate(TEST_VECTORS, start=2):
         # make equal tests
diff --git a/generated_tests/gen_constant_array_size_tests.py b/generated_tests/gen_constant_array_size_tests.py
index c395b1a..c6f75ab 100644
--- a/generated_tests/gen_constant_array_size_tests.py
+++ b/generated_tests/gen_constant_array_size_tests.py
@@ -40,6 +40,7 @@ import optparse
 import os
 import os.path
 
+from modules import utils
 
 class ParserTest(object):
     """Class used to build a test of a single built-in.  This is an
@@ -198,8 +199,7 @@ class ParserTest(object):
         parser_test += self.make_shader()
         filename = self.filename()
         dirname = os.path.dirname(filename)
-        if not os.path.exists(dirname):
-            os.makedirs(dirname)
+        utils.safe_makedirs(dirname)
         with open(filename, 'w') as f:
             f.write(parser_test)
 
diff --git a/generated_tests/gen_constant_array_size_tests_fp64.py b/generated_tests/gen_constant_array_size_tests_fp64.py
index a54598b..8d4a785 100644
--- a/generated_tests/gen_constant_array_size_tests_fp64.py
+++ b/generated_tests/gen_constant_array_size_tests_fp64.py
@@ -40,6 +40,7 @@ import optparse
 import os
 import os.path
 
+from modules import utils
 
 class ParserTest(object):
     """Class used to build a test of a single built-in.  This is an
@@ -199,8 +200,7 @@ class ParserTest(object):
         parser_test += self.make_shader()
         filename = self.filename()
         dirname = os.path.dirname(filename)
-        if not os.path.exists(dirname):
-            os.makedirs(dirname)
+        utils.safe_makedirs(dirname)
         with open(filename, 'w') as f:
             f.write(parser_test)
 
diff --git a/generated_tests/gen_interpolation_tests.py b/generated_tests/gen_interpolation_tests.py
index 4f1428b..1700a20 100644
--- a/generated_tests/gen_interpolation_tests.py
+++ b/generated_tests/gen_interpolation_tests.py
@@ -71,6 +71,7 @@ from __future__ import print_function
 import os
 
 from templates import template_file
+from modules import utils
 
 TEMPLATE = template_file(os.path.basename(os.path.splitext(__file__)[0]),
                          'template.shader_test.mako')
@@ -267,8 +268,7 @@ class Test(object):
     def generate(self):
         filename = self.filename()
         dirname = os.path.dirname(filename)
-        if not os.path.exists(dirname):
-            os.makedirs(dirname)
+        utils.safe_makedirs(dirname)
         with open(filename, 'w') as f:
             f.write(TEMPLATE.render(args=self))
 
diff --git a/generated_tests/gen_non-lvalue_tests.py b/generated_tests/gen_non-lvalue_tests.py
index 95381f8..8b90353 100644
--- a/generated_tests/gen_non-lvalue_tests.py
+++ b/generated_tests/gen_non-lvalue_tests.py
@@ -26,6 +26,7 @@ import os
 import itertools
 
 from templates import template_dir
+from modules import utils
 
 TEMPLATES = template_dir(os.path.basename(os.path.splitext(__file__)[0]))
 
@@ -89,8 +90,7 @@ def all_tests():
 
 def main():
     dirname = os.path.join('spec', 'glsl-1.10', 'compiler', 'expressions')
-    if not os.path.exists(dirname):
-        os.makedirs(dirname)
+    utils.safe_makedirs(dirname)
 
     for args in all_tests():
         generate(dirname, *args)
diff --git a/generated_tests/gen_outerproduct_invalid_params.py b/generated_tests/gen_outerproduct_invalid_params.py
index c872e9f..bdf03d6 100644
--- a/generated_tests/gen_outerproduct_invalid_params.py
+++ b/generated_tests/gen_outerproduct_invalid_params.py
@@ -22,6 +22,7 @@ from __future__ import print_function
 import os
 
 from templates import template_file
+from modules import utils
 
 TEMPLATE = template_file(os.path.basename(os.path.splitext(__file__)[0]),
                          'template.vert.mako')
@@ -31,8 +32,7 @@ def main():
     """ Generate tests """
     dirname = os.path.join('spec', 'glsl-1.20', 'compiler',
                            'built-in-functions')
-    if not os.path.exists(dirname):
-        os.makedirs(dirname)
+    utils.safe_makedirs(dirname)
 
     for type_ in ['int', 'float', 'bool', 'bvec2', 'bvec3', 'bvec4', 'mat2',
                   'mat2x2', 'mat2x3', 'mat2x4', 'mat3', 'mat3x2', 'mat3x3',
diff --git a/generated_tests/gen_outerproduct_tests.py b/generated_tests/gen_outerproduct_tests.py
index 80a6ad8..42c527b 100644
--- a/generated_tests/gen_outerproduct_tests.py
+++ b/generated_tests/gen_outerproduct_tests.py
@@ -26,6 +26,7 @@ import itertools
 import collections
 
 from templates import template_file
+from modules import utils
 
 TEMPLATE = template_file(os.path.splitext(os.path.basename(__file__))[0],
                          'template.shader_test.mako')
@@ -37,8 +38,7 @@ Parameters = collections.namedtuple(
 def main():
     """ Generate tests """
     dirname = os.path.join('spec', 'glsl-1.20', 'execution')
-    if not os.path.exists(dirname):
-        os.makedirs(dirname)
+    utils.safe_makedirs(dirname)
 
 
     for c, r in itertools.product(xrange(2, 5), repeat=2):
diff --git a/generated_tests/gen_shader_bit_encoding_tests.py b/generated_tests/gen_shader_bit_encoding_tests.py
index c879255..4656755 100644
--- a/generated_tests/gen_shader_bit_encoding_tests.py
+++ b/generated_tests/gen_shader_bit_encoding_tests.py
@@ -27,6 +27,7 @@ import os
 from operator import neg
 
 from templates import template_file
+from modules import utils
 
 TEMPLATE = template_file(os.path.basename(os.path.splitext(__file__)[0]),
                          'template.shader_test.mako')
@@ -139,8 +140,7 @@ def main():
 
         dirname = os.path.join('spec', api.lower(), 'execution',
                                'built-in-functions')
-        if not os.path.exists(dirname):
-            os.makedirs(dirname)
+        utils.safe_makedirs(dirname)
 
         for func, attrib in FUNCS.iteritems():
             for execution_stage in ('vs', 'fs'):
diff --git a/generated_tests/gen_shader_image_load_store_tests.py b/generated_tests/gen_shader_image_load_store_tests.py
index b292809..1ffa078 100644
--- a/generated_tests/gen_shader_image_load_store_tests.py
+++ b/generated_tests/gen_shader_image_load_store_tests.py
@@ -26,6 +26,8 @@ import os.path
 from mako.template import Template
 from textwrap import dedent
 
+from modules import utils
+
 
 def gen_header(status):
     """
@@ -143,8 +145,7 @@ def gen(name, src, tests):
         print(filename)
 
         dirname = os.path.dirname(filename)
-        if not os.path.exists(dirname):
-            os.makedirs(dirname)
+        utils.safe_makedirs(dirname)
 
         with open(filename, 'w') as f:
             f.write(template.render(header = gen_header, **t))
diff --git a/generated_tests/gen_texture_lod_tests.py b/generated_tests/gen_texture_lod_tests.py
index a2f43fa..53b2568 100644
--- a/generated_tests/gen_texture_lod_tests.py
+++ b/generated_tests/gen_texture_lod_tests.py
@@ -25,6 +25,7 @@ import os
 import collections
 
 from templates import template_dir
+from modules import utils
 
 TEMPLATES = template_dir(os.path.basename(os.path.splitext(__file__)[0]))
 
@@ -75,8 +76,7 @@ def main():
 
     """
     dirname = 'spec/arb_shader_texture_lod/compiler'
-    if not os.path.exists(dirname):
-        os.makedirs(dirname)
+    utils.safe_makedirs(dirname)
 
     for params in LOD_TESTS:
         name = os.path.join(
diff --git a/generated_tests/gen_texture_query_lod_tests.py b/generated_tests/gen_texture_query_lod_tests.py
index 1605204..c786a10 100644
--- a/generated_tests/gen_texture_query_lod_tests.py
+++ b/generated_tests/gen_texture_query_lod_tests.py
@@ -26,6 +26,7 @@ import os
 import os.path
 
 from templates import template_file
+from modules import utils
 
 TEMPLATE = template_file(os.path.basename(os.path.splitext(__file__)[0]),
                          'template.glsl_parser_test.mako')
@@ -85,8 +86,7 @@ def main():
         lod = 'Lod' if api == 'glsl-4.00' else 'LOD'
         dirname = os.path.join("spec", api.lower(), "compiler",
                                "built-in-functions")
-        if not os.path.exists(dirname):
-            os.makedirs(dirname)
+        utils.safe_makedirs(dirname)
 
         for sampler_type, coord_type in SAMPLER_TYPE_TO_COORD_TYPE.iteritems():
             requirements = [requirement['extensions']] if requirement['extensions'] else []
diff --git a/generated_tests/gen_uniform_initializer_tests.py b/generated_tests/gen_uniform_initializer_tests.py
index 0a1d908..1605876 100644
--- a/generated_tests/gen_uniform_initializer_tests.py
+++ b/generated_tests/gen_uniform_initializer_tests.py
@@ -25,6 +25,7 @@ from __future__ import print_function
 import os
 
 from templates import template_dir
+from modules import utils
 
 TEMPLATES = template_dir(os.path.splitext(os.path.basename(__file__))[0])
 
@@ -77,8 +78,7 @@ def generate_tests(type_list, base_name, major, minor):
                            'glsl-{0}.{1}'.format(major, minor),
                            'execution',
                            'uniform-initializer')
-    if not os.path.exists(dirname):
-        os.makedirs(dirname)
+    utils.safe_makedirs(dirname)
 
     for target in ("vs", "fs"):
         for t in ALL_TEMPLATES:
@@ -133,8 +133,7 @@ def generate_array_tests(type_list, base_name, major, minor):
                            'glsl-{0}.{1}'.format(major, minor),
                            'execution',
                            'uniform-initializer')
-    if not os.path.exists(dirname):
-        os.makedirs(dirname)
+    utils.safe_makedirs(dirname)
 
     def parts():
         """Generate parts."""
diff --git a/generated_tests/genclbuiltins.py b/generated_tests/genclbuiltins.py
index 571930d..681aa29 100644
--- a/generated_tests/genclbuiltins.py
+++ b/generated_tests/genclbuiltins.py
@@ -3,6 +3,8 @@ __all__ = ['gen', 'DATA_SIZES', 'MAX_VALUES', 'MAX', 'MIN', 'BMIN', 'BMAX',
 
 import os
 
+from modules import utils
+
 
 DATA_SIZES = {
     'char': 8,
@@ -354,8 +356,7 @@ def print_test(f, fnName, argType, functionDef, tests, numTests, vecSize, tss):
 
 def gen(types, minVersions, functions, testDefs, dirName):
     # Create the output directory if required
-    if not os.path.exists(dirName):
-        os.makedirs(dirName)
+    utils.safe_makedirs(dirName)
 
     # Loop over all data types being tested. Create one output file per data
     # type
diff --git a/generated_tests/generate-cl-store-tests.py b/generated_tests/generate-cl-store-tests.py
index 0aa7fe6..586c4f0 100644
--- a/generated_tests/generate-cl-store-tests.py
+++ b/generated_tests/generate-cl-store-tests.py
@@ -26,12 +26,13 @@
 import os
 import textwrap
 
+from modules import utils
+
 TYPES = ['char', 'uchar', 'short', 'ushort', 'int', 'uint', 'long', 'ulong', 'float', 'double']
 VEC_SIZES = ['', '2', '4', '8', '16']
 
 dirName = os.path.join("cl", "store")
-if not os.path.exists(dirName):
-    os.makedirs(dirName)
+utils.safe_makedirs(dirname)
 
 
 def gen_array(size):
diff --git a/generated_tests/interpolation-qualifier-built-in-variable.py b/generated_tests/interpolation-qualifier-built-in-variable.py
index 96def23..45fc42c 100644
--- a/generated_tests/interpolation-qualifier-built-in-variable.py
+++ b/generated_tests/interpolation-qualifier-built-in-variable.py
@@ -28,6 +28,7 @@ import os
 import itertools
 
 from templates import template_dir
+from modules import utils
 
 TEMPLATES = template_dir(os.path.basename(os.path.splitext(__file__)[0]))
 
@@ -171,9 +172,7 @@ def main():
     """main function."""
     dirname = os.path.join('spec', 'glsl-1.30', 'linker',
                            'interpolation-qualifiers')
-
-    if not os.path.exists(dirname):
-        os.makedirs(dirname)
+    utils.safe_makedirs(dirname)
 
     for fs_mode, vs_mode in itertools.product(INTERPOLATION_MODES, repeat=2):
         make_fs_vs_tests(fs_mode, vs_mode, dirname)
diff --git a/generated_tests/modules/__init__.py b/generated_tests/modules/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/generated_tests/gen_outerproduct_invalid_params.py b/generated_tests/modules/utils.py
similarity index 56%
copy from generated_tests/gen_outerproduct_invalid_params.py
copy to generated_tests/modules/utils.py
index c872e9f..d8fd914 100644
--- a/generated_tests/gen_outerproduct_invalid_params.py
+++ b/generated_tests/modules/utils.py
@@ -18,30 +18,26 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
-from __future__ import print_function
-import os
-
-from templates import template_file
-
-TEMPLATE = template_file(os.path.basename(os.path.splitext(__file__)[0]),
-                         'template.vert.mako')
-
+"""Helper functions for test generators."""
 
-def main():
-    """ Generate tests """
-    dirname = os.path.join('spec', 'glsl-1.20', 'compiler',
-                           'built-in-functions')
-    if not os.path.exists(dirname):
-        os.makedirs(dirname)
-
-    for type_ in ['int', 'float', 'bool', 'bvec2', 'bvec3', 'bvec4', 'mat2',
-                  'mat2x2', 'mat2x3', 'mat2x4', 'mat3', 'mat3x2', 'mat3x3',
-                  'mat3x4', 'mat4', 'mat4x2', 'mat4x3', 'mat4x4']:
-        name = os.path.join(dirname, 'outerProduct-{0}.vert'.format(type_))
-        print(name)
-        with open(name, 'w+') as f:
-            f.write(TEMPLATE.render_unicode(type=type_))
-
-
-if __name__ == '__main__':
-    main()
+import os
+import errno
+
+
+def safe_makedirs(dirs):
+    """A safe function for creating a directory tree.
+
+    This function wraps os.makedirs, and provides a couple of sanity checks,
+    first, if the directory already exists it doesn't try to create it, and
+    second if the directory comes into existence between the check and creation
+    time (like if another generator creates it) then the exception will be
+    caught.
+
+    """
+    if not os.path.exists(dirs):
+        try:
+            os.makedirs(dirs)
+        except OSError as e:
+            if e.errno == errno.EEXIST:
+                pass
+            raise
-- 
2.2.0



More information about the Piglit mailing list