[Piglit] [Patch v2] generated_tests: Catch exception in os.makedirs
Dylan Baker
baker.dylan.c at gmail.com
Fri Dec 12 11:09:25 PST 2014
Currently we check for the existence of a directory, and if it doesn't
exist we create it. This would be fine, except that multiple generators
can create the same directory trees and when os.makedirs tries to create
a tree that already exists it raises and OSError exception, with the
errno set to 17 (or errno.EEXIST). Since between the time that a check
is made and the time the os.makedirs completes a second generator can
create the same directory tree we can have exceptions raised
sporadically, especially as the number of threads being used to generate
goes up.
We maintain the check for existence before trying to create the
directories because os.makedirs is slow, so not even trying is better
than trying in vain.
v2: - Use a helper function to reduce code duplication
- use errno.EEXIST instead of magic number 17 (Mark)
Signed-off-by: Dylan Baker <dylanx.c.baker at intel.com>
---
generated_tests/gen_builtin_packing_tests.py | 4 +-
generated_tests/gen_builtin_uniform_tests.py | 5 ++-
generated_tests/gen_builtin_uniform_tests_fp64.py | 5 ++-
generated_tests/gen_const_builtin_equal_tests.py | 4 +-
generated_tests/gen_constant_array_size_tests.py | 4 +-
.../gen_constant_array_size_tests_fp64.py | 4 +-
generated_tests/gen_interpolation_tests.py | 4 +-
generated_tests/gen_non-lvalue_tests.py | 4 +-
generated_tests/gen_outerproduct_invalid_params.py | 4 +-
generated_tests/gen_outerproduct_tests.py | 4 +-
generated_tests/gen_shader_bit_encoding_tests.py | 4 +-
.../gen_shader_image_load_store_tests.py | 5 ++-
generated_tests/gen_texture_lod_tests.py | 4 +-
generated_tests/gen_texture_query_lod_tests.py | 4 +-
generated_tests/gen_uniform_initializer_tests.py | 7 ++--
generated_tests/genclbuiltins.py | 5 ++-
generated_tests/generate-cl-store-tests.py | 5 ++-
.../interpolation-qualifier-built-in-variable.py | 5 +--
generated_tests/modules/__init__.py | 0
.../utils.py} | 48 ++++++++++------------
20 files changed, 64 insertions(+), 65 deletions(-)
create mode 100644 generated_tests/modules/__init__.py
copy generated_tests/{gen_outerproduct_invalid_params.py => modules/utils.py} (56%)
diff --git a/generated_tests/gen_builtin_packing_tests.py b/generated_tests/gen_builtin_packing_tests.py
index 52bf1a0..901f493 100644
--- a/generated_tests/gen_builtin_packing_tests.py
+++ b/generated_tests/gen_builtin_packing_tests.py
@@ -43,6 +43,7 @@ from math import copysign, fabs, fmod, frexp, isinf, isnan, modf
from numpy import int8, int16, uint8, uint16, uint32, float32
from templates import template_dir
+from modules import utils
TEMPLATES = template_dir(os.path.basename(os.path.splitext(__file__)[0]))
@@ -1027,8 +1028,7 @@ class ShaderTest(object):
def write_file(self):
dirname = os.path.dirname(self.filename)
- if not os.path.exists(dirname):
- os.makedirs(dirname)
+ utils.safe_makedirs(dirname)
with open(self.filename, "w") as f:
f.write(self.__template.render(func=self.__func_info))
diff --git a/generated_tests/gen_builtin_uniform_tests.py b/generated_tests/gen_builtin_uniform_tests.py
index d89a6f3..9bb3075 100644
--- a/generated_tests/gen_builtin_uniform_tests.py
+++ b/generated_tests/gen_builtin_uniform_tests.py
@@ -52,6 +52,8 @@ import os
import os.path
import sys
+from modules import utils
+
def compute_offset_and_scale(test_vectors):
"""Compute scale and offset values such that for each result in
@@ -537,8 +539,7 @@ class ShaderTest(object):
shader_test += self.make_test()
filename = self.filename()
dirname = os.path.dirname(filename)
- if not os.path.exists(dirname):
- os.makedirs(dirname)
+ utils.safe_makedirs(dirname)
with open(filename, 'w') as f:
f.write(shader_test)
diff --git a/generated_tests/gen_builtin_uniform_tests_fp64.py b/generated_tests/gen_builtin_uniform_tests_fp64.py
index ec12650..23032bd 100644
--- a/generated_tests/gen_builtin_uniform_tests_fp64.py
+++ b/generated_tests/gen_builtin_uniform_tests_fp64.py
@@ -52,6 +52,8 @@ import os
import os.path
import sys
+from modules import utils
+
def compute_offset_and_scale(test_vectors):
"""Compute scale and offset values such that for each result in
@@ -507,8 +509,7 @@ class ShaderTest(object):
shader_test += self.make_test()
filename = self.filename()
dirname = os.path.dirname(filename)
- if not os.path.exists(dirname):
- os.makedirs(dirname)
+ utils.safe_makedirs(dirname)
with open(filename, 'w') as f:
f.write(shader_test)
diff --git a/generated_tests/gen_const_builtin_equal_tests.py b/generated_tests/gen_const_builtin_equal_tests.py
index eb2856c..47a238c 100644
--- a/generated_tests/gen_const_builtin_equal_tests.py
+++ b/generated_tests/gen_const_builtin_equal_tests.py
@@ -25,6 +25,7 @@ import re
import os
from templates import template_file
+from modules import utils
TEMPLATE = template_file(os.path.basename(os.path.splitext(__file__)[0]),
'template.shader_test.mako')
@@ -73,8 +74,7 @@ def main():
""" Main function """
dirname = os.path.join('spec', 'glsl-1.20', 'execution',
'built-in-functions')
- if not os.path.exists(dirname):
- os.makedirs(dirname)
+ utils.safe_makedirs(dirname)
for test_id, x in enumerate(TEST_VECTORS, start=2):
# make equal tests
diff --git a/generated_tests/gen_constant_array_size_tests.py b/generated_tests/gen_constant_array_size_tests.py
index c395b1a..c6f75ab 100644
--- a/generated_tests/gen_constant_array_size_tests.py
+++ b/generated_tests/gen_constant_array_size_tests.py
@@ -40,6 +40,7 @@ import optparse
import os
import os.path
+from modules import utils
class ParserTest(object):
"""Class used to build a test of a single built-in. This is an
@@ -198,8 +199,7 @@ class ParserTest(object):
parser_test += self.make_shader()
filename = self.filename()
dirname = os.path.dirname(filename)
- if not os.path.exists(dirname):
- os.makedirs(dirname)
+ utils.safe_makedirs(dirname)
with open(filename, 'w') as f:
f.write(parser_test)
diff --git a/generated_tests/gen_constant_array_size_tests_fp64.py b/generated_tests/gen_constant_array_size_tests_fp64.py
index a54598b..8d4a785 100644
--- a/generated_tests/gen_constant_array_size_tests_fp64.py
+++ b/generated_tests/gen_constant_array_size_tests_fp64.py
@@ -40,6 +40,7 @@ import optparse
import os
import os.path
+from modules import utils
class ParserTest(object):
"""Class used to build a test of a single built-in. This is an
@@ -199,8 +200,7 @@ class ParserTest(object):
parser_test += self.make_shader()
filename = self.filename()
dirname = os.path.dirname(filename)
- if not os.path.exists(dirname):
- os.makedirs(dirname)
+ utils.safe_makedirs(dirname)
with open(filename, 'w') as f:
f.write(parser_test)
diff --git a/generated_tests/gen_interpolation_tests.py b/generated_tests/gen_interpolation_tests.py
index 4f1428b..1700a20 100644
--- a/generated_tests/gen_interpolation_tests.py
+++ b/generated_tests/gen_interpolation_tests.py
@@ -71,6 +71,7 @@ from __future__ import print_function
import os
from templates import template_file
+from modules import utils
TEMPLATE = template_file(os.path.basename(os.path.splitext(__file__)[0]),
'template.shader_test.mako')
@@ -267,8 +268,7 @@ class Test(object):
def generate(self):
filename = self.filename()
dirname = os.path.dirname(filename)
- if not os.path.exists(dirname):
- os.makedirs(dirname)
+ utils.safe_makedirs(dirname)
with open(filename, 'w') as f:
f.write(TEMPLATE.render(args=self))
diff --git a/generated_tests/gen_non-lvalue_tests.py b/generated_tests/gen_non-lvalue_tests.py
index 95381f8..8b90353 100644
--- a/generated_tests/gen_non-lvalue_tests.py
+++ b/generated_tests/gen_non-lvalue_tests.py
@@ -26,6 +26,7 @@ import os
import itertools
from templates import template_dir
+from modules import utils
TEMPLATES = template_dir(os.path.basename(os.path.splitext(__file__)[0]))
@@ -89,8 +90,7 @@ def all_tests():
def main():
dirname = os.path.join('spec', 'glsl-1.10', 'compiler', 'expressions')
- if not os.path.exists(dirname):
- os.makedirs(dirname)
+ utils.safe_makedirs(dirname)
for args in all_tests():
generate(dirname, *args)
diff --git a/generated_tests/gen_outerproduct_invalid_params.py b/generated_tests/gen_outerproduct_invalid_params.py
index c872e9f..bdf03d6 100644
--- a/generated_tests/gen_outerproduct_invalid_params.py
+++ b/generated_tests/gen_outerproduct_invalid_params.py
@@ -22,6 +22,7 @@ from __future__ import print_function
import os
from templates import template_file
+from modules import utils
TEMPLATE = template_file(os.path.basename(os.path.splitext(__file__)[0]),
'template.vert.mako')
@@ -31,8 +32,7 @@ def main():
""" Generate tests """
dirname = os.path.join('spec', 'glsl-1.20', 'compiler',
'built-in-functions')
- if not os.path.exists(dirname):
- os.makedirs(dirname)
+ utils.safe_makedirs(dirname)
for type_ in ['int', 'float', 'bool', 'bvec2', 'bvec3', 'bvec4', 'mat2',
'mat2x2', 'mat2x3', 'mat2x4', 'mat3', 'mat3x2', 'mat3x3',
diff --git a/generated_tests/gen_outerproduct_tests.py b/generated_tests/gen_outerproduct_tests.py
index 80a6ad8..42c527b 100644
--- a/generated_tests/gen_outerproduct_tests.py
+++ b/generated_tests/gen_outerproduct_tests.py
@@ -26,6 +26,7 @@ import itertools
import collections
from templates import template_file
+from modules import utils
TEMPLATE = template_file(os.path.splitext(os.path.basename(__file__))[0],
'template.shader_test.mako')
@@ -37,8 +38,7 @@ Parameters = collections.namedtuple(
def main():
""" Generate tests """
dirname = os.path.join('spec', 'glsl-1.20', 'execution')
- if not os.path.exists(dirname):
- os.makedirs(dirname)
+ utils.safe_makedirs(dirname)
for c, r in itertools.product(xrange(2, 5), repeat=2):
diff --git a/generated_tests/gen_shader_bit_encoding_tests.py b/generated_tests/gen_shader_bit_encoding_tests.py
index c879255..4656755 100644
--- a/generated_tests/gen_shader_bit_encoding_tests.py
+++ b/generated_tests/gen_shader_bit_encoding_tests.py
@@ -27,6 +27,7 @@ import os
from operator import neg
from templates import template_file
+from modules import utils
TEMPLATE = template_file(os.path.basename(os.path.splitext(__file__)[0]),
'template.shader_test.mako')
@@ -139,8 +140,7 @@ def main():
dirname = os.path.join('spec', api.lower(), 'execution',
'built-in-functions')
- if not os.path.exists(dirname):
- os.makedirs(dirname)
+ utils.safe_makedirs(dirname)
for func, attrib in FUNCS.iteritems():
for execution_stage in ('vs', 'fs'):
diff --git a/generated_tests/gen_shader_image_load_store_tests.py b/generated_tests/gen_shader_image_load_store_tests.py
index b292809..1ffa078 100644
--- a/generated_tests/gen_shader_image_load_store_tests.py
+++ b/generated_tests/gen_shader_image_load_store_tests.py
@@ -26,6 +26,8 @@ import os.path
from mako.template import Template
from textwrap import dedent
+from modules import utils
+
def gen_header(status):
"""
@@ -143,8 +145,7 @@ def gen(name, src, tests):
print(filename)
dirname = os.path.dirname(filename)
- if not os.path.exists(dirname):
- os.makedirs(dirname)
+ utils.safe_makedirs(dirname)
with open(filename, 'w') as f:
f.write(template.render(header = gen_header, **t))
diff --git a/generated_tests/gen_texture_lod_tests.py b/generated_tests/gen_texture_lod_tests.py
index a2f43fa..53b2568 100644
--- a/generated_tests/gen_texture_lod_tests.py
+++ b/generated_tests/gen_texture_lod_tests.py
@@ -25,6 +25,7 @@ import os
import collections
from templates import template_dir
+from modules import utils
TEMPLATES = template_dir(os.path.basename(os.path.splitext(__file__)[0]))
@@ -75,8 +76,7 @@ def main():
"""
dirname = 'spec/arb_shader_texture_lod/compiler'
- if not os.path.exists(dirname):
- os.makedirs(dirname)
+ utils.safe_makedirs(dirname)
for params in LOD_TESTS:
name = os.path.join(
diff --git a/generated_tests/gen_texture_query_lod_tests.py b/generated_tests/gen_texture_query_lod_tests.py
index 1605204..c786a10 100644
--- a/generated_tests/gen_texture_query_lod_tests.py
+++ b/generated_tests/gen_texture_query_lod_tests.py
@@ -26,6 +26,7 @@ import os
import os.path
from templates import template_file
+from modules import utils
TEMPLATE = template_file(os.path.basename(os.path.splitext(__file__)[0]),
'template.glsl_parser_test.mako')
@@ -85,8 +86,7 @@ def main():
lod = 'Lod' if api == 'glsl-4.00' else 'LOD'
dirname = os.path.join("spec", api.lower(), "compiler",
"built-in-functions")
- if not os.path.exists(dirname):
- os.makedirs(dirname)
+ utils.safe_makedirs(dirname)
for sampler_type, coord_type in SAMPLER_TYPE_TO_COORD_TYPE.iteritems():
requirements = [requirement['extensions']] if requirement['extensions'] else []
diff --git a/generated_tests/gen_uniform_initializer_tests.py b/generated_tests/gen_uniform_initializer_tests.py
index 0a1d908..1605876 100644
--- a/generated_tests/gen_uniform_initializer_tests.py
+++ b/generated_tests/gen_uniform_initializer_tests.py
@@ -25,6 +25,7 @@ from __future__ import print_function
import os
from templates import template_dir
+from modules import utils
TEMPLATES = template_dir(os.path.splitext(os.path.basename(__file__))[0])
@@ -77,8 +78,7 @@ def generate_tests(type_list, base_name, major, minor):
'glsl-{0}.{1}'.format(major, minor),
'execution',
'uniform-initializer')
- if not os.path.exists(dirname):
- os.makedirs(dirname)
+ utils.safe_makedirs(dirname)
for target in ("vs", "fs"):
for t in ALL_TEMPLATES:
@@ -133,8 +133,7 @@ def generate_array_tests(type_list, base_name, major, minor):
'glsl-{0}.{1}'.format(major, minor),
'execution',
'uniform-initializer')
- if not os.path.exists(dirname):
- os.makedirs(dirname)
+ utils.safe_makedirs(dirname)
def parts():
"""Generate parts."""
diff --git a/generated_tests/genclbuiltins.py b/generated_tests/genclbuiltins.py
index 571930d..681aa29 100644
--- a/generated_tests/genclbuiltins.py
+++ b/generated_tests/genclbuiltins.py
@@ -3,6 +3,8 @@ __all__ = ['gen', 'DATA_SIZES', 'MAX_VALUES', 'MAX', 'MIN', 'BMIN', 'BMAX',
import os
+from modules import utils
+
DATA_SIZES = {
'char': 8,
@@ -354,8 +356,7 @@ def print_test(f, fnName, argType, functionDef, tests, numTests, vecSize, tss):
def gen(types, minVersions, functions, testDefs, dirName):
# Create the output directory if required
- if not os.path.exists(dirName):
- os.makedirs(dirName)
+ utils.safe_makedirs(dirName)
# Loop over all data types being tested. Create one output file per data
# type
diff --git a/generated_tests/generate-cl-store-tests.py b/generated_tests/generate-cl-store-tests.py
index 0aa7fe6..586c4f0 100644
--- a/generated_tests/generate-cl-store-tests.py
+++ b/generated_tests/generate-cl-store-tests.py
@@ -26,12 +26,13 @@
import os
import textwrap
+from modules import utils
+
TYPES = ['char', 'uchar', 'short', 'ushort', 'int', 'uint', 'long', 'ulong', 'float', 'double']
VEC_SIZES = ['', '2', '4', '8', '16']
dirName = os.path.join("cl", "store")
-if not os.path.exists(dirName):
- os.makedirs(dirName)
+utils.safe_makedirs(dirname)
def gen_array(size):
diff --git a/generated_tests/interpolation-qualifier-built-in-variable.py b/generated_tests/interpolation-qualifier-built-in-variable.py
index 96def23..45fc42c 100644
--- a/generated_tests/interpolation-qualifier-built-in-variable.py
+++ b/generated_tests/interpolation-qualifier-built-in-variable.py
@@ -28,6 +28,7 @@ import os
import itertools
from templates import template_dir
+from modules import utils
TEMPLATES = template_dir(os.path.basename(os.path.splitext(__file__)[0]))
@@ -171,9 +172,7 @@ def main():
"""main function."""
dirname = os.path.join('spec', 'glsl-1.30', 'linker',
'interpolation-qualifiers')
-
- if not os.path.exists(dirname):
- os.makedirs(dirname)
+ utils.safe_makedirs(dirname)
for fs_mode, vs_mode in itertools.product(INTERPOLATION_MODES, repeat=2):
make_fs_vs_tests(fs_mode, vs_mode, dirname)
diff --git a/generated_tests/modules/__init__.py b/generated_tests/modules/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/generated_tests/gen_outerproduct_invalid_params.py b/generated_tests/modules/utils.py
similarity index 56%
copy from generated_tests/gen_outerproduct_invalid_params.py
copy to generated_tests/modules/utils.py
index c872e9f..d8fd914 100644
--- a/generated_tests/gen_outerproduct_invalid_params.py
+++ b/generated_tests/modules/utils.py
@@ -18,30 +18,26 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
-from __future__ import print_function
-import os
-
-from templates import template_file
-
-TEMPLATE = template_file(os.path.basename(os.path.splitext(__file__)[0]),
- 'template.vert.mako')
-
+"""Helper functions for test generators."""
-def main():
- """ Generate tests """
- dirname = os.path.join('spec', 'glsl-1.20', 'compiler',
- 'built-in-functions')
- if not os.path.exists(dirname):
- os.makedirs(dirname)
-
- for type_ in ['int', 'float', 'bool', 'bvec2', 'bvec3', 'bvec4', 'mat2',
- 'mat2x2', 'mat2x3', 'mat2x4', 'mat3', 'mat3x2', 'mat3x3',
- 'mat3x4', 'mat4', 'mat4x2', 'mat4x3', 'mat4x4']:
- name = os.path.join(dirname, 'outerProduct-{0}.vert'.format(type_))
- print(name)
- with open(name, 'w+') as f:
- f.write(TEMPLATE.render_unicode(type=type_))
-
-
-if __name__ == '__main__':
- main()
+import os
+import errno
+
+
+def safe_makedirs(dirs):
+ """A safe function for creating a directory tree.
+
+ This function wraps os.makedirs, and provides a couple of sanity checks,
+ first, if the directory already exists it doesn't try to create it, and
+ second if the directory comes into existence between the check and creation
+ time (like if another generator creates it) then the exception will be
+ caught.
+
+ """
+ if not os.path.exists(dirs):
+ try:
+ os.makedirs(dirs)
+ except OSError as e:
+ if e.errno == errno.EEXIST:
+ pass
+ raise
--
2.2.0
More information about the Piglit
mailing list