[Piglit] [PATCH] framework/test/deqp: handle deqp extra args properly in more cases

[Mark Janes]

Reviewed-by: Mark Janes <mark.a.janes at intel.com>

Dylan Baker <baker.dylan.c at gmail.com> writes:

> Currently we blindly pass deqp extra args to each test case, and only to
> the test case.
>
> There are two cases where this is problematic:
> 1) if the extra args need to be passed to the deqp command that
>    generates the test list (it isn't before this patch)
> 2) if there are any --deqp-case options passed into the Test derived
>    classes this will cause the tests to fail (conflicting options)
>
> Both of these are resolved by this patch.
>
> CC: Mark Janes <mark.a.janes at intel.com>
> Signed-off-by: Dylan Baker <dylanx.c.baker at intel.com>
> ---
>  framework/test/deqp.py | 11 ++++++++---
>  tests/deqp_gles2.py    | 12 +++++++-----
>  tests/deqp_gles3.py    | 11 +++++++----
>  tests/deqp_gles31.py   | 12 +++++++-----
>  4 files changed, 29 insertions(+), 17 deletions(-)
>
> diff --git a/framework/test/deqp.py b/framework/test/deqp.py
> index 843cde8..1462ca2 100644
> --- a/framework/test/deqp.py
> +++ b/framework/test/deqp.py
> @@ -61,8 +61,12 @@ def get_option(env_varname, config_option, default=None):
>      return opt or default
>  
>  
> -def gen_caselist_txt(bin_, caselist):
> -    """Generate a caselist.txt and return its path."""
> +def gen_caselist_txt(bin_, caselist, extra_args):
> +    """Generate a caselist.txt and return its path.
> +
> +    Extra args should be a list of extra arguments to pass to deqp.
> +
> +    """
>      # dEQP is stupid (2014-12-07):
>      #   1. To generate the caselist file, dEQP requires that the process's
>      #      current directory must be that same as that of the executable.
> @@ -76,9 +80,10 @@ def gen_caselist_txt(bin_, caselist):
>      #      we must *dynamically* generate it during the testrun.
>      basedir = os.path.dirname(bin_)
>      caselist_path = os.path.join(basedir, caselist)
> +
>      # TODO: need to catch some exceptions here...
>      subprocess.check_call(
> -        [bin_, '--deqp-runmode=txt-caselist'], cwd=basedir)
> +        [bin_, '--deqp-runmode=txt-caselist'] + extra_args, cwd=basedir)
>      assert os.path.exists(caselist_path)
>      return caselist_path
>  
> diff --git a/tests/deqp_gles2.py b/tests/deqp_gles2.py
> index b97cb5d..c4bacb2 100644
> --- a/tests/deqp_gles2.py
> +++ b/tests/deqp_gles2.py
> @@ -24,20 +24,22 @@ from framework.test import deqp
>  
>  __all__ = ['profile']
>  
> -
>  # Path to the deqp-gles2 executable.
>  _DEQP_GLES2_BIN = deqp.get_option('PIGLIT_DEQP_GLES2_BIN',
>                                    ('deqp-gles2', 'bin'))
>  
> +_EXTRA_ARGS = deqp.get_option('PIGLIT_DEQP_GLES2_EXTRA_ARGS',
> +                              ('deqp-gles2', 'extra_args'),
> +                              default='').split()
> +
>  
>  class DEQPGLES2Test(deqp.DEQPBaseTest):
>      deqp_bin = _DEQP_GLES2_BIN
> -    extra_args = deqp.get_option('PIGLIT_DEQP_GLES2_EXTRA_ARGS',
> -                                 ('deqp-gles2', 'extra_args'),
> -                                 default='').split()
> +    extra_args = [x for x in _EXTRA_ARGS if not x.startswith('--deqp-case')]
>  
>  
>  profile = deqp.make_profile(  # pylint: disable=invalid-name
>      deqp.iter_deqp_test_cases(
> -        deqp.gen_caselist_txt(_DEQP_GLES2_BIN, 'dEQP-GLES2-cases.txt')),
> +        deqp.gen_caselist_txt(_DEQP_GLES2_BIN, 'dEQP-GLES2-cases.txt',
> +                              _EXTRA_ARGS)),
>      DEQPGLES2Test)
> diff --git a/tests/deqp_gles3.py b/tests/deqp_gles3.py
> index dfb82c9..d353899 100644
> --- a/tests/deqp_gles3.py
> +++ b/tests/deqp_gles3.py
> @@ -35,6 +35,10 @@ _DEQP_GLES3_EXE = deqp.get_option('PIGLIT_DEQP_GLES3_EXE',
>  _DEQP_MUSTPASS = deqp.get_option('PIGLIT_DEQP_MUSTPASS',
>                                   ('deqp-gles3', 'mustpasslist'))
>  
> +_EXTRA_ARGS = deqp.get_option('PIGLIT_DEQP_GLES3_EXTRA_ARGS',
> +                              ('deqp-gles3', 'extra_args'),
> +                              default='').split()
> +
>  
>  def _get_test_case(root, root_group, outputfile):
>      """Parser the test case list of Google Android CTS,
> @@ -73,9 +77,7 @@ def filter_mustpass(caselist_path):
>  
>  class DEQPGLES3Test(deqp.DEQPBaseTest):
>      deqp_bin = _DEQP_GLES3_EXE
> -    extra_args = deqp.get_option('PIGLIT_DEQP_GLES3_EXTRA_ARGS',
> -                                 ('deqp-gles3', 'extra_args'),
> -                                 default='').split()
> +    extra_args = [x for x in _EXTRA_ARGS if not x.startswith('--deqp-case')]
>  
>  
>      def __init__(self, *args, **kwargs):
> @@ -84,5 +86,6 @@ class DEQPGLES3Test(deqp.DEQPBaseTest):
>  
>  profile = deqp.make_profile(  # pylint: disable=invalid-name
>      deqp.iter_deqp_test_cases(filter_mustpass(
> -        deqp.gen_caselist_txt(_DEQP_GLES3_EXE, 'dEQP-GLES3-cases.txt'))),
> +        deqp.gen_caselist_txt(_DEQP_GLES3_EXE, 'dEQP-GLES3-cases.txt',
> +                              _EXTRA_ARGS))),
>      DEQPGLES3Test)
> diff --git a/tests/deqp_gles31.py b/tests/deqp_gles31.py
> index 423c1bf..b98d2be 100644
> --- a/tests/deqp_gles31.py
> +++ b/tests/deqp_gles31.py
> @@ -24,20 +24,22 @@ from framework.test import deqp
>  
>  __all__ = ['profile']
>  
> -
>  # Path to the deqp-gles3 executable.
>  _DEQP_GLES31_BIN = deqp.get_option('PIGLIT_DEQP_GLES31_BIN',
>                                     ('deqp-gles31', 'bin'))
>  
> +_EXTRA_ARGS = deqp.get_option('PIGLIT_DEQP_GLES31_EXTRA_ARGS',
> +                              ('deqp-gles31', 'extra_args'),
> +                              default='').split()
> +
>  
>  class DEQPGLES31Test(deqp.DEQPBaseTest):
>      deqp_bin = _DEQP_GLES31_BIN
> -    extra_args = deqp.get_option('PIGLIT_DEQP_GLES31_EXTRA_ARGS',
> -                                 ('deqp-gles31', 'extra_args'),
> -                                 default='').split()
> +    extra_args = [x for x in _EXTRA_ARGS if not x.startsiwth('--deqp-case')]
>  
>  
>  profile = deqp.make_profile(  # pylint: disable=invalid-name
>      deqp.iter_deqp_test_cases(
> -        deqp.gen_caselist_txt(_DEQP_GLES31_BIN, 'dEQP-GLES31-cases.txt')),
> +        deqp.gen_caselist_txt(_DEQP_GLES31_BIN, 'dEQP-GLES31-cases.txt',
> +                              _EXTRA_ARGS)),
>      DEQPGLES31Test)
> -- 
> 2.4.3