[igt-dev] [PATCH i-g-t] .gitlab-ci: Produce a list of undocumented tests

[Petri Latvala]

On Wed, Sep 04, 2019 at 01:43:21PM +0300, Arkadiusz Hiler wrote:
> We have a requirement that all new tests should be documented using
> igt_describe() & family since 2f273018ac42 ("CONTRIBUTING: Rework a bit and update").
> 
> Let's start actually enforcing that by having this as a part of the CI.
> 
> For consumption by:
> https://gitlab.freedesktop.org/gfx-ci/i915-infra/merge_requests/55
> 
> Cc: Petri Latvala <petri.latvala at intel.com>
> Signed-off-by: Arkadiusz Hiler <arkadiusz.hiler at intel.com>

Reviewed-by: Petri Latvala <petri.latvala at intel.com>

> ---
>  .gitlab-ci.yml                        |  9 ++++
>  .gitlab-ci/list_undocumented_tests.py | 61 +++++++++++++++++++++++++++
>  2 files changed, 70 insertions(+)
>  create mode 100755 .gitlab-ci/list_undocumented_tests.py
> 
> diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
> index 17378df5..d465c79a 100644
> --- a/.gitlab-ci.yml
> +++ b/.gitlab-ci.yml
> @@ -237,6 +237,15 @@ test:test-list-diff:
>    stage: test
>    script: diff <(sed "s/ /\n/g" meson-test-list.txt| grep -v 'vc4\|v3d\|panfrost' | sort) <(sed "s/ /\n/g" autotools-test-list.txt | sort)
>  
> +test:list-undocumented-tests:
> +  dependencies:
> +    - build:tests-fedora
> +  stage: test
> +  script: .gitlab-ci/list_undocumented_tests.py build/tests/test-list.txt > undocumented_tests.txt
> +  artifacts:
> +    paths:
> +      - undocumented_tests.txt
> +
>  ################### DEPLOY #########################
>  
>  pages:
> diff --git a/.gitlab-ci/list_undocumented_tests.py b/.gitlab-ci/list_undocumented_tests.py
> new file mode 100755
> index 00000000..ee836559
> --- /dev/null
> +++ b/.gitlab-ci/list_undocumented_tests.py
> @@ -0,0 +1,61 @@
> +#!/usr/bin/env python3
> +
> +import re
> +import sys
> +import os.path
> +import subprocess
> +
> +from collections import namedtuple
> +
> +Subtest = namedtuple("Subtest", "name description")
> +
> +def get_testlist(path):
> +    "read binaries' names from test-list.txt"
> +    with open(path, 'r') as f:
> +        assert(f.readline() == "TESTLIST\n")
> +        tests = f.readline().strip().split(" ")
> +        assert(f.readline() == "END TESTLIST\n")
> +
> +    return tests
> +
> +def get_subtests(testdir, test):
> +    "execute test and get subtests with their descriptions via --describe"
> +    output = []
> +    full_test_path = os.path.join(testdir, test)
> +    proc = subprocess.run([full_test_path, "--describe"], stdout=subprocess.PIPE)
> +    description = ""
> +    current_subtest = None
> +
> +    for line in proc.stdout.decode().splitlines():
> +        if line.startswith("SUB "):
> +            output += [Subtest(current_subtest, description)]
> +            description = ""
> +            current_subtest = line.split(' ')[1]
> +        else:
> +            description += line
> +
> +    output += [Subtest(current_subtest, description)]
> +
> +    return output
> +
> +def main():
> +    testlist_file = sys.argv[1]
> +    testdir       = os.path.abspath(os.path.dirname(testlist_file))
> +
> +    tests = get_testlist(testlist_file)
> +
> +    for test in tests:
> +        subtests = get_subtests(testdir, test)
> +
> +        if subtests and subtests[0].name:
> +            # no top level description
> +            print(test)
> +
> +        for name, description in subtests:
> +            if not name:
> +                continue
> +
> +            if "NO DOCUMENTATION!" in description:
> +                print("{}@{}".format(test, name))
> +
> +main()
> -- 
> 2.21.0
>