[igt-dev] [PATCH i-g-t] .gitlab-ci: Produce a list of undocumented tests

[Arkadiusz Hiler]

We have a requirement that all new tests should be documented using
igt_describe() & family since 2f273018ac42 ("CONTRIBUTING: Rework a bit and update").

Let's start actually enforcing that by having this as a part of the CI.

For consumption by:
https://gitlab.freedesktop.org/gfx-ci/i915-infra/merge_requests/55

Cc: Petri Latvala <petri.latvala at intel.com>
Signed-off-by: Arkadiusz Hiler <arkadiusz.hiler at intel.com>
---
 .gitlab-ci.yml                        |  9 ++++
 .gitlab-ci/list_undocumented_tests.py | 61 +++++++++++++++++++++++++++
 2 files changed, 70 insertions(+)
 create mode 100755 .gitlab-ci/list_undocumented_tests.py

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 17378df5..d465c79a 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -237,6 +237,15 @@ test:test-list-diff:
   stage: test
   script: diff <(sed "s/ /\n/g" meson-test-list.txt| grep -v 'vc4\|v3d\|panfrost' | sort) <(sed "s/ /\n/g" autotools-test-list.txt | sort)
 
+test:list-undocumented-tests:
+  dependencies:
+    - build:tests-fedora
+  stage: test
+  script: .gitlab-ci/list_undocumented_tests.py build/tests/test-list.txt > undocumented_tests.txt
+  artifacts:
+    paths:
+      - undocumented_tests.txt
+
 ################### DEPLOY #########################
 
 pages:
diff --git a/.gitlab-ci/list_undocumented_tests.py b/.gitlab-ci/list_undocumented_tests.py
new file mode 100755
index 00000000..ee836559
--- /dev/null
+++ b/.gitlab-ci/list_undocumented_tests.py
@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+
+import re
+import sys
+import os.path
+import subprocess
+
+from collections import namedtuple
+
+Subtest = namedtuple("Subtest", "name description")
+
+def get_testlist(path):
+    "read binaries' names from test-list.txt"
+    with open(path, 'r') as f:
+        assert(f.readline() == "TESTLIST\n")
+        tests = f.readline().strip().split(" ")
+        assert(f.readline() == "END TESTLIST\n")
+
+    return tests
+
+def get_subtests(testdir, test):
+    "execute test and get subtests with their descriptions via --describe"
+    output = []
+    full_test_path = os.path.join(testdir, test)
+    proc = subprocess.run([full_test_path, "--describe"], stdout=subprocess.PIPE)
+    description = ""
+    current_subtest = None
+
+    for line in proc.stdout.decode().splitlines():
+        if line.startswith("SUB "):
+            output += [Subtest(current_subtest, description)]
+            description = ""
+            current_subtest = line.split(' ')[1]
+        else:
+            description += line
+
+    output += [Subtest(current_subtest, description)]
+
+    return output
+
+def main():
+    testlist_file = sys.argv[1]
+    testdir       = os.path.abspath(os.path.dirname(testlist_file))
+
+    tests = get_testlist(testlist_file)
+
+    for test in tests:
+        subtests = get_subtests(testdir, test)
+
+        if subtests and subtests[0].name:
+            # no top level description
+            print(test)
+
+        for name, description in subtests:
+            if not name:
+                continue
+
+            if "NO DOCUMENTATION!" in description:
+                print("{}@{}".format(test, name))
+
+main()
-- 
2.21.0