[igt-dev] [PATCH 1/1] scripts:igt-doc.py: add a parser to document tests inlined
Mauro Carvalho Chehab
mauro.chehab at linux.intel.com
Tue Feb 7 10:10:34 UTC 2023
From: Mauro Carvalho Chehab <mchehab at kernel.org>
Tests need to be documentation, as otherwise its goal will be
lost with time. Keeping documentation out of the sources is also
not such a good idea, as they tend to bitrot.
So, add a script to allow keeping the documentation inlined, and
add tools to verify if the documentation has gaps.
Signed-off-by: Mauro Carvalho Chehab <mchehab at kernel.org>
---
scripts/igt-doc.py | 516 +++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 516 insertions(+)
create mode 100755 scripts/igt-doc.py
diff --git a/scripts/igt-doc.py b/scripts/igt-doc.py
new file mode 100755
index 000000000000..57a729624225
--- /dev/null
+++ b/scripts/igt-doc.py
@@ -0,0 +1,516 @@
+#!/usr/bin/env python3
+# pylint: disable=C0103,C0116,C0114,C0301,R0914,R0912,R0915,R1702,W0603,W0602
+# SPDX-License-Identifier: GPL-2.0
+
+## Copyright (C) 2023 Intel Corporation ##
+## Author: Mauro Carvalho Chehab <mchehab at kernel.org> ##
+## ##
+## Allow keeping inlined test documentation and validate ##
+## if the documentation is kept updated. ##
+
+import argparse
+import fileinput
+import re
+import subprocess
+import sys
+
+igt_build_path = 'build/'
+igt_runner = '/runner/igt_runner'
+
+# Fields that mat be inside TEST and SUBTEST macros
+fields = [
+ 'Category', # Hardware building block / Software building block / ...
+ 'Sub-category', # waitfence / dmabuf/ sysfs / debugfs / ...
+ 'Coverered functionality', # basic test / ...
+ 'Test type', # functionality test / pereformance / stress
+ 'Run type', # BAT / workarouds / developer-specific / ...
+ 'Issues?', # Bug tracker issue(s)
+ 'Platforms?', # all / DG1 / DG2 / TGL / MTL / PVC / ATS-M / ...
+ 'Platform requirements?', # Any other specific platform requirements
+ 'Depends on', # some other IGT test, like igt at test@subtes
+ 'Requirements?', # other non-platform requirements
+ 'Description'] # Description of the test
+
+doc = {}
+test_number = 0
+
+#
+# ReST print routines
+#
+
+def print_subtest(fname, test_name, test, subtest):
+ summary = test_name + '@' + doc[test]["subtest"][subtest]["Summary"]
+
+ if not summary:
+ return
+
+ num_vars = summary.count('%')
+
+ if num_vars == 0:
+ print()
+ print(summary)
+ print(len(summary) * '=')
+ print("")
+
+ for k in sorted(doc[test]["subtest"][subtest].keys()):
+ if k == 'Summary':
+ continue
+ if k == 'arg':
+ continue
+
+ if doc[test]["subtest"][subtest][k] == doc[test][k]:
+ continue
+
+ print(f":{k}:", doc[test]["subtest"][subtest][k])
+ print()
+
+ return
+
+ # Convert arguments into an array
+ arg_array = {}
+ arg_ref = doc[test]["subtest"][subtest]["arg"]
+
+ for n in doc[test]["arg"][arg_ref].keys():
+ arg_array[n] = []
+ if int(n) > num_vars:
+ continue
+
+ for el in sorted(doc[test]["arg"][arg_ref][n].keys()):
+ arg_array[n].append(el)
+
+ size = len(arg_array)
+
+ if size < num_vars:
+ sys.exit(f"{fname}:subtest {summary} needs {num_vars} arguments but only {size} are defined.")
+
+
+ for j in range(0, num_vars):
+ if arg_array[j] is None:
+ sys.exit(f"{fname}:subtest{summary} needs arg[{j}], but this is not defined.")
+
+
+ # convert numeric wildcards to string ones
+ summary = re.sub(r'%(d|ld|lld|i|u|lu|llu)','%s', summary)
+
+ pos = [ 0 ] * num_vars
+ args = [ 0 ] * num_vars
+ arg_map = [ 0 ] * num_vars
+
+ while 1:
+ for j in range(0, num_vars):
+ a = arg_array[j][pos[j]]
+ args[j] = a
+
+ if a in doc[test]["arg"][arg_ref][j]:
+ arg_map[j] = doc[test]["arg"][arg_ref][j][a]
+ if re.match(r"\<.*\>", doc[test]["arg"][arg_ref][j][a]):
+ args[j] = "<" + a + ">"
+ else:
+ arg_map[j] = a
+
+ arg_summary = summary % tuple(args)
+
+ # Output the elements
+ print()
+ print(arg_summary)
+ print('=' * len(arg_summary))
+ print()
+
+ for field in sorted(doc[test]["subtest"][subtest].keys()):
+ if field == 'Summary':
+ continue
+ if field == 'arg':
+ continue
+
+ s = doc[test]["subtest"][subtest][field]
+ s = re.sub(r"%?\barg\[(\d+)\]", lambda m: arg_map[int(m.group(1)) - 1], s)
+ if s == doc[test][field]:
+ continue
+
+ print(f":{field}: %s\n" % s)
+
+ print()
+
+ # Increment variable inside the array
+ p = 0
+ while pos[p] + 1 >= len(arg_array[p]):
+ pos[p] = 0
+ p += 1
+ if p >= num_vars:
+ break
+
+ if p >= num_vars:
+ break
+
+ pos[p] += 1
+
+def print_test():
+ for test in sorted(doc.keys()):
+ fname = doc[test]["File"]
+
+ name = re.sub(r'.*tests/', '', fname)
+ name = re.sub(r'\.[ch]', '', name)
+ name = "igt@" + name
+
+ print(len(name) * '=')
+ print(name)
+ print(len(name) * '=')
+ print()
+
+ for field in sorted(doc[test].keys()):
+ if field == "subtest":
+ continue
+ if field == "arg":
+ continue
+
+ print(f":{field}: {doc[test][field]}")
+
+ for subtest in doc[test]["subtest"].keys():
+ print_subtest(fname, name, test, subtest)
+
+ print()
+ print()
+
+#
+# Get a list of subtests
+#
+
+def get_subtests():
+ subtests = []
+
+ for test in sorted(doc.keys()):
+ fname = doc[test]["File"]
+
+ test_name = re.sub(r'.*tests/', '', fname)
+ test_name = re.sub(r'\.[ch]', '', test_name)
+ test_name = "igt@" + test_name
+
+ for subtest in doc[test]["subtest"].keys():
+ summary = test_name + '@' + doc[test]["subtest"][subtest]["Summary"]
+
+ if not summary:
+ continue
+
+ num_vars = summary.count('%')
+
+ if num_vars == 0:
+ subtests.append(summary)
+ else:
+ # Convert arguments into an array
+ arg_array = {}
+ arg_ref = doc[test]["subtest"][subtest]["arg"]
+
+ for n in doc[test]["arg"][arg_ref].keys():
+ arg_array[n] = []
+ if int(n) > num_vars:
+ continue
+
+ for el in sorted(doc[test]["arg"][arg_ref][n].keys()):
+ arg_array[n].append(el)
+
+ size = len(arg_array)
+
+ if size < num_vars:
+ sys.exit(f"{fname}:subtest {summary} needs {num_vars} arguments but only {size} are defined.")
+
+ for j in range(0, num_vars):
+ if arg_array[j] is None:
+ sys.exit(f"{fname}:subtest{summary} needs arg[{j}], but this is not defined.")
+
+ # convert numeric wildcard,s to string ones
+ summary = re.sub('%(d|ld|lld|i|u|lu|llu)','%s', summary)
+
+ pos = [ 0 ] * num_vars
+ args = [ 0 ] * num_vars
+
+ while 1:
+ for j in range(0, num_vars):
+ a = arg_array[j][pos[j]]
+
+ if re.match(r"\<.*\>", doc[test]["arg"][arg_ref][j][a]):
+ args[j] = "<" + a + ">"
+ else:
+ args[j] = a
+
+ arg_summary = summary % tuple(args)
+ subtests.append(arg_summary)
+
+ # Increment variable inside the array
+ p = 0
+ while pos[p] + 1 >= len(arg_array[p]):
+ pos[p] = 0
+ p += 1
+ if p >= num_vars:
+ break
+
+ if p >= num_vars:
+ break
+
+ pos[p] += 1
+
+ return subtests
+
+#
+# Compare testlists from IGT with the documentation from source code
+#
+def check_tests():
+ global min_test_prefix
+
+ doc_subtests = sorted(get_subtests())
+
+ for i in range(0, len(doc_subtests)):
+ doc_subtests[i] = re.sub(r'\<[^\>]+\>', r'\\d+', doc_subtests[i])
+
+ # Get a list of tests from
+ result = subprocess.run([ f"{igt_build_path}/{igt_runner}",
+ "-L", "-t", min_test_prefix,
+ f"{igt_build_path}/tests"],
+ capture_output = True, text = True)
+ if result.returncode:
+ print( result.stdout)
+ print("Error:", result.stderr)
+ sys.exit(result.returncode)
+
+ run_subtests = sorted(result.stdout.splitlines())
+
+ # Compare arrays
+
+ run_missing = []
+ doc_uneeded = []
+
+ for t1 in doc_subtests:
+ found = False
+ for t2 in run_subtests:
+ if re.match(r'^' + t1 + r'$', t2):
+ found = True
+ break
+ if not found:
+ doc_uneeded.append(t1)
+
+ for t2 in run_subtests:
+ found = False
+ for t1 in doc_subtests:
+ if re.match(r'^' + t1 + r'$', t2):
+ found = True
+ break
+ if not found:
+ run_missing.append(t2)
+
+ if doc_uneeded:
+ print("Unused documentation")
+ for t in doc_uneeded:
+ print(t)
+
+ if run_missing:
+ if doc_uneeded:
+ print()
+ print("Missing documentation")
+ for t in run_missing:
+ print(t)
+
+#
+# Parse a filename, adding documentation inside the doc dictionary
+#
+
+def add_fname_to_dict(fname):
+ global test_number
+
+ current_test = None
+ current_subtest = None
+
+ handle_section = ''
+ current_field = ''
+ arg_number = 1
+ cur_arg = -1
+ cur_arg_element = 0
+
+ with open(fname, 'r', encoding='utf8') as handle:
+ arg_ref = None
+ current_test = ''
+ subtest_number = 0
+
+ for ln in handle:
+ if re.match(r'^\s*\*$', ln):
+ continue
+
+ if re.match(r'^\s*\*/$', ln):
+ handle_section = ''
+ current_subtest = None
+ arg_ref = None
+ cur_arg = -1
+
+ continue
+
+ if re.match(r'^\s*/\*\*$', ln):
+ handle_section = '1'
+ continue
+
+ if not handle_section:
+ continue
+
+ ln = re.sub(r'^\s*\*\s*', '', ln)
+
+ # Handle only known sections
+ if handle_section == '1':
+ current_field = ''
+
+ # Check if it is a new TEST section
+ if (match := re.match(r'^TEST:\s*(.*)', ln)):
+ current_test = test_number
+ test_number += 1
+
+ handle_section = 'test'
+
+ doc[current_test] = {}
+ doc[current_test]["arg"] = {}
+ doc[current_test]["Summary"] = match.group(1)
+ doc[current_test]["File"] = fname
+ doc[current_test]["subtest"] = {}
+ current_subtest = None
+
+ continue
+
+ # Check if it is a new SUBTEST section
+ if (match := re.match(r'^SUBTESTS?:\s*(.*)', ln)):
+ current_subtest = subtest_number
+ subtest_number += 1
+
+ current_field = ''
+ handle_section = 'subtest'
+
+ doc[current_test]["subtest"][current_subtest] = {}
+
+ doc[current_test]["subtest"][current_subtest]["Summary"] = match.group(1)
+ doc[current_test]["subtest"][current_subtest]["Description"] = ''
+
+ if not arg_ref:
+ arg_ref = arg_number
+ arg_number += 1
+ doc[current_test]["arg"][arg_ref] = {}
+
+ doc[current_test]["subtest"][current_subtest]["arg"] = arg_ref
+
+ continue
+
+ # It is a known section. Parse its contents
+ if (match := re.match(field_re, ln)):
+ current_field = match.group(1).lower().capitalize()
+ v = match.group(2)
+
+ if handle_section == 'test':
+ doc[current_test][current_field] = v
+ else:
+ doc[current_test]["subtest"][current_subtest][current_field] = v
+
+ cur_arg = -1
+
+ continue
+
+ # Use hashes for arguments to avoid duplication
+ if (match := re.match(r'arg\[(\d+)\]:\s*(.*)', ln)):
+ current_field = ''
+ if arg_ref is None:
+ sys.exit(f"{fname}:{fileinput.filelineno()}: arguments should be defined after one or more subtests, at the same comment")
+
+ cur_arg = int(match.group(1)) - 1
+ if cur_arg not in doc[current_test]["arg"][arg_ref]:
+ doc[current_test]["arg"][arg_ref][cur_arg] = {}
+
+ cur_arg_element = match.group(2)
+
+ if match.group(2):
+ # Should be used only for numeric values
+ doc[current_test]["arg"][arg_ref][cur_arg][cur_arg_element] = "<" + match.group(2) + ">"
+
+ continue
+
+ if (match := re.match(r'\@(\S+):\s*(.*)', ln)):
+ if cur_arg >= 0:
+ current_field = ''
+ if arg_ref is None:
+ sys.exit(f"{fname}:{fileinput.filelineno()}: arguments should be defined after one or more subtests, at the same comment")
+
+ cur_arg_element = match.group(1)
+ doc[current_test]["arg"][arg_ref][cur_arg][cur_arg_element] = match.group(2)
+
+ else:
+ print(f"{fname}: Warning: invalid argument: @%s: %s" %
+ (match.group(1), match.group(2)))
+
+ continue
+
+ # Handle multi-line field contents
+ if current_field:
+ if (match := re.match(r'\s*(.*)', ln)):
+ if handle_section == 'test':
+ doc[current_test][current_field] += " " + \
+ match.group(1)
+ else:
+ doc[current_test]["subtest"][current_subtest][current_field] += " " + \
+ match.group(1)
+
+ continue
+
+ # Handle multi-line argument contents
+ if cur_arg >= 0 and arg_ref is not None:
+ if (match := re.match(r'\s*\*?\s*(.*)', ln)):
+ v = match.group(1)
+
+ if (match := re.match(r'(.*)>$', doc[current_test]["arg"][arg_ref][cur_arg][cur_arg_element])):
+ doc[current_test]["arg"][arg_ref][cur_arg][cur_arg_element] = '<' + v + ">"
+ else:
+ doc[current_test]["arg"][arg_ref][cur_arg][cur_arg_element] = v
+
+ continue
+
+#
+# Main: parse the files and fill %doc struct
+#
+
+parser = argparse.ArgumentParser(description = "Print formatted kernel documentation to stdout.",
+ formatter_class = argparse.ArgumentDefaultsHelpFormatter,
+ epilog = 'If no action specified, assume --rest.')
+parser.add_argument("--rest", action="store_true",
+ help="Generate documentation from the source files, in ReST file format.")
+parser.add_argument("--show-subtests", action="store_true",
+ help="Shows the name of the documented subtests in alphabetical order.")
+parser.add_argument("--check-testlist", action="store_true",
+ help="Compare documentation against IGT runner testlist.")
+parser.add_argument("--igt-build-path",
+ help="Path where the IGT runner is sitting. Used by --check-testlist.",
+ default=igt_build_path)
+parser.add_argument('--files', nargs='+',
+ help="File name(s) to be processed")
+
+parse_args = parser.parse_args()
+
+field_re = re.compile(r"(" + '|'.join(fields) + r'):\s*(.*)', re.I)
+
+min_test_prefix = ''
+
+for filename in parse_args.files:
+ # Dynamically build a test prefix from the file name
+ prefix = re.sub(r'.*tests/', '', filename)
+ prefix = r'igt\@' + re.sub(r'(.*/).*', r'\1', prefix)
+
+ if min_test_prefix == '':
+ min_test_prefix = prefix
+ elif len(prefix) < len(min_test_prefix):
+ min_test_prefix = prefix
+
+ add_fname_to_dict(filename)
+
+run = 0
+if parse_args.show_subtests:
+ run = 1
+ subtests = get_subtests()
+
+ for subtest in subtests:
+ print (subtest)
+
+if parse_args.check_testlist:
+ run = 1
+ check_tests()
+
+if not run or parse_args.rest:
+ print_test()
--
2.39.0
More information about the igt-dev
mailing list