[igt-dev] [PATCH i-g-t v2 1/1] scripts:igt-doc.py: add a parser to document tests inlined
Kamil Konieczny
kamil.konieczny at linux.intel.com
Mon Feb 13 16:15:38 UTC 2023
Hi Mauro,
On 2023-02-09 at 12:57:12 +0100, Mauro Carvalho Chehab wrote:
> From: Mauro Carvalho Chehab <mchehab at kernel.org>
>
> Tests need to be documentation, as otherwise its goal will be
> lost with time. Keeping documentation out of the sources is also
> not such a good idea, as they tend to bitrot.
>
> So, add a script to allow keeping the documentation inlined, and
> add tools to verify if the documentation has gaps.
>
> Signed-off-by: Mauro Carvalho Chehab <mchehab at kernel.org>
> ---
> scripts/igt_doc.py | 487 +++++++++++++++++++++++++++++++++++++++++++++
> 1 file changed, 487 insertions(+)
> create mode 100755 scripts/igt_doc.py
>
> diff --git a/scripts/igt_doc.py b/scripts/igt_doc.py
> new file mode 100755
> index 000000000000..292d99c1ef92
> --- /dev/null
> +++ b/scripts/igt_doc.py
> @@ -0,0 +1,487 @@
> +#!/usr/bin/env python3
> +# pylint: disable=C0301,R0914,R0912,R0915
> +# SPDX-License-Identifier: GPL-2.0
----------------------------- ^
This should be MIT
> +
> +## Copyright (C) 2023 Intel Corporation ##
> +## Author: Mauro Carvalho Chehab <mchehab at kernel.org> ##
> +## ##
> +## Allow keeping inlined test documentation and validate ##
> +## if the documentation is kept updated. ##
> +
> +"""Maintain test plan and test implementation documentation on IGT."""
> +
> +import argparse
> +import fileinput
> +import re
> +import subprocess
> +import sys
> +
> +IGT_BUILD_PATH = 'build/'
> +IGT_RUNNER = '/runner/igt_runner'
> +
> +# Fields that mat be inside TEST and SUBTEST macros
> +fields = [
> + 'Category', # Hardware building block / Software building block / ...
> + 'Sub-category', # waitfence / dmabuf/ sysfs / debugfs / ...
> + 'Coverered functionality', # basic test / ...
------------^^
Covered
> + 'Test type', # functionality test / pereformance / stress
-------------------------------------------------- ^
performance
> + 'Run type', # BAT / workarouds / stress / developer-specific / ...
------------------------------------------------------ ^
I am not sure what is it.
+cc Petri
Regards,
Kamil
> + 'Issues?', # Bug tracker issue(s)
> + 'Platforms?', # all / DG1 / DG2 / TGL / MTL / PVC / ATS-M / ...
> + 'Platform requirements?', # Any other specific platform requirements
> + 'Depends on', # some other IGT test, like igt at test@subtes
> + 'Requirements?', # other non-platform requirements
> + 'Description'] # Description of the test
> +
> +#
> +# Expand subtests
> +#
> +
> +class TestList:
> + """Parse and handle test lists with test/subtest documentation."""
> + def __init__(self):
> + self.doc = {}
> + self.test_number = 0
> + self.min_test_prefix = ''
> +
> + def expand_subtest(self, fname, test_name, test):
> + """Expand subtest wildcards providing an array with subtests"""
> + subtest_array = []
> +
> + for subtest in self.doc[test]["subtest"].keys():
> + summary = test_name + '@' + self.doc[test]["subtest"][subtest]["Summary"]
> +
> + if not summary:
> + continue
> +
> + num_vars = summary.count('%')
> +
> + if num_vars == 0:
> + subtest_dict = {}
> +
> + # Trivial case: no need to expand anything
> + subtest_dict["Summary"] = summary
> +
> + for k in sorted(self.doc[test]["subtest"][subtest].keys()):
> + if k == 'Summary':
> + continue
> + if k == 'arg':
> + continue
> +
> + if self.doc[test]["subtest"][subtest][k] == self.doc[test][k]:
> + continue
> +
> + subtest_dict[k] = self.doc[test]["subtest"][subtest][k]
> + subtest_array.append(subtest_dict)
> +
> + continue
> +
> + # Handle summaries with wildcards
> +
> + # Convert arguments into an array
> + arg_array = {}
> + arg_ref = self.doc[test]["subtest"][subtest]["arg"]
> +
> + for arg_k in self.doc[test]["arg"][arg_ref].keys():
> + arg_array[arg_k] = []
> + if int(arg_k) > num_vars:
> + continue
> +
> + for arg_el in sorted(self.doc[test]["arg"][arg_ref][arg_k].keys()):
> + arg_array[arg_k].append(arg_el)
> +
> + size = len(arg_array)
> +
> + if size < num_vars:
> + sys.exit(f"{fname}:subtest {summary} needs {num_vars} arguments but only {size} are defined.")
> +
> + for j in range(0, num_vars):
> + if arg_array[j] is None:
> + sys.exit(f"{fname}:subtest{summary} needs arg[{j}], but this is not defined.")
> +
> +
> + # convert numeric wildcards to string ones
> + summary = re.sub(r'%(d|ld|lld|i|u|lu|llu)','%s', summary)
> +
> + pos = [ 0 ] * num_vars
> + args = [ 0 ] * num_vars
> + arg_map = [ 0 ] * num_vars
> +
> + while 1:
> + for j in range(0, num_vars):
> + arg_val = arg_array[j][pos[j]]
> + args[j] = arg_val
> +
> + if arg_val in self.doc[test]["arg"][arg_ref][j]:
> + arg_map[j] = self.doc[test]["arg"][arg_ref][j][arg_val]
> + if re.match(r"\<.*\>", self.doc[test]["arg"][arg_ref][j][arg_val]):
> + args[j] = "<" + arg_val + ">"
> + else:
> + arg_map[j] = arg_val
> +
> + arg_summary = summary % tuple(args)
> +
> + # Store the element
> + subtest_dict = {}
> + subtest_dict["Summary"] = arg_summary
> +
> + for field in sorted(self.doc[test]["subtest"][subtest].keys()):
> + if field == 'Summary':
> + continue
> + if field == 'arg':
> + continue
> +
> + sub_field = self.doc[test]["subtest"][subtest][field]
> + sub_field = re.sub(r"%?\barg\[(\d+)\]", lambda m: arg_map[int(m.group(1)) - 1], sub_field) # pylint: disable=W0640
> + if sub_field == self.doc[test][field]:
> + continue
> +
> + subtest_dict[field] = sub_field
> +
> + subtest_array.append(subtest_dict)
> +
> + # Increment variable inside the array
> + arr_pos = 0
> + while pos[arr_pos] + 1 >= len(arg_array[arr_pos]):
> + pos[arr_pos] = 0
> + arr_pos += 1
> + if arr_pos >= num_vars:
> + break
> +
> + if arr_pos >= num_vars:
> + break
> +
> + pos[arr_pos] += 1
> +
> + return subtest_array
> +
> + #
> + # ReST print routines
> + #
> +
> + def print_test(self):
> + """Print tests and subtests"""
> + for test in sorted(self.doc.keys()):
> + fname = self.doc[test]["File"]
> +
> + name = re.sub(r'.*tests/', '', fname)
> + name = re.sub(r'\.[ch]', '', name)
> + name = "igt@" + name
> +
> + print(len(name) * '=')
> + print(name)
> + print(len(name) * '=')
> + print()
> +
> + for field in sorted(self.doc[test].keys()):
> + if field == "subtest":
> + continue
> + if field == "arg":
> + continue
> +
> + print(f":{field}: {self.doc[test][field]}")
> +
> + subtest_array = self.expand_subtest(fname, name, test)
> +
> + for subtest in subtest_array:
> + print()
> + print(subtest["Summary"])
> + print(len(subtest["Summary"]) * '=')
> + print("")
> +
> + for field in sorted(subtest.keys()):
> + if field == 'Summary':
> + continue
> + if field == 'arg':
> + continue
> +
> + print(f":{field}:", subtest[field])
> +
> + print()
> +
> + print()
> + print()
> +
> + #
> + # Get a list of subtests
> + #
> +
> + def get_subtests(self):
> + """Return an array with all subtests"""
> + subtests = []
> +
> + for test in sorted(self.doc.keys()):
> + fname = self.doc[test]["File"]
> +
> + test_name = re.sub(r'.*tests/', '', fname)
> + test_name = re.sub(r'\.[ch]', '', test_name)
> + test_name = "igt@" + test_name
> +
> + subtest_array = self.expand_subtest(fname, test_name, test)
> +
> + for subtest in subtest_array:
> + subtests.append(subtest["Summary"])
> +
> + return subtests
> +
> + #
> + # Compare testlists from IGT with the documentation from source code
> + #
> + def check_tests(self):
> + """Compare documented subtests with the IGT test list"""
> + doc_subtests = sorted(self.get_subtests())
> +
> + for i in range(0, len(doc_subtests)): # pylint: disable=C0200
> + doc_subtests[i] = re.sub(r'\<[^\>]+\>', r'\\d+', doc_subtests[i])
> +
> + # Get a list of tests from
> + result = subprocess.run([ f"{IGT_BUILD_PATH}/{IGT_RUNNER}", # pylint: disable=W1510
> + "-L", "-t", self.min_test_prefix,
> + f"{IGT_BUILD_PATH}/tests"],
> + capture_output = True, text = True)
> + if result.returncode:
> + print( result.stdout)
> + print("Error:", result.stderr)
> + sys.exit(result.returncode)
> +
> + run_subtests = sorted(result.stdout.splitlines())
> +
> + # Compare arrays
> +
> + run_missing = []
> + doc_uneeded = []
> +
> + for doc_test in doc_subtests:
> + found = False
> + for run_test in run_subtests:
> + if re.match(r'^' + doc_test + r'$', run_test):
> + found = True
> + break
> + if not found:
> + doc_uneeded.append(doc_test)
> +
> + for run_test in run_subtests:
> + found = False
> + for doc_test in doc_subtests:
> + if re.match(r'^' + doc_test + r'$', run_test):
> + found = True
> + break
> + if not found:
> + run_missing.append(run_test)
> +
> + if doc_uneeded:
> + print("Unused documentation")
> + for test_name in doc_uneeded:
> + print(test_name)
> +
> + if run_missing:
> + if doc_uneeded:
> + print()
> + print("Missing documentation")
> + for test_name in run_missing:
> + print(test_name)
> +
> + #
> + # Parse a filename, adding documentation inside the doc dictionary
> + #
> +
> + def add_file_documentation(self, fname):
> + """Adds the contents of test/subtest documentation form a file"""
> + current_test = None
> + current_subtest = None
> +
> + handle_section = ''
> + current_field = ''
> + arg_number = 1
> + cur_arg = -1
> + cur_arg_element = 0
> +
> + prefix = re.sub(r'.*tests/', '', filename)
> + prefix = r'igt\@' + re.sub(r'(.*/).*', r'\1', prefix)
> +
> + if self.min_test_prefix == '':
> + self.min_test_prefix = prefix
> + elif len(prefix) < len(self.min_test_prefix):
> + self.min_test_prefix = prefix
> +
> + with open(fname, 'r', encoding='utf8') as handle:
> + arg_ref = None
> + current_test = ''
> + subtest_number = 0
> +
> + for file_line in handle:
> + if re.match(r'^\s*\*$', file_line):
> + continue
> +
> + if re.match(r'^\s*\*/$', file_line):
> + handle_section = ''
> + current_subtest = None
> + arg_ref = None
> + cur_arg = -1
> +
> + continue
> +
> + if re.match(r'^\s*/\*\*$', file_line):
> + handle_section = '1'
> + continue
> +
> + if not handle_section:
> + continue
> +
> + file_line = re.sub(r'^\s*\*\s*', '', file_line)
> +
> + # Handle only known sections
> + if handle_section == '1':
> + current_field = ''
> +
> + # Check if it is a new TEST section
> + if (match := re.match(r'^TEST:\s*(.*)', file_line)):
> + current_test = self.test_number
> + self.test_number += 1
> +
> + handle_section = 'test'
> +
> + self.doc[current_test] = {}
> + self.doc[current_test]["arg"] = {}
> + self.doc[current_test]["Summary"] = match.group(1)
> + self.doc[current_test]["File"] = fname
> + self.doc[current_test]["subtest"] = {}
> + current_subtest = None
> +
> + continue
> +
> + # Check if it is a new SUBTEST section
> + if (match := re.match(r'^SUBTESTS?:\s*(.*)', file_line)):
> + current_subtest = subtest_number
> + subtest_number += 1
> +
> + current_field = ''
> + handle_section = 'subtest'
> +
> + self.doc[current_test]["subtest"][current_subtest] = {}
> +
> + self.doc[current_test]["subtest"][current_subtest]["Summary"] = match.group(1)
> + self.doc[current_test]["subtest"][current_subtest]["Description"] = ''
> +
> + if not arg_ref:
> + arg_ref = arg_number
> + arg_number += 1
> + self.doc[current_test]["arg"][arg_ref] = {}
> +
> + self.doc[current_test]["subtest"][current_subtest]["arg"] = arg_ref
> +
> + continue
> +
> + # It is a known section. Parse its contents
> + if (match := re.match(field_re, file_line)):
> + current_field = match.group(1).lower().capitalize()
> + match_val = match.group(2)
> +
> + if handle_section == 'test':
> + self.doc[current_test][current_field] = match_val
> + else:
> + self.doc[current_test]["subtest"][current_subtest][current_field] = match_val
> +
> + cur_arg = -1
> +
> + continue
> +
> + # Use hashes for arguments to avoid duplication
> + if (match := re.match(r'arg\[(\d+)\]:\s*(.*)', file_line)):
> + current_field = ''
> + if arg_ref is None:
> + sys.exit(f"{fname}:{fileinput.filelineno()}: arguments should be defined after one or more subtests, at the same comment")
> +
> + cur_arg = int(match.group(1)) - 1
> + if cur_arg not in self.doc[current_test]["arg"][arg_ref]:
> + self.doc[current_test]["arg"][arg_ref][cur_arg] = {}
> +
> + cur_arg_element = match.group(2)
> +
> + if match.group(2):
> + # Should be used only for numeric values
> + self.doc[current_test]["arg"][arg_ref][cur_arg][cur_arg_element] = "<" + match.group(2) + ">"
> +
> + continue
> +
> + if (match := re.match(r'\@(\S+):\s*(.*)', file_line)):
> + if cur_arg >= 0:
> + current_field = ''
> + if arg_ref is None:
> + sys.exit(f"{fname}:{fileinput.filelineno()}: arguments should be defined after one or more subtests, at the same comment")
> +
> + cur_arg_element = match.group(1)
> + self.doc[current_test]["arg"][arg_ref][cur_arg][cur_arg_element] = match.group(2)
> +
> + else:
> + print(f"{fname}: Warning: invalid argument: @%s: %s" %
> + (match.group(1), match.group(2)))
> +
> + continue
> +
> + # Handle multi-line field contents
> + if current_field:
> + if (match := re.match(r'\s*(.*)', file_line)):
> + if handle_section == 'test':
> + self.doc[current_test][current_field] += " " + \
> + match.group(1)
> + else:
> + self.doc[current_test]["subtest"][current_subtest][current_field] += " " + \
> + match.group(1)
> +
> + continue
> +
> + # Handle multi-line argument contents
> + if cur_arg >= 0 and arg_ref is not None:
> + if (match := re.match(r'\s*\*?\s*(.*)', file_line)):
> + match_val = match.group(1)
> +
> + match = re.match(r'^(\<.*)\>$',self.doc[current_test]["arg"][arg_ref][cur_arg][cur_arg_element])
> + if match:
> + self.doc[current_test]["arg"][arg_ref][cur_arg][cur_arg_element] = match.group(1) + ' ' + match_val + ">"
> + else:
> + self.doc[current_test]["arg"][arg_ref][cur_arg][cur_arg_element] += ' ' + match_val
> +
> + continue
> +
> +#
> +# Main: parse the files and fill %doc struct
> +#
> +
> +parser = argparse.ArgumentParser(description = "Print formatted kernel documentation to stdout.",
> + formatter_class = argparse.ArgumentDefaultsHelpFormatter,
> + epilog = 'If no action specified, assume --rest.')
> +parser.add_argument("--rest", action="store_true",
> + help="Generate documentation from the source files, in ReST file format.")
> +parser.add_argument("--show-subtests", action="store_true",
> + help="Shows the name of the documented subtests in alphabetical order.")
> +parser.add_argument("--check-testlist", action="store_true",
> + help="Compare documentation against IGT runner testlist.")
> +parser.add_argument("--igt-build-path",
> + help="Path where the IGT runner is sitting. Used by --check-testlist.",
> + default=IGT_BUILD_PATH)
> +parser.add_argument('--files', nargs='+', required=True,
> + help="File name(s) to be processed")
> +
> +parse_args = parser.parse_args()
> +
> +field_re = re.compile(r"(" + '|'.join(fields) + r'):\s*(.*)', re.I)
> +
> +tests = TestList()
> +
> +for filename in parse_args.files:
> + tests.add_file_documentation(filename)
> +
> +RUN = 0
> +if parse_args.show_subtests:
> + RUN = 1
> +
> + for sub in tests.get_subtests():
> + print (sub)
> +
> +if parse_args.check_testlist:
> + RUN = 1
> + tests.check_tests()
> +
> +if not RUN or parse_args.rest:
> + tests.print_test()
> --
> 2.39.0
>
More information about the igt-dev
mailing list