[igt-dev] [PATCH i-g-t 1/5] scripts/test_list.py: better parse list of tests
Kamil Konieczny
kamil.konieczny at linux.intel.com
Thu Nov 2 14:57:36 UTC 2023
Hi Mauro,
On 2023-11-02 at 14:06:23 +0100, Mauro Carvalho Chehab wrote:
> From: Mauro Carvalho Chehab <mchehab at kernel.org>
>
> The JSON config items for using a list of tests to fill a
> field is confusing:
> - there are several fields defined under __properties__, but
> they're not grouped altogether;
> - the names of such fields are badly defined;
> - the "FULL" testlist is not really a list of all tests, as
> it currently exclude tests used on other platforms.
>
> Rewrite the logic to group them into a dict inside a field,
> like:
>
> { "Field": {
> "_properties_": {
> "update-from-file": {
> "type": "subtest-match", # or "regex", "regex-ignorecase"
> "default-if-not-excluded": "not on testlists",
> "append-value-if-not-excluded": "FULL",
> "include": [
> { "BAT": "../ci-dir/driver-bat.testlist" },
> { "func1": "../ci-dir/driver-func1.testlist" },
> ],
> "exclude": [
> { "Xe BAT, FULL": "../ci-dir/driver.blocklist" }
> { "func1, FULL": "../ci-dir/driver-func1.blocklist" }
> ]
> }
> }
> }
>
> This will update all tests from ../ci-dir/driver-bat.testlist setting
> "Field" with "BAT", excluding the ones at ../ci-dir/driver.blocklist.
> Similarly, the "func1" value will be parsed.
>
> The "FULL" testlist will contain all tests but the ones inside the
> two exclude lists: ../ci-dir/driver.blocklist and ../ci-dir/driver-func1.blocklist.
>
> Signed-off-by: Mauro Carvalho Chehab <mchehab at kernel.org>
Acked-by: Kamil Konieczny <kamil.konieczny at linux.intel.com>
> ---
> scripts/test_list.py | 70 ++++++++++++++++++--------------
> tests/intel/kms_test_config.json | 28 +++++++------
> tests/intel/xe_test_config.json | 16 ++++----
> 3 files changed, 63 insertions(+), 51 deletions(-)
>
> diff --git a/scripts/test_list.py b/scripts/test_list.py
> index 741ec5f4b5b5..bd609feeb1f1 100644
> --- a/scripts/test_list.py
> +++ b/scripts/test_list.py
> @@ -328,23 +328,25 @@ class TestList:
> del item["_properties_"]["level"]
> del item["_properties_"]["sublevel"]
>
> - # Read testlist files if any
> - if "include" in item["_properties_"]:
> - testlist = {}
> - for value in item["_properties_"]["include"]:
> - for name in value.keys():
> - self.read_testlist(field, item, testlist, name, cfg_path + value[name])
> + update = self.props[field]["_properties_"].get("update-from-file")
> + if update:
> + # Read testlist files if any
> + if "include" in update:
> + testlist = {}
> + for value in update["include"]:
> + for name in value.keys():
> + self.read_testlist(update, field, item, testlist, name, cfg_path + value[name])
>
> - item["_properties_"]["include"] = testlist
> + update["include"] = testlist
>
> - # Read blocklist files if any
> - if "exclude" in item["_properties_"]:
> - testlist = {}
> - for value in item["_properties_"]["exclude"]:
> - for name in value.keys():
> - self.read_testlist(field, item, testlist, name, cfg_path + value[name])
> + # Read blocklist files if any
> + if "exclude" in update:
> + testlist = {}
> + for value in update["exclude"]:
> + for name in value.keys():
> + self.read_testlist(update, field, item, testlist, name, cfg_path + value[name])
>
> - item["_properties_"]["exclude"] = testlist
> + update["exclude"] = testlist
>
> if "_properties_" in self.props:
> del self.props["_properties_"]
> @@ -446,9 +448,9 @@ class TestList:
>
> self.__add_field(key, sublevel, hierarchy_level, field[key])
>
> - def read_testlist(self, field, item, testlist, name, filename):
> + def read_testlist(self, update, field, item, testlist, name, filename):
>
> - match_type = item["_properties_"].get("match-type", "subtest-match")
> + match_type = update.get("type", "subtest-match")
>
> match_type_regex = set(["regex", "regex-ignorecase"])
> match_type_str = set(["subtest-match"])
> @@ -514,10 +516,13 @@ class TestList:
> if "_properties_" not in self.props[field]:
> continue
>
> - if "include" not in self.props[field]["_properties_"]:
> + update = self.props[field]["_properties_"].get("update-from-file")
> + if not update:
> continue
>
> - default_value = self.props[field]["_properties_"].get("default-testlist")
> + match_type = update.get("type", "subtest-match")
> + default_value = update.get("default--if-not-excluded")
> + append_value = update.get("append-value-if-not-excluded")
>
> testname = subtest_dict["_summary_"]
>
> @@ -527,7 +532,11 @@ class TestList:
> else:
> values = set()
>
> - for names, regex_array in self.props[field]["_properties_"]["include"].items():
> + if append_value:
> + include_names = set(re.split(",\s*", append_value))
> + values.update(include_names)
> +
> + for names, regex_array in update.get("include", {}).items():
> name = set(re.split(",\s*", names))
> for regex in regex_array:
> if regex.fullmatch(testname):
> @@ -535,20 +544,19 @@ class TestList:
> break
>
> # If test is at a global blocklist, ignore it
> - set_full_if_empty = True
> - if "exclude" in self.props[field]["_properties_"]:
> - for names, regex_array in self.props[field]["_properties_"]["exclude"].items():
> - deleted_names = set(re.split(",\s*", names))
> - for regex in regex_array:
> - if regex.fullmatch(testname):
> - if sorted(deleted_names) == sorted(values):
> - set_full_if_empty = False
> - values.discard(deleted_names)
> + set_default = True
> + for names, regex_array in update.get("exclude", {}).items():
> + deleted_names = set(re.split(",\s*", names))
> + for regex in regex_array:
> + if regex.fullmatch(testname):
> + if sorted(deleted_names) == sorted(values):
> + set_default = False
> + values -= deleted_names
>
> - if default_value and set_full_if_empty and not values:
> - values = set([default_value])
> + if default_value and set_default and not values:
> + values.update([default_value])
>
> - if values:
> + if values or self.props[field]["_properties_"].get("mandatory"):
> subtest_dict[field] = ", ".join(sorted(values))
>
> def expand_subtest(self, fname, test_name, test, allow_inherit, with_lines = False, with_subtest_nr = False):
> diff --git a/tests/intel/kms_test_config.json b/tests/intel/kms_test_config.json
> index 837380ee7fae..53465c534bdc 100644
> --- a/tests/intel/kms_test_config.json
> +++ b/tests/intel/kms_test_config.json
> @@ -22,20 +22,22 @@
> "Run type": {
> "_properties_": {
> "description": "Defines what category of testlist it belongs",
> - "default-testlist": "FULL",
> - "match-type": "subtest-match",
> - "include": [
> - { "i915 BAT": "../intel-ci/fast-feedback.testlist" },
> - { "i915 BAT chamelium": "../intel-ci/fast-feedback-chamelium-only.testlist" },
> - { "i915 chamelium": "../intel-ci/chamelium-only.testlist" },
> + "update-from-file": {
> + "append-value-if-not-excluded": "FULL",
> + "match-type": "subtest-match",
> + "include": [
> + { "i915 BAT": "../intel-ci/fast-feedback.testlist" },
> + { "i915 BAT chamelium": "../intel-ci/fast-feedback-chamelium-only.testlist" },
> + { "i915 chamelium": "../intel-ci/chamelium-only.testlist" },
>
> - { "Xe BAT": "../intel-ci/xe-fast-feedback.testlist" },
> - { "Xe BAT chamelium": "../intel-ci/xe-fast-feedback-chamelium-only.testlist" }
> - ],
> - "exclude": [
> - { "i915 BAT, i915 BAT chamelium, i915 chamelium": "../intel-ci/blacklist.txt" },
> - { "Xe BAT, Xe BAT chamelium": "../intel-ci/xe.blocklist.txt" }
> - ]
> + { "Xe BAT": "../intel-ci/xe-fast-feedback.testlist" },
> + { "Xe BAT chamelium": "../intel-ci/xe-fast-feedback-chamelium-only.testlist" }
> + ],
> + "exclude": [
> + { "i915 BAT, i915 BAT chamelium, i915 chamelium, FULL": "../intel-ci/blacklist.txt" },
> + { "Xe BAT, Xe BAT chamelium, FULL": "../intel-ci/xe.blocklist.txt" }
> + ]
> + }
> }
> }
> },
> diff --git a/tests/intel/xe_test_config.json b/tests/intel/xe_test_config.json
> index dd7aa4776ec6..6c8059b6f21f 100644
> --- a/tests/intel/xe_test_config.json
> +++ b/tests/intel/xe_test_config.json
> @@ -33,13 +33,15 @@
> "mandatory": true,
> "description": "Defines what category of testlist it belongs",
> "default-testlist": "FULL",
> - "match-type": "subtest-match",
> - "include": [
> - { "Xe BAT": "../intel-ci/xe-fast-feedback.testlist" }
> - ],
> - "exclude": [
> - { "Xe BAT": "../intel-ci/xe.blocklist.txt" }
> - ],
> + "update-from-file": {
> + "type": "subtest-match",
> + "include": [
> + { "Xe BAT": "../intel-ci/xe-fast-feedback.testlist" }
> + ],
> + "exclude": [
> + { "Xe BAT": "../intel-ci/xe.blocklist.txt" }
> + ],
> + },
> "order": [
> "boot",
> "__all__",
> --
> 2.41.0
>
More information about the igt-dev
mailing list